aspera-cli 4.27.0 → 4.27.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +29 -0
- data/CONTRIBUTING.md +1 -7
- data/TODO.md +50 -0
- data/docs/README.md +708 -488
- data/lib/aspera/agent/base.rb +10 -2
- data/lib/aspera/agent/connect.rb +7 -7
- data/lib/aspera/agent/desktop.rb +3 -3
- data/lib/aspera/agent/direct.rb +48 -19
- data/lib/aspera/agent/factory.rb +3 -3
- data/lib/aspera/agent/httpgw.rb +55 -14
- data/lib/aspera/agent/node.rb +2 -2
- data/lib/aspera/agent/transferd.rb +12 -12
- data/lib/aspera/api/aoc.rb +28 -28
- data/lib/aspera/api/cos_node.rb +10 -9
- data/lib/aspera/api/faspex.rb +2 -2
- data/lib/aspera/api/httpgw.rb +20 -20
- data/lib/aspera/api/node.rb +31 -53
- data/lib/aspera/ascmd.rb +13 -13
- data/lib/aspera/ascp/installation.rb +43 -36
- data/lib/aspera/ascp/management.rb +6 -6
- data/lib/aspera/assert.rb +12 -13
- data/lib/aspera/cli/ascp_actions.rb +7 -7
- data/lib/aspera/cli/async_transfer_store.rb +49 -5
- data/lib/aspera/cli/bootstrapper.rb +9 -9
- data/lib/aspera/cli/command_registry.rb +1 -1
- data/lib/aspera/cli/command_spec.rb +6 -3
- data/lib/aspera/cli/context.rb +3 -3
- data/lib/aspera/cli/extended_value.rb +26 -26
- data/lib/aspera/cli/formatter.rb +76 -54
- data/lib/aspera/cli/gem_checker.rb +1 -1
- data/lib/aspera/cli/hints.rb +2 -2
- data/lib/aspera/cli/http.rb +22 -29
- data/lib/aspera/cli/mailer.rb +7 -7
- data/lib/aspera/cli/mcp_tool.rb +31 -4
- data/lib/aspera/cli/option_declarator.rb +77 -0
- data/lib/aspera/cli/options.schema.yaml +273 -3
- data/lib/aspera/cli/parser.rb +539 -280
- data/lib/aspera/cli/plugins/alee.rb +1 -1
- data/lib/aspera/cli/plugins/aoc.rb +297 -313
- data/lib/aspera/cli/plugins/ats.rb +25 -21
- data/lib/aspera/cli/plugins/base.rb +357 -186
- data/lib/aspera/cli/plugins/basic_auth.rb +0 -12
- data/lib/aspera/cli/plugins/config.rb +76 -73
- data/lib/aspera/cli/plugins/console.rb +8 -8
- data/lib/aspera/cli/plugins/cos.rb +4 -3
- data/lib/aspera/cli/plugins/factory.rb +2 -2
- data/lib/aspera/cli/plugins/faspex5.rb +259 -248
- data/lib/aspera/cli/plugins/faspio.rb +7 -2
- data/lib/aspera/cli/plugins/httpgw.rb +1 -1
- data/lib/aspera/cli/plugins/mcp.rb +11 -11
- data/lib/aspera/cli/plugins/node.rb +227 -283
- data/lib/aspera/cli/plugins/orchestrator.rb +23 -23
- data/lib/aspera/cli/plugins/preview.rb +61 -64
- data/lib/aspera/cli/plugins/server.rb +36 -31
- data/lib/aspera/cli/plugins/shares.rb +174 -135
- data/lib/aspera/cli/preset_actions.rb +39 -19
- data/lib/aspera/cli/preset_manager.rb +74 -32
- data/lib/aspera/cli/result.rb +18 -16
- data/lib/aspera/cli/runner.rb +75 -39
- data/lib/aspera/cli/secret_finder.rb +1 -1
- data/lib/aspera/cli/sync_actions.rb +36 -35
- data/lib/aspera/cli/terminal_formatter.rb +2 -2
- data/lib/aspera/cli/transfer_actions.rb +17 -10
- data/lib/aspera/cli/transfer_agent.rb +44 -44
- data/lib/aspera/cli/transfer_progress.rb +7 -7
- data/lib/aspera/cli/vault_manager.rb +34 -12
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +17 -20
- data/lib/aspera/colors.rb +3 -3
- data/lib/aspera/command_line_builder.rb +16 -16
- data/lib/aspera/command_line_converter.rb +1 -1
- data/lib/aspera/coverage.rb +1 -2
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/dot_container.rb +8 -8
- data/lib/aspera/environment.rb +12 -12
- data/lib/aspera/faspex_gw.rb +4 -4
- data/lib/aspera/faspex_postproc.rb +14 -9
- data/lib/aspera/graphql.rb +4 -2
- data/lib/aspera/hash_ext.rb +2 -2
- data/lib/aspera/json_rpc/client.rb +1 -1
- data/lib/aspera/keychain/base.rb +10 -3
- data/lib/aspera/keychain/encrypted_hash.rb +92 -17
- data/lib/aspera/keychain/factory.rb +17 -5
- data/lib/aspera/keychain/hashicorp_vault.rb +1 -1
- data/lib/aspera/keychain/macos_security.rb +23 -25
- data/lib/aspera/keychain/one_password_api.rb +86 -0
- data/lib/aspera/keychain/one_password_base.rb +34 -0
- data/lib/aspera/keychain/one_password_cli.rb +98 -0
- data/lib/aspera/link_header.rb +2 -2
- data/lib/aspera/log.rb +13 -13
- data/lib/aspera/markdown.rb +5 -5
- data/lib/aspera/nagios.rb +8 -8
- data/lib/aspera/node_simulator.rb +33 -27
- data/lib/aspera/oauth/base.rb +4 -5
- data/lib/aspera/oauth/boot.rb +3 -3
- data/lib/aspera/oauth/factory.rb +5 -5
- data/lib/aspera/oauth/jwt.rb +5 -7
- data/lib/aspera/oauth/web.rb +4 -4
- data/lib/aspera/persistency_action_once.rb +4 -4
- data/lib/aspera/persistency_folder.rb +8 -8
- data/lib/aspera/preview/file_types.rb +4 -4
- data/lib/aspera/preview/generator.rb +49 -46
- data/lib/aspera/preview/options.rb +3 -2
- data/lib/aspera/preview/terminal.rb +5 -5
- data/lib/aspera/preview/utils.rb +65 -18
- data/lib/aspera/products/connect.rb +2 -2
- data/lib/aspera/products/desktop.rb +1 -1
- data/lib/aspera/products/other.rb +2 -2
- data/lib/aspera/products/transferd.rb +3 -3
- data/lib/aspera/proxy_auto_config.rb +10 -10
- data/lib/aspera/rest.rb +149 -115
- data/lib/aspera/rest_error_analyzer.rb +3 -3
- data/lib/aspera/rest_list.rb +10 -9
- data/lib/aspera/schema/IBM Aspera Node API-4.4.6.yaml +6232 -0
- data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +249 -0
- data/lib/aspera/schema/IBM_Aspera_Shares.yaml +4653 -0
- data/lib/aspera/schema/documentation.rb +17 -11
- data/lib/aspera/schema/reader.rb +78 -4
- data/lib/aspera/schema/registry.rb +26 -7
- data/lib/aspera/secret_hider.rb +5 -5
- data/lib/aspera/ssh.rb +64 -31
- data/lib/aspera/ssl.rb +3 -3
- data/lib/aspera/sync/database.rb +2 -2
- data/lib/aspera/sync/operations.rb +13 -13
- data/lib/aspera/temp_file_manager.rb +24 -4
- data/lib/aspera/transfer/faux_file.rb +2 -2
- data/lib/aspera/transfer/parameters.rb +15 -15
- data/lib/aspera/transfer/resumer.rb +9 -9
- data/lib/aspera/transfer/spec.rb +1 -1
- data/lib/aspera/uri_reader.rb +52 -15
- data/lib/aspera/web_auth.rb +6 -5
- data/lib/aspera/web_server_simple.rb +9 -9
- data/lib/aspera/yaml.rb +4 -4
- data.tar.gz.sig +0 -0
- metadata +10 -4
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/plugins/faspex.rb +0 -576
- data/lib/aspera/transfer/uri.rb +0 -56
|
@@ -28,7 +28,7 @@ module Aspera
|
|
|
28
28
|
@agent_columns = agent_columns
|
|
29
29
|
@code_highlight = code_highlight
|
|
30
30
|
@columns = %w[name type description]
|
|
31
|
-
@columns.insert(-2, *Agent::Factory::ALL.values.map{ |i| i[:short].to_s}.sort) if @agent_columns
|
|
31
|
+
@columns.insert(-2, *Agent::Factory::ALL.values.map { |i| i[:short].to_s }.sort) if @agent_columns
|
|
32
32
|
# Sections: each entry is {header: row_or_nil, rows: []}
|
|
33
33
|
# A flat schema produces a single section with no header.
|
|
34
34
|
@sections = [{header: nil, rows: []}]
|
|
@@ -36,7 +36,7 @@ module Aspera
|
|
|
36
36
|
|
|
37
37
|
def rows
|
|
38
38
|
@sections.flat_map do |section|
|
|
39
|
-
sorted = section[:rows].sort_by{ |i| i['name']}
|
|
39
|
+
sorted = section[:rows].sort_by { |i| i['name'] }
|
|
40
40
|
section[:header] ? [section[:header]] + sorted : sorted
|
|
41
41
|
end
|
|
42
42
|
end
|
|
@@ -47,7 +47,7 @@ module Aspera
|
|
|
47
47
|
# First row is the titles (for Markdown table generation)
|
|
48
48
|
# @return [Array<Array<String>>]
|
|
49
49
|
def table
|
|
50
|
-
[@columns] + rows.map{ |row| @columns.map{ |field| row[field]}}
|
|
50
|
+
[@columns] + rows.map { |row| @columns.map { |field| row[field] } }
|
|
51
51
|
end
|
|
52
52
|
|
|
53
53
|
# Generate a documentation table from a JSON schema for transfer specifications
|
|
@@ -58,9 +58,9 @@ module Aspera
|
|
|
58
58
|
# @param schema [Reader] The JSON schema to process
|
|
59
59
|
# @return [Documentation]
|
|
60
60
|
def build(schema = nil)
|
|
61
|
-
code = @code_highlight ? ->(c){"`#{c}`"} : ->(c){c}
|
|
61
|
+
code = @code_highlight ? ->(c) { "`#{c}`" } : ->(c) { c }
|
|
62
62
|
schema ||= @schema
|
|
63
|
-
render_title = ->(title){title.gsub(Markdown::FORMATS){@formatter.markdown_text(Regexp.last_match)}}
|
|
63
|
+
render_title = ->(title) { title.gsub(Markdown::FORMATS) { @formatter.markdown_text(Regexp.last_match) } }
|
|
64
64
|
on_variant = ->(variant_reader, discriminant_property, discriminant_value) do
|
|
65
65
|
title = variant_reader.current['title'] || variant_reader.current['description']
|
|
66
66
|
header =
|
|
@@ -80,8 +80,14 @@ module Aspera
|
|
|
80
80
|
schema.each_property(on_variant: on_variant) do |property_schema, _name, property_full_name|
|
|
81
81
|
node = property_schema.current
|
|
82
82
|
# Manual table
|
|
83
|
-
item_type =
|
|
84
|
-
|
|
83
|
+
item_type =
|
|
84
|
+
if node['type'].is_a?(Array)
|
|
85
|
+
node['type'].map { |t| JSON_TYPE_TO_DOC.fetch(t, t) }.join(', ')
|
|
86
|
+
elsif node['type'].eql?('array') && node.dig('items', 'type').is_a?(String)
|
|
87
|
+
"#{JSON_TYPE_TO_DOC.fetch(node['type'], node['type'])}[#{JSON_TYPE_TO_DOC.fetch(node.dig('items', 'type'), node.dig('items', 'type'))}]"
|
|
88
|
+
else
|
|
89
|
+
JSON_TYPE_TO_DOC.fetch(node['type'], node['type'])
|
|
90
|
+
end
|
|
85
91
|
item = {
|
|
86
92
|
'name' => code.call(property_full_name),
|
|
87
93
|
'type' => code.call(item_type),
|
|
@@ -90,7 +96,7 @@ module Aspera
|
|
|
90
96
|
# Render Markdown formatting and split lines
|
|
91
97
|
item['description'] =
|
|
92
98
|
node['description']
|
|
93
|
-
.gsub(Markdown::FORMATS){@formatter.markdown_text(Regexp.last_match)}
|
|
99
|
+
.gsub(Markdown::FORMATS) { @formatter.markdown_text(Regexp.last_match) }
|
|
94
100
|
.split("\n") if node.key?('description')
|
|
95
101
|
item['description'].unshift("DEPRECATED: #{node['x-deprecation']}") if node.key?('x-deprecation')
|
|
96
102
|
# Add flags for supported agents in doc
|
|
@@ -98,17 +104,17 @@ module Aspera
|
|
|
98
104
|
Agent::Factory::ALL.each_key do |sym|
|
|
99
105
|
agents.push(sym) if node['x-agents'].nil? || node['x-agents'].include?(sym.to_s)
|
|
100
106
|
end
|
|
101
|
-
Aspera.assert(agents.include?(:direct)){"#{property_full_name}: x-cli-option requires agent direct (or nil)"} if node['x-cli-option']
|
|
107
|
+
Aspera.assert(agents.include?(:direct)) { "#{property_full_name}: x-cli-option requires agent direct (or nil)" } if node['x-cli-option']
|
|
102
108
|
if @agent_columns
|
|
103
109
|
Agent::Factory::ALL.each do |sym, names|
|
|
104
110
|
item[names[:short].to_s] = @formatter.tick(agents.include?(sym))
|
|
105
111
|
end
|
|
106
112
|
else
|
|
107
|
-
item['description'].push("(#{agents.map{ |i| Agent::Factory::ALL[i][:short].to_s.upcase}.sort.join(', ')})") unless agents.length.eql?(Agent::Factory::ALL.length)
|
|
113
|
+
item['description'].push("(#{agents.map { |i| Agent::Factory::ALL[i][:short].to_s.upcase }.sort.join(', ')})") unless agents.length.eql?(Agent::Factory::ALL.length)
|
|
108
114
|
end
|
|
109
115
|
# Only keep lines that are usable in supported agents
|
|
110
116
|
next if agents.empty?
|
|
111
|
-
item['description'].push("Allowed values: #{node['enum'].map{ |v| @formatter.markdown_text("`#{v}`")}.join(', ')}.") if node.key?('enum')
|
|
117
|
+
item['description'].push("Allowed values: #{node['enum'].map { |v| @formatter.markdown_text("`#{v}`") }.join(', ')}.") if node.key?('enum')
|
|
112
118
|
item['description'].push("Default: #{@formatter.markdown_text("`#{node['default']}`")}.") if node.key?('default')
|
|
113
119
|
item['description'].push("Example: #{@formatter.markdown_text("`#{node['example']}`")}.") if node.key?('example')
|
|
114
120
|
if @include_option
|
data/lib/aspera/schema/reader.rb
CHANGED
|
@@ -42,12 +42,12 @@ module Aspera
|
|
|
42
42
|
def dig(*path)
|
|
43
43
|
current = @current
|
|
44
44
|
path.each do |p|
|
|
45
|
-
Aspera.assert(current.key?(p)){"schema: #{p} in #{path}"}
|
|
45
|
+
Aspera.assert(current.key?(p)) { "schema: #{p} in #{path}" }
|
|
46
46
|
current = current[p]
|
|
47
|
-
Aspera.assert_type(current, Hash){'schema'}
|
|
47
|
+
Aspera.assert_type(current, Hash) { 'schema' }
|
|
48
48
|
if current.key?('$ref')
|
|
49
49
|
ref = current['$ref']
|
|
50
|
-
Aspera.assert(ref.start_with?('#/')){"schema $ref must start with '#/': #{ref}"}
|
|
50
|
+
Aspera.assert(ref.start_with?('#/')) { "schema $ref must start with '#/': #{ref}" }
|
|
51
51
|
current = @root.dig(*ref[2..].split('/'))
|
|
52
52
|
end
|
|
53
53
|
end
|
|
@@ -56,7 +56,7 @@ module Aspera
|
|
|
56
56
|
|
|
57
57
|
# Resolve a $ref string to a Reader
|
|
58
58
|
def resolve_ref(ref)
|
|
59
|
-
Aspera.assert(ref.start_with?('#/')){"schema $ref must start with '#/': #{ref}"}
|
|
59
|
+
Aspera.assert(ref.start_with?('#/')) { "schema $ref must start with '#/': #{ref}" }
|
|
60
60
|
Reader.new(@root, @root.dig(*ref[2..].split('/')))
|
|
61
61
|
end
|
|
62
62
|
|
|
@@ -98,6 +98,15 @@ module Aspera
|
|
|
98
98
|
end
|
|
99
99
|
return
|
|
100
100
|
end
|
|
101
|
+
if @current.key?('allOf')
|
|
102
|
+
# Merge all branches: each branch contributes its properties (no variants)
|
|
103
|
+
@current['allOf'].each do |branch_node|
|
|
104
|
+
ref = branch_node['$ref']
|
|
105
|
+
branch_reader = ref ? resolve_ref(ref) : Reader.new(@root, branch_node)
|
|
106
|
+
branch_reader.each_property(prefix, on_variant: on_variant, &block)
|
|
107
|
+
end
|
|
108
|
+
return
|
|
109
|
+
end
|
|
101
110
|
properties = dig('properties')
|
|
102
111
|
properties.current.each_key do |name|
|
|
103
112
|
property_full_name = "#{prefix}#{name}"
|
|
@@ -117,6 +126,71 @@ module Aspera
|
|
|
117
126
|
array_item_schema.each_property("#{property_full_name}[].", on_variant: on_variant, &block) if array_item_schema.current['properties']
|
|
118
127
|
end
|
|
119
128
|
end
|
|
129
|
+
# allOf without explicit type: object — recurse to merge all branches
|
|
130
|
+
property_schema.each_property("#{property_full_name}.", on_variant: on_variant, &block) if node['allOf']
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# Convert this schema to a flat array of field descriptors.
|
|
135
|
+
# Returns raw semantic fields with no ANSI or formatting, suitable for
|
|
136
|
+
# JSON/YAML output or MCP consumption. An AI or script can use the result
|
|
137
|
+
# directly to build a valid payload.
|
|
138
|
+
#
|
|
139
|
+
# Each Hash entry contains:
|
|
140
|
+
# name [String] dot/bracket-path of the field (e.g. "recipients[].name")
|
|
141
|
+
# type [String] JSON type string (e.g. "string", "boolean", "Array[object]")
|
|
142
|
+
# required [Boolean] true when the field is in its immediate parent's required list
|
|
143
|
+
# description [String] human description (may contain Markdown **bold** / `code`)
|
|
144
|
+
# default [Object] (optional) default value as native Ruby type
|
|
145
|
+
# enum [Array] (optional) list of allowed string values
|
|
146
|
+
#
|
|
147
|
+
# @return [Array<Hash>]
|
|
148
|
+
def to_rows
|
|
149
|
+
rows = []
|
|
150
|
+
collect_rows(rows, self, '')
|
|
151
|
+
rows
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
private
|
|
155
|
+
|
|
156
|
+
# Recursively collect rows from a schema node, passing each property's own
|
|
157
|
+
# parent `required` array so that only direct-parent membership is checked.
|
|
158
|
+
def collect_rows(rows, reader, prefix)
|
|
159
|
+
return unless reader.current.key?('properties')
|
|
160
|
+
parent_required = Set.new(Array(reader.current['required']))
|
|
161
|
+
props_reader = reader.dig('properties')
|
|
162
|
+
props_reader.current.each_key do |name|
|
|
163
|
+
prop_reader = props_reader.dig(name)
|
|
164
|
+
node = prop_reader.current
|
|
165
|
+
full_name = "#{prefix}#{name}"
|
|
166
|
+
type_val =
|
|
167
|
+
if node['type'].is_a?(Array)
|
|
168
|
+
node['type'].join(', ')
|
|
169
|
+
elsif node['type'].eql?('array') && node.dig('items', 'type').is_a?(String)
|
|
170
|
+
"Array[#{node.dig('items', 'type')}]"
|
|
171
|
+
else
|
|
172
|
+
node['type'].to_s
|
|
173
|
+
end
|
|
174
|
+
row = {
|
|
175
|
+
'name' => full_name,
|
|
176
|
+
'type' => type_val,
|
|
177
|
+
'required' => parent_required.include?(name),
|
|
178
|
+
'description' => node['description'].to_s
|
|
179
|
+
}
|
|
180
|
+
row['default'] = node['default'] if node.key?('default')
|
|
181
|
+
row['enum'] = node['enum'] if node.key?('enum')
|
|
182
|
+
rows << row
|
|
183
|
+
# Recurse into nested object or array-of-objects
|
|
184
|
+
case node['type']
|
|
185
|
+
when 'object'
|
|
186
|
+
collect_rows(rows, prop_reader, "#{full_name}.") if node['properties']
|
|
187
|
+
when 'array'
|
|
188
|
+
if node['items']
|
|
189
|
+
item_reader = prop_reader.dig('items')
|
|
190
|
+
collect_rows(rows, item_reader, "#{full_name}[].") if item_reader.current['properties']
|
|
191
|
+
end
|
|
192
|
+
end
|
|
193
|
+
collect_rows(rows, prop_reader, "#{full_name}.") if node['allOf']
|
|
120
194
|
end
|
|
121
195
|
end
|
|
122
196
|
end
|
|
@@ -18,21 +18,32 @@ module Aspera
|
|
|
18
18
|
LOCATIONS.key?(sym)
|
|
19
19
|
end
|
|
20
20
|
|
|
21
|
+
# Split a component string into registry key and optional path prefix.
|
|
22
|
+
# Syntax: 'key' or 'key+/prefix' (e.g. 'shares+/api/v1')
|
|
23
|
+
# @param component [String] registry key with optional '+/prefix'
|
|
24
|
+
# @return [Array(String, String)] [key, prefix] where prefix may be ''
|
|
25
|
+
def split_component(component)
|
|
26
|
+
key, prefix = component.split('+', 2)
|
|
27
|
+
[key, prefix || '']
|
|
28
|
+
end
|
|
29
|
+
|
|
21
30
|
# Get path to request body schema, no check if it exists
|
|
22
|
-
# @param component [String] registry key (e.g. '
|
|
23
|
-
# @param endpoint [String] endpoint path without leading slash (e.g. '
|
|
31
|
+
# @param component [String] registry key, optionally with path prefix (e.g. 'shares+/api/v1')
|
|
32
|
+
# @param endpoint [String] endpoint path without leading slash (e.g. 'data/shares.post')
|
|
24
33
|
# @return [String] schema path usable in schema: keyword
|
|
25
34
|
def req_body(component, endpoint)
|
|
26
|
-
|
|
35
|
+
key, prefix = split_component(component)
|
|
36
|
+
"#{key}:paths.#{prefix}/#{endpoint}.requestBody.content.application/json.schema"
|
|
27
37
|
end
|
|
28
38
|
|
|
29
39
|
# Get path to query parameters for a GET endpoint
|
|
30
|
-
# @param component [String] registry key (e.g. '
|
|
31
|
-
# @param endpoint [String] resource path without leading slash (e.g. '
|
|
40
|
+
# @param component [String] registry key, optionally with path prefix (e.g. 'shares+/api/v1')
|
|
41
|
+
# @param endpoint [String] resource path without leading slash (e.g. 'data/shares')
|
|
32
42
|
# @param method [String] HTTP method (default: 'get')
|
|
33
43
|
# @return [String] schema path usable in query_schema: keyword
|
|
34
44
|
def query_params(component, endpoint, method: 'get')
|
|
35
|
-
|
|
45
|
+
key, prefix = split_component(component)
|
|
46
|
+
"#{key}:paths.#{prefix}/#{endpoint}.#{method}#{QUERY_PARAMS_SUFFIX}"
|
|
36
47
|
end
|
|
37
48
|
end
|
|
38
49
|
|
|
@@ -43,6 +54,9 @@ module Aspera
|
|
|
43
54
|
opts: 'aspera/cli/options.schema.yaml',
|
|
44
55
|
aoc: 'aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml',
|
|
45
56
|
faspex: 'aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml',
|
|
57
|
+
faspio: 'aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml',
|
|
58
|
+
node: 'aspera/schema/IBM Aspera Node API-4.4.6.yaml',
|
|
59
|
+
shares: 'aspera/schema/IBM_Aspera_Shares.yaml',
|
|
46
60
|
async_tables: 'aspera/schema/async_tables.yaml'
|
|
47
61
|
}
|
|
48
62
|
|
|
@@ -52,6 +66,9 @@ module Aspera
|
|
|
52
66
|
SYNC_ARGS = 'args'
|
|
53
67
|
AOC = 'aoc'
|
|
54
68
|
FASPEX = 'faspex'
|
|
69
|
+
FASPIO = 'faspio'
|
|
70
|
+
NODE = 'node'
|
|
71
|
+
SHARES = 'shares+/api/v1'
|
|
55
72
|
ASYNC_TABLES = 'async_tables'
|
|
56
73
|
LOG_OPTIONS = "#{OPTIONS}:components.schemas.LogOptions"
|
|
57
74
|
DIRECT_AGENT_OPTIONS = "#{OPTIONS}:components.schemas.DirectAgentOptions"
|
|
@@ -62,7 +79,9 @@ module Aspera
|
|
|
62
79
|
SMTP_OPTIONS = "#{OPTIONS}:components.schemas.SmtpOptions"
|
|
63
80
|
HTTP_OPTIONS = "#{OPTIONS}:components.schemas.HttpOptions"
|
|
64
81
|
VAULT_OPTIONS = "#{OPTIONS}:components.schemas.VaultOptions"
|
|
82
|
+
VAULT_SECRET = "#{OPTIONS}:components.schemas.VaultSecret"
|
|
65
83
|
IMAGE_OPTIONS = "#{OPTIONS}:components.schemas.ImageOptions"
|
|
84
|
+
OUT_OPTIONS = "#{OPTIONS}:components.schemas.OutOptions"
|
|
66
85
|
PACKAGE_FOLDER_OPTIONS = "#{OPTIONS}:components.schemas.PackageFolderOptions"
|
|
67
86
|
|
|
68
87
|
REQ_BODY = '.requestBody.content.application/json.schema'
|
|
@@ -84,7 +103,7 @@ module Aspera
|
|
|
84
103
|
def reader(name_path)
|
|
85
104
|
name, path = name_path.split(':', 2)
|
|
86
105
|
sym = name.to_sym
|
|
87
|
-
Aspera.assert(Registry.known?(sym)){"schema: #{sym}"}
|
|
106
|
+
Aspera.assert(Registry.known?(sym)) { "schema: #{sym}" }
|
|
88
107
|
spec_file = File.join(@main_folder, LOCATIONS[sym])
|
|
89
108
|
@cache[sym] = Yaml.safe_load(File.read(spec_file)) if spec_file.end_with?('.yaml') && !@cache.key?(sym)
|
|
90
109
|
@cache[sym] = JSON.parse(File.read(spec_file)) if spec_file.end_with?('.json') && !@cache.key?(sym)
|
data/lib/aspera/secret_hider.rb
CHANGED
|
@@ -52,7 +52,7 @@ module Aspera
|
|
|
52
52
|
return lambda do |severity, date_time, program_name, msg|
|
|
53
53
|
if msg.is_a?(String) && !@log_secrets
|
|
54
54
|
REGEX_LOG_REPLACES.each do |reg_ex|
|
|
55
|
-
msg = msg.gsub(reg_ex){"#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}"}
|
|
55
|
+
msg = msg.gsub(reg_ex) { "#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}" }
|
|
56
56
|
end
|
|
57
57
|
end
|
|
58
58
|
original_formatter.call(severity, date_time, program_name, msg)
|
|
@@ -63,7 +63,7 @@ module Aspera
|
|
|
63
63
|
# @param value [String] Input string possibly containing a private key
|
|
64
64
|
# @return [String] String with private key replaced by placeholder
|
|
65
65
|
def hide_secrets_in_string(value)
|
|
66
|
-
return value.gsub(REGEX_LOG_REPLACES.first){"#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}"}
|
|
66
|
+
return value.gsub(REGEX_LOG_REPLACES.first) { "#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}" }
|
|
67
67
|
end
|
|
68
68
|
|
|
69
69
|
# @param keyword [String, Symbol] Key name to check
|
|
@@ -74,17 +74,17 @@ module Aspera
|
|
|
74
74
|
# only Strings can be secrets, not booleans, or hash, arrays
|
|
75
75
|
return false unless keyword.is_a?(String) && value.is_a?(String)
|
|
76
76
|
# those are not secrets
|
|
77
|
-
return false if KEY_FALSE_POSITIVES.any?{ |f| f.match?(keyword)}
|
|
77
|
+
return false if KEY_FALSE_POSITIVES.any? { |f| f.match?(keyword) }
|
|
78
78
|
return true if ADDITIONAL_KEYS_TO_HIDE.include?(keyword)
|
|
79
79
|
# check if keyword (name) contains an element that designate it as a secret
|
|
80
|
-
ALL_SECRETS.any?{ |kw| keyword.include?(kw)}
|
|
80
|
+
ALL_SECRETS.any? { |kw| keyword.include?(kw) }
|
|
81
81
|
end
|
|
82
82
|
|
|
83
83
|
# Hides recursively secrets in Hash or Array of Hash
|
|
84
84
|
def deep_remove_secret(obj)
|
|
85
85
|
case obj
|
|
86
86
|
when Array
|
|
87
|
-
obj.each{ |i| deep_remove_secret(i)}
|
|
87
|
+
obj.each { |i| deep_remove_secret(i) }
|
|
88
88
|
when Hash
|
|
89
89
|
obj.each do |k, v|
|
|
90
90
|
if secret?(k, v)
|
data/lib/aspera/ssh.rb
CHANGED
|
@@ -8,77 +8,110 @@ module Aspera
|
|
|
8
8
|
# A simple wrapper around Net::SSH
|
|
9
9
|
# executes one command and get its result from stdout
|
|
10
10
|
class Ssh
|
|
11
|
+
# Raised when the remote command fails or cannot be started
|
|
11
12
|
class Error < Aspera::Error
|
|
12
13
|
end
|
|
14
|
+
# Regexp matching ecdsa/ecdh-sha2 algorithm names (JRuby workaround)
|
|
15
|
+
EXCLUDE_ECDSHA2 = /^ecd(sa|h)-sha2/
|
|
16
|
+
|
|
13
17
|
class << self
|
|
14
|
-
#
|
|
18
|
+
# Removes ed25519 keys from the default identity list used by Net::SSH.
|
|
19
|
+
# Called when the `ed25519` gem is absent or explicitly disabled via
|
|
20
|
+
# `ASCLI_ENABLE_ED25519=false`.
|
|
21
|
+
# @return [void]
|
|
15
22
|
def disable_ed25519_keys
|
|
16
23
|
Log.log.debug('Disabling SSH ed25519 user keys')
|
|
17
24
|
old_verbose = $VERBOSE
|
|
18
25
|
$VERBOSE = nil
|
|
19
26
|
Net::SSH::Authentication::Session.class_eval do
|
|
20
27
|
define_method(:default_keys) do
|
|
21
|
-
%w[.ssh .ssh2].product(%w[rsa dsa ecdsa]).map{"~/#{_1}/id_#{_2}"}.freeze
|
|
28
|
+
%w[.ssh .ssh2].product(%w[rsa dsa ecdsa]).map { "~/#{_1}/id_#{_2}" }.freeze
|
|
22
29
|
end
|
|
23
30
|
private(:default_keys)
|
|
24
|
-
end
|
|
31
|
+
end
|
|
25
32
|
$VERBOSE = old_verbose
|
|
26
33
|
end
|
|
27
34
|
|
|
28
|
-
#
|
|
35
|
+
# Returns Net::SSH option overrides that exclude all ecdsa/ecdh-sha2 algorithms.
|
|
36
|
+
# Merge the result into `ssh_options` passed to {Ssh#initialize} to avoid
|
|
37
|
+
# mutating Net::SSH internal constants.
|
|
38
|
+
# @return [Hash{Symbol => Array<String>}] `:host_key` and `:kex` filtered lists
|
|
39
|
+
def no_ecd_sha2_options
|
|
40
|
+
Log.log.debug('Building SSH options without ecdsa/ecdh-sha2')
|
|
41
|
+
{
|
|
42
|
+
host_key: Net::SSH::Transport::Algorithms::ALGORITHMS[:host_key].reject { |a| a.match?(EXCLUDE_ECDSHA2) },
|
|
43
|
+
kex: Net::SSH::Transport::Algorithms::ALGORITHMS[:kex].reject { |a| a.match?(EXCLUDE_ECDSHA2) }
|
|
44
|
+
}
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Mutates Net::SSH internal algorithm lists to remove ecdsa/ecdh-sha2 entries globally.
|
|
48
|
+
# Kept for backwards compatibility and JRuby usage; prefer {no_ecd_sha2_options} for new code.
|
|
49
|
+
# @return [void]
|
|
29
50
|
def disable_ecd_sha2_algorithms
|
|
30
|
-
Log.log.debug('Disabling SSH ecdsa')
|
|
31
|
-
Net::SSH::Transport::Algorithms::ALGORITHMS.each_value{ |a| a.reject!{ |a| a
|
|
32
|
-
Net::SSH::KnownHosts::SUPPORTED_TYPE.reject!{ |t| t
|
|
51
|
+
Log.log.debug('Disabling SSH ecdsa (global)')
|
|
52
|
+
Net::SSH::Transport::Algorithms::ALGORITHMS.each_value { |a| a.reject! { |a| a.match?(EXCLUDE_ECDSHA2) } }
|
|
53
|
+
Net::SSH::KnownHosts::SUPPORTED_TYPE.reject! { |t| t.match?(EXCLUDE_ECDSHA2) }
|
|
33
54
|
end
|
|
34
55
|
end
|
|
35
|
-
#
|
|
36
|
-
#
|
|
56
|
+
# @param host [String] remote server address
|
|
57
|
+
# @param username [String] SSH user name
|
|
58
|
+
# @param ssh_options [Hash{Symbol => Object}] options forwarded to `Net::SSH.start`
|
|
59
|
+
# (see {https://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start Net::SSH.start}).
|
|
60
|
+
# Defaults are injected for `:logger`, `:verbose`, and `:use_agent` if absent.
|
|
37
61
|
def initialize(host, username, ssh_options)
|
|
38
62
|
Aspera.assert_type(host, String)
|
|
39
63
|
Aspera.assert_type(username, String)
|
|
40
64
|
Aspera.assert_type(ssh_options, Hash)
|
|
65
|
+
Aspera.assert_hash_all(ssh_options, Symbol, nil)
|
|
41
66
|
@host = host
|
|
42
67
|
@username = username
|
|
43
68
|
@ssh_options = ssh_options.dup
|
|
44
69
|
@ssh_options[:logger] = Log.log unless @ssh_options.key?(:logger)
|
|
45
70
|
@ssh_options[:verbose] = :warn unless @ssh_options.key?(:verbose)
|
|
71
|
+
# @ssh_options[:verbose] = :debug
|
|
46
72
|
@ssh_options[:use_agent] = false unless @ssh_options.key?(:use_agent)
|
|
47
|
-
Log.log.debug{"ssh:#{@username}@#{@host}"}
|
|
73
|
+
Log.log.debug { "ssh:#{@username}@#{@host}" }
|
|
48
74
|
Log.dump(:ssh_options, @ssh_options)
|
|
49
75
|
end
|
|
50
76
|
|
|
51
|
-
#
|
|
52
|
-
|
|
77
|
+
# Executes a single command on the remote host over a new SSH session.
|
|
78
|
+
# @param cmd [String] shell command to execute remotely
|
|
79
|
+
# @param input [String, nil] data written to the command's stdin, or `nil`
|
|
80
|
+
# @return [String] concatenated stdout of the remote command
|
|
81
|
+
# @raise [Error] if the channel cannot be opened or the remote command exits with a non-zero status
|
|
82
|
+
def execute(cmd, input: nil)
|
|
53
83
|
Aspera.assert_type(cmd, String)
|
|
54
|
-
Log.log.debug{"cmd=#{cmd}"}
|
|
84
|
+
Log.log.debug { "cmd=#{cmd}" }
|
|
85
|
+
# @type response [Array<String>]
|
|
55
86
|
response = []
|
|
87
|
+
# @type error [Array<String>]
|
|
56
88
|
error = []
|
|
89
|
+
exit_code = nil
|
|
90
|
+
# @param session [Net::SSH::Connection::Session]
|
|
57
91
|
Net::SSH.start(@host, @username, @ssh_options) do |session|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
channel.on_extended_data
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
92
|
+
# @param channel [Net::SSH::Connection::Channel]
|
|
93
|
+
session.open_channel do |channel|
|
|
94
|
+
# Register stdout/stderr before exec so no data is missed (e.g. ForceCommand errors)
|
|
95
|
+
channel.on_data { |_ch, data| response.push(data) }
|
|
96
|
+
channel.on_extended_data { |_ch, type, data| error.push(data) if type.eql?(1) }
|
|
97
|
+
# @param data [Net::SSH::Buffer]
|
|
98
|
+
channel.on_request('exit-status') do |_channel, data|
|
|
99
|
+
Log.dump(:data, data, level: :trace1)
|
|
66
100
|
exit_code = data.read_long
|
|
67
|
-
next if exit_code.zero?
|
|
68
|
-
error_message = "#{cmd}: exit #{exit_code}, #{error.join.chomp}"
|
|
69
|
-
raise Error, error_message if exception
|
|
70
|
-
# Happens when windows user hasn't logged in and created home account.
|
|
71
|
-
error_message += "\nHint: home not created in Windows?" if data.include?('Could not chdir to home directory')
|
|
72
|
-
Log.log.debug(error_message)
|
|
73
101
|
end
|
|
74
102
|
# send command to SSH channel (execute) cspell: disable-next-line
|
|
75
|
-
channel.send('cexe'.reverse, cmd)
|
|
103
|
+
channel.send('cexe'.reverse, cmd) do |_ch, success|
|
|
104
|
+
raise Error, "could not execute command: #{cmd}" unless success
|
|
105
|
+
channel.send_data(input) unless input.nil?
|
|
106
|
+
end
|
|
76
107
|
end
|
|
77
|
-
# wait for channel to finish
|
|
78
|
-
ssh_channel.wait
|
|
79
|
-
# main SSH session loop
|
|
108
|
+
# wait for channel to finish and flush the session
|
|
80
109
|
session.loop
|
|
81
110
|
end
|
|
111
|
+
error_text = error.join
|
|
112
|
+
hint = error_text.include?('Could not chdir to home directory') ? "\nHint: home not created in Windows?" : ''
|
|
113
|
+
raise Error, "#{cmd}: exit #{exit_code}, #{error_text.chomp}#{hint}" if exit_code&.nonzero?
|
|
114
|
+
Log.log.error { "#{error_text}#{hint}" } unless error_text.empty?
|
|
82
115
|
# response as single string
|
|
83
116
|
return response.join
|
|
84
117
|
end
|
data/lib/aspera/ssl.rb
CHANGED
|
@@ -13,15 +13,15 @@ module Aspera
|
|
|
13
13
|
attr_reader :extra_options
|
|
14
14
|
|
|
15
15
|
def option_list=(options)
|
|
16
|
-
Aspera.assert_type(options, Array){'ssl_options'}
|
|
16
|
+
Aspera.assert_type(options, Array) { 'ssl_options' }
|
|
17
17
|
options.each do |opt|
|
|
18
|
-
Aspera.assert_type(opt, String, Integer){'Expected String or Integer in ssl_options'}
|
|
18
|
+
Aspera.assert_type(opt, String, Integer) { 'Expected String or Integer in ssl_options' }
|
|
19
19
|
case opt
|
|
20
20
|
when Integer
|
|
21
21
|
@extra_options = opt
|
|
22
22
|
when String
|
|
23
23
|
name = "OP_#{opt.start_with?('-') ? opt[1..] : opt}".upcase
|
|
24
|
-
Aspera.assert(OpenSSL::SSL.const_defined?(name), type: Cli::BadArgument){"Unknown ssl_option: #{name}, use one of: #{OpenSSL::SSL.constants.grep(/^OP_/).map{ |c| c.to_s.sub(/^OP_/, '')}.join(', ')}"}
|
|
24
|
+
Aspera.assert(OpenSSL::SSL.const_defined?(name), type: Cli::BadArgument) { "Unknown ssl_option: #{name}, use one of: #{OpenSSL::SSL.constants.grep(/^OP_/).map { |c| c.to_s.sub(/^OP_/, '') }.join(', ')}" }
|
|
25
25
|
if opt.start_with?('-')
|
|
26
26
|
@extra_options &= ~OpenSSL::SSL.const_get(name)
|
|
27
27
|
else
|
data/lib/aspera/sync/database.rb
CHANGED
|
@@ -25,7 +25,7 @@ class SqLite3Wrapper
|
|
|
25
25
|
end
|
|
26
26
|
else
|
|
27
27
|
def execute(sql)
|
|
28
|
-
db = SQLite3::Database.new(@db_path).tap{ |d| d.results_as_hash = true}
|
|
28
|
+
db = SQLite3::Database.new(@db_path).tap { |d| d.results_as_hash = true }
|
|
29
29
|
begin
|
|
30
30
|
normalize_rows(db.execute(sql))
|
|
31
31
|
ensure
|
|
@@ -46,7 +46,7 @@ class SqLite3Wrapper
|
|
|
46
46
|
private
|
|
47
47
|
|
|
48
48
|
def normalize_rows(rows)
|
|
49
|
-
rows.map{ |r| r.transform_keys(&:to_s)}
|
|
49
|
+
rows.map { |r| r.transform_keys(&:to_s) }
|
|
50
50
|
end
|
|
51
51
|
end
|
|
52
52
|
|
|
@@ -90,14 +90,14 @@ module Aspera
|
|
|
90
90
|
def start(sync_info, opt_ts = nil)
|
|
91
91
|
Log.dump(:sync_params_initial, sync_info)
|
|
92
92
|
Aspera.assert_type(sync_info, Hash)
|
|
93
|
-
Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
|
|
93
|
+
Aspera.assert(PARAM_KEYS.any? { |k| sync_info.key?(k) }, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
|
|
94
94
|
exec_spec = ExecSpec.new(exec: :async)
|
|
95
95
|
if sync_info.key?('local')
|
|
96
96
|
# `conf` format
|
|
97
|
-
Aspera.assert_type(sync_info['local'], Hash){'local'}
|
|
97
|
+
Aspera.assert_type(sync_info['local'], Hash) { 'local' }
|
|
98
98
|
remote = sync_info['remote']
|
|
99
|
-
Aspera.assert_type(remote, Hash){'remote'}
|
|
100
|
-
Aspera.assert_type(remote['path'], String){'remote path'}
|
|
99
|
+
Aspera.assert_type(remote, Hash) { 'remote' }
|
|
100
|
+
Aspera.assert_type(remote['path'], String) { 'remote path' }
|
|
101
101
|
# get transfer spec if possible, and feed back to new structure
|
|
102
102
|
if block_given?
|
|
103
103
|
transfer_spec = yield(direction_sym(sync_info), sync_info['local']['path'], remote['path'])
|
|
@@ -125,8 +125,8 @@ module Aspera
|
|
|
125
125
|
Aspera.assert_type(sync_info['sessions'].first, Hash)
|
|
126
126
|
if block_given?
|
|
127
127
|
sync_info['sessions'].each do |session|
|
|
128
|
-
Aspera.assert_type(session['local_dir'], String){'local_dir'}
|
|
129
|
-
Aspera.assert_type(session['remote_dir'], String){'remote_dir'}
|
|
128
|
+
Aspera.assert_type(session['local_dir'], String) { 'local_dir' }
|
|
129
|
+
Aspera.assert_type(session['remote_dir'], String) { 'remote_dir' }
|
|
130
130
|
transfer_spec = yield(direction_sym(session), session['local_dir'], session['remote_dir'])
|
|
131
131
|
Log.dump(:auth_ts, transfer_spec)
|
|
132
132
|
transfer_spec.deep_merge!(opt_ts) unless opt_ts.nil?
|
|
@@ -155,7 +155,7 @@ module Aspera
|
|
|
155
155
|
|
|
156
156
|
# Parse output of asyncadmin
|
|
157
157
|
def parse_status(stdout)
|
|
158
|
-
Log.log.trace1{"stdout=#{stdout}"}
|
|
158
|
+
Log.log.trace1 { "stdout=#{stdout}" }
|
|
159
159
|
result = {}
|
|
160
160
|
ids = nil
|
|
161
161
|
stdout.split("\n").each do |line|
|
|
@@ -175,7 +175,7 @@ module Aspera
|
|
|
175
175
|
# @param sync_info [Hash] sync parameters in conf or args format
|
|
176
176
|
# @return [Hash] parsed output of asyncadmin
|
|
177
177
|
def admin_status(sync_info)
|
|
178
|
-
Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
|
|
178
|
+
Aspera.assert(PARAM_KEYS.any? { |k| sync_info.key?(k) }, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
|
|
179
179
|
arguments = [ASYNC_ADMIN_EXECUTABLE, '--quiet']
|
|
180
180
|
if sync_info.key?('local')
|
|
181
181
|
# `conf` format
|
|
@@ -207,7 +207,7 @@ module Aspera
|
|
|
207
207
|
# @param sync_info [Hash] sync parameters in conf or args format
|
|
208
208
|
# @return [String, nil] Path to "local DB dir", i.e. folder that contains folders that contain `snap.db`
|
|
209
209
|
def local_db_folder(sync_info)
|
|
210
|
-
Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
|
|
210
|
+
Aspera.assert(PARAM_KEYS.any? { |k| sync_info.key?(k) }, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
|
|
211
211
|
if sync_info.key?('local')
|
|
212
212
|
# `conf` format
|
|
213
213
|
if sync_info.key?('local_db_dir')
|
|
@@ -231,7 +231,7 @@ module Aspera
|
|
|
231
231
|
end
|
|
232
232
|
|
|
233
233
|
def session_name(sync_info)
|
|
234
|
-
Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
|
|
234
|
+
Aspera.assert(PARAM_KEYS.any? { |k| sync_info.key?(k) }, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
|
|
235
235
|
if sync_info.key?('local')
|
|
236
236
|
# `conf` format
|
|
237
237
|
return sync_info['name']
|
|
@@ -243,7 +243,7 @@ module Aspera
|
|
|
243
243
|
|
|
244
244
|
def session_db_file(sync_info)
|
|
245
245
|
db_file = File.join(local_db_folder(sync_info), PRIVATE_FOLDER, session_name(sync_info), ASYNC_DB)
|
|
246
|
-
Aspera.assert(File.exist?(db_file)){"Database file #{db_file} does not exist"}
|
|
246
|
+
Aspera.assert(File.exist?(db_file)) { "Database file #{db_file} does not exist" }
|
|
247
247
|
db_file
|
|
248
248
|
end
|
|
249
249
|
|
|
@@ -284,7 +284,7 @@ module Aspera
|
|
|
284
284
|
# @param option [String] Option to search
|
|
285
285
|
# @return [Array,nil] with path/schema for that option
|
|
286
286
|
def find_option(schema, path, option)
|
|
287
|
-
if %w[x-cli-option x-cli-short].any?{ |i| schema[i].eql?(option)}
|
|
287
|
+
if %w[x-cli-option x-cli-short].any? { |i| schema[i].eql?(option) }
|
|
288
288
|
Log.log.debug('Special') if schema['x-cli-special']
|
|
289
289
|
return [path, schema]
|
|
290
290
|
end
|
|
@@ -322,7 +322,7 @@ module Aspera
|
|
|
322
322
|
end
|
|
323
323
|
end
|
|
324
324
|
path, props = find_option(CONF_SCHEMA, [], option)
|
|
325
|
-
Aspera.assert(!path.nil?){"Option not found: #{option}"}
|
|
325
|
+
Aspera.assert(!path.nil?) { "Option not found: #{option}" }
|
|
326
326
|
last_key = path.pop
|
|
327
327
|
# navigate in the current result to insert the value
|
|
328
328
|
current = result
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require 'singleton'
|
|
4
4
|
require 'fileutils'
|
|
5
5
|
require 'etc'
|
|
6
|
+
require 'tempfile'
|
|
6
7
|
|
|
7
8
|
module Aspera
|
|
8
9
|
# create a temp file name for a given folder
|
|
@@ -40,7 +41,7 @@ module Aspera
|
|
|
40
41
|
def delete_file(filepath)
|
|
41
42
|
File.delete(filepath) if @cleanup_on_exit
|
|
42
43
|
rescue => e
|
|
43
|
-
Log.log.warn{"Problem deleting file: #{filepath}: #{e.message}"}
|
|
44
|
+
Log.log.warn { "Problem deleting file: #{filepath}: #{e.message}" }
|
|
44
45
|
end
|
|
45
46
|
|
|
46
47
|
# Call this on process exit
|
|
@@ -51,11 +52,16 @@ module Aspera
|
|
|
51
52
|
@created_files = []
|
|
52
53
|
end
|
|
53
54
|
|
|
54
|
-
# Ensure that provided folder exists, or create it,
|
|
55
|
+
# Ensure that provided folder exists, or create it, atomically create and reserve a unique file.
|
|
56
|
+
# The file is created immediately (O_EXCL) to prevent TOCTOU race conditions.
|
|
55
57
|
# @return [String] path to that unique file
|
|
56
58
|
def new_file_path_in_folder(temp_folder, prefix: nil, suffix: nil)
|
|
57
59
|
FileUtils.mkdir_p(temp_folder)
|
|
58
|
-
|
|
60
|
+
basename = [prefix || 'aspera', SecureRandom.uuid].join('-')
|
|
61
|
+
# Tempfile.new creates the file atomically with O_EXCL; close keeps it on disk
|
|
62
|
+
tmp = Tempfile.new([basename, suffix ? "-#{suffix}" : ''], temp_folder)
|
|
63
|
+
tmp.close
|
|
64
|
+
new_file = tmp.path
|
|
59
65
|
@created_files.push(new_file)
|
|
60
66
|
new_file
|
|
61
67
|
end
|
|
@@ -72,6 +78,20 @@ module Aspera
|
|
|
72
78
|
new_file_path_in_folder(@global_temp, prefix: prefix, suffix: suffix)
|
|
73
79
|
end
|
|
74
80
|
|
|
81
|
+
# Generate a unique directory path inside the global temp folder (does NOT create it).
|
|
82
|
+
# Unlike new_file_path_global, the caller is expected to use the path as a directory.
|
|
83
|
+
# @return [String] path to a unique (not yet existing) directory
|
|
84
|
+
def new_dir_path_global(prefix = nil)
|
|
85
|
+
username =
|
|
86
|
+
begin
|
|
87
|
+
Etc.getlogin || Etc.getpwuid(Process.uid).name || 'unknown_user'
|
|
88
|
+
rescue StandardError
|
|
89
|
+
'unknown_user'
|
|
90
|
+
end
|
|
91
|
+
dir_name = [prefix, username, SecureRandom.uuid].compact.join('-')
|
|
92
|
+
File.join(@global_temp, dir_name)
|
|
93
|
+
end
|
|
94
|
+
|
|
75
95
|
# Garbage collect undeleted files
|
|
76
96
|
def cleanup_expired(temp_folder)
|
|
77
97
|
Dir.entries(temp_folder).each do |name|
|
|
@@ -79,7 +99,7 @@ module Aspera
|
|
|
79
99
|
age_sec = (Time.now - File.stat(file_path).mtime).to_i
|
|
80
100
|
# check age of file, delete too old
|
|
81
101
|
if File.file?(file_path) && (age_sec > FILE_LIST_AGE_MAX_SEC)
|
|
82
|
-
Log.log.debug{"garbage collecting #{name}"}
|
|
102
|
+
Log.log.debug { "garbage collecting #{name}" }
|
|
83
103
|
delete_file(file_path)
|
|
84
104
|
end
|
|
85
105
|
end
|