aspera-cli 4.27.0 → 4.27.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +29 -0
  4. data/CONTRIBUTING.md +1 -7
  5. data/TODO.md +50 -0
  6. data/docs/README.md +708 -488
  7. data/lib/aspera/agent/base.rb +10 -2
  8. data/lib/aspera/agent/connect.rb +7 -7
  9. data/lib/aspera/agent/desktop.rb +3 -3
  10. data/lib/aspera/agent/direct.rb +48 -19
  11. data/lib/aspera/agent/factory.rb +3 -3
  12. data/lib/aspera/agent/httpgw.rb +55 -14
  13. data/lib/aspera/agent/node.rb +2 -2
  14. data/lib/aspera/agent/transferd.rb +12 -12
  15. data/lib/aspera/api/aoc.rb +28 -28
  16. data/lib/aspera/api/cos_node.rb +10 -9
  17. data/lib/aspera/api/faspex.rb +2 -2
  18. data/lib/aspera/api/httpgw.rb +20 -20
  19. data/lib/aspera/api/node.rb +31 -53
  20. data/lib/aspera/ascmd.rb +13 -13
  21. data/lib/aspera/ascp/installation.rb +43 -36
  22. data/lib/aspera/ascp/management.rb +6 -6
  23. data/lib/aspera/assert.rb +12 -13
  24. data/lib/aspera/cli/ascp_actions.rb +7 -7
  25. data/lib/aspera/cli/async_transfer_store.rb +49 -5
  26. data/lib/aspera/cli/bootstrapper.rb +9 -9
  27. data/lib/aspera/cli/command_registry.rb +1 -1
  28. data/lib/aspera/cli/command_spec.rb +6 -3
  29. data/lib/aspera/cli/context.rb +3 -3
  30. data/lib/aspera/cli/extended_value.rb +26 -26
  31. data/lib/aspera/cli/formatter.rb +76 -54
  32. data/lib/aspera/cli/gem_checker.rb +1 -1
  33. data/lib/aspera/cli/hints.rb +2 -2
  34. data/lib/aspera/cli/http.rb +22 -29
  35. data/lib/aspera/cli/mailer.rb +7 -7
  36. data/lib/aspera/cli/mcp_tool.rb +31 -4
  37. data/lib/aspera/cli/option_declarator.rb +77 -0
  38. data/lib/aspera/cli/options.schema.yaml +273 -3
  39. data/lib/aspera/cli/parser.rb +539 -280
  40. data/lib/aspera/cli/plugins/alee.rb +1 -1
  41. data/lib/aspera/cli/plugins/aoc.rb +297 -313
  42. data/lib/aspera/cli/plugins/ats.rb +25 -21
  43. data/lib/aspera/cli/plugins/base.rb +357 -186
  44. data/lib/aspera/cli/plugins/basic_auth.rb +0 -12
  45. data/lib/aspera/cli/plugins/config.rb +76 -73
  46. data/lib/aspera/cli/plugins/console.rb +8 -8
  47. data/lib/aspera/cli/plugins/cos.rb +4 -3
  48. data/lib/aspera/cli/plugins/factory.rb +2 -2
  49. data/lib/aspera/cli/plugins/faspex5.rb +259 -248
  50. data/lib/aspera/cli/plugins/faspio.rb +7 -2
  51. data/lib/aspera/cli/plugins/httpgw.rb +1 -1
  52. data/lib/aspera/cli/plugins/mcp.rb +11 -11
  53. data/lib/aspera/cli/plugins/node.rb +227 -283
  54. data/lib/aspera/cli/plugins/orchestrator.rb +23 -23
  55. data/lib/aspera/cli/plugins/preview.rb +61 -64
  56. data/lib/aspera/cli/plugins/server.rb +36 -31
  57. data/lib/aspera/cli/plugins/shares.rb +174 -135
  58. data/lib/aspera/cli/preset_actions.rb +39 -19
  59. data/lib/aspera/cli/preset_manager.rb +74 -32
  60. data/lib/aspera/cli/result.rb +18 -16
  61. data/lib/aspera/cli/runner.rb +75 -39
  62. data/lib/aspera/cli/secret_finder.rb +1 -1
  63. data/lib/aspera/cli/sync_actions.rb +36 -35
  64. data/lib/aspera/cli/terminal_formatter.rb +2 -2
  65. data/lib/aspera/cli/transfer_actions.rb +17 -10
  66. data/lib/aspera/cli/transfer_agent.rb +44 -44
  67. data/lib/aspera/cli/transfer_progress.rb +7 -7
  68. data/lib/aspera/cli/vault_manager.rb +34 -12
  69. data/lib/aspera/cli/version.rb +1 -1
  70. data/lib/aspera/cli/wizard.rb +17 -20
  71. data/lib/aspera/colors.rb +3 -3
  72. data/lib/aspera/command_line_builder.rb +16 -16
  73. data/lib/aspera/command_line_converter.rb +1 -1
  74. data/lib/aspera/coverage.rb +1 -2
  75. data/lib/aspera/data_repository.rb +1 -1
  76. data/lib/aspera/dot_container.rb +8 -8
  77. data/lib/aspera/environment.rb +12 -12
  78. data/lib/aspera/faspex_gw.rb +4 -4
  79. data/lib/aspera/faspex_postproc.rb +14 -9
  80. data/lib/aspera/graphql.rb +4 -2
  81. data/lib/aspera/hash_ext.rb +2 -2
  82. data/lib/aspera/json_rpc/client.rb +1 -1
  83. data/lib/aspera/keychain/base.rb +10 -3
  84. data/lib/aspera/keychain/encrypted_hash.rb +92 -17
  85. data/lib/aspera/keychain/factory.rb +17 -5
  86. data/lib/aspera/keychain/hashicorp_vault.rb +1 -1
  87. data/lib/aspera/keychain/macos_security.rb +23 -25
  88. data/lib/aspera/keychain/one_password_api.rb +86 -0
  89. data/lib/aspera/keychain/one_password_base.rb +34 -0
  90. data/lib/aspera/keychain/one_password_cli.rb +98 -0
  91. data/lib/aspera/link_header.rb +2 -2
  92. data/lib/aspera/log.rb +13 -13
  93. data/lib/aspera/markdown.rb +5 -5
  94. data/lib/aspera/nagios.rb +8 -8
  95. data/lib/aspera/node_simulator.rb +33 -27
  96. data/lib/aspera/oauth/base.rb +4 -5
  97. data/lib/aspera/oauth/boot.rb +3 -3
  98. data/lib/aspera/oauth/factory.rb +5 -5
  99. data/lib/aspera/oauth/jwt.rb +5 -7
  100. data/lib/aspera/oauth/web.rb +4 -4
  101. data/lib/aspera/persistency_action_once.rb +4 -4
  102. data/lib/aspera/persistency_folder.rb +8 -8
  103. data/lib/aspera/preview/file_types.rb +4 -4
  104. data/lib/aspera/preview/generator.rb +49 -46
  105. data/lib/aspera/preview/options.rb +3 -2
  106. data/lib/aspera/preview/terminal.rb +5 -5
  107. data/lib/aspera/preview/utils.rb +65 -18
  108. data/lib/aspera/products/connect.rb +2 -2
  109. data/lib/aspera/products/desktop.rb +1 -1
  110. data/lib/aspera/products/other.rb +2 -2
  111. data/lib/aspera/products/transferd.rb +3 -3
  112. data/lib/aspera/proxy_auto_config.rb +10 -10
  113. data/lib/aspera/rest.rb +149 -115
  114. data/lib/aspera/rest_error_analyzer.rb +3 -3
  115. data/lib/aspera/rest_list.rb +10 -9
  116. data/lib/aspera/schema/IBM Aspera Node API-4.4.6.yaml +6232 -0
  117. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +249 -0
  118. data/lib/aspera/schema/IBM_Aspera_Shares.yaml +4653 -0
  119. data/lib/aspera/schema/documentation.rb +17 -11
  120. data/lib/aspera/schema/reader.rb +78 -4
  121. data/lib/aspera/schema/registry.rb +26 -7
  122. data/lib/aspera/secret_hider.rb +5 -5
  123. data/lib/aspera/ssh.rb +64 -31
  124. data/lib/aspera/ssl.rb +3 -3
  125. data/lib/aspera/sync/database.rb +2 -2
  126. data/lib/aspera/sync/operations.rb +13 -13
  127. data/lib/aspera/temp_file_manager.rb +24 -4
  128. data/lib/aspera/transfer/faux_file.rb +2 -2
  129. data/lib/aspera/transfer/parameters.rb +15 -15
  130. data/lib/aspera/transfer/resumer.rb +9 -9
  131. data/lib/aspera/transfer/spec.rb +1 -1
  132. data/lib/aspera/uri_reader.rb +52 -15
  133. data/lib/aspera/web_auth.rb +6 -5
  134. data/lib/aspera/web_server_simple.rb +9 -9
  135. data/lib/aspera/yaml.rb +4 -4
  136. data.tar.gz.sig +0 -0
  137. metadata +10 -4
  138. metadata.gz.sig +0 -0
  139. data/lib/aspera/cli/plugins/faspex.rb +0 -576
  140. data/lib/aspera/transfer/uri.rb +0 -56
@@ -28,7 +28,7 @@ module Aspera
28
28
  @agent_columns = agent_columns
29
29
  @code_highlight = code_highlight
30
30
  @columns = %w[name type description]
31
- @columns.insert(-2, *Agent::Factory::ALL.values.map{ |i| i[:short].to_s}.sort) if @agent_columns
31
+ @columns.insert(-2, *Agent::Factory::ALL.values.map { |i| i[:short].to_s }.sort) if @agent_columns
32
32
  # Sections: each entry is {header: row_or_nil, rows: []}
33
33
  # A flat schema produces a single section with no header.
34
34
  @sections = [{header: nil, rows: []}]
@@ -36,7 +36,7 @@ module Aspera
36
36
 
37
37
  def rows
38
38
  @sections.flat_map do |section|
39
- sorted = section[:rows].sort_by{ |i| i['name']}
39
+ sorted = section[:rows].sort_by { |i| i['name'] }
40
40
  section[:header] ? [section[:header]] + sorted : sorted
41
41
  end
42
42
  end
@@ -47,7 +47,7 @@ module Aspera
47
47
  # First row is the titles (for Markdown table generation)
48
48
  # @return [Array<Array<String>>]
49
49
  def table
50
- [@columns] + rows.map{ |row| @columns.map{ |field| row[field]}}
50
+ [@columns] + rows.map { |row| @columns.map { |field| row[field] } }
51
51
  end
52
52
 
53
53
  # Generate a documentation table from a JSON schema for transfer specifications
@@ -58,9 +58,9 @@ module Aspera
58
58
  # @param schema [Reader] The JSON schema to process
59
59
  # @return [Documentation]
60
60
  def build(schema = nil)
61
- code = @code_highlight ? ->(c){"`#{c}`"} : ->(c){c}
61
+ code = @code_highlight ? ->(c) { "`#{c}`" } : ->(c) { c }
62
62
  schema ||= @schema
63
- render_title = ->(title){title.gsub(Markdown::FORMATS){@formatter.markdown_text(Regexp.last_match)}}
63
+ render_title = ->(title) { title.gsub(Markdown::FORMATS) { @formatter.markdown_text(Regexp.last_match) } }
64
64
  on_variant = ->(variant_reader, discriminant_property, discriminant_value) do
65
65
  title = variant_reader.current['title'] || variant_reader.current['description']
66
66
  header =
@@ -80,8 +80,14 @@ module Aspera
80
80
  schema.each_property(on_variant: on_variant) do |property_schema, _name, property_full_name|
81
81
  node = property_schema.current
82
82
  # Manual table
83
- item_type = JSON_TYPE_TO_DOC.fetch(node['type'], node['type'])
84
- item_type = "#{item_type}[#{JSON_TYPE_TO_DOC.fetch(node.dig('items', 'type'), node.dig('items', 'type'))}]" if node['type'].eql?('array') && node.dig('items', 'type').is_a?(String)
83
+ item_type =
84
+ if node['type'].is_a?(Array)
85
+ node['type'].map { |t| JSON_TYPE_TO_DOC.fetch(t, t) }.join(', ')
86
+ elsif node['type'].eql?('array') && node.dig('items', 'type').is_a?(String)
87
+ "#{JSON_TYPE_TO_DOC.fetch(node['type'], node['type'])}[#{JSON_TYPE_TO_DOC.fetch(node.dig('items', 'type'), node.dig('items', 'type'))}]"
88
+ else
89
+ JSON_TYPE_TO_DOC.fetch(node['type'], node['type'])
90
+ end
85
91
  item = {
86
92
  'name' => code.call(property_full_name),
87
93
  'type' => code.call(item_type),
@@ -90,7 +96,7 @@ module Aspera
90
96
  # Render Markdown formatting and split lines
91
97
  item['description'] =
92
98
  node['description']
93
- .gsub(Markdown::FORMATS){@formatter.markdown_text(Regexp.last_match)}
99
+ .gsub(Markdown::FORMATS) { @formatter.markdown_text(Regexp.last_match) }
94
100
  .split("\n") if node.key?('description')
95
101
  item['description'].unshift("DEPRECATED: #{node['x-deprecation']}") if node.key?('x-deprecation')
96
102
  # Add flags for supported agents in doc
@@ -98,17 +104,17 @@ module Aspera
98
104
  Agent::Factory::ALL.each_key do |sym|
99
105
  agents.push(sym) if node['x-agents'].nil? || node['x-agents'].include?(sym.to_s)
100
106
  end
101
- Aspera.assert(agents.include?(:direct)){"#{property_full_name}: x-cli-option requires agent direct (or nil)"} if node['x-cli-option']
107
+ Aspera.assert(agents.include?(:direct)) { "#{property_full_name}: x-cli-option requires agent direct (or nil)" } if node['x-cli-option']
102
108
  if @agent_columns
103
109
  Agent::Factory::ALL.each do |sym, names|
104
110
  item[names[:short].to_s] = @formatter.tick(agents.include?(sym))
105
111
  end
106
112
  else
107
- item['description'].push("(#{agents.map{ |i| Agent::Factory::ALL[i][:short].to_s.upcase}.sort.join(', ')})") unless agents.length.eql?(Agent::Factory::ALL.length)
113
+ item['description'].push("(#{agents.map { |i| Agent::Factory::ALL[i][:short].to_s.upcase }.sort.join(', ')})") unless agents.length.eql?(Agent::Factory::ALL.length)
108
114
  end
109
115
  # Only keep lines that are usable in supported agents
110
116
  next if agents.empty?
111
- item['description'].push("Allowed values: #{node['enum'].map{ |v| @formatter.markdown_text("`#{v}`")}.join(', ')}.") if node.key?('enum')
117
+ item['description'].push("Allowed values: #{node['enum'].map { |v| @formatter.markdown_text("`#{v}`") }.join(', ')}.") if node.key?('enum')
112
118
  item['description'].push("Default: #{@formatter.markdown_text("`#{node['default']}`")}.") if node.key?('default')
113
119
  item['description'].push("Example: #{@formatter.markdown_text("`#{node['example']}`")}.") if node.key?('example')
114
120
  if @include_option
@@ -42,12 +42,12 @@ module Aspera
42
42
  def dig(*path)
43
43
  current = @current
44
44
  path.each do |p|
45
- Aspera.assert(current.key?(p)){"schema: #{p} in #{path}"}
45
+ Aspera.assert(current.key?(p)) { "schema: #{p} in #{path}" }
46
46
  current = current[p]
47
- Aspera.assert_type(current, Hash){'schema'}
47
+ Aspera.assert_type(current, Hash) { 'schema' }
48
48
  if current.key?('$ref')
49
49
  ref = current['$ref']
50
- Aspera.assert(ref.start_with?('#/')){"schema $ref must start with '#/': #{ref}"}
50
+ Aspera.assert(ref.start_with?('#/')) { "schema $ref must start with '#/': #{ref}" }
51
51
  current = @root.dig(*ref[2..].split('/'))
52
52
  end
53
53
  end
@@ -56,7 +56,7 @@ module Aspera
56
56
 
57
57
  # Resolve a $ref string to a Reader
58
58
  def resolve_ref(ref)
59
- Aspera.assert(ref.start_with?('#/')){"schema $ref must start with '#/': #{ref}"}
59
+ Aspera.assert(ref.start_with?('#/')) { "schema $ref must start with '#/': #{ref}" }
60
60
  Reader.new(@root, @root.dig(*ref[2..].split('/')))
61
61
  end
62
62
 
@@ -98,6 +98,15 @@ module Aspera
98
98
  end
99
99
  return
100
100
  end
101
+ if @current.key?('allOf')
102
+ # Merge all branches: each branch contributes its properties (no variants)
103
+ @current['allOf'].each do |branch_node|
104
+ ref = branch_node['$ref']
105
+ branch_reader = ref ? resolve_ref(ref) : Reader.new(@root, branch_node)
106
+ branch_reader.each_property(prefix, on_variant: on_variant, &block)
107
+ end
108
+ return
109
+ end
101
110
  properties = dig('properties')
102
111
  properties.current.each_key do |name|
103
112
  property_full_name = "#{prefix}#{name}"
@@ -117,6 +126,71 @@ module Aspera
117
126
  array_item_schema.each_property("#{property_full_name}[].", on_variant: on_variant, &block) if array_item_schema.current['properties']
118
127
  end
119
128
  end
129
+ # allOf without explicit type: object — recurse to merge all branches
130
+ property_schema.each_property("#{property_full_name}.", on_variant: on_variant, &block) if node['allOf']
131
+ end
132
+ end
133
+
134
+ # Convert this schema to a flat array of field descriptors.
135
+ # Returns raw semantic fields with no ANSI or formatting, suitable for
136
+ # JSON/YAML output or MCP consumption. An AI or script can use the result
137
+ # directly to build a valid payload.
138
+ #
139
+ # Each Hash entry contains:
140
+ # name [String] dot/bracket-path of the field (e.g. "recipients[].name")
141
+ # type [String] JSON type string (e.g. "string", "boolean", "Array[object]")
142
+ # required [Boolean] true when the field is in its immediate parent's required list
143
+ # description [String] human description (may contain Markdown **bold** / `code`)
144
+ # default [Object] (optional) default value as native Ruby type
145
+ # enum [Array] (optional) list of allowed string values
146
+ #
147
+ # @return [Array<Hash>]
148
+ def to_rows
149
+ rows = []
150
+ collect_rows(rows, self, '')
151
+ rows
152
+ end
153
+
154
+ private
155
+
156
+ # Recursively collect rows from a schema node, passing each property's own
157
+ # parent `required` array so that only direct-parent membership is checked.
158
+ def collect_rows(rows, reader, prefix)
159
+ return unless reader.current.key?('properties')
160
+ parent_required = Set.new(Array(reader.current['required']))
161
+ props_reader = reader.dig('properties')
162
+ props_reader.current.each_key do |name|
163
+ prop_reader = props_reader.dig(name)
164
+ node = prop_reader.current
165
+ full_name = "#{prefix}#{name}"
166
+ type_val =
167
+ if node['type'].is_a?(Array)
168
+ node['type'].join(', ')
169
+ elsif node['type'].eql?('array') && node.dig('items', 'type').is_a?(String)
170
+ "Array[#{node.dig('items', 'type')}]"
171
+ else
172
+ node['type'].to_s
173
+ end
174
+ row = {
175
+ 'name' => full_name,
176
+ 'type' => type_val,
177
+ 'required' => parent_required.include?(name),
178
+ 'description' => node['description'].to_s
179
+ }
180
+ row['default'] = node['default'] if node.key?('default')
181
+ row['enum'] = node['enum'] if node.key?('enum')
182
+ rows << row
183
+ # Recurse into nested object or array-of-objects
184
+ case node['type']
185
+ when 'object'
186
+ collect_rows(rows, prop_reader, "#{full_name}.") if node['properties']
187
+ when 'array'
188
+ if node['items']
189
+ item_reader = prop_reader.dig('items')
190
+ collect_rows(rows, item_reader, "#{full_name}[].") if item_reader.current['properties']
191
+ end
192
+ end
193
+ collect_rows(rows, prop_reader, "#{full_name}.") if node['allOf']
120
194
  end
121
195
  end
122
196
  end
@@ -18,21 +18,32 @@ module Aspera
18
18
  LOCATIONS.key?(sym)
19
19
  end
20
20
 
21
+ # Split a component string into registry key and optional path prefix.
22
+ # Syntax: 'key' or 'key+/prefix' (e.g. 'shares+/api/v1')
23
+ # @param component [String] registry key with optional '+/prefix'
24
+ # @return [Array(String, String)] [key, prefix] where prefix may be ''
25
+ def split_component(component)
26
+ key, prefix = component.split('+', 2)
27
+ [key, prefix || '']
28
+ end
29
+
21
30
  # Get path to request body schema, no check if it exists
22
- # @param component [String] registry key (e.g. 'faspex', 'aoc')
23
- # @param endpoint [String] endpoint path without leading slash (e.g. 'packages.post')
31
+ # @param component [String] registry key, optionally with path prefix (e.g. 'shares+/api/v1')
32
+ # @param endpoint [String] endpoint path without leading slash (e.g. 'data/shares.post')
24
33
  # @return [String] schema path usable in schema: keyword
25
34
  def req_body(component, endpoint)
26
- "#{component}:paths./#{endpoint}.requestBody.content.application/json.schema"
35
+ key, prefix = split_component(component)
36
+ "#{key}:paths.#{prefix}/#{endpoint}.requestBody.content.application/json.schema"
27
37
  end
28
38
 
29
39
  # Get path to query parameters for a GET endpoint
30
- # @param component [String] registry key (e.g. 'faspex', 'aoc')
31
- # @param endpoint [String] resource path without leading slash (e.g. 'packages')
40
+ # @param component [String] registry key, optionally with path prefix (e.g. 'shares+/api/v1')
41
+ # @param endpoint [String] resource path without leading slash (e.g. 'data/shares')
32
42
  # @param method [String] HTTP method (default: 'get')
33
43
  # @return [String] schema path usable in query_schema: keyword
34
44
  def query_params(component, endpoint, method: 'get')
35
- "#{component}:paths./#{endpoint}.#{method}#{QUERY_PARAMS_SUFFIX}"
45
+ key, prefix = split_component(component)
46
+ "#{key}:paths.#{prefix}/#{endpoint}.#{method}#{QUERY_PARAMS_SUFFIX}"
36
47
  end
37
48
  end
38
49
 
@@ -43,6 +54,9 @@ module Aspera
43
54
  opts: 'aspera/cli/options.schema.yaml',
44
55
  aoc: 'aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml',
45
56
  faspex: 'aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml',
57
+ faspio: 'aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml',
58
+ node: 'aspera/schema/IBM Aspera Node API-4.4.6.yaml',
59
+ shares: 'aspera/schema/IBM_Aspera_Shares.yaml',
46
60
  async_tables: 'aspera/schema/async_tables.yaml'
47
61
  }
48
62
 
@@ -52,6 +66,9 @@ module Aspera
52
66
  SYNC_ARGS = 'args'
53
67
  AOC = 'aoc'
54
68
  FASPEX = 'faspex'
69
+ FASPIO = 'faspio'
70
+ NODE = 'node'
71
+ SHARES = 'shares+/api/v1'
55
72
  ASYNC_TABLES = 'async_tables'
56
73
  LOG_OPTIONS = "#{OPTIONS}:components.schemas.LogOptions"
57
74
  DIRECT_AGENT_OPTIONS = "#{OPTIONS}:components.schemas.DirectAgentOptions"
@@ -62,7 +79,9 @@ module Aspera
62
79
  SMTP_OPTIONS = "#{OPTIONS}:components.schemas.SmtpOptions"
63
80
  HTTP_OPTIONS = "#{OPTIONS}:components.schemas.HttpOptions"
64
81
  VAULT_OPTIONS = "#{OPTIONS}:components.schemas.VaultOptions"
82
+ VAULT_SECRET = "#{OPTIONS}:components.schemas.VaultSecret"
65
83
  IMAGE_OPTIONS = "#{OPTIONS}:components.schemas.ImageOptions"
84
+ OUT_OPTIONS = "#{OPTIONS}:components.schemas.OutOptions"
66
85
  PACKAGE_FOLDER_OPTIONS = "#{OPTIONS}:components.schemas.PackageFolderOptions"
67
86
 
68
87
  REQ_BODY = '.requestBody.content.application/json.schema'
@@ -84,7 +103,7 @@ module Aspera
84
103
  def reader(name_path)
85
104
  name, path = name_path.split(':', 2)
86
105
  sym = name.to_sym
87
- Aspera.assert(Registry.known?(sym)){"schema: #{sym}"}
106
+ Aspera.assert(Registry.known?(sym)) { "schema: #{sym}" }
88
107
  spec_file = File.join(@main_folder, LOCATIONS[sym])
89
108
  @cache[sym] = Yaml.safe_load(File.read(spec_file)) if spec_file.end_with?('.yaml') && !@cache.key?(sym)
90
109
  @cache[sym] = JSON.parse(File.read(spec_file)) if spec_file.end_with?('.json') && !@cache.key?(sym)
@@ -52,7 +52,7 @@ module Aspera
52
52
  return lambda do |severity, date_time, program_name, msg|
53
53
  if msg.is_a?(String) && !@log_secrets
54
54
  REGEX_LOG_REPLACES.each do |reg_ex|
55
- msg = msg.gsub(reg_ex){"#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}"}
55
+ msg = msg.gsub(reg_ex) { "#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}" }
56
56
  end
57
57
  end
58
58
  original_formatter.call(severity, date_time, program_name, msg)
@@ -63,7 +63,7 @@ module Aspera
63
63
  # @param value [String] Input string possibly containing a private key
64
64
  # @return [String] String with private key replaced by placeholder
65
65
  def hide_secrets_in_string(value)
66
- return value.gsub(REGEX_LOG_REPLACES.first){"#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}"}
66
+ return value.gsub(REGEX_LOG_REPLACES.first) { "#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}" }
67
67
  end
68
68
 
69
69
  # @param keyword [String, Symbol] Key name to check
@@ -74,17 +74,17 @@ module Aspera
74
74
  # only Strings can be secrets, not booleans, or hash, arrays
75
75
  return false unless keyword.is_a?(String) && value.is_a?(String)
76
76
  # those are not secrets
77
- return false if KEY_FALSE_POSITIVES.any?{ |f| f.match?(keyword)}
77
+ return false if KEY_FALSE_POSITIVES.any? { |f| f.match?(keyword) }
78
78
  return true if ADDITIONAL_KEYS_TO_HIDE.include?(keyword)
79
79
  # check if keyword (name) contains an element that designate it as a secret
80
- ALL_SECRETS.any?{ |kw| keyword.include?(kw)}
80
+ ALL_SECRETS.any? { |kw| keyword.include?(kw) }
81
81
  end
82
82
 
83
83
  # Hides recursively secrets in Hash or Array of Hash
84
84
  def deep_remove_secret(obj)
85
85
  case obj
86
86
  when Array
87
- obj.each{ |i| deep_remove_secret(i)}
87
+ obj.each { |i| deep_remove_secret(i) }
88
88
  when Hash
89
89
  obj.each do |k, v|
90
90
  if secret?(k, v)
data/lib/aspera/ssh.rb CHANGED
@@ -8,77 +8,110 @@ module Aspera
8
8
  # A simple wrapper around Net::SSH
9
9
  # executes one command and get its result from stdout
10
10
  class Ssh
11
+ # Raised when the remote command fails or cannot be started
11
12
  class Error < Aspera::Error
12
13
  end
14
+ # Regexp matching ecdsa/ecdh-sha2 algorithm names (JRuby workaround)
15
+ EXCLUDE_ECDSHA2 = /^ecd(sa|h)-sha2/
16
+
13
17
  class << self
14
- # HACK: disable some key type
18
+ # Removes ed25519 keys from the default identity list used by Net::SSH.
19
+ # Called when the `ed25519` gem is absent or explicitly disabled via
20
+ # `ASCLI_ENABLE_ED25519=false`.
21
+ # @return [void]
15
22
  def disable_ed25519_keys
16
23
  Log.log.debug('Disabling SSH ed25519 user keys')
17
24
  old_verbose = $VERBOSE
18
25
  $VERBOSE = nil
19
26
  Net::SSH::Authentication::Session.class_eval do
20
27
  define_method(:default_keys) do
21
- %w[.ssh .ssh2].product(%w[rsa dsa ecdsa]).map{"~/#{_1}/id_#{_2}"}.freeze
28
+ %w[.ssh .ssh2].product(%w[rsa dsa ecdsa]).map { "~/#{_1}/id_#{_2}" }.freeze
22
29
  end
23
30
  private(:default_keys)
24
- end rescue nil
31
+ end
25
32
  $VERBOSE = old_verbose
26
33
  end
27
34
 
28
- # HACK: disable some algorithms
35
+ # Returns Net::SSH option overrides that exclude all ecdsa/ecdh-sha2 algorithms.
36
+ # Merge the result into `ssh_options` passed to {Ssh#initialize} to avoid
37
+ # mutating Net::SSH internal constants.
38
+ # @return [Hash{Symbol => Array<String>}] `:host_key` and `:kex` filtered lists
39
+ def no_ecd_sha2_options
40
+ Log.log.debug('Building SSH options without ecdsa/ecdh-sha2')
41
+ {
42
+ host_key: Net::SSH::Transport::Algorithms::ALGORITHMS[:host_key].reject { |a| a.match?(EXCLUDE_ECDSHA2) },
43
+ kex: Net::SSH::Transport::Algorithms::ALGORITHMS[:kex].reject { |a| a.match?(EXCLUDE_ECDSHA2) }
44
+ }
45
+ end
46
+
47
+ # Mutates Net::SSH internal algorithm lists to remove ecdsa/ecdh-sha2 entries globally.
48
+ # Kept for backwards compatibility and JRuby usage; prefer {no_ecd_sha2_options} for new code.
49
+ # @return [void]
29
50
  def disable_ecd_sha2_algorithms
30
- Log.log.debug('Disabling SSH ecdsa')
31
- Net::SSH::Transport::Algorithms::ALGORITHMS.each_value{ |a| a.reject!{ |a| a =~ /^ecd(sa|h)-sha2/}}
32
- Net::SSH::KnownHosts::SUPPORTED_TYPE.reject!{ |t| t =~ /^ecd(sa|h)-sha2/}
51
+ Log.log.debug('Disabling SSH ecdsa (global)')
52
+ Net::SSH::Transport::Algorithms::ALGORITHMS.each_value { |a| a.reject! { |a| a.match?(EXCLUDE_ECDSHA2) } }
53
+ Net::SSH::KnownHosts::SUPPORTED_TYPE.reject! { |t| t.match?(EXCLUDE_ECDSHA2) }
33
54
  end
34
55
  end
35
- # ssh_options: same as Net::SSH.start
36
- # see: https://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start
56
+ # @param host [String] remote server address
57
+ # @param username [String] SSH user name
58
+ # @param ssh_options [Hash{Symbol => Object}] options forwarded to `Net::SSH.start`
59
+ # (see {https://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start Net::SSH.start}).
60
+ # Defaults are injected for `:logger`, `:verbose`, and `:use_agent` if absent.
37
61
  def initialize(host, username, ssh_options)
38
62
  Aspera.assert_type(host, String)
39
63
  Aspera.assert_type(username, String)
40
64
  Aspera.assert_type(ssh_options, Hash)
65
+ Aspera.assert_hash_all(ssh_options, Symbol, nil)
41
66
  @host = host
42
67
  @username = username
43
68
  @ssh_options = ssh_options.dup
44
69
  @ssh_options[:logger] = Log.log unless @ssh_options.key?(:logger)
45
70
  @ssh_options[:verbose] = :warn unless @ssh_options.key?(:verbose)
71
+ # @ssh_options[:verbose] = :debug
46
72
  @ssh_options[:use_agent] = false unless @ssh_options.key?(:use_agent)
47
- Log.log.debug{"ssh:#{@username}@#{@host}"}
73
+ Log.log.debug { "ssh:#{@username}@#{@host}" }
48
74
  Log.dump(:ssh_options, @ssh_options)
49
75
  end
50
76
 
51
- # Anything on stderr raises an exception
52
- def execute(cmd, input: nil, exception: false)
77
+ # Executes a single command on the remote host over a new SSH session.
78
+ # @param cmd [String] shell command to execute remotely
79
+ # @param input [String, nil] data written to the command's stdin, or `nil`
80
+ # @return [String] concatenated stdout of the remote command
81
+ # @raise [Error] if the channel cannot be opened or the remote command exits with a non-zero status
82
+ def execute(cmd, input: nil)
53
83
  Aspera.assert_type(cmd, String)
54
- Log.log.debug{"cmd=#{cmd}"}
84
+ Log.log.debug { "cmd=#{cmd}" }
85
+ # @type response [Array<String>]
55
86
  response = []
87
+ # @type error [Array<String>]
56
88
  error = []
89
+ exit_code = nil
90
+ # @param session [Net::SSH::Connection::Session]
57
91
  Net::SSH.start(@host, @username, @ssh_options) do |session|
58
- ssh_channel = session.open_channel do |channel|
59
- # prepare stdout processing
60
- channel.on_data{ |_chan, data| response.push(data)}
61
- # prepare stderr processing, stderr if type = 1
62
- channel.on_extended_data do |_chan, _type, data|
63
- error.push(data)
64
- end
65
- channel.on_request('exit-status') do |_ch, data|
92
+ # @param channel [Net::SSH::Connection::Channel]
93
+ session.open_channel do |channel|
94
+ # Register stdout/stderr before exec so no data is missed (e.g. ForceCommand errors)
95
+ channel.on_data { |_ch, data| response.push(data) }
96
+ channel.on_extended_data { |_ch, type, data| error.push(data) if type.eql?(1) }
97
+ # @param data [Net::SSH::Buffer]
98
+ channel.on_request('exit-status') do |_channel, data|
99
+ Log.dump(:data, data, level: :trace1)
66
100
  exit_code = data.read_long
67
- next if exit_code.zero?
68
- error_message = "#{cmd}: exit #{exit_code}, #{error.join.chomp}"
69
- raise Error, error_message if exception
70
- # Happens when windows user hasn't logged in and created home account.
71
- error_message += "\nHint: home not created in Windows?" if data.include?('Could not chdir to home directory')
72
- Log.log.debug(error_message)
73
101
  end
74
102
  # send command to SSH channel (execute) cspell: disable-next-line
75
- channel.send('cexe'.reverse, cmd){ |_ch, _success| channel.send_data(input) unless input.nil?}
103
+ channel.send('cexe'.reverse, cmd) do |_ch, success|
104
+ raise Error, "could not execute command: #{cmd}" unless success
105
+ channel.send_data(input) unless input.nil?
106
+ end
76
107
  end
77
- # wait for channel to finish (command exit)
78
- ssh_channel.wait
79
- # main SSH session loop
108
+ # wait for channel to finish and flush the session
80
109
  session.loop
81
110
  end
111
+ error_text = error.join
112
+ hint = error_text.include?('Could not chdir to home directory') ? "\nHint: home not created in Windows?" : ''
113
+ raise Error, "#{cmd}: exit #{exit_code}, #{error_text.chomp}#{hint}" if exit_code&.nonzero?
114
+ Log.log.error { "#{error_text}#{hint}" } unless error_text.empty?
82
115
  # response as single string
83
116
  return response.join
84
117
  end
data/lib/aspera/ssl.rb CHANGED
@@ -13,15 +13,15 @@ module Aspera
13
13
  attr_reader :extra_options
14
14
 
15
15
  def option_list=(options)
16
- Aspera.assert_type(options, Array){'ssl_options'}
16
+ Aspera.assert_type(options, Array) { 'ssl_options' }
17
17
  options.each do |opt|
18
- Aspera.assert_type(opt, String, Integer){'Expected String or Integer in ssl_options'}
18
+ Aspera.assert_type(opt, String, Integer) { 'Expected String or Integer in ssl_options' }
19
19
  case opt
20
20
  when Integer
21
21
  @extra_options = opt
22
22
  when String
23
23
  name = "OP_#{opt.start_with?('-') ? opt[1..] : opt}".upcase
24
- Aspera.assert(OpenSSL::SSL.const_defined?(name), type: Cli::BadArgument){"Unknown ssl_option: #{name}, use one of: #{OpenSSL::SSL.constants.grep(/^OP_/).map{ |c| c.to_s.sub(/^OP_/, '')}.join(', ')}"}
24
+ Aspera.assert(OpenSSL::SSL.const_defined?(name), type: Cli::BadArgument) { "Unknown ssl_option: #{name}, use one of: #{OpenSSL::SSL.constants.grep(/^OP_/).map { |c| c.to_s.sub(/^OP_/, '') }.join(', ')}" }
25
25
  if opt.start_with?('-')
26
26
  @extra_options &= ~OpenSSL::SSL.const_get(name)
27
27
  else
@@ -25,7 +25,7 @@ class SqLite3Wrapper
25
25
  end
26
26
  else
27
27
  def execute(sql)
28
- db = SQLite3::Database.new(@db_path).tap{ |d| d.results_as_hash = true}
28
+ db = SQLite3::Database.new(@db_path).tap { |d| d.results_as_hash = true }
29
29
  begin
30
30
  normalize_rows(db.execute(sql))
31
31
  ensure
@@ -46,7 +46,7 @@ class SqLite3Wrapper
46
46
  private
47
47
 
48
48
  def normalize_rows(rows)
49
- rows.map{ |r| r.transform_keys(&:to_s)}
49
+ rows.map { |r| r.transform_keys(&:to_s) }
50
50
  end
51
51
  end
52
52
 
@@ -90,14 +90,14 @@ module Aspera
90
90
  def start(sync_info, opt_ts = nil)
91
91
  Log.dump(:sync_params_initial, sync_info)
92
92
  Aspera.assert_type(sync_info, Hash)
93
- Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
93
+ Aspera.assert(PARAM_KEYS.any? { |k| sync_info.key?(k) }, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
94
94
  exec_spec = ExecSpec.new(exec: :async)
95
95
  if sync_info.key?('local')
96
96
  # `conf` format
97
- Aspera.assert_type(sync_info['local'], Hash){'local'}
97
+ Aspera.assert_type(sync_info['local'], Hash) { 'local' }
98
98
  remote = sync_info['remote']
99
- Aspera.assert_type(remote, Hash){'remote'}
100
- Aspera.assert_type(remote['path'], String){'remote path'}
99
+ Aspera.assert_type(remote, Hash) { 'remote' }
100
+ Aspera.assert_type(remote['path'], String) { 'remote path' }
101
101
  # get transfer spec if possible, and feed back to new structure
102
102
  if block_given?
103
103
  transfer_spec = yield(direction_sym(sync_info), sync_info['local']['path'], remote['path'])
@@ -125,8 +125,8 @@ module Aspera
125
125
  Aspera.assert_type(sync_info['sessions'].first, Hash)
126
126
  if block_given?
127
127
  sync_info['sessions'].each do |session|
128
- Aspera.assert_type(session['local_dir'], String){'local_dir'}
129
- Aspera.assert_type(session['remote_dir'], String){'remote_dir'}
128
+ Aspera.assert_type(session['local_dir'], String) { 'local_dir' }
129
+ Aspera.assert_type(session['remote_dir'], String) { 'remote_dir' }
130
130
  transfer_spec = yield(direction_sym(session), session['local_dir'], session['remote_dir'])
131
131
  Log.dump(:auth_ts, transfer_spec)
132
132
  transfer_spec.deep_merge!(opt_ts) unless opt_ts.nil?
@@ -155,7 +155,7 @@ module Aspera
155
155
 
156
156
  # Parse output of asyncadmin
157
157
  def parse_status(stdout)
158
- Log.log.trace1{"stdout=#{stdout}"}
158
+ Log.log.trace1 { "stdout=#{stdout}" }
159
159
  result = {}
160
160
  ids = nil
161
161
  stdout.split("\n").each do |line|
@@ -175,7 +175,7 @@ module Aspera
175
175
  # @param sync_info [Hash] sync parameters in conf or args format
176
176
  # @return [Hash] parsed output of asyncadmin
177
177
  def admin_status(sync_info)
178
- Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
178
+ Aspera.assert(PARAM_KEYS.any? { |k| sync_info.key?(k) }, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
179
179
  arguments = [ASYNC_ADMIN_EXECUTABLE, '--quiet']
180
180
  if sync_info.key?('local')
181
181
  # `conf` format
@@ -207,7 +207,7 @@ module Aspera
207
207
  # @param sync_info [Hash] sync parameters in conf or args format
208
208
  # @return [String, nil] Path to "local DB dir", i.e. folder that contains folders that contain `snap.db`
209
209
  def local_db_folder(sync_info)
210
- Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
210
+ Aspera.assert(PARAM_KEYS.any? { |k| sync_info.key?(k) }, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
211
211
  if sync_info.key?('local')
212
212
  # `conf` format
213
213
  if sync_info.key?('local_db_dir')
@@ -231,7 +231,7 @@ module Aspera
231
231
  end
232
232
 
233
233
  def session_name(sync_info)
234
- Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
234
+ Aspera.assert(PARAM_KEYS.any? { |k| sync_info.key?(k) }, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
235
235
  if sync_info.key?('local')
236
236
  # `conf` format
237
237
  return sync_info['name']
@@ -243,7 +243,7 @@ module Aspera
243
243
 
244
244
  def session_db_file(sync_info)
245
245
  db_file = File.join(local_db_folder(sync_info), PRIVATE_FOLDER, session_name(sync_info), ASYNC_DB)
246
- Aspera.assert(File.exist?(db_file)){"Database file #{db_file} does not exist"}
246
+ Aspera.assert(File.exist?(db_file)) { "Database file #{db_file} does not exist" }
247
247
  db_file
248
248
  end
249
249
 
@@ -284,7 +284,7 @@ module Aspera
284
284
  # @param option [String] Option to search
285
285
  # @return [Array,nil] with path/schema for that option
286
286
  def find_option(schema, path, option)
287
- if %w[x-cli-option x-cli-short].any?{ |i| schema[i].eql?(option)}
287
+ if %w[x-cli-option x-cli-short].any? { |i| schema[i].eql?(option) }
288
288
  Log.log.debug('Special') if schema['x-cli-special']
289
289
  return [path, schema]
290
290
  end
@@ -322,7 +322,7 @@ module Aspera
322
322
  end
323
323
  end
324
324
  path, props = find_option(CONF_SCHEMA, [], option)
325
- Aspera.assert(!path.nil?){"Option not found: #{option}"}
325
+ Aspera.assert(!path.nil?) { "Option not found: #{option}" }
326
326
  last_key = path.pop
327
327
  # navigate in the current result to insert the value
328
328
  current = result
@@ -3,6 +3,7 @@
3
3
  require 'singleton'
4
4
  require 'fileutils'
5
5
  require 'etc'
6
+ require 'tempfile'
6
7
 
7
8
  module Aspera
8
9
  # create a temp file name for a given folder
@@ -40,7 +41,7 @@ module Aspera
40
41
  def delete_file(filepath)
41
42
  File.delete(filepath) if @cleanup_on_exit
42
43
  rescue => e
43
- Log.log.warn{"Problem deleting file: #{filepath}: #{e.message}"}
44
+ Log.log.warn { "Problem deleting file: #{filepath}: #{e.message}" }
44
45
  end
45
46
 
46
47
  # Call this on process exit
@@ -51,11 +52,16 @@ module Aspera
51
52
  @created_files = []
52
53
  end
53
54
 
54
- # Ensure that provided folder exists, or create it, generate a unique filename
55
+ # Ensure that provided folder exists, or create it, atomically create and reserve a unique file.
56
+ # The file is created immediately (O_EXCL) to prevent TOCTOU race conditions.
55
57
  # @return [String] path to that unique file
56
58
  def new_file_path_in_folder(temp_folder, prefix: nil, suffix: nil)
57
59
  FileUtils.mkdir_p(temp_folder)
58
- new_file = File.join(temp_folder, [prefix, SecureRandom.uuid, suffix].compact.join('-'))
60
+ basename = [prefix || 'aspera', SecureRandom.uuid].join('-')
61
+ # Tempfile.new creates the file atomically with O_EXCL; close keeps it on disk
62
+ tmp = Tempfile.new([basename, suffix ? "-#{suffix}" : ''], temp_folder)
63
+ tmp.close
64
+ new_file = tmp.path
59
65
  @created_files.push(new_file)
60
66
  new_file
61
67
  end
@@ -72,6 +78,20 @@ module Aspera
72
78
  new_file_path_in_folder(@global_temp, prefix: prefix, suffix: suffix)
73
79
  end
74
80
 
81
+ # Generate a unique directory path inside the global temp folder (does NOT create it).
82
+ # Unlike new_file_path_global, the caller is expected to use the path as a directory.
83
+ # @return [String] path to a unique (not yet existing) directory
84
+ def new_dir_path_global(prefix = nil)
85
+ username =
86
+ begin
87
+ Etc.getlogin || Etc.getpwuid(Process.uid).name || 'unknown_user'
88
+ rescue StandardError
89
+ 'unknown_user'
90
+ end
91
+ dir_name = [prefix, username, SecureRandom.uuid].compact.join('-')
92
+ File.join(@global_temp, dir_name)
93
+ end
94
+
75
95
  # Garbage collect undeleted files
76
96
  def cleanup_expired(temp_folder)
77
97
  Dir.entries(temp_folder).each do |name|
@@ -79,7 +99,7 @@ module Aspera
79
99
  age_sec = (Time.now - File.stat(file_path).mtime).to_i
80
100
  # check age of file, delete too old
81
101
  if File.file?(file_path) && (age_sec > FILE_LIST_AGE_MAX_SEC)
82
- Log.log.debug{"garbage collecting #{name}"}
102
+ Log.log.debug { "garbage collecting #{name}" }
83
103
  delete_file(file_path)
84
104
  end
85
105
  end