aspera-cli 4.26.2 → 4.27.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +310 -21
  4. data/CONTRIBUTING.md +99 -14
  5. data/TODO.md +50 -0
  6. data/bin/ascli +3 -3
  7. data/docs/README.md +10977 -0
  8. data/docs/test-mcp-with-ai.md +202 -0
  9. data/lib/aspera/agent/base.rb +29 -7
  10. data/lib/aspera/agent/connect.rb +58 -21
  11. data/lib/aspera/agent/desktop.rb +53 -18
  12. data/lib/aspera/agent/direct.rb +71 -40
  13. data/lib/aspera/agent/factory.rb +3 -3
  14. data/lib/aspera/agent/httpgw.rb +55 -14
  15. data/lib/aspera/agent/node.rb +34 -2
  16. data/lib/aspera/agent/transferd.rb +48 -12
  17. data/lib/aspera/api/alee.rb +1 -1
  18. data/lib/aspera/api/aoc.rb +42 -44
  19. data/lib/aspera/api/cos_node.rb +10 -9
  20. data/lib/aspera/api/faspex.rb +16 -15
  21. data/lib/aspera/api/httpgw.rb +23 -23
  22. data/lib/aspera/api/node.rb +51 -70
  23. data/lib/aspera/api/queries/bss_subscription_account.graphql +18 -0
  24. data/lib/aspera/api/queries/bss_subscription_usage.graphql +18 -0
  25. data/lib/aspera/ascmd.rb +32 -30
  26. data/lib/aspera/ascp/installation.rb +75 -69
  27. data/lib/aspera/ascp/management.rb +7 -8
  28. data/lib/aspera/assert.rb +12 -13
  29. data/lib/aspera/cli/ascp_actions.rb +155 -0
  30. data/lib/aspera/cli/async_transfer_store.rb +125 -0
  31. data/lib/aspera/cli/bootstrapper.rb +197 -0
  32. data/lib/aspera/cli/command_registry.rb +145 -0
  33. data/lib/aspera/cli/command_spec.rb +141 -0
  34. data/lib/aspera/cli/context.rb +19 -6
  35. data/lib/aspera/cli/error.rb +16 -0
  36. data/lib/aspera/cli/extended_value.rb +27 -26
  37. data/lib/aspera/cli/formatter.rb +95 -59
  38. data/lib/aspera/cli/gem_checker.rb +65 -0
  39. data/lib/aspera/cli/hints.rb +3 -3
  40. data/lib/aspera/cli/http.rb +74 -30
  41. data/lib/aspera/cli/info.rb +2 -0
  42. data/lib/aspera/cli/mailer.rb +97 -0
  43. data/lib/aspera/cli/mcp_tool.rb +225 -0
  44. data/lib/aspera/cli/option_declarator.rb +77 -0
  45. data/lib/aspera/cli/options.schema.yaml +634 -7
  46. data/lib/aspera/cli/parser.rb +1217 -0
  47. data/lib/aspera/cli/plugins/alee.rb +20 -22
  48. data/lib/aspera/cli/plugins/aoc.rb +1178 -881
  49. data/lib/aspera/cli/plugins/ats.rb +205 -162
  50. data/lib/aspera/cli/plugins/base.rb +703 -175
  51. data/lib/aspera/cli/plugins/basic_auth.rb +7 -9
  52. data/lib/aspera/cli/plugins/config.rb +434 -757
  53. data/lib/aspera/cli/plugins/console.rb +107 -65
  54. data/lib/aspera/cli/plugins/cos.rb +46 -33
  55. data/lib/aspera/cli/plugins/factory.rb +6 -6
  56. data/lib/aspera/cli/plugins/faspex5.rb +627 -388
  57. data/lib/aspera/cli/plugins/faspio.rb +54 -51
  58. data/lib/aspera/cli/plugins/httpgw.rb +18 -25
  59. data/lib/aspera/cli/plugins/mcp.rb +279 -0
  60. data/lib/aspera/cli/plugins/node.rb +988 -840
  61. data/lib/aspera/cli/plugins/oauth.rb +7 -10
  62. data/lib/aspera/cli/plugins/orchestrator.rb +112 -135
  63. data/lib/aspera/cli/plugins/preview.rb +195 -157
  64. data/lib/aspera/cli/plugins/server.rb +141 -91
  65. data/lib/aspera/cli/plugins/shares.rb +343 -110
  66. data/lib/aspera/cli/preset_actions.rb +159 -0
  67. data/lib/aspera/cli/preset_manager.rb +82 -39
  68. data/lib/aspera/cli/result.rb +78 -26
  69. data/lib/aspera/cli/runner.rb +323 -153
  70. data/lib/aspera/cli/secret_finder.rb +40 -0
  71. data/lib/aspera/cli/special_values.rb +1 -0
  72. data/lib/aspera/cli/sync_actions.rb +89 -64
  73. data/lib/aspera/cli/terminal_formatter.rb +3 -3
  74. data/lib/aspera/cli/transfer_actions.rb +90 -0
  75. data/lib/aspera/cli/transfer_agent.rb +129 -64
  76. data/lib/aspera/cli/transfer_progress.rb +9 -9
  77. data/lib/aspera/cli/vault_manager.rb +79 -0
  78. data/lib/aspera/cli/version.rb +1 -1
  79. data/lib/aspera/cli/wizard.rb +28 -30
  80. data/lib/aspera/colors.rb +3 -3
  81. data/lib/aspera/command_line_builder.rb +34 -34
  82. data/lib/aspera/command_line_converter.rb +1 -1
  83. data/lib/aspera/coverage.rb +1 -2
  84. data/lib/aspera/data_repository.rb +2 -1
  85. data/lib/aspera/dot_container.rb +12 -12
  86. data/lib/aspera/environment.rb +30 -23
  87. data/lib/aspera/exec_spec.rb +13 -0
  88. data/lib/aspera/faspex_gw.rb +5 -5
  89. data/lib/aspera/faspex_postproc.rb +16 -10
  90. data/lib/aspera/graphql.rb +37 -0
  91. data/lib/aspera/hash_ext.rb +8 -2
  92. data/lib/aspera/json_rpc/client.rb +62 -0
  93. data/lib/aspera/json_rpc/version.rb +7 -0
  94. data/lib/aspera/keychain/base.rb +10 -3
  95. data/lib/aspera/keychain/encrypted_hash.rb +92 -17
  96. data/lib/aspera/keychain/factory.rb +20 -8
  97. data/lib/aspera/keychain/hashicorp_vault.rb +1 -1
  98. data/lib/aspera/keychain/macos_security.rb +23 -25
  99. data/lib/aspera/keychain/one_password_api.rb +86 -0
  100. data/lib/aspera/keychain/one_password_base.rb +34 -0
  101. data/lib/aspera/keychain/one_password_cli.rb +98 -0
  102. data/lib/aspera/link_header.rb +82 -0
  103. data/lib/aspera/log.rb +29 -15
  104. data/lib/aspera/markdown.rb +90 -7
  105. data/lib/aspera/nagios.rb +8 -8
  106. data/lib/aspera/node_simulator.rb +35 -27
  107. data/lib/aspera/oauth/base.rb +9 -10
  108. data/lib/aspera/oauth/boot.rb +5 -5
  109. data/lib/aspera/oauth/factory.rb +12 -10
  110. data/lib/aspera/oauth/jwt.rb +9 -11
  111. data/lib/aspera/oauth/web.rb +6 -6
  112. data/lib/aspera/persistency_action_once.rb +12 -12
  113. data/lib/aspera/persistency_folder.rb +18 -11
  114. data/lib/aspera/preview/file_types.rb +5 -5
  115. data/lib/aspera/preview/generator.rb +52 -49
  116. data/lib/aspera/preview/options.rb +3 -2
  117. data/lib/aspera/preview/terminal.rb +5 -5
  118. data/lib/aspera/preview/utils.rb +66 -19
  119. data/lib/aspera/products/connect.rb +2 -2
  120. data/lib/aspera/products/desktop.rb +1 -1
  121. data/lib/aspera/products/other.rb +3 -3
  122. data/lib/aspera/products/transferd.rb +4 -4
  123. data/lib/aspera/proxy_auto_config.rb +15 -14
  124. data/lib/aspera/rest.rb +159 -151
  125. data/lib/aspera/rest_error_analyzer.rb +6 -6
  126. data/lib/aspera/rest_errors_aspera.rb +0 -10
  127. data/lib/aspera/rest_list.rb +11 -10
  128. data/lib/aspera/schema/IBM Aspera Node API-4.4.6.yaml +6232 -0
  129. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +249 -0
  130. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +1853 -510
  131. data/lib/aspera/schema/IBM_Aspera_Shares.yaml +4653 -0
  132. data/lib/aspera/schema/documentation.rb +65 -28
  133. data/lib/aspera/schema/reader.rb +136 -13
  134. data/lib/aspera/schema/registry.rb +60 -8
  135. data/lib/aspera/secret_hider.rb +12 -7
  136. data/lib/aspera/ssh.rb +64 -31
  137. data/lib/aspera/ssl.rb +5 -5
  138. data/lib/aspera/sync/conf.schema.yaml +2 -2
  139. data/lib/aspera/sync/database.rb +2 -2
  140. data/lib/aspera/sync/operations.rb +22 -25
  141. data/lib/aspera/temp_file_manager.rb +25 -5
  142. data/lib/aspera/timer_limiter.rb +1 -1
  143. data/lib/aspera/transfer/faux_file.rb +24 -11
  144. data/lib/aspera/transfer/parameters.rb +36 -34
  145. data/lib/aspera/transfer/result.rb +74 -0
  146. data/lib/aspera/transfer/resumer.rb +10 -10
  147. data/lib/aspera/transfer/spec.rb +18 -1
  148. data/lib/aspera/transfer/spec.schema.yaml +12 -3
  149. data/lib/aspera/uri_reader.rb +52 -15
  150. data/lib/aspera/web_auth.rb +8 -7
  151. data/lib/aspera/web_server_simple.rb +15 -12
  152. data/lib/aspera/yaml.rb +5 -4
  153. data.tar.gz.sig +0 -0
  154. metadata +34 -6
  155. metadata.gz.sig +0 -0
  156. data/lib/aspera/cli/manager.rb +0 -766
  157. data/lib/aspera/cli/plugins/faspex.rb +0 -539
  158. data/lib/aspera/json_rpc.rb +0 -52
  159. data/lib/aspera/transfer/uri.rb +0 -56
data/lib/aspera/rest.rb CHANGED
@@ -77,6 +77,9 @@ module Aspera
77
77
  # and error are analyzed in RestErrorAnalyzer
78
78
  class Rest
79
79
  class << self
80
+ # Build a Basic authentication header value
81
+ # @param user [String] Username
82
+ # @param pass [String] Password
80
83
  # @return [String] Basic auth token
81
84
  def basic_authorization(user, pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}"; end
82
85
 
@@ -84,7 +87,7 @@ module Aspera
84
87
  # @param query [Hash] A key can have Array value and result will use PHP format: a[]=1&a[]=2
85
88
  # @return [Hash] The query parameters.
86
89
  def php_style(query)
87
- Aspera.assert_type(query, Hash){'query'}
90
+ Aspera.assert_type(query, Hash) { 'query' }
88
91
  query[:x_array_php_style] = true
89
92
  query
90
93
  end
@@ -93,10 +96,11 @@ module Aspera
93
96
  # Check if php style is specified.
94
97
  # `nil` values in query result in key without value, e.g. `?a`, while empty string values result in `?a=`.
95
98
  # @param url [String] The URL without query.
96
- # @param query [Hash,Array,String] The query.
99
+ # @param query [Hash,Array,String] The query parameters.
100
+ # @return [URI] The built URI.
97
101
  def build_uri(url, query)
98
102
  uri = URI.parse(url)
99
- Aspera.assert_values(uri.scheme, %w[http https]){'URI scheme'}
103
+ Aspera.assert_values(uri.scheme, %w[http https]) { 'URI scheme' }
100
104
  return uri if query.nil? || query.respond_to?(:empty?) && query.empty?
101
105
  Log.dump(:query, query)
102
106
  uri.query =
@@ -106,9 +110,9 @@ module Aspera
106
110
  when Hash
107
111
  URI.encode_www_form(h_to_query_array(query))
108
112
  when Array
109
- Aspera.assert(query.all?{ |i| i.is_a?(Array) && i.length.eql?(2)}, 'Query must be array of arrays of 2 elements')
113
+ Aspera.assert(query.all? { |i| i.is_a?(Array) && i.length.eql?(2) }, 'Query must be array of arrays of 2 elements')
110
114
  URI.encode_www_form(query) # remove nil values
111
- else Aspera.error_unexpected_value(query.class){'query type'}
115
+ else Aspera.error_unexpected_value(query.class) { 'query type' }
112
116
  end.gsub('%5B%5D=', '[]=')
113
117
  # [] is allowed in url parameters
114
118
  uri
@@ -118,6 +122,7 @@ module Aspera
118
122
  # Either p=1&p=2 (default)
119
123
  # or p[]=1&p[]=2 (if `:x_array_php_style` is set to true in query)
120
124
  # @param query [Hash] HTTP query as hash
125
+ # @return [Array<Array>] Array of [key, value] pairs suitable for URI.encode_www_form
121
126
  def h_to_query_array(query)
122
127
  Aspera.assert_type(query, Hash)
123
128
  suffix = query.delete(:x_array_php_style) ? '[]' : nil
@@ -155,43 +160,12 @@ module Aspera
155
160
  end
156
161
  end
157
162
 
158
- # Parse Link header according to RFC 8288 to extract a specific relation
159
- # @param link_header [String, nil] The Link header value
160
- # @param rel [String] The relation to look for (default: 'next')
161
- # @return [String, nil] The URL of the link with the specified relation, or nil
162
- def parse_link_header(link_header, rel: 'next')
163
- return if link_header.nil? || link_header.empty?
164
- # RFC 8288: Link header format is: <URI>; param1=value1; param2=value2, <URI2>; ...
165
- # We look for the link with the specified rel
166
- link_header.split(',').each do |link_part|
167
- link_part = link_part.strip
168
- # Extract URL between < and >
169
- url_match = link_part.match(/<([^>]+)>/)
170
- next unless url_match
171
- url = url_match[1]
172
- # Extract parameters after the URL
173
- params_str = link_part[url_match.end(0)..]
174
- # Check if this link has the specified rel (with or without quotes, case insensitive)
175
- next unless /;\s*rel\s*=\s*"?#{Regexp.escape(rel)}"?/i.match?(params_str)
176
- return url
177
- end
178
- # Fallback: if no rel found and looking for 'next', try the first link (backward compatibility)
179
- if rel.eql?('next')
180
- first_link = link_header.split(',').first&.strip
181
- if first_link
182
- url_match = first_link.match(/<([^>]+)>/)
183
- return url_match[1] if url_match
184
- end
185
- end
186
- nil
187
- end
188
-
189
163
  # Start a HTTP/S session, also used for web sockets
190
164
  # @param base_url [String] Base url of HTTP/S session
191
165
  # @return [Net::HTTP] A started HTTP session
192
166
  def start_http_session(base_url)
193
167
  uri = URI.parse(base_url)
194
- Aspera.assert_values(uri.scheme, %w[http https]){'URI scheme'}
168
+ Aspera.assert_values(uri.scheme, %w[http https]) { 'URI scheme' }
195
169
  # This honors http_proxy env var
196
170
  http_session = Net::HTTP.new(uri.host, uri.port)
197
171
  http_session.use_ssl = uri.scheme.eql?('https')
@@ -211,7 +185,7 @@ module Aspera
211
185
  Aspera.assert_type(http_session, Net::HTTP)
212
186
  # Net::BufferedIO in net/protocol.rb
213
187
  result = http_session.instance_variable_get(:@socket)
214
- Aspera.assert(!result.nil?){"no socket for #{http_session}"}
188
+ Aspera.assert(!result.nil?) { "no socket for #{http_session}" }
215
189
  return result
216
190
  end
217
191
 
@@ -279,7 +253,7 @@ module Aspera
279
253
  # Base common headers of API
280
254
  attr_reader :headers
281
255
 
282
- # @return creation parameters
256
+ # @return [Hash] creation parameters
283
257
  def params
284
258
  return {
285
259
  base_url: @base_url, # String
@@ -316,7 +290,7 @@ module Aspera
316
290
  # remove trailing port if it is 443 and scheme is https
317
291
  @base_url = @base_url.gsub(/:443$/, '') if @base_url.start_with?('https://')
318
292
  @base_url = @base_url.gsub(/:80$/, '') if @base_url.start_with?('http://')
319
- Log.log.debug{"Rest.new(#{@base_url})"}
293
+ Log.log.debug { "Rest.new(#{@base_url})" }
320
294
  # default is no auth
321
295
  @auth_params = auth
322
296
  Aspera.assert_type(@auth_params, Hash)
@@ -334,11 +308,11 @@ module Aspera
334
308
  @oauth = nil
335
309
  end
336
310
 
337
- # @return the OAuth object (create, or cached if already created)
311
+ # @return [OAuth::Base] the OAuth object (create, or cached if already created)
338
312
  def oauth
339
313
  if @oauth.nil?
340
314
  Aspera.assert(@auth_params[:type].eql?(:oauth2), 'no OAuth defined')
341
- oauth_parameters = @auth_params.reject{ |k, _v| k.eql?(:type)}
315
+ oauth_parameters = @auth_params.reject { |k, _v| k.eql?(:type) }
342
316
  Log.dump(:oauth_parameters, oauth_parameters)
343
317
  @oauth = OAuth::Factory.instance.create(**oauth_parameters)
344
318
  end
@@ -352,9 +326,9 @@ module Aspera
352
326
  # @param content_type [String, nil] Type of body parameters (one of MIME_*) and serialization, else use headers
353
327
  # @param body [Hash, String, nil] Body parameters
354
328
  # @param headers [Hash{String => String}] Additional headers (override Content-Type)
355
- # @param save_to_file [String, nil] File path to save response body
329
+ # @param save_to [String, IO, nil] File path or IO object to save response body; progress bar is used when set
356
330
  # @param exception [Boolean] Whether to raise an exception on HTTP error
357
- # @param ret [Symbol] One of :data, :resp, :both controls return value
331
+ # @param ret [Symbol] One of :data, :resp, :both - controls return value
358
332
  # @return [Array(Hash, Net::HTTPResponse)] When `ret` is :both
359
333
  # @return [Net::HTTPResponse] When `ret` is :resp
360
334
  # @return [Hash] When `ret` is :data
@@ -366,42 +340,20 @@ module Aspera
366
340
  content_type: nil,
367
341
  body: nil,
368
342
  headers: nil,
369
- save_to_file: nil,
343
+ save_to: nil,
370
344
  exception: true,
371
345
  ret: :data
372
346
  )
373
347
  subpath = subpath.to_s if subpath.is_a?(Symbol)
374
348
  subpath = '' if subpath.nil?
375
- Log.log.debug{"call #{operation} [#{subpath}]".red.bold.bg_green}
349
+ Log.log.debug { "call #{operation} [#{subpath}]".red.bold.bg_green }
376
350
  Log.dump(:body, body, level: :trace1)
377
351
  Log.dump(:query, query, level: :trace1)
378
352
  Log.dump(:headers, headers, level: :trace1)
379
353
  Aspera.assert_type(subpath, String)
380
354
  # We must have a way to check return code
381
355
  Aspera.assert(exception || !ret.eql?(:data), 'ret: :data requires exception handler')
382
- if headers.nil?
383
- headers = @headers.clone
384
- else
385
- h = headers
386
- headers = @headers.clone
387
- headers.merge!(h)
388
- end
389
- Aspera.assert_type(headers, Hash)
390
- case @auth_params[:type]
391
- when :none
392
- # no auth
393
- when :basic
394
- Log.log.debug('using Basic auth')
395
- # done in build_req
396
- when :oauth2
397
- headers['Authorization'] = oauth.authorization unless headers.key?('Authorization')
398
- when :url
399
- query ||= {}
400
- @auth_params[:url_query].each do |key, value|
401
- query[key] = value
402
- end
403
- else Aspera.error_unexpected_value(@auth_params[:type])
404
- end
356
+ headers, query = prepare_call(headers, query)
405
357
  result_http = nil
406
358
  result_data = nil
407
359
  # initialize with number of initial retries allowed, nil gives zero
@@ -409,37 +361,7 @@ module Aspera
409
361
  # start a block to be able to retry the actual HTTP request in case of OAuth token expiration
410
362
  begin
411
363
  Log.log.debug("send request (retries=#{tries_remain_redirect})")
412
- # TODO: shall we percent encode subpath (spaces) test with access key delete with space in id
413
- # URI.escape()
414
- separator = ['', '/'].include?(subpath) ? '' : '/'
415
- uri = self.class.build_uri("#{@base_url}#{separator}#{subpath}", query)
416
- Log.log.debug{"URI=#{uri}"}
417
- begin
418
- # instantiate request object based on string name
419
- req = Net::HTTP.const_get(operation.capitalize).new(uri)
420
- rescue NameError
421
- raise "unsupported operation : #{operation}"
422
- end
423
- case content_type
424
- when nil # ignore
425
- when Mime::JSON
426
- req.body = JSON.generate(body) # , ascii_only: true
427
- req['Content-Type'] = Mime::JSON
428
- when Mime::WWW
429
- req.body = URI.encode_www_form(body)
430
- req['Content-Type'] = Mime::WWW
431
- when Mime::TEXT
432
- req.body = body
433
- req['Content-Type'] = Mime::TEXT
434
- else Aspera.error_unexpected_value(content_type){'body type'}
435
- end
436
- # set headers
437
- headers.each do |key, value|
438
- req[key] = value
439
- end
440
- # :type = :basic
441
- req.basic_auth(@auth_params[:username], @auth_params[:password]) if @auth_params[:type].eql?(:basic)
442
- Log.dump(:req_body, req.body, level: :trace1)
364
+ req = build_request(operation, subpath, query, content_type, body, headers)
443
365
  # we try the call, and will retry on some error types
444
366
  error_tries ||= 1 + RestParameters.instance.retry_max
445
367
  result_mime = nil
@@ -448,58 +370,21 @@ module Aspera
448
370
  http_session.request(req) do |response|
449
371
  result_http = response
450
372
  result_mime = self.class.parse_header(result_http['Content-Type'] || Mime::TEXT)[:type]
451
- Log.log.debug{"response: code=#{result_http.code}, mime=#{result_mime}, content-type=#{response['Content-Type']}"}
373
+ Log.log.debug { "response: code=#{result_http.code}, mime=#{result_mime}, content-type=#{response['Content-Type']}" }
452
374
  # JSON data needs to be parsed, in case it contains an error code
453
- if !save_to_file.nil? &&
454
- result_http.code.to_s.start_with?('2') &&
455
- !Mime.json?(result_mime)
456
- total_size = result_http['Content-Length']&.to_i
457
- Log.log.debug('before write file')
458
- target_file = save_to_file
459
- # override user's path to path in header
460
- if !response['Content-Disposition'].nil?
461
- disposition = self.class.parse_header(response['Content-Disposition'])
462
- target_file = File.join(File.dirname(target_file), disposition[:parameters][:filename]) if disposition[:parameters].key?(:filename) && !disposition[:parameters][:filename].eql?('.')
463
- end
464
- # download with temp filename
465
- target_file_tmp = "#{target_file}#{RestParameters.instance.download_partial_suffix}"
466
- Log.log.debug{"saving to: #{target_file}"}
467
- written_size = 0
468
- session_id = SecureRandom.uuid.freeze
469
- RestParameters.instance.progress_bar&.event(:session_start, session_id: session_id)
470
- RestParameters.instance.progress_bar&.event(:session_size, session_id: session_id, info: total_size) if total_size
471
- FileUtils.mkdir_p(File.dirname(target_file_tmp))
472
- limiter = TimerLimiter.new(0.5)
473
- File.open(target_file_tmp, 'wb') do |file|
474
- result_http.read_body do |fragment|
475
- file.write(fragment)
476
- written_size += fragment.length
477
- RestParameters.instance.progress_bar&.event(:transfer, session_id: session_id, info: written_size) if limiter.trigger?
478
- end
479
- end
480
- RestParameters.instance.progress_bar&.event(:session_end, session_id: session_id)
481
- RestParameters.instance.progress_bar&.event(:end)
482
- # rename at the end
483
- File.rename(target_file_tmp, target_file)
484
- file_saved = true
485
- end
375
+ file_saved = save_response(response, result_http, result_mime, save_to)
486
376
  end
487
- Log.log.debug{"result: code=#{result_http.code} mime=#{result_mime}"}
488
- # sometimes there is a UTF8 char (e.g. © )
377
+ Log.log.debug { "result: code=#{result_http.code} mime=#{result_mime}" }
378
+ # sometimes there is a UTF8 char (e.g. (c) )
489
379
  # TODO : related to mime type encoding ?
490
380
  # result_http.body.force_encoding('UTF-8') if result_http.body.is_a?(String)
491
381
  # Log.log.debug{"result: body=#{result_http.body}"}
492
- result_data = result_http.body
493
- Log.dump(:result_data_raw, result_data, level: :trace1)
494
- # TODO: Remove next 2 lines when bug in async node api is fixed. (Aspera/core/issues/4490)
495
- node_api_bug = result_data&.index('}HTTP/1.1 400 Bad Request') if result_data.is_a?(String)
496
- result_data = result_data[0..node_api_bug] if node_api_bug
497
- result_data = JSON.parse(result_data) if Mime.json?(result_mime) && !result_data.nil? && !result_data.empty?
498
- Log.dump(:result_data, result_data)
382
+ result_data = parse_response(result_http, result_mime)
499
383
  RestErrorAnalyzer.instance.raise_on_error(req, result_data, result_http)
500
- unless file_saved || save_to_file.nil?
501
- FileUtils.mkdir_p(File.dirname(save_to_file))
502
- File.write(save_to_file, result_http.body, binmode: true)
384
+ unless file_saved || save_to.nil?
385
+ raise 'save_to: IO object requires a streaming response' if save_to.respond_to?(:write)
386
+ FileUtils.mkdir_p(File.dirname(save_to))
387
+ File.write(save_to, result_http.body, binmode: true)
503
388
  end
504
389
  rescue RestCallError => e
505
390
  do_retry = false
@@ -520,7 +405,7 @@ module Aspera
520
405
  # regenerate a brand new token
521
406
  req['Authorization'] = oauth.authorization(cache: false)
522
407
  end
523
- Log.log.debug{"using new token=#{headers['Authorization']}"}
408
+ Log.log.debug('using new token')
524
409
  do_retry ||= true
525
410
  end
526
411
  if do_retry && (error_tries -= 1).positive?
@@ -535,7 +420,7 @@ module Aspera
535
420
  # special case: relative redirect
536
421
  if URI.parse(new_url).host.nil?
537
422
  # we don't manage relative redirects with non-absolute path
538
- Aspera.assert(new_url.start_with?('/')){"redirect location is relative: #{new_url}, but does not start with /."}
423
+ Aspera.assert(new_url.start_with?('/')) { "redirect location is relative: #{new_url}, but does not start with /." }
539
424
  new_url = "#{current_uri.scheme}://#{current_uri.host}#{new_url}"
540
425
  end
541
426
  # forwards the request to the new location
@@ -548,7 +433,7 @@ module Aspera
548
433
  query: query,
549
434
  body: body,
550
435
  content_type: content_type,
551
- save_to_file: save_to_file,
436
+ save_to: save_to,
552
437
  exception: exception,
553
438
  headers: headers,
554
439
  ret: ret
@@ -557,15 +442,138 @@ module Aspera
557
442
  # raise exception if could not retry and not return error in result
558
443
  raise e if exception
559
444
  end
560
- Log.log.debug{"result=http:#{result_http}, data:#{result_data.class}"}
445
+ Log.log.debug { "result=http:#{result_http}, data:#{result_data.class}" }
561
446
  return case ret
562
447
  when :data then result_data
563
448
  when :resp then result_http
564
449
  when :both then [result_data, result_http]
565
- else Aspera.error_unexpected_value(ret){'Type of result for REST'}
450
+ else Aspera.error_unexpected_value(ret) { 'Type of result for REST' }
566
451
  end
567
452
  end
568
453
 
454
+ private
455
+
456
+ def prepare_call(headers, query)
457
+ if headers.nil?
458
+ headers = @headers.clone
459
+ else
460
+ h = headers
461
+ headers = @headers.clone
462
+ headers.merge!(h)
463
+ end
464
+ Aspera.assert_type(headers, Hash)
465
+ case @auth_params[:type]
466
+ when :none
467
+ # no auth
468
+ when :basic
469
+ Log.log.debug('using Basic auth')
470
+ # done in build_req
471
+ when :oauth2
472
+ headers['Authorization'] = oauth.authorization unless headers.key?('Authorization')
473
+ when :url
474
+ query ||= {}
475
+ @auth_params[:url_query].each do |key, value|
476
+ query[key] = value
477
+ end
478
+ else Aspera.error_unexpected_value(@auth_params[:type])
479
+ end
480
+ [headers, query]
481
+ end
482
+
483
+ def build_request(operation, subpath, query, content_type, body, headers)
484
+ # TODO: shall we percent encode subpath (spaces) test with access key delete with space in id
485
+ # URI.escape()
486
+ separator = ['', '/'].include?(subpath) ? '' : '/'
487
+ uri = self.class.build_uri("#{@base_url}#{separator}#{subpath}", query)
488
+ Log.log.debug { "URI=#{uri}" }
489
+ begin
490
+ # instantiate request object based on string name
491
+ req = Net::HTTP.const_get(operation.capitalize).new(uri)
492
+ rescue NameError
493
+ raise "unsupported operation : #{operation}"
494
+ end
495
+ case content_type
496
+ when nil # ignore
497
+ when Mime::JSON
498
+ req.body = JSON.generate(body) # , ascii_only: true
499
+ req['Content-Type'] = Mime::JSON
500
+ when Mime::WWW
501
+ req.body = URI.encode_www_form(body)
502
+ req['Content-Type'] = Mime::WWW
503
+ when Mime::TEXT
504
+ req.body = body
505
+ req['Content-Type'] = Mime::TEXT
506
+ else Aspera.error_unexpected_value(content_type) { 'body type' }
507
+ end
508
+ # set headers
509
+ headers.each do |key, value|
510
+ req[key] = value
511
+ end
512
+ # :type = :basic
513
+ req.basic_auth(@auth_params[:username], @auth_params[:password]) if @auth_params[:type].eql?(:basic)
514
+ Log.dump(:req_body, req.body, level: :trace1)
515
+ req
516
+ end
517
+
518
+ def save_response(response, result_http, result_mime, save_to)
519
+ return false unless !save_to.nil? && result_http.code.to_s.start_with?('2') && !Mime.json?(result_mime)
520
+
521
+ total_size = result_http['Content-Length']&.to_i
522
+ Log.log.debug('before write file')
523
+ target_file = save_to
524
+ # override user's path to path in header
525
+ unless response['Content-Disposition'].nil?
526
+ disposition = self.class.parse_header(response['Content-Disposition'])
527
+ if disposition[:parameters].key?(:filename) && !disposition[:parameters][:filename].eql?('.')
528
+ # Use only the basename to prevent path traversal via a server-controlled Content-Disposition header
529
+ safe_filename = File.basename(disposition[:parameters][:filename])
530
+ target_file = File.join(File.dirname(target_file), safe_filename) unless safe_filename.empty?
531
+ end
532
+ end
533
+ Log.log.debug { "saving to: #{target_file}" }
534
+ written_size = 0
535
+ session_id = SecureRandom.uuid.freeze
536
+ RestParameters.instance.progress_bar&.event(:session_start, session_id: session_id)
537
+ RestParameters.instance.progress_bar&.event(:session_size, session_id: session_id, info: total_size) if total_size
538
+ limiter = TimerLimiter.new(0.5)
539
+ if target_file.respond_to?(:write)
540
+ # IO object: stream directly into it
541
+ result_http.read_body do |fragment|
542
+ target_file.write(fragment)
543
+ written_size += fragment.length
544
+ RestParameters.instance.progress_bar&.event(:transfer, session_id: session_id, info: written_size) if limiter.trigger?
545
+ end
546
+ else
547
+ # file path: download to partial name first, then rename atomically
548
+ target_file_tmp = "#{target_file}#{RestParameters.instance.download_partial_suffix}"
549
+ FileUtils.mkdir_p(File.dirname(target_file_tmp))
550
+ File.open(target_file_tmp, 'wb') do |file|
551
+ result_http.read_body do |fragment|
552
+ file.write(fragment)
553
+ written_size += fragment.length
554
+ RestParameters.instance.progress_bar&.event(:transfer, session_id: session_id, info: written_size) if limiter.trigger?
555
+ end
556
+ end
557
+ File.rename(target_file_tmp, target_file)
558
+ end
559
+ RestParameters.instance.progress_bar&.event(:session_end, session_id: session_id)
560
+ RestParameters.instance.progress_bar&.event(:end)
561
+ true
562
+ end
563
+
564
+ def parse_response(result_http, result_mime)
565
+ result_data = result_http.body
566
+ Log.dump(:result_data_raw, result_data, level: :trace1)
567
+ # TODO: Remove next 2 lines when bug in async node api is fixed. (Aspera/core/issues/4490)
568
+ node_api_bug = result_data&.index('}HTTP/1.1 400 Bad Request') if result_data.is_a?(String)
569
+ result_data = result_data[0..node_api_bug] if node_api_bug
570
+ result_data = JSON.parse(result_data) if Mime.json?(result_mime) && !result_data.nil? && !result_data.empty?
571
+ Log.dump(:result_data, result_data)
572
+ result_data
573
+ end
574
+
575
+ public
576
+
569
577
  #
570
578
  # CRUD simplified methods here
571
579
  # If specific elements are needed, then use the full `call` method
@@ -32,11 +32,11 @@ module Aspera
32
32
  # Use this method to analyze a EST result and raise an exception
33
33
  # Analyzes REST call response and raises a RestCallError exception
34
34
  # if HTTP result code is not 2XX
35
- # @param req [Net::HTTPRequest]
36
- # @param data [Object]
37
- # @param http [Net::HTTPResponse]
35
+ # @param req [Net::HTTPRequest] the HTTP request
36
+ # @param data [Object] the parsed response body
37
+ # @param http [Net::HTTPResponse] the raw HTTP response
38
38
  def raise_on_error(req, data, http)
39
- Log.log.debug{"raise_on_error #{req.method} #{req.path} #{http.code}"}
39
+ Log.log.debug { "raise_on_error #{req.method} #{req.path} #{http.code}" }
40
40
  context = {
41
41
  messages: [],
42
42
  request: req,
@@ -49,7 +49,7 @@ module Aspera
49
49
  @error_handlers.each do |handler|
50
50
  handler[:block].call(handler[:name], context)
51
51
  rescue StandardError => e
52
- Log.log.error{"ERROR in handler:\n#{e.message}\n#{e.backtrace}"}
52
+ Log.log.error { "ERROR in handler:\n#{e.message}\n#{e.backtrace}" }
53
53
  end
54
54
  raise RestCallError, context unless context[:messages].empty?
55
55
  end
@@ -97,7 +97,7 @@ module Aspera
97
97
  # @param message [String] one error message to add to list
98
98
  def add_error(context, type, message)
99
99
  context[:messages].push(message)
100
- Log.log.trace1{"Found error: #{type}: #{message}"}
100
+ Log.log.trace1 { "Found error: #{type}: #{message}" }
101
101
  log_file = instance.log_file
102
102
  # log error for further analysis (file must exist to activate)
103
103
  return if log_file.nil? || !File.exist?(log_file)
@@ -41,16 +41,6 @@ module Aspera
41
41
  end
42
42
  RestErrorAnalyzer.instance.add_simple_handler(name: 'T9:IBM cloud IAM', path: ['errorMessage'])
43
43
  RestErrorAnalyzer.instance.add_simple_handler(name: 'T10:faspex v4', path: ['user_message'])
44
- RestErrorAnalyzer.instance.add_handler('bss graphql') do |type, context|
45
- next unless context[:data].is_a?(Hash)
46
- d_t_s = context[:data]['errors']
47
- next unless d_t_s.is_a?(Array)
48
- d_t_s.each do |res|
49
- r_err = res['message']
50
- next unless r_err.is_a?(String)
51
- RestErrorAnalyzer.add_error(context, type, r_err)
52
- end
53
- end
54
44
  RestErrorAnalyzer.instance.add_handler('Orchestrator') do |type, context|
55
45
  next if context[:response].code.start_with?('2')
56
46
  data = context[:data]
@@ -15,10 +15,10 @@ module Aspera
15
15
  # @param value [String] Value of field of searched entity
16
16
  # @param field [String] Field of searched entity
17
17
  # @param query [Hash] Optional additional search query parameters
18
- # @returns [Hash] A single entity matching the search, or an exception if not found or multiple found
18
+ # @return [Hash] A single entity matching the search, or an exception if not found or multiple found
19
19
  def lookup_with_q(entity, value:, field: 'name', query: {})
20
- Aspera.assert_type(query, Hash){'query'}
21
- Aspera.assert_type(field, String){'field'}
20
+ Aspera.assert_type(query, Hash) { 'query' }
21
+ Aspera.assert_type(field, String) { 'field' }
22
22
  # returns entities matching the query (it matches against several fields in case insensitive way)
23
23
  # We don't do paging, as anyway, we look for only one match
24
24
  matching_items = read(entity, query.merge({'q' => value}))
@@ -32,11 +32,11 @@ module Aspera
32
32
  else
33
33
  # multiple case insensitive partial matches, try case insensitive full match
34
34
  # (anyway AoC does not allow creation of 2 entities with same case insensitive field value)
35
- value_matches = matching_items.select{ |i| i[field].casecmp?(value)}
35
+ value_matches = matching_items.select { |i| i[field].casecmp?(value) }
36
36
  case value_matches.length
37
37
  when 1 then return value_matches.first
38
- when 0 then raise %Q(#{entity}: Multiple case insensitive partial match for: "#{value}" in #{matching_items.map{ |i| i[field]}.join(', ')} but no case insensitive full match. Please be more specific or give exact #{field}.)
39
- else raise "Two entities cannot have the same case insensitive #{field}: #{value_matches.map{ |i| i[field]}}"
38
+ when 0 then raise Error, "#{entity}: Multiple case insensitive partial match for: \"#{value}\" in #{matching_items.map { |i| i[field] }.join(', ')} but no case insensitive full match. Please be more specific or give exact #{field}."
39
+ else raise Error, "Two entities cannot have the same case insensitive #{field}: #{value_matches.map { |i| i[field] }}"
40
40
  end
41
41
  end
42
42
  end
@@ -60,7 +60,8 @@ module Aspera
60
60
  Aspera.assert_type(entity, String)
61
61
  Aspera.assert_type(items_key, String)
62
62
  Aspera.assert_type(query, Hash)
63
- Log.log.debug{"list_entities t=#{entity} k=#{items_key} q=#{query}"}
63
+ Log.log.debug { "list_entities t=#{entity} k=#{items_key}" }
64
+ Log.dump(:query, query)
64
65
  result = []
65
66
  offset = 0
66
67
  max_items = query.delete(MAX_ITEMS)
@@ -104,10 +105,10 @@ module Aspera
104
105
  # @param query [Hash] Additional query parameters (Default: `:default`)
105
106
  def lookup_entity_by_field(entity:, value:, field: 'name', items_key: nil, query: :default)
106
107
  if query.eql?(:default)
107
- Aspera.assert_values(field, ['name']){'Default query field'}
108
+ Aspera.assert_values(field, ['name']) { 'Default query field' }
108
109
  query = {'q'=> value}
109
110
  end
110
- lookup_entity_generic(entity: entity, field: field, value: value){list_entities_limit_offset_total_count(entity: entity, items_key: items_key, query: query).first}
111
+ lookup_entity_generic(entity: entity, field: field, value: value) { list_entities_limit_offset_total_count(entity: entity, items_key: items_key, query: query).first }
111
112
  end
112
113
 
113
114
  # Lookup entity by field and value.
@@ -123,7 +124,7 @@ module Aspera
123
124
  Aspera.assert(block_given?, 'block required for lookup_entity_generic')
124
125
  found = yield
125
126
  Aspera.assert_array_all(found, Hash)
126
- found = found.select{ |i| i[field].eql?(value)}
127
+ found = found.select { |i| i[field].eql?(value) }
127
128
  return found.first if found.length.eql?(1)
128
129
  raise Cli::BadIdentifier.new(entity, value, field: field, count: found.length)
129
130
  end