aspera-cli 4.27.2 → 4.27.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +133 -0
  4. data/CONTRIBUTING.md +5 -2
  5. data/bin/ascli +3 -1
  6. data/docs/README.md +1006 -766
  7. data/lib/aspera/agent/base.rb +7 -2
  8. data/lib/aspera/agent/connect.rb +6 -8
  9. data/lib/aspera/agent/desktop.rb +2 -6
  10. data/lib/aspera/agent/direct.rb +52 -22
  11. data/lib/aspera/agent/node.rb +9 -8
  12. data/lib/aspera/agent/transferd.rb +2 -2
  13. data/lib/aspera/api/alee.rb +1 -1
  14. data/lib/aspera/api/aoc.rb +14 -12
  15. data/lib/aspera/api/ats.rb +1 -1
  16. data/lib/aspera/api/cos_node.rb +2 -2
  17. data/lib/aspera/api/faspex.rb +11 -7
  18. data/lib/aspera/api/httpgw.rb +38 -35
  19. data/lib/aspera/api/node.rb +39 -34
  20. data/lib/aspera/ascmd.rb +3 -1
  21. data/lib/aspera/ascp/installation.rb +63 -28
  22. data/lib/aspera/ascp/management.rb +1 -0
  23. data/lib/aspera/assert.rb +4 -0
  24. data/lib/aspera/cli/ascp_actions.rb +20 -41
  25. data/lib/aspera/cli/async_transfer_store.rb +12 -11
  26. data/lib/aspera/cli/bootstrapper.rb +14 -16
  27. data/lib/aspera/cli/command_line.rb +252 -0
  28. data/lib/aspera/cli/command_registry.rb +215 -37
  29. data/lib/aspera/cli/command_spec.rb +104 -15
  30. data/lib/aspera/cli/completion/ascli.bash +12 -0
  31. data/lib/aspera/cli/completion/ascli.fish +16 -0
  32. data/lib/aspera/cli/completion/ascli.zsh +19 -0
  33. data/lib/aspera/cli/context.rb +3 -0
  34. data/lib/aspera/cli/deprecation.rb +37 -0
  35. data/lib/aspera/cli/extended_value.rb +6 -3
  36. data/lib/aspera/cli/formatter.rb +94 -80
  37. data/lib/aspera/cli/gem_checker.rb +1 -1
  38. data/lib/aspera/cli/hints.rb +7 -6
  39. data/lib/aspera/cli/http.rb +22 -34
  40. data/lib/aspera/cli/info.rb +3 -0
  41. data/lib/aspera/cli/mcp_tool.rb +47 -83
  42. data/lib/aspera/cli/option_declarator.rb +33 -42
  43. data/lib/aspera/cli/option_registry.rb +69 -0
  44. data/lib/aspera/cli/option_types.rb +105 -0
  45. data/lib/aspera/cli/option_value.rb +278 -0
  46. data/lib/aspera/cli/options.schema.yaml +124 -15
  47. data/lib/aspera/cli/parser.rb +333 -862
  48. data/lib/aspera/cli/plugins/alee.rb +7 -4
  49. data/lib/aspera/cli/plugins/aoc.rb +545 -518
  50. data/lib/aspera/cli/plugins/ats.rb +59 -80
  51. data/lib/aspera/cli/plugins/base.rb +221 -265
  52. data/lib/aspera/cli/plugins/basic_auth.rb +2 -10
  53. data/lib/aspera/cli/plugins/config.rb +263 -184
  54. data/lib/aspera/cli/plugins/console.rb +103 -39
  55. data/lib/aspera/cli/plugins/cos.rb +6 -23
  56. data/lib/aspera/cli/plugins/factory.rb +3 -0
  57. data/lib/aspera/cli/plugins/faspex5.rb +204 -182
  58. data/lib/aspera/cli/plugins/faspio.rb +6 -11
  59. data/lib/aspera/cli/plugins/httpgw.rb +8 -11
  60. data/lib/aspera/cli/plugins/mcp.rb +20 -55
  61. data/lib/aspera/cli/plugins/node.rb +300 -327
  62. data/lib/aspera/cli/plugins/orchestrator.rb +152 -110
  63. data/lib/aspera/cli/plugins/preview.rb +96 -105
  64. data/lib/aspera/cli/plugins/server.rb +78 -53
  65. data/lib/aspera/cli/plugins/shares.rb +80 -131
  66. data/lib/aspera/cli/preset_actions.rb +44 -27
  67. data/lib/aspera/cli/preset_manager.rb +44 -19
  68. data/lib/aspera/cli/prompt.rb +36 -0
  69. data/lib/aspera/cli/result.rb +42 -36
  70. data/lib/aspera/cli/runner.rb +32 -59
  71. data/lib/aspera/cli/special_values.rb +5 -0
  72. data/lib/aspera/cli/sync_actions.rb +51 -46
  73. data/lib/aspera/cli/terminal_formatter.rb +9 -3
  74. data/lib/aspera/cli/transfer_actions.rb +14 -9
  75. data/lib/aspera/cli/transfer_agent.rb +34 -38
  76. data/lib/aspera/cli/transfer_progress.rb +290 -55
  77. data/lib/aspera/cli/vault_manager.rb +0 -17
  78. data/lib/aspera/cli/version.rb +1 -1
  79. data/lib/aspera/cli/wizard.rb +5 -3
  80. data/lib/aspera/coverage.rb +1 -1
  81. data/lib/aspera/environment.rb +35 -5
  82. data/lib/aspera/faspex_gw.rb +2 -1
  83. data/lib/aspera/faspex_postproc.rb +1 -0
  84. data/lib/aspera/graphql.rb +5 -5
  85. data/lib/aspera/json_rpc/client.rb +5 -5
  86. data/lib/aspera/keychain/encrypted_hash.rb +2 -2
  87. data/lib/aspera/keychain/factory.rb +2 -1
  88. data/lib/aspera/keychain/one_password_api.rb +1 -1
  89. data/lib/aspera/link_header.rb +2 -2
  90. data/lib/aspera/log.rb +47 -27
  91. data/lib/aspera/markdown.rb +2 -0
  92. data/lib/aspera/mime.rb +25 -0
  93. data/lib/aspera/node_emulator.rb +759 -0
  94. data/lib/aspera/oauth/base.rb +37 -26
  95. data/lib/aspera/oauth/factory.rb +7 -3
  96. data/lib/aspera/oauth/generic.rb +1 -1
  97. data/lib/aspera/oauth/json_credentials.rb +34 -0
  98. data/lib/aspera/oauth/jwt.rb +4 -5
  99. data/lib/aspera/oauth/web.rb +9 -8
  100. data/lib/aspera/oauth.rb +1 -0
  101. data/lib/aspera/persistency_folder.rb +1 -3
  102. data/lib/aspera/preview/file_types.rb +4 -4
  103. data/lib/aspera/preview/generator.rb +11 -1
  104. data/lib/aspera/preview/options.schema.yaml +119 -0
  105. data/lib/aspera/preview/terminal.rb +4 -3
  106. data/lib/aspera/preview/utils.rb +9 -6
  107. data/lib/aspera/products/connect.rb +1 -1
  108. data/lib/aspera/rainbow.rb +7 -0
  109. data/lib/aspera/rest/aspera_errors.rb +72 -0
  110. data/lib/aspera/rest/call_error.rb +27 -0
  111. data/lib/aspera/rest/client.rb +523 -0
  112. data/lib/aspera/rest/error_analyzer.rb +113 -0
  113. data/lib/aspera/rest/list.rb +149 -0
  114. data/lib/aspera/rest/parameters.rb +55 -0
  115. data/lib/aspera/rest/util.rb +176 -0
  116. data/lib/aspera/rest.rb +7 -621
  117. data/lib/aspera/schema/IBM Aspera Console-enhanced.yaml +1125 -0
  118. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +0 -20
  119. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +1784 -0
  120. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +1230 -137
  121. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2395 -0
  122. data/lib/aspera/schema/IBM_Aspera_Shares.yaml +14 -13
  123. data/lib/aspera/schema/documentation.rb +13 -3
  124. data/lib/aspera/schema/reader.rb +12 -18
  125. data/lib/aspera/schema/registry.rb +23 -1
  126. data/lib/aspera/schema/validator.rb +92 -0
  127. data/lib/aspera/secret_hider.rb +36 -25
  128. data/lib/aspera/string_ext.rb +15 -0
  129. data/lib/aspera/temp_file_manager.rb +6 -5
  130. data/lib/aspera/transfer/parameters.rb +2 -0
  131. data/lib/aspera/transfer/spec.rb +1 -0
  132. data/lib/aspera/uri_reader.rb +11 -11
  133. data/lib/aspera/web_auth/index.html +147 -0
  134. data/lib/aspera/web_auth/server.rb +81 -0
  135. data.tar.gz.sig +0 -0
  136. metadata +43 -9
  137. metadata.gz.sig +0 -0
  138. data/lib/aspera/colors.rb +0 -79
  139. data/lib/aspera/node_simulator.rb +0 -344
  140. data/lib/aspera/preview/options.rb +0 -45
  141. data/lib/aspera/rest_call_error.rb +0 -25
  142. data/lib/aspera/rest_error_analyzer.rb +0 -111
  143. data/lib/aspera/rest_errors_aspera.rb +0 -58
  144. data/lib/aspera/rest_list.rb +0 -136
  145. data/lib/aspera/web_auth.rb +0 -211
@@ -23,14 +23,19 @@ module Aspera
23
23
  end
24
24
 
25
25
  # Wait for all sessions to terminate and return a typed Transfer::Result.
26
+ # Notifies the end of transfer to the progress bar (agents do not).
26
27
  # @return [Transfer::Result::Success, Transfer::Result::Error]
27
28
  def wait_for_completion
29
+ success = false
28
30
  statuses = wait_for_transfers_completion
29
- @progress&.reset
30
31
  Aspera.assert_type(statuses, Array)
31
32
  Aspera.assert(statuses.none? { |i| !i.eql?(:success) && !i.is_a?(StandardError) }) { "bad statuses content: #{statuses}" }
32
33
  errors = statuses.reject { |i| i.eql?(:success) }
33
- return errors.empty? ? Transfer::Result.success : Transfer::Result.error(errors.first)
34
+ success = errors.empty?
35
+ return success ? Transfer::Result.success : Transfer::Result.error(errors.first)
36
+ ensure
37
+ # Also when the agent raises an exception
38
+ notify_progress(:end, info: success)
34
39
  end
35
40
 
36
41
  # Return the job_id of the last transfer submitted by this agent.
@@ -6,6 +6,8 @@ require 'aspera/products/other'
6
6
  require 'aspera/rest'
7
7
  require 'aspera/environment'
8
8
  require 'securerandom'
9
+ require 'aspera/rainbow'
10
+ using Rainbow
9
11
 
10
12
  module Aspera
11
13
  module Agent
@@ -21,9 +23,9 @@ module Aspera
21
23
  # agent_params must contain 'app_id'; the Connect URL is auto-discovered.
22
24
  # @return [Hash] normalized status hash
23
25
  def transfer_status(transfer_id, agent_params)
24
- connect_api = Rest.new(
26
+ connect_api = Rest::Client.new(
25
27
  base_url: "#{connect_api_url}/v5/connect",
26
- headers: {'Origin' => RestParameters.instance.user_agent}
28
+ headers: {'Origin' => Rest::Parameters.instance.user_agent}
27
29
  )
28
30
  tr_info = connect_api.create("transfers/info/#{transfer_id}", {'aspera_connect_settings' => {'app_id' => agent_params['app_id']}})
29
31
  transfer = tr_info['transfer_info']
@@ -72,9 +74,9 @@ module Aspera
72
74
  # raise exception if connect not started and file does not exist
73
75
  connect_url = self.class.connect_api_url
74
76
  Log.log.debug { "found: #{connect_url}" }
75
- @connect_api = Rest.new(
77
+ @connect_api = Rest::Client.new(
76
78
  base_url: "#{connect_url}/v5/connect", # could use v6 also now
77
- headers: {'Origin' => RestParameters.instance.user_agent}
79
+ headers: {'Origin' => Rest::Parameters.instance.user_agent}
78
80
  )
79
81
  connect_info = @connect_api.read('info/version')
80
82
  Log.log.debug('Connect was reached') if method_index > 0
@@ -158,19 +160,15 @@ module Aspera
158
160
  end
159
161
  when 'completed'
160
162
  notify_progress(:session_end, session_id: @transfer_id)
161
- notify_progress(:end)
162
163
  break
163
164
  when 'failed'
164
165
  notify_progress(:session_end, session_id: @transfer_id)
165
- notify_progress(:end)
166
166
  raise Transfer::Error, transfer['error_desc']
167
167
  when 'cancelled'
168
168
  notify_progress(:session_end, session_id: @transfer_id)
169
- notify_progress(:end)
170
169
  raise Transfer::Error, 'Transfer cancelled by user'
171
170
  else
172
171
  notify_progress(:session_end, session_id: @transfer_id)
173
- notify_progress(:end)
174
172
  raise Transfer::Error, "unknown status: #{transfer['status']}: #{transfer['error_desc']}"
175
173
  end
176
174
  end
@@ -24,7 +24,7 @@ module Aspera
24
24
  # @return [Hash] normalized status hash
25
25
  def transfer_status(transfer_id, agent_params)
26
26
  app_id = agent_params['application_id']
27
- client = Aspera::JsonRpc::Client.new(Aspera::Rest.new(base_url: desktop_api_url))
27
+ client = Aspera::JsonRpc::Client.new(Aspera::Rest::Client.new(base_url: desktop_api_url))
28
28
  raw = client.get_transfer(app_id: app_id, transfer_id: transfer_id)
29
29
  normalize_status(raw['status'], bytes: raw['bytes_written'].to_i, error: raw['error_desc'])
30
30
  end
@@ -70,7 +70,7 @@ module Aspera
70
70
  begin
71
71
  # curl 'http://127.0.0.1:33024/' -X POST -H 'content-type: application/json' --data-raw '{"jsonrpc":"2.0","params":[],"id":999999,"method":"rpc.discover"}'
72
72
  # https://playground.open-rpc.org/?schemaUrl=http://127.0.0.1:33024
73
- @client_app_api = Aspera::JsonRpc::Client.new(Aspera::Rest.new(base_url: aspera_client_api_url))
73
+ @client_app_api = Aspera::JsonRpc::Client.new(Aspera::Rest::Client.new(base_url: aspera_client_api_url))
74
74
  client_info = @client_app_api.get_info
75
75
  Log.dump(:client_version, client_info)
76
76
  Log.log.debug('Client was reached') if method_index > 0
@@ -121,19 +121,15 @@ module Aspera
121
121
  end
122
122
  when 'completed'
123
123
  notify_progress(:session_end, session_id: @transfer_id)
124
- notify_progress(:end)
125
124
  break
126
125
  when 'failed'
127
126
  notify_progress(:session_end, session_id: @transfer_id)
128
- notify_progress(:end)
129
127
  raise Transfer::Error, transfer['error_desc']
130
128
  when 'cancelled'
131
129
  notify_progress(:session_end, session_id: @transfer_id)
132
- notify_progress(:end)
133
130
  raise Transfer::Error, 'Transfer cancelled by user'
134
131
  else
135
132
  notify_progress(:session_end, session_id: @transfer_id)
136
- notify_progress(:end)
137
133
  raise Transfer::Error, "unknown status: #{transfer['status']}: #{transfer['error_desc']}"
138
134
  end
139
135
  sleep(1)
@@ -14,6 +14,8 @@ require 'socket'
14
14
  require 'securerandom'
15
15
  require 'shellwords'
16
16
  require 'English'
17
+ require 'aspera/rainbow'
18
+ using Rainbow
17
19
 
18
20
  module Aspera
19
21
  module Agent
@@ -105,8 +107,6 @@ module Aspera
105
107
  @sessions = []
106
108
  # mutex protects global data accessed by threads
107
109
  @mutex = Mutex.new
108
- @pre_calc_sent = false
109
- @pre_calc_last_size = nil
110
110
  # Check on all management messages if that file exists, and if so, read commands from it
111
111
  @command_file = File.join(config_dir || '.', "send_#{$PROCESS_ID}")
112
112
  end
@@ -162,6 +162,11 @@ module Aspera
162
162
  error: nil, # exception if failed
163
163
  io: nil, # management port server socket
164
164
  token_regenerator: token_regenerator, # regenerate bearer token with oauth
165
+ progress_id: nil, # session identifier for progress, one per `ascp` process, same on resume
166
+ size_sent: false, # `true` when total size was notified (PreTransferBytes)
167
+ last_size: nil, # last notified transferred size
168
+ skipped: 0, # size of whole files already at destination
169
+ last_skipped: nil, # last notified size not transferred (already at destination)
165
170
  # env vars and args for ascp (from transfer spec)
166
171
  exec_spec: Transfer::Parameters.new(transfer_spec, **@tr_opts).ascp_args
167
172
  }
@@ -208,7 +213,6 @@ module Aspera
208
213
  session[:thread].join
209
214
  result.push(session[:error] || :success)
210
215
  end
211
- notify_progress(:end)
212
216
  Log.log.debug('all transfers joined')
213
217
  # since all are finished and we return the result, clear statuses
214
218
  @sessions.clear
@@ -258,7 +262,7 @@ module Aspera
258
262
  @resume_policy.execute_with_resume do
259
263
  start_and_monitor_process(session: session, exec_spec: session[:exec_spec])
260
264
  end
261
- Log.log.debug('transfer ok'.bg_green)
265
+ Log.log.debug('transfer ok'.bg(:green))
262
266
  rescue => e
263
267
  session[:error] = e
264
268
  raise if Log.log.debug? || !e.is_a?(Transfer::Error)
@@ -345,7 +349,7 @@ module Aspera
345
349
  # store session identifier
346
350
  session[:id] = event['SessionId'] if event['Type'].eql?('INIT')
347
351
  @management_cb&.call(event)
348
- process_progress(event)
352
+ process_progress(session, event)
349
353
  next unless File.exist?(@command_file)
350
354
  begin
351
355
  commands = JSON.parse(File.read(@command_file))
@@ -371,7 +375,9 @@ module Aspera
371
375
  env['ASPERA_SCP_TOKEN'] = session[:token_regenerator].refreshed_transfer_token
372
376
  end
373
377
  raise Transfer::Error.new(last_event['Description'], code: last_event['Code'].to_i)
374
- else Aspera.error_unexpected_value(last_event['Type'], :error) { 'last event type' }
378
+ else
379
+ # e.g. process was killed: like a lost connection (code 16), so that transfer is resumed
380
+ raise Transfer::Error.new("#{exec} ended without final status (last event: #{last_event['Type']})", code: 16)
375
381
  end
376
382
  rescue SystemCallError => e
377
383
  # Process.spawn failed, or socket error
@@ -416,39 +422,63 @@ module Aspera
416
422
  attr_reader :sessions
417
423
 
418
424
  # Notify progress to callback
419
- # @param event [Hash] management port event
420
- def process_progress(event)
421
- session_id = event['SessionId']
425
+ # @param session [Hash] This session information, progress state is stored in it
426
+ # @param event [Hash] management port event
427
+ def process_progress(session, event)
428
+ # Not the `ascp` session id: must be the same when session is resumed
429
+ progress_id = session[:progress_id] ||= SecureRandom.uuid
422
430
  case event['Type']
423
431
  when 'INIT'
424
- @pre_calc_sent = false
425
- @pre_calc_last_size = nil
426
- notify_progress(:session_start, session_id: session_id)
432
+ session[:size_sent] = false
433
+ session[:last_size] = nil
434
+ session[:skipped] = 0
435
+ session[:last_skipped] = nil
436
+ notify_progress(:session_start, session_id: progress_id)
427
437
  when 'NOTIFICATION' # sent from remote
428
438
  if event.key?('PreTransferBytes')
429
- @pre_calc_sent = true
430
- notify_progress(:session_size, session_id: session_id, info: event['PreTransferBytes'])
439
+ session[:size_sent] = true
440
+ notify_progress(:session_size, session_id: progress_id, info: event['PreTransferBytes'])
431
441
  end
432
442
  when 'STATS' # during transfer
433
- @pre_calc_last_size = event['TransferBytes'].to_i + event['StartByte'].to_i
434
- notify_progress(:transfer, session_id: session_id, info: @pre_calc_last_size)
443
+ notify_transferred(session, event)
444
+ when 'STOP' # one file is completed
445
+ size = event['Size'].to_i
446
+ # Whole file already at destination (resume): not transferred, and not in `FileBytes`.
447
+ # Note: in multi-session, each session notifies all such files: progress bar limits progress to size of session.
448
+ session[:skipped] += size if size.positive? && event['StartByte'].to_i.eql?(size)
449
+ notify_transferred(session, event)
435
450
  when 'DONE', 'ERROR' # end of session
436
- total_size = event['TransferBytes'].to_i + event['StartByte'].to_i
437
- notify_progress(:session_size, session_id: session_id, info: total_size) if !@pre_calc_sent && !total_size.zero?
438
- notify_progress(:transfer, session_id: session_id, info: total_size) if @pre_calc_last_size != total_size
439
- notify_progress(:session_end, session_id: session_id)
451
+ notify_transferred(session, event)
452
+ total_size = session[:last_size] + session[:last_skipped]
453
+ notify_progress(:session_size, session_id: progress_id, info: total_size) if !session[:size_sent] && !total_size.zero?
454
+ notify_progress(:session_end, session_id: progress_id)
440
455
  # cspell:disable
441
456
  when 'SESSION'
442
457
  when 'ARGSTOP'
443
458
  when 'FILEERROR'
444
459
  Log.log.error { "#{event['Type']} #{event['Description']}" }
445
- when 'STOP'
446
460
  # cspell:enable
447
- # stop event when one file is completed
448
461
  else
449
462
  Log.log.debug { "Unknown event type for progress: #{event['Type']}" }
450
463
  end
451
464
  end
465
+
466
+ # Notify transferred size, and size not transferred because already at destination (resume), if changed
467
+ # @param session [Hash] This session information, progress state is stored in it
468
+ # @param event [Hash] management port event, with `TransferBytes` and `FileBytes`
469
+ def notify_transferred(session, event)
470
+ transferred = event['TransferBytes'].to_i
471
+ # `FileBytes` also includes the part of files already at destination.
472
+ # Note: `StartByte` is not used: in multi-session, it is the offset of the part of file transferred by the session.
473
+ skipped = session[:skipped] + [event['FileBytes'].to_i - transferred, 0].max
474
+ if session[:last_skipped] != skipped
475
+ session[:last_skipped] = skipped
476
+ notify_progress(:skip, session_id: session[:progress_id], info: skipped)
477
+ end
478
+ return if session[:last_size].eql?(transferred)
479
+ session[:last_size] = transferred
480
+ notify_progress(:transfer, session_id: session[:progress_id], info: transferred)
481
+ end
452
482
  end
453
483
  end
454
484
  end
@@ -24,7 +24,7 @@ module Aspera
24
24
  else
25
25
  rest_params[:auth] = {type: :basic, username: agent_params['username'], password: agent_params['password']}
26
26
  end
27
- node_api = Rest.new(**rest_params)
27
+ node_api = Rest::Client.new(**rest_params)
28
28
  data = node_api.read("ops/transfers/#{transfer_id}") || {'status' => 'unknown'}
29
29
  normalize_status(data)
30
30
  end
@@ -38,6 +38,9 @@ module Aspera
38
38
  result['ended_at'] = Time.now.utc.iso8601
39
39
  when 'failed'
40
40
  result.merge!('status' => 'failed', 'ended_at' => Time.now.utc.iso8601, 'error' => data['error_desc'].to_s)
41
+ when 'canceled'
42
+ result['status'] = 'cancelled'
43
+ result['ended_at'] = Time.now.utc.iso8601
41
44
  else
42
45
  result['status'] = 'running'
43
46
  end
@@ -71,7 +74,7 @@ module Aspera
71
74
  password: password
72
75
  }
73
76
  end
74
- @node_api = Rest.new(**rest_params)
77
+ @node_api = Rest::Client.new(**rest_params)
75
78
  # TODO: currently only supports one transfer. This is bad shortcut. but ok for CLI.
76
79
  @transfer_id = nil
77
80
  end
@@ -109,7 +112,7 @@ module Aspera
109
112
  # status is empty sometimes with status 200...
110
113
  transfer_data = node_api_.read("ops/transfers/#{@transfer_id}") || {'status' => 'unknown'} rescue {'status' => 'waiting(api error)'}
111
114
  case transfer_data['status']
112
- when 'waiting', 'partially_completed', 'unknown', 'waiting(read error)', 'waiting(api error)'
115
+ when 'waiting', 'partially_completed', 'paused', 'unknown', 'waiting(read error)', 'waiting(api error)'
113
116
  notify_progress(:sessions_init, info: transfer_data['status'])
114
117
  when 'running'
115
118
  if !session_started
@@ -129,13 +132,11 @@ module Aspera
129
132
  when 'completed'
130
133
  notify_progress(:transfer, session_id: @transfer_id, info: bytes_expected) if bytes_expected
131
134
  notify_progress(:session_end, session_id: @transfer_id)
132
- notify_progress(:end)
133
135
  break
134
- when 'failed'
136
+ when 'failed', 'canceled'
135
137
  notify_progress(:session_end, session_id: @transfer_id)
136
- notify_progress(:end)
137
138
  # Bug in HSTS ? transfer is marked failed, but there is no reason
138
- break if transfer_data['error_code'].eql?(0) && transfer_data['error_desc'].empty?
139
+ break if transfer_data['status'].eql?('failed') && transfer_data['error_code'].eql?(0) && transfer_data['error_desc'].empty?
139
140
  raise Transfer::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
140
141
  else Aspera.error_unexpected_value(transfer_data['status']) { "transfer_data -> #{transfer_data}" }
141
142
  end
@@ -149,7 +150,7 @@ module Aspera
149
150
 
150
151
  # used internally to ensure node api is set before using.
151
152
  def node_api_
152
- Aspera.assert(!@node_api.nil?, 'Before using this object, set the node_api attribute to a Aspera::Rest object')
153
+ Aspera.assert(!@node_api.nil?, 'Before using this object, set the node_api attribute to a Aspera::Rest::Client object')
153
154
  return @node_api
154
155
  end
155
156
  end
@@ -140,6 +140,8 @@ module Aspera
140
140
 
141
141
  # Actual endpoint the daemon is listening on (resolved after connect, e.g. when port 0 was used)
142
142
  attr_reader :daemon_endpoint
143
+ # gRPC client of the daemon
144
+ attr_reader :transfer_client
143
145
 
144
146
  # :reek:UnusedParameters token_regenerator
145
147
  def start_transfer(transfer_spec, token_regenerator: nil)
@@ -181,11 +183,9 @@ module Aspera
181
183
  when :COMPLETED
182
184
  notify_progress(:transfer, session_id: @transfer_id, info: bytes_expected) if bytes_expected
183
185
  notify_progress(:session_end, session_id: @transfer_id)
184
- notify_progress(:end)
185
186
  break
186
187
  when :FAILED, :CANCELED
187
188
  notify_progress(:session_end, session_id: @transfer_id)
188
- notify_progress(:end)
189
189
  raise Transfer::Error, JSON.parse(response.message)['Description']
190
190
  when :QUEUED, :UNKNOWN_STATUS, :PAUSED, :ORPHANED
191
191
  notify_progress(:sessions_init, info: response.status.to_s.downcase)
@@ -4,7 +4,7 @@ require 'aspera/api/aoc'
4
4
  module Aspera
5
5
  module Api
6
6
  # Aspera License Entitlement Engine API client
7
- class Alee < Aspera::Rest
7
+ class Alee < Aspera::Rest::Client
8
8
  def initialize(entitlement_id, customer_id, api_domain: AoC::SAAS_DOMAIN_PROD, version: 'v1')
9
9
  super(
10
10
  base_url: "https://api.#{api_domain}/metering/#{version}",
@@ -7,16 +7,18 @@ require 'aspera/hash_ext'
7
7
  require 'aspera/data_repository'
8
8
  require 'aspera/transfer/spec'
9
9
  require 'aspera/api/node'
10
- require 'aspera/rest_list'
10
+ require 'aspera/rest/list'
11
11
  require 'base64'
12
+ require 'aspera/rainbow'
13
+ using Rainbow
12
14
 
13
15
  module Aspera
14
16
  module Api
15
17
  # information relative to Files or Packages App
16
18
 
17
19
  # Aspera on Cloud API client
18
- class AoC < Rest
19
- include RestList
20
+ class AoC < Rest::Client
21
+ include Rest::List
20
22
 
21
23
  module AppType
22
24
  FILES = 'files'
@@ -140,7 +142,7 @@ module Aspera
140
142
  # @param url [String] URL of AoC public link
141
143
  # @return [Hash{Symbol => String, Hash}] Information about public link, or nil if not a public link
142
144
  def link_info(url)
143
- final_uri = Rest.new(base_url: url, redirect_max: MAX_AOC_URL_REDIRECT).call(operation: 'GET', ret: :resp).uri
145
+ final_uri = Rest::Client.new(base_url: url, redirect_max: MAX_AOC_URL_REDIRECT).call(operation: 'GET', ret: :resp).uri
144
146
  Log.dump(:final_uri, final_uri, level: :trace1)
145
147
  org_domain = split_org_domain(final_uri)
146
148
  if (m = final_uri.path.match(%r{/oauth2/([^/]+)/login$}))
@@ -202,8 +204,8 @@ module Aspera
202
204
  Aspera.assert(block_given?, 'block required for call_paging')
203
205
  # set default large page if user does not specify own parameters. AoC Caps to 1000 anyway
204
206
  query['per_page'] = 1000 unless query.key?('per_page')
205
- max_items = query.delete(RestList::MAX_ITEMS)
206
- max_pages = query.delete(RestList::MAX_PAGES)
207
+ max_items = query.delete(Rest::List::MAX_ITEMS)
208
+ max_pages = query.delete(Rest::List::MAX_PAGES)
207
209
  item_list = []
208
210
  total_count = nil
209
211
  current_page = query['page']
@@ -225,9 +227,9 @@ module Aspera
225
227
  break if !max_items.nil? && item_list.count >= max_items
226
228
  break if !max_pages.nil? && page_count >= max_pages
227
229
  break if total_count&.<=(item_list.count)
228
- RestParameters.instance.spinner_cb.call("#{item_list.count} / #{total_count}") unless total_count.eql?(item_list.count.to_s)
230
+ Rest::Parameters.instance.spinner_cb.call("#{item_list.count} / #{total_count}") unless total_count.eql?(item_list.count.to_s)
229
231
  end
230
- RestParameters.instance.spinner_cb.call(action: :success)
232
+ Rest::Parameters.instance.spinner_cb.call(action: :success)
231
233
  item_list = item_list[0..max_items - 1] if !max_items.nil? && item_list.count > max_items
232
234
  return {items: item_list, total: total_count}
233
235
  end
@@ -394,7 +396,7 @@ module Aspera
394
396
  @cache_user_info =
395
397
  begin
396
398
  read('self')
397
- rescue Aspera::RestCallError => e
399
+ rescue Aspera::Rest::CallError => e
398
400
  raise if exception || e.message.include?('invalid_grant')
399
401
  Log.log.debug { "Ignoring error: (#{e.class}) #{e}" }
400
402
  {}
@@ -496,9 +498,9 @@ module Aspera
496
498
  }
497
499
  else
498
500
  # OAuth bearer token
499
- node_params[:auth] = auth_params.clone
500
- node_params[:auth][:params] ||= {}
501
- node_params[:auth][:params][:scope] = Node.token_scope(node_info['access_key'], scope)
501
+ # copy nested params, so that auth params of this API are not modified
502
+ node_params[:auth] = auth_params.merge(params: auth_params[:params]&.dup || {})
503
+ node_params[:auth][:params][:scope] = Node::Scope.join(access_key: node_info['access_key'], scope: scope)
502
504
  node_params[:auth][:params][:owner_access] = true if scope.eql?(Node::Scope::ADMIN)
503
505
  # Special header required for bearer token only
504
506
  node_params[:headers] = {Node::HEADER_X_ASPERA_ACCESS_KEY => node_info['access_key']}
@@ -6,7 +6,7 @@ require 'aspera/rest'
6
6
  module Aspera
7
7
  module Api
8
8
  # Aspera Transfer Service API client without authentication
9
- class Ats < Aspera::Rest
9
+ class Ats < Aspera::Rest::Client
10
10
  SERVICE_BASE_URL = 'https://ats.aspera.io'
11
11
  # currently supported clouds
12
12
  # Note to Aspera: shall be an API call
@@ -22,7 +22,7 @@ module Aspera
22
22
  Aspera.assert(service_credentials.key?(field)) { "service_credentials must have a field: #{field}" }
23
23
  end
24
24
  # read endpoints from service provided in service credentials
25
- endpoints = Aspera::Rest.new(base_url: service_credentials['endpoints']).read('')
25
+ endpoints = Aspera::Rest::Client.new(base_url: service_credentials['endpoints']).read('')
26
26
  Log.dump(:endpoints, endpoints)
27
27
  endpoint = endpoints.dig('service-endpoints', 'regional', bucket_region, 'public', bucket_region)
28
28
  Aspera.assert(!endpoint.nil?) { "no such region: #{bucket_region}" }
@@ -40,7 +40,7 @@ module Aspera
40
40
  endpoint = "https://#{endpoint}" unless endpoint.start_with?('http')
41
41
  @auth_url = auth_url
42
42
  @api_key = api_key
43
- s3_api = Aspera::Rest.new(
43
+ s3_api = Aspera::Rest::Client.new(
44
44
  base_url: endpoint,
45
45
  not_auth_codes: %w[401 403], # error codes when not authorized
46
46
  headers: {'ibm-service-instance-id' => instance_id},
@@ -3,7 +3,7 @@
3
3
  require 'aspera/assert'
4
4
  require 'aspera/rest'
5
5
  require 'aspera/oauth/base'
6
- require 'aspera/rest_list'
6
+ require 'aspera/rest/list'
7
7
  require 'digest'
8
8
 
9
9
  module Aspera
@@ -15,6 +15,7 @@ module Aspera
15
15
  # @param context [String] The `context` query parameter in public link
16
16
  # @param redirect_uri [String] URI of web UI login
17
17
  # @param path_authorize [String] Path to provide passcode
18
+ # @param base_params [Hash] Parameters for base class
18
19
  def initialize(
19
20
  context:,
20
21
  redirect_uri:,
@@ -48,18 +49,18 @@ module Aspera
48
49
  raise Error, info['action_message'] if info['action_message']
49
50
  Aspera.assert(info['code'], 'Missing code in answer')
50
51
  # Exchange code for token
51
- return create_token_call(base_params.merge(
52
+ create_token_base(
52
53
  grant_type: 'authorization_code',
53
54
  code: info['code'],
54
55
  redirect_uri: @redirect_uri
55
- ))
56
+ )
56
57
  end
57
58
  end
58
59
  OAuth::Factory.instance.register_token_creator(FaspexPubLink)
59
60
  module Api
60
61
  # Aspera Faspex 5 API Client
61
- class Faspex < Aspera::Rest
62
- include RestList
62
+ class Faspex < Aspera::Rest::Client
63
+ include Rest::List
63
64
 
64
65
  # endpoint for authentication API
65
66
  PATH_AUTH = 'auth'
@@ -137,10 +138,12 @@ module Aspera
137
138
  SENT_MAILBOX_TYPES.include?(box) || box == 'ALL' ? :sent : :received
138
139
  end
139
140
  end
141
+ # @return [Hash, nil] public link context decoded from URL (keys: resource, type, id, passcode, package_id, email), or nil if not a public link
140
142
  attr_reader :pub_link_context
141
143
 
142
144
  # @param url [String] Faspex URL, can be a public link
143
145
  # @param auth [Symbol] Authentication method: :boot (token in header), :web (open browser), :jwt (client_id + private key), :public_link (context in URL)
146
+ # @param root [String] Root path for API
144
147
  # @param password [String, nil] For :boot auth, the token copied directly from browser in developer mode
145
148
  # @param client_id [String, nil] For :web and :jwt auth, the client_id of web UI application
146
149
  # @param client_secret [String, nil] For :web auth, the client_secret of web UI application (not needed for :jwt)
@@ -161,12 +164,13 @@ module Aspera
161
164
  passphrase: nil
162
165
  )
163
166
  auth = :public_link if self.class.public_link?(url)
167
+ # @type [Hash, nil]
164
168
  @pub_link_context = nil
165
169
  super(**
166
170
  case auth
167
171
  when :public_link
168
172
  # Get URL of final redirect of provided public link
169
- final_url = Rest.new(base_url: url, redirect_max: 3).call(operation: 'GET', ret: :resp).uri.to_s
173
+ final_url = Rest::Client.new(base_url: url, redirect_max: 3).call(operation: 'GET', ret: :resp).uri.to_s
170
174
  Log.dump(:final_url, final_url, level: :trace1)
171
175
  # Get context from query
172
176
  encoded_context = Rest.query_to_h(URI.parse(final_url).query)['context']
@@ -178,7 +182,7 @@ module Aspera
178
182
  base_url = final_url.gsub(%r{/public/.*}, '').gsub(/\?.*/, '')
179
183
  # Get web UI client_id and redirect_uri
180
184
  # TODO: change this for something more reliable
181
- config = JSON.parse(Rest.new(base_url: "#{base_url}/config.js", redirect_max: 3).call(operation: 'GET').sub(/^[^=]+=/, '').gsub(/([a-z_]+):/, '"\1":').delete("\n ").tr("'", '"')).symbolize_keys
185
+ config = JSON.parse(Rest::Client.new(base_url: "#{base_url}/config.js", redirect_max: 3).call(operation: 'GET').sub(/^[^=]+=/, '').gsub(/([a-z_]+):/, '"\1":').delete("\n ").tr("'", '"')).symbolize_keys
182
186
  Log.dump(:config_js, config)
183
187
  {
184
188
  base_url: "#{base_url}/#{root}",