aspera-cli 4.27.0 → 4.27.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +46 -1
  4. data/CONTRIBUTING.md +1 -7
  5. data/TODO.md +50 -0
  6. data/docs/README.md +709 -489
  7. data/lib/aspera/agent/base.rb +10 -2
  8. data/lib/aspera/agent/connect.rb +7 -7
  9. data/lib/aspera/agent/desktop.rb +3 -3
  10. data/lib/aspera/agent/direct.rb +48 -19
  11. data/lib/aspera/agent/factory.rb +3 -3
  12. data/lib/aspera/agent/httpgw.rb +55 -14
  13. data/lib/aspera/agent/node.rb +2 -2
  14. data/lib/aspera/agent/transferd.rb +12 -12
  15. data/lib/aspera/api/aoc.rb +28 -28
  16. data/lib/aspera/api/cos_node.rb +10 -9
  17. data/lib/aspera/api/faspex.rb +2 -2
  18. data/lib/aspera/api/httpgw.rb +20 -20
  19. data/lib/aspera/api/node.rb +31 -53
  20. data/lib/aspera/ascmd.rb +13 -13
  21. data/lib/aspera/ascp/installation.rb +43 -36
  22. data/lib/aspera/ascp/management.rb +6 -6
  23. data/lib/aspera/assert.rb +12 -13
  24. data/lib/aspera/cli/ascp_actions.rb +7 -7
  25. data/lib/aspera/cli/async_transfer_store.rb +49 -5
  26. data/lib/aspera/cli/bootstrapper.rb +9 -9
  27. data/lib/aspera/cli/command_registry.rb +1 -1
  28. data/lib/aspera/cli/command_spec.rb +6 -3
  29. data/lib/aspera/cli/context.rb +3 -3
  30. data/lib/aspera/cli/extended_value.rb +26 -26
  31. data/lib/aspera/cli/formatter.rb +76 -54
  32. data/lib/aspera/cli/gem_checker.rb +1 -1
  33. data/lib/aspera/cli/hints.rb +2 -2
  34. data/lib/aspera/cli/http.rb +22 -29
  35. data/lib/aspera/cli/mailer.rb +7 -7
  36. data/lib/aspera/cli/mcp_tool.rb +31 -4
  37. data/lib/aspera/cli/option_declarator.rb +77 -0
  38. data/lib/aspera/cli/options.schema.yaml +273 -3
  39. data/lib/aspera/cli/parser.rb +539 -280
  40. data/lib/aspera/cli/plugins/alee.rb +1 -1
  41. data/lib/aspera/cli/plugins/aoc.rb +303 -314
  42. data/lib/aspera/cli/plugins/ats.rb +25 -21
  43. data/lib/aspera/cli/plugins/base.rb +357 -186
  44. data/lib/aspera/cli/plugins/basic_auth.rb +0 -12
  45. data/lib/aspera/cli/plugins/config.rb +76 -73
  46. data/lib/aspera/cli/plugins/console.rb +8 -8
  47. data/lib/aspera/cli/plugins/cos.rb +4 -3
  48. data/lib/aspera/cli/plugins/factory.rb +2 -2
  49. data/lib/aspera/cli/plugins/faspex5.rb +259 -248
  50. data/lib/aspera/cli/plugins/faspio.rb +7 -2
  51. data/lib/aspera/cli/plugins/httpgw.rb +1 -1
  52. data/lib/aspera/cli/plugins/mcp.rb +11 -11
  53. data/lib/aspera/cli/plugins/node.rb +227 -283
  54. data/lib/aspera/cli/plugins/orchestrator.rb +23 -23
  55. data/lib/aspera/cli/plugins/preview.rb +61 -64
  56. data/lib/aspera/cli/plugins/server.rb +36 -31
  57. data/lib/aspera/cli/plugins/shares.rb +174 -135
  58. data/lib/aspera/cli/preset_actions.rb +39 -19
  59. data/lib/aspera/cli/preset_manager.rb +74 -32
  60. data/lib/aspera/cli/result.rb +18 -16
  61. data/lib/aspera/cli/runner.rb +75 -39
  62. data/lib/aspera/cli/secret_finder.rb +1 -1
  63. data/lib/aspera/cli/sync_actions.rb +36 -35
  64. data/lib/aspera/cli/terminal_formatter.rb +2 -2
  65. data/lib/aspera/cli/transfer_actions.rb +17 -10
  66. data/lib/aspera/cli/transfer_agent.rb +44 -44
  67. data/lib/aspera/cli/transfer_progress.rb +7 -7
  68. data/lib/aspera/cli/vault_manager.rb +34 -12
  69. data/lib/aspera/cli/version.rb +1 -1
  70. data/lib/aspera/cli/wizard.rb +17 -20
  71. data/lib/aspera/colors.rb +3 -3
  72. data/lib/aspera/command_line_builder.rb +17 -16
  73. data/lib/aspera/command_line_converter.rb +1 -1
  74. data/lib/aspera/coverage.rb +1 -2
  75. data/lib/aspera/data_repository.rb +1 -1
  76. data/lib/aspera/dot_container.rb +8 -8
  77. data/lib/aspera/environment.rb +13 -13
  78. data/lib/aspera/faspex_gw.rb +4 -4
  79. data/lib/aspera/faspex_postproc.rb +14 -9
  80. data/lib/aspera/graphql.rb +4 -2
  81. data/lib/aspera/hash_ext.rb +2 -2
  82. data/lib/aspera/json_rpc/client.rb +1 -1
  83. data/lib/aspera/keychain/base.rb +10 -3
  84. data/lib/aspera/keychain/encrypted_hash.rb +92 -17
  85. data/lib/aspera/keychain/factory.rb +17 -5
  86. data/lib/aspera/keychain/hashicorp_vault.rb +1 -1
  87. data/lib/aspera/keychain/macos_security.rb +23 -25
  88. data/lib/aspera/keychain/one_password_api.rb +86 -0
  89. data/lib/aspera/keychain/one_password_base.rb +34 -0
  90. data/lib/aspera/keychain/one_password_cli.rb +98 -0
  91. data/lib/aspera/link_header.rb +2 -2
  92. data/lib/aspera/log.rb +13 -13
  93. data/lib/aspera/markdown.rb +5 -5
  94. data/lib/aspera/nagios.rb +8 -8
  95. data/lib/aspera/node_simulator.rb +33 -27
  96. data/lib/aspera/oauth/base.rb +4 -5
  97. data/lib/aspera/oauth/boot.rb +3 -3
  98. data/lib/aspera/oauth/factory.rb +5 -5
  99. data/lib/aspera/oauth/jwt.rb +5 -7
  100. data/lib/aspera/oauth/web.rb +4 -4
  101. data/lib/aspera/persistency_action_once.rb +4 -4
  102. data/lib/aspera/persistency_folder.rb +8 -8
  103. data/lib/aspera/preview/file_types.rb +4 -4
  104. data/lib/aspera/preview/generator.rb +49 -46
  105. data/lib/aspera/preview/options.rb +3 -2
  106. data/lib/aspera/preview/terminal.rb +5 -5
  107. data/lib/aspera/preview/utils.rb +65 -18
  108. data/lib/aspera/products/connect.rb +2 -2
  109. data/lib/aspera/products/desktop.rb +1 -1
  110. data/lib/aspera/products/other.rb +2 -2
  111. data/lib/aspera/products/transferd.rb +3 -3
  112. data/lib/aspera/proxy_auto_config.rb +10 -10
  113. data/lib/aspera/rest.rb +149 -115
  114. data/lib/aspera/rest_error_analyzer.rb +3 -3
  115. data/lib/aspera/rest_list.rb +10 -9
  116. data/lib/aspera/schema/IBM Aspera Node API-4.4.6.yaml +6232 -0
  117. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +249 -0
  118. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +39 -0
  119. data/lib/aspera/schema/IBM_Aspera_Shares.yaml +4653 -0
  120. data/lib/aspera/schema/documentation.rb +17 -11
  121. data/lib/aspera/schema/reader.rb +78 -4
  122. data/lib/aspera/schema/registry.rb +26 -7
  123. data/lib/aspera/secret_hider.rb +5 -5
  124. data/lib/aspera/ssh.rb +64 -31
  125. data/lib/aspera/ssl.rb +3 -3
  126. data/lib/aspera/sync/database.rb +2 -2
  127. data/lib/aspera/sync/operations.rb +13 -13
  128. data/lib/aspera/temp_file_manager.rb +24 -4
  129. data/lib/aspera/transfer/faux_file.rb +2 -2
  130. data/lib/aspera/transfer/parameters.rb +15 -15
  131. data/lib/aspera/transfer/resumer.rb +9 -9
  132. data/lib/aspera/transfer/spec.rb +1 -1
  133. data/lib/aspera/transfer/spec.schema.yaml +1 -0
  134. data/lib/aspera/uri_reader.rb +52 -15
  135. data/lib/aspera/web_auth.rb +6 -5
  136. data/lib/aspera/web_server_simple.rb +9 -9
  137. data/lib/aspera/yaml.rb +4 -4
  138. data.tar.gz.sig +0 -0
  139. metadata +10 -4
  140. metadata.gz.sig +0 -0
  141. data/lib/aspera/cli/plugins/faspex.rb +0 -576
  142. data/lib/aspera/transfer/uri.rb +0 -56
@@ -28,11 +28,19 @@ module Aspera
28
28
  statuses = wait_for_transfers_completion
29
29
  @progress&.reset
30
30
  Aspera.assert_type(statuses, Array)
31
- Aspera.assert(statuses.none?{ |i| !i.eql?(:success) && !i.is_a?(StandardError)}){"bad statuses content: #{statuses}"}
32
- errors = statuses.reject{ |i| i.eql?(:success)}
31
+ Aspera.assert(statuses.none? { |i| !i.eql?(:success) && !i.is_a?(StandardError) }) { "bad statuses content: #{statuses}" }
32
+ errors = statuses.reject { |i| i.eql?(:success) }
33
33
  return errors.empty? ? Transfer::Result.success : Transfer::Result.error(errors.first)
34
34
  end
35
35
 
36
+ # Return the job_id of the last transfer submitted by this agent.
37
+ # In-process agents (direct, httpgw) override this to expose their internal job id.
38
+ # Remote-daemon agents do not need to override it: TransferAgent generates a UUID itself.
39
+ # @return [String, nil]
40
+ def last_job_id
41
+ nil
42
+ end
43
+
36
44
  private
37
45
 
38
46
  Aspera.require_method!(:start_transfer)
@@ -66,12 +66,12 @@ module Aspera
66
66
  @connect_settings = {
67
67
  'app_id' => SecureRandom.uuid
68
68
  }
69
- Aspera.assert(Environment.instance.graphical?, type: Error){'Using connect requires a graphical environment'}
69
+ Aspera.assert(Environment.instance.graphical?, type: Error) { 'Using connect requires a graphical environment' }
70
70
  method_index = 0
71
71
  begin
72
72
  # raise exception if connect not started and file does not exist
73
73
  connect_url = self.class.connect_api_url
74
- Log.log.debug{"found: #{connect_url}"}
74
+ Log.log.debug { "found: #{connect_url}" }
75
75
  @connect_api = Rest.new(
76
76
  base_url: "#{connect_url}/v5/connect", # could use v6 also now
77
77
  headers: {'Origin' => RestParameters.instance.user_agent}
@@ -80,11 +80,11 @@ module Aspera
80
80
  Log.log.debug('Connect was reached') if method_index > 0
81
81
  Log.dump(:connect_version, connect_info)
82
82
  rescue StandardError => e # Errno::ECONNREFUSED
83
- Log.log.debug{"Exception: #{e}"}
83
+ Log.log.debug { "Exception: #{e}" }
84
84
  start_url = CONNECT_START_URIS[method_index]
85
85
  method_index += 1
86
- Aspera.assert(!start_url.nil?){"Unable to start connect #{method_index} times"}
87
- Log.log.warn{"Aspera Connect is not started (#{e}). Trying to start it ##{method_index}..."}
86
+ Aspera.assert(!start_url.nil?) { "Unable to start connect #{method_index} times" }
87
+ Log.log.warn { "Aspera Connect is not started (#{e}). Trying to start it ##{method_index}..." }
88
88
  Environment.instance.open_uri_graphical(start_url)
89
89
  sleep(SLEEP_SEC_BETWEEN_RETRY)
90
90
  retry
@@ -99,7 +99,7 @@ module Aspera
99
99
  # :reek:UnusedParameters token_regenerator
100
100
  def start_transfer(transfer_spec, token_regenerator: nil)
101
101
  if transfer_spec['direction'] == 'send'
102
- Log.log.warn{"Connect requires upload selection using GUI, ignoring #{transfer_spec['paths']}".red}
102
+ Log.log.warn { "Connect requires upload selection using GUI, ignoring #{transfer_spec['paths']}".red }
103
103
  transfer_spec.delete('paths')
104
104
  selection = @connect_api.create('windows/select-open-file-dialog/', {
105
105
  'aspera_connect_settings' => @connect_settings,
@@ -108,7 +108,7 @@ module Aspera
108
108
  'allowMultipleSelection' => true,
109
109
  'allowedFileTypes' => ''
110
110
  })
111
- transfer_spec['paths'] = selection['dataTransfer']['files'].map{ |i| {'source' => i['name']}}
111
+ transfer_spec['paths'] = selection['dataTransfer']['files'].map { |i| {'source' => i['name']} }
112
112
  end
113
113
  # if there is a token, we ask connect client to use well known ssh private keys
114
114
  # instead of asking password
@@ -65,7 +65,7 @@ module Aspera
65
65
  @application_id = SecureRandom.uuid
66
66
  @transfer_id = nil
67
67
  super
68
- Aspera.assert(Environment.instance.graphical?, type: Error){'Using client requires a graphical environment'}
68
+ Aspera.assert(Environment.instance.graphical?, type: Error) { 'Using client requires a graphical environment' }
69
69
  method_index = 0
70
70
  begin
71
71
  # curl 'http://127.0.0.1:33024/' -X POST -H 'content-type: application/json' --data-raw '{"jsonrpc":"2.0","params":[],"id":999999,"method":"rpc.discover"}'
@@ -77,8 +77,8 @@ module Aspera
77
77
  rescue Errno::ECONNREFUSED => e
78
78
  start_url = START_URIS[method_index]
79
79
  method_index += 1
80
- Aspera.assert(!start_url.nil?){"Unable to start #{Products::Desktop::APP_NAME} #{method_index} times"}
81
- Log.log.warn{"#{Products::Desktop::APP_NAME} is not started (#{e}). Trying to start it ##{method_index}..."}
80
+ Aspera.assert(!start_url.nil?) { "Unable to start #{Products::Desktop::APP_NAME} #{method_index} times" }
81
+ Log.log.warn { "#{Products::Desktop::APP_NAME} is not started (#{e}). Trying to start it ##{method_index}..." }
82
82
  Environment.instance.open_uri_graphical(start_url)
83
83
  sleep(SLEEP_SEC_BETWEEN_RETRY)
84
84
  retry
@@ -19,6 +19,30 @@ module Aspera
19
19
  module Agent
20
20
  # Execute a local `ascp` and use its management port to monitor progress
21
21
  class Direct < Base
22
+ class << self
23
+ # Re-query the status of a direct transfer that was started in-process.
24
+ # Only works if the same Ruby process is still alive (e.g. MCP mode).
25
+ # agent_params must contain '_agent_ref' pointing to the live Direct instance.
26
+ # @param transfer_id [String] job_id returned by start_transfer / last_job_id
27
+ # @param agent_params [Hash] connection parameters; '_agent_ref' is the live agent
28
+ # @return [Hash] status hash with at least 'status' key
29
+ def transfer_status(transfer_id, agent_params)
30
+ agent = agent_params['_agent_ref']
31
+ return {'status' => 'unknown', 'note' => 'direct agent not available (different process?)'} if agent.nil?
32
+ sessions = agent.sessions_by_job(transfer_id)
33
+ return {'status' => 'unknown', 'note' => "no sessions found for job #{transfer_id}"} if sessions.empty?
34
+ errors = sessions.filter_map { |s| s[:error] }
35
+ running = sessions.any? { |s| s[:thread]&.alive? }
36
+ if errors.any?
37
+ {'status' => 'failed', 'error' => errors.first.message}
38
+ elsif running
39
+ {'status' => 'running'}
40
+ else
41
+ {'status' => 'completed'}
42
+ end
43
+ end
44
+ end
45
+
22
46
  # `ascp` started locally, so listen local
23
47
  LISTEN_LOCAL_ADDRESS = '127.0.0.1'
24
48
  # 0 means: use any available port
@@ -75,7 +99,7 @@ module Aspera
75
99
  @multi_incr_udp = multi_incr_udp.nil? ? Environment.instance.os.eql?(Environment::OS_WINDOWS) : multi_incr_udp
76
100
  @monitor = monitor
77
101
  @management_cb = management_cb
78
- Aspera.assert_type(resume, Hash){'resume'}
102
+ Aspera.assert_type(resume, Hash) { 'resume' }
79
103
  @resume_policy = Transfer::Resumer.new(**resume.symbolize_keys)
80
104
  # all transfer jobs, key = SecureRandom.uuid, protected by mutex, cond var on change
81
105
  @sessions = []
@@ -101,7 +125,7 @@ module Aspera
101
125
  # using a non unique id results in discard of tags in AoC, and a package is never finalized
102
126
  # all sessions in a multi-session transfer must have the same xfer_id (see admin manual)
103
127
  transfer_spec['tags'][Transfer::Spec::TAG_RESERVED]['xfer_id'] ||= SecureRandom.uuid
104
- Log.log.debug{"xfer id=#{transfer_spec['xfer_id']}"}
128
+ Log.log.debug { "xfer id=#{transfer_spec['xfer_id']}" }
105
129
  # TODO: useful ? node only ? seems to be a timeout for retry in node
106
130
  transfer_spec['tags'][Transfer::Spec::TAG_RESERVED]['xfer_retry'] ||= 3600
107
131
  end
@@ -115,7 +139,7 @@ module Aspera
115
139
  count: transfer_spec.delete('multi_session').to_i
116
140
  }
117
141
  if multi_session_info[:count].negative?
118
- Log.log.error{"multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0"}
142
+ Log.log.error { "multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0" }
119
143
  multi_session_info = nil
120
144
  elsif multi_session_info[:count].eql?(0)
121
145
  Log.log.debug('multi_session count is zero: no multi session')
@@ -145,7 +169,7 @@ module Aspera
145
169
  if multi_session_info.nil?
146
170
  Log.log.debug('Starting single session thread')
147
171
  # single session for transfer : simple
148
- session[:thread] = Thread.new{transfer_thread_entry(session)}
172
+ session[:thread] = Thread.new { transfer_thread_entry(session) }
149
173
  @sessions.push(session)
150
174
  else
151
175
  Log.log.debug('Starting multi session threads')
@@ -161,13 +185,18 @@ module Aspera
161
185
  # option: increment (default as per ascp manual) or not (cluster on other side ?)
162
186
  exec_spec.args.unshift('-O', (multi_session_info[:udp_base] + i - 1).to_s) if @multi_incr_udp
163
187
  # finally start the thread
164
- this_session[:thread] = Thread.new{transfer_thread_entry(this_session)}
188
+ this_session[:thread] = Thread.new { transfer_thread_entry(this_session) }
165
189
  @sessions.push(this_session)
166
190
  end
167
191
  end
168
192
  return session[:job_id]
169
193
  end
170
194
 
195
+ # @return [String, nil] job_id of the last submitted transfer
196
+ def last_job_id
197
+ @sessions.last&.dig(:job_id)
198
+ end
199
+
171
200
  # wait for completion of all jobs started
172
201
  # @return [Array] list of :success or error message
173
202
  def wait_for_transfers_completion
@@ -175,7 +204,7 @@ module Aspera
175
204
  # set to non-nil to exit loop
176
205
  result = []
177
206
  @sessions.each do |session|
178
- Log.log.debug{"join #{session[:thread]}"}
207
+ Log.log.debug { "join #{session[:thread]}" }
179
208
  session[:thread].join
180
209
  result.push(session[:error] || :success)
181
210
  end
@@ -194,7 +223,7 @@ module Aspera
194
223
  # @param id [String] Transfer session identifier
195
224
  # @return [Array] list of sessions for a job
196
225
  def sessions_by_job(id)
197
- @sessions.select{ |session| session[:job_id].eql?(id)}
226
+ @sessions.select { |session| session[:job_id].eql?(id) }
198
227
  end
199
228
 
200
229
  # Send command to management port of command (used in `asession).
@@ -224,7 +253,7 @@ module Aspera
224
253
  begin
225
254
  # set name for logging
226
255
  Thread.current[:name] = 'transfer'
227
- Log.log.debug{"ENTER (#{Thread.current[:name]})"}
256
+ Log.log.debug { "ENTER (#{Thread.current[:name]})" }
228
257
  # start transfer with selected resumer policy
229
258
  @resume_policy.execute_with_resume do
230
259
  start_and_monitor_process(session: session, exec_spec: session[:exec_spec])
@@ -234,7 +263,7 @@ module Aspera
234
263
  session[:error] = e
235
264
  raise if Log.log.debug? || !e.is_a?(Transfer::Error)
236
265
  end
237
- Log.log.debug{"EXIT (#{Thread.current[:name]})"}
266
+ Log.log.debug { "EXIT (#{Thread.current[:name]})" }
238
267
  end
239
268
 
240
269
  public
@@ -293,7 +322,7 @@ module Aspera
293
322
  return unless @monitor
294
323
  # TODO: timeout does not work when Process.spawn is used... until process exits, then it works
295
324
  # So we use select to detect that anything happens on the socket (connection)
296
- Log.log.debug{"before select, timeout: #{@spawn_timeout_sec}"}
325
+ Log.log.debug { "before select, timeout: #{@spawn_timeout_sec}" }
297
326
  readable, _, _ = IO.select([mgt_server_socket], nil, nil, @spawn_timeout_sec)
298
327
  Log.log.debug('after select, before accept')
299
328
  Aspera.assert(readable, 'timeout waiting mgt port connect (select not readable)', type: Transfer::Error)
@@ -322,17 +351,17 @@ module Aspera
322
351
  commands = JSON.parse(File.read(@command_file))
323
352
  send_command(commands)
324
353
  rescue => e
325
- Log.log.error{e.to_s}
354
+ Log.log.error { e.to_s }
326
355
  end
327
356
  File.delete(@command_file)
328
357
  end
329
358
  Log.log.debug('management io closed')
330
359
  # check that last status was received before process exit
331
360
  last_event = processor.last_event
332
- Aspera.assert_type(last_event, Hash, type: Transfer::Error){'management event'}
361
+ Aspera.assert_type(last_event, Hash, type: Transfer::Error) { 'management event' }
333
362
  case last_event['Type']
334
363
  when 'DONE'
335
- Log.log.trace1{'Graceful shutdown, DONE message received'}
364
+ Log.log.trace1 { 'Graceful shutdown, DONE message received' }
336
365
  when 'ERROR'
337
366
  if /bearer token/i.match?(last_event['Description']) &&
338
367
  session[:token_regenerator].respond_to?(:refreshed_transfer_token)
@@ -342,7 +371,7 @@ module Aspera
342
371
  env['ASPERA_SCP_TOKEN'] = session[:token_regenerator].refreshed_transfer_token
343
372
  end
344
373
  raise Transfer::Error.new(last_event['Description'], code: last_event['Code'].to_i)
345
- else Aspera.error_unexpected_value(last_event['Type'], :error){'last event type'}
374
+ else Aspera.error_unexpected_value(last_event['Type'], :error) { 'last event type' }
346
375
  end
347
376
  rescue SystemCallError => e
348
377
  # Process.spawn failed, or socket error
@@ -361,11 +390,11 @@ module Aspera
361
390
  # process stderr of ascp
362
391
  stderr_flag = false
363
392
  stderr_r.each_line do |line|
364
- Log.log.error{"BEGIN stderr #{exec}"} unless stderr_flag
365
- Log.log.error{line.chomp}
393
+ Log.log.error { "BEGIN stderr #{exec}" } unless stderr_flag
394
+ Log.log.error { line.chomp }
366
395
  stderr_flag = true
367
396
  end
368
- Log.log.error{"END stderr #{exec}"} if stderr_flag
397
+ Log.log.error { "END stderr #{exec}" } if stderr_flag
369
398
  stderr_r.close
370
399
  end
371
400
  # status is nil if an exception occurred before starting command
@@ -412,12 +441,12 @@ module Aspera
412
441
  when 'SESSION'
413
442
  when 'ARGSTOP'
414
443
  when 'FILEERROR'
415
- Log.log.error{"#{event['Type']} #{event['Description']}"}
444
+ Log.log.error { "#{event['Type']} #{event['Description']}" }
416
445
  when 'STOP'
417
446
  # cspell:enable
418
447
  # stop event when one file is completed
419
448
  else
420
- Log.log.debug{"Unknown event type for progress: #{event['Type']}"}
449
+ Log.log.debug { "Unknown event type for progress: #{event['Type']}" }
421
450
  end
422
451
  end
423
452
  end
@@ -24,9 +24,9 @@ module Aspera
24
24
  # Available agents: :long : Capitalized name string, :short : single character symbol
25
25
  ALL =
26
26
  Dir.children(File.dirname(File.expand_path(__FILE__)))
27
- .select{ |file| file.end_with?(Environment::RB_EXT)}
28
- .map{ |file| File.basename(file, Environment::RB_EXT).to_sym}
29
- .reject{ |item| IGNORED_ITEMS.include?(item)}.to_h do |agent_sym|
27
+ .select { |file| file.end_with?(Environment::RB_EXT) }
28
+ .map { |file| File.basename(file, Environment::RB_EXT).to_sym }
29
+ .reject { |item| IGNORED_ITEMS.include?(item) }.to_h do |agent_sym|
30
30
  [agent_sym, {
31
31
  long: agent_sym.to_s.capitalize,
32
32
  short: agent_sym.eql?(:direct) ? :a : agent_sym.to_s[0].to_sym
@@ -5,10 +5,32 @@ require 'aspera/transfer/spec'
5
5
  require 'aspera/api/httpgw'
6
6
  require 'aspera/log'
7
7
  require 'aspera/assert'
8
+ require 'securerandom'
8
9
 
9
10
  module Aspera
10
11
  module Agent
11
12
  class Httpgw < Base
13
+ class << self
14
+ # Re-query the status of an httpgw transfer that was started in-process.
15
+ # Only works if the same Ruby process is still alive (e.g. MCP mode).
16
+ # agent_params must contain '_agent_ref' pointing to the live Httpgw instance.
17
+ # @param transfer_id [String] job_id returned by last_job_id
18
+ # @param agent_params [Hash] '_agent_ref' is the live agent instance
19
+ # @return [Hash] status hash with at least 'status' key
20
+ def transfer_status(transfer_id, agent_params)
21
+ agent = agent_params['_agent_ref']
22
+ return {'status' => 'unknown', 'note' => 'httpgw agent not available (different process?)'} if agent.nil?
23
+ return {'status' => 'unknown', 'note' => 'no transfer started'} if agent.instance_variable_get(:@transfer_thread).nil?
24
+ if agent.instance_variable_get(:@transfer_thread).alive?
25
+ {'status' => 'running'}
26
+ elsif (err = agent.instance_variable_get(:@transfer_error))
27
+ {'status' => 'failed', 'error' => err.message}
28
+ else
29
+ {'status' => 'completed'}
30
+ end
31
+ end
32
+ end
33
+
12
34
  def initialize(
13
35
  url:,
14
36
  api_version: Api::Httpgw::API_V2,
@@ -23,34 +45,53 @@ module Aspera
23
45
  api_version: api_version,
24
46
  upload_chunk_size: upload_chunk_size,
25
47
  synchronous: synchronous,
26
- notify_cb: ->(*pa, **ka){notify_progress(*pa, **ka)}
48
+ notify_cb: ->(*pa, **ka) { notify_progress(*pa, **ka) }
27
49
  )
50
+ @last_job_id = nil
51
+ @transfer_thread = nil
52
+ @transfer_error = nil
28
53
  end
29
54
 
30
- # Start FASP transfer based on transfer spec (hash table)
31
- # note that this should be asynchronous, but it is not
32
- # HTTP download only supports file list
55
+ # Start FASP transfer based on transfer spec (hash table).
56
+ # Non-blocking: the actual upload/download runs in a dedicated thread.
57
+ # HTTP download only supports file list.
33
58
  # :reek:UnusedParameters token_regenerator
34
59
  def start_transfer(transfer_spec, token_regenerator: nil)
35
60
  Aspera.assert(!@gw_api.nil?, 'GW URL must be set')
36
- Aspera.assert_type(transfer_spec['paths'], Array){'paths'}
37
- Aspera.assert_type(transfer_spec['token'], String){'only token based transfer is supported in GW'}
61
+ Aspera.assert_type(transfer_spec['paths'], Array) { 'paths' }
62
+ Aspera.assert_type(transfer_spec['token'], String) { 'only token based transfer is supported in GW' }
38
63
  Log.dump(:user_spec, transfer_spec)
39
64
  transfer_spec['authentication'] ||= 'token'
65
+ @last_job_id = SecureRandom.uuid
66
+ @transfer_error = nil
67
+ @transfer_thread = Thread.new { run_transfer(transfer_spec) }
68
+ @last_job_id
69
+ end
70
+
71
+ # Wait for the transfer thread to complete.
72
+ # @return [Array] list of :success or error
73
+ def wait_for_transfers_completion
74
+ @transfer_thread&.join
75
+ return [@transfer_error || :success]
76
+ end
77
+
78
+ # @return [String, nil] job_id of the last submitted transfer
79
+ attr_reader :last_job_id
80
+
81
+ private
82
+
83
+ # Execute the actual transfer (runs inside the transfer thread).
84
+ def run_transfer(transfer_spec)
40
85
  case transfer_spec['direction']
41
86
  when Transfer::Spec::DIRECTION_SEND
42
87
  @gw_api.upload(transfer_spec)
43
88
  when Transfer::Spec::DIRECTION_RECEIVE
44
89
  @gw_api.download(transfer_spec)
45
- else Aspera.error_unexpected_value(transfer_spec['direction']){'direction'}
90
+ else Aspera.error_unexpected_value(transfer_spec['direction']) { 'direction' }
46
91
  end
47
- end
48
-
49
- # Wait for completion of all jobs started
50
- # @return [Array] list of :success or error message
51
- def wait_for_transfers_completion
52
- # well ... transfer was done in "start"
53
- return [:success]
92
+ rescue => e
93
+ @transfer_error = e
94
+ Log.log.error { "httpgw transfer thread error: #{e}" }
54
95
  end
55
96
  end
56
97
  end
@@ -95,7 +95,7 @@ module Aspera
95
95
  transfer_spec['remote_host'] = '127.0.0.1' if !transfer_spec['wss_enabled'] && transfer_spec['remote_host'].eql?(URI.parse(node_api_.base_url).host)
96
96
  resp = node_api_.create('ops/transfers', transfer_spec)
97
97
  @transfer_id = resp['id']
98
- Log.log.debug{"tr_id=#{@transfer_id}"}
98
+ Log.log.debug { "tr_id=#{@transfer_id}" }
99
99
  return @transfer_id
100
100
  end
101
101
 
@@ -137,7 +137,7 @@ module Aspera
137
137
  # Bug in HSTS ? transfer is marked failed, but there is no reason
138
138
  break if transfer_data['error_code'].eql?(0) && transfer_data['error_desc'].empty?
139
139
  raise Transfer::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
140
- else Aspera.error_unexpected_value(transfer_data['status']){"transfer_data -> #{transfer_data}"}
140
+ else Aspera.error_unexpected_value(transfer_data['status']) { "transfer_data -> #{transfer_data}" }
141
141
  end
142
142
  sleep(1.0)
143
143
  end
@@ -66,7 +66,7 @@ module Aspera
66
66
  @transfer_id = nil
67
67
  @stop = stop
68
68
  is_local_auto_port = url.eql?(AUTO_LOCAL_TCP_PORT)
69
- Aspera.assert(!(is_local_auto_port && (!@stop || !start)), type: Error){'Cannot set options `stop` or `start` to false with port zero'}
69
+ Aspera.assert(!(is_local_auto_port && (!@stop || !start)), type: Error) { 'Cannot set options `stop` or `start` to false with port zero' }
70
70
  # keep PID for optional shutdown
71
71
  @daemon_pid = nil
72
72
  @daemon_endpoint = nil
@@ -81,18 +81,18 @@ module Aspera
81
81
  # Initiate actual connection
82
82
  get_info_response = @transfer_client.get_info(::Transferd::Api::InstanceInfoRequest.new)
83
83
  @daemon_endpoint = daemon_endpoint
84
- Log.log.debug{"Daemon info: #{get_info_response}"}
84
+ Log.log.debug { "Daemon info: #{get_info_response}" }
85
85
  Log.log.warn('Attached to existing daemon') unless @daemon_pid || !start || !@stop
86
- at_exit{shutdown}
86
+ at_exit { shutdown }
87
87
  rescue GRPC::Unavailable => e
88
88
  # if transferd is external: do not start it, or other error
89
89
  raise if !start || !e.message.include?('failed to connect')
90
90
  # we already tried to start a daemon, but it failed
91
- Aspera.assert(@daemon_pid.nil?){"Daemon started with PID #{@daemon_pid}, but connection failed to #{daemon_endpoint}"}
91
+ Aspera.assert(@daemon_pid.nil?) { "Daemon started with PID #{@daemon_pid}, but connection failed to #{daemon_endpoint}" }
92
92
  Log.log.warn('no daemon present, starting daemon...') if !start
93
93
  # transferd only supports local ip and port
94
94
  daemon_uri = URI.parse("ipv4://#{daemon_endpoint}")
95
- Aspera.assert(daemon_uri.scheme.eql?('ipv4')){"Invalid scheme daemon URI #{daemon_endpoint}"}
95
+ Aspera.assert(daemon_uri.scheme.eql?('ipv4')) { "Invalid scheme daemon URI #{daemon_endpoint}" }
96
96
  # create a config file for daemon
97
97
  config = {
98
98
  address: daemon_uri.host,
@@ -107,7 +107,7 @@ module Aspera
107
107
  }
108
108
  # config file and logs are created in same folder
109
109
  transferd_base_tmp = TempFileManager.instance.new_file_path_global('transferd')
110
- Log.log.debug{"transferd base tmp #{transferd_base_tmp}"}
110
+ Log.log.debug { "transferd base tmp #{transferd_base_tmp}" }
111
111
  conf_file = "#{transferd_base_tmp}.conf"
112
112
  log_stdout = "#{transferd_base_tmp}.out"
113
113
  log_stderr = "#{transferd_base_tmp}.err"
@@ -128,9 +128,9 @@ module Aspera
128
128
  rescue Timeout::Error
129
129
  nil
130
130
  end
131
- Log.log.debug{"Daemon started with pid #{@daemon_pid}"}
131
+ Log.log.debug { "Daemon started with pid #{@daemon_pid}" }
132
132
  Process.detach(@daemon_pid) unless @stop
133
- at_exit{shutdown}
133
+ at_exit { shutdown }
134
134
  # update port for next connection attempt (if auto high port was requested)
135
135
  daemon_endpoint = "#{LOCAL_SOCKET_ADDR}#{PORT_SEP}#{Products::Transferd.daemon_port_from_log(log_stdout)}" if is_local_auto_port
136
136
  # local daemon started, try again
@@ -152,10 +152,10 @@ module Aspera
152
152
  ) # transfer definition
153
153
  # send start transfer request to the transfer manager daemon
154
154
  start_response = @transfer_client.start_transfer(transfer_request)
155
- Aspera.assert(!start_response.status.eql?(:FAILED), type: Transfer::Error){start_response.error.description}
156
- Log.log.debug{"start transfer response #{start_response}"}
155
+ Aspera.assert(!start_response.status.eql?(:FAILED), type: Transfer::Error) { start_response.error.description }
156
+ Log.log.debug { "start transfer response #{start_response}" }
157
157
  @transfer_id = start_response.transferId
158
- Log.log.debug{"transfer started with id #{@transfer_id}"}
158
+ Log.log.debug { "transfer started with id #{@transfer_id}" }
159
159
  end
160
160
 
161
161
  def wait_for_transfers_completion
@@ -190,7 +190,7 @@ module Aspera
190
190
  when :QUEUED, :UNKNOWN_STATUS, :PAUSED, :ORPHANED
191
191
  notify_progress(:sessions_init, info: response.status.to_s.downcase)
192
192
  else
193
- Log.log.error{"unknown status#{response.status}"}
193
+ Log.log.error { "unknown status#{response.status}" }
194
194
  end
195
195
  end
196
196
  # TODO: return status