aspera-cli 4.26.2 → 4.27.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +310 -21
  4. data/CONTRIBUTING.md +99 -14
  5. data/TODO.md +50 -0
  6. data/bin/ascli +3 -3
  7. data/docs/README.md +10977 -0
  8. data/docs/test-mcp-with-ai.md +202 -0
  9. data/lib/aspera/agent/base.rb +29 -7
  10. data/lib/aspera/agent/connect.rb +58 -21
  11. data/lib/aspera/agent/desktop.rb +53 -18
  12. data/lib/aspera/agent/direct.rb +71 -40
  13. data/lib/aspera/agent/factory.rb +3 -3
  14. data/lib/aspera/agent/httpgw.rb +55 -14
  15. data/lib/aspera/agent/node.rb +34 -2
  16. data/lib/aspera/agent/transferd.rb +48 -12
  17. data/lib/aspera/api/alee.rb +1 -1
  18. data/lib/aspera/api/aoc.rb +42 -44
  19. data/lib/aspera/api/cos_node.rb +10 -9
  20. data/lib/aspera/api/faspex.rb +16 -15
  21. data/lib/aspera/api/httpgw.rb +23 -23
  22. data/lib/aspera/api/node.rb +51 -70
  23. data/lib/aspera/api/queries/bss_subscription_account.graphql +18 -0
  24. data/lib/aspera/api/queries/bss_subscription_usage.graphql +18 -0
  25. data/lib/aspera/ascmd.rb +32 -30
  26. data/lib/aspera/ascp/installation.rb +75 -69
  27. data/lib/aspera/ascp/management.rb +7 -8
  28. data/lib/aspera/assert.rb +12 -13
  29. data/lib/aspera/cli/ascp_actions.rb +155 -0
  30. data/lib/aspera/cli/async_transfer_store.rb +125 -0
  31. data/lib/aspera/cli/bootstrapper.rb +197 -0
  32. data/lib/aspera/cli/command_registry.rb +145 -0
  33. data/lib/aspera/cli/command_spec.rb +141 -0
  34. data/lib/aspera/cli/context.rb +19 -6
  35. data/lib/aspera/cli/error.rb +16 -0
  36. data/lib/aspera/cli/extended_value.rb +27 -26
  37. data/lib/aspera/cli/formatter.rb +95 -59
  38. data/lib/aspera/cli/gem_checker.rb +65 -0
  39. data/lib/aspera/cli/hints.rb +3 -3
  40. data/lib/aspera/cli/http.rb +74 -30
  41. data/lib/aspera/cli/info.rb +2 -0
  42. data/lib/aspera/cli/mailer.rb +97 -0
  43. data/lib/aspera/cli/mcp_tool.rb +225 -0
  44. data/lib/aspera/cli/option_declarator.rb +77 -0
  45. data/lib/aspera/cli/options.schema.yaml +634 -7
  46. data/lib/aspera/cli/parser.rb +1217 -0
  47. data/lib/aspera/cli/plugins/alee.rb +20 -22
  48. data/lib/aspera/cli/plugins/aoc.rb +1178 -881
  49. data/lib/aspera/cli/plugins/ats.rb +205 -162
  50. data/lib/aspera/cli/plugins/base.rb +703 -175
  51. data/lib/aspera/cli/plugins/basic_auth.rb +7 -9
  52. data/lib/aspera/cli/plugins/config.rb +434 -757
  53. data/lib/aspera/cli/plugins/console.rb +107 -65
  54. data/lib/aspera/cli/plugins/cos.rb +46 -33
  55. data/lib/aspera/cli/plugins/factory.rb +6 -6
  56. data/lib/aspera/cli/plugins/faspex5.rb +627 -388
  57. data/lib/aspera/cli/plugins/faspio.rb +54 -51
  58. data/lib/aspera/cli/plugins/httpgw.rb +18 -25
  59. data/lib/aspera/cli/plugins/mcp.rb +279 -0
  60. data/lib/aspera/cli/plugins/node.rb +988 -840
  61. data/lib/aspera/cli/plugins/oauth.rb +7 -10
  62. data/lib/aspera/cli/plugins/orchestrator.rb +112 -135
  63. data/lib/aspera/cli/plugins/preview.rb +195 -157
  64. data/lib/aspera/cli/plugins/server.rb +141 -91
  65. data/lib/aspera/cli/plugins/shares.rb +343 -110
  66. data/lib/aspera/cli/preset_actions.rb +159 -0
  67. data/lib/aspera/cli/preset_manager.rb +82 -39
  68. data/lib/aspera/cli/result.rb +78 -26
  69. data/lib/aspera/cli/runner.rb +323 -153
  70. data/lib/aspera/cli/secret_finder.rb +40 -0
  71. data/lib/aspera/cli/special_values.rb +1 -0
  72. data/lib/aspera/cli/sync_actions.rb +89 -64
  73. data/lib/aspera/cli/terminal_formatter.rb +3 -3
  74. data/lib/aspera/cli/transfer_actions.rb +90 -0
  75. data/lib/aspera/cli/transfer_agent.rb +129 -64
  76. data/lib/aspera/cli/transfer_progress.rb +9 -9
  77. data/lib/aspera/cli/vault_manager.rb +79 -0
  78. data/lib/aspera/cli/version.rb +1 -1
  79. data/lib/aspera/cli/wizard.rb +28 -30
  80. data/lib/aspera/colors.rb +3 -3
  81. data/lib/aspera/command_line_builder.rb +34 -34
  82. data/lib/aspera/command_line_converter.rb +1 -1
  83. data/lib/aspera/coverage.rb +1 -2
  84. data/lib/aspera/data_repository.rb +2 -1
  85. data/lib/aspera/dot_container.rb +12 -12
  86. data/lib/aspera/environment.rb +30 -23
  87. data/lib/aspera/exec_spec.rb +13 -0
  88. data/lib/aspera/faspex_gw.rb +5 -5
  89. data/lib/aspera/faspex_postproc.rb +16 -10
  90. data/lib/aspera/graphql.rb +37 -0
  91. data/lib/aspera/hash_ext.rb +8 -2
  92. data/lib/aspera/json_rpc/client.rb +62 -0
  93. data/lib/aspera/json_rpc/version.rb +7 -0
  94. data/lib/aspera/keychain/base.rb +10 -3
  95. data/lib/aspera/keychain/encrypted_hash.rb +92 -17
  96. data/lib/aspera/keychain/factory.rb +20 -8
  97. data/lib/aspera/keychain/hashicorp_vault.rb +1 -1
  98. data/lib/aspera/keychain/macos_security.rb +23 -25
  99. data/lib/aspera/keychain/one_password_api.rb +86 -0
  100. data/lib/aspera/keychain/one_password_base.rb +34 -0
  101. data/lib/aspera/keychain/one_password_cli.rb +98 -0
  102. data/lib/aspera/link_header.rb +82 -0
  103. data/lib/aspera/log.rb +29 -15
  104. data/lib/aspera/markdown.rb +90 -7
  105. data/lib/aspera/nagios.rb +8 -8
  106. data/lib/aspera/node_simulator.rb +35 -27
  107. data/lib/aspera/oauth/base.rb +9 -10
  108. data/lib/aspera/oauth/boot.rb +5 -5
  109. data/lib/aspera/oauth/factory.rb +12 -10
  110. data/lib/aspera/oauth/jwt.rb +9 -11
  111. data/lib/aspera/oauth/web.rb +6 -6
  112. data/lib/aspera/persistency_action_once.rb +12 -12
  113. data/lib/aspera/persistency_folder.rb +18 -11
  114. data/lib/aspera/preview/file_types.rb +5 -5
  115. data/lib/aspera/preview/generator.rb +52 -49
  116. data/lib/aspera/preview/options.rb +3 -2
  117. data/lib/aspera/preview/terminal.rb +5 -5
  118. data/lib/aspera/preview/utils.rb +66 -19
  119. data/lib/aspera/products/connect.rb +2 -2
  120. data/lib/aspera/products/desktop.rb +1 -1
  121. data/lib/aspera/products/other.rb +3 -3
  122. data/lib/aspera/products/transferd.rb +4 -4
  123. data/lib/aspera/proxy_auto_config.rb +15 -14
  124. data/lib/aspera/rest.rb +159 -151
  125. data/lib/aspera/rest_error_analyzer.rb +6 -6
  126. data/lib/aspera/rest_errors_aspera.rb +0 -10
  127. data/lib/aspera/rest_list.rb +11 -10
  128. data/lib/aspera/schema/IBM Aspera Node API-4.4.6.yaml +6232 -0
  129. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +249 -0
  130. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +1853 -510
  131. data/lib/aspera/schema/IBM_Aspera_Shares.yaml +4653 -0
  132. data/lib/aspera/schema/documentation.rb +65 -28
  133. data/lib/aspera/schema/reader.rb +136 -13
  134. data/lib/aspera/schema/registry.rb +60 -8
  135. data/lib/aspera/secret_hider.rb +12 -7
  136. data/lib/aspera/ssh.rb +64 -31
  137. data/lib/aspera/ssl.rb +5 -5
  138. data/lib/aspera/sync/conf.schema.yaml +2 -2
  139. data/lib/aspera/sync/database.rb +2 -2
  140. data/lib/aspera/sync/operations.rb +22 -25
  141. data/lib/aspera/temp_file_manager.rb +25 -5
  142. data/lib/aspera/timer_limiter.rb +1 -1
  143. data/lib/aspera/transfer/faux_file.rb +24 -11
  144. data/lib/aspera/transfer/parameters.rb +36 -34
  145. data/lib/aspera/transfer/result.rb +74 -0
  146. data/lib/aspera/transfer/resumer.rb +10 -10
  147. data/lib/aspera/transfer/spec.rb +18 -1
  148. data/lib/aspera/transfer/spec.schema.yaml +12 -3
  149. data/lib/aspera/uri_reader.rb +52 -15
  150. data/lib/aspera/web_auth.rb +8 -7
  151. data/lib/aspera/web_server_simple.rb +15 -12
  152. data/lib/aspera/yaml.rb +5 -4
  153. data.tar.gz.sig +0 -0
  154. metadata +34 -6
  155. metadata.gz.sig +0 -0
  156. data/lib/aspera/cli/manager.rb +0 -766
  157. data/lib/aspera/cli/plugins/faspex.rb +0 -539
  158. data/lib/aspera/json_rpc.rb +0 -52
  159. data/lib/aspera/transfer/uri.rb +0 -56
@@ -3,6 +3,7 @@
3
3
  require 'aspera/agent/base'
4
4
  require 'aspera/ascp/installation'
5
5
  require 'aspera/ascp/management'
6
+ require 'aspera/exec_spec'
6
7
  require 'aspera/transfer/parameters'
7
8
  require 'aspera/transfer/error'
8
9
  require 'aspera/transfer/spec'
@@ -18,18 +19,43 @@ module Aspera
18
19
  module Agent
19
20
  # Execute a local `ascp` and use its management port to monitor progress
20
21
  class Direct < Base
22
+ class << self
23
+ # Re-query the status of a direct transfer that was started in-process.
24
+ # Only works if the same Ruby process is still alive (e.g. MCP mode).
25
+ # agent_params must contain '_agent_ref' pointing to the live Direct instance.
26
+ # @param transfer_id [String] job_id returned by start_transfer / last_job_id
27
+ # @param agent_params [Hash] connection parameters; '_agent_ref' is the live agent
28
+ # @return [Hash] status hash with at least 'status' key
29
+ def transfer_status(transfer_id, agent_params)
30
+ agent = agent_params['_agent_ref']
31
+ return {'status' => 'unknown', 'note' => 'direct agent not available (different process?)'} if agent.nil?
32
+ sessions = agent.sessions_by_job(transfer_id)
33
+ return {'status' => 'unknown', 'note' => "no sessions found for job #{transfer_id}"} if sessions.empty?
34
+ errors = sessions.filter_map { |s| s[:error] }
35
+ running = sessions.any? { |s| s[:thread]&.alive? }
36
+ if errors.any?
37
+ {'status' => 'failed', 'error' => errors.first.message}
38
+ elsif running
39
+ {'status' => 'running'}
40
+ else
41
+ {'status' => 'completed'}
42
+ end
43
+ end
44
+ end
45
+
21
46
  # `ascp` started locally, so listen local
22
47
  LISTEN_LOCAL_ADDRESS = '127.0.0.1'
23
48
  # 0 means: use any available port
24
49
  SELECT_AVAILABLE_PORT = 0
25
50
  private_constant :LISTEN_LOCAL_ADDRESS, :SELECT_AVAILABLE_PORT
26
51
 
27
- # Options: same as values in option `transfer_info`
52
+ # Options: same as values in option `transfer`
28
53
  # (Args) : Options that influence `ascp` arguments.
29
54
  #
30
55
  # @param ascp_args [Array] (Args) Optional Additional arguments to ascp
31
56
  # @param wss [Boolean] (Args) `true`: if both SSH and wss in ts: prefer wss
32
57
  # @param quiet [Boolean] (Args) By default no native `ascp` progress bar
58
+ # @param file_list [Boolean] (Args) true: provide file list in a file, else on command line.
33
59
  # @param client_ssh_key [String] (Args) Client SSH key option (from CLIENT_SSH_KEY_OPTIONS)
34
60
  # @param trusted_certs [Array<String>] (Args) (WSS) Optional list of files with trusted certificates (stores)
35
61
  # @param check_ignore_cb [Proc] (Args) (WSS) Callback with host,port to check if WSS connection shall ignore certificate validity
@@ -44,6 +70,7 @@ module Aspera
44
70
  ascp_args: nil,
45
71
  wss: true,
46
72
  quiet: true,
73
+ file_list: true,
47
74
  client_ssh_key: nil,
48
75
  trusted_certs: nil,
49
76
  check_ignore_cb: nil,
@@ -61,6 +88,7 @@ module Aspera
61
88
  ascp_args: ascp_args,
62
89
  wss: wss,
63
90
  quiet: quiet,
91
+ file_list: file_list,
64
92
  trusted_certs: trusted_certs,
65
93
  client_ssh_key: client_ssh_key,
66
94
  check_ignore_cb: check_ignore_cb
@@ -71,7 +99,7 @@ module Aspera
71
99
  @multi_incr_udp = multi_incr_udp.nil? ? Environment.instance.os.eql?(Environment::OS_WINDOWS) : multi_incr_udp
72
100
  @monitor = monitor
73
101
  @management_cb = management_cb
74
- Aspera.assert_type(resume, Hash){'resume'}
102
+ Aspera.assert_type(resume, Hash) { 'resume' }
75
103
  @resume_policy = Transfer::Resumer.new(**resume.symbolize_keys)
76
104
  # all transfer jobs, key = SecureRandom.uuid, protected by mutex, cond var on change
77
105
  @sessions = []
@@ -97,7 +125,7 @@ module Aspera
97
125
  # using a non unique id results in discard of tags in AoC, and a package is never finalized
98
126
  # all sessions in a multi-session transfer must have the same xfer_id (see admin manual)
99
127
  transfer_spec['tags'][Transfer::Spec::TAG_RESERVED]['xfer_id'] ||= SecureRandom.uuid
100
- Log.log.debug{"xfer id=#{transfer_spec['xfer_id']}"}
128
+ Log.log.debug { "xfer id=#{transfer_spec['xfer_id']}" }
101
129
  # TODO: useful ? node only ? seems to be a timeout for retry in node
102
130
  transfer_spec['tags'][Transfer::Spec::TAG_RESERVED]['xfer_retry'] ||= 3600
103
131
  end
@@ -111,7 +139,7 @@ module Aspera
111
139
  count: transfer_spec.delete('multi_session').to_i
112
140
  }
113
141
  if multi_session_info[:count].negative?
114
- Log.log.error{"multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0"}
142
+ Log.log.error { "multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0" }
115
143
  multi_session_info = nil
116
144
  elsif multi_session_info[:count].eql?(0)
117
145
  Log.log.debug('multi_session count is zero: no multi session')
@@ -135,13 +163,13 @@ module Aspera
135
163
  io: nil, # management port server socket
136
164
  token_regenerator: token_regenerator, # regenerate bearer token with oauth
137
165
  # env vars and args for ascp (from transfer spec)
138
- env_args: Transfer::Parameters.new(transfer_spec, **@tr_opts).ascp_args
166
+ exec_spec: Transfer::Parameters.new(transfer_spec, **@tr_opts).ascp_args
139
167
  }
140
168
 
141
169
  if multi_session_info.nil?
142
170
  Log.log.debug('Starting single session thread')
143
171
  # single session for transfer : simple
144
- session[:thread] = Thread.new{transfer_thread_entry(session)}
172
+ session[:thread] = Thread.new { transfer_thread_entry(session) }
145
173
  @sessions.push(session)
146
174
  else
147
175
  Log.log.debug('Starting multi session threads')
@@ -151,28 +179,32 @@ module Aspera
151
179
  # do deep copy (each thread has its own copy because it is modified here below and in thread)
152
180
  this_session = session.clone
153
181
  this_session[:ts] = this_session[:ts].clone
154
- env_args = this_session[:env_args] = this_session[:env_args].clone
155
- args = env_args[:args] = env_args[:args].clone
182
+ exec_spec = this_session[:exec_spec] = this_session[:exec_spec].deep_clone
156
183
  # set multi session part
157
- args.unshift("-C#{i}:#{multi_session_info[:count]}")
184
+ exec_spec.args.unshift("-C#{i}:#{multi_session_info[:count]}")
158
185
  # option: increment (default as per ascp manual) or not (cluster on other side ?)
159
- args.unshift('-O', (multi_session_info[:udp_base] + i - 1).to_s) if @multi_incr_udp
186
+ exec_spec.args.unshift('-O', (multi_session_info[:udp_base] + i - 1).to_s) if @multi_incr_udp
160
187
  # finally start the thread
161
- this_session[:thread] = Thread.new{transfer_thread_entry(this_session)}
188
+ this_session[:thread] = Thread.new { transfer_thread_entry(this_session) }
162
189
  @sessions.push(this_session)
163
190
  end
164
191
  end
165
192
  return session[:job_id]
166
193
  end
167
194
 
195
+ # @return [String, nil] job_id of the last submitted transfer
196
+ def last_job_id
197
+ @sessions.last&.dig(:job_id)
198
+ end
199
+
168
200
  # wait for completion of all jobs started
169
- # @return list of :success or error message
201
+ # @return [Array] list of :success or error message
170
202
  def wait_for_transfers_completion
171
203
  Log.log.debug('wait_for_transfers_completion')
172
204
  # set to non-nil to exit loop
173
205
  result = []
174
206
  @sessions.each do |session|
175
- Log.log.debug{"join #{session[:thread]}"}
207
+ Log.log.debug { "join #{session[:thread]}" }
176
208
  session[:thread].join
177
209
  result.push(session[:error] || :success)
178
210
  end
@@ -191,7 +223,7 @@ module Aspera
191
223
  # @param id [String] Transfer session identifier
192
224
  # @return [Array] list of sessions for a job
193
225
  def sessions_by_job(id)
194
- @sessions.select{ |session| session[:job_id].eql?(id)}
226
+ @sessions.select { |session| session[:job_id].eql?(id) }
195
227
  end
196
228
 
197
229
  # Send command to management port of command (used in `asession).
@@ -216,22 +248,22 @@ module Aspera
216
248
  private
217
249
 
218
250
  # transfer thread entry
219
- # @param session information
251
+ # @param session [Hash] transfer session information
220
252
  def transfer_thread_entry(session)
221
253
  begin
222
254
  # set name for logging
223
255
  Thread.current[:name] = 'transfer'
224
- Log.log.debug{"ENTER (#{Thread.current[:name]})"}
256
+ Log.log.debug { "ENTER (#{Thread.current[:name]})" }
225
257
  # start transfer with selected resumer policy
226
258
  @resume_policy.execute_with_resume do
227
- start_and_monitor_process(session: session, **session[:env_args])
259
+ start_and_monitor_process(session: session, exec_spec: session[:exec_spec])
228
260
  end
229
261
  Log.log.debug('transfer ok'.bg_green)
230
262
  rescue => e
231
263
  session[:error] = e
232
264
  raise if Log.log.debug? || !e.is_a?(Transfer::Error)
233
265
  end
234
- Log.log.debug{"EXIT (#{Thread.current[:name]})"}
266
+ Log.log.debug { "EXIT (#{Thread.current[:name]})" }
235
267
  end
236
268
 
237
269
  public
@@ -239,18 +271,17 @@ module Aspera
239
271
  # This is the low level method to start the transfer process.
240
272
  # Typically started in a thread.
241
273
  # Start process with management port.
242
- # @param session [Hash] This session information, keys :io and :token_regenerator
243
- # @param name [Symbol] Name of executable: :ascp, :ascp4 or :async (comes from ascp_args)
244
- # @param env [Hash] Environment variables (comes from ascp_args)
245
- # @param args [Array] Command line arguments (comes from ascp_args)
274
+ # @param session [Hash] This session information, keys :io and :token_regenerator
275
+ # @param exec_spec [ExecSpec] Executable, environment variables and arguments (comes from ascp_args)
246
276
  # @return [nil] when process has exited
247
- # @raise FaspError on error
277
+ # @raise [Transfer::FaspError] on error
248
278
  def start_and_monitor_process(
249
279
  session:,
250
- name:,
251
- env:,
252
- args:
280
+ exec_spec:
253
281
  )
282
+ exec = exec_spec.exec
283
+ env = exec_spec.env
284
+ args = exec_spec.args
254
285
  Aspera.assert_type(session, Hash)
255
286
  notify_progress(:sessions_init, info: 'starting')
256
287
  # Do not use `capture_stderr`
@@ -259,7 +290,7 @@ module Aspera
259
290
  spawn_args = {}
260
291
  command_pid = nil
261
292
  # Get location of command executable (ascp, async)
262
- command_path = Ascp::Installation.instance.path(name)
293
+ command_path = Ascp::Installation.instance.path(exec)
263
294
  command_arguments = [command_path]
264
295
  begin
265
296
  if @monitor
@@ -271,7 +302,7 @@ module Aspera
271
302
  # make port ready to accept connections, before starting ascp
272
303
  mgt_server_socket.listen(1)
273
304
  # build arguments and add mgt port
274
- command_arguments = if name.eql?(:async)
305
+ command_arguments = if exec.eql?(:async)
275
306
  [command_path, "--exclusive-mgmt-port=#{mgt_server_socket.local_address.ip_port}"]
276
307
  else
277
308
  [command_path, '-M', mgt_server_socket.local_address.ip_port.to_s]
@@ -286,12 +317,12 @@ module Aspera
286
317
  command_pid = Environment.secure_execute(*command_arguments, mode: :background, env: env, **spawn_args)
287
318
  # close here, but still used in other process (pipe)
288
319
  stderr_w&.close
289
- notify_progress(:sessions_init, info: "waiting for #{name} to start")
320
+ notify_progress(:sessions_init, info: "waiting for #{exec} to start")
290
321
  # "ensure" block will wait for process
291
322
  return unless @monitor
292
323
  # TODO: timeout does not work when Process.spawn is used... until process exits, then it works
293
324
  # So we use select to detect that anything happens on the socket (connection)
294
- Log.log.debug{"before select, timeout: #{@spawn_timeout_sec}"}
325
+ Log.log.debug { "before select, timeout: #{@spawn_timeout_sec}" }
295
326
  readable, _, _ = IO.select([mgt_server_socket], nil, nil, @spawn_timeout_sec)
296
327
  Log.log.debug('after select, before accept')
297
328
  Aspera.assert(readable, 'timeout waiting mgt port connect (select not readable)', type: Transfer::Error)
@@ -320,17 +351,17 @@ module Aspera
320
351
  commands = JSON.parse(File.read(@command_file))
321
352
  send_command(commands)
322
353
  rescue => e
323
- Log.log.error{e.to_s}
354
+ Log.log.error { e.to_s }
324
355
  end
325
356
  File.delete(@command_file)
326
357
  end
327
358
  Log.log.debug('management io closed')
328
359
  # check that last status was received before process exit
329
360
  last_event = processor.last_event
330
- raise Transfer::Error, "No management event (#{last_event.class})" unless last_event.is_a?(Hash)
361
+ Aspera.assert_type(last_event, Hash, type: Transfer::Error) { 'management event' }
331
362
  case last_event['Type']
332
363
  when 'DONE'
333
- Log.log.trace1{'Graceful shutdown, DONE message received'}
364
+ Log.log.trace1 { 'Graceful shutdown, DONE message received' }
334
365
  when 'ERROR'
335
366
  if /bearer token/i.match?(last_event['Description']) &&
336
367
  session[:token_regenerator].respond_to?(:refreshed_transfer_token)
@@ -340,7 +371,7 @@ module Aspera
340
371
  env['ASPERA_SCP_TOKEN'] = session[:token_regenerator].refreshed_transfer_token
341
372
  end
342
373
  raise Transfer::Error.new(last_event['Description'], code: last_event['Code'].to_i)
343
- else Aspera.error_unexpected_value(last_event['Type'], :error){'last event type'}
374
+ else Aspera.error_unexpected_value(last_event['Type'], :error) { 'last event type' }
344
375
  end
345
376
  rescue SystemCallError => e
346
377
  # Process.spawn failed, or socket error
@@ -359,16 +390,16 @@ module Aspera
359
390
  # process stderr of ascp
360
391
  stderr_flag = false
361
392
  stderr_r.each_line do |line|
362
- Log.log.error{"BEGIN stderr #{name}"} unless stderr_flag
363
- Log.log.error{line.chomp}
393
+ Log.log.error { "BEGIN stderr #{exec}" } unless stderr_flag
394
+ Log.log.error { line.chomp }
364
395
  stderr_flag = true
365
396
  end
366
- Log.log.error{"END stderr #{name}"} if stderr_flag
397
+ Log.log.error { "END stderr #{exec}" } if stderr_flag
367
398
  stderr_r.close
368
399
  end
369
400
  # status is nil if an exception occurred before starting command
370
401
  if !status&.success?
371
- message = "#{name} failed (#{status})"
402
+ message = "#{exec} failed (#{status})"
372
403
  # raise error only if there was not already an exception (`$ERROR_INFO`)
373
404
  raise Transfer::Error, message unless $ERROR_INFO
374
405
  # else display this message also, as main exception is already here
@@ -410,12 +441,12 @@ module Aspera
410
441
  when 'SESSION'
411
442
  when 'ARGSTOP'
412
443
  when 'FILEERROR'
413
- Log.log.error{"#{event['Type']} #{event['Description']}"}
444
+ Log.log.error { "#{event['Type']} #{event['Description']}" }
414
445
  when 'STOP'
415
446
  # cspell:enable
416
447
  # stop event when one file is completed
417
448
  else
418
- Log.log.debug{"Unknown event type for progress: #{event['Type']}"}
449
+ Log.log.debug { "Unknown event type for progress: #{event['Type']}" }
419
450
  end
420
451
  end
421
452
  end
@@ -24,9 +24,9 @@ module Aspera
24
24
  # Available agents: :long : Capitalized name string, :short : single character symbol
25
25
  ALL =
26
26
  Dir.children(File.dirname(File.expand_path(__FILE__)))
27
- .select{ |file| file.end_with?(Environment::RB_EXT)}
28
- .map{ |file| File.basename(file, Environment::RB_EXT).to_sym}
29
- .reject{ |item| IGNORED_ITEMS.include?(item)}.to_h do |agent_sym|
27
+ .select { |file| file.end_with?(Environment::RB_EXT) }
28
+ .map { |file| File.basename(file, Environment::RB_EXT).to_sym }
29
+ .reject { |item| IGNORED_ITEMS.include?(item) }.to_h do |agent_sym|
30
30
  [agent_sym, {
31
31
  long: agent_sym.to_s.capitalize,
32
32
  short: agent_sym.eql?(:direct) ? :a : agent_sym.to_s[0].to_sym
@@ -5,10 +5,32 @@ require 'aspera/transfer/spec'
5
5
  require 'aspera/api/httpgw'
6
6
  require 'aspera/log'
7
7
  require 'aspera/assert'
8
+ require 'securerandom'
8
9
 
9
10
  module Aspera
10
11
  module Agent
11
12
  class Httpgw < Base
13
+ class << self
14
+ # Re-query the status of an httpgw transfer that was started in-process.
15
+ # Only works if the same Ruby process is still alive (e.g. MCP mode).
16
+ # agent_params must contain '_agent_ref' pointing to the live Httpgw instance.
17
+ # @param transfer_id [String] job_id returned by last_job_id
18
+ # @param agent_params [Hash] '_agent_ref' is the live agent instance
19
+ # @return [Hash] status hash with at least 'status' key
20
+ def transfer_status(transfer_id, agent_params)
21
+ agent = agent_params['_agent_ref']
22
+ return {'status' => 'unknown', 'note' => 'httpgw agent not available (different process?)'} if agent.nil?
23
+ return {'status' => 'unknown', 'note' => 'no transfer started'} if agent.instance_variable_get(:@transfer_thread).nil?
24
+ if agent.instance_variable_get(:@transfer_thread).alive?
25
+ {'status' => 'running'}
26
+ elsif (err = agent.instance_variable_get(:@transfer_error))
27
+ {'status' => 'failed', 'error' => err.message}
28
+ else
29
+ {'status' => 'completed'}
30
+ end
31
+ end
32
+ end
33
+
12
34
  def initialize(
13
35
  url:,
14
36
  api_version: Api::Httpgw::API_V2,
@@ -23,34 +45,53 @@ module Aspera
23
45
  api_version: api_version,
24
46
  upload_chunk_size: upload_chunk_size,
25
47
  synchronous: synchronous,
26
- notify_cb: ->(*pa, **ka){notify_progress(*pa, **ka)}
48
+ notify_cb: ->(*pa, **ka) { notify_progress(*pa, **ka) }
27
49
  )
50
+ @last_job_id = nil
51
+ @transfer_thread = nil
52
+ @transfer_error = nil
28
53
  end
29
54
 
30
- # Start FASP transfer based on transfer spec (hash table)
31
- # note that this should be asynchronous, but it is not
32
- # HTTP download only supports file list
55
+ # Start FASP transfer based on transfer spec (hash table).
56
+ # Non-blocking: the actual upload/download runs in a dedicated thread.
57
+ # HTTP download only supports file list.
33
58
  # :reek:UnusedParameters token_regenerator
34
59
  def start_transfer(transfer_spec, token_regenerator: nil)
35
60
  Aspera.assert(!@gw_api.nil?, 'GW URL must be set')
36
- Aspera.assert_type(transfer_spec['paths'], Array){'paths'}
37
- Aspera.assert_type(transfer_spec['token'], String){'only token based transfer is supported in GW'}
61
+ Aspera.assert_type(transfer_spec['paths'], Array) { 'paths' }
62
+ Aspera.assert_type(transfer_spec['token'], String) { 'only token based transfer is supported in GW' }
38
63
  Log.dump(:user_spec, transfer_spec)
39
64
  transfer_spec['authentication'] ||= 'token'
65
+ @last_job_id = SecureRandom.uuid
66
+ @transfer_error = nil
67
+ @transfer_thread = Thread.new { run_transfer(transfer_spec) }
68
+ @last_job_id
69
+ end
70
+
71
+ # Wait for the transfer thread to complete.
72
+ # @return [Array] list of :success or error
73
+ def wait_for_transfers_completion
74
+ @transfer_thread&.join
75
+ return [@transfer_error || :success]
76
+ end
77
+
78
+ # @return [String, nil] job_id of the last submitted transfer
79
+ attr_reader :last_job_id
80
+
81
+ private
82
+
83
+ # Execute the actual transfer (runs inside the transfer thread).
84
+ def run_transfer(transfer_spec)
40
85
  case transfer_spec['direction']
41
86
  when Transfer::Spec::DIRECTION_SEND
42
87
  @gw_api.upload(transfer_spec)
43
88
  when Transfer::Spec::DIRECTION_RECEIVE
44
89
  @gw_api.download(transfer_spec)
45
- else Aspera.error_unexpected_value(transfer_spec['direction']){'direction'}
90
+ else Aspera.error_unexpected_value(transfer_spec['direction']) { 'direction' }
46
91
  end
47
- end
48
-
49
- # Wait for completion of all jobs started
50
- # @return list of :success or error message
51
- def wait_for_transfers_completion
52
- # well ... transfer was done in "start"
53
- return [:success]
92
+ rescue => e
93
+ @transfer_error = e
94
+ Log.log.error { "httpgw transfer thread error: #{e}" }
54
95
  end
55
96
  end
56
97
  end
@@ -13,6 +13,38 @@ module Aspera
13
13
  # this singleton class is used by the CLI to provide a common interface to start a transfer
14
14
  # before using it, the use must set the `node_api` member.
15
15
  class Node < Base
16
+ class << self
17
+ # Re-query a previously started transfer via the Node REST API.
18
+ # agent_params must contain 'url', 'username', 'password' (and optionally 'root_id').
19
+ # @return [Hash] normalized status hash
20
+ def transfer_status(transfer_id, agent_params)
21
+ rest_params = {base_url: agent_params['url']}
22
+ if OAuth::Factory.bearer_auth?(agent_params['password'])
23
+ rest_params[:headers] = Api::Node.bearer_headers(agent_params['password'], access_key: agent_params['username'])
24
+ else
25
+ rest_params[:auth] = {type: :basic, username: agent_params['username'], password: agent_params['password']}
26
+ end
27
+ node_api = Rest.new(**rest_params)
28
+ data = node_api.read("ops/transfers/#{transfer_id}") || {'status' => 'unknown'}
29
+ normalize_status(data)
30
+ end
31
+
32
+ # Normalize a node API transfer hash into the standard async-transfer fields.
33
+ def normalize_status(data)
34
+ result = {'bytes_transferred' => data['bytes_transferred'].to_i}
35
+ case data['status']
36
+ when 'completed'
37
+ result['status'] = 'completed'
38
+ result['ended_at'] = Time.now.utc.iso8601
39
+ when 'failed'
40
+ result.merge!('status' => 'failed', 'ended_at' => Time.now.utc.iso8601, 'error' => data['error_desc'].to_s)
41
+ else
42
+ result['status'] = 'running'
43
+ end
44
+ result
45
+ end
46
+ end
47
+
16
48
  # @param url [String] the base url of the node api
17
49
  # @param username [String] the username to use for the node api
18
50
  # @param password [String] the password to use for the node api
@@ -63,7 +95,7 @@ module Aspera
63
95
  transfer_spec['remote_host'] = '127.0.0.1' if !transfer_spec['wss_enabled'] && transfer_spec['remote_host'].eql?(URI.parse(node_api_.base_url).host)
64
96
  resp = node_api_.create('ops/transfers', transfer_spec)
65
97
  @transfer_id = resp['id']
66
- Log.log.debug{"tr_id=#{@transfer_id}"}
98
+ Log.log.debug { "tr_id=#{@transfer_id}" }
67
99
  return @transfer_id
68
100
  end
69
101
 
@@ -105,7 +137,7 @@ module Aspera
105
137
  # Bug in HSTS ? transfer is marked failed, but there is no reason
106
138
  break if transfer_data['error_code'].eql?(0) && transfer_data['error_desc'].empty?
107
139
  raise Transfer::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
108
- else Aspera.error_unexpected_value(transfer_data['status']){"transfer_data -> #{transfer_data}"}
140
+ else Aspera.error_unexpected_value(transfer_data['status']) { "transfer_data -> #{transfer_data}" }
109
141
  end
110
142
  sleep(1.0)
111
143
  end
@@ -21,6 +21,37 @@ module Aspera
21
21
 
22
22
  private_constant :LOCAL_SOCKET_ADDR, :PORT_SEP, :AUTO_LOCAL_TCP_PORT
23
23
 
24
+ class << self
25
+ # Re-query a previously started transfer from a running transferd daemon.
26
+ # agent_params must contain 'url' (the daemon endpoint, e.g. "127.0.0.1:12345").
27
+ # @return [Hash] normalized status hash
28
+ def transfer_status(transfer_id, agent_params)
29
+ client = ::Transferd::Api::TransferService::Stub.new(agent_params['url'], :this_channel_is_insecure)
30
+ result = {'bytes_transferred' => 0}
31
+ client.monitor_transfers(::Transferd::Api::RegistrationRequest.new(transferId: [transfer_id])) do |response|
32
+ result = case response.status
33
+ when :COMPLETED
34
+ {
35
+ 'status' => 'completed', 'ended_at' => Time.now.utc.iso8601,
36
+ 'bytes_transferred' => response.transferInfo.bytesTransferred
37
+ }
38
+ when :FAILED, :CANCELED
39
+ {
40
+ 'status' => response.status == :FAILED ? 'failed' : 'cancelled',
41
+ 'ended_at' => Time.now.utc.iso8601, 'bytes_transferred' => 0,
42
+ 'error' => JSON.parse(response.message.to_s)['Description']
43
+ }
44
+ else
45
+ {'status' => 'running', 'bytes_transferred' => response.transferInfo.bytesTransferred}
46
+ end
47
+ break
48
+ end
49
+ result
50
+ rescue StandardError => e
51
+ {'status' => 'running', 'bytes_transferred' => 0, 'error' => e.message}
52
+ end
53
+ end
54
+
24
55
  # @param url [String] URL of the transfer manager daemon
25
56
  # @param start [Boolean] If `false`, expect that an external daemon is already running
26
57
  # @param stop [Boolean] If `false`, do not shutdown daemon on exit
@@ -35,9 +66,10 @@ module Aspera
35
66
  @transfer_id = nil
36
67
  @stop = stop
37
68
  is_local_auto_port = url.eql?(AUTO_LOCAL_TCP_PORT)
38
- raise Error, 'Cannot set options `stop` or `start` to false with port zero' if is_local_auto_port && (!@stop || !start)
69
+ Aspera.assert(!(is_local_auto_port && (!@stop || !start)), type: Error) { 'Cannot set options `stop` or `start` to false with port zero' }
39
70
  # keep PID for optional shutdown
40
71
  @daemon_pid = nil
72
+ @daemon_endpoint = nil
41
73
  daemon_endpoint = url
42
74
  Log.dump(:daemon_endpoint, daemon_endpoint)
43
75
  # retry loop
@@ -48,18 +80,19 @@ module Aspera
48
80
  @transfer_client = ::Transferd::Api::TransferService::Stub.new(daemon_endpoint, :this_channel_is_insecure)
49
81
  # Initiate actual connection
50
82
  get_info_response = @transfer_client.get_info(::Transferd::Api::InstanceInfoRequest.new)
51
- Log.log.debug{"Daemon info: #{get_info_response}"}
83
+ @daemon_endpoint = daemon_endpoint
84
+ Log.log.debug { "Daemon info: #{get_info_response}" }
52
85
  Log.log.warn('Attached to existing daemon') unless @daemon_pid || !start || !@stop
53
- at_exit{shutdown}
86
+ at_exit { shutdown }
54
87
  rescue GRPC::Unavailable => e
55
88
  # if transferd is external: do not start it, or other error
56
89
  raise if !start || !e.message.include?('failed to connect')
57
90
  # we already tried to start a daemon, but it failed
58
- Aspera.assert(@daemon_pid.nil?){"Daemon started with PID #{@daemon_pid}, but connection failed to #{daemon_endpoint}}"}
91
+ Aspera.assert(@daemon_pid.nil?) { "Daemon started with PID #{@daemon_pid}, but connection failed to #{daemon_endpoint}" }
59
92
  Log.log.warn('no daemon present, starting daemon...') if !start
60
93
  # transferd only supports local ip and port
61
94
  daemon_uri = URI.parse("ipv4://#{daemon_endpoint}")
62
- Aspera.assert(daemon_uri.scheme.eql?('ipv4')){"Invalid scheme daemon URI #{daemon_endpoint}"}
95
+ Aspera.assert(daemon_uri.scheme.eql?('ipv4')) { "Invalid scheme daemon URI #{daemon_endpoint}" }
63
96
  # create a config file for daemon
64
97
  config = {
65
98
  address: daemon_uri.host,
@@ -74,7 +107,7 @@ module Aspera
74
107
  }
75
108
  # config file and logs are created in same folder
76
109
  transferd_base_tmp = TempFileManager.instance.new_file_path_global('transferd')
77
- Log.log.debug{"transferd base tmp #{transferd_base_tmp}"}
110
+ Log.log.debug { "transferd base tmp #{transferd_base_tmp}" }
78
111
  conf_file = "#{transferd_base_tmp}.conf"
79
112
  log_stdout = "#{transferd_base_tmp}.out"
80
113
  log_stderr = "#{transferd_base_tmp}.err"
@@ -95,9 +128,9 @@ module Aspera
95
128
  rescue Timeout::Error
96
129
  nil
97
130
  end
98
- Log.log.debug{"Daemon started with pid #{@daemon_pid}"}
131
+ Log.log.debug { "Daemon started with pid #{@daemon_pid}" }
99
132
  Process.detach(@daemon_pid) unless @stop
100
- at_exit{shutdown}
133
+ at_exit { shutdown }
101
134
  # update port for next connection attempt (if auto high port was requested)
102
135
  daemon_endpoint = "#{LOCAL_SOCKET_ADDR}#{PORT_SEP}#{Products::Transferd.daemon_port_from_log(log_stdout)}" if is_local_auto_port
103
136
  # local daemon started, try again
@@ -105,6 +138,9 @@ module Aspera
105
138
  end
106
139
  end
107
140
 
141
+ # Actual endpoint the daemon is listening on (resolved after connect, e.g. when port 0 was used)
142
+ attr_reader :daemon_endpoint
143
+
108
144
  # :reek:UnusedParameters token_regenerator
109
145
  def start_transfer(transfer_spec, token_regenerator: nil)
110
146
  Transfer::Spec.fix_transferd_resume_policy(transfer_spec)
@@ -116,10 +152,10 @@ module Aspera
116
152
  ) # transfer definition
117
153
  # send start transfer request to the transfer manager daemon
118
154
  start_response = @transfer_client.start_transfer(transfer_request)
119
- raise Transfer::Error, start_response.error.description if start_response.status.eql?(:FAILED)
120
- Log.log.debug{"start transfer response #{start_response}"}
155
+ Aspera.assert(!start_response.status.eql?(:FAILED), type: Transfer::Error) { start_response.error.description }
156
+ Log.log.debug { "start transfer response #{start_response}" }
121
157
  @transfer_id = start_response.transferId
122
- Log.log.debug{"transfer started with id #{@transfer_id}"}
158
+ Log.log.debug { "transfer started with id #{@transfer_id}" }
123
159
  end
124
160
 
125
161
  def wait_for_transfers_completion
@@ -154,7 +190,7 @@ module Aspera
154
190
  when :QUEUED, :UNKNOWN_STATUS, :PAUSED, :ORPHANED
155
191
  notify_progress(:sessions_init, info: response.status.to_s.downcase)
156
192
  else
157
- Log.log.error{"unknown status#{response.status}"}
193
+ Log.log.error { "unknown status#{response.status}" }
158
194
  end
159
195
  end
160
196
  # TODO: return status
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/api/aoc.rb'
3
+ require 'aspera/api/aoc'
4
4
  module Aspera
5
5
  module Api
6
6
  # Aspera License Entitlement Engine API client