aspera-cli 4.21.1 → 4.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +1 -1
  4. data/CHANGELOG.md +52 -22
  5. data/CONTRIBUTING.md +69 -148
  6. data/README.md +929 -668
  7. data/bin/ascli +5 -14
  8. data/bin/asession +1 -3
  9. data/examples/get_proto_file.rb +4 -3
  10. data/examples/proxy.pac +20 -20
  11. data/lib/aspera/agent/base.rb +11 -5
  12. data/lib/aspera/agent/connect.rb +30 -28
  13. data/lib/aspera/agent/{alpha.rb → desktop.rb} +35 -31
  14. data/lib/aspera/agent/direct.rb +141 -121
  15. data/lib/aspera/agent/httpgw.rb +22 -26
  16. data/lib/aspera/agent/node.rb +14 -11
  17. data/lib/aspera/agent/transferd.rb +30 -19
  18. data/lib/aspera/api/alee.rb +1 -1
  19. data/lib/aspera/api/aoc.rb +6 -6
  20. data/lib/aspera/api/cos_node.rb +2 -2
  21. data/lib/aspera/api/httpgw.rb +7 -3
  22. data/lib/aspera/api/node.rb +10 -8
  23. data/lib/aspera/ascmd.rb +3 -3
  24. data/lib/aspera/ascp/installation.rb +53 -72
  25. data/lib/aspera/ascp/management.rb +1 -1
  26. data/lib/aspera/assert.rb +11 -2
  27. data/lib/aspera/cli/error.rb +2 -2
  28. data/lib/aspera/cli/extended_value.rb +46 -21
  29. data/lib/aspera/cli/formatter.rb +55 -48
  30. data/lib/aspera/cli/hints.rb +1 -1
  31. data/lib/aspera/cli/info.rb +1 -0
  32. data/lib/aspera/cli/main.rb +192 -170
  33. data/lib/aspera/cli/manager.rb +18 -18
  34. data/lib/aspera/cli/plugin.rb +23 -20
  35. data/lib/aspera/cli/plugin_factory.rb +1 -1
  36. data/lib/aspera/cli/plugins/alee.rb +1 -1
  37. data/lib/aspera/cli/plugins/aoc.rb +247 -159
  38. data/lib/aspera/cli/plugins/ats.rb +19 -17
  39. data/lib/aspera/cli/plugins/config.rb +76 -113
  40. data/lib/aspera/cli/plugins/console.rb +5 -3
  41. data/lib/aspera/cli/plugins/faspex.rb +39 -35
  42. data/lib/aspera/cli/plugins/faspex5.rb +111 -84
  43. data/lib/aspera/cli/plugins/faspio.rb +13 -1
  44. data/lib/aspera/cli/plugins/httpgw.rb +13 -1
  45. data/lib/aspera/cli/plugins/node.rb +312 -182
  46. data/lib/aspera/cli/plugins/orchestrator.rb +34 -40
  47. data/lib/aspera/cli/plugins/preview.rb +3 -3
  48. data/lib/aspera/cli/plugins/server.rb +6 -6
  49. data/lib/aspera/cli/plugins/shares.rb +5 -5
  50. data/lib/aspera/cli/sync_actions.rb +19 -18
  51. data/lib/aspera/cli/transfer_agent.rb +5 -5
  52. data/lib/aspera/cli/transfer_progress.rb +2 -2
  53. data/lib/aspera/cli/version.rb +1 -1
  54. data/lib/aspera/command_line_builder.rb +116 -95
  55. data/lib/aspera/coverage.rb +8 -5
  56. data/lib/aspera/environment.rb +26 -17
  57. data/lib/aspera/faspex_gw.rb +14 -14
  58. data/lib/aspera/faspex_postproc.rb +10 -11
  59. data/lib/aspera/hash_ext.rb +4 -14
  60. data/lib/aspera/json_rpc.rb +1 -1
  61. data/lib/aspera/keychain/encrypted_hash.rb +47 -34
  62. data/lib/aspera/keychain/factory.rb +41 -0
  63. data/lib/aspera/keychain/hashicorp_vault.rb +71 -0
  64. data/lib/aspera/keychain/macos_security.rb +19 -11
  65. data/lib/aspera/log.rb +28 -34
  66. data/lib/aspera/nagios.rb +6 -6
  67. data/lib/aspera/node_simulator.rb +8 -8
  68. data/lib/aspera/oauth/base.rb +14 -7
  69. data/lib/aspera/oauth/factory.rb +5 -6
  70. data/lib/aspera/oauth/url_json.rb +6 -6
  71. data/lib/aspera/persistency_action_once.rb +6 -4
  72. data/lib/aspera/persistency_folder.rb +2 -2
  73. data/lib/aspera/preview/generator.rb +13 -10
  74. data/lib/aspera/preview/options.rb +16 -16
  75. data/lib/aspera/preview/terminal.rb +4 -4
  76. data/lib/aspera/preview/utils.rb +15 -17
  77. data/lib/aspera/products/connect.rb +35 -1
  78. data/lib/aspera/products/{alpha.rb → desktop.rb} +3 -3
  79. data/lib/aspera/products/transferd.rb +9 -2
  80. data/lib/aspera/proxy_auto_config.rb +2 -2
  81. data/lib/aspera/rest.rb +56 -47
  82. data/lib/aspera/rest_errors_aspera.rb +1 -1
  83. data/lib/aspera/secret_hider.rb +12 -5
  84. data/lib/aspera/ssh.rb +4 -4
  85. data/lib/aspera/temp_file_manager.rb +5 -4
  86. data/lib/aspera/transfer/convert.rb +29 -0
  87. data/lib/aspera/transfer/error_info.rb +66 -66
  88. data/lib/aspera/transfer/parameters.rb +13 -68
  89. data/lib/aspera/transfer/spec.rb +5 -6
  90. data/lib/aspera/transfer/spec.schema.yaml +753 -0
  91. data/lib/aspera/transfer/spec_doc.rb +62 -0
  92. data/lib/aspera/transfer/sync.rb +23 -72
  93. data/lib/aspera/transfer/sync_instance.schema.yaml +13 -0
  94. data/lib/aspera/transfer/sync_session.schema.yaml +79 -0
  95. data/lib/aspera/transfer/uri.rb +6 -6
  96. data/lib/aspera/uri_reader.rb +18 -1
  97. data/lib/aspera/web_auth.rb +1 -1
  98. data/lib/aspera/web_server_simple.rb +53 -44
  99. data.tar.gz.sig +0 -0
  100. metadata +28 -165
  101. metadata.gz.sig +0 -0
  102. data/examples/build_exec +0 -74
  103. data/examples/build_exec_rubyc +0 -40
  104. data/examples/build_package.sh +0 -28
  105. data/lib/aspera/transfer/spec.yaml +0 -718
@@ -16,15 +16,68 @@ require 'English'
16
16
 
17
17
  module Aspera
18
18
  module Agent
19
- # executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
19
+ # executes a local "ascp", create mgt port
20
20
  class Direct < Base
21
+ # ascp started locally, so listen local
21
22
  LISTEN_LOCAL_ADDRESS = '127.0.0.1'
22
- # 0 means: select an available port
23
+ # 0 means: use any available port
23
24
  SELECT_AVAILABLE_PORT = 0
24
- # spellchecker: enable
25
25
  private_constant :LISTEN_LOCAL_ADDRESS, :SELECT_AVAILABLE_PORT
26
26
 
27
- # method of Base
27
+ # options for initialize (same as values in option transfer_info)
28
+ # @param ascp_args [Array] additional arguments to ascp
29
+ # @param wss [Boolean] true: if both SSH and wss in ts: prefer wss
30
+ # @param quiet [Boolean] by default no native ascp progress bar
31
+ # @param monitor [Boolean] set to false to eliminate management port
32
+ # @param trusted_certs [Array,NilClass] list of files with trusted certificates (stores)
33
+ # @param client_ssh_key [String] client ssh key option (from CLIENT_SSH_KEY_OPTIONS)
34
+ # @param check_ignore_cb [Proc] callback with host,port
35
+ # @param spawn_timeout_sec [Integer] timeout for ascp spawn
36
+ # @param spawn_delay_sec [Integer] optional delay to start between sessions
37
+ # @param multi_incr_udp [Boolean,NilClass] true: increment udp port for each session
38
+ # @param resume [Hash,NilClass] resume policy
39
+ # @param management_cb [Proc] callback for management events
40
+ # @param base_options [Hash] other options for base class
41
+ def initialize(
42
+ ascp_args: nil,
43
+ wss: true,
44
+ quiet: true,
45
+ trusted_certs: nil,
46
+ client_ssh_key: nil,
47
+ check_ignore_cb: nil,
48
+ spawn_timeout_sec: 2,
49
+ spawn_delay_sec: 2,
50
+ multi_incr_udp: nil,
51
+ resume: nil,
52
+ monitor: true,
53
+ management_cb: nil,
54
+ **base_options
55
+ )
56
+ super(**base_options)
57
+ # special transfer parameters
58
+ @tr_opts = {
59
+ ascp_args: ascp_args,
60
+ wss: wss,
61
+ quiet: quiet,
62
+ trusted_certs: trusted_certs,
63
+ client_ssh_key: client_ssh_key,
64
+ check_ignore_cb: check_ignore_cb
65
+ }
66
+ @spawn_timeout_sec = spawn_timeout_sec
67
+ @spawn_delay_sec = spawn_delay_sec
68
+ # default is true on Windows, false on other OSes
69
+ @multi_incr_udp = multi_incr_udp.nil? ? Environment.os.eql?(Environment::OS_WINDOWS) : multi_incr_udp
70
+ @monitor = monitor
71
+ @management_cb = management_cb
72
+ @resume_policy = Resumer.new(resume.nil? ? {} : resume.symbolize_keys)
73
+ # all transfer jobs, key = SecureRandom.uuid, protected by mutex, cond var on change
74
+ @sessions = []
75
+ # mutex protects global data accessed by threads
76
+ @mutex = Mutex.new
77
+ @pre_calc_sent = false
78
+ @pre_calc_last_size = nil
79
+ end
80
+
28
81
  # start ascp transfer(s) (non blocking), single or multi-session
29
82
  # session information added to @sessions
30
83
  # @param transfer_spec [Hash] aspera transfer specification
@@ -48,11 +101,10 @@ module Aspera
48
101
  # (even if the var is not used in single session)
49
102
  multi_session_info = nil
50
103
  if transfer_spec.key?('multi_session')
104
+ # Managed by multi-session, so delete from transfer spec
51
105
  multi_session_info = {
52
- count: transfer_spec['multi_session'].to_i
106
+ count: transfer_spec.delete('multi_session').to_i
53
107
  }
54
- # Managed by multi-session, so delete from transfer spec
55
- transfer_spec.delete('multi_session')
56
108
  if multi_session_info[:count].negative?
57
109
  Log.log.error{"multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0"}
58
110
  multi_session_info = nil
@@ -77,14 +129,14 @@ module Aspera
77
129
  error: nil, # exception if failed
78
130
  io: nil, # management port server socket
79
131
  token_regenerator: token_regenerator, # regenerate bearer token with oauth
80
- # env vars and args to ascp (from transfer spec)
132
+ # env vars and args for ascp (from transfer spec)
81
133
  env_args: Transfer::Parameters.new(transfer_spec, **@tr_opts).ascp_args
82
134
  }
83
135
 
84
136
  if multi_session_info.nil?
85
137
  Log.log.debug('Starting single session thread')
86
138
  # single session for transfer : simple
87
- session[:thread] = Thread.new {transfer_thread_entry(session)}
139
+ session[:thread] = Thread.new{transfer_thread_entry(session)}
88
140
  @sessions.push(session)
89
141
  else
90
142
  Log.log.debug('Starting multi session threads')
@@ -101,7 +153,7 @@ module Aspera
101
153
  # option: increment (default as per ascp manual) or not (cluster on other side ?)
102
154
  args.unshift('-O', (multi_session_info[:udp_base] + i - 1).to_s) if @multi_incr_udp
103
155
  # finally start the thread
104
- this_session[:thread] = Thread.new {transfer_thread_entry(this_session)}
156
+ this_session[:thread] = Thread.new{transfer_thread_entry(this_session)}
105
157
  @sessions.push(this_session)
106
158
  end
107
159
  end
@@ -132,9 +184,44 @@ module Aspera
132
184
 
133
185
  # @return [Array] list of sessions for a job
134
186
  def sessions_by_job(job_id)
135
- @sessions.select{|session| session[:job_id].eql?(job_id)}
187
+ @sessions.select{ |session| session[:job_id].eql?(job_id)}
136
188
  end
137
189
 
190
+ # send command to management port of command (used in `asession)
191
+ # @param job_id identified transfer process
192
+ # @param session_index index of session (for multi session)
193
+ # @param data command on mgt port, examples:
194
+ # {'type'=>'START','source'=>_path_,'destination'=>_path_}
195
+ # {'type'=>'DONE'}
196
+ def send_command(job_id, data)
197
+ session = @sessions.find{ |session| session[:job_id].eql?(job_id)}
198
+ Log.log.debug{"command: #{data}"}
199
+ session[:io].puts(Ascp::Management.command_to_stream(data))
200
+ end
201
+
202
+ private
203
+
204
+ # transfer thread entry
205
+ # @param session information
206
+ def transfer_thread_entry(session)
207
+ begin
208
+ # set name for logging
209
+ Thread.current[:name] = 'transfer'
210
+ Log.log.debug{"ENTER (#{Thread.current[:name]})"}
211
+ # start transfer with selected resumer policy
212
+ @resume_policy.execute_with_resume do
213
+ start_and_monitor_process(session: session, **session[:env_args])
214
+ end
215
+ Log.log.debug('transfer ok'.bg_green)
216
+ rescue StandardError => e
217
+ session[:error] = e
218
+ Log.log.error{"Transfer thread error: #{e.class}:\n#{e.message}:\n#{e.backtrace.join("\n")}".red} if Log.instance.level.eql?(:debug)
219
+ end
220
+ Log.log.debug{"EXIT (#{Thread.current[:name]})"}
221
+ end
222
+
223
+ public
224
+
138
225
  # This is the low level method to start the transfer process.
139
226
  # Typically started in a thread.
140
227
  # Start process with management port.
@@ -154,24 +241,32 @@ module Aspera
154
241
  notify_progress(:pre_start, session_id: nil, info: 'starting')
155
242
  begin
156
243
  command_pid = nil
157
- # we use Socket directly, instead of TCPServer, as it gives access to lower level options
158
- socket_class = defined?(JRUBY_VERSION) ? ServerSocket : Socket
159
- mgt_server_socket = socket_class.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
160
- # open any available (0) local TCP port for use as management port
161
- mgt_server_socket.bind(Addrinfo.tcp(LISTEN_LOCAL_ADDRESS, SELECT_AVAILABLE_PORT))
162
- # make port ready to accept connections, before starting ascp
163
- mgt_server_socket.listen(1)
164
- # build arguments and add mgt port
165
- command_arguments = if name.eql?(:async)
166
- ["--exclusive-mgmt-port=#{mgt_server_socket.local_address.ip_port}"]
167
- else
168
- ['-M', mgt_server_socket.local_address.ip_port.to_s]
244
+ command_arguments = []
245
+ if @monitor
246
+ # we use Socket directly, instead of TCPServer, as it gives access to lower level options
247
+ socket_class = defined?(JRUBY_VERSION) ? ServerSocket : Socket
248
+ mgt_server_socket = socket_class.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
249
+ # open any available (0) local TCP port for use as management port
250
+ mgt_server_socket.bind(Addrinfo.tcp(LISTEN_LOCAL_ADDRESS, SELECT_AVAILABLE_PORT))
251
+ # make port ready to accept connections, before starting ascp
252
+ mgt_server_socket.listen(1)
253
+ # build arguments and add mgt port
254
+ command_arguments = if name.eql?(:async)
255
+ ["--exclusive-mgmt-port=#{mgt_server_socket.local_address.ip_port}"]
256
+ else
257
+ ['-M', mgt_server_socket.local_address.ip_port.to_s]
258
+ end
169
259
  end
170
260
  command_arguments.concat(args)
261
+ # capture process stderr
262
+ stderr_r, stderr_w = IO.pipe
171
263
  # get location of command executable (ascp, async)
172
264
  command_path = Ascp::Installation.instance.path(name)
173
- command_pid = Environment.secure_spawn(env: env, exec: command_path, args: command_arguments)
265
+ command_pid = Environment.secure_spawn(env: env, exec: command_path, args: command_arguments, err: stderr_w)
266
+ stderr_w.close
174
267
  notify_progress(:pre_start, session_id: nil, info: "waiting for #{name} to start")
268
+ # "ensure" block will wait for process
269
+ return unless @monitor
175
270
  # TODO: timeout does not work when Process.spawn is used... until process exits, then it works
176
271
  # So we use select to detect that anything happens on the socket (connection)
177
272
  Log.log.debug{"before select, timeout: #{@spawn_timeout_sec}"}
@@ -182,9 +277,9 @@ module Aspera
182
277
  client_socket, _client_addrinfo = mgt_server_socket.accept
183
278
  Log.log.debug('after accept')
184
279
  management_port_io = client_socket.to_io
185
- # management messages include file names which may be utf8
186
280
  # by default socket is US-ASCII
187
- # TODO: use same value as Encoding.default_external
281
+ # management messages include file names which may be UTF-8
282
+ # TODO: use same value as Encoding.default_external ?
188
283
  management_port_io.set_encoding(Encoding::UTF_8)
189
284
  session[:io] = management_port_io
190
285
  processor = Ascp::Management.new
@@ -194,7 +289,7 @@ module Aspera
194
289
  next unless event
195
290
  # event is ready
196
291
  Log.log.trace1{Log.dump(:management_port, event)}
197
- # store latest event by type
292
+ # store session identifier
198
293
  session[:id] = event['SessionId'] if event['Type'].eql?('INIT')
199
294
  @management_cb&.call(event)
200
295
  process_progress(event)
@@ -203,8 +298,10 @@ module Aspera
203
298
  Log.log.debug('management io closed')
204
299
  # check that last status was received before process exit
205
300
  last_event = processor.last_event
206
- raise Transfer::Error, "internal: no management event (#{last_event.class})" unless last_event.is_a?(Hash)
301
+ raise Transfer::Error, "No management event (#{last_event.class})" unless last_event.is_a?(Hash)
207
302
  case last_event['Type']
303
+ when 'DONE'
304
+ Log.log.trace1{'Graceful shutdown, DONE message received'}
208
305
  when 'ERROR'
209
306
  if /bearer token/i.match?(last_event['Description']) &&
210
307
  session[:token_regenerator].respond_to?(:refreshed_transfer_token)
@@ -214,10 +311,9 @@ module Aspera
214
311
  env['ASPERA_SCP_TOKEN'] = session[:token_regenerator].refreshed_transfer_token
215
312
  end
216
313
  raise Transfer::Error.new(last_event['Description'], last_event['Code'].to_i)
217
- when 'DONE'
218
- nil
219
314
  else
220
- raise Transfer::Error, "unexpected last event type: #{last_event['Type']}, #{last_event['Description']}"
315
+ Log.log.error{"unexpected last event type: #{last_event['Type']}"}
316
+ # raise Transfer::Error, "unexpected last event type: #{last_event['Type']}, #{last_event['Description']}"
221
317
  end
222
318
  rescue SystemCallError => e
223
319
  # Process.spawn failed, or socket error
@@ -225,17 +321,21 @@ module Aspera
225
321
  rescue Interrupt
226
322
  raise Transfer::Error, 'transfer interrupted by user'
227
323
  ensure
228
- mgt_server_socket.close
324
+ mgt_server_socket&.close
325
+ session.delete(:io)
229
326
  # if command was successfully started, check its status
230
327
  unless command_pid.nil?
231
- # "wait" for process to avoid zombie
232
- Process.wait(command_pid)
233
- status = $CHILD_STATUS
234
- # command_pid = nil
235
- session.delete(:io)
328
+ Process.kill(:INT, command_pid) if @monitor
329
+ # collect process exit status or wait for termination
330
+ _, status = Process.wait2(command_pid)
331
+ # process stderr of ascp
332
+ stderr_r.each_line do |line|
333
+ Log.log.error(line.chomp)
334
+ end
335
+ stderr_r.close
236
336
  # status is nil if an exception occurred before starting command
237
337
  if !status&.success?
238
- message = status.nil? ? "#{name} not started" : "#{name} failed (#{status})"
338
+ message = "#{name} failed (#{status})"
239
339
  # raise error only if there was not already an exception (ERROR_INFO)
240
340
  raise Transfer::Error, message unless $ERROR_INFO
241
341
  # else display this message also, as main exception is already here
@@ -246,10 +346,10 @@ module Aspera
246
346
  nil
247
347
  end
248
348
 
249
- attr_reader :sessions
250
-
251
349
  private
252
350
 
351
+ attr_reader :sessions
352
+
253
353
  # notify progress to callback
254
354
  # @param event management port event
255
355
  # @param session sessin object
@@ -285,88 +385,8 @@ module Aspera
285
385
  # cspell:enable
286
386
  # stop event when one file is completed
287
387
  else
288
- Log.log.debug{"unknown event type #{event['Type']}"}
289
- end
290
- end
291
-
292
- # send command to management port of command (used in `asession)
293
- # @param job_id identified transfer process
294
- # @param session_index index of session (for multi session)
295
- # @param data command on mgt port, examples:
296
- # {'type'=>'START','source'=>_path_,'destination'=>_path_}
297
- # {'type'=>'DONE'}
298
- def send_command(job_id, data)
299
- session = @sessions.find{|session| session[:job_id].eql?(job_id)}
300
- Log.log.debug{"command: #{data}"}
301
- session[:io].puts(Ascp::Management.command_to_stream(data))
302
- end
303
-
304
- # options for initialize (same as values in option transfer_info)
305
- # @param ascp_args [Array] additional arguments to ascp
306
- # @param wss [Boolean] true: if both SSH and wss in ts: prefer wss
307
- # @param quiet [Boolean] by default no native ascp progress bar
308
- # @param trusted_certs [Array,NilClass] list of files with trusted certificates (stores)
309
- # @param client_ssh_key [String] client ssh key option (from CLIENT_SSH_KEY_OPTIONS)
310
- # @param check_ignore_cb [Proc] callback with host,port
311
- # @param spawn_timeout_sec [Integer] timeout for ascp spawn
312
- # @param spawn_delay_sec [Integer] optional delay to start between sessions
313
- # @param multi_incr_udp [Boolean,NilClass] true: increment udp port for each session
314
- # @param resume [Hash,NilClass] resume policy
315
- # @param management_cb [Proc] callback for management events
316
- # @param base_options [Hash] other options for base class
317
- def initialize(
318
- ascp_args: nil,
319
- wss: true,
320
- quiet: true,
321
- trusted_certs: nil,
322
- client_ssh_key: nil,
323
- check_ignore_cb: nil,
324
- spawn_timeout_sec: 2,
325
- spawn_delay_sec: 2,
326
- multi_incr_udp: nil,
327
- resume: nil,
328
- management_cb: nil,
329
- **base_options
330
- )
331
- super(**base_options)
332
- # special transfer parameters
333
- @tr_opts = {
334
- ascp_args: ascp_args,
335
- wss: wss,
336
- quiet: quiet,
337
- trusted_certs: trusted_certs,
338
- client_ssh_key: client_ssh_key,
339
- check_ignore_cb: check_ignore_cb
340
- }
341
- @spawn_timeout_sec = spawn_timeout_sec
342
- @spawn_delay_sec = spawn_delay_sec
343
- # default is true on windows, false on other platforms
344
- @multi_incr_udp = multi_incr_udp.nil? ? Environment.os.eql?(Environment::OS_WINDOWS) : multi_incr_udp
345
- @management_cb = management_cb
346
- @resume_policy = Resumer.new(resume.nil? ? {} : resume.symbolize_keys)
347
- # all transfer jobs, key = SecureRandom.uuid, protected by mutex, cond var on change
348
- @sessions = []
349
- # mutex protects global data accessed by threads
350
- @mutex = Mutex.new
351
- end
352
-
353
- # transfer thread entry
354
- # @param session information
355
- def transfer_thread_entry(session)
356
- begin
357
- # set name for logging
358
- Thread.current[:name] = 'transfer'
359
- Log.log.debug{"ENTER (#{Thread.current[:name]})"}
360
- # start transfer with selected resumer policy
361
- @resume_policy.execute_with_resume do
362
- start_and_monitor_process(session: session, **session[:env_args])
363
- end
364
- Log.log.debug('transfer ok'.bg_green)
365
- rescue StandardError => e
366
- session[:error] = e
367
- Log.log.error{"Transfer thread error: #{e.class}:\n#{e.message}:\n#{e.backtrace.join("\n")}".red} if Log.instance.level.eql?(:debug)
388
+ Log.log.debug{"Unknown event type for progress: #{event['Type']}"}
368
389
  end
369
- Log.log.debug{"EXIT (#{Thread.current[:name]})"}
370
390
  end
371
391
  end
372
392
  end
@@ -9,9 +9,28 @@ require 'aspera/assert'
9
9
  module Aspera
10
10
  module Agent
11
11
  class Httpgw < Base
12
- # start FASP transfer based on transfer spec (hash table)
13
- # note that it is asynchronous
12
+ def initialize(
13
+ url:,
14
+ api_version: Api::Httpgw::API_V2,
15
+ upload_chunk_size: 64_000,
16
+ synchronous: false,
17
+ **base_options
18
+ )
19
+ super(**base_options)
20
+ @gw_api = Api::Httpgw.new(
21
+ # remove /v1 from end of user-provided GW url: we need the base url only
22
+ url: url,
23
+ api_version: api_version,
24
+ upload_chunk_size: upload_chunk_size,
25
+ synchronous: synchronous,
26
+ notify_cb: ->(*pa, **ka){notify_progress(*pa, **ka)}
27
+ )
28
+ end
29
+
30
+ # Start FASP transfer based on transfer spec (hash table)
31
+ # note that this should be asynchronous, but it is not
14
32
  # HTTP download only supports file list
33
+ # :reek:UnusedParameters token_regenerator
15
34
  def start_transfer(transfer_spec, token_regenerator: nil)
16
35
  raise 'GW URL must be set' if @gw_api.nil?
17
36
  Aspera.assert_type(transfer_spec['paths'], Array){'paths'}
@@ -27,35 +46,12 @@ module Aspera
27
46
  end
28
47
  end
29
48
 
30
- # wait for completion of all jobs started
49
+ # Wait for completion of all jobs started
31
50
  # @return list of :success or error message
32
51
  def wait_for_transfers_completion
33
52
  # well ... transfer was done in "start"
34
53
  return [:success]
35
54
  end
36
-
37
- # TODO: is that useful?
38
- # def url=(api_url); end
39
-
40
- private
41
-
42
- def initialize(
43
- url:,
44
- api_version: Api::Httpgw::API_V2,
45
- upload_chunk_size: 64_000,
46
- synchronous: false,
47
- **base_options
48
- )
49
- super(**base_options)
50
- @gw_api = Api::Httpgw.new(
51
- # remove /v1 from end of user-provided GW url: we need the base url only
52
- url: url,
53
- api_version: api_version,
54
- upload_chunk_size: upload_chunk_size,
55
- synchronous: synchronous,
56
- notify_cb: ->(*pa, **ka) { notify_progress(*pa, **ka) }
57
- )
58
- end
59
55
  end
60
56
  end
61
57
  end
@@ -13,10 +13,10 @@ module Aspera
13
13
  # this singleton class is used by the CLI to provide a common interface to start a transfer
14
14
  # before using it, the use must set the `node_api` member.
15
15
  class Node < Base
16
- # @param url [String] the base url of the node api
17
- # @param username [String] the username to use for the node api
18
- # @param password [String] the password to use for the node api
19
- # @param root_id [String] root file id if the node is an access key
16
+ # @param url [String] the base url of the node api
17
+ # @param username [String] the username to use for the node api
18
+ # @param password [String] the password to use for the node api
19
+ # @param root_id [String] root file id if the node is an access key
20
20
  # @param base_options [Hash] options for base class
21
21
  def initialize(
22
22
  url:,
@@ -28,7 +28,7 @@ module Aspera
28
28
  super(**base_options)
29
29
  # root id is required for access key
30
30
  @root_id = root_id
31
- rest_params = { base_url: url}
31
+ rest_params = {base_url: url}
32
32
  if OAuth::Factory.bearer?(password)
33
33
  Aspera.assert(!@root_id.nil?){'root_id not allowed for access key'}
34
34
  rest_params[:headers] = Api::Node.bearer_headers(password, access_key: username)
@@ -44,13 +44,8 @@ module Aspera
44
44
  @transfer_id = nil
45
45
  end
46
46
 
47
- # used internally to ensure node api is set before using.
48
- def node_api_
49
- Aspera.assert(!@node_api.nil?){'Before using this object, set the node_api attribute to a Aspera::Rest object'}
50
- return @node_api
51
- end
52
-
53
47
  # generic method
48
+ # :reek:UnusedParameters token_regenerator
54
49
  def start_transfer(transfer_spec, token_regenerator: nil)
55
50
  # add root id if access key
56
51
  if !@root_id.nil?
@@ -119,6 +114,14 @@ module Aspera
119
114
  # TODO: get status of sessions
120
115
  return []
121
116
  end
117
+
118
+ private
119
+
120
+ # used internally to ensure node api is set before using.
121
+ def node_api_
122
+ Aspera.assert(!@node_api.nil?){'Before using this object, set the node_api attribute to a Aspera::Rest object'}
123
+ return @node_api
124
+ end
122
125
  end
123
126
  end
124
127
  end
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'aspera/environment'
3
4
  require 'aspera/agent/base'
4
5
  require 'aspera/products/transferd'
5
6
  require 'aspera/temp_file_manager'
@@ -17,20 +18,23 @@ module Aspera
17
18
  # port zero means select a random available high port
18
19
  AUTO_LOCAL_TCP_PORT = "#{PORT_SEP}0"
19
20
 
20
- # @param url [String] URL of the transfer manager daemon
21
- # @param external [Boolean] if true, expect that an external daemon is already running
22
- # @param keep [Boolean] if true, do not shutdown daemon on exit
23
- # @param base_options [Hash] base options
21
+ private_constant :LOCAL_SOCKET_ADDR, :PORT_SEP, :AUTO_LOCAL_TCP_PORT
22
+
23
+ # @param url [String] URL of the transfer manager daemon
24
+ # @param start [Bool] if false, expect that an external daemon is already running
25
+ # @param stop [Bool] if false, do not shutdown daemon on exit
26
+ # @param base [Hash] base class options
24
27
  def initialize(
25
- url: AUTO_LOCAL_TCP_PORT,
26
- external: false,
27
- keep: false,
28
- **base_options
28
+ url: AUTO_LOCAL_TCP_PORT,
29
+ start: true,
30
+ stop: true,
31
+ **base
29
32
  )
30
- super(**base_options)
31
- @keep = keep
33
+ super(**base)
34
+ @transfer_id = nil
35
+ @stop = stop
32
36
  is_local_auto_port = url.eql?(AUTO_LOCAL_TCP_PORT)
33
- raise 'Cannot use options `keep` or `external` with port zero' if is_local_auto_port && (@keep || external)
37
+ raise 'Cannot set options `stop` or `start` to false with port zero' if is_local_auto_port && (!@stop || !start)
34
38
  # keep PID for optional shutdown
35
39
  @daemon_pid = nil
36
40
  daemon_endpoint = url
@@ -44,14 +48,14 @@ module Aspera
44
48
  # Initiate actual connection
45
49
  get_info_response = @transfer_client.get_info(::Transferd::Api::InstanceInfoRequest.new)
46
50
  Log.log.debug{"Daemon info: #{get_info_response}"}
47
- Log.log.warn{'Attached to existing daemon'} unless @daemon_pid || external || @keep
51
+ Log.log.warn('Attached to existing daemon') unless @daemon_pid || !start || !@stop
48
52
  at_exit{shutdown}
49
53
  rescue GRPC::Unavailable => e
50
54
  # if transferd is external: do not start it, or other error
51
- raise if external || !e.message.include?('failed to connect')
55
+ raise if !start || !e.message.include?('failed to connect')
52
56
  # we already tried to start a daemon, but it failed
53
57
  Aspera.assert(@daemon_pid.nil?){"Daemon started with PID #{@daemon_pid}, but connection failed to #{daemon_endpoint}}"}
54
- Log.log.warn('no daemon present, starting daemon...') if external
58
+ Log.log.warn('no daemon present, starting daemon...') if !start
55
59
  # transferd only supports local ip and port
56
60
  daemon_uri = URI.parse("ipv4://#{daemon_endpoint}")
57
61
  Aspera.assert(daemon_uri.scheme.eql?('ipv4')){"Invalid scheme daemon URI #{daemon_endpoint}"}
@@ -74,7 +78,11 @@ module Aspera
74
78
  log_stdout = "#{transferd_base_tmp}.out"
75
79
  log_stderr = "#{transferd_base_tmp}.err"
76
80
  File.write(conf_file, config.to_json)
77
- @daemon_pid = Process.spawn(Ascp::Installation.instance.path(:transferd), '--config', conf_file, out: log_stdout, err: log_stderr)
81
+ @daemon_pid = Environment.secure_spawn(
82
+ exec: Ascp::Installation.instance.path(:transferd),
83
+ args: ['--config', conf_file],
84
+ out: log_stdout,
85
+ err: log_stderr)
78
86
  begin
79
87
  # wait for process to initialize, max 2 seconds
80
88
  Timeout.timeout(2.0) do
@@ -86,8 +94,8 @@ module Aspera
86
94
  nil
87
95
  end
88
96
  Log.log.debug{"Daemon started with pid #{@daemon_pid}"}
89
- Process.detach(@daemon_pid) if @keep
90
- at_exit {shutdown}
97
+ Process.detach(@daemon_pid) unless @stop
98
+ at_exit{shutdown}
91
99
  # update port for next connection attempt (if auto high port was requested)
92
100
  daemon_endpoint = "#{LOCAL_SOCKET_ADDR}#{PORT_SEP}#{Products::Transferd.daemon_port_from_log(log_stdout)}" if is_local_auto_port
93
101
  # local daemon started, try again
@@ -95,6 +103,7 @@ module Aspera
95
103
  end
96
104
  end
97
105
 
106
+ # :reek:UnusedParameters token_regenerator
98
107
  def start_transfer(transfer_spec, token_regenerator: nil)
99
108
  # create a transfer request
100
109
  transfer_request = ::Transferd::Api::TransferRequest.new(
@@ -146,13 +155,15 @@ module Aspera
146
155
  end
147
156
 
148
157
  def shutdown
149
- stop_daemon unless @keep
158
+ stop_daemon if @stop
150
159
  end
151
160
 
161
+ private
162
+
152
163
  def stop_daemon
153
164
  if !@daemon_pid.nil?
154
165
  Log.log.debug("Stopping daemon #{@daemon_pid}")
155
- Process.kill('INT', @daemon_pid)
166
+ Process.kill(:INT, @daemon_pid)
156
167
  _, status = Process.wait2(@daemon_pid)
157
168
  Log.log.debug("daemon stopped #{status}")
158
169
  @daemon_pid = nil
@@ -7,7 +7,7 @@ module Aspera
7
7
  def initialize(entitlement_id, customer_id, api_domain: AoC::SAAS_DOMAIN_PROD, version: 'v1')
8
8
  super(
9
9
  base_url: "https://api.#{api_domain}/metering/#{version}",
10
- headers: {'X-Aspera-Entitlement-Authorization' => Rest.basic_token(entitlement_id, customer_id)}
10
+ headers: {'X-Aspera-Entitlement-Authorization' => Rest.basic_authorization(entitlement_id, customer_id)}
11
11
  )
12
12
  end
13
13
  end
@@ -22,7 +22,7 @@ module Aspera
22
22
  MAX_AOC_URL_REDIRECT = 10
23
23
  CLIENT_ID_PREFIX = 'aspera.'
24
24
  # Well-known AoC global client apps
25
- GLOBAL_CLIENT_APPS = DataRepository::ELEMENTS.select{|i|i.to_s.start_with?(CLIENT_ID_PREFIX)}.freeze
25
+ GLOBAL_CLIENT_APPS = DataRepository::ELEMENTS.select{ |i| i.to_s.start_with?(CLIENT_ID_PREFIX)}.freeze
26
26
  # cookie prefix so that console can decode identity
27
27
  COOKIE_PREFIX_CONSOLE_AOC = 'aspera.aoc'
28
28
  # path in URL of public links
@@ -237,7 +237,7 @@ module Aspera
237
237
  Log.log.debug{"ignoring error: #{e}"}
238
238
  {}
239
239
  end
240
- USER_INFO_FIELDS_MIN.each{|f|@cache_user_info[f] = nil if @cache_user_info[f].nil?}
240
+ USER_INFO_FIELDS_MIN.each{ |f| @cache_user_info[f] = nil if @cache_user_info[f].nil?}
241
241
  return @cache_user_info
242
242
  end
243
243
 
@@ -283,7 +283,7 @@ module Aspera
283
283
  if ws_info.nil?
284
284
  {
285
285
  id: nil,
286
- name: 'Shared folders'
286
+ name: "Shared #{application}"
287
287
  }
288
288
  else
289
289
  {
@@ -386,10 +386,10 @@ module Aspera
386
386
  Aspera.assert(field.key?('name')){'metadata field must have name'}
387
387
  Aspera.assert(field.key?('values')){'metadata field must have values'}
388
388
  Aspera.assert_type(field['values'], Array){'metadata field values'}
389
- Aspera.assert(!meta_schema.none?{|i|i['name'].eql?(field['name'])}){"unknown metadata field: #{field['name']}"}
389
+ Aspera.assert(!meta_schema.none?{ |i| i['name'].eql?(field['name'])}){"unknown metadata field: #{field['name']}"}
390
390
  end
391
391
  meta_schema.each do |field|
392
- provided = pkg_meta.select{|i|i['name'].eql?(field['name'])}
392
+ provided = pkg_meta.select{ |i| i['name'].eql?(field['name'])}
393
393
  raise "only one field with name #{field['name']} allowed" if provided.count > 1
394
394
  raise "missing mandatory field: #{field['name']}" if field['required'] && provided.empty?
395
395
  end
@@ -519,7 +519,7 @@ module Aspera
519
519
  # Console cookie
520
520
  ################
521
521
  # we are sure that fields are not nil
522
- cookie_elements = [app_info[:app], current_user_info['name'] || 'public link', current_user_info['email'] || 'none'].map{|e|Base64.strict_encode64(e)}
522
+ cookie_elements = [app_info[:app], current_user_info['name'] || 'public link', current_user_info['email'] || 'none'].map{ |e| Base64.strict_encode64(e)}
523
523
  cookie_elements.unshift(COOKIE_PREFIX_CONSOLE_AOC)
524
524
  transfer_spec['cookie'] = cookie_elements.join(':')
525
525
  # Application tags