aspera-cli 4.18.1 → 4.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +33 -0
  4. data/CONTRIBUTING.md +17 -12
  5. data/README.md +396 -185
  6. data/bin/asession +26 -19
  7. data/examples/build_exec +74 -0
  8. data/examples/{rubyc → build_exec_rubyc} +18 -2
  9. data/examples/get_proto_file.rb +7 -0
  10. data/lib/aspera/agent/alpha.rb +8 -8
  11. data/lib/aspera/agent/base.rb +4 -18
  12. data/lib/aspera/agent/connect.rb +14 -13
  13. data/lib/aspera/agent/direct.rb +123 -120
  14. data/lib/aspera/agent/httpgw.rb +2 -3
  15. data/lib/aspera/agent/node.rb +10 -10
  16. data/lib/aspera/agent/trsdk.rb +17 -20
  17. data/lib/aspera/api/alee.rb +15 -0
  18. data/lib/aspera/api/aoc.rb +128 -99
  19. data/lib/aspera/api/ats.rb +1 -1
  20. data/lib/aspera/api/cos_node.rb +1 -1
  21. data/lib/aspera/api/httpgw.rb +104 -64
  22. data/lib/aspera/api/node.rb +33 -12
  23. data/lib/aspera/ascmd.rb +56 -48
  24. data/lib/aspera/ascp/installation.rb +142 -70
  25. data/lib/aspera/ascp/management.rb +7 -3
  26. data/lib/aspera/ascp/products.rb +13 -7
  27. data/lib/aspera/assert.rb +10 -5
  28. data/lib/aspera/cli/formatter.rb +42 -26
  29. data/lib/aspera/cli/hints.rb +2 -1
  30. data/lib/aspera/cli/info.rb +12 -10
  31. data/lib/aspera/cli/main.rb +16 -13
  32. data/lib/aspera/cli/manager.rb +15 -10
  33. data/lib/aspera/cli/plugin.rb +17 -31
  34. data/lib/aspera/cli/plugin_factory.rb +10 -1
  35. data/lib/aspera/cli/plugins/alee.rb +3 -3
  36. data/lib/aspera/cli/plugins/aoc.rb +222 -194
  37. data/lib/aspera/cli/plugins/ats.rb +16 -14
  38. data/lib/aspera/cli/plugins/config.rb +66 -53
  39. data/lib/aspera/cli/plugins/console.rb +3 -3
  40. data/lib/aspera/cli/plugins/faspex.rb +11 -21
  41. data/lib/aspera/cli/plugins/faspex5.rb +44 -42
  42. data/lib/aspera/cli/plugins/faspio.rb +2 -2
  43. data/lib/aspera/cli/plugins/httpgw.rb +1 -1
  44. data/lib/aspera/cli/plugins/node.rb +155 -96
  45. data/lib/aspera/cli/plugins/orchestrator.rb +14 -13
  46. data/lib/aspera/cli/plugins/preview.rb +8 -9
  47. data/lib/aspera/cli/plugins/server.rb +6 -10
  48. data/lib/aspera/cli/plugins/shares.rb +13 -9
  49. data/lib/aspera/cli/sync_actions.rb +72 -31
  50. data/lib/aspera/cli/transfer_agent.rb +13 -14
  51. data/lib/aspera/cli/transfer_progress.rb +36 -18
  52. data/lib/aspera/cli/version.rb +1 -1
  53. data/lib/aspera/command_line_builder.rb +3 -4
  54. data/lib/aspera/coverage.rb +13 -1
  55. data/lib/aspera/environment.rb +59 -10
  56. data/lib/aspera/faspex_gw.rb +3 -3
  57. data/lib/aspera/json_rpc.rb +1 -1
  58. data/lib/aspera/keychain/encrypted_hash.rb +2 -0
  59. data/lib/aspera/keychain/macos_security.rb +7 -12
  60. data/lib/aspera/log.rb +4 -4
  61. data/lib/aspera/node_simulator.rb +1 -1
  62. data/lib/aspera/oauth/base.rb +39 -45
  63. data/lib/aspera/oauth/factory.rb +11 -4
  64. data/lib/aspera/oauth/generic.rb +4 -8
  65. data/lib/aspera/oauth/jwt.rb +4 -4
  66. data/lib/aspera/oauth/url_json.rb +3 -2
  67. data/lib/aspera/oauth/web.rb +10 -6
  68. data/lib/aspera/persistency_action_once.rb +16 -8
  69. data/lib/aspera/preview/utils.rb +5 -16
  70. data/lib/aspera/rest.rb +100 -76
  71. data/lib/aspera/secret_hider.rb +3 -2
  72. data/lib/aspera/ssh.rb +1 -1
  73. data/lib/aspera/transfer/faux_file.rb +7 -5
  74. data/lib/aspera/transfer/parameters.rb +41 -35
  75. data/lib/aspera/transfer/spec.rb +16 -18
  76. data/lib/aspera/transfer/sync.rb +51 -50
  77. data/lib/aspera/transfer/uri.rb +1 -1
  78. data/lib/aspera/uri_reader.rb +1 -1
  79. data/lib/aspera/web_auth.rb +166 -18
  80. data/lib/aspera/web_server_simple.rb +27 -15
  81. data/lib/transfer_pb.rb +84 -0
  82. data/lib/transfer_services_pb.rb +82 -0
  83. data.tar.gz.sig +0 -0
  84. metadata +25 -6
  85. metadata.gz.sig +0 -0
@@ -19,9 +19,10 @@ module Aspera
19
19
  # executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
20
20
  class Direct < Base
21
21
  LISTEN_LOCAL_ADDRESS = '127.0.0.1'
22
- ANY_AVAILABLE_PORT = 0 # 0 means any available port
22
+ # 0 means: select an available port
23
+ SELECT_AVAILABLE_PORT = 0
23
24
  # spellchecker: enable
24
- private_constant :LISTEN_LOCAL_ADDRESS, :ANY_AVAILABLE_PORT
25
+ private_constant :LISTEN_LOCAL_ADDRESS, :SELECT_AVAILABLE_PORT
25
26
 
26
27
  # method of Base
27
28
  # start ascp transfer(s) (non blocking), single or multi-session
@@ -93,12 +94,12 @@ module Aspera
93
94
  # do deep copy (each thread has its own copy because it is modified here below and in thread)
94
95
  this_session = session.clone
95
96
  this_session[:ts] = this_session[:ts].clone
96
- this_session[:env_args] = this_session[:env_args].clone
97
- this_session[:env_args][:args] = this_session[:env_args][:args].clone
97
+ env_args = this_session[:env_args] = this_session[:env_args].clone
98
+ args = env_args[:args] = env_args[:args].clone
98
99
  # set multi session part
99
- this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_info[:count]}")
100
+ args.unshift("-C#{i}:#{multi_session_info[:count]}")
100
101
  # option: increment (default as per ascp manual) or not (cluster on other side ?)
101
- this_session[:env_args][:args].unshift('-O', (multi_session_info[:udp_base] + i - 1).to_s) if @multi_incr_udp
102
+ args.unshift('-O', (multi_session_info[:udp_base] + i - 1).to_s) if @multi_incr_udp
102
103
  # finally start the thread
103
104
  this_session[:thread] = Thread.new(this_session) {|session_info|transfer_thread_entry(session_info)}
104
105
  @sessions.push(this_session)
@@ -129,81 +130,46 @@ module Aspera
129
130
  Log.log.debug('fasp local shutdown')
130
131
  end
131
132
 
132
- # @param event management port event
133
- def process_progress(event)
134
- session_id = event['SessionId']
135
- case event['Type']
136
- when 'INIT'
137
- @pre_calc_sent = false
138
- @pre_calc_last_size = nil
139
- notify_progress(session_id: session_id, type: :session_start)
140
- when 'NOTIFICATION' # sent from remote
141
- if event.key?('PreTransferBytes')
142
- @pre_calc_sent = true
143
- notify_progress(session_id: session_id, type: :session_size, info: event['PreTransferBytes'])
144
- end
145
- when 'STATS' # during transfer
146
- @pre_calc_last_size = event['TransferBytes'].to_i + event['StartByte'].to_i
147
- notify_progress(session_id: session_id, type: :transfer, info: @pre_calc_last_size)
148
- when 'DONE', 'ERROR' # end of session
149
- total_size = event['TransferBytes'].to_i + event['StartByte'].to_i
150
- if !@pre_calc_sent && !total_size.zero?
151
- notify_progress(session_id: session_id, type: :session_size, info: total_size)
152
- end
153
- if @pre_calc_last_size != total_size
154
- notify_progress(session_id: session_id, type: :transfer, info: total_size)
155
- end
156
- notify_progress(session_id: session_id, type: :end)
157
- # cspell:disable
158
- when 'SESSION'
159
- when 'ARGSTOP'
160
- when 'FILEERROR'
161
- when 'STOP'
162
- # cspell:enable
163
- # stop event when one file is completed
164
- else
165
- Log.log.debug{"unknown event type #{event['Type']}"}
166
- end
133
+ # @return [Array] list of sessions for a job
134
+ def sessions_by_job(job_id)
135
+ @sessions.select{|session_info| session_info[:job_id].eql?(job_id)}
167
136
  end
168
137
 
169
- # This is the low level method to start the "ascp" process
170
- # currently, relies on command line arguments
171
- # start ascp with management port.
138
+ # This is the low level method to start the transfer process
139
+ # Start process with management port.
140
+ # returns
172
141
  # raises FaspError on error
173
- # if there is a thread info: set and broadcast session id
174
- # runs in separate thread
175
- # @param env_args a hash containing :args :env :ascp_version
176
- # @param session this session information
177
- # could be private method
178
- def start_transfer_with_args_env(env_args, session)
179
- Aspera.assert_type(env_args, Hash)
142
+ # @param session this session information, keys :io and :token_regenerator
143
+ # @param env [Hash] environment variables
144
+ # @param name [Symbol] name of executable: :ascp, :ascp4 or :async
145
+ # @param args [Array] command line arguments
146
+ # @return [nil] when process has exited
147
+ def start_and_monitor_process(
148
+ session:,
149
+ env:,
150
+ name:,
151
+ args:
152
+ )
180
153
  Aspera.assert_type(session, Hash)
181
- Log.log.debug{"env_args=#{env_args.inspect}"}
182
- notify_progress(session_id: nil, type: :pre_start, info: 'starting')
154
+ notify_progress(:pre_start, session_id: nil, info: 'starting')
183
155
  begin
184
- ascp_pid = nil
156
+ command_pid = nil
185
157
  # we use Socket directly, instead of TCPServer, as it gives access to lower level options
186
- socket_class = RUBY_ENGINE.eql?('jruby') ? ServerSocket : Socket
158
+ socket_class = defined?(JRUBY_VERSION) ? ServerSocket : Socket
187
159
  mgt_server_socket = socket_class.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
188
- # open any available (0) local TCP port for use as ascp management port
189
- mgt_server_socket.bind(Addrinfo.tcp(LISTEN_LOCAL_ADDRESS, ANY_AVAILABLE_PORT))
160
+ # open any available (0) local TCP port for use as management port
161
+ mgt_server_socket.bind(Addrinfo.tcp(LISTEN_LOCAL_ADDRESS, SELECT_AVAILABLE_PORT))
190
162
  # build arguments and add mgt port
191
- ascp_arguments = ['-M', mgt_server_socket.local_address.ip_port.to_s].concat(env_args[:args])
192
- # get location of ascp executable
193
- ascp_path = Ascp::Installation.instance.path(env_args[:ascp_version])
194
- # display ascp command line
195
- Log.log.debug do
196
- [
197
- 'execute:'.red,
198
- env_args[:env].map{|k, v| "#{k}=#{Shellwords.shellescape(v)}"},
199
- Shellwords.shellescape(ascp_path),
200
- ascp_arguments.map{|a|Shellwords.shellescape(a)}
201
- ].flatten.join(' ')
163
+ command_arguments = if name.eql?(:async)
164
+ ["--exclusive-mgmt-port=#{mgt_server_socket.local_address.ip_port}"]
165
+ else
166
+ ['-M', mgt_server_socket.local_address.ip_port.to_s]
202
167
  end
203
- # start ascp in separate process
204
- ascp_pid = Process.spawn(env_args[:env], [ascp_path, ascp_path], *ascp_arguments, close_others: true)
205
- Log.log.debug{"spawned ascp pid #{ascp_pid}"}
206
- notify_progress(session_id: nil, type: :pre_start, info: 'waiting for ascp')
168
+ command_arguments.concat(args)
169
+ # get location of command executable (ascp, async)
170
+ command_path = Ascp::Installation.instance.path(name)
171
+ command_pid = Environment.secure_spawn(env: env, exec: command_path, args: command_arguments)
172
+ notify_progress(:pre_start, session_id: nil, info: "waiting for #{name} to start")
207
173
  mgt_server_socket.listen(1)
208
174
  # TODO: timeout does not work when Process.spawn is used... until process exits, then it works
209
175
  Log.log.debug{"before select, timeout: #{@spawn_timeout_sec}"}
@@ -213,15 +179,15 @@ module Aspera
213
179
  # There is a connection to accept
214
180
  client_socket, _client_addrinfo = mgt_server_socket.accept
215
181
  Log.log.debug('after accept')
216
- ascp_mgt_io = client_socket.to_io
182
+ management_port_io = client_socket.to_io
217
183
  # management messages include file names which may be utf8
218
184
  # by default socket is US-ASCII
219
185
  # TODO: use same value as Encoding.default_external
220
- ascp_mgt_io.set_encoding(Encoding::UTF_8)
221
- session[:io] = ascp_mgt_io
186
+ management_port_io.set_encoding(Encoding::UTF_8)
187
+ session[:io] = management_port_io
222
188
  processor = Ascp::Management.new
223
189
  # read management port, until socket is closed (gets returns nil)
224
- while (line = ascp_mgt_io.gets)
190
+ while (line = management_port_io.gets)
225
191
  event = processor.process_line(line.chomp)
226
192
  next unless event
227
193
  # event is ready
@@ -231,24 +197,23 @@ module Aspera
231
197
  Log.log.error((event['Description']).to_s) if event['Type'].eql?('FILEERROR') # cspell:disable-line
232
198
  end
233
199
  Log.log.debug('management io closed')
234
- last_event = processor.last_event
235
200
  # check that last status was received before process exit
236
- if last_event.is_a?(Hash)
237
- case last_event['Type']
238
- when 'ERROR'
239
- if /bearer token/i.match?(last_event['Description']) &&
240
- session[:token_regenerator].respond_to?(:refreshed_transfer_token)
241
- # regenerate token here, expired, or error on it
242
- # Note: in multi-session, each session will have a different one.
243
- Log.log.warn('Regenerating token for transfer')
244
- env_args[:env]['ASPERA_SCP_TOKEN'] = session[:token_regenerator].refreshed_transfer_token
245
- end
246
- raise Transfer::Error.new(last_event['Description'], last_event['Code'].to_i)
247
- when 'DONE'
248
- nil
249
- else
250
- raise "unexpected last event type: #{last_event['Type']}"
201
+ last_event = processor.last_event
202
+ raise Transfer::Error, "internal: no management event (#{last_event.class})" unless last_event.is_a?(Hash)
203
+ case last_event['Type']
204
+ when 'ERROR'
205
+ if /bearer token/i.match?(last_event['Description']) &&
206
+ session[:token_regenerator].respond_to?(:refreshed_transfer_token)
207
+ # regenerate token here, expired, or error on it
208
+ # Note: in multi-session, each session will have a different one.
209
+ Log.log.warn('Regenerating token for transfer')
210
+ env['ASPERA_SCP_TOKEN'] = session[:token_regenerator].refreshed_transfer_token
251
211
  end
212
+ raise Transfer::Error.new(last_event['Description'], last_event['Code'].to_i)
213
+ when 'DONE'
214
+ nil
215
+ else
216
+ raise Transfer::Error, "unexpected last event type: #{last_event['Type']}, #{last_event['Description']}"
252
217
  end
253
218
  rescue SystemCallError => e
254
219
  # Process.spawn failed, or socket error
@@ -257,16 +222,16 @@ module Aspera
257
222
  raise Transfer::Error, 'transfer interrupted by user'
258
223
  ensure
259
224
  mgt_server_socket.close
260
- # if ascp was successfully started, check its status
261
- unless ascp_pid.nil?
225
+ # if command was successfully started, check its status
226
+ unless command_pid.nil?
262
227
  # "wait" for process to avoid zombie
263
- Process.wait(ascp_pid)
228
+ Process.wait(command_pid)
264
229
  status = $CHILD_STATUS
265
- ascp_pid = nil
230
+ # command_pid = nil
266
231
  session.delete(:io)
267
- # status is nil if an exception occurred before starting ascp
232
+ # status is nil if an exception occurred before starting command
268
233
  if !status&.success?
269
- message = status.nil? ? 'ascp not started' : "ascp failed (#{status})"
234
+ message = status.nil? ? "#{name} not started" : "#{name} failed (#{status})"
270
235
  # raise error only if there was not already an exception (ERROR_INFO)
271
236
  raise Transfer::Error, message unless $ERROR_INFO
272
237
  # else display this message also, as main exception is already here
@@ -274,11 +239,47 @@ module Aspera
274
239
  end
275
240
  end
276
241
  end
242
+ nil
277
243
  end
278
244
 
279
- # @return [Array] list of sessions for a job
280
- def sessions_by_job(job_id)
281
- @sessions.select{|session_info| session_info[:job_id].eql?(job_id)}
245
+ private
246
+
247
+ # notify progress to callback
248
+ # @param event management port event
249
+ def process_progress(event)
250
+ session_id = event['SessionId']
251
+ case event['Type']
252
+ when 'INIT'
253
+ @pre_calc_sent = false
254
+ @pre_calc_last_size = nil
255
+ notify_progress(:session_start, session_id: session_id)
256
+ when 'NOTIFICATION' # sent from remote
257
+ if event.key?('PreTransferBytes')
258
+ @pre_calc_sent = true
259
+ notify_progress(:session_size, session_id: session_id, info: event['PreTransferBytes'])
260
+ end
261
+ when 'STATS' # during transfer
262
+ @pre_calc_last_size = event['TransferBytes'].to_i + event['StartByte'].to_i
263
+ notify_progress(:transfer, session_id: session_id, info: @pre_calc_last_size)
264
+ when 'DONE', 'ERROR' # end of session
265
+ total_size = event['TransferBytes'].to_i + event['StartByte'].to_i
266
+ if !@pre_calc_sent && !total_size.zero?
267
+ notify_progress(:session_size, session_id: session_id, info: total_size)
268
+ end
269
+ if @pre_calc_last_size != total_size
270
+ notify_progress(:transfer, session_id: session_id, info: total_size)
271
+ end
272
+ notify_progress(:end, session_id: session_id)
273
+ # cspell:disable
274
+ when 'SESSION'
275
+ when 'ARGSTOP'
276
+ when 'FILEERROR'
277
+ when 'STOP'
278
+ # cspell:enable
279
+ # stop event when one file is completed
280
+ else
281
+ Log.log.debug{"unknown event type #{event['Type']}"}
282
+ end
282
283
  end
283
284
 
284
285
  # @return [Hash] session information
@@ -289,7 +290,7 @@ module Aspera
289
290
  return matches.first
290
291
  end
291
292
 
292
- # send command of management port to ascp session (used in `asession)
293
+ # send command to management port of command (used in `asession)
293
294
  # @param job_id identified transfer process
294
295
  # @param session_index index of session (for multi session)
295
296
  # @param data command on mgt port, examples:
@@ -309,47 +310,49 @@ module Aspera
309
310
  end
310
311
  attr_reader :sessions
311
312
 
312
- private
313
-
314
313
  # options for initialize (same as values in option transfer_info)
315
- # @param wss [Boolean] true: if both SSH and wss in ts: prefer wss
316
314
  # @param ascp_args [Array] additional arguments to ascp
317
- # @param spawn_timeout_sec [Integer] timeout for ascp spawn
318
- # @param spawn_delay_sec [Integer] optional delay to start between sessions
319
- # @param multi_incr_udp [Boolean] true: increment udp port for each session
320
- # @param trusted_certs [Array] list of files with trusted certificates (stores)
321
- # @param resume [Hash] resume policy
315
+ # @param wss [Boolean] true: if both SSH and wss in ts: prefer wss
322
316
  # @param quiet [Boolean] by default no native ascp progress bar
317
+ # @param trusted_certs [Array,NilClass] list of files with trusted certificates (stores)
318
+ # @param client_ssh_key [String] client ssh key option (from CLIENT_SSH_KEY_OPTIONS)
323
319
  # @param check_ignore_cb [Proc] callback with host,port
320
+ # @param spawn_timeout_sec [Integer] timeout for ascp spawn
321
+ # @param spawn_delay_sec [Integer] optional delay to start between sessions
322
+ # @param multi_incr_udp [Boolean,NilClass] true: increment udp port for each session
323
+ # @param resume [Hash,NilClass] resume policy
324
324
  # @param management_cb [Proc] callback for management events
325
325
  # @param base_options [Hash] other options for base class
326
326
  def initialize(
327
+ ascp_args: nil,
327
328
  wss: true,
328
- ascp_args: [],
329
- spawn_timeout_sec: 2,
330
- spawn_delay_sec: 2,
331
- multi_incr_udp: true,
332
- trusted_certs: [],
333
- resume: {},
334
329
  quiet: true,
330
+ trusted_certs: nil,
331
+ client_ssh_key: nil,
335
332
  check_ignore_cb: nil,
333
+ spawn_timeout_sec: 2,
334
+ spawn_delay_sec: 2,
335
+ multi_incr_udp: nil,
336
+ resume: nil,
336
337
  management_cb: nil,
337
338
  **base_options
338
339
  )
339
340
  super(**base_options)
341
+ # special transfer parameters
340
342
  @tr_opts = {
341
343
  ascp_args: ascp_args,
342
344
  wss: wss,
343
345
  quiet: quiet,
344
346
  trusted_certs: trusted_certs,
347
+ client_ssh_key: client_ssh_key,
345
348
  check_ignore_cb: check_ignore_cb
346
349
  }
347
350
  @spawn_timeout_sec = spawn_timeout_sec
348
351
  @spawn_delay_sec = spawn_delay_sec
349
- @multi_incr_udp = multi_incr_udp
350
- @resume = resume
352
+ # default is true on windows, false on other platforms
353
+ @multi_incr_udp = multi_incr_udp.nil? ? Environment.os.eql?(Environment::OS_WINDOWS) : multi_incr_udp
351
354
  @management_cb = management_cb
352
- @resume_policy = Resumer.new(@resume.symbolize_keys)
355
+ @resume_policy = Resumer.new(resume.nil? ? {} : resume.symbolize_keys)
353
356
  # all transfer jobs, key = SecureRandom.uuid, protected by mutex, cond var on change
354
357
  @sessions = []
355
358
  # mutex protects global data accessed by threads
@@ -365,7 +368,7 @@ module Aspera
365
368
  Log.log.debug{"ENTER (#{Thread.current[:name]})"}
366
369
  # start transfer with selected resumer policy
367
370
  @resume_policy.execute_with_resume do
368
- start_transfer_with_args_env(session[:env_args], session)
371
+ start_and_monitor_process(session: session, **session[:env_args])
369
372
  end
370
373
  Log.log.debug('transfer ok'.bg_green)
371
374
  rescue StandardError => e
@@ -23,8 +23,7 @@ module Aspera
23
23
  @gw_api.upload(transfer_spec)
24
24
  when Transfer::Spec::DIRECTION_RECEIVE
25
25
  @gw_api.download(transfer_spec)
26
- else
27
- raise "unexpected direction: [#{transfer_spec['direction']}]"
26
+ else Aspera.error_unexpected_value(transfer_spec['direction']){'direction'}
28
27
  end
29
28
  end
30
29
 
@@ -54,7 +53,7 @@ module Aspera
54
53
  api_version: api_version,
55
54
  upload_chunk_size: upload_chunk_size,
56
55
  synchronous: synchronous,
57
- notify_cb: ->(**event) { notify_progress(**event) }
56
+ notify_cb: ->(*pa, **ka) { notify_progress(*pa, **ka) }
58
57
  )
59
58
  end
60
59
  end
@@ -78,7 +78,7 @@ module Aspera
78
78
  if !transfer_spec['wss_enabled'] && transfer_spec['remote_host'].eql?(URI.parse(node_api_.base_url).host)
79
79
  transfer_spec['remote_host'] = '127.0.0.1'
80
80
  end
81
- resp = node_api_.create('ops/transfers', transfer_spec)[:data]
81
+ resp = node_api_.create('ops/transfers', transfer_spec)
82
82
  @transfer_id = resp['id']
83
83
  Log.log.debug{"tr_id=#{@transfer_id}"}
84
84
  return @transfer_id
@@ -92,31 +92,31 @@ module Aspera
92
92
  # lets emulate management events to display progress bar
93
93
  loop do
94
94
  # status is empty sometimes with status 200...
95
- transfer_data = node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {'status' => 'unknown'} rescue {'status' => 'waiting(api error)'}
95
+ transfer_data = node_api_.read("ops/transfers/#{@transfer_id}") || {'status' => 'unknown'} rescue {'status' => 'waiting(api error)'}
96
96
  case transfer_data['status']
97
97
  when 'waiting', 'partially_completed', 'unknown', 'waiting(read error)'
98
- notify_progress(session_id: nil, type: :pre_start, info: transfer_data['status'])
98
+ notify_progress(:pre_start, session_id: nil, info: transfer_data['status'])
99
99
  when 'running'
100
100
  if !session_started
101
- notify_progress(session_id: @transfer_id, type: :session_start)
101
+ notify_progress(:session_start, session_id: @transfer_id)
102
102
  session_started = true
103
103
  end
104
104
  message = transfer_data['status']
105
105
  message = "#{message} (#{transfer_data['error_desc']})" if !transfer_data['error_desc']&.empty?
106
- notify_progress(session_id: nil, type: :pre_start, info: message)
106
+ notify_progress(:pre_start, session_id: nil, info: message)
107
107
  if bytes_expected.nil? &&
108
108
  transfer_data['precalc'].is_a?(Hash) &&
109
109
  transfer_data['precalc']['status'].eql?('ready')
110
110
  bytes_expected = transfer_data['precalc']['bytes_expected']
111
- notify_progress(type: :session_size, session_id: @transfer_id, info: bytes_expected)
111
+ notify_progress(:session_size, session_id: @transfer_id, info: bytes_expected)
112
112
  end
113
- notify_progress(type: :transfer, session_id: @transfer_id, info: transfer_data['bytes_transferred'])
113
+ notify_progress(:transfer, session_id: @transfer_id, info: transfer_data['bytes_transferred'])
114
114
  when 'completed'
115
- notify_progress(type: :transfer, session_id: @transfer_id, info: bytes_expected) if bytes_expected
116
- notify_progress(type: :end, session_id: @transfer_id)
115
+ notify_progress(:transfer, session_id: @transfer_id, info: bytes_expected) if bytes_expected
116
+ notify_progress(:end, session_id: @transfer_id)
117
117
  break
118
118
  when 'failed'
119
- notify_progress(type: :end, session_id: @transfer_id)
119
+ notify_progress(:end, session_id: @transfer_id)
120
120
  # Bug in HSTS ? transfer is marked failed, but there is no reason
121
121
  break if transfer_data['error_code'].eql?(0) && transfer_data['error_desc'].empty?
122
122
  raise Transfer::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
@@ -7,6 +7,7 @@ require 'aspera/log'
7
7
  require 'aspera/assert'
8
8
  require 'json'
9
9
  require 'uri'
10
+ require 'transfer_services_pb'
10
11
 
11
12
  module Aspera
12
13
  module Agent
@@ -48,14 +49,12 @@ module Aspera
48
49
  **base_options
49
50
  )
50
51
  super(**base_options)
51
- is_local_auto_port = @url.eql?(AUTO_LOCAL_TCP_PORT)
52
- raise 'Cannot use options `keep` or `external` with port zero' if is_local_auto_port && (@keep || @external)
53
- # load SDK stub class on demand, as it's an optional gem
54
- $LOAD_PATH.unshift(Ascp::Installation.instance.sdk_ruby_folder)
55
- require 'transfer_services_pb'
52
+ @keep = keep
53
+ is_local_auto_port = url.eql?(AUTO_LOCAL_TCP_PORT)
54
+ raise 'Cannot use options `keep` or `external` with port zero' if is_local_auto_port && (@keep || external)
56
55
  # keep PID for optional shutdown
57
56
  @daemon_pid = nil
58
- daemon_endpoint = @url
57
+ daemon_endpoint = url
59
58
  Log.log.debug{Log.dump(:daemon_endpoint, daemon_endpoint)}
60
59
  # retry loop
61
60
  begin
@@ -66,16 +65,14 @@ module Aspera
66
65
  # Initiate actual connection
67
66
  get_info_response = @transfer_client.get_info(Transfersdk::InstanceInfoRequest.new)
68
67
  Log.log.debug{"Daemon info: #{get_info_response}"}
69
- Log.log.warn{'Attached to existing daemon'} unless @daemon_pid || @external || @keep
68
+ Log.log.warn{'Attached to existing daemon'} unless @daemon_pid || external || @keep
70
69
  at_exit{shutdown}
71
70
  rescue GRPC::Unavailable => e
72
71
  # if transferd is external: do not start it, or other error
73
- raise if @external || !e.message.include?('failed to connect')
72
+ raise if external || !e.message.include?('failed to connect')
74
73
  # we already tried to start a daemon, but it failed
75
74
  Aspera.assert(@daemon_pid.nil?){"Daemon started with PID #{@daemon_pid}, but connection failed to #{daemon_endpoint}}"}
76
- Log.log.warn('no daemon present, starting daemon...') if @external
77
- # location of daemon binary
78
- sdk_folder = File.realpath(File.join(Ascp::Installation.instance.sdk_ruby_folder, '..'))
75
+ Log.log.warn('no daemon present, starting daemon...') if external
79
76
  # transferd only supports local ip and port
80
77
  daemon_uri = URI.parse("ipv4://#{daemon_endpoint}")
81
78
  Aspera.assert(daemon_uri.scheme.eql?('ipv4')){"Invalid scheme daemon URI #{daemon_endpoint}"}
@@ -86,8 +83,8 @@ module Aspera
86
83
  fasp_runtime: {
87
84
  use_embedded: false,
88
85
  user_defined: {
89
- bin: sdk_folder,
90
- etc: sdk_folder
86
+ bin: Ascp::Installation.instance.sdk_folder,
87
+ etc: Ascp::Installation.instance.sdk_folder
91
88
  }
92
89
  }
93
90
  }
@@ -143,24 +140,24 @@ module Aspera
143
140
  case response.status
144
141
  when :RUNNING
145
142
  if !session_started
146
- notify_progress(session_id: @transfer_id, type: :session_start)
143
+ notify_progress(:session_start, session_id: @transfer_id)
147
144
  session_started = true
148
145
  end
149
146
  if bytes_expected.nil? &&
150
147
  !response.sessionInfo.preTransferBytes.eql?(0)
151
148
  bytes_expected = response.sessionInfo.preTransferBytes
152
- notify_progress(type: :session_size, session_id: @transfer_id, info: bytes_expected)
149
+ notify_progress(:session_size, session_id: @transfer_id, info: bytes_expected)
153
150
  end
154
- notify_progress(type: :transfer, session_id: @transfer_id, info: response.transferInfo.bytesTransferred)
151
+ notify_progress(:transfer, session_id: @transfer_id, info: response.transferInfo.bytesTransferred)
155
152
  when :COMPLETED
156
- notify_progress(type: :transfer, session_id: @transfer_id, info: bytes_expected) if bytes_expected
157
- notify_progress(type: :end, session_id: @transfer_id)
153
+ notify_progress(:transfer, session_id: @transfer_id, info: bytes_expected) if bytes_expected
154
+ notify_progress(:end, session_id: @transfer_id)
158
155
  break
159
156
  when :FAILED, :CANCELED
160
- notify_progress(type: :end, session_id: @transfer_id)
157
+ notify_progress(:end, session_id: @transfer_id)
161
158
  raise Transfer::Error, JSON.parse(response.message)['Description']
162
159
  when :QUEUED, :UNKNOWN_STATUS, :PAUSED, :ORPHANED
163
- notify_progress(session_id: nil, type: :pre_start, info: response.status.to_s.downcase)
160
+ notify_progress(:pre_start, session_id: nil, info: response.status.to_s.downcase)
164
161
  else
165
162
  Log.log.error{"unknown status#{response.status}"}
166
163
  end
@@ -0,0 +1,15 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aspera/api/aoc.rb'
4
+ module Aspera
5
+ module Api
6
+ class Alee < Aspera::Rest
7
+ def initialize(entitlement_id, customer_id, api_domain: AoC::SAAS_DOMAIN_PROD, version: 'v1')
8
+ super(
9
+ base_url: "https://api.#{api_domain}/metering/#{version}",
10
+ headers: {'X-Aspera-Entitlement-Authorization' => Rest.basic_token(entitlement_id, customer_id)}
11
+ )
12
+ end
13
+ end
14
+ end
15
+ end