aspera-cli 4.4.0 → 4.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2095 -1503
  3. data/bin/ascli +2 -1
  4. data/bin/asession +4 -5
  5. data/docs/test_env.conf +3 -0
  6. data/examples/aoc.rb +4 -3
  7. data/examples/faspex4.rb +25 -25
  8. data/examples/proxy.pac +1 -1
  9. data/examples/transfer.rb +17 -17
  10. data/lib/aspera/aoc.rb +238 -185
  11. data/lib/aspera/ascmd.rb +93 -83
  12. data/lib/aspera/ats_api.rb +11 -10
  13. data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
  14. data/lib/aspera/cli/extended_value.rb +42 -33
  15. data/lib/aspera/cli/formater.rb +142 -108
  16. data/lib/aspera/cli/info.rb +17 -0
  17. data/lib/aspera/cli/listener/line_dump.rb +3 -2
  18. data/lib/aspera/cli/listener/logger.rb +2 -1
  19. data/lib/aspera/cli/listener/progress.rb +16 -18
  20. data/lib/aspera/cli/listener/progress_multi.rb +18 -21
  21. data/lib/aspera/cli/main.rb +173 -149
  22. data/lib/aspera/cli/manager.rb +163 -168
  23. data/lib/aspera/cli/plugin.rb +43 -31
  24. data/lib/aspera/cli/plugins/alee.rb +6 -6
  25. data/lib/aspera/cli/plugins/aoc.rb +405 -370
  26. data/lib/aspera/cli/plugins/ats.rb +86 -79
  27. data/lib/aspera/cli/plugins/bss.rb +14 -16
  28. data/lib/aspera/cli/plugins/config.rb +580 -362
  29. data/lib/aspera/cli/plugins/console.rb +23 -19
  30. data/lib/aspera/cli/plugins/cos.rb +18 -18
  31. data/lib/aspera/cli/plugins/faspex.rb +201 -158
  32. data/lib/aspera/cli/plugins/faspex5.rb +80 -57
  33. data/lib/aspera/cli/plugins/node.rb +183 -166
  34. data/lib/aspera/cli/plugins/orchestrator.rb +71 -67
  35. data/lib/aspera/cli/plugins/preview.rb +92 -96
  36. data/lib/aspera/cli/plugins/server.rb +79 -75
  37. data/lib/aspera/cli/plugins/shares.rb +35 -19
  38. data/lib/aspera/cli/plugins/sync.rb +20 -22
  39. data/lib/aspera/cli/transfer_agent.rb +76 -113
  40. data/lib/aspera/cli/version.rb +2 -1
  41. data/lib/aspera/colors.rb +35 -27
  42. data/lib/aspera/command_line_builder.rb +48 -34
  43. data/lib/aspera/cos_node.rb +29 -21
  44. data/lib/aspera/data_repository.rb +3 -2
  45. data/lib/aspera/environment.rb +50 -45
  46. data/lib/aspera/fasp/{manager.rb → agent_base.rb} +28 -25
  47. data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +52 -43
  48. data/lib/aspera/fasp/{local.rb → agent_direct.rb} +58 -72
  49. data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +37 -43
  50. data/lib/aspera/fasp/{node.rb → agent_node.rb} +35 -16
  51. data/lib/aspera/fasp/agent_trsdk.rb +104 -0
  52. data/lib/aspera/fasp/error.rb +2 -1
  53. data/lib/aspera/fasp/error_info.rb +68 -52
  54. data/lib/aspera/fasp/installation.rb +152 -124
  55. data/lib/aspera/fasp/listener.rb +1 -0
  56. data/lib/aspera/fasp/parameters.rb +87 -92
  57. data/lib/aspera/fasp/parameters.yaml +305 -249
  58. data/lib/aspera/fasp/resume_policy.rb +11 -14
  59. data/lib/aspera/fasp/transfer_spec.rb +26 -0
  60. data/lib/aspera/fasp/uri.rb +22 -21
  61. data/lib/aspera/faspex_gw.rb +55 -89
  62. data/lib/aspera/hash_ext.rb +4 -3
  63. data/lib/aspera/id_generator.rb +8 -7
  64. data/lib/aspera/keychain/encrypted_hash.rb +121 -0
  65. data/lib/aspera/keychain/macos_security.rb +90 -0
  66. data/lib/aspera/log.rb +55 -37
  67. data/lib/aspera/nagios.rb +13 -12
  68. data/lib/aspera/node.rb +30 -25
  69. data/lib/aspera/oauth.rb +175 -226
  70. data/lib/aspera/open_application.rb +4 -3
  71. data/lib/aspera/persistency_action_once.rb +6 -6
  72. data/lib/aspera/persistency_folder.rb +5 -9
  73. data/lib/aspera/preview/file_types.rb +6 -5
  74. data/lib/aspera/preview/generator.rb +25 -24
  75. data/lib/aspera/preview/options.rb +16 -14
  76. data/lib/aspera/preview/utils.rb +98 -98
  77. data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
  78. data/lib/aspera/proxy_auto_config.rb +111 -20
  79. data/lib/aspera/rest.rb +154 -135
  80. data/lib/aspera/rest_call_error.rb +2 -2
  81. data/lib/aspera/rest_error_analyzer.rb +23 -25
  82. data/lib/aspera/rest_errors_aspera.rb +15 -14
  83. data/lib/aspera/ssh.rb +12 -10
  84. data/lib/aspera/sync.rb +42 -41
  85. data/lib/aspera/temp_file_manager.rb +18 -14
  86. data/lib/aspera/timer_limiter.rb +2 -1
  87. data/lib/aspera/uri_reader.rb +7 -5
  88. data/lib/aspera/web_auth.rb +79 -76
  89. metadata +116 -29
  90. data/docs/Makefile +0 -66
  91. data/docs/README.erb.md +0 -3973
  92. data/docs/README.md +0 -13
  93. data/docs/diagrams.txt +0 -49
  94. data/docs/doc_tools.rb +0 -58
  95. data/lib/aspera/api_detector.rb +0 -60
  96. data/lib/aspera/cli/plugins/shares2.rb +0 -114
  97. data/lib/aspera/secrets.rb +0 -20
@@ -1,36 +1,31 @@
1
- #!/bin/echo this is a ruby class:
2
- #
3
- # FASP manager for Ruby
4
- # Aspera 2016
5
- # Laurent Martin
6
- #
7
- ##############################################################################
8
- require 'aspera/fasp/manager'
1
+ # frozen_string_literal: true
2
+ require 'English'
3
+ require 'aspera/fasp/agent_base'
9
4
  require 'aspera/fasp/error'
10
5
  require 'aspera/fasp/parameters'
11
6
  require 'aspera/fasp/installation'
12
7
  require 'aspera/fasp/resume_policy'
8
+ require 'aspera/fasp/transfer_spec'
13
9
  require 'aspera/log'
14
10
  require 'socket'
15
11
  require 'timeout'
16
12
  require 'securerandom'
13
+ require 'shellwords'
17
14
 
18
15
  module Aspera
19
16
  module Fasp
20
17
  # executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
21
- class Local < Manager
18
+ class AgentDirect < AgentBase
22
19
  # options for initialize (same as values in option transfer_info)
23
20
  DEFAULT_OPTIONS = {
24
- :spawn_timeout_sec => 3,
25
- :spawn_delay_sec => 2,
26
- :wss => false,
27
- :multi_incr_udp => true,
28
- :resume => {}
21
+ spawn_timeout_sec: 3,
22
+ spawn_delay_sec: 2,
23
+ wss: false,
24
+ multi_incr_udp: true,
25
+ resume: {},
26
+ quiet: true # by default no interactive progress bar
29
27
  }
30
- DEFAULT_UDP_PORT=33001
31
28
  private_constant :DEFAULT_OPTIONS
32
- # set to false to keep ascp progress bar display ("true" adds ascp's option -q)
33
- attr_accessor :quiet
34
29
 
35
30
  # start ascp transfer (non blocking), single or multi-session
36
31
  # job information added to @jobs
@@ -44,10 +39,11 @@ module Aspera
44
39
  # clone transfer spec because we modify it (first level keys)
45
40
  transfer_spec=transfer_spec.clone
46
41
  # if there is aspera tags
47
- if transfer_spec['tags'].is_a?(Hash) and transfer_spec['tags']['aspera'].is_a?(Hash)
42
+ if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags']['aspera'].is_a?(Hash)
48
43
  # TODO: what is this for ? only on local ascp ?
49
44
  # NOTE: important: transfer id must be unique: generate random id
50
45
  # using a non unique id results in discard of tags in AoC, and a package is never finalized
46
+ # all sessions in a multi-session transfer must have the same xfer_id (see admin manual)
51
47
  transfer_spec['tags']['aspera']['xfer_id']||=SecureRandom.uuid
52
48
  Log.log.debug("xfer id=#{transfer_spec['xfer_id']}")
53
49
  # TODO: useful ? node only ?
@@ -56,8 +52,8 @@ module Aspera
56
52
  Log.dump('ts',transfer_spec)
57
53
 
58
54
  # add bypass keys when authentication is token and no auth is provided
59
- if transfer_spec.has_key?('token') and
60
- !transfer_spec.has_key?('remote_password') and
55
+ if transfer_spec.has_key?('token') &&
56
+ !transfer_spec.has_key?('remote_password') &&
61
57
  !transfer_spec.has_key?('EX_ssh_key_paths')
62
58
  # transfer_spec['remote_password'] = Installation.instance.bypass_pass # not used
63
59
  transfer_spec['EX_ssh_key_paths'] = Installation.instance.bypass_keys
@@ -68,24 +64,22 @@ module Aspera
68
64
  multi_session_info=nil
69
65
  if transfer_spec.has_key?('multi_session')
70
66
  multi_session_info={
71
- count: transfer_spec['multi_session'].to_i,
67
+ count: transfer_spec['multi_session'].to_i
72
68
  }
73
69
  # Managed by multi-session, so delete from transfer spec
74
70
  transfer_spec.delete('multi_session')
75
- if multi_session_info[:count] < 0
71
+ if multi_session_info[:count].negative?
76
72
  Log.log.error("multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0")
77
73
  multi_session_info = nil
78
74
  elsif multi_session_info[:count].eql?(0)
79
- Log.log.debug("multi_session count is zero: no multisession")
75
+ Log.log.debug('multi_session count is zero: no multisession')
80
76
  multi_session_info = nil
81
- else # multi_session_info[:count] > 0
77
+ elsif @options[:multi_incr_udp] # multi_session_info[:count] > 0
82
78
  # if option not true: keep default udp port for all sessions
83
- if @options[:multi_incr_udp]
84
- # override if specified, else use default value
85
- multi_session_info[:udp_base]=transfer_spec.has_key?('fasp_port') ? transfer_spec['fasp_port'] : DEFAULT_UDP_PORT
86
- # delete from original transfer spec, as we will increment values
87
- transfer_spec.delete('fasp_port')
88
- end
79
+ multi_session_info[:udp_base]=transfer_spec.has_key?('fasp_port') ? transfer_spec['fasp_port'] : TransferSpec::UDP_PORT
80
+ # delete from original transfer spec, as we will increment values
81
+ transfer_spec.delete('fasp_port')
82
+ # override if specified, else use default value
89
83
  end
90
84
  end
91
85
 
@@ -98,22 +92,22 @@ module Aspera
98
92
  env_args[:args].unshift('-I',Installation.instance.path(:fallback_cert))
99
93
  end
100
94
 
101
- env_args[:args].unshift('-q') if @quiet
95
+ env_args[:args].unshift('-q') if @options[:quiet]
102
96
 
103
97
  # transfer job can be multi session
104
98
  xfer_job={
105
- :id => job_options[:job_id],
106
- :sessions => [] # all sessions as below
99
+ id: job_options[:job_id],
100
+ sessions: [] # all sessions as below
107
101
  }
108
102
 
109
103
  # generic session information
110
104
  session={
111
- :thread => nil, # Thread object monitoring management port, not nil when pushed to :sessions
112
- :error => nil, # exception if failed
113
- :io => nil, # management port server socket
114
- :id => nil, # SessionId from INIT message in mgt port
115
- :env_args => env_args, # env vars and args to ascp (from transfer spec)
116
- :options => job_options # [Hash]
105
+ thread: nil, # Thread object monitoring management port, not nil when pushed to :sessions
106
+ error: nil, # exception if failed
107
+ io: nil, # management port server socket
108
+ id: nil, # SessionId from INIT message in mgt port
109
+ env_args: env_args, # env vars and args to ascp (from transfer spec)
110
+ options: job_options # [Hash]
117
111
  }
118
112
 
119
113
  if multi_session_info.nil?
@@ -132,7 +126,7 @@ module Aspera
132
126
  this_session[:env_args][:args]=this_session[:env_args][:args].clone()
133
127
  this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_info[:count]}")
134
128
  # option: increment (default as per ascp manual) or not (cluster on other side ?)
135
- this_session[:env_args][:args].unshift('-O',"#{multi_session_info[:udp_base]+i-1}") if @options[:multi_incr_udp]
129
+ this_session[:env_args][:args].unshift('-O',(multi_session_info[:udp_base]+i-1).to_s) if @options[:multi_incr_udp]
136
130
  this_session[:thread] = Thread.new(this_session) {|s|transfer_thread_entry(s)}
137
131
  xfer_job[:sessions].push(this_session)
138
132
  end
@@ -152,11 +146,11 @@ module Aspera
152
146
  Log.log.debug('wait_for_transfers_completion')
153
147
  # set to non-nil to exit loop
154
148
  result=[]
155
- @jobs.each do |id,job|
149
+ @jobs.each do |_id,job|
156
150
  job[:sessions].each do |session|
157
151
  Log.log.debug("join #{session[:thread]}")
158
152
  session[:thread].join
159
- result.push(session[:error] ? session[:error] : :success)
153
+ result.push(session[:error] || :success)
160
154
  end
161
155
  end
162
156
  Log.log.debug('all transfers joined')
@@ -190,7 +184,7 @@ module Aspera
190
184
  Fasp::Installation.instance.path(env_args[:ascp_version])
191
185
  end
192
186
  # (optional) check it exists
193
- raise Fasp::Error.new("no such file: #{ascp_path}") unless File.exist?(ascp_path)
187
+ raise Fasp::Error, "no such file: #{ascp_path}" unless File.exist?(ascp_path)
194
188
  # open random local TCP port for listening for ascp management
195
189
  mgt_sock = TCPServer.new('127.0.0.1',0)
196
190
  # clone arguments as we eed to modify with mgt port
@@ -198,7 +192,7 @@ module Aspera
198
192
  # add management port
199
193
  ascp_arguments.unshift('-M', mgt_sock.addr[1].to_s)
200
194
  # start ascp in sub process
201
- Log.log.debug("execute: #{env_args[:env].map{|k,v| "#{k}=\"#{v}\""}.join(' ')} \"#{ascp_path}\" \"#{ascp_arguments.join('" "')}\"")
195
+ Log.log.debug("execute: #{env_args[:env].map{|k,v| "#{k}=#{Shellwords.shellescape(v)}"}.join(' ')} #{Shellwords.shellescape(ascp_path)} #{ascp_arguments.map{|a|Shellwords.shellescape(a)}.join(' ')}")
202
196
  # start process
203
197
  ascp_pid = Process.spawn(env_args[:env],[ascp_path,ascp_path],*ascp_arguments)
204
198
  # in parent, wait for connection to socket max 3 seconds
@@ -226,21 +220,21 @@ module Aspera
226
220
  line = ascp_mgt_io.gets
227
221
  # nil when ascp process exits
228
222
  break if line.nil?
229
- current_event_text=current_event_text+line
223
+ current_event_text+=line
230
224
  line.chomp!
231
225
  Log.log.debug("line=[#{line}]")
232
226
  case line
233
227
  when 'FASPMGR 2'
234
228
  # begin event
235
- current_event_data = Hash.new
229
+ current_event_data = {}
236
230
  current_event_text = ''
237
231
  when /^([^:]+): (.*)$/
238
232
  # event field
239
- current_event_data[$1] = $2
233
+ current_event_data[Regexp.last_match(1)] = Regexp.last_match(2)
240
234
  when ''
241
235
  # empty line is separator to end event information
242
236
  raise 'unexpected empty line' if current_event_data.nil?
243
- current_event_data[Manager::LISTENER_SESSION_ID_B]=ascp_pid
237
+ current_event_data[AgentBase::LISTENER_SESSION_ID_B]=ascp_pid
244
238
  notify_listeners(current_event_text,current_event_data)
245
239
  case current_event_data['Type']
246
240
  when 'INIT'
@@ -262,9 +256,9 @@ module Aspera
262
256
  exception_raised=false
263
257
  when 'ERROR'
264
258
  Log.log.error("code: #{last_status_event['Code']}")
265
- if last_status_event['Description'] =~ /bearer token/i
259
+ if last_status_event['Description'] =~ /bearer token/i
266
260
  Log.log.error('need to regenerate token'.red)
267
- if session[:options].is_a?(Hash) and session[:options].has_key?(:regenerate_token)
261
+ if session[:options].is_a?(Hash) && session[:options].has_key?(:regenerate_token)
268
262
  # regenerate token here, expired, or error on it
269
263
  # Note: in multi-session, each session will have a different one.
270
264
  env_args[:env]['ASPERA_SCP_TOKEN']=session[:options][:regenerate_token].call(true)
@@ -280,27 +274,25 @@ module Aspera
280
274
  end
281
275
  rescue SystemCallError => e
282
276
  # Process.spawn
283
- raise Fasp::Error.new(e.message)
284
- rescue Timeout::Error => e
285
- raise Fasp::Error.new('timeout waiting mgt port connect')
286
- rescue Interrupt => e
287
- raise Fasp::Error.new('transfer interrupted by user')
277
+ raise Fasp::Error, e.message
278
+ rescue Timeout::Error
279
+ raise Fasp::Error, 'timeout waiting mgt port connect'
280
+ rescue Interrupt
281
+ raise Fasp::Error, 'transfer interrupted by user'
288
282
  ensure
289
283
  # if ascp was successfully started
290
284
  unless ascp_pid.nil?
291
285
  # "wait" for process to avoid zombie
292
286
  Process.wait(ascp_pid)
293
- status=$?
287
+ status=$CHILD_STATUS
294
288
  ascp_pid=nil
295
289
  session.delete(:io)
296
290
  if !status.success?
297
291
  message="ascp failed with code #{status.exitstatus}"
298
- if exception_raised
299
- # just debug, as main exception is already here
300
- Log.log.debug(message)
301
- else
302
- raise Fasp::Error.new(message)
303
- end
292
+ # raise error only if there was not already an exception
293
+ raise Fasp::Error, message unless exception_raised
294
+ # else just debug, as main exception is already here
295
+ Log.log.debug(message)
304
296
  end
305
297
  end
306
298
  end # begin-ensure
@@ -333,8 +325,6 @@ module Aspera
333
325
  # @param options : keys(symbol): see DEFAULT_OPTIONS
334
326
  def initialize(options=nil)
335
327
  super()
336
- # by default no interactive progress bar
337
- @quiet=true
338
328
  # all transfer jobs, key = SecureRandom.uuid, protected by mutex, condvar on change
339
329
  @jobs={}
340
330
  # mutex protects global data accessed by threads
@@ -344,11 +334,8 @@ module Aspera
344
334
  if !options.nil?
345
335
  raise "expecting Hash (or nil), but have #{options.class}" unless options.is_a?(Hash)
346
336
  options.each do |k,v|
347
- if DEFAULT_OPTIONS.has_key?(k)
348
- @options[k]=v
349
- else
350
- raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map{|i|i.to_s}.join(",")}"
351
- end
337
+ raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.has_key?(k)
338
+ @options[k]=v
352
339
  end
353
340
  end
354
341
  Log.log.debug("local options= #{options}")
@@ -367,13 +354,12 @@ module Aspera
367
354
  start_transfer_with_args_env(session[:env_args],session)
368
355
  end
369
356
  Log.log.debug('transfer ok'.bg_green)
370
- rescue => e
357
+ rescue StandardError => e
371
358
  session[:error]=e
372
359
  Log.log.error("Transfer thread error: #{e.class}:\n#{e.message}:\n#{e.backtrace.join("\n")}".red) if Log.instance.level.eql?(:debug)
373
360
  end
374
361
  Log.log.debug("EXIT (#{Thread.current[:name]})")
375
362
  end
376
-
377
- end # Local
363
+ end # AgentDirect
378
364
  end
379
365
  end
@@ -1,5 +1,6 @@
1
- #!/bin/echo this is a ruby class:
2
- require 'aspera/fasp/manager'
1
+ # frozen_string_literal: true
2
+ require 'aspera/fasp/agent_base'
3
+ require 'aspera/fasp/transfer_spec'
3
4
  require 'aspera/log'
4
5
  require 'aspera/rest'
5
6
  require 'websocket-client-simple'
@@ -12,7 +13,7 @@ require 'json'
12
13
  module Aspera
13
14
  module Fasp
14
15
  # start a transfer using Aspera HTTP Gateway, using web socket session
15
- class HttpGW < Manager
16
+ class AgentHttpgw < AgentBase
16
17
  # message returned by HTTP GW in case of success
17
18
  OK_MESSAGE='end upload'
18
19
  # refresh rate for progress
@@ -29,7 +30,7 @@ module Aspera
29
30
  # we need to keep track of actual file path because transfer spec is modified to be sent in web socket
30
31
  source_paths=[]
31
32
  # get source root or nil
32
- source_root = (transfer_spec.has_key?('source_root') and !transfer_spec['source_root'].empty?) ? transfer_spec['source_root'] : nil
33
+ source_root = transfer_spec.has_key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] : nil
33
34
  # source root is ignored by GW, used only here
34
35
  transfer_spec.delete('source_root')
35
36
  # compute total size of files to upload (for progress)
@@ -61,27 +62,23 @@ module Aspera
61
62
  received+=1
62
63
  else
63
64
  message.chomp!
64
- if message[0].eql?('"') and message[-1].eql?('"')
65
- error=JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message']
66
- else
67
- error="expecting quotes in [#{message}]"
68
- end
65
+ error=message.start_with?('"') && message.end_with?('"') ? JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message'] : "expecting quotes in [#{message}]"
69
66
  end
70
67
  end
71
68
  ws.on :error do |e|
72
69
  error=e
73
70
  end
74
71
  ws.on :open do
75
- Log.log.info("ws: open")
72
+ Log.log.info('ws: open')
76
73
  end
77
74
  ws.on :close do
78
- Log.log.info("ws: close")
75
+ Log.log.info('ws: close')
79
76
  end
80
77
  # open web socket to end point
81
78
  ws.connect("#{@gw_api.params[:base_url]}/upload")
82
79
  # async wait ready
83
- while !ws.open? and error.nil? do
84
- Log.log.info("ws: wait")
80
+ while !ws.open? && error.nil?
81
+ Log.log.info('ws: wait')
85
82
  sleep(0.2)
86
83
  end
87
84
  # notify progress bar
@@ -101,11 +98,11 @@ module Aspera
101
98
  file_size=item['file_size']
102
99
  file_name=File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
103
100
  # compute total number of slices
104
- numslices=1+(file_size-1)/@upload_chunksize
101
+ numslices=1+((file_size-1)/@upload_chunksize)
105
102
  File.open(source_paths[file_index]) do |file|
106
103
  # current slice index
107
104
  slicenum=0
108
- while !file.eof? do
105
+ while !file.eof?
109
106
  data=file.read(@upload_chunksize)
110
107
  slice_data={
111
108
  name: file_name,
@@ -117,11 +114,11 @@ module Aspera
117
114
  fileIndex: file_index
118
115
  }
119
116
  # log without data
120
- Log.dump(:slide_data,slice_data.keys.inject({}){|m,i|m[i]=i.eql?(:data)?'base64 data':slice_data[i];m}) if slicenum.eql?(0)
117
+ Log.dump(:slide_data,slice_data.keys.each_with_object({}){|i,m|m[i]=i.eql?(:data)?'base64 data':slice_data[i];}) if slicenum.eql?(0)
121
118
  ws_send(ws,:slice_upload, slice_data)
122
119
  sent_bytes+=data.length
123
120
  currenttime=Time.now
124
- if lastevent.nil? or (currenttime-lastevent)>UPLOAD_REFRESH_SEC
121
+ if lastevent.nil? || ((currenttime-lastevent)>UPLOAD_REFRESH_SEC)
125
122
  notify_progress(session_id,sent_bytes)
126
123
  lastevent=currenttime
127
124
  end
@@ -146,40 +143,40 @@ module Aspera
146
143
  dname=dname.gsub(/\.@gw_api.*$/,'')
147
144
  # ands add indication of number of files if there is more than one
148
145
  if transfer_spec['paths'].length > 1
149
- dname=dname+" #{transfer_spec['paths'].length} Files"
146
+ dname+=" #{transfer_spec['paths'].length} Files"
150
147
  end
151
148
  transfer_spec['download_name']=dname
152
149
  end
153
150
  creation=@gw_api.create('download',{'transfer_spec'=>transfer_spec})[:data]
154
151
  transfer_uuid=creation['url'].split('/').last
155
- if transfer_spec['zip_required'] or transfer_spec['paths'].length > 1
156
- # it is a zip file if zip is required or there is more than 1 file
157
- file_dest=transfer_spec['download_name']+'.zip'
158
- else
159
- # it is a plain file if we don't require zip and there is only one file
160
- file_dest=File.basename(transfer_spec['paths'].first['source'])
152
+ file_dest=if transfer_spec['zip_required'] || transfer_spec['paths'].length > 1
153
+ # it is a zip file if zip is required or there is more than 1 file
154
+ transfer_spec['download_name']+'.zip'
155
+ else
156
+ # it is a plain file if we don't require zip and there is only one file
157
+ File.basename(transfer_spec['paths'].first['source'])
161
158
  end
162
159
  file_dest=File.join(transfer_spec['destination_root'],file_dest)
163
- @gw_api.call({:operation=>'GET',:subpath=>"download/#{transfer_uuid}",:save_to_file=>file_dest})
160
+ @gw_api.call({operation: 'GET',subpath: "download/#{transfer_uuid}",save_to_file: file_dest})
164
161
  end
165
162
 
166
163
  # start FASP transfer based on transfer spec (hash table)
167
164
  # note that it is asynchronous
168
165
  # HTTP download only supports file list
169
166
  def start_transfer(transfer_spec,options={})
170
- raise "GW URL must be set" unless !@gw_api.nil?
171
- raise "option: must be hash (or nil)" unless options.is_a?(Hash)
172
- raise "paths: must be Array" unless transfer_spec['paths'].is_a?(Array)
173
- raise "only token based transfer is supported in GW" unless transfer_spec['token'].is_a?(String)
167
+ raise 'GW URL must be set' if @gw_api.nil?
168
+ raise 'option: must be hash (or nil)' unless options.is_a?(Hash)
169
+ raise 'paths: must be Array' unless transfer_spec['paths'].is_a?(Array)
170
+ raise 'only token based transfer is supported in GW' unless transfer_spec['token'].is_a?(String)
174
171
  Log.dump(:user_spec,transfer_spec)
175
172
  transfer_spec['authentication']||='token'
176
173
  case transfer_spec['direction']
177
- when 'send'
174
+ when Fasp::TransferSpec::DIRECTION_SEND
178
175
  upload(transfer_spec)
179
- when 'receive'
176
+ when Fasp::TransferSpec::DIRECTION_RECEIVE
180
177
  download(transfer_spec)
181
178
  else
182
- raise "error"
179
+ raise "unexpected direction: [#{transfer_spec['direction']}]"
183
180
  end
184
181
  end # start_transfer
185
182
 
@@ -190,25 +187,22 @@ module Aspera
190
187
  end
191
188
 
192
189
  # terminates monitor thread
193
- def shutdown
194
- end
190
+ def shutdown; end
195
191
 
196
- def url=(api_url)
197
- end
192
+ def url=(api_url); end
198
193
 
199
194
  private
200
195
 
201
196
  def initialize(params)
202
- raise "params must be Hash" unless params.is_a?(Hash)
197
+ raise 'params must be Hash' unless params.is_a?(Hash)
203
198
  params=params.symbolize_keys
204
- raise "must have only one param: url" unless params.keys.eql?([:url])
199
+ raise 'must have only one param: url' unless params.keys.eql?([:url])
205
200
  super()
206
- @gw_api=Rest.new({:base_url => params[:url]})
201
+ @gw_api=Rest.new({base_url: params[:url]})
207
202
  api_info = @gw_api.read('info')[:data]
208
- Log.log.info("#{api_info}")
209
- @upload_chunksize=128000 # TODO: configurable ?
203
+ Log.log.info(api_info.to_s)
204
+ @upload_chunksize=128_000 # TODO: configurable ?
210
205
  end
211
-
212
- end # HttpGW
206
+ end # AgentHttpgw
213
207
  end
214
208
  end
@@ -1,4 +1,6 @@
1
- require 'aspera/fasp/manager'
1
+ # frozen_string_literal: true
2
+ require 'aspera/fasp/agent_base'
3
+ require 'aspera/fasp/transfer_spec'
2
4
  require 'aspera/log'
3
5
  require 'tty-spinner'
4
6
 
@@ -6,13 +8,30 @@ module Aspera
6
8
  module Fasp
7
9
  # this singleton class is used by the CLI to provide a common interface to start a transfer
8
10
  # before using it, the use must set the `node_api` member.
9
- class Node < Manager
11
+ class AgentNode < AgentBase
10
12
  # option include: root_id if the node is an access key
11
13
  attr_writer :options
12
- def initialize(node_api,options={})
14
+ def initialize(options)
15
+ raise 'node specification must be Hash' unless options.is_a?(Hash)
16
+ [:url,:username,:password].each { |k| raise "missing parameter [#{k}] in node specification: #{options}" unless options.has_key?(k) }
13
17
  super()
14
- @node_api=node_api
15
- @options=options
18
+ # root id is required for access key
19
+ @root_id=options[:root_id]
20
+ rest_params={ base_url: options[:url]}
21
+ if options[:password].match(/^Bearer /)
22
+ rest_params[:headers]={
23
+ 'X-Aspera-AccessKey'=>options[:username],
24
+ 'Authorization' =>options[:password]
25
+ }
26
+ raise 'root_id is required for access key' if @root_id.nil?
27
+ else
28
+ rest_params[:auth]={
29
+ type: :basic,
30
+ username: options[:username],
31
+ password: options[:password]
32
+ }
33
+ end
34
+ @node_api=Rest.new(rest_params)
16
35
  # TODO: currently only supports one transfer. This is bad shortcut. but ok for CLI.
17
36
  @transfer_id=nil
18
37
  end
@@ -27,24 +46,24 @@ module Aspera
27
46
 
28
47
  # use this to set the node_api end point before using the class.
29
48
  def node_api=(new_value)
30
- if !@node_api.nil? and !new_value.nil?
49
+ if !@node_api.nil? && !new_value.nil?
31
50
  Log.log.warn('overriding existing node api value')
32
51
  end
33
52
  @node_api=new_value
34
53
  end
35
54
 
36
55
  # generic method
37
- def start_transfer(transfer_spec,options=nil)
56
+ def start_transfer(transfer_spec,_options=nil)
38
57
  # add root id if access key
39
- if @options.has_key?(:root_id)
58
+ if !@root_id.nil?
40
59
  case transfer_spec['direction']
41
- when 'send';transfer_spec['source_root_id']=@options[:root_id]
42
- when 'receive';transfer_spec['destination_root_id']=@options[:root_id]
60
+ when Fasp::TransferSpec::DIRECTION_SEND then transfer_spec['source_root_id']=@root_id
61
+ when Fasp::TransferSpec::DIRECTION_RECEIVE then transfer_spec['destination_root_id']=@root_id
43
62
  else raise "unexpected direction in ts: #{transfer_spec['direction']}"
44
63
  end
45
64
  end
46
65
  # manage special additional parameter
47
- if transfer_spec.has_key?('EX_ssh_key_paths') and transfer_spec['EX_ssh_key_paths'].is_a?(Array) and !transfer_spec['EX_ssh_key_paths'].empty?
66
+ if transfer_spec.has_key?('EX_ssh_key_paths') && transfer_spec['EX_ssh_key_paths'].is_a?(Array) && !transfer_spec['EX_ssh_key_paths'].empty?
48
67
  # not standard, so place standard field
49
68
  if transfer_spec.has_key?('ssh_private_key')
50
69
  Log.log.warn('Both ssh_private_key and EX_ssh_key_paths are present, using ssh_private_key')
@@ -54,7 +73,7 @@ module Aspera
54
73
  transfer_spec.delete('EX_ssh_key_paths')
55
74
  end
56
75
  end
57
- if transfer_spec['tags'].is_a?(Hash) and transfer_spec['tags']['aspera'].is_a?(Hash)
76
+ if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags']['aspera'].is_a?(Hash)
58
77
  transfer_spec['tags']['aspera']['xfer_retry']||=150
59
78
  end
60
79
  # optimisation in case of sending to the same node
@@ -74,14 +93,14 @@ module Aspera
74
93
  # lets emulate management events to display progress bar
75
94
  loop do
76
95
  # status is empty sometimes with status 200...
77
- trdata=node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {"status"=>"unknown"} rescue {"status"=>"waiting(read error)"}
96
+ trdata=node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {'status'=>'unknown'} rescue {'status'=>'waiting(read error)'}
78
97
  case trdata['status']
79
98
  when 'completed'
80
99
  notify_end(@transfer_id)
81
100
  break
82
101
  when 'waiting','partially_completed','unknown','waiting(read error)'
83
102
  if spinner.nil?
84
- spinner = TTY::Spinner.new("[:spinner] :title", format: :classic)
103
+ spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
85
104
  spinner.start
86
105
  end
87
106
  spinner.update(title: trdata['status'])
@@ -89,7 +108,7 @@ module Aspera
89
108
  #puts trdata
90
109
  when 'running'
91
110
  #puts "running: sessions:#{trdata["sessions"].length}, #{trdata["sessions"].map{|i| i['bytes_transferred']}.join(',')}"
92
- if !started and trdata['precalc'].is_a?(Hash) and
111
+ if !started && trdata['precalc'].is_a?(Hash) &&
93
112
  trdata['precalc']['status'].eql?('ready')
94
113
  notify_begin(@transfer_id,trdata['precalc']['bytes_expected'])
95
114
  started=true
@@ -98,7 +117,7 @@ module Aspera
98
117
  end
99
118
  else
100
119
  Log.log.warn("trdata -> #{trdata}")
101
- raise Fasp::Error.new("#{trdata['status']}: #{trdata['error_desc']}")
120
+ raise Fasp::Error, "#{trdata['status']}: #{trdata['error_desc']}"
102
121
  end
103
122
  sleep 1
104
123
  end