aspera-cli 4.4.0 → 4.7.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2095 -1503
  3. data/bin/ascli +2 -1
  4. data/bin/asession +4 -5
  5. data/docs/test_env.conf +3 -0
  6. data/examples/aoc.rb +4 -3
  7. data/examples/faspex4.rb +25 -25
  8. data/examples/proxy.pac +1 -1
  9. data/examples/transfer.rb +17 -17
  10. data/lib/aspera/aoc.rb +238 -185
  11. data/lib/aspera/ascmd.rb +93 -83
  12. data/lib/aspera/ats_api.rb +11 -10
  13. data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
  14. data/lib/aspera/cli/extended_value.rb +42 -33
  15. data/lib/aspera/cli/formater.rb +142 -108
  16. data/lib/aspera/cli/info.rb +17 -0
  17. data/lib/aspera/cli/listener/line_dump.rb +3 -2
  18. data/lib/aspera/cli/listener/logger.rb +2 -1
  19. data/lib/aspera/cli/listener/progress.rb +16 -18
  20. data/lib/aspera/cli/listener/progress_multi.rb +18 -21
  21. data/lib/aspera/cli/main.rb +173 -149
  22. data/lib/aspera/cli/manager.rb +163 -168
  23. data/lib/aspera/cli/plugin.rb +43 -31
  24. data/lib/aspera/cli/plugins/alee.rb +6 -6
  25. data/lib/aspera/cli/plugins/aoc.rb +405 -370
  26. data/lib/aspera/cli/plugins/ats.rb +86 -79
  27. data/lib/aspera/cli/plugins/bss.rb +14 -16
  28. data/lib/aspera/cli/plugins/config.rb +580 -362
  29. data/lib/aspera/cli/plugins/console.rb +23 -19
  30. data/lib/aspera/cli/plugins/cos.rb +18 -18
  31. data/lib/aspera/cli/plugins/faspex.rb +201 -158
  32. data/lib/aspera/cli/plugins/faspex5.rb +80 -57
  33. data/lib/aspera/cli/plugins/node.rb +183 -166
  34. data/lib/aspera/cli/plugins/orchestrator.rb +71 -67
  35. data/lib/aspera/cli/plugins/preview.rb +92 -96
  36. data/lib/aspera/cli/plugins/server.rb +79 -75
  37. data/lib/aspera/cli/plugins/shares.rb +35 -19
  38. data/lib/aspera/cli/plugins/sync.rb +20 -22
  39. data/lib/aspera/cli/transfer_agent.rb +76 -113
  40. data/lib/aspera/cli/version.rb +2 -1
  41. data/lib/aspera/colors.rb +35 -27
  42. data/lib/aspera/command_line_builder.rb +48 -34
  43. data/lib/aspera/cos_node.rb +29 -21
  44. data/lib/aspera/data_repository.rb +3 -2
  45. data/lib/aspera/environment.rb +50 -45
  46. data/lib/aspera/fasp/{manager.rb → agent_base.rb} +28 -25
  47. data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +52 -43
  48. data/lib/aspera/fasp/{local.rb → agent_direct.rb} +58 -72
  49. data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +37 -43
  50. data/lib/aspera/fasp/{node.rb → agent_node.rb} +35 -16
  51. data/lib/aspera/fasp/agent_trsdk.rb +104 -0
  52. data/lib/aspera/fasp/error.rb +2 -1
  53. data/lib/aspera/fasp/error_info.rb +68 -52
  54. data/lib/aspera/fasp/installation.rb +152 -124
  55. data/lib/aspera/fasp/listener.rb +1 -0
  56. data/lib/aspera/fasp/parameters.rb +87 -92
  57. data/lib/aspera/fasp/parameters.yaml +305 -249
  58. data/lib/aspera/fasp/resume_policy.rb +11 -14
  59. data/lib/aspera/fasp/transfer_spec.rb +26 -0
  60. data/lib/aspera/fasp/uri.rb +22 -21
  61. data/lib/aspera/faspex_gw.rb +55 -89
  62. data/lib/aspera/hash_ext.rb +4 -3
  63. data/lib/aspera/id_generator.rb +8 -7
  64. data/lib/aspera/keychain/encrypted_hash.rb +121 -0
  65. data/lib/aspera/keychain/macos_security.rb +90 -0
  66. data/lib/aspera/log.rb +55 -37
  67. data/lib/aspera/nagios.rb +13 -12
  68. data/lib/aspera/node.rb +30 -25
  69. data/lib/aspera/oauth.rb +175 -226
  70. data/lib/aspera/open_application.rb +4 -3
  71. data/lib/aspera/persistency_action_once.rb +6 -6
  72. data/lib/aspera/persistency_folder.rb +5 -9
  73. data/lib/aspera/preview/file_types.rb +6 -5
  74. data/lib/aspera/preview/generator.rb +25 -24
  75. data/lib/aspera/preview/options.rb +16 -14
  76. data/lib/aspera/preview/utils.rb +98 -98
  77. data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
  78. data/lib/aspera/proxy_auto_config.rb +111 -20
  79. data/lib/aspera/rest.rb +154 -135
  80. data/lib/aspera/rest_call_error.rb +2 -2
  81. data/lib/aspera/rest_error_analyzer.rb +23 -25
  82. data/lib/aspera/rest_errors_aspera.rb +15 -14
  83. data/lib/aspera/ssh.rb +12 -10
  84. data/lib/aspera/sync.rb +42 -41
  85. data/lib/aspera/temp_file_manager.rb +18 -14
  86. data/lib/aspera/timer_limiter.rb +2 -1
  87. data/lib/aspera/uri_reader.rb +7 -5
  88. data/lib/aspera/web_auth.rb +79 -76
  89. metadata +116 -29
  90. data/docs/Makefile +0 -66
  91. data/docs/README.erb.md +0 -3973
  92. data/docs/README.md +0 -13
  93. data/docs/diagrams.txt +0 -49
  94. data/docs/doc_tools.rb +0 -58
  95. data/lib/aspera/api_detector.rb +0 -60
  96. data/lib/aspera/cli/plugins/shares2.rb +0 -114
  97. data/lib/aspera/secrets.rb +0 -20
@@ -1,36 +1,31 @@
1
- #!/bin/echo this is a ruby class:
2
- #
3
- # FASP manager for Ruby
4
- # Aspera 2016
5
- # Laurent Martin
6
- #
7
- ##############################################################################
8
- require 'aspera/fasp/manager'
1
+ # frozen_string_literal: true
2
+ require 'English'
3
+ require 'aspera/fasp/agent_base'
9
4
  require 'aspera/fasp/error'
10
5
  require 'aspera/fasp/parameters'
11
6
  require 'aspera/fasp/installation'
12
7
  require 'aspera/fasp/resume_policy'
8
+ require 'aspera/fasp/transfer_spec'
13
9
  require 'aspera/log'
14
10
  require 'socket'
15
11
  require 'timeout'
16
12
  require 'securerandom'
13
+ require 'shellwords'
17
14
 
18
15
  module Aspera
19
16
  module Fasp
20
17
  # executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
21
- class Local < Manager
18
+ class AgentDirect < AgentBase
22
19
  # options for initialize (same as values in option transfer_info)
23
20
  DEFAULT_OPTIONS = {
24
- :spawn_timeout_sec => 3,
25
- :spawn_delay_sec => 2,
26
- :wss => false,
27
- :multi_incr_udp => true,
28
- :resume => {}
21
+ spawn_timeout_sec: 3,
22
+ spawn_delay_sec: 2,
23
+ wss: false,
24
+ multi_incr_udp: true,
25
+ resume: {},
26
+ quiet: true # by default no interactive progress bar
29
27
  }
30
- DEFAULT_UDP_PORT=33001
31
28
  private_constant :DEFAULT_OPTIONS
32
- # set to false to keep ascp progress bar display ("true" adds ascp's option -q)
33
- attr_accessor :quiet
34
29
 
35
30
  # start ascp transfer (non blocking), single or multi-session
36
31
  # job information added to @jobs
@@ -44,10 +39,11 @@ module Aspera
44
39
  # clone transfer spec because we modify it (first level keys)
45
40
  transfer_spec=transfer_spec.clone
46
41
  # if there is aspera tags
47
- if transfer_spec['tags'].is_a?(Hash) and transfer_spec['tags']['aspera'].is_a?(Hash)
42
+ if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags']['aspera'].is_a?(Hash)
48
43
  # TODO: what is this for ? only on local ascp ?
49
44
  # NOTE: important: transfer id must be unique: generate random id
50
45
  # using a non unique id results in discard of tags in AoC, and a package is never finalized
46
+ # all sessions in a multi-session transfer must have the same xfer_id (see admin manual)
51
47
  transfer_spec['tags']['aspera']['xfer_id']||=SecureRandom.uuid
52
48
  Log.log.debug("xfer id=#{transfer_spec['xfer_id']}")
53
49
  # TODO: useful ? node only ?
@@ -56,8 +52,8 @@ module Aspera
56
52
  Log.dump('ts',transfer_spec)
57
53
 
58
54
  # add bypass keys when authentication is token and no auth is provided
59
- if transfer_spec.has_key?('token') and
60
- !transfer_spec.has_key?('remote_password') and
55
+ if transfer_spec.has_key?('token') &&
56
+ !transfer_spec.has_key?('remote_password') &&
61
57
  !transfer_spec.has_key?('EX_ssh_key_paths')
62
58
  # transfer_spec['remote_password'] = Installation.instance.bypass_pass # not used
63
59
  transfer_spec['EX_ssh_key_paths'] = Installation.instance.bypass_keys
@@ -68,24 +64,22 @@ module Aspera
68
64
  multi_session_info=nil
69
65
  if transfer_spec.has_key?('multi_session')
70
66
  multi_session_info={
71
- count: transfer_spec['multi_session'].to_i,
67
+ count: transfer_spec['multi_session'].to_i
72
68
  }
73
69
  # Managed by multi-session, so delete from transfer spec
74
70
  transfer_spec.delete('multi_session')
75
- if multi_session_info[:count] < 0
71
+ if multi_session_info[:count].negative?
76
72
  Log.log.error("multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0")
77
73
  multi_session_info = nil
78
74
  elsif multi_session_info[:count].eql?(0)
79
- Log.log.debug("multi_session count is zero: no multisession")
75
+ Log.log.debug('multi_session count is zero: no multisession')
80
76
  multi_session_info = nil
81
- else # multi_session_info[:count] > 0
77
+ elsif @options[:multi_incr_udp] # multi_session_info[:count] > 0
82
78
  # if option not true: keep default udp port for all sessions
83
- if @options[:multi_incr_udp]
84
- # override if specified, else use default value
85
- multi_session_info[:udp_base]=transfer_spec.has_key?('fasp_port') ? transfer_spec['fasp_port'] : DEFAULT_UDP_PORT
86
- # delete from original transfer spec, as we will increment values
87
- transfer_spec.delete('fasp_port')
88
- end
79
+ multi_session_info[:udp_base]=transfer_spec.has_key?('fasp_port') ? transfer_spec['fasp_port'] : TransferSpec::UDP_PORT
80
+ # delete from original transfer spec, as we will increment values
81
+ transfer_spec.delete('fasp_port')
82
+ # override if specified, else use default value
89
83
  end
90
84
  end
91
85
 
@@ -98,22 +92,22 @@ module Aspera
98
92
  env_args[:args].unshift('-I',Installation.instance.path(:fallback_cert))
99
93
  end
100
94
 
101
- env_args[:args].unshift('-q') if @quiet
95
+ env_args[:args].unshift('-q') if @options[:quiet]
102
96
 
103
97
  # transfer job can be multi session
104
98
  xfer_job={
105
- :id => job_options[:job_id],
106
- :sessions => [] # all sessions as below
99
+ id: job_options[:job_id],
100
+ sessions: [] # all sessions as below
107
101
  }
108
102
 
109
103
  # generic session information
110
104
  session={
111
- :thread => nil, # Thread object monitoring management port, not nil when pushed to :sessions
112
- :error => nil, # exception if failed
113
- :io => nil, # management port server socket
114
- :id => nil, # SessionId from INIT message in mgt port
115
- :env_args => env_args, # env vars and args to ascp (from transfer spec)
116
- :options => job_options # [Hash]
105
+ thread: nil, # Thread object monitoring management port, not nil when pushed to :sessions
106
+ error: nil, # exception if failed
107
+ io: nil, # management port server socket
108
+ id: nil, # SessionId from INIT message in mgt port
109
+ env_args: env_args, # env vars and args to ascp (from transfer spec)
110
+ options: job_options # [Hash]
117
111
  }
118
112
 
119
113
  if multi_session_info.nil?
@@ -132,7 +126,7 @@ module Aspera
132
126
  this_session[:env_args][:args]=this_session[:env_args][:args].clone()
133
127
  this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_info[:count]}")
134
128
  # option: increment (default as per ascp manual) or not (cluster on other side ?)
135
- this_session[:env_args][:args].unshift('-O',"#{multi_session_info[:udp_base]+i-1}") if @options[:multi_incr_udp]
129
+ this_session[:env_args][:args].unshift('-O',(multi_session_info[:udp_base]+i-1).to_s) if @options[:multi_incr_udp]
136
130
  this_session[:thread] = Thread.new(this_session) {|s|transfer_thread_entry(s)}
137
131
  xfer_job[:sessions].push(this_session)
138
132
  end
@@ -152,11 +146,11 @@ module Aspera
152
146
  Log.log.debug('wait_for_transfers_completion')
153
147
  # set to non-nil to exit loop
154
148
  result=[]
155
- @jobs.each do |id,job|
149
+ @jobs.each do |_id,job|
156
150
  job[:sessions].each do |session|
157
151
  Log.log.debug("join #{session[:thread]}")
158
152
  session[:thread].join
159
- result.push(session[:error] ? session[:error] : :success)
153
+ result.push(session[:error] || :success)
160
154
  end
161
155
  end
162
156
  Log.log.debug('all transfers joined')
@@ -190,7 +184,7 @@ module Aspera
190
184
  Fasp::Installation.instance.path(env_args[:ascp_version])
191
185
  end
192
186
  # (optional) check it exists
193
- raise Fasp::Error.new("no such file: #{ascp_path}") unless File.exist?(ascp_path)
187
+ raise Fasp::Error, "no such file: #{ascp_path}" unless File.exist?(ascp_path)
194
188
  # open random local TCP port for listening for ascp management
195
189
  mgt_sock = TCPServer.new('127.0.0.1',0)
196
190
  # clone arguments as we eed to modify with mgt port
@@ -198,7 +192,7 @@ module Aspera
198
192
  # add management port
199
193
  ascp_arguments.unshift('-M', mgt_sock.addr[1].to_s)
200
194
  # start ascp in sub process
201
- Log.log.debug("execute: #{env_args[:env].map{|k,v| "#{k}=\"#{v}\""}.join(' ')} \"#{ascp_path}\" \"#{ascp_arguments.join('" "')}\"")
195
+ Log.log.debug("execute: #{env_args[:env].map{|k,v| "#{k}=#{Shellwords.shellescape(v)}"}.join(' ')} #{Shellwords.shellescape(ascp_path)} #{ascp_arguments.map{|a|Shellwords.shellescape(a)}.join(' ')}")
202
196
  # start process
203
197
  ascp_pid = Process.spawn(env_args[:env],[ascp_path,ascp_path],*ascp_arguments)
204
198
  # in parent, wait for connection to socket max 3 seconds
@@ -226,21 +220,21 @@ module Aspera
226
220
  line = ascp_mgt_io.gets
227
221
  # nil when ascp process exits
228
222
  break if line.nil?
229
- current_event_text=current_event_text+line
223
+ current_event_text+=line
230
224
  line.chomp!
231
225
  Log.log.debug("line=[#{line}]")
232
226
  case line
233
227
  when 'FASPMGR 2'
234
228
  # begin event
235
- current_event_data = Hash.new
229
+ current_event_data = {}
236
230
  current_event_text = ''
237
231
  when /^([^:]+): (.*)$/
238
232
  # event field
239
- current_event_data[$1] = $2
233
+ current_event_data[Regexp.last_match(1)] = Regexp.last_match(2)
240
234
  when ''
241
235
  # empty line is separator to end event information
242
236
  raise 'unexpected empty line' if current_event_data.nil?
243
- current_event_data[Manager::LISTENER_SESSION_ID_B]=ascp_pid
237
+ current_event_data[AgentBase::LISTENER_SESSION_ID_B]=ascp_pid
244
238
  notify_listeners(current_event_text,current_event_data)
245
239
  case current_event_data['Type']
246
240
  when 'INIT'
@@ -262,9 +256,9 @@ module Aspera
262
256
  exception_raised=false
263
257
  when 'ERROR'
264
258
  Log.log.error("code: #{last_status_event['Code']}")
265
- if last_status_event['Description'] =~ /bearer token/i
259
+ if last_status_event['Description'] =~ /bearer token/i
266
260
  Log.log.error('need to regenerate token'.red)
267
- if session[:options].is_a?(Hash) and session[:options].has_key?(:regenerate_token)
261
+ if session[:options].is_a?(Hash) && session[:options].has_key?(:regenerate_token)
268
262
  # regenerate token here, expired, or error on it
269
263
  # Note: in multi-session, each session will have a different one.
270
264
  env_args[:env]['ASPERA_SCP_TOKEN']=session[:options][:regenerate_token].call(true)
@@ -280,27 +274,25 @@ module Aspera
280
274
  end
281
275
  rescue SystemCallError => e
282
276
  # Process.spawn
283
- raise Fasp::Error.new(e.message)
284
- rescue Timeout::Error => e
285
- raise Fasp::Error.new('timeout waiting mgt port connect')
286
- rescue Interrupt => e
287
- raise Fasp::Error.new('transfer interrupted by user')
277
+ raise Fasp::Error, e.message
278
+ rescue Timeout::Error
279
+ raise Fasp::Error, 'timeout waiting mgt port connect'
280
+ rescue Interrupt
281
+ raise Fasp::Error, 'transfer interrupted by user'
288
282
  ensure
289
283
  # if ascp was successfully started
290
284
  unless ascp_pid.nil?
291
285
  # "wait" for process to avoid zombie
292
286
  Process.wait(ascp_pid)
293
- status=$?
287
+ status=$CHILD_STATUS
294
288
  ascp_pid=nil
295
289
  session.delete(:io)
296
290
  if !status.success?
297
291
  message="ascp failed with code #{status.exitstatus}"
298
- if exception_raised
299
- # just debug, as main exception is already here
300
- Log.log.debug(message)
301
- else
302
- raise Fasp::Error.new(message)
303
- end
292
+ # raise error only if there was not already an exception
293
+ raise Fasp::Error, message unless exception_raised
294
+ # else just debug, as main exception is already here
295
+ Log.log.debug(message)
304
296
  end
305
297
  end
306
298
  end # begin-ensure
@@ -333,8 +325,6 @@ module Aspera
333
325
  # @param options : keys(symbol): see DEFAULT_OPTIONS
334
326
  def initialize(options=nil)
335
327
  super()
336
- # by default no interactive progress bar
337
- @quiet=true
338
328
  # all transfer jobs, key = SecureRandom.uuid, protected by mutex, condvar on change
339
329
  @jobs={}
340
330
  # mutex protects global data accessed by threads
@@ -344,11 +334,8 @@ module Aspera
344
334
  if !options.nil?
345
335
  raise "expecting Hash (or nil), but have #{options.class}" unless options.is_a?(Hash)
346
336
  options.each do |k,v|
347
- if DEFAULT_OPTIONS.has_key?(k)
348
- @options[k]=v
349
- else
350
- raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map{|i|i.to_s}.join(",")}"
351
- end
337
+ raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.has_key?(k)
338
+ @options[k]=v
352
339
  end
353
340
  end
354
341
  Log.log.debug("local options= #{options}")
@@ -367,13 +354,12 @@ module Aspera
367
354
  start_transfer_with_args_env(session[:env_args],session)
368
355
  end
369
356
  Log.log.debug('transfer ok'.bg_green)
370
- rescue => e
357
+ rescue StandardError => e
371
358
  session[:error]=e
372
359
  Log.log.error("Transfer thread error: #{e.class}:\n#{e.message}:\n#{e.backtrace.join("\n")}".red) if Log.instance.level.eql?(:debug)
373
360
  end
374
361
  Log.log.debug("EXIT (#{Thread.current[:name]})")
375
362
  end
376
-
377
- end # Local
363
+ end # AgentDirect
378
364
  end
379
365
  end
@@ -1,5 +1,6 @@
1
- #!/bin/echo this is a ruby class:
2
- require 'aspera/fasp/manager'
1
+ # frozen_string_literal: true
2
+ require 'aspera/fasp/agent_base'
3
+ require 'aspera/fasp/transfer_spec'
3
4
  require 'aspera/log'
4
5
  require 'aspera/rest'
5
6
  require 'websocket-client-simple'
@@ -12,7 +13,7 @@ require 'json'
12
13
  module Aspera
13
14
  module Fasp
14
15
  # start a transfer using Aspera HTTP Gateway, using web socket session
15
- class HttpGW < Manager
16
+ class AgentHttpgw < AgentBase
16
17
  # message returned by HTTP GW in case of success
17
18
  OK_MESSAGE='end upload'
18
19
  # refresh rate for progress
@@ -29,7 +30,7 @@ module Aspera
29
30
  # we need to keep track of actual file path because transfer spec is modified to be sent in web socket
30
31
  source_paths=[]
31
32
  # get source root or nil
32
- source_root = (transfer_spec.has_key?('source_root') and !transfer_spec['source_root'].empty?) ? transfer_spec['source_root'] : nil
33
+ source_root = transfer_spec.has_key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] : nil
33
34
  # source root is ignored by GW, used only here
34
35
  transfer_spec.delete('source_root')
35
36
  # compute total size of files to upload (for progress)
@@ -61,27 +62,23 @@ module Aspera
61
62
  received+=1
62
63
  else
63
64
  message.chomp!
64
- if message[0].eql?('"') and message[-1].eql?('"')
65
- error=JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message']
66
- else
67
- error="expecting quotes in [#{message}]"
68
- end
65
+ error=message.start_with?('"') && message.end_with?('"') ? JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message'] : "expecting quotes in [#{message}]"
69
66
  end
70
67
  end
71
68
  ws.on :error do |e|
72
69
  error=e
73
70
  end
74
71
  ws.on :open do
75
- Log.log.info("ws: open")
72
+ Log.log.info('ws: open')
76
73
  end
77
74
  ws.on :close do
78
- Log.log.info("ws: close")
75
+ Log.log.info('ws: close')
79
76
  end
80
77
  # open web socket to end point
81
78
  ws.connect("#{@gw_api.params[:base_url]}/upload")
82
79
  # async wait ready
83
- while !ws.open? and error.nil? do
84
- Log.log.info("ws: wait")
80
+ while !ws.open? && error.nil?
81
+ Log.log.info('ws: wait')
85
82
  sleep(0.2)
86
83
  end
87
84
  # notify progress bar
@@ -101,11 +98,11 @@ module Aspera
101
98
  file_size=item['file_size']
102
99
  file_name=File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
103
100
  # compute total number of slices
104
- numslices=1+(file_size-1)/@upload_chunksize
101
+ numslices=1+((file_size-1)/@upload_chunksize)
105
102
  File.open(source_paths[file_index]) do |file|
106
103
  # current slice index
107
104
  slicenum=0
108
- while !file.eof? do
105
+ while !file.eof?
109
106
  data=file.read(@upload_chunksize)
110
107
  slice_data={
111
108
  name: file_name,
@@ -117,11 +114,11 @@ module Aspera
117
114
  fileIndex: file_index
118
115
  }
119
116
  # log without data
120
- Log.dump(:slide_data,slice_data.keys.inject({}){|m,i|m[i]=i.eql?(:data)?'base64 data':slice_data[i];m}) if slicenum.eql?(0)
117
+ Log.dump(:slide_data,slice_data.keys.each_with_object({}){|i,m|m[i]=i.eql?(:data)?'base64 data':slice_data[i];}) if slicenum.eql?(0)
121
118
  ws_send(ws,:slice_upload, slice_data)
122
119
  sent_bytes+=data.length
123
120
  currenttime=Time.now
124
- if lastevent.nil? or (currenttime-lastevent)>UPLOAD_REFRESH_SEC
121
+ if lastevent.nil? || ((currenttime-lastevent)>UPLOAD_REFRESH_SEC)
125
122
  notify_progress(session_id,sent_bytes)
126
123
  lastevent=currenttime
127
124
  end
@@ -146,40 +143,40 @@ module Aspera
146
143
  dname=dname.gsub(/\.@gw_api.*$/,'')
147
144
  # ands add indication of number of files if there is more than one
148
145
  if transfer_spec['paths'].length > 1
149
- dname=dname+" #{transfer_spec['paths'].length} Files"
146
+ dname+=" #{transfer_spec['paths'].length} Files"
150
147
  end
151
148
  transfer_spec['download_name']=dname
152
149
  end
153
150
  creation=@gw_api.create('download',{'transfer_spec'=>transfer_spec})[:data]
154
151
  transfer_uuid=creation['url'].split('/').last
155
- if transfer_spec['zip_required'] or transfer_spec['paths'].length > 1
156
- # it is a zip file if zip is required or there is more than 1 file
157
- file_dest=transfer_spec['download_name']+'.zip'
158
- else
159
- # it is a plain file if we don't require zip and there is only one file
160
- file_dest=File.basename(transfer_spec['paths'].first['source'])
152
+ file_dest=if transfer_spec['zip_required'] || transfer_spec['paths'].length > 1
153
+ # it is a zip file if zip is required or there is more than 1 file
154
+ transfer_spec['download_name']+'.zip'
155
+ else
156
+ # it is a plain file if we don't require zip and there is only one file
157
+ File.basename(transfer_spec['paths'].first['source'])
161
158
  end
162
159
  file_dest=File.join(transfer_spec['destination_root'],file_dest)
163
- @gw_api.call({:operation=>'GET',:subpath=>"download/#{transfer_uuid}",:save_to_file=>file_dest})
160
+ @gw_api.call({operation: 'GET',subpath: "download/#{transfer_uuid}",save_to_file: file_dest})
164
161
  end
165
162
 
166
163
  # start FASP transfer based on transfer spec (hash table)
167
164
  # note that it is asynchronous
168
165
  # HTTP download only supports file list
169
166
  def start_transfer(transfer_spec,options={})
170
- raise "GW URL must be set" unless !@gw_api.nil?
171
- raise "option: must be hash (or nil)" unless options.is_a?(Hash)
172
- raise "paths: must be Array" unless transfer_spec['paths'].is_a?(Array)
173
- raise "only token based transfer is supported in GW" unless transfer_spec['token'].is_a?(String)
167
+ raise 'GW URL must be set' if @gw_api.nil?
168
+ raise 'option: must be hash (or nil)' unless options.is_a?(Hash)
169
+ raise 'paths: must be Array' unless transfer_spec['paths'].is_a?(Array)
170
+ raise 'only token based transfer is supported in GW' unless transfer_spec['token'].is_a?(String)
174
171
  Log.dump(:user_spec,transfer_spec)
175
172
  transfer_spec['authentication']||='token'
176
173
  case transfer_spec['direction']
177
- when 'send'
174
+ when Fasp::TransferSpec::DIRECTION_SEND
178
175
  upload(transfer_spec)
179
- when 'receive'
176
+ when Fasp::TransferSpec::DIRECTION_RECEIVE
180
177
  download(transfer_spec)
181
178
  else
182
- raise "error"
179
+ raise "unexpected direction: [#{transfer_spec['direction']}]"
183
180
  end
184
181
  end # start_transfer
185
182
 
@@ -190,25 +187,22 @@ module Aspera
190
187
  end
191
188
 
192
189
  # terminates monitor thread
193
- def shutdown
194
- end
190
+ def shutdown; end
195
191
 
196
- def url=(api_url)
197
- end
192
+ def url=(api_url); end
198
193
 
199
194
  private
200
195
 
201
196
  def initialize(params)
202
- raise "params must be Hash" unless params.is_a?(Hash)
197
+ raise 'params must be Hash' unless params.is_a?(Hash)
203
198
  params=params.symbolize_keys
204
- raise "must have only one param: url" unless params.keys.eql?([:url])
199
+ raise 'must have only one param: url' unless params.keys.eql?([:url])
205
200
  super()
206
- @gw_api=Rest.new({:base_url => params[:url]})
201
+ @gw_api=Rest.new({base_url: params[:url]})
207
202
  api_info = @gw_api.read('info')[:data]
208
- Log.log.info("#{api_info}")
209
- @upload_chunksize=128000 # TODO: configurable ?
203
+ Log.log.info(api_info.to_s)
204
+ @upload_chunksize=128_000 # TODO: configurable ?
210
205
  end
211
-
212
- end # HttpGW
206
+ end # AgentHttpgw
213
207
  end
214
208
  end
@@ -1,4 +1,6 @@
1
- require 'aspera/fasp/manager'
1
+ # frozen_string_literal: true
2
+ require 'aspera/fasp/agent_base'
3
+ require 'aspera/fasp/transfer_spec'
2
4
  require 'aspera/log'
3
5
  require 'tty-spinner'
4
6
 
@@ -6,13 +8,30 @@ module Aspera
6
8
  module Fasp
7
9
  # this singleton class is used by the CLI to provide a common interface to start a transfer
8
10
  # before using it, the use must set the `node_api` member.
9
- class Node < Manager
11
+ class AgentNode < AgentBase
10
12
  # option include: root_id if the node is an access key
11
13
  attr_writer :options
12
- def initialize(node_api,options={})
14
+ def initialize(options)
15
+ raise 'node specification must be Hash' unless options.is_a?(Hash)
16
+ [:url,:username,:password].each { |k| raise "missing parameter [#{k}] in node specification: #{options}" unless options.has_key?(k) }
13
17
  super()
14
- @node_api=node_api
15
- @options=options
18
+ # root id is required for access key
19
+ @root_id=options[:root_id]
20
+ rest_params={ base_url: options[:url]}
21
+ if options[:password].match(/^Bearer /)
22
+ rest_params[:headers]={
23
+ 'X-Aspera-AccessKey'=>options[:username],
24
+ 'Authorization' =>options[:password]
25
+ }
26
+ raise 'root_id is required for access key' if @root_id.nil?
27
+ else
28
+ rest_params[:auth]={
29
+ type: :basic,
30
+ username: options[:username],
31
+ password: options[:password]
32
+ }
33
+ end
34
+ @node_api=Rest.new(rest_params)
16
35
  # TODO: currently only supports one transfer. This is bad shortcut. but ok for CLI.
17
36
  @transfer_id=nil
18
37
  end
@@ -27,24 +46,24 @@ module Aspera
27
46
 
28
47
  # use this to set the node_api end point before using the class.
29
48
  def node_api=(new_value)
30
- if !@node_api.nil? and !new_value.nil?
49
+ if !@node_api.nil? && !new_value.nil?
31
50
  Log.log.warn('overriding existing node api value')
32
51
  end
33
52
  @node_api=new_value
34
53
  end
35
54
 
36
55
  # generic method
37
- def start_transfer(transfer_spec,options=nil)
56
+ def start_transfer(transfer_spec,_options=nil)
38
57
  # add root id if access key
39
- if @options.has_key?(:root_id)
58
+ if !@root_id.nil?
40
59
  case transfer_spec['direction']
41
- when 'send';transfer_spec['source_root_id']=@options[:root_id]
42
- when 'receive';transfer_spec['destination_root_id']=@options[:root_id]
60
+ when Fasp::TransferSpec::DIRECTION_SEND then transfer_spec['source_root_id']=@root_id
61
+ when Fasp::TransferSpec::DIRECTION_RECEIVE then transfer_spec['destination_root_id']=@root_id
43
62
  else raise "unexpected direction in ts: #{transfer_spec['direction']}"
44
63
  end
45
64
  end
46
65
  # manage special additional parameter
47
- if transfer_spec.has_key?('EX_ssh_key_paths') and transfer_spec['EX_ssh_key_paths'].is_a?(Array) and !transfer_spec['EX_ssh_key_paths'].empty?
66
+ if transfer_spec.has_key?('EX_ssh_key_paths') && transfer_spec['EX_ssh_key_paths'].is_a?(Array) && !transfer_spec['EX_ssh_key_paths'].empty?
48
67
  # not standard, so place standard field
49
68
  if transfer_spec.has_key?('ssh_private_key')
50
69
  Log.log.warn('Both ssh_private_key and EX_ssh_key_paths are present, using ssh_private_key')
@@ -54,7 +73,7 @@ module Aspera
54
73
  transfer_spec.delete('EX_ssh_key_paths')
55
74
  end
56
75
  end
57
- if transfer_spec['tags'].is_a?(Hash) and transfer_spec['tags']['aspera'].is_a?(Hash)
76
+ if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags']['aspera'].is_a?(Hash)
58
77
  transfer_spec['tags']['aspera']['xfer_retry']||=150
59
78
  end
60
79
  # optimisation in case of sending to the same node
@@ -74,14 +93,14 @@ module Aspera
74
93
  # lets emulate management events to display progress bar
75
94
  loop do
76
95
  # status is empty sometimes with status 200...
77
- trdata=node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {"status"=>"unknown"} rescue {"status"=>"waiting(read error)"}
96
+ trdata=node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {'status'=>'unknown'} rescue {'status'=>'waiting(read error)'}
78
97
  case trdata['status']
79
98
  when 'completed'
80
99
  notify_end(@transfer_id)
81
100
  break
82
101
  when 'waiting','partially_completed','unknown','waiting(read error)'
83
102
  if spinner.nil?
84
- spinner = TTY::Spinner.new("[:spinner] :title", format: :classic)
103
+ spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
85
104
  spinner.start
86
105
  end
87
106
  spinner.update(title: trdata['status'])
@@ -89,7 +108,7 @@ module Aspera
89
108
  #puts trdata
90
109
  when 'running'
91
110
  #puts "running: sessions:#{trdata["sessions"].length}, #{trdata["sessions"].map{|i| i['bytes_transferred']}.join(',')}"
92
- if !started and trdata['precalc'].is_a?(Hash) and
111
+ if !started && trdata['precalc'].is_a?(Hash) &&
93
112
  trdata['precalc']['status'].eql?('ready')
94
113
  notify_begin(@transfer_id,trdata['precalc']['bytes_expected'])
95
114
  started=true
@@ -98,7 +117,7 @@ module Aspera
98
117
  end
99
118
  else
100
119
  Log.log.warn("trdata -> #{trdata}")
101
- raise Fasp::Error.new("#{trdata['status']}: #{trdata['error_desc']}")
120
+ raise Fasp::Error, "#{trdata['status']}: #{trdata['error_desc']}"
102
121
  end
103
122
  sleep 1
104
123
  end