aspera-cli 4.6.0 → 4.7.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (96) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +427 -300
  3. data/bin/ascli +2 -1
  4. data/bin/asession +1 -0
  5. data/docs/test_env.conf +2 -0
  6. data/examples/aoc.rb +4 -3
  7. data/examples/faspex4.rb +21 -19
  8. data/examples/proxy.pac +1 -1
  9. data/examples/transfer.rb +15 -15
  10. data/lib/aspera/aoc.rb +135 -124
  11. data/lib/aspera/ascmd.rb +85 -75
  12. data/lib/aspera/ats_api.rb +11 -10
  13. data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
  14. data/lib/aspera/cli/extended_value.rb +42 -33
  15. data/lib/aspera/cli/formater.rb +138 -111
  16. data/lib/aspera/cli/info.rb +17 -0
  17. data/lib/aspera/cli/listener/line_dump.rb +3 -2
  18. data/lib/aspera/cli/listener/logger.rb +2 -1
  19. data/lib/aspera/cli/listener/progress.rb +16 -18
  20. data/lib/aspera/cli/listener/progress_multi.rb +13 -16
  21. data/lib/aspera/cli/main.rb +122 -130
  22. data/lib/aspera/cli/manager.rb +146 -154
  23. data/lib/aspera/cli/plugin.rb +38 -34
  24. data/lib/aspera/cli/plugins/alee.rb +6 -6
  25. data/lib/aspera/cli/plugins/aoc.rb +273 -276
  26. data/lib/aspera/cli/plugins/ats.rb +82 -76
  27. data/lib/aspera/cli/plugins/bss.rb +14 -16
  28. data/lib/aspera/cli/plugins/config.rb +350 -306
  29. data/lib/aspera/cli/plugins/console.rb +23 -19
  30. data/lib/aspera/cli/plugins/cos.rb +18 -18
  31. data/lib/aspera/cli/plugins/faspex.rb +180 -159
  32. data/lib/aspera/cli/plugins/faspex5.rb +64 -54
  33. data/lib/aspera/cli/plugins/node.rb +147 -140
  34. data/lib/aspera/cli/plugins/orchestrator.rb +68 -66
  35. data/lib/aspera/cli/plugins/preview.rb +92 -96
  36. data/lib/aspera/cli/plugins/server.rb +79 -75
  37. data/lib/aspera/cli/plugins/shares.rb +23 -24
  38. data/lib/aspera/cli/plugins/sync.rb +20 -22
  39. data/lib/aspera/cli/transfer_agent.rb +40 -39
  40. data/lib/aspera/cli/version.rb +2 -1
  41. data/lib/aspera/colors.rb +35 -27
  42. data/lib/aspera/command_line_builder.rb +48 -34
  43. data/lib/aspera/cos_node.rb +29 -21
  44. data/lib/aspera/data_repository.rb +3 -2
  45. data/lib/aspera/environment.rb +50 -45
  46. data/lib/aspera/fasp/agent_base.rb +22 -20
  47. data/lib/aspera/fasp/agent_connect.rb +13 -11
  48. data/lib/aspera/fasp/agent_direct.rb +48 -59
  49. data/lib/aspera/fasp/agent_httpgw.rb +33 -39
  50. data/lib/aspera/fasp/agent_node.rb +15 -13
  51. data/lib/aspera/fasp/agent_trsdk.rb +12 -14
  52. data/lib/aspera/fasp/error.rb +2 -1
  53. data/lib/aspera/fasp/error_info.rb +68 -52
  54. data/lib/aspera/fasp/installation.rb +106 -94
  55. data/lib/aspera/fasp/listener.rb +1 -0
  56. data/lib/aspera/fasp/parameters.rb +83 -92
  57. data/lib/aspera/fasp/parameters.yaml +305 -249
  58. data/lib/aspera/fasp/resume_policy.rb +11 -14
  59. data/lib/aspera/fasp/transfer_spec.rb +26 -0
  60. data/lib/aspera/fasp/uri.rb +22 -21
  61. data/lib/aspera/faspex_gw.rb +55 -90
  62. data/lib/aspera/hash_ext.rb +4 -3
  63. data/lib/aspera/id_generator.rb +8 -7
  64. data/lib/aspera/keychain/encrypted_hash.rb +17 -16
  65. data/lib/aspera/keychain/macos_security.rb +6 -10
  66. data/lib/aspera/log.rb +25 -20
  67. data/lib/aspera/nagios.rb +13 -12
  68. data/lib/aspera/node.rb +30 -22
  69. data/lib/aspera/oauth.rb +175 -226
  70. data/lib/aspera/open_application.rb +4 -3
  71. data/lib/aspera/persistency_action_once.rb +6 -6
  72. data/lib/aspera/persistency_folder.rb +5 -9
  73. data/lib/aspera/preview/file_types.rb +6 -5
  74. data/lib/aspera/preview/generator.rb +25 -24
  75. data/lib/aspera/preview/options.rb +16 -14
  76. data/lib/aspera/preview/utils.rb +98 -98
  77. data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
  78. data/lib/aspera/proxy_auto_config.rb +111 -20
  79. data/lib/aspera/rest.rb +115 -113
  80. data/lib/aspera/rest_call_error.rb +2 -2
  81. data/lib/aspera/rest_error_analyzer.rb +23 -25
  82. data/lib/aspera/rest_errors_aspera.rb +15 -14
  83. data/lib/aspera/ssh.rb +12 -10
  84. data/lib/aspera/sync.rb +42 -41
  85. data/lib/aspera/temp_file_manager.rb +18 -14
  86. data/lib/aspera/timer_limiter.rb +2 -1
  87. data/lib/aspera/uri_reader.rb +7 -5
  88. data/lib/aspera/web_auth.rb +79 -76
  89. metadata +64 -21
  90. data/docs/Makefile +0 -65
  91. data/docs/README.erb.md +0 -4424
  92. data/docs/README.md +0 -13
  93. data/docs/diagrams.txt +0 -49
  94. data/docs/doc_tools.rb +0 -58
  95. data/lib/aspera/cli/plugins/shares2.rb +0 -114
  96. data/lib/aspera/fasp/default.rb +0 -17
@@ -1,30 +1,32 @@
1
+ # frozen_string_literal: true
1
2
  module Aspera
2
3
  module Fasp
3
4
  # Base class for FASP transfer agents
4
5
  # sub classes shall implement start_transfer and shutdown
5
6
  class AgentBase
7
+ # fields description for JSON generation
8
+ INTEGER_FIELDS=%w[Bytescont FaspFileArgIndex StartByte Rate MinRate Port Priority RateCap MinRateCap TCPPort CreatePolicy TimePolicy DatagramSize XoptFlags VLinkVersion
9
+ PeerVLinkVersion DSPipelineDepth PeerDSPipelineDepth ReadBlockSize WriteBlockSize ClusterNumNodes ClusterNodeId Size Written Loss FileBytes PreTransferBytes TransferBytes PMTU Elapsedusec ArgScansAttempted ArgScansCompleted PathScansAttempted FileScansCompleted TransfersAttempted TransfersPassed Delay].freeze
10
+ BOOLEAN_FIELDS=%w[Encryption Remote RateLock MinRateLock PolicyLock FilesEncrypt FilesDecrypt VLinkLocalEnabled VLinkRemoteEnabled MoveRange Keepalive TestLogin UseProxy
11
+ Precalc RTTAutocorrect].freeze
12
+ EXPECTED_METHODS=%i[text struct enhanced].freeze
13
+ private_constant :INTEGER_FIELDS,:BOOLEAN_FIELDS,:EXPECTED_METHODS
6
14
 
7
15
  private
8
16
 
9
- # fields description for JSON generation
10
- IntegerFields=['Bytescont','FaspFileArgIndex','StartByte','Rate','MinRate','Port','Priority','RateCap','MinRateCap','TCPPort','CreatePolicy','TimePolicy','DatagramSize','XoptFlags','VLinkVersion','PeerVLinkVersion','DSPipelineDepth','PeerDSPipelineDepth','ReadBlockSize','WriteBlockSize','ClusterNumNodes','ClusterNodeId','Size','Written','Loss','FileBytes','PreTransferBytes','TransferBytes','PMTU','Elapsedusec','ArgScansAttempted','ArgScansCompleted','PathScansAttempted','FileScansCompleted','TransfersAttempted','TransfersPassed','Delay']
11
- BooleanFields=['Encryption','Remote','RateLock','MinRateLock','PolicyLock','FilesEncrypt','FilesDecrypt','VLinkLocalEnabled','VLinkRemoteEnabled','MoveRange','Keepalive','TestLogin','UseProxy','Precalc','RTTAutocorrect']
12
- ExpectedMethod=[:text,:struct,:enhanced]
13
-
14
17
  # translates legacy event into enhanced (JSON) event
15
18
  def enhanced_event_format(event)
16
- return event.keys.inject({}) do |h,e|
19
+ return event.keys.each_with_object({}) do |e,h|
17
20
  # capital_to_snake_case
18
21
  new_name=e.
19
- gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
20
- gsub(/([a-z\d])([A-Z])/,'\1_\2').
21
- gsub(/([a-z\d])(usec)$/,'\1_\2').
22
- downcase
22
+ gsub(/([a-z\d])([A-Z])/,'\1_\2').
23
+ gsub(/([a-z\d])(usec)$/,'\1_\2').
24
+ gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
25
+ downcase
23
26
  value=event[e]
24
- value=value.to_i if IntegerFields.include?(e)
25
- value=value.eql?('Yes') ? true : false if BooleanFields.include?(e)
27
+ value=value.to_i if INTEGER_FIELDS.include?(e)
28
+ value=value.eql?('Yes') if BOOLEAN_FIELDS.include?(e)
26
29
  h[new_name]=value
27
- h
28
30
  end
29
31
  end
30
32
 
@@ -33,14 +35,14 @@ module Aspera
33
35
  end
34
36
 
35
37
  def notify_listeners(current_event_text,current_event_data)
36
- Log.log.debug("send event to listeners")
38
+ Log.log.debug('send event to listeners')
37
39
  enhanced_event=nil
38
40
  @listeners.each do |listener|
39
- listener.send(:event_text,current_event_text) if listener.respond_to?(:event_text)
40
- listener.send(:event_struct,current_event_data) if listener.respond_to?(:event_struct)
41
+ listener.event_text(current_event_text) if listener.respond_to?(:event_text)
42
+ listener.event_struct(current_event_data) if listener.respond_to?(:event_struct)
41
43
  if listener.respond_to?(:event_enhanced)
42
44
  enhanced_event=enhanced_event_format(current_event_data) if enhanced_event.nil?
43
- listener.send(:event_enhanced,enhanced_event)
45
+ listener.event_enhanced(enhanced_event)
44
46
  end
45
47
  end
46
48
  end # notify_listeners
@@ -58,12 +60,13 @@ module Aspera
58
60
  end
59
61
 
60
62
  public
63
+
61
64
  LISTENER_SESSION_ID_B='ListenerSessionId'
62
65
  LISTENER_SESSION_ID_S='listener_session_id'
63
66
 
64
67
  # listener receives events
65
68
  def add_listener(listener)
66
- raise "expect one of #{ExpectedMethod}" if ExpectedMethod.inject(0){|m,e|m+=listener.respond_to?("event_#{e}")?1:0;m}.eql?(0)
69
+ raise "expect one of #{EXPECTED_METHODS}" if EXPECTED_METHODS.inject(0){|m,e|m+=listener.respond_to?("event_#{e}")?1:0;m}.eql?(0)
67
70
  @listeners.push(listener)
68
71
  self
69
72
  end
@@ -72,14 +75,13 @@ module Aspera
72
75
  # it must be a list of :success or exception
73
76
  def self.validate_status_list(statuses)
74
77
  raise "internal error: bad statuses type: #{statuses.class}" unless statuses.is_a?(Array)
75
- raise "internal error: bad statuses content: #{statuses}" unless statuses.select{|i|!i.eql?(:success) and !i.is_a?(StandardError)}.empty?
78
+ raise "internal error: bad statuses content: #{statuses}" unless statuses.select{|i|!i.eql?(:success) && !i.is_a?(StandardError)}.empty?
76
79
  end
77
80
 
78
81
  # the following methods must be implemented by subclass:
79
82
  # start_transfer(transfer_spec,options) : start and wait for completion
80
83
  # wait_for_transfers_completion : wait for termination of all transfers, @return list of : :success or error message
81
84
  # optional: shutdown
82
-
83
85
  end
84
86
  end
85
87
  end
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  require 'aspera/fasp/agent_base'
2
3
  require 'aspera/rest'
3
4
  require 'aspera/open_application'
@@ -10,7 +11,7 @@ module Aspera
10
11
  MAX_CONNECT_START_RETRY=3
11
12
  SLEEP_SEC_BETWEEN_RETRY=2
12
13
  private_constant :MAX_CONNECT_START_RETRY,:SLEEP_SEC_BETWEEN_RETRY
13
- def initialize(options)
14
+ def initialize(_options)
14
15
  super()
15
16
  @connect_settings={
16
17
  'app_id' => SecureRandom.uuid
@@ -23,7 +24,7 @@ module Aspera
23
24
  @connect_api=Rest.new({base_url: "#{connect_url}/v5/connect",headers: {'Origin'=>Rest.user_agent}}) # could use v6 also now
24
25
  cinfo=@connect_api.read('info/version')[:data]
25
26
  Log.dump(:connect_version,cinfo)
26
- rescue => e # Errno::ECONNREFUSED
27
+ rescue StandardError => e # Errno::ECONNREFUSED
27
28
  raise StandardError,"Unable to start connect after #{trynumber} try" if trynumber >= MAX_CONNECT_START_RETRY
28
29
  Log.log.warn("connect is not started. Retry ##{trynumber}, err=#{e}")
29
30
  trynumber+=1
@@ -36,11 +37,12 @@ module Aspera
36
37
  end
37
38
  end
38
39
 
39
- def start_transfer(transfer_spec,options=nil)
40
+ def start_transfer(transfer_spec,_options=nil)
40
41
  if transfer_spec['direction'] == 'send'
41
42
  Log.log.warn("Connect requires upload selection using GUI, ignoring #{transfer_spec['paths']}".red)
42
43
  transfer_spec.delete('paths')
43
- resdata=@connect_api.create('windows/select-open-file-dialog/',{'aspera_connect_settings'=>@connect_settings,'title'=>'Select Files','suggestedName'=>'','allowMultipleSelection'=>true,'allowedFileTypes'=>''})[:data]
44
+ resdata=@connect_api.create('windows/select-open-file-dialog/',
45
+ {'aspera_connect_settings'=>@connect_settings,'title'=>'Select Files','suggestedName'=>'','allowMultipleSelection'=>true,'allowedFileTypes'=>''})[:data]
44
46
  transfer_spec['paths']=resdata['dataTransfer']['files'].map { |i| {'source'=>i['name']}}
45
47
  end
46
48
  @request_id=SecureRandom.uuid
@@ -50,10 +52,10 @@ module Aspera
50
52
  connect_transfer_args={
51
53
  'aspera_connect_settings'=>@connect_settings.merge({
52
54
  'request_id' =>@request_id,
53
- 'allow_dialogs' =>true,
55
+ 'allow_dialogs' =>true
54
56
  }),
55
57
  'transfer_specs' =>[{
56
- 'transfer_spec' =>transfer_spec,
58
+ 'transfer_spec' =>transfer_spec
57
59
  }]}
58
60
  # asynchronous anyway
59
61
  res=@connect_api.create('transfers/start',connect_transfer_args)[:data]
@@ -70,7 +72,7 @@ module Aspera
70
72
  if tr_info['transfer_info'].is_a?(Hash)
71
73
  trdata=tr_info['transfer_info']
72
74
  if trdata.nil?
73
- Log.log.warn("no session in Connect")
75
+ Log.log.warn('no session in Connect')
74
76
  break
75
77
  end
76
78
  # TODO: get session id
@@ -87,7 +89,7 @@ module Aspera
87
89
  spinner.spin
88
90
  when 'running'
89
91
  #puts "running: sessions:#{trdata['sessions'].length}, #{trdata['sessions'].map{|i| i['bytes_transferred']}.join(',')}"
90
- if !started and trdata['bytes_expected'] != 0
92
+ if !started && (trdata['bytes_expected'] != 0)
91
93
  spinner.success unless spinner.nil?
92
94
  notify_begin(@connect_settings['app_id'],trdata['bytes_expected'])
93
95
  started=true
@@ -96,14 +98,14 @@ module Aspera
96
98
  end
97
99
  when 'failed'
98
100
  spinner.error unless spinner.nil?
99
- raise Fasp::Error.new(trdata['error_desc'])
101
+ raise Fasp::Error, trdata['error_desc']
100
102
  else
101
- raise Fasp::Error.new("unknown status: #{trdata['status']}: #{trdata['error_desc']}")
103
+ raise Fasp::Error, "unknown status: #{trdata['status']}: #{trdata['error_desc']}"
102
104
  end
103
105
  end
104
106
  sleep 1
105
107
  end
106
- rescue => e
108
+ rescue StandardError => e
107
109
  return [e]
108
110
  end
109
111
  return [:success]
@@ -1,20 +1,16 @@
1
- #!/bin/echo this is a ruby class:
2
- #
3
- # FASP manager for Ruby
4
- # Aspera 2016
5
- # Laurent Martin
6
- #
7
- ##############################################################################
1
+ # frozen_string_literal: true
2
+ require 'English'
8
3
  require 'aspera/fasp/agent_base'
9
4
  require 'aspera/fasp/error'
10
5
  require 'aspera/fasp/parameters'
11
6
  require 'aspera/fasp/installation'
12
7
  require 'aspera/fasp/resume_policy'
13
- require 'aspera/fasp/default'
8
+ require 'aspera/fasp/transfer_spec'
14
9
  require 'aspera/log'
15
10
  require 'socket'
16
11
  require 'timeout'
17
12
  require 'securerandom'
13
+ require 'shellwords'
18
14
 
19
15
  module Aspera
20
16
  module Fasp
@@ -27,7 +23,7 @@ module Aspera
27
23
  wss: false,
28
24
  multi_incr_udp: true,
29
25
  resume: {},
30
- quiet: true # by default no interactive progress bar
26
+ quiet: true # by default no interactive progress bar
31
27
  }
32
28
  private_constant :DEFAULT_OPTIONS
33
29
 
@@ -43,10 +39,11 @@ module Aspera
43
39
  # clone transfer spec because we modify it (first level keys)
44
40
  transfer_spec=transfer_spec.clone
45
41
  # if there is aspera tags
46
- if transfer_spec['tags'].is_a?(Hash) and transfer_spec['tags']['aspera'].is_a?(Hash)
42
+ if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags']['aspera'].is_a?(Hash)
47
43
  # TODO: what is this for ? only on local ascp ?
48
44
  # NOTE: important: transfer id must be unique: generate random id
49
45
  # using a non unique id results in discard of tags in AoC, and a package is never finalized
46
+ # all sessions in a multi-session transfer must have the same xfer_id (see admin manual)
50
47
  transfer_spec['tags']['aspera']['xfer_id']||=SecureRandom.uuid
51
48
  Log.log.debug("xfer id=#{transfer_spec['xfer_id']}")
52
49
  # TODO: useful ? node only ?
@@ -55,8 +52,8 @@ module Aspera
55
52
  Log.dump('ts',transfer_spec)
56
53
 
57
54
  # add bypass keys when authentication is token and no auth is provided
58
- if transfer_spec.has_key?('token') and
59
- !transfer_spec.has_key?('remote_password') and
55
+ if transfer_spec.has_key?('token') &&
56
+ !transfer_spec.has_key?('remote_password') &&
60
57
  !transfer_spec.has_key?('EX_ssh_key_paths')
61
58
  # transfer_spec['remote_password'] = Installation.instance.bypass_pass # not used
62
59
  transfer_spec['EX_ssh_key_paths'] = Installation.instance.bypass_keys
@@ -67,24 +64,22 @@ module Aspera
67
64
  multi_session_info=nil
68
65
  if transfer_spec.has_key?('multi_session')
69
66
  multi_session_info={
70
- count: transfer_spec['multi_session'].to_i,
67
+ count: transfer_spec['multi_session'].to_i
71
68
  }
72
69
  # Managed by multi-session, so delete from transfer spec
73
70
  transfer_spec.delete('multi_session')
74
- if multi_session_info[:count] < 0
71
+ if multi_session_info[:count].negative?
75
72
  Log.log.error("multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0")
76
73
  multi_session_info = nil
77
74
  elsif multi_session_info[:count].eql?(0)
78
- Log.log.debug("multi_session count is zero: no multisession")
75
+ Log.log.debug('multi_session count is zero: no multisession')
79
76
  multi_session_info = nil
80
- else # multi_session_info[:count] > 0
77
+ elsif @options[:multi_incr_udp] # multi_session_info[:count] > 0
81
78
  # if option not true: keep default udp port for all sessions
82
- if @options[:multi_incr_udp]
83
- # override if specified, else use default value
84
- multi_session_info[:udp_base]=transfer_spec.has_key?('fasp_port') ? transfer_spec['fasp_port'] : Default::UDP_PORT
85
- # delete from original transfer spec, as we will increment values
86
- transfer_spec.delete('fasp_port')
87
- end
79
+ multi_session_info[:udp_base]=transfer_spec.has_key?('fasp_port') ? transfer_spec['fasp_port'] : TransferSpec::UDP_PORT
80
+ # delete from original transfer spec, as we will increment values
81
+ transfer_spec.delete('fasp_port')
82
+ # override if specified, else use default value
88
83
  end
89
84
  end
90
85
 
@@ -101,18 +96,18 @@ module Aspera
101
96
 
102
97
  # transfer job can be multi session
103
98
  xfer_job={
104
- :id => job_options[:job_id],
105
- :sessions => [] # all sessions as below
99
+ id: job_options[:job_id],
100
+ sessions: [] # all sessions as below
106
101
  }
107
102
 
108
103
  # generic session information
109
104
  session={
110
- :thread => nil, # Thread object monitoring management port, not nil when pushed to :sessions
111
- :error => nil, # exception if failed
112
- :io => nil, # management port server socket
113
- :id => nil, # SessionId from INIT message in mgt port
114
- :env_args => env_args, # env vars and args to ascp (from transfer spec)
115
- :options => job_options # [Hash]
105
+ thread: nil, # Thread object monitoring management port, not nil when pushed to :sessions
106
+ error: nil, # exception if failed
107
+ io: nil, # management port server socket
108
+ id: nil, # SessionId from INIT message in mgt port
109
+ env_args: env_args, # env vars and args to ascp (from transfer spec)
110
+ options: job_options # [Hash]
116
111
  }
117
112
 
118
113
  if multi_session_info.nil?
@@ -131,7 +126,7 @@ module Aspera
131
126
  this_session[:env_args][:args]=this_session[:env_args][:args].clone()
132
127
  this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_info[:count]}")
133
128
  # option: increment (default as per ascp manual) or not (cluster on other side ?)
134
- this_session[:env_args][:args].unshift('-O',"#{multi_session_info[:udp_base]+i-1}") if @options[:multi_incr_udp]
129
+ this_session[:env_args][:args].unshift('-O',(multi_session_info[:udp_base]+i-1).to_s) if @options[:multi_incr_udp]
135
130
  this_session[:thread] = Thread.new(this_session) {|s|transfer_thread_entry(s)}
136
131
  xfer_job[:sessions].push(this_session)
137
132
  end
@@ -151,11 +146,11 @@ module Aspera
151
146
  Log.log.debug('wait_for_transfers_completion')
152
147
  # set to non-nil to exit loop
153
148
  result=[]
154
- @jobs.each do |id,job|
149
+ @jobs.each do |_id,job|
155
150
  job[:sessions].each do |session|
156
151
  Log.log.debug("join #{session[:thread]}")
157
152
  session[:thread].join
158
- result.push(session[:error] ? session[:error] : :success)
153
+ result.push(session[:error] || :success)
159
154
  end
160
155
  end
161
156
  Log.log.debug('all transfers joined')
@@ -189,7 +184,7 @@ module Aspera
189
184
  Fasp::Installation.instance.path(env_args[:ascp_version])
190
185
  end
191
186
  # (optional) check it exists
192
- raise Fasp::Error.new("no such file: #{ascp_path}") unless File.exist?(ascp_path)
187
+ raise Fasp::Error, "no such file: #{ascp_path}" unless File.exist?(ascp_path)
193
188
  # open random local TCP port for listening for ascp management
194
189
  mgt_sock = TCPServer.new('127.0.0.1',0)
195
190
  # clone arguments as we eed to modify with mgt port
@@ -197,7 +192,7 @@ module Aspera
197
192
  # add management port
198
193
  ascp_arguments.unshift('-M', mgt_sock.addr[1].to_s)
199
194
  # start ascp in sub process
200
- Log.log.debug("execute: #{env_args[:env].map{|k,v| "#{k}=\"#{v}\""}.join(' ')} \"#{ascp_path}\" \"#{ascp_arguments.join('" "')}\"")
195
+ Log.log.debug("execute: #{env_args[:env].map{|k,v| "#{k}=#{Shellwords.shellescape(v)}"}.join(' ')} #{Shellwords.shellescape(ascp_path)} #{ascp_arguments.map{|a|Shellwords.shellescape(a)}.join(' ')}")
201
196
  # start process
202
197
  ascp_pid = Process.spawn(env_args[:env],[ascp_path,ascp_path],*ascp_arguments)
203
198
  # in parent, wait for connection to socket max 3 seconds
@@ -225,17 +220,17 @@ module Aspera
225
220
  line = ascp_mgt_io.gets
226
221
  # nil when ascp process exits
227
222
  break if line.nil?
228
- current_event_text=current_event_text+line
223
+ current_event_text+=line
229
224
  line.chomp!
230
225
  Log.log.debug("line=[#{line}]")
231
226
  case line
232
227
  when 'FASPMGR 2'
233
228
  # begin event
234
- current_event_data = Hash.new
229
+ current_event_data = {}
235
230
  current_event_text = ''
236
231
  when /^([^:]+): (.*)$/
237
232
  # event field
238
- current_event_data[$1] = $2
233
+ current_event_data[Regexp.last_match(1)] = Regexp.last_match(2)
239
234
  when ''
240
235
  # empty line is separator to end event information
241
236
  raise 'unexpected empty line' if current_event_data.nil?
@@ -261,9 +256,9 @@ module Aspera
261
256
  exception_raised=false
262
257
  when 'ERROR'
263
258
  Log.log.error("code: #{last_status_event['Code']}")
264
- if last_status_event['Description'] =~ /bearer token/i
259
+ if last_status_event['Description'] =~ /bearer token/i
265
260
  Log.log.error('need to regenerate token'.red)
266
- if session[:options].is_a?(Hash) and session[:options].has_key?(:regenerate_token)
261
+ if session[:options].is_a?(Hash) && session[:options].has_key?(:regenerate_token)
267
262
  # regenerate token here, expired, or error on it
268
263
  # Note: in multi-session, each session will have a different one.
269
264
  env_args[:env]['ASPERA_SCP_TOKEN']=session[:options][:regenerate_token].call(true)
@@ -279,27 +274,25 @@ module Aspera
279
274
  end
280
275
  rescue SystemCallError => e
281
276
  # Process.spawn
282
- raise Fasp::Error.new(e.message)
283
- rescue Timeout::Error => e
284
- raise Fasp::Error.new('timeout waiting mgt port connect')
285
- rescue Interrupt => e
286
- raise Fasp::Error.new('transfer interrupted by user')
277
+ raise Fasp::Error, e.message
278
+ rescue Timeout::Error
279
+ raise Fasp::Error, 'timeout waiting mgt port connect'
280
+ rescue Interrupt
281
+ raise Fasp::Error, 'transfer interrupted by user'
287
282
  ensure
288
283
  # if ascp was successfully started
289
284
  unless ascp_pid.nil?
290
285
  # "wait" for process to avoid zombie
291
286
  Process.wait(ascp_pid)
292
- status=$?
287
+ status=$CHILD_STATUS
293
288
  ascp_pid=nil
294
289
  session.delete(:io)
295
290
  if !status.success?
296
291
  message="ascp failed with code #{status.exitstatus}"
297
- if exception_raised
298
- # just debug, as main exception is already here
299
- Log.log.debug(message)
300
- else
301
- raise Fasp::Error.new(message)
302
- end
292
+ # raise error only if there was not already an exception
293
+ raise Fasp::Error, message unless exception_raised
294
+ # else just debug, as main exception is already here
295
+ Log.log.debug(message)
303
296
  end
304
297
  end
305
298
  end # begin-ensure
@@ -341,11 +334,8 @@ module Aspera
341
334
  if !options.nil?
342
335
  raise "expecting Hash (or nil), but have #{options.class}" unless options.is_a?(Hash)
343
336
  options.each do |k,v|
344
- if DEFAULT_OPTIONS.has_key?(k)
345
- @options[k]=v
346
- else
347
- raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map{|i|i.to_s}.join(",")}"
348
- end
337
+ raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.has_key?(k)
338
+ @options[k]=v
349
339
  end
350
340
  end
351
341
  Log.log.debug("local options= #{options}")
@@ -364,13 +354,12 @@ module Aspera
364
354
  start_transfer_with_args_env(session[:env_args],session)
365
355
  end
366
356
  Log.log.debug('transfer ok'.bg_green)
367
- rescue => e
357
+ rescue StandardError => e
368
358
  session[:error]=e
369
359
  Log.log.error("Transfer thread error: #{e.class}:\n#{e.message}:\n#{e.backtrace.join("\n")}".red) if Log.instance.level.eql?(:debug)
370
360
  end
371
361
  Log.log.debug("EXIT (#{Thread.current[:name]})")
372
362
  end
373
-
374
363
  end # AgentDirect
375
364
  end
376
365
  end
@@ -1,5 +1,6 @@
1
- #!/bin/echo this is a ruby class:
1
+ # frozen_string_literal: true
2
2
  require 'aspera/fasp/agent_base'
3
+ require 'aspera/fasp/transfer_spec'
3
4
  require 'aspera/log'
4
5
  require 'aspera/rest'
5
6
  require 'websocket-client-simple'
@@ -29,7 +30,7 @@ module Aspera
29
30
  # we need to keep track of actual file path because transfer spec is modified to be sent in web socket
30
31
  source_paths=[]
31
32
  # get source root or nil
32
- source_root = (transfer_spec.has_key?('source_root') and !transfer_spec['source_root'].empty?) ? transfer_spec['source_root'] : nil
33
+ source_root = transfer_spec.has_key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] : nil
33
34
  # source root is ignored by GW, used only here
34
35
  transfer_spec.delete('source_root')
35
36
  # compute total size of files to upload (for progress)
@@ -61,27 +62,23 @@ module Aspera
61
62
  received+=1
62
63
  else
63
64
  message.chomp!
64
- if message[0].eql?('"') and message[-1].eql?('"')
65
- error=JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message']
66
- else
67
- error="expecting quotes in [#{message}]"
68
- end
65
+ error=message.start_with?('"') && message.end_with?('"') ? JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message'] : "expecting quotes in [#{message}]"
69
66
  end
70
67
  end
71
68
  ws.on :error do |e|
72
69
  error=e
73
70
  end
74
71
  ws.on :open do
75
- Log.log.info("ws: open")
72
+ Log.log.info('ws: open')
76
73
  end
77
74
  ws.on :close do
78
- Log.log.info("ws: close")
75
+ Log.log.info('ws: close')
79
76
  end
80
77
  # open web socket to end point
81
78
  ws.connect("#{@gw_api.params[:base_url]}/upload")
82
79
  # async wait ready
83
- while !ws.open? and error.nil? do
84
- Log.log.info("ws: wait")
80
+ while !ws.open? && error.nil?
81
+ Log.log.info('ws: wait')
85
82
  sleep(0.2)
86
83
  end
87
84
  # notify progress bar
@@ -101,11 +98,11 @@ module Aspera
101
98
  file_size=item['file_size']
102
99
  file_name=File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
103
100
  # compute total number of slices
104
- numslices=1+(file_size-1)/@upload_chunksize
101
+ numslices=1+((file_size-1)/@upload_chunksize)
105
102
  File.open(source_paths[file_index]) do |file|
106
103
  # current slice index
107
104
  slicenum=0
108
- while !file.eof? do
105
+ while !file.eof?
109
106
  data=file.read(@upload_chunksize)
110
107
  slice_data={
111
108
  name: file_name,
@@ -117,11 +114,11 @@ module Aspera
117
114
  fileIndex: file_index
118
115
  }
119
116
  # log without data
120
- Log.dump(:slide_data,slice_data.keys.inject({}){|m,i|m[i]=i.eql?(:data)?'base64 data':slice_data[i];m}) if slicenum.eql?(0)
117
+ Log.dump(:slide_data,slice_data.keys.each_with_object({}){|i,m|m[i]=i.eql?(:data)?'base64 data':slice_data[i];}) if slicenum.eql?(0)
121
118
  ws_send(ws,:slice_upload, slice_data)
122
119
  sent_bytes+=data.length
123
120
  currenttime=Time.now
124
- if lastevent.nil? or (currenttime-lastevent)>UPLOAD_REFRESH_SEC
121
+ if lastevent.nil? || ((currenttime-lastevent)>UPLOAD_REFRESH_SEC)
125
122
  notify_progress(session_id,sent_bytes)
126
123
  lastevent=currenttime
127
124
  end
@@ -146,37 +143,37 @@ module Aspera
146
143
  dname=dname.gsub(/\.@gw_api.*$/,'')
147
144
  # ands add indication of number of files if there is more than one
148
145
  if transfer_spec['paths'].length > 1
149
- dname=dname+" #{transfer_spec['paths'].length} Files"
146
+ dname+=" #{transfer_spec['paths'].length} Files"
150
147
  end
151
148
  transfer_spec['download_name']=dname
152
149
  end
153
150
  creation=@gw_api.create('download',{'transfer_spec'=>transfer_spec})[:data]
154
151
  transfer_uuid=creation['url'].split('/').last
155
- if transfer_spec['zip_required'] or transfer_spec['paths'].length > 1
156
- # it is a zip file if zip is required or there is more than 1 file
157
- file_dest=transfer_spec['download_name']+'.zip'
158
- else
159
- # it is a plain file if we don't require zip and there is only one file
160
- file_dest=File.basename(transfer_spec['paths'].first['source'])
152
+ file_dest=if transfer_spec['zip_required'] || transfer_spec['paths'].length > 1
153
+ # it is a zip file if zip is required or there is more than 1 file
154
+ transfer_spec['download_name']+'.zip'
155
+ else
156
+ # it is a plain file if we don't require zip and there is only one file
157
+ File.basename(transfer_spec['paths'].first['source'])
161
158
  end
162
159
  file_dest=File.join(transfer_spec['destination_root'],file_dest)
163
- @gw_api.call({:operation=>'GET',:subpath=>"download/#{transfer_uuid}",:save_to_file=>file_dest})
160
+ @gw_api.call({operation: 'GET',subpath: "download/#{transfer_uuid}",save_to_file: file_dest})
164
161
  end
165
162
 
166
163
  # start FASP transfer based on transfer spec (hash table)
167
164
  # note that it is asynchronous
168
165
  # HTTP download only supports file list
169
166
  def start_transfer(transfer_spec,options={})
170
- raise "GW URL must be set" unless !@gw_api.nil?
171
- raise "option: must be hash (or nil)" unless options.is_a?(Hash)
172
- raise "paths: must be Array" unless transfer_spec['paths'].is_a?(Array)
173
- raise "only token based transfer is supported in GW" unless transfer_spec['token'].is_a?(String)
167
+ raise 'GW URL must be set' if @gw_api.nil?
168
+ raise 'option: must be hash (or nil)' unless options.is_a?(Hash)
169
+ raise 'paths: must be Array' unless transfer_spec['paths'].is_a?(Array)
170
+ raise 'only token based transfer is supported in GW' unless transfer_spec['token'].is_a?(String)
174
171
  Log.dump(:user_spec,transfer_spec)
175
172
  transfer_spec['authentication']||='token'
176
173
  case transfer_spec['direction']
177
- when 'send'
174
+ when Fasp::TransferSpec::DIRECTION_SEND
178
175
  upload(transfer_spec)
179
- when 'receive'
176
+ when Fasp::TransferSpec::DIRECTION_RECEIVE
180
177
  download(transfer_spec)
181
178
  else
182
179
  raise "unexpected direction: [#{transfer_spec['direction']}]"
@@ -190,25 +187,22 @@ module Aspera
190
187
  end
191
188
 
192
189
  # terminates monitor thread
193
- def shutdown
194
- end
190
+ def shutdown; end
195
191
 
196
- def url=(api_url)
197
- end
192
+ def url=(api_url); end
198
193
 
199
194
  private
200
195
 
201
196
  def initialize(params)
202
- raise "params must be Hash" unless params.is_a?(Hash)
197
+ raise 'params must be Hash' unless params.is_a?(Hash)
203
198
  params=params.symbolize_keys
204
- raise "must have only one param: url" unless params.keys.eql?([:url])
199
+ raise 'must have only one param: url' unless params.keys.eql?([:url])
205
200
  super()
206
- @gw_api=Rest.new({:base_url => params[:url]})
201
+ @gw_api=Rest.new({base_url: params[:url]})
207
202
  api_info = @gw_api.read('info')[:data]
208
- Log.log.info("#{api_info}")
209
- @upload_chunksize=128000 # TODO: configurable ?
203
+ Log.log.info(api_info.to_s)
204
+ @upload_chunksize=128_000 # TODO: configurable ?
210
205
  end
211
-
212
206
  end # AgentHttpgw
213
207
  end
214
208
  end