aspera-cli 4.6.0 → 4.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +427 -300
  3. data/bin/ascli +2 -1
  4. data/bin/asession +1 -0
  5. data/docs/test_env.conf +2 -0
  6. data/examples/aoc.rb +4 -3
  7. data/examples/faspex4.rb +21 -19
  8. data/examples/proxy.pac +1 -1
  9. data/examples/transfer.rb +15 -15
  10. data/lib/aspera/aoc.rb +135 -124
  11. data/lib/aspera/ascmd.rb +85 -75
  12. data/lib/aspera/ats_api.rb +11 -10
  13. data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
  14. data/lib/aspera/cli/extended_value.rb +42 -33
  15. data/lib/aspera/cli/formater.rb +138 -111
  16. data/lib/aspera/cli/info.rb +17 -0
  17. data/lib/aspera/cli/listener/line_dump.rb +3 -2
  18. data/lib/aspera/cli/listener/logger.rb +2 -1
  19. data/lib/aspera/cli/listener/progress.rb +16 -18
  20. data/lib/aspera/cli/listener/progress_multi.rb +13 -16
  21. data/lib/aspera/cli/main.rb +122 -130
  22. data/lib/aspera/cli/manager.rb +146 -154
  23. data/lib/aspera/cli/plugin.rb +38 -34
  24. data/lib/aspera/cli/plugins/alee.rb +6 -6
  25. data/lib/aspera/cli/plugins/aoc.rb +273 -276
  26. data/lib/aspera/cli/plugins/ats.rb +82 -76
  27. data/lib/aspera/cli/plugins/bss.rb +14 -16
  28. data/lib/aspera/cli/plugins/config.rb +350 -306
  29. data/lib/aspera/cli/plugins/console.rb +23 -19
  30. data/lib/aspera/cli/plugins/cos.rb +18 -18
  31. data/lib/aspera/cli/plugins/faspex.rb +180 -159
  32. data/lib/aspera/cli/plugins/faspex5.rb +64 -54
  33. data/lib/aspera/cli/plugins/node.rb +147 -140
  34. data/lib/aspera/cli/plugins/orchestrator.rb +68 -66
  35. data/lib/aspera/cli/plugins/preview.rb +92 -96
  36. data/lib/aspera/cli/plugins/server.rb +79 -75
  37. data/lib/aspera/cli/plugins/shares.rb +23 -24
  38. data/lib/aspera/cli/plugins/sync.rb +20 -22
  39. data/lib/aspera/cli/transfer_agent.rb +40 -39
  40. data/lib/aspera/cli/version.rb +2 -1
  41. data/lib/aspera/colors.rb +35 -27
  42. data/lib/aspera/command_line_builder.rb +48 -34
  43. data/lib/aspera/cos_node.rb +29 -21
  44. data/lib/aspera/data_repository.rb +3 -2
  45. data/lib/aspera/environment.rb +50 -45
  46. data/lib/aspera/fasp/agent_base.rb +22 -20
  47. data/lib/aspera/fasp/agent_connect.rb +13 -11
  48. data/lib/aspera/fasp/agent_direct.rb +48 -59
  49. data/lib/aspera/fasp/agent_httpgw.rb +33 -39
  50. data/lib/aspera/fasp/agent_node.rb +15 -13
  51. data/lib/aspera/fasp/agent_trsdk.rb +12 -14
  52. data/lib/aspera/fasp/error.rb +2 -1
  53. data/lib/aspera/fasp/error_info.rb +68 -52
  54. data/lib/aspera/fasp/installation.rb +106 -94
  55. data/lib/aspera/fasp/listener.rb +1 -0
  56. data/lib/aspera/fasp/parameters.rb +83 -92
  57. data/lib/aspera/fasp/parameters.yaml +305 -249
  58. data/lib/aspera/fasp/resume_policy.rb +11 -14
  59. data/lib/aspera/fasp/transfer_spec.rb +26 -0
  60. data/lib/aspera/fasp/uri.rb +22 -21
  61. data/lib/aspera/faspex_gw.rb +55 -90
  62. data/lib/aspera/hash_ext.rb +4 -3
  63. data/lib/aspera/id_generator.rb +8 -7
  64. data/lib/aspera/keychain/encrypted_hash.rb +17 -16
  65. data/lib/aspera/keychain/macos_security.rb +6 -10
  66. data/lib/aspera/log.rb +25 -20
  67. data/lib/aspera/nagios.rb +13 -12
  68. data/lib/aspera/node.rb +30 -22
  69. data/lib/aspera/oauth.rb +175 -226
  70. data/lib/aspera/open_application.rb +4 -3
  71. data/lib/aspera/persistency_action_once.rb +6 -6
  72. data/lib/aspera/persistency_folder.rb +5 -9
  73. data/lib/aspera/preview/file_types.rb +6 -5
  74. data/lib/aspera/preview/generator.rb +25 -24
  75. data/lib/aspera/preview/options.rb +16 -14
  76. data/lib/aspera/preview/utils.rb +98 -98
  77. data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
  78. data/lib/aspera/proxy_auto_config.rb +111 -20
  79. data/lib/aspera/rest.rb +115 -113
  80. data/lib/aspera/rest_call_error.rb +2 -2
  81. data/lib/aspera/rest_error_analyzer.rb +23 -25
  82. data/lib/aspera/rest_errors_aspera.rb +15 -14
  83. data/lib/aspera/ssh.rb +12 -10
  84. data/lib/aspera/sync.rb +42 -41
  85. data/lib/aspera/temp_file_manager.rb +18 -14
  86. data/lib/aspera/timer_limiter.rb +2 -1
  87. data/lib/aspera/uri_reader.rb +7 -5
  88. data/lib/aspera/web_auth.rb +79 -76
  89. metadata +64 -21
  90. data/docs/Makefile +0 -65
  91. data/docs/README.erb.md +0 -4424
  92. data/docs/README.md +0 -13
  93. data/docs/diagrams.txt +0 -49
  94. data/docs/doc_tools.rb +0 -58
  95. data/lib/aspera/cli/plugins/shares2.rb +0 -114
  96. data/lib/aspera/fasp/default.rb +0 -17
@@ -1,30 +1,32 @@
1
+ # frozen_string_literal: true
1
2
  module Aspera
2
3
  module Fasp
3
4
  # Base class for FASP transfer agents
4
5
  # sub classes shall implement start_transfer and shutdown
5
6
  class AgentBase
7
+ # fields description for JSON generation
8
+ INTEGER_FIELDS=%w[Bytescont FaspFileArgIndex StartByte Rate MinRate Port Priority RateCap MinRateCap TCPPort CreatePolicy TimePolicy DatagramSize XoptFlags VLinkVersion
9
+ PeerVLinkVersion DSPipelineDepth PeerDSPipelineDepth ReadBlockSize WriteBlockSize ClusterNumNodes ClusterNodeId Size Written Loss FileBytes PreTransferBytes TransferBytes PMTU Elapsedusec ArgScansAttempted ArgScansCompleted PathScansAttempted FileScansCompleted TransfersAttempted TransfersPassed Delay].freeze
10
+ BOOLEAN_FIELDS=%w[Encryption Remote RateLock MinRateLock PolicyLock FilesEncrypt FilesDecrypt VLinkLocalEnabled VLinkRemoteEnabled MoveRange Keepalive TestLogin UseProxy
11
+ Precalc RTTAutocorrect].freeze
12
+ EXPECTED_METHODS=%i[text struct enhanced].freeze
13
+ private_constant :INTEGER_FIELDS,:BOOLEAN_FIELDS,:EXPECTED_METHODS
6
14
 
7
15
  private
8
16
 
9
- # fields description for JSON generation
10
- IntegerFields=['Bytescont','FaspFileArgIndex','StartByte','Rate','MinRate','Port','Priority','RateCap','MinRateCap','TCPPort','CreatePolicy','TimePolicy','DatagramSize','XoptFlags','VLinkVersion','PeerVLinkVersion','DSPipelineDepth','PeerDSPipelineDepth','ReadBlockSize','WriteBlockSize','ClusterNumNodes','ClusterNodeId','Size','Written','Loss','FileBytes','PreTransferBytes','TransferBytes','PMTU','Elapsedusec','ArgScansAttempted','ArgScansCompleted','PathScansAttempted','FileScansCompleted','TransfersAttempted','TransfersPassed','Delay']
11
- BooleanFields=['Encryption','Remote','RateLock','MinRateLock','PolicyLock','FilesEncrypt','FilesDecrypt','VLinkLocalEnabled','VLinkRemoteEnabled','MoveRange','Keepalive','TestLogin','UseProxy','Precalc','RTTAutocorrect']
12
- ExpectedMethod=[:text,:struct,:enhanced]
13
-
14
17
  # translates legacy event into enhanced (JSON) event
15
18
  def enhanced_event_format(event)
16
- return event.keys.inject({}) do |h,e|
19
+ return event.keys.each_with_object({}) do |e,h|
17
20
  # capital_to_snake_case
18
21
  new_name=e.
19
- gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
20
- gsub(/([a-z\d])([A-Z])/,'\1_\2').
21
- gsub(/([a-z\d])(usec)$/,'\1_\2').
22
- downcase
22
+ gsub(/([a-z\d])([A-Z])/,'\1_\2').
23
+ gsub(/([a-z\d])(usec)$/,'\1_\2').
24
+ gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
25
+ downcase
23
26
  value=event[e]
24
- value=value.to_i if IntegerFields.include?(e)
25
- value=value.eql?('Yes') ? true : false if BooleanFields.include?(e)
27
+ value=value.to_i if INTEGER_FIELDS.include?(e)
28
+ value=value.eql?('Yes') if BOOLEAN_FIELDS.include?(e)
26
29
  h[new_name]=value
27
- h
28
30
  end
29
31
  end
30
32
 
@@ -33,14 +35,14 @@ module Aspera
33
35
  end
34
36
 
35
37
  def notify_listeners(current_event_text,current_event_data)
36
- Log.log.debug("send event to listeners")
38
+ Log.log.debug('send event to listeners')
37
39
  enhanced_event=nil
38
40
  @listeners.each do |listener|
39
- listener.send(:event_text,current_event_text) if listener.respond_to?(:event_text)
40
- listener.send(:event_struct,current_event_data) if listener.respond_to?(:event_struct)
41
+ listener.event_text(current_event_text) if listener.respond_to?(:event_text)
42
+ listener.event_struct(current_event_data) if listener.respond_to?(:event_struct)
41
43
  if listener.respond_to?(:event_enhanced)
42
44
  enhanced_event=enhanced_event_format(current_event_data) if enhanced_event.nil?
43
- listener.send(:event_enhanced,enhanced_event)
45
+ listener.event_enhanced(enhanced_event)
44
46
  end
45
47
  end
46
48
  end # notify_listeners
@@ -58,12 +60,13 @@ module Aspera
58
60
  end
59
61
 
60
62
  public
63
+
61
64
  LISTENER_SESSION_ID_B='ListenerSessionId'
62
65
  LISTENER_SESSION_ID_S='listener_session_id'
63
66
 
64
67
  # listener receives events
65
68
  def add_listener(listener)
66
- raise "expect one of #{ExpectedMethod}" if ExpectedMethod.inject(0){|m,e|m+=listener.respond_to?("event_#{e}")?1:0;m}.eql?(0)
69
+ raise "expect one of #{EXPECTED_METHODS}" if EXPECTED_METHODS.inject(0){|m,e|m+=listener.respond_to?("event_#{e}")?1:0;m}.eql?(0)
67
70
  @listeners.push(listener)
68
71
  self
69
72
  end
@@ -72,14 +75,13 @@ module Aspera
72
75
  # it must be a list of :success or exception
73
76
  def self.validate_status_list(statuses)
74
77
  raise "internal error: bad statuses type: #{statuses.class}" unless statuses.is_a?(Array)
75
- raise "internal error: bad statuses content: #{statuses}" unless statuses.select{|i|!i.eql?(:success) and !i.is_a?(StandardError)}.empty?
78
+ raise "internal error: bad statuses content: #{statuses}" unless statuses.select{|i|!i.eql?(:success) && !i.is_a?(StandardError)}.empty?
76
79
  end
77
80
 
78
81
  # the following methods must be implemented by subclass:
79
82
  # start_transfer(transfer_spec,options) : start and wait for completion
80
83
  # wait_for_transfers_completion : wait for termination of all transfers, @return list of : :success or error message
81
84
  # optional: shutdown
82
-
83
85
  end
84
86
  end
85
87
  end
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  require 'aspera/fasp/agent_base'
2
3
  require 'aspera/rest'
3
4
  require 'aspera/open_application'
@@ -10,7 +11,7 @@ module Aspera
10
11
  MAX_CONNECT_START_RETRY=3
11
12
  SLEEP_SEC_BETWEEN_RETRY=2
12
13
  private_constant :MAX_CONNECT_START_RETRY,:SLEEP_SEC_BETWEEN_RETRY
13
- def initialize(options)
14
+ def initialize(_options)
14
15
  super()
15
16
  @connect_settings={
16
17
  'app_id' => SecureRandom.uuid
@@ -23,7 +24,7 @@ module Aspera
23
24
  @connect_api=Rest.new({base_url: "#{connect_url}/v5/connect",headers: {'Origin'=>Rest.user_agent}}) # could use v6 also now
24
25
  cinfo=@connect_api.read('info/version')[:data]
25
26
  Log.dump(:connect_version,cinfo)
26
- rescue => e # Errno::ECONNREFUSED
27
+ rescue StandardError => e # Errno::ECONNREFUSED
27
28
  raise StandardError,"Unable to start connect after #{trynumber} try" if trynumber >= MAX_CONNECT_START_RETRY
28
29
  Log.log.warn("connect is not started. Retry ##{trynumber}, err=#{e}")
29
30
  trynumber+=1
@@ -36,11 +37,12 @@ module Aspera
36
37
  end
37
38
  end
38
39
 
39
- def start_transfer(transfer_spec,options=nil)
40
+ def start_transfer(transfer_spec,_options=nil)
40
41
  if transfer_spec['direction'] == 'send'
41
42
  Log.log.warn("Connect requires upload selection using GUI, ignoring #{transfer_spec['paths']}".red)
42
43
  transfer_spec.delete('paths')
43
- resdata=@connect_api.create('windows/select-open-file-dialog/',{'aspera_connect_settings'=>@connect_settings,'title'=>'Select Files','suggestedName'=>'','allowMultipleSelection'=>true,'allowedFileTypes'=>''})[:data]
44
+ resdata=@connect_api.create('windows/select-open-file-dialog/',
45
+ {'aspera_connect_settings'=>@connect_settings,'title'=>'Select Files','suggestedName'=>'','allowMultipleSelection'=>true,'allowedFileTypes'=>''})[:data]
44
46
  transfer_spec['paths']=resdata['dataTransfer']['files'].map { |i| {'source'=>i['name']}}
45
47
  end
46
48
  @request_id=SecureRandom.uuid
@@ -50,10 +52,10 @@ module Aspera
50
52
  connect_transfer_args={
51
53
  'aspera_connect_settings'=>@connect_settings.merge({
52
54
  'request_id' =>@request_id,
53
- 'allow_dialogs' =>true,
55
+ 'allow_dialogs' =>true
54
56
  }),
55
57
  'transfer_specs' =>[{
56
- 'transfer_spec' =>transfer_spec,
58
+ 'transfer_spec' =>transfer_spec
57
59
  }]}
58
60
  # asynchronous anyway
59
61
  res=@connect_api.create('transfers/start',connect_transfer_args)[:data]
@@ -70,7 +72,7 @@ module Aspera
70
72
  if tr_info['transfer_info'].is_a?(Hash)
71
73
  trdata=tr_info['transfer_info']
72
74
  if trdata.nil?
73
- Log.log.warn("no session in Connect")
75
+ Log.log.warn('no session in Connect')
74
76
  break
75
77
  end
76
78
  # TODO: get session id
@@ -87,7 +89,7 @@ module Aspera
87
89
  spinner.spin
88
90
  when 'running'
89
91
  #puts "running: sessions:#{trdata['sessions'].length}, #{trdata['sessions'].map{|i| i['bytes_transferred']}.join(',')}"
90
- if !started and trdata['bytes_expected'] != 0
92
+ if !started && (trdata['bytes_expected'] != 0)
91
93
  spinner.success unless spinner.nil?
92
94
  notify_begin(@connect_settings['app_id'],trdata['bytes_expected'])
93
95
  started=true
@@ -96,14 +98,14 @@ module Aspera
96
98
  end
97
99
  when 'failed'
98
100
  spinner.error unless spinner.nil?
99
- raise Fasp::Error.new(trdata['error_desc'])
101
+ raise Fasp::Error, trdata['error_desc']
100
102
  else
101
- raise Fasp::Error.new("unknown status: #{trdata['status']}: #{trdata['error_desc']}")
103
+ raise Fasp::Error, "unknown status: #{trdata['status']}: #{trdata['error_desc']}"
102
104
  end
103
105
  end
104
106
  sleep 1
105
107
  end
106
- rescue => e
108
+ rescue StandardError => e
107
109
  return [e]
108
110
  end
109
111
  return [:success]
@@ -1,20 +1,16 @@
1
- #!/bin/echo this is a ruby class:
2
- #
3
- # FASP manager for Ruby
4
- # Aspera 2016
5
- # Laurent Martin
6
- #
7
- ##############################################################################
1
+ # frozen_string_literal: true
2
+ require 'English'
8
3
  require 'aspera/fasp/agent_base'
9
4
  require 'aspera/fasp/error'
10
5
  require 'aspera/fasp/parameters'
11
6
  require 'aspera/fasp/installation'
12
7
  require 'aspera/fasp/resume_policy'
13
- require 'aspera/fasp/default'
8
+ require 'aspera/fasp/transfer_spec'
14
9
  require 'aspera/log'
15
10
  require 'socket'
16
11
  require 'timeout'
17
12
  require 'securerandom'
13
+ require 'shellwords'
18
14
 
19
15
  module Aspera
20
16
  module Fasp
@@ -27,7 +23,7 @@ module Aspera
27
23
  wss: false,
28
24
  multi_incr_udp: true,
29
25
  resume: {},
30
- quiet: true # by default no interactive progress bar
26
+ quiet: true # by default no interactive progress bar
31
27
  }
32
28
  private_constant :DEFAULT_OPTIONS
33
29
 
@@ -43,10 +39,11 @@ module Aspera
43
39
  # clone transfer spec because we modify it (first level keys)
44
40
  transfer_spec=transfer_spec.clone
45
41
  # if there is aspera tags
46
- if transfer_spec['tags'].is_a?(Hash) and transfer_spec['tags']['aspera'].is_a?(Hash)
42
+ if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags']['aspera'].is_a?(Hash)
47
43
  # TODO: what is this for ? only on local ascp ?
48
44
  # NOTE: important: transfer id must be unique: generate random id
49
45
  # using a non unique id results in discard of tags in AoC, and a package is never finalized
46
+ # all sessions in a multi-session transfer must have the same xfer_id (see admin manual)
50
47
  transfer_spec['tags']['aspera']['xfer_id']||=SecureRandom.uuid
51
48
  Log.log.debug("xfer id=#{transfer_spec['xfer_id']}")
52
49
  # TODO: useful ? node only ?
@@ -55,8 +52,8 @@ module Aspera
55
52
  Log.dump('ts',transfer_spec)
56
53
 
57
54
  # add bypass keys when authentication is token and no auth is provided
58
- if transfer_spec.has_key?('token') and
59
- !transfer_spec.has_key?('remote_password') and
55
+ if transfer_spec.has_key?('token') &&
56
+ !transfer_spec.has_key?('remote_password') &&
60
57
  !transfer_spec.has_key?('EX_ssh_key_paths')
61
58
  # transfer_spec['remote_password'] = Installation.instance.bypass_pass # not used
62
59
  transfer_spec['EX_ssh_key_paths'] = Installation.instance.bypass_keys
@@ -67,24 +64,22 @@ module Aspera
67
64
  multi_session_info=nil
68
65
  if transfer_spec.has_key?('multi_session')
69
66
  multi_session_info={
70
- count: transfer_spec['multi_session'].to_i,
67
+ count: transfer_spec['multi_session'].to_i
71
68
  }
72
69
  # Managed by multi-session, so delete from transfer spec
73
70
  transfer_spec.delete('multi_session')
74
- if multi_session_info[:count] < 0
71
+ if multi_session_info[:count].negative?
75
72
  Log.log.error("multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0")
76
73
  multi_session_info = nil
77
74
  elsif multi_session_info[:count].eql?(0)
78
- Log.log.debug("multi_session count is zero: no multisession")
75
+ Log.log.debug('multi_session count is zero: no multisession')
79
76
  multi_session_info = nil
80
- else # multi_session_info[:count] > 0
77
+ elsif @options[:multi_incr_udp] # multi_session_info[:count] > 0
81
78
  # if option not true: keep default udp port for all sessions
82
- if @options[:multi_incr_udp]
83
- # override if specified, else use default value
84
- multi_session_info[:udp_base]=transfer_spec.has_key?('fasp_port') ? transfer_spec['fasp_port'] : Default::UDP_PORT
85
- # delete from original transfer spec, as we will increment values
86
- transfer_spec.delete('fasp_port')
87
- end
79
+ multi_session_info[:udp_base]=transfer_spec.has_key?('fasp_port') ? transfer_spec['fasp_port'] : TransferSpec::UDP_PORT
80
+ # delete from original transfer spec, as we will increment values
81
+ transfer_spec.delete('fasp_port')
82
+ # override if specified, else use default value
88
83
  end
89
84
  end
90
85
 
@@ -101,18 +96,18 @@ module Aspera
101
96
 
102
97
  # transfer job can be multi session
103
98
  xfer_job={
104
- :id => job_options[:job_id],
105
- :sessions => [] # all sessions as below
99
+ id: job_options[:job_id],
100
+ sessions: [] # all sessions as below
106
101
  }
107
102
 
108
103
  # generic session information
109
104
  session={
110
- :thread => nil, # Thread object monitoring management port, not nil when pushed to :sessions
111
- :error => nil, # exception if failed
112
- :io => nil, # management port server socket
113
- :id => nil, # SessionId from INIT message in mgt port
114
- :env_args => env_args, # env vars and args to ascp (from transfer spec)
115
- :options => job_options # [Hash]
105
+ thread: nil, # Thread object monitoring management port, not nil when pushed to :sessions
106
+ error: nil, # exception if failed
107
+ io: nil, # management port server socket
108
+ id: nil, # SessionId from INIT message in mgt port
109
+ env_args: env_args, # env vars and args to ascp (from transfer spec)
110
+ options: job_options # [Hash]
116
111
  }
117
112
 
118
113
  if multi_session_info.nil?
@@ -131,7 +126,7 @@ module Aspera
131
126
  this_session[:env_args][:args]=this_session[:env_args][:args].clone()
132
127
  this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_info[:count]}")
133
128
  # option: increment (default as per ascp manual) or not (cluster on other side ?)
134
- this_session[:env_args][:args].unshift('-O',"#{multi_session_info[:udp_base]+i-1}") if @options[:multi_incr_udp]
129
+ this_session[:env_args][:args].unshift('-O',(multi_session_info[:udp_base]+i-1).to_s) if @options[:multi_incr_udp]
135
130
  this_session[:thread] = Thread.new(this_session) {|s|transfer_thread_entry(s)}
136
131
  xfer_job[:sessions].push(this_session)
137
132
  end
@@ -151,11 +146,11 @@ module Aspera
151
146
  Log.log.debug('wait_for_transfers_completion')
152
147
  # set to non-nil to exit loop
153
148
  result=[]
154
- @jobs.each do |id,job|
149
+ @jobs.each do |_id,job|
155
150
  job[:sessions].each do |session|
156
151
  Log.log.debug("join #{session[:thread]}")
157
152
  session[:thread].join
158
- result.push(session[:error] ? session[:error] : :success)
153
+ result.push(session[:error] || :success)
159
154
  end
160
155
  end
161
156
  Log.log.debug('all transfers joined')
@@ -189,7 +184,7 @@ module Aspera
189
184
  Fasp::Installation.instance.path(env_args[:ascp_version])
190
185
  end
191
186
  # (optional) check it exists
192
- raise Fasp::Error.new("no such file: #{ascp_path}") unless File.exist?(ascp_path)
187
+ raise Fasp::Error, "no such file: #{ascp_path}" unless File.exist?(ascp_path)
193
188
  # open random local TCP port for listening for ascp management
194
189
  mgt_sock = TCPServer.new('127.0.0.1',0)
195
190
  # clone arguments as we eed to modify with mgt port
@@ -197,7 +192,7 @@ module Aspera
197
192
  # add management port
198
193
  ascp_arguments.unshift('-M', mgt_sock.addr[1].to_s)
199
194
  # start ascp in sub process
200
- Log.log.debug("execute: #{env_args[:env].map{|k,v| "#{k}=\"#{v}\""}.join(' ')} \"#{ascp_path}\" \"#{ascp_arguments.join('" "')}\"")
195
+ Log.log.debug("execute: #{env_args[:env].map{|k,v| "#{k}=#{Shellwords.shellescape(v)}"}.join(' ')} #{Shellwords.shellescape(ascp_path)} #{ascp_arguments.map{|a|Shellwords.shellescape(a)}.join(' ')}")
201
196
  # start process
202
197
  ascp_pid = Process.spawn(env_args[:env],[ascp_path,ascp_path],*ascp_arguments)
203
198
  # in parent, wait for connection to socket max 3 seconds
@@ -225,17 +220,17 @@ module Aspera
225
220
  line = ascp_mgt_io.gets
226
221
  # nil when ascp process exits
227
222
  break if line.nil?
228
- current_event_text=current_event_text+line
223
+ current_event_text+=line
229
224
  line.chomp!
230
225
  Log.log.debug("line=[#{line}]")
231
226
  case line
232
227
  when 'FASPMGR 2'
233
228
  # begin event
234
- current_event_data = Hash.new
229
+ current_event_data = {}
235
230
  current_event_text = ''
236
231
  when /^([^:]+): (.*)$/
237
232
  # event field
238
- current_event_data[$1] = $2
233
+ current_event_data[Regexp.last_match(1)] = Regexp.last_match(2)
239
234
  when ''
240
235
  # empty line is separator to end event information
241
236
  raise 'unexpected empty line' if current_event_data.nil?
@@ -261,9 +256,9 @@ module Aspera
261
256
  exception_raised=false
262
257
  when 'ERROR'
263
258
  Log.log.error("code: #{last_status_event['Code']}")
264
- if last_status_event['Description'] =~ /bearer token/i
259
+ if last_status_event['Description'] =~ /bearer token/i
265
260
  Log.log.error('need to regenerate token'.red)
266
- if session[:options].is_a?(Hash) and session[:options].has_key?(:regenerate_token)
261
+ if session[:options].is_a?(Hash) && session[:options].has_key?(:regenerate_token)
267
262
  # regenerate token here, expired, or error on it
268
263
  # Note: in multi-session, each session will have a different one.
269
264
  env_args[:env]['ASPERA_SCP_TOKEN']=session[:options][:regenerate_token].call(true)
@@ -279,27 +274,25 @@ module Aspera
279
274
  end
280
275
  rescue SystemCallError => e
281
276
  # Process.spawn
282
- raise Fasp::Error.new(e.message)
283
- rescue Timeout::Error => e
284
- raise Fasp::Error.new('timeout waiting mgt port connect')
285
- rescue Interrupt => e
286
- raise Fasp::Error.new('transfer interrupted by user')
277
+ raise Fasp::Error, e.message
278
+ rescue Timeout::Error
279
+ raise Fasp::Error, 'timeout waiting mgt port connect'
280
+ rescue Interrupt
281
+ raise Fasp::Error, 'transfer interrupted by user'
287
282
  ensure
288
283
  # if ascp was successfully started
289
284
  unless ascp_pid.nil?
290
285
  # "wait" for process to avoid zombie
291
286
  Process.wait(ascp_pid)
292
- status=$?
287
+ status=$CHILD_STATUS
293
288
  ascp_pid=nil
294
289
  session.delete(:io)
295
290
  if !status.success?
296
291
  message="ascp failed with code #{status.exitstatus}"
297
- if exception_raised
298
- # just debug, as main exception is already here
299
- Log.log.debug(message)
300
- else
301
- raise Fasp::Error.new(message)
302
- end
292
+ # raise error only if there was not already an exception
293
+ raise Fasp::Error, message unless exception_raised
294
+ # else just debug, as main exception is already here
295
+ Log.log.debug(message)
303
296
  end
304
297
  end
305
298
  end # begin-ensure
@@ -341,11 +334,8 @@ module Aspera
341
334
  if !options.nil?
342
335
  raise "expecting Hash (or nil), but have #{options.class}" unless options.is_a?(Hash)
343
336
  options.each do |k,v|
344
- if DEFAULT_OPTIONS.has_key?(k)
345
- @options[k]=v
346
- else
347
- raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map{|i|i.to_s}.join(",")}"
348
- end
337
+ raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.has_key?(k)
338
+ @options[k]=v
349
339
  end
350
340
  end
351
341
  Log.log.debug("local options= #{options}")
@@ -364,13 +354,12 @@ module Aspera
364
354
  start_transfer_with_args_env(session[:env_args],session)
365
355
  end
366
356
  Log.log.debug('transfer ok'.bg_green)
367
- rescue => e
357
+ rescue StandardError => e
368
358
  session[:error]=e
369
359
  Log.log.error("Transfer thread error: #{e.class}:\n#{e.message}:\n#{e.backtrace.join("\n")}".red) if Log.instance.level.eql?(:debug)
370
360
  end
371
361
  Log.log.debug("EXIT (#{Thread.current[:name]})")
372
362
  end
373
-
374
363
  end # AgentDirect
375
364
  end
376
365
  end
@@ -1,5 +1,6 @@
1
- #!/bin/echo this is a ruby class:
1
+ # frozen_string_literal: true
2
2
  require 'aspera/fasp/agent_base'
3
+ require 'aspera/fasp/transfer_spec'
3
4
  require 'aspera/log'
4
5
  require 'aspera/rest'
5
6
  require 'websocket-client-simple'
@@ -29,7 +30,7 @@ module Aspera
29
30
  # we need to keep track of actual file path because transfer spec is modified to be sent in web socket
30
31
  source_paths=[]
31
32
  # get source root or nil
32
- source_root = (transfer_spec.has_key?('source_root') and !transfer_spec['source_root'].empty?) ? transfer_spec['source_root'] : nil
33
+ source_root = transfer_spec.has_key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] : nil
33
34
  # source root is ignored by GW, used only here
34
35
  transfer_spec.delete('source_root')
35
36
  # compute total size of files to upload (for progress)
@@ -61,27 +62,23 @@ module Aspera
61
62
  received+=1
62
63
  else
63
64
  message.chomp!
64
- if message[0].eql?('"') and message[-1].eql?('"')
65
- error=JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message']
66
- else
67
- error="expecting quotes in [#{message}]"
68
- end
65
+ error=message.start_with?('"') && message.end_with?('"') ? JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message'] : "expecting quotes in [#{message}]"
69
66
  end
70
67
  end
71
68
  ws.on :error do |e|
72
69
  error=e
73
70
  end
74
71
  ws.on :open do
75
- Log.log.info("ws: open")
72
+ Log.log.info('ws: open')
76
73
  end
77
74
  ws.on :close do
78
- Log.log.info("ws: close")
75
+ Log.log.info('ws: close')
79
76
  end
80
77
  # open web socket to end point
81
78
  ws.connect("#{@gw_api.params[:base_url]}/upload")
82
79
  # async wait ready
83
- while !ws.open? and error.nil? do
84
- Log.log.info("ws: wait")
80
+ while !ws.open? && error.nil?
81
+ Log.log.info('ws: wait')
85
82
  sleep(0.2)
86
83
  end
87
84
  # notify progress bar
@@ -101,11 +98,11 @@ module Aspera
101
98
  file_size=item['file_size']
102
99
  file_name=File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
103
100
  # compute total number of slices
104
- numslices=1+(file_size-1)/@upload_chunksize
101
+ numslices=1+((file_size-1)/@upload_chunksize)
105
102
  File.open(source_paths[file_index]) do |file|
106
103
  # current slice index
107
104
  slicenum=0
108
- while !file.eof? do
105
+ while !file.eof?
109
106
  data=file.read(@upload_chunksize)
110
107
  slice_data={
111
108
  name: file_name,
@@ -117,11 +114,11 @@ module Aspera
117
114
  fileIndex: file_index
118
115
  }
119
116
  # log without data
120
- Log.dump(:slide_data,slice_data.keys.inject({}){|m,i|m[i]=i.eql?(:data)?'base64 data':slice_data[i];m}) if slicenum.eql?(0)
117
+ Log.dump(:slide_data,slice_data.keys.each_with_object({}){|i,m|m[i]=i.eql?(:data)?'base64 data':slice_data[i];}) if slicenum.eql?(0)
121
118
  ws_send(ws,:slice_upload, slice_data)
122
119
  sent_bytes+=data.length
123
120
  currenttime=Time.now
124
- if lastevent.nil? or (currenttime-lastevent)>UPLOAD_REFRESH_SEC
121
+ if lastevent.nil? || ((currenttime-lastevent)>UPLOAD_REFRESH_SEC)
125
122
  notify_progress(session_id,sent_bytes)
126
123
  lastevent=currenttime
127
124
  end
@@ -146,37 +143,37 @@ module Aspera
146
143
  dname=dname.gsub(/\.@gw_api.*$/,'')
147
144
  # ands add indication of number of files if there is more than one
148
145
  if transfer_spec['paths'].length > 1
149
- dname=dname+" #{transfer_spec['paths'].length} Files"
146
+ dname+=" #{transfer_spec['paths'].length} Files"
150
147
  end
151
148
  transfer_spec['download_name']=dname
152
149
  end
153
150
  creation=@gw_api.create('download',{'transfer_spec'=>transfer_spec})[:data]
154
151
  transfer_uuid=creation['url'].split('/').last
155
- if transfer_spec['zip_required'] or transfer_spec['paths'].length > 1
156
- # it is a zip file if zip is required or there is more than 1 file
157
- file_dest=transfer_spec['download_name']+'.zip'
158
- else
159
- # it is a plain file if we don't require zip and there is only one file
160
- file_dest=File.basename(transfer_spec['paths'].first['source'])
152
+ file_dest=if transfer_spec['zip_required'] || transfer_spec['paths'].length > 1
153
+ # it is a zip file if zip is required or there is more than 1 file
154
+ transfer_spec['download_name']+'.zip'
155
+ else
156
+ # it is a plain file if we don't require zip and there is only one file
157
+ File.basename(transfer_spec['paths'].first['source'])
161
158
  end
162
159
  file_dest=File.join(transfer_spec['destination_root'],file_dest)
163
- @gw_api.call({:operation=>'GET',:subpath=>"download/#{transfer_uuid}",:save_to_file=>file_dest})
160
+ @gw_api.call({operation: 'GET',subpath: "download/#{transfer_uuid}",save_to_file: file_dest})
164
161
  end
165
162
 
166
163
  # start FASP transfer based on transfer spec (hash table)
167
164
  # note that it is asynchronous
168
165
  # HTTP download only supports file list
169
166
  def start_transfer(transfer_spec,options={})
170
- raise "GW URL must be set" unless !@gw_api.nil?
171
- raise "option: must be hash (or nil)" unless options.is_a?(Hash)
172
- raise "paths: must be Array" unless transfer_spec['paths'].is_a?(Array)
173
- raise "only token based transfer is supported in GW" unless transfer_spec['token'].is_a?(String)
167
+ raise 'GW URL must be set' if @gw_api.nil?
168
+ raise 'option: must be hash (or nil)' unless options.is_a?(Hash)
169
+ raise 'paths: must be Array' unless transfer_spec['paths'].is_a?(Array)
170
+ raise 'only token based transfer is supported in GW' unless transfer_spec['token'].is_a?(String)
174
171
  Log.dump(:user_spec,transfer_spec)
175
172
  transfer_spec['authentication']||='token'
176
173
  case transfer_spec['direction']
177
- when 'send'
174
+ when Fasp::TransferSpec::DIRECTION_SEND
178
175
  upload(transfer_spec)
179
- when 'receive'
176
+ when Fasp::TransferSpec::DIRECTION_RECEIVE
180
177
  download(transfer_spec)
181
178
  else
182
179
  raise "unexpected direction: [#{transfer_spec['direction']}]"
@@ -190,25 +187,22 @@ module Aspera
190
187
  end
191
188
 
192
189
  # terminates monitor thread
193
- def shutdown
194
- end
190
+ def shutdown; end
195
191
 
196
- def url=(api_url)
197
- end
192
+ def url=(api_url); end
198
193
 
199
194
  private
200
195
 
201
196
  def initialize(params)
202
- raise "params must be Hash" unless params.is_a?(Hash)
197
+ raise 'params must be Hash' unless params.is_a?(Hash)
203
198
  params=params.symbolize_keys
204
- raise "must have only one param: url" unless params.keys.eql?([:url])
199
+ raise 'must have only one param: url' unless params.keys.eql?([:url])
205
200
  super()
206
- @gw_api=Rest.new({:base_url => params[:url]})
201
+ @gw_api=Rest.new({base_url: params[:url]})
207
202
  api_info = @gw_api.read('info')[:data]
208
- Log.log.info("#{api_info}")
209
- @upload_chunksize=128000 # TODO: configurable ?
203
+ Log.log.info(api_info.to_s)
204
+ @upload_chunksize=128_000 # TODO: configurable ?
210
205
  end
211
-
212
206
  end # AgentHttpgw
213
207
  end
214
208
  end