aspera-cli 4.9.0 → 4.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +20 -0
  4. data/CHANGELOG.md +509 -0
  5. data/CONTRIBUTING.md +118 -0
  6. data/README.md +1241 -916
  7. data/bin/ascli +4 -4
  8. data/bin/asession +11 -11
  9. data/docs/test_env.conf +32 -21
  10. data/examples/aoc.rb +4 -4
  11. data/examples/dascli +16 -9
  12. data/examples/faspex4.rb +8 -8
  13. data/examples/node.rb +12 -12
  14. data/examples/server.rb +10 -10
  15. data/lib/aspera/aoc.rb +273 -266
  16. data/lib/aspera/ascmd.rb +56 -54
  17. data/lib/aspera/ats_api.rb +4 -4
  18. data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
  19. data/lib/aspera/cli/extended_value.rb +5 -5
  20. data/lib/aspera/cli/formater.rb +64 -64
  21. data/lib/aspera/cli/info.rb +2 -2
  22. data/lib/aspera/cli/listener/line_dump.rb +1 -1
  23. data/lib/aspera/cli/listener/logger.rb +1 -1
  24. data/lib/aspera/cli/listener/progress.rb +5 -6
  25. data/lib/aspera/cli/listener/progress_multi.rb +14 -19
  26. data/lib/aspera/cli/main.rb +66 -67
  27. data/lib/aspera/cli/manager.rb +112 -110
  28. data/lib/aspera/cli/plugin.rb +57 -36
  29. data/lib/aspera/cli/plugins/alee.rb +4 -4
  30. data/lib/aspera/cli/plugins/aoc.rb +309 -670
  31. data/lib/aspera/cli/plugins/ats.rb +44 -46
  32. data/lib/aspera/cli/plugins/bss.rb +10 -10
  33. data/lib/aspera/cli/plugins/config.rb +497 -378
  34. data/lib/aspera/cli/plugins/console.rb +12 -12
  35. data/lib/aspera/cli/plugins/cos.rb +18 -20
  36. data/lib/aspera/cli/plugins/faspex.rb +112 -114
  37. data/lib/aspera/cli/plugins/faspex5.rb +71 -46
  38. data/lib/aspera/cli/plugins/node.rb +379 -283
  39. data/lib/aspera/cli/plugins/orchestrator.rb +46 -46
  40. data/lib/aspera/cli/plugins/preview.rb +122 -114
  41. data/lib/aspera/cli/plugins/server.rb +137 -83
  42. data/lib/aspera/cli/plugins/shares.rb +30 -29
  43. data/lib/aspera/cli/plugins/sync.rb +13 -33
  44. data/lib/aspera/cli/transfer_agent.rb +60 -59
  45. data/lib/aspera/cli/version.rb +1 -1
  46. data/lib/aspera/colors.rb +3 -3
  47. data/lib/aspera/command_line_builder.rb +27 -27
  48. data/lib/aspera/cos_node.rb +22 -20
  49. data/lib/aspera/data_repository.rb +1 -1
  50. data/lib/aspera/environment.rb +35 -15
  51. data/lib/aspera/fasp/agent_base.rb +15 -15
  52. data/lib/aspera/fasp/agent_connect.rb +23 -21
  53. data/lib/aspera/fasp/agent_direct.rb +66 -64
  54. data/lib/aspera/fasp/agent_httpgw.rb +141 -78
  55. data/lib/aspera/fasp/agent_node.rb +23 -21
  56. data/lib/aspera/fasp/agent_trsdk.rb +20 -20
  57. data/lib/aspera/fasp/error.rb +3 -2
  58. data/lib/aspera/fasp/error_info.rb +11 -8
  59. data/lib/aspera/fasp/installation.rb +79 -79
  60. data/lib/aspera/fasp/listener.rb +1 -1
  61. data/lib/aspera/fasp/parameters.rb +86 -71
  62. data/lib/aspera/fasp/parameters.yaml +7 -4
  63. data/lib/aspera/fasp/resume_policy.rb +8 -8
  64. data/lib/aspera/fasp/transfer_spec.rb +35 -2
  65. data/lib/aspera/fasp/uri.rb +7 -7
  66. data/lib/aspera/faspex_gw.rb +7 -5
  67. data/lib/aspera/hash_ext.rb +3 -3
  68. data/lib/aspera/id_generator.rb +5 -5
  69. data/lib/aspera/keychain/encrypted_hash.rb +38 -105
  70. data/lib/aspera/keychain/macos_security.rb +128 -57
  71. data/lib/aspera/log.rb +7 -7
  72. data/lib/aspera/nagios.rb +19 -18
  73. data/lib/aspera/node.rb +209 -35
  74. data/lib/aspera/oauth.rb +37 -36
  75. data/lib/aspera/open_application.rb +19 -11
  76. data/lib/aspera/persistency_action_once.rb +4 -4
  77. data/lib/aspera/persistency_folder.rb +16 -15
  78. data/lib/aspera/preview/file_types.rb +8 -8
  79. data/lib/aspera/preview/generator.rb +67 -67
  80. data/lib/aspera/preview/utils.rb +27 -27
  81. data/lib/aspera/proxy_auto_config.js +41 -41
  82. data/lib/aspera/proxy_auto_config.rb +21 -14
  83. data/lib/aspera/rest.rb +72 -67
  84. data/lib/aspera/rest_call_error.rb +2 -1
  85. data/lib/aspera/rest_error_analyzer.rb +18 -17
  86. data/lib/aspera/rest_errors_aspera.rb +16 -16
  87. data/lib/aspera/secret_hider.rb +15 -13
  88. data/lib/aspera/ssh.rb +11 -10
  89. data/lib/aspera/sync.rb +158 -44
  90. data/lib/aspera/temp_file_manager.rb +2 -2
  91. data/lib/aspera/uri_reader.rb +4 -4
  92. data/lib/aspera/web_auth.rb +14 -13
  93. data.tar.gz.sig +0 -0
  94. metadata +11 -36
  95. metadata.gz.sig +0 -0
@@ -13,7 +13,7 @@ module Aspera
13
13
  BOOLEAN_FIELDS = %w[Encryption Remote RateLock MinRateLock PolicyLock FilesEncrypt FilesDecrypt VLinkLocalEnabled VLinkRemoteEnabled
14
14
  MoveRange Keepalive TestLogin UseProxy Precalc RTTAutocorrect].freeze
15
15
  EXPECTED_METHODS = %i[text struct enhanced].freeze
16
- private_constant :INTEGER_FIELDS,:BOOLEAN_FIELDS,:EXPECTED_METHODS
16
+ private_constant :INTEGER_FIELDS, :BOOLEAN_FIELDS, :EXPECTED_METHODS
17
17
 
18
18
  class << self
19
19
  # This checks the validity of the value returned by wait_for_transfers_completion
@@ -28,13 +28,13 @@ module Aspera
28
28
 
29
29
  # translates legacy event into enhanced (JSON) event
30
30
  def enhanced_event_format(event)
31
- return event.keys.each_with_object({}) do |e,h|
31
+ return event.keys.each_with_object({}) do |e, h|
32
32
  # capital_to_snake_case
33
- new_name = e.
34
- gsub(/([a-z\d])([A-Z])/,'\1_\2').
35
- gsub(/([a-z\d])(usec)$/,'\1_\2').
36
- gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
37
- downcase
33
+ new_name = e
34
+ .gsub(/([a-z\d])([A-Z])/, '\1_\2')
35
+ .gsub(/([a-z\d])(usec)$/, '\1_\2')
36
+ .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
37
+ .downcase
38
38
  value = event[e]
39
39
  value = value.to_i if INTEGER_FIELDS.include?(e)
40
40
  value = value.eql?('Yes') if BOOLEAN_FIELDS.include?(e)
@@ -46,7 +46,7 @@ module Aspera
46
46
  @listeners = []
47
47
  end
48
48
 
49
- def notify_listeners(current_event_text,current_event_data)
49
+ def notify_listeners(current_event_text, current_event_data)
50
50
  Log.log.debug('send event to listeners')
51
51
  enhanced_event = nil
52
52
  @listeners.each do |listener|
@@ -59,16 +59,16 @@ module Aspera
59
59
  end
60
60
  end # notify_listeners
61
61
 
62
- def notify_begin(id,size)
63
- notify_listeners('emulated',{LISTENER_SESSION_ID_B => id,'Type' => 'NOTIFICATION','PreTransferBytes' => size})
62
+ def notify_begin(id, size)
63
+ notify_listeners('emulated', {LISTENER_SESSION_ID_B => id, 'Type' => 'NOTIFICATION', 'PreTransferBytes' => size})
64
64
  end
65
65
 
66
- def notify_progress(id,size)
67
- notify_listeners('emulated',{LISTENER_SESSION_ID_B => id,'Type' => 'STATS','Bytescont' => size})
66
+ def notify_progress(id, size)
67
+ notify_listeners('emulated', {LISTENER_SESSION_ID_B => id, 'Type' => 'STATS', 'Bytescont' => size})
68
68
  end
69
69
 
70
70
  def notify_end(id)
71
- notify_listeners('emulated',{LISTENER_SESSION_ID_B => id,'Type' => 'DONE'})
71
+ notify_listeners('emulated', {LISTENER_SESSION_ID_B => id, 'Type' => 'DONE'})
72
72
  end
73
73
 
74
74
  public
@@ -78,13 +78,13 @@ module Aspera
78
78
 
79
79
  # listener receives events
80
80
  def add_listener(listener)
81
- raise "expect one of #{EXPECTED_METHODS}" if EXPECTED_METHODS.inject(0){|m,e|m += listener.respond_to?("event_#{e}") ? 1 : 0;m}.eql?(0)
81
+ raise "expect one of #{EXPECTED_METHODS}" if EXPECTED_METHODS.inject(0){|m, e|m + (listener.respond_to?("event_#{e}") ? 1 : 0)}.eql?(0)
82
82
  @listeners.push(listener)
83
83
  self
84
84
  end
85
85
 
86
86
  # the following methods must be implemented by subclass:
87
- # start_transfer(transfer_spec,options) : start and wait for completion
87
+ # start_transfer(transfer_spec) : start and wait for completion
88
88
  # wait_for_transfers_completion : wait for termination of all transfers, @return list of : :success or error message
89
89
  # optional: shutdown
90
90
  end
@@ -8,10 +8,10 @@ require 'tty-spinner'
8
8
 
9
9
  module Aspera
10
10
  module Fasp
11
- class AgentConnect < AgentBase
12
- MAX_CONNECT_START_RETRY = 3
13
- SLEEP_SEC_BETWEEN_RETRY = 2
14
- private_constant :MAX_CONNECT_START_RETRY,:SLEEP_SEC_BETWEEN_RETRY
11
+ class AgentConnect < Aspera::Fasp::AgentBase
12
+ MAX_CONNECT_START_RETRY = 4
13
+ SLEEP_SEC_BETWEEN_RETRY = 3
14
+ private_constant :MAX_CONNECT_START_RETRY, :SLEEP_SEC_BETWEEN_RETRY
15
15
  def initialize(_options)
16
16
  super()
17
17
  @connect_settings = {
@@ -21,28 +21,30 @@ module Aspera
21
21
  trynumber = 0
22
22
  begin
23
23
  connect_url = Installation.instance.connect_uri
24
- Log.log.debug("found: #{connect_url}")
25
- @connect_api = Rest.new({base_url: "#{connect_url}/v5/connect",headers: {'Origin' => Rest.user_agent}}) # could use v6 also now
24
+ Log.log.debug{"found: #{connect_url}"}
25
+ @connect_api = Rest.new({base_url: "#{connect_url}/v5/connect", headers: {'Origin' => Rest.user_agent}}) # could use v6 also now
26
26
  cinfo = @connect_api.read('info/version')[:data]
27
- Log.dump(:connect_version,cinfo)
27
+ Log.log.info('Connect was reached') if trynumber > 0
28
+ Log.dump(:connect_version, cinfo)
28
29
  rescue StandardError => e # Errno::ECONNREFUSED
29
- raise StandardError,"Unable to start connect after #{trynumber} try" if trynumber >= MAX_CONNECT_START_RETRY
30
- Log.log.warn("connect is not started. Retry ##{trynumber}, err=#{e}")
30
+ raise StandardError, "Unable to start connect #{trynumber} times" if trynumber >= MAX_CONNECT_START_RETRY
31
+ Log.log.warn{"Aspera Connect is not started (#{e}). Trying to start it ##{trynumber}..."}
31
32
  trynumber += 1
32
- if !OpenApplication.uri_graphical('fasp://initialize')
33
+ start_url = trynumber <= 2 ? 'fasp://initialize' : 'https://test-connect.ibmaspera.com/'
34
+ if !OpenApplication.uri_graphical(start_url)
33
35
  OpenApplication.uri_graphical('https://downloads.asperasoft.com/connect2/')
34
- raise StandardError,'Connect is not installed'
36
+ raise StandardError, 'Connect is not installed'
35
37
  end
36
38
  sleep(SLEEP_SEC_BETWEEN_RETRY)
37
39
  retry
38
40
  end
39
41
  end
40
42
 
41
- def start_transfer(transfer_spec,_options=nil)
43
+ def start_transfer(transfer_spec)
42
44
  if transfer_spec['direction'] == 'send'
43
- Log.log.warn("Connect requires upload selection using GUI, ignoring #{transfer_spec['paths']}".red)
45
+ Log.log.warn{"Connect requires upload selection using GUI, ignoring #{transfer_spec['paths']}".red}
44
46
  transfer_spec.delete('paths')
45
- resdata = @connect_api.create('windows/select-open-file-dialog/',{
47
+ resdata = @connect_api.create('windows/select-open-file-dialog/', {
46
48
  'aspera_connect_settings' => @connect_settings,
47
49
  'title' => 'Select Files',
48
50
  'suggestedName' => '',
@@ -53,7 +55,7 @@ module Aspera
53
55
  @request_id = SecureRandom.uuid
54
56
  # if there is a token, we ask connect client to use well known ssh private keys
55
57
  # instead of asking password
56
- transfer_spec['authentication'] = 'token' if transfer_spec.has_key?('token')
58
+ transfer_spec['authentication'] = 'token' if transfer_spec.key?('token')
57
59
  connect_transfer_args = {
58
60
  'aspera_connect_settings' => @connect_settings.merge({
59
61
  'request_id' => @request_id,
@@ -63,7 +65,7 @@ module Aspera
63
65
  'transfer_spec' => transfer_spec
64
66
  }]}
65
67
  # asynchronous anyway
66
- res = @connect_api.create('transfers/start',connect_transfer_args)[:data]
68
+ res = @connect_api.create('transfers/start', connect_transfer_args)[:data]
67
69
  @xfer_id = res['transfer_specs'].first['transfer_spec']['tags']['aspera']['xfer_id']
68
70
  end
69
71
 
@@ -73,7 +75,7 @@ module Aspera
73
75
  spinner = nil
74
76
  begin
75
77
  loop do
76
- tr_info = @connect_api.create("transfers/info/#{@xfer_id}",connect_activity_args)[:data]
78
+ tr_info = @connect_api.create("transfers/info/#{@xfer_id}", connect_activity_args)[:data]
77
79
  if tr_info['transfer_info'].is_a?(Hash)
78
80
  trdata = tr_info['transfer_info']
79
81
  if trdata.nil?
@@ -85,7 +87,7 @@ module Aspera
85
87
  when 'completed'
86
88
  notify_end(@connect_settings['app_id'])
87
89
  break
88
- when 'initiating','queued'
90
+ when 'initiating', 'queued'
89
91
  if spinner.nil?
90
92
  spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
91
93
  spinner.start
@@ -93,13 +95,13 @@ module Aspera
93
95
  spinner.update(title: trdata['status'])
94
96
  spinner.spin
95
97
  when 'running'
96
- #puts "running: sessions:#{trdata['sessions'].length}, #{trdata['sessions'].map{|i| i['bytes_transferred']}.join(',')}"
98
+ # puts "running: sessions:#{trdata['sessions'].length}, #{trdata['sessions'].map{|i| i['bytes_transferred']}.join(',')}"
97
99
  if !started && (trdata['bytes_expected'] != 0)
98
100
  spinner&.success
99
- notify_begin(@connect_settings['app_id'],trdata['bytes_expected'])
101
+ notify_begin(@connect_settings['app_id'], trdata['bytes_expected'])
100
102
  started = true
101
103
  else
102
- notify_progress(@connect_settings['app_id'],trdata['bytes_written'])
104
+ notify_progress(@connect_settings['app_id'], trdata['bytes_written'])
103
105
  end
104
106
  when 'failed'
105
107
  spinner&.error
@@ -16,12 +16,12 @@ require 'shellwords'
16
16
  module Aspera
17
17
  module Fasp
18
18
  # executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
19
- class AgentDirect < AgentBase
19
+ class AgentDirect < Aspera::Fasp::AgentBase
20
20
  # options for initialize (same as values in option transfer_info)
21
21
  DEFAULT_OPTIONS = {
22
22
  spawn_timeout_sec: 3,
23
23
  spawn_delay_sec: 2,
24
- wss: false,
24
+ wss: true, # true: if both SSH and wss in ts: prefer wss
25
25
  multi_incr_udp: true,
26
26
  resume: {},
27
27
  quiet: true # by default no interactive progress bar
@@ -31,12 +31,8 @@ module Aspera
31
31
  # start ascp transfer (non blocking), single or multi-session
32
32
  # job information added to @jobs
33
33
  # @param transfer_spec [Hash] aspera transfer specification
34
- # @param options [Hash] :resumer, :regenerate_token
35
- def start_transfer(transfer_spec,options={})
36
- raise 'option: must be hash (or nil)' unless options.is_a?(Hash)
37
- job_options = options.clone
38
- job_options[:resumer] ||= @resume_policy
39
- job_options[:job_id] ||= SecureRandom.uuid
34
+ def start_transfer(transfer_spec)
35
+ the_job_id = SecureRandom.uuid
40
36
  # clone transfer spec because we modify it (first level keys)
41
37
  transfer_spec = transfer_spec.clone
42
38
  # if there is aspera tags
@@ -46,16 +42,16 @@ module Aspera
46
42
  # using a non unique id results in discard of tags in AoC, and a package is never finalized
47
43
  # all sessions in a multi-session transfer must have the same xfer_id (see admin manual)
48
44
  transfer_spec['tags']['aspera']['xfer_id'] ||= SecureRandom.uuid
49
- Log.log.debug("xfer id=#{transfer_spec['xfer_id']}")
45
+ Log.log.debug{"xfer id=#{transfer_spec['xfer_id']}"}
50
46
  # TODO: useful ? node only ?
51
47
  transfer_spec['tags']['aspera']['xfer_retry'] ||= 3600
52
48
  end
53
- Log.dump('ts',transfer_spec)
49
+ Log.dump('ts', transfer_spec)
54
50
 
55
51
  # add bypass keys when authentication is token and no auth is provided
56
- if transfer_spec.has_key?('token') &&
57
- !transfer_spec.has_key?('remote_password') &&
58
- !transfer_spec.has_key?('EX_ssh_key_paths')
52
+ if transfer_spec.key?('token') &&
53
+ !transfer_spec.key?('remote_password') &&
54
+ !transfer_spec.key?('EX_ssh_key_paths')
59
55
  # transfer_spec['remote_password'] = Installation.instance.bypass_pass # not used
60
56
  transfer_spec['EX_ssh_key_paths'] = Installation.instance.bypass_keys
61
57
  end
@@ -63,21 +59,21 @@ module Aspera
63
59
  # Compute this before using transfer spec because it potentially modifies the transfer spec
64
60
  # (even if the var is not used in single session)
65
61
  multi_session_info = nil
66
- if transfer_spec.has_key?('multi_session')
62
+ if transfer_spec.key?('multi_session')
67
63
  multi_session_info = {
68
64
  count: transfer_spec['multi_session'].to_i
69
65
  }
70
66
  # Managed by multi-session, so delete from transfer spec
71
67
  transfer_spec.delete('multi_session')
72
68
  if multi_session_info[:count].negative?
73
- Log.log.error("multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0")
69
+ Log.log.error{"multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0"}
74
70
  multi_session_info = nil
75
71
  elsif multi_session_info[:count].eql?(0)
76
72
  Log.log.debug('multi_session count is zero: no multisession')
77
73
  multi_session_info = nil
78
74
  elsif @options[:multi_incr_udp] # multi_session_info[:count] > 0
79
75
  # if option not true: keep default udp port for all sessions
80
- multi_session_info[:udp_base] = transfer_spec.has_key?('fasp_port') ? transfer_spec['fasp_port'] : TransferSpec::UDP_PORT
76
+ multi_session_info[:udp_base] = transfer_spec.key?('fasp_port') ? transfer_spec['fasp_port'] : TransferSpec::UDP_PORT
81
77
  # delete from original transfer spec, as we will increment values
82
78
  transfer_spec.delete('fasp_port')
83
79
  # override if specified, else use default value
@@ -85,19 +81,19 @@ module Aspera
85
81
  end
86
82
 
87
83
  # compute known args
88
- env_args = Parameters.ts_to_env_args(transfer_spec,wss: @options[:wss])
84
+ env_args = Parameters.ts_to_env_args(transfer_spec, wss: @options[:wss])
89
85
 
90
86
  # add fallback cert and key as arguments if needed
91
87
  if %w[1 force].include?(transfer_spec['http_fallback'])
92
- env_args[:args].unshift('-Y',Installation.instance.path(:fallback_key))
93
- env_args[:args].unshift('-I',Installation.instance.path(:fallback_cert))
88
+ env_args[:args].unshift('-Y', Installation.instance.path(:fallback_key))
89
+ env_args[:args].unshift('-I', Installation.instance.path(:fallback_cert))
94
90
  end
95
91
 
96
92
  env_args[:args].unshift('-q') if @options[:quiet]
97
93
 
98
94
  # transfer job can be multi session
99
95
  xfer_job = {
100
- id: job_options[:job_id],
96
+ id: the_job_id,
101
97
  sessions: [] # all sessions as below
102
98
  }
103
99
 
@@ -107,8 +103,7 @@ module Aspera
107
103
  error: nil, # exception if failed
108
104
  io: nil, # management port server socket
109
105
  id: nil, # SessionId from INIT message in mgt port
110
- env_args: env_args, # env vars and args to ascp (from transfer spec)
111
- options: job_options # [Hash]
106
+ env_args: env_args # env vars and args to ascp (from transfer spec)
112
107
  }
113
108
 
114
109
  if multi_session_info.nil?
@@ -127,7 +122,7 @@ module Aspera
127
122
  this_session[:env_args][:args] = this_session[:env_args][:args].clone
128
123
  this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_info[:count]}")
129
124
  # option: increment (default as per ascp manual) or not (cluster on other side ?)
130
- this_session[:env_args][:args].unshift('-O',(multi_session_info[:udp_base] + i - 1).to_s) if @options[:multi_incr_udp]
125
+ this_session[:env_args][:args].unshift('-O', (multi_session_info[:udp_base] + i - 1).to_s) if @options[:multi_incr_udp]
131
126
  this_session[:thread] = Thread.new(this_session) {|s|transfer_thread_entry(s)}
132
127
  xfer_job[:sessions].push(this_session)
133
128
  end
@@ -135,10 +130,10 @@ module Aspera
135
130
  Log.log.debug('started session thread(s)')
136
131
 
137
132
  # add job to list of jobs
138
- @jobs[job_options[:job_id]] = xfer_job
139
- Log.log.debug("jobs: #{@jobs.keys.count}")
133
+ @jobs[the_job_id] = xfer_job
134
+ Log.log.debug{"jobs: #{@jobs.keys.count}"}
140
135
 
141
- return job_options[:job_id]
136
+ return the_job_id
142
137
  end # start_transfer
143
138
 
144
139
  # wait for completion of all jobs started
@@ -147,9 +142,9 @@ module Aspera
147
142
  Log.log.debug('wait_for_transfers_completion')
148
143
  # set to non-nil to exit loop
149
144
  result = []
150
- @jobs.each do |_id,job|
145
+ @jobs.each do |_id, job|
151
146
  job[:sessions].each do |session|
152
- Log.log.debug("join #{session[:thread]}")
147
+ Log.log.debug{"join #{session[:thread]}"}
153
148
  session[:thread].join
154
149
  result.push(session[:error] || :success)
155
150
  end
@@ -173,13 +168,13 @@ module Aspera
173
168
  # @param env_args a hash containing :args :env :ascp_version
174
169
  # @param session this session information
175
170
  # could be private method
176
- def start_transfer_with_args_env(env_args,session)
171
+ def start_transfer_with_args_env(env_args, session)
177
172
  raise 'env_args must be Hash' unless env_args.is_a?(Hash)
178
173
  raise 'session must be Hash' unless session.is_a?(Hash)
179
174
  # by default we assume an exception will be raised (for ensure block)
180
175
  exception_raised = true
181
176
  begin
182
- Log.log.debug("env_args=#{env_args.inspect}")
177
+ Log.log.debug{"env_args=#{env_args.inspect}"}
183
178
  # get location of ascp executable
184
179
  ascp_path = @mutex.synchronize do
185
180
  Fasp::Installation.instance.path(env_args[:ascp_version])
@@ -187,24 +182,24 @@ module Aspera
187
182
  # (optional) check it exists
188
183
  raise Fasp::Error, "no such file: #{ascp_path}" unless File.exist?(ascp_path)
189
184
  # open random local TCP port for listening for ascp management
190
- mgt_sock = TCPServer.new('127.0.0.1',0)
185
+ mgt_sock = TCPServer.new('127.0.0.1', 0)
191
186
  # clone arguments as we eed to modify with mgt port
192
187
  ascp_arguments = env_args[:args].clone
193
188
  # add management port
194
189
  ascp_arguments.unshift('-M', mgt_sock.addr[1].to_s)
195
190
  # start ascp in sub process
196
191
  Log.log.debug do
197
- 'execute: '+
198
- env_args[:env].map{|k,v| "#{k}=#{Shellwords.shellescape(v)}"}.join(' ')+
199
- ' '+
200
- Shellwords.shellescape(ascp_path)+
201
- ' '+
202
- ascp_arguments.map{|a|Shellwords.shellescape(a)}.join(' ')
192
+ 'execute: ' +
193
+ env_args[:env].map{|k, v| "#{k}=#{Shellwords.shellescape(v)}"}.join(' ') +
194
+ ' ' +
195
+ Shellwords.shellescape(ascp_path) +
196
+ ' ' +
197
+ ascp_arguments.map{|a|Shellwords.shellescape(a)}.join(' ')
203
198
  end
204
199
  # start process
205
- ascp_pid = Process.spawn(env_args[:env],[ascp_path,ascp_path],*ascp_arguments)
200
+ ascp_pid = Process.spawn(env_args[:env], [ascp_path, ascp_path], *ascp_arguments)
206
201
  # in parent, wait for connection to socket max 3 seconds
207
- Log.log.debug("before accept for pid (#{ascp_pid})")
202
+ Log.log.debug{"before accept for pid (#{ascp_pid})"}
208
203
  # init management socket
209
204
  ascp_mgt_io = nil
210
205
  Timeout.timeout(@options[:spawn_timeout_sec]) do
@@ -214,7 +209,7 @@ module Aspera
214
209
  # TODO: use same value as Encoding.default_external
215
210
  ascp_mgt_io.set_encoding(Encoding::UTF_8)
216
211
  end
217
- Log.log.debug("after accept (#{ascp_mgt_io})")
212
+ Log.log.debug{"after accept (#{ascp_mgt_io})"}
218
213
  session[:io] = ascp_mgt_io
219
214
  # exact text for event, with \n
220
215
  current_event_text = ''
@@ -230,7 +225,7 @@ module Aspera
230
225
  break if line.nil?
231
226
  current_event_text += line
232
227
  line.chomp!
233
- Log.log.debug("line=[#{line}]")
228
+ Log.log.debug{"line=[#{line}]"}
234
229
  case line
235
230
  when 'FASPMGR 2'
236
231
  # begin event
@@ -243,12 +238,12 @@ module Aspera
243
238
  # empty line is separator to end event information
244
239
  raise 'unexpected empty line' if current_event_data.nil?
245
240
  current_event_data[AgentBase::LISTENER_SESSION_ID_B] = ascp_pid
246
- notify_listeners(current_event_text,current_event_data)
241
+ notify_listeners(current_event_text, current_event_data)
247
242
  case current_event_data['Type']
248
243
  when 'INIT'
249
244
  session[:id] = current_event_data['SessionId']
250
- Log.log.debug("session id: #{session[:id]}")
251
- when 'DONE','ERROR'
245
+ Log.log.debug{"session id: #{session[:id]}"}
246
+ when 'DONE', 'ERROR'
252
247
  # TODO: check if this is always the last event
253
248
  last_status_event = current_event_data
254
249
  end # event type
@@ -263,16 +258,20 @@ module Aspera
263
258
  # all went well
264
259
  exception_raised = false
265
260
  when 'ERROR'
266
- Log.log.error("code: #{last_status_event['Code']}")
261
+ Log.log.error{"code: #{last_status_event['Code']}"}
267
262
  if /bearer token/i.match?(last_status_event['Description'])
268
263
  Log.log.error('need to regenerate token'.red)
269
- if session[:options].is_a?(Hash) && session[:options].has_key?(:regenerate_token)
264
+ if !@token_regenerator.nil?
270
265
  # regenerate token here, expired, or error on it
271
266
  # Note: in multi-session, each session will have a different one.
272
- env_args[:env]['ASPERA_SCP_TOKEN'] = session[:options][:regenerate_token].call(true)
267
+ env_args[:env]['ASPERA_SCP_TOKEN'] = @token_regenerator.call(true)
273
268
  end
274
269
  end
275
- raise Fasp::Error.new(last_status_event['Description'],last_status_event['Code'].to_i)
270
+ # cannot resolve address
271
+ # if last_status_event['Code'].to_i.eql?(14)
272
+ # Log.log.warn{"host: #{}"}
273
+ # end
274
+ raise Fasp::Error.new(last_status_event['Description'], last_status_event['Code'].to_i)
276
275
  else # case
277
276
  raise "unexpected last event type: #{last_status_event['Type']}"
278
277
  end
@@ -312,22 +311,24 @@ module Aspera
312
311
  # @param data command on mgt port, examples:
313
312
  # {'type'=>'START','source'=>_path_,'destination'=>_path_}
314
313
  # {'type'=>'DONE'}
315
- def send_command(job_id,session_index,data)
314
+ def send_command(job_id, session_index, data)
316
315
  job = @jobs[job_id]
317
316
  raise 'no such job' if job.nil?
318
317
  session = job[:sessions][session_index]
319
318
  raise 'no such session' if session.nil?
320
- Log.log.debug("command: #{data}")
319
+ Log.log.debug{"command: #{data}"}
321
320
  # build command
322
- command = data.
323
- keys.
324
- map{|k|"#{k.capitalize}: #{data[k]}"}.
325
- unshift('FASPMGR 2').
326
- push('','').
327
- join("\n")
321
+ command = data
322
+ .keys
323
+ .map{|k|"#{k.capitalize}: #{data[k]}"}
324
+ .unshift('FASPMGR 2')
325
+ .push('', '')
326
+ .join("\n")
328
327
  session[:io].puts(command)
329
328
  end
330
329
 
330
+ attr_writer :token_regenerator
331
+
331
332
  private
332
333
 
333
334
  # @param options : keys(symbol): see DEFAULT_OPTIONS
@@ -341,13 +342,14 @@ module Aspera
341
342
  @options = DEFAULT_OPTIONS.dup
342
343
  if !options.nil?
343
344
  raise "expecting Hash (or nil), but have #{options.class}" unless options.is_a?(Hash)
344
- options.each do |k,v|
345
- raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.has_key?(k)
345
+ options.each do |k, v|
346
+ raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.key?(k)
346
347
  @options[k] = v
347
348
  end
348
349
  end
349
- Log.log.debug("local options= #{options}")
350
+ Log.log.debug{"local options= #{options}"}
350
351
  @resume_policy = ResumePolicy.new(@options[:resume].symbolize_keys)
352
+ @token_regenerator = nil
351
353
  end
352
354
 
353
355
  # transfer thread entry
@@ -356,17 +358,17 @@ module Aspera
356
358
  begin
357
359
  # set name for logging
358
360
  Thread.current[:name] = 'transfer'
359
- Log.log.debug("ENTER (#{Thread.current[:name]})")
361
+ Log.log.debug{"ENTER (#{Thread.current[:name]})"}
360
362
  # start transfer with selected resumer policy
361
- session[:options][:resumer].execute_with_resume do
362
- start_transfer_with_args_env(session[:env_args],session)
363
+ @resume_policy.execute_with_resume do
364
+ start_transfer_with_args_env(session[:env_args], session)
363
365
  end
364
366
  Log.log.debug('transfer ok'.bg_green)
365
367
  rescue StandardError => e
366
368
  session[:error] = e
367
- Log.log.error("Transfer thread error: #{e.class}:\n#{e.message}:\n#{e.backtrace.join("\n")}".red) if Log.instance.level.eql?(:debug)
369
+ Log.log.error{"Transfer thread error: #{e.class}:\n#{e.message}:\n#{e.backtrace.join("\n")}".red} if Log.instance.level.eql?(:debug)
368
370
  end
369
- Log.log.debug("EXIT (#{Thread.current[:name]})")
371
+ Log.log.debug{"EXIT (#{Thread.current[:name]})"}
370
372
  end
371
373
  end # AgentDirect
372
374
  end