aspera-cli 4.9.0 → 4.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +20 -0
- data/CHANGELOG.md +509 -0
- data/CONTRIBUTING.md +118 -0
- data/README.md +1241 -916
- data/bin/ascli +4 -4
- data/bin/asession +11 -11
- data/docs/test_env.conf +32 -21
- data/examples/aoc.rb +4 -4
- data/examples/dascli +16 -9
- data/examples/faspex4.rb +8 -8
- data/examples/node.rb +12 -12
- data/examples/server.rb +10 -10
- data/lib/aspera/aoc.rb +273 -266
- data/lib/aspera/ascmd.rb +56 -54
- data/lib/aspera/ats_api.rb +4 -4
- data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
- data/lib/aspera/cli/extended_value.rb +5 -5
- data/lib/aspera/cli/formater.rb +64 -64
- data/lib/aspera/cli/info.rb +2 -2
- data/lib/aspera/cli/listener/line_dump.rb +1 -1
- data/lib/aspera/cli/listener/logger.rb +1 -1
- data/lib/aspera/cli/listener/progress.rb +5 -6
- data/lib/aspera/cli/listener/progress_multi.rb +14 -19
- data/lib/aspera/cli/main.rb +66 -67
- data/lib/aspera/cli/manager.rb +112 -110
- data/lib/aspera/cli/plugin.rb +57 -36
- data/lib/aspera/cli/plugins/alee.rb +4 -4
- data/lib/aspera/cli/plugins/aoc.rb +309 -670
- data/lib/aspera/cli/plugins/ats.rb +44 -46
- data/lib/aspera/cli/plugins/bss.rb +10 -10
- data/lib/aspera/cli/plugins/config.rb +497 -378
- data/lib/aspera/cli/plugins/console.rb +12 -12
- data/lib/aspera/cli/plugins/cos.rb +18 -20
- data/lib/aspera/cli/plugins/faspex.rb +112 -114
- data/lib/aspera/cli/plugins/faspex5.rb +71 -46
- data/lib/aspera/cli/plugins/node.rb +379 -283
- data/lib/aspera/cli/plugins/orchestrator.rb +46 -46
- data/lib/aspera/cli/plugins/preview.rb +122 -114
- data/lib/aspera/cli/plugins/server.rb +137 -83
- data/lib/aspera/cli/plugins/shares.rb +30 -29
- data/lib/aspera/cli/plugins/sync.rb +13 -33
- data/lib/aspera/cli/transfer_agent.rb +60 -59
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +3 -3
- data/lib/aspera/command_line_builder.rb +27 -27
- data/lib/aspera/cos_node.rb +22 -20
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +35 -15
- data/lib/aspera/fasp/agent_base.rb +15 -15
- data/lib/aspera/fasp/agent_connect.rb +23 -21
- data/lib/aspera/fasp/agent_direct.rb +66 -64
- data/lib/aspera/fasp/agent_httpgw.rb +141 -78
- data/lib/aspera/fasp/agent_node.rb +23 -21
- data/lib/aspera/fasp/agent_trsdk.rb +20 -20
- data/lib/aspera/fasp/error.rb +3 -2
- data/lib/aspera/fasp/error_info.rb +11 -8
- data/lib/aspera/fasp/installation.rb +79 -79
- data/lib/aspera/fasp/listener.rb +1 -1
- data/lib/aspera/fasp/parameters.rb +86 -71
- data/lib/aspera/fasp/parameters.yaml +7 -4
- data/lib/aspera/fasp/resume_policy.rb +8 -8
- data/lib/aspera/fasp/transfer_spec.rb +35 -2
- data/lib/aspera/fasp/uri.rb +7 -7
- data/lib/aspera/faspex_gw.rb +7 -5
- data/lib/aspera/hash_ext.rb +3 -3
- data/lib/aspera/id_generator.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +38 -105
- data/lib/aspera/keychain/macos_security.rb +128 -57
- data/lib/aspera/log.rb +7 -7
- data/lib/aspera/nagios.rb +19 -18
- data/lib/aspera/node.rb +209 -35
- data/lib/aspera/oauth.rb +37 -36
- data/lib/aspera/open_application.rb +19 -11
- data/lib/aspera/persistency_action_once.rb +4 -4
- data/lib/aspera/persistency_folder.rb +16 -15
- data/lib/aspera/preview/file_types.rb +8 -8
- data/lib/aspera/preview/generator.rb +67 -67
- data/lib/aspera/preview/utils.rb +27 -27
- data/lib/aspera/proxy_auto_config.js +41 -41
- data/lib/aspera/proxy_auto_config.rb +21 -14
- data/lib/aspera/rest.rb +72 -67
- data/lib/aspera/rest_call_error.rb +2 -1
- data/lib/aspera/rest_error_analyzer.rb +18 -17
- data/lib/aspera/rest_errors_aspera.rb +16 -16
- data/lib/aspera/secret_hider.rb +15 -13
- data/lib/aspera/ssh.rb +11 -10
- data/lib/aspera/sync.rb +158 -44
- data/lib/aspera/temp_file_manager.rb +2 -2
- data/lib/aspera/uri_reader.rb +4 -4
- data/lib/aspera/web_auth.rb +14 -13
- data.tar.gz.sig +0 -0
- metadata +11 -36
- metadata.gz.sig +0 -0
@@ -13,7 +13,7 @@ module Aspera
|
|
13
13
|
BOOLEAN_FIELDS = %w[Encryption Remote RateLock MinRateLock PolicyLock FilesEncrypt FilesDecrypt VLinkLocalEnabled VLinkRemoteEnabled
|
14
14
|
MoveRange Keepalive TestLogin UseProxy Precalc RTTAutocorrect].freeze
|
15
15
|
EXPECTED_METHODS = %i[text struct enhanced].freeze
|
16
|
-
private_constant :INTEGER_FIELDS
|
16
|
+
private_constant :INTEGER_FIELDS, :BOOLEAN_FIELDS, :EXPECTED_METHODS
|
17
17
|
|
18
18
|
class << self
|
19
19
|
# This checks the validity of the value returned by wait_for_transfers_completion
|
@@ -28,13 +28,13 @@ module Aspera
|
|
28
28
|
|
29
29
|
# translates legacy event into enhanced (JSON) event
|
30
30
|
def enhanced_event_format(event)
|
31
|
-
return event.keys.each_with_object({}) do |e,h|
|
31
|
+
return event.keys.each_with_object({}) do |e, h|
|
32
32
|
# capital_to_snake_case
|
33
|
-
new_name = e
|
34
|
-
gsub(/([a-z\d])([A-Z])/,'\1_\2')
|
35
|
-
gsub(/([a-z\d])(usec)$/,'\1_\2')
|
36
|
-
gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2')
|
37
|
-
downcase
|
33
|
+
new_name = e
|
34
|
+
.gsub(/([a-z\d])([A-Z])/, '\1_\2')
|
35
|
+
.gsub(/([a-z\d])(usec)$/, '\1_\2')
|
36
|
+
.gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
|
37
|
+
.downcase
|
38
38
|
value = event[e]
|
39
39
|
value = value.to_i if INTEGER_FIELDS.include?(e)
|
40
40
|
value = value.eql?('Yes') if BOOLEAN_FIELDS.include?(e)
|
@@ -46,7 +46,7 @@ module Aspera
|
|
46
46
|
@listeners = []
|
47
47
|
end
|
48
48
|
|
49
|
-
def notify_listeners(current_event_text,current_event_data)
|
49
|
+
def notify_listeners(current_event_text, current_event_data)
|
50
50
|
Log.log.debug('send event to listeners')
|
51
51
|
enhanced_event = nil
|
52
52
|
@listeners.each do |listener|
|
@@ -59,16 +59,16 @@ module Aspera
|
|
59
59
|
end
|
60
60
|
end # notify_listeners
|
61
61
|
|
62
|
-
def notify_begin(id,size)
|
63
|
-
notify_listeners('emulated',{LISTENER_SESSION_ID_B => id,'Type' => 'NOTIFICATION','PreTransferBytes' => size})
|
62
|
+
def notify_begin(id, size)
|
63
|
+
notify_listeners('emulated', {LISTENER_SESSION_ID_B => id, 'Type' => 'NOTIFICATION', 'PreTransferBytes' => size})
|
64
64
|
end
|
65
65
|
|
66
|
-
def notify_progress(id,size)
|
67
|
-
notify_listeners('emulated',{LISTENER_SESSION_ID_B => id,'Type' => 'STATS','Bytescont' => size})
|
66
|
+
def notify_progress(id, size)
|
67
|
+
notify_listeners('emulated', {LISTENER_SESSION_ID_B => id, 'Type' => 'STATS', 'Bytescont' => size})
|
68
68
|
end
|
69
69
|
|
70
70
|
def notify_end(id)
|
71
|
-
notify_listeners('emulated',{LISTENER_SESSION_ID_B => id,'Type' => 'DONE'})
|
71
|
+
notify_listeners('emulated', {LISTENER_SESSION_ID_B => id, 'Type' => 'DONE'})
|
72
72
|
end
|
73
73
|
|
74
74
|
public
|
@@ -78,13 +78,13 @@ module Aspera
|
|
78
78
|
|
79
79
|
# listener receives events
|
80
80
|
def add_listener(listener)
|
81
|
-
raise "expect one of #{EXPECTED_METHODS}" if EXPECTED_METHODS.inject(0){|m,e|m
|
81
|
+
raise "expect one of #{EXPECTED_METHODS}" if EXPECTED_METHODS.inject(0){|m, e|m + (listener.respond_to?("event_#{e}") ? 1 : 0)}.eql?(0)
|
82
82
|
@listeners.push(listener)
|
83
83
|
self
|
84
84
|
end
|
85
85
|
|
86
86
|
# the following methods must be implemented by subclass:
|
87
|
-
# start_transfer(transfer_spec
|
87
|
+
# start_transfer(transfer_spec) : start and wait for completion
|
88
88
|
# wait_for_transfers_completion : wait for termination of all transfers, @return list of : :success or error message
|
89
89
|
# optional: shutdown
|
90
90
|
end
|
@@ -8,10 +8,10 @@ require 'tty-spinner'
|
|
8
8
|
|
9
9
|
module Aspera
|
10
10
|
module Fasp
|
11
|
-
class AgentConnect < AgentBase
|
12
|
-
MAX_CONNECT_START_RETRY =
|
13
|
-
SLEEP_SEC_BETWEEN_RETRY =
|
14
|
-
private_constant :MAX_CONNECT_START_RETRY
|
11
|
+
class AgentConnect < Aspera::Fasp::AgentBase
|
12
|
+
MAX_CONNECT_START_RETRY = 4
|
13
|
+
SLEEP_SEC_BETWEEN_RETRY = 3
|
14
|
+
private_constant :MAX_CONNECT_START_RETRY, :SLEEP_SEC_BETWEEN_RETRY
|
15
15
|
def initialize(_options)
|
16
16
|
super()
|
17
17
|
@connect_settings = {
|
@@ -21,28 +21,30 @@ module Aspera
|
|
21
21
|
trynumber = 0
|
22
22
|
begin
|
23
23
|
connect_url = Installation.instance.connect_uri
|
24
|
-
Log.log.debug
|
25
|
-
@connect_api = Rest.new({base_url: "#{connect_url}/v5/connect",headers: {'Origin' => Rest.user_agent}}) # could use v6 also now
|
24
|
+
Log.log.debug{"found: #{connect_url}"}
|
25
|
+
@connect_api = Rest.new({base_url: "#{connect_url}/v5/connect", headers: {'Origin' => Rest.user_agent}}) # could use v6 also now
|
26
26
|
cinfo = @connect_api.read('info/version')[:data]
|
27
|
-
Log.
|
27
|
+
Log.log.info('Connect was reached') if trynumber > 0
|
28
|
+
Log.dump(:connect_version, cinfo)
|
28
29
|
rescue StandardError => e # Errno::ECONNREFUSED
|
29
|
-
raise StandardError,"Unable to start connect
|
30
|
-
Log.log.warn
|
30
|
+
raise StandardError, "Unable to start connect #{trynumber} times" if trynumber >= MAX_CONNECT_START_RETRY
|
31
|
+
Log.log.warn{"Aspera Connect is not started (#{e}). Trying to start it ##{trynumber}..."}
|
31
32
|
trynumber += 1
|
32
|
-
|
33
|
+
start_url = trynumber <= 2 ? 'fasp://initialize' : 'https://test-connect.ibmaspera.com/'
|
34
|
+
if !OpenApplication.uri_graphical(start_url)
|
33
35
|
OpenApplication.uri_graphical('https://downloads.asperasoft.com/connect2/')
|
34
|
-
raise StandardError,'Connect is not installed'
|
36
|
+
raise StandardError, 'Connect is not installed'
|
35
37
|
end
|
36
38
|
sleep(SLEEP_SEC_BETWEEN_RETRY)
|
37
39
|
retry
|
38
40
|
end
|
39
41
|
end
|
40
42
|
|
41
|
-
def start_transfer(transfer_spec
|
43
|
+
def start_transfer(transfer_spec)
|
42
44
|
if transfer_spec['direction'] == 'send'
|
43
|
-
Log.log.warn
|
45
|
+
Log.log.warn{"Connect requires upload selection using GUI, ignoring #{transfer_spec['paths']}".red}
|
44
46
|
transfer_spec.delete('paths')
|
45
|
-
resdata = @connect_api.create('windows/select-open-file-dialog/',{
|
47
|
+
resdata = @connect_api.create('windows/select-open-file-dialog/', {
|
46
48
|
'aspera_connect_settings' => @connect_settings,
|
47
49
|
'title' => 'Select Files',
|
48
50
|
'suggestedName' => '',
|
@@ -53,7 +55,7 @@ module Aspera
|
|
53
55
|
@request_id = SecureRandom.uuid
|
54
56
|
# if there is a token, we ask connect client to use well known ssh private keys
|
55
57
|
# instead of asking password
|
56
|
-
transfer_spec['authentication'] = 'token' if transfer_spec.
|
58
|
+
transfer_spec['authentication'] = 'token' if transfer_spec.key?('token')
|
57
59
|
connect_transfer_args = {
|
58
60
|
'aspera_connect_settings' => @connect_settings.merge({
|
59
61
|
'request_id' => @request_id,
|
@@ -63,7 +65,7 @@ module Aspera
|
|
63
65
|
'transfer_spec' => transfer_spec
|
64
66
|
}]}
|
65
67
|
# asynchronous anyway
|
66
|
-
res = @connect_api.create('transfers/start',connect_transfer_args)[:data]
|
68
|
+
res = @connect_api.create('transfers/start', connect_transfer_args)[:data]
|
67
69
|
@xfer_id = res['transfer_specs'].first['transfer_spec']['tags']['aspera']['xfer_id']
|
68
70
|
end
|
69
71
|
|
@@ -73,7 +75,7 @@ module Aspera
|
|
73
75
|
spinner = nil
|
74
76
|
begin
|
75
77
|
loop do
|
76
|
-
tr_info = @connect_api.create("transfers/info/#{@xfer_id}",connect_activity_args)[:data]
|
78
|
+
tr_info = @connect_api.create("transfers/info/#{@xfer_id}", connect_activity_args)[:data]
|
77
79
|
if tr_info['transfer_info'].is_a?(Hash)
|
78
80
|
trdata = tr_info['transfer_info']
|
79
81
|
if trdata.nil?
|
@@ -85,7 +87,7 @@ module Aspera
|
|
85
87
|
when 'completed'
|
86
88
|
notify_end(@connect_settings['app_id'])
|
87
89
|
break
|
88
|
-
when 'initiating','queued'
|
90
|
+
when 'initiating', 'queued'
|
89
91
|
if spinner.nil?
|
90
92
|
spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
|
91
93
|
spinner.start
|
@@ -93,13 +95,13 @@ module Aspera
|
|
93
95
|
spinner.update(title: trdata['status'])
|
94
96
|
spinner.spin
|
95
97
|
when 'running'
|
96
|
-
#puts "running: sessions:#{trdata['sessions'].length}, #{trdata['sessions'].map{|i| i['bytes_transferred']}.join(',')}"
|
98
|
+
# puts "running: sessions:#{trdata['sessions'].length}, #{trdata['sessions'].map{|i| i['bytes_transferred']}.join(',')}"
|
97
99
|
if !started && (trdata['bytes_expected'] != 0)
|
98
100
|
spinner&.success
|
99
|
-
notify_begin(@connect_settings['app_id'],trdata['bytes_expected'])
|
101
|
+
notify_begin(@connect_settings['app_id'], trdata['bytes_expected'])
|
100
102
|
started = true
|
101
103
|
else
|
102
|
-
notify_progress(@connect_settings['app_id'],trdata['bytes_written'])
|
104
|
+
notify_progress(@connect_settings['app_id'], trdata['bytes_written'])
|
103
105
|
end
|
104
106
|
when 'failed'
|
105
107
|
spinner&.error
|
@@ -16,12 +16,12 @@ require 'shellwords'
|
|
16
16
|
module Aspera
|
17
17
|
module Fasp
|
18
18
|
# executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
|
19
|
-
class AgentDirect < AgentBase
|
19
|
+
class AgentDirect < Aspera::Fasp::AgentBase
|
20
20
|
# options for initialize (same as values in option transfer_info)
|
21
21
|
DEFAULT_OPTIONS = {
|
22
22
|
spawn_timeout_sec: 3,
|
23
23
|
spawn_delay_sec: 2,
|
24
|
-
wss:
|
24
|
+
wss: true, # true: if both SSH and wss in ts: prefer wss
|
25
25
|
multi_incr_udp: true,
|
26
26
|
resume: {},
|
27
27
|
quiet: true # by default no interactive progress bar
|
@@ -31,12 +31,8 @@ module Aspera
|
|
31
31
|
# start ascp transfer (non blocking), single or multi-session
|
32
32
|
# job information added to @jobs
|
33
33
|
# @param transfer_spec [Hash] aspera transfer specification
|
34
|
-
|
35
|
-
|
36
|
-
raise 'option: must be hash (or nil)' unless options.is_a?(Hash)
|
37
|
-
job_options = options.clone
|
38
|
-
job_options[:resumer] ||= @resume_policy
|
39
|
-
job_options[:job_id] ||= SecureRandom.uuid
|
34
|
+
def start_transfer(transfer_spec)
|
35
|
+
the_job_id = SecureRandom.uuid
|
40
36
|
# clone transfer spec because we modify it (first level keys)
|
41
37
|
transfer_spec = transfer_spec.clone
|
42
38
|
# if there is aspera tags
|
@@ -46,16 +42,16 @@ module Aspera
|
|
46
42
|
# using a non unique id results in discard of tags in AoC, and a package is never finalized
|
47
43
|
# all sessions in a multi-session transfer must have the same xfer_id (see admin manual)
|
48
44
|
transfer_spec['tags']['aspera']['xfer_id'] ||= SecureRandom.uuid
|
49
|
-
Log.log.debug
|
45
|
+
Log.log.debug{"xfer id=#{transfer_spec['xfer_id']}"}
|
50
46
|
# TODO: useful ? node only ?
|
51
47
|
transfer_spec['tags']['aspera']['xfer_retry'] ||= 3600
|
52
48
|
end
|
53
|
-
Log.dump('ts',transfer_spec)
|
49
|
+
Log.dump('ts', transfer_spec)
|
54
50
|
|
55
51
|
# add bypass keys when authentication is token and no auth is provided
|
56
|
-
if transfer_spec.
|
57
|
-
|
58
|
-
|
52
|
+
if transfer_spec.key?('token') &&
|
53
|
+
!transfer_spec.key?('remote_password') &&
|
54
|
+
!transfer_spec.key?('EX_ssh_key_paths')
|
59
55
|
# transfer_spec['remote_password'] = Installation.instance.bypass_pass # not used
|
60
56
|
transfer_spec['EX_ssh_key_paths'] = Installation.instance.bypass_keys
|
61
57
|
end
|
@@ -63,21 +59,21 @@ module Aspera
|
|
63
59
|
# Compute this before using transfer spec because it potentially modifies the transfer spec
|
64
60
|
# (even if the var is not used in single session)
|
65
61
|
multi_session_info = nil
|
66
|
-
if transfer_spec.
|
62
|
+
if transfer_spec.key?('multi_session')
|
67
63
|
multi_session_info = {
|
68
64
|
count: transfer_spec['multi_session'].to_i
|
69
65
|
}
|
70
66
|
# Managed by multi-session, so delete from transfer spec
|
71
67
|
transfer_spec.delete('multi_session')
|
72
68
|
if multi_session_info[:count].negative?
|
73
|
-
Log.log.error
|
69
|
+
Log.log.error{"multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0"}
|
74
70
|
multi_session_info = nil
|
75
71
|
elsif multi_session_info[:count].eql?(0)
|
76
72
|
Log.log.debug('multi_session count is zero: no multisession')
|
77
73
|
multi_session_info = nil
|
78
74
|
elsif @options[:multi_incr_udp] # multi_session_info[:count] > 0
|
79
75
|
# if option not true: keep default udp port for all sessions
|
80
|
-
multi_session_info[:udp_base] = transfer_spec.
|
76
|
+
multi_session_info[:udp_base] = transfer_spec.key?('fasp_port') ? transfer_spec['fasp_port'] : TransferSpec::UDP_PORT
|
81
77
|
# delete from original transfer spec, as we will increment values
|
82
78
|
transfer_spec.delete('fasp_port')
|
83
79
|
# override if specified, else use default value
|
@@ -85,19 +81,19 @@ module Aspera
|
|
85
81
|
end
|
86
82
|
|
87
83
|
# compute known args
|
88
|
-
env_args = Parameters.ts_to_env_args(transfer_spec,wss: @options[:wss])
|
84
|
+
env_args = Parameters.ts_to_env_args(transfer_spec, wss: @options[:wss])
|
89
85
|
|
90
86
|
# add fallback cert and key as arguments if needed
|
91
87
|
if %w[1 force].include?(transfer_spec['http_fallback'])
|
92
|
-
env_args[:args].unshift('-Y',Installation.instance.path(:fallback_key))
|
93
|
-
env_args[:args].unshift('-I',Installation.instance.path(:fallback_cert))
|
88
|
+
env_args[:args].unshift('-Y', Installation.instance.path(:fallback_key))
|
89
|
+
env_args[:args].unshift('-I', Installation.instance.path(:fallback_cert))
|
94
90
|
end
|
95
91
|
|
96
92
|
env_args[:args].unshift('-q') if @options[:quiet]
|
97
93
|
|
98
94
|
# transfer job can be multi session
|
99
95
|
xfer_job = {
|
100
|
-
id:
|
96
|
+
id: the_job_id,
|
101
97
|
sessions: [] # all sessions as below
|
102
98
|
}
|
103
99
|
|
@@ -107,8 +103,7 @@ module Aspera
|
|
107
103
|
error: nil, # exception if failed
|
108
104
|
io: nil, # management port server socket
|
109
105
|
id: nil, # SessionId from INIT message in mgt port
|
110
|
-
env_args: env_args
|
111
|
-
options: job_options # [Hash]
|
106
|
+
env_args: env_args # env vars and args to ascp (from transfer spec)
|
112
107
|
}
|
113
108
|
|
114
109
|
if multi_session_info.nil?
|
@@ -127,7 +122,7 @@ module Aspera
|
|
127
122
|
this_session[:env_args][:args] = this_session[:env_args][:args].clone
|
128
123
|
this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_info[:count]}")
|
129
124
|
# option: increment (default as per ascp manual) or not (cluster on other side ?)
|
130
|
-
this_session[:env_args][:args].unshift('-O',(multi_session_info[:udp_base] + i - 1).to_s) if @options[:multi_incr_udp]
|
125
|
+
this_session[:env_args][:args].unshift('-O', (multi_session_info[:udp_base] + i - 1).to_s) if @options[:multi_incr_udp]
|
131
126
|
this_session[:thread] = Thread.new(this_session) {|s|transfer_thread_entry(s)}
|
132
127
|
xfer_job[:sessions].push(this_session)
|
133
128
|
end
|
@@ -135,10 +130,10 @@ module Aspera
|
|
135
130
|
Log.log.debug('started session thread(s)')
|
136
131
|
|
137
132
|
# add job to list of jobs
|
138
|
-
@jobs[
|
139
|
-
Log.log.debug
|
133
|
+
@jobs[the_job_id] = xfer_job
|
134
|
+
Log.log.debug{"jobs: #{@jobs.keys.count}"}
|
140
135
|
|
141
|
-
return
|
136
|
+
return the_job_id
|
142
137
|
end # start_transfer
|
143
138
|
|
144
139
|
# wait for completion of all jobs started
|
@@ -147,9 +142,9 @@ module Aspera
|
|
147
142
|
Log.log.debug('wait_for_transfers_completion')
|
148
143
|
# set to non-nil to exit loop
|
149
144
|
result = []
|
150
|
-
@jobs.each do |_id,job|
|
145
|
+
@jobs.each do |_id, job|
|
151
146
|
job[:sessions].each do |session|
|
152
|
-
Log.log.debug
|
147
|
+
Log.log.debug{"join #{session[:thread]}"}
|
153
148
|
session[:thread].join
|
154
149
|
result.push(session[:error] || :success)
|
155
150
|
end
|
@@ -173,13 +168,13 @@ module Aspera
|
|
173
168
|
# @param env_args a hash containing :args :env :ascp_version
|
174
169
|
# @param session this session information
|
175
170
|
# could be private method
|
176
|
-
def start_transfer_with_args_env(env_args,session)
|
171
|
+
def start_transfer_with_args_env(env_args, session)
|
177
172
|
raise 'env_args must be Hash' unless env_args.is_a?(Hash)
|
178
173
|
raise 'session must be Hash' unless session.is_a?(Hash)
|
179
174
|
# by default we assume an exception will be raised (for ensure block)
|
180
175
|
exception_raised = true
|
181
176
|
begin
|
182
|
-
Log.log.debug
|
177
|
+
Log.log.debug{"env_args=#{env_args.inspect}"}
|
183
178
|
# get location of ascp executable
|
184
179
|
ascp_path = @mutex.synchronize do
|
185
180
|
Fasp::Installation.instance.path(env_args[:ascp_version])
|
@@ -187,24 +182,24 @@ module Aspera
|
|
187
182
|
# (optional) check it exists
|
188
183
|
raise Fasp::Error, "no such file: #{ascp_path}" unless File.exist?(ascp_path)
|
189
184
|
# open random local TCP port for listening for ascp management
|
190
|
-
mgt_sock = TCPServer.new('127.0.0.1',0)
|
185
|
+
mgt_sock = TCPServer.new('127.0.0.1', 0)
|
191
186
|
# clone arguments as we eed to modify with mgt port
|
192
187
|
ascp_arguments = env_args[:args].clone
|
193
188
|
# add management port
|
194
189
|
ascp_arguments.unshift('-M', mgt_sock.addr[1].to_s)
|
195
190
|
# start ascp in sub process
|
196
191
|
Log.log.debug do
|
197
|
-
'execute: '+
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
192
|
+
'execute: ' +
|
193
|
+
env_args[:env].map{|k, v| "#{k}=#{Shellwords.shellescape(v)}"}.join(' ') +
|
194
|
+
' ' +
|
195
|
+
Shellwords.shellescape(ascp_path) +
|
196
|
+
' ' +
|
197
|
+
ascp_arguments.map{|a|Shellwords.shellescape(a)}.join(' ')
|
203
198
|
end
|
204
199
|
# start process
|
205
|
-
ascp_pid = Process.spawn(env_args[:env],[ascp_path,ascp_path]
|
200
|
+
ascp_pid = Process.spawn(env_args[:env], [ascp_path, ascp_path], *ascp_arguments)
|
206
201
|
# in parent, wait for connection to socket max 3 seconds
|
207
|
-
Log.log.debug
|
202
|
+
Log.log.debug{"before accept for pid (#{ascp_pid})"}
|
208
203
|
# init management socket
|
209
204
|
ascp_mgt_io = nil
|
210
205
|
Timeout.timeout(@options[:spawn_timeout_sec]) do
|
@@ -214,7 +209,7 @@ module Aspera
|
|
214
209
|
# TODO: use same value as Encoding.default_external
|
215
210
|
ascp_mgt_io.set_encoding(Encoding::UTF_8)
|
216
211
|
end
|
217
|
-
Log.log.debug
|
212
|
+
Log.log.debug{"after accept (#{ascp_mgt_io})"}
|
218
213
|
session[:io] = ascp_mgt_io
|
219
214
|
# exact text for event, with \n
|
220
215
|
current_event_text = ''
|
@@ -230,7 +225,7 @@ module Aspera
|
|
230
225
|
break if line.nil?
|
231
226
|
current_event_text += line
|
232
227
|
line.chomp!
|
233
|
-
Log.log.debug
|
228
|
+
Log.log.debug{"line=[#{line}]"}
|
234
229
|
case line
|
235
230
|
when 'FASPMGR 2'
|
236
231
|
# begin event
|
@@ -243,12 +238,12 @@ module Aspera
|
|
243
238
|
# empty line is separator to end event information
|
244
239
|
raise 'unexpected empty line' if current_event_data.nil?
|
245
240
|
current_event_data[AgentBase::LISTENER_SESSION_ID_B] = ascp_pid
|
246
|
-
notify_listeners(current_event_text,current_event_data)
|
241
|
+
notify_listeners(current_event_text, current_event_data)
|
247
242
|
case current_event_data['Type']
|
248
243
|
when 'INIT'
|
249
244
|
session[:id] = current_event_data['SessionId']
|
250
|
-
Log.log.debug
|
251
|
-
when 'DONE','ERROR'
|
245
|
+
Log.log.debug{"session id: #{session[:id]}"}
|
246
|
+
when 'DONE', 'ERROR'
|
252
247
|
# TODO: check if this is always the last event
|
253
248
|
last_status_event = current_event_data
|
254
249
|
end # event type
|
@@ -263,16 +258,20 @@ module Aspera
|
|
263
258
|
# all went well
|
264
259
|
exception_raised = false
|
265
260
|
when 'ERROR'
|
266
|
-
Log.log.error
|
261
|
+
Log.log.error{"code: #{last_status_event['Code']}"}
|
267
262
|
if /bearer token/i.match?(last_status_event['Description'])
|
268
263
|
Log.log.error('need to regenerate token'.red)
|
269
|
-
if
|
264
|
+
if !@token_regenerator.nil?
|
270
265
|
# regenerate token here, expired, or error on it
|
271
266
|
# Note: in multi-session, each session will have a different one.
|
272
|
-
env_args[:env]['ASPERA_SCP_TOKEN'] =
|
267
|
+
env_args[:env]['ASPERA_SCP_TOKEN'] = @token_regenerator.call(true)
|
273
268
|
end
|
274
269
|
end
|
275
|
-
|
270
|
+
# cannot resolve address
|
271
|
+
# if last_status_event['Code'].to_i.eql?(14)
|
272
|
+
# Log.log.warn{"host: #{}"}
|
273
|
+
# end
|
274
|
+
raise Fasp::Error.new(last_status_event['Description'], last_status_event['Code'].to_i)
|
276
275
|
else # case
|
277
276
|
raise "unexpected last event type: #{last_status_event['Type']}"
|
278
277
|
end
|
@@ -312,22 +311,24 @@ module Aspera
|
|
312
311
|
# @param data command on mgt port, examples:
|
313
312
|
# {'type'=>'START','source'=>_path_,'destination'=>_path_}
|
314
313
|
# {'type'=>'DONE'}
|
315
|
-
def send_command(job_id,session_index,data)
|
314
|
+
def send_command(job_id, session_index, data)
|
316
315
|
job = @jobs[job_id]
|
317
316
|
raise 'no such job' if job.nil?
|
318
317
|
session = job[:sessions][session_index]
|
319
318
|
raise 'no such session' if session.nil?
|
320
|
-
Log.log.debug
|
319
|
+
Log.log.debug{"command: #{data}"}
|
321
320
|
# build command
|
322
|
-
command = data
|
323
|
-
keys
|
324
|
-
map{|k|"#{k.capitalize}: #{data[k]}"}
|
325
|
-
unshift('FASPMGR 2')
|
326
|
-
push('','')
|
327
|
-
join("\n")
|
321
|
+
command = data
|
322
|
+
.keys
|
323
|
+
.map{|k|"#{k.capitalize}: #{data[k]}"}
|
324
|
+
.unshift('FASPMGR 2')
|
325
|
+
.push('', '')
|
326
|
+
.join("\n")
|
328
327
|
session[:io].puts(command)
|
329
328
|
end
|
330
329
|
|
330
|
+
attr_writer :token_regenerator
|
331
|
+
|
331
332
|
private
|
332
333
|
|
333
334
|
# @param options : keys(symbol): see DEFAULT_OPTIONS
|
@@ -341,13 +342,14 @@ module Aspera
|
|
341
342
|
@options = DEFAULT_OPTIONS.dup
|
342
343
|
if !options.nil?
|
343
344
|
raise "expecting Hash (or nil), but have #{options.class}" unless options.is_a?(Hash)
|
344
|
-
options.each do |k,v|
|
345
|
-
raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.
|
345
|
+
options.each do |k, v|
|
346
|
+
raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.key?(k)
|
346
347
|
@options[k] = v
|
347
348
|
end
|
348
349
|
end
|
349
|
-
Log.log.debug
|
350
|
+
Log.log.debug{"local options= #{options}"}
|
350
351
|
@resume_policy = ResumePolicy.new(@options[:resume].symbolize_keys)
|
352
|
+
@token_regenerator = nil
|
351
353
|
end
|
352
354
|
|
353
355
|
# transfer thread entry
|
@@ -356,17 +358,17 @@ module Aspera
|
|
356
358
|
begin
|
357
359
|
# set name for logging
|
358
360
|
Thread.current[:name] = 'transfer'
|
359
|
-
Log.log.debug
|
361
|
+
Log.log.debug{"ENTER (#{Thread.current[:name]})"}
|
360
362
|
# start transfer with selected resumer policy
|
361
|
-
|
362
|
-
start_transfer_with_args_env(session[:env_args],session)
|
363
|
+
@resume_policy.execute_with_resume do
|
364
|
+
start_transfer_with_args_env(session[:env_args], session)
|
363
365
|
end
|
364
366
|
Log.log.debug('transfer ok'.bg_green)
|
365
367
|
rescue StandardError => e
|
366
368
|
session[:error] = e
|
367
|
-
Log.log.error
|
369
|
+
Log.log.error{"Transfer thread error: #{e.class}:\n#{e.message}:\n#{e.backtrace.join("\n")}".red} if Log.instance.level.eql?(:debug)
|
368
370
|
end
|
369
|
-
Log.log.debug
|
371
|
+
Log.log.debug{"EXIT (#{Thread.current[:name]})"}
|
370
372
|
end
|
371
373
|
end # AgentDirect
|
372
374
|
end
|