aspera-cli 4.2.1 → 4.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1580 -946
- data/bin/ascli +1 -1
- data/bin/asession +3 -5
- data/docs/Makefile +8 -11
- data/docs/README.erb.md +1521 -829
- data/docs/doc_tools.rb +58 -0
- data/docs/test_env.conf +3 -1
- data/examples/faspex4.rb +28 -19
- data/examples/transfer.rb +2 -2
- data/lib/aspera/aoc.rb +157 -134
- data/lib/aspera/cli/listener/progress_multi.rb +5 -5
- data/lib/aspera/cli/main.rb +106 -48
- data/lib/aspera/cli/manager.rb +19 -20
- data/lib/aspera/cli/plugin.rb +22 -7
- data/lib/aspera/cli/plugins/aoc.rb +260 -208
- data/lib/aspera/cli/plugins/ats.rb +11 -10
- data/lib/aspera/cli/plugins/bss.rb +2 -2
- data/lib/aspera/cli/plugins/config.rb +360 -189
- data/lib/aspera/cli/plugins/faspex.rb +119 -56
- data/lib/aspera/cli/plugins/faspex5.rb +32 -17
- data/lib/aspera/cli/plugins/node.rb +72 -31
- data/lib/aspera/cli/plugins/orchestrator.rb +5 -3
- data/lib/aspera/cli/plugins/preview.rb +94 -68
- data/lib/aspera/cli/plugins/server.rb +16 -5
- data/lib/aspera/cli/plugins/shares.rb +17 -0
- data/lib/aspera/cli/transfer_agent.rb +64 -82
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +48 -31
- data/lib/aspera/cos_node.rb +4 -3
- data/lib/aspera/environment.rb +4 -4
- data/lib/aspera/fasp/{manager.rb → agent_base.rb} +7 -6
- data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +46 -39
- data/lib/aspera/fasp/{local.rb → agent_direct.rb} +42 -38
- data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +50 -29
- data/lib/aspera/fasp/{node.rb → agent_node.rb} +43 -4
- data/lib/aspera/fasp/agent_trsdk.rb +106 -0
- data/lib/aspera/fasp/default.rb +17 -0
- data/lib/aspera/fasp/installation.rb +64 -48
- data/lib/aspera/fasp/parameters.rb +78 -91
- data/lib/aspera/fasp/parameters.yaml +531 -0
- data/lib/aspera/fasp/uri.rb +1 -1
- data/lib/aspera/faspex_gw.rb +12 -11
- data/lib/aspera/id_generator.rb +22 -0
- data/lib/aspera/keychain/encrypted_hash.rb +120 -0
- data/lib/aspera/keychain/macos_security.rb +94 -0
- data/lib/aspera/log.rb +45 -32
- data/lib/aspera/node.rb +9 -4
- data/lib/aspera/oauth.rb +116 -100
- data/lib/aspera/persistency_action_once.rb +11 -7
- data/lib/aspera/persistency_folder.rb +6 -26
- data/lib/aspera/rest.rb +66 -50
- data/lib/aspera/sync.rb +40 -35
- data/lib/aspera/timer_limiter.rb +22 -0
- metadata +86 -29
- data/docs/transfer_spec.html +0 -99
- data/lib/aspera/api_detector.rb +0 -60
- data/lib/aspera/fasp/aoc.rb +0 -24
- data/lib/aspera/secrets.rb +0 -20
@@ -4,21 +4,45 @@ module Aspera
|
|
4
4
|
# process_param is called repeatedly with all known parameters
|
5
5
|
# add_env_args is called to get resulting param list and env var (also checks that all params were used)
|
6
6
|
class CommandLineBuilder
|
7
|
+
# transform yes/no to trye/false
|
8
|
+
def self.yes_to_true(value)
|
9
|
+
case value
|
10
|
+
when 'yes'; return true
|
11
|
+
when 'no'; return false
|
12
|
+
end
|
13
|
+
raise "unsupported value: #{value}"
|
14
|
+
end
|
7
15
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
16
|
+
# Called by provider of definition before constructor of this class so that params_definition has all mandatory fields
|
17
|
+
def self.normalize_description(d)
|
18
|
+
d.each do |param_name,options|
|
19
|
+
raise "Expecting Hash, but have #{options.class} in #{param_name}" unless options.is_a?(Hash)
|
20
|
+
#options[:accepted_types]=:bool if options[:cltype].eql?(:envvar) and !options.has_key?(:accepted_types)
|
21
|
+
# by default : not mandatory
|
22
|
+
options[:mandatory]||=false
|
23
|
+
options[:desc]||=''
|
24
|
+
# by default : string, unless it's without arg
|
25
|
+
if ! options.has_key?(:accepted_types)
|
26
|
+
options[:accepted_types]=options[:cltype].eql?(:opt_without_arg) ? :bool : :string
|
27
|
+
end
|
28
|
+
# single type is placed in array
|
29
|
+
options[:accepted_types]=[options[:accepted_types]] unless options[:accepted_types].is_a?(Array)
|
30
|
+
if !options.has_key?(:option_switch) and options.has_key?(:cltype) and [:opt_without_arg,:opt_with_arg].include?(options[:cltype])
|
31
|
+
options[:option_switch]='--'+param_name.to_s.gsub('_','-')
|
32
|
+
end
|
33
|
+
end
|
13
34
|
end
|
14
35
|
|
36
|
+
private
|
37
|
+
|
38
|
+
# clvarname : command line variable name
|
15
39
|
def env_name(param_name,options)
|
16
|
-
return options[:
|
40
|
+
return options[:clvarname]
|
17
41
|
end
|
18
42
|
|
19
43
|
public
|
20
44
|
|
21
|
-
|
45
|
+
attr_reader :params_definition
|
22
46
|
|
23
47
|
# @param param_hash
|
24
48
|
def initialize(param_hash,params_definition)
|
@@ -44,15 +68,6 @@ module Aspera
|
|
44
68
|
return nil
|
45
69
|
end
|
46
70
|
|
47
|
-
# transform yes/no to trye/false
|
48
|
-
def self.yes_to_true(value)
|
49
|
-
case value
|
50
|
-
when 'yes'; return true
|
51
|
-
when 'no'; return false
|
52
|
-
end
|
53
|
-
raise "unsupported value: #{value}"
|
54
|
-
end
|
55
|
-
|
56
71
|
# add options directly to ascp command line
|
57
72
|
def add_command_line_options(options)
|
58
73
|
return if options.nil?
|
@@ -71,24 +86,25 @@ module Aspera
|
|
71
86
|
# @param options : options for type
|
72
87
|
def process_param(param_name,action=nil)
|
73
88
|
options=@params_definition[param_name]
|
74
|
-
action=options[:
|
89
|
+
action=options[:cltype] if action.nil?
|
75
90
|
# should not happen
|
76
91
|
raise "Internal error: ask processing of param #{param_name}" if options.nil?
|
77
|
-
# by default : not mandatory
|
78
|
-
options[:mandatory]||=false
|
79
|
-
if options.has_key?(:accepted_types)
|
80
|
-
# single type is placed in array
|
81
|
-
options[:accepted_types]=[options[:accepted_types]] unless options[:accepted_types].is_a?(Array)
|
82
|
-
else
|
83
|
-
# by default : string, unless it's without arg
|
84
|
-
options[:accepted_types]=action.eql?(:opt_without_arg) ? BOOLEAN_CLASSES : [String]
|
85
|
-
end
|
86
92
|
# check mandatory parameter (nil is valid value)
|
87
93
|
raise Fasp::Error.new("mandatory parameter: #{param_name}") if options[:mandatory] and !@param_hash.has_key?(param_name)
|
88
94
|
parameter_value=@param_hash[param_name]
|
89
|
-
parameter_value=options[:default] if parameter_value.nil? and options.has_key?(:default)
|
95
|
+
#parameter_value=options[:default] if parameter_value.nil? and options.has_key?(:default)
|
96
|
+
expected_classes=options[:accepted_types].map do |s|
|
97
|
+
case s
|
98
|
+
when :string; String
|
99
|
+
when :array; Array
|
100
|
+
when :hash; Hash
|
101
|
+
when :int; Integer
|
102
|
+
when :bool; [TrueClass,FalseClass]
|
103
|
+
else raise "INTERNAL: unexpected value: #{s}"
|
104
|
+
end
|
105
|
+
end.flatten
|
90
106
|
# check provided type
|
91
|
-
raise Fasp::Error.new("#{param_name} is : #{parameter_value.class} (#{parameter_value}), shall be #{options[:accepted_types]}, ") unless parameter_value.nil? or
|
107
|
+
raise Fasp::Error.new("#{param_name} is : #{parameter_value.class} (#{parameter_value}), shall be #{options[:accepted_types]}, ") unless parameter_value.nil? or expected_classes.include?(parameter_value.class)
|
92
108
|
@used_param_names.push(param_name) unless action.eql?(:defer)
|
93
109
|
|
94
110
|
# process only non-nil values
|
@@ -102,7 +118,8 @@ module Aspera
|
|
102
118
|
end
|
103
119
|
raise "unsupported value: #{parameter_value}" unless options[:accepted_values].nil? or options[:accepted_values].include?(parameter_value)
|
104
120
|
if options[:encode]
|
105
|
-
|
121
|
+
# :encode has name of class with encoding method
|
122
|
+
newvalue=Kernel.const_get(options[:encode]).send("encode_#{param_name}",parameter_value)
|
106
123
|
raise Fasp::Error.new("unsupported #{param_name}: #{parameter_value}") if newvalue.nil?
|
107
124
|
parameter_value=newvalue
|
108
125
|
end
|
@@ -123,12 +140,12 @@ module Aspera
|
|
123
140
|
else raise Fasp::Error.new("unsupported #{param_name}: #{parameter_value}")
|
124
141
|
end
|
125
142
|
add_param=!add_param if options[:add_on_false]
|
126
|
-
add_command_line_options([
|
143
|
+
add_command_line_options([options[:option_switch]]) if add_param
|
127
144
|
when :opt_with_arg # transform into command line option with value
|
128
145
|
#parameter_value=parameter_value.to_s if parameter_value.is_a?(Integer)
|
129
146
|
parameter_value=[parameter_value] unless parameter_value.is_a?(Array)
|
130
147
|
# if transfer_spec value is an array, applies option many times
|
131
|
-
parameter_value.each{|v|add_command_line_options([
|
148
|
+
parameter_value.each{|v|add_command_line_options([options[:option_switch],v])}
|
132
149
|
else
|
133
150
|
raise "Error"
|
134
151
|
end
|
data/lib/aspera/cos_node.rb
CHANGED
@@ -6,6 +6,7 @@ module Aspera
|
|
6
6
|
class CosNode < Rest
|
7
7
|
attr_reader :add_ts
|
8
8
|
IBM_CLOUD_TOKEN_URL='https://iam.cloud.ibm.com/identity'
|
9
|
+
TOKEN_FIELD='delegated_refresh_token'
|
9
10
|
def initialize(bucket_name,storage_endpoint,instance_id,api_key,auth_url=IBM_CLOUD_TOKEN_URL)
|
10
11
|
@auth_url=auth_url
|
11
12
|
@api_key=api_key
|
@@ -32,7 +33,7 @@ module Aspera
|
|
32
33
|
# prepare transfer spec addition
|
33
34
|
@add_ts={'tags'=>{'aspera'=>{'node'=>{'storage_credentials'=>{
|
34
35
|
'type' => 'token',
|
35
|
-
'token' => {
|
36
|
+
'token' => {TOKEN_FIELD=>nil}
|
36
37
|
}}}}}
|
37
38
|
generate_token
|
38
39
|
end
|
@@ -45,10 +46,10 @@ module Aspera
|
|
45
46
|
:base_url => @auth_url,
|
46
47
|
:grant => :delegated_refresh,
|
47
48
|
:api_key => @api_key,
|
48
|
-
:token_field=>
|
49
|
+
:token_field=> TOKEN_FIELD
|
49
50
|
})
|
50
51
|
# get delagated token to be placed in rest call header and in transfer tags
|
51
|
-
@add_ts['tags']['aspera']['node']['storage_credentials']['token'][
|
52
|
+
@add_ts['tags']['aspera']['node']['storage_credentials']['token'][TOKEN_FIELD]=delegated_oauth.get_authorization().gsub(/^Bearer /,'')
|
52
53
|
@params[:headers]={'X-Aspera-Storage-Credentials'=>JSON.generate(@add_ts['tags']['aspera']['node']['storage_credentials'])}
|
53
54
|
end
|
54
55
|
end
|
data/lib/aspera/environment.rb
CHANGED
@@ -33,12 +33,12 @@ module Aspera
|
|
33
33
|
def self.cpu
|
34
34
|
case RbConfig::CONFIG['host_cpu']
|
35
35
|
when /x86_64/,/x64/
|
36
|
-
return
|
36
|
+
return CPU_X86_64
|
37
37
|
when /powerpc/
|
38
|
-
return
|
39
|
-
return
|
38
|
+
return CPU_PPC64LE if os.eql?(OS_LINUX)
|
39
|
+
return CPU_PPC64
|
40
40
|
when /s390/
|
41
|
-
return
|
41
|
+
return CPU_S390
|
42
42
|
else # other
|
43
43
|
raise "Unknown CPU: #{RbConfig::CONFIG['host_cpu']}"
|
44
44
|
end
|
@@ -2,7 +2,7 @@ module Aspera
|
|
2
2
|
module Fasp
|
3
3
|
# Base class for FASP transfer agents
|
4
4
|
# sub classes shall implement start_transfer and shutdown
|
5
|
-
class
|
5
|
+
class AgentBase
|
6
6
|
|
7
7
|
private
|
8
8
|
|
@@ -68,17 +68,18 @@ module Aspera
|
|
68
68
|
self
|
69
69
|
end
|
70
70
|
|
71
|
-
# the following methods must be implemented by subclass:
|
72
|
-
# start_transfer(transfer_spec,options) : start and wait for completion
|
73
|
-
# wait_for_transfers_completion : wait for termination of all transfers, @return list of : :success or error message
|
74
|
-
# optional: shutdown
|
75
|
-
|
76
71
|
# This checks the validity of the value returned by wait_for_transfers_completion
|
77
72
|
# it must be a list of :success or exception
|
78
73
|
def self.validate_status_list(statuses)
|
79
74
|
raise "internal error: bad statuses type: #{statuses.class}" unless statuses.is_a?(Array)
|
80
75
|
raise "internal error: bad statuses content: #{statuses}" unless statuses.select{|i|!i.eql?(:success) and !i.is_a?(StandardError)}.empty?
|
81
76
|
end
|
77
|
+
|
78
|
+
# the following methods must be implemented by subclass:
|
79
|
+
# start_transfer(transfer_spec,options) : start and wait for completion
|
80
|
+
# wait_for_transfers_completion : wait for termination of all transfers, @return list of : :success or error message
|
81
|
+
# optional: shutdown
|
82
|
+
|
82
83
|
end
|
83
84
|
end
|
84
85
|
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require 'aspera/fasp/
|
1
|
+
require 'aspera/fasp/agent_base'
|
2
2
|
require 'aspera/rest'
|
3
3
|
require 'aspera/open_application'
|
4
4
|
require 'securerandom'
|
@@ -6,19 +6,15 @@ require 'tty-spinner'
|
|
6
6
|
|
7
7
|
module Aspera
|
8
8
|
module Fasp
|
9
|
-
class
|
9
|
+
class AgentConnect < AgentBase
|
10
10
|
MAX_CONNECT_START_RETRY=3
|
11
11
|
SLEEP_SEC_BETWEEN_RETRY=2
|
12
12
|
private_constant :MAX_CONNECT_START_RETRY,:SLEEP_SEC_BETWEEN_RETRY
|
13
|
-
def initialize
|
14
|
-
super
|
13
|
+
def initialize(options)
|
14
|
+
super()
|
15
15
|
@connect_settings={
|
16
16
|
'app_id' => SecureRandom.uuid
|
17
17
|
}
|
18
|
-
# TODO: start here and create monitor
|
19
|
-
end
|
20
|
-
|
21
|
-
def start_transfer(transfer_spec,options=nil)
|
22
18
|
raise 'Using connect requires a graphical environment' if !OpenApplication.default_gui_mode.eql?(:graphical)
|
23
19
|
trynumber=0
|
24
20
|
begin
|
@@ -26,6 +22,7 @@ module Aspera
|
|
26
22
|
Log.log.debug("found: #{connect_url}")
|
27
23
|
@connect_api=Rest.new({base_url: "#{connect_url}/v5/connect",headers: {'Origin'=>Rest.user_agent}}) # could use v6 also now
|
28
24
|
cinfo=@connect_api.read('info/version')[:data]
|
25
|
+
Log.dump(:connect_version,cinfo)
|
29
26
|
rescue => e # Errno::ECONNREFUSED
|
30
27
|
raise StandardError,"Unable to start connect after #{trynumber} try" if trynumber >= MAX_CONNECT_START_RETRY
|
31
28
|
Log.log.warn("connect is not started. Retry ##{trynumber}, err=#{e}")
|
@@ -37,6 +34,9 @@ module Aspera
|
|
37
34
|
sleep(SLEEP_SEC_BETWEEN_RETRY)
|
38
35
|
retry
|
39
36
|
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def start_transfer(transfer_spec,options=nil)
|
40
40
|
if transfer_spec['direction'] == 'send'
|
41
41
|
Log.log.warn("Connect requires upload selection using GUI, ignoring #{transfer_spec['paths']}".red)
|
42
42
|
transfer_spec.delete('paths')
|
@@ -47,7 +47,6 @@ module Aspera
|
|
47
47
|
# if there is a token, we ask connect client to use well known ssh private keys
|
48
48
|
# instead of asking password
|
49
49
|
transfer_spec['authentication']='token' if transfer_spec.has_key?('token')
|
50
|
-
connect_settings=
|
51
50
|
connect_transfer_args={
|
52
51
|
'aspera_connect_settings'=>@connect_settings.merge({
|
53
52
|
'request_id' =>@request_id,
|
@@ -65,42 +64,50 @@ module Aspera
|
|
65
64
|
connect_activity_args={'aspera_connect_settings'=>@connect_settings}
|
66
65
|
started=false
|
67
66
|
spinner=nil
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
# TODO: get session id
|
77
|
-
case trdata['status']
|
78
|
-
when 'completed'
|
79
|
-
notify_end(@connect_settings['app_id'])
|
80
|
-
break
|
81
|
-
when 'initiating','queued'
|
82
|
-
if spinner.nil?
|
83
|
-
spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
|
84
|
-
spinner.start
|
67
|
+
begin
|
68
|
+
loop do
|
69
|
+
tr_info=@connect_api.create("transfers/info/#{@xfer_id}",connect_activity_args)[:data]
|
70
|
+
if tr_info['transfer_info'].is_a?(Hash)
|
71
|
+
trdata=tr_info['transfer_info']
|
72
|
+
if trdata.nil?
|
73
|
+
Log.log.warn("no session in Connect")
|
74
|
+
break
|
85
75
|
end
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
76
|
+
# TODO: get session id
|
77
|
+
case trdata['status']
|
78
|
+
when 'completed'
|
79
|
+
notify_end(@connect_settings['app_id'])
|
80
|
+
break
|
81
|
+
when 'initiating','queued'
|
82
|
+
if spinner.nil?
|
83
|
+
spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
|
84
|
+
spinner.start
|
85
|
+
end
|
86
|
+
spinner.update(title: trdata['status'])
|
87
|
+
spinner.spin
|
88
|
+
when 'running'
|
89
|
+
#puts "running: sessions:#{trdata['sessions'].length}, #{trdata['sessions'].map{|i| i['bytes_transferred']}.join(',')}"
|
90
|
+
if !started and trdata['bytes_expected'] != 0
|
91
|
+
spinner.success unless spinner.nil?
|
92
|
+
notify_begin(@connect_settings['app_id'],trdata['bytes_expected'])
|
93
|
+
started=true
|
94
|
+
else
|
95
|
+
notify_progress(@connect_settings['app_id'],trdata['bytes_written'])
|
96
|
+
end
|
97
|
+
when 'failed'
|
98
|
+
spinner.error unless spinner.nil?
|
99
|
+
raise Fasp::Error.new(trdata['error_desc'])
|
93
100
|
else
|
94
|
-
|
101
|
+
raise Fasp::Error.new("unknown status: #{trdata['status']}: #{trdata['error_desc']}")
|
95
102
|
end
|
96
|
-
else
|
97
|
-
raise Fasp::Error.new("unknown status: #{trdata['status']}: #{trdata['error_desc']}")
|
98
103
|
end
|
104
|
+
sleep 1
|
99
105
|
end
|
100
|
-
|
106
|
+
rescue => e
|
107
|
+
return [e]
|
101
108
|
end
|
102
|
-
return []
|
109
|
+
return [:success]
|
103
110
|
end # wait
|
104
|
-
end #
|
111
|
+
end # AgentConnect
|
105
112
|
end
|
106
113
|
end
|
@@ -5,11 +5,12 @@
|
|
5
5
|
# Laurent Martin
|
6
6
|
#
|
7
7
|
##############################################################################
|
8
|
-
require 'aspera/fasp/
|
8
|
+
require 'aspera/fasp/agent_base'
|
9
9
|
require 'aspera/fasp/error'
|
10
10
|
require 'aspera/fasp/parameters'
|
11
11
|
require 'aspera/fasp/installation'
|
12
12
|
require 'aspera/fasp/resume_policy'
|
13
|
+
require 'aspera/fasp/default'
|
13
14
|
require 'aspera/log'
|
14
15
|
require 'socket'
|
15
16
|
require 'timeout'
|
@@ -17,21 +18,18 @@ require 'securerandom'
|
|
17
18
|
|
18
19
|
module Aspera
|
19
20
|
module Fasp
|
20
|
-
# (public) default transfer username for access key based transfers
|
21
|
-
ACCESS_KEY_TRANSFER_USER='xfer'
|
22
21
|
# executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
|
23
|
-
class
|
24
|
-
# options for initialize
|
22
|
+
class AgentDirect < AgentBase
|
23
|
+
# options for initialize (same as values in option transfer_info)
|
25
24
|
DEFAULT_OPTIONS = {
|
26
|
-
:
|
27
|
-
:
|
28
|
-
:
|
29
|
-
:
|
25
|
+
spawn_timeout_sec: 3,
|
26
|
+
spawn_delay_sec: 2,
|
27
|
+
wss: false,
|
28
|
+
multi_incr_udp: true,
|
29
|
+
resume: {},
|
30
|
+
quiet: true # by default no interactive progress bar
|
30
31
|
}
|
31
|
-
DEFAULT_UDP_PORT=33001
|
32
32
|
private_constant :DEFAULT_OPTIONS
|
33
|
-
# set to false to keep ascp progress bar display ("true" adds ascp's option -q)
|
34
|
-
attr_accessor :quiet
|
35
33
|
|
36
34
|
# start ascp transfer (non blocking), single or multi-session
|
37
35
|
# job information added to @jobs
|
@@ -64,21 +62,27 @@ module Aspera
|
|
64
62
|
transfer_spec['EX_ssh_key_paths'] = Installation.instance.bypass_keys
|
65
63
|
end
|
66
64
|
|
67
|
-
#
|
68
|
-
#
|
69
|
-
|
70
|
-
multi_session_number=0
|
65
|
+
# Compute this before using transfer spec because it potentially modifies the transfer spec
|
66
|
+
# (even if the var is not used in single session)
|
67
|
+
multi_session_info=nil
|
71
68
|
if transfer_spec.has_key?('multi_session')
|
72
|
-
|
73
|
-
|
69
|
+
multi_session_info={
|
70
|
+
count: transfer_spec['multi_session'].to_i,
|
71
|
+
}
|
72
|
+
# Managed by multi-session, so delete from transfer spec
|
73
|
+
transfer_spec.delete('multi_session')
|
74
|
+
if multi_session_info[:count] < 0
|
74
75
|
Log.log.error("multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0")
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
if
|
81
|
-
|
76
|
+
multi_session_info = nil
|
77
|
+
elsif multi_session_info[:count].eql?(0)
|
78
|
+
Log.log.debug("multi_session count is zero: no multisession")
|
79
|
+
multi_session_info = nil
|
80
|
+
else # multi_session_info[:count] > 0
|
81
|
+
# if option not true: keep default udp port for all sessions
|
82
|
+
if @options[:multi_incr_udp]
|
83
|
+
# override if specified, else use default value
|
84
|
+
multi_session_info[:udp_base]=transfer_spec.has_key?('fasp_port') ? transfer_spec['fasp_port'] : Default::UDP_PORT
|
85
|
+
# delete from original transfer spec, as we will increment values
|
82
86
|
transfer_spec.delete('fasp_port')
|
83
87
|
end
|
84
88
|
end
|
@@ -93,7 +97,7 @@ module Aspera
|
|
93
97
|
env_args[:args].unshift('-I',Installation.instance.path(:fallback_cert))
|
94
98
|
end
|
95
99
|
|
96
|
-
env_args[:args].unshift('-q') if @quiet
|
100
|
+
env_args[:args].unshift('-q') if @options[:quiet]
|
97
101
|
|
98
102
|
# transfer job can be multi session
|
99
103
|
xfer_job={
|
@@ -111,22 +115,23 @@ module Aspera
|
|
111
115
|
:options => job_options # [Hash]
|
112
116
|
}
|
113
117
|
|
114
|
-
if
|
118
|
+
if multi_session_info.nil?
|
115
119
|
Log.log.debug('Starting single session thread')
|
116
120
|
# single session for transfer : simple
|
117
121
|
session[:thread] = Thread.new(session) {|s|transfer_thread_entry(s)}
|
118
122
|
xfer_job[:sessions].push(session)
|
119
123
|
else
|
120
124
|
Log.log.debug('Starting multi session threads')
|
121
|
-
1.upto(
|
125
|
+
1.upto(multi_session_info[:count]) do |i|
|
126
|
+
# do not delay the first session
|
122
127
|
sleep(@options[:spawn_delay_sec]) unless i.eql?(1)
|
123
128
|
# do deep copy (each thread has its own copy because it is modified here below and in thread)
|
124
129
|
this_session=session.clone()
|
125
130
|
this_session[:env_args]=this_session[:env_args].clone()
|
126
131
|
this_session[:env_args][:args]=this_session[:env_args][:args].clone()
|
127
|
-
this_session[:env_args][:args].unshift("-C#{i}:#{
|
128
|
-
#
|
129
|
-
this_session[:env_args][:args].unshift('-O',"#{
|
132
|
+
this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_info[:count]}")
|
133
|
+
# option: increment (default as per ascp manual) or not (cluster on other side ?)
|
134
|
+
this_session[:env_args][:args].unshift('-O',"#{multi_session_info[:udp_base]+i-1}") if @options[:multi_incr_udp]
|
130
135
|
this_session[:thread] = Thread.new(this_session) {|s|transfer_thread_entry(s)}
|
131
136
|
xfer_job[:sessions].push(this_session)
|
132
137
|
end
|
@@ -234,7 +239,7 @@ module Aspera
|
|
234
239
|
when ''
|
235
240
|
# empty line is separator to end event information
|
236
241
|
raise 'unexpected empty line' if current_event_data.nil?
|
237
|
-
current_event_data[
|
242
|
+
current_event_data[AgentBase::LISTENER_SESSION_ID_B]=ascp_pid
|
238
243
|
notify_listeners(current_event_text,current_event_data)
|
239
244
|
case current_event_data['Type']
|
240
245
|
when 'INIT'
|
@@ -260,6 +265,7 @@ module Aspera
|
|
260
265
|
Log.log.error('need to regenerate token'.red)
|
261
266
|
if session[:options].is_a?(Hash) and session[:options].has_key?(:regenerate_token)
|
262
267
|
# regenerate token here, expired, or error on it
|
268
|
+
# Note: in multi-session, each session will have a different one.
|
263
269
|
env_args[:env]['ASPERA_SCP_TOKEN']=session[:options][:regenerate_token].call(true)
|
264
270
|
end
|
265
271
|
end
|
@@ -323,16 +329,14 @@ module Aspera
|
|
323
329
|
|
324
330
|
private
|
325
331
|
|
326
|
-
# @param options : keys(symbol):
|
332
|
+
# @param options : keys(symbol): see DEFAULT_OPTIONS
|
327
333
|
def initialize(options=nil)
|
328
334
|
super()
|
329
|
-
# by default no interactive progress bar
|
330
|
-
@quiet=true
|
331
335
|
# all transfer jobs, key = SecureRandom.uuid, protected by mutex, condvar on change
|
332
336
|
@jobs={}
|
333
337
|
# mutex protects global data accessed by threads
|
334
338
|
@mutex=Mutex.new
|
335
|
-
#
|
339
|
+
# set default options and override if specified
|
336
340
|
@options=DEFAULT_OPTIONS.clone
|
337
341
|
if !options.nil?
|
338
342
|
raise "expecting Hash (or nil), but have #{options.class}" unless options.is_a?(Hash)
|
@@ -340,7 +344,7 @@ module Aspera
|
|
340
344
|
if DEFAULT_OPTIONS.has_key?(k)
|
341
345
|
@options[k]=v
|
342
346
|
else
|
343
|
-
raise "
|
347
|
+
raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map{|i|i.to_s}.join(",")}"
|
344
348
|
end
|
345
349
|
end
|
346
350
|
end
|
@@ -367,6 +371,6 @@ module Aspera
|
|
367
371
|
Log.log.debug("EXIT (#{Thread.current[:name]})")
|
368
372
|
end
|
369
373
|
|
370
|
-
end #
|
374
|
+
end # AgentDirect
|
371
375
|
end
|
372
376
|
end
|