aspera-cli 4.4.0 → 4.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1042 -787
- data/bin/ascli +1 -1
- data/bin/asession +3 -5
- data/docs/Makefile +4 -7
- data/docs/README.erb.md +988 -740
- data/examples/faspex4.rb +4 -6
- data/examples/transfer.rb +2 -2
- data/lib/aspera/aoc.rb +139 -118
- data/lib/aspera/cli/listener/progress_multi.rb +5 -5
- data/lib/aspera/cli/main.rb +64 -34
- data/lib/aspera/cli/manager.rb +19 -20
- data/lib/aspera/cli/plugin.rb +9 -1
- data/lib/aspera/cli/plugins/aoc.rb +156 -143
- data/lib/aspera/cli/plugins/ats.rb +11 -10
- data/lib/aspera/cli/plugins/bss.rb +2 -2
- data/lib/aspera/cli/plugins/config.rb +236 -112
- data/lib/aspera/cli/plugins/faspex.rb +29 -7
- data/lib/aspera/cli/plugins/faspex5.rb +21 -8
- data/lib/aspera/cli/plugins/node.rb +21 -9
- data/lib/aspera/cli/plugins/orchestrator.rb +5 -3
- data/lib/aspera/cli/plugins/preview.rb +2 -2
- data/lib/aspera/cli/plugins/server.rb +3 -3
- data/lib/aspera/cli/plugins/shares.rb +17 -0
- data/lib/aspera/cli/transfer_agent.rb +47 -85
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/environment.rb +4 -4
- data/lib/aspera/fasp/{manager.rb → agent_base.rb} +7 -6
- data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +46 -39
- data/lib/aspera/fasp/{local.rb → agent_direct.rb} +14 -17
- data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +4 -4
- data/lib/aspera/fasp/{node.rb → agent_node.rb} +25 -8
- data/lib/aspera/fasp/agent_trsdk.rb +106 -0
- data/lib/aspera/fasp/default.rb +17 -0
- data/lib/aspera/fasp/installation.rb +64 -48
- data/lib/aspera/fasp/parameters.rb +7 -3
- data/lib/aspera/faspex_gw.rb +6 -6
- data/lib/aspera/keychain/encrypted_hash.rb +120 -0
- data/lib/aspera/keychain/macos_security.rb +94 -0
- data/lib/aspera/log.rb +45 -32
- data/lib/aspera/node.rb +3 -6
- data/lib/aspera/rest.rb +65 -49
- metadata +68 -27
- data/lib/aspera/api_detector.rb +0 -60
- data/lib/aspera/secrets.rb +0 -20
@@ -1,4 +1,4 @@
|
|
1
|
-
require 'aspera/fasp/
|
1
|
+
require 'aspera/fasp/agent_base'
|
2
2
|
require 'aspera/rest'
|
3
3
|
require 'aspera/open_application'
|
4
4
|
require 'securerandom'
|
@@ -6,19 +6,15 @@ require 'tty-spinner'
|
|
6
6
|
|
7
7
|
module Aspera
|
8
8
|
module Fasp
|
9
|
-
class
|
9
|
+
class AgentConnect < AgentBase
|
10
10
|
MAX_CONNECT_START_RETRY=3
|
11
11
|
SLEEP_SEC_BETWEEN_RETRY=2
|
12
12
|
private_constant :MAX_CONNECT_START_RETRY,:SLEEP_SEC_BETWEEN_RETRY
|
13
|
-
def initialize
|
14
|
-
super
|
13
|
+
def initialize(options)
|
14
|
+
super()
|
15
15
|
@connect_settings={
|
16
16
|
'app_id' => SecureRandom.uuid
|
17
17
|
}
|
18
|
-
# TODO: start here and create monitor
|
19
|
-
end
|
20
|
-
|
21
|
-
def start_transfer(transfer_spec,options=nil)
|
22
18
|
raise 'Using connect requires a graphical environment' if !OpenApplication.default_gui_mode.eql?(:graphical)
|
23
19
|
trynumber=0
|
24
20
|
begin
|
@@ -26,6 +22,7 @@ module Aspera
|
|
26
22
|
Log.log.debug("found: #{connect_url}")
|
27
23
|
@connect_api=Rest.new({base_url: "#{connect_url}/v5/connect",headers: {'Origin'=>Rest.user_agent}}) # could use v6 also now
|
28
24
|
cinfo=@connect_api.read('info/version')[:data]
|
25
|
+
Log.dump(:connect_version,cinfo)
|
29
26
|
rescue => e # Errno::ECONNREFUSED
|
30
27
|
raise StandardError,"Unable to start connect after #{trynumber} try" if trynumber >= MAX_CONNECT_START_RETRY
|
31
28
|
Log.log.warn("connect is not started. Retry ##{trynumber}, err=#{e}")
|
@@ -37,6 +34,9 @@ module Aspera
|
|
37
34
|
sleep(SLEEP_SEC_BETWEEN_RETRY)
|
38
35
|
retry
|
39
36
|
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def start_transfer(transfer_spec,options=nil)
|
40
40
|
if transfer_spec['direction'] == 'send'
|
41
41
|
Log.log.warn("Connect requires upload selection using GUI, ignoring #{transfer_spec['paths']}".red)
|
42
42
|
transfer_spec.delete('paths')
|
@@ -47,7 +47,6 @@ module Aspera
|
|
47
47
|
# if there is a token, we ask connect client to use well known ssh private keys
|
48
48
|
# instead of asking password
|
49
49
|
transfer_spec['authentication']='token' if transfer_spec.has_key?('token')
|
50
|
-
connect_settings=
|
51
50
|
connect_transfer_args={
|
52
51
|
'aspera_connect_settings'=>@connect_settings.merge({
|
53
52
|
'request_id' =>@request_id,
|
@@ -65,42 +64,50 @@ module Aspera
|
|
65
64
|
connect_activity_args={'aspera_connect_settings'=>@connect_settings}
|
66
65
|
started=false
|
67
66
|
spinner=nil
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
# TODO: get session id
|
77
|
-
case trdata['status']
|
78
|
-
when 'completed'
|
79
|
-
notify_end(@connect_settings['app_id'])
|
80
|
-
break
|
81
|
-
when 'initiating','queued'
|
82
|
-
if spinner.nil?
|
83
|
-
spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
|
84
|
-
spinner.start
|
67
|
+
begin
|
68
|
+
loop do
|
69
|
+
tr_info=@connect_api.create("transfers/info/#{@xfer_id}",connect_activity_args)[:data]
|
70
|
+
if tr_info['transfer_info'].is_a?(Hash)
|
71
|
+
trdata=tr_info['transfer_info']
|
72
|
+
if trdata.nil?
|
73
|
+
Log.log.warn("no session in Connect")
|
74
|
+
break
|
85
75
|
end
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
76
|
+
# TODO: get session id
|
77
|
+
case trdata['status']
|
78
|
+
when 'completed'
|
79
|
+
notify_end(@connect_settings['app_id'])
|
80
|
+
break
|
81
|
+
when 'initiating','queued'
|
82
|
+
if spinner.nil?
|
83
|
+
spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
|
84
|
+
spinner.start
|
85
|
+
end
|
86
|
+
spinner.update(title: trdata['status'])
|
87
|
+
spinner.spin
|
88
|
+
when 'running'
|
89
|
+
#puts "running: sessions:#{trdata['sessions'].length}, #{trdata['sessions'].map{|i| i['bytes_transferred']}.join(',')}"
|
90
|
+
if !started and trdata['bytes_expected'] != 0
|
91
|
+
spinner.success unless spinner.nil?
|
92
|
+
notify_begin(@connect_settings['app_id'],trdata['bytes_expected'])
|
93
|
+
started=true
|
94
|
+
else
|
95
|
+
notify_progress(@connect_settings['app_id'],trdata['bytes_written'])
|
96
|
+
end
|
97
|
+
when 'failed'
|
98
|
+
spinner.error unless spinner.nil?
|
99
|
+
raise Fasp::Error.new(trdata['error_desc'])
|
93
100
|
else
|
94
|
-
|
101
|
+
raise Fasp::Error.new("unknown status: #{trdata['status']}: #{trdata['error_desc']}")
|
95
102
|
end
|
96
|
-
else
|
97
|
-
raise Fasp::Error.new("unknown status: #{trdata['status']}: #{trdata['error_desc']}")
|
98
103
|
end
|
104
|
+
sleep 1
|
99
105
|
end
|
100
|
-
|
106
|
+
rescue => e
|
107
|
+
return [e]
|
101
108
|
end
|
102
|
-
return []
|
109
|
+
return [:success]
|
103
110
|
end # wait
|
104
|
-
end #
|
111
|
+
end # AgentConnect
|
105
112
|
end
|
106
113
|
end
|
@@ -5,11 +5,12 @@
|
|
5
5
|
# Laurent Martin
|
6
6
|
#
|
7
7
|
##############################################################################
|
8
|
-
require 'aspera/fasp/
|
8
|
+
require 'aspera/fasp/agent_base'
|
9
9
|
require 'aspera/fasp/error'
|
10
10
|
require 'aspera/fasp/parameters'
|
11
11
|
require 'aspera/fasp/installation'
|
12
12
|
require 'aspera/fasp/resume_policy'
|
13
|
+
require 'aspera/fasp/default'
|
13
14
|
require 'aspera/log'
|
14
15
|
require 'socket'
|
15
16
|
require 'timeout'
|
@@ -18,19 +19,17 @@ require 'securerandom'
|
|
18
19
|
module Aspera
|
19
20
|
module Fasp
|
20
21
|
# executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
|
21
|
-
class
|
22
|
+
class AgentDirect < AgentBase
|
22
23
|
# options for initialize (same as values in option transfer_info)
|
23
24
|
DEFAULT_OPTIONS = {
|
24
|
-
:
|
25
|
-
:
|
26
|
-
:
|
27
|
-
:
|
28
|
-
:
|
25
|
+
spawn_timeout_sec: 3,
|
26
|
+
spawn_delay_sec: 2,
|
27
|
+
wss: false,
|
28
|
+
multi_incr_udp: true,
|
29
|
+
resume: {},
|
30
|
+
quiet: true # by default no interactive progress bar
|
29
31
|
}
|
30
|
-
DEFAULT_UDP_PORT=33001
|
31
32
|
private_constant :DEFAULT_OPTIONS
|
32
|
-
# set to false to keep ascp progress bar display ("true" adds ascp's option -q)
|
33
|
-
attr_accessor :quiet
|
34
33
|
|
35
34
|
# start ascp transfer (non blocking), single or multi-session
|
36
35
|
# job information added to @jobs
|
@@ -76,13 +75,13 @@ module Aspera
|
|
76
75
|
Log.log.error("multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0")
|
77
76
|
multi_session_info = nil
|
78
77
|
elsif multi_session_info[:count].eql?(0)
|
79
|
-
Log.log.debug("multi_session
|
78
|
+
Log.log.debug("multi_session count is zero: no multisession")
|
80
79
|
multi_session_info = nil
|
81
80
|
else # multi_session_info[:count] > 0
|
82
81
|
# if option not true: keep default udp port for all sessions
|
83
82
|
if @options[:multi_incr_udp]
|
84
83
|
# override if specified, else use default value
|
85
|
-
multi_session_info[:udp_base]=transfer_spec.has_key?('fasp_port') ? transfer_spec['fasp_port'] :
|
84
|
+
multi_session_info[:udp_base]=transfer_spec.has_key?('fasp_port') ? transfer_spec['fasp_port'] : Default::UDP_PORT
|
86
85
|
# delete from original transfer spec, as we will increment values
|
87
86
|
transfer_spec.delete('fasp_port')
|
88
87
|
end
|
@@ -98,7 +97,7 @@ module Aspera
|
|
98
97
|
env_args[:args].unshift('-I',Installation.instance.path(:fallback_cert))
|
99
98
|
end
|
100
99
|
|
101
|
-
env_args[:args].unshift('-q') if @quiet
|
100
|
+
env_args[:args].unshift('-q') if @options[:quiet]
|
102
101
|
|
103
102
|
# transfer job can be multi session
|
104
103
|
xfer_job={
|
@@ -240,7 +239,7 @@ module Aspera
|
|
240
239
|
when ''
|
241
240
|
# empty line is separator to end event information
|
242
241
|
raise 'unexpected empty line' if current_event_data.nil?
|
243
|
-
current_event_data[
|
242
|
+
current_event_data[AgentBase::LISTENER_SESSION_ID_B]=ascp_pid
|
244
243
|
notify_listeners(current_event_text,current_event_data)
|
245
244
|
case current_event_data['Type']
|
246
245
|
when 'INIT'
|
@@ -333,8 +332,6 @@ module Aspera
|
|
333
332
|
# @param options : keys(symbol): see DEFAULT_OPTIONS
|
334
333
|
def initialize(options=nil)
|
335
334
|
super()
|
336
|
-
# by default no interactive progress bar
|
337
|
-
@quiet=true
|
338
335
|
# all transfer jobs, key = SecureRandom.uuid, protected by mutex, condvar on change
|
339
336
|
@jobs={}
|
340
337
|
# mutex protects global data accessed by threads
|
@@ -374,6 +371,6 @@ module Aspera
|
|
374
371
|
Log.log.debug("EXIT (#{Thread.current[:name]})")
|
375
372
|
end
|
376
373
|
|
377
|
-
end #
|
374
|
+
end # AgentDirect
|
378
375
|
end
|
379
376
|
end
|
@@ -1,5 +1,5 @@
|
|
1
1
|
#!/bin/echo this is a ruby class:
|
2
|
-
require 'aspera/fasp/
|
2
|
+
require 'aspera/fasp/agent_base'
|
3
3
|
require 'aspera/log'
|
4
4
|
require 'aspera/rest'
|
5
5
|
require 'websocket-client-simple'
|
@@ -12,7 +12,7 @@ require 'json'
|
|
12
12
|
module Aspera
|
13
13
|
module Fasp
|
14
14
|
# start a transfer using Aspera HTTP Gateway, using web socket session
|
15
|
-
class
|
15
|
+
class AgentHttpgw < AgentBase
|
16
16
|
# message returned by HTTP GW in case of success
|
17
17
|
OK_MESSAGE='end upload'
|
18
18
|
# refresh rate for progress
|
@@ -179,7 +179,7 @@ module Aspera
|
|
179
179
|
when 'receive'
|
180
180
|
download(transfer_spec)
|
181
181
|
else
|
182
|
-
raise "
|
182
|
+
raise "unexpected direction: [#{transfer_spec['direction']}]"
|
183
183
|
end
|
184
184
|
end # start_transfer
|
185
185
|
|
@@ -209,6 +209,6 @@ module Aspera
|
|
209
209
|
@upload_chunksize=128000 # TODO: configurable ?
|
210
210
|
end
|
211
211
|
|
212
|
-
end #
|
212
|
+
end # AgentHttpgw
|
213
213
|
end
|
214
214
|
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require 'aspera/fasp/
|
1
|
+
require 'aspera/fasp/agent_base'
|
2
2
|
require 'aspera/log'
|
3
3
|
require 'tty-spinner'
|
4
4
|
|
@@ -6,13 +6,30 @@ module Aspera
|
|
6
6
|
module Fasp
|
7
7
|
# this singleton class is used by the CLI to provide a common interface to start a transfer
|
8
8
|
# before using it, the use must set the `node_api` member.
|
9
|
-
class
|
9
|
+
class AgentNode < AgentBase
|
10
10
|
# option include: root_id if the node is an access key
|
11
11
|
attr_writer :options
|
12
|
-
def initialize(
|
12
|
+
def initialize(options)
|
13
|
+
raise "node specification must be Hash" unless options.is_a?(Hash)
|
14
|
+
[:url,:username,:password].each { |k| raise "missing parameter [#{k}] in node specification: #{options}" unless options.has_key?(k) }
|
13
15
|
super()
|
14
|
-
|
15
|
-
@
|
16
|
+
# root id is required for access key
|
17
|
+
@root_id=options[:root_id]
|
18
|
+
rest_params={ base_url: options[:url]}
|
19
|
+
if options[:password].match(/^Bearer /)
|
20
|
+
rest_params[:headers]={
|
21
|
+
'X-Aspera-AccessKey'=>options[:username],
|
22
|
+
'Authorization' =>options[:password]
|
23
|
+
}
|
24
|
+
raise "root_id is required for access key" if @root_id.nil?
|
25
|
+
else
|
26
|
+
rest_params[:auth]={
|
27
|
+
type: :basic,
|
28
|
+
username: options[:username],
|
29
|
+
password: options[:password]
|
30
|
+
}
|
31
|
+
end
|
32
|
+
@node_api=Rest.new(rest_params)
|
16
33
|
# TODO: currently only supports one transfer. This is bad shortcut. but ok for CLI.
|
17
34
|
@transfer_id=nil
|
18
35
|
end
|
@@ -36,10 +53,10 @@ module Aspera
|
|
36
53
|
# generic method
|
37
54
|
def start_transfer(transfer_spec,options=nil)
|
38
55
|
# add root id if access key
|
39
|
-
if @
|
56
|
+
if ! @root_id.nil?
|
40
57
|
case transfer_spec['direction']
|
41
|
-
when 'send';transfer_spec['source_root_id']=@
|
42
|
-
when 'receive';transfer_spec['destination_root_id']=@
|
58
|
+
when 'send';transfer_spec['source_root_id']=@root_id
|
59
|
+
when 'receive';transfer_spec['destination_root_id']=@root_id
|
43
60
|
else raise "unexpected direction in ts: #{transfer_spec['direction']}"
|
44
61
|
end
|
45
62
|
end
|
@@ -0,0 +1,106 @@
|
|
1
|
+
require 'aspera/fasp/agent_base'
|
2
|
+
require 'aspera/fasp/installation'
|
3
|
+
require 'json'
|
4
|
+
|
5
|
+
module Aspera
|
6
|
+
module Fasp
|
7
|
+
class AgentTrsdk < AgentBase
|
8
|
+
DEFAULT_OPTIONS = {
|
9
|
+
address: '127.0.0.1',
|
10
|
+
port: 55002,
|
11
|
+
}
|
12
|
+
private_constant :DEFAULT_OPTIONS
|
13
|
+
|
14
|
+
# options come from transfer_info
|
15
|
+
def initialize(user_opts)
|
16
|
+
raise "expecting Hash (or nil), but have #{user_opts.class}" unless user_opts.nil? or user_opts.is_a?(Hash)
|
17
|
+
# set default options and override if specified
|
18
|
+
options=DEFAULT_OPTIONS.clone
|
19
|
+
if !user_opts.nil?
|
20
|
+
user_opts.each do |k,v|
|
21
|
+
if DEFAULT_OPTIONS.has_key?(k)
|
22
|
+
options[k]=v
|
23
|
+
else
|
24
|
+
raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map{|i|i.to_s}.join(",")}"
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
Log.log.debug("options= #{options}")
|
29
|
+
super()
|
30
|
+
# load and create SDK stub
|
31
|
+
$LOAD_PATH.unshift(Installation.instance.sdk_ruby_folder)
|
32
|
+
require 'transfer_services_pb'
|
33
|
+
@transfer_client = Transfersdk::TransferService::Stub.new("#{options[:address]}:#{options[:port]}",:this_channel_is_insecure)
|
34
|
+
begin
|
35
|
+
get_info_response = @transfer_client.get_info(Transfersdk::InstanceInfoRequest.new)
|
36
|
+
Log.log.debug("daemon info: #{get_info_response}")
|
37
|
+
rescue GRPC::Unavailable => e
|
38
|
+
Log.log.warn("no daemon present, starting daemon...")
|
39
|
+
# location of daemon binary
|
40
|
+
bin_folder=File.realpath(File.join(Installation.instance.sdk_ruby_folder,'..'))
|
41
|
+
# config file and logs are created in same folder
|
42
|
+
conf_file = File.join(bin_folder,'sdk.conf')
|
43
|
+
log_base = File.join(bin_folder,'transferd')
|
44
|
+
# create a config file for daemon
|
45
|
+
config = {
|
46
|
+
address: options[:address],
|
47
|
+
port: options[:port],
|
48
|
+
fasp_runtime: {
|
49
|
+
use_embedded: false,
|
50
|
+
user_defined: {
|
51
|
+
bin: bin_folder,
|
52
|
+
etc: bin_folder,
|
53
|
+
}
|
54
|
+
}
|
55
|
+
}
|
56
|
+
File.write(conf_file,config.to_json)
|
57
|
+
trd_pid = Process.spawn(Installation.instance.path(:transferd),'--config' , conf_file, out: "#{log_base}.out", err: "#{log_base}.err")
|
58
|
+
Process.detach(trd_pid)
|
59
|
+
sleep(2.0)
|
60
|
+
retry
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def start_transfer(transfer_spec,options=nil)
|
65
|
+
# create a transfer request
|
66
|
+
transfer_request = Transfersdk::TransferRequest.new(
|
67
|
+
transferType: Transfersdk::TransferType::FILE_REGULAR, # transfer type (file/stream)
|
68
|
+
config: Transfersdk::TransferConfig.new, # transfer configuration
|
69
|
+
transferSpec: transfer_spec.to_json) # transfer definition
|
70
|
+
# send start transfer request to the transfer manager daemon
|
71
|
+
start_transfer_response = @transfer_client.start_transfer(transfer_request)
|
72
|
+
Log.log.debug("start transfer response #{start_transfer_response}")
|
73
|
+
@transfer_id = start_transfer_response.transferId
|
74
|
+
Log.log.debug("transfer started with id #{@transfer_id}")
|
75
|
+
end
|
76
|
+
|
77
|
+
def wait_for_transfers_completion
|
78
|
+
started=false
|
79
|
+
# monitor transfer status
|
80
|
+
@transfer_client.monitor_transfers(Transfersdk::RegistrationRequest.new(transferId: [@transfer_id])) do |response|
|
81
|
+
Log.dump(:response, response.to_h)
|
82
|
+
#Log.log.debug("#{response.sessionInfo.preTransferBytes} #{response.transferInfo.bytesTransferred}")
|
83
|
+
case response.status
|
84
|
+
when :RUNNING
|
85
|
+
if !started and !response.sessionInfo.preTransferBytes.eql?(0)
|
86
|
+
notify_begin(@transfer_id,response.sessionInfo.preTransferBytes)
|
87
|
+
started=true
|
88
|
+
elsif started
|
89
|
+
notify_progress(@transfer_id,response.transferInfo.bytesTransferred)
|
90
|
+
end
|
91
|
+
when :FAILED, :COMPLETED, :CANCELED
|
92
|
+
notify_end(@transfer_id)
|
93
|
+
raise Fasp::Error.new(JSON.parse(response.message)['Description']) unless :COMPLETED.eql?(response.status)
|
94
|
+
break
|
95
|
+
when :QUEUED,:UNKNOWN_STATUS,:PAUSED,:ORPHANED
|
96
|
+
# ignore
|
97
|
+
else
|
98
|
+
Log.log.error("unknown status#{response.status}")
|
99
|
+
end
|
100
|
+
end
|
101
|
+
# TODO return status
|
102
|
+
return []
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module Aspera
|
2
|
+
module Fasp
|
3
|
+
# default parameters
|
4
|
+
class Default
|
5
|
+
# (public) default transfer username for access key based transfers
|
6
|
+
ACCESS_KEY_TRANSFER_USER='xfer'
|
7
|
+
SSH_PORT=33001
|
8
|
+
UDP_PORT=33001
|
9
|
+
AK_TSPEC_BASE={
|
10
|
+
'remote_user' => ACCESS_KEY_TRANSFER_USER,
|
11
|
+
'ssh_port' => SSH_PORT,
|
12
|
+
'fasp_port' => UDP_PORT
|
13
|
+
}
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
@@ -10,7 +10,7 @@ require 'fileutils'
|
|
10
10
|
module Aspera
|
11
11
|
module Fasp
|
12
12
|
# Singleton that tells where to find ascp and other local resources (keys..) , using the "path(symb)" method.
|
13
|
-
# It is used by object :
|
13
|
+
# It is used by object : AgentDirect to find necessary resources
|
14
14
|
# By default it takes the first Aspera product found specified in product_locations
|
15
15
|
# but the user can specify ascp location by calling:
|
16
16
|
# Installation.instance.use_ascp_from_product(product_name)
|
@@ -22,6 +22,9 @@ module Aspera
|
|
22
22
|
PRODUCT_CLI_V1='Aspera CLI'
|
23
23
|
PRODUCT_DRIVE='Aspera Drive'
|
24
24
|
PRODUCT_ENTSRV='Enterprise Server'
|
25
|
+
# protobuf generated files from sdk
|
26
|
+
EXT_RUBY_PROTOBUF='_pb.rb'
|
27
|
+
RB_SDK_FOLDER='lib'
|
25
28
|
MAX_REDIRECT_SDK=2
|
26
29
|
private_constant :MAX_REDIRECT_SDK
|
27
30
|
# set ascp executable path
|
@@ -29,9 +32,10 @@ module Aspera
|
|
29
32
|
@path_to_ascp=v
|
30
33
|
end
|
31
34
|
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
+
def sdk_ruby_folder
|
36
|
+
ruby_pb_folder=File.join(folder_path,RB_SDK_FOLDER)
|
37
|
+
FileUtils.mkdir_p(ruby_pb_folder) unless Dir.exist?(ruby_pb_folder)
|
38
|
+
return ruby_pb_folder
|
35
39
|
end
|
36
40
|
|
37
41
|
# location of SDK files
|
@@ -100,6 +104,8 @@ module Aspera
|
|
100
104
|
file=@path_to_ascp
|
101
105
|
# note that there might be a .exe at the end
|
102
106
|
file=file.gsub('ascp','ascp4') if k.eql?(:ascp4)
|
107
|
+
when :transferd
|
108
|
+
file=transferd_filepath
|
103
109
|
when :ssh_bypass_key_dsa
|
104
110
|
file=File.join(folder_path,'aspera_bypass_dsa.pem')
|
105
111
|
File.write(file,get_key('dsa',1)) unless File.exist?(file)
|
@@ -182,16 +188,22 @@ module Aspera
|
|
182
188
|
return [:ssh_bypass_key_dsa,:ssh_bypass_key_rsa].map{|i|Installation.instance.path(i)}
|
183
189
|
end
|
184
190
|
|
191
|
+
# use in plugin `config`
|
192
|
+
def get_ascp_version(exe_path)
|
193
|
+
return get_exe_version(exe_path,'-A')
|
194
|
+
end
|
195
|
+
|
185
196
|
# Check that specified path is ascp and get version
|
186
|
-
def
|
187
|
-
raise "
|
188
|
-
|
189
|
-
|
190
|
-
cmd_out=%x{"#{
|
197
|
+
def get_exe_version(exe_path,vers_arg)
|
198
|
+
raise "ERROR: nil arg" if exe_path.nil?
|
199
|
+
return nil unless File.exist?(exe_path)
|
200
|
+
exe_version=nil
|
201
|
+
cmd_out=%x{"#{exe_path}" #{vers_arg}}
|
191
202
|
raise "An error occured when testing #{ascp_filename}: #{cmd_out}" unless $? == 0
|
192
203
|
# get version from ascp, only after full extract, as windows requires DLLs (SSL/TLS/etc...)
|
193
|
-
m=cmd_out.match(/
|
194
|
-
|
204
|
+
m=cmd_out.match(/ version ([0-9\.]+)/)
|
205
|
+
exe_version=m[1] unless m.nil?
|
206
|
+
return exe_version
|
195
207
|
end
|
196
208
|
|
197
209
|
# download aspera SDK or use local file
|
@@ -205,54 +217,49 @@ module Aspera
|
|
205
217
|
raise 'use format: file:///<path>' unless sdk_url.start_with?('file:///')
|
206
218
|
sdk_zip_path=sdk_url.gsub(%r{^file:///},'')
|
207
219
|
else
|
208
|
-
|
209
|
-
begin
|
210
|
-
Aspera::Rest.new(base_url: sdk_url).call(operation: 'GET',save_to_file: sdk_zip_path)
|
211
|
-
rescue Aspera::RestCallError => e
|
212
|
-
if e.response.is_a?(Net::HTTPRedirection)
|
213
|
-
if redirect_remain > 0
|
214
|
-
redirect_remain-=1
|
215
|
-
sdk_url=e.response['location']
|
216
|
-
retry
|
217
|
-
else
|
218
|
-
raise "Too many redirect"
|
219
|
-
end
|
220
|
-
else
|
221
|
-
raise e
|
222
|
-
end
|
223
|
-
end
|
220
|
+
Aspera::Rest.new(base_url: sdk_url, redirect_max: 3).call(operation: 'GET',save_to_file: sdk_zip_path)
|
224
221
|
end
|
225
|
-
# SDK is organized by architecture
|
226
|
-
filter="/#{Environment.architecture}/"
|
227
|
-
ascp_path=nil
|
228
|
-
sdk_path=folder_path
|
229
222
|
# rename old install
|
230
|
-
if
|
231
|
-
Log.log.warn("Previous install exists, renaming.")
|
232
|
-
File.rename(
|
223
|
+
if ! Dir.empty?(folder_path)
|
224
|
+
Log.log.warn("Previous install exists, renaming folder.")
|
225
|
+
File.rename(folder_path,"#{folder_path}.#{Time.now.strftime("%Y%m%d%H%M%S")}")
|
226
|
+
# TODO: delete old archives ?
|
233
227
|
end
|
234
|
-
#
|
235
|
-
|
236
|
-
|
228
|
+
# SDK is organized by architecture
|
229
|
+
arch_filter="#{Environment.architecture}/"
|
230
|
+
# extract files from archive
|
237
231
|
Zip::File.open(sdk_zip_path) do |zip_file|
|
238
232
|
zip_file.each do |entry|
|
239
|
-
#
|
240
|
-
if entry.name.
|
241
|
-
|
242
|
-
|
233
|
+
# skip folder entries
|
234
|
+
next if entry.name.end_with?('/')
|
235
|
+
dest_folder=nil
|
236
|
+
# binaries
|
237
|
+
dest_folder=folder_path if entry.name.include?(arch_filter)
|
238
|
+
# ruby adapters
|
239
|
+
dest_folder=sdk_ruby_folder if entry.name.end_with?(EXT_RUBY_PROTOBUF)
|
240
|
+
if !dest_folder.nil?
|
241
|
+
File.open(File.join(dest_folder,File.basename(entry.name)), 'wb') do |output_stream|
|
243
242
|
IO.copy_stream(entry.get_input_stream, output_stream)
|
244
243
|
end
|
245
|
-
if File.basename(entry.name).eql?(ascp_filename)
|
246
|
-
FileUtils.chmod(0755,archive_file)
|
247
|
-
ascp_path=archive_file
|
248
|
-
end
|
249
244
|
end
|
250
245
|
end
|
251
246
|
end
|
252
247
|
File.unlink(sdk_zip_path) rescue nil # Windows may give error
|
253
|
-
|
254
|
-
|
255
|
-
|
248
|
+
# ensure license file are generated so that ascp invokation for version works
|
249
|
+
self.path(:aspera_license)
|
250
|
+
self.path(:aspera_conf)
|
251
|
+
ascp_path=File.join(folder_path,ascp_filename)
|
252
|
+
raise "No #{ascp_filename} found in SDK archive" unless File.exist?(ascp_path)
|
253
|
+
FileUtils.chmod(0755,ascp_path)
|
254
|
+
FileUtils.chmod(0755,ascp_path.gsub('ascp','ascp4'))
|
255
|
+
ascp_version=get_ascp_version(File.join(folder_path,ascp_filename))
|
256
|
+
trd_path=transferd_filepath
|
257
|
+
Log.log.warn("No #{trd_path} in SDK archive") unless File.exist?(trd_path)
|
258
|
+
FileUtils.chmod(0755,trd_path) if File.exist?(trd_path)
|
259
|
+
transferd_version=get_exe_version(trd_path,'version')
|
260
|
+
sdk_version = transferd_version||ascp_version
|
261
|
+
File.write(File.join(folder_path,PRODUCT_INFO),"<product><name>IBM Aspera SDK</name><version>#{sdk_version}</version></product>")
|
262
|
+
return sdk_version
|
256
263
|
end
|
257
264
|
|
258
265
|
private
|
@@ -288,6 +295,15 @@ module Aspera
|
|
288
295
|
@sdk_folder
|
289
296
|
end
|
290
297
|
|
298
|
+
# filename for ascp with optional extension (Windows)
|
299
|
+
def ascp_filename
|
300
|
+
return 'ascp'+Environment.exe_extension
|
301
|
+
end
|
302
|
+
|
303
|
+
def transferd_filepath
|
304
|
+
return File.join(folder_path,'asperatransferd'+Environment.exe_extension)
|
305
|
+
end
|
306
|
+
|
291
307
|
# @return product folders depending on OS fields
|
292
308
|
# :expected M app name is taken from the manifest if present, else defaults to this value
|
293
309
|
# :app_root M main folder for the application
|
@@ -7,6 +7,7 @@ require 'json'
|
|
7
7
|
require 'yaml'
|
8
8
|
require 'securerandom'
|
9
9
|
require 'fileutils'
|
10
|
+
require 'openssl'
|
10
11
|
|
11
12
|
module Aspera
|
12
13
|
module Fasp
|
@@ -26,7 +27,7 @@ module Aspera
|
|
26
27
|
Aspera::CommandLineBuilder.normalize_description(@@param_description_cache)
|
27
28
|
end
|
28
29
|
|
29
|
-
# Agents shown in manual
|
30
|
+
# Agents shown in manual for parameters (sub list)
|
30
31
|
SUPPORTED_AGENTS=[:direct,:node,:connect]
|
31
32
|
# Short names of columns in manual
|
32
33
|
SUPPORTED_AGENTS_SHORT=SUPPORTED_AGENTS.map{|a|a.to_s[0].to_sym}
|
@@ -111,6 +112,9 @@ module Aspera
|
|
111
112
|
@job_spec.delete('fasp_port')
|
112
113
|
@job_spec.delete('EX_ssh_key_paths')
|
113
114
|
@job_spec.delete('sshfp')
|
115
|
+
# set default location for CA bundle, see env var SSL_CERT_FILE / SSL_CERT_DIR
|
116
|
+
@job_spec['EX_ssh_key_paths']=[OpenSSL::X509::DEFAULT_CERT_FILE]
|
117
|
+
Log.log.debug("CA certs: EX_ssh_key_paths <- DEFAULT_CERT_FILE from openssl")
|
114
118
|
else
|
115
119
|
# remove unused parameter (avoid warning)
|
116
120
|
@job_spec.delete('wss_port')
|
@@ -163,8 +167,8 @@ module Aspera
|
|
163
167
|
lines=paths_array.map{|i|i['source']}
|
164
168
|
end
|
165
169
|
file_list_file=Aspera::TempFileManager.instance.new_file_path_in_folder(@@file_list_folder)
|
166
|
-
File.open(file_list_file, 'w+'){|f|f.
|
167
|
-
Log.log.debug
|
170
|
+
File.open(file_list_file, 'w+'){|f|f.write(lines.join("\n"))}
|
171
|
+
Log.log.debug{"#{option}=\n#{File.read(file_list_file)}".red}
|
168
172
|
end
|
169
173
|
end
|
170
174
|
@builder.add_command_line_options(["#{option}=#{file_list_file}"])
|