aspera-cli 4.4.0 → 4.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +1042 -787
- data/bin/ascli +1 -1
- data/bin/asession +3 -5
- data/docs/Makefile +4 -7
- data/docs/README.erb.md +988 -740
- data/examples/faspex4.rb +4 -6
- data/examples/transfer.rb +2 -2
- data/lib/aspera/aoc.rb +139 -118
- data/lib/aspera/cli/listener/progress_multi.rb +5 -5
- data/lib/aspera/cli/main.rb +64 -34
- data/lib/aspera/cli/manager.rb +19 -20
- data/lib/aspera/cli/plugin.rb +9 -1
- data/lib/aspera/cli/plugins/aoc.rb +156 -143
- data/lib/aspera/cli/plugins/ats.rb +11 -10
- data/lib/aspera/cli/plugins/bss.rb +2 -2
- data/lib/aspera/cli/plugins/config.rb +236 -112
- data/lib/aspera/cli/plugins/faspex.rb +29 -7
- data/lib/aspera/cli/plugins/faspex5.rb +21 -8
- data/lib/aspera/cli/plugins/node.rb +21 -9
- data/lib/aspera/cli/plugins/orchestrator.rb +5 -3
- data/lib/aspera/cli/plugins/preview.rb +2 -2
- data/lib/aspera/cli/plugins/server.rb +3 -3
- data/lib/aspera/cli/plugins/shares.rb +17 -0
- data/lib/aspera/cli/transfer_agent.rb +47 -85
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/environment.rb +4 -4
- data/lib/aspera/fasp/{manager.rb → agent_base.rb} +7 -6
- data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +46 -39
- data/lib/aspera/fasp/{local.rb → agent_direct.rb} +14 -17
- data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +4 -4
- data/lib/aspera/fasp/{node.rb → agent_node.rb} +25 -8
- data/lib/aspera/fasp/agent_trsdk.rb +106 -0
- data/lib/aspera/fasp/default.rb +17 -0
- data/lib/aspera/fasp/installation.rb +64 -48
- data/lib/aspera/fasp/parameters.rb +7 -3
- data/lib/aspera/faspex_gw.rb +6 -6
- data/lib/aspera/keychain/encrypted_hash.rb +120 -0
- data/lib/aspera/keychain/macos_security.rb +94 -0
- data/lib/aspera/log.rb +45 -32
- data/lib/aspera/node.rb +3 -6
- data/lib/aspera/rest.rb +65 -49
- metadata +68 -27
- data/lib/aspera/api_detector.rb +0 -60
- data/lib/aspera/secrets.rb +0 -20
@@ -1,4 +1,4 @@
|
|
1
|
-
require 'aspera/fasp/
|
1
|
+
require 'aspera/fasp/agent_base'
|
2
2
|
require 'aspera/rest'
|
3
3
|
require 'aspera/open_application'
|
4
4
|
require 'securerandom'
|
@@ -6,19 +6,15 @@ require 'tty-spinner'
|
|
6
6
|
|
7
7
|
module Aspera
|
8
8
|
module Fasp
|
9
|
-
class
|
9
|
+
class AgentConnect < AgentBase
|
10
10
|
MAX_CONNECT_START_RETRY=3
|
11
11
|
SLEEP_SEC_BETWEEN_RETRY=2
|
12
12
|
private_constant :MAX_CONNECT_START_RETRY,:SLEEP_SEC_BETWEEN_RETRY
|
13
|
-
def initialize
|
14
|
-
super
|
13
|
+
def initialize(options)
|
14
|
+
super()
|
15
15
|
@connect_settings={
|
16
16
|
'app_id' => SecureRandom.uuid
|
17
17
|
}
|
18
|
-
# TODO: start here and create monitor
|
19
|
-
end
|
20
|
-
|
21
|
-
def start_transfer(transfer_spec,options=nil)
|
22
18
|
raise 'Using connect requires a graphical environment' if !OpenApplication.default_gui_mode.eql?(:graphical)
|
23
19
|
trynumber=0
|
24
20
|
begin
|
@@ -26,6 +22,7 @@ module Aspera
|
|
26
22
|
Log.log.debug("found: #{connect_url}")
|
27
23
|
@connect_api=Rest.new({base_url: "#{connect_url}/v5/connect",headers: {'Origin'=>Rest.user_agent}}) # could use v6 also now
|
28
24
|
cinfo=@connect_api.read('info/version')[:data]
|
25
|
+
Log.dump(:connect_version,cinfo)
|
29
26
|
rescue => e # Errno::ECONNREFUSED
|
30
27
|
raise StandardError,"Unable to start connect after #{trynumber} try" if trynumber >= MAX_CONNECT_START_RETRY
|
31
28
|
Log.log.warn("connect is not started. Retry ##{trynumber}, err=#{e}")
|
@@ -37,6 +34,9 @@ module Aspera
|
|
37
34
|
sleep(SLEEP_SEC_BETWEEN_RETRY)
|
38
35
|
retry
|
39
36
|
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def start_transfer(transfer_spec,options=nil)
|
40
40
|
if transfer_spec['direction'] == 'send'
|
41
41
|
Log.log.warn("Connect requires upload selection using GUI, ignoring #{transfer_spec['paths']}".red)
|
42
42
|
transfer_spec.delete('paths')
|
@@ -47,7 +47,6 @@ module Aspera
|
|
47
47
|
# if there is a token, we ask connect client to use well known ssh private keys
|
48
48
|
# instead of asking password
|
49
49
|
transfer_spec['authentication']='token' if transfer_spec.has_key?('token')
|
50
|
-
connect_settings=
|
51
50
|
connect_transfer_args={
|
52
51
|
'aspera_connect_settings'=>@connect_settings.merge({
|
53
52
|
'request_id' =>@request_id,
|
@@ -65,42 +64,50 @@ module Aspera
|
|
65
64
|
connect_activity_args={'aspera_connect_settings'=>@connect_settings}
|
66
65
|
started=false
|
67
66
|
spinner=nil
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
# TODO: get session id
|
77
|
-
case trdata['status']
|
78
|
-
when 'completed'
|
79
|
-
notify_end(@connect_settings['app_id'])
|
80
|
-
break
|
81
|
-
when 'initiating','queued'
|
82
|
-
if spinner.nil?
|
83
|
-
spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
|
84
|
-
spinner.start
|
67
|
+
begin
|
68
|
+
loop do
|
69
|
+
tr_info=@connect_api.create("transfers/info/#{@xfer_id}",connect_activity_args)[:data]
|
70
|
+
if tr_info['transfer_info'].is_a?(Hash)
|
71
|
+
trdata=tr_info['transfer_info']
|
72
|
+
if trdata.nil?
|
73
|
+
Log.log.warn("no session in Connect")
|
74
|
+
break
|
85
75
|
end
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
76
|
+
# TODO: get session id
|
77
|
+
case trdata['status']
|
78
|
+
when 'completed'
|
79
|
+
notify_end(@connect_settings['app_id'])
|
80
|
+
break
|
81
|
+
when 'initiating','queued'
|
82
|
+
if spinner.nil?
|
83
|
+
spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
|
84
|
+
spinner.start
|
85
|
+
end
|
86
|
+
spinner.update(title: trdata['status'])
|
87
|
+
spinner.spin
|
88
|
+
when 'running'
|
89
|
+
#puts "running: sessions:#{trdata['sessions'].length}, #{trdata['sessions'].map{|i| i['bytes_transferred']}.join(',')}"
|
90
|
+
if !started and trdata['bytes_expected'] != 0
|
91
|
+
spinner.success unless spinner.nil?
|
92
|
+
notify_begin(@connect_settings['app_id'],trdata['bytes_expected'])
|
93
|
+
started=true
|
94
|
+
else
|
95
|
+
notify_progress(@connect_settings['app_id'],trdata['bytes_written'])
|
96
|
+
end
|
97
|
+
when 'failed'
|
98
|
+
spinner.error unless spinner.nil?
|
99
|
+
raise Fasp::Error.new(trdata['error_desc'])
|
93
100
|
else
|
94
|
-
|
101
|
+
raise Fasp::Error.new("unknown status: #{trdata['status']}: #{trdata['error_desc']}")
|
95
102
|
end
|
96
|
-
else
|
97
|
-
raise Fasp::Error.new("unknown status: #{trdata['status']}: #{trdata['error_desc']}")
|
98
103
|
end
|
104
|
+
sleep 1
|
99
105
|
end
|
100
|
-
|
106
|
+
rescue => e
|
107
|
+
return [e]
|
101
108
|
end
|
102
|
-
return []
|
109
|
+
return [:success]
|
103
110
|
end # wait
|
104
|
-
end #
|
111
|
+
end # AgentConnect
|
105
112
|
end
|
106
113
|
end
|
@@ -5,11 +5,12 @@
|
|
5
5
|
# Laurent Martin
|
6
6
|
#
|
7
7
|
##############################################################################
|
8
|
-
require 'aspera/fasp/
|
8
|
+
require 'aspera/fasp/agent_base'
|
9
9
|
require 'aspera/fasp/error'
|
10
10
|
require 'aspera/fasp/parameters'
|
11
11
|
require 'aspera/fasp/installation'
|
12
12
|
require 'aspera/fasp/resume_policy'
|
13
|
+
require 'aspera/fasp/default'
|
13
14
|
require 'aspera/log'
|
14
15
|
require 'socket'
|
15
16
|
require 'timeout'
|
@@ -18,19 +19,17 @@ require 'securerandom'
|
|
18
19
|
module Aspera
|
19
20
|
module Fasp
|
20
21
|
# executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
|
21
|
-
class
|
22
|
+
class AgentDirect < AgentBase
|
22
23
|
# options for initialize (same as values in option transfer_info)
|
23
24
|
DEFAULT_OPTIONS = {
|
24
|
-
:
|
25
|
-
:
|
26
|
-
:
|
27
|
-
:
|
28
|
-
:
|
25
|
+
spawn_timeout_sec: 3,
|
26
|
+
spawn_delay_sec: 2,
|
27
|
+
wss: false,
|
28
|
+
multi_incr_udp: true,
|
29
|
+
resume: {},
|
30
|
+
quiet: true # by default no interactive progress bar
|
29
31
|
}
|
30
|
-
DEFAULT_UDP_PORT=33001
|
31
32
|
private_constant :DEFAULT_OPTIONS
|
32
|
-
# set to false to keep ascp progress bar display ("true" adds ascp's option -q)
|
33
|
-
attr_accessor :quiet
|
34
33
|
|
35
34
|
# start ascp transfer (non blocking), single or multi-session
|
36
35
|
# job information added to @jobs
|
@@ -76,13 +75,13 @@ module Aspera
|
|
76
75
|
Log.log.error("multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0")
|
77
76
|
multi_session_info = nil
|
78
77
|
elsif multi_session_info[:count].eql?(0)
|
79
|
-
Log.log.debug("multi_session
|
78
|
+
Log.log.debug("multi_session count is zero: no multisession")
|
80
79
|
multi_session_info = nil
|
81
80
|
else # multi_session_info[:count] > 0
|
82
81
|
# if option not true: keep default udp port for all sessions
|
83
82
|
if @options[:multi_incr_udp]
|
84
83
|
# override if specified, else use default value
|
85
|
-
multi_session_info[:udp_base]=transfer_spec.has_key?('fasp_port') ? transfer_spec['fasp_port'] :
|
84
|
+
multi_session_info[:udp_base]=transfer_spec.has_key?('fasp_port') ? transfer_spec['fasp_port'] : Default::UDP_PORT
|
86
85
|
# delete from original transfer spec, as we will increment values
|
87
86
|
transfer_spec.delete('fasp_port')
|
88
87
|
end
|
@@ -98,7 +97,7 @@ module Aspera
|
|
98
97
|
env_args[:args].unshift('-I',Installation.instance.path(:fallback_cert))
|
99
98
|
end
|
100
99
|
|
101
|
-
env_args[:args].unshift('-q') if @quiet
|
100
|
+
env_args[:args].unshift('-q') if @options[:quiet]
|
102
101
|
|
103
102
|
# transfer job can be multi session
|
104
103
|
xfer_job={
|
@@ -240,7 +239,7 @@ module Aspera
|
|
240
239
|
when ''
|
241
240
|
# empty line is separator to end event information
|
242
241
|
raise 'unexpected empty line' if current_event_data.nil?
|
243
|
-
current_event_data[
|
242
|
+
current_event_data[AgentBase::LISTENER_SESSION_ID_B]=ascp_pid
|
244
243
|
notify_listeners(current_event_text,current_event_data)
|
245
244
|
case current_event_data['Type']
|
246
245
|
when 'INIT'
|
@@ -333,8 +332,6 @@ module Aspera
|
|
333
332
|
# @param options : keys(symbol): see DEFAULT_OPTIONS
|
334
333
|
def initialize(options=nil)
|
335
334
|
super()
|
336
|
-
# by default no interactive progress bar
|
337
|
-
@quiet=true
|
338
335
|
# all transfer jobs, key = SecureRandom.uuid, protected by mutex, condvar on change
|
339
336
|
@jobs={}
|
340
337
|
# mutex protects global data accessed by threads
|
@@ -374,6 +371,6 @@ module Aspera
|
|
374
371
|
Log.log.debug("EXIT (#{Thread.current[:name]})")
|
375
372
|
end
|
376
373
|
|
377
|
-
end #
|
374
|
+
end # AgentDirect
|
378
375
|
end
|
379
376
|
end
|
@@ -1,5 +1,5 @@
|
|
1
1
|
#!/bin/echo this is a ruby class:
|
2
|
-
require 'aspera/fasp/
|
2
|
+
require 'aspera/fasp/agent_base'
|
3
3
|
require 'aspera/log'
|
4
4
|
require 'aspera/rest'
|
5
5
|
require 'websocket-client-simple'
|
@@ -12,7 +12,7 @@ require 'json'
|
|
12
12
|
module Aspera
|
13
13
|
module Fasp
|
14
14
|
# start a transfer using Aspera HTTP Gateway, using web socket session
|
15
|
-
class
|
15
|
+
class AgentHttpgw < AgentBase
|
16
16
|
# message returned by HTTP GW in case of success
|
17
17
|
OK_MESSAGE='end upload'
|
18
18
|
# refresh rate for progress
|
@@ -179,7 +179,7 @@ module Aspera
|
|
179
179
|
when 'receive'
|
180
180
|
download(transfer_spec)
|
181
181
|
else
|
182
|
-
raise "
|
182
|
+
raise "unexpected direction: [#{transfer_spec['direction']}]"
|
183
183
|
end
|
184
184
|
end # start_transfer
|
185
185
|
|
@@ -209,6 +209,6 @@ module Aspera
|
|
209
209
|
@upload_chunksize=128000 # TODO: configurable ?
|
210
210
|
end
|
211
211
|
|
212
|
-
end #
|
212
|
+
end # AgentHttpgw
|
213
213
|
end
|
214
214
|
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require 'aspera/fasp/
|
1
|
+
require 'aspera/fasp/agent_base'
|
2
2
|
require 'aspera/log'
|
3
3
|
require 'tty-spinner'
|
4
4
|
|
@@ -6,13 +6,30 @@ module Aspera
|
|
6
6
|
module Fasp
|
7
7
|
# this singleton class is used by the CLI to provide a common interface to start a transfer
|
8
8
|
# before using it, the use must set the `node_api` member.
|
9
|
-
class
|
9
|
+
class AgentNode < AgentBase
|
10
10
|
# option include: root_id if the node is an access key
|
11
11
|
attr_writer :options
|
12
|
-
def initialize(
|
12
|
+
def initialize(options)
|
13
|
+
raise "node specification must be Hash" unless options.is_a?(Hash)
|
14
|
+
[:url,:username,:password].each { |k| raise "missing parameter [#{k}] in node specification: #{options}" unless options.has_key?(k) }
|
13
15
|
super()
|
14
|
-
|
15
|
-
@
|
16
|
+
# root id is required for access key
|
17
|
+
@root_id=options[:root_id]
|
18
|
+
rest_params={ base_url: options[:url]}
|
19
|
+
if options[:password].match(/^Bearer /)
|
20
|
+
rest_params[:headers]={
|
21
|
+
'X-Aspera-AccessKey'=>options[:username],
|
22
|
+
'Authorization' =>options[:password]
|
23
|
+
}
|
24
|
+
raise "root_id is required for access key" if @root_id.nil?
|
25
|
+
else
|
26
|
+
rest_params[:auth]={
|
27
|
+
type: :basic,
|
28
|
+
username: options[:username],
|
29
|
+
password: options[:password]
|
30
|
+
}
|
31
|
+
end
|
32
|
+
@node_api=Rest.new(rest_params)
|
16
33
|
# TODO: currently only supports one transfer. This is bad shortcut. but ok for CLI.
|
17
34
|
@transfer_id=nil
|
18
35
|
end
|
@@ -36,10 +53,10 @@ module Aspera
|
|
36
53
|
# generic method
|
37
54
|
def start_transfer(transfer_spec,options=nil)
|
38
55
|
# add root id if access key
|
39
|
-
if @
|
56
|
+
if ! @root_id.nil?
|
40
57
|
case transfer_spec['direction']
|
41
|
-
when 'send';transfer_spec['source_root_id']=@
|
42
|
-
when 'receive';transfer_spec['destination_root_id']=@
|
58
|
+
when 'send';transfer_spec['source_root_id']=@root_id
|
59
|
+
when 'receive';transfer_spec['destination_root_id']=@root_id
|
43
60
|
else raise "unexpected direction in ts: #{transfer_spec['direction']}"
|
44
61
|
end
|
45
62
|
end
|
@@ -0,0 +1,106 @@
|
|
1
|
+
require 'aspera/fasp/agent_base'
|
2
|
+
require 'aspera/fasp/installation'
|
3
|
+
require 'json'
|
4
|
+
|
5
|
+
module Aspera
|
6
|
+
module Fasp
|
7
|
+
class AgentTrsdk < AgentBase
|
8
|
+
DEFAULT_OPTIONS = {
|
9
|
+
address: '127.0.0.1',
|
10
|
+
port: 55002,
|
11
|
+
}
|
12
|
+
private_constant :DEFAULT_OPTIONS
|
13
|
+
|
14
|
+
# options come from transfer_info
|
15
|
+
def initialize(user_opts)
|
16
|
+
raise "expecting Hash (or nil), but have #{user_opts.class}" unless user_opts.nil? or user_opts.is_a?(Hash)
|
17
|
+
# set default options and override if specified
|
18
|
+
options=DEFAULT_OPTIONS.clone
|
19
|
+
if !user_opts.nil?
|
20
|
+
user_opts.each do |k,v|
|
21
|
+
if DEFAULT_OPTIONS.has_key?(k)
|
22
|
+
options[k]=v
|
23
|
+
else
|
24
|
+
raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map{|i|i.to_s}.join(",")}"
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
Log.log.debug("options= #{options}")
|
29
|
+
super()
|
30
|
+
# load and create SDK stub
|
31
|
+
$LOAD_PATH.unshift(Installation.instance.sdk_ruby_folder)
|
32
|
+
require 'transfer_services_pb'
|
33
|
+
@transfer_client = Transfersdk::TransferService::Stub.new("#{options[:address]}:#{options[:port]}",:this_channel_is_insecure)
|
34
|
+
begin
|
35
|
+
get_info_response = @transfer_client.get_info(Transfersdk::InstanceInfoRequest.new)
|
36
|
+
Log.log.debug("daemon info: #{get_info_response}")
|
37
|
+
rescue GRPC::Unavailable => e
|
38
|
+
Log.log.warn("no daemon present, starting daemon...")
|
39
|
+
# location of daemon binary
|
40
|
+
bin_folder=File.realpath(File.join(Installation.instance.sdk_ruby_folder,'..'))
|
41
|
+
# config file and logs are created in same folder
|
42
|
+
conf_file = File.join(bin_folder,'sdk.conf')
|
43
|
+
log_base = File.join(bin_folder,'transferd')
|
44
|
+
# create a config file for daemon
|
45
|
+
config = {
|
46
|
+
address: options[:address],
|
47
|
+
port: options[:port],
|
48
|
+
fasp_runtime: {
|
49
|
+
use_embedded: false,
|
50
|
+
user_defined: {
|
51
|
+
bin: bin_folder,
|
52
|
+
etc: bin_folder,
|
53
|
+
}
|
54
|
+
}
|
55
|
+
}
|
56
|
+
File.write(conf_file,config.to_json)
|
57
|
+
trd_pid = Process.spawn(Installation.instance.path(:transferd),'--config' , conf_file, out: "#{log_base}.out", err: "#{log_base}.err")
|
58
|
+
Process.detach(trd_pid)
|
59
|
+
sleep(2.0)
|
60
|
+
retry
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def start_transfer(transfer_spec,options=nil)
|
65
|
+
# create a transfer request
|
66
|
+
transfer_request = Transfersdk::TransferRequest.new(
|
67
|
+
transferType: Transfersdk::TransferType::FILE_REGULAR, # transfer type (file/stream)
|
68
|
+
config: Transfersdk::TransferConfig.new, # transfer configuration
|
69
|
+
transferSpec: transfer_spec.to_json) # transfer definition
|
70
|
+
# send start transfer request to the transfer manager daemon
|
71
|
+
start_transfer_response = @transfer_client.start_transfer(transfer_request)
|
72
|
+
Log.log.debug("start transfer response #{start_transfer_response}")
|
73
|
+
@transfer_id = start_transfer_response.transferId
|
74
|
+
Log.log.debug("transfer started with id #{@transfer_id}")
|
75
|
+
end
|
76
|
+
|
77
|
+
def wait_for_transfers_completion
|
78
|
+
started=false
|
79
|
+
# monitor transfer status
|
80
|
+
@transfer_client.monitor_transfers(Transfersdk::RegistrationRequest.new(transferId: [@transfer_id])) do |response|
|
81
|
+
Log.dump(:response, response.to_h)
|
82
|
+
#Log.log.debug("#{response.sessionInfo.preTransferBytes} #{response.transferInfo.bytesTransferred}")
|
83
|
+
case response.status
|
84
|
+
when :RUNNING
|
85
|
+
if !started and !response.sessionInfo.preTransferBytes.eql?(0)
|
86
|
+
notify_begin(@transfer_id,response.sessionInfo.preTransferBytes)
|
87
|
+
started=true
|
88
|
+
elsif started
|
89
|
+
notify_progress(@transfer_id,response.transferInfo.bytesTransferred)
|
90
|
+
end
|
91
|
+
when :FAILED, :COMPLETED, :CANCELED
|
92
|
+
notify_end(@transfer_id)
|
93
|
+
raise Fasp::Error.new(JSON.parse(response.message)['Description']) unless :COMPLETED.eql?(response.status)
|
94
|
+
break
|
95
|
+
when :QUEUED,:UNKNOWN_STATUS,:PAUSED,:ORPHANED
|
96
|
+
# ignore
|
97
|
+
else
|
98
|
+
Log.log.error("unknown status#{response.status}")
|
99
|
+
end
|
100
|
+
end
|
101
|
+
# TODO return status
|
102
|
+
return []
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module Aspera
|
2
|
+
module Fasp
|
3
|
+
# default parameters
|
4
|
+
class Default
|
5
|
+
# (public) default transfer username for access key based transfers
|
6
|
+
ACCESS_KEY_TRANSFER_USER='xfer'
|
7
|
+
SSH_PORT=33001
|
8
|
+
UDP_PORT=33001
|
9
|
+
AK_TSPEC_BASE={
|
10
|
+
'remote_user' => ACCESS_KEY_TRANSFER_USER,
|
11
|
+
'ssh_port' => SSH_PORT,
|
12
|
+
'fasp_port' => UDP_PORT
|
13
|
+
}
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
@@ -10,7 +10,7 @@ require 'fileutils'
|
|
10
10
|
module Aspera
|
11
11
|
module Fasp
|
12
12
|
# Singleton that tells where to find ascp and other local resources (keys..) , using the "path(symb)" method.
|
13
|
-
# It is used by object :
|
13
|
+
# It is used by object : AgentDirect to find necessary resources
|
14
14
|
# By default it takes the first Aspera product found specified in product_locations
|
15
15
|
# but the user can specify ascp location by calling:
|
16
16
|
# Installation.instance.use_ascp_from_product(product_name)
|
@@ -22,6 +22,9 @@ module Aspera
|
|
22
22
|
PRODUCT_CLI_V1='Aspera CLI'
|
23
23
|
PRODUCT_DRIVE='Aspera Drive'
|
24
24
|
PRODUCT_ENTSRV='Enterprise Server'
|
25
|
+
# protobuf generated files from sdk
|
26
|
+
EXT_RUBY_PROTOBUF='_pb.rb'
|
27
|
+
RB_SDK_FOLDER='lib'
|
25
28
|
MAX_REDIRECT_SDK=2
|
26
29
|
private_constant :MAX_REDIRECT_SDK
|
27
30
|
# set ascp executable path
|
@@ -29,9 +32,10 @@ module Aspera
|
|
29
32
|
@path_to_ascp=v
|
30
33
|
end
|
31
34
|
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
+
def sdk_ruby_folder
|
36
|
+
ruby_pb_folder=File.join(folder_path,RB_SDK_FOLDER)
|
37
|
+
FileUtils.mkdir_p(ruby_pb_folder) unless Dir.exist?(ruby_pb_folder)
|
38
|
+
return ruby_pb_folder
|
35
39
|
end
|
36
40
|
|
37
41
|
# location of SDK files
|
@@ -100,6 +104,8 @@ module Aspera
|
|
100
104
|
file=@path_to_ascp
|
101
105
|
# note that there might be a .exe at the end
|
102
106
|
file=file.gsub('ascp','ascp4') if k.eql?(:ascp4)
|
107
|
+
when :transferd
|
108
|
+
file=transferd_filepath
|
103
109
|
when :ssh_bypass_key_dsa
|
104
110
|
file=File.join(folder_path,'aspera_bypass_dsa.pem')
|
105
111
|
File.write(file,get_key('dsa',1)) unless File.exist?(file)
|
@@ -182,16 +188,22 @@ module Aspera
|
|
182
188
|
return [:ssh_bypass_key_dsa,:ssh_bypass_key_rsa].map{|i|Installation.instance.path(i)}
|
183
189
|
end
|
184
190
|
|
191
|
+
# use in plugin `config`
|
192
|
+
def get_ascp_version(exe_path)
|
193
|
+
return get_exe_version(exe_path,'-A')
|
194
|
+
end
|
195
|
+
|
185
196
|
# Check that specified path is ascp and get version
|
186
|
-
def
|
187
|
-
raise "
|
188
|
-
|
189
|
-
|
190
|
-
cmd_out=%x{"#{
|
197
|
+
def get_exe_version(exe_path,vers_arg)
|
198
|
+
raise "ERROR: nil arg" if exe_path.nil?
|
199
|
+
return nil unless File.exist?(exe_path)
|
200
|
+
exe_version=nil
|
201
|
+
cmd_out=%x{"#{exe_path}" #{vers_arg}}
|
191
202
|
raise "An error occured when testing #{ascp_filename}: #{cmd_out}" unless $? == 0
|
192
203
|
# get version from ascp, only after full extract, as windows requires DLLs (SSL/TLS/etc...)
|
193
|
-
m=cmd_out.match(/
|
194
|
-
|
204
|
+
m=cmd_out.match(/ version ([0-9\.]+)/)
|
205
|
+
exe_version=m[1] unless m.nil?
|
206
|
+
return exe_version
|
195
207
|
end
|
196
208
|
|
197
209
|
# download aspera SDK or use local file
|
@@ -205,54 +217,49 @@ module Aspera
|
|
205
217
|
raise 'use format: file:///<path>' unless sdk_url.start_with?('file:///')
|
206
218
|
sdk_zip_path=sdk_url.gsub(%r{^file:///},'')
|
207
219
|
else
|
208
|
-
|
209
|
-
begin
|
210
|
-
Aspera::Rest.new(base_url: sdk_url).call(operation: 'GET',save_to_file: sdk_zip_path)
|
211
|
-
rescue Aspera::RestCallError => e
|
212
|
-
if e.response.is_a?(Net::HTTPRedirection)
|
213
|
-
if redirect_remain > 0
|
214
|
-
redirect_remain-=1
|
215
|
-
sdk_url=e.response['location']
|
216
|
-
retry
|
217
|
-
else
|
218
|
-
raise "Too many redirect"
|
219
|
-
end
|
220
|
-
else
|
221
|
-
raise e
|
222
|
-
end
|
223
|
-
end
|
220
|
+
Aspera::Rest.new(base_url: sdk_url, redirect_max: 3).call(operation: 'GET',save_to_file: sdk_zip_path)
|
224
221
|
end
|
225
|
-
# SDK is organized by architecture
|
226
|
-
filter="/#{Environment.architecture}/"
|
227
|
-
ascp_path=nil
|
228
|
-
sdk_path=folder_path
|
229
222
|
# rename old install
|
230
|
-
if
|
231
|
-
Log.log.warn("Previous install exists, renaming.")
|
232
|
-
File.rename(
|
223
|
+
if ! Dir.empty?(folder_path)
|
224
|
+
Log.log.warn("Previous install exists, renaming folder.")
|
225
|
+
File.rename(folder_path,"#{folder_path}.#{Time.now.strftime("%Y%m%d%H%M%S")}")
|
226
|
+
# TODO: delete old archives ?
|
233
227
|
end
|
234
|
-
#
|
235
|
-
|
236
|
-
|
228
|
+
# SDK is organized by architecture
|
229
|
+
arch_filter="#{Environment.architecture}/"
|
230
|
+
# extract files from archive
|
237
231
|
Zip::File.open(sdk_zip_path) do |zip_file|
|
238
232
|
zip_file.each do |entry|
|
239
|
-
#
|
240
|
-
if entry.name.
|
241
|
-
|
242
|
-
|
233
|
+
# skip folder entries
|
234
|
+
next if entry.name.end_with?('/')
|
235
|
+
dest_folder=nil
|
236
|
+
# binaries
|
237
|
+
dest_folder=folder_path if entry.name.include?(arch_filter)
|
238
|
+
# ruby adapters
|
239
|
+
dest_folder=sdk_ruby_folder if entry.name.end_with?(EXT_RUBY_PROTOBUF)
|
240
|
+
if !dest_folder.nil?
|
241
|
+
File.open(File.join(dest_folder,File.basename(entry.name)), 'wb') do |output_stream|
|
243
242
|
IO.copy_stream(entry.get_input_stream, output_stream)
|
244
243
|
end
|
245
|
-
if File.basename(entry.name).eql?(ascp_filename)
|
246
|
-
FileUtils.chmod(0755,archive_file)
|
247
|
-
ascp_path=archive_file
|
248
|
-
end
|
249
244
|
end
|
250
245
|
end
|
251
246
|
end
|
252
247
|
File.unlink(sdk_zip_path) rescue nil # Windows may give error
|
253
|
-
|
254
|
-
|
255
|
-
|
248
|
+
# ensure license file are generated so that ascp invokation for version works
|
249
|
+
self.path(:aspera_license)
|
250
|
+
self.path(:aspera_conf)
|
251
|
+
ascp_path=File.join(folder_path,ascp_filename)
|
252
|
+
raise "No #{ascp_filename} found in SDK archive" unless File.exist?(ascp_path)
|
253
|
+
FileUtils.chmod(0755,ascp_path)
|
254
|
+
FileUtils.chmod(0755,ascp_path.gsub('ascp','ascp4'))
|
255
|
+
ascp_version=get_ascp_version(File.join(folder_path,ascp_filename))
|
256
|
+
trd_path=transferd_filepath
|
257
|
+
Log.log.warn("No #{trd_path} in SDK archive") unless File.exist?(trd_path)
|
258
|
+
FileUtils.chmod(0755,trd_path) if File.exist?(trd_path)
|
259
|
+
transferd_version=get_exe_version(trd_path,'version')
|
260
|
+
sdk_version = transferd_version||ascp_version
|
261
|
+
File.write(File.join(folder_path,PRODUCT_INFO),"<product><name>IBM Aspera SDK</name><version>#{sdk_version}</version></product>")
|
262
|
+
return sdk_version
|
256
263
|
end
|
257
264
|
|
258
265
|
private
|
@@ -288,6 +295,15 @@ module Aspera
|
|
288
295
|
@sdk_folder
|
289
296
|
end
|
290
297
|
|
298
|
+
# filename for ascp with optional extension (Windows)
|
299
|
+
def ascp_filename
|
300
|
+
return 'ascp'+Environment.exe_extension
|
301
|
+
end
|
302
|
+
|
303
|
+
def transferd_filepath
|
304
|
+
return File.join(folder_path,'asperatransferd'+Environment.exe_extension)
|
305
|
+
end
|
306
|
+
|
291
307
|
# @return product folders depending on OS fields
|
292
308
|
# :expected M app name is taken from the manifest if present, else defaults to this value
|
293
309
|
# :app_root M main folder for the application
|
@@ -7,6 +7,7 @@ require 'json'
|
|
7
7
|
require 'yaml'
|
8
8
|
require 'securerandom'
|
9
9
|
require 'fileutils'
|
10
|
+
require 'openssl'
|
10
11
|
|
11
12
|
module Aspera
|
12
13
|
module Fasp
|
@@ -26,7 +27,7 @@ module Aspera
|
|
26
27
|
Aspera::CommandLineBuilder.normalize_description(@@param_description_cache)
|
27
28
|
end
|
28
29
|
|
29
|
-
# Agents shown in manual
|
30
|
+
# Agents shown in manual for parameters (sub list)
|
30
31
|
SUPPORTED_AGENTS=[:direct,:node,:connect]
|
31
32
|
# Short names of columns in manual
|
32
33
|
SUPPORTED_AGENTS_SHORT=SUPPORTED_AGENTS.map{|a|a.to_s[0].to_sym}
|
@@ -111,6 +112,9 @@ module Aspera
|
|
111
112
|
@job_spec.delete('fasp_port')
|
112
113
|
@job_spec.delete('EX_ssh_key_paths')
|
113
114
|
@job_spec.delete('sshfp')
|
115
|
+
# set default location for CA bundle, see env var SSL_CERT_FILE / SSL_CERT_DIR
|
116
|
+
@job_spec['EX_ssh_key_paths']=[OpenSSL::X509::DEFAULT_CERT_FILE]
|
117
|
+
Log.log.debug("CA certs: EX_ssh_key_paths <- DEFAULT_CERT_FILE from openssl")
|
114
118
|
else
|
115
119
|
# remove unused parameter (avoid warning)
|
116
120
|
@job_spec.delete('wss_port')
|
@@ -163,8 +167,8 @@ module Aspera
|
|
163
167
|
lines=paths_array.map{|i|i['source']}
|
164
168
|
end
|
165
169
|
file_list_file=Aspera::TempFileManager.instance.new_file_path_in_folder(@@file_list_folder)
|
166
|
-
File.open(file_list_file, 'w+'){|f|f.
|
167
|
-
Log.log.debug
|
170
|
+
File.open(file_list_file, 'w+'){|f|f.write(lines.join("\n"))}
|
171
|
+
Log.log.debug{"#{option}=\n#{File.read(file_list_file)}".red}
|
168
172
|
end
|
169
173
|
end
|
170
174
|
@builder.add_command_line_options(["#{option}=#{file_list_file}"])
|