aspera-cli 4.6.0 → 4.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +427 -300
- data/bin/ascli +2 -1
- data/bin/asession +1 -0
- data/docs/test_env.conf +2 -0
- data/examples/aoc.rb +4 -3
- data/examples/faspex4.rb +21 -19
- data/examples/proxy.pac +1 -1
- data/examples/transfer.rb +15 -15
- data/lib/aspera/aoc.rb +135 -124
- data/lib/aspera/ascmd.rb +85 -75
- data/lib/aspera/ats_api.rb +11 -10
- data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
- data/lib/aspera/cli/extended_value.rb +42 -33
- data/lib/aspera/cli/formater.rb +138 -111
- data/lib/aspera/cli/info.rb +17 -0
- data/lib/aspera/cli/listener/line_dump.rb +3 -2
- data/lib/aspera/cli/listener/logger.rb +2 -1
- data/lib/aspera/cli/listener/progress.rb +16 -18
- data/lib/aspera/cli/listener/progress_multi.rb +13 -16
- data/lib/aspera/cli/main.rb +122 -130
- data/lib/aspera/cli/manager.rb +146 -154
- data/lib/aspera/cli/plugin.rb +38 -34
- data/lib/aspera/cli/plugins/alee.rb +6 -6
- data/lib/aspera/cli/plugins/aoc.rb +273 -276
- data/lib/aspera/cli/plugins/ats.rb +82 -76
- data/lib/aspera/cli/plugins/bss.rb +14 -16
- data/lib/aspera/cli/plugins/config.rb +350 -306
- data/lib/aspera/cli/plugins/console.rb +23 -19
- data/lib/aspera/cli/plugins/cos.rb +18 -18
- data/lib/aspera/cli/plugins/faspex.rb +180 -159
- data/lib/aspera/cli/plugins/faspex5.rb +64 -54
- data/lib/aspera/cli/plugins/node.rb +147 -140
- data/lib/aspera/cli/plugins/orchestrator.rb +68 -66
- data/lib/aspera/cli/plugins/preview.rb +92 -96
- data/lib/aspera/cli/plugins/server.rb +79 -75
- data/lib/aspera/cli/plugins/shares.rb +23 -24
- data/lib/aspera/cli/plugins/sync.rb +20 -22
- data/lib/aspera/cli/transfer_agent.rb +40 -39
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +35 -27
- data/lib/aspera/command_line_builder.rb +48 -34
- data/lib/aspera/cos_node.rb +29 -21
- data/lib/aspera/data_repository.rb +3 -2
- data/lib/aspera/environment.rb +50 -45
- data/lib/aspera/fasp/agent_base.rb +22 -20
- data/lib/aspera/fasp/agent_connect.rb +13 -11
- data/lib/aspera/fasp/agent_direct.rb +48 -59
- data/lib/aspera/fasp/agent_httpgw.rb +33 -39
- data/lib/aspera/fasp/agent_node.rb +15 -13
- data/lib/aspera/fasp/agent_trsdk.rb +12 -14
- data/lib/aspera/fasp/error.rb +2 -1
- data/lib/aspera/fasp/error_info.rb +68 -52
- data/lib/aspera/fasp/installation.rb +106 -94
- data/lib/aspera/fasp/listener.rb +1 -0
- data/lib/aspera/fasp/parameters.rb +83 -92
- data/lib/aspera/fasp/parameters.yaml +305 -249
- data/lib/aspera/fasp/resume_policy.rb +11 -14
- data/lib/aspera/fasp/transfer_spec.rb +26 -0
- data/lib/aspera/fasp/uri.rb +22 -21
- data/lib/aspera/faspex_gw.rb +55 -90
- data/lib/aspera/hash_ext.rb +4 -3
- data/lib/aspera/id_generator.rb +8 -7
- data/lib/aspera/keychain/encrypted_hash.rb +17 -16
- data/lib/aspera/keychain/macos_security.rb +6 -10
- data/lib/aspera/log.rb +25 -20
- data/lib/aspera/nagios.rb +13 -12
- data/lib/aspera/node.rb +30 -22
- data/lib/aspera/oauth.rb +175 -226
- data/lib/aspera/open_application.rb +4 -3
- data/lib/aspera/persistency_action_once.rb +6 -6
- data/lib/aspera/persistency_folder.rb +5 -9
- data/lib/aspera/preview/file_types.rb +6 -5
- data/lib/aspera/preview/generator.rb +25 -24
- data/lib/aspera/preview/options.rb +16 -14
- data/lib/aspera/preview/utils.rb +98 -98
- data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
- data/lib/aspera/proxy_auto_config.rb +111 -20
- data/lib/aspera/rest.rb +115 -113
- data/lib/aspera/rest_call_error.rb +2 -2
- data/lib/aspera/rest_error_analyzer.rb +23 -25
- data/lib/aspera/rest_errors_aspera.rb +15 -14
- data/lib/aspera/ssh.rb +12 -10
- data/lib/aspera/sync.rb +42 -41
- data/lib/aspera/temp_file_manager.rb +18 -14
- data/lib/aspera/timer_limiter.rb +2 -1
- data/lib/aspera/uri_reader.rb +7 -5
- data/lib/aspera/web_auth.rb +79 -76
- metadata +64 -21
- data/docs/Makefile +0 -65
- data/docs/README.erb.md +0 -4424
- data/docs/README.md +0 -13
- data/docs/diagrams.txt +0 -49
- data/docs/doc_tools.rb +0 -58
- data/lib/aspera/cli/plugins/shares2.rb +0 -114
- data/lib/aspera/fasp/default.rb +0 -17
@@ -1,30 +1,32 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
module Aspera
|
2
3
|
module Fasp
|
3
4
|
# Base class for FASP transfer agents
|
4
5
|
# sub classes shall implement start_transfer and shutdown
|
5
6
|
class AgentBase
|
7
|
+
# fields description for JSON generation
|
8
|
+
INTEGER_FIELDS=%w[Bytescont FaspFileArgIndex StartByte Rate MinRate Port Priority RateCap MinRateCap TCPPort CreatePolicy TimePolicy DatagramSize XoptFlags VLinkVersion
|
9
|
+
PeerVLinkVersion DSPipelineDepth PeerDSPipelineDepth ReadBlockSize WriteBlockSize ClusterNumNodes ClusterNodeId Size Written Loss FileBytes PreTransferBytes TransferBytes PMTU Elapsedusec ArgScansAttempted ArgScansCompleted PathScansAttempted FileScansCompleted TransfersAttempted TransfersPassed Delay].freeze
|
10
|
+
BOOLEAN_FIELDS=%w[Encryption Remote RateLock MinRateLock PolicyLock FilesEncrypt FilesDecrypt VLinkLocalEnabled VLinkRemoteEnabled MoveRange Keepalive TestLogin UseProxy
|
11
|
+
Precalc RTTAutocorrect].freeze
|
12
|
+
EXPECTED_METHODS=%i[text struct enhanced].freeze
|
13
|
+
private_constant :INTEGER_FIELDS,:BOOLEAN_FIELDS,:EXPECTED_METHODS
|
6
14
|
|
7
15
|
private
|
8
16
|
|
9
|
-
# fields description for JSON generation
|
10
|
-
IntegerFields=['Bytescont','FaspFileArgIndex','StartByte','Rate','MinRate','Port','Priority','RateCap','MinRateCap','TCPPort','CreatePolicy','TimePolicy','DatagramSize','XoptFlags','VLinkVersion','PeerVLinkVersion','DSPipelineDepth','PeerDSPipelineDepth','ReadBlockSize','WriteBlockSize','ClusterNumNodes','ClusterNodeId','Size','Written','Loss','FileBytes','PreTransferBytes','TransferBytes','PMTU','Elapsedusec','ArgScansAttempted','ArgScansCompleted','PathScansAttempted','FileScansCompleted','TransfersAttempted','TransfersPassed','Delay']
|
11
|
-
BooleanFields=['Encryption','Remote','RateLock','MinRateLock','PolicyLock','FilesEncrypt','FilesDecrypt','VLinkLocalEnabled','VLinkRemoteEnabled','MoveRange','Keepalive','TestLogin','UseProxy','Precalc','RTTAutocorrect']
|
12
|
-
ExpectedMethod=[:text,:struct,:enhanced]
|
13
|
-
|
14
17
|
# translates legacy event into enhanced (JSON) event
|
15
18
|
def enhanced_event_format(event)
|
16
|
-
return event.keys.
|
19
|
+
return event.keys.each_with_object({}) do |e,h|
|
17
20
|
# capital_to_snake_case
|
18
21
|
new_name=e.
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
22
|
+
gsub(/([a-z\d])([A-Z])/,'\1_\2').
|
23
|
+
gsub(/([a-z\d])(usec)$/,'\1_\2').
|
24
|
+
gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
|
25
|
+
downcase
|
23
26
|
value=event[e]
|
24
|
-
value=value.to_i if
|
25
|
-
value=value.eql?('Yes')
|
27
|
+
value=value.to_i if INTEGER_FIELDS.include?(e)
|
28
|
+
value=value.eql?('Yes') if BOOLEAN_FIELDS.include?(e)
|
26
29
|
h[new_name]=value
|
27
|
-
h
|
28
30
|
end
|
29
31
|
end
|
30
32
|
|
@@ -33,14 +35,14 @@ module Aspera
|
|
33
35
|
end
|
34
36
|
|
35
37
|
def notify_listeners(current_event_text,current_event_data)
|
36
|
-
Log.log.debug(
|
38
|
+
Log.log.debug('send event to listeners')
|
37
39
|
enhanced_event=nil
|
38
40
|
@listeners.each do |listener|
|
39
|
-
listener.
|
40
|
-
listener.
|
41
|
+
listener.event_text(current_event_text) if listener.respond_to?(:event_text)
|
42
|
+
listener.event_struct(current_event_data) if listener.respond_to?(:event_struct)
|
41
43
|
if listener.respond_to?(:event_enhanced)
|
42
44
|
enhanced_event=enhanced_event_format(current_event_data) if enhanced_event.nil?
|
43
|
-
listener.
|
45
|
+
listener.event_enhanced(enhanced_event)
|
44
46
|
end
|
45
47
|
end
|
46
48
|
end # notify_listeners
|
@@ -58,12 +60,13 @@ module Aspera
|
|
58
60
|
end
|
59
61
|
|
60
62
|
public
|
63
|
+
|
61
64
|
LISTENER_SESSION_ID_B='ListenerSessionId'
|
62
65
|
LISTENER_SESSION_ID_S='listener_session_id'
|
63
66
|
|
64
67
|
# listener receives events
|
65
68
|
def add_listener(listener)
|
66
|
-
raise "expect one of #{
|
69
|
+
raise "expect one of #{EXPECTED_METHODS}" if EXPECTED_METHODS.inject(0){|m,e|m+=listener.respond_to?("event_#{e}")?1:0;m}.eql?(0)
|
67
70
|
@listeners.push(listener)
|
68
71
|
self
|
69
72
|
end
|
@@ -72,14 +75,13 @@ module Aspera
|
|
72
75
|
# it must be a list of :success or exception
|
73
76
|
def self.validate_status_list(statuses)
|
74
77
|
raise "internal error: bad statuses type: #{statuses.class}" unless statuses.is_a?(Array)
|
75
|
-
raise "internal error: bad statuses content: #{statuses}" unless statuses.select{|i|!i.eql?(:success)
|
78
|
+
raise "internal error: bad statuses content: #{statuses}" unless statuses.select{|i|!i.eql?(:success) && !i.is_a?(StandardError)}.empty?
|
76
79
|
end
|
77
80
|
|
78
81
|
# the following methods must be implemented by subclass:
|
79
82
|
# start_transfer(transfer_spec,options) : start and wait for completion
|
80
83
|
# wait_for_transfers_completion : wait for termination of all transfers, @return list of : :success or error message
|
81
84
|
# optional: shutdown
|
82
|
-
|
83
85
|
end
|
84
86
|
end
|
85
87
|
end
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'aspera/fasp/agent_base'
|
2
3
|
require 'aspera/rest'
|
3
4
|
require 'aspera/open_application'
|
@@ -10,7 +11,7 @@ module Aspera
|
|
10
11
|
MAX_CONNECT_START_RETRY=3
|
11
12
|
SLEEP_SEC_BETWEEN_RETRY=2
|
12
13
|
private_constant :MAX_CONNECT_START_RETRY,:SLEEP_SEC_BETWEEN_RETRY
|
13
|
-
def initialize(
|
14
|
+
def initialize(_options)
|
14
15
|
super()
|
15
16
|
@connect_settings={
|
16
17
|
'app_id' => SecureRandom.uuid
|
@@ -23,7 +24,7 @@ module Aspera
|
|
23
24
|
@connect_api=Rest.new({base_url: "#{connect_url}/v5/connect",headers: {'Origin'=>Rest.user_agent}}) # could use v6 also now
|
24
25
|
cinfo=@connect_api.read('info/version')[:data]
|
25
26
|
Log.dump(:connect_version,cinfo)
|
26
|
-
rescue => e # Errno::ECONNREFUSED
|
27
|
+
rescue StandardError => e # Errno::ECONNREFUSED
|
27
28
|
raise StandardError,"Unable to start connect after #{trynumber} try" if trynumber >= MAX_CONNECT_START_RETRY
|
28
29
|
Log.log.warn("connect is not started. Retry ##{trynumber}, err=#{e}")
|
29
30
|
trynumber+=1
|
@@ -36,11 +37,12 @@ module Aspera
|
|
36
37
|
end
|
37
38
|
end
|
38
39
|
|
39
|
-
def start_transfer(transfer_spec,
|
40
|
+
def start_transfer(transfer_spec,_options=nil)
|
40
41
|
if transfer_spec['direction'] == 'send'
|
41
42
|
Log.log.warn("Connect requires upload selection using GUI, ignoring #{transfer_spec['paths']}".red)
|
42
43
|
transfer_spec.delete('paths')
|
43
|
-
resdata=@connect_api.create('windows/select-open-file-dialog/',
|
44
|
+
resdata=@connect_api.create('windows/select-open-file-dialog/',
|
45
|
+
{'aspera_connect_settings'=>@connect_settings,'title'=>'Select Files','suggestedName'=>'','allowMultipleSelection'=>true,'allowedFileTypes'=>''})[:data]
|
44
46
|
transfer_spec['paths']=resdata['dataTransfer']['files'].map { |i| {'source'=>i['name']}}
|
45
47
|
end
|
46
48
|
@request_id=SecureRandom.uuid
|
@@ -50,10 +52,10 @@ module Aspera
|
|
50
52
|
connect_transfer_args={
|
51
53
|
'aspera_connect_settings'=>@connect_settings.merge({
|
52
54
|
'request_id' =>@request_id,
|
53
|
-
'allow_dialogs' =>true
|
55
|
+
'allow_dialogs' =>true
|
54
56
|
}),
|
55
57
|
'transfer_specs' =>[{
|
56
|
-
'transfer_spec' =>transfer_spec
|
58
|
+
'transfer_spec' =>transfer_spec
|
57
59
|
}]}
|
58
60
|
# asynchronous anyway
|
59
61
|
res=@connect_api.create('transfers/start',connect_transfer_args)[:data]
|
@@ -70,7 +72,7 @@ module Aspera
|
|
70
72
|
if tr_info['transfer_info'].is_a?(Hash)
|
71
73
|
trdata=tr_info['transfer_info']
|
72
74
|
if trdata.nil?
|
73
|
-
Log.log.warn(
|
75
|
+
Log.log.warn('no session in Connect')
|
74
76
|
break
|
75
77
|
end
|
76
78
|
# TODO: get session id
|
@@ -87,7 +89,7 @@ module Aspera
|
|
87
89
|
spinner.spin
|
88
90
|
when 'running'
|
89
91
|
#puts "running: sessions:#{trdata['sessions'].length}, #{trdata['sessions'].map{|i| i['bytes_transferred']}.join(',')}"
|
90
|
-
if !started
|
92
|
+
if !started && (trdata['bytes_expected'] != 0)
|
91
93
|
spinner.success unless spinner.nil?
|
92
94
|
notify_begin(@connect_settings['app_id'],trdata['bytes_expected'])
|
93
95
|
started=true
|
@@ -96,14 +98,14 @@ module Aspera
|
|
96
98
|
end
|
97
99
|
when 'failed'
|
98
100
|
spinner.error unless spinner.nil?
|
99
|
-
raise Fasp::Error
|
101
|
+
raise Fasp::Error, trdata['error_desc']
|
100
102
|
else
|
101
|
-
raise Fasp::Error
|
103
|
+
raise Fasp::Error, "unknown status: #{trdata['status']}: #{trdata['error_desc']}"
|
102
104
|
end
|
103
105
|
end
|
104
106
|
sleep 1
|
105
107
|
end
|
106
|
-
rescue => e
|
108
|
+
rescue StandardError => e
|
107
109
|
return [e]
|
108
110
|
end
|
109
111
|
return [:success]
|
@@ -1,20 +1,16 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
# FASP manager for Ruby
|
4
|
-
# Aspera 2016
|
5
|
-
# Laurent Martin
|
6
|
-
#
|
7
|
-
##############################################################################
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'English'
|
8
3
|
require 'aspera/fasp/agent_base'
|
9
4
|
require 'aspera/fasp/error'
|
10
5
|
require 'aspera/fasp/parameters'
|
11
6
|
require 'aspera/fasp/installation'
|
12
7
|
require 'aspera/fasp/resume_policy'
|
13
|
-
require 'aspera/fasp/
|
8
|
+
require 'aspera/fasp/transfer_spec'
|
14
9
|
require 'aspera/log'
|
15
10
|
require 'socket'
|
16
11
|
require 'timeout'
|
17
12
|
require 'securerandom'
|
13
|
+
require 'shellwords'
|
18
14
|
|
19
15
|
module Aspera
|
20
16
|
module Fasp
|
@@ -27,7 +23,7 @@ module Aspera
|
|
27
23
|
wss: false,
|
28
24
|
multi_incr_udp: true,
|
29
25
|
resume: {},
|
30
|
-
quiet: true
|
26
|
+
quiet: true # by default no interactive progress bar
|
31
27
|
}
|
32
28
|
private_constant :DEFAULT_OPTIONS
|
33
29
|
|
@@ -43,10 +39,11 @@ module Aspera
|
|
43
39
|
# clone transfer spec because we modify it (first level keys)
|
44
40
|
transfer_spec=transfer_spec.clone
|
45
41
|
# if there is aspera tags
|
46
|
-
if transfer_spec['tags'].is_a?(Hash)
|
42
|
+
if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags']['aspera'].is_a?(Hash)
|
47
43
|
# TODO: what is this for ? only on local ascp ?
|
48
44
|
# NOTE: important: transfer id must be unique: generate random id
|
49
45
|
# using a non unique id results in discard of tags in AoC, and a package is never finalized
|
46
|
+
# all sessions in a multi-session transfer must have the same xfer_id (see admin manual)
|
50
47
|
transfer_spec['tags']['aspera']['xfer_id']||=SecureRandom.uuid
|
51
48
|
Log.log.debug("xfer id=#{transfer_spec['xfer_id']}")
|
52
49
|
# TODO: useful ? node only ?
|
@@ -55,8 +52,8 @@ module Aspera
|
|
55
52
|
Log.dump('ts',transfer_spec)
|
56
53
|
|
57
54
|
# add bypass keys when authentication is token and no auth is provided
|
58
|
-
if transfer_spec.has_key?('token')
|
59
|
-
!transfer_spec.has_key?('remote_password')
|
55
|
+
if transfer_spec.has_key?('token') &&
|
56
|
+
!transfer_spec.has_key?('remote_password') &&
|
60
57
|
!transfer_spec.has_key?('EX_ssh_key_paths')
|
61
58
|
# transfer_spec['remote_password'] = Installation.instance.bypass_pass # not used
|
62
59
|
transfer_spec['EX_ssh_key_paths'] = Installation.instance.bypass_keys
|
@@ -67,24 +64,22 @@ module Aspera
|
|
67
64
|
multi_session_info=nil
|
68
65
|
if transfer_spec.has_key?('multi_session')
|
69
66
|
multi_session_info={
|
70
|
-
count: transfer_spec['multi_session'].to_i
|
67
|
+
count: transfer_spec['multi_session'].to_i
|
71
68
|
}
|
72
69
|
# Managed by multi-session, so delete from transfer spec
|
73
70
|
transfer_spec.delete('multi_session')
|
74
|
-
if multi_session_info[:count]
|
71
|
+
if multi_session_info[:count].negative?
|
75
72
|
Log.log.error("multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0")
|
76
73
|
multi_session_info = nil
|
77
74
|
elsif multi_session_info[:count].eql?(0)
|
78
|
-
Log.log.debug(
|
75
|
+
Log.log.debug('multi_session count is zero: no multisession')
|
79
76
|
multi_session_info = nil
|
80
|
-
|
77
|
+
elsif @options[:multi_incr_udp] # multi_session_info[:count] > 0
|
81
78
|
# if option not true: keep default udp port for all sessions
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
transfer_spec.delete('fasp_port')
|
87
|
-
end
|
79
|
+
multi_session_info[:udp_base]=transfer_spec.has_key?('fasp_port') ? transfer_spec['fasp_port'] : TransferSpec::UDP_PORT
|
80
|
+
# delete from original transfer spec, as we will increment values
|
81
|
+
transfer_spec.delete('fasp_port')
|
82
|
+
# override if specified, else use default value
|
88
83
|
end
|
89
84
|
end
|
90
85
|
|
@@ -101,18 +96,18 @@ module Aspera
|
|
101
96
|
|
102
97
|
# transfer job can be multi session
|
103
98
|
xfer_job={
|
104
|
-
:
|
105
|
-
:
|
99
|
+
id: job_options[:job_id],
|
100
|
+
sessions: [] # all sessions as below
|
106
101
|
}
|
107
102
|
|
108
103
|
# generic session information
|
109
104
|
session={
|
110
|
-
:
|
111
|
-
:
|
112
|
-
:
|
113
|
-
:
|
114
|
-
:env_args
|
115
|
-
:
|
105
|
+
thread: nil, # Thread object monitoring management port, not nil when pushed to :sessions
|
106
|
+
error: nil, # exception if failed
|
107
|
+
io: nil, # management port server socket
|
108
|
+
id: nil, # SessionId from INIT message in mgt port
|
109
|
+
env_args: env_args, # env vars and args to ascp (from transfer spec)
|
110
|
+
options: job_options # [Hash]
|
116
111
|
}
|
117
112
|
|
118
113
|
if multi_session_info.nil?
|
@@ -131,7 +126,7 @@ module Aspera
|
|
131
126
|
this_session[:env_args][:args]=this_session[:env_args][:args].clone()
|
132
127
|
this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_info[:count]}")
|
133
128
|
# option: increment (default as per ascp manual) or not (cluster on other side ?)
|
134
|
-
this_session[:env_args][:args].unshift('-O',
|
129
|
+
this_session[:env_args][:args].unshift('-O',(multi_session_info[:udp_base]+i-1).to_s) if @options[:multi_incr_udp]
|
135
130
|
this_session[:thread] = Thread.new(this_session) {|s|transfer_thread_entry(s)}
|
136
131
|
xfer_job[:sessions].push(this_session)
|
137
132
|
end
|
@@ -151,11 +146,11 @@ module Aspera
|
|
151
146
|
Log.log.debug('wait_for_transfers_completion')
|
152
147
|
# set to non-nil to exit loop
|
153
148
|
result=[]
|
154
|
-
@jobs.each do |
|
149
|
+
@jobs.each do |_id,job|
|
155
150
|
job[:sessions].each do |session|
|
156
151
|
Log.log.debug("join #{session[:thread]}")
|
157
152
|
session[:thread].join
|
158
|
-
result.push(session[:error]
|
153
|
+
result.push(session[:error] || :success)
|
159
154
|
end
|
160
155
|
end
|
161
156
|
Log.log.debug('all transfers joined')
|
@@ -189,7 +184,7 @@ module Aspera
|
|
189
184
|
Fasp::Installation.instance.path(env_args[:ascp_version])
|
190
185
|
end
|
191
186
|
# (optional) check it exists
|
192
|
-
raise Fasp::Error
|
187
|
+
raise Fasp::Error, "no such file: #{ascp_path}" unless File.exist?(ascp_path)
|
193
188
|
# open random local TCP port for listening for ascp management
|
194
189
|
mgt_sock = TCPServer.new('127.0.0.1',0)
|
195
190
|
# clone arguments as we eed to modify with mgt port
|
@@ -197,7 +192,7 @@ module Aspera
|
|
197
192
|
# add management port
|
198
193
|
ascp_arguments.unshift('-M', mgt_sock.addr[1].to_s)
|
199
194
|
# start ascp in sub process
|
200
|
-
Log.log.debug("execute: #{env_args[:env].map{|k,v| "#{k}
|
195
|
+
Log.log.debug("execute: #{env_args[:env].map{|k,v| "#{k}=#{Shellwords.shellescape(v)}"}.join(' ')} #{Shellwords.shellescape(ascp_path)} #{ascp_arguments.map{|a|Shellwords.shellescape(a)}.join(' ')}")
|
201
196
|
# start process
|
202
197
|
ascp_pid = Process.spawn(env_args[:env],[ascp_path,ascp_path],*ascp_arguments)
|
203
198
|
# in parent, wait for connection to socket max 3 seconds
|
@@ -225,17 +220,17 @@ module Aspera
|
|
225
220
|
line = ascp_mgt_io.gets
|
226
221
|
# nil when ascp process exits
|
227
222
|
break if line.nil?
|
228
|
-
current_event_text
|
223
|
+
current_event_text+=line
|
229
224
|
line.chomp!
|
230
225
|
Log.log.debug("line=[#{line}]")
|
231
226
|
case line
|
232
227
|
when 'FASPMGR 2'
|
233
228
|
# begin event
|
234
|
-
current_event_data =
|
229
|
+
current_event_data = {}
|
235
230
|
current_event_text = ''
|
236
231
|
when /^([^:]+): (.*)$/
|
237
232
|
# event field
|
238
|
-
current_event_data[
|
233
|
+
current_event_data[Regexp.last_match(1)] = Regexp.last_match(2)
|
239
234
|
when ''
|
240
235
|
# empty line is separator to end event information
|
241
236
|
raise 'unexpected empty line' if current_event_data.nil?
|
@@ -261,9 +256,9 @@ module Aspera
|
|
261
256
|
exception_raised=false
|
262
257
|
when 'ERROR'
|
263
258
|
Log.log.error("code: #{last_status_event['Code']}")
|
264
|
-
if last_status_event['Description']
|
259
|
+
if last_status_event['Description'] =~ /bearer token/i
|
265
260
|
Log.log.error('need to regenerate token'.red)
|
266
|
-
if session[:options].is_a?(Hash)
|
261
|
+
if session[:options].is_a?(Hash) && session[:options].has_key?(:regenerate_token)
|
267
262
|
# regenerate token here, expired, or error on it
|
268
263
|
# Note: in multi-session, each session will have a different one.
|
269
264
|
env_args[:env]['ASPERA_SCP_TOKEN']=session[:options][:regenerate_token].call(true)
|
@@ -279,27 +274,25 @@ module Aspera
|
|
279
274
|
end
|
280
275
|
rescue SystemCallError => e
|
281
276
|
# Process.spawn
|
282
|
-
raise Fasp::Error
|
283
|
-
rescue Timeout::Error
|
284
|
-
raise Fasp::Error
|
285
|
-
rescue Interrupt
|
286
|
-
raise Fasp::Error
|
277
|
+
raise Fasp::Error, e.message
|
278
|
+
rescue Timeout::Error
|
279
|
+
raise Fasp::Error, 'timeout waiting mgt port connect'
|
280
|
+
rescue Interrupt
|
281
|
+
raise Fasp::Error, 'transfer interrupted by user'
|
287
282
|
ensure
|
288
283
|
# if ascp was successfully started
|
289
284
|
unless ascp_pid.nil?
|
290
285
|
# "wait" for process to avoid zombie
|
291
286
|
Process.wait(ascp_pid)
|
292
|
-
status
|
287
|
+
status=$CHILD_STATUS
|
293
288
|
ascp_pid=nil
|
294
289
|
session.delete(:io)
|
295
290
|
if !status.success?
|
296
291
|
message="ascp failed with code #{status.exitstatus}"
|
297
|
-
if
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
-
raise Fasp::Error.new(message)
|
302
|
-
end
|
292
|
+
# raise error only if there was not already an exception
|
293
|
+
raise Fasp::Error, message unless exception_raised
|
294
|
+
# else just debug, as main exception is already here
|
295
|
+
Log.log.debug(message)
|
303
296
|
end
|
304
297
|
end
|
305
298
|
end # begin-ensure
|
@@ -341,11 +334,8 @@ module Aspera
|
|
341
334
|
if !options.nil?
|
342
335
|
raise "expecting Hash (or nil), but have #{options.class}" unless options.is_a?(Hash)
|
343
336
|
options.each do |k,v|
|
344
|
-
|
345
|
-
|
346
|
-
else
|
347
|
-
raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map{|i|i.to_s}.join(",")}"
|
348
|
-
end
|
337
|
+
raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.has_key?(k)
|
338
|
+
@options[k]=v
|
349
339
|
end
|
350
340
|
end
|
351
341
|
Log.log.debug("local options= #{options}")
|
@@ -364,13 +354,12 @@ module Aspera
|
|
364
354
|
start_transfer_with_args_env(session[:env_args],session)
|
365
355
|
end
|
366
356
|
Log.log.debug('transfer ok'.bg_green)
|
367
|
-
rescue => e
|
357
|
+
rescue StandardError => e
|
368
358
|
session[:error]=e
|
369
359
|
Log.log.error("Transfer thread error: #{e.class}:\n#{e.message}:\n#{e.backtrace.join("\n")}".red) if Log.instance.level.eql?(:debug)
|
370
360
|
end
|
371
361
|
Log.log.debug("EXIT (#{Thread.current[:name]})")
|
372
362
|
end
|
373
|
-
|
374
363
|
end # AgentDirect
|
375
364
|
end
|
376
365
|
end
|
@@ -1,5 +1,6 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
2
|
require 'aspera/fasp/agent_base'
|
3
|
+
require 'aspera/fasp/transfer_spec'
|
3
4
|
require 'aspera/log'
|
4
5
|
require 'aspera/rest'
|
5
6
|
require 'websocket-client-simple'
|
@@ -29,7 +30,7 @@ module Aspera
|
|
29
30
|
# we need to keep track of actual file path because transfer spec is modified to be sent in web socket
|
30
31
|
source_paths=[]
|
31
32
|
# get source root or nil
|
32
|
-
source_root =
|
33
|
+
source_root = transfer_spec.has_key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] : nil
|
33
34
|
# source root is ignored by GW, used only here
|
34
35
|
transfer_spec.delete('source_root')
|
35
36
|
# compute total size of files to upload (for progress)
|
@@ -61,27 +62,23 @@ module Aspera
|
|
61
62
|
received+=1
|
62
63
|
else
|
63
64
|
message.chomp!
|
64
|
-
|
65
|
-
error=JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message']
|
66
|
-
else
|
67
|
-
error="expecting quotes in [#{message}]"
|
68
|
-
end
|
65
|
+
error=message.start_with?('"') && message.end_with?('"') ? JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message'] : "expecting quotes in [#{message}]"
|
69
66
|
end
|
70
67
|
end
|
71
68
|
ws.on :error do |e|
|
72
69
|
error=e
|
73
70
|
end
|
74
71
|
ws.on :open do
|
75
|
-
Log.log.info(
|
72
|
+
Log.log.info('ws: open')
|
76
73
|
end
|
77
74
|
ws.on :close do
|
78
|
-
Log.log.info(
|
75
|
+
Log.log.info('ws: close')
|
79
76
|
end
|
80
77
|
# open web socket to end point
|
81
78
|
ws.connect("#{@gw_api.params[:base_url]}/upload")
|
82
79
|
# async wait ready
|
83
|
-
while !ws.open?
|
84
|
-
Log.log.info(
|
80
|
+
while !ws.open? && error.nil?
|
81
|
+
Log.log.info('ws: wait')
|
85
82
|
sleep(0.2)
|
86
83
|
end
|
87
84
|
# notify progress bar
|
@@ -101,11 +98,11 @@ module Aspera
|
|
101
98
|
file_size=item['file_size']
|
102
99
|
file_name=File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
|
103
100
|
# compute total number of slices
|
104
|
-
numslices=1+(file_size-1)/@upload_chunksize
|
101
|
+
numslices=1+((file_size-1)/@upload_chunksize)
|
105
102
|
File.open(source_paths[file_index]) do |file|
|
106
103
|
# current slice index
|
107
104
|
slicenum=0
|
108
|
-
while !file.eof?
|
105
|
+
while !file.eof?
|
109
106
|
data=file.read(@upload_chunksize)
|
110
107
|
slice_data={
|
111
108
|
name: file_name,
|
@@ -117,11 +114,11 @@ module Aspera
|
|
117
114
|
fileIndex: file_index
|
118
115
|
}
|
119
116
|
# log without data
|
120
|
-
Log.dump(:slide_data,slice_data.keys.
|
117
|
+
Log.dump(:slide_data,slice_data.keys.each_with_object({}){|i,m|m[i]=i.eql?(:data)?'base64 data':slice_data[i];}) if slicenum.eql?(0)
|
121
118
|
ws_send(ws,:slice_upload, slice_data)
|
122
119
|
sent_bytes+=data.length
|
123
120
|
currenttime=Time.now
|
124
|
-
if lastevent.nil?
|
121
|
+
if lastevent.nil? || ((currenttime-lastevent)>UPLOAD_REFRESH_SEC)
|
125
122
|
notify_progress(session_id,sent_bytes)
|
126
123
|
lastevent=currenttime
|
127
124
|
end
|
@@ -146,37 +143,37 @@ module Aspera
|
|
146
143
|
dname=dname.gsub(/\.@gw_api.*$/,'')
|
147
144
|
# ands add indication of number of files if there is more than one
|
148
145
|
if transfer_spec['paths'].length > 1
|
149
|
-
dname
|
146
|
+
dname+=" #{transfer_spec['paths'].length} Files"
|
150
147
|
end
|
151
148
|
transfer_spec['download_name']=dname
|
152
149
|
end
|
153
150
|
creation=@gw_api.create('download',{'transfer_spec'=>transfer_spec})[:data]
|
154
151
|
transfer_uuid=creation['url'].split('/').last
|
155
|
-
if transfer_spec['zip_required']
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
152
|
+
file_dest=if transfer_spec['zip_required'] || transfer_spec['paths'].length > 1
|
153
|
+
# it is a zip file if zip is required or there is more than 1 file
|
154
|
+
transfer_spec['download_name']+'.zip'
|
155
|
+
else
|
156
|
+
# it is a plain file if we don't require zip and there is only one file
|
157
|
+
File.basename(transfer_spec['paths'].first['source'])
|
161
158
|
end
|
162
159
|
file_dest=File.join(transfer_spec['destination_root'],file_dest)
|
163
|
-
@gw_api.call({:
|
160
|
+
@gw_api.call({operation: 'GET',subpath: "download/#{transfer_uuid}",save_to_file: file_dest})
|
164
161
|
end
|
165
162
|
|
166
163
|
# start FASP transfer based on transfer spec (hash table)
|
167
164
|
# note that it is asynchronous
|
168
165
|
# HTTP download only supports file list
|
169
166
|
def start_transfer(transfer_spec,options={})
|
170
|
-
raise
|
171
|
-
raise
|
172
|
-
raise
|
173
|
-
raise
|
167
|
+
raise 'GW URL must be set' if @gw_api.nil?
|
168
|
+
raise 'option: must be hash (or nil)' unless options.is_a?(Hash)
|
169
|
+
raise 'paths: must be Array' unless transfer_spec['paths'].is_a?(Array)
|
170
|
+
raise 'only token based transfer is supported in GW' unless transfer_spec['token'].is_a?(String)
|
174
171
|
Log.dump(:user_spec,transfer_spec)
|
175
172
|
transfer_spec['authentication']||='token'
|
176
173
|
case transfer_spec['direction']
|
177
|
-
when
|
174
|
+
when Fasp::TransferSpec::DIRECTION_SEND
|
178
175
|
upload(transfer_spec)
|
179
|
-
when
|
176
|
+
when Fasp::TransferSpec::DIRECTION_RECEIVE
|
180
177
|
download(transfer_spec)
|
181
178
|
else
|
182
179
|
raise "unexpected direction: [#{transfer_spec['direction']}]"
|
@@ -190,25 +187,22 @@ module Aspera
|
|
190
187
|
end
|
191
188
|
|
192
189
|
# terminates monitor thread
|
193
|
-
def shutdown
|
194
|
-
end
|
190
|
+
def shutdown; end
|
195
191
|
|
196
|
-
def url=(api_url)
|
197
|
-
end
|
192
|
+
def url=(api_url); end
|
198
193
|
|
199
194
|
private
|
200
195
|
|
201
196
|
def initialize(params)
|
202
|
-
raise
|
197
|
+
raise 'params must be Hash' unless params.is_a?(Hash)
|
203
198
|
params=params.symbolize_keys
|
204
|
-
raise
|
199
|
+
raise 'must have only one param: url' unless params.keys.eql?([:url])
|
205
200
|
super()
|
206
|
-
@gw_api=Rest.new({:
|
201
|
+
@gw_api=Rest.new({base_url: params[:url]})
|
207
202
|
api_info = @gw_api.read('info')[:data]
|
208
|
-
Log.log.info(
|
209
|
-
@upload_chunksize=
|
203
|
+
Log.log.info(api_info.to_s)
|
204
|
+
@upload_chunksize=128_000 # TODO: configurable ?
|
210
205
|
end
|
211
|
-
|
212
206
|
end # AgentHttpgw
|
213
207
|
end
|
214
208
|
end
|