aspera-cli 4.6.0 → 4.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +427 -300
- data/bin/ascli +2 -1
- data/bin/asession +1 -0
- data/docs/test_env.conf +2 -0
- data/examples/aoc.rb +4 -3
- data/examples/faspex4.rb +21 -19
- data/examples/proxy.pac +1 -1
- data/examples/transfer.rb +15 -15
- data/lib/aspera/aoc.rb +135 -124
- data/lib/aspera/ascmd.rb +85 -75
- data/lib/aspera/ats_api.rb +11 -10
- data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
- data/lib/aspera/cli/extended_value.rb +42 -33
- data/lib/aspera/cli/formater.rb +138 -111
- data/lib/aspera/cli/info.rb +17 -0
- data/lib/aspera/cli/listener/line_dump.rb +3 -2
- data/lib/aspera/cli/listener/logger.rb +2 -1
- data/lib/aspera/cli/listener/progress.rb +16 -18
- data/lib/aspera/cli/listener/progress_multi.rb +13 -16
- data/lib/aspera/cli/main.rb +122 -130
- data/lib/aspera/cli/manager.rb +146 -154
- data/lib/aspera/cli/plugin.rb +38 -34
- data/lib/aspera/cli/plugins/alee.rb +6 -6
- data/lib/aspera/cli/plugins/aoc.rb +273 -276
- data/lib/aspera/cli/plugins/ats.rb +82 -76
- data/lib/aspera/cli/plugins/bss.rb +14 -16
- data/lib/aspera/cli/plugins/config.rb +350 -306
- data/lib/aspera/cli/plugins/console.rb +23 -19
- data/lib/aspera/cli/plugins/cos.rb +18 -18
- data/lib/aspera/cli/plugins/faspex.rb +180 -159
- data/lib/aspera/cli/plugins/faspex5.rb +64 -54
- data/lib/aspera/cli/plugins/node.rb +147 -140
- data/lib/aspera/cli/plugins/orchestrator.rb +68 -66
- data/lib/aspera/cli/plugins/preview.rb +92 -96
- data/lib/aspera/cli/plugins/server.rb +79 -75
- data/lib/aspera/cli/plugins/shares.rb +23 -24
- data/lib/aspera/cli/plugins/sync.rb +20 -22
- data/lib/aspera/cli/transfer_agent.rb +40 -39
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +35 -27
- data/lib/aspera/command_line_builder.rb +48 -34
- data/lib/aspera/cos_node.rb +29 -21
- data/lib/aspera/data_repository.rb +3 -2
- data/lib/aspera/environment.rb +50 -45
- data/lib/aspera/fasp/agent_base.rb +22 -20
- data/lib/aspera/fasp/agent_connect.rb +13 -11
- data/lib/aspera/fasp/agent_direct.rb +48 -59
- data/lib/aspera/fasp/agent_httpgw.rb +33 -39
- data/lib/aspera/fasp/agent_node.rb +15 -13
- data/lib/aspera/fasp/agent_trsdk.rb +12 -14
- data/lib/aspera/fasp/error.rb +2 -1
- data/lib/aspera/fasp/error_info.rb +68 -52
- data/lib/aspera/fasp/installation.rb +106 -94
- data/lib/aspera/fasp/listener.rb +1 -0
- data/lib/aspera/fasp/parameters.rb +83 -92
- data/lib/aspera/fasp/parameters.yaml +305 -249
- data/lib/aspera/fasp/resume_policy.rb +11 -14
- data/lib/aspera/fasp/transfer_spec.rb +26 -0
- data/lib/aspera/fasp/uri.rb +22 -21
- data/lib/aspera/faspex_gw.rb +55 -90
- data/lib/aspera/hash_ext.rb +4 -3
- data/lib/aspera/id_generator.rb +8 -7
- data/lib/aspera/keychain/encrypted_hash.rb +17 -16
- data/lib/aspera/keychain/macos_security.rb +6 -10
- data/lib/aspera/log.rb +25 -20
- data/lib/aspera/nagios.rb +13 -12
- data/lib/aspera/node.rb +30 -22
- data/lib/aspera/oauth.rb +175 -226
- data/lib/aspera/open_application.rb +4 -3
- data/lib/aspera/persistency_action_once.rb +6 -6
- data/lib/aspera/persistency_folder.rb +5 -9
- data/lib/aspera/preview/file_types.rb +6 -5
- data/lib/aspera/preview/generator.rb +25 -24
- data/lib/aspera/preview/options.rb +16 -14
- data/lib/aspera/preview/utils.rb +98 -98
- data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
- data/lib/aspera/proxy_auto_config.rb +111 -20
- data/lib/aspera/rest.rb +115 -113
- data/lib/aspera/rest_call_error.rb +2 -2
- data/lib/aspera/rest_error_analyzer.rb +23 -25
- data/lib/aspera/rest_errors_aspera.rb +15 -14
- data/lib/aspera/ssh.rb +12 -10
- data/lib/aspera/sync.rb +42 -41
- data/lib/aspera/temp_file_manager.rb +18 -14
- data/lib/aspera/timer_limiter.rb +2 -1
- data/lib/aspera/uri_reader.rb +7 -5
- data/lib/aspera/web_auth.rb +79 -76
- metadata +64 -21
- data/docs/Makefile +0 -65
- data/docs/README.erb.md +0 -4424
- data/docs/README.md +0 -13
- data/docs/diagrams.txt +0 -49
- data/docs/doc_tools.rb +0 -58
- data/lib/aspera/cli/plugins/shares2.rb +0 -114
- data/lib/aspera/fasp/default.rb +0 -17
|
@@ -1,30 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
module Aspera
|
|
2
3
|
module Fasp
|
|
3
4
|
# Base class for FASP transfer agents
|
|
4
5
|
# sub classes shall implement start_transfer and shutdown
|
|
5
6
|
class AgentBase
|
|
7
|
+
# fields description for JSON generation
|
|
8
|
+
INTEGER_FIELDS=%w[Bytescont FaspFileArgIndex StartByte Rate MinRate Port Priority RateCap MinRateCap TCPPort CreatePolicy TimePolicy DatagramSize XoptFlags VLinkVersion
|
|
9
|
+
PeerVLinkVersion DSPipelineDepth PeerDSPipelineDepth ReadBlockSize WriteBlockSize ClusterNumNodes ClusterNodeId Size Written Loss FileBytes PreTransferBytes TransferBytes PMTU Elapsedusec ArgScansAttempted ArgScansCompleted PathScansAttempted FileScansCompleted TransfersAttempted TransfersPassed Delay].freeze
|
|
10
|
+
BOOLEAN_FIELDS=%w[Encryption Remote RateLock MinRateLock PolicyLock FilesEncrypt FilesDecrypt VLinkLocalEnabled VLinkRemoteEnabled MoveRange Keepalive TestLogin UseProxy
|
|
11
|
+
Precalc RTTAutocorrect].freeze
|
|
12
|
+
EXPECTED_METHODS=%i[text struct enhanced].freeze
|
|
13
|
+
private_constant :INTEGER_FIELDS,:BOOLEAN_FIELDS,:EXPECTED_METHODS
|
|
6
14
|
|
|
7
15
|
private
|
|
8
16
|
|
|
9
|
-
# fields description for JSON generation
|
|
10
|
-
IntegerFields=['Bytescont','FaspFileArgIndex','StartByte','Rate','MinRate','Port','Priority','RateCap','MinRateCap','TCPPort','CreatePolicy','TimePolicy','DatagramSize','XoptFlags','VLinkVersion','PeerVLinkVersion','DSPipelineDepth','PeerDSPipelineDepth','ReadBlockSize','WriteBlockSize','ClusterNumNodes','ClusterNodeId','Size','Written','Loss','FileBytes','PreTransferBytes','TransferBytes','PMTU','Elapsedusec','ArgScansAttempted','ArgScansCompleted','PathScansAttempted','FileScansCompleted','TransfersAttempted','TransfersPassed','Delay']
|
|
11
|
-
BooleanFields=['Encryption','Remote','RateLock','MinRateLock','PolicyLock','FilesEncrypt','FilesDecrypt','VLinkLocalEnabled','VLinkRemoteEnabled','MoveRange','Keepalive','TestLogin','UseProxy','Precalc','RTTAutocorrect']
|
|
12
|
-
ExpectedMethod=[:text,:struct,:enhanced]
|
|
13
|
-
|
|
14
17
|
# translates legacy event into enhanced (JSON) event
|
|
15
18
|
def enhanced_event_format(event)
|
|
16
|
-
return event.keys.
|
|
19
|
+
return event.keys.each_with_object({}) do |e,h|
|
|
17
20
|
# capital_to_snake_case
|
|
18
21
|
new_name=e.
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
22
|
+
gsub(/([a-z\d])([A-Z])/,'\1_\2').
|
|
23
|
+
gsub(/([a-z\d])(usec)$/,'\1_\2').
|
|
24
|
+
gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
|
|
25
|
+
downcase
|
|
23
26
|
value=event[e]
|
|
24
|
-
value=value.to_i if
|
|
25
|
-
value=value.eql?('Yes')
|
|
27
|
+
value=value.to_i if INTEGER_FIELDS.include?(e)
|
|
28
|
+
value=value.eql?('Yes') if BOOLEAN_FIELDS.include?(e)
|
|
26
29
|
h[new_name]=value
|
|
27
|
-
h
|
|
28
30
|
end
|
|
29
31
|
end
|
|
30
32
|
|
|
@@ -33,14 +35,14 @@ module Aspera
|
|
|
33
35
|
end
|
|
34
36
|
|
|
35
37
|
def notify_listeners(current_event_text,current_event_data)
|
|
36
|
-
Log.log.debug(
|
|
38
|
+
Log.log.debug('send event to listeners')
|
|
37
39
|
enhanced_event=nil
|
|
38
40
|
@listeners.each do |listener|
|
|
39
|
-
listener.
|
|
40
|
-
listener.
|
|
41
|
+
listener.event_text(current_event_text) if listener.respond_to?(:event_text)
|
|
42
|
+
listener.event_struct(current_event_data) if listener.respond_to?(:event_struct)
|
|
41
43
|
if listener.respond_to?(:event_enhanced)
|
|
42
44
|
enhanced_event=enhanced_event_format(current_event_data) if enhanced_event.nil?
|
|
43
|
-
listener.
|
|
45
|
+
listener.event_enhanced(enhanced_event)
|
|
44
46
|
end
|
|
45
47
|
end
|
|
46
48
|
end # notify_listeners
|
|
@@ -58,12 +60,13 @@ module Aspera
|
|
|
58
60
|
end
|
|
59
61
|
|
|
60
62
|
public
|
|
63
|
+
|
|
61
64
|
LISTENER_SESSION_ID_B='ListenerSessionId'
|
|
62
65
|
LISTENER_SESSION_ID_S='listener_session_id'
|
|
63
66
|
|
|
64
67
|
# listener receives events
|
|
65
68
|
def add_listener(listener)
|
|
66
|
-
raise "expect one of #{
|
|
69
|
+
raise "expect one of #{EXPECTED_METHODS}" if EXPECTED_METHODS.inject(0){|m,e|m+=listener.respond_to?("event_#{e}")?1:0;m}.eql?(0)
|
|
67
70
|
@listeners.push(listener)
|
|
68
71
|
self
|
|
69
72
|
end
|
|
@@ -72,14 +75,13 @@ module Aspera
|
|
|
72
75
|
# it must be a list of :success or exception
|
|
73
76
|
def self.validate_status_list(statuses)
|
|
74
77
|
raise "internal error: bad statuses type: #{statuses.class}" unless statuses.is_a?(Array)
|
|
75
|
-
raise "internal error: bad statuses content: #{statuses}" unless statuses.select{|i|!i.eql?(:success)
|
|
78
|
+
raise "internal error: bad statuses content: #{statuses}" unless statuses.select{|i|!i.eql?(:success) && !i.is_a?(StandardError)}.empty?
|
|
76
79
|
end
|
|
77
80
|
|
|
78
81
|
# the following methods must be implemented by subclass:
|
|
79
82
|
# start_transfer(transfer_spec,options) : start and wait for completion
|
|
80
83
|
# wait_for_transfers_completion : wait for termination of all transfers, @return list of : :success or error message
|
|
81
84
|
# optional: shutdown
|
|
82
|
-
|
|
83
85
|
end
|
|
84
86
|
end
|
|
85
87
|
end
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
require 'aspera/fasp/agent_base'
|
|
2
3
|
require 'aspera/rest'
|
|
3
4
|
require 'aspera/open_application'
|
|
@@ -10,7 +11,7 @@ module Aspera
|
|
|
10
11
|
MAX_CONNECT_START_RETRY=3
|
|
11
12
|
SLEEP_SEC_BETWEEN_RETRY=2
|
|
12
13
|
private_constant :MAX_CONNECT_START_RETRY,:SLEEP_SEC_BETWEEN_RETRY
|
|
13
|
-
def initialize(
|
|
14
|
+
def initialize(_options)
|
|
14
15
|
super()
|
|
15
16
|
@connect_settings={
|
|
16
17
|
'app_id' => SecureRandom.uuid
|
|
@@ -23,7 +24,7 @@ module Aspera
|
|
|
23
24
|
@connect_api=Rest.new({base_url: "#{connect_url}/v5/connect",headers: {'Origin'=>Rest.user_agent}}) # could use v6 also now
|
|
24
25
|
cinfo=@connect_api.read('info/version')[:data]
|
|
25
26
|
Log.dump(:connect_version,cinfo)
|
|
26
|
-
rescue => e # Errno::ECONNREFUSED
|
|
27
|
+
rescue StandardError => e # Errno::ECONNREFUSED
|
|
27
28
|
raise StandardError,"Unable to start connect after #{trynumber} try" if trynumber >= MAX_CONNECT_START_RETRY
|
|
28
29
|
Log.log.warn("connect is not started. Retry ##{trynumber}, err=#{e}")
|
|
29
30
|
trynumber+=1
|
|
@@ -36,11 +37,12 @@ module Aspera
|
|
|
36
37
|
end
|
|
37
38
|
end
|
|
38
39
|
|
|
39
|
-
def start_transfer(transfer_spec,
|
|
40
|
+
def start_transfer(transfer_spec,_options=nil)
|
|
40
41
|
if transfer_spec['direction'] == 'send'
|
|
41
42
|
Log.log.warn("Connect requires upload selection using GUI, ignoring #{transfer_spec['paths']}".red)
|
|
42
43
|
transfer_spec.delete('paths')
|
|
43
|
-
resdata=@connect_api.create('windows/select-open-file-dialog/',
|
|
44
|
+
resdata=@connect_api.create('windows/select-open-file-dialog/',
|
|
45
|
+
{'aspera_connect_settings'=>@connect_settings,'title'=>'Select Files','suggestedName'=>'','allowMultipleSelection'=>true,'allowedFileTypes'=>''})[:data]
|
|
44
46
|
transfer_spec['paths']=resdata['dataTransfer']['files'].map { |i| {'source'=>i['name']}}
|
|
45
47
|
end
|
|
46
48
|
@request_id=SecureRandom.uuid
|
|
@@ -50,10 +52,10 @@ module Aspera
|
|
|
50
52
|
connect_transfer_args={
|
|
51
53
|
'aspera_connect_settings'=>@connect_settings.merge({
|
|
52
54
|
'request_id' =>@request_id,
|
|
53
|
-
'allow_dialogs' =>true
|
|
55
|
+
'allow_dialogs' =>true
|
|
54
56
|
}),
|
|
55
57
|
'transfer_specs' =>[{
|
|
56
|
-
'transfer_spec' =>transfer_spec
|
|
58
|
+
'transfer_spec' =>transfer_spec
|
|
57
59
|
}]}
|
|
58
60
|
# asynchronous anyway
|
|
59
61
|
res=@connect_api.create('transfers/start',connect_transfer_args)[:data]
|
|
@@ -70,7 +72,7 @@ module Aspera
|
|
|
70
72
|
if tr_info['transfer_info'].is_a?(Hash)
|
|
71
73
|
trdata=tr_info['transfer_info']
|
|
72
74
|
if trdata.nil?
|
|
73
|
-
Log.log.warn(
|
|
75
|
+
Log.log.warn('no session in Connect')
|
|
74
76
|
break
|
|
75
77
|
end
|
|
76
78
|
# TODO: get session id
|
|
@@ -87,7 +89,7 @@ module Aspera
|
|
|
87
89
|
spinner.spin
|
|
88
90
|
when 'running'
|
|
89
91
|
#puts "running: sessions:#{trdata['sessions'].length}, #{trdata['sessions'].map{|i| i['bytes_transferred']}.join(',')}"
|
|
90
|
-
if !started
|
|
92
|
+
if !started && (trdata['bytes_expected'] != 0)
|
|
91
93
|
spinner.success unless spinner.nil?
|
|
92
94
|
notify_begin(@connect_settings['app_id'],trdata['bytes_expected'])
|
|
93
95
|
started=true
|
|
@@ -96,14 +98,14 @@ module Aspera
|
|
|
96
98
|
end
|
|
97
99
|
when 'failed'
|
|
98
100
|
spinner.error unless spinner.nil?
|
|
99
|
-
raise Fasp::Error
|
|
101
|
+
raise Fasp::Error, trdata['error_desc']
|
|
100
102
|
else
|
|
101
|
-
raise Fasp::Error
|
|
103
|
+
raise Fasp::Error, "unknown status: #{trdata['status']}: #{trdata['error_desc']}"
|
|
102
104
|
end
|
|
103
105
|
end
|
|
104
106
|
sleep 1
|
|
105
107
|
end
|
|
106
|
-
rescue => e
|
|
108
|
+
rescue StandardError => e
|
|
107
109
|
return [e]
|
|
108
110
|
end
|
|
109
111
|
return [:success]
|
|
@@ -1,20 +1,16 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
# FASP manager for Ruby
|
|
4
|
-
# Aspera 2016
|
|
5
|
-
# Laurent Martin
|
|
6
|
-
#
|
|
7
|
-
##############################################################################
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require 'English'
|
|
8
3
|
require 'aspera/fasp/agent_base'
|
|
9
4
|
require 'aspera/fasp/error'
|
|
10
5
|
require 'aspera/fasp/parameters'
|
|
11
6
|
require 'aspera/fasp/installation'
|
|
12
7
|
require 'aspera/fasp/resume_policy'
|
|
13
|
-
require 'aspera/fasp/
|
|
8
|
+
require 'aspera/fasp/transfer_spec'
|
|
14
9
|
require 'aspera/log'
|
|
15
10
|
require 'socket'
|
|
16
11
|
require 'timeout'
|
|
17
12
|
require 'securerandom'
|
|
13
|
+
require 'shellwords'
|
|
18
14
|
|
|
19
15
|
module Aspera
|
|
20
16
|
module Fasp
|
|
@@ -27,7 +23,7 @@ module Aspera
|
|
|
27
23
|
wss: false,
|
|
28
24
|
multi_incr_udp: true,
|
|
29
25
|
resume: {},
|
|
30
|
-
quiet: true
|
|
26
|
+
quiet: true # by default no interactive progress bar
|
|
31
27
|
}
|
|
32
28
|
private_constant :DEFAULT_OPTIONS
|
|
33
29
|
|
|
@@ -43,10 +39,11 @@ module Aspera
|
|
|
43
39
|
# clone transfer spec because we modify it (first level keys)
|
|
44
40
|
transfer_spec=transfer_spec.clone
|
|
45
41
|
# if there is aspera tags
|
|
46
|
-
if transfer_spec['tags'].is_a?(Hash)
|
|
42
|
+
if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags']['aspera'].is_a?(Hash)
|
|
47
43
|
# TODO: what is this for ? only on local ascp ?
|
|
48
44
|
# NOTE: important: transfer id must be unique: generate random id
|
|
49
45
|
# using a non unique id results in discard of tags in AoC, and a package is never finalized
|
|
46
|
+
# all sessions in a multi-session transfer must have the same xfer_id (see admin manual)
|
|
50
47
|
transfer_spec['tags']['aspera']['xfer_id']||=SecureRandom.uuid
|
|
51
48
|
Log.log.debug("xfer id=#{transfer_spec['xfer_id']}")
|
|
52
49
|
# TODO: useful ? node only ?
|
|
@@ -55,8 +52,8 @@ module Aspera
|
|
|
55
52
|
Log.dump('ts',transfer_spec)
|
|
56
53
|
|
|
57
54
|
# add bypass keys when authentication is token and no auth is provided
|
|
58
|
-
if transfer_spec.has_key?('token')
|
|
59
|
-
!transfer_spec.has_key?('remote_password')
|
|
55
|
+
if transfer_spec.has_key?('token') &&
|
|
56
|
+
!transfer_spec.has_key?('remote_password') &&
|
|
60
57
|
!transfer_spec.has_key?('EX_ssh_key_paths')
|
|
61
58
|
# transfer_spec['remote_password'] = Installation.instance.bypass_pass # not used
|
|
62
59
|
transfer_spec['EX_ssh_key_paths'] = Installation.instance.bypass_keys
|
|
@@ -67,24 +64,22 @@ module Aspera
|
|
|
67
64
|
multi_session_info=nil
|
|
68
65
|
if transfer_spec.has_key?('multi_session')
|
|
69
66
|
multi_session_info={
|
|
70
|
-
count: transfer_spec['multi_session'].to_i
|
|
67
|
+
count: transfer_spec['multi_session'].to_i
|
|
71
68
|
}
|
|
72
69
|
# Managed by multi-session, so delete from transfer spec
|
|
73
70
|
transfer_spec.delete('multi_session')
|
|
74
|
-
if multi_session_info[:count]
|
|
71
|
+
if multi_session_info[:count].negative?
|
|
75
72
|
Log.log.error("multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0")
|
|
76
73
|
multi_session_info = nil
|
|
77
74
|
elsif multi_session_info[:count].eql?(0)
|
|
78
|
-
Log.log.debug(
|
|
75
|
+
Log.log.debug('multi_session count is zero: no multisession')
|
|
79
76
|
multi_session_info = nil
|
|
80
|
-
|
|
77
|
+
elsif @options[:multi_incr_udp] # multi_session_info[:count] > 0
|
|
81
78
|
# if option not true: keep default udp port for all sessions
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
transfer_spec.delete('fasp_port')
|
|
87
|
-
end
|
|
79
|
+
multi_session_info[:udp_base]=transfer_spec.has_key?('fasp_port') ? transfer_spec['fasp_port'] : TransferSpec::UDP_PORT
|
|
80
|
+
# delete from original transfer spec, as we will increment values
|
|
81
|
+
transfer_spec.delete('fasp_port')
|
|
82
|
+
# override if specified, else use default value
|
|
88
83
|
end
|
|
89
84
|
end
|
|
90
85
|
|
|
@@ -101,18 +96,18 @@ module Aspera
|
|
|
101
96
|
|
|
102
97
|
# transfer job can be multi session
|
|
103
98
|
xfer_job={
|
|
104
|
-
:
|
|
105
|
-
:
|
|
99
|
+
id: job_options[:job_id],
|
|
100
|
+
sessions: [] # all sessions as below
|
|
106
101
|
}
|
|
107
102
|
|
|
108
103
|
# generic session information
|
|
109
104
|
session={
|
|
110
|
-
:
|
|
111
|
-
:
|
|
112
|
-
:
|
|
113
|
-
:
|
|
114
|
-
:env_args
|
|
115
|
-
:
|
|
105
|
+
thread: nil, # Thread object monitoring management port, not nil when pushed to :sessions
|
|
106
|
+
error: nil, # exception if failed
|
|
107
|
+
io: nil, # management port server socket
|
|
108
|
+
id: nil, # SessionId from INIT message in mgt port
|
|
109
|
+
env_args: env_args, # env vars and args to ascp (from transfer spec)
|
|
110
|
+
options: job_options # [Hash]
|
|
116
111
|
}
|
|
117
112
|
|
|
118
113
|
if multi_session_info.nil?
|
|
@@ -131,7 +126,7 @@ module Aspera
|
|
|
131
126
|
this_session[:env_args][:args]=this_session[:env_args][:args].clone()
|
|
132
127
|
this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_info[:count]}")
|
|
133
128
|
# option: increment (default as per ascp manual) or not (cluster on other side ?)
|
|
134
|
-
this_session[:env_args][:args].unshift('-O',
|
|
129
|
+
this_session[:env_args][:args].unshift('-O',(multi_session_info[:udp_base]+i-1).to_s) if @options[:multi_incr_udp]
|
|
135
130
|
this_session[:thread] = Thread.new(this_session) {|s|transfer_thread_entry(s)}
|
|
136
131
|
xfer_job[:sessions].push(this_session)
|
|
137
132
|
end
|
|
@@ -151,11 +146,11 @@ module Aspera
|
|
|
151
146
|
Log.log.debug('wait_for_transfers_completion')
|
|
152
147
|
# set to non-nil to exit loop
|
|
153
148
|
result=[]
|
|
154
|
-
@jobs.each do |
|
|
149
|
+
@jobs.each do |_id,job|
|
|
155
150
|
job[:sessions].each do |session|
|
|
156
151
|
Log.log.debug("join #{session[:thread]}")
|
|
157
152
|
session[:thread].join
|
|
158
|
-
result.push(session[:error]
|
|
153
|
+
result.push(session[:error] || :success)
|
|
159
154
|
end
|
|
160
155
|
end
|
|
161
156
|
Log.log.debug('all transfers joined')
|
|
@@ -189,7 +184,7 @@ module Aspera
|
|
|
189
184
|
Fasp::Installation.instance.path(env_args[:ascp_version])
|
|
190
185
|
end
|
|
191
186
|
# (optional) check it exists
|
|
192
|
-
raise Fasp::Error
|
|
187
|
+
raise Fasp::Error, "no such file: #{ascp_path}" unless File.exist?(ascp_path)
|
|
193
188
|
# open random local TCP port for listening for ascp management
|
|
194
189
|
mgt_sock = TCPServer.new('127.0.0.1',0)
|
|
195
190
|
# clone arguments as we eed to modify with mgt port
|
|
@@ -197,7 +192,7 @@ module Aspera
|
|
|
197
192
|
# add management port
|
|
198
193
|
ascp_arguments.unshift('-M', mgt_sock.addr[1].to_s)
|
|
199
194
|
# start ascp in sub process
|
|
200
|
-
Log.log.debug("execute: #{env_args[:env].map{|k,v| "#{k}
|
|
195
|
+
Log.log.debug("execute: #{env_args[:env].map{|k,v| "#{k}=#{Shellwords.shellescape(v)}"}.join(' ')} #{Shellwords.shellescape(ascp_path)} #{ascp_arguments.map{|a|Shellwords.shellescape(a)}.join(' ')}")
|
|
201
196
|
# start process
|
|
202
197
|
ascp_pid = Process.spawn(env_args[:env],[ascp_path,ascp_path],*ascp_arguments)
|
|
203
198
|
# in parent, wait for connection to socket max 3 seconds
|
|
@@ -225,17 +220,17 @@ module Aspera
|
|
|
225
220
|
line = ascp_mgt_io.gets
|
|
226
221
|
# nil when ascp process exits
|
|
227
222
|
break if line.nil?
|
|
228
|
-
current_event_text
|
|
223
|
+
current_event_text+=line
|
|
229
224
|
line.chomp!
|
|
230
225
|
Log.log.debug("line=[#{line}]")
|
|
231
226
|
case line
|
|
232
227
|
when 'FASPMGR 2'
|
|
233
228
|
# begin event
|
|
234
|
-
current_event_data =
|
|
229
|
+
current_event_data = {}
|
|
235
230
|
current_event_text = ''
|
|
236
231
|
when /^([^:]+): (.*)$/
|
|
237
232
|
# event field
|
|
238
|
-
current_event_data[
|
|
233
|
+
current_event_data[Regexp.last_match(1)] = Regexp.last_match(2)
|
|
239
234
|
when ''
|
|
240
235
|
# empty line is separator to end event information
|
|
241
236
|
raise 'unexpected empty line' if current_event_data.nil?
|
|
@@ -261,9 +256,9 @@ module Aspera
|
|
|
261
256
|
exception_raised=false
|
|
262
257
|
when 'ERROR'
|
|
263
258
|
Log.log.error("code: #{last_status_event['Code']}")
|
|
264
|
-
if last_status_event['Description']
|
|
259
|
+
if last_status_event['Description'] =~ /bearer token/i
|
|
265
260
|
Log.log.error('need to regenerate token'.red)
|
|
266
|
-
if session[:options].is_a?(Hash)
|
|
261
|
+
if session[:options].is_a?(Hash) && session[:options].has_key?(:regenerate_token)
|
|
267
262
|
# regenerate token here, expired, or error on it
|
|
268
263
|
# Note: in multi-session, each session will have a different one.
|
|
269
264
|
env_args[:env]['ASPERA_SCP_TOKEN']=session[:options][:regenerate_token].call(true)
|
|
@@ -279,27 +274,25 @@ module Aspera
|
|
|
279
274
|
end
|
|
280
275
|
rescue SystemCallError => e
|
|
281
276
|
# Process.spawn
|
|
282
|
-
raise Fasp::Error
|
|
283
|
-
rescue Timeout::Error
|
|
284
|
-
raise Fasp::Error
|
|
285
|
-
rescue Interrupt
|
|
286
|
-
raise Fasp::Error
|
|
277
|
+
raise Fasp::Error, e.message
|
|
278
|
+
rescue Timeout::Error
|
|
279
|
+
raise Fasp::Error, 'timeout waiting mgt port connect'
|
|
280
|
+
rescue Interrupt
|
|
281
|
+
raise Fasp::Error, 'transfer interrupted by user'
|
|
287
282
|
ensure
|
|
288
283
|
# if ascp was successfully started
|
|
289
284
|
unless ascp_pid.nil?
|
|
290
285
|
# "wait" for process to avoid zombie
|
|
291
286
|
Process.wait(ascp_pid)
|
|
292
|
-
status
|
|
287
|
+
status=$CHILD_STATUS
|
|
293
288
|
ascp_pid=nil
|
|
294
289
|
session.delete(:io)
|
|
295
290
|
if !status.success?
|
|
296
291
|
message="ascp failed with code #{status.exitstatus}"
|
|
297
|
-
if
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
raise Fasp::Error.new(message)
|
|
302
|
-
end
|
|
292
|
+
# raise error only if there was not already an exception
|
|
293
|
+
raise Fasp::Error, message unless exception_raised
|
|
294
|
+
# else just debug, as main exception is already here
|
|
295
|
+
Log.log.debug(message)
|
|
303
296
|
end
|
|
304
297
|
end
|
|
305
298
|
end # begin-ensure
|
|
@@ -341,11 +334,8 @@ module Aspera
|
|
|
341
334
|
if !options.nil?
|
|
342
335
|
raise "expecting Hash (or nil), but have #{options.class}" unless options.is_a?(Hash)
|
|
343
336
|
options.each do |k,v|
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
else
|
|
347
|
-
raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map{|i|i.to_s}.join(",")}"
|
|
348
|
-
end
|
|
337
|
+
raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.has_key?(k)
|
|
338
|
+
@options[k]=v
|
|
349
339
|
end
|
|
350
340
|
end
|
|
351
341
|
Log.log.debug("local options= #{options}")
|
|
@@ -364,13 +354,12 @@ module Aspera
|
|
|
364
354
|
start_transfer_with_args_env(session[:env_args],session)
|
|
365
355
|
end
|
|
366
356
|
Log.log.debug('transfer ok'.bg_green)
|
|
367
|
-
rescue => e
|
|
357
|
+
rescue StandardError => e
|
|
368
358
|
session[:error]=e
|
|
369
359
|
Log.log.error("Transfer thread error: #{e.class}:\n#{e.message}:\n#{e.backtrace.join("\n")}".red) if Log.instance.level.eql?(:debug)
|
|
370
360
|
end
|
|
371
361
|
Log.log.debug("EXIT (#{Thread.current[:name]})")
|
|
372
362
|
end
|
|
373
|
-
|
|
374
363
|
end # AgentDirect
|
|
375
364
|
end
|
|
376
365
|
end
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
2
|
require 'aspera/fasp/agent_base'
|
|
3
|
+
require 'aspera/fasp/transfer_spec'
|
|
3
4
|
require 'aspera/log'
|
|
4
5
|
require 'aspera/rest'
|
|
5
6
|
require 'websocket-client-simple'
|
|
@@ -29,7 +30,7 @@ module Aspera
|
|
|
29
30
|
# we need to keep track of actual file path because transfer spec is modified to be sent in web socket
|
|
30
31
|
source_paths=[]
|
|
31
32
|
# get source root or nil
|
|
32
|
-
source_root =
|
|
33
|
+
source_root = transfer_spec.has_key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] : nil
|
|
33
34
|
# source root is ignored by GW, used only here
|
|
34
35
|
transfer_spec.delete('source_root')
|
|
35
36
|
# compute total size of files to upload (for progress)
|
|
@@ -61,27 +62,23 @@ module Aspera
|
|
|
61
62
|
received+=1
|
|
62
63
|
else
|
|
63
64
|
message.chomp!
|
|
64
|
-
|
|
65
|
-
error=JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message']
|
|
66
|
-
else
|
|
67
|
-
error="expecting quotes in [#{message}]"
|
|
68
|
-
end
|
|
65
|
+
error=message.start_with?('"') && message.end_with?('"') ? JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message'] : "expecting quotes in [#{message}]"
|
|
69
66
|
end
|
|
70
67
|
end
|
|
71
68
|
ws.on :error do |e|
|
|
72
69
|
error=e
|
|
73
70
|
end
|
|
74
71
|
ws.on :open do
|
|
75
|
-
Log.log.info(
|
|
72
|
+
Log.log.info('ws: open')
|
|
76
73
|
end
|
|
77
74
|
ws.on :close do
|
|
78
|
-
Log.log.info(
|
|
75
|
+
Log.log.info('ws: close')
|
|
79
76
|
end
|
|
80
77
|
# open web socket to end point
|
|
81
78
|
ws.connect("#{@gw_api.params[:base_url]}/upload")
|
|
82
79
|
# async wait ready
|
|
83
|
-
while !ws.open?
|
|
84
|
-
Log.log.info(
|
|
80
|
+
while !ws.open? && error.nil?
|
|
81
|
+
Log.log.info('ws: wait')
|
|
85
82
|
sleep(0.2)
|
|
86
83
|
end
|
|
87
84
|
# notify progress bar
|
|
@@ -101,11 +98,11 @@ module Aspera
|
|
|
101
98
|
file_size=item['file_size']
|
|
102
99
|
file_name=File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
|
|
103
100
|
# compute total number of slices
|
|
104
|
-
numslices=1+(file_size-1)/@upload_chunksize
|
|
101
|
+
numslices=1+((file_size-1)/@upload_chunksize)
|
|
105
102
|
File.open(source_paths[file_index]) do |file|
|
|
106
103
|
# current slice index
|
|
107
104
|
slicenum=0
|
|
108
|
-
while !file.eof?
|
|
105
|
+
while !file.eof?
|
|
109
106
|
data=file.read(@upload_chunksize)
|
|
110
107
|
slice_data={
|
|
111
108
|
name: file_name,
|
|
@@ -117,11 +114,11 @@ module Aspera
|
|
|
117
114
|
fileIndex: file_index
|
|
118
115
|
}
|
|
119
116
|
# log without data
|
|
120
|
-
Log.dump(:slide_data,slice_data.keys.
|
|
117
|
+
Log.dump(:slide_data,slice_data.keys.each_with_object({}){|i,m|m[i]=i.eql?(:data)?'base64 data':slice_data[i];}) if slicenum.eql?(0)
|
|
121
118
|
ws_send(ws,:slice_upload, slice_data)
|
|
122
119
|
sent_bytes+=data.length
|
|
123
120
|
currenttime=Time.now
|
|
124
|
-
if lastevent.nil?
|
|
121
|
+
if lastevent.nil? || ((currenttime-lastevent)>UPLOAD_REFRESH_SEC)
|
|
125
122
|
notify_progress(session_id,sent_bytes)
|
|
126
123
|
lastevent=currenttime
|
|
127
124
|
end
|
|
@@ -146,37 +143,37 @@ module Aspera
|
|
|
146
143
|
dname=dname.gsub(/\.@gw_api.*$/,'')
|
|
147
144
|
# ands add indication of number of files if there is more than one
|
|
148
145
|
if transfer_spec['paths'].length > 1
|
|
149
|
-
dname
|
|
146
|
+
dname+=" #{transfer_spec['paths'].length} Files"
|
|
150
147
|
end
|
|
151
148
|
transfer_spec['download_name']=dname
|
|
152
149
|
end
|
|
153
150
|
creation=@gw_api.create('download',{'transfer_spec'=>transfer_spec})[:data]
|
|
154
151
|
transfer_uuid=creation['url'].split('/').last
|
|
155
|
-
if transfer_spec['zip_required']
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
152
|
+
file_dest=if transfer_spec['zip_required'] || transfer_spec['paths'].length > 1
|
|
153
|
+
# it is a zip file if zip is required or there is more than 1 file
|
|
154
|
+
transfer_spec['download_name']+'.zip'
|
|
155
|
+
else
|
|
156
|
+
# it is a plain file if we don't require zip and there is only one file
|
|
157
|
+
File.basename(transfer_spec['paths'].first['source'])
|
|
161
158
|
end
|
|
162
159
|
file_dest=File.join(transfer_spec['destination_root'],file_dest)
|
|
163
|
-
@gw_api.call({:
|
|
160
|
+
@gw_api.call({operation: 'GET',subpath: "download/#{transfer_uuid}",save_to_file: file_dest})
|
|
164
161
|
end
|
|
165
162
|
|
|
166
163
|
# start FASP transfer based on transfer spec (hash table)
|
|
167
164
|
# note that it is asynchronous
|
|
168
165
|
# HTTP download only supports file list
|
|
169
166
|
def start_transfer(transfer_spec,options={})
|
|
170
|
-
raise
|
|
171
|
-
raise
|
|
172
|
-
raise
|
|
173
|
-
raise
|
|
167
|
+
raise 'GW URL must be set' if @gw_api.nil?
|
|
168
|
+
raise 'option: must be hash (or nil)' unless options.is_a?(Hash)
|
|
169
|
+
raise 'paths: must be Array' unless transfer_spec['paths'].is_a?(Array)
|
|
170
|
+
raise 'only token based transfer is supported in GW' unless transfer_spec['token'].is_a?(String)
|
|
174
171
|
Log.dump(:user_spec,transfer_spec)
|
|
175
172
|
transfer_spec['authentication']||='token'
|
|
176
173
|
case transfer_spec['direction']
|
|
177
|
-
when
|
|
174
|
+
when Fasp::TransferSpec::DIRECTION_SEND
|
|
178
175
|
upload(transfer_spec)
|
|
179
|
-
when
|
|
176
|
+
when Fasp::TransferSpec::DIRECTION_RECEIVE
|
|
180
177
|
download(transfer_spec)
|
|
181
178
|
else
|
|
182
179
|
raise "unexpected direction: [#{transfer_spec['direction']}]"
|
|
@@ -190,25 +187,22 @@ module Aspera
|
|
|
190
187
|
end
|
|
191
188
|
|
|
192
189
|
# terminates monitor thread
|
|
193
|
-
def shutdown
|
|
194
|
-
end
|
|
190
|
+
def shutdown; end
|
|
195
191
|
|
|
196
|
-
def url=(api_url)
|
|
197
|
-
end
|
|
192
|
+
def url=(api_url); end
|
|
198
193
|
|
|
199
194
|
private
|
|
200
195
|
|
|
201
196
|
def initialize(params)
|
|
202
|
-
raise
|
|
197
|
+
raise 'params must be Hash' unless params.is_a?(Hash)
|
|
203
198
|
params=params.symbolize_keys
|
|
204
|
-
raise
|
|
199
|
+
raise 'must have only one param: url' unless params.keys.eql?([:url])
|
|
205
200
|
super()
|
|
206
|
-
@gw_api=Rest.new({:
|
|
201
|
+
@gw_api=Rest.new({base_url: params[:url]})
|
|
207
202
|
api_info = @gw_api.read('info')[:data]
|
|
208
|
-
Log.log.info(
|
|
209
|
-
@upload_chunksize=
|
|
203
|
+
Log.log.info(api_info.to_s)
|
|
204
|
+
@upload_chunksize=128_000 # TODO: configurable ?
|
|
210
205
|
end
|
|
211
|
-
|
|
212
206
|
end # AgentHttpgw
|
|
213
207
|
end
|
|
214
208
|
end
|