aspera-cli 4.7.0 → 4.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/README.md +1267 -999
- data/bin/ascli +20 -1
- data/bin/asession +37 -34
- data/docs/test_env.conf +7 -3
- data/examples/aoc.rb +13 -12
- data/examples/dascli +23 -0
- data/examples/faspex4.rb +34 -29
- data/examples/{transfer.rb → node.rb} +31 -59
- data/examples/server.rb +93 -0
- data/lib/aspera/aoc.rb +153 -143
- data/lib/aspera/ascmd.rb +56 -45
- data/lib/aspera/ats_api.rb +9 -6
- data/lib/aspera/cli/basic_auth_plugin.rb +18 -16
- data/lib/aspera/cli/extended_value.rb +33 -30
- data/lib/aspera/cli/formater.rb +105 -111
- data/lib/aspera/cli/info.rb +3 -2
- data/lib/aspera/cli/listener/line_dump.rb +1 -0
- data/lib/aspera/cli/listener/logger.rb +1 -0
- data/lib/aspera/cli/listener/progress.rb +13 -12
- data/lib/aspera/cli/listener/progress_multi.rb +21 -20
- data/lib/aspera/cli/main.rb +110 -90
- data/lib/aspera/cli/manager.rb +99 -88
- data/lib/aspera/cli/plugin.rb +98 -39
- data/lib/aspera/cli/plugins/alee.rb +6 -5
- data/lib/aspera/cli/plugins/aoc.rb +581 -450
- data/lib/aspera/cli/plugins/ats.rb +84 -83
- data/lib/aspera/cli/plugins/bss.rb +30 -27
- data/lib/aspera/cli/plugins/config.rb +488 -397
- data/lib/aspera/cli/plugins/console.rb +17 -15
- data/lib/aspera/cli/plugins/cos.rb +26 -35
- data/lib/aspera/cli/plugins/faspex.rb +206 -172
- data/lib/aspera/cli/plugins/faspex5.rb +109 -74
- data/lib/aspera/cli/plugins/node.rb +379 -189
- data/lib/aspera/cli/plugins/orchestrator.rb +71 -65
- data/lib/aspera/cli/plugins/preview.rb +131 -122
- data/lib/aspera/cli/plugins/server.rb +50 -150
- data/lib/aspera/cli/plugins/shares.rb +61 -27
- data/lib/aspera/cli/plugins/sync.rb +15 -14
- data/lib/aspera/cli/transfer_agent.rb +75 -64
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +29 -28
- data/lib/aspera/command_line_builder.rb +50 -43
- data/lib/aspera/cos_node.rb +64 -38
- data/lib/aspera/data_repository.rb +1 -0
- data/lib/aspera/environment.rb +33 -10
- data/lib/aspera/fasp/agent_base.rb +35 -30
- data/lib/aspera/fasp/agent_connect.rb +35 -30
- data/lib/aspera/fasp/agent_direct.rb +68 -60
- data/lib/aspera/fasp/agent_httpgw.rb +71 -64
- data/lib/aspera/fasp/agent_node.rb +24 -23
- data/lib/aspera/fasp/agent_trsdk.rb +19 -20
- data/lib/aspera/fasp/error.rb +2 -1
- data/lib/aspera/fasp/error_info.rb +79 -68
- data/lib/aspera/fasp/installation.rb +130 -126
- data/lib/aspera/fasp/listener.rb +1 -0
- data/lib/aspera/fasp/parameters.rb +71 -60
- data/lib/aspera/fasp/parameters.yaml +69 -17
- data/lib/aspera/fasp/resume_policy.rb +14 -11
- data/lib/aspera/fasp/transfer_spec.rb +6 -5
- data/lib/aspera/fasp/uri.rb +25 -24
- data/lib/aspera/faspex_gw.rb +83 -72
- data/lib/aspera/hash_ext.rb +23 -13
- data/lib/aspera/id_generator.rb +16 -13
- data/lib/aspera/keychain/encrypted_hash.rb +61 -46
- data/lib/aspera/keychain/macos_security.rb +26 -24
- data/lib/aspera/log.rb +35 -39
- data/lib/aspera/nagios.rb +36 -28
- data/lib/aspera/node.rb +19 -19
- data/lib/aspera/oauth.rb +120 -100
- data/lib/aspera/open_application.rb +25 -22
- data/lib/aspera/persistency_action_once.rb +9 -8
- data/lib/aspera/persistency_folder.rb +13 -9
- data/lib/aspera/preview/file_types.rb +261 -266
- data/lib/aspera/preview/generator.rb +74 -73
- data/lib/aspera/preview/image_error.png +0 -0
- data/lib/aspera/preview/options.rb +7 -6
- data/lib/aspera/preview/utils.rb +30 -33
- data/lib/aspera/preview/video_error.png +0 -0
- data/lib/aspera/proxy_auto_config.rb +27 -23
- data/lib/aspera/rest.rb +73 -74
- data/lib/aspera/rest_call_error.rb +1 -0
- data/lib/aspera/rest_error_analyzer.rb +23 -19
- data/lib/aspera/rest_errors_aspera.rb +43 -40
- data/lib/aspera/secret_hider.rb +74 -0
- data/lib/aspera/ssh.rb +13 -10
- data/lib/aspera/sync.rb +49 -47
- data/lib/aspera/temp_file_manager.rb +7 -5
- data/lib/aspera/timer_limiter.rb +9 -8
- data/lib/aspera/uri_reader.rb +17 -18
- data/lib/aspera/web_auth.rb +17 -15
- data.tar.gz.sig +5 -0
- metadata +119 -35
- metadata.gz.sig +0 -0
- data/bin/dascli +0 -13
data/lib/aspera/environment.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
require 'aspera/log'
|
3
4
|
require 'rbconfig'
|
4
5
|
|
@@ -9,14 +10,14 @@ module Aspera
|
|
9
10
|
OS_X = :osx
|
10
11
|
OS_LINUX = :linux
|
11
12
|
OS_AIX = :aix
|
12
|
-
OS_LIST=[OS_WINDOWS,OS_X,OS_LINUX,OS_AIX].freeze
|
13
|
-
CPU_X86_64
|
14
|
-
CPU_PPC64
|
15
|
-
CPU_PPC64LE
|
16
|
-
CPU_S390
|
17
|
-
CPU_LIST=[CPU_X86_64,CPU_PPC64,CPU_PPC64LE,CPU_S390].freeze
|
13
|
+
OS_LIST = [OS_WINDOWS,OS_X,OS_LINUX,OS_AIX].freeze
|
14
|
+
CPU_X86_64 = :x86_64
|
15
|
+
CPU_PPC64 = :ppc64
|
16
|
+
CPU_PPC64LE = :ppc64le
|
17
|
+
CPU_S390 = :s390
|
18
|
+
CPU_LIST = [CPU_X86_64,CPU_PPC64,CPU_PPC64LE,CPU_S390].freeze
|
18
19
|
|
19
|
-
class<<self
|
20
|
+
class << self
|
20
21
|
def ruby_version
|
21
22
|
return RbConfig::CONFIG['RUBY_PROGRAM_VERSION']
|
22
23
|
end
|
@@ -45,9 +46,11 @@ module Aspera
|
|
45
46
|
return CPU_PPC64
|
46
47
|
when /s390/
|
47
48
|
return CPU_S390
|
48
|
-
|
49
|
-
|
49
|
+
when /arm/
|
50
|
+
# arm on mac has rosetta 2
|
51
|
+
return CPU_X86_64 if os.eql?(OS_X)
|
50
52
|
end
|
53
|
+
raise "Unknown CPU: #{RbConfig::CONFIG['host_cpu']}"
|
51
54
|
end
|
52
55
|
|
53
56
|
def architecture
|
@@ -63,9 +66,29 @@ module Aspera
|
|
63
66
|
# so, tell Ruby the right way
|
64
67
|
def fix_home
|
65
68
|
return unless os.eql?(OS_WINDOWS) && ENV.has_key?('USERPROFILE') && Dir.exist?(ENV['USERPROFILE'])
|
66
|
-
ENV['HOME']=ENV['USERPROFILE']
|
69
|
+
ENV['HOME'] = ENV['USERPROFILE']
|
67
70
|
Log.log.debug("Windows: set home to USERPROFILE: #{ENV['HOME']}")
|
68
71
|
end
|
72
|
+
|
73
|
+
def empty_binding
|
74
|
+
return Kernel.binding
|
75
|
+
end
|
76
|
+
|
77
|
+
# secure execution of Ruby code
|
78
|
+
def secure_eval(code)
|
79
|
+
Kernel.send('lave'.reverse,code,empty_binding, __FILE__, __LINE__)
|
80
|
+
end
|
81
|
+
|
82
|
+
# value is provided in block
|
83
|
+
def write_file_restricted(path,force: false)
|
84
|
+
raise 'coding error, missing content block' unless block_given?
|
85
|
+
if force || !File.exist?(path)
|
86
|
+
File.unlink(path) rescue nil # Windows may give error
|
87
|
+
File.write(path,yield)
|
88
|
+
File.chmod(0400,path)
|
89
|
+
end
|
90
|
+
return path
|
91
|
+
end
|
69
92
|
end
|
70
93
|
end
|
71
94
|
end
|
@@ -1,83 +1,88 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
module Aspera
|
3
4
|
module Fasp
|
4
5
|
# Base class for FASP transfer agents
|
5
6
|
# sub classes shall implement start_transfer and shutdown
|
6
7
|
class AgentBase
|
7
8
|
# fields description for JSON generation
|
8
|
-
INTEGER_FIELDS
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
9
|
+
INTEGER_FIELDS = %w[Bytescont FaspFileArgIndex StartByte Rate MinRate Port Priority RateCap MinRateCap TCPPort CreatePolicy TimePolicy
|
10
|
+
DatagramSize XoptFlags VLinkVersion PeerVLinkVersion DSPipelineDepth PeerDSPipelineDepth ReadBlockSize WriteBlockSize
|
11
|
+
ClusterNumNodes ClusterNodeId Size Written Loss FileBytes PreTransferBytes TransferBytes PMTU Elapsedusec ArgScansAttempted
|
12
|
+
ArgScansCompleted PathScansAttempted FileScansCompleted TransfersAttempted TransfersPassed Delay].freeze
|
13
|
+
BOOLEAN_FIELDS = %w[Encryption Remote RateLock MinRateLock PolicyLock FilesEncrypt FilesDecrypt VLinkLocalEnabled VLinkRemoteEnabled
|
14
|
+
MoveRange Keepalive TestLogin UseProxy Precalc RTTAutocorrect].freeze
|
15
|
+
EXPECTED_METHODS = %i[text struct enhanced].freeze
|
13
16
|
private_constant :INTEGER_FIELDS,:BOOLEAN_FIELDS,:EXPECTED_METHODS
|
14
17
|
|
18
|
+
class << self
|
19
|
+
# This checks the validity of the value returned by wait_for_transfers_completion
|
20
|
+
# it must be a list of :success or exception
|
21
|
+
def validate_status_list(statuses)
|
22
|
+
raise "internal error: bad statuses type: #{statuses.class}" unless statuses.is_a?(Array)
|
23
|
+
raise "internal error: bad statuses content: #{statuses}" unless statuses.select{|i|!i.eql?(:success) && !i.is_a?(StandardError)}.empty?
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
15
27
|
private
|
16
28
|
|
17
29
|
# translates legacy event into enhanced (JSON) event
|
18
30
|
def enhanced_event_format(event)
|
19
31
|
return event.keys.each_with_object({}) do |e,h|
|
20
32
|
# capital_to_snake_case
|
21
|
-
new_name=e.
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
value=event[e]
|
27
|
-
value=value.to_i if INTEGER_FIELDS.include?(e)
|
28
|
-
value=value.eql?('Yes') if BOOLEAN_FIELDS.include?(e)
|
29
|
-
h[new_name]=value
|
33
|
+
new_name = e.
|
34
|
+
gsub(/([a-z\d])([A-Z])/,'\1_\2').
|
35
|
+
gsub(/([a-z\d])(usec)$/,'\1_\2').
|
36
|
+
gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
|
37
|
+
downcase
|
38
|
+
value = event[e]
|
39
|
+
value = value.to_i if INTEGER_FIELDS.include?(e)
|
40
|
+
value = value.eql?('Yes') if BOOLEAN_FIELDS.include?(e)
|
41
|
+
h[new_name] = value
|
30
42
|
end
|
31
43
|
end
|
32
44
|
|
33
45
|
def initialize
|
34
|
-
@listeners=[]
|
46
|
+
@listeners = []
|
35
47
|
end
|
36
48
|
|
37
49
|
def notify_listeners(current_event_text,current_event_data)
|
38
50
|
Log.log.debug('send event to listeners')
|
39
|
-
enhanced_event=nil
|
51
|
+
enhanced_event = nil
|
40
52
|
@listeners.each do |listener|
|
41
53
|
listener.event_text(current_event_text) if listener.respond_to?(:event_text)
|
42
54
|
listener.event_struct(current_event_data) if listener.respond_to?(:event_struct)
|
43
55
|
if listener.respond_to?(:event_enhanced)
|
44
|
-
enhanced_event=enhanced_event_format(current_event_data) if enhanced_event.nil?
|
56
|
+
enhanced_event = enhanced_event_format(current_event_data) if enhanced_event.nil?
|
45
57
|
listener.event_enhanced(enhanced_event)
|
46
58
|
end
|
47
59
|
end
|
48
60
|
end # notify_listeners
|
49
61
|
|
50
62
|
def notify_begin(id,size)
|
51
|
-
notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'NOTIFICATION','PreTransferBytes'=>size})
|
63
|
+
notify_listeners('emulated',{LISTENER_SESSION_ID_B => id,'Type' => 'NOTIFICATION','PreTransferBytes' => size})
|
52
64
|
end
|
53
65
|
|
54
66
|
def notify_progress(id,size)
|
55
|
-
notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'STATS','Bytescont'=>size})
|
67
|
+
notify_listeners('emulated',{LISTENER_SESSION_ID_B => id,'Type' => 'STATS','Bytescont' => size})
|
56
68
|
end
|
57
69
|
|
58
70
|
def notify_end(id)
|
59
|
-
notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'DONE'})
|
71
|
+
notify_listeners('emulated',{LISTENER_SESSION_ID_B => id,'Type' => 'DONE'})
|
60
72
|
end
|
61
73
|
|
62
74
|
public
|
63
75
|
|
64
|
-
LISTENER_SESSION_ID_B='ListenerSessionId'
|
65
|
-
LISTENER_SESSION_ID_S='listener_session_id'
|
76
|
+
LISTENER_SESSION_ID_B = 'ListenerSessionId'
|
77
|
+
LISTENER_SESSION_ID_S = 'listener_session_id'
|
66
78
|
|
67
79
|
# listener receives events
|
68
80
|
def add_listener(listener)
|
69
|
-
raise "expect one of #{EXPECTED_METHODS}" if EXPECTED_METHODS.inject(0){|m,e|m+=listener.respond_to?("event_#{e}")?1:0;m}.eql?(0)
|
81
|
+
raise "expect one of #{EXPECTED_METHODS}" if EXPECTED_METHODS.inject(0){|m,e|m += listener.respond_to?("event_#{e}") ? 1 : 0;m}.eql?(0)
|
70
82
|
@listeners.push(listener)
|
71
83
|
self
|
72
84
|
end
|
73
85
|
|
74
|
-
# This checks the validity of the value returned by wait_for_transfers_completion
|
75
|
-
# it must be a list of :success or exception
|
76
|
-
def self.validate_status_list(statuses)
|
77
|
-
raise "internal error: bad statuses type: #{statuses.class}" unless statuses.is_a?(Array)
|
78
|
-
raise "internal error: bad statuses content: #{statuses}" unless statuses.select{|i|!i.eql?(:success) && !i.is_a?(StandardError)}.empty?
|
79
|
-
end
|
80
|
-
|
81
86
|
# the following methods must be implemented by subclass:
|
82
87
|
# start_transfer(transfer_spec,options) : start and wait for completion
|
83
88
|
# wait_for_transfers_completion : wait for termination of all transfers, @return list of : :success or error message
|
@@ -1,4 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
require 'aspera/fasp/agent_base'
|
3
4
|
require 'aspera/rest'
|
4
5
|
require 'aspera/open_application'
|
@@ -8,26 +9,26 @@ require 'tty-spinner'
|
|
8
9
|
module Aspera
|
9
10
|
module Fasp
|
10
11
|
class AgentConnect < AgentBase
|
11
|
-
MAX_CONNECT_START_RETRY=3
|
12
|
-
SLEEP_SEC_BETWEEN_RETRY=2
|
12
|
+
MAX_CONNECT_START_RETRY = 3
|
13
|
+
SLEEP_SEC_BETWEEN_RETRY = 2
|
13
14
|
private_constant :MAX_CONNECT_START_RETRY,:SLEEP_SEC_BETWEEN_RETRY
|
14
15
|
def initialize(_options)
|
15
16
|
super()
|
16
|
-
@connect_settings={
|
17
|
+
@connect_settings = {
|
17
18
|
'app_id' => SecureRandom.uuid
|
18
19
|
}
|
19
20
|
raise 'Using connect requires a graphical environment' if !OpenApplication.default_gui_mode.eql?(:graphical)
|
20
|
-
trynumber=0
|
21
|
+
trynumber = 0
|
21
22
|
begin
|
22
|
-
connect_url=Installation.instance.connect_uri
|
23
|
+
connect_url = Installation.instance.connect_uri
|
23
24
|
Log.log.debug("found: #{connect_url}")
|
24
|
-
@connect_api=Rest.new({base_url: "#{connect_url}/v5/connect",headers: {'Origin'=>Rest.user_agent}}) # could use v6 also now
|
25
|
-
cinfo
|
25
|
+
@connect_api = Rest.new({base_url: "#{connect_url}/v5/connect",headers: {'Origin' => Rest.user_agent}}) # could use v6 also now
|
26
|
+
cinfo = @connect_api.read('info/version')[:data]
|
26
27
|
Log.dump(:connect_version,cinfo)
|
27
28
|
rescue StandardError => e # Errno::ECONNREFUSED
|
28
29
|
raise StandardError,"Unable to start connect after #{trynumber} try" if trynumber >= MAX_CONNECT_START_RETRY
|
29
30
|
Log.log.warn("connect is not started. Retry ##{trynumber}, err=#{e}")
|
30
|
-
trynumber+=1
|
31
|
+
trynumber += 1
|
31
32
|
if !OpenApplication.uri_graphical('fasp://initialize')
|
32
33
|
OpenApplication.uri_graphical('https://downloads.asperasoft.com/connect2/')
|
33
34
|
raise StandardError,'Connect is not installed'
|
@@ -41,36 +42,40 @@ module Aspera
|
|
41
42
|
if transfer_spec['direction'] == 'send'
|
42
43
|
Log.log.warn("Connect requires upload selection using GUI, ignoring #{transfer_spec['paths']}".red)
|
43
44
|
transfer_spec.delete('paths')
|
44
|
-
resdata
|
45
|
-
|
46
|
-
|
45
|
+
resdata = @connect_api.create('windows/select-open-file-dialog/',{
|
46
|
+
'aspera_connect_settings' => @connect_settings,
|
47
|
+
'title' => 'Select Files',
|
48
|
+
'suggestedName' => '',
|
49
|
+
'allowMultipleSelection' => true,
|
50
|
+
'allowedFileTypes' => ''})[:data]
|
51
|
+
transfer_spec['paths'] = resdata['dataTransfer']['files'].map { |i| {'source' => i['name']}}
|
47
52
|
end
|
48
|
-
@request_id=SecureRandom.uuid
|
53
|
+
@request_id = SecureRandom.uuid
|
49
54
|
# if there is a token, we ask connect client to use well known ssh private keys
|
50
55
|
# instead of asking password
|
51
|
-
transfer_spec['authentication']='token' if transfer_spec.has_key?('token')
|
52
|
-
connect_transfer_args={
|
53
|
-
'aspera_connect_settings'
|
54
|
-
|
55
|
-
|
56
|
+
transfer_spec['authentication'] = 'token' if transfer_spec.has_key?('token')
|
57
|
+
connect_transfer_args = {
|
58
|
+
'aspera_connect_settings' => @connect_settings.merge({
|
59
|
+
'request_id' => @request_id,
|
60
|
+
'allow_dialogs' => true
|
56
61
|
}),
|
57
|
-
'transfer_specs'
|
58
|
-
|
62
|
+
'transfer_specs' => [{
|
63
|
+
'transfer_spec' => transfer_spec
|
59
64
|
}]}
|
60
65
|
# asynchronous anyway
|
61
|
-
res
|
62
|
-
@xfer_id=res['transfer_specs'].first['transfer_spec']['tags']['aspera']['xfer_id']
|
66
|
+
res = @connect_api.create('transfers/start',connect_transfer_args)[:data]
|
67
|
+
@xfer_id = res['transfer_specs'].first['transfer_spec']['tags']['aspera']['xfer_id']
|
63
68
|
end
|
64
69
|
|
65
70
|
def wait_for_transfers_completion
|
66
|
-
connect_activity_args={'aspera_connect_settings'
|
67
|
-
started=false
|
68
|
-
spinner=nil
|
71
|
+
connect_activity_args = {'aspera_connect_settings' => @connect_settings}
|
72
|
+
started = false
|
73
|
+
spinner = nil
|
69
74
|
begin
|
70
75
|
loop do
|
71
|
-
tr_info
|
76
|
+
tr_info = @connect_api.create("transfers/info/#{@xfer_id}",connect_activity_args)[:data]
|
72
77
|
if tr_info['transfer_info'].is_a?(Hash)
|
73
|
-
trdata=tr_info['transfer_info']
|
78
|
+
trdata = tr_info['transfer_info']
|
74
79
|
if trdata.nil?
|
75
80
|
Log.log.warn('no session in Connect')
|
76
81
|
break
|
@@ -90,20 +95,20 @@ module Aspera
|
|
90
95
|
when 'running'
|
91
96
|
#puts "running: sessions:#{trdata['sessions'].length}, #{trdata['sessions'].map{|i| i['bytes_transferred']}.join(',')}"
|
92
97
|
if !started && (trdata['bytes_expected'] != 0)
|
93
|
-
spinner
|
98
|
+
spinner&.success
|
94
99
|
notify_begin(@connect_settings['app_id'],trdata['bytes_expected'])
|
95
|
-
started=true
|
100
|
+
started = true
|
96
101
|
else
|
97
102
|
notify_progress(@connect_settings['app_id'],trdata['bytes_written'])
|
98
103
|
end
|
99
104
|
when 'failed'
|
100
|
-
spinner
|
105
|
+
spinner&.error
|
101
106
|
raise Fasp::Error, trdata['error_desc']
|
102
107
|
else
|
103
108
|
raise Fasp::Error, "unknown status: #{trdata['status']}: #{trdata['error_desc']}"
|
104
109
|
end
|
105
110
|
end
|
106
|
-
sleep
|
111
|
+
sleep(1)
|
107
112
|
end
|
108
113
|
rescue StandardError => e
|
109
114
|
return [e]
|
@@ -1,4 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
require 'English'
|
3
4
|
require 'aspera/fasp/agent_base'
|
4
5
|
require 'aspera/fasp/error'
|
@@ -24,7 +25,7 @@ module Aspera
|
|
24
25
|
multi_incr_udp: true,
|
25
26
|
resume: {},
|
26
27
|
quiet: true # by default no interactive progress bar
|
27
|
-
}
|
28
|
+
}.freeze
|
28
29
|
private_constant :DEFAULT_OPTIONS
|
29
30
|
|
30
31
|
# start ascp transfer (non blocking), single or multi-session
|
@@ -37,17 +38,17 @@ module Aspera
|
|
37
38
|
job_options[:resumer] ||= @resume_policy
|
38
39
|
job_options[:job_id] ||= SecureRandom.uuid
|
39
40
|
# clone transfer spec because we modify it (first level keys)
|
40
|
-
transfer_spec=transfer_spec.clone
|
41
|
+
transfer_spec = transfer_spec.clone
|
41
42
|
# if there is aspera tags
|
42
43
|
if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags']['aspera'].is_a?(Hash)
|
43
44
|
# TODO: what is this for ? only on local ascp ?
|
44
45
|
# NOTE: important: transfer id must be unique: generate random id
|
45
46
|
# using a non unique id results in discard of tags in AoC, and a package is never finalized
|
46
47
|
# all sessions in a multi-session transfer must have the same xfer_id (see admin manual)
|
47
|
-
transfer_spec['tags']['aspera']['xfer_id']||=SecureRandom.uuid
|
48
|
+
transfer_spec['tags']['aspera']['xfer_id'] ||= SecureRandom.uuid
|
48
49
|
Log.log.debug("xfer id=#{transfer_spec['xfer_id']}")
|
49
50
|
# TODO: useful ? node only ?
|
50
|
-
transfer_spec['tags']['aspera']['xfer_retry']||=3600
|
51
|
+
transfer_spec['tags']['aspera']['xfer_retry'] ||= 3600
|
51
52
|
end
|
52
53
|
Log.dump('ts',transfer_spec)
|
53
54
|
|
@@ -61,9 +62,9 @@ module Aspera
|
|
61
62
|
|
62
63
|
# Compute this before using transfer spec because it potentially modifies the transfer spec
|
63
64
|
# (even if the var is not used in single session)
|
64
|
-
multi_session_info=nil
|
65
|
+
multi_session_info = nil
|
65
66
|
if transfer_spec.has_key?('multi_session')
|
66
|
-
multi_session_info={
|
67
|
+
multi_session_info = {
|
67
68
|
count: transfer_spec['multi_session'].to_i
|
68
69
|
}
|
69
70
|
# Managed by multi-session, so delete from transfer spec
|
@@ -76,7 +77,7 @@ module Aspera
|
|
76
77
|
multi_session_info = nil
|
77
78
|
elsif @options[:multi_incr_udp] # multi_session_info[:count] > 0
|
78
79
|
# if option not true: keep default udp port for all sessions
|
79
|
-
multi_session_info[:udp_base]=transfer_spec.has_key?('fasp_port') ? transfer_spec['fasp_port'] : TransferSpec::UDP_PORT
|
80
|
+
multi_session_info[:udp_base] = transfer_spec.has_key?('fasp_port') ? transfer_spec['fasp_port'] : TransferSpec::UDP_PORT
|
80
81
|
# delete from original transfer spec, as we will increment values
|
81
82
|
transfer_spec.delete('fasp_port')
|
82
83
|
# override if specified, else use default value
|
@@ -84,10 +85,10 @@ module Aspera
|
|
84
85
|
end
|
85
86
|
|
86
87
|
# compute known args
|
87
|
-
env_args=Parameters.ts_to_env_args(transfer_spec,wss: @options[:wss])
|
88
|
+
env_args = Parameters.ts_to_env_args(transfer_spec,wss: @options[:wss])
|
88
89
|
|
89
90
|
# add fallback cert and key as arguments if needed
|
90
|
-
if [
|
91
|
+
if %w[1 force].include?(transfer_spec['http_fallback'])
|
91
92
|
env_args[:args].unshift('-Y',Installation.instance.path(:fallback_key))
|
92
93
|
env_args[:args].unshift('-I',Installation.instance.path(:fallback_cert))
|
93
94
|
end
|
@@ -95,19 +96,19 @@ module Aspera
|
|
95
96
|
env_args[:args].unshift('-q') if @options[:quiet]
|
96
97
|
|
97
98
|
# transfer job can be multi session
|
98
|
-
xfer_job={
|
99
|
-
id:
|
100
|
-
sessions:
|
99
|
+
xfer_job = {
|
100
|
+
id: job_options[:job_id],
|
101
|
+
sessions: [] # all sessions as below
|
101
102
|
}
|
102
103
|
|
103
104
|
# generic session information
|
104
|
-
session={
|
105
|
-
thread:
|
106
|
-
error:
|
107
|
-
io:
|
108
|
-
id:
|
109
|
-
env_args:
|
110
|
-
options:
|
105
|
+
session = {
|
106
|
+
thread: nil, # Thread object monitoring management port, not nil when pushed to :sessions
|
107
|
+
error: nil, # exception if failed
|
108
|
+
io: nil, # management port server socket
|
109
|
+
id: nil, # SessionId from INIT message in mgt port
|
110
|
+
env_args: env_args, # env vars and args to ascp (from transfer spec)
|
111
|
+
options: job_options # [Hash]
|
111
112
|
}
|
112
113
|
|
113
114
|
if multi_session_info.nil?
|
@@ -121,12 +122,12 @@ module Aspera
|
|
121
122
|
# do not delay the first session
|
122
123
|
sleep(@options[:spawn_delay_sec]) unless i.eql?(1)
|
123
124
|
# do deep copy (each thread has its own copy because it is modified here below and in thread)
|
124
|
-
this_session=session.clone
|
125
|
-
this_session[:env_args]=this_session[:env_args].clone
|
126
|
-
this_session[:env_args][:args]=this_session[:env_args][:args].clone
|
125
|
+
this_session = session.clone
|
126
|
+
this_session[:env_args] = this_session[:env_args].clone
|
127
|
+
this_session[:env_args][:args] = this_session[:env_args][:args].clone
|
127
128
|
this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_info[:count]}")
|
128
129
|
# option: increment (default as per ascp manual) or not (cluster on other side ?)
|
129
|
-
this_session[:env_args][:args].unshift('-O',(multi_session_info[:udp_base]+i-1).to_s) if @options[:multi_incr_udp]
|
130
|
+
this_session[:env_args][:args].unshift('-O',(multi_session_info[:udp_base] + i - 1).to_s) if @options[:multi_incr_udp]
|
130
131
|
this_session[:thread] = Thread.new(this_session) {|s|transfer_thread_entry(s)}
|
131
132
|
xfer_job[:sessions].push(this_session)
|
132
133
|
end
|
@@ -134,7 +135,7 @@ module Aspera
|
|
134
135
|
Log.log.debug('started session thread(s)')
|
135
136
|
|
136
137
|
# add job to list of jobs
|
137
|
-
@jobs[job_options[:job_id]]=xfer_job
|
138
|
+
@jobs[job_options[:job_id]] = xfer_job
|
138
139
|
Log.log.debug("jobs: #{@jobs.keys.count}")
|
139
140
|
|
140
141
|
return job_options[:job_id]
|
@@ -145,7 +146,7 @@ module Aspera
|
|
145
146
|
def wait_for_transfers_completion
|
146
147
|
Log.log.debug('wait_for_transfers_completion')
|
147
148
|
# set to non-nil to exit loop
|
148
|
-
result=[]
|
149
|
+
result = []
|
149
150
|
@jobs.each do |_id,job|
|
150
151
|
job[:sessions].each do |session|
|
151
152
|
Log.log.debug("join #{session[:thread]}")
|
@@ -176,11 +177,11 @@ module Aspera
|
|
176
177
|
raise 'env_args must be Hash' unless env_args.is_a?(Hash)
|
177
178
|
raise 'session must be Hash' unless session.is_a?(Hash)
|
178
179
|
# by default we assume an exception will be raised (for ensure block)
|
179
|
-
exception_raised=true
|
180
|
+
exception_raised = true
|
180
181
|
begin
|
181
182
|
Log.log.debug("env_args=#{env_args.inspect}")
|
182
183
|
# get location of ascp executable
|
183
|
-
ascp_path
|
184
|
+
ascp_path = @mutex.synchronize do
|
184
185
|
Fasp::Installation.instance.path(env_args[:ascp_version])
|
185
186
|
end
|
186
187
|
# (optional) check it exists
|
@@ -188,17 +189,24 @@ module Aspera
|
|
188
189
|
# open random local TCP port for listening for ascp management
|
189
190
|
mgt_sock = TCPServer.new('127.0.0.1',0)
|
190
191
|
# clone arguments as we eed to modify with mgt port
|
191
|
-
ascp_arguments=env_args[:args].clone
|
192
|
+
ascp_arguments = env_args[:args].clone
|
192
193
|
# add management port
|
193
194
|
ascp_arguments.unshift('-M', mgt_sock.addr[1].to_s)
|
194
195
|
# start ascp in sub process
|
195
|
-
Log.log.debug
|
196
|
+
Log.log.debug do
|
197
|
+
'execute: '+
|
198
|
+
env_args[:env].map{|k,v| "#{k}=#{Shellwords.shellescape(v)}"}.join(' ')+
|
199
|
+
' '+
|
200
|
+
Shellwords.shellescape(ascp_path)+
|
201
|
+
' '+
|
202
|
+
ascp_arguments.map{|a|Shellwords.shellescape(a)}.join(' ')
|
203
|
+
end
|
196
204
|
# start process
|
197
205
|
ascp_pid = Process.spawn(env_args[:env],[ascp_path,ascp_path],*ascp_arguments)
|
198
206
|
# in parent, wait for connection to socket max 3 seconds
|
199
207
|
Log.log.debug("before accept for pid (#{ascp_pid})")
|
200
208
|
# init management socket
|
201
|
-
ascp_mgt_io=nil
|
209
|
+
ascp_mgt_io = nil
|
202
210
|
Timeout.timeout(@options[:spawn_timeout_sec]) do
|
203
211
|
ascp_mgt_io = mgt_sock.accept
|
204
212
|
# management messages include file names which may be utf8
|
@@ -207,20 +215,20 @@ module Aspera
|
|
207
215
|
ascp_mgt_io.set_encoding(Encoding::UTF_8)
|
208
216
|
end
|
209
217
|
Log.log.debug("after accept (#{ascp_mgt_io})")
|
210
|
-
session[:io]=ascp_mgt_io
|
218
|
+
session[:io] = ascp_mgt_io
|
211
219
|
# exact text for event, with \n
|
212
|
-
current_event_text=''
|
220
|
+
current_event_text = ''
|
213
221
|
# parsed event (hash)
|
214
|
-
current_event_data=nil
|
222
|
+
current_event_data = nil
|
215
223
|
# this is the last full status
|
216
|
-
last_status_event=nil
|
224
|
+
last_status_event = nil
|
217
225
|
# read management port
|
218
226
|
loop do
|
219
227
|
# TODO: timeout here ?
|
220
228
|
line = ascp_mgt_io.gets
|
221
229
|
# nil when ascp process exits
|
222
230
|
break if line.nil?
|
223
|
-
current_event_text+=line
|
231
|
+
current_event_text += line
|
224
232
|
line.chomp!
|
225
233
|
Log.log.debug("line=[#{line}]")
|
226
234
|
case line
|
@@ -234,11 +242,11 @@ module Aspera
|
|
234
242
|
when ''
|
235
243
|
# empty line is separator to end event information
|
236
244
|
raise 'unexpected empty line' if current_event_data.nil?
|
237
|
-
current_event_data[AgentBase::LISTENER_SESSION_ID_B]=ascp_pid
|
245
|
+
current_event_data[AgentBase::LISTENER_SESSION_ID_B] = ascp_pid
|
238
246
|
notify_listeners(current_event_text,current_event_data)
|
239
247
|
case current_event_data['Type']
|
240
248
|
when 'INIT'
|
241
|
-
session[:id]=current_event_data['SessionId']
|
249
|
+
session[:id] = current_event_data['SessionId']
|
242
250
|
Log.log.debug("session id: #{session[:id]}")
|
243
251
|
when 'DONE','ERROR'
|
244
252
|
# TODO: check if this is always the last event
|
@@ -253,15 +261,15 @@ module Aspera
|
|
253
261
|
case last_status_event['Type']
|
254
262
|
when 'DONE'
|
255
263
|
# all went well
|
256
|
-
exception_raised=false
|
264
|
+
exception_raised = false
|
257
265
|
when 'ERROR'
|
258
266
|
Log.log.error("code: #{last_status_event['Code']}")
|
259
|
-
if last_status_event['Description']
|
267
|
+
if /bearer token/i.match?(last_status_event['Description'])
|
260
268
|
Log.log.error('need to regenerate token'.red)
|
261
269
|
if session[:options].is_a?(Hash) && session[:options].has_key?(:regenerate_token)
|
262
270
|
# regenerate token here, expired, or error on it
|
263
271
|
# Note: in multi-session, each session will have a different one.
|
264
|
-
env_args[:env]['ASPERA_SCP_TOKEN']=session[:options][:regenerate_token].call(true)
|
272
|
+
env_args[:env]['ASPERA_SCP_TOKEN'] = session[:options][:regenerate_token].call(true)
|
265
273
|
end
|
266
274
|
end
|
267
275
|
raise Fasp::Error.new(last_status_event['Description'],last_status_event['Code'].to_i)
|
@@ -269,7 +277,7 @@ module Aspera
|
|
269
277
|
raise "unexpected last event type: #{last_status_event['Type']}"
|
270
278
|
end
|
271
279
|
else
|
272
|
-
exception_raised=false
|
280
|
+
exception_raised = false
|
273
281
|
Log.log.debug('no status read from ascp mgt port')
|
274
282
|
end
|
275
283
|
rescue SystemCallError => e
|
@@ -284,11 +292,11 @@ module Aspera
|
|
284
292
|
unless ascp_pid.nil?
|
285
293
|
# "wait" for process to avoid zombie
|
286
294
|
Process.wait(ascp_pid)
|
287
|
-
status
|
288
|
-
ascp_pid=nil
|
295
|
+
status = $CHILD_STATUS
|
296
|
+
ascp_pid = nil
|
289
297
|
session.delete(:io)
|
290
298
|
if !status.success?
|
291
|
-
message="ascp failed with code #{status.exitstatus}"
|
299
|
+
message = "ascp failed with code #{status.exitstatus}"
|
292
300
|
# raise error only if there was not already an exception
|
293
301
|
raise Fasp::Error, message unless exception_raised
|
294
302
|
# else just debug, as main exception is already here
|
@@ -305,18 +313,18 @@ module Aspera
|
|
305
313
|
# {'type'=>'START','source'=>_path_,'destination'=>_path_}
|
306
314
|
# {'type'=>'DONE'}
|
307
315
|
def send_command(job_id,session_index,data)
|
308
|
-
job
|
316
|
+
job = @jobs[job_id]
|
309
317
|
raise 'no such job' if job.nil?
|
310
|
-
session=job[:sessions][session_index]
|
318
|
+
session = job[:sessions][session_index]
|
311
319
|
raise 'no such session' if session.nil?
|
312
320
|
Log.log.debug("command: #{data}")
|
313
321
|
# build command
|
314
|
-
command=data.
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
322
|
+
command = data.
|
323
|
+
keys.
|
324
|
+
map{|k|"#{k.capitalize}: #{data[k]}"}.
|
325
|
+
unshift('FASPMGR 2').
|
326
|
+
push('','').
|
327
|
+
join("\n")
|
320
328
|
session[:io].puts(command)
|
321
329
|
end
|
322
330
|
|
@@ -326,20 +334,20 @@ module Aspera
|
|
326
334
|
def initialize(options=nil)
|
327
335
|
super()
|
328
336
|
# all transfer jobs, key = SecureRandom.uuid, protected by mutex, condvar on change
|
329
|
-
@jobs={}
|
337
|
+
@jobs = {}
|
330
338
|
# mutex protects global data accessed by threads
|
331
|
-
@mutex=Mutex.new
|
339
|
+
@mutex = Mutex.new
|
332
340
|
# set default options and override if specified
|
333
|
-
@options=DEFAULT_OPTIONS.
|
341
|
+
@options = DEFAULT_OPTIONS.dup
|
334
342
|
if !options.nil?
|
335
343
|
raise "expecting Hash (or nil), but have #{options.class}" unless options.is_a?(Hash)
|
336
344
|
options.each do |k,v|
|
337
345
|
raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.has_key?(k)
|
338
|
-
@options[k]=v
|
346
|
+
@options[k] = v
|
339
347
|
end
|
340
348
|
end
|
341
349
|
Log.log.debug("local options= #{options}")
|
342
|
-
@resume_policy=ResumePolicy.new(@options[:resume].symbolize_keys)
|
350
|
+
@resume_policy = ResumePolicy.new(@options[:resume].symbolize_keys)
|
343
351
|
end
|
344
352
|
|
345
353
|
# transfer thread entry
|
@@ -347,15 +355,15 @@ module Aspera
|
|
347
355
|
def transfer_thread_entry(session)
|
348
356
|
begin
|
349
357
|
# set name for logging
|
350
|
-
Thread.current[:name]='transfer'
|
358
|
+
Thread.current[:name] = 'transfer'
|
351
359
|
Log.log.debug("ENTER (#{Thread.current[:name]})")
|
352
360
|
# start transfer with selected resumer policy
|
353
|
-
session[:options][:resumer].
|
361
|
+
session[:options][:resumer].execute_with_resume do
|
354
362
|
start_transfer_with_args_env(session[:env_args],session)
|
355
363
|
end
|
356
364
|
Log.log.debug('transfer ok'.bg_green)
|
357
365
|
rescue StandardError => e
|
358
|
-
session[:error]=e
|
366
|
+
session[:error] = e
|
359
367
|
Log.log.error("Transfer thread error: #{e.class}:\n#{e.message}:\n#{e.backtrace.join("\n")}".red) if Log.instance.level.eql?(:debug)
|
360
368
|
end
|
361
369
|
Log.log.debug("EXIT (#{Thread.current[:name]})")
|