aspera-cli 4.21.2 → 4.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +1 -1
- data/CHANGELOG.md +34 -16
- data/CONTRIBUTING.md +6 -10
- data/README.md +805 -574
- data/examples/get_proto_file.rb +1 -1
- data/lib/aspera/agent/base.rb +9 -5
- data/lib/aspera/agent/connect.rb +30 -28
- data/lib/aspera/agent/desktop.rb +29 -25
- data/lib/aspera/agent/direct.rb +137 -125
- data/lib/aspera/agent/httpgw.rb +22 -26
- data/lib/aspera/agent/node.rb +14 -11
- data/lib/aspera/agent/transferd.rb +6 -2
- data/lib/aspera/api/aoc.rb +6 -6
- data/lib/aspera/api/cos_node.rb +1 -1
- data/lib/aspera/api/httpgw.rb +7 -3
- data/lib/aspera/api/node.rb +6 -4
- data/lib/aspera/ascmd.rb +3 -3
- data/lib/aspera/ascp/installation.rb +15 -16
- data/lib/aspera/ascp/management.rb +1 -1
- data/lib/aspera/assert.rb +11 -2
- data/lib/aspera/cli/error.rb +2 -2
- data/lib/aspera/cli/extended_value.rb +38 -19
- data/lib/aspera/cli/formatter.rb +48 -48
- data/lib/aspera/cli/hints.rb +1 -1
- data/lib/aspera/cli/main.rb +190 -168
- data/lib/aspera/cli/manager.rb +15 -15
- data/lib/aspera/cli/plugin.rb +23 -20
- data/lib/aspera/cli/plugin_factory.rb +1 -1
- data/lib/aspera/cli/plugins/alee.rb +1 -1
- data/lib/aspera/cli/plugins/aoc.rb +144 -107
- data/lib/aspera/cli/plugins/ats.rb +19 -17
- data/lib/aspera/cli/plugins/config.rb +67 -83
- data/lib/aspera/cli/plugins/console.rb +5 -3
- data/lib/aspera/cli/plugins/faspex.rb +39 -35
- data/lib/aspera/cli/plugins/faspex5.rb +104 -80
- data/lib/aspera/cli/plugins/faspio.rb +13 -1
- data/lib/aspera/cli/plugins/httpgw.rb +13 -1
- data/lib/aspera/cli/plugins/node.rb +306 -179
- data/lib/aspera/cli/plugins/orchestrator.rb +34 -40
- data/lib/aspera/cli/plugins/preview.rb +3 -3
- data/lib/aspera/cli/plugins/server.rb +6 -6
- data/lib/aspera/cli/plugins/shares.rb +5 -5
- data/lib/aspera/cli/sync_actions.rb +19 -18
- data/lib/aspera/cli/transfer_agent.rb +5 -5
- data/lib/aspera/cli/transfer_progress.rb +2 -2
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +116 -95
- data/lib/aspera/coverage.rb +4 -3
- data/lib/aspera/environment.rb +6 -6
- data/lib/aspera/faspex_gw.rb +14 -14
- data/lib/aspera/faspex_postproc.rb +7 -6
- data/lib/aspera/hash_ext.rb +2 -2
- data/lib/aspera/json_rpc.rb +1 -1
- data/lib/aspera/keychain/encrypted_hash.rb +47 -34
- data/lib/aspera/keychain/factory.rb +41 -0
- data/lib/aspera/keychain/hashicorp_vault.rb +71 -0
- data/lib/aspera/keychain/macos_security.rb +19 -11
- data/lib/aspera/log.rb +28 -34
- data/lib/aspera/nagios.rb +6 -6
- data/lib/aspera/node_simulator.rb +8 -8
- data/lib/aspera/oauth/base.rb +8 -6
- data/lib/aspera/oauth/factory.rb +5 -6
- data/lib/aspera/oauth/url_json.rb +6 -6
- data/lib/aspera/persistency_action_once.rb +6 -4
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/generator.rb +1 -1
- data/lib/aspera/preview/options.rb +16 -16
- data/lib/aspera/preview/terminal.rb +3 -3
- data/lib/aspera/preview/utils.rb +11 -13
- data/lib/aspera/products/connect.rb +1 -1
- data/lib/aspera/products/desktop.rb +1 -1
- data/lib/aspera/products/transferd.rb +1 -1
- data/lib/aspera/proxy_auto_config.rb +2 -2
- data/lib/aspera/rest.rb +52 -43
- data/lib/aspera/rest_errors_aspera.rb +1 -1
- data/lib/aspera/secret_hider.rb +5 -5
- data/lib/aspera/ssh.rb +4 -4
- data/lib/aspera/transfer/convert.rb +29 -0
- data/lib/aspera/transfer/error_info.rb +66 -66
- data/lib/aspera/transfer/parameters.rb +13 -68
- data/lib/aspera/transfer/spec.rb +5 -6
- data/lib/aspera/transfer/spec.schema.yaml +753 -0
- data/lib/aspera/transfer/spec_doc.rb +62 -0
- data/lib/aspera/transfer/sync.rb +23 -72
- data/lib/aspera/transfer/sync_instance.schema.yaml +13 -0
- data/lib/aspera/transfer/sync_session.schema.yaml +79 -0
- data/lib/aspera/transfer/uri.rb +6 -6
- data/lib/aspera/uri_reader.rb +1 -1
- data/lib/aspera/web_auth.rb +1 -1
- data/lib/aspera/web_server_simple.rb +53 -44
- data.tar.gz.sig +1 -2
- metadata +37 -4
- metadata.gz.sig +0 -0
- data/examples/build_package.sh +0 -28
- data/lib/aspera/transfer/spec.yaml +0 -718
data/examples/get_proto_file.rb
CHANGED
@@ -5,4 +5,4 @@ require 'aspera/ascp/installation'
|
|
5
5
|
require 'aspera/cli/transfer_progress'
|
6
6
|
Aspera::RestParameters.instance.progress_bar = Aspera::Cli::TransferProgress.new
|
7
7
|
# Retrieve `transfer.proto` from the web
|
8
|
-
Aspera::Ascp::Installation.instance.install_sdk(folder: ARGV.first, backup: false, with_exe: false)
|
8
|
+
Aspera::Ascp::Installation.instance.install_sdk(folder: ARGV.first, backup: false, with_exe: false){ |name| name.end_with?('.proto') ? '/' : nil}
|
data/lib/aspera/agent/base.rb
CHANGED
@@ -21,7 +21,7 @@ module Aspera
|
|
21
21
|
base_class = File.basename(__FILE__)
|
22
22
|
Dir.entries(File.dirname(File.expand_path(__FILE__))).select do |file|
|
23
23
|
file.end_with?(RUBY_EXT) && !file.eql?(base_class)
|
24
|
-
end.map{|file|file[0..(-1 - RUBY_EXT.length)].to_sym}
|
24
|
+
end.map{ |file| file[0..(-1 - RUBY_EXT.length)].to_sym}
|
25
25
|
end
|
26
26
|
end
|
27
27
|
|
@@ -31,16 +31,20 @@ module Aspera
|
|
31
31
|
statuses = wait_for_transfers_completion
|
32
32
|
@progress&.reset
|
33
33
|
Aspera.assert_type(statuses, Array)
|
34
|
-
Aspera.assert(statuses.none?{|i
|
34
|
+
Aspera.assert(statuses.none?{ |i| !i.eql?(:success) && !i.is_a?(StandardError)}){"bad statuses content: #{statuses}"}
|
35
35
|
return statuses
|
36
36
|
end
|
37
37
|
|
38
38
|
private
|
39
39
|
|
40
|
+
Aspera.require_method!(:start_transfer)
|
41
|
+
Aspera.require_method!(:wait_for_transfers_completion)
|
42
|
+
# method `shutdown` is optional
|
43
|
+
def shutdown
|
44
|
+
nil
|
45
|
+
end
|
46
|
+
|
40
47
|
def initialize(progress: nil)
|
41
|
-
# method `shutdown` is optional
|
42
|
-
Aspera.assert(respond_to?(:start_transfer))
|
43
|
-
Aspera.assert(respond_to?(:wait_for_transfers_completion))
|
44
48
|
@progress = progress
|
45
49
|
end
|
46
50
|
|
data/lib/aspera/agent/connect.rb
CHANGED
@@ -17,6 +17,7 @@ module Aspera
|
|
17
17
|
private_constant :CONNECT_START_URIS, :SLEEP_SEC_BETWEEN_RETRY
|
18
18
|
def initialize(**base_options)
|
19
19
|
super
|
20
|
+
@transfer_id = nil
|
20
21
|
@connect_settings = {
|
21
22
|
'app_id' => SecureRandom.uuid
|
22
23
|
}
|
@@ -46,21 +47,7 @@ module Aspera
|
|
46
47
|
end
|
47
48
|
end
|
48
49
|
|
49
|
-
#
|
50
|
-
def connect_api_url
|
51
|
-
connect_locations = Products::Other.find(Products::Connect.locations).first
|
52
|
-
raise "Product: #{name} not found, please install." if connect_locations.nil?
|
53
|
-
folder = File.join(connect_locations[:run_root], 'var', 'run')
|
54
|
-
['', 's'].each do |ext|
|
55
|
-
uri_file = File.join(folder, "http#{ext}.uri")
|
56
|
-
Log.log.debug{"checking connect port file: #{uri_file}"}
|
57
|
-
if File.exist?(uri_file)
|
58
|
-
return File.open(uri_file, &:gets).strip
|
59
|
-
end
|
60
|
-
end
|
61
|
-
raise "no connect uri file found in #{folder}"
|
62
|
-
end
|
63
|
-
|
50
|
+
# :reek:UnusedParameters token_regenerator
|
64
51
|
def start_transfer(transfer_spec, token_regenerator: nil)
|
65
52
|
if transfer_spec['direction'] == 'send'
|
66
53
|
Log.log.warn{"Connect requires upload selection using GUI, ignoring #{transfer_spec['paths']}".red}
|
@@ -71,15 +58,14 @@ module Aspera
|
|
71
58
|
'suggestedName' => '',
|
72
59
|
'allowMultipleSelection' => true,
|
73
60
|
'allowedFileTypes' => ''})
|
74
|
-
transfer_spec['paths'] = selection['dataTransfer']['files'].map
|
61
|
+
transfer_spec['paths'] = selection['dataTransfer']['files'].map{ |i| {'source' => i['name']}}
|
75
62
|
end
|
76
|
-
@request_id = SecureRandom.uuid
|
77
63
|
# if there is a token, we ask connect client to use well known ssh private keys
|
78
64
|
# instead of asking password
|
79
65
|
transfer_spec['authentication'] = 'token' if transfer_spec.key?('token')
|
80
66
|
connect_transfer_args = {
|
81
67
|
'aspera_connect_settings' => @connect_settings.merge({
|
82
|
-
'request_id' =>
|
68
|
+
'request_id' => SecureRandom.uuid,
|
83
69
|
'allow_dialogs' => true
|
84
70
|
}),
|
85
71
|
'transfer_specs' => [{
|
@@ -87,17 +73,16 @@ module Aspera
|
|
87
73
|
}]}
|
88
74
|
# asynchronous anyway
|
89
75
|
res = @connect_api.create('transfers/start', connect_transfer_args)
|
90
|
-
@
|
76
|
+
@transfer_id = res['transfer_specs'].first['transfer_spec']['tags'][Transfer::Spec::TAG_RESERVED]['xfer_id']
|
91
77
|
end
|
92
78
|
|
93
79
|
def wait_for_transfers_completion
|
94
80
|
connect_activity_args = {'aspera_connect_settings' => @connect_settings}
|
95
81
|
started = false
|
96
82
|
pre_calc = false
|
97
|
-
session_id = @xfer_id
|
98
83
|
begin
|
99
84
|
loop do
|
100
|
-
tr_info = @connect_api.create("transfers/info/#{@
|
85
|
+
tr_info = @connect_api.create("transfers/info/#{@transfer_id}", connect_activity_args)
|
101
86
|
Log.log.trace1{Log.dump(:tr_info, tr_info)}
|
102
87
|
if tr_info['transfer_info'].is_a?(Hash)
|
103
88
|
transfer = tr_info['transfer_info']
|
@@ -111,26 +96,26 @@ module Aspera
|
|
111
96
|
notify_progress(:pre_start, session_id: nil, info: transfer['status'])
|
112
97
|
when 'running'
|
113
98
|
if !started
|
114
|
-
notify_progress(:session_start, session_id:
|
99
|
+
notify_progress(:session_start, session_id: @transfer_id)
|
115
100
|
started = true
|
116
101
|
end
|
117
102
|
if !pre_calc && (transfer['bytes_expected'] != 0)
|
118
|
-
notify_progress(:session_size, session_id:
|
103
|
+
notify_progress(:session_size, session_id: @transfer_id, info: transfer['bytes_expected'])
|
119
104
|
pre_calc = true
|
120
105
|
else
|
121
|
-
notify_progress(:transfer, session_id:
|
106
|
+
notify_progress(:transfer, session_id: @transfer_id, info: transfer['bytes_written'])
|
122
107
|
end
|
123
108
|
when 'completed'
|
124
|
-
notify_progress(:end, session_id:
|
109
|
+
notify_progress(:end, session_id: @transfer_id)
|
125
110
|
break
|
126
111
|
when 'failed'
|
127
|
-
notify_progress(:end, session_id:
|
112
|
+
notify_progress(:end, session_id: @transfer_id)
|
128
113
|
raise Transfer::Error, transfer['error_desc']
|
129
114
|
when 'cancelled'
|
130
|
-
notify_progress(:end, session_id:
|
115
|
+
notify_progress(:end, session_id: @transfer_id)
|
131
116
|
raise Transfer::Error, 'Transfer cancelled by user'
|
132
117
|
else
|
133
|
-
notify_progress(:end, session_id:
|
118
|
+
notify_progress(:end, session_id: @transfer_id)
|
134
119
|
raise Transfer::Error, "unknown status: #{transfer['status']}: #{transfer['error_desc']}"
|
135
120
|
end
|
136
121
|
end
|
@@ -141,6 +126,23 @@ module Aspera
|
|
141
126
|
end
|
142
127
|
return [:success]
|
143
128
|
end
|
129
|
+
|
130
|
+
private
|
131
|
+
|
132
|
+
# @return the file path of local connect where API's URI can be read
|
133
|
+
def connect_api_url
|
134
|
+
connect_locations = Products::Other.find(Products::Connect.locations).first
|
135
|
+
raise "Product: #{name} not found, please install." if connect_locations.nil?
|
136
|
+
folder = File.join(connect_locations[:run_root], 'var', 'run')
|
137
|
+
['', 's'].each do |ext|
|
138
|
+
uri_file = File.join(folder, "http#{ext}.uri")
|
139
|
+
Log.log.debug{"checking connect port file: #{uri_file}"}
|
140
|
+
if File.exist?(uri_file)
|
141
|
+
return File.open(uri_file, &:gets).strip
|
142
|
+
end
|
143
|
+
end
|
144
|
+
raise "no connect uri file found in #{folder}"
|
145
|
+
end
|
144
146
|
end
|
145
147
|
end
|
146
148
|
end
|
data/lib/aspera/agent/desktop.rb
CHANGED
@@ -19,7 +19,7 @@ module Aspera
|
|
19
19
|
|
20
20
|
def initialize(**base_options)
|
21
21
|
@application_id = SecureRandom.uuid
|
22
|
-
@
|
22
|
+
@transfer_id = nil
|
23
23
|
super
|
24
24
|
raise 'Using client requires a graphical environment' if !Environment.default_gui_mode.eql?(:graphical)
|
25
25
|
method_index = 0
|
@@ -44,28 +44,14 @@ module Aspera
|
|
44
44
|
end
|
45
45
|
end
|
46
46
|
|
47
|
-
|
48
|
-
log_file = Products::Desktop.log_file
|
49
|
-
url = 'http://127.0.0.1:33024'
|
50
|
-
File.open(log_file, 'r') do |file|
|
51
|
-
file.each_line do |line|
|
52
|
-
line = line.chomp
|
53
|
-
if (m = line.match(/JSON-RPC server listening on (.*)/))
|
54
|
-
url = "http://#{m[1]}"
|
55
|
-
end
|
56
|
-
end
|
57
|
-
end
|
58
|
-
# raise StandardError, "Unable to find the JSON-RPC server URL in #{log_file}" if url.nil?
|
59
|
-
return url
|
60
|
-
end
|
61
|
-
|
47
|
+
# :reek:UnusedParameters token_regenerator
|
62
48
|
def start_transfer(transfer_spec, token_regenerator: nil)
|
63
49
|
@request_id = SecureRandom.uuid
|
64
50
|
# if there is a token, we ask the client app to use well known ssh private keys
|
65
51
|
# instead of asking password
|
66
52
|
transfer_spec['authentication'] = 'token' if transfer_spec.key?('token')
|
67
53
|
result = @client_app_api.start_transfer(app_id: @application_id, desktop_spec: {}, transfer_spec: transfer_spec)
|
68
|
-
@
|
54
|
+
@transfer_id = result['uuid']
|
69
55
|
end
|
70
56
|
|
71
57
|
def wait_for_transfers_completion
|
@@ -73,32 +59,32 @@ module Aspera
|
|
73
59
|
pre_calc = false
|
74
60
|
begin
|
75
61
|
loop do
|
76
|
-
transfer = @client_app_api.get_transfer(app_id: @application_id, transfer_id: @
|
62
|
+
transfer = @client_app_api.get_transfer(app_id: @application_id, transfer_id: @transfer_id)
|
77
63
|
case transfer['status']
|
78
64
|
when 'initiating', 'queued'
|
79
65
|
notify_progress(:pre_start, session_id: nil, info: transfer['status'])
|
80
66
|
when 'running'
|
81
67
|
if !started
|
82
|
-
notify_progress(:session_start, session_id: @
|
68
|
+
notify_progress(:session_start, session_id: @transfer_id)
|
83
69
|
started = true
|
84
70
|
end
|
85
71
|
if !pre_calc && (transfer['bytes_expected'] != 0)
|
86
|
-
notify_progress(:session_size, session_id: @
|
72
|
+
notify_progress(:session_size, session_id: @transfer_id, info: transfer['bytes_expected'])
|
87
73
|
pre_calc = true
|
88
74
|
else
|
89
|
-
notify_progress(:transfer, session_id: @
|
75
|
+
notify_progress(:transfer, session_id: @transfer_id, info: transfer['bytes_written'])
|
90
76
|
end
|
91
77
|
when 'completed'
|
92
|
-
notify_progress(:end, session_id: @
|
78
|
+
notify_progress(:end, session_id: @transfer_id)
|
93
79
|
break
|
94
80
|
when 'failed'
|
95
|
-
notify_progress(:end, session_id: @
|
81
|
+
notify_progress(:end, session_id: @transfer_id)
|
96
82
|
raise Transfer::Error, transfer['error_desc']
|
97
83
|
when 'cancelled'
|
98
|
-
notify_progress(:end, session_id: @
|
84
|
+
notify_progress(:end, session_id: @transfer_id)
|
99
85
|
raise Transfer::Error, 'Transfer cancelled by user'
|
100
86
|
else
|
101
|
-
notify_progress(:end, session_id: @
|
87
|
+
notify_progress(:end, session_id: @transfer_id)
|
102
88
|
raise Transfer::Error, "unknown status: #{transfer['status']}: #{transfer['error_desc']}"
|
103
89
|
end
|
104
90
|
sleep(1)
|
@@ -108,6 +94,24 @@ module Aspera
|
|
108
94
|
end
|
109
95
|
return [:success]
|
110
96
|
end
|
97
|
+
|
98
|
+
private
|
99
|
+
|
100
|
+
# @return [String] the url where transferd is listening
|
101
|
+
def aspera_client_api_url
|
102
|
+
log_file = Products::Desktop.log_file
|
103
|
+
url = 'http://127.0.0.1:33024'
|
104
|
+
File.open(log_file, 'r') do |file|
|
105
|
+
file.each_line do |line|
|
106
|
+
line = line.chomp
|
107
|
+
if (m = line.match(/JSON-RPC server listening on (.*)/))
|
108
|
+
url = "http://#{m[1]}"
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
# raise StandardError, "Unable to find the JSON-RPC server URL in #{log_file}" if url.nil?
|
113
|
+
return url
|
114
|
+
end
|
111
115
|
end
|
112
116
|
end
|
113
117
|
end
|
data/lib/aspera/agent/direct.rb
CHANGED
@@ -16,15 +16,68 @@ require 'English'
|
|
16
16
|
|
17
17
|
module Aspera
|
18
18
|
module Agent
|
19
|
-
# executes a local "ascp",
|
19
|
+
# executes a local "ascp", create mgt port
|
20
20
|
class Direct < Base
|
21
|
+
# ascp started locally, so listen local
|
21
22
|
LISTEN_LOCAL_ADDRESS = '127.0.0.1'
|
22
|
-
# 0 means:
|
23
|
+
# 0 means: use any available port
|
23
24
|
SELECT_AVAILABLE_PORT = 0
|
24
|
-
# spellchecker: enable
|
25
25
|
private_constant :LISTEN_LOCAL_ADDRESS, :SELECT_AVAILABLE_PORT
|
26
26
|
|
27
|
-
#
|
27
|
+
# options for initialize (same as values in option transfer_info)
|
28
|
+
# @param ascp_args [Array] additional arguments to ascp
|
29
|
+
# @param wss [Boolean] true: if both SSH and wss in ts: prefer wss
|
30
|
+
# @param quiet [Boolean] by default no native ascp progress bar
|
31
|
+
# @param monitor [Boolean] set to false to eliminate management port
|
32
|
+
# @param trusted_certs [Array,NilClass] list of files with trusted certificates (stores)
|
33
|
+
# @param client_ssh_key [String] client ssh key option (from CLIENT_SSH_KEY_OPTIONS)
|
34
|
+
# @param check_ignore_cb [Proc] callback with host,port
|
35
|
+
# @param spawn_timeout_sec [Integer] timeout for ascp spawn
|
36
|
+
# @param spawn_delay_sec [Integer] optional delay to start between sessions
|
37
|
+
# @param multi_incr_udp [Boolean,NilClass] true: increment udp port for each session
|
38
|
+
# @param resume [Hash,NilClass] resume policy
|
39
|
+
# @param management_cb [Proc] callback for management events
|
40
|
+
# @param base_options [Hash] other options for base class
|
41
|
+
def initialize(
|
42
|
+
ascp_args: nil,
|
43
|
+
wss: true,
|
44
|
+
quiet: true,
|
45
|
+
trusted_certs: nil,
|
46
|
+
client_ssh_key: nil,
|
47
|
+
check_ignore_cb: nil,
|
48
|
+
spawn_timeout_sec: 2,
|
49
|
+
spawn_delay_sec: 2,
|
50
|
+
multi_incr_udp: nil,
|
51
|
+
resume: nil,
|
52
|
+
monitor: true,
|
53
|
+
management_cb: nil,
|
54
|
+
**base_options
|
55
|
+
)
|
56
|
+
super(**base_options)
|
57
|
+
# special transfer parameters
|
58
|
+
@tr_opts = {
|
59
|
+
ascp_args: ascp_args,
|
60
|
+
wss: wss,
|
61
|
+
quiet: quiet,
|
62
|
+
trusted_certs: trusted_certs,
|
63
|
+
client_ssh_key: client_ssh_key,
|
64
|
+
check_ignore_cb: check_ignore_cb
|
65
|
+
}
|
66
|
+
@spawn_timeout_sec = spawn_timeout_sec
|
67
|
+
@spawn_delay_sec = spawn_delay_sec
|
68
|
+
# default is true on Windows, false on other OSes
|
69
|
+
@multi_incr_udp = multi_incr_udp.nil? ? Environment.os.eql?(Environment::OS_WINDOWS) : multi_incr_udp
|
70
|
+
@monitor = monitor
|
71
|
+
@management_cb = management_cb
|
72
|
+
@resume_policy = Resumer.new(resume.nil? ? {} : resume.symbolize_keys)
|
73
|
+
# all transfer jobs, key = SecureRandom.uuid, protected by mutex, cond var on change
|
74
|
+
@sessions = []
|
75
|
+
# mutex protects global data accessed by threads
|
76
|
+
@mutex = Mutex.new
|
77
|
+
@pre_calc_sent = false
|
78
|
+
@pre_calc_last_size = nil
|
79
|
+
end
|
80
|
+
|
28
81
|
# start ascp transfer(s) (non blocking), single or multi-session
|
29
82
|
# session information added to @sessions
|
30
83
|
# @param transfer_spec [Hash] aspera transfer specification
|
@@ -48,11 +101,10 @@ module Aspera
|
|
48
101
|
# (even if the var is not used in single session)
|
49
102
|
multi_session_info = nil
|
50
103
|
if transfer_spec.key?('multi_session')
|
104
|
+
# Managed by multi-session, so delete from transfer spec
|
51
105
|
multi_session_info = {
|
52
|
-
count: transfer_spec
|
106
|
+
count: transfer_spec.delete('multi_session').to_i
|
53
107
|
}
|
54
|
-
# Managed by multi-session, so delete from transfer spec
|
55
|
-
transfer_spec.delete('multi_session')
|
56
108
|
if multi_session_info[:count].negative?
|
57
109
|
Log.log.error{"multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0"}
|
58
110
|
multi_session_info = nil
|
@@ -77,14 +129,14 @@ module Aspera
|
|
77
129
|
error: nil, # exception if failed
|
78
130
|
io: nil, # management port server socket
|
79
131
|
token_regenerator: token_regenerator, # regenerate bearer token with oauth
|
80
|
-
# env vars and args
|
132
|
+
# env vars and args for ascp (from transfer spec)
|
81
133
|
env_args: Transfer::Parameters.new(transfer_spec, **@tr_opts).ascp_args
|
82
134
|
}
|
83
135
|
|
84
136
|
if multi_session_info.nil?
|
85
137
|
Log.log.debug('Starting single session thread')
|
86
138
|
# single session for transfer : simple
|
87
|
-
session[:thread] = Thread.new
|
139
|
+
session[:thread] = Thread.new{transfer_thread_entry(session)}
|
88
140
|
@sessions.push(session)
|
89
141
|
else
|
90
142
|
Log.log.debug('Starting multi session threads')
|
@@ -101,7 +153,7 @@ module Aspera
|
|
101
153
|
# option: increment (default as per ascp manual) or not (cluster on other side ?)
|
102
154
|
args.unshift('-O', (multi_session_info[:udp_base] + i - 1).to_s) if @multi_incr_udp
|
103
155
|
# finally start the thread
|
104
|
-
this_session[:thread] = Thread.new
|
156
|
+
this_session[:thread] = Thread.new{transfer_thread_entry(this_session)}
|
105
157
|
@sessions.push(this_session)
|
106
158
|
end
|
107
159
|
end
|
@@ -132,9 +184,44 @@ module Aspera
|
|
132
184
|
|
133
185
|
# @return [Array] list of sessions for a job
|
134
186
|
def sessions_by_job(job_id)
|
135
|
-
@sessions.select{|session| session[:job_id].eql?(job_id)}
|
187
|
+
@sessions.select{ |session| session[:job_id].eql?(job_id)}
|
136
188
|
end
|
137
189
|
|
190
|
+
# send command to management port of command (used in `asession)
|
191
|
+
# @param job_id identified transfer process
|
192
|
+
# @param session_index index of session (for multi session)
|
193
|
+
# @param data command on mgt port, examples:
|
194
|
+
# {'type'=>'START','source'=>_path_,'destination'=>_path_}
|
195
|
+
# {'type'=>'DONE'}
|
196
|
+
def send_command(job_id, data)
|
197
|
+
session = @sessions.find{ |session| session[:job_id].eql?(job_id)}
|
198
|
+
Log.log.debug{"command: #{data}"}
|
199
|
+
session[:io].puts(Ascp::Management.command_to_stream(data))
|
200
|
+
end
|
201
|
+
|
202
|
+
private
|
203
|
+
|
204
|
+
# transfer thread entry
|
205
|
+
# @param session information
|
206
|
+
def transfer_thread_entry(session)
|
207
|
+
begin
|
208
|
+
# set name for logging
|
209
|
+
Thread.current[:name] = 'transfer'
|
210
|
+
Log.log.debug{"ENTER (#{Thread.current[:name]})"}
|
211
|
+
# start transfer with selected resumer policy
|
212
|
+
@resume_policy.execute_with_resume do
|
213
|
+
start_and_monitor_process(session: session, **session[:env_args])
|
214
|
+
end
|
215
|
+
Log.log.debug('transfer ok'.bg_green)
|
216
|
+
rescue StandardError => e
|
217
|
+
session[:error] = e
|
218
|
+
Log.log.error{"Transfer thread error: #{e.class}:\n#{e.message}:\n#{e.backtrace.join("\n")}".red} if Log.instance.level.eql?(:debug)
|
219
|
+
end
|
220
|
+
Log.log.debug{"EXIT (#{Thread.current[:name]})"}
|
221
|
+
end
|
222
|
+
|
223
|
+
public
|
224
|
+
|
138
225
|
# This is the low level method to start the transfer process.
|
139
226
|
# Typically started in a thread.
|
140
227
|
# Start process with management port.
|
@@ -154,18 +241,21 @@ module Aspera
|
|
154
241
|
notify_progress(:pre_start, session_id: nil, info: 'starting')
|
155
242
|
begin
|
156
243
|
command_pid = nil
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
244
|
+
command_arguments = []
|
245
|
+
if @monitor
|
246
|
+
# we use Socket directly, instead of TCPServer, as it gives access to lower level options
|
247
|
+
socket_class = defined?(JRUBY_VERSION) ? ServerSocket : Socket
|
248
|
+
mgt_server_socket = socket_class.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
|
249
|
+
# open any available (0) local TCP port for use as management port
|
250
|
+
mgt_server_socket.bind(Addrinfo.tcp(LISTEN_LOCAL_ADDRESS, SELECT_AVAILABLE_PORT))
|
251
|
+
# make port ready to accept connections, before starting ascp
|
252
|
+
mgt_server_socket.listen(1)
|
253
|
+
# build arguments and add mgt port
|
254
|
+
command_arguments = if name.eql?(:async)
|
255
|
+
["--exclusive-mgmt-port=#{mgt_server_socket.local_address.ip_port}"]
|
256
|
+
else
|
257
|
+
['-M', mgt_server_socket.local_address.ip_port.to_s]
|
258
|
+
end
|
169
259
|
end
|
170
260
|
command_arguments.concat(args)
|
171
261
|
# capture process stderr
|
@@ -175,6 +265,8 @@ module Aspera
|
|
175
265
|
command_pid = Environment.secure_spawn(env: env, exec: command_path, args: command_arguments, err: stderr_w)
|
176
266
|
stderr_w.close
|
177
267
|
notify_progress(:pre_start, session_id: nil, info: "waiting for #{name} to start")
|
268
|
+
# "ensure" block will wait for process
|
269
|
+
return unless @monitor
|
178
270
|
# TODO: timeout does not work when Process.spawn is used... until process exits, then it works
|
179
271
|
# So we use select to detect that anything happens on the socket (connection)
|
180
272
|
Log.log.debug{"before select, timeout: #{@spawn_timeout_sec}"}
|
@@ -185,9 +277,9 @@ module Aspera
|
|
185
277
|
client_socket, _client_addrinfo = mgt_server_socket.accept
|
186
278
|
Log.log.debug('after accept')
|
187
279
|
management_port_io = client_socket.to_io
|
188
|
-
# management messages include file names which may be utf8
|
189
280
|
# by default socket is US-ASCII
|
190
|
-
#
|
281
|
+
# management messages include file names which may be UTF-8
|
282
|
+
# TODO: use same value as Encoding.default_external ?
|
191
283
|
management_port_io.set_encoding(Encoding::UTF_8)
|
192
284
|
session[:io] = management_port_io
|
193
285
|
processor = Ascp::Management.new
|
@@ -197,7 +289,7 @@ module Aspera
|
|
197
289
|
next unless event
|
198
290
|
# event is ready
|
199
291
|
Log.log.trace1{Log.dump(:management_port, event)}
|
200
|
-
# store
|
292
|
+
# store session identifier
|
201
293
|
session[:id] = event['SessionId'] if event['Type'].eql?('INIT')
|
202
294
|
@management_cb&.call(event)
|
203
295
|
process_progress(event)
|
@@ -206,12 +298,10 @@ module Aspera
|
|
206
298
|
Log.log.debug('management io closed')
|
207
299
|
# check that last status was received before process exit
|
208
300
|
last_event = processor.last_event
|
209
|
-
|
210
|
-
stderr_r&.each_line do |line|
|
211
|
-
Log.log.error(line.chomp)
|
212
|
-
end
|
213
|
-
raise Transfer::Error, "internal: no management event (#{last_event.class})" unless last_event.is_a?(Hash)
|
301
|
+
raise Transfer::Error, "No management event (#{last_event.class})" unless last_event.is_a?(Hash)
|
214
302
|
case last_event['Type']
|
303
|
+
when 'DONE'
|
304
|
+
Log.log.trace1{'Graceful shutdown, DONE message received'}
|
215
305
|
when 'ERROR'
|
216
306
|
if /bearer token/i.match?(last_event['Description']) &&
|
217
307
|
session[:token_regenerator].respond_to?(:refreshed_transfer_token)
|
@@ -221,10 +311,9 @@ module Aspera
|
|
221
311
|
env['ASPERA_SCP_TOKEN'] = session[:token_regenerator].refreshed_transfer_token
|
222
312
|
end
|
223
313
|
raise Transfer::Error.new(last_event['Description'], last_event['Code'].to_i)
|
224
|
-
when 'DONE'
|
225
|
-
nil
|
226
314
|
else
|
227
|
-
|
315
|
+
Log.log.error{"unexpected last event type: #{last_event['Type']}"}
|
316
|
+
# raise Transfer::Error, "unexpected last event type: #{last_event['Type']}, #{last_event['Description']}"
|
228
317
|
end
|
229
318
|
rescue SystemCallError => e
|
230
319
|
# Process.spawn failed, or socket error
|
@@ -232,18 +321,21 @@ module Aspera
|
|
232
321
|
rescue Interrupt
|
233
322
|
raise Transfer::Error, 'transfer interrupted by user'
|
234
323
|
ensure
|
235
|
-
mgt_server_socket
|
236
|
-
|
324
|
+
mgt_server_socket&.close
|
325
|
+
session.delete(:io)
|
237
326
|
# if command was successfully started, check its status
|
238
327
|
unless command_pid.nil?
|
239
|
-
|
240
|
-
|
241
|
-
status =
|
242
|
-
#
|
243
|
-
|
328
|
+
Process.kill(:INT, command_pid) if @monitor
|
329
|
+
# collect process exit status or wait for termination
|
330
|
+
_, status = Process.wait2(command_pid)
|
331
|
+
# process stderr of ascp
|
332
|
+
stderr_r.each_line do |line|
|
333
|
+
Log.log.error(line.chomp)
|
334
|
+
end
|
335
|
+
stderr_r.close
|
244
336
|
# status is nil if an exception occurred before starting command
|
245
337
|
if !status&.success?
|
246
|
-
message =
|
338
|
+
message = "#{name} failed (#{status})"
|
247
339
|
# raise error only if there was not already an exception (ERROR_INFO)
|
248
340
|
raise Transfer::Error, message unless $ERROR_INFO
|
249
341
|
# else display this message also, as main exception is already here
|
@@ -254,10 +346,10 @@ module Aspera
|
|
254
346
|
nil
|
255
347
|
end
|
256
348
|
|
257
|
-
attr_reader :sessions
|
258
|
-
|
259
349
|
private
|
260
350
|
|
351
|
+
attr_reader :sessions
|
352
|
+
|
261
353
|
# notify progress to callback
|
262
354
|
# @param event management port event
|
263
355
|
# @param session sessin object
|
@@ -293,88 +385,8 @@ module Aspera
|
|
293
385
|
# cspell:enable
|
294
386
|
# stop event when one file is completed
|
295
387
|
else
|
296
|
-
Log.log.debug{"
|
297
|
-
end
|
298
|
-
end
|
299
|
-
|
300
|
-
# send command to management port of command (used in `asession)
|
301
|
-
# @param job_id identified transfer process
|
302
|
-
# @param session_index index of session (for multi session)
|
303
|
-
# @param data command on mgt port, examples:
|
304
|
-
# {'type'=>'START','source'=>_path_,'destination'=>_path_}
|
305
|
-
# {'type'=>'DONE'}
|
306
|
-
def send_command(job_id, data)
|
307
|
-
session = @sessions.find{|session| session[:job_id].eql?(job_id)}
|
308
|
-
Log.log.debug{"command: #{data}"}
|
309
|
-
session[:io].puts(Ascp::Management.command_to_stream(data))
|
310
|
-
end
|
311
|
-
|
312
|
-
# options for initialize (same as values in option transfer_info)
|
313
|
-
# @param ascp_args [Array] additional arguments to ascp
|
314
|
-
# @param wss [Boolean] true: if both SSH and wss in ts: prefer wss
|
315
|
-
# @param quiet [Boolean] by default no native ascp progress bar
|
316
|
-
# @param trusted_certs [Array,NilClass] list of files with trusted certificates (stores)
|
317
|
-
# @param client_ssh_key [String] client ssh key option (from CLIENT_SSH_KEY_OPTIONS)
|
318
|
-
# @param check_ignore_cb [Proc] callback with host,port
|
319
|
-
# @param spawn_timeout_sec [Integer] timeout for ascp spawn
|
320
|
-
# @param spawn_delay_sec [Integer] optional delay to start between sessions
|
321
|
-
# @param multi_incr_udp [Boolean,NilClass] true: increment udp port for each session
|
322
|
-
# @param resume [Hash,NilClass] resume policy
|
323
|
-
# @param management_cb [Proc] callback for management events
|
324
|
-
# @param base_options [Hash] other options for base class
|
325
|
-
def initialize(
|
326
|
-
ascp_args: nil,
|
327
|
-
wss: true,
|
328
|
-
quiet: true,
|
329
|
-
trusted_certs: nil,
|
330
|
-
client_ssh_key: nil,
|
331
|
-
check_ignore_cb: nil,
|
332
|
-
spawn_timeout_sec: 2,
|
333
|
-
spawn_delay_sec: 2,
|
334
|
-
multi_incr_udp: nil,
|
335
|
-
resume: nil,
|
336
|
-
management_cb: nil,
|
337
|
-
**base_options
|
338
|
-
)
|
339
|
-
super(**base_options)
|
340
|
-
# special transfer parameters
|
341
|
-
@tr_opts = {
|
342
|
-
ascp_args: ascp_args,
|
343
|
-
wss: wss,
|
344
|
-
quiet: quiet,
|
345
|
-
trusted_certs: trusted_certs,
|
346
|
-
client_ssh_key: client_ssh_key,
|
347
|
-
check_ignore_cb: check_ignore_cb
|
348
|
-
}
|
349
|
-
@spawn_timeout_sec = spawn_timeout_sec
|
350
|
-
@spawn_delay_sec = spawn_delay_sec
|
351
|
-
# default is true on windows, false on other platforms
|
352
|
-
@multi_incr_udp = multi_incr_udp.nil? ? Environment.os.eql?(Environment::OS_WINDOWS) : multi_incr_udp
|
353
|
-
@management_cb = management_cb
|
354
|
-
@resume_policy = Resumer.new(resume.nil? ? {} : resume.symbolize_keys)
|
355
|
-
# all transfer jobs, key = SecureRandom.uuid, protected by mutex, cond var on change
|
356
|
-
@sessions = []
|
357
|
-
# mutex protects global data accessed by threads
|
358
|
-
@mutex = Mutex.new
|
359
|
-
end
|
360
|
-
|
361
|
-
# transfer thread entry
|
362
|
-
# @param session information
|
363
|
-
def transfer_thread_entry(session)
|
364
|
-
begin
|
365
|
-
# set name for logging
|
366
|
-
Thread.current[:name] = 'transfer'
|
367
|
-
Log.log.debug{"ENTER (#{Thread.current[:name]})"}
|
368
|
-
# start transfer with selected resumer policy
|
369
|
-
@resume_policy.execute_with_resume do
|
370
|
-
start_and_monitor_process(session: session, **session[:env_args])
|
371
|
-
end
|
372
|
-
Log.log.debug('transfer ok'.bg_green)
|
373
|
-
rescue StandardError => e
|
374
|
-
session[:error] = e
|
375
|
-
Log.log.error{"Transfer thread error: #{e.class}:\n#{e.message}:\n#{e.backtrace.join("\n")}".red} if Log.instance.level.eql?(:debug)
|
388
|
+
Log.log.debug{"Unknown event type for progress: #{event['Type']}"}
|
376
389
|
end
|
377
|
-
Log.log.debug{"EXIT (#{Thread.current[:name]})"}
|
378
390
|
end
|
379
391
|
end
|
380
392
|
end
|