aspera-cli 4.20.0 → 4.21.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +41 -3
- data/CONTRIBUTING.md +69 -142
- data/README.md +687 -461
- data/bin/ascli +5 -14
- data/bin/asession +3 -5
- data/examples/get_proto_file.rb +4 -3
- data/examples/proxy.pac +20 -20
- data/lib/aspera/agent/base.rb +2 -0
- data/lib/aspera/agent/connect.rb +20 -2
- data/lib/aspera/agent/{alpha.rb → desktop.rb} +12 -18
- data/lib/aspera/agent/direct.rb +30 -31
- data/lib/aspera/agent/node.rb +1 -11
- data/lib/aspera/agent/{trsdk.rb → transferd.rb} +37 -51
- data/lib/aspera/api/alee.rb +1 -1
- data/lib/aspera/api/aoc.rb +13 -8
- data/lib/aspera/api/cos_node.rb +1 -1
- data/lib/aspera/api/node.rb +49 -32
- data/lib/aspera/ascp/installation.rb +98 -77
- data/lib/aspera/ascp/management.rb +27 -6
- data/lib/aspera/cli/extended_value.rb +9 -3
- data/lib/aspera/cli/formatter.rb +155 -154
- data/lib/aspera/cli/info.rb +2 -1
- data/lib/aspera/cli/main.rb +12 -0
- data/lib/aspera/cli/manager.rb +4 -4
- data/lib/aspera/cli/plugin.rb +2 -2
- data/lib/aspera/cli/plugins/aoc.rb +134 -73
- data/lib/aspera/cli/plugins/config.rb +114 -83
- data/lib/aspera/cli/plugins/cos.rb +1 -0
- data/lib/aspera/cli/plugins/faspex.rb +4 -2
- data/lib/aspera/cli/plugins/faspex5.rb +29 -14
- data/lib/aspera/cli/plugins/node.rb +51 -41
- data/lib/aspera/cli/transfer_progress.rb +2 -0
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +1 -1
- data/lib/aspera/coverage.rb +5 -3
- data/lib/aspera/environment.rb +59 -16
- data/lib/aspera/faspex_postproc.rb +3 -5
- data/lib/aspera/hash_ext.rb +2 -12
- data/lib/aspera/node_simulator.rb +230 -112
- data/lib/aspera/oauth/base.rb +40 -48
- data/lib/aspera/oauth/factory.rb +41 -2
- data/lib/aspera/oauth/jwt.rb +4 -1
- data/lib/aspera/persistency_action_once.rb +1 -1
- data/lib/aspera/persistency_folder.rb +20 -2
- data/lib/aspera/preview/generator.rb +13 -10
- data/lib/aspera/preview/options.rb +2 -2
- data/lib/aspera/preview/terminal.rb +1 -1
- data/lib/aspera/preview/utils.rb +11 -6
- data/lib/aspera/products/connect.rb +82 -0
- data/lib/aspera/products/desktop.rb +30 -0
- data/lib/aspera/products/other.rb +82 -0
- data/lib/aspera/products/transferd.rb +61 -0
- data/lib/aspera/rest.rb +22 -17
- data/lib/aspera/secret_hider.rb +9 -2
- data/lib/aspera/ssh.rb +31 -24
- data/lib/aspera/temp_file_manager.rb +5 -4
- data/lib/aspera/transfer/parameters.rb +2 -1
- data/lib/aspera/transfer/spec.yaml +22 -20
- data/lib/aspera/transfer/sync.rb +1 -5
- data/lib/aspera/transfer/uri.rb +2 -2
- data/lib/aspera/uri_reader.rb +18 -1
- data/lib/transferd_pb.rb +86 -0
- data/lib/transferd_services_pb.rb +84 -0
- data.tar.gz.sig +0 -0
- metadata +13 -166
- metadata.gz.sig +0 -0
- data/examples/build_exec +0 -74
- data/examples/build_exec_rubyc +0 -40
- data/lib/aspera/ascp/products.rb +0 -168
- data/lib/transfer_pb.rb +0 -84
- data/lib/transfer_services_pb.rb +0 -82
data/bin/ascli
CHANGED
@@ -9,17 +9,8 @@ Encoding.default_internal = Encoding::UTF_8
|
|
9
9
|
Encoding.default_external = Encoding::UTF_8
|
10
10
|
$VERBOSE = old_verbose
|
11
11
|
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
rescue LoadError
|
18
|
-
# if in development, add path toward gem
|
19
|
-
$LOAD_PATH.unshift(gem_lib_folder)
|
20
|
-
require 'aspera/cli/main'
|
21
|
-
end
|
22
|
-
require 'aspera/environment'
|
23
|
-
Aspera::Environment.fix_home
|
24
|
-
Aspera::Cli::Main.new(ARGV).process_command_line
|
25
|
-
end
|
12
|
+
require 'aspera/coverage'
|
13
|
+
require 'aspera/environment'
|
14
|
+
require 'aspera/cli/main'
|
15
|
+
Aspera::Environment.fix_home
|
16
|
+
Aspera::Cli::Main.new(ARGV).process_command_line
|
data/bin/asession
CHANGED
@@ -1,12 +1,10 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
# frozen_string_literal: true
|
3
3
|
|
4
|
-
|
5
|
-
Kernel.load(File.join(gem_lib_folder, 'aspera/coverage.rb'))
|
6
|
-
$LOAD_PATH.unshift(gem_lib_folder)
|
4
|
+
require 'aspera/coverage'
|
7
5
|
require 'aspera/agent/direct'
|
8
6
|
require 'aspera/cli/extended_value'
|
9
|
-
require 'aspera/
|
7
|
+
require 'aspera/products/transferd'
|
10
8
|
require 'aspera/log'
|
11
9
|
require 'json'
|
12
10
|
# extended transfer spec parameter (only used in asession)
|
@@ -77,7 +75,7 @@ if session_spec.key?(PARAM_TMP_FILE_LIST_FOLDER)
|
|
77
75
|
Aspera::Transfer::Parameters.file_list_folder = session_spec[PARAM_TMP_FILE_LIST_FOLDER]
|
78
76
|
end
|
79
77
|
session_spec[PARAM_SDK] = File.join(Dir.home, '.aspera', 'sdk') unless session_spec.key?(PARAM_SDK)
|
80
|
-
Aspera::
|
78
|
+
Aspera::Products::Transferd.sdk_directory = session_spec[PARAM_SDK]
|
81
79
|
session_spec[PARAM_AGENT] = {} unless session_spec.key?(PARAM_AGENT)
|
82
80
|
agent_params = session_spec[PARAM_AGENT]
|
83
81
|
agent_params['quiet'] = true
|
data/examples/get_proto_file.rb
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
# frozen_string_literal: true
|
3
3
|
|
4
|
-
# Retrieve `transfer.proto` from the web
|
5
|
-
$LOAD_PATH.unshift(File.join(File.dirname(File.dirname(File.realpath(__FILE__))), 'lib'))
|
6
4
|
require 'aspera/ascp/installation'
|
7
|
-
|
5
|
+
require 'aspera/cli/transfer_progress'
|
6
|
+
Aspera::RestParameters.instance.progress_bar = Aspera::Cli::TransferProgress.new
|
7
|
+
# Retrieve `transfer.proto` from the web
|
8
|
+
Aspera::Ascp::Installation.instance.install_sdk(folder: ARGV.first, backup: false, with_exe: false) {|name| name.end_with?('.proto') ? '/' : nil }
|
data/examples/proxy.pac
CHANGED
@@ -11,20 +11,20 @@ function FindProxyForURL(url, host) {
|
|
11
11
|
|
12
12
|
/* Don't proxy local domains */
|
13
13
|
if (dnsDomainIs(host, ".example1.com") || (host == "example1.com")
|
14
|
-
|
15
|
-
|
14
|
+
|| dnsDomainIs(host, ".example2.com") || (host == "example2.com")
|
15
|
+
|| dnsDomainIs(host, ".example3.com") || (host == "example3.com")) {
|
16
16
|
return 'DIRECT';
|
17
17
|
}
|
18
18
|
|
19
19
|
/* Don't proxy Windows Update */
|
20
20
|
if ((host == "download.microsoft.com")
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
21
|
+
|| (host == "ntservicepack.microsoft.com")
|
22
|
+
|| (host == "cdm.microsoft.com") || (host == "wustat.windows.com")
|
23
|
+
|| (host == "windowsupdate.microsoft.com")
|
24
|
+
|| (dnsDomainIs(host, ".windowsupdate.microsoft.com"))
|
25
|
+
|| (host == "update.microsoft.com")
|
26
|
+
|| (dnsDomainIs(host, ".update.microsoft.com"))
|
27
|
+
|| (dnsDomainIs(host, ".windowsupdate.com"))) {
|
28
28
|
return 'DIRECT';
|
29
29
|
}
|
30
30
|
|
@@ -33,22 +33,22 @@ function FindProxyForURL(url, host) {
|
|
33
33
|
|
34
34
|
/* Don't proxy private addresses (RFC 3330) */
|
35
35
|
if (isInNet(hostIP, '0.0.0.0', '255.0.0.0')
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
36
|
+
|| isInNet(hostIP, '10.0.0.0', '255.0.0.0')
|
37
|
+
|| isInNet(hostIP, '127.0.0.0', '255.0.0.0')
|
38
|
+
|| isInNet(hostIP, '169.254.0.0', '255.255.0.0')
|
39
|
+
|| isInNet(hostIP, '172.16.0.0', '255.240.0.0')
|
40
|
+
|| isInNet(hostIP, '192.0.2.0', '255.255.255.0')
|
41
|
+
|| isInNet(hostIP, '192.88.99.0', '255.255.255.0')
|
42
|
+
|| isInNet(hostIP, '192.168.0.0', '255.255.0.0')
|
43
|
+
|| isInNet(hostIP, '198.18.0.0', '255.254.0.0')
|
44
|
+
|| isInNet(hostIP, '224.0.0.0', '240.0.0.0')
|
45
|
+
|| isInNet(hostIP, '240.0.0.0', '240.0.0.0')) {
|
46
46
|
return 'DIRECT';
|
47
47
|
}
|
48
48
|
}
|
49
49
|
|
50
50
|
if (url.substring(0, 5) == 'http:' || url.substring(0, 6) == 'https:'
|
51
|
-
|
51
|
+
|| url.substring(0, 4) == 'ftp:') {
|
52
52
|
return 'PROXY proxy.example.com:8080';
|
53
53
|
}
|
54
54
|
|
data/lib/aspera/agent/base.rb
CHANGED
@@ -7,6 +7,7 @@ module Aspera
|
|
7
7
|
# Base class for transfer agents
|
8
8
|
class Base
|
9
9
|
RUBY_EXT = '.rb'
|
10
|
+
private_constant :RUBY_EXT
|
10
11
|
class << self
|
11
12
|
def factory_create(agent, options)
|
12
13
|
# Aspera.assert_values(agent, agent_list)
|
@@ -15,6 +16,7 @@ module Aspera
|
|
15
16
|
end
|
16
17
|
|
17
18
|
# discover available agents
|
19
|
+
# @return [Array] list of symbols of agents
|
18
20
|
def agent_list
|
19
21
|
base_class = File.basename(__FILE__)
|
20
22
|
Dir.entries(File.dirname(File.expand_path(__FILE__))).select do |file|
|
data/lib/aspera/agent/connect.rb
CHANGED
@@ -1,7 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'aspera/agent/base'
|
4
|
+
require 'aspera/products/connect'
|
5
|
+
require 'aspera/products/other'
|
4
6
|
require 'aspera/rest'
|
7
|
+
require 'aspera/environment'
|
5
8
|
require 'securerandom'
|
6
9
|
|
7
10
|
module Aspera
|
@@ -13,14 +16,14 @@ module Aspera
|
|
13
16
|
SLEEP_SEC_BETWEEN_RETRY = 5
|
14
17
|
private_constant :CONNECT_START_URIS, :SLEEP_SEC_BETWEEN_RETRY
|
15
18
|
def initialize(**base_options)
|
16
|
-
super
|
19
|
+
super
|
17
20
|
@connect_settings = {
|
18
21
|
'app_id' => SecureRandom.uuid
|
19
22
|
}
|
20
23
|
raise 'Using connect requires a graphical environment' if !Environment.default_gui_mode.eql?(:graphical)
|
21
24
|
method_index = 0
|
22
25
|
begin
|
23
|
-
connect_url =
|
26
|
+
connect_url = connect_api_url
|
24
27
|
Log.log.debug{"found: #{connect_url}"}
|
25
28
|
@connect_api = Rest.new(
|
26
29
|
base_url: "#{connect_url}/v5/connect", # could use v6 also now
|
@@ -43,6 +46,21 @@ module Aspera
|
|
43
46
|
end
|
44
47
|
end
|
45
48
|
|
49
|
+
# @return the file path of local connect where API's URI can be read
|
50
|
+
def connect_api_url
|
51
|
+
connect_locations = Products::Other.find(Products::Connect.locations).first
|
52
|
+
raise "Product: #{name} not found, please install." if connect_locations.nil?
|
53
|
+
folder = File.join(connect_locations[:run_root], 'var', 'run')
|
54
|
+
['', 's'].each do |ext|
|
55
|
+
uri_file = File.join(folder, "http#{ext}.uri")
|
56
|
+
Log.log.debug{"checking connect port file: #{uri_file}"}
|
57
|
+
if File.exist?(uri_file)
|
58
|
+
return File.open(uri_file, &:gets).strip
|
59
|
+
end
|
60
|
+
end
|
61
|
+
raise "no connect uri file found in #{folder}"
|
62
|
+
end
|
63
|
+
|
46
64
|
def start_transfer(transfer_spec, token_regenerator: nil)
|
47
65
|
if transfer_spec['direction'] == 'send'
|
48
66
|
Log.log.warn{"Connect requires upload selection using GUI, ignoring #{transfer_spec['paths']}".red}
|
@@ -2,26 +2,25 @@
|
|
2
2
|
|
3
3
|
require 'aspera/agent/base'
|
4
4
|
require 'aspera/rest'
|
5
|
-
require 'aspera/log'
|
6
|
-
require 'aspera/json_rpc'
|
7
5
|
require 'aspera/environment'
|
6
|
+
require 'aspera/json_rpc'
|
7
|
+
require 'aspera/products/desktop'
|
8
8
|
require 'securerandom'
|
9
9
|
|
10
10
|
module Aspera
|
11
11
|
module Agent
|
12
|
-
# Aspera Desktop
|
13
|
-
class
|
12
|
+
# Client: Aspera for Desktop
|
13
|
+
class Desktop < Base
|
14
14
|
# try twice the main init url in sequence
|
15
15
|
START_URIS = ['aspera://', 'aspera://', 'aspera://']
|
16
16
|
# delay between each try to start the app
|
17
17
|
SLEEP_SEC_BETWEEN_RETRY = 5
|
18
|
-
APP_IDENTIFIER = 'com.ibm.software.aspera.desktop'
|
19
|
-
APP_NAME = 'Aspera Desktop Alpha Client'
|
20
18
|
private_constant :START_URIS, :SLEEP_SEC_BETWEEN_RETRY
|
19
|
+
|
21
20
|
def initialize(**base_options)
|
22
21
|
@application_id = SecureRandom.uuid
|
23
22
|
@xfer_id = nil
|
24
|
-
super
|
23
|
+
super
|
25
24
|
raise 'Using client requires a graphical environment' if !Environment.default_gui_mode.eql?(:graphical)
|
26
25
|
method_index = 0
|
27
26
|
begin
|
@@ -34,24 +33,20 @@ module Aspera
|
|
34
33
|
rescue Errno::ECONNREFUSED => e
|
35
34
|
start_url = START_URIS[method_index]
|
36
35
|
method_index += 1
|
37
|
-
raise StandardError, "Unable to start #{APP_NAME} #{method_index} times" if start_url.nil?
|
38
|
-
Log.log.warn{"#{APP_NAME} is not started (#{e}). Trying to start it ##{method_index}..."}
|
36
|
+
raise StandardError, "Unable to start #{Products::Desktop::APP_NAME} #{method_index} times" if start_url.nil?
|
37
|
+
Log.log.warn{"#{Products::Desktop::APP_NAME} is not started (#{e}). Trying to start it ##{method_index}..."}
|
39
38
|
if !Environment.open_uri_graphical(start_url)
|
40
39
|
Environment.open_uri_graphical('https://www.ibm.com/aspera/connect/')
|
41
|
-
raise StandardError, "#{APP_NAME} is not installed"
|
40
|
+
raise StandardError, "#{Products::Desktop::APP_NAME} is not installed"
|
42
41
|
end
|
43
42
|
sleep(SLEEP_SEC_BETWEEN_RETRY)
|
44
43
|
retry
|
45
44
|
end
|
46
45
|
end
|
47
46
|
|
48
|
-
def sdk_log_file
|
49
|
-
File.join(Dir.home, 'Library', 'Logs', APP_IDENTIFIER, 'ibm-aspera-desktop.log')
|
50
|
-
end
|
51
|
-
|
52
47
|
def aspera_client_api_url
|
53
|
-
log_file =
|
54
|
-
url =
|
48
|
+
log_file = Products::Desktop.log_file
|
49
|
+
url = 'http://127.0.0.1:33024'
|
55
50
|
File.open(log_file, 'r') do |file|
|
56
51
|
file.each_line do |line|
|
57
52
|
line = line.chomp
|
@@ -60,8 +55,7 @@ module Aspera
|
|
60
55
|
end
|
61
56
|
end
|
62
57
|
end
|
63
|
-
|
64
|
-
raise StandardError, "Unable to find the JSON-RPC server URL in #{log_file}" if url.nil?
|
58
|
+
# raise StandardError, "Unable to find the JSON-RPC server URL in #{log_file}" if url.nil?
|
65
59
|
return url
|
66
60
|
end
|
67
61
|
|
data/lib/aspera/agent/direct.rb
CHANGED
@@ -72,7 +72,7 @@ module Aspera
|
|
72
72
|
session = {
|
73
73
|
id: nil, # SessionId from INIT message in mgt port
|
74
74
|
job_id: SecureRandom.uuid, # job id (regroup sessions)
|
75
|
-
ts: transfer_spec, # transfer spec
|
75
|
+
ts: transfer_spec, # global transfer spec
|
76
76
|
thread: nil, # Thread object monitoring management port, not nil when pushed to :sessions
|
77
77
|
error: nil, # exception if failed
|
78
78
|
io: nil, # management port server socket
|
@@ -84,7 +84,7 @@ module Aspera
|
|
84
84
|
if multi_session_info.nil?
|
85
85
|
Log.log.debug('Starting single session thread')
|
86
86
|
# single session for transfer : simple
|
87
|
-
session[:thread] = Thread.new
|
87
|
+
session[:thread] = Thread.new {transfer_thread_entry(session)}
|
88
88
|
@sessions.push(session)
|
89
89
|
else
|
90
90
|
Log.log.debug('Starting multi session threads')
|
@@ -101,7 +101,7 @@ module Aspera
|
|
101
101
|
# option: increment (default as per ascp manual) or not (cluster on other side ?)
|
102
102
|
args.unshift('-O', (multi_session_info[:udp_base] + i - 1).to_s) if @multi_incr_udp
|
103
103
|
# finally start the thread
|
104
|
-
this_session[:thread] = Thread.new
|
104
|
+
this_session[:thread] = Thread.new {transfer_thread_entry(this_session)}
|
105
105
|
@sessions.push(this_session)
|
106
106
|
end
|
107
107
|
end
|
@@ -132,18 +132,18 @@ module Aspera
|
|
132
132
|
|
133
133
|
# @return [Array] list of sessions for a job
|
134
134
|
def sessions_by_job(job_id)
|
135
|
-
@sessions.select{|
|
135
|
+
@sessions.select{|session| session[:job_id].eql?(job_id)}
|
136
136
|
end
|
137
137
|
|
138
|
-
# This is the low level method to start the transfer process
|
138
|
+
# This is the low level method to start the transfer process.
|
139
|
+
# Typically started in a thread.
|
139
140
|
# Start process with management port.
|
140
|
-
# returns
|
141
|
-
# raises FaspError on error
|
142
141
|
# @param session this session information, keys :io and :token_regenerator
|
143
|
-
# @param env [Hash] environment variables
|
144
|
-
# @param name [Symbol] name of executable: :ascp, :ascp4 or :async
|
145
|
-
# @param args [Array] command line arguments
|
142
|
+
# @param env [Hash] environment variables (comes from ascp_args)
|
143
|
+
# @param name [Symbol] name of executable: :ascp, :ascp4 or :async (comes from ascp_args)
|
144
|
+
# @param args [Array] command line arguments (comes from ascp_args)
|
146
145
|
# @return [nil] when process has exited
|
146
|
+
# @throw FaspError on error
|
147
147
|
def start_and_monitor_process(
|
148
148
|
session:,
|
149
149
|
env:,
|
@@ -159,6 +159,8 @@ module Aspera
|
|
159
159
|
mgt_server_socket = socket_class.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
|
160
160
|
# open any available (0) local TCP port for use as management port
|
161
161
|
mgt_server_socket.bind(Addrinfo.tcp(LISTEN_LOCAL_ADDRESS, SELECT_AVAILABLE_PORT))
|
162
|
+
# make port ready to accept connections, before starting ascp
|
163
|
+
mgt_server_socket.listen(1)
|
162
164
|
# build arguments and add mgt port
|
163
165
|
command_arguments = if name.eql?(:async)
|
164
166
|
["--exclusive-mgmt-port=#{mgt_server_socket.local_address.ip_port}"]
|
@@ -166,12 +168,15 @@ module Aspera
|
|
166
168
|
['-M', mgt_server_socket.local_address.ip_port.to_s]
|
167
169
|
end
|
168
170
|
command_arguments.concat(args)
|
171
|
+
# capture process stderr
|
172
|
+
stderr_r, stderr_w = IO.pipe
|
169
173
|
# get location of command executable (ascp, async)
|
170
174
|
command_path = Ascp::Installation.instance.path(name)
|
171
|
-
command_pid = Environment.secure_spawn(env: env, exec: command_path, args: command_arguments)
|
175
|
+
command_pid = Environment.secure_spawn(env: env, exec: command_path, args: command_arguments, err: stderr_w)
|
176
|
+
stderr_w.close
|
172
177
|
notify_progress(:pre_start, session_id: nil, info: "waiting for #{name} to start")
|
173
|
-
mgt_server_socket.listen(1)
|
174
178
|
# TODO: timeout does not work when Process.spawn is used... until process exits, then it works
|
179
|
+
# So we use select to detect that anything happens on the socket (connection)
|
175
180
|
Log.log.debug{"before select, timeout: #{@spawn_timeout_sec}"}
|
176
181
|
readable, _, _ = IO.select([mgt_server_socket], nil, nil, @spawn_timeout_sec)
|
177
182
|
Log.log.debug('after select, before accept')
|
@@ -192,13 +197,19 @@ module Aspera
|
|
192
197
|
next unless event
|
193
198
|
# event is ready
|
194
199
|
Log.log.trace1{Log.dump(:management_port, event)}
|
200
|
+
# store latest event by type
|
201
|
+
session[:id] = event['SessionId'] if event['Type'].eql?('INIT')
|
195
202
|
@management_cb&.call(event)
|
196
203
|
process_progress(event)
|
197
|
-
Log.log.error(
|
204
|
+
Log.log.error(event['Description'].to_s) if event['Type'].eql?('FILEERROR') # cspell:disable-line
|
198
205
|
end
|
199
206
|
Log.log.debug('management io closed')
|
200
207
|
# check that last status was received before process exit
|
201
208
|
last_event = processor.last_event
|
209
|
+
# process stderr of ascp
|
210
|
+
stderr_r&.each_line do |line|
|
211
|
+
Log.log.error(line.chomp)
|
212
|
+
end
|
202
213
|
raise Transfer::Error, "internal: no management event (#{last_event.class})" unless last_event.is_a?(Hash)
|
203
214
|
case last_event['Type']
|
204
215
|
when 'ERROR'
|
@@ -222,6 +233,7 @@ module Aspera
|
|
222
233
|
raise Transfer::Error, 'transfer interrupted by user'
|
223
234
|
ensure
|
224
235
|
mgt_server_socket.close
|
236
|
+
stderr_r&.close
|
225
237
|
# if command was successfully started, check its status
|
226
238
|
unless command_pid.nil?
|
227
239
|
# "wait" for process to avoid zombie
|
@@ -242,10 +254,13 @@ module Aspera
|
|
242
254
|
nil
|
243
255
|
end
|
244
256
|
|
257
|
+
attr_reader :sessions
|
258
|
+
|
245
259
|
private
|
246
260
|
|
247
261
|
# notify progress to callback
|
248
262
|
# @param event management port event
|
263
|
+
# @param session sessin object
|
249
264
|
def process_progress(event)
|
250
265
|
session_id = event['SessionId']
|
251
266
|
case event['Type']
|
@@ -282,14 +297,6 @@ module Aspera
|
|
282
297
|
end
|
283
298
|
end
|
284
299
|
|
285
|
-
# @return [Hash] session information
|
286
|
-
def session_by_id(id)
|
287
|
-
matches = @sessions.select{|session_info| session_info[:id].eql?(id)}
|
288
|
-
raise 'no such session' if matches.empty?
|
289
|
-
raise 'more than one session' if matches.length > 1
|
290
|
-
return matches.first
|
291
|
-
end
|
292
|
-
|
293
300
|
# send command to management port of command (used in `asession)
|
294
301
|
# @param job_id identified transfer process
|
295
302
|
# @param session_index index of session (for multi session)
|
@@ -297,18 +304,10 @@ module Aspera
|
|
297
304
|
# {'type'=>'START','source'=>_path_,'destination'=>_path_}
|
298
305
|
# {'type'=>'DONE'}
|
299
306
|
def send_command(job_id, data)
|
300
|
-
session =
|
307
|
+
session = @sessions.find{|session| session[:job_id].eql?(job_id)}
|
301
308
|
Log.log.debug{"command: #{data}"}
|
302
|
-
|
303
|
-
command = data
|
304
|
-
.keys
|
305
|
-
.map{|k|"#{k.capitalize}: #{data[k]}"}
|
306
|
-
.unshift(MGT_HEADER)
|
307
|
-
.push('', '')
|
308
|
-
.join("\n")
|
309
|
-
session[:io].puts(command)
|
309
|
+
session[:io].puts(Ascp::Management.command_to_stream(data))
|
310
310
|
end
|
311
|
-
attr_reader :sessions
|
312
311
|
|
313
312
|
# options for initialize (same as values in option transfer_info)
|
314
313
|
# @param ascp_args [Array] additional arguments to ascp
|
data/lib/aspera/agent/node.rb
CHANGED
@@ -50,14 +50,6 @@ module Aspera
|
|
50
50
|
return @node_api
|
51
51
|
end
|
52
52
|
|
53
|
-
# use this to set the node_api end point before using the class.
|
54
|
-
def node_api=(new_value)
|
55
|
-
if !@node_api.nil? && !new_value.nil?
|
56
|
-
Log.log.warn('overriding existing node api value')
|
57
|
-
end
|
58
|
-
@node_api = new_value
|
59
|
-
end
|
60
|
-
|
61
53
|
# generic method
|
62
54
|
def start_transfer(transfer_spec, token_regenerator: nil)
|
63
55
|
# add root id if access key
|
@@ -120,9 +112,7 @@ module Aspera
|
|
120
112
|
# Bug in HSTS ? transfer is marked failed, but there is no reason
|
121
113
|
break if transfer_data['error_code'].eql?(0) && transfer_data['error_desc'].empty?
|
122
114
|
raise Transfer::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
|
123
|
-
else
|
124
|
-
Log.log.warn{"transfer_data -> #{transfer_data}"}
|
125
|
-
raise Transfer::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
|
115
|
+
else Aspera.error_unexpected_value(transfer_data['status']){"transfer_data -> #{transfer_data}"}
|
126
116
|
end
|
127
117
|
sleep(1.0)
|
128
118
|
end
|
@@ -1,57 +1,39 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'aspera/environment'
|
3
4
|
require 'aspera/agent/base'
|
4
|
-
require 'aspera/
|
5
|
+
require 'aspera/products/transferd'
|
5
6
|
require 'aspera/temp_file_manager'
|
6
|
-
require 'aspera/log'
|
7
|
-
require 'aspera/assert'
|
8
7
|
require 'json'
|
9
8
|
require 'uri'
|
10
|
-
require '
|
9
|
+
require 'transferd_services_pb'
|
11
10
|
|
12
11
|
module Aspera
|
13
12
|
module Agent
|
14
|
-
class
|
15
|
-
#
|
13
|
+
class Transferd < Base
|
14
|
+
# https://github.com/grpc/grpc/blob/master/doc/naming.md
|
16
15
|
# https://grpc.io/docs/guides/custom-name-resolution/
|
17
16
|
LOCAL_SOCKET_ADDR = '127.0.0.1'
|
18
17
|
PORT_SEP = ':'
|
19
18
|
# port zero means select a random available high port
|
20
19
|
AUTO_LOCAL_TCP_PORT = "#{PORT_SEP}0"
|
21
|
-
class << self
|
22
|
-
# Well, the port number is only in log file
|
23
|
-
def daemon_port_from_log(log_file)
|
24
|
-
result = nil
|
25
|
-
# if port is zero, a dynamic port was created, get it
|
26
|
-
File.open(log_file, 'r') do |file|
|
27
|
-
file.each_line do |line|
|
28
|
-
# Well, it's tricky to depend on log
|
29
|
-
if (m = line.match(/Info: API Server: Listening on ([^:]+):(\d+) /))
|
30
|
-
result = m[2].to_i
|
31
|
-
# no "break" , need to read last matching log line
|
32
|
-
end
|
33
|
-
end
|
34
|
-
end
|
35
|
-
raise 'Port not found in daemon logs' if result.nil?
|
36
|
-
Log.log.debug{"Got port #{result} from log"}
|
37
|
-
return result
|
38
|
-
end
|
39
|
-
end
|
40
20
|
|
41
|
-
|
42
|
-
|
43
|
-
# @param
|
44
|
-
# @param
|
21
|
+
private_constant :LOCAL_SOCKET_ADDR, :PORT_SEP, :AUTO_LOCAL_TCP_PORT
|
22
|
+
|
23
|
+
# @param url [String] URL of the transfer manager daemon
|
24
|
+
# @param start [Bool] if false, expect that an external daemon is already running
|
25
|
+
# @param stop [Bool] if false, do not shutdown daemon on exit
|
26
|
+
# @param base [Hash] base class options
|
45
27
|
def initialize(
|
46
|
-
url:
|
47
|
-
|
48
|
-
|
49
|
-
**
|
28
|
+
url: AUTO_LOCAL_TCP_PORT,
|
29
|
+
start: true,
|
30
|
+
stop: true,
|
31
|
+
**base
|
50
32
|
)
|
51
|
-
super(**
|
52
|
-
@
|
33
|
+
super(**base)
|
34
|
+
@stop = stop
|
53
35
|
is_local_auto_port = url.eql?(AUTO_LOCAL_TCP_PORT)
|
54
|
-
raise 'Cannot
|
36
|
+
raise 'Cannot set options `stop` or `start` to false with port zero' if is_local_auto_port && (!@stop || !start)
|
55
37
|
# keep PID for optional shutdown
|
56
38
|
@daemon_pid = nil
|
57
39
|
daemon_endpoint = url
|
@@ -61,18 +43,18 @@ module Aspera
|
|
61
43
|
# no address: local bind
|
62
44
|
daemon_endpoint = "#{LOCAL_SOCKET_ADDR}#{daemon_endpoint}" if daemon_endpoint.match?(/^#{PORT_SEP}[0-9]+$/o)
|
63
45
|
# Create stub (without credentials)
|
64
|
-
@transfer_client =
|
46
|
+
@transfer_client = ::Transferd::Api::TransferService::Stub.new(daemon_endpoint, :this_channel_is_insecure)
|
65
47
|
# Initiate actual connection
|
66
|
-
get_info_response = @transfer_client.get_info(
|
48
|
+
get_info_response = @transfer_client.get_info(::Transferd::Api::InstanceInfoRequest.new)
|
67
49
|
Log.log.debug{"Daemon info: #{get_info_response}"}
|
68
|
-
Log.log.warn
|
50
|
+
Log.log.warn('Attached to existing daemon') unless @daemon_pid || !start || !@stop
|
69
51
|
at_exit{shutdown}
|
70
52
|
rescue GRPC::Unavailable => e
|
71
53
|
# if transferd is external: do not start it, or other error
|
72
|
-
raise if
|
54
|
+
raise if !start || !e.message.include?('failed to connect')
|
73
55
|
# we already tried to start a daemon, but it failed
|
74
56
|
Aspera.assert(@daemon_pid.nil?){"Daemon started with PID #{@daemon_pid}, but connection failed to #{daemon_endpoint}}"}
|
75
|
-
Log.log.warn('no daemon present, starting daemon...') if
|
57
|
+
Log.log.warn('no daemon present, starting daemon...') if !start
|
76
58
|
# transferd only supports local ip and port
|
77
59
|
daemon_uri = URI.parse("ipv4://#{daemon_endpoint}")
|
78
60
|
Aspera.assert(daemon_uri.scheme.eql?('ipv4')){"Invalid scheme daemon URI #{daemon_endpoint}"}
|
@@ -83,8 +65,8 @@ module Aspera
|
|
83
65
|
fasp_runtime: {
|
84
66
|
use_embedded: false,
|
85
67
|
user_defined: {
|
86
|
-
bin:
|
87
|
-
etc:
|
68
|
+
bin: Products::Transferd.sdk_directory,
|
69
|
+
etc: Products::Transferd.sdk_directory
|
88
70
|
}
|
89
71
|
}
|
90
72
|
}
|
@@ -95,7 +77,11 @@ module Aspera
|
|
95
77
|
log_stdout = "#{transferd_base_tmp}.out"
|
96
78
|
log_stderr = "#{transferd_base_tmp}.err"
|
97
79
|
File.write(conf_file, config.to_json)
|
98
|
-
@daemon_pid =
|
80
|
+
@daemon_pid = Environment.secure_spawn(
|
81
|
+
exec: Ascp::Installation.instance.path(:transferd),
|
82
|
+
args: ['--config', conf_file],
|
83
|
+
out: log_stdout,
|
84
|
+
err: log_stderr)
|
99
85
|
begin
|
100
86
|
# wait for process to initialize, max 2 seconds
|
101
87
|
Timeout.timeout(2.0) do
|
@@ -107,10 +93,10 @@ module Aspera
|
|
107
93
|
nil
|
108
94
|
end
|
109
95
|
Log.log.debug{"Daemon started with pid #{@daemon_pid}"}
|
110
|
-
Process.detach(@daemon_pid)
|
96
|
+
Process.detach(@daemon_pid) unless @stop
|
111
97
|
at_exit {shutdown}
|
112
98
|
# update port for next connection attempt (if auto high port was requested)
|
113
|
-
daemon_endpoint = "#{LOCAL_SOCKET_ADDR}#{PORT_SEP}#{
|
99
|
+
daemon_endpoint = "#{LOCAL_SOCKET_ADDR}#{PORT_SEP}#{Products::Transferd.daemon_port_from_log(log_stdout)}" if is_local_auto_port
|
114
100
|
# local daemon started, try again
|
115
101
|
retry
|
116
102
|
end
|
@@ -118,9 +104,9 @@ module Aspera
|
|
118
104
|
|
119
105
|
def start_transfer(transfer_spec, token_regenerator: nil)
|
120
106
|
# create a transfer request
|
121
|
-
transfer_request =
|
122
|
-
transferType:
|
123
|
-
config:
|
107
|
+
transfer_request = ::Transferd::Api::TransferRequest.new(
|
108
|
+
transferType: ::Transferd::Api::TransferType::FILE_REGULAR, # transfer type (file/stream)
|
109
|
+
config: ::Transferd::Api::TransferConfig.new, # transfer configuration
|
124
110
|
transferSpec: transfer_spec.to_json) # transfer definition
|
125
111
|
# send start transfer request to the transfer manager daemon
|
126
112
|
start_transfer_response = @transfer_client.start_transfer(transfer_request)
|
@@ -134,7 +120,7 @@ module Aspera
|
|
134
120
|
session_started = false
|
135
121
|
bytes_expected = nil
|
136
122
|
# monitor transfer status
|
137
|
-
@transfer_client.monitor_transfers(
|
123
|
+
@transfer_client.monitor_transfers(::Transferd::Api::RegistrationRequest.new(transferId: [@transfer_id])) do |response|
|
138
124
|
Log.log.debug{Log.dump(:response, response.to_h)}
|
139
125
|
# Log.log.debug{"#{response.sessionInfo.preTransferBytes} #{response.transferInfo.bytesTransferred}"}
|
140
126
|
case response.status
|
@@ -167,7 +153,7 @@ module Aspera
|
|
167
153
|
end
|
168
154
|
|
169
155
|
def shutdown
|
170
|
-
stop_daemon
|
156
|
+
stop_daemon if @stop
|
171
157
|
end
|
172
158
|
|
173
159
|
def stop_daemon
|
data/lib/aspera/api/alee.rb
CHANGED
@@ -7,7 +7,7 @@ module Aspera
|
|
7
7
|
def initialize(entitlement_id, customer_id, api_domain: AoC::SAAS_DOMAIN_PROD, version: 'v1')
|
8
8
|
super(
|
9
9
|
base_url: "https://api.#{api_domain}/metering/#{version}",
|
10
|
-
headers: {'X-Aspera-Entitlement-Authorization' => Rest.
|
10
|
+
headers: {'X-Aspera-Entitlement-Authorization' => Rest.basic_authorization(entitlement_id, customer_id)}
|
11
11
|
)
|
12
12
|
end
|
13
13
|
end
|