aspera-cli 4.10.0 → 4.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +19 -0
- data/CHANGELOG.md +528 -0
- data/CONTRIBUTING.md +143 -0
- data/README.md +977 -589
- data/bin/ascli +4 -4
- data/bin/asession +12 -12
- data/docs/test_env.conf +29 -19
- data/examples/aoc.rb +6 -6
- data/examples/dascli +18 -16
- data/examples/faspex4.rb +15 -15
- data/examples/node.rb +12 -12
- data/examples/proxy.pac +2 -2
- data/examples/server.rb +12 -12
- data/lib/aspera/aoc.rb +344 -272
- data/lib/aspera/ascmd.rb +56 -54
- data/lib/aspera/ats_api.rb +4 -4
- data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
- data/lib/aspera/cli/extended_value.rb +9 -9
- data/lib/aspera/cli/{formater.rb → formatter.rb} +69 -69
- data/lib/aspera/cli/listener/line_dump.rb +1 -1
- data/lib/aspera/cli/listener/logger.rb +1 -1
- data/lib/aspera/cli/listener/progress.rb +5 -6
- data/lib/aspera/cli/listener/progress_multi.rb +16 -21
- data/lib/aspera/cli/main.rb +72 -73
- data/lib/aspera/cli/manager.rb +112 -112
- data/lib/aspera/cli/plugin.rb +68 -48
- data/lib/aspera/cli/plugins/alee.rb +4 -4
- data/lib/aspera/cli/plugins/aoc.rb +322 -720
- data/lib/aspera/cli/plugins/ats.rb +50 -52
- data/lib/aspera/cli/plugins/bss.rb +10 -10
- data/lib/aspera/cli/plugins/config.rb +514 -410
- data/lib/aspera/cli/plugins/console.rb +12 -12
- data/lib/aspera/cli/plugins/cos.rb +18 -20
- data/lib/aspera/cli/plugins/faspex.rb +134 -136
- data/lib/aspera/cli/plugins/faspex5.rb +235 -70
- data/lib/aspera/cli/plugins/node.rb +378 -309
- data/lib/aspera/cli/plugins/orchestrator.rb +52 -49
- data/lib/aspera/cli/plugins/preview.rb +129 -120
- data/lib/aspera/cli/plugins/server.rb +137 -83
- data/lib/aspera/cli/plugins/shares.rb +77 -52
- data/lib/aspera/cli/plugins/sync.rb +13 -33
- data/lib/aspera/cli/transfer_agent.rb +61 -61
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +3 -3
- data/lib/aspera/command_line_builder.rb +78 -74
- data/lib/aspera/cos_node.rb +31 -29
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +30 -28
- data/lib/aspera/fasp/agent_base.rb +17 -15
- data/lib/aspera/fasp/agent_connect.rb +34 -32
- data/lib/aspera/fasp/agent_direct.rb +70 -73
- data/lib/aspera/fasp/agent_httpgw.rb +79 -74
- data/lib/aspera/fasp/agent_node.rb +26 -26
- data/lib/aspera/fasp/agent_trsdk.rb +20 -20
- data/lib/aspera/fasp/error.rb +3 -2
- data/lib/aspera/fasp/error_info.rb +11 -8
- data/lib/aspera/fasp/installation.rb +80 -80
- data/lib/aspera/fasp/listener.rb +2 -2
- data/lib/aspera/fasp/parameters.rb +103 -92
- data/lib/aspera/fasp/parameters.yaml +313 -214
- data/lib/aspera/fasp/resume_policy.rb +10 -10
- data/lib/aspera/fasp/transfer_spec.rb +22 -2
- data/lib/aspera/fasp/uri.rb +7 -7
- data/lib/aspera/faspex_gw.rb +80 -159
- data/lib/aspera/faspex_postproc.rb +77 -0
- data/lib/aspera/hash_ext.rb +3 -3
- data/lib/aspera/id_generator.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +23 -28
- data/lib/aspera/keychain/macos_security.rb +21 -20
- data/lib/aspera/log.rb +13 -13
- data/lib/aspera/nagios.rb +24 -23
- data/lib/aspera/node.rb +217 -38
- data/lib/aspera/oauth.rb +78 -74
- data/lib/aspera/open_application.rb +19 -11
- data/lib/aspera/persistency_action_once.rb +4 -4
- data/lib/aspera/persistency_folder.rb +13 -13
- data/lib/aspera/preview/file_types.rb +8 -8
- data/lib/aspera/preview/generator.rb +67 -67
- data/lib/aspera/preview/utils.rb +27 -27
- data/lib/aspera/proxy_auto_config.js +63 -63
- data/lib/aspera/proxy_auto_config.rb +19 -19
- data/lib/aspera/rest.rb +65 -67
- data/lib/aspera/rest_call_error.rb +2 -1
- data/lib/aspera/rest_error_analyzer.rb +22 -21
- data/lib/aspera/rest_errors_aspera.rb +16 -16
- data/lib/aspera/secret_hider.rb +17 -14
- data/lib/aspera/ssh.rb +15 -14
- data/lib/aspera/sync.rb +177 -62
- data/lib/aspera/temp_file_manager.rb +2 -2
- data/lib/aspera/uri_reader.rb +4 -4
- data/lib/aspera/web_auth.rb +13 -64
- data/lib/aspera/web_server_simple.rb +76 -0
- data.tar.gz.sig +0 -0
- metadata +11 -6
- metadata.gz.sig +0 -0
@@ -2,6 +2,7 @@
|
|
2
2
|
|
3
3
|
require 'aspera/fasp/agent_base'
|
4
4
|
require 'aspera/fasp/transfer_spec'
|
5
|
+
require 'aspera/node'
|
5
6
|
require 'aspera/log'
|
6
7
|
require 'tty-spinner'
|
7
8
|
|
@@ -9,20 +10,21 @@ module Aspera
|
|
9
10
|
module Fasp
|
10
11
|
# this singleton class is used by the CLI to provide a common interface to start a transfer
|
11
12
|
# before using it, the use must set the `node_api` member.
|
12
|
-
class AgentNode < AgentBase
|
13
|
+
class AgentNode < Aspera::Fasp::AgentBase
|
13
14
|
# option include: root_id if the node is an access key
|
14
15
|
attr_writer :options
|
16
|
+
|
15
17
|
def initialize(options)
|
16
18
|
raise 'node specification must be Hash' unless options.is_a?(Hash)
|
17
|
-
%i[url username password].each { |k| raise "missing parameter [#{k}] in node specification: #{options}" unless options.
|
19
|
+
%i[url username password].each { |k| raise "missing parameter [#{k}] in node specification: #{options}" unless options.key?(k) }
|
18
20
|
super()
|
19
21
|
# root id is required for access key
|
20
22
|
@root_id = options[:root_id]
|
21
23
|
rest_params = { base_url: options[:url]}
|
22
24
|
if /^Bearer /.match?(options[:password])
|
23
25
|
rest_params[:headers] = {
|
24
|
-
|
25
|
-
'Authorization'
|
26
|
+
Aspera::Node::HEADER_X_ASPERA_ACCESS_KEY => options[:username],
|
27
|
+
'Authorization' => options[:password]
|
26
28
|
}
|
27
29
|
raise 'root_id is required for access key' if @root_id.nil?
|
28
30
|
else
|
@@ -39,7 +41,7 @@ module Aspera
|
|
39
41
|
|
40
42
|
# used internally to ensure node api is set before using.
|
41
43
|
def node_api_
|
42
|
-
raise StandardError,'Before using this object, set the node_api attribute to a Aspera::Rest object' if @node_api.nil?
|
44
|
+
raise StandardError, 'Before using this object, set the node_api attribute to a Aspera::Rest object' if @node_api.nil?
|
43
45
|
return @node_api
|
44
46
|
end
|
45
47
|
# use this to read the node_api end point.
|
@@ -54,7 +56,7 @@ module Aspera
|
|
54
56
|
end
|
55
57
|
|
56
58
|
# generic method
|
57
|
-
def start_transfer(transfer_spec,
|
59
|
+
def start_transfer(transfer_spec, token_regenerator: nil)
|
58
60
|
# add root id if access key
|
59
61
|
if !@root_id.nil?
|
60
62
|
case transfer_spec['direction']
|
@@ -64,9 +66,9 @@ module Aspera
|
|
64
66
|
end
|
65
67
|
end
|
66
68
|
# manage special additional parameter
|
67
|
-
if transfer_spec.
|
69
|
+
if transfer_spec.key?('EX_ssh_key_paths') && transfer_spec['EX_ssh_key_paths'].is_a?(Array) && !transfer_spec['EX_ssh_key_paths'].empty?
|
68
70
|
# not standard, so place standard field
|
69
|
-
if transfer_spec.
|
71
|
+
if transfer_spec.key?('ssh_private_key')
|
70
72
|
Log.log.warn('Both ssh_private_key and EX_ssh_key_paths are present, using ssh_private_key')
|
71
73
|
else
|
72
74
|
Log.log.warn('EX_ssh_key_paths has multiple keys, using first one only') unless transfer_spec['EX_ssh_key_paths'].length.eql?(1)
|
@@ -77,13 +79,13 @@ module Aspera
|
|
77
79
|
if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags']['aspera'].is_a?(Hash)
|
78
80
|
transfer_spec['tags']['aspera']['xfer_retry'] ||= 150
|
79
81
|
end
|
80
|
-
#
|
81
|
-
if transfer_spec['remote_host'].eql?(URI.parse(node_api_.params[:base_url]).host)
|
82
|
-
transfer_spec['remote_host'] = '
|
82
|
+
# Optimization in case of sending to the same node (TODO: probably remove this, as /etc/hosts shall be used for that)
|
83
|
+
if !transfer_spec['wss_enabled'] && transfer_spec['remote_host'].eql?(URI.parse(node_api_.params[:base_url]).host)
|
84
|
+
transfer_spec['remote_host'] = '127.0.0.1'
|
83
85
|
end
|
84
|
-
resp = node_api_.create('ops/transfers',transfer_spec)[:data]
|
86
|
+
resp = node_api_.create('ops/transfers', transfer_spec)[:data]
|
85
87
|
@transfer_id = resp['id']
|
86
|
-
Log.log.debug
|
88
|
+
Log.log.debug{"tr_id=#{@transfer_id}"}
|
87
89
|
return @transfer_id
|
88
90
|
end
|
89
91
|
|
@@ -94,35 +96,33 @@ module Aspera
|
|
94
96
|
# lets emulate management events to display progress bar
|
95
97
|
loop do
|
96
98
|
# status is empty sometimes with status 200...
|
97
|
-
|
98
|
-
case
|
99
|
+
transfer_data = node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {'status' => 'unknown'} rescue {'status' => 'waiting(read error)'}
|
100
|
+
case transfer_data['status']
|
99
101
|
when 'completed'
|
100
102
|
notify_end(@transfer_id)
|
101
103
|
break
|
102
|
-
when 'waiting','partially_completed','unknown','waiting(read error)'
|
104
|
+
when 'waiting', 'partially_completed', 'unknown', 'waiting(read error)'
|
103
105
|
if spinner.nil?
|
104
106
|
spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
|
105
107
|
spinner.start
|
106
108
|
end
|
107
|
-
spinner.update(title:
|
109
|
+
spinner.update(title: transfer_data['status'])
|
108
110
|
spinner.spin
|
109
|
-
#puts trdata
|
110
111
|
when 'running'
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
notify_begin(@transfer_id,trdata['precalc']['bytes_expected'])
|
112
|
+
if !started && transfer_data['precalc'].is_a?(Hash) &&
|
113
|
+
transfer_data['precalc']['status'].eql?('ready')
|
114
|
+
notify_begin(@transfer_id, transfer_data['precalc']['bytes_expected'])
|
115
115
|
started = true
|
116
116
|
else
|
117
|
-
notify_progress(@transfer_id,
|
117
|
+
notify_progress(@transfer_id, transfer_data['bytes_transferred'])
|
118
118
|
end
|
119
119
|
else
|
120
|
-
Log.log.warn
|
121
|
-
raise Fasp::Error, "#{
|
120
|
+
Log.log.warn{"transfer_data -> #{transfer_data}"}
|
121
|
+
raise Fasp::Error, "#{transfer_data['status']}: #{transfer_data['error_desc']}"
|
122
122
|
end
|
123
123
|
sleep(1)
|
124
124
|
end
|
125
|
-
#TODO get status of sessions
|
125
|
+
# TODO: get status of sessions
|
126
126
|
return []
|
127
127
|
end
|
128
128
|
end
|
@@ -6,7 +6,7 @@ require 'json'
|
|
6
6
|
|
7
7
|
module Aspera
|
8
8
|
module Fasp
|
9
|
-
class AgentTrsdk < AgentBase
|
9
|
+
class AgentTrsdk < Aspera::Fasp::AgentBase
|
10
10
|
DEFAULT_OPTIONS = {
|
11
11
|
address: '127.0.0.1',
|
12
12
|
port: 55_002
|
@@ -18,26 +18,26 @@ module Aspera
|
|
18
18
|
raise "expecting Hash (or nil), but have #{user_opts.class}" unless user_opts.nil? || user_opts.is_a?(Hash)
|
19
19
|
# set default options and override if specified
|
20
20
|
options = DEFAULT_OPTIONS.dup
|
21
|
-
user_opts&.each do |k,v|
|
22
|
-
raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.
|
21
|
+
user_opts&.each do |k, v|
|
22
|
+
raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.key?(k)
|
23
23
|
options[k] = v
|
24
24
|
end
|
25
|
-
Log.log.debug
|
25
|
+
Log.log.debug{"options= #{options}"}
|
26
26
|
super()
|
27
27
|
# load and create SDK stub
|
28
28
|
$LOAD_PATH.unshift(Installation.instance.sdk_ruby_folder)
|
29
29
|
require 'transfer_services_pb'
|
30
|
-
@transfer_client = Transfersdk::TransferService::Stub.new("#{options[:address]}:#{options[:port]}"
|
30
|
+
@transfer_client = Transfersdk::TransferService::Stub.new("#{options[:address]}:#{options[:port]}", :this_channel_is_insecure)
|
31
31
|
begin
|
32
32
|
get_info_response = @transfer_client.get_info(Transfersdk::InstanceInfoRequest.new)
|
33
|
-
Log.log.debug
|
33
|
+
Log.log.debug{"daemon info: #{get_info_response}"}
|
34
34
|
rescue GRPC::Unavailable
|
35
35
|
Log.log.warn('no daemon present, starting daemon...')
|
36
36
|
# location of daemon binary
|
37
|
-
bin_folder = File.realpath(File.join(Installation.instance.sdk_ruby_folder,'..'))
|
37
|
+
bin_folder = File.realpath(File.join(Installation.instance.sdk_ruby_folder, '..'))
|
38
38
|
# config file and logs are created in same folder
|
39
|
-
conf_file = File.join(bin_folder,'sdk.conf')
|
40
|
-
log_base = File.join(bin_folder,'transferd')
|
39
|
+
conf_file = File.join(bin_folder, 'sdk.conf')
|
40
|
+
log_base = File.join(bin_folder, 'transferd')
|
41
41
|
# create a config file for daemon
|
42
42
|
config = {
|
43
43
|
address: options[:address],
|
@@ -50,15 +50,15 @@ module Aspera
|
|
50
50
|
}
|
51
51
|
}
|
52
52
|
}
|
53
|
-
File.write(conf_file,config.to_json)
|
54
|
-
trd_pid = Process.spawn(Installation.instance.path(:transferd),'--config', conf_file, out: "#{log_base}.out", err: "#{log_base}.err")
|
53
|
+
File.write(conf_file, config.to_json)
|
54
|
+
trd_pid = Process.spawn(Installation.instance.path(:transferd), '--config', conf_file, out: "#{log_base}.out", err: "#{log_base}.err")
|
55
55
|
Process.detach(trd_pid)
|
56
56
|
sleep(2.0)
|
57
57
|
retry
|
58
58
|
end
|
59
59
|
end
|
60
60
|
|
61
|
-
def start_transfer(transfer_spec,
|
61
|
+
def start_transfer(transfer_spec, token_regenerator: nil)
|
62
62
|
# create a transfer request
|
63
63
|
transfer_request = Transfersdk::TransferRequest.new(
|
64
64
|
transferType: Transfersdk::TransferType::FILE_REGULAR, # transfer type (file/stream)
|
@@ -66,9 +66,9 @@ module Aspera
|
|
66
66
|
transferSpec: transfer_spec.to_json) # transfer definition
|
67
67
|
# send start transfer request to the transfer manager daemon
|
68
68
|
start_transfer_response = @transfer_client.start_transfer(transfer_request)
|
69
|
-
Log.log.debug
|
69
|
+
Log.log.debug{"start transfer response #{start_transfer_response}"}
|
70
70
|
@transfer_id = start_transfer_response.transferId
|
71
|
-
Log.log.debug
|
71
|
+
Log.log.debug{"transfer started with id #{@transfer_id}"}
|
72
72
|
end
|
73
73
|
|
74
74
|
def wait_for_transfers_completion
|
@@ -76,26 +76,26 @@ module Aspera
|
|
76
76
|
# monitor transfer status
|
77
77
|
@transfer_client.monitor_transfers(Transfersdk::RegistrationRequest.new(transferId: [@transfer_id])) do |response|
|
78
78
|
Log.dump(:response, response.to_h)
|
79
|
-
#Log.log.debug
|
79
|
+
# Log.log.debug{"#{response.sessionInfo.preTransferBytes} #{response.transferInfo.bytesTransferred}"}
|
80
80
|
case response.status
|
81
81
|
when :RUNNING
|
82
82
|
if !started && !response.sessionInfo.preTransferBytes.eql?(0)
|
83
|
-
notify_begin(@transfer_id,response.sessionInfo.preTransferBytes)
|
83
|
+
notify_begin(@transfer_id, response.sessionInfo.preTransferBytes)
|
84
84
|
started = true
|
85
85
|
elsif started
|
86
|
-
notify_progress(@transfer_id,response.transferInfo.bytesTransferred)
|
86
|
+
notify_progress(@transfer_id, response.transferInfo.bytesTransferred)
|
87
87
|
end
|
88
88
|
when :FAILED, :COMPLETED, :CANCELED
|
89
89
|
notify_end(@transfer_id)
|
90
90
|
raise Fasp::Error, JSON.parse(response.message)['Description'] unless :COMPLETED.eql?(response.status)
|
91
91
|
break
|
92
|
-
when :QUEUED
|
92
|
+
when :QUEUED, :UNKNOWN_STATUS, :PAUSED, :ORPHANED
|
93
93
|
# ignore
|
94
94
|
else
|
95
|
-
Log.log.error
|
95
|
+
Log.log.error{"unknown status#{response.status}"}
|
96
96
|
end
|
97
97
|
end
|
98
|
-
# TODO return status
|
98
|
+
# TODO: return status
|
99
99
|
return []
|
100
100
|
end
|
101
101
|
end
|
data/lib/aspera/fasp/error.rb
CHANGED
@@ -7,7 +7,8 @@ module Aspera
|
|
7
7
|
# error raised if transfer fails
|
8
8
|
class Error < StandardError
|
9
9
|
attr_reader :err_code
|
10
|
-
|
10
|
+
|
11
|
+
def initialize(message, err_code=nil)
|
11
12
|
super(message)
|
12
13
|
@err_code = err_code
|
13
14
|
end
|
@@ -17,7 +18,7 @@ module Aspera
|
|
17
18
|
return r.merge({i: @err_code})
|
18
19
|
end
|
19
20
|
|
20
|
-
def retryable?; info[:r];end
|
21
|
+
def retryable?; info[:r]; end
|
21
22
|
end
|
22
23
|
end
|
23
24
|
end
|
@@ -4,6 +4,8 @@ module Aspera
|
|
4
4
|
module Fasp
|
5
5
|
# from https://www.google.com/search?q=FASP+error+codes
|
6
6
|
# Note that the fact that an error is retryable is not internally defined by protocol, it's client-side responsibility
|
7
|
+
# rubocop:disable Layout/MultilineHashKeyLineBreaks
|
8
|
+
# rubocop:disable Layout/FirstHashElementLineBreak
|
7
9
|
ERROR_INFO = {
|
8
10
|
# id retryable mnemo message additional info
|
9
11
|
1 => { r: false, c: 'FASP_PROTO', m: 'Generic fasp(tm) protocol error', a: 'fasp(tm) error'},
|
@@ -18,13 +20,13 @@ module Aspera
|
|
18
20
|
10 => { r: false, c: 'LIC_RATE_EXCEEDED', m: 'Rate exceeds the cap imposed by license', a: 'Rate exceeds cap imposed by license'},
|
19
21
|
11 => { r: false, c: 'INTERNAL_ERROR', m: 'Internal error (unexpected error)', a: 'Internal error'},
|
20
22
|
12 => { r: true, c: 'TRANSFER_ERROR', m: 'Error establishing control connection',
|
21
|
-
|
23
|
+
a: 'Error establishing SSH connection (check SSH port and firewall)'},
|
22
24
|
13 => { r: true, c: 'TRANSFER_TIMEOUT', m: 'Timeout establishing control connection',
|
23
|
-
|
25
|
+
a: 'Timeout establishing SSH connection (check SSH port and firewall)'},
|
24
26
|
14 => { r: true, c: 'CONNECTION_ERROR', m: 'Error establishing data connection',
|
25
|
-
|
27
|
+
a: 'Error establishing UDP connection (check UDP port and firewall)'},
|
26
28
|
15 => { r: true, c: 'CONNECTION_TIMEOUT', m: 'Timeout establishing data connection',
|
27
|
-
|
29
|
+
a: 'Timeout establishing UDP connection (check UDP port and firewall)'},
|
28
30
|
16 => { r: true, c: 'CONNECTION_LOST', m: 'Connection lost', a: 'Connection lost'},
|
29
31
|
17 => { r: true, c: 'RCVR_SEND_ERROR', m: 'Receiver fails to send feedback', a: 'Network failure (receiver can\'t send feedback)'},
|
30
32
|
18 => { r: true, c: 'RCVR_RECV_ERROR', m: 'Receiver fails to receive data packets', a: 'Network failure (receiver can\'t receive UDP data)'},
|
@@ -68,20 +70,21 @@ module Aspera
|
|
68
70
|
54 => { r: false, c: 'THRESHOLD_VALIDATION_FAILED', m: 'File threshold validation failed', a: 'File threshold validation failed'},
|
69
71
|
55 => { r: false, c: 'FILEPATH_TOO_LONG', m: 'File path/name too long for underlying file system', a: 'File path exceeds underlying file system limit'},
|
70
72
|
56 => { r: false, c: 'ILLEGAL_CHARS_IN_PATH', m: 'Windows path contains illegal characters',
|
71
|
-
|
73
|
+
a: 'Path being written to Windows file system contains illegal characters'},
|
72
74
|
57 => { r: false, c: 'CHUNK_MUST_MATCH_ALIGNMENT', m: 'Chunk size/start must be aligned with storage', a: 'Chunk size/start must be aligned with storage'},
|
73
75
|
58 => { r: false, c: 'VALIDATION_SESSION_ABORT', m: 'Session aborted to due to validation error', a: 'Session aborted to due validation error'},
|
74
76
|
59 => { r: false, c: 'REMOTE_STORAGE_ERROR', m: 'Remote storage errored', a: 'Remote storage errored'},
|
75
77
|
60 => { r: false, c: 'LUA_SCRIPT_ABORTED_SESSION', m: 'Session aborted due to Lua script abort', a: 'Session aborted due to Lua script abort'},
|
76
78
|
61 => { r: true, c: 'SSEAR_RETRYABLE', m: 'Transfer failed because of a retryable Encryption at Rest error',
|
77
|
-
|
79
|
+
a: 'Transfer failed because of a retryable Encryption at Rest error'},
|
78
80
|
62 => { r: false, c: 'SSEAR_FATAL', m: 'Transfer failed because of a fatal Encryption at Rest error',
|
79
|
-
|
81
|
+
a: 'Transfer failed because of a fatal Encryption at Rest error'},
|
80
82
|
63 => { r: false, c: 'LINK_LOOP', m: 'Path refers to a symbolic link loop', a: 'Path refers to a symbolic link loop'},
|
81
83
|
64 => { r: false, c: 'CANNOT_RENAME_PARTIAL_FILES', m: 'Can\'t rename a partial file', a: 'Can\'t rename a partial file.'},
|
82
84
|
65 => { r: false, c: 'CIPHER_NON_COMPAT_FIPS', m: 'Can\'t use this cipher with FIPS mode enabled', a: 'Can\'t use this cipher with FIPS mode enabled'},
|
83
85
|
66 => { r: false, c: 'PEER_REQUIRES_FIPS', m: 'Peer rejects cipher due to FIPS mode enabled on peer',
|
84
|
-
|
86
|
+
a: 'Peer rejects cipher due to FIPS mode enabled on peer'}
|
85
87
|
}.freeze
|
88
|
+
# rubocop:enable Layout/MultilineHashKeyLineBreaks
|
86
89
|
end
|
87
90
|
end
|
@@ -42,9 +42,9 @@ module Aspera
|
|
42
42
|
</default>
|
43
43
|
</CONF>
|
44
44
|
END_OF_CONFIG_FILE
|
45
|
-
DUMMY_CERT_INFO='/C=US/ST=California/L=Emeryville/O=Aspera Inc./OU=Corporate/CN=Aspera Inc./emailAddress=info@asperasoft.com'
|
46
|
-
private_constant :PRODUCT_CONNECT
|
47
|
-
:ONE_YEAR_SECONDS
|
45
|
+
DUMMY_CERT_INFO = '/C=US/ST=California/L=Emeryville/O=Aspera Inc./OU=Corporate/CN=Aspera Inc./emailAddress=info@asperasoft.com'
|
46
|
+
private_constant :PRODUCT_CONNECT, :PRODUCT_CLI_V1, :PRODUCT_DRIVE, :PRODUCT_ENTSRV, :EXT_RUBY_PROTOBUF, :RB_SDK_FOLDER,
|
47
|
+
:ONE_YEAR_SECONDS, :DEFAULT_ASPERA_CONF, :DUMMY_CERT_INFO
|
48
48
|
# set ascp executable path
|
49
49
|
def ascp_path=(v)
|
50
50
|
@path_to_ascp = v
|
@@ -55,7 +55,7 @@ module Aspera
|
|
55
55
|
end
|
56
56
|
|
57
57
|
def sdk_ruby_folder
|
58
|
-
ruby_pb_folder = File.join(sdk_folder,RB_SDK_FOLDER)
|
58
|
+
ruby_pb_folder = File.join(sdk_folder, RB_SDK_FOLDER)
|
59
59
|
FileUtils.mkdir_p(ruby_pb_folder) unless Dir.exist?(ruby_pb_folder)
|
60
60
|
return ruby_pb_folder
|
61
61
|
end
|
@@ -87,7 +87,7 @@ module Aspera
|
|
87
87
|
raise "no such product installed: #{product_name}" if pl.nil?
|
88
88
|
end
|
89
89
|
self.ascp_path = pl[:ascp_path]
|
90
|
-
Log.log.debug
|
90
|
+
Log.log.debug{"ascp_path=#{@path_to_ascp}"}
|
91
91
|
end
|
92
92
|
|
93
93
|
# @return the list of installed products in format of product_locations
|
@@ -104,15 +104,15 @@ module Aspera
|
|
104
104
|
@found_products = scan_locations.select! do |item|
|
105
105
|
# skip if not main folder
|
106
106
|
next false unless Dir.exist?(item[:app_root])
|
107
|
-
Log.log.debug
|
107
|
+
Log.log.debug{"Found #{item[:app_root]}"}
|
108
108
|
sub_bin = item[:sub_bin] || BIN_SUBFOLDER
|
109
|
-
item[:ascp_path] = File.join(item[:app_root],sub_bin,ascp_filename)
|
109
|
+
item[:ascp_path] = File.join(item[:app_root], sub_bin, ascp_filename)
|
110
110
|
# skip if no ascp
|
111
111
|
next false unless File.exist?(item[:ascp_path])
|
112
112
|
# read info from product info file if present
|
113
113
|
product_info_file = "#{item[:app_root]}/#{PRODUCT_INFO}"
|
114
114
|
if File.exist?(product_info_file)
|
115
|
-
res_s = XmlSimple.xml_in(File.read(product_info_file),{'ForceArray' => false})
|
115
|
+
res_s = XmlSimple.xml_in(File.read(product_info_file), {'ForceArray' => false})
|
116
116
|
item[:name] = res_s['name']
|
117
117
|
item[:version] = res_s['version']
|
118
118
|
else
|
@@ -128,23 +128,23 @@ module Aspera
|
|
128
128
|
FILES = %i[ascp ascp4 ssh_bypass_key_dsa ssh_bypass_key_rsa aspera_license aspera_conf fallback_cert fallback_key].freeze
|
129
129
|
|
130
130
|
# get path of one resource file of currently activated product
|
131
|
-
# keys and certs are generated locally... (they are well known values, arch.
|
131
|
+
# keys and certs are generated locally... (they are well known values, arch. independent)
|
132
132
|
def path(k)
|
133
133
|
case k
|
134
|
-
when :ascp
|
134
|
+
when :ascp, :ascp4
|
135
135
|
use_ascp_from_product(FIRST_FOUND) if @path_to_ascp.nil?
|
136
136
|
file = @path_to_ascp
|
137
|
-
#
|
138
|
-
file = file.gsub('ascp','ascp4') if k.eql?(:ascp4)
|
137
|
+
# NOTE: that there might be a .exe at the end
|
138
|
+
file = file.gsub('ascp', 'ascp4') if k.eql?(:ascp4)
|
139
139
|
when :transferd
|
140
140
|
file = transferd_filepath
|
141
141
|
when :ssh_bypass_key_dsa
|
142
|
-
file=Environment.write_file_restricted(File.join(sdk_folder,'aspera_bypass_dsa.pem')) {get_key('dsa',1)}
|
142
|
+
file = Environment.write_file_restricted(File.join(sdk_folder, 'aspera_bypass_dsa.pem')) {get_key('dsa', 1)}
|
143
143
|
when :ssh_bypass_key_rsa
|
144
|
-
file=Environment.write_file_restricted(File.join(sdk_folder,'aspera_bypass_rsa.pem')) {get_key('rsa',2)}
|
144
|
+
file = Environment.write_file_restricted(File.join(sdk_folder, 'aspera_bypass_rsa.pem')) {get_key('rsa', 2)}
|
145
145
|
when :aspera_license
|
146
|
-
file=Environment.write_file_restricted(File.join(sdk_folder,'aspera-license')) do
|
147
|
-
clear=[
|
146
|
+
file = Environment.write_file_restricted(File.join(sdk_folder, 'aspera-license')) do
|
147
|
+
clear = [
|
148
148
|
Zlib::Inflate.inflate(DataRepository.instance.data(6)),
|
149
149
|
"==SIGNATURE==\n",
|
150
150
|
Base64.strict_encode64(DataRepository.instance.data(7))
|
@@ -152,10 +152,10 @@ module Aspera
|
|
152
152
|
Base64.strict_encode64(clear.join)
|
153
153
|
end
|
154
154
|
when :aspera_conf
|
155
|
-
file=Environment.write_file_restricted(File.join(sdk_folder,'aspera.conf')) {DEFAULT_ASPERA_CONF}
|
156
|
-
when :fallback_cert
|
157
|
-
file_key = File.join(sdk_folder,'aspera_fallback_key.pem')
|
158
|
-
file_cert = File.join(sdk_folder,'aspera_fallback_cert.pem')
|
155
|
+
file = Environment.write_file_restricted(File.join(sdk_folder, 'aspera.conf')) {DEFAULT_ASPERA_CONF}
|
156
|
+
when :fallback_cert, :fallback_key
|
157
|
+
file_key = File.join(sdk_folder, 'aspera_fallback_key.pem')
|
158
|
+
file_cert = File.join(sdk_folder, 'aspera_fallback_cert.pem')
|
159
159
|
if !File.exist?(file_key) || !File.exist?(file_cert)
|
160
160
|
require 'openssl'
|
161
161
|
# create new self signed certificate for http fallback
|
@@ -182,10 +182,10 @@ module Aspera
|
|
182
182
|
# @return the file path of local connect where API's URI can be read
|
183
183
|
def connect_uri
|
184
184
|
connect = get_product_folders(PRODUCT_CONNECT)
|
185
|
-
folder = File.join(connect[:run_root],
|
186
|
-
['','s'].each do |ext|
|
187
|
-
uri_file = File.join(folder,"http#{ext}.uri")
|
188
|
-
Log.log.debug
|
185
|
+
folder = File.join(connect[:run_root], VAR_RUN_SUBFOLDER)
|
186
|
+
['', 's'].each do |ext|
|
187
|
+
uri_file = File.join(folder, "http#{ext}.uri")
|
188
|
+
Log.log.debug{"checking connect port file: #{uri_file}"}
|
189
189
|
if File.exist?(uri_file)
|
190
190
|
return File.open(uri_file, &:gets).strip
|
191
191
|
end
|
@@ -196,12 +196,12 @@ module Aspera
|
|
196
196
|
# @ return path to configuration file of aspera CLI
|
197
197
|
def cli_conf_file
|
198
198
|
connect = get_product_folders(PRODUCT_CLI_V1)
|
199
|
-
return File.join(connect[:app_root],BIN_SUBFOLDER,'.aspera_cli_conf')
|
199
|
+
return File.join(connect[:app_root], BIN_SUBFOLDER, '.aspera_cli_conf')
|
200
200
|
end
|
201
201
|
|
202
202
|
# default bypass key phrase
|
203
203
|
def bypass_pass
|
204
|
-
return format('%08x-%04x-%04x-%04x-%04x%08x'
|
204
|
+
return format('%08x-%04x-%04x-%04x-%04x%08x', *DataRepository.instance.data(3).unpack('NnnnnN'))
|
205
205
|
end
|
206
206
|
|
207
207
|
def bypass_keys
|
@@ -210,11 +210,11 @@ module Aspera
|
|
210
210
|
|
211
211
|
# use in plugin `config`
|
212
212
|
def get_ascp_version(exe_path)
|
213
|
-
return get_exe_version(exe_path,'-A')
|
213
|
+
return get_exe_version(exe_path, '-A')
|
214
214
|
end
|
215
215
|
|
216
216
|
# Check that specified path is ascp and get version
|
217
|
-
def get_exe_version(exe_path,vers_arg)
|
217
|
+
def get_exe_version(exe_path, vers_arg)
|
218
218
|
raise 'ERROR: nil arg' if exe_path.nil?
|
219
219
|
return nil unless File.exist?(exe_path)
|
220
220
|
exe_version = nil
|
@@ -233,18 +233,18 @@ module Aspera
|
|
233
233
|
# SDK is organized by architecture, check this first, in case architecture is not supported
|
234
234
|
arch_filter = "#{Environment.architecture}/"
|
235
235
|
require 'zip'
|
236
|
-
sdk_zip_path = File.join(Dir.tmpdir,'sdk.zip')
|
236
|
+
sdk_zip_path = File.join(Dir.tmpdir, 'sdk.zip')
|
237
237
|
if sdk_url.start_with?('file:')
|
238
238
|
# require specific file scheme: the path part is "relative", or absolute if there are 4 slash
|
239
239
|
raise 'use format: file:///<path>' unless sdk_url.start_with?('file:///')
|
240
|
-
sdk_zip_path = sdk_url.gsub(%r{^file:///},'')
|
240
|
+
sdk_zip_path = sdk_url.gsub(%r{^file:///}, '')
|
241
241
|
else
|
242
|
-
Aspera::Rest.new(base_url: sdk_url, redirect_max: 3).call(operation: 'GET',save_to_file: sdk_zip_path)
|
242
|
+
Aspera::Rest.new(base_url: sdk_url, redirect_max: 3).call(operation: 'GET', save_to_file: sdk_zip_path)
|
243
243
|
end
|
244
244
|
# rename old install
|
245
245
|
if !Dir.empty?(sdk_folder)
|
246
246
|
Log.log.warn('Previous install exists, renaming folder.')
|
247
|
-
File.rename(sdk_folder,"#{sdk_folder}.#{Time.now.strftime('%Y%m%d%H%M%S')}")
|
247
|
+
File.rename(sdk_folder, "#{sdk_folder}.#{Time.now.strftime('%Y%m%d%H%M%S')}")
|
248
248
|
# TODO: delete old archives ?
|
249
249
|
end
|
250
250
|
# extract files from archive
|
@@ -258,26 +258,26 @@ module Aspera
|
|
258
258
|
# ruby adapters
|
259
259
|
dest_folder = sdk_ruby_folder if entry.name.end_with?(EXT_RUBY_PROTOBUF)
|
260
260
|
next if dest_folder.nil?
|
261
|
-
File.open(File.join(dest_folder,File.basename(entry.name)), 'wb') do |output_stream|
|
261
|
+
File.open(File.join(dest_folder, File.basename(entry.name)), 'wb') do |output_stream|
|
262
262
|
IO.copy_stream(entry.get_input_stream, output_stream)
|
263
263
|
end
|
264
264
|
end
|
265
265
|
end
|
266
266
|
File.unlink(sdk_zip_path) rescue nil # Windows may give error
|
267
|
-
# ensure license file are generated so that ascp
|
267
|
+
# ensure license file are generated so that ascp invocation for version works
|
268
268
|
path(:aspera_license)
|
269
269
|
path(:aspera_conf)
|
270
|
-
ascp_path = File.join(sdk_folder,ascp_filename)
|
270
|
+
ascp_path = File.join(sdk_folder, ascp_filename)
|
271
271
|
raise "No #{ascp_filename} found in SDK archive" unless File.exist?(ascp_path)
|
272
|
-
Environment.restrict_file_access(ascp_path, mode:
|
273
|
-
Environment.restrict_file_access(ascp_path.gsub('ascp','ascp4'), mode:
|
274
|
-
ascp_version = get_ascp_version(File.join(sdk_folder,ascp_filename))
|
272
|
+
Environment.restrict_file_access(ascp_path, mode: 0o755)
|
273
|
+
Environment.restrict_file_access(ascp_path.gsub('ascp', 'ascp4'), mode: 0o755)
|
274
|
+
ascp_version = get_ascp_version(File.join(sdk_folder, ascp_filename))
|
275
275
|
trd_path = transferd_filepath
|
276
|
-
Log.log.warn
|
277
|
-
Environment.restrict_file_access(trd_path, mode:
|
278
|
-
transferd_version = get_exe_version(trd_path,'version')
|
276
|
+
Log.log.warn{"No #{trd_path} in SDK archive"} unless File.exist?(trd_path)
|
277
|
+
Environment.restrict_file_access(trd_path, mode: 0o755) if File.exist?(trd_path)
|
278
|
+
transferd_version = get_exe_version(trd_path, 'version')
|
279
279
|
sdk_version = transferd_version || ascp_version
|
280
|
-
File.write(File.join(sdk_folder,PRODUCT_INFO),"<product><name>IBM Aspera SDK</name><version>#{sdk_version}</version></product>")
|
280
|
+
File.write(File.join(sdk_folder, PRODUCT_INFO), "<product><name>IBM Aspera SDK</name><version>#{sdk_version}</version></product>")
|
281
281
|
return sdk_version
|
282
282
|
end
|
283
283
|
|
@@ -285,13 +285,13 @@ module Aspera
|
|
285
285
|
|
286
286
|
BIN_SUBFOLDER = 'bin'
|
287
287
|
ETC_SUBFOLDER = 'etc'
|
288
|
-
|
288
|
+
VAR_RUN_SUBFOLDER = File.join('var', 'run')
|
289
289
|
# product information manifest: XML (part of aspera product)
|
290
290
|
PRODUCT_INFO = 'product-info.mf'
|
291
291
|
# policy for product selection
|
292
292
|
FIRST_FOUND = 'FIRST'
|
293
293
|
|
294
|
-
private_constant :BIN_SUBFOLDER
|
294
|
+
private_constant :BIN_SUBFOLDER, :ETC_SUBFOLDER, :VAR_RUN_SUBFOLDER, :PRODUCT_INFO
|
295
295
|
|
296
296
|
def initialize
|
297
297
|
@path_to_ascp = nil
|
@@ -313,7 +313,7 @@ module Aspera
|
|
313
313
|
end
|
314
314
|
|
315
315
|
def transferd_filepath
|
316
|
-
return File.join(sdk_folder,'asperatransferd' + Environment.exe_extension)
|
316
|
+
return File.join(sdk_folder, 'asperatransferd' + Environment.exe_extension)
|
317
317
|
end
|
318
318
|
|
319
319
|
# @return product folders depending on OS fields
|
@@ -326,54 +326,54 @@ module Aspera
|
|
326
326
|
case Aspera::Environment.os
|
327
327
|
when Aspera::Environment::OS_WINDOWS; return [{
|
328
328
|
expected: PRODUCT_CONNECT,
|
329
|
-
app_root: File.join(ENV['LOCALAPPDATA'],'Programs','Aspera','Aspera Connect'),
|
330
|
-
log_root: File.join(ENV['LOCALAPPDATA'],'Aspera','Aspera Connect','var','log'),
|
331
|
-
run_root: File.join(ENV['LOCALAPPDATA'],'Aspera','Aspera Connect')
|
332
|
-
},{
|
329
|
+
app_root: File.join(ENV['LOCALAPPDATA'], 'Programs', 'Aspera', 'Aspera Connect'),
|
330
|
+
log_root: File.join(ENV['LOCALAPPDATA'], 'Aspera', 'Aspera Connect', 'var', 'log'),
|
331
|
+
run_root: File.join(ENV['LOCALAPPDATA'], 'Aspera', 'Aspera Connect')
|
332
|
+
}, {
|
333
333
|
expected: PRODUCT_CLI_V1,
|
334
|
-
app_root: File.join('C:','Program Files','Aspera','cli'),
|
335
|
-
log_root: File.join('C:','Program Files','Aspera','cli','var','log')
|
336
|
-
},{
|
334
|
+
app_root: File.join('C:', 'Program Files', 'Aspera', 'cli'),
|
335
|
+
log_root: File.join('C:', 'Program Files', 'Aspera', 'cli', 'var', 'log')
|
336
|
+
}, {
|
337
337
|
expected: PRODUCT_ENTSRV,
|
338
|
-
app_root: File.join('C:','Program Files','Aspera','Enterprise Server'),
|
339
|
-
log_root: File.join('C:','Program Files','Aspera','Enterprise Server','var','log')
|
338
|
+
app_root: File.join('C:', 'Program Files', 'Aspera', 'Enterprise Server'),
|
339
|
+
log_root: File.join('C:', 'Program Files', 'Aspera', 'Enterprise Server', 'var', 'log')
|
340
340
|
}]
|
341
341
|
when Aspera::Environment::OS_X; return [{
|
342
342
|
expected: PRODUCT_CONNECT,
|
343
|
-
app_root: File.join(Dir.home,'Applications','Aspera Connect.app'),
|
344
|
-
log_root: File.join(Dir.home,'Library','Logs','Aspera_Connect'),
|
345
|
-
run_root: File.join(Dir.home,'Library','Application Support','Aspera','Aspera Connect'),
|
346
|
-
sub_bin: File.join('Contents','Resources')
|
347
|
-
},{
|
343
|
+
app_root: File.join(Dir.home, 'Applications', 'Aspera Connect.app'),
|
344
|
+
log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera_Connect'),
|
345
|
+
run_root: File.join(Dir.home, 'Library', 'Application Support', 'Aspera', 'Aspera Connect'),
|
346
|
+
sub_bin: File.join('Contents', 'Resources')
|
347
|
+
}, {
|
348
348
|
expected: PRODUCT_CONNECT,
|
349
|
-
app_root: File.join('','Applications','Aspera Connect.app'),
|
350
|
-
log_root: File.join(Dir.home,'Library','Logs','Aspera_Connect'),
|
351
|
-
run_root: File.join(Dir.home,'Library','Application Support','Aspera','Aspera Connect'),
|
352
|
-
sub_bin: File.join('Contents','Resources')
|
353
|
-
},{
|
349
|
+
app_root: File.join('', 'Applications', 'Aspera Connect.app'),
|
350
|
+
log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera_Connect'),
|
351
|
+
run_root: File.join(Dir.home, 'Library', 'Application Support', 'Aspera', 'Aspera Connect'),
|
352
|
+
sub_bin: File.join('Contents', 'Resources')
|
353
|
+
}, {
|
354
354
|
expected: PRODUCT_CLI_V1,
|
355
|
-
app_root: File.join(Dir.home,'Applications','Aspera CLI'),
|
356
|
-
log_root: File.join(Dir.home,'Library','Logs','Aspera')
|
357
|
-
},{
|
355
|
+
app_root: File.join(Dir.home, 'Applications', 'Aspera CLI'),
|
356
|
+
log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera')
|
357
|
+
}, {
|
358
358
|
expected: PRODUCT_ENTSRV,
|
359
|
-
app_root: File.join('','Library','Aspera'),
|
360
|
-
log_root: File.join(Dir.home,'Library','Logs','Aspera')
|
361
|
-
},{
|
359
|
+
app_root: File.join('', 'Library', 'Aspera'),
|
360
|
+
log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera')
|
361
|
+
}, {
|
362
362
|
expected: PRODUCT_DRIVE,
|
363
|
-
app_root: File.join('','Applications','Aspera Drive.app'),
|
364
|
-
log_root: File.join(Dir.home,'Library','Logs','Aspera_Drive'),
|
365
|
-
sub_bin: File.join('Contents','Resources')
|
363
|
+
app_root: File.join('', 'Applications', 'Aspera Drive.app'),
|
364
|
+
log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera_Drive'),
|
365
|
+
sub_bin: File.join('Contents', 'Resources')
|
366
366
|
}]
|
367
367
|
else; return [{ # other: Linux and Unix family
|
368
368
|
expected: PRODUCT_CONNECT,
|
369
|
-
app_root: File.join(Dir.home,'.aspera','connect'),
|
370
|
-
run_root: File.join(Dir.home,'.aspera','connect')
|
371
|
-
},{
|
369
|
+
app_root: File.join(Dir.home, '.aspera', 'connect'),
|
370
|
+
run_root: File.join(Dir.home, '.aspera', 'connect')
|
371
|
+
}, {
|
372
372
|
expected: PRODUCT_CLI_V1,
|
373
|
-
app_root: File.join(Dir.home,'.aspera','cli')
|
374
|
-
},{
|
373
|
+
app_root: File.join(Dir.home, '.aspera', 'cli')
|
374
|
+
}, {
|
375
375
|
expected: PRODUCT_ENTSRV,
|
376
|
-
app_root: File.join('','opt','aspera')
|
376
|
+
app_root: File.join('', 'opt', 'aspera')
|
377
377
|
}]
|
378
378
|
end
|
379
379
|
end
|
@@ -381,7 +381,7 @@ module Aspera
|
|
381
381
|
# @return a standard bypass key
|
382
382
|
# @param type rsa or dsa
|
383
383
|
# @param id in repository 1 for dsa, 2 for rsa
|
384
|
-
def get_key(type,id)
|
384
|
+
def get_key(type, id)
|
385
385
|
# generate PEM from DER
|
386
386
|
OpenSSL::PKey.const_get(type.upcase).new(DataRepository.instance.data(id)).to_pem
|
387
387
|
end
|