aspera-cli 4.10.0 → 4.11.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +20 -0
- data/CHANGELOG.md +509 -0
- data/CONTRIBUTING.md +118 -0
- data/README.md +621 -378
- data/bin/ascli +4 -4
- data/bin/asession +11 -11
- data/docs/test_env.conf +28 -19
- data/examples/aoc.rb +4 -4
- data/examples/dascli +11 -9
- data/examples/faspex4.rb +8 -8
- data/examples/node.rb +11 -11
- data/examples/server.rb +9 -9
- data/lib/aspera/aoc.rb +273 -266
- data/lib/aspera/ascmd.rb +56 -54
- data/lib/aspera/ats_api.rb +4 -4
- data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
- data/lib/aspera/cli/extended_value.rb +5 -5
- data/lib/aspera/cli/formater.rb +64 -64
- data/lib/aspera/cli/listener/line_dump.rb +1 -1
- data/lib/aspera/cli/listener/logger.rb +1 -1
- data/lib/aspera/cli/listener/progress.rb +5 -6
- data/lib/aspera/cli/listener/progress_multi.rb +14 -19
- data/lib/aspera/cli/main.rb +66 -67
- data/lib/aspera/cli/manager.rb +110 -110
- data/lib/aspera/cli/plugin.rb +54 -37
- data/lib/aspera/cli/plugins/alee.rb +4 -4
- data/lib/aspera/cli/plugins/aoc.rb +308 -669
- data/lib/aspera/cli/plugins/ats.rb +44 -46
- data/lib/aspera/cli/plugins/bss.rb +10 -10
- data/lib/aspera/cli/plugins/config.rb +447 -344
- data/lib/aspera/cli/plugins/console.rb +12 -12
- data/lib/aspera/cli/plugins/cos.rb +18 -20
- data/lib/aspera/cli/plugins/faspex.rb +110 -112
- data/lib/aspera/cli/plugins/faspex5.rb +67 -46
- data/lib/aspera/cli/plugins/node.rb +364 -288
- data/lib/aspera/cli/plugins/orchestrator.rb +46 -46
- data/lib/aspera/cli/plugins/preview.rb +122 -114
- data/lib/aspera/cli/plugins/server.rb +137 -83
- data/lib/aspera/cli/plugins/shares.rb +30 -29
- data/lib/aspera/cli/plugins/sync.rb +13 -33
- data/lib/aspera/cli/transfer_agent.rb +57 -57
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +3 -3
- data/lib/aspera/command_line_builder.rb +27 -27
- data/lib/aspera/cos_node.rb +22 -20
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +30 -28
- data/lib/aspera/fasp/agent_base.rb +15 -15
- data/lib/aspera/fasp/agent_connect.rb +23 -21
- data/lib/aspera/fasp/agent_direct.rb +65 -67
- data/lib/aspera/fasp/agent_httpgw.rb +72 -68
- data/lib/aspera/fasp/agent_node.rb +23 -21
- data/lib/aspera/fasp/agent_trsdk.rb +20 -20
- data/lib/aspera/fasp/error.rb +3 -2
- data/lib/aspera/fasp/error_info.rb +11 -8
- data/lib/aspera/fasp/installation.rb +78 -78
- data/lib/aspera/fasp/listener.rb +1 -1
- data/lib/aspera/fasp/parameters.rb +75 -72
- data/lib/aspera/fasp/parameters.yaml +2 -2
- data/lib/aspera/fasp/resume_policy.rb +8 -8
- data/lib/aspera/fasp/transfer_spec.rb +35 -2
- data/lib/aspera/fasp/uri.rb +7 -7
- data/lib/aspera/faspex_gw.rb +7 -5
- data/lib/aspera/hash_ext.rb +3 -3
- data/lib/aspera/id_generator.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +23 -28
- data/lib/aspera/keychain/macos_security.rb +21 -20
- data/lib/aspera/log.rb +7 -7
- data/lib/aspera/nagios.rb +19 -18
- data/lib/aspera/node.rb +209 -35
- data/lib/aspera/oauth.rb +37 -36
- data/lib/aspera/open_application.rb +19 -11
- data/lib/aspera/persistency_action_once.rb +4 -4
- data/lib/aspera/persistency_folder.rb +13 -13
- data/lib/aspera/preview/file_types.rb +8 -8
- data/lib/aspera/preview/generator.rb +67 -67
- data/lib/aspera/preview/utils.rb +27 -27
- data/lib/aspera/proxy_auto_config.js +41 -41
- data/lib/aspera/proxy_auto_config.rb +16 -16
- data/lib/aspera/rest.rb +56 -60
- data/lib/aspera/rest_call_error.rb +2 -1
- data/lib/aspera/rest_error_analyzer.rb +18 -17
- data/lib/aspera/rest_errors_aspera.rb +16 -16
- data/lib/aspera/secret_hider.rb +15 -13
- data/lib/aspera/ssh.rb +11 -10
- data/lib/aspera/sync.rb +158 -44
- data/lib/aspera/temp_file_manager.rb +2 -2
- data/lib/aspera/uri_reader.rb +4 -4
- data/lib/aspera/web_auth.rb +14 -13
- data.tar.gz.sig +0 -0
- metadata +8 -5
- metadata.gz.sig +0 -0
@@ -14,7 +14,7 @@ require 'json'
|
|
14
14
|
module Aspera
|
15
15
|
module Fasp
|
16
16
|
# start a transfer using Aspera HTTP Gateway, using web socket session for uploads
|
17
|
-
class AgentHttpgw < AgentBase
|
17
|
+
class AgentHttpgw < Aspera::Fasp::AgentBase
|
18
18
|
# message returned by HTTP GW in case of success
|
19
19
|
MSG_END_UPLOAD = 'end upload'
|
20
20
|
MSG_END_SLICE = 'end_slice_upload'
|
@@ -23,15 +23,15 @@ module Aspera
|
|
23
23
|
upload_chunksize: 64_000,
|
24
24
|
upload_bar_refresh_sec: 0.5
|
25
25
|
}.freeze
|
26
|
-
DEFAULT_BASE_PATH='/aspera/http-gwy'
|
26
|
+
DEFAULT_BASE_PATH = '/aspera/http-gwy'
|
27
27
|
# upload endpoints
|
28
|
-
V1_UPLOAD='/v1/upload'
|
29
|
-
V2_UPLOAD='/v2/upload'
|
30
|
-
private_constant :DEFAULT_OPTIONS
|
28
|
+
V1_UPLOAD = '/v1/upload'
|
29
|
+
V2_UPLOAD = '/v2/upload'
|
30
|
+
private_constant :DEFAULT_OPTIONS, :MSG_END_UPLOAD, :MSG_END_SLICE, :V1_UPLOAD, :V2_UPLOAD
|
31
31
|
|
32
32
|
# send message on http gw web socket
|
33
33
|
def ws_snd_json(data)
|
34
|
-
@slice_uploads += 1 if data.
|
34
|
+
@slice_uploads += 1 if data.key?(:slice_upload)
|
35
35
|
Log.log.debug{JSON.generate(data)}
|
36
36
|
ws_send(JSON.generate(data))
|
37
37
|
end
|
@@ -47,7 +47,7 @@ module Aspera
|
|
47
47
|
# we need to keep track of actual file path because transfer spec is modified to be sent in web socket
|
48
48
|
source_paths = []
|
49
49
|
# get source root or nil
|
50
|
-
source_root = transfer_spec.
|
50
|
+
source_root = transfer_spec.key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] : nil
|
51
51
|
# source root is ignored by GW, used only here
|
52
52
|
transfer_spec.delete('source_root')
|
53
53
|
# compute total size of files to upload (for progress)
|
@@ -56,7 +56,7 @@ module Aspera
|
|
56
56
|
# save actual file location to be able read contents later
|
57
57
|
full_src_filepath = item['source']
|
58
58
|
# add source root if needed
|
59
|
-
full_src_filepath = File.join(source_root,full_src_filepath) unless source_root.nil?
|
59
|
+
full_src_filepath = File.join(source_root, full_src_filepath) unless source_root.nil?
|
60
60
|
# GW expects a simple file name in 'source' but if user wants to change the name, we take it
|
61
61
|
item['source'] = File.basename(item['destination'].nil? ? item['source'] : item['destination'])
|
62
62
|
item['file_size'] = File.size(full_src_filepath)
|
@@ -66,18 +66,18 @@ module Aspera
|
|
66
66
|
end
|
67
67
|
# identify this session uniquely
|
68
68
|
session_id = SecureRandom.uuid
|
69
|
-
@slice_uploads=0
|
69
|
+
@slice_uploads = 0
|
70
70
|
# web socket endpoint: by default use v2 (newer gateways), without base64 encoding
|
71
71
|
upload_api_version = V2_UPLOAD
|
72
72
|
# is the latest supported? else revert to old api
|
73
|
-
upload_api_version=V1_UPLOAD unless @api_info['endpoints'].any?{|i|i.include?(upload_api_version)}
|
73
|
+
upload_api_version = V1_UPLOAD unless @api_info['endpoints'].any?{|i|i.include?(upload_api_version)}
|
74
74
|
Log.log.debug{"api version: #{upload_api_version}"}
|
75
|
-
url=File.join(@gw_api.params[:base_url],upload_api_version)
|
76
|
-
#uri = URI.parse(url)
|
75
|
+
url = File.join(@gw_api.params[:base_url], upload_api_version)
|
76
|
+
# uri = URI.parse(url)
|
77
77
|
# open web socket to end point (equivalent to Net::HTTP.start)
|
78
78
|
http_socket = Rest.start_http_session(url)
|
79
79
|
@ws_io = http_socket.instance_variable_get(:@socket)
|
80
|
-
|
80
|
+
# @ws_io.debug_output = Log.log
|
81
81
|
@ws_handshake = ::WebSocket::Handshake::Client.new(url: url, headers: {})
|
82
82
|
@ws_io.write(@ws_handshake.to_s)
|
83
83
|
sleep(0.1)
|
@@ -85,21 +85,21 @@ module Aspera
|
|
85
85
|
raise 'Error in websocket handshake' unless @ws_handshake.finished?
|
86
86
|
Log.log.debug('ws: handshake success')
|
87
87
|
# data shared between main thread and read thread
|
88
|
-
shared_info={
|
88
|
+
shared_info = {
|
89
89
|
read_exception: nil, # error message if any in callback
|
90
90
|
end_uploads: 0 # number of files totally sent
|
91
|
-
#mutex: Mutex.new
|
92
|
-
#cond_var: ConditionVariable.new
|
91
|
+
# mutex: Mutex.new
|
92
|
+
# cond_var: ConditionVariable.new
|
93
93
|
}
|
94
94
|
# start read thread
|
95
95
|
ws_read_thread = Thread.new do
|
96
96
|
Log.log.debug('ws: thread: started')
|
97
97
|
frame = ::WebSocket::Frame::Incoming::Client.new
|
98
98
|
loop do
|
99
|
-
begin
|
99
|
+
begin # rubocop:disable Style/RedundantBegin
|
100
100
|
frame << @ws_io.readuntil("\n")
|
101
101
|
while (msg = frame.next)
|
102
|
-
Log.log.debug
|
102
|
+
Log.log.debug{"ws: thread: message: #{msg.data} #{shared_info[:end_uploads]}"}
|
103
103
|
message = msg.data
|
104
104
|
if message.eql?(MSG_END_UPLOAD)
|
105
105
|
shared_info[:end_uploads] += 1
|
@@ -122,12 +122,12 @@ module Aspera
|
|
122
122
|
break
|
123
123
|
end
|
124
124
|
end
|
125
|
-
Log.log.debug
|
125
|
+
Log.log.debug{"ws: thread: stopping (exc=#{shared_info[:read_exception]},cls=#{shared_info[:read_exception].class})"}
|
126
126
|
end
|
127
127
|
# notify progress bar
|
128
|
-
notify_begin(session_id,total_size)
|
128
|
+
notify_begin(session_id, total_size)
|
129
129
|
# first step send transfer spec
|
130
|
-
Log.dump(:ws_spec,transfer_spec)
|
130
|
+
Log.dump(:ws_spec, transfer_spec)
|
131
131
|
ws_snd_json(transfer_spec: transfer_spec)
|
132
132
|
# current file index
|
133
133
|
file_index = 0
|
@@ -135,58 +135,63 @@ module Aspera
|
|
135
135
|
sent_bytes = 0
|
136
136
|
# last progress event
|
137
137
|
last_progress_time = nil
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
138
|
+
|
139
|
+
transfer_spec['paths'].each do |item|
|
140
|
+
# TODO: get mime type?
|
141
|
+
file_mime_type = ''
|
142
|
+
file_size = item['file_size']
|
143
|
+
file_name = File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
|
144
|
+
# compute total number of slices
|
145
|
+
numslices = ((file_size - 1) / @options[:upload_chunksize]) + 1
|
146
|
+
File.open(source_paths[file_index]) do |file|
|
147
|
+
# current slice index
|
148
|
+
slicenum = 0
|
149
|
+
until file.eof?
|
150
|
+
data = file.read(@options[:upload_chunksize])
|
151
|
+
slice_data = {
|
152
|
+
name: file_name,
|
153
|
+
type: file_mime_type,
|
154
|
+
size: file_size,
|
155
|
+
slice: slicenum,
|
156
|
+
total_slices: numslices,
|
157
|
+
fileIndex: file_index
|
158
|
+
}
|
159
|
+
# Log.dump(:slice_data,slice_data) #if slicenum.eql?(0)
|
160
|
+
# interrupt main thread if read thread failed
|
161
|
+
raise shared_info[:read_exception] unless shared_info[:read_exception].nil?
|
162
|
+
begin
|
162
163
|
if upload_api_version.eql?(V1_UPLOAD)
|
163
164
|
slice_data[:data] = Base64.strict_encode64(data)
|
164
165
|
ws_snd_json(slice_upload: slice_data)
|
165
166
|
else
|
166
167
|
ws_snd_json(slice_upload: slice_data) if slicenum.eql?(0)
|
167
|
-
ws_send(data,type: :binary)
|
168
|
+
ws_send(data, type: :binary)
|
168
169
|
Log.log.debug{"ws: sent buffer: #{file_index} / #{slicenum}"}
|
169
|
-
ws_snd_json(slice_upload: slice_data) if slicenum.eql?(numslices-1)
|
170
|
-
end
|
171
|
-
sent_bytes += data.length
|
172
|
-
currenttime = Time.now
|
173
|
-
if last_progress_time.nil? || ((currenttime - last_progress_time) > @options[:upload_bar_refresh_sec])
|
174
|
-
notify_progress(session_id,sent_bytes)
|
175
|
-
last_progress_time = currenttime
|
170
|
+
ws_snd_json(slice_upload: slice_data) if slicenum.eql?(numslices - 1)
|
176
171
|
end
|
177
|
-
|
172
|
+
rescue Errno::EPIPE => e
|
173
|
+
raise shared_info[:read_exception] unless shared_info[:read_exception].nil?
|
174
|
+
raise e
|
175
|
+
end
|
176
|
+
sent_bytes += data.length
|
177
|
+
currenttime = Time.now
|
178
|
+
if last_progress_time.nil? || ((currenttime - last_progress_time) > @options[:upload_bar_refresh_sec])
|
179
|
+
notify_progress(session_id, sent_bytes)
|
180
|
+
last_progress_time = currenttime
|
178
181
|
end
|
182
|
+
slicenum += 1
|
179
183
|
end
|
180
|
-
file_index += 1
|
181
184
|
end
|
185
|
+
file_index += 1
|
182
186
|
end
|
187
|
+
|
183
188
|
Log.log.debug('Finished upload')
|
184
189
|
ws_read_thread.join
|
185
190
|
Log.log.debug{"result: #{shared_info[:end_uploads]} / #{@slice_uploads}"}
|
186
191
|
ws_send(nil, type: :close) unless @ws_io.nil?
|
187
192
|
@ws_io = nil
|
188
193
|
http_socket&.finish
|
189
|
-
notify_progress(session_id,sent_bytes)
|
194
|
+
notify_progress(session_id, sent_bytes)
|
190
195
|
notify_end(session_id)
|
191
196
|
end
|
192
197
|
|
@@ -194,18 +199,18 @@ module Aspera
|
|
194
199
|
transfer_spec['zip_required'] ||= false
|
195
200
|
transfer_spec['source_root'] ||= '/'
|
196
201
|
# is normally provided by application, like package name
|
197
|
-
if !transfer_spec.
|
202
|
+
if !transfer_spec.key?('download_name')
|
198
203
|
# by default it is the name of first file
|
199
204
|
dname = File.basename(transfer_spec['paths'].first['source'])
|
200
205
|
# we remove extension
|
201
|
-
dname = dname.gsub(/\.@gw_api.*$/,'')
|
206
|
+
dname = dname.gsub(/\.@gw_api.*$/, '')
|
202
207
|
# ands add indication of number of files if there is more than one
|
203
208
|
if transfer_spec['paths'].length > 1
|
204
209
|
dname += " #{transfer_spec['paths'].length} Files"
|
205
210
|
end
|
206
211
|
transfer_spec['download_name'] = dname
|
207
212
|
end
|
208
|
-
creation = @gw_api.create('v1/download',{'transfer_spec' => transfer_spec})[:data]
|
213
|
+
creation = @gw_api.create('v1/download', {'transfer_spec' => transfer_spec})[:data]
|
209
214
|
transfer_uuid = creation['url'].split('/').last
|
210
215
|
file_dest =
|
211
216
|
if transfer_spec['zip_required'] || transfer_spec['paths'].length > 1
|
@@ -215,19 +220,18 @@ module Aspera
|
|
215
220
|
# it is a plain file if we don't require zip and there is only one file
|
216
221
|
File.basename(transfer_spec['paths'].first['source'])
|
217
222
|
end
|
218
|
-
file_dest = File.join(transfer_spec['destination_root'],file_dest)
|
219
|
-
@gw_api.call({operation: 'GET',subpath: "v1/download/#{transfer_uuid}",save_to_file: file_dest})
|
223
|
+
file_dest = File.join(transfer_spec['destination_root'], file_dest)
|
224
|
+
@gw_api.call({operation: 'GET', subpath: "v1/download/#{transfer_uuid}", save_to_file: file_dest})
|
220
225
|
end
|
221
226
|
|
222
227
|
# start FASP transfer based on transfer spec (hash table)
|
223
228
|
# note that it is asynchronous
|
224
229
|
# HTTP download only supports file list
|
225
|
-
def start_transfer(transfer_spec
|
230
|
+
def start_transfer(transfer_spec)
|
226
231
|
raise 'GW URL must be set' if @gw_api.nil?
|
227
|
-
raise 'option: must be hash (or nil)' unless options.is_a?(Hash)
|
228
232
|
raise 'paths: must be Array' unless transfer_spec['paths'].is_a?(Array)
|
229
233
|
raise 'only token based transfer is supported in GW' unless transfer_spec['token'].is_a?(String)
|
230
|
-
Log.dump(:user_spec,transfer_spec)
|
234
|
+
Log.dump(:user_spec, transfer_spec)
|
231
235
|
transfer_spec['authentication'] ||= 'token'
|
232
236
|
case transfer_spec['direction']
|
233
237
|
when Fasp::TransferSpec::DIRECTION_SEND
|
@@ -253,17 +257,17 @@ module Aspera
|
|
253
257
|
private
|
254
258
|
|
255
259
|
def initialize(opts)
|
256
|
-
Log.log.debug
|
260
|
+
Log.log.debug{"local options= #{opts}"}
|
257
261
|
# set default options and override if specified
|
258
262
|
@options = DEFAULT_OPTIONS.dup
|
259
263
|
raise "httpgw agent parameters (transfer_info): expecting Hash, but have #{opts.class}" unless opts.is_a?(Hash)
|
260
|
-
opts.symbolize_keys.each do |k,v|
|
261
|
-
raise "httpgw agent parameter: Unknown: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.
|
264
|
+
opts.symbolize_keys.each do |k, v|
|
265
|
+
raise "httpgw agent parameter: Unknown: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.key?(k)
|
262
266
|
@options[k] = v
|
263
267
|
end
|
264
268
|
raise 'missing param: url' if @options[:url].nil?
|
265
269
|
# remove /v1 from end
|
266
|
-
@options[:url].gsub(%r{/v1/*$},'')
|
270
|
+
@options[:url].gsub(%r{/v1/*$}, '')
|
267
271
|
super()
|
268
272
|
@gw_api = Rest.new({base_url: @options[:url]})
|
269
273
|
@api_info = @gw_api.read('v1/info')[:data]
|
@@ -2,6 +2,7 @@
|
|
2
2
|
|
3
3
|
require 'aspera/fasp/agent_base'
|
4
4
|
require 'aspera/fasp/transfer_spec'
|
5
|
+
require 'aspera/node'
|
5
6
|
require 'aspera/log'
|
6
7
|
require 'tty-spinner'
|
7
8
|
|
@@ -9,20 +10,21 @@ module Aspera
|
|
9
10
|
module Fasp
|
10
11
|
# this singleton class is used by the CLI to provide a common interface to start a transfer
|
11
12
|
# before using it, the use must set the `node_api` member.
|
12
|
-
class AgentNode < AgentBase
|
13
|
+
class AgentNode < Aspera::Fasp::AgentBase
|
13
14
|
# option include: root_id if the node is an access key
|
14
15
|
attr_writer :options
|
16
|
+
|
15
17
|
def initialize(options)
|
16
18
|
raise 'node specification must be Hash' unless options.is_a?(Hash)
|
17
|
-
%i[url username password].each { |k| raise "missing parameter [#{k}] in node specification: #{options}" unless options.
|
19
|
+
%i[url username password].each { |k| raise "missing parameter [#{k}] in node specification: #{options}" unless options.key?(k) }
|
18
20
|
super()
|
19
21
|
# root id is required for access key
|
20
22
|
@root_id = options[:root_id]
|
21
23
|
rest_params = { base_url: options[:url]}
|
22
24
|
if /^Bearer /.match?(options[:password])
|
23
25
|
rest_params[:headers] = {
|
24
|
-
|
25
|
-
'Authorization'
|
26
|
+
Aspera::Node::X_ASPERA_ACCESSKEY => options[:username],
|
27
|
+
'Authorization' => options[:password]
|
26
28
|
}
|
27
29
|
raise 'root_id is required for access key' if @root_id.nil?
|
28
30
|
else
|
@@ -39,7 +41,7 @@ module Aspera
|
|
39
41
|
|
40
42
|
# used internally to ensure node api is set before using.
|
41
43
|
def node_api_
|
42
|
-
raise StandardError,'Before using this object, set the node_api attribute to a Aspera::Rest object' if @node_api.nil?
|
44
|
+
raise StandardError, 'Before using this object, set the node_api attribute to a Aspera::Rest object' if @node_api.nil?
|
43
45
|
return @node_api
|
44
46
|
end
|
45
47
|
# use this to read the node_api end point.
|
@@ -54,7 +56,7 @@ module Aspera
|
|
54
56
|
end
|
55
57
|
|
56
58
|
# generic method
|
57
|
-
def start_transfer(transfer_spec
|
59
|
+
def start_transfer(transfer_spec)
|
58
60
|
# add root id if access key
|
59
61
|
if !@root_id.nil?
|
60
62
|
case transfer_spec['direction']
|
@@ -64,9 +66,9 @@ module Aspera
|
|
64
66
|
end
|
65
67
|
end
|
66
68
|
# manage special additional parameter
|
67
|
-
if transfer_spec.
|
69
|
+
if transfer_spec.key?('EX_ssh_key_paths') && transfer_spec['EX_ssh_key_paths'].is_a?(Array) && !transfer_spec['EX_ssh_key_paths'].empty?
|
68
70
|
# not standard, so place standard field
|
69
|
-
if transfer_spec.
|
71
|
+
if transfer_spec.key?('ssh_private_key')
|
70
72
|
Log.log.warn('Both ssh_private_key and EX_ssh_key_paths are present, using ssh_private_key')
|
71
73
|
else
|
72
74
|
Log.log.warn('EX_ssh_key_paths has multiple keys, using first one only') unless transfer_spec['EX_ssh_key_paths'].length.eql?(1)
|
@@ -77,13 +79,13 @@ module Aspera
|
|
77
79
|
if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags']['aspera'].is_a?(Hash)
|
78
80
|
transfer_spec['tags']['aspera']['xfer_retry'] ||= 150
|
79
81
|
end
|
80
|
-
#
|
81
|
-
if transfer_spec['remote_host'].eql?(URI.parse(node_api_.params[:base_url]).host)
|
82
|
-
transfer_spec['remote_host'] = '
|
82
|
+
# Optimisation in case of sending to the same node (TODO: probably remove this, as /etc/hosts shall be used for that)
|
83
|
+
if !transfer_spec['wss_enabled'] && transfer_spec['remote_host'].eql?(URI.parse(node_api_.params[:base_url]).host)
|
84
|
+
transfer_spec['remote_host'] = '127.0.0.1'
|
83
85
|
end
|
84
|
-
resp = node_api_.create('ops/transfers',transfer_spec)[:data]
|
86
|
+
resp = node_api_.create('ops/transfers', transfer_spec)[:data]
|
85
87
|
@transfer_id = resp['id']
|
86
|
-
Log.log.debug
|
88
|
+
Log.log.debug{"tr_id=#{@transfer_id}"}
|
87
89
|
return @transfer_id
|
88
90
|
end
|
89
91
|
|
@@ -99,30 +101,30 @@ module Aspera
|
|
99
101
|
when 'completed'
|
100
102
|
notify_end(@transfer_id)
|
101
103
|
break
|
102
|
-
when 'waiting','partially_completed','unknown','waiting(read error)'
|
104
|
+
when 'waiting', 'partially_completed', 'unknown', 'waiting(read error)'
|
103
105
|
if spinner.nil?
|
104
106
|
spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
|
105
107
|
spinner.start
|
106
108
|
end
|
107
109
|
spinner.update(title: trdata['status'])
|
108
110
|
spinner.spin
|
109
|
-
#puts trdata
|
111
|
+
# puts trdata
|
110
112
|
when 'running'
|
111
|
-
#puts "running: sessions:#{trdata["sessions"].length}, #{trdata["sessions"].map{|i| i['bytes_transferred']}.join(',')}"
|
113
|
+
# puts "running: sessions:#{trdata["sessions"].length}, #{trdata["sessions"].map{|i| i['bytes_transferred']}.join(',')}"
|
112
114
|
if !started && trdata['precalc'].is_a?(Hash) &&
|
113
|
-
|
114
|
-
notify_begin(@transfer_id,trdata['precalc']['bytes_expected'])
|
115
|
+
trdata['precalc']['status'].eql?('ready')
|
116
|
+
notify_begin(@transfer_id, trdata['precalc']['bytes_expected'])
|
115
117
|
started = true
|
116
118
|
else
|
117
|
-
notify_progress(@transfer_id,trdata['bytes_transferred'])
|
119
|
+
notify_progress(@transfer_id, trdata['bytes_transferred'])
|
118
120
|
end
|
119
121
|
else
|
120
|
-
Log.log.warn
|
122
|
+
Log.log.warn{"trdata -> #{trdata}"}
|
121
123
|
raise Fasp::Error, "#{trdata['status']}: #{trdata['error_desc']}"
|
122
124
|
end
|
123
125
|
sleep(1)
|
124
126
|
end
|
125
|
-
#TODO get status of sessions
|
127
|
+
# TODO: get status of sessions
|
126
128
|
return []
|
127
129
|
end
|
128
130
|
end
|
@@ -6,7 +6,7 @@ require 'json'
|
|
6
6
|
|
7
7
|
module Aspera
|
8
8
|
module Fasp
|
9
|
-
class AgentTrsdk < AgentBase
|
9
|
+
class AgentTrsdk < Aspera::Fasp::AgentBase
|
10
10
|
DEFAULT_OPTIONS = {
|
11
11
|
address: '127.0.0.1',
|
12
12
|
port: 55_002
|
@@ -18,26 +18,26 @@ module Aspera
|
|
18
18
|
raise "expecting Hash (or nil), but have #{user_opts.class}" unless user_opts.nil? || user_opts.is_a?(Hash)
|
19
19
|
# set default options and override if specified
|
20
20
|
options = DEFAULT_OPTIONS.dup
|
21
|
-
user_opts&.each do |k,v|
|
22
|
-
raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.
|
21
|
+
user_opts&.each do |k, v|
|
22
|
+
raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.key?(k)
|
23
23
|
options[k] = v
|
24
24
|
end
|
25
|
-
Log.log.debug
|
25
|
+
Log.log.debug{"options= #{options}"}
|
26
26
|
super()
|
27
27
|
# load and create SDK stub
|
28
28
|
$LOAD_PATH.unshift(Installation.instance.sdk_ruby_folder)
|
29
29
|
require 'transfer_services_pb'
|
30
|
-
@transfer_client = Transfersdk::TransferService::Stub.new("#{options[:address]}:#{options[:port]}"
|
30
|
+
@transfer_client = Transfersdk::TransferService::Stub.new("#{options[:address]}:#{options[:port]}", :this_channel_is_insecure)
|
31
31
|
begin
|
32
32
|
get_info_response = @transfer_client.get_info(Transfersdk::InstanceInfoRequest.new)
|
33
|
-
Log.log.debug
|
33
|
+
Log.log.debug{"daemon info: #{get_info_response}"}
|
34
34
|
rescue GRPC::Unavailable
|
35
35
|
Log.log.warn('no daemon present, starting daemon...')
|
36
36
|
# location of daemon binary
|
37
|
-
bin_folder = File.realpath(File.join(Installation.instance.sdk_ruby_folder,'..'))
|
37
|
+
bin_folder = File.realpath(File.join(Installation.instance.sdk_ruby_folder, '..'))
|
38
38
|
# config file and logs are created in same folder
|
39
|
-
conf_file = File.join(bin_folder,'sdk.conf')
|
40
|
-
log_base = File.join(bin_folder,'transferd')
|
39
|
+
conf_file = File.join(bin_folder, 'sdk.conf')
|
40
|
+
log_base = File.join(bin_folder, 'transferd')
|
41
41
|
# create a config file for daemon
|
42
42
|
config = {
|
43
43
|
address: options[:address],
|
@@ -50,15 +50,15 @@ module Aspera
|
|
50
50
|
}
|
51
51
|
}
|
52
52
|
}
|
53
|
-
File.write(conf_file,config.to_json)
|
54
|
-
trd_pid = Process.spawn(Installation.instance.path(:transferd),'--config', conf_file, out: "#{log_base}.out", err: "#{log_base}.err")
|
53
|
+
File.write(conf_file, config.to_json)
|
54
|
+
trd_pid = Process.spawn(Installation.instance.path(:transferd), '--config', conf_file, out: "#{log_base}.out", err: "#{log_base}.err")
|
55
55
|
Process.detach(trd_pid)
|
56
56
|
sleep(2.0)
|
57
57
|
retry
|
58
58
|
end
|
59
59
|
end
|
60
60
|
|
61
|
-
def start_transfer(transfer_spec
|
61
|
+
def start_transfer(transfer_spec)
|
62
62
|
# create a transfer request
|
63
63
|
transfer_request = Transfersdk::TransferRequest.new(
|
64
64
|
transferType: Transfersdk::TransferType::FILE_REGULAR, # transfer type (file/stream)
|
@@ -66,9 +66,9 @@ module Aspera
|
|
66
66
|
transferSpec: transfer_spec.to_json) # transfer definition
|
67
67
|
# send start transfer request to the transfer manager daemon
|
68
68
|
start_transfer_response = @transfer_client.start_transfer(transfer_request)
|
69
|
-
Log.log.debug
|
69
|
+
Log.log.debug{"start transfer response #{start_transfer_response}"}
|
70
70
|
@transfer_id = start_transfer_response.transferId
|
71
|
-
Log.log.debug
|
71
|
+
Log.log.debug{"transfer started with id #{@transfer_id}"}
|
72
72
|
end
|
73
73
|
|
74
74
|
def wait_for_transfers_completion
|
@@ -76,26 +76,26 @@ module Aspera
|
|
76
76
|
# monitor transfer status
|
77
77
|
@transfer_client.monitor_transfers(Transfersdk::RegistrationRequest.new(transferId: [@transfer_id])) do |response|
|
78
78
|
Log.dump(:response, response.to_h)
|
79
|
-
#Log.log.debug
|
79
|
+
# Log.log.debug{"#{response.sessionInfo.preTransferBytes} #{response.transferInfo.bytesTransferred}"}
|
80
80
|
case response.status
|
81
81
|
when :RUNNING
|
82
82
|
if !started && !response.sessionInfo.preTransferBytes.eql?(0)
|
83
|
-
notify_begin(@transfer_id,response.sessionInfo.preTransferBytes)
|
83
|
+
notify_begin(@transfer_id, response.sessionInfo.preTransferBytes)
|
84
84
|
started = true
|
85
85
|
elsif started
|
86
|
-
notify_progress(@transfer_id,response.transferInfo.bytesTransferred)
|
86
|
+
notify_progress(@transfer_id, response.transferInfo.bytesTransferred)
|
87
87
|
end
|
88
88
|
when :FAILED, :COMPLETED, :CANCELED
|
89
89
|
notify_end(@transfer_id)
|
90
90
|
raise Fasp::Error, JSON.parse(response.message)['Description'] unless :COMPLETED.eql?(response.status)
|
91
91
|
break
|
92
|
-
when :QUEUED
|
92
|
+
when :QUEUED, :UNKNOWN_STATUS, :PAUSED, :ORPHANED
|
93
93
|
# ignore
|
94
94
|
else
|
95
|
-
Log.log.error
|
95
|
+
Log.log.error{"unknown status#{response.status}"}
|
96
96
|
end
|
97
97
|
end
|
98
|
-
# TODO return status
|
98
|
+
# TODO: return status
|
99
99
|
return []
|
100
100
|
end
|
101
101
|
end
|
data/lib/aspera/fasp/error.rb
CHANGED
@@ -7,7 +7,8 @@ module Aspera
|
|
7
7
|
# error raised if transfer fails
|
8
8
|
class Error < StandardError
|
9
9
|
attr_reader :err_code
|
10
|
-
|
10
|
+
|
11
|
+
def initialize(message, err_code=nil)
|
11
12
|
super(message)
|
12
13
|
@err_code = err_code
|
13
14
|
end
|
@@ -17,7 +18,7 @@ module Aspera
|
|
17
18
|
return r.merge({i: @err_code})
|
18
19
|
end
|
19
20
|
|
20
|
-
def retryable?; info[:r];end
|
21
|
+
def retryable?; info[:r]; end
|
21
22
|
end
|
22
23
|
end
|
23
24
|
end
|
@@ -4,6 +4,8 @@ module Aspera
|
|
4
4
|
module Fasp
|
5
5
|
# from https://www.google.com/search?q=FASP+error+codes
|
6
6
|
# Note that the fact that an error is retryable is not internally defined by protocol, it's client-side responsibility
|
7
|
+
# rubocop:disable Layout/MultilineHashKeyLineBreaks
|
8
|
+
# rubocop:disable Layout/FirstHashElementLineBreak
|
7
9
|
ERROR_INFO = {
|
8
10
|
# id retryable mnemo message additional info
|
9
11
|
1 => { r: false, c: 'FASP_PROTO', m: 'Generic fasp(tm) protocol error', a: 'fasp(tm) error'},
|
@@ -18,13 +20,13 @@ module Aspera
|
|
18
20
|
10 => { r: false, c: 'LIC_RATE_EXCEEDED', m: 'Rate exceeds the cap imposed by license', a: 'Rate exceeds cap imposed by license'},
|
19
21
|
11 => { r: false, c: 'INTERNAL_ERROR', m: 'Internal error (unexpected error)', a: 'Internal error'},
|
20
22
|
12 => { r: true, c: 'TRANSFER_ERROR', m: 'Error establishing control connection',
|
21
|
-
|
23
|
+
a: 'Error establishing SSH connection (check SSH port and firewall)'},
|
22
24
|
13 => { r: true, c: 'TRANSFER_TIMEOUT', m: 'Timeout establishing control connection',
|
23
|
-
|
25
|
+
a: 'Timeout establishing SSH connection (check SSH port and firewall)'},
|
24
26
|
14 => { r: true, c: 'CONNECTION_ERROR', m: 'Error establishing data connection',
|
25
|
-
|
27
|
+
a: 'Error establishing UDP connection (check UDP port and firewall)'},
|
26
28
|
15 => { r: true, c: 'CONNECTION_TIMEOUT', m: 'Timeout establishing data connection',
|
27
|
-
|
29
|
+
a: 'Timeout establishing UDP connection (check UDP port and firewall)'},
|
28
30
|
16 => { r: true, c: 'CONNECTION_LOST', m: 'Connection lost', a: 'Connection lost'},
|
29
31
|
17 => { r: true, c: 'RCVR_SEND_ERROR', m: 'Receiver fails to send feedback', a: 'Network failure (receiver can\'t send feedback)'},
|
30
32
|
18 => { r: true, c: 'RCVR_RECV_ERROR', m: 'Receiver fails to receive data packets', a: 'Network failure (receiver can\'t receive UDP data)'},
|
@@ -68,20 +70,21 @@ module Aspera
|
|
68
70
|
54 => { r: false, c: 'THRESHOLD_VALIDATION_FAILED', m: 'File threshold validation failed', a: 'File threshold validation failed'},
|
69
71
|
55 => { r: false, c: 'FILEPATH_TOO_LONG', m: 'File path/name too long for underlying file system', a: 'File path exceeds underlying file system limit'},
|
70
72
|
56 => { r: false, c: 'ILLEGAL_CHARS_IN_PATH', m: 'Windows path contains illegal characters',
|
71
|
-
|
73
|
+
a: 'Path being written to Windows file system contains illegal characters'},
|
72
74
|
57 => { r: false, c: 'CHUNK_MUST_MATCH_ALIGNMENT', m: 'Chunk size/start must be aligned with storage', a: 'Chunk size/start must be aligned with storage'},
|
73
75
|
58 => { r: false, c: 'VALIDATION_SESSION_ABORT', m: 'Session aborted to due to validation error', a: 'Session aborted to due validation error'},
|
74
76
|
59 => { r: false, c: 'REMOTE_STORAGE_ERROR', m: 'Remote storage errored', a: 'Remote storage errored'},
|
75
77
|
60 => { r: false, c: 'LUA_SCRIPT_ABORTED_SESSION', m: 'Session aborted due to Lua script abort', a: 'Session aborted due to Lua script abort'},
|
76
78
|
61 => { r: true, c: 'SSEAR_RETRYABLE', m: 'Transfer failed because of a retryable Encryption at Rest error',
|
77
|
-
|
79
|
+
a: 'Transfer failed because of a retryable Encryption at Rest error'},
|
78
80
|
62 => { r: false, c: 'SSEAR_FATAL', m: 'Transfer failed because of a fatal Encryption at Rest error',
|
79
|
-
|
81
|
+
a: 'Transfer failed because of a fatal Encryption at Rest error'},
|
80
82
|
63 => { r: false, c: 'LINK_LOOP', m: 'Path refers to a symbolic link loop', a: 'Path refers to a symbolic link loop'},
|
81
83
|
64 => { r: false, c: 'CANNOT_RENAME_PARTIAL_FILES', m: 'Can\'t rename a partial file', a: 'Can\'t rename a partial file.'},
|
82
84
|
65 => { r: false, c: 'CIPHER_NON_COMPAT_FIPS', m: 'Can\'t use this cipher with FIPS mode enabled', a: 'Can\'t use this cipher with FIPS mode enabled'},
|
83
85
|
66 => { r: false, c: 'PEER_REQUIRES_FIPS', m: 'Peer rejects cipher due to FIPS mode enabled on peer',
|
84
|
-
|
86
|
+
a: 'Peer rejects cipher due to FIPS mode enabled on peer'}
|
85
87
|
}.freeze
|
88
|
+
# rubocop:enable Layout/MultilineHashKeyLineBreaks
|
86
89
|
end
|
87
90
|
end
|