aspera-cli 4.10.0 → 4.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +20 -0
- data/CHANGELOG.md +509 -0
- data/CONTRIBUTING.md +118 -0
- data/README.md +621 -378
- data/bin/ascli +4 -4
- data/bin/asession +11 -11
- data/docs/test_env.conf +28 -19
- data/examples/aoc.rb +4 -4
- data/examples/dascli +11 -9
- data/examples/faspex4.rb +8 -8
- data/examples/node.rb +11 -11
- data/examples/server.rb +9 -9
- data/lib/aspera/aoc.rb +273 -266
- data/lib/aspera/ascmd.rb +56 -54
- data/lib/aspera/ats_api.rb +4 -4
- data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
- data/lib/aspera/cli/extended_value.rb +5 -5
- data/lib/aspera/cli/formater.rb +64 -64
- data/lib/aspera/cli/listener/line_dump.rb +1 -1
- data/lib/aspera/cli/listener/logger.rb +1 -1
- data/lib/aspera/cli/listener/progress.rb +5 -6
- data/lib/aspera/cli/listener/progress_multi.rb +14 -19
- data/lib/aspera/cli/main.rb +66 -67
- data/lib/aspera/cli/manager.rb +110 -110
- data/lib/aspera/cli/plugin.rb +54 -37
- data/lib/aspera/cli/plugins/alee.rb +4 -4
- data/lib/aspera/cli/plugins/aoc.rb +308 -669
- data/lib/aspera/cli/plugins/ats.rb +44 -46
- data/lib/aspera/cli/plugins/bss.rb +10 -10
- data/lib/aspera/cli/plugins/config.rb +447 -344
- data/lib/aspera/cli/plugins/console.rb +12 -12
- data/lib/aspera/cli/plugins/cos.rb +18 -20
- data/lib/aspera/cli/plugins/faspex.rb +110 -112
- data/lib/aspera/cli/plugins/faspex5.rb +67 -46
- data/lib/aspera/cli/plugins/node.rb +364 -288
- data/lib/aspera/cli/plugins/orchestrator.rb +46 -46
- data/lib/aspera/cli/plugins/preview.rb +122 -114
- data/lib/aspera/cli/plugins/server.rb +137 -83
- data/lib/aspera/cli/plugins/shares.rb +30 -29
- data/lib/aspera/cli/plugins/sync.rb +13 -33
- data/lib/aspera/cli/transfer_agent.rb +57 -57
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +3 -3
- data/lib/aspera/command_line_builder.rb +27 -27
- data/lib/aspera/cos_node.rb +22 -20
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +30 -28
- data/lib/aspera/fasp/agent_base.rb +15 -15
- data/lib/aspera/fasp/agent_connect.rb +23 -21
- data/lib/aspera/fasp/agent_direct.rb +65 -67
- data/lib/aspera/fasp/agent_httpgw.rb +72 -68
- data/lib/aspera/fasp/agent_node.rb +23 -21
- data/lib/aspera/fasp/agent_trsdk.rb +20 -20
- data/lib/aspera/fasp/error.rb +3 -2
- data/lib/aspera/fasp/error_info.rb +11 -8
- data/lib/aspera/fasp/installation.rb +78 -78
- data/lib/aspera/fasp/listener.rb +1 -1
- data/lib/aspera/fasp/parameters.rb +75 -72
- data/lib/aspera/fasp/parameters.yaml +2 -2
- data/lib/aspera/fasp/resume_policy.rb +8 -8
- data/lib/aspera/fasp/transfer_spec.rb +35 -2
- data/lib/aspera/fasp/uri.rb +7 -7
- data/lib/aspera/faspex_gw.rb +7 -5
- data/lib/aspera/hash_ext.rb +3 -3
- data/lib/aspera/id_generator.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +23 -28
- data/lib/aspera/keychain/macos_security.rb +21 -20
- data/lib/aspera/log.rb +7 -7
- data/lib/aspera/nagios.rb +19 -18
- data/lib/aspera/node.rb +209 -35
- data/lib/aspera/oauth.rb +37 -36
- data/lib/aspera/open_application.rb +19 -11
- data/lib/aspera/persistency_action_once.rb +4 -4
- data/lib/aspera/persistency_folder.rb +13 -13
- data/lib/aspera/preview/file_types.rb +8 -8
- data/lib/aspera/preview/generator.rb +67 -67
- data/lib/aspera/preview/utils.rb +27 -27
- data/lib/aspera/proxy_auto_config.js +41 -41
- data/lib/aspera/proxy_auto_config.rb +16 -16
- data/lib/aspera/rest.rb +56 -60
- data/lib/aspera/rest_call_error.rb +2 -1
- data/lib/aspera/rest_error_analyzer.rb +18 -17
- data/lib/aspera/rest_errors_aspera.rb +16 -16
- data/lib/aspera/secret_hider.rb +15 -13
- data/lib/aspera/ssh.rb +11 -10
- data/lib/aspera/sync.rb +158 -44
- data/lib/aspera/temp_file_manager.rb +2 -2
- data/lib/aspera/uri_reader.rb +4 -4
- data/lib/aspera/web_auth.rb +14 -13
- data.tar.gz.sig +0 -0
- metadata +8 -5
- metadata.gz.sig +0 -0
|
@@ -14,7 +14,7 @@ require 'json'
|
|
|
14
14
|
module Aspera
|
|
15
15
|
module Fasp
|
|
16
16
|
# start a transfer using Aspera HTTP Gateway, using web socket session for uploads
|
|
17
|
-
class AgentHttpgw < AgentBase
|
|
17
|
+
class AgentHttpgw < Aspera::Fasp::AgentBase
|
|
18
18
|
# message returned by HTTP GW in case of success
|
|
19
19
|
MSG_END_UPLOAD = 'end upload'
|
|
20
20
|
MSG_END_SLICE = 'end_slice_upload'
|
|
@@ -23,15 +23,15 @@ module Aspera
|
|
|
23
23
|
upload_chunksize: 64_000,
|
|
24
24
|
upload_bar_refresh_sec: 0.5
|
|
25
25
|
}.freeze
|
|
26
|
-
DEFAULT_BASE_PATH='/aspera/http-gwy'
|
|
26
|
+
DEFAULT_BASE_PATH = '/aspera/http-gwy'
|
|
27
27
|
# upload endpoints
|
|
28
|
-
V1_UPLOAD='/v1/upload'
|
|
29
|
-
V2_UPLOAD='/v2/upload'
|
|
30
|
-
private_constant :DEFAULT_OPTIONS
|
|
28
|
+
V1_UPLOAD = '/v1/upload'
|
|
29
|
+
V2_UPLOAD = '/v2/upload'
|
|
30
|
+
private_constant :DEFAULT_OPTIONS, :MSG_END_UPLOAD, :MSG_END_SLICE, :V1_UPLOAD, :V2_UPLOAD
|
|
31
31
|
|
|
32
32
|
# send message on http gw web socket
|
|
33
33
|
def ws_snd_json(data)
|
|
34
|
-
@slice_uploads += 1 if data.
|
|
34
|
+
@slice_uploads += 1 if data.key?(:slice_upload)
|
|
35
35
|
Log.log.debug{JSON.generate(data)}
|
|
36
36
|
ws_send(JSON.generate(data))
|
|
37
37
|
end
|
|
@@ -47,7 +47,7 @@ module Aspera
|
|
|
47
47
|
# we need to keep track of actual file path because transfer spec is modified to be sent in web socket
|
|
48
48
|
source_paths = []
|
|
49
49
|
# get source root or nil
|
|
50
|
-
source_root = transfer_spec.
|
|
50
|
+
source_root = transfer_spec.key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] : nil
|
|
51
51
|
# source root is ignored by GW, used only here
|
|
52
52
|
transfer_spec.delete('source_root')
|
|
53
53
|
# compute total size of files to upload (for progress)
|
|
@@ -56,7 +56,7 @@ module Aspera
|
|
|
56
56
|
# save actual file location to be able read contents later
|
|
57
57
|
full_src_filepath = item['source']
|
|
58
58
|
# add source root if needed
|
|
59
|
-
full_src_filepath = File.join(source_root,full_src_filepath) unless source_root.nil?
|
|
59
|
+
full_src_filepath = File.join(source_root, full_src_filepath) unless source_root.nil?
|
|
60
60
|
# GW expects a simple file name in 'source' but if user wants to change the name, we take it
|
|
61
61
|
item['source'] = File.basename(item['destination'].nil? ? item['source'] : item['destination'])
|
|
62
62
|
item['file_size'] = File.size(full_src_filepath)
|
|
@@ -66,18 +66,18 @@ module Aspera
|
|
|
66
66
|
end
|
|
67
67
|
# identify this session uniquely
|
|
68
68
|
session_id = SecureRandom.uuid
|
|
69
|
-
@slice_uploads=0
|
|
69
|
+
@slice_uploads = 0
|
|
70
70
|
# web socket endpoint: by default use v2 (newer gateways), without base64 encoding
|
|
71
71
|
upload_api_version = V2_UPLOAD
|
|
72
72
|
# is the latest supported? else revert to old api
|
|
73
|
-
upload_api_version=V1_UPLOAD unless @api_info['endpoints'].any?{|i|i.include?(upload_api_version)}
|
|
73
|
+
upload_api_version = V1_UPLOAD unless @api_info['endpoints'].any?{|i|i.include?(upload_api_version)}
|
|
74
74
|
Log.log.debug{"api version: #{upload_api_version}"}
|
|
75
|
-
url=File.join(@gw_api.params[:base_url],upload_api_version)
|
|
76
|
-
#uri = URI.parse(url)
|
|
75
|
+
url = File.join(@gw_api.params[:base_url], upload_api_version)
|
|
76
|
+
# uri = URI.parse(url)
|
|
77
77
|
# open web socket to end point (equivalent to Net::HTTP.start)
|
|
78
78
|
http_socket = Rest.start_http_session(url)
|
|
79
79
|
@ws_io = http_socket.instance_variable_get(:@socket)
|
|
80
|
-
|
|
80
|
+
# @ws_io.debug_output = Log.log
|
|
81
81
|
@ws_handshake = ::WebSocket::Handshake::Client.new(url: url, headers: {})
|
|
82
82
|
@ws_io.write(@ws_handshake.to_s)
|
|
83
83
|
sleep(0.1)
|
|
@@ -85,21 +85,21 @@ module Aspera
|
|
|
85
85
|
raise 'Error in websocket handshake' unless @ws_handshake.finished?
|
|
86
86
|
Log.log.debug('ws: handshake success')
|
|
87
87
|
# data shared between main thread and read thread
|
|
88
|
-
shared_info={
|
|
88
|
+
shared_info = {
|
|
89
89
|
read_exception: nil, # error message if any in callback
|
|
90
90
|
end_uploads: 0 # number of files totally sent
|
|
91
|
-
#mutex: Mutex.new
|
|
92
|
-
#cond_var: ConditionVariable.new
|
|
91
|
+
# mutex: Mutex.new
|
|
92
|
+
# cond_var: ConditionVariable.new
|
|
93
93
|
}
|
|
94
94
|
# start read thread
|
|
95
95
|
ws_read_thread = Thread.new do
|
|
96
96
|
Log.log.debug('ws: thread: started')
|
|
97
97
|
frame = ::WebSocket::Frame::Incoming::Client.new
|
|
98
98
|
loop do
|
|
99
|
-
begin
|
|
99
|
+
begin # rubocop:disable Style/RedundantBegin
|
|
100
100
|
frame << @ws_io.readuntil("\n")
|
|
101
101
|
while (msg = frame.next)
|
|
102
|
-
Log.log.debug
|
|
102
|
+
Log.log.debug{"ws: thread: message: #{msg.data} #{shared_info[:end_uploads]}"}
|
|
103
103
|
message = msg.data
|
|
104
104
|
if message.eql?(MSG_END_UPLOAD)
|
|
105
105
|
shared_info[:end_uploads] += 1
|
|
@@ -122,12 +122,12 @@ module Aspera
|
|
|
122
122
|
break
|
|
123
123
|
end
|
|
124
124
|
end
|
|
125
|
-
Log.log.debug
|
|
125
|
+
Log.log.debug{"ws: thread: stopping (exc=#{shared_info[:read_exception]},cls=#{shared_info[:read_exception].class})"}
|
|
126
126
|
end
|
|
127
127
|
# notify progress bar
|
|
128
|
-
notify_begin(session_id,total_size)
|
|
128
|
+
notify_begin(session_id, total_size)
|
|
129
129
|
# first step send transfer spec
|
|
130
|
-
Log.dump(:ws_spec,transfer_spec)
|
|
130
|
+
Log.dump(:ws_spec, transfer_spec)
|
|
131
131
|
ws_snd_json(transfer_spec: transfer_spec)
|
|
132
132
|
# current file index
|
|
133
133
|
file_index = 0
|
|
@@ -135,58 +135,63 @@ module Aspera
|
|
|
135
135
|
sent_bytes = 0
|
|
136
136
|
# last progress event
|
|
137
137
|
last_progress_time = nil
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
138
|
+
|
|
139
|
+
transfer_spec['paths'].each do |item|
|
|
140
|
+
# TODO: get mime type?
|
|
141
|
+
file_mime_type = ''
|
|
142
|
+
file_size = item['file_size']
|
|
143
|
+
file_name = File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
|
|
144
|
+
# compute total number of slices
|
|
145
|
+
numslices = ((file_size - 1) / @options[:upload_chunksize]) + 1
|
|
146
|
+
File.open(source_paths[file_index]) do |file|
|
|
147
|
+
# current slice index
|
|
148
|
+
slicenum = 0
|
|
149
|
+
until file.eof?
|
|
150
|
+
data = file.read(@options[:upload_chunksize])
|
|
151
|
+
slice_data = {
|
|
152
|
+
name: file_name,
|
|
153
|
+
type: file_mime_type,
|
|
154
|
+
size: file_size,
|
|
155
|
+
slice: slicenum,
|
|
156
|
+
total_slices: numslices,
|
|
157
|
+
fileIndex: file_index
|
|
158
|
+
}
|
|
159
|
+
# Log.dump(:slice_data,slice_data) #if slicenum.eql?(0)
|
|
160
|
+
# interrupt main thread if read thread failed
|
|
161
|
+
raise shared_info[:read_exception] unless shared_info[:read_exception].nil?
|
|
162
|
+
begin
|
|
162
163
|
if upload_api_version.eql?(V1_UPLOAD)
|
|
163
164
|
slice_data[:data] = Base64.strict_encode64(data)
|
|
164
165
|
ws_snd_json(slice_upload: slice_data)
|
|
165
166
|
else
|
|
166
167
|
ws_snd_json(slice_upload: slice_data) if slicenum.eql?(0)
|
|
167
|
-
ws_send(data,type: :binary)
|
|
168
|
+
ws_send(data, type: :binary)
|
|
168
169
|
Log.log.debug{"ws: sent buffer: #{file_index} / #{slicenum}"}
|
|
169
|
-
ws_snd_json(slice_upload: slice_data) if slicenum.eql?(numslices-1)
|
|
170
|
-
end
|
|
171
|
-
sent_bytes += data.length
|
|
172
|
-
currenttime = Time.now
|
|
173
|
-
if last_progress_time.nil? || ((currenttime - last_progress_time) > @options[:upload_bar_refresh_sec])
|
|
174
|
-
notify_progress(session_id,sent_bytes)
|
|
175
|
-
last_progress_time = currenttime
|
|
170
|
+
ws_snd_json(slice_upload: slice_data) if slicenum.eql?(numslices - 1)
|
|
176
171
|
end
|
|
177
|
-
|
|
172
|
+
rescue Errno::EPIPE => e
|
|
173
|
+
raise shared_info[:read_exception] unless shared_info[:read_exception].nil?
|
|
174
|
+
raise e
|
|
175
|
+
end
|
|
176
|
+
sent_bytes += data.length
|
|
177
|
+
currenttime = Time.now
|
|
178
|
+
if last_progress_time.nil? || ((currenttime - last_progress_time) > @options[:upload_bar_refresh_sec])
|
|
179
|
+
notify_progress(session_id, sent_bytes)
|
|
180
|
+
last_progress_time = currenttime
|
|
178
181
|
end
|
|
182
|
+
slicenum += 1
|
|
179
183
|
end
|
|
180
|
-
file_index += 1
|
|
181
184
|
end
|
|
185
|
+
file_index += 1
|
|
182
186
|
end
|
|
187
|
+
|
|
183
188
|
Log.log.debug('Finished upload')
|
|
184
189
|
ws_read_thread.join
|
|
185
190
|
Log.log.debug{"result: #{shared_info[:end_uploads]} / #{@slice_uploads}"}
|
|
186
191
|
ws_send(nil, type: :close) unless @ws_io.nil?
|
|
187
192
|
@ws_io = nil
|
|
188
193
|
http_socket&.finish
|
|
189
|
-
notify_progress(session_id,sent_bytes)
|
|
194
|
+
notify_progress(session_id, sent_bytes)
|
|
190
195
|
notify_end(session_id)
|
|
191
196
|
end
|
|
192
197
|
|
|
@@ -194,18 +199,18 @@ module Aspera
|
|
|
194
199
|
transfer_spec['zip_required'] ||= false
|
|
195
200
|
transfer_spec['source_root'] ||= '/'
|
|
196
201
|
# is normally provided by application, like package name
|
|
197
|
-
if !transfer_spec.
|
|
202
|
+
if !transfer_spec.key?('download_name')
|
|
198
203
|
# by default it is the name of first file
|
|
199
204
|
dname = File.basename(transfer_spec['paths'].first['source'])
|
|
200
205
|
# we remove extension
|
|
201
|
-
dname = dname.gsub(/\.@gw_api.*$/,'')
|
|
206
|
+
dname = dname.gsub(/\.@gw_api.*$/, '')
|
|
202
207
|
# ands add indication of number of files if there is more than one
|
|
203
208
|
if transfer_spec['paths'].length > 1
|
|
204
209
|
dname += " #{transfer_spec['paths'].length} Files"
|
|
205
210
|
end
|
|
206
211
|
transfer_spec['download_name'] = dname
|
|
207
212
|
end
|
|
208
|
-
creation = @gw_api.create('v1/download',{'transfer_spec' => transfer_spec})[:data]
|
|
213
|
+
creation = @gw_api.create('v1/download', {'transfer_spec' => transfer_spec})[:data]
|
|
209
214
|
transfer_uuid = creation['url'].split('/').last
|
|
210
215
|
file_dest =
|
|
211
216
|
if transfer_spec['zip_required'] || transfer_spec['paths'].length > 1
|
|
@@ -215,19 +220,18 @@ module Aspera
|
|
|
215
220
|
# it is a plain file if we don't require zip and there is only one file
|
|
216
221
|
File.basename(transfer_spec['paths'].first['source'])
|
|
217
222
|
end
|
|
218
|
-
file_dest = File.join(transfer_spec['destination_root'],file_dest)
|
|
219
|
-
@gw_api.call({operation: 'GET',subpath: "v1/download/#{transfer_uuid}",save_to_file: file_dest})
|
|
223
|
+
file_dest = File.join(transfer_spec['destination_root'], file_dest)
|
|
224
|
+
@gw_api.call({operation: 'GET', subpath: "v1/download/#{transfer_uuid}", save_to_file: file_dest})
|
|
220
225
|
end
|
|
221
226
|
|
|
222
227
|
# start FASP transfer based on transfer spec (hash table)
|
|
223
228
|
# note that it is asynchronous
|
|
224
229
|
# HTTP download only supports file list
|
|
225
|
-
def start_transfer(transfer_spec
|
|
230
|
+
def start_transfer(transfer_spec)
|
|
226
231
|
raise 'GW URL must be set' if @gw_api.nil?
|
|
227
|
-
raise 'option: must be hash (or nil)' unless options.is_a?(Hash)
|
|
228
232
|
raise 'paths: must be Array' unless transfer_spec['paths'].is_a?(Array)
|
|
229
233
|
raise 'only token based transfer is supported in GW' unless transfer_spec['token'].is_a?(String)
|
|
230
|
-
Log.dump(:user_spec,transfer_spec)
|
|
234
|
+
Log.dump(:user_spec, transfer_spec)
|
|
231
235
|
transfer_spec['authentication'] ||= 'token'
|
|
232
236
|
case transfer_spec['direction']
|
|
233
237
|
when Fasp::TransferSpec::DIRECTION_SEND
|
|
@@ -253,17 +257,17 @@ module Aspera
|
|
|
253
257
|
private
|
|
254
258
|
|
|
255
259
|
def initialize(opts)
|
|
256
|
-
Log.log.debug
|
|
260
|
+
Log.log.debug{"local options= #{opts}"}
|
|
257
261
|
# set default options and override if specified
|
|
258
262
|
@options = DEFAULT_OPTIONS.dup
|
|
259
263
|
raise "httpgw agent parameters (transfer_info): expecting Hash, but have #{opts.class}" unless opts.is_a?(Hash)
|
|
260
|
-
opts.symbolize_keys.each do |k,v|
|
|
261
|
-
raise "httpgw agent parameter: Unknown: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.
|
|
264
|
+
opts.symbolize_keys.each do |k, v|
|
|
265
|
+
raise "httpgw agent parameter: Unknown: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.key?(k)
|
|
262
266
|
@options[k] = v
|
|
263
267
|
end
|
|
264
268
|
raise 'missing param: url' if @options[:url].nil?
|
|
265
269
|
# remove /v1 from end
|
|
266
|
-
@options[:url].gsub(%r{/v1/*$},'')
|
|
270
|
+
@options[:url].gsub(%r{/v1/*$}, '')
|
|
267
271
|
super()
|
|
268
272
|
@gw_api = Rest.new({base_url: @options[:url]})
|
|
269
273
|
@api_info = @gw_api.read('v1/info')[:data]
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require 'aspera/fasp/agent_base'
|
|
4
4
|
require 'aspera/fasp/transfer_spec'
|
|
5
|
+
require 'aspera/node'
|
|
5
6
|
require 'aspera/log'
|
|
6
7
|
require 'tty-spinner'
|
|
7
8
|
|
|
@@ -9,20 +10,21 @@ module Aspera
|
|
|
9
10
|
module Fasp
|
|
10
11
|
# this singleton class is used by the CLI to provide a common interface to start a transfer
|
|
11
12
|
# before using it, the use must set the `node_api` member.
|
|
12
|
-
class AgentNode < AgentBase
|
|
13
|
+
class AgentNode < Aspera::Fasp::AgentBase
|
|
13
14
|
# option include: root_id if the node is an access key
|
|
14
15
|
attr_writer :options
|
|
16
|
+
|
|
15
17
|
def initialize(options)
|
|
16
18
|
raise 'node specification must be Hash' unless options.is_a?(Hash)
|
|
17
|
-
%i[url username password].each { |k| raise "missing parameter [#{k}] in node specification: #{options}" unless options.
|
|
19
|
+
%i[url username password].each { |k| raise "missing parameter [#{k}] in node specification: #{options}" unless options.key?(k) }
|
|
18
20
|
super()
|
|
19
21
|
# root id is required for access key
|
|
20
22
|
@root_id = options[:root_id]
|
|
21
23
|
rest_params = { base_url: options[:url]}
|
|
22
24
|
if /^Bearer /.match?(options[:password])
|
|
23
25
|
rest_params[:headers] = {
|
|
24
|
-
|
|
25
|
-
'Authorization'
|
|
26
|
+
Aspera::Node::X_ASPERA_ACCESSKEY => options[:username],
|
|
27
|
+
'Authorization' => options[:password]
|
|
26
28
|
}
|
|
27
29
|
raise 'root_id is required for access key' if @root_id.nil?
|
|
28
30
|
else
|
|
@@ -39,7 +41,7 @@ module Aspera
|
|
|
39
41
|
|
|
40
42
|
# used internally to ensure node api is set before using.
|
|
41
43
|
def node_api_
|
|
42
|
-
raise StandardError,'Before using this object, set the node_api attribute to a Aspera::Rest object' if @node_api.nil?
|
|
44
|
+
raise StandardError, 'Before using this object, set the node_api attribute to a Aspera::Rest object' if @node_api.nil?
|
|
43
45
|
return @node_api
|
|
44
46
|
end
|
|
45
47
|
# use this to read the node_api end point.
|
|
@@ -54,7 +56,7 @@ module Aspera
|
|
|
54
56
|
end
|
|
55
57
|
|
|
56
58
|
# generic method
|
|
57
|
-
def start_transfer(transfer_spec
|
|
59
|
+
def start_transfer(transfer_spec)
|
|
58
60
|
# add root id if access key
|
|
59
61
|
if !@root_id.nil?
|
|
60
62
|
case transfer_spec['direction']
|
|
@@ -64,9 +66,9 @@ module Aspera
|
|
|
64
66
|
end
|
|
65
67
|
end
|
|
66
68
|
# manage special additional parameter
|
|
67
|
-
if transfer_spec.
|
|
69
|
+
if transfer_spec.key?('EX_ssh_key_paths') && transfer_spec['EX_ssh_key_paths'].is_a?(Array) && !transfer_spec['EX_ssh_key_paths'].empty?
|
|
68
70
|
# not standard, so place standard field
|
|
69
|
-
if transfer_spec.
|
|
71
|
+
if transfer_spec.key?('ssh_private_key')
|
|
70
72
|
Log.log.warn('Both ssh_private_key and EX_ssh_key_paths are present, using ssh_private_key')
|
|
71
73
|
else
|
|
72
74
|
Log.log.warn('EX_ssh_key_paths has multiple keys, using first one only') unless transfer_spec['EX_ssh_key_paths'].length.eql?(1)
|
|
@@ -77,13 +79,13 @@ module Aspera
|
|
|
77
79
|
if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags']['aspera'].is_a?(Hash)
|
|
78
80
|
transfer_spec['tags']['aspera']['xfer_retry'] ||= 150
|
|
79
81
|
end
|
|
80
|
-
#
|
|
81
|
-
if transfer_spec['remote_host'].eql?(URI.parse(node_api_.params[:base_url]).host)
|
|
82
|
-
transfer_spec['remote_host'] = '
|
|
82
|
+
# Optimisation in case of sending to the same node (TODO: probably remove this, as /etc/hosts shall be used for that)
|
|
83
|
+
if !transfer_spec['wss_enabled'] && transfer_spec['remote_host'].eql?(URI.parse(node_api_.params[:base_url]).host)
|
|
84
|
+
transfer_spec['remote_host'] = '127.0.0.1'
|
|
83
85
|
end
|
|
84
|
-
resp = node_api_.create('ops/transfers',transfer_spec)[:data]
|
|
86
|
+
resp = node_api_.create('ops/transfers', transfer_spec)[:data]
|
|
85
87
|
@transfer_id = resp['id']
|
|
86
|
-
Log.log.debug
|
|
88
|
+
Log.log.debug{"tr_id=#{@transfer_id}"}
|
|
87
89
|
return @transfer_id
|
|
88
90
|
end
|
|
89
91
|
|
|
@@ -99,30 +101,30 @@ module Aspera
|
|
|
99
101
|
when 'completed'
|
|
100
102
|
notify_end(@transfer_id)
|
|
101
103
|
break
|
|
102
|
-
when 'waiting','partially_completed','unknown','waiting(read error)'
|
|
104
|
+
when 'waiting', 'partially_completed', 'unknown', 'waiting(read error)'
|
|
103
105
|
if spinner.nil?
|
|
104
106
|
spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
|
|
105
107
|
spinner.start
|
|
106
108
|
end
|
|
107
109
|
spinner.update(title: trdata['status'])
|
|
108
110
|
spinner.spin
|
|
109
|
-
#puts trdata
|
|
111
|
+
# puts trdata
|
|
110
112
|
when 'running'
|
|
111
|
-
#puts "running: sessions:#{trdata["sessions"].length}, #{trdata["sessions"].map{|i| i['bytes_transferred']}.join(',')}"
|
|
113
|
+
# puts "running: sessions:#{trdata["sessions"].length}, #{trdata["sessions"].map{|i| i['bytes_transferred']}.join(',')}"
|
|
112
114
|
if !started && trdata['precalc'].is_a?(Hash) &&
|
|
113
|
-
|
|
114
|
-
notify_begin(@transfer_id,trdata['precalc']['bytes_expected'])
|
|
115
|
+
trdata['precalc']['status'].eql?('ready')
|
|
116
|
+
notify_begin(@transfer_id, trdata['precalc']['bytes_expected'])
|
|
115
117
|
started = true
|
|
116
118
|
else
|
|
117
|
-
notify_progress(@transfer_id,trdata['bytes_transferred'])
|
|
119
|
+
notify_progress(@transfer_id, trdata['bytes_transferred'])
|
|
118
120
|
end
|
|
119
121
|
else
|
|
120
|
-
Log.log.warn
|
|
122
|
+
Log.log.warn{"trdata -> #{trdata}"}
|
|
121
123
|
raise Fasp::Error, "#{trdata['status']}: #{trdata['error_desc']}"
|
|
122
124
|
end
|
|
123
125
|
sleep(1)
|
|
124
126
|
end
|
|
125
|
-
#TODO get status of sessions
|
|
127
|
+
# TODO: get status of sessions
|
|
126
128
|
return []
|
|
127
129
|
end
|
|
128
130
|
end
|
|
@@ -6,7 +6,7 @@ require 'json'
|
|
|
6
6
|
|
|
7
7
|
module Aspera
|
|
8
8
|
module Fasp
|
|
9
|
-
class AgentTrsdk < AgentBase
|
|
9
|
+
class AgentTrsdk < Aspera::Fasp::AgentBase
|
|
10
10
|
DEFAULT_OPTIONS = {
|
|
11
11
|
address: '127.0.0.1',
|
|
12
12
|
port: 55_002
|
|
@@ -18,26 +18,26 @@ module Aspera
|
|
|
18
18
|
raise "expecting Hash (or nil), but have #{user_opts.class}" unless user_opts.nil? || user_opts.is_a?(Hash)
|
|
19
19
|
# set default options and override if specified
|
|
20
20
|
options = DEFAULT_OPTIONS.dup
|
|
21
|
-
user_opts&.each do |k,v|
|
|
22
|
-
raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.
|
|
21
|
+
user_opts&.each do |k, v|
|
|
22
|
+
raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.key?(k)
|
|
23
23
|
options[k] = v
|
|
24
24
|
end
|
|
25
|
-
Log.log.debug
|
|
25
|
+
Log.log.debug{"options= #{options}"}
|
|
26
26
|
super()
|
|
27
27
|
# load and create SDK stub
|
|
28
28
|
$LOAD_PATH.unshift(Installation.instance.sdk_ruby_folder)
|
|
29
29
|
require 'transfer_services_pb'
|
|
30
|
-
@transfer_client = Transfersdk::TransferService::Stub.new("#{options[:address]}:#{options[:port]}"
|
|
30
|
+
@transfer_client = Transfersdk::TransferService::Stub.new("#{options[:address]}:#{options[:port]}", :this_channel_is_insecure)
|
|
31
31
|
begin
|
|
32
32
|
get_info_response = @transfer_client.get_info(Transfersdk::InstanceInfoRequest.new)
|
|
33
|
-
Log.log.debug
|
|
33
|
+
Log.log.debug{"daemon info: #{get_info_response}"}
|
|
34
34
|
rescue GRPC::Unavailable
|
|
35
35
|
Log.log.warn('no daemon present, starting daemon...')
|
|
36
36
|
# location of daemon binary
|
|
37
|
-
bin_folder = File.realpath(File.join(Installation.instance.sdk_ruby_folder,'..'))
|
|
37
|
+
bin_folder = File.realpath(File.join(Installation.instance.sdk_ruby_folder, '..'))
|
|
38
38
|
# config file and logs are created in same folder
|
|
39
|
-
conf_file = File.join(bin_folder,'sdk.conf')
|
|
40
|
-
log_base = File.join(bin_folder,'transferd')
|
|
39
|
+
conf_file = File.join(bin_folder, 'sdk.conf')
|
|
40
|
+
log_base = File.join(bin_folder, 'transferd')
|
|
41
41
|
# create a config file for daemon
|
|
42
42
|
config = {
|
|
43
43
|
address: options[:address],
|
|
@@ -50,15 +50,15 @@ module Aspera
|
|
|
50
50
|
}
|
|
51
51
|
}
|
|
52
52
|
}
|
|
53
|
-
File.write(conf_file,config.to_json)
|
|
54
|
-
trd_pid = Process.spawn(Installation.instance.path(:transferd),'--config', conf_file, out: "#{log_base}.out", err: "#{log_base}.err")
|
|
53
|
+
File.write(conf_file, config.to_json)
|
|
54
|
+
trd_pid = Process.spawn(Installation.instance.path(:transferd), '--config', conf_file, out: "#{log_base}.out", err: "#{log_base}.err")
|
|
55
55
|
Process.detach(trd_pid)
|
|
56
56
|
sleep(2.0)
|
|
57
57
|
retry
|
|
58
58
|
end
|
|
59
59
|
end
|
|
60
60
|
|
|
61
|
-
def start_transfer(transfer_spec
|
|
61
|
+
def start_transfer(transfer_spec)
|
|
62
62
|
# create a transfer request
|
|
63
63
|
transfer_request = Transfersdk::TransferRequest.new(
|
|
64
64
|
transferType: Transfersdk::TransferType::FILE_REGULAR, # transfer type (file/stream)
|
|
@@ -66,9 +66,9 @@ module Aspera
|
|
|
66
66
|
transferSpec: transfer_spec.to_json) # transfer definition
|
|
67
67
|
# send start transfer request to the transfer manager daemon
|
|
68
68
|
start_transfer_response = @transfer_client.start_transfer(transfer_request)
|
|
69
|
-
Log.log.debug
|
|
69
|
+
Log.log.debug{"start transfer response #{start_transfer_response}"}
|
|
70
70
|
@transfer_id = start_transfer_response.transferId
|
|
71
|
-
Log.log.debug
|
|
71
|
+
Log.log.debug{"transfer started with id #{@transfer_id}"}
|
|
72
72
|
end
|
|
73
73
|
|
|
74
74
|
def wait_for_transfers_completion
|
|
@@ -76,26 +76,26 @@ module Aspera
|
|
|
76
76
|
# monitor transfer status
|
|
77
77
|
@transfer_client.monitor_transfers(Transfersdk::RegistrationRequest.new(transferId: [@transfer_id])) do |response|
|
|
78
78
|
Log.dump(:response, response.to_h)
|
|
79
|
-
#Log.log.debug
|
|
79
|
+
# Log.log.debug{"#{response.sessionInfo.preTransferBytes} #{response.transferInfo.bytesTransferred}"}
|
|
80
80
|
case response.status
|
|
81
81
|
when :RUNNING
|
|
82
82
|
if !started && !response.sessionInfo.preTransferBytes.eql?(0)
|
|
83
|
-
notify_begin(@transfer_id,response.sessionInfo.preTransferBytes)
|
|
83
|
+
notify_begin(@transfer_id, response.sessionInfo.preTransferBytes)
|
|
84
84
|
started = true
|
|
85
85
|
elsif started
|
|
86
|
-
notify_progress(@transfer_id,response.transferInfo.bytesTransferred)
|
|
86
|
+
notify_progress(@transfer_id, response.transferInfo.bytesTransferred)
|
|
87
87
|
end
|
|
88
88
|
when :FAILED, :COMPLETED, :CANCELED
|
|
89
89
|
notify_end(@transfer_id)
|
|
90
90
|
raise Fasp::Error, JSON.parse(response.message)['Description'] unless :COMPLETED.eql?(response.status)
|
|
91
91
|
break
|
|
92
|
-
when :QUEUED
|
|
92
|
+
when :QUEUED, :UNKNOWN_STATUS, :PAUSED, :ORPHANED
|
|
93
93
|
# ignore
|
|
94
94
|
else
|
|
95
|
-
Log.log.error
|
|
95
|
+
Log.log.error{"unknown status#{response.status}"}
|
|
96
96
|
end
|
|
97
97
|
end
|
|
98
|
-
# TODO return status
|
|
98
|
+
# TODO: return status
|
|
99
99
|
return []
|
|
100
100
|
end
|
|
101
101
|
end
|
data/lib/aspera/fasp/error.rb
CHANGED
|
@@ -7,7 +7,8 @@ module Aspera
|
|
|
7
7
|
# error raised if transfer fails
|
|
8
8
|
class Error < StandardError
|
|
9
9
|
attr_reader :err_code
|
|
10
|
-
|
|
10
|
+
|
|
11
|
+
def initialize(message, err_code=nil)
|
|
11
12
|
super(message)
|
|
12
13
|
@err_code = err_code
|
|
13
14
|
end
|
|
@@ -17,7 +18,7 @@ module Aspera
|
|
|
17
18
|
return r.merge({i: @err_code})
|
|
18
19
|
end
|
|
19
20
|
|
|
20
|
-
def retryable?; info[:r];end
|
|
21
|
+
def retryable?; info[:r]; end
|
|
21
22
|
end
|
|
22
23
|
end
|
|
23
24
|
end
|
|
@@ -4,6 +4,8 @@ module Aspera
|
|
|
4
4
|
module Fasp
|
|
5
5
|
# from https://www.google.com/search?q=FASP+error+codes
|
|
6
6
|
# Note that the fact that an error is retryable is not internally defined by protocol, it's client-side responsibility
|
|
7
|
+
# rubocop:disable Layout/MultilineHashKeyLineBreaks
|
|
8
|
+
# rubocop:disable Layout/FirstHashElementLineBreak
|
|
7
9
|
ERROR_INFO = {
|
|
8
10
|
# id retryable mnemo message additional info
|
|
9
11
|
1 => { r: false, c: 'FASP_PROTO', m: 'Generic fasp(tm) protocol error', a: 'fasp(tm) error'},
|
|
@@ -18,13 +20,13 @@ module Aspera
|
|
|
18
20
|
10 => { r: false, c: 'LIC_RATE_EXCEEDED', m: 'Rate exceeds the cap imposed by license', a: 'Rate exceeds cap imposed by license'},
|
|
19
21
|
11 => { r: false, c: 'INTERNAL_ERROR', m: 'Internal error (unexpected error)', a: 'Internal error'},
|
|
20
22
|
12 => { r: true, c: 'TRANSFER_ERROR', m: 'Error establishing control connection',
|
|
21
|
-
|
|
23
|
+
a: 'Error establishing SSH connection (check SSH port and firewall)'},
|
|
22
24
|
13 => { r: true, c: 'TRANSFER_TIMEOUT', m: 'Timeout establishing control connection',
|
|
23
|
-
|
|
25
|
+
a: 'Timeout establishing SSH connection (check SSH port and firewall)'},
|
|
24
26
|
14 => { r: true, c: 'CONNECTION_ERROR', m: 'Error establishing data connection',
|
|
25
|
-
|
|
27
|
+
a: 'Error establishing UDP connection (check UDP port and firewall)'},
|
|
26
28
|
15 => { r: true, c: 'CONNECTION_TIMEOUT', m: 'Timeout establishing data connection',
|
|
27
|
-
|
|
29
|
+
a: 'Timeout establishing UDP connection (check UDP port and firewall)'},
|
|
28
30
|
16 => { r: true, c: 'CONNECTION_LOST', m: 'Connection lost', a: 'Connection lost'},
|
|
29
31
|
17 => { r: true, c: 'RCVR_SEND_ERROR', m: 'Receiver fails to send feedback', a: 'Network failure (receiver can\'t send feedback)'},
|
|
30
32
|
18 => { r: true, c: 'RCVR_RECV_ERROR', m: 'Receiver fails to receive data packets', a: 'Network failure (receiver can\'t receive UDP data)'},
|
|
@@ -68,20 +70,21 @@ module Aspera
|
|
|
68
70
|
54 => { r: false, c: 'THRESHOLD_VALIDATION_FAILED', m: 'File threshold validation failed', a: 'File threshold validation failed'},
|
|
69
71
|
55 => { r: false, c: 'FILEPATH_TOO_LONG', m: 'File path/name too long for underlying file system', a: 'File path exceeds underlying file system limit'},
|
|
70
72
|
56 => { r: false, c: 'ILLEGAL_CHARS_IN_PATH', m: 'Windows path contains illegal characters',
|
|
71
|
-
|
|
73
|
+
a: 'Path being written to Windows file system contains illegal characters'},
|
|
72
74
|
57 => { r: false, c: 'CHUNK_MUST_MATCH_ALIGNMENT', m: 'Chunk size/start must be aligned with storage', a: 'Chunk size/start must be aligned with storage'},
|
|
73
75
|
58 => { r: false, c: 'VALIDATION_SESSION_ABORT', m: 'Session aborted to due to validation error', a: 'Session aborted to due validation error'},
|
|
74
76
|
59 => { r: false, c: 'REMOTE_STORAGE_ERROR', m: 'Remote storage errored', a: 'Remote storage errored'},
|
|
75
77
|
60 => { r: false, c: 'LUA_SCRIPT_ABORTED_SESSION', m: 'Session aborted due to Lua script abort', a: 'Session aborted due to Lua script abort'},
|
|
76
78
|
61 => { r: true, c: 'SSEAR_RETRYABLE', m: 'Transfer failed because of a retryable Encryption at Rest error',
|
|
77
|
-
|
|
79
|
+
a: 'Transfer failed because of a retryable Encryption at Rest error'},
|
|
78
80
|
62 => { r: false, c: 'SSEAR_FATAL', m: 'Transfer failed because of a fatal Encryption at Rest error',
|
|
79
|
-
|
|
81
|
+
a: 'Transfer failed because of a fatal Encryption at Rest error'},
|
|
80
82
|
63 => { r: false, c: 'LINK_LOOP', m: 'Path refers to a symbolic link loop', a: 'Path refers to a symbolic link loop'},
|
|
81
83
|
64 => { r: false, c: 'CANNOT_RENAME_PARTIAL_FILES', m: 'Can\'t rename a partial file', a: 'Can\'t rename a partial file.'},
|
|
82
84
|
65 => { r: false, c: 'CIPHER_NON_COMPAT_FIPS', m: 'Can\'t use this cipher with FIPS mode enabled', a: 'Can\'t use this cipher with FIPS mode enabled'},
|
|
83
85
|
66 => { r: false, c: 'PEER_REQUIRES_FIPS', m: 'Peer rejects cipher due to FIPS mode enabled on peer',
|
|
84
|
-
|
|
86
|
+
a: 'Peer rejects cipher due to FIPS mode enabled on peer'}
|
|
85
87
|
}.freeze
|
|
88
|
+
# rubocop:enable Layout/MultilineHashKeyLineBreaks
|
|
86
89
|
end
|
|
87
90
|
end
|