aspera-cli 4.13.0 → 4.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +28 -5
- data/CONTRIBUTING.md +17 -1
- data/README.md +782 -401
- data/examples/dascli +1 -1
- data/examples/rubyc +24 -0
- data/lib/aspera/aoc.rb +21 -32
- data/lib/aspera/ascmd.rb +1 -0
- data/lib/aspera/cli/basic_auth_plugin.rb +6 -6
- data/lib/aspera/cli/formatter.rb +17 -25
- data/lib/aspera/cli/main.rb +21 -27
- data/lib/aspera/cli/manager.rb +128 -114
- data/lib/aspera/cli/plugin.rb +87 -38
- data/lib/aspera/cli/plugins/alee.rb +2 -2
- data/lib/aspera/cli/plugins/aoc.rb +216 -102
- data/lib/aspera/cli/plugins/ats.rb +16 -18
- data/lib/aspera/cli/plugins/bss.rb +3 -3
- data/lib/aspera/cli/plugins/config.rb +177 -367
- data/lib/aspera/cli/plugins/console.rb +4 -6
- data/lib/aspera/cli/plugins/cos.rb +12 -13
- data/lib/aspera/cli/plugins/faspex.rb +17 -18
- data/lib/aspera/cli/plugins/faspex5.rb +332 -216
- data/lib/aspera/cli/plugins/node.rb +171 -142
- data/lib/aspera/cli/plugins/orchestrator.rb +15 -18
- data/lib/aspera/cli/plugins/preview.rb +38 -60
- data/lib/aspera/cli/plugins/server.rb +22 -15
- data/lib/aspera/cli/plugins/shares.rb +24 -33
- data/lib/aspera/cli/plugins/sync.rb +3 -3
- data/lib/aspera/cli/transfer_agent.rb +29 -26
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +9 -7
- data/lib/aspera/data/6 +0 -0
- data/lib/aspera/environment.rb +7 -3
- data/lib/aspera/fasp/agent_connect.rb +5 -0
- data/lib/aspera/fasp/agent_direct.rb +5 -5
- data/lib/aspera/fasp/agent_httpgw.rb +138 -60
- data/lib/aspera/fasp/agent_trsdk.rb +2 -0
- data/lib/aspera/fasp/error_info.rb +2 -0
- data/lib/aspera/fasp/installation.rb +18 -19
- data/lib/aspera/fasp/parameters.rb +18 -17
- data/lib/aspera/fasp/parameters.yaml +2 -1
- data/lib/aspera/fasp/resume_policy.rb +3 -3
- data/lib/aspera/fasp/transfer_spec.rb +6 -5
- data/lib/aspera/fasp/uri.rb +23 -21
- data/lib/aspera/faspex_postproc.rb +1 -1
- data/lib/aspera/hash_ext.rb +12 -2
- data/lib/aspera/keychain/macos_security.rb +13 -13
- data/lib/aspera/log.rb +1 -0
- data/lib/aspera/node.rb +62 -80
- data/lib/aspera/oauth.rb +1 -1
- data/lib/aspera/persistency_action_once.rb +1 -1
- data/lib/aspera/preview/terminal.rb +61 -15
- data/lib/aspera/preview/utils.rb +3 -3
- data/lib/aspera/proxy_auto_config.js +2 -2
- data/lib/aspera/rest.rb +37 -0
- data/lib/aspera/secret_hider.rb +6 -1
- data/lib/aspera/ssh.rb +1 -1
- data/lib/aspera/sync.rb +2 -0
- data.tar.gz.sig +0 -0
- metadata +3 -4
- metadata.gz.sig +0 -0
- data/docs/test_env.conf +0 -186
- data/lib/aspera/data/7 +0 -0
@@ -25,7 +25,7 @@ module Aspera
|
|
25
25
|
multi_incr_udp: true,
|
26
26
|
resume: {},
|
27
27
|
ascp_args: [],
|
28
|
-
quiet: true # by default no
|
28
|
+
quiet: true # by default no native ascp progress bar
|
29
29
|
}.freeze
|
30
30
|
private_constant :DEFAULT_OPTIONS
|
31
31
|
|
@@ -82,12 +82,12 @@ module Aspera
|
|
82
82
|
end
|
83
83
|
|
84
84
|
# compute known args
|
85
|
-
env_args =
|
85
|
+
env_args = Parameters.new(transfer_spec, @options).ascp_args
|
86
86
|
|
87
87
|
# add fallback cert and key as arguments if needed
|
88
88
|
if ['1', 1, true, 'force'].include?(transfer_spec['http_fallback'])
|
89
|
-
env_args[:args].unshift('-Y', Installation.instance.path(:
|
90
|
-
env_args[:args].unshift('-I', Installation.instance.path(:
|
89
|
+
env_args[:args].unshift('-Y', Installation.instance.path(:fallback_cert_privkey))
|
90
|
+
env_args[:args].unshift('-I', Installation.instance.path(:fallback_certificate))
|
91
91
|
end
|
92
92
|
|
93
93
|
env_args[:args].unshift('-q') if @options[:quiet]
|
@@ -334,7 +334,7 @@ module Aspera
|
|
334
334
|
# @param options : keys(symbol): see DEFAULT_OPTIONS
|
335
335
|
def initialize(options=nil)
|
336
336
|
super()
|
337
|
-
# all transfer jobs, key = SecureRandom.uuid, protected by mutex,
|
337
|
+
# all transfer jobs, key = SecureRandom.uuid, protected by mutex, cond var on change
|
338
338
|
@jobs = {}
|
339
339
|
# mutex protects global data accessed by threads
|
340
340
|
@mutex = Mutex.new
|
@@ -9,6 +9,17 @@ require 'websocket'
|
|
9
9
|
require 'base64'
|
10
10
|
require 'json'
|
11
11
|
|
12
|
+
# HTTP GW Upload protocol
|
13
|
+
# -----------------------
|
14
|
+
# v1
|
15
|
+
# 1 - MessageType: String (Transfer Spec) JSON : type: transfer_spec, acknowledged with "end upload"
|
16
|
+
# 2.. - MessageType: String (Slice Upload start) JSON : type: slice_upload, acknowledged with "end upload"
|
17
|
+
# v2
|
18
|
+
# 1 - MessageType: String (Transfer Spec) JSON : type: transfer_spec, acknowledged with "end upload"
|
19
|
+
# 2 - MessageType: String (Slice Upload start) JSON : type: slice_upload, acknowledged with "end_slice_upload"
|
20
|
+
# 3.. - MessageType: ByteArray (File Size) Chunks : acknowledged with "end upload"
|
21
|
+
# last - MessageType: String (Slice Upload end) JSON : type: slice_upload, acknowledged with "end_slice_upload"
|
22
|
+
|
12
23
|
# ref: https://api.ibm.com/explorer/catalog/aspera/product/ibm-aspera/api/http-gateway-api/doc/guides-toc
|
13
24
|
# https://developer.ibm.com/apis/catalog?search=%22aspera%20http%22
|
14
25
|
module Aspera
|
@@ -16,35 +27,66 @@ module Aspera
|
|
16
27
|
# start a transfer using Aspera HTTP Gateway, using web socket session for uploads
|
17
28
|
class AgentHttpgw < Aspera::Fasp::AgentBase
|
18
29
|
# message returned by HTTP GW in case of success
|
19
|
-
|
20
|
-
|
30
|
+
MSG_RECV_DATA_RECEIVED_SIGNAL = 'end upload'
|
31
|
+
MSG_RECV_SLICE_UPLOAD_SIGNAL = 'end_slice_upload'
|
32
|
+
MSG_SEND_SLICE_UPLOAD = 'slice_upload'
|
33
|
+
MSG_SEND_TRANSFER_SPEC = 'transfer_spec'
|
34
|
+
# upload API versions
|
35
|
+
API_V1 = 'v1'
|
36
|
+
API_V2 = 'v2'
|
21
37
|
# options available in CLI (transfer_info)
|
22
38
|
DEFAULT_OPTIONS = {
|
23
39
|
url: nil,
|
24
40
|
upload_chunk_size: 64_000,
|
25
|
-
upload_bar_refresh_sec: 0.5
|
41
|
+
upload_bar_refresh_sec: 0.5,
|
42
|
+
api_version: API_V2,
|
43
|
+
synchronous: true
|
26
44
|
}.freeze
|
27
45
|
DEFAULT_BASE_PATH = '/aspera/http-gwy'
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
private_constant :DEFAULT_OPTIONS, :MSG_END_UPLOAD, :MSG_END_SLICE, :V1_UPLOAD, :V2_UPLOAD
|
46
|
+
LOG_WS_MAIN = 'ws: send: '.green
|
47
|
+
LOG_WS_THREAD = 'ws: ack: '.red
|
48
|
+
private_constant :DEFAULT_OPTIONS, :MSG_RECV_DATA_RECEIVED_SIGNAL, :MSG_RECV_SLICE_UPLOAD_SIGNAL, :API_V1, :API_V2
|
32
49
|
|
33
50
|
# send message on http gw web socket
|
34
|
-
def ws_snd_json(
|
35
|
-
|
36
|
-
|
37
|
-
|
51
|
+
def ws_snd_json(msg_type, payload)
|
52
|
+
if msg_type.eql?(MSG_SEND_SLICE_UPLOAD) && @options[:api_version].eql?(API_V2)
|
53
|
+
@shared_info[:count][:sent_v2_slice] += 1
|
54
|
+
else
|
55
|
+
@shared_info[:count][:sent_other] += 1
|
56
|
+
end
|
57
|
+
Log.log.debug do
|
58
|
+
log_data = payload.dup
|
59
|
+
log_data[:data] = "[data #{log_data[:data].length} bytes]" if log_data.key?(:data)
|
60
|
+
"send_txt: #{msg_type}: #{JSON.generate(log_data)}"
|
61
|
+
end
|
62
|
+
ws_send(JSON.generate({msg_type => payload}))
|
38
63
|
end
|
39
64
|
|
40
|
-
def ws_send(
|
41
|
-
|
65
|
+
def ws_send(data_to_send, type: :text)
|
66
|
+
Log.log.debug{"#{LOG_WS_MAIN}send low: type: #{type}"}
|
67
|
+
@shared_info[:count][:sent_other] += 1 if type.eql?(:binary)
|
68
|
+
Log.log.debug{"#{LOG_WS_MAIN}counts: #{@shared_info[:count]}"}
|
69
|
+
frame = ::WebSocket::Frame::Outgoing::Client.new(data: data_to_send, type: type, version: @ws_handshake.version)
|
42
70
|
@ws_io.write(frame.to_s)
|
43
71
|
end
|
44
72
|
|
73
|
+
# wait for all message sent to be acknowledged by HTTPGW server
|
74
|
+
def wait_for_sent_msg_ack_or_exception
|
75
|
+
return unless @options[:synchronous]
|
76
|
+
@shared_info[:mutex].synchronize do
|
77
|
+
while (@shared_info[:count][:received_data] != @shared_info[:count][:sent_other]) ||
|
78
|
+
(@shared_info[:count][:received_v2_slice] != @shared_info[:count][:sent_v2_slice])
|
79
|
+
Log.log.debug{"#{LOG_WS_MAIN}wait: counts: #{@shared_info[:count]}"}
|
80
|
+
@shared_info[:cond_var].wait(@shared_info[:mutex], 1.0)
|
81
|
+
raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
|
82
|
+
end
|
83
|
+
end
|
84
|
+
Log.log.debug{"#{LOG_WS_MAIN}sync ok: counts: #{@shared_info[:count]}"}
|
85
|
+
end
|
86
|
+
|
45
87
|
def upload(transfer_spec)
|
46
88
|
# total size of all files
|
47
|
-
|
89
|
+
total_bytes_to_transfer = 0
|
48
90
|
# we need to keep track of actual file path because transfer spec is modified to be sent in web socket
|
49
91
|
source_paths = []
|
50
92
|
# get source root or nil
|
@@ -61,50 +103,65 @@ module Aspera
|
|
61
103
|
# GW expects a simple file name in 'source' but if user wants to change the name, we take it
|
62
104
|
item['source'] = File.basename(item['destination'].nil? ? item['source'] : item['destination'])
|
63
105
|
item['file_size'] = File.size(full_src_filepath)
|
64
|
-
|
106
|
+
total_bytes_to_transfer += item['file_size']
|
65
107
|
# save so that we can actually read the file later
|
66
108
|
source_paths.push(full_src_filepath)
|
67
109
|
end
|
68
110
|
# identify this session uniquely
|
69
111
|
session_id = SecureRandom.uuid
|
70
|
-
|
71
|
-
#
|
72
|
-
upload_api_version = V2_UPLOAD
|
73
|
-
# is the latest supported? else revert to old api
|
74
|
-
upload_api_version = V1_UPLOAD unless @api_info['endpoints'].any?{|i|i.include?(upload_api_version)}
|
75
|
-
Log.log.debug{"api version: #{upload_api_version}"}
|
76
|
-
url = File.join(@gw_api.params[:base_url], upload_api_version)
|
77
|
-
# uri = URI.parse(url)
|
112
|
+
upload_url = File.join(@gw_api.params[:base_url], @options[:api_version], 'upload')
|
113
|
+
# uri = URI.parse(upload_url)
|
78
114
|
# open web socket to end point (equivalent to Net::HTTP.start)
|
79
|
-
http_socket = Rest.start_http_session(
|
115
|
+
http_socket = Rest.start_http_session(upload_url)
|
116
|
+
# little hack to get the socket opened for HTTP, handy because HTTP debug will be available
|
80
117
|
@ws_io = http_socket.instance_variable_get(:@socket)
|
81
118
|
# @ws_io.debug_output = Log.log
|
82
|
-
@ws_handshake = ::WebSocket::Handshake::Client.new(url:
|
119
|
+
@ws_handshake = ::WebSocket::Handshake::Client.new(url: upload_url, headers: {})
|
83
120
|
@ws_io.write(@ws_handshake.to_s)
|
84
121
|
sleep(0.1)
|
85
122
|
@ws_handshake << @ws_io.readuntil("\r\n\r\n")
|
86
123
|
raise 'Error in websocket handshake' unless @ws_handshake.finished?
|
87
|
-
Log.log.debug
|
124
|
+
Log.log.debug{"#{LOG_WS_MAIN}handshake success"}
|
88
125
|
# data shared between main thread and read thread
|
89
|
-
shared_info = {
|
126
|
+
@shared_info = {
|
90
127
|
read_exception: nil, # error message if any in callback
|
91
|
-
|
92
|
-
|
93
|
-
|
128
|
+
count: {
|
129
|
+
received_data: 0, # number of files received on other side
|
130
|
+
received_v2_slice: 0, # number of slices received on other side
|
131
|
+
sent_other: 0,
|
132
|
+
sent_v2_slice: 0
|
133
|
+
},
|
134
|
+
mutex: Mutex.new,
|
135
|
+
cond_var: ConditionVariable.new
|
94
136
|
}
|
95
137
|
# start read thread
|
96
138
|
ws_read_thread = Thread.new do
|
97
|
-
Log.log.debug
|
98
|
-
frame = ::WebSocket::Frame::Incoming::Client.new
|
139
|
+
Log.log.debug{"#{LOG_WS_THREAD}read started"}
|
140
|
+
frame = ::WebSocket::Frame::Incoming::Client.new(version: @ws_handshake.version)
|
99
141
|
loop do
|
100
142
|
begin # rubocop:disable Style/RedundantBegin
|
101
|
-
|
143
|
+
# unless (recv_data = @ws_io.getc)
|
144
|
+
# sleep(0.1)
|
145
|
+
# next
|
146
|
+
# end
|
147
|
+
# frame << recv_data
|
148
|
+
# frame << @ws_io.readuntil("\n")
|
149
|
+
# frame << @ws_io.read_all
|
150
|
+
frame << @ws_io.read(1)
|
102
151
|
while (msg = frame.next)
|
103
|
-
Log.log.debug{"
|
152
|
+
Log.log.debug{"#{LOG_WS_THREAD}type: #{msg.class}"}
|
104
153
|
message = msg.data
|
105
|
-
|
106
|
-
|
107
|
-
|
154
|
+
Log.log.debug{"#{LOG_WS_THREAD}message: [#{message}]"}
|
155
|
+
if message.eql?(MSG_RECV_DATA_RECEIVED_SIGNAL)
|
156
|
+
@shared_info[:mutex].synchronize do
|
157
|
+
@shared_info[:count][:received_data] += 1
|
158
|
+
@shared_info[:cond_var].signal
|
159
|
+
end
|
160
|
+
elsif message.eql?(MSG_RECV_SLICE_UPLOAD_SIGNAL)
|
161
|
+
@shared_info[:mutex].synchronize do
|
162
|
+
@shared_info[:count][:received_v2_slice] += 1
|
163
|
+
@shared_info[:cond_var].signal
|
164
|
+
end
|
108
165
|
else
|
109
166
|
message.chomp!
|
110
167
|
error_message =
|
@@ -117,19 +174,25 @@ module Aspera
|
|
117
174
|
end
|
118
175
|
raise error_message
|
119
176
|
end
|
120
|
-
|
177
|
+
Log.log.debug{"#{LOG_WS_THREAD}counts: #{@shared_info[:count]}"}
|
178
|
+
end # while
|
121
179
|
rescue => e
|
122
|
-
|
180
|
+
Log.log.debug{"#{LOG_WS_THREAD}Exception: #{e}"}
|
181
|
+
@shared_info[:mutex].synchronize do
|
182
|
+
@shared_info[:read_exception] = e unless e.is_a?(EOFError)
|
183
|
+
@shared_info[:cond_var].signal
|
184
|
+
end
|
123
185
|
break
|
124
|
-
end
|
125
|
-
end
|
126
|
-
Log.log.debug{"
|
186
|
+
end # begin
|
187
|
+
end # loop
|
188
|
+
Log.log.debug{"#{LOG_WS_THREAD}stopping (exc=#{@shared_info[:read_exception]},cls=#{@shared_info[:read_exception].class})"}
|
127
189
|
end
|
128
190
|
# notify progress bar
|
129
|
-
notify_begin(session_id,
|
191
|
+
notify_begin(session_id, total_bytes_to_transfer)
|
130
192
|
# first step send transfer spec
|
131
193
|
Log.dump(:ws_spec, transfer_spec)
|
132
|
-
ws_snd_json(
|
194
|
+
ws_snd_json(MSG_SEND_TRANSFER_SPEC, transfer_spec)
|
195
|
+
wait_for_sent_msg_ack_or_exception
|
133
196
|
# current file index
|
134
197
|
file_index = 0
|
135
198
|
# aggregate size sent
|
@@ -148,7 +211,7 @@ module Aspera
|
|
148
211
|
# current slice index
|
149
212
|
slice_index = 0
|
150
213
|
until file.eof?
|
151
|
-
|
214
|
+
file_bin_data = file.read(@options[:upload_chunk_size])
|
152
215
|
slice_data = {
|
153
216
|
name: file_name,
|
154
217
|
type: file_mime_type,
|
@@ -159,22 +222,26 @@ module Aspera
|
|
159
222
|
}
|
160
223
|
# Log.dump(:slice_data,slice_data) #if slice_index.eql?(0)
|
161
224
|
# interrupt main thread if read thread failed
|
162
|
-
raise shared_info[:read_exception] unless shared_info[:read_exception].nil?
|
225
|
+
raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
|
163
226
|
begin
|
164
|
-
if
|
165
|
-
slice_data[:data] = Base64.strict_encode64(
|
166
|
-
ws_snd_json(
|
227
|
+
if @options[:api_version].eql?(API_V1)
|
228
|
+
slice_data[:data] = Base64.strict_encode64(file_bin_data)
|
229
|
+
ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_data)
|
167
230
|
else
|
168
|
-
ws_snd_json(
|
169
|
-
ws_send(
|
170
|
-
Log.log.debug{"
|
171
|
-
ws_snd_json(
|
231
|
+
ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_data) if slice_index.eql?(0)
|
232
|
+
ws_send(file_bin_data, type: :binary)
|
233
|
+
Log.log.debug{"#{LOG_WS_MAIN}sent bin buffer: #{file_index} / #{slice_index}"}
|
234
|
+
ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_data) if slice_index.eql?(slice_total - 1)
|
172
235
|
end
|
236
|
+
wait_for_sent_msg_ack_or_exception
|
173
237
|
rescue Errno::EPIPE => e
|
174
|
-
raise shared_info[:read_exception] unless shared_info[:read_exception].nil?
|
238
|
+
raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
|
239
|
+
raise e
|
240
|
+
rescue Net::ReadTimeout => e
|
241
|
+
Log.log.warn{'A timeout condition using HTTPGW may signal a permission problem on destination. Check ascp logs on httpgw.'}
|
175
242
|
raise e
|
176
243
|
end
|
177
|
-
sent_bytes +=
|
244
|
+
sent_bytes += file_bin_data.length
|
178
245
|
current_time = Time.now
|
179
246
|
if last_progress_time.nil? || ((current_time - last_progress_time) > @options[:upload_bar_refresh_sec])
|
180
247
|
notify_progress(session_id, sent_bytes)
|
@@ -186,9 +253,9 @@ module Aspera
|
|
186
253
|
file_index += 1
|
187
254
|
end
|
188
255
|
|
189
|
-
Log.log.debug('Finished upload')
|
256
|
+
Log.log.debug('Finished upload, waiting for end of read thread.')
|
190
257
|
ws_read_thread.join
|
191
|
-
Log.log.debug{"result: #{shared_info[:
|
258
|
+
Log.log.debug{"Read thread joined, result: #{@shared_info[:count][:received_data]} / #{@shared_info[:count][:sent_other]}"}
|
192
259
|
ws_send(nil, type: :close) unless @ws_io.nil?
|
193
260
|
@ws_io = nil
|
194
261
|
http_socket&.finish
|
@@ -258,7 +325,7 @@ module Aspera
|
|
258
325
|
private
|
259
326
|
|
260
327
|
def initialize(opts)
|
261
|
-
Log.
|
328
|
+
Log.dump(:in_options, opts)
|
262
329
|
# set default options and override if specified
|
263
330
|
@options = DEFAULT_OPTIONS.dup
|
264
331
|
raise "httpgw agent parameters (transfer_info): expecting Hash, but have #{opts.class}" unless opts.is_a?(Hash)
|
@@ -266,13 +333,24 @@ module Aspera
|
|
266
333
|
raise "httpgw agent parameter: Unknown: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.key?(k)
|
267
334
|
@options[k] = v
|
268
335
|
end
|
269
|
-
|
336
|
+
if @options[:url].nil?
|
337
|
+
available = DEFAULT_OPTIONS.map { |k, v| "#{k}(#{v})"}.join(', ')
|
338
|
+
raise "Missing mandatory parameter for HTTP GW in transfer_info: url. Allowed parameters: #{available}."
|
339
|
+
end
|
270
340
|
# remove /v1 from end
|
271
341
|
@options[:url].gsub(%r{/v1/*$}, '')
|
272
342
|
super()
|
273
343
|
@gw_api = Rest.new({base_url: @options[:url]})
|
274
344
|
@api_info = @gw_api.read('v1/info')[:data]
|
275
|
-
Log.
|
345
|
+
Log.dump(:api_info, @api_info)
|
346
|
+
if @options[:api_version].nil?
|
347
|
+
# web socket endpoint: by default use v2 (newer gateways), without base64 encoding
|
348
|
+
@options[:api_version] = API_V2
|
349
|
+
# is the latest supported? else revert to old api
|
350
|
+
@options[:api_version] = API_V1 unless @api_info['endpoints'].any?{|i|i.include?(@options[:api_version])}
|
351
|
+
end
|
352
|
+
@options.freeze
|
353
|
+
Log.dump(:final_options, @options)
|
276
354
|
end
|
277
355
|
end # AgentHttpgw
|
278
356
|
end
|
@@ -126,7 +126,11 @@ module Aspera
|
|
126
126
|
end
|
127
127
|
|
128
128
|
# all ascp files (in SDK)
|
129
|
-
FILES = %i[ascp ascp4
|
129
|
+
FILES = %i[ascp ascp4 ssh_bypass_dsa_privkey ssh_bypass_rsa_privkey aspera_license aspera_conf fallback_certificate fallback_cert_privkey].freeze
|
130
|
+
|
131
|
+
def check_or_create_sdk_file(filename, force: false, &block)
|
132
|
+
return Environment.write_file_restricted(File.join(sdk_folder, filename), force: force, mode: 0o644, &block)
|
133
|
+
end
|
130
134
|
|
131
135
|
# get path of one resource file of currently activated product
|
132
136
|
# keys and certs are generated locally... (they are well known values, arch. independent)
|
@@ -139,23 +143,18 @@ module Aspera
|
|
139
143
|
file = file.gsub('ascp', 'ascp4') if k.eql?(:ascp4)
|
140
144
|
when :transferd
|
141
145
|
file = transferd_filepath
|
142
|
-
when :
|
143
|
-
file =
|
144
|
-
when :
|
145
|
-
file =
|
146
|
+
when :ssh_bypass_dsa_privkey
|
147
|
+
file = check_or_create_sdk_file('aspera_bypass_dsa.pem') {get_key('dsa', 1)}
|
148
|
+
when :ssh_bypass_rsa_privkey
|
149
|
+
file = check_or_create_sdk_file('aspera_bypass_rsa.pem') {get_key('rsa', 2)}
|
146
150
|
when :aspera_license
|
147
|
-
file =
|
148
|
-
|
149
|
-
Zlib::Inflate.inflate(DataRepository.instance.data(6)),
|
150
|
-
"==SIGNATURE==\n",
|
151
|
-
Base64.strict_encode64(DataRepository.instance.data(7))
|
152
|
-
]
|
153
|
-
Base64.strict_encode64(clear.join)
|
151
|
+
file = check_or_create_sdk_file('aspera-license') do
|
152
|
+
Zlib::Inflate.inflate(DataRepository.instance.data(6))
|
154
153
|
end
|
155
154
|
when :aspera_conf
|
156
|
-
file =
|
157
|
-
when :
|
158
|
-
file_key = File.join(sdk_folder, '
|
155
|
+
file = check_or_create_sdk_file('aspera.conf') {DEFAULT_ASPERA_CONF}
|
156
|
+
when :fallback_certificate, :fallback_cert_privkey
|
157
|
+
file_key = File.join(sdk_folder, 'aspera_fallback_cert_private_key.pem')
|
159
158
|
file_cert = File.join(sdk_folder, 'aspera_fallback_cert.pem')
|
160
159
|
if !File.exist?(file_key) || !File.exist?(file_cert)
|
161
160
|
require 'openssl'
|
@@ -169,10 +168,10 @@ module Aspera
|
|
169
168
|
cert.serial = 0x0
|
170
169
|
cert.version = 2
|
171
170
|
cert.sign(private_key, OpenSSL::Digest.new('SHA1'))
|
172
|
-
|
173
|
-
|
171
|
+
check_or_create_sdk_file('aspera_fallback_cert_private_key.pem', force: true) {private_key.to_pem}
|
172
|
+
check_or_create_sdk_file('aspera_fallback_cert.pem', force: true) {cert.to_pem}
|
174
173
|
end
|
175
|
-
file = k.eql?(:
|
174
|
+
file = k.eql?(:fallback_certificate) ? file_cert : file_key
|
176
175
|
else
|
177
176
|
raise "INTERNAL ERROR: #{k}"
|
178
177
|
end
|
@@ -206,7 +205,7 @@ module Aspera
|
|
206
205
|
end
|
207
206
|
|
208
207
|
def bypass_keys
|
209
|
-
return %i[
|
208
|
+
return %i[ssh_bypass_dsa_privkey ssh_bypass_rsa_privkey].map{|i|Installation.instance.path(i)}
|
210
209
|
end
|
211
210
|
|
212
211
|
# use in plugin `config`
|
@@ -3,6 +3,7 @@
|
|
3
3
|
require 'aspera/log'
|
4
4
|
require 'aspera/command_line_builder'
|
5
5
|
require 'aspera/temp_file_manager'
|
6
|
+
require 'aspera/fasp/error'
|
6
7
|
require 'securerandom'
|
7
8
|
require 'base64'
|
8
9
|
require 'json'
|
@@ -19,6 +20,7 @@ module Aspera
|
|
19
20
|
# Short names of columns in manual
|
20
21
|
SUPPORTED_AGENTS_SHORT = SUPPORTED_AGENTS.map{|a|a.to_s[0].to_sym}
|
21
22
|
FILE_LIST_OPTIONS = ['--file-list', '--file-pair-list'].freeze
|
23
|
+
SUPPORTED_OPTIONS = %i[ascp_args wss].freeze
|
22
24
|
|
23
25
|
private_constant :SUPPORTED_AGENTS, :FILE_LIST_OPTIONS
|
24
26
|
|
@@ -103,10 +105,6 @@ module Aspera
|
|
103
105
|
ts.key?('EX_file_pair_list')
|
104
106
|
end
|
105
107
|
|
106
|
-
def ts_to_env_args(transfer_spec, wss:, ascp_args:)
|
107
|
-
return Parameters.new(transfer_spec, wss: wss, ascp_args: ascp_args).ascp_args
|
108
|
-
end
|
109
|
-
|
110
108
|
# temp file list files are created here
|
111
109
|
def file_list_folder=(v)
|
112
110
|
@file_list_folder = v
|
@@ -120,19 +118,20 @@ module Aspera
|
|
120
118
|
end # self
|
121
119
|
|
122
120
|
# @param options [Hash] key: :wss: bool, :ascp_args: array of strings
|
123
|
-
def initialize(job_spec,
|
121
|
+
def initialize(job_spec, options)
|
124
122
|
@job_spec = job_spec
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
raise 'ascp args must be an Array
|
123
|
+
# check necessary options
|
124
|
+
raise 'Internal: missing options' unless (SUPPORTED_OPTIONS - options.keys).empty?
|
125
|
+
@options = SUPPORTED_OPTIONS.each_with_object({}){|o, h| h[o] = options[o]}
|
126
|
+
Log.dump(:options, @options)
|
127
|
+
raise 'ascp args must be an Array' unless @options[:ascp_args].is_a?(Array)
|
128
|
+
raise 'ascp args must be an Array of String' if @options[:ascp_args].any?{|i|!i.is_a?(String)}
|
130
129
|
@builder = Aspera::CommandLineBuilder.new(@job_spec, self.class.description)
|
131
130
|
end
|
132
131
|
|
133
132
|
def process_file_list
|
134
133
|
# is the file list provided through EX_ parameters?
|
135
|
-
ascp_file_list_provided = self.class.ts_has_ascp_file_list(@job_spec, @
|
134
|
+
ascp_file_list_provided = self.class.ts_has_ascp_file_list(@job_spec, @options[:ascp_args])
|
136
135
|
# set if paths is mandatory in ts
|
137
136
|
@builder.params_definition['paths'][:mandatory] = !@job_spec.key?('keepalive') && !ascp_file_list_provided
|
138
137
|
# get paths in transfer spec (after setting if it is mandatory)
|
@@ -154,12 +153,14 @@ module Aspera
|
|
154
153
|
Log.log.debug('placing source file list on command line (no file list file)')
|
155
154
|
@builder.add_command_line_options(ts_paths_array.map{|i|i['source']})
|
156
155
|
else
|
156
|
+
raise "All elements of paths must have a 'source' key" unless ts_paths_array.all?{|i|i.key?('source')}
|
157
|
+
is_pair_list = ts_paths_array.any?{|i|i.key?('destination')}
|
158
|
+
raise "All elements of paths must be consistent with 'destination' key" if is_pair_list && !ts_paths_array.all?{|i|i.key?('destination')}
|
157
159
|
# safer option: generate a file list file if there is storage defined for it
|
158
|
-
# if there is destination in paths, then use file-pair-list
|
159
|
-
|
160
|
-
if ts_paths_array.first.key?('destination')
|
160
|
+
# if there is one destination in paths, then use file-pair-list
|
161
|
+
if is_pair_list
|
161
162
|
option = '--file-pair-list'
|
162
|
-
lines = ts_paths_array.each_with_object([]){|e, m|m.push(e['source'], e['destination']); }
|
163
|
+
lines = ts_paths_array.each_with_object([]){|e, m|m.push(e['source'], e['destination'] || e['source']); }
|
163
164
|
else
|
164
165
|
option = '--file-list'
|
165
166
|
lines = ts_paths_array.map{|i|i['source']}
|
@@ -194,7 +195,7 @@ module Aspera
|
|
194
195
|
@job_spec.delete('source_root') if @job_spec.key?('source_root') && @job_spec['source_root'].empty?
|
195
196
|
|
196
197
|
# use web socket session initiation ?
|
197
|
-
if @builder.read_param('wss_enabled') && (@
|
198
|
+
if @builder.read_param('wss_enabled') && (@options[:wss] || !@job_spec.key?('fasp_port'))
|
198
199
|
# by default use web socket session if available, unless removed by user
|
199
200
|
@builder.add_command_line_options(['--ws-connect'])
|
200
201
|
# TODO: option to give order ssh,ws (legacy http is implied bu ssh)
|
@@ -226,7 +227,7 @@ module Aspera
|
|
226
227
|
process_file_list
|
227
228
|
# optional args, at the end to override previous ones (to allow override)
|
228
229
|
@builder.add_command_line_options(@builder.read_param('EX_ascp_args'))
|
229
|
-
@builder.add_command_line_options(@
|
230
|
+
@builder.add_command_line_options(@options[:ascp_args])
|
230
231
|
# process destination folder
|
231
232
|
destination_folder = @builder.read_param('destination_root') || '/'
|
232
233
|
# ascp4 does not support base64 encoding of destination
|
@@ -8,6 +8,7 @@
|
|
8
8
|
# cli.switch : ascp: switch for ascp command line
|
9
9
|
# cli.convert : ascp: transform value: either a Hash with conversion values, or name of class
|
10
10
|
# cli.variable : ascp: name of env var
|
11
|
+
# cspell:words dgram
|
11
12
|
---
|
12
13
|
cipher:
|
13
14
|
:desc: "In transit encryption type."
|
@@ -569,7 +570,7 @@ EX_at_rest_password:
|
|
569
570
|
EX_proxy_password:
|
570
571
|
:desc: |-
|
571
572
|
Password used for Aspera proxy server authentication.
|
572
|
-
May be overridden by password in URL
|
573
|
+
May be overridden by password in URL provided in parameter: proxy.
|
573
574
|
|
574
575
|
|
575
576
|
:agents:
|
@@ -49,11 +49,11 @@ module Aspera
|
|
49
49
|
# failure in ascp
|
50
50
|
if e.retryable?
|
51
51
|
# exit if we exceed the max number of retry
|
52
|
-
raise Fasp::Error,
|
52
|
+
raise Fasp::Error, "Maximum number of retry reached (#{@parameters[:iter_max]})" if remaining_resumes <= 0
|
53
53
|
else
|
54
54
|
# give one chance only to non retryable errors
|
55
55
|
unless remaining_resumes.eql?(@parameters[:iter_max])
|
56
|
-
Log.log.error('non-retryable error')
|
56
|
+
Log.log.error('non-retryable error'.red.blink)
|
57
57
|
raise e
|
58
58
|
end
|
59
59
|
end
|
@@ -61,7 +61,7 @@ module Aspera
|
|
61
61
|
|
62
62
|
# take this retry in account
|
63
63
|
remaining_resumes -= 1
|
64
|
-
Log.log.warn{"
|
64
|
+
Log.log.warn{"Resuming in #{sleep_seconds} seconds (retry left:#{remaining_resumes})"}
|
65
65
|
|
66
66
|
# wait a bit before retrying, maybe network condition will be better
|
67
67
|
sleep(sleep_seconds)
|
@@ -17,11 +17,12 @@ module Aspera
|
|
17
17
|
}.freeze
|
18
18
|
# reserved tag for Aspera
|
19
19
|
TAG_RESERVED = 'aspera'
|
20
|
-
# define constants for enums of parameters: <parameter>_<enum>, e.g. CIPHER_AES_128
|
21
|
-
Aspera::Fasp::Parameters.description.each do |
|
22
|
-
next unless
|
23
|
-
|
24
|
-
|
20
|
+
# define constants for enums of parameters: <parameter>_<enum>, e.g. CIPHER_AES_128, DIRECTION_SEND, ...
|
21
|
+
Aspera::Fasp::Parameters.description.each do |name, description|
|
22
|
+
next unless description[:enum].is_a?(Array)
|
23
|
+
TransferSpec.const_set("#{name.to_s.upcase}_ENUM_VALUES", description[:enum])
|
24
|
+
description[:enum].each do |enum|
|
25
|
+
TransferSpec.const_set("#{name.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/, '_')}", enum.freeze)
|
25
26
|
end
|
26
27
|
end
|
27
28
|
class << self
|