aspera-cli 4.13.0 → 4.14.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +28 -5
- data/CONTRIBUTING.md +17 -1
- data/README.md +782 -401
- data/examples/dascli +1 -1
- data/examples/rubyc +24 -0
- data/lib/aspera/aoc.rb +21 -32
- data/lib/aspera/ascmd.rb +1 -0
- data/lib/aspera/cli/basic_auth_plugin.rb +6 -6
- data/lib/aspera/cli/formatter.rb +17 -25
- data/lib/aspera/cli/main.rb +21 -27
- data/lib/aspera/cli/manager.rb +128 -114
- data/lib/aspera/cli/plugin.rb +87 -38
- data/lib/aspera/cli/plugins/alee.rb +2 -2
- data/lib/aspera/cli/plugins/aoc.rb +216 -102
- data/lib/aspera/cli/plugins/ats.rb +16 -18
- data/lib/aspera/cli/plugins/bss.rb +3 -3
- data/lib/aspera/cli/plugins/config.rb +177 -367
- data/lib/aspera/cli/plugins/console.rb +4 -6
- data/lib/aspera/cli/plugins/cos.rb +12 -13
- data/lib/aspera/cli/plugins/faspex.rb +17 -18
- data/lib/aspera/cli/plugins/faspex5.rb +332 -216
- data/lib/aspera/cli/plugins/node.rb +171 -142
- data/lib/aspera/cli/plugins/orchestrator.rb +15 -18
- data/lib/aspera/cli/plugins/preview.rb +38 -60
- data/lib/aspera/cli/plugins/server.rb +22 -15
- data/lib/aspera/cli/plugins/shares.rb +24 -33
- data/lib/aspera/cli/plugins/sync.rb +3 -3
- data/lib/aspera/cli/transfer_agent.rb +29 -26
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +9 -7
- data/lib/aspera/data/6 +0 -0
- data/lib/aspera/environment.rb +7 -3
- data/lib/aspera/fasp/agent_connect.rb +5 -0
- data/lib/aspera/fasp/agent_direct.rb +5 -5
- data/lib/aspera/fasp/agent_httpgw.rb +138 -60
- data/lib/aspera/fasp/agent_trsdk.rb +2 -0
- data/lib/aspera/fasp/error_info.rb +2 -0
- data/lib/aspera/fasp/installation.rb +18 -19
- data/lib/aspera/fasp/parameters.rb +18 -17
- data/lib/aspera/fasp/parameters.yaml +2 -1
- data/lib/aspera/fasp/resume_policy.rb +3 -3
- data/lib/aspera/fasp/transfer_spec.rb +6 -5
- data/lib/aspera/fasp/uri.rb +23 -21
- data/lib/aspera/faspex_postproc.rb +1 -1
- data/lib/aspera/hash_ext.rb +12 -2
- data/lib/aspera/keychain/macos_security.rb +13 -13
- data/lib/aspera/log.rb +1 -0
- data/lib/aspera/node.rb +62 -80
- data/lib/aspera/oauth.rb +1 -1
- data/lib/aspera/persistency_action_once.rb +1 -1
- data/lib/aspera/preview/terminal.rb +61 -15
- data/lib/aspera/preview/utils.rb +3 -3
- data/lib/aspera/proxy_auto_config.js +2 -2
- data/lib/aspera/rest.rb +37 -0
- data/lib/aspera/secret_hider.rb +6 -1
- data/lib/aspera/ssh.rb +1 -1
- data/lib/aspera/sync.rb +2 -0
- data.tar.gz.sig +0 -0
- metadata +3 -4
- metadata.gz.sig +0 -0
- data/docs/test_env.conf +0 -186
- data/lib/aspera/data/7 +0 -0
@@ -25,7 +25,7 @@ module Aspera
|
|
25
25
|
multi_incr_udp: true,
|
26
26
|
resume: {},
|
27
27
|
ascp_args: [],
|
28
|
-
quiet: true # by default no
|
28
|
+
quiet: true # by default no native ascp progress bar
|
29
29
|
}.freeze
|
30
30
|
private_constant :DEFAULT_OPTIONS
|
31
31
|
|
@@ -82,12 +82,12 @@ module Aspera
|
|
82
82
|
end
|
83
83
|
|
84
84
|
# compute known args
|
85
|
-
env_args =
|
85
|
+
env_args = Parameters.new(transfer_spec, @options).ascp_args
|
86
86
|
|
87
87
|
# add fallback cert and key as arguments if needed
|
88
88
|
if ['1', 1, true, 'force'].include?(transfer_spec['http_fallback'])
|
89
|
-
env_args[:args].unshift('-Y', Installation.instance.path(:
|
90
|
-
env_args[:args].unshift('-I', Installation.instance.path(:
|
89
|
+
env_args[:args].unshift('-Y', Installation.instance.path(:fallback_cert_privkey))
|
90
|
+
env_args[:args].unshift('-I', Installation.instance.path(:fallback_certificate))
|
91
91
|
end
|
92
92
|
|
93
93
|
env_args[:args].unshift('-q') if @options[:quiet]
|
@@ -334,7 +334,7 @@ module Aspera
|
|
334
334
|
# @param options : keys(symbol): see DEFAULT_OPTIONS
|
335
335
|
def initialize(options=nil)
|
336
336
|
super()
|
337
|
-
# all transfer jobs, key = SecureRandom.uuid, protected by mutex,
|
337
|
+
# all transfer jobs, key = SecureRandom.uuid, protected by mutex, cond var on change
|
338
338
|
@jobs = {}
|
339
339
|
# mutex protects global data accessed by threads
|
340
340
|
@mutex = Mutex.new
|
@@ -9,6 +9,17 @@ require 'websocket'
|
|
9
9
|
require 'base64'
|
10
10
|
require 'json'
|
11
11
|
|
12
|
+
# HTTP GW Upload protocol
|
13
|
+
# -----------------------
|
14
|
+
# v1
|
15
|
+
# 1 - MessageType: String (Transfer Spec) JSON : type: transfer_spec, acknowledged with "end upload"
|
16
|
+
# 2.. - MessageType: String (Slice Upload start) JSON : type: slice_upload, acknowledged with "end upload"
|
17
|
+
# v2
|
18
|
+
# 1 - MessageType: String (Transfer Spec) JSON : type: transfer_spec, acknowledged with "end upload"
|
19
|
+
# 2 - MessageType: String (Slice Upload start) JSON : type: slice_upload, acknowledged with "end_slice_upload"
|
20
|
+
# 3.. - MessageType: ByteArray (File Size) Chunks : acknowledged with "end upload"
|
21
|
+
# last - MessageType: String (Slice Upload end) JSON : type: slice_upload, acknowledged with "end_slice_upload"
|
22
|
+
|
12
23
|
# ref: https://api.ibm.com/explorer/catalog/aspera/product/ibm-aspera/api/http-gateway-api/doc/guides-toc
|
13
24
|
# https://developer.ibm.com/apis/catalog?search=%22aspera%20http%22
|
14
25
|
module Aspera
|
@@ -16,35 +27,66 @@ module Aspera
|
|
16
27
|
# start a transfer using Aspera HTTP Gateway, using web socket session for uploads
|
17
28
|
class AgentHttpgw < Aspera::Fasp::AgentBase
|
18
29
|
# message returned by HTTP GW in case of success
|
19
|
-
|
20
|
-
|
30
|
+
MSG_RECV_DATA_RECEIVED_SIGNAL = 'end upload'
|
31
|
+
MSG_RECV_SLICE_UPLOAD_SIGNAL = 'end_slice_upload'
|
32
|
+
MSG_SEND_SLICE_UPLOAD = 'slice_upload'
|
33
|
+
MSG_SEND_TRANSFER_SPEC = 'transfer_spec'
|
34
|
+
# upload API versions
|
35
|
+
API_V1 = 'v1'
|
36
|
+
API_V2 = 'v2'
|
21
37
|
# options available in CLI (transfer_info)
|
22
38
|
DEFAULT_OPTIONS = {
|
23
39
|
url: nil,
|
24
40
|
upload_chunk_size: 64_000,
|
25
|
-
upload_bar_refresh_sec: 0.5
|
41
|
+
upload_bar_refresh_sec: 0.5,
|
42
|
+
api_version: API_V2,
|
43
|
+
synchronous: true
|
26
44
|
}.freeze
|
27
45
|
DEFAULT_BASE_PATH = '/aspera/http-gwy'
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
private_constant :DEFAULT_OPTIONS, :MSG_END_UPLOAD, :MSG_END_SLICE, :V1_UPLOAD, :V2_UPLOAD
|
46
|
+
LOG_WS_MAIN = 'ws: send: '.green
|
47
|
+
LOG_WS_THREAD = 'ws: ack: '.red
|
48
|
+
private_constant :DEFAULT_OPTIONS, :MSG_RECV_DATA_RECEIVED_SIGNAL, :MSG_RECV_SLICE_UPLOAD_SIGNAL, :API_V1, :API_V2
|
32
49
|
|
33
50
|
# send message on http gw web socket
|
34
|
-
def ws_snd_json(
|
35
|
-
|
36
|
-
|
37
|
-
|
51
|
+
def ws_snd_json(msg_type, payload)
|
52
|
+
if msg_type.eql?(MSG_SEND_SLICE_UPLOAD) && @options[:api_version].eql?(API_V2)
|
53
|
+
@shared_info[:count][:sent_v2_slice] += 1
|
54
|
+
else
|
55
|
+
@shared_info[:count][:sent_other] += 1
|
56
|
+
end
|
57
|
+
Log.log.debug do
|
58
|
+
log_data = payload.dup
|
59
|
+
log_data[:data] = "[data #{log_data[:data].length} bytes]" if log_data.key?(:data)
|
60
|
+
"send_txt: #{msg_type}: #{JSON.generate(log_data)}"
|
61
|
+
end
|
62
|
+
ws_send(JSON.generate({msg_type => payload}))
|
38
63
|
end
|
39
64
|
|
40
|
-
def ws_send(
|
41
|
-
|
65
|
+
def ws_send(data_to_send, type: :text)
|
66
|
+
Log.log.debug{"#{LOG_WS_MAIN}send low: type: #{type}"}
|
67
|
+
@shared_info[:count][:sent_other] += 1 if type.eql?(:binary)
|
68
|
+
Log.log.debug{"#{LOG_WS_MAIN}counts: #{@shared_info[:count]}"}
|
69
|
+
frame = ::WebSocket::Frame::Outgoing::Client.new(data: data_to_send, type: type, version: @ws_handshake.version)
|
42
70
|
@ws_io.write(frame.to_s)
|
43
71
|
end
|
44
72
|
|
73
|
+
# wait for all message sent to be acknowledged by HTTPGW server
|
74
|
+
def wait_for_sent_msg_ack_or_exception
|
75
|
+
return unless @options[:synchronous]
|
76
|
+
@shared_info[:mutex].synchronize do
|
77
|
+
while (@shared_info[:count][:received_data] != @shared_info[:count][:sent_other]) ||
|
78
|
+
(@shared_info[:count][:received_v2_slice] != @shared_info[:count][:sent_v2_slice])
|
79
|
+
Log.log.debug{"#{LOG_WS_MAIN}wait: counts: #{@shared_info[:count]}"}
|
80
|
+
@shared_info[:cond_var].wait(@shared_info[:mutex], 1.0)
|
81
|
+
raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
|
82
|
+
end
|
83
|
+
end
|
84
|
+
Log.log.debug{"#{LOG_WS_MAIN}sync ok: counts: #{@shared_info[:count]}"}
|
85
|
+
end
|
86
|
+
|
45
87
|
def upload(transfer_spec)
|
46
88
|
# total size of all files
|
47
|
-
|
89
|
+
total_bytes_to_transfer = 0
|
48
90
|
# we need to keep track of actual file path because transfer spec is modified to be sent in web socket
|
49
91
|
source_paths = []
|
50
92
|
# get source root or nil
|
@@ -61,50 +103,65 @@ module Aspera
|
|
61
103
|
# GW expects a simple file name in 'source' but if user wants to change the name, we take it
|
62
104
|
item['source'] = File.basename(item['destination'].nil? ? item['source'] : item['destination'])
|
63
105
|
item['file_size'] = File.size(full_src_filepath)
|
64
|
-
|
106
|
+
total_bytes_to_transfer += item['file_size']
|
65
107
|
# save so that we can actually read the file later
|
66
108
|
source_paths.push(full_src_filepath)
|
67
109
|
end
|
68
110
|
# identify this session uniquely
|
69
111
|
session_id = SecureRandom.uuid
|
70
|
-
|
71
|
-
#
|
72
|
-
upload_api_version = V2_UPLOAD
|
73
|
-
# is the latest supported? else revert to old api
|
74
|
-
upload_api_version = V1_UPLOAD unless @api_info['endpoints'].any?{|i|i.include?(upload_api_version)}
|
75
|
-
Log.log.debug{"api version: #{upload_api_version}"}
|
76
|
-
url = File.join(@gw_api.params[:base_url], upload_api_version)
|
77
|
-
# uri = URI.parse(url)
|
112
|
+
upload_url = File.join(@gw_api.params[:base_url], @options[:api_version], 'upload')
|
113
|
+
# uri = URI.parse(upload_url)
|
78
114
|
# open web socket to end point (equivalent to Net::HTTP.start)
|
79
|
-
http_socket = Rest.start_http_session(
|
115
|
+
http_socket = Rest.start_http_session(upload_url)
|
116
|
+
# little hack to get the socket opened for HTTP, handy because HTTP debug will be available
|
80
117
|
@ws_io = http_socket.instance_variable_get(:@socket)
|
81
118
|
# @ws_io.debug_output = Log.log
|
82
|
-
@ws_handshake = ::WebSocket::Handshake::Client.new(url:
|
119
|
+
@ws_handshake = ::WebSocket::Handshake::Client.new(url: upload_url, headers: {})
|
83
120
|
@ws_io.write(@ws_handshake.to_s)
|
84
121
|
sleep(0.1)
|
85
122
|
@ws_handshake << @ws_io.readuntil("\r\n\r\n")
|
86
123
|
raise 'Error in websocket handshake' unless @ws_handshake.finished?
|
87
|
-
Log.log.debug
|
124
|
+
Log.log.debug{"#{LOG_WS_MAIN}handshake success"}
|
88
125
|
# data shared between main thread and read thread
|
89
|
-
shared_info = {
|
126
|
+
@shared_info = {
|
90
127
|
read_exception: nil, # error message if any in callback
|
91
|
-
|
92
|
-
|
93
|
-
|
128
|
+
count: {
|
129
|
+
received_data: 0, # number of files received on other side
|
130
|
+
received_v2_slice: 0, # number of slices received on other side
|
131
|
+
sent_other: 0,
|
132
|
+
sent_v2_slice: 0
|
133
|
+
},
|
134
|
+
mutex: Mutex.new,
|
135
|
+
cond_var: ConditionVariable.new
|
94
136
|
}
|
95
137
|
# start read thread
|
96
138
|
ws_read_thread = Thread.new do
|
97
|
-
Log.log.debug
|
98
|
-
frame = ::WebSocket::Frame::Incoming::Client.new
|
139
|
+
Log.log.debug{"#{LOG_WS_THREAD}read started"}
|
140
|
+
frame = ::WebSocket::Frame::Incoming::Client.new(version: @ws_handshake.version)
|
99
141
|
loop do
|
100
142
|
begin # rubocop:disable Style/RedundantBegin
|
101
|
-
|
143
|
+
# unless (recv_data = @ws_io.getc)
|
144
|
+
# sleep(0.1)
|
145
|
+
# next
|
146
|
+
# end
|
147
|
+
# frame << recv_data
|
148
|
+
# frame << @ws_io.readuntil("\n")
|
149
|
+
# frame << @ws_io.read_all
|
150
|
+
frame << @ws_io.read(1)
|
102
151
|
while (msg = frame.next)
|
103
|
-
Log.log.debug{"
|
152
|
+
Log.log.debug{"#{LOG_WS_THREAD}type: #{msg.class}"}
|
104
153
|
message = msg.data
|
105
|
-
|
106
|
-
|
107
|
-
|
154
|
+
Log.log.debug{"#{LOG_WS_THREAD}message: [#{message}]"}
|
155
|
+
if message.eql?(MSG_RECV_DATA_RECEIVED_SIGNAL)
|
156
|
+
@shared_info[:mutex].synchronize do
|
157
|
+
@shared_info[:count][:received_data] += 1
|
158
|
+
@shared_info[:cond_var].signal
|
159
|
+
end
|
160
|
+
elsif message.eql?(MSG_RECV_SLICE_UPLOAD_SIGNAL)
|
161
|
+
@shared_info[:mutex].synchronize do
|
162
|
+
@shared_info[:count][:received_v2_slice] += 1
|
163
|
+
@shared_info[:cond_var].signal
|
164
|
+
end
|
108
165
|
else
|
109
166
|
message.chomp!
|
110
167
|
error_message =
|
@@ -117,19 +174,25 @@ module Aspera
|
|
117
174
|
end
|
118
175
|
raise error_message
|
119
176
|
end
|
120
|
-
|
177
|
+
Log.log.debug{"#{LOG_WS_THREAD}counts: #{@shared_info[:count]}"}
|
178
|
+
end # while
|
121
179
|
rescue => e
|
122
|
-
|
180
|
+
Log.log.debug{"#{LOG_WS_THREAD}Exception: #{e}"}
|
181
|
+
@shared_info[:mutex].synchronize do
|
182
|
+
@shared_info[:read_exception] = e unless e.is_a?(EOFError)
|
183
|
+
@shared_info[:cond_var].signal
|
184
|
+
end
|
123
185
|
break
|
124
|
-
end
|
125
|
-
end
|
126
|
-
Log.log.debug{"
|
186
|
+
end # begin
|
187
|
+
end # loop
|
188
|
+
Log.log.debug{"#{LOG_WS_THREAD}stopping (exc=#{@shared_info[:read_exception]},cls=#{@shared_info[:read_exception].class})"}
|
127
189
|
end
|
128
190
|
# notify progress bar
|
129
|
-
notify_begin(session_id,
|
191
|
+
notify_begin(session_id, total_bytes_to_transfer)
|
130
192
|
# first step send transfer spec
|
131
193
|
Log.dump(:ws_spec, transfer_spec)
|
132
|
-
ws_snd_json(
|
194
|
+
ws_snd_json(MSG_SEND_TRANSFER_SPEC, transfer_spec)
|
195
|
+
wait_for_sent_msg_ack_or_exception
|
133
196
|
# current file index
|
134
197
|
file_index = 0
|
135
198
|
# aggregate size sent
|
@@ -148,7 +211,7 @@ module Aspera
|
|
148
211
|
# current slice index
|
149
212
|
slice_index = 0
|
150
213
|
until file.eof?
|
151
|
-
|
214
|
+
file_bin_data = file.read(@options[:upload_chunk_size])
|
152
215
|
slice_data = {
|
153
216
|
name: file_name,
|
154
217
|
type: file_mime_type,
|
@@ -159,22 +222,26 @@ module Aspera
|
|
159
222
|
}
|
160
223
|
# Log.dump(:slice_data,slice_data) #if slice_index.eql?(0)
|
161
224
|
# interrupt main thread if read thread failed
|
162
|
-
raise shared_info[:read_exception] unless shared_info[:read_exception].nil?
|
225
|
+
raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
|
163
226
|
begin
|
164
|
-
if
|
165
|
-
slice_data[:data] = Base64.strict_encode64(
|
166
|
-
ws_snd_json(
|
227
|
+
if @options[:api_version].eql?(API_V1)
|
228
|
+
slice_data[:data] = Base64.strict_encode64(file_bin_data)
|
229
|
+
ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_data)
|
167
230
|
else
|
168
|
-
ws_snd_json(
|
169
|
-
ws_send(
|
170
|
-
Log.log.debug{"
|
171
|
-
ws_snd_json(
|
231
|
+
ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_data) if slice_index.eql?(0)
|
232
|
+
ws_send(file_bin_data, type: :binary)
|
233
|
+
Log.log.debug{"#{LOG_WS_MAIN}sent bin buffer: #{file_index} / #{slice_index}"}
|
234
|
+
ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_data) if slice_index.eql?(slice_total - 1)
|
172
235
|
end
|
236
|
+
wait_for_sent_msg_ack_or_exception
|
173
237
|
rescue Errno::EPIPE => e
|
174
|
-
raise shared_info[:read_exception] unless shared_info[:read_exception].nil?
|
238
|
+
raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
|
239
|
+
raise e
|
240
|
+
rescue Net::ReadTimeout => e
|
241
|
+
Log.log.warn{'A timeout condition using HTTPGW may signal a permission problem on destination. Check ascp logs on httpgw.'}
|
175
242
|
raise e
|
176
243
|
end
|
177
|
-
sent_bytes +=
|
244
|
+
sent_bytes += file_bin_data.length
|
178
245
|
current_time = Time.now
|
179
246
|
if last_progress_time.nil? || ((current_time - last_progress_time) > @options[:upload_bar_refresh_sec])
|
180
247
|
notify_progress(session_id, sent_bytes)
|
@@ -186,9 +253,9 @@ module Aspera
|
|
186
253
|
file_index += 1
|
187
254
|
end
|
188
255
|
|
189
|
-
Log.log.debug('Finished upload')
|
256
|
+
Log.log.debug('Finished upload, waiting for end of read thread.')
|
190
257
|
ws_read_thread.join
|
191
|
-
Log.log.debug{"result: #{shared_info[:
|
258
|
+
Log.log.debug{"Read thread joined, result: #{@shared_info[:count][:received_data]} / #{@shared_info[:count][:sent_other]}"}
|
192
259
|
ws_send(nil, type: :close) unless @ws_io.nil?
|
193
260
|
@ws_io = nil
|
194
261
|
http_socket&.finish
|
@@ -258,7 +325,7 @@ module Aspera
|
|
258
325
|
private
|
259
326
|
|
260
327
|
def initialize(opts)
|
261
|
-
Log.
|
328
|
+
Log.dump(:in_options, opts)
|
262
329
|
# set default options and override if specified
|
263
330
|
@options = DEFAULT_OPTIONS.dup
|
264
331
|
raise "httpgw agent parameters (transfer_info): expecting Hash, but have #{opts.class}" unless opts.is_a?(Hash)
|
@@ -266,13 +333,24 @@ module Aspera
|
|
266
333
|
raise "httpgw agent parameter: Unknown: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.key?(k)
|
267
334
|
@options[k] = v
|
268
335
|
end
|
269
|
-
|
336
|
+
if @options[:url].nil?
|
337
|
+
available = DEFAULT_OPTIONS.map { |k, v| "#{k}(#{v})"}.join(', ')
|
338
|
+
raise "Missing mandatory parameter for HTTP GW in transfer_info: url. Allowed parameters: #{available}."
|
339
|
+
end
|
270
340
|
# remove /v1 from end
|
271
341
|
@options[:url].gsub(%r{/v1/*$}, '')
|
272
342
|
super()
|
273
343
|
@gw_api = Rest.new({base_url: @options[:url]})
|
274
344
|
@api_info = @gw_api.read('v1/info')[:data]
|
275
|
-
Log.
|
345
|
+
Log.dump(:api_info, @api_info)
|
346
|
+
if @options[:api_version].nil?
|
347
|
+
# web socket endpoint: by default use v2 (newer gateways), without base64 encoding
|
348
|
+
@options[:api_version] = API_V2
|
349
|
+
# is the latest supported? else revert to old api
|
350
|
+
@options[:api_version] = API_V1 unless @api_info['endpoints'].any?{|i|i.include?(@options[:api_version])}
|
351
|
+
end
|
352
|
+
@options.freeze
|
353
|
+
Log.dump(:final_options, @options)
|
276
354
|
end
|
277
355
|
end # AgentHttpgw
|
278
356
|
end
|
@@ -126,7 +126,11 @@ module Aspera
|
|
126
126
|
end
|
127
127
|
|
128
128
|
# all ascp files (in SDK)
|
129
|
-
FILES = %i[ascp ascp4
|
129
|
+
FILES = %i[ascp ascp4 ssh_bypass_dsa_privkey ssh_bypass_rsa_privkey aspera_license aspera_conf fallback_certificate fallback_cert_privkey].freeze
|
130
|
+
|
131
|
+
def check_or_create_sdk_file(filename, force: false, &block)
|
132
|
+
return Environment.write_file_restricted(File.join(sdk_folder, filename), force: force, mode: 0o644, &block)
|
133
|
+
end
|
130
134
|
|
131
135
|
# get path of one resource file of currently activated product
|
132
136
|
# keys and certs are generated locally... (they are well known values, arch. independent)
|
@@ -139,23 +143,18 @@ module Aspera
|
|
139
143
|
file = file.gsub('ascp', 'ascp4') if k.eql?(:ascp4)
|
140
144
|
when :transferd
|
141
145
|
file = transferd_filepath
|
142
|
-
when :
|
143
|
-
file =
|
144
|
-
when :
|
145
|
-
file =
|
146
|
+
when :ssh_bypass_dsa_privkey
|
147
|
+
file = check_or_create_sdk_file('aspera_bypass_dsa.pem') {get_key('dsa', 1)}
|
148
|
+
when :ssh_bypass_rsa_privkey
|
149
|
+
file = check_or_create_sdk_file('aspera_bypass_rsa.pem') {get_key('rsa', 2)}
|
146
150
|
when :aspera_license
|
147
|
-
file =
|
148
|
-
|
149
|
-
Zlib::Inflate.inflate(DataRepository.instance.data(6)),
|
150
|
-
"==SIGNATURE==\n",
|
151
|
-
Base64.strict_encode64(DataRepository.instance.data(7))
|
152
|
-
]
|
153
|
-
Base64.strict_encode64(clear.join)
|
151
|
+
file = check_or_create_sdk_file('aspera-license') do
|
152
|
+
Zlib::Inflate.inflate(DataRepository.instance.data(6))
|
154
153
|
end
|
155
154
|
when :aspera_conf
|
156
|
-
file =
|
157
|
-
when :
|
158
|
-
file_key = File.join(sdk_folder, '
|
155
|
+
file = check_or_create_sdk_file('aspera.conf') {DEFAULT_ASPERA_CONF}
|
156
|
+
when :fallback_certificate, :fallback_cert_privkey
|
157
|
+
file_key = File.join(sdk_folder, 'aspera_fallback_cert_private_key.pem')
|
159
158
|
file_cert = File.join(sdk_folder, 'aspera_fallback_cert.pem')
|
160
159
|
if !File.exist?(file_key) || !File.exist?(file_cert)
|
161
160
|
require 'openssl'
|
@@ -169,10 +168,10 @@ module Aspera
|
|
169
168
|
cert.serial = 0x0
|
170
169
|
cert.version = 2
|
171
170
|
cert.sign(private_key, OpenSSL::Digest.new('SHA1'))
|
172
|
-
|
173
|
-
|
171
|
+
check_or_create_sdk_file('aspera_fallback_cert_private_key.pem', force: true) {private_key.to_pem}
|
172
|
+
check_or_create_sdk_file('aspera_fallback_cert.pem', force: true) {cert.to_pem}
|
174
173
|
end
|
175
|
-
file = k.eql?(:
|
174
|
+
file = k.eql?(:fallback_certificate) ? file_cert : file_key
|
176
175
|
else
|
177
176
|
raise "INTERNAL ERROR: #{k}"
|
178
177
|
end
|
@@ -206,7 +205,7 @@ module Aspera
|
|
206
205
|
end
|
207
206
|
|
208
207
|
def bypass_keys
|
209
|
-
return %i[
|
208
|
+
return %i[ssh_bypass_dsa_privkey ssh_bypass_rsa_privkey].map{|i|Installation.instance.path(i)}
|
210
209
|
end
|
211
210
|
|
212
211
|
# use in plugin `config`
|
@@ -3,6 +3,7 @@
|
|
3
3
|
require 'aspera/log'
|
4
4
|
require 'aspera/command_line_builder'
|
5
5
|
require 'aspera/temp_file_manager'
|
6
|
+
require 'aspera/fasp/error'
|
6
7
|
require 'securerandom'
|
7
8
|
require 'base64'
|
8
9
|
require 'json'
|
@@ -19,6 +20,7 @@ module Aspera
|
|
19
20
|
# Short names of columns in manual
|
20
21
|
SUPPORTED_AGENTS_SHORT = SUPPORTED_AGENTS.map{|a|a.to_s[0].to_sym}
|
21
22
|
FILE_LIST_OPTIONS = ['--file-list', '--file-pair-list'].freeze
|
23
|
+
SUPPORTED_OPTIONS = %i[ascp_args wss].freeze
|
22
24
|
|
23
25
|
private_constant :SUPPORTED_AGENTS, :FILE_LIST_OPTIONS
|
24
26
|
|
@@ -103,10 +105,6 @@ module Aspera
|
|
103
105
|
ts.key?('EX_file_pair_list')
|
104
106
|
end
|
105
107
|
|
106
|
-
def ts_to_env_args(transfer_spec, wss:, ascp_args:)
|
107
|
-
return Parameters.new(transfer_spec, wss: wss, ascp_args: ascp_args).ascp_args
|
108
|
-
end
|
109
|
-
|
110
108
|
# temp file list files are created here
|
111
109
|
def file_list_folder=(v)
|
112
110
|
@file_list_folder = v
|
@@ -120,19 +118,20 @@ module Aspera
|
|
120
118
|
end # self
|
121
119
|
|
122
120
|
# @param options [Hash] key: :wss: bool, :ascp_args: array of strings
|
123
|
-
def initialize(job_spec,
|
121
|
+
def initialize(job_spec, options)
|
124
122
|
@job_spec = job_spec
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
raise 'ascp args must be an Array
|
123
|
+
# check necessary options
|
124
|
+
raise 'Internal: missing options' unless (SUPPORTED_OPTIONS - options.keys).empty?
|
125
|
+
@options = SUPPORTED_OPTIONS.each_with_object({}){|o, h| h[o] = options[o]}
|
126
|
+
Log.dump(:options, @options)
|
127
|
+
raise 'ascp args must be an Array' unless @options[:ascp_args].is_a?(Array)
|
128
|
+
raise 'ascp args must be an Array of String' if @options[:ascp_args].any?{|i|!i.is_a?(String)}
|
130
129
|
@builder = Aspera::CommandLineBuilder.new(@job_spec, self.class.description)
|
131
130
|
end
|
132
131
|
|
133
132
|
def process_file_list
|
134
133
|
# is the file list provided through EX_ parameters?
|
135
|
-
ascp_file_list_provided = self.class.ts_has_ascp_file_list(@job_spec, @
|
134
|
+
ascp_file_list_provided = self.class.ts_has_ascp_file_list(@job_spec, @options[:ascp_args])
|
136
135
|
# set if paths is mandatory in ts
|
137
136
|
@builder.params_definition['paths'][:mandatory] = !@job_spec.key?('keepalive') && !ascp_file_list_provided
|
138
137
|
# get paths in transfer spec (after setting if it is mandatory)
|
@@ -154,12 +153,14 @@ module Aspera
|
|
154
153
|
Log.log.debug('placing source file list on command line (no file list file)')
|
155
154
|
@builder.add_command_line_options(ts_paths_array.map{|i|i['source']})
|
156
155
|
else
|
156
|
+
raise "All elements of paths must have a 'source' key" unless ts_paths_array.all?{|i|i.key?('source')}
|
157
|
+
is_pair_list = ts_paths_array.any?{|i|i.key?('destination')}
|
158
|
+
raise "All elements of paths must be consistent with 'destination' key" if is_pair_list && !ts_paths_array.all?{|i|i.key?('destination')}
|
157
159
|
# safer option: generate a file list file if there is storage defined for it
|
158
|
-
# if there is destination in paths, then use file-pair-list
|
159
|
-
|
160
|
-
if ts_paths_array.first.key?('destination')
|
160
|
+
# if there is one destination in paths, then use file-pair-list
|
161
|
+
if is_pair_list
|
161
162
|
option = '--file-pair-list'
|
162
|
-
lines = ts_paths_array.each_with_object([]){|e, m|m.push(e['source'], e['destination']); }
|
163
|
+
lines = ts_paths_array.each_with_object([]){|e, m|m.push(e['source'], e['destination'] || e['source']); }
|
163
164
|
else
|
164
165
|
option = '--file-list'
|
165
166
|
lines = ts_paths_array.map{|i|i['source']}
|
@@ -194,7 +195,7 @@ module Aspera
|
|
194
195
|
@job_spec.delete('source_root') if @job_spec.key?('source_root') && @job_spec['source_root'].empty?
|
195
196
|
|
196
197
|
# use web socket session initiation ?
|
197
|
-
if @builder.read_param('wss_enabled') && (@
|
198
|
+
if @builder.read_param('wss_enabled') && (@options[:wss] || !@job_spec.key?('fasp_port'))
|
198
199
|
# by default use web socket session if available, unless removed by user
|
199
200
|
@builder.add_command_line_options(['--ws-connect'])
|
200
201
|
# TODO: option to give order ssh,ws (legacy http is implied bu ssh)
|
@@ -226,7 +227,7 @@ module Aspera
|
|
226
227
|
process_file_list
|
227
228
|
# optional args, at the end to override previous ones (to allow override)
|
228
229
|
@builder.add_command_line_options(@builder.read_param('EX_ascp_args'))
|
229
|
-
@builder.add_command_line_options(@
|
230
|
+
@builder.add_command_line_options(@options[:ascp_args])
|
230
231
|
# process destination folder
|
231
232
|
destination_folder = @builder.read_param('destination_root') || '/'
|
232
233
|
# ascp4 does not support base64 encoding of destination
|
@@ -8,6 +8,7 @@
|
|
8
8
|
# cli.switch : ascp: switch for ascp command line
|
9
9
|
# cli.convert : ascp: transform value: either a Hash with conversion values, or name of class
|
10
10
|
# cli.variable : ascp: name of env var
|
11
|
+
# cspell:words dgram
|
11
12
|
---
|
12
13
|
cipher:
|
13
14
|
:desc: "In transit encryption type."
|
@@ -569,7 +570,7 @@ EX_at_rest_password:
|
|
569
570
|
EX_proxy_password:
|
570
571
|
:desc: |-
|
571
572
|
Password used for Aspera proxy server authentication.
|
572
|
-
May be overridden by password in URL
|
573
|
+
May be overridden by password in URL provided in parameter: proxy.
|
573
574
|
|
574
575
|
|
575
576
|
:agents:
|
@@ -49,11 +49,11 @@ module Aspera
|
|
49
49
|
# failure in ascp
|
50
50
|
if e.retryable?
|
51
51
|
# exit if we exceed the max number of retry
|
52
|
-
raise Fasp::Error,
|
52
|
+
raise Fasp::Error, "Maximum number of retry reached (#{@parameters[:iter_max]})" if remaining_resumes <= 0
|
53
53
|
else
|
54
54
|
# give one chance only to non retryable errors
|
55
55
|
unless remaining_resumes.eql?(@parameters[:iter_max])
|
56
|
-
Log.log.error('non-retryable error')
|
56
|
+
Log.log.error('non-retryable error'.red.blink)
|
57
57
|
raise e
|
58
58
|
end
|
59
59
|
end
|
@@ -61,7 +61,7 @@ module Aspera
|
|
61
61
|
|
62
62
|
# take this retry in account
|
63
63
|
remaining_resumes -= 1
|
64
|
-
Log.log.warn{"
|
64
|
+
Log.log.warn{"Resuming in #{sleep_seconds} seconds (retry left:#{remaining_resumes})"}
|
65
65
|
|
66
66
|
# wait a bit before retrying, maybe network condition will be better
|
67
67
|
sleep(sleep_seconds)
|
@@ -17,11 +17,12 @@ module Aspera
|
|
17
17
|
}.freeze
|
18
18
|
# reserved tag for Aspera
|
19
19
|
TAG_RESERVED = 'aspera'
|
20
|
-
# define constants for enums of parameters: <parameter>_<enum>, e.g. CIPHER_AES_128
|
21
|
-
Aspera::Fasp::Parameters.description.each do |
|
22
|
-
next unless
|
23
|
-
|
24
|
-
|
20
|
+
# define constants for enums of parameters: <parameter>_<enum>, e.g. CIPHER_AES_128, DIRECTION_SEND, ...
|
21
|
+
Aspera::Fasp::Parameters.description.each do |name, description|
|
22
|
+
next unless description[:enum].is_a?(Array)
|
23
|
+
TransferSpec.const_set("#{name.to_s.upcase}_ENUM_VALUES", description[:enum])
|
24
|
+
description[:enum].each do |enum|
|
25
|
+
TransferSpec.const_set("#{name.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/, '_')}", enum.freeze)
|
25
26
|
end
|
26
27
|
end
|
27
28
|
class << self
|