aspera-cli 4.17.0 → 4.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +2 -4
- data/CHANGELOG.md +23 -0
- data/CONTRIBUTING.md +15 -1
- data/README.md +620 -378
- data/bin/ascli +5 -0
- data/bin/asession +2 -2
- data/lib/aspera/agent/alpha.rb +6 -4
- data/lib/aspera/agent/base.rb +9 -6
- data/lib/aspera/agent/connect.rb +4 -4
- data/lib/aspera/agent/direct.rb +56 -37
- data/lib/aspera/agent/httpgw.rb +23 -324
- data/lib/aspera/agent/node.rb +19 -20
- data/lib/aspera/agent/trsdk.rb +19 -20
- data/lib/aspera/api/aoc.rb +17 -14
- data/lib/aspera/api/cos_node.rb +4 -4
- data/lib/aspera/api/httpgw.rb +339 -0
- data/lib/aspera/api/node.rb +34 -21
- data/lib/aspera/ascmd.rb +4 -3
- data/lib/aspera/ascp/installation.rb +15 -7
- data/lib/aspera/ascp/management.rb +2 -2
- data/lib/aspera/cli/basic_auth_plugin.rb +5 -9
- data/lib/aspera/cli/extended_value.rb +12 -6
- data/lib/aspera/cli/formatter.rb +155 -65
- data/lib/aspera/cli/hints.rb +18 -0
- data/lib/aspera/cli/main.rb +22 -29
- data/lib/aspera/cli/manager.rb +53 -36
- data/lib/aspera/cli/plugin.rb +26 -17
- data/lib/aspera/cli/plugin_factory.rb +31 -20
- data/lib/aspera/cli/plugins/alee.rb +14 -2
- data/lib/aspera/cli/plugins/aoc.rb +141 -131
- data/lib/aspera/cli/plugins/ats.rb +1 -1
- data/lib/aspera/cli/plugins/config.rb +52 -46
- data/lib/aspera/cli/plugins/console.rb +8 -5
- data/lib/aspera/cli/plugins/faspex.rb +27 -19
- data/lib/aspera/cli/plugins/faspex5.rb +222 -149
- data/lib/aspera/cli/plugins/faspio.rb +85 -0
- data/lib/aspera/cli/plugins/httpgw.rb +55 -0
- data/lib/aspera/cli/plugins/node.rb +86 -29
- data/lib/aspera/cli/plugins/orchestrator.rb +31 -29
- data/lib/aspera/cli/plugins/preview.rb +6 -2
- data/lib/aspera/cli/plugins/server.rb +5 -5
- data/lib/aspera/cli/plugins/shares.rb +16 -14
- data/lib/aspera/cli/sync_actions.rb +6 -6
- data/lib/aspera/cli/transfer_agent.rb +5 -4
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/environment.rb +7 -6
- data/lib/aspera/faspex_gw.rb +5 -4
- data/lib/aspera/faspex_postproc.rb +2 -2
- data/lib/aspera/log.rb +6 -3
- data/lib/aspera/node_simulator.rb +2 -2
- data/lib/aspera/oauth/base.rb +31 -19
- data/lib/aspera/oauth/factory.rb +12 -13
- data/lib/aspera/oauth/generic.rb +1 -0
- data/lib/aspera/oauth/jwt.rb +18 -15
- data/lib/aspera/oauth/url_json.rb +8 -6
- data/lib/aspera/open_application.rb +5 -7
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/generator.rb +3 -3
- data/lib/aspera/preview/options.rb +3 -3
- data/lib/aspera/preview/terminal.rb +4 -4
- data/lib/aspera/preview/utils.rb +3 -3
- data/lib/aspera/proxy_auto_config.rb +5 -1
- data/lib/aspera/rest.rb +60 -74
- data/lib/aspera/rest_call_error.rb +1 -1
- data/lib/aspera/rest_error_analyzer.rb +2 -2
- data/lib/aspera/rest_errors_aspera.rb +1 -1
- data/lib/aspera/resumer.rb +1 -1
- data/lib/aspera/secret_hider.rb +2 -4
- data/lib/aspera/ssh.rb +1 -1
- data/lib/aspera/transfer/parameters.rb +39 -36
- data/lib/aspera/transfer/spec.rb +2 -0
- data/lib/aspera/transfer/sync.rb +2 -1
- data/lib/aspera/transfer/uri.rb +1 -1
- data/lib/aspera/uri_reader.rb +5 -4
- data/lib/aspera/web_auth.rb +1 -1
- data/lib/aspera/web_server_simple.rb +4 -3
- data.tar.gz.sig +0 -0
- metadata +5 -3
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/plugins/bss.rb +0 -71
data/lib/aspera/agent/httpgw.rb
CHANGED
@@ -2,312 +2,13 @@
|
|
2
2
|
|
3
3
|
require 'aspera/agent/base'
|
4
4
|
require 'aspera/transfer/spec'
|
5
|
-
require 'aspera/
|
5
|
+
require 'aspera/api/httpgw'
|
6
6
|
require 'aspera/log'
|
7
7
|
require 'aspera/assert'
|
8
|
-
require 'aspera/rest'
|
9
|
-
require 'securerandom'
|
10
|
-
require 'websocket'
|
11
|
-
require 'base64'
|
12
|
-
require 'json'
|
13
8
|
|
14
9
|
module Aspera
|
15
10
|
module Agent
|
16
|
-
# Start a transfer using Aspera HTTP Gateway, using web socket secure for uploads
|
17
|
-
# ref: https://api.ibm.com/explorer/catalog/aspera/product/ibm-aspera/api/http-gateway-api/doc/guides-toc
|
18
|
-
# https://developer.ibm.com/apis/catalog?search=%22aspera%20http%22
|
19
|
-
# HTTP GW Upload protocol:
|
20
|
-
# # type Contents Ack Counter
|
21
|
-
# v1
|
22
|
-
# 0 JSON.transfer_spec Transfer Spec "end upload" sent_general
|
23
|
-
# 1.. JSON.slice_upload File base64 chunks "end upload" sent_general
|
24
|
-
# v2
|
25
|
-
# 0 JSON.transfer_spec Transfer Spec "end upload" sent_general
|
26
|
-
# 1 JSON.slice_upload File start "end_slice_upload" sent_v2_delimiter
|
27
|
-
# 2.. Binary File binary chunks "end upload" sent_general
|
28
|
-
# last JSON.slice_upload File end "end_slice_upload" sent_v2_delimiter
|
29
11
|
class Httpgw < Base
|
30
|
-
MSG_SEND_TRANSFER_SPEC = 'transfer_spec'
|
31
|
-
MSG_SEND_SLICE_UPLOAD = 'slice_upload'
|
32
|
-
MSG_RECV_DATA_RECEIVED_SIGNAL = 'end upload'
|
33
|
-
MSG_RECV_SLICE_UPLOAD_SIGNAL = 'end_slice_upload'
|
34
|
-
# upload API versions
|
35
|
-
API_V1 = 'v1'
|
36
|
-
API_V2 = 'v2'
|
37
|
-
# options available in CLI (transfer_info)
|
38
|
-
DEFAULT_OPTIONS = {
|
39
|
-
url: :required,
|
40
|
-
upload_chunk_size: 64_000,
|
41
|
-
api_version: API_V2,
|
42
|
-
synchronous: false
|
43
|
-
}.freeze
|
44
|
-
DEFAULT_BASE_PATH = '/aspera/http-gwy'
|
45
|
-
THR_RECV = 'recv'
|
46
|
-
LOG_WS_SEND = 'ws: send: '.red
|
47
|
-
LOG_WS_RECV = "ws: #{THR_RECV}: ".green
|
48
|
-
private_constant :DEFAULT_OPTIONS, :MSG_RECV_DATA_RECEIVED_SIGNAL, :MSG_RECV_SLICE_UPLOAD_SIGNAL, :API_V1, :API_V2
|
49
|
-
|
50
|
-
# send message on http gw web socket
|
51
|
-
def ws_snd_json(msg_type, payload)
|
52
|
-
if msg_type.eql?(MSG_SEND_SLICE_UPLOAD) && @options[:api_version].eql?(API_V2)
|
53
|
-
@shared_info[:count][:sent_v2_delimiter] += 1
|
54
|
-
else
|
55
|
-
@shared_info[:count][:sent_general] += 1
|
56
|
-
end
|
57
|
-
Log.log.debug do
|
58
|
-
log_data = payload.dup
|
59
|
-
log_data[:data] = "[data #{log_data[:data].length} bytes]" if log_data.key?(:data)
|
60
|
-
"#{LOG_WS_SEND}json: #{msg_type}: #{JSON.generate(log_data)}"
|
61
|
-
end
|
62
|
-
ws_send(ws_type: :text, data: JSON.generate({msg_type => payload}))
|
63
|
-
end
|
64
|
-
|
65
|
-
# send data on http gw web socket
|
66
|
-
def ws_send(ws_type:, data:)
|
67
|
-
Log.log.debug{"#{LOG_WS_SEND}sending: #{ws_type} (#{data&.length || 0} bytes)"}
|
68
|
-
@shared_info[:count][:sent_general] += 1 if ws_type.eql?(:binary)
|
69
|
-
frame_generator = ::WebSocket::Frame::Outgoing::Client.new(data: data, type: ws_type, version: @ws_handshake.version)
|
70
|
-
@ws_io.write(frame_generator.to_s)
|
71
|
-
if @options[:synchronous]
|
72
|
-
@shared_info[:mutex].synchronize do
|
73
|
-
# if read thread exited, there will be no more updates
|
74
|
-
# we allow for 1 of difference else it stays blocked
|
75
|
-
while @ws_read_thread.alive? &&
|
76
|
-
@shared_info[:read_exception].nil? &&
|
77
|
-
(((@shared_info[:count][:sent_general] - @shared_info[:count][:received_general]) > 1) ||
|
78
|
-
((@shared_info[:count][:received_v2_delimiter] - @shared_info[:count][:sent_v2_delimiter]) > 1))
|
79
|
-
if !@shared_info[:cond_var].wait(@shared_info[:mutex], 2.0)
|
80
|
-
Log.log.debug{"#{LOG_WS_SEND}#{'timeout'.blue}: #{@shared_info[:count]}"}
|
81
|
-
end
|
82
|
-
end
|
83
|
-
end
|
84
|
-
end
|
85
|
-
raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
|
86
|
-
Log.log.debug{"#{LOG_WS_SEND}counts: #{@shared_info[:count]}"}
|
87
|
-
end
|
88
|
-
|
89
|
-
# message processing for read thread
|
90
|
-
def process_received_message(message)
|
91
|
-
Log.log.debug{"#{LOG_WS_RECV}message: [#{message}] (#{message.class})"}
|
92
|
-
if message.eql?(MSG_RECV_DATA_RECEIVED_SIGNAL)
|
93
|
-
@shared_info[:mutex].synchronize do
|
94
|
-
@shared_info[:count][:received_general] += 1
|
95
|
-
@shared_info[:cond_var].signal
|
96
|
-
end
|
97
|
-
elsif message.eql?(MSG_RECV_SLICE_UPLOAD_SIGNAL)
|
98
|
-
@shared_info[:mutex].synchronize do
|
99
|
-
@shared_info[:count][:received_v2_delimiter] += 1
|
100
|
-
@shared_info[:cond_var].signal
|
101
|
-
end
|
102
|
-
else
|
103
|
-
message.chomp!
|
104
|
-
error_message =
|
105
|
-
if message.start_with?('"') && message.end_with?('"')
|
106
|
-
# remove double quotes : 1..-2
|
107
|
-
JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message']
|
108
|
-
elsif message.start_with?('{') && message.end_with?('}')
|
109
|
-
JSON.parse(message)['message']
|
110
|
-
else
|
111
|
-
"unknown message from gateway: [#{message}]"
|
112
|
-
end
|
113
|
-
raise error_message
|
114
|
-
end
|
115
|
-
end
|
116
|
-
|
117
|
-
# main function of read thread
|
118
|
-
def process_read_thread
|
119
|
-
Log.log.debug{"#{LOG_WS_RECV}read thread started"}
|
120
|
-
frame_parser = ::WebSocket::Frame::Incoming::Client.new(version: @ws_handshake.version)
|
121
|
-
until @ws_io.eof?
|
122
|
-
begin # rubocop:disable Style/RedundantBegin
|
123
|
-
# ready byte by byte until frame is ready
|
124
|
-
# blocking read
|
125
|
-
byte = @ws_io.read(1)
|
126
|
-
Log.log.trace1{"#{LOG_WS_RECV}read: #{byte} (#{byte.class}) eof=#{@ws_io.eof?}"}
|
127
|
-
frame_parser << byte
|
128
|
-
frame_ok = frame_parser.next
|
129
|
-
next if frame_ok.nil?
|
130
|
-
process_received_message(frame_ok.data.to_s)
|
131
|
-
Log.log.debug{"#{LOG_WS_RECV}counts: #{@shared_info[:count]}"}
|
132
|
-
rescue => e
|
133
|
-
Log.log.debug{"#{LOG_WS_RECV}Exception: #{e}"}
|
134
|
-
@shared_info[:mutex].synchronize do
|
135
|
-
@shared_info[:read_exception] = e
|
136
|
-
@shared_info[:cond_var].signal
|
137
|
-
end
|
138
|
-
break
|
139
|
-
end # begin/rescue
|
140
|
-
end # loop
|
141
|
-
Log.log.debug do
|
142
|
-
"#{LOG_WS_RECV}exception: #{@shared_info[:read_exception]},cls=#{@shared_info[:read_exception].class})"
|
143
|
-
end unless @shared_info[:read_exception].nil?
|
144
|
-
Log.log.debug{"#{LOG_WS_RECV}read thread stopped (ws eof=#{@ws_io.eof?})"}
|
145
|
-
end
|
146
|
-
|
147
|
-
def upload(transfer_spec)
|
148
|
-
# identify this session uniquely
|
149
|
-
session_id = SecureRandom.uuid
|
150
|
-
notify_progress(session_id: nil, type: :pre_start, info: 'starting')
|
151
|
-
# total size of all files
|
152
|
-
total_bytes_to_transfer = 0
|
153
|
-
# we need to keep track of actual file path because transfer spec is modified to be sent in web socket
|
154
|
-
files_to_read = []
|
155
|
-
# get source root or nil
|
156
|
-
source_root = transfer_spec.key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] : nil
|
157
|
-
# source root is ignored by GW, used only here
|
158
|
-
transfer_spec.delete('source_root')
|
159
|
-
# compute total size of files to upload (for progress)
|
160
|
-
# modify transfer spec to be suitable for GW
|
161
|
-
transfer_spec['paths'].each do |item|
|
162
|
-
# save actual file location to be able read contents later
|
163
|
-
file_to_add = Transfer::FauxFile.open(item['source'])
|
164
|
-
if file_to_add
|
165
|
-
item['source'] = file_to_add.path
|
166
|
-
item['file_size'] = file_to_add.size
|
167
|
-
else
|
168
|
-
file_to_add = item['source']
|
169
|
-
# add source root if needed
|
170
|
-
file_to_add = File.join(source_root, file_to_add) unless source_root.nil?
|
171
|
-
# GW expects a simple file name in 'source' but if user wants to change the name, we take it
|
172
|
-
item['source'] = File.basename(item['destination'].nil? ? item['source'] : item['destination'])
|
173
|
-
item['file_size'] = File.size(file_to_add)
|
174
|
-
end
|
175
|
-
# save so that we can actually read the file later
|
176
|
-
files_to_read.push(file_to_add)
|
177
|
-
total_bytes_to_transfer += item['file_size']
|
178
|
-
end
|
179
|
-
# TODO: check that this is available in endpoints: @api_info['endpoints']
|
180
|
-
upload_url = File.join(@gw_base_url, @options[:api_version], 'upload')
|
181
|
-
notify_progress(session_id: nil, type: :pre_start, info: 'connecting wss')
|
182
|
-
# open web socket to end point (equivalent to Net::HTTP.start)
|
183
|
-
http_session = Rest.start_http_session(upload_url)
|
184
|
-
# get the underlying socket i/o
|
185
|
-
@ws_io = Rest.io_http_session(http_session)
|
186
|
-
@ws_handshake = ::WebSocket::Handshake::Client.new(url: upload_url, headers: {})
|
187
|
-
@ws_io.write(@ws_handshake.to_s)
|
188
|
-
sleep(0.1)
|
189
|
-
@ws_handshake << @ws_io.readuntil("\r\n\r\n")
|
190
|
-
Aspera.assert(@ws_handshake.finished?){'Error in websocket handshake'}
|
191
|
-
Log.log.debug{"#{LOG_WS_SEND}handshake success"}
|
192
|
-
# start read thread after handshake
|
193
|
-
@ws_read_thread = Thread.new {process_read_thread}
|
194
|
-
notify_progress(session_id: session_id, type: :session_start)
|
195
|
-
notify_progress(session_id: session_id, type: :session_size, info: total_bytes_to_transfer)
|
196
|
-
sleep(1)
|
197
|
-
# data shared between main thread and read thread
|
198
|
-
@shared_info = {
|
199
|
-
read_exception: nil, # error message if any in callback
|
200
|
-
count: {
|
201
|
-
sent_general: 0,
|
202
|
-
received_general: 0,
|
203
|
-
sent_v2_delimiter: 0,
|
204
|
-
received_v2_delimiter: 0
|
205
|
-
},
|
206
|
-
mutex: Mutex.new,
|
207
|
-
cond_var: ConditionVariable.new
|
208
|
-
}
|
209
|
-
# notify progress bar
|
210
|
-
notify_progress(type: :session_size, session_id: session_id, info: total_bytes_to_transfer)
|
211
|
-
# first step send transfer spec
|
212
|
-
Log.log.debug{Log.dump(:ws_spec, transfer_spec)}
|
213
|
-
ws_snd_json(MSG_SEND_TRANSFER_SPEC, transfer_spec)
|
214
|
-
# current file index
|
215
|
-
file_index = 0
|
216
|
-
# aggregate size sent
|
217
|
-
session_sent_bytes = 0
|
218
|
-
# process each file
|
219
|
-
transfer_spec['paths'].each do |item|
|
220
|
-
slice_info = {
|
221
|
-
name: nil,
|
222
|
-
# TODO: get mime type?
|
223
|
-
type: 'application/octet-stream',
|
224
|
-
size: item['file_size'],
|
225
|
-
slice: 0, # current slice index
|
226
|
-
# index of last slice (i.e number of slices - 1)
|
227
|
-
last_slice: (item['file_size'] - 1) / @options[:upload_chunk_size],
|
228
|
-
fileIndex: file_index
|
229
|
-
}
|
230
|
-
file = files_to_read[file_index]
|
231
|
-
if file.is_a?(Transfer::FauxFile)
|
232
|
-
slice_info[:name] = file.path
|
233
|
-
else
|
234
|
-
file = File.open(file)
|
235
|
-
slice_info[:name] = File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
|
236
|
-
end
|
237
|
-
begin
|
238
|
-
until file.eof?
|
239
|
-
slice_bin_data = file.read(@options[:upload_chunk_size])
|
240
|
-
# interrupt main thread if read thread failed
|
241
|
-
raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
|
242
|
-
begin
|
243
|
-
if @options[:api_version].eql?(API_V1)
|
244
|
-
slice_info[:data] = Base64.strict_encode64(slice_bin_data)
|
245
|
-
ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_info)
|
246
|
-
else
|
247
|
-
# send once, before data, at beginning
|
248
|
-
ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_info) if slice_info[:slice].eql?(0)
|
249
|
-
ws_send(ws_type: :binary, data: slice_bin_data)
|
250
|
-
Log.log.debug{"#{LOG_WS_SEND}buffer: file: #{file_index}, slice: #{slice_info[:slice]}/#{slice_info[:last_slice]}"}
|
251
|
-
# send once, after data, at end
|
252
|
-
ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_info) if slice_info[:slice].eql?(slice_info[:last_slice])
|
253
|
-
end
|
254
|
-
rescue Errno::EPIPE => e
|
255
|
-
raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
|
256
|
-
raise e
|
257
|
-
rescue Net::ReadTimeout => e
|
258
|
-
Log.log.warn{'A timeout condition using HTTPGW may signal a permission problem on destination. Check ascp logs on httpgw.'}
|
259
|
-
raise e
|
260
|
-
end
|
261
|
-
session_sent_bytes += slice_bin_data.length
|
262
|
-
notify_progress(type: :transfer, session_id: session_id, info: session_sent_bytes)
|
263
|
-
slice_info[:slice] += 1
|
264
|
-
end
|
265
|
-
ensure
|
266
|
-
file.close
|
267
|
-
end
|
268
|
-
file_index += 1
|
269
|
-
end # loop on files
|
270
|
-
# throttling may have skipped last one
|
271
|
-
notify_progress(type: :transfer, session_id: session_id, info: session_sent_bytes)
|
272
|
-
notify_progress(type: :end, session_id: session_id)
|
273
|
-
ws_send(ws_type: :close, data: nil)
|
274
|
-
Log.log.debug("Finished upload, waiting for end of #{THR_RECV} thread.")
|
275
|
-
@ws_read_thread.join
|
276
|
-
Log.log.debug{'Read thread joined'}
|
277
|
-
# session no more used
|
278
|
-
@ws_io = nil
|
279
|
-
http_session&.finish
|
280
|
-
end
|
281
|
-
|
282
|
-
def download(transfer_spec)
|
283
|
-
transfer_spec['zip_required'] ||= false
|
284
|
-
transfer_spec['source_root'] ||= '/'
|
285
|
-
# is normally provided by application, like package name
|
286
|
-
if !transfer_spec.key?('download_name')
|
287
|
-
# by default it is the name of first file
|
288
|
-
download_name = File.basename(transfer_spec['paths'].first['source'])
|
289
|
-
# we remove extension
|
290
|
-
download_name = download_name.gsub(/\.@gw_api.*$/, '')
|
291
|
-
# ands add indication of number of files if there is more than one
|
292
|
-
if transfer_spec['paths'].length > 1
|
293
|
-
download_name += " #{transfer_spec['paths'].length} Files"
|
294
|
-
end
|
295
|
-
transfer_spec['download_name'] = download_name
|
296
|
-
end
|
297
|
-
creation = @gw_api.create('v1/download', {'transfer_spec' => transfer_spec})[:data]
|
298
|
-
transfer_uuid = creation['url'].split('/').last
|
299
|
-
file_name =
|
300
|
-
if transfer_spec['zip_required'] || transfer_spec['paths'].length > 1
|
301
|
-
# it is a zip file if zip is required or there is more than 1 file
|
302
|
-
transfer_spec['download_name'] + '.zip'
|
303
|
-
else
|
304
|
-
# it is a plain file if we don't require zip and there is only one file
|
305
|
-
File.basename(transfer_spec['paths'].first['source'])
|
306
|
-
end
|
307
|
-
file_path = File.join(transfer_spec['destination_root'], file_name)
|
308
|
-
@gw_api.call(operation: 'GET', subpath: "v1/download/#{transfer_uuid}", save_to_file: file_path)
|
309
|
-
end
|
310
|
-
|
311
12
|
# start FASP transfer based on transfer spec (hash table)
|
312
13
|
# note that it is asynchronous
|
313
14
|
# HTTP download only supports file list
|
@@ -319,13 +20,13 @@ module Aspera
|
|
319
20
|
transfer_spec['authentication'] ||= 'token'
|
320
21
|
case transfer_spec['direction']
|
321
22
|
when Transfer::Spec::DIRECTION_SEND
|
322
|
-
upload(transfer_spec)
|
23
|
+
@gw_api.upload(transfer_spec)
|
323
24
|
when Transfer::Spec::DIRECTION_RECEIVE
|
324
|
-
download(transfer_spec)
|
25
|
+
@gw_api.download(transfer_spec)
|
325
26
|
else
|
326
27
|
raise "unexpected direction: [#{transfer_spec['direction']}]"
|
327
28
|
end
|
328
|
-
end
|
29
|
+
end
|
329
30
|
|
330
31
|
# wait for completion of all jobs started
|
331
32
|
# @return list of :success or error message
|
@@ -335,29 +36,27 @@ module Aspera
|
|
335
36
|
end
|
336
37
|
|
337
38
|
# TODO: is that useful?
|
338
|
-
def url=(api_url); end
|
39
|
+
# def url=(api_url); end
|
339
40
|
|
340
41
|
private
|
341
42
|
|
342
|
-
def initialize(
|
343
|
-
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
end
|
361
|
-
end # AgentHttpgw
|
43
|
+
def initialize(
|
44
|
+
url:,
|
45
|
+
api_version: Api::Httpgw::API_V2,
|
46
|
+
upload_chunk_size: 64_000,
|
47
|
+
synchronous: false,
|
48
|
+
**base_options
|
49
|
+
)
|
50
|
+
super(**base_options)
|
51
|
+
@gw_api = Api::Httpgw.new(
|
52
|
+
# remove /v1 from end of user-provided GW url: we need the base url only
|
53
|
+
url: url.gsub(%r{/#{Api::Httpgw::API_V1}/*$}o, ''),
|
54
|
+
api_version: api_version,
|
55
|
+
upload_chunk_size: upload_chunk_size,
|
56
|
+
synchronous: synchronous,
|
57
|
+
notify_cb: ->(**event) { notify_progress(**event) }
|
58
|
+
)
|
59
|
+
end
|
60
|
+
end
|
362
61
|
end
|
363
62
|
end
|
data/lib/aspera/agent/node.rb
CHANGED
@@ -13,36 +13,35 @@ module Aspera
|
|
13
13
|
# this singleton class is used by the CLI to provide a common interface to start a transfer
|
14
14
|
# before using it, the use must set the `node_api` member.
|
15
15
|
class Node < Base
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
16
|
+
# @param url [String] the base url of the node api
|
17
|
+
# @param username [String] the username to use for the node api
|
18
|
+
# @param password [String] the password to use for the node api
|
19
|
+
# @param root_id [String] root file id if the node is an access key
|
20
|
+
# @param base_options [Hash] options for base class
|
21
|
+
def initialize(
|
22
|
+
url:,
|
23
|
+
username:,
|
24
|
+
password:,
|
25
|
+
root_id: nil,
|
26
|
+
**base_options
|
27
|
+
)
|
28
|
+
super(**base_options)
|
29
29
|
# root id is required for access key
|
30
|
-
@root_id =
|
31
|
-
rest_params = { base_url:
|
32
|
-
if OAuth::Factory.bearer?(
|
30
|
+
@root_id = root_id
|
31
|
+
rest_params = { base_url: url}
|
32
|
+
if OAuth::Factory.bearer?(password)
|
33
33
|
Aspera.assert(!@root_id.nil?){'root_id not allowed for access key'}
|
34
|
-
rest_params[:headers] = Api::Node.bearer_headers(
|
34
|
+
rest_params[:headers] = Api::Node.bearer_headers(password, access_key: username)
|
35
35
|
else
|
36
36
|
rest_params[:auth] = {
|
37
37
|
type: :basic,
|
38
|
-
username:
|
39
|
-
password:
|
38
|
+
username: username,
|
39
|
+
password: password
|
40
40
|
}
|
41
41
|
end
|
42
42
|
@node_api = Rest.new(**rest_params)
|
43
43
|
# TODO: currently only supports one transfer. This is bad shortcut. but ok for CLI.
|
44
44
|
@transfer_id = nil
|
45
|
-
# Log.log.debug{Log.dump(:agent_options, @options)}
|
46
45
|
end
|
47
46
|
|
48
47
|
# used internally to ensure node api is set before using.
|
data/lib/aspera/agent/trsdk.rb
CHANGED
@@ -17,13 +17,6 @@ module Aspera
|
|
17
17
|
PORT_SEP = ':'
|
18
18
|
# port zero means select a random available high port
|
19
19
|
AUTO_LOCAL_TCP_PORT = "#{PORT_SEP}0"
|
20
|
-
DEFAULT_OPTIONS = {
|
21
|
-
url: AUTO_LOCAL_TCP_PORT,
|
22
|
-
external: false, # expect that an external daemon is already running
|
23
|
-
keep: false # do not shutdown daemon on exit
|
24
|
-
}.freeze
|
25
|
-
private_constant :DEFAULT_OPTIONS
|
26
|
-
|
27
20
|
class << self
|
28
21
|
# Well, the port number is only in log file
|
29
22
|
def daemon_port_from_log(log_file)
|
@@ -44,19 +37,25 @@ module Aspera
|
|
44
37
|
end
|
45
38
|
end
|
46
39
|
|
47
|
-
#
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
40
|
+
# @param url [String] URL of the transfer manager daemon
|
41
|
+
# @param external [Boolean] if true, expect that an external daemon is already running
|
42
|
+
# @param keep [Boolean] if true, do not shutdown daemon on exit
|
43
|
+
# @param base_options [Hash] base options
|
44
|
+
def initialize(
|
45
|
+
url: AUTO_LOCAL_TCP_PORT,
|
46
|
+
external: false,
|
47
|
+
keep: false,
|
48
|
+
**base_options
|
49
|
+
)
|
50
|
+
super(**base_options)
|
51
|
+
is_local_auto_port = @url.eql?(AUTO_LOCAL_TCP_PORT)
|
52
|
+
raise 'Cannot use options `keep` or `external` with port zero' if is_local_auto_port && (@keep || @external)
|
54
53
|
# load SDK stub class on demand, as it's an optional gem
|
55
54
|
$LOAD_PATH.unshift(Ascp::Installation.instance.sdk_ruby_folder)
|
56
55
|
require 'transfer_services_pb'
|
57
56
|
# keep PID for optional shutdown
|
58
57
|
@daemon_pid = nil
|
59
|
-
daemon_endpoint = @
|
58
|
+
daemon_endpoint = @url
|
60
59
|
Log.log.debug{Log.dump(:daemon_endpoint, daemon_endpoint)}
|
61
60
|
# retry loop
|
62
61
|
begin
|
@@ -67,14 +66,14 @@ module Aspera
|
|
67
66
|
# Initiate actual connection
|
68
67
|
get_info_response = @transfer_client.get_info(Transfersdk::InstanceInfoRequest.new)
|
69
68
|
Log.log.debug{"Daemon info: #{get_info_response}"}
|
70
|
-
Log.log.warn{'Attached to existing daemon'} unless @daemon_pid || @
|
69
|
+
Log.log.warn{'Attached to existing daemon'} unless @daemon_pid || @external || @keep
|
71
70
|
at_exit{shutdown}
|
72
71
|
rescue GRPC::Unavailable => e
|
73
72
|
# if transferd is external: do not start it, or other error
|
74
|
-
raise if @
|
73
|
+
raise if @external || !e.message.include?('failed to connect')
|
75
74
|
# we already tried to start a daemon, but it failed
|
76
75
|
Aspera.assert(@daemon_pid.nil?){"Daemon started with PID #{@daemon_pid}, but connection failed to #{daemon_endpoint}}"}
|
77
|
-
Log.log.warn('no daemon present, starting daemon...') if @
|
76
|
+
Log.log.warn('no daemon present, starting daemon...') if @external
|
78
77
|
# location of daemon binary
|
79
78
|
sdk_folder = File.realpath(File.join(Ascp::Installation.instance.sdk_ruby_folder, '..'))
|
80
79
|
# transferd only supports local ip and port
|
@@ -111,7 +110,7 @@ module Aspera
|
|
111
110
|
nil
|
112
111
|
end
|
113
112
|
Log.log.debug{"Daemon started with pid #{@daemon_pid}"}
|
114
|
-
Process.detach(@daemon_pid) if @
|
113
|
+
Process.detach(@daemon_pid) if @keep
|
115
114
|
at_exit {shutdown}
|
116
115
|
# update port for next connection attempt (if auto high port was requested)
|
117
116
|
daemon_endpoint = "#{LOCAL_SOCKET_ADDR}#{PORT_SEP}#{self.class.daemon_port_from_log(log_stdout)}" if is_local_auto_port
|
@@ -171,7 +170,7 @@ module Aspera
|
|
171
170
|
end
|
172
171
|
|
173
172
|
def shutdown
|
174
|
-
stop_daemon unless @
|
173
|
+
stop_daemon unless @keep
|
175
174
|
end
|
176
175
|
|
177
176
|
def stop_daemon
|
data/lib/aspera/api/aoc.rb
CHANGED
@@ -12,15 +12,25 @@ require 'cgi'
|
|
12
12
|
|
13
13
|
module Aspera
|
14
14
|
module Api
|
15
|
+
SAAS_DOMAIN_PROD = 'ibmaspera.com'
|
16
|
+
class Alee < Aspera::Rest
|
17
|
+
def initialize(entitlement_id, customer_id, api_domain: SAAS_DOMAIN_PROD, version: 'v1')
|
18
|
+
super(
|
19
|
+
base_url: "https://api.#{api_domain}/metering/#{version}",
|
20
|
+
headers: {'X-Aspera-Entitlement-Authorization' => Rest.basic_token(entitlement_id, customer_id)}
|
21
|
+
)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
15
25
|
class AoC < Aspera::Rest
|
16
26
|
PRODUCT_NAME = 'Aspera on Cloud'
|
17
27
|
DEFAULT_WORKSPACE = ''
|
18
28
|
# Production domain of AoC
|
19
|
-
|
29
|
+
SAAS_DOMAIN_PROD = 'ibmaspera.com' # cspell:disable-line
|
20
30
|
# to avoid infinite loop in pub link redirection
|
21
31
|
MAX_AOC_URL_REDIRECT = 10
|
22
32
|
CLIENT_ID_PREFIX = 'aspera.'
|
23
|
-
# Well-known AoC
|
33
|
+
# Well-known AoC global client apps
|
24
34
|
GLOBAL_CLIENT_APPS = DataRepository::ELEMENTS.select{|i|i.to_s.start_with?(CLIENT_ID_PREFIX)}.freeze
|
25
35
|
# cookie prefix so that console can decode identity
|
26
36
|
COOKIE_PREFIX_CONSOLE_AOC = 'aspera.aoc'
|
@@ -63,20 +73,13 @@ module Aspera
|
|
63
73
|
end
|
64
74
|
|
65
75
|
# base API url depends on domain, which could be "qa.xxx"
|
66
|
-
def api_base_url(organization: 'api', api_domain:
|
76
|
+
def api_base_url(organization: 'api', api_domain: SAAS_DOMAIN_PROD)
|
67
77
|
return "https://#{organization}.#{api_domain}"
|
68
78
|
end
|
69
79
|
|
70
|
-
def metering_api(entitlement_id, customer_id, api_domain=PROD_DOMAIN)
|
71
|
-
return Rest.new(
|
72
|
-
base_url: "#{api_base_url(api_domain: api_domain)}/metering/v1",
|
73
|
-
headers: {'X-Aspera-Entitlement-Authorization' => Rest.basic_token(entitlement_id, customer_id)}
|
74
|
-
)
|
75
|
-
end
|
76
|
-
|
77
80
|
# split host of http://myorg.asperafiles.com in org and domain
|
78
81
|
def url_parts(uri)
|
79
|
-
raise "No host found in URL.Please check URL format: https://myorg.#{
|
82
|
+
raise "No host found in URL.Please check URL format: https://myorg.#{SAAS_DOMAIN_PROD}" if uri.host.nil?
|
80
83
|
parts = uri.host.split('.', 2)
|
81
84
|
Aspera.assert(parts.length == 2){"expecting a public FQDN for #{PRODUCT_NAME}"}
|
82
85
|
return parts
|
@@ -128,11 +131,11 @@ module Aspera
|
|
128
131
|
organization: parts[0]
|
129
132
|
}
|
130
133
|
end
|
131
|
-
end
|
134
|
+
end
|
132
135
|
|
133
136
|
attr_reader :private_link
|
134
137
|
|
135
|
-
def initialize(subpath: API_V1,
|
138
|
+
def initialize(url:, auth:, subpath: API_V1, client_id: nil, client_secret: nil, scope: nil, redirect_uri: nil, private_key: nil, passphrase: nil, username: nil,
|
136
139
|
password: nil, workspace: nil, secret_finder: nil)
|
137
140
|
# test here because link may set url
|
138
141
|
raise ArgumentError, 'Missing mandatory option: url' if url.nil?
|
@@ -405,7 +408,7 @@ module Aspera
|
|
405
408
|
end
|
406
409
|
else # unexpected extended value, must be String or Hash
|
407
410
|
raise "#{recipient_list_field} item must be a String (email, shared inbox) or Hash (id,type)"
|
408
|
-
end
|
411
|
+
end
|
409
412
|
# add original or resolved recipient info
|
410
413
|
resolved_list.push(short_recipient_info)
|
411
414
|
end
|
data/lib/aspera/api/cos_node.rb
CHANGED
@@ -52,9 +52,9 @@ module Aspera
|
|
52
52
|
# read FASP connection information for bucket
|
53
53
|
xml_result_text = s3_api.call(
|
54
54
|
operation: 'GET',
|
55
|
-
subpath:
|
56
|
-
headers:
|
57
|
-
|
55
|
+
subpath: bucket,
|
56
|
+
headers: {'Accept' => 'application/xml'},
|
57
|
+
query: {'faspConnectionInfo' => nil}
|
58
58
|
)[:http].body
|
59
59
|
ats_info = XmlSimple.xml_in(xml_result_text, {'ForceArray' => false})
|
60
60
|
Aspera::Log.dump('ats_info', ats_info)
|
@@ -87,7 +87,7 @@ module Aspera
|
|
87
87
|
receiver_client_ids: 'aspera_ats'
|
88
88
|
)
|
89
89
|
# get delegated token to be placed in rest call header and in transfer tags
|
90
|
-
@storage_credentials['token'][TOKEN_FIELD] = OAuth::Factory.bearer_extract(delegated_oauth.
|
90
|
+
@storage_credentials['token'][TOKEN_FIELD] = OAuth::Factory.bearer_extract(delegated_oauth.token)
|
91
91
|
@headers['X-Aspera-Storage-Credentials'] = JSON.generate(@storage_credentials)
|
92
92
|
end
|
93
93
|
end
|