aspera-cli 4.14.0 → 4.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +54 -3
- data/CONTRIBUTING.md +7 -7
- data/README.md +1457 -880
- data/bin/ascli +18 -9
- data/bin/asession +12 -14
- data/examples/proxy.pac +1 -1
- data/lib/aspera/aoc.rb +198 -127
- data/lib/aspera/ascmd.rb +24 -14
- data/lib/aspera/cli/basic_auth_plugin.rb +9 -6
- data/lib/aspera/cli/error.rb +17 -0
- data/lib/aspera/cli/extended_value.rb +47 -12
- data/lib/aspera/cli/formatter.rb +260 -171
- data/lib/aspera/cli/hints.rb +80 -0
- data/lib/aspera/cli/main.rb +101 -147
- data/lib/aspera/cli/manager.rb +160 -124
- data/lib/aspera/cli/plugin.rb +70 -59
- data/lib/aspera/cli/plugins/alee.rb +0 -1
- data/lib/aspera/cli/plugins/aoc.rb +239 -273
- data/lib/aspera/cli/plugins/ats.rb +8 -5
- data/lib/aspera/cli/plugins/bss.rb +2 -2
- data/lib/aspera/cli/plugins/config.rb +516 -375
- data/lib/aspera/cli/plugins/console.rb +40 -0
- data/lib/aspera/cli/plugins/cos.rb +4 -5
- data/lib/aspera/cli/plugins/faspex.rb +99 -84
- data/lib/aspera/cli/plugins/faspex5.rb +179 -148
- data/lib/aspera/cli/plugins/node.rb +219 -153
- data/lib/aspera/cli/plugins/orchestrator.rb +52 -17
- data/lib/aspera/cli/plugins/preview.rb +46 -32
- data/lib/aspera/cli/plugins/server.rb +57 -17
- data/lib/aspera/cli/plugins/shares.rb +34 -12
- data/lib/aspera/cli/sync_actions.rb +68 -0
- data/lib/aspera/cli/transfer_agent.rb +45 -55
- data/lib/aspera/cli/transfer_progress.rb +74 -0
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +3 -1
- data/lib/aspera/command_line_builder.rb +14 -11
- data/lib/aspera/cos_node.rb +3 -2
- data/lib/aspera/environment.rb +17 -6
- data/lib/aspera/fasp/agent_aspera.rb +126 -0
- data/lib/aspera/fasp/agent_base.rb +31 -77
- data/lib/aspera/fasp/agent_connect.rb +21 -22
- data/lib/aspera/fasp/agent_direct.rb +88 -102
- data/lib/aspera/fasp/agent_httpgw.rb +196 -192
- data/lib/aspera/fasp/agent_node.rb +41 -34
- data/lib/aspera/fasp/agent_trsdk.rb +75 -34
- data/lib/aspera/fasp/error_info.rb +2 -2
- data/lib/aspera/fasp/faux_file.rb +52 -0
- data/lib/aspera/fasp/installation.rb +43 -184
- data/lib/aspera/fasp/management.rb +244 -0
- data/lib/aspera/fasp/parameters.rb +59 -26
- data/lib/aspera/fasp/parameters.yaml +75 -8
- data/lib/aspera/fasp/products.rb +162 -0
- data/lib/aspera/fasp/transfer_spec.rb +1 -1
- data/lib/aspera/fasp/uri.rb +4 -4
- data/lib/aspera/faspex_gw.rb +2 -2
- data/lib/aspera/faspex_postproc.rb +2 -2
- data/lib/aspera/hash_ext.rb +2 -2
- data/lib/aspera/json_rpc.rb +49 -0
- data/lib/aspera/line_logger.rb +23 -0
- data/lib/aspera/log.rb +57 -16
- data/lib/aspera/node.rb +97 -14
- data/lib/aspera/oauth.rb +36 -18
- data/lib/aspera/open_application.rb +4 -4
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/file_types.rb +4 -2
- data/lib/aspera/preview/generator.rb +22 -35
- data/lib/aspera/preview/options.rb +2 -0
- data/lib/aspera/preview/terminal.rb +24 -13
- data/lib/aspera/preview/utils.rb +19 -26
- data/lib/aspera/rest.rb +103 -72
- data/lib/aspera/rest_call_error.rb +1 -1
- data/lib/aspera/rest_error_analyzer.rb +15 -14
- data/lib/aspera/rest_errors_aspera.rb +37 -34
- data/lib/aspera/secret_hider.rb +14 -16
- data/lib/aspera/ssh.rb +4 -1
- data/lib/aspera/sync.rb +128 -122
- data/lib/aspera/temp_file_manager.rb +10 -3
- data/lib/aspera/web_auth.rb +10 -7
- data/lib/aspera/web_server_simple.rb +9 -4
- data.tar.gz.sig +0 -0
- metadata +33 -15
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/listener/line_dump.rb +0 -19
- data/lib/aspera/cli/listener/logger.rb +0 -22
- data/lib/aspera/cli/listener/progress.rb +0 -50
- data/lib/aspera/cli/listener/progress_multi.rb +0 -84
- data/lib/aspera/cli/plugins/sync.rb +0 -44
- data/lib/aspera/fasp/listener.rb +0 -13
@@ -2,6 +2,7 @@
|
|
2
2
|
|
3
3
|
require 'aspera/fasp/agent_base'
|
4
4
|
require 'aspera/fasp/transfer_spec'
|
5
|
+
require 'aspera/fasp/faux_file'
|
5
6
|
require 'aspera/log'
|
6
7
|
require 'aspera/rest'
|
7
8
|
require 'securerandom'
|
@@ -9,86 +10,147 @@ require 'websocket'
|
|
9
10
|
require 'base64'
|
10
11
|
require 'json'
|
11
12
|
|
12
|
-
# HTTP GW Upload protocol
|
13
|
-
# -----------------------
|
14
|
-
# v1
|
15
|
-
# 1 - MessageType: String (Transfer Spec) JSON : type: transfer_spec, acknowledged with "end upload"
|
16
|
-
# 2.. - MessageType: String (Slice Upload start) JSON : type: slice_upload, acknowledged with "end upload"
|
17
|
-
# v2
|
18
|
-
# 1 - MessageType: String (Transfer Spec) JSON : type: transfer_spec, acknowledged with "end upload"
|
19
|
-
# 2 - MessageType: String (Slice Upload start) JSON : type: slice_upload, acknowledged with "end_slice_upload"
|
20
|
-
# 3.. - MessageType: ByteArray (File Size) Chunks : acknowledged with "end upload"
|
21
|
-
# last - MessageType: String (Slice Upload end) JSON : type: slice_upload, acknowledged with "end_slice_upload"
|
22
|
-
|
23
|
-
# ref: https://api.ibm.com/explorer/catalog/aspera/product/ibm-aspera/api/http-gateway-api/doc/guides-toc
|
24
|
-
# https://developer.ibm.com/apis/catalog?search=%22aspera%20http%22
|
25
13
|
module Aspera
|
26
14
|
module Fasp
|
27
|
-
#
|
15
|
+
# Start a transfer using Aspera HTTP Gateway, using web socket secure for uploads
|
16
|
+
# ref: https://api.ibm.com/explorer/catalog/aspera/product/ibm-aspera/api/http-gateway-api/doc/guides-toc
|
17
|
+
# https://developer.ibm.com/apis/catalog?search=%22aspera%20http%22
|
18
|
+
# HTTP GW Upload protocol:
|
19
|
+
# # type Contents Ack Counter
|
20
|
+
# v1
|
21
|
+
# 0 JSON.transfer_spec Transfer Spec "end upload" sent_general
|
22
|
+
# 1.. JSON.slice_upload File base64 chunks "end upload" sent_general
|
23
|
+
# v2
|
24
|
+
# 0 JSON.transfer_spec Transfer Spec "end upload" sent_general
|
25
|
+
# 1 JSON.slice_upload File start "end_slice_upload" sent_v2_delimiter
|
26
|
+
# 2.. Binary File binary chunks "end upload" sent_general
|
27
|
+
# last JSON.slice_upload File end "end_slice_upload" sent_v2_delimiter
|
28
28
|
class AgentHttpgw < Aspera::Fasp::AgentBase
|
29
|
-
|
29
|
+
MSG_SEND_TRANSFER_SPEC = 'transfer_spec'
|
30
|
+
MSG_SEND_SLICE_UPLOAD = 'slice_upload'
|
30
31
|
MSG_RECV_DATA_RECEIVED_SIGNAL = 'end upload'
|
31
32
|
MSG_RECV_SLICE_UPLOAD_SIGNAL = 'end_slice_upload'
|
32
|
-
MSG_SEND_SLICE_UPLOAD = 'slice_upload'
|
33
|
-
MSG_SEND_TRANSFER_SPEC = 'transfer_spec'
|
34
33
|
# upload API versions
|
35
34
|
API_V1 = 'v1'
|
36
35
|
API_V2 = 'v2'
|
37
36
|
# options available in CLI (transfer_info)
|
38
37
|
DEFAULT_OPTIONS = {
|
39
|
-
url:
|
40
|
-
upload_chunk_size:
|
41
|
-
|
42
|
-
|
43
|
-
synchronous: true
|
38
|
+
url: :required,
|
39
|
+
upload_chunk_size: 64_000,
|
40
|
+
api_version: API_V2,
|
41
|
+
synchronous: false
|
44
42
|
}.freeze
|
45
43
|
DEFAULT_BASE_PATH = '/aspera/http-gwy'
|
46
|
-
|
47
|
-
|
44
|
+
THR_RECV = 'recv'
|
45
|
+
LOG_WS_SEND = 'ws: send: '.red
|
46
|
+
LOG_WS_RECV = "ws: #{THR_RECV}: ".green
|
48
47
|
private_constant :DEFAULT_OPTIONS, :MSG_RECV_DATA_RECEIVED_SIGNAL, :MSG_RECV_SLICE_UPLOAD_SIGNAL, :API_V1, :API_V2
|
49
48
|
|
50
49
|
# send message on http gw web socket
|
51
50
|
def ws_snd_json(msg_type, payload)
|
52
51
|
if msg_type.eql?(MSG_SEND_SLICE_UPLOAD) && @options[:api_version].eql?(API_V2)
|
53
|
-
@shared_info[:count][:
|
52
|
+
@shared_info[:count][:sent_v2_delimiter] += 1
|
54
53
|
else
|
55
|
-
@shared_info[:count][:
|
54
|
+
@shared_info[:count][:sent_general] += 1
|
56
55
|
end
|
57
56
|
Log.log.debug do
|
58
57
|
log_data = payload.dup
|
59
58
|
log_data[:data] = "[data #{log_data[:data].length} bytes]" if log_data.key?(:data)
|
60
|
-
"
|
59
|
+
"#{LOG_WS_SEND}json: #{msg_type}: #{JSON.generate(log_data)}"
|
61
60
|
end
|
62
|
-
ws_send(JSON.generate({msg_type => payload}))
|
61
|
+
ws_send(ws_type: :text, data: JSON.generate({msg_type => payload}))
|
63
62
|
end
|
64
63
|
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
@ws_io.write(
|
64
|
+
# send data on http gw web socket
|
65
|
+
def ws_send(ws_type:, data:)
|
66
|
+
Log.log.debug{"#{LOG_WS_SEND}sending: #{ws_type} (#{data&.length || 0} bytes)"}
|
67
|
+
@shared_info[:count][:sent_general] += 1 if ws_type.eql?(:binary)
|
68
|
+
frame_generator = ::WebSocket::Frame::Outgoing::Client.new(data: data, type: ws_type, version: @ws_handshake.version)
|
69
|
+
@ws_io.write(frame_generator.to_s)
|
70
|
+
if @options[:synchronous]
|
71
|
+
@shared_info[:mutex].synchronize do
|
72
|
+
# if read thread exited, there will be no more updates
|
73
|
+
# we allow for 1 of difference else it stays blocked
|
74
|
+
while @ws_read_thread.alive? &&
|
75
|
+
@shared_info[:read_exception].nil? &&
|
76
|
+
(((@shared_info[:count][:sent_general] - @shared_info[:count][:received_general]) > 1) ||
|
77
|
+
((@shared_info[:count][:received_v2_delimiter] - @shared_info[:count][:sent_v2_delimiter]) > 1))
|
78
|
+
if !@shared_info[:cond_var].wait(@shared_info[:mutex], 2.0)
|
79
|
+
Log.log.debug{"#{LOG_WS_SEND}#{'timeout'.blue}: #{@shared_info[:count]}"}
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
|
85
|
+
Log.log.debug{"#{LOG_WS_SEND}counts: #{@shared_info[:count]}"}
|
71
86
|
end
|
72
87
|
|
73
|
-
#
|
74
|
-
def
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
88
|
+
# message processing for read thread
|
89
|
+
def process_received_message(message)
|
90
|
+
Log.log.debug{"#{LOG_WS_RECV}message: [#{message}] (#{message.class})"}
|
91
|
+
if message.eql?(MSG_RECV_DATA_RECEIVED_SIGNAL)
|
92
|
+
@shared_info[:mutex].synchronize do
|
93
|
+
@shared_info[:count][:received_general] += 1
|
94
|
+
@shared_info[:cond_var].signal
|
95
|
+
end
|
96
|
+
elsif message.eql?(MSG_RECV_SLICE_UPLOAD_SIGNAL)
|
97
|
+
@shared_info[:mutex].synchronize do
|
98
|
+
@shared_info[:count][:received_v2_delimiter] += 1
|
99
|
+
@shared_info[:cond_var].signal
|
82
100
|
end
|
101
|
+
else
|
102
|
+
message.chomp!
|
103
|
+
error_message =
|
104
|
+
if message.start_with?('"') && message.end_with?('"')
|
105
|
+
# remove double quotes : 1..-2
|
106
|
+
JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message']
|
107
|
+
elsif message.start_with?('{') && message.end_with?('}')
|
108
|
+
JSON.parse(message)['message']
|
109
|
+
else
|
110
|
+
"unknown message from gateway: [#{message}]"
|
111
|
+
end
|
112
|
+
raise error_message
|
83
113
|
end
|
84
|
-
|
114
|
+
end
|
115
|
+
|
116
|
+
# main function of read thread
|
117
|
+
def process_read_thread
|
118
|
+
Log.log.debug{"#{LOG_WS_RECV}read thread started"}
|
119
|
+
frame_parser = ::WebSocket::Frame::Incoming::Client.new(version: @ws_handshake.version)
|
120
|
+
until @ws_io.eof?
|
121
|
+
begin # rubocop:disable Style/RedundantBegin
|
122
|
+
# ready byte by byte until frame is ready
|
123
|
+
# blocking read
|
124
|
+
byte = @ws_io.read(1)
|
125
|
+
Log.log.trace1{"#{LOG_WS_RECV}read: #{byte} (#{byte.class}) eof=#{@ws_io.eof?}"}
|
126
|
+
frame_parser << byte
|
127
|
+
frame_ok = frame_parser.next
|
128
|
+
next if frame_ok.nil?
|
129
|
+
process_received_message(frame_ok.data.to_s)
|
130
|
+
Log.log.debug{"#{LOG_WS_RECV}counts: #{@shared_info[:count]}"}
|
131
|
+
rescue => e
|
132
|
+
Log.log.debug{"#{LOG_WS_RECV}Exception: #{e}"}
|
133
|
+
@shared_info[:mutex].synchronize do
|
134
|
+
@shared_info[:read_exception] = e
|
135
|
+
@shared_info[:cond_var].signal
|
136
|
+
end
|
137
|
+
break
|
138
|
+
end # begin/rescue
|
139
|
+
end # loop
|
140
|
+
Log.log.debug do
|
141
|
+
"#{LOG_WS_RECV}exception: #{@shared_info[:read_exception]},cls=#{@shared_info[:read_exception].class})"
|
142
|
+
end unless @shared_info[:read_exception].nil?
|
143
|
+
Log.log.debug{"#{LOG_WS_RECV}read thread stopped (ws eof=#{@ws_io.eof?})"}
|
85
144
|
end
|
86
145
|
|
87
146
|
def upload(transfer_spec)
|
147
|
+
# identify this session uniquely
|
148
|
+
session_id = SecureRandom.uuid
|
149
|
+
notify_progress(session_id: nil, type: :pre_start, info: 'starting')
|
88
150
|
# total size of all files
|
89
151
|
total_bytes_to_transfer = 0
|
90
152
|
# we need to keep track of actual file path because transfer spec is modified to be sent in web socket
|
91
|
-
|
153
|
+
files_to_read = []
|
92
154
|
# get source root or nil
|
93
155
|
source_root = transfer_spec.key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] : nil
|
94
156
|
# source root is ignored by GW, used only here
|
@@ -97,143 +159,96 @@ module Aspera
|
|
97
159
|
# modify transfer spec to be suitable for GW
|
98
160
|
transfer_spec['paths'].each do |item|
|
99
161
|
# save actual file location to be able read contents later
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
162
|
+
file_to_add = FauxFile.open(item['source'])
|
163
|
+
if file_to_add
|
164
|
+
item['source'] = file_to_add.path
|
165
|
+
item['file_size'] = file_to_add.size
|
166
|
+
else
|
167
|
+
file_to_add = item['source']
|
168
|
+
# add source root if needed
|
169
|
+
file_to_add = File.join(source_root, file_to_add) unless source_root.nil?
|
170
|
+
# GW expects a simple file name in 'source' but if user wants to change the name, we take it
|
171
|
+
item['source'] = File.basename(item['destination'].nil? ? item['source'] : item['destination'])
|
172
|
+
item['file_size'] = File.size(file_to_add)
|
173
|
+
end
|
107
174
|
# save so that we can actually read the file later
|
108
|
-
|
175
|
+
files_to_read.push(file_to_add)
|
176
|
+
total_bytes_to_transfer += item['file_size']
|
109
177
|
end
|
110
|
-
# identify this session uniquely
|
111
|
-
session_id = SecureRandom.uuid
|
112
178
|
upload_url = File.join(@gw_api.params[:base_url], @options[:api_version], 'upload')
|
113
|
-
|
179
|
+
notify_progress(session_id: nil, type: :pre_start, info: 'connecting wss')
|
114
180
|
# open web socket to end point (equivalent to Net::HTTP.start)
|
115
|
-
|
116
|
-
#
|
117
|
-
@ws_io =
|
118
|
-
# @ws_io.debug_output = Log.log
|
181
|
+
http_session = Rest.start_http_session(upload_url)
|
182
|
+
# get the underlying socket i/o
|
183
|
+
@ws_io = Rest.io_http_session(http_session)
|
119
184
|
@ws_handshake = ::WebSocket::Handshake::Client.new(url: upload_url, headers: {})
|
120
185
|
@ws_io.write(@ws_handshake.to_s)
|
121
186
|
sleep(0.1)
|
122
187
|
@ws_handshake << @ws_io.readuntil("\r\n\r\n")
|
123
188
|
raise 'Error in websocket handshake' unless @ws_handshake.finished?
|
124
|
-
Log.log.debug{"#{
|
189
|
+
Log.log.debug{"#{LOG_WS_SEND}handshake success"}
|
190
|
+
# start read thread after handshake
|
191
|
+
@ws_read_thread = Thread.new {process_read_thread}
|
192
|
+
notify_progress(session_id: session_id, type: :session_start)
|
193
|
+
notify_progress(session_id: session_id, type: :session_size, info: total_bytes_to_transfer)
|
194
|
+
sleep(1)
|
125
195
|
# data shared between main thread and read thread
|
126
196
|
@shared_info = {
|
127
197
|
read_exception: nil, # error message if any in callback
|
128
198
|
count: {
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
199
|
+
sent_general: 0,
|
200
|
+
received_general: 0,
|
201
|
+
sent_v2_delimiter: 0,
|
202
|
+
received_v2_delimiter: 0
|
133
203
|
},
|
134
204
|
mutex: Mutex.new,
|
135
205
|
cond_var: ConditionVariable.new
|
136
206
|
}
|
137
|
-
# start read thread
|
138
|
-
ws_read_thread = Thread.new do
|
139
|
-
Log.log.debug{"#{LOG_WS_THREAD}read started"}
|
140
|
-
frame = ::WebSocket::Frame::Incoming::Client.new(version: @ws_handshake.version)
|
141
|
-
loop do
|
142
|
-
begin # rubocop:disable Style/RedundantBegin
|
143
|
-
# unless (recv_data = @ws_io.getc)
|
144
|
-
# sleep(0.1)
|
145
|
-
# next
|
146
|
-
# end
|
147
|
-
# frame << recv_data
|
148
|
-
# frame << @ws_io.readuntil("\n")
|
149
|
-
# frame << @ws_io.read_all
|
150
|
-
frame << @ws_io.read(1)
|
151
|
-
while (msg = frame.next)
|
152
|
-
Log.log.debug{"#{LOG_WS_THREAD}type: #{msg.class}"}
|
153
|
-
message = msg.data
|
154
|
-
Log.log.debug{"#{LOG_WS_THREAD}message: [#{message}]"}
|
155
|
-
if message.eql?(MSG_RECV_DATA_RECEIVED_SIGNAL)
|
156
|
-
@shared_info[:mutex].synchronize do
|
157
|
-
@shared_info[:count][:received_data] += 1
|
158
|
-
@shared_info[:cond_var].signal
|
159
|
-
end
|
160
|
-
elsif message.eql?(MSG_RECV_SLICE_UPLOAD_SIGNAL)
|
161
|
-
@shared_info[:mutex].synchronize do
|
162
|
-
@shared_info[:count][:received_v2_slice] += 1
|
163
|
-
@shared_info[:cond_var].signal
|
164
|
-
end
|
165
|
-
else
|
166
|
-
message.chomp!
|
167
|
-
error_message =
|
168
|
-
if message.start_with?('"') && message.end_with?('"')
|
169
|
-
JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message']
|
170
|
-
elsif message.start_with?('{') && message.end_with?('}')
|
171
|
-
JSON.parse(message)['message']
|
172
|
-
else
|
173
|
-
"unknown message from gateway: [#{message}]"
|
174
|
-
end
|
175
|
-
raise error_message
|
176
|
-
end
|
177
|
-
Log.log.debug{"#{LOG_WS_THREAD}counts: #{@shared_info[:count]}"}
|
178
|
-
end # while
|
179
|
-
rescue => e
|
180
|
-
Log.log.debug{"#{LOG_WS_THREAD}Exception: #{e}"}
|
181
|
-
@shared_info[:mutex].synchronize do
|
182
|
-
@shared_info[:read_exception] = e unless e.is_a?(EOFError)
|
183
|
-
@shared_info[:cond_var].signal
|
184
|
-
end
|
185
|
-
break
|
186
|
-
end # begin
|
187
|
-
end # loop
|
188
|
-
Log.log.debug{"#{LOG_WS_THREAD}stopping (exc=#{@shared_info[:read_exception]},cls=#{@shared_info[:read_exception].class})"}
|
189
|
-
end
|
190
207
|
# notify progress bar
|
191
|
-
|
208
|
+
notify_progress(type: :session_size, session_id: session_id, info: total_bytes_to_transfer)
|
192
209
|
# first step send transfer spec
|
193
|
-
Log.dump(:ws_spec, transfer_spec)
|
210
|
+
Log.log.debug{Log.dump(:ws_spec, transfer_spec)}
|
194
211
|
ws_snd_json(MSG_SEND_TRANSFER_SPEC, transfer_spec)
|
195
|
-
wait_for_sent_msg_ack_or_exception
|
196
212
|
# current file index
|
197
213
|
file_index = 0
|
198
214
|
# aggregate size sent
|
199
|
-
|
200
|
-
#
|
201
|
-
last_progress_time = nil
|
202
|
-
|
215
|
+
session_sent_bytes = 0
|
216
|
+
# process each file
|
203
217
|
transfer_spec['paths'].each do |item|
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
218
|
+
slice_info = {
|
219
|
+
name: nil,
|
220
|
+
# TODO: get mime type?
|
221
|
+
type: 'application/octet-stream',
|
222
|
+
size: item['file_size'],
|
223
|
+
slice: 0, # current slice index
|
224
|
+
# index of last slice (i.e number of slices - 1)
|
225
|
+
last_slice: (item['file_size'] - 1) / @options[:upload_chunk_size],
|
226
|
+
fileIndex: file_index
|
227
|
+
}
|
228
|
+
file = files_to_read[file_index]
|
229
|
+
if file.is_a?(FauxFile)
|
230
|
+
slice_info[:name] = file.path
|
231
|
+
else
|
232
|
+
file = File.open(file)
|
233
|
+
slice_info[:name] = File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
|
234
|
+
end
|
235
|
+
begin
|
213
236
|
until file.eof?
|
214
|
-
|
215
|
-
slice_data = {
|
216
|
-
name: file_name,
|
217
|
-
type: file_mime_type,
|
218
|
-
size: file_size,
|
219
|
-
slice: slice_index,
|
220
|
-
total_slices: slice_total,
|
221
|
-
fileIndex: file_index
|
222
|
-
}
|
223
|
-
# Log.dump(:slice_data,slice_data) #if slice_index.eql?(0)
|
237
|
+
slice_bin_data = file.read(@options[:upload_chunk_size])
|
224
238
|
# interrupt main thread if read thread failed
|
225
239
|
raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
|
226
240
|
begin
|
227
241
|
if @options[:api_version].eql?(API_V1)
|
228
|
-
|
229
|
-
ws_snd_json(MSG_SEND_SLICE_UPLOAD,
|
242
|
+
slice_info[:data] = Base64.strict_encode64(slice_bin_data)
|
243
|
+
ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_info)
|
230
244
|
else
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
245
|
+
# send once, before data, at beginning
|
246
|
+
ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_info) if slice_info[:slice].eql?(0)
|
247
|
+
ws_send(ws_type: :binary, data: slice_bin_data)
|
248
|
+
Log.log.debug{"#{LOG_WS_SEND}buffer: file: #{file_index}, slice: #{slice_info[:slice]}/#{slice_info[:last_slice]}"}
|
249
|
+
# send once, after data, at end
|
250
|
+
ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_info) if slice_info[:slice].eql?(slice_info[:last_slice])
|
235
251
|
end
|
236
|
-
wait_for_sent_msg_ack_or_exception
|
237
252
|
rescue Errno::EPIPE => e
|
238
253
|
raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
|
239
254
|
raise e
|
@@ -241,26 +256,25 @@ module Aspera
|
|
241
256
|
Log.log.warn{'A timeout condition using HTTPGW may signal a permission problem on destination. Check ascp logs on httpgw.'}
|
242
257
|
raise e
|
243
258
|
end
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
notify_progress(session_id, sent_bytes)
|
248
|
-
last_progress_time = current_time
|
249
|
-
end
|
250
|
-
slice_index += 1
|
259
|
+
session_sent_bytes += slice_bin_data.length
|
260
|
+
notify_progress(type: :transfer, session_id: session_id, info: session_sent_bytes)
|
261
|
+
slice_info[:slice] += 1
|
251
262
|
end
|
263
|
+
ensure
|
264
|
+
file.close
|
252
265
|
end
|
253
266
|
file_index += 1
|
254
|
-
end
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
267
|
+
end # loop on files
|
268
|
+
# throttling may have skipped last one
|
269
|
+
notify_progress(type: :transfer, session_id: session_id, info: session_sent_bytes)
|
270
|
+
notify_progress(type: :end, session_id: session_id)
|
271
|
+
ws_send(ws_type: :close, data: nil)
|
272
|
+
Log.log.debug("Finished upload, waiting for end of #{THR_RECV} thread.")
|
273
|
+
@ws_read_thread.join
|
274
|
+
Log.log.debug{'Read thread joined'}
|
275
|
+
# session no more used
|
260
276
|
@ws_io = nil
|
261
|
-
|
262
|
-
notify_progress(session_id, sent_bytes)
|
263
|
-
notify_end(session_id)
|
277
|
+
http_session&.finish
|
264
278
|
end
|
265
279
|
|
266
280
|
def download(transfer_spec)
|
@@ -299,7 +313,7 @@ module Aspera
|
|
299
313
|
raise 'GW URL must be set' if @gw_api.nil?
|
300
314
|
raise 'paths: must be Array' unless transfer_spec['paths'].is_a?(Array)
|
301
315
|
raise 'only token based transfer is supported in GW' unless transfer_spec['token'].is_a?(String)
|
302
|
-
Log.dump(:user_spec, transfer_spec)
|
316
|
+
Log.log.debug{Log.dump(:user_spec, transfer_spec)}
|
303
317
|
transfer_spec['authentication'] ||= 'token'
|
304
318
|
case transfer_spec['direction']
|
305
319
|
when Fasp::TransferSpec::DIRECTION_SEND
|
@@ -314,43 +328,33 @@ module Aspera
|
|
314
328
|
# wait for completion of all jobs started
|
315
329
|
# @return list of :success or error message
|
316
330
|
def wait_for_transfers_completion
|
331
|
+
# well ... transfer was done in "start"
|
317
332
|
return [:success]
|
318
333
|
end
|
319
334
|
|
320
|
-
#
|
321
|
-
def shutdown; end
|
322
|
-
|
335
|
+
# TODO: is that useful?
|
323
336
|
def url=(api_url); end
|
324
337
|
|
325
338
|
private
|
326
339
|
|
327
340
|
def initialize(opts)
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
raise "httpgw agent parameters (transfer_info): expecting Hash, but have #{opts.class}" unless opts.is_a?(Hash)
|
332
|
-
opts.symbolize_keys.each do |k, v|
|
333
|
-
raise "httpgw agent parameter: Unknown: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.key?(k)
|
334
|
-
@options[k] = v
|
335
|
-
end
|
336
|
-
if @options[:url].nil?
|
337
|
-
available = DEFAULT_OPTIONS.map { |k, v| "#{k}(#{v})"}.join(', ')
|
338
|
-
raise "Missing mandatory parameter for HTTP GW in transfer_info: url. Allowed parameters: #{available}."
|
339
|
-
end
|
340
|
-
# remove /v1 from end
|
341
|
+
super(opts)
|
342
|
+
@options = AgentBase.options(default: DEFAULT_OPTIONS, options: opts)
|
343
|
+
# remove /v1 from end of user-provided GW url: we need the base url only
|
341
344
|
@options[:url].gsub(%r{/v1/*$}, '')
|
342
|
-
super()
|
343
345
|
@gw_api = Rest.new({base_url: @options[:url]})
|
344
346
|
@api_info = @gw_api.read('v1/info')[:data]
|
345
|
-
Log.dump(:api_info, @api_info)
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
347
|
+
Log.log.debug{Log.dump(:api_info, @api_info)}
|
348
|
+
# web socket endpoint: by default use v2 (newer gateways), without base64 encoding
|
349
|
+
# is the latest supported? else revert to old api
|
350
|
+
if !@options[:api_version].eql?(API_V1)
|
351
|
+
if !@api_info['endpoints'].any?{|i|i.include?(@options[:api_version])}
|
352
|
+
Log.log.warn{"API version #{@options[:api_version]} not supported, reverting to #{API_V1}"}
|
353
|
+
@options[:api_version] = API_V1
|
354
|
+
end
|
351
355
|
end
|
352
356
|
@options.freeze
|
353
|
-
Log.dump(:
|
357
|
+
Log.log.debug{Log.dump(:agent_options, @options)}
|
354
358
|
end
|
355
359
|
end # AgentHttpgw
|
356
360
|
end
|
@@ -1,32 +1,36 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
# cspell:ignore precalc
|
3
4
|
require 'aspera/fasp/agent_base'
|
4
5
|
require 'aspera/fasp/transfer_spec'
|
5
6
|
require 'aspera/node'
|
6
7
|
require 'aspera/log'
|
7
|
-
require '
|
8
|
+
require 'aspera/oauth'
|
8
9
|
|
9
10
|
module Aspera
|
10
11
|
module Fasp
|
11
12
|
# this singleton class is used by the CLI to provide a common interface to start a transfer
|
12
13
|
# before using it, the use must set the `node_api` member.
|
13
14
|
class AgentNode < Aspera::Fasp::AgentBase
|
15
|
+
DEFAULT_OPTIONS = {
|
16
|
+
url: :required,
|
17
|
+
username: :required,
|
18
|
+
password: :required,
|
19
|
+
root_id: nil
|
20
|
+
}.freeze
|
14
21
|
# option include: root_id if the node is an access key
|
15
|
-
attr_writer :options
|
22
|
+
# attr_writer :options
|
16
23
|
|
17
|
-
def initialize(
|
18
|
-
raise 'node specification must be Hash' unless
|
19
|
-
|
20
|
-
|
24
|
+
def initialize(opts)
|
25
|
+
raise 'node specification must be Hash' unless opts.is_a?(Hash)
|
26
|
+
super(opts)
|
27
|
+
options = AgentBase.options(default: DEFAULT_OPTIONS, options: opts)
|
21
28
|
# root id is required for access key
|
22
29
|
@root_id = options[:root_id]
|
23
30
|
rest_params = { base_url: options[:url]}
|
24
|
-
if
|
25
|
-
rest_params[:headers] = {
|
26
|
-
Aspera::Node::HEADER_X_ASPERA_ACCESS_KEY => options[:username],
|
27
|
-
'Authorization' => options[:password]
|
28
|
-
}
|
31
|
+
if Oauth.bearer?(options[:password])
|
29
32
|
raise 'root_id is required for access key' if @root_id.nil?
|
33
|
+
rest_params[:headers] = Aspera::Node.bearer_headers(options[:password], access_key: options[:username])
|
30
34
|
else
|
31
35
|
rest_params[:auth] = {
|
32
36
|
type: :basic,
|
@@ -37,6 +41,7 @@ module Aspera
|
|
37
41
|
@node_api = Rest.new(rest_params)
|
38
42
|
# TODO: currently only supports one transfer. This is bad shortcut. but ok for CLI.
|
39
43
|
@transfer_id = nil
|
44
|
+
# Log.log.debug{Log.dump(:agent_options, @options)}
|
40
45
|
end
|
41
46
|
|
42
47
|
# used internally to ensure node api is set before using.
|
@@ -45,7 +50,7 @@ module Aspera
|
|
45
50
|
return @node_api
|
46
51
|
end
|
47
52
|
# use this to read the node_api end point.
|
48
|
-
attr_reader :node_api
|
53
|
+
# attr_reader :node_api
|
49
54
|
|
50
55
|
# use this to set the node_api end point before using the class.
|
51
56
|
def node_api=(new_value)
|
@@ -94,43 +99,45 @@ module Aspera
|
|
94
99
|
|
95
100
|
# generic method
|
96
101
|
def wait_for_transfers_completion
|
97
|
-
|
98
|
-
|
102
|
+
# set to true when we know the total size of the transfer
|
103
|
+
session_started = false
|
104
|
+
bytes_expected = nil
|
99
105
|
# lets emulate management events to display progress bar
|
100
106
|
loop do
|
101
107
|
# status is empty sometimes with status 200...
|
102
|
-
transfer_data = node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {'status' => 'unknown'} rescue {'status' => 'waiting(
|
108
|
+
transfer_data = node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {'status' => 'unknown'} rescue {'status' => 'waiting(api error)'}
|
103
109
|
case transfer_data['status']
|
104
|
-
when 'completed'
|
105
|
-
notify_end(@transfer_id)
|
106
|
-
break
|
107
110
|
when 'waiting', 'partially_completed', 'unknown', 'waiting(read error)'
|
108
|
-
|
109
|
-
spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
|
110
|
-
spinner.start
|
111
|
-
end
|
112
|
-
spinner.update(title: transfer_data['status'])
|
113
|
-
spinner.spin
|
111
|
+
notify_progress(session_id: nil, type: :pre_start, info: transfer_data['status'])
|
114
112
|
when 'running'
|
115
|
-
if !
|
113
|
+
if !session_started
|
114
|
+
notify_progress(session_id: @transfer_id, type: :session_start)
|
115
|
+
session_started = true
|
116
|
+
end
|
117
|
+
message = transfer_data['status']
|
118
|
+
message = "#{message} (#{transfer_data['error_desc']})" if !transfer_data['error_desc']&.empty?
|
119
|
+
notify_progress(session_id: nil, type: :pre_start, info: message)
|
120
|
+
if bytes_expected.nil? &&
|
121
|
+
transfer_data['precalc'].is_a?(Hash) &&
|
116
122
|
transfer_data['precalc']['status'].eql?('ready')
|
117
|
-
|
118
|
-
|
119
|
-
else
|
120
|
-
notify_progress(@transfer_id, transfer_data['bytes_transferred'])
|
123
|
+
bytes_expected = transfer_data['precalc']['bytes_expected']
|
124
|
+
notify_progress(type: :session_size, session_id: @transfer_id, info: bytes_expected)
|
121
125
|
end
|
126
|
+
notify_progress(type: :transfer, session_id: @transfer_id, info: transfer_data['bytes_transferred'])
|
127
|
+
when 'completed'
|
128
|
+
notify_progress(type: :transfer, session_id: @transfer_id, info: bytes_expected) if bytes_expected
|
129
|
+
notify_progress(type: :end, session_id: @transfer_id)
|
130
|
+
break
|
122
131
|
when 'failed'
|
132
|
+
notify_progress(type: :end, session_id: @transfer_id)
|
123
133
|
# Bug in HSTS ? transfer is marked failed, but there is no reason
|
124
|
-
if transfer_data['error_code'].eql?(0) && transfer_data['error_desc'].empty?
|
125
|
-
notify_end(@transfer_id)
|
126
|
-
break
|
127
|
-
end
|
134
|
+
break if transfer_data['error_code'].eql?(0) && transfer_data['error_desc'].empty?
|
128
135
|
raise Fasp::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
|
129
136
|
else
|
130
137
|
Log.log.warn{"transfer_data -> #{transfer_data}"}
|
131
138
|
raise Fasp::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
|
132
139
|
end
|
133
|
-
sleep(1)
|
140
|
+
sleep(1.0)
|
134
141
|
end
|
135
142
|
# TODO: get status of sessions
|
136
143
|
return []
|