aspera-cli 4.14.0 → 4.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +54 -3
  4. data/CONTRIBUTING.md +7 -7
  5. data/README.md +1457 -880
  6. data/bin/ascli +18 -9
  7. data/bin/asession +12 -14
  8. data/examples/proxy.pac +1 -1
  9. data/lib/aspera/aoc.rb +198 -127
  10. data/lib/aspera/ascmd.rb +24 -14
  11. data/lib/aspera/cli/basic_auth_plugin.rb +9 -6
  12. data/lib/aspera/cli/error.rb +17 -0
  13. data/lib/aspera/cli/extended_value.rb +47 -12
  14. data/lib/aspera/cli/formatter.rb +260 -171
  15. data/lib/aspera/cli/hints.rb +80 -0
  16. data/lib/aspera/cli/main.rb +101 -147
  17. data/lib/aspera/cli/manager.rb +160 -124
  18. data/lib/aspera/cli/plugin.rb +70 -59
  19. data/lib/aspera/cli/plugins/alee.rb +0 -1
  20. data/lib/aspera/cli/plugins/aoc.rb +239 -273
  21. data/lib/aspera/cli/plugins/ats.rb +8 -5
  22. data/lib/aspera/cli/plugins/bss.rb +2 -2
  23. data/lib/aspera/cli/plugins/config.rb +516 -375
  24. data/lib/aspera/cli/plugins/console.rb +40 -0
  25. data/lib/aspera/cli/plugins/cos.rb +4 -5
  26. data/lib/aspera/cli/plugins/faspex.rb +99 -84
  27. data/lib/aspera/cli/plugins/faspex5.rb +179 -148
  28. data/lib/aspera/cli/plugins/node.rb +219 -153
  29. data/lib/aspera/cli/plugins/orchestrator.rb +52 -17
  30. data/lib/aspera/cli/plugins/preview.rb +46 -32
  31. data/lib/aspera/cli/plugins/server.rb +57 -17
  32. data/lib/aspera/cli/plugins/shares.rb +34 -12
  33. data/lib/aspera/cli/sync_actions.rb +68 -0
  34. data/lib/aspera/cli/transfer_agent.rb +45 -55
  35. data/lib/aspera/cli/transfer_progress.rb +74 -0
  36. data/lib/aspera/cli/version.rb +1 -1
  37. data/lib/aspera/colors.rb +3 -1
  38. data/lib/aspera/command_line_builder.rb +14 -11
  39. data/lib/aspera/cos_node.rb +3 -2
  40. data/lib/aspera/environment.rb +17 -6
  41. data/lib/aspera/fasp/agent_aspera.rb +126 -0
  42. data/lib/aspera/fasp/agent_base.rb +31 -77
  43. data/lib/aspera/fasp/agent_connect.rb +21 -22
  44. data/lib/aspera/fasp/agent_direct.rb +88 -102
  45. data/lib/aspera/fasp/agent_httpgw.rb +196 -192
  46. data/lib/aspera/fasp/agent_node.rb +41 -34
  47. data/lib/aspera/fasp/agent_trsdk.rb +75 -34
  48. data/lib/aspera/fasp/error_info.rb +2 -2
  49. data/lib/aspera/fasp/faux_file.rb +52 -0
  50. data/lib/aspera/fasp/installation.rb +43 -184
  51. data/lib/aspera/fasp/management.rb +244 -0
  52. data/lib/aspera/fasp/parameters.rb +59 -26
  53. data/lib/aspera/fasp/parameters.yaml +75 -8
  54. data/lib/aspera/fasp/products.rb +162 -0
  55. data/lib/aspera/fasp/transfer_spec.rb +1 -1
  56. data/lib/aspera/fasp/uri.rb +4 -4
  57. data/lib/aspera/faspex_gw.rb +2 -2
  58. data/lib/aspera/faspex_postproc.rb +2 -2
  59. data/lib/aspera/hash_ext.rb +2 -2
  60. data/lib/aspera/json_rpc.rb +49 -0
  61. data/lib/aspera/line_logger.rb +23 -0
  62. data/lib/aspera/log.rb +57 -16
  63. data/lib/aspera/node.rb +97 -14
  64. data/lib/aspera/oauth.rb +36 -18
  65. data/lib/aspera/open_application.rb +4 -4
  66. data/lib/aspera/persistency_folder.rb +2 -2
  67. data/lib/aspera/preview/file_types.rb +4 -2
  68. data/lib/aspera/preview/generator.rb +22 -35
  69. data/lib/aspera/preview/options.rb +2 -0
  70. data/lib/aspera/preview/terminal.rb +24 -13
  71. data/lib/aspera/preview/utils.rb +19 -26
  72. data/lib/aspera/rest.rb +103 -72
  73. data/lib/aspera/rest_call_error.rb +1 -1
  74. data/lib/aspera/rest_error_analyzer.rb +15 -14
  75. data/lib/aspera/rest_errors_aspera.rb +37 -34
  76. data/lib/aspera/secret_hider.rb +14 -16
  77. data/lib/aspera/ssh.rb +4 -1
  78. data/lib/aspera/sync.rb +128 -122
  79. data/lib/aspera/temp_file_manager.rb +10 -3
  80. data/lib/aspera/web_auth.rb +10 -7
  81. data/lib/aspera/web_server_simple.rb +9 -4
  82. data.tar.gz.sig +0 -0
  83. metadata +33 -15
  84. metadata.gz.sig +0 -0
  85. data/lib/aspera/cli/listener/line_dump.rb +0 -19
  86. data/lib/aspera/cli/listener/logger.rb +0 -22
  87. data/lib/aspera/cli/listener/progress.rb +0 -50
  88. data/lib/aspera/cli/listener/progress_multi.rb +0 -84
  89. data/lib/aspera/cli/plugins/sync.rb +0 -44
  90. data/lib/aspera/fasp/listener.rb +0 -13
@@ -2,6 +2,7 @@
2
2
 
3
3
  require 'aspera/fasp/agent_base'
4
4
  require 'aspera/fasp/transfer_spec'
5
+ require 'aspera/fasp/faux_file'
5
6
  require 'aspera/log'
6
7
  require 'aspera/rest'
7
8
  require 'securerandom'
@@ -9,86 +10,147 @@ require 'websocket'
9
10
  require 'base64'
10
11
  require 'json'
11
12
 
12
- # HTTP GW Upload protocol
13
- # -----------------------
14
- # v1
15
- # 1 - MessageType: String (Transfer Spec) JSON : type: transfer_spec, acknowledged with "end upload"
16
- # 2.. - MessageType: String (Slice Upload start) JSON : type: slice_upload, acknowledged with "end upload"
17
- # v2
18
- # 1 - MessageType: String (Transfer Spec) JSON : type: transfer_spec, acknowledged with "end upload"
19
- # 2 - MessageType: String (Slice Upload start) JSON : type: slice_upload, acknowledged with "end_slice_upload"
20
- # 3.. - MessageType: ByteArray (File Size) Chunks : acknowledged with "end upload"
21
- # last - MessageType: String (Slice Upload end) JSON : type: slice_upload, acknowledged with "end_slice_upload"
22
-
23
- # ref: https://api.ibm.com/explorer/catalog/aspera/product/ibm-aspera/api/http-gateway-api/doc/guides-toc
24
- # https://developer.ibm.com/apis/catalog?search=%22aspera%20http%22
25
13
  module Aspera
26
14
  module Fasp
27
- # start a transfer using Aspera HTTP Gateway, using web socket session for uploads
15
+ # Start a transfer using Aspera HTTP Gateway, using web socket secure for uploads
16
+ # ref: https://api.ibm.com/explorer/catalog/aspera/product/ibm-aspera/api/http-gateway-api/doc/guides-toc
17
+ # https://developer.ibm.com/apis/catalog?search=%22aspera%20http%22
18
+ # HTTP GW Upload protocol:
19
+ # # type Contents Ack Counter
20
+ # v1
21
+ # 0 JSON.transfer_spec Transfer Spec "end upload" sent_general
22
+ # 1.. JSON.slice_upload File base64 chunks "end upload" sent_general
23
+ # v2
24
+ # 0 JSON.transfer_spec Transfer Spec "end upload" sent_general
25
+ # 1 JSON.slice_upload File start "end_slice_upload" sent_v2_delimiter
26
+ # 2.. Binary File binary chunks "end upload" sent_general
27
+ # last JSON.slice_upload File end "end_slice_upload" sent_v2_delimiter
28
28
  class AgentHttpgw < Aspera::Fasp::AgentBase
29
- # message returned by HTTP GW in case of success
29
+ MSG_SEND_TRANSFER_SPEC = 'transfer_spec'
30
+ MSG_SEND_SLICE_UPLOAD = 'slice_upload'
30
31
  MSG_RECV_DATA_RECEIVED_SIGNAL = 'end upload'
31
32
  MSG_RECV_SLICE_UPLOAD_SIGNAL = 'end_slice_upload'
32
- MSG_SEND_SLICE_UPLOAD = 'slice_upload'
33
- MSG_SEND_TRANSFER_SPEC = 'transfer_spec'
34
33
  # upload API versions
35
34
  API_V1 = 'v1'
36
35
  API_V2 = 'v2'
37
36
  # options available in CLI (transfer_info)
38
37
  DEFAULT_OPTIONS = {
39
- url: nil,
40
- upload_chunk_size: 64_000,
41
- upload_bar_refresh_sec: 0.5,
42
- api_version: API_V2,
43
- synchronous: true
38
+ url: :required,
39
+ upload_chunk_size: 64_000,
40
+ api_version: API_V2,
41
+ synchronous: false
44
42
  }.freeze
45
43
  DEFAULT_BASE_PATH = '/aspera/http-gwy'
46
- LOG_WS_MAIN = 'ws: send: '.green
47
- LOG_WS_THREAD = 'ws: ack: '.red
44
+ THR_RECV = 'recv'
45
+ LOG_WS_SEND = 'ws: send: '.red
46
+ LOG_WS_RECV = "ws: #{THR_RECV}: ".green
48
47
  private_constant :DEFAULT_OPTIONS, :MSG_RECV_DATA_RECEIVED_SIGNAL, :MSG_RECV_SLICE_UPLOAD_SIGNAL, :API_V1, :API_V2
49
48
 
50
49
  # send message on http gw web socket
51
50
  def ws_snd_json(msg_type, payload)
52
51
  if msg_type.eql?(MSG_SEND_SLICE_UPLOAD) && @options[:api_version].eql?(API_V2)
53
- @shared_info[:count][:sent_v2_slice] += 1
52
+ @shared_info[:count][:sent_v2_delimiter] += 1
54
53
  else
55
- @shared_info[:count][:sent_other] += 1
54
+ @shared_info[:count][:sent_general] += 1
56
55
  end
57
56
  Log.log.debug do
58
57
  log_data = payload.dup
59
58
  log_data[:data] = "[data #{log_data[:data].length} bytes]" if log_data.key?(:data)
60
- "send_txt: #{msg_type}: #{JSON.generate(log_data)}"
59
+ "#{LOG_WS_SEND}json: #{msg_type}: #{JSON.generate(log_data)}"
61
60
  end
62
- ws_send(JSON.generate({msg_type => payload}))
61
+ ws_send(ws_type: :text, data: JSON.generate({msg_type => payload}))
63
62
  end
64
63
 
65
- def ws_send(data_to_send, type: :text)
66
- Log.log.debug{"#{LOG_WS_MAIN}send low: type: #{type}"}
67
- @shared_info[:count][:sent_other] += 1 if type.eql?(:binary)
68
- Log.log.debug{"#{LOG_WS_MAIN}counts: #{@shared_info[:count]}"}
69
- frame = ::WebSocket::Frame::Outgoing::Client.new(data: data_to_send, type: type, version: @ws_handshake.version)
70
- @ws_io.write(frame.to_s)
64
+ # send data on http gw web socket
65
+ def ws_send(ws_type:, data:)
66
+ Log.log.debug{"#{LOG_WS_SEND}sending: #{ws_type} (#{data&.length || 0} bytes)"}
67
+ @shared_info[:count][:sent_general] += 1 if ws_type.eql?(:binary)
68
+ frame_generator = ::WebSocket::Frame::Outgoing::Client.new(data: data, type: ws_type, version: @ws_handshake.version)
69
+ @ws_io.write(frame_generator.to_s)
70
+ if @options[:synchronous]
71
+ @shared_info[:mutex].synchronize do
72
+ # if read thread exited, there will be no more updates
73
+ # we allow for 1 of difference else it stays blocked
74
+ while @ws_read_thread.alive? &&
75
+ @shared_info[:read_exception].nil? &&
76
+ (((@shared_info[:count][:sent_general] - @shared_info[:count][:received_general]) > 1) ||
77
+ ((@shared_info[:count][:received_v2_delimiter] - @shared_info[:count][:sent_v2_delimiter]) > 1))
78
+ if !@shared_info[:cond_var].wait(@shared_info[:mutex], 2.0)
79
+ Log.log.debug{"#{LOG_WS_SEND}#{'timeout'.blue}: #{@shared_info[:count]}"}
80
+ end
81
+ end
82
+ end
83
+ end
84
+ raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
85
+ Log.log.debug{"#{LOG_WS_SEND}counts: #{@shared_info[:count]}"}
71
86
  end
72
87
 
73
- # wait for all message sent to be acknowledged by HTTPGW server
74
- def wait_for_sent_msg_ack_or_exception
75
- return unless @options[:synchronous]
76
- @shared_info[:mutex].synchronize do
77
- while (@shared_info[:count][:received_data] != @shared_info[:count][:sent_other]) ||
78
- (@shared_info[:count][:received_v2_slice] != @shared_info[:count][:sent_v2_slice])
79
- Log.log.debug{"#{LOG_WS_MAIN}wait: counts: #{@shared_info[:count]}"}
80
- @shared_info[:cond_var].wait(@shared_info[:mutex], 1.0)
81
- raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
88
+ # message processing for read thread
89
+ def process_received_message(message)
90
+ Log.log.debug{"#{LOG_WS_RECV}message: [#{message}] (#{message.class})"}
91
+ if message.eql?(MSG_RECV_DATA_RECEIVED_SIGNAL)
92
+ @shared_info[:mutex].synchronize do
93
+ @shared_info[:count][:received_general] += 1
94
+ @shared_info[:cond_var].signal
95
+ end
96
+ elsif message.eql?(MSG_RECV_SLICE_UPLOAD_SIGNAL)
97
+ @shared_info[:mutex].synchronize do
98
+ @shared_info[:count][:received_v2_delimiter] += 1
99
+ @shared_info[:cond_var].signal
82
100
  end
101
+ else
102
+ message.chomp!
103
+ error_message =
104
+ if message.start_with?('"') && message.end_with?('"')
105
+ # remove double quotes : 1..-2
106
+ JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message']
107
+ elsif message.start_with?('{') && message.end_with?('}')
108
+ JSON.parse(message)['message']
109
+ else
110
+ "unknown message from gateway: [#{message}]"
111
+ end
112
+ raise error_message
83
113
  end
84
- Log.log.debug{"#{LOG_WS_MAIN}sync ok: counts: #{@shared_info[:count]}"}
114
+ end
115
+
116
+ # main function of read thread
117
+ def process_read_thread
118
+ Log.log.debug{"#{LOG_WS_RECV}read thread started"}
119
+ frame_parser = ::WebSocket::Frame::Incoming::Client.new(version: @ws_handshake.version)
120
+ until @ws_io.eof?
121
+ begin # rubocop:disable Style/RedundantBegin
122
+ # ready byte by byte until frame is ready
123
+ # blocking read
124
+ byte = @ws_io.read(1)
125
+ Log.log.trace1{"#{LOG_WS_RECV}read: #{byte} (#{byte.class}) eof=#{@ws_io.eof?}"}
126
+ frame_parser << byte
127
+ frame_ok = frame_parser.next
128
+ next if frame_ok.nil?
129
+ process_received_message(frame_ok.data.to_s)
130
+ Log.log.debug{"#{LOG_WS_RECV}counts: #{@shared_info[:count]}"}
131
+ rescue => e
132
+ Log.log.debug{"#{LOG_WS_RECV}Exception: #{e}"}
133
+ @shared_info[:mutex].synchronize do
134
+ @shared_info[:read_exception] = e
135
+ @shared_info[:cond_var].signal
136
+ end
137
+ break
138
+ end # begin/rescue
139
+ end # loop
140
+ Log.log.debug do
141
+ "#{LOG_WS_RECV}exception: #{@shared_info[:read_exception]},cls=#{@shared_info[:read_exception].class})"
142
+ end unless @shared_info[:read_exception].nil?
143
+ Log.log.debug{"#{LOG_WS_RECV}read thread stopped (ws eof=#{@ws_io.eof?})"}
85
144
  end
86
145
 
87
146
  def upload(transfer_spec)
147
+ # identify this session uniquely
148
+ session_id = SecureRandom.uuid
149
+ notify_progress(session_id: nil, type: :pre_start, info: 'starting')
88
150
  # total size of all files
89
151
  total_bytes_to_transfer = 0
90
152
  # we need to keep track of actual file path because transfer spec is modified to be sent in web socket
91
- source_paths = []
153
+ files_to_read = []
92
154
  # get source root or nil
93
155
  source_root = transfer_spec.key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] : nil
94
156
  # source root is ignored by GW, used only here
@@ -97,143 +159,96 @@ module Aspera
97
159
  # modify transfer spec to be suitable for GW
98
160
  transfer_spec['paths'].each do |item|
99
161
  # save actual file location to be able read contents later
100
- full_src_filepath = item['source']
101
- # add source root if needed
102
- full_src_filepath = File.join(source_root, full_src_filepath) unless source_root.nil?
103
- # GW expects a simple file name in 'source' but if user wants to change the name, we take it
104
- item['source'] = File.basename(item['destination'].nil? ? item['source'] : item['destination'])
105
- item['file_size'] = File.size(full_src_filepath)
106
- total_bytes_to_transfer += item['file_size']
162
+ file_to_add = FauxFile.open(item['source'])
163
+ if file_to_add
164
+ item['source'] = file_to_add.path
165
+ item['file_size'] = file_to_add.size
166
+ else
167
+ file_to_add = item['source']
168
+ # add source root if needed
169
+ file_to_add = File.join(source_root, file_to_add) unless source_root.nil?
170
+ # GW expects a simple file name in 'source' but if user wants to change the name, we take it
171
+ item['source'] = File.basename(item['destination'].nil? ? item['source'] : item['destination'])
172
+ item['file_size'] = File.size(file_to_add)
173
+ end
107
174
  # save so that we can actually read the file later
108
- source_paths.push(full_src_filepath)
175
+ files_to_read.push(file_to_add)
176
+ total_bytes_to_transfer += item['file_size']
109
177
  end
110
- # identify this session uniquely
111
- session_id = SecureRandom.uuid
112
178
  upload_url = File.join(@gw_api.params[:base_url], @options[:api_version], 'upload')
113
- # uri = URI.parse(upload_url)
179
+ notify_progress(session_id: nil, type: :pre_start, info: 'connecting wss')
114
180
  # open web socket to end point (equivalent to Net::HTTP.start)
115
- http_socket = Rest.start_http_session(upload_url)
116
- # little hack to get the socket opened for HTTP, handy because HTTP debug will be available
117
- @ws_io = http_socket.instance_variable_get(:@socket)
118
- # @ws_io.debug_output = Log.log
181
+ http_session = Rest.start_http_session(upload_url)
182
+ # get the underlying socket i/o
183
+ @ws_io = Rest.io_http_session(http_session)
119
184
  @ws_handshake = ::WebSocket::Handshake::Client.new(url: upload_url, headers: {})
120
185
  @ws_io.write(@ws_handshake.to_s)
121
186
  sleep(0.1)
122
187
  @ws_handshake << @ws_io.readuntil("\r\n\r\n")
123
188
  raise 'Error in websocket handshake' unless @ws_handshake.finished?
124
- Log.log.debug{"#{LOG_WS_MAIN}handshake success"}
189
+ Log.log.debug{"#{LOG_WS_SEND}handshake success"}
190
+ # start read thread after handshake
191
+ @ws_read_thread = Thread.new {process_read_thread}
192
+ notify_progress(session_id: session_id, type: :session_start)
193
+ notify_progress(session_id: session_id, type: :session_size, info: total_bytes_to_transfer)
194
+ sleep(1)
125
195
  # data shared between main thread and read thread
126
196
  @shared_info = {
127
197
  read_exception: nil, # error message if any in callback
128
198
  count: {
129
- received_data: 0, # number of files received on other side
130
- received_v2_slice: 0, # number of slices received on other side
131
- sent_other: 0,
132
- sent_v2_slice: 0
199
+ sent_general: 0,
200
+ received_general: 0,
201
+ sent_v2_delimiter: 0,
202
+ received_v2_delimiter: 0
133
203
  },
134
204
  mutex: Mutex.new,
135
205
  cond_var: ConditionVariable.new
136
206
  }
137
- # start read thread
138
- ws_read_thread = Thread.new do
139
- Log.log.debug{"#{LOG_WS_THREAD}read started"}
140
- frame = ::WebSocket::Frame::Incoming::Client.new(version: @ws_handshake.version)
141
- loop do
142
- begin # rubocop:disable Style/RedundantBegin
143
- # unless (recv_data = @ws_io.getc)
144
- # sleep(0.1)
145
- # next
146
- # end
147
- # frame << recv_data
148
- # frame << @ws_io.readuntil("\n")
149
- # frame << @ws_io.read_all
150
- frame << @ws_io.read(1)
151
- while (msg = frame.next)
152
- Log.log.debug{"#{LOG_WS_THREAD}type: #{msg.class}"}
153
- message = msg.data
154
- Log.log.debug{"#{LOG_WS_THREAD}message: [#{message}]"}
155
- if message.eql?(MSG_RECV_DATA_RECEIVED_SIGNAL)
156
- @shared_info[:mutex].synchronize do
157
- @shared_info[:count][:received_data] += 1
158
- @shared_info[:cond_var].signal
159
- end
160
- elsif message.eql?(MSG_RECV_SLICE_UPLOAD_SIGNAL)
161
- @shared_info[:mutex].synchronize do
162
- @shared_info[:count][:received_v2_slice] += 1
163
- @shared_info[:cond_var].signal
164
- end
165
- else
166
- message.chomp!
167
- error_message =
168
- if message.start_with?('"') && message.end_with?('"')
169
- JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message']
170
- elsif message.start_with?('{') && message.end_with?('}')
171
- JSON.parse(message)['message']
172
- else
173
- "unknown message from gateway: [#{message}]"
174
- end
175
- raise error_message
176
- end
177
- Log.log.debug{"#{LOG_WS_THREAD}counts: #{@shared_info[:count]}"}
178
- end # while
179
- rescue => e
180
- Log.log.debug{"#{LOG_WS_THREAD}Exception: #{e}"}
181
- @shared_info[:mutex].synchronize do
182
- @shared_info[:read_exception] = e unless e.is_a?(EOFError)
183
- @shared_info[:cond_var].signal
184
- end
185
- break
186
- end # begin
187
- end # loop
188
- Log.log.debug{"#{LOG_WS_THREAD}stopping (exc=#{@shared_info[:read_exception]},cls=#{@shared_info[:read_exception].class})"}
189
- end
190
207
  # notify progress bar
191
- notify_begin(session_id, total_bytes_to_transfer)
208
+ notify_progress(type: :session_size, session_id: session_id, info: total_bytes_to_transfer)
192
209
  # first step send transfer spec
193
- Log.dump(:ws_spec, transfer_spec)
210
+ Log.log.debug{Log.dump(:ws_spec, transfer_spec)}
194
211
  ws_snd_json(MSG_SEND_TRANSFER_SPEC, transfer_spec)
195
- wait_for_sent_msg_ack_or_exception
196
212
  # current file index
197
213
  file_index = 0
198
214
  # aggregate size sent
199
- sent_bytes = 0
200
- # last progress event
201
- last_progress_time = nil
202
-
215
+ session_sent_bytes = 0
216
+ # process each file
203
217
  transfer_spec['paths'].each do |item|
204
- # TODO: get mime type?
205
- file_mime_type = ''
206
- file_size = item['file_size']
207
- file_name = File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
208
- # compute total number of slices
209
- slice_total = ((file_size - 1) / @options[:upload_chunk_size]) + 1
210
- File.open(source_paths[file_index]) do |file|
211
- # current slice index
212
- slice_index = 0
218
+ slice_info = {
219
+ name: nil,
220
+ # TODO: get mime type?
221
+ type: 'application/octet-stream',
222
+ size: item['file_size'],
223
+ slice: 0, # current slice index
224
+ # index of last slice (i.e number of slices - 1)
225
+ last_slice: (item['file_size'] - 1) / @options[:upload_chunk_size],
226
+ fileIndex: file_index
227
+ }
228
+ file = files_to_read[file_index]
229
+ if file.is_a?(FauxFile)
230
+ slice_info[:name] = file.path
231
+ else
232
+ file = File.open(file)
233
+ slice_info[:name] = File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
234
+ end
235
+ begin
213
236
  until file.eof?
214
- file_bin_data = file.read(@options[:upload_chunk_size])
215
- slice_data = {
216
- name: file_name,
217
- type: file_mime_type,
218
- size: file_size,
219
- slice: slice_index,
220
- total_slices: slice_total,
221
- fileIndex: file_index
222
- }
223
- # Log.dump(:slice_data,slice_data) #if slice_index.eql?(0)
237
+ slice_bin_data = file.read(@options[:upload_chunk_size])
224
238
  # interrupt main thread if read thread failed
225
239
  raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
226
240
  begin
227
241
  if @options[:api_version].eql?(API_V1)
228
- slice_data[:data] = Base64.strict_encode64(file_bin_data)
229
- ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_data)
242
+ slice_info[:data] = Base64.strict_encode64(slice_bin_data)
243
+ ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_info)
230
244
  else
231
- ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_data) if slice_index.eql?(0)
232
- ws_send(file_bin_data, type: :binary)
233
- Log.log.debug{"#{LOG_WS_MAIN}sent bin buffer: #{file_index} / #{slice_index}"}
234
- ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_data) if slice_index.eql?(slice_total - 1)
245
+ # send once, before data, at beginning
246
+ ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_info) if slice_info[:slice].eql?(0)
247
+ ws_send(ws_type: :binary, data: slice_bin_data)
248
+ Log.log.debug{"#{LOG_WS_SEND}buffer: file: #{file_index}, slice: #{slice_info[:slice]}/#{slice_info[:last_slice]}"}
249
+ # send once, after data, at end
250
+ ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_info) if slice_info[:slice].eql?(slice_info[:last_slice])
235
251
  end
236
- wait_for_sent_msg_ack_or_exception
237
252
  rescue Errno::EPIPE => e
238
253
  raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
239
254
  raise e
@@ -241,26 +256,25 @@ module Aspera
241
256
  Log.log.warn{'A timeout condition using HTTPGW may signal a permission problem on destination. Check ascp logs on httpgw.'}
242
257
  raise e
243
258
  end
244
- sent_bytes += file_bin_data.length
245
- current_time = Time.now
246
- if last_progress_time.nil? || ((current_time - last_progress_time) > @options[:upload_bar_refresh_sec])
247
- notify_progress(session_id, sent_bytes)
248
- last_progress_time = current_time
249
- end
250
- slice_index += 1
259
+ session_sent_bytes += slice_bin_data.length
260
+ notify_progress(type: :transfer, session_id: session_id, info: session_sent_bytes)
261
+ slice_info[:slice] += 1
251
262
  end
263
+ ensure
264
+ file.close
252
265
  end
253
266
  file_index += 1
254
- end
255
-
256
- Log.log.debug('Finished upload, waiting for end of read thread.')
257
- ws_read_thread.join
258
- Log.log.debug{"Read thread joined, result: #{@shared_info[:count][:received_data]} / #{@shared_info[:count][:sent_other]}"}
259
- ws_send(nil, type: :close) unless @ws_io.nil?
267
+ end # loop on files
268
+ # throttling may have skipped last one
269
+ notify_progress(type: :transfer, session_id: session_id, info: session_sent_bytes)
270
+ notify_progress(type: :end, session_id: session_id)
271
+ ws_send(ws_type: :close, data: nil)
272
+ Log.log.debug("Finished upload, waiting for end of #{THR_RECV} thread.")
273
+ @ws_read_thread.join
274
+ Log.log.debug{'Read thread joined'}
275
+ # session no more used
260
276
  @ws_io = nil
261
- http_socket&.finish
262
- notify_progress(session_id, sent_bytes)
263
- notify_end(session_id)
277
+ http_session&.finish
264
278
  end
265
279
 
266
280
  def download(transfer_spec)
@@ -299,7 +313,7 @@ module Aspera
299
313
  raise 'GW URL must be set' if @gw_api.nil?
300
314
  raise 'paths: must be Array' unless transfer_spec['paths'].is_a?(Array)
301
315
  raise 'only token based transfer is supported in GW' unless transfer_spec['token'].is_a?(String)
302
- Log.dump(:user_spec, transfer_spec)
316
+ Log.log.debug{Log.dump(:user_spec, transfer_spec)}
303
317
  transfer_spec['authentication'] ||= 'token'
304
318
  case transfer_spec['direction']
305
319
  when Fasp::TransferSpec::DIRECTION_SEND
@@ -314,43 +328,33 @@ module Aspera
314
328
  # wait for completion of all jobs started
315
329
  # @return list of :success or error message
316
330
  def wait_for_transfers_completion
331
+ # well ... transfer was done in "start"
317
332
  return [:success]
318
333
  end
319
334
 
320
- # terminates monitor thread
321
- def shutdown; end
322
-
335
+ # TODO: is that useful?
323
336
  def url=(api_url); end
324
337
 
325
338
  private
326
339
 
327
340
  def initialize(opts)
328
- Log.dump(:in_options, opts)
329
- # set default options and override if specified
330
- @options = DEFAULT_OPTIONS.dup
331
- raise "httpgw agent parameters (transfer_info): expecting Hash, but have #{opts.class}" unless opts.is_a?(Hash)
332
- opts.symbolize_keys.each do |k, v|
333
- raise "httpgw agent parameter: Unknown: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.key?(k)
334
- @options[k] = v
335
- end
336
- if @options[:url].nil?
337
- available = DEFAULT_OPTIONS.map { |k, v| "#{k}(#{v})"}.join(', ')
338
- raise "Missing mandatory parameter for HTTP GW in transfer_info: url. Allowed parameters: #{available}."
339
- end
340
- # remove /v1 from end
341
+ super(opts)
342
+ @options = AgentBase.options(default: DEFAULT_OPTIONS, options: opts)
343
+ # remove /v1 from end of user-provided GW url: we need the base url only
341
344
  @options[:url].gsub(%r{/v1/*$}, '')
342
- super()
343
345
  @gw_api = Rest.new({base_url: @options[:url]})
344
346
  @api_info = @gw_api.read('v1/info')[:data]
345
- Log.dump(:api_info, @api_info)
346
- if @options[:api_version].nil?
347
- # web socket endpoint: by default use v2 (newer gateways), without base64 encoding
348
- @options[:api_version] = API_V2
349
- # is the latest supported? else revert to old api
350
- @options[:api_version] = API_V1 unless @api_info['endpoints'].any?{|i|i.include?(@options[:api_version])}
347
+ Log.log.debug{Log.dump(:api_info, @api_info)}
348
+ # web socket endpoint: by default use v2 (newer gateways), without base64 encoding
349
+ # is the latest supported? else revert to old api
350
+ if !@options[:api_version].eql?(API_V1)
351
+ if !@api_info['endpoints'].any?{|i|i.include?(@options[:api_version])}
352
+ Log.log.warn{"API version #{@options[:api_version]} not supported, reverting to #{API_V1}"}
353
+ @options[:api_version] = API_V1
354
+ end
351
355
  end
352
356
  @options.freeze
353
- Log.dump(:final_options, @options)
357
+ Log.log.debug{Log.dump(:agent_options, @options)}
354
358
  end
355
359
  end # AgentHttpgw
356
360
  end
@@ -1,32 +1,36 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # cspell:ignore precalc
3
4
  require 'aspera/fasp/agent_base'
4
5
  require 'aspera/fasp/transfer_spec'
5
6
  require 'aspera/node'
6
7
  require 'aspera/log'
7
- require 'tty-spinner'
8
+ require 'aspera/oauth'
8
9
 
9
10
  module Aspera
10
11
  module Fasp
11
12
  # this singleton class is used by the CLI to provide a common interface to start a transfer
12
13
  # before using it, the use must set the `node_api` member.
13
14
  class AgentNode < Aspera::Fasp::AgentBase
15
+ DEFAULT_OPTIONS = {
16
+ url: :required,
17
+ username: :required,
18
+ password: :required,
19
+ root_id: nil
20
+ }.freeze
14
21
  # option include: root_id if the node is an access key
15
- attr_writer :options
22
+ # attr_writer :options
16
23
 
17
- def initialize(options)
18
- raise 'node specification must be Hash' unless options.is_a?(Hash)
19
- %i[url username password].each { |k| raise "missing parameter [#{k}] in node specification: #{options}" unless options.key?(k) }
20
- super()
24
+ def initialize(opts)
25
+ raise 'node specification must be Hash' unless opts.is_a?(Hash)
26
+ super(opts)
27
+ options = AgentBase.options(default: DEFAULT_OPTIONS, options: opts)
21
28
  # root id is required for access key
22
29
  @root_id = options[:root_id]
23
30
  rest_params = { base_url: options[:url]}
24
- if /^Bearer /.match?(options[:password])
25
- rest_params[:headers] = {
26
- Aspera::Node::HEADER_X_ASPERA_ACCESS_KEY => options[:username],
27
- 'Authorization' => options[:password]
28
- }
31
+ if Oauth.bearer?(options[:password])
29
32
  raise 'root_id is required for access key' if @root_id.nil?
33
+ rest_params[:headers] = Aspera::Node.bearer_headers(options[:password], access_key: options[:username])
30
34
  else
31
35
  rest_params[:auth] = {
32
36
  type: :basic,
@@ -37,6 +41,7 @@ module Aspera
37
41
  @node_api = Rest.new(rest_params)
38
42
  # TODO: currently only supports one transfer. This is bad shortcut. but ok for CLI.
39
43
  @transfer_id = nil
44
+ # Log.log.debug{Log.dump(:agent_options, @options)}
40
45
  end
41
46
 
42
47
  # used internally to ensure node api is set before using.
@@ -45,7 +50,7 @@ module Aspera
45
50
  return @node_api
46
51
  end
47
52
  # use this to read the node_api end point.
48
- attr_reader :node_api
53
+ # attr_reader :node_api
49
54
 
50
55
  # use this to set the node_api end point before using the class.
51
56
  def node_api=(new_value)
@@ -94,43 +99,45 @@ module Aspera
94
99
 
95
100
  # generic method
96
101
  def wait_for_transfers_completion
97
- started = false
98
- spinner = nil
102
+ # set to true when we know the total size of the transfer
103
+ session_started = false
104
+ bytes_expected = nil
99
105
  # lets emulate management events to display progress bar
100
106
  loop do
101
107
  # status is empty sometimes with status 200...
102
- transfer_data = node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {'status' => 'unknown'} rescue {'status' => 'waiting(read error)'}
108
+ transfer_data = node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {'status' => 'unknown'} rescue {'status' => 'waiting(api error)'}
103
109
  case transfer_data['status']
104
- when 'completed'
105
- notify_end(@transfer_id)
106
- break
107
110
  when 'waiting', 'partially_completed', 'unknown', 'waiting(read error)'
108
- if spinner.nil?
109
- spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
110
- spinner.start
111
- end
112
- spinner.update(title: transfer_data['status'])
113
- spinner.spin
111
+ notify_progress(session_id: nil, type: :pre_start, info: transfer_data['status'])
114
112
  when 'running'
115
- if !started && transfer_data['precalc'].is_a?(Hash) &&
113
+ if !session_started
114
+ notify_progress(session_id: @transfer_id, type: :session_start)
115
+ session_started = true
116
+ end
117
+ message = transfer_data['status']
118
+ message = "#{message} (#{transfer_data['error_desc']})" if !transfer_data['error_desc']&.empty?
119
+ notify_progress(session_id: nil, type: :pre_start, info: message)
120
+ if bytes_expected.nil? &&
121
+ transfer_data['precalc'].is_a?(Hash) &&
116
122
  transfer_data['precalc']['status'].eql?('ready')
117
- notify_begin(@transfer_id, transfer_data['precalc']['bytes_expected'])
118
- started = true
119
- else
120
- notify_progress(@transfer_id, transfer_data['bytes_transferred'])
123
+ bytes_expected = transfer_data['precalc']['bytes_expected']
124
+ notify_progress(type: :session_size, session_id: @transfer_id, info: bytes_expected)
121
125
  end
126
+ notify_progress(type: :transfer, session_id: @transfer_id, info: transfer_data['bytes_transferred'])
127
+ when 'completed'
128
+ notify_progress(type: :transfer, session_id: @transfer_id, info: bytes_expected) if bytes_expected
129
+ notify_progress(type: :end, session_id: @transfer_id)
130
+ break
122
131
  when 'failed'
132
+ notify_progress(type: :end, session_id: @transfer_id)
123
133
  # Bug in HSTS ? transfer is marked failed, but there is no reason
124
- if transfer_data['error_code'].eql?(0) && transfer_data['error_desc'].empty?
125
- notify_end(@transfer_id)
126
- break
127
- end
134
+ break if transfer_data['error_code'].eql?(0) && transfer_data['error_desc'].empty?
128
135
  raise Fasp::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
129
136
  else
130
137
  Log.log.warn{"transfer_data -> #{transfer_data}"}
131
138
  raise Fasp::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
132
139
  end
133
- sleep(1)
140
+ sleep(1.0)
134
141
  end
135
142
  # TODO: get status of sessions
136
143
  return []