aspera-cli 4.14.0 → 4.16.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (104) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +29 -3
  4. data/CHANGELOG.md +300 -185
  5. data/CONTRIBUTING.md +74 -23
  6. data/README.md +2346 -1619
  7. data/bin/ascli +16 -25
  8. data/bin/asession +15 -15
  9. data/examples/dascli +2 -2
  10. data/examples/proxy.pac +1 -1
  11. data/lib/aspera/aoc.rb +216 -150
  12. data/lib/aspera/ascmd.rb +25 -18
  13. data/lib/aspera/assert.rb +45 -0
  14. data/lib/aspera/cli/basic_auth_plugin.rb +9 -6
  15. data/lib/aspera/cli/error.rb +17 -0
  16. data/lib/aspera/cli/extended_value.rb +51 -16
  17. data/lib/aspera/cli/formatter.rb +276 -174
  18. data/lib/aspera/cli/hints.rb +81 -0
  19. data/lib/aspera/cli/main.rb +114 -147
  20. data/lib/aspera/cli/manager.rb +181 -136
  21. data/lib/aspera/cli/plugin.rb +82 -64
  22. data/lib/aspera/cli/plugins/alee.rb +0 -1
  23. data/lib/aspera/cli/plugins/aoc.rb +327 -331
  24. data/lib/aspera/cli/plugins/ats.rb +12 -8
  25. data/lib/aspera/cli/plugins/bss.rb +2 -2
  26. data/lib/aspera/cli/plugins/config.rb +575 -439
  27. data/lib/aspera/cli/plugins/console.rb +40 -0
  28. data/lib/aspera/cli/plugins/cos.rb +4 -5
  29. data/lib/aspera/cli/plugins/faspex.rb +111 -92
  30. data/lib/aspera/cli/plugins/faspex5.rb +245 -182
  31. data/lib/aspera/cli/plugins/node.rb +239 -160
  32. data/lib/aspera/cli/plugins/orchestrator.rb +56 -19
  33. data/lib/aspera/cli/plugins/preview.rb +54 -38
  34. data/lib/aspera/cli/plugins/server.rb +63 -20
  35. data/lib/aspera/cli/plugins/shares.rb +64 -38
  36. data/lib/aspera/cli/sync_actions.rb +68 -0
  37. data/lib/aspera/cli/transfer_agent.rb +64 -67
  38. data/lib/aspera/cli/transfer_progress.rb +73 -0
  39. data/lib/aspera/cli/version.rb +1 -1
  40. data/lib/aspera/colors.rb +3 -1
  41. data/lib/aspera/command_line_builder.rb +27 -22
  42. data/lib/aspera/cos_node.rb +6 -4
  43. data/lib/aspera/coverage.rb +22 -0
  44. data/lib/aspera/data_repository.rb +33 -2
  45. data/lib/aspera/environment.rb +21 -8
  46. data/lib/aspera/fasp/agent_alpha.rb +116 -0
  47. data/lib/aspera/fasp/agent_base.rb +40 -76
  48. data/lib/aspera/fasp/agent_connect.rb +21 -22
  49. data/lib/aspera/fasp/agent_direct.rb +169 -179
  50. data/lib/aspera/fasp/agent_httpgw.rb +200 -195
  51. data/lib/aspera/fasp/agent_node.rb +43 -35
  52. data/lib/aspera/fasp/agent_trsdk.rb +124 -41
  53. data/lib/aspera/fasp/error_info.rb +2 -2
  54. data/lib/aspera/fasp/faux_file.rb +52 -0
  55. data/lib/aspera/fasp/installation.rb +89 -191
  56. data/lib/aspera/fasp/management.rb +249 -0
  57. data/lib/aspera/fasp/parameters.rb +86 -47
  58. data/lib/aspera/fasp/parameters.yaml +75 -8
  59. data/lib/aspera/fasp/products.rb +162 -0
  60. data/lib/aspera/fasp/resume_policy.rb +7 -5
  61. data/lib/aspera/fasp/sync.rb +273 -0
  62. data/lib/aspera/fasp/transfer_spec.rb +10 -8
  63. data/lib/aspera/fasp/uri.rb +6 -6
  64. data/lib/aspera/faspex_gw.rb +11 -8
  65. data/lib/aspera/faspex_postproc.rb +8 -7
  66. data/lib/aspera/hash_ext.rb +2 -2
  67. data/lib/aspera/id_generator.rb +3 -1
  68. data/lib/aspera/json_rpc.rb +51 -0
  69. data/lib/aspera/keychain/encrypted_hash.rb +46 -11
  70. data/lib/aspera/keychain/macos_security.rb +15 -13
  71. data/lib/aspera/line_logger.rb +23 -0
  72. data/lib/aspera/log.rb +61 -19
  73. data/lib/aspera/nagios.rb +7 -2
  74. data/lib/aspera/node.rb +105 -21
  75. data/lib/aspera/node_simulator.rb +214 -0
  76. data/lib/aspera/oauth.rb +57 -36
  77. data/lib/aspera/open_application.rb +4 -4
  78. data/lib/aspera/persistency_action_once.rb +13 -14
  79. data/lib/aspera/persistency_folder.rb +5 -4
  80. data/lib/aspera/preview/file_types.rb +56 -268
  81. data/lib/aspera/preview/generator.rb +28 -39
  82. data/lib/aspera/preview/options.rb +2 -0
  83. data/lib/aspera/preview/terminal.rb +36 -16
  84. data/lib/aspera/preview/utils.rb +23 -29
  85. data/lib/aspera/proxy_auto_config.rb +6 -3
  86. data/lib/aspera/rest.rb +127 -80
  87. data/lib/aspera/rest_call_error.rb +1 -1
  88. data/lib/aspera/rest_error_analyzer.rb +16 -14
  89. data/lib/aspera/rest_errors_aspera.rb +39 -34
  90. data/lib/aspera/secret_hider.rb +18 -17
  91. data/lib/aspera/ssh.rb +10 -5
  92. data/lib/aspera/temp_file_manager.rb +11 -4
  93. data/lib/aspera/web_auth.rb +10 -7
  94. data/lib/aspera/web_server_simple.rb +11 -5
  95. data.tar.gz.sig +0 -0
  96. metadata +108 -39
  97. metadata.gz.sig +0 -0
  98. data/lib/aspera/cli/listener/line_dump.rb +0 -19
  99. data/lib/aspera/cli/listener/logger.rb +0 -22
  100. data/lib/aspera/cli/listener/progress.rb +0 -50
  101. data/lib/aspera/cli/listener/progress_multi.rb +0 -84
  102. data/lib/aspera/cli/plugins/sync.rb +0 -44
  103. data/lib/aspera/fasp/listener.rb +0 -13
  104. data/lib/aspera/sync.rb +0 -213
@@ -2,93 +2,156 @@
2
2
 
3
3
  require 'aspera/fasp/agent_base'
4
4
  require 'aspera/fasp/transfer_spec'
5
+ require 'aspera/fasp/faux_file'
5
6
  require 'aspera/log'
7
+ require 'aspera/assert'
6
8
  require 'aspera/rest'
7
9
  require 'securerandom'
8
10
  require 'websocket'
9
11
  require 'base64'
10
12
  require 'json'
11
13
 
12
- # HTTP GW Upload protocol
13
- # -----------------------
14
- # v1
15
- # 1 - MessageType: String (Transfer Spec) JSON : type: transfer_spec, acknowledged with "end upload"
16
- # 2.. - MessageType: String (Slice Upload start) JSON : type: slice_upload, acknowledged with "end upload"
17
- # v2
18
- # 1 - MessageType: String (Transfer Spec) JSON : type: transfer_spec, acknowledged with "end upload"
19
- # 2 - MessageType: String (Slice Upload start) JSON : type: slice_upload, acknowledged with "end_slice_upload"
20
- # 3.. - MessageType: ByteArray (File Size) Chunks : acknowledged with "end upload"
21
- # last - MessageType: String (Slice Upload end) JSON : type: slice_upload, acknowledged with "end_slice_upload"
22
-
23
- # ref: https://api.ibm.com/explorer/catalog/aspera/product/ibm-aspera/api/http-gateway-api/doc/guides-toc
24
- # https://developer.ibm.com/apis/catalog?search=%22aspera%20http%22
25
14
  module Aspera
26
15
  module Fasp
27
- # start a transfer using Aspera HTTP Gateway, using web socket session for uploads
16
+ # Start a transfer using Aspera HTTP Gateway, using web socket secure for uploads
17
+ # ref: https://api.ibm.com/explorer/catalog/aspera/product/ibm-aspera/api/http-gateway-api/doc/guides-toc
18
+ # https://developer.ibm.com/apis/catalog?search=%22aspera%20http%22
19
+ # HTTP GW Upload protocol:
20
+ # # type Contents Ack Counter
21
+ # v1
22
+ # 0 JSON.transfer_spec Transfer Spec "end upload" sent_general
23
+ # 1.. JSON.slice_upload File base64 chunks "end upload" sent_general
24
+ # v2
25
+ # 0 JSON.transfer_spec Transfer Spec "end upload" sent_general
26
+ # 1 JSON.slice_upload File start "end_slice_upload" sent_v2_delimiter
27
+ # 2.. Binary File binary chunks "end upload" sent_general
28
+ # last JSON.slice_upload File end "end_slice_upload" sent_v2_delimiter
28
29
  class AgentHttpgw < Aspera::Fasp::AgentBase
29
- # message returned by HTTP GW in case of success
30
+ MSG_SEND_TRANSFER_SPEC = 'transfer_spec'
31
+ MSG_SEND_SLICE_UPLOAD = 'slice_upload'
30
32
  MSG_RECV_DATA_RECEIVED_SIGNAL = 'end upload'
31
33
  MSG_RECV_SLICE_UPLOAD_SIGNAL = 'end_slice_upload'
32
- MSG_SEND_SLICE_UPLOAD = 'slice_upload'
33
- MSG_SEND_TRANSFER_SPEC = 'transfer_spec'
34
34
  # upload API versions
35
35
  API_V1 = 'v1'
36
36
  API_V2 = 'v2'
37
37
  # options available in CLI (transfer_info)
38
38
  DEFAULT_OPTIONS = {
39
- url: nil,
40
- upload_chunk_size: 64_000,
41
- upload_bar_refresh_sec: 0.5,
42
- api_version: API_V2,
43
- synchronous: true
39
+ url: :required,
40
+ upload_chunk_size: 64_000,
41
+ api_version: API_V2,
42
+ synchronous: false
44
43
  }.freeze
45
44
  DEFAULT_BASE_PATH = '/aspera/http-gwy'
46
- LOG_WS_MAIN = 'ws: send: '.green
47
- LOG_WS_THREAD = 'ws: ack: '.red
45
+ THR_RECV = 'recv'
46
+ LOG_WS_SEND = 'ws: send: '.red
47
+ LOG_WS_RECV = "ws: #{THR_RECV}: ".green
48
48
  private_constant :DEFAULT_OPTIONS, :MSG_RECV_DATA_RECEIVED_SIGNAL, :MSG_RECV_SLICE_UPLOAD_SIGNAL, :API_V1, :API_V2
49
49
 
50
50
  # send message on http gw web socket
51
51
  def ws_snd_json(msg_type, payload)
52
52
  if msg_type.eql?(MSG_SEND_SLICE_UPLOAD) && @options[:api_version].eql?(API_V2)
53
- @shared_info[:count][:sent_v2_slice] += 1
53
+ @shared_info[:count][:sent_v2_delimiter] += 1
54
54
  else
55
- @shared_info[:count][:sent_other] += 1
55
+ @shared_info[:count][:sent_general] += 1
56
56
  end
57
57
  Log.log.debug do
58
58
  log_data = payload.dup
59
59
  log_data[:data] = "[data #{log_data[:data].length} bytes]" if log_data.key?(:data)
60
- "send_txt: #{msg_type}: #{JSON.generate(log_data)}"
60
+ "#{LOG_WS_SEND}json: #{msg_type}: #{JSON.generate(log_data)}"
61
61
  end
62
- ws_send(JSON.generate({msg_type => payload}))
62
+ ws_send(ws_type: :text, data: JSON.generate({msg_type => payload}))
63
63
  end
64
64
 
65
- def ws_send(data_to_send, type: :text)
66
- Log.log.debug{"#{LOG_WS_MAIN}send low: type: #{type}"}
67
- @shared_info[:count][:sent_other] += 1 if type.eql?(:binary)
68
- Log.log.debug{"#{LOG_WS_MAIN}counts: #{@shared_info[:count]}"}
69
- frame = ::WebSocket::Frame::Outgoing::Client.new(data: data_to_send, type: type, version: @ws_handshake.version)
70
- @ws_io.write(frame.to_s)
65
+ # send data on http gw web socket
66
+ def ws_send(ws_type:, data:)
67
+ Log.log.debug{"#{LOG_WS_SEND}sending: #{ws_type} (#{data&.length || 0} bytes)"}
68
+ @shared_info[:count][:sent_general] += 1 if ws_type.eql?(:binary)
69
+ frame_generator = ::WebSocket::Frame::Outgoing::Client.new(data: data, type: ws_type, version: @ws_handshake.version)
70
+ @ws_io.write(frame_generator.to_s)
71
+ if @options[:synchronous]
72
+ @shared_info[:mutex].synchronize do
73
+ # if read thread exited, there will be no more updates
74
+ # we allow for 1 of difference else it stays blocked
75
+ while @ws_read_thread.alive? &&
76
+ @shared_info[:read_exception].nil? &&
77
+ (((@shared_info[:count][:sent_general] - @shared_info[:count][:received_general]) > 1) ||
78
+ ((@shared_info[:count][:received_v2_delimiter] - @shared_info[:count][:sent_v2_delimiter]) > 1))
79
+ if !@shared_info[:cond_var].wait(@shared_info[:mutex], 2.0)
80
+ Log.log.debug{"#{LOG_WS_SEND}#{'timeout'.blue}: #{@shared_info[:count]}"}
81
+ end
82
+ end
83
+ end
84
+ end
85
+ raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
86
+ Log.log.debug{"#{LOG_WS_SEND}counts: #{@shared_info[:count]}"}
71
87
  end
72
88
 
73
- # wait for all message sent to be acknowledged by HTTPGW server
74
- def wait_for_sent_msg_ack_or_exception
75
- return unless @options[:synchronous]
76
- @shared_info[:mutex].synchronize do
77
- while (@shared_info[:count][:received_data] != @shared_info[:count][:sent_other]) ||
78
- (@shared_info[:count][:received_v2_slice] != @shared_info[:count][:sent_v2_slice])
79
- Log.log.debug{"#{LOG_WS_MAIN}wait: counts: #{@shared_info[:count]}"}
80
- @shared_info[:cond_var].wait(@shared_info[:mutex], 1.0)
81
- raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
89
+ # message processing for read thread
90
+ def process_received_message(message)
91
+ Log.log.debug{"#{LOG_WS_RECV}message: [#{message}] (#{message.class})"}
92
+ if message.eql?(MSG_RECV_DATA_RECEIVED_SIGNAL)
93
+ @shared_info[:mutex].synchronize do
94
+ @shared_info[:count][:received_general] += 1
95
+ @shared_info[:cond_var].signal
96
+ end
97
+ elsif message.eql?(MSG_RECV_SLICE_UPLOAD_SIGNAL)
98
+ @shared_info[:mutex].synchronize do
99
+ @shared_info[:count][:received_v2_delimiter] += 1
100
+ @shared_info[:cond_var].signal
82
101
  end
102
+ else
103
+ message.chomp!
104
+ error_message =
105
+ if message.start_with?('"') && message.end_with?('"')
106
+ # remove double quotes : 1..-2
107
+ JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message']
108
+ elsif message.start_with?('{') && message.end_with?('}')
109
+ JSON.parse(message)['message']
110
+ else
111
+ "unknown message from gateway: [#{message}]"
112
+ end
113
+ raise error_message
83
114
  end
84
- Log.log.debug{"#{LOG_WS_MAIN}sync ok: counts: #{@shared_info[:count]}"}
115
+ end
116
+
117
+ # main function of read thread
118
+ def process_read_thread
119
+ Log.log.debug{"#{LOG_WS_RECV}read thread started"}
120
+ frame_parser = ::WebSocket::Frame::Incoming::Client.new(version: @ws_handshake.version)
121
+ until @ws_io.eof?
122
+ begin # rubocop:disable Style/RedundantBegin
123
+ # ready byte by byte until frame is ready
124
+ # blocking read
125
+ byte = @ws_io.read(1)
126
+ Log.log.trace1{"#{LOG_WS_RECV}read: #{byte} (#{byte.class}) eof=#{@ws_io.eof?}"}
127
+ frame_parser << byte
128
+ frame_ok = frame_parser.next
129
+ next if frame_ok.nil?
130
+ process_received_message(frame_ok.data.to_s)
131
+ Log.log.debug{"#{LOG_WS_RECV}counts: #{@shared_info[:count]}"}
132
+ rescue => e
133
+ Log.log.debug{"#{LOG_WS_RECV}Exception: #{e}"}
134
+ @shared_info[:mutex].synchronize do
135
+ @shared_info[:read_exception] = e
136
+ @shared_info[:cond_var].signal
137
+ end
138
+ break
139
+ end # begin/rescue
140
+ end # loop
141
+ Log.log.debug do
142
+ "#{LOG_WS_RECV}exception: #{@shared_info[:read_exception]},cls=#{@shared_info[:read_exception].class})"
143
+ end unless @shared_info[:read_exception].nil?
144
+ Log.log.debug{"#{LOG_WS_RECV}read thread stopped (ws eof=#{@ws_io.eof?})"}
85
145
  end
86
146
 
87
147
  def upload(transfer_spec)
148
+ # identify this session uniquely
149
+ session_id = SecureRandom.uuid
150
+ notify_progress(session_id: nil, type: :pre_start, info: 'starting')
88
151
  # total size of all files
89
152
  total_bytes_to_transfer = 0
90
153
  # we need to keep track of actual file path because transfer spec is modified to be sent in web socket
91
- source_paths = []
154
+ files_to_read = []
92
155
  # get source root or nil
93
156
  source_root = transfer_spec.key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] : nil
94
157
  # source root is ignored by GW, used only here
@@ -97,143 +160,96 @@ module Aspera
97
160
  # modify transfer spec to be suitable for GW
98
161
  transfer_spec['paths'].each do |item|
99
162
  # save actual file location to be able read contents later
100
- full_src_filepath = item['source']
101
- # add source root if needed
102
- full_src_filepath = File.join(source_root, full_src_filepath) unless source_root.nil?
103
- # GW expects a simple file name in 'source' but if user wants to change the name, we take it
104
- item['source'] = File.basename(item['destination'].nil? ? item['source'] : item['destination'])
105
- item['file_size'] = File.size(full_src_filepath)
106
- total_bytes_to_transfer += item['file_size']
163
+ file_to_add = FauxFile.open(item['source'])
164
+ if file_to_add
165
+ item['source'] = file_to_add.path
166
+ item['file_size'] = file_to_add.size
167
+ else
168
+ file_to_add = item['source']
169
+ # add source root if needed
170
+ file_to_add = File.join(source_root, file_to_add) unless source_root.nil?
171
+ # GW expects a simple file name in 'source' but if user wants to change the name, we take it
172
+ item['source'] = File.basename(item['destination'].nil? ? item['source'] : item['destination'])
173
+ item['file_size'] = File.size(file_to_add)
174
+ end
107
175
  # save so that we can actually read the file later
108
- source_paths.push(full_src_filepath)
176
+ files_to_read.push(file_to_add)
177
+ total_bytes_to_transfer += item['file_size']
109
178
  end
110
- # identify this session uniquely
111
- session_id = SecureRandom.uuid
112
179
  upload_url = File.join(@gw_api.params[:base_url], @options[:api_version], 'upload')
113
- # uri = URI.parse(upload_url)
180
+ notify_progress(session_id: nil, type: :pre_start, info: 'connecting wss')
114
181
  # open web socket to end point (equivalent to Net::HTTP.start)
115
- http_socket = Rest.start_http_session(upload_url)
116
- # little hack to get the socket opened for HTTP, handy because HTTP debug will be available
117
- @ws_io = http_socket.instance_variable_get(:@socket)
118
- # @ws_io.debug_output = Log.log
182
+ http_session = Rest.start_http_session(upload_url)
183
+ # get the underlying socket i/o
184
+ @ws_io = Rest.io_http_session(http_session)
119
185
  @ws_handshake = ::WebSocket::Handshake::Client.new(url: upload_url, headers: {})
120
186
  @ws_io.write(@ws_handshake.to_s)
121
187
  sleep(0.1)
122
188
  @ws_handshake << @ws_io.readuntil("\r\n\r\n")
123
- raise 'Error in websocket handshake' unless @ws_handshake.finished?
124
- Log.log.debug{"#{LOG_WS_MAIN}handshake success"}
189
+ assert(@ws_handshake.finished?){'Error in websocket handshake'}
190
+ Log.log.debug{"#{LOG_WS_SEND}handshake success"}
191
+ # start read thread after handshake
192
+ @ws_read_thread = Thread.new {process_read_thread}
193
+ notify_progress(session_id: session_id, type: :session_start)
194
+ notify_progress(session_id: session_id, type: :session_size, info: total_bytes_to_transfer)
195
+ sleep(1)
125
196
  # data shared between main thread and read thread
126
197
  @shared_info = {
127
198
  read_exception: nil, # error message if any in callback
128
199
  count: {
129
- received_data: 0, # number of files received on other side
130
- received_v2_slice: 0, # number of slices received on other side
131
- sent_other: 0,
132
- sent_v2_slice: 0
200
+ sent_general: 0,
201
+ received_general: 0,
202
+ sent_v2_delimiter: 0,
203
+ received_v2_delimiter: 0
133
204
  },
134
205
  mutex: Mutex.new,
135
206
  cond_var: ConditionVariable.new
136
207
  }
137
- # start read thread
138
- ws_read_thread = Thread.new do
139
- Log.log.debug{"#{LOG_WS_THREAD}read started"}
140
- frame = ::WebSocket::Frame::Incoming::Client.new(version: @ws_handshake.version)
141
- loop do
142
- begin # rubocop:disable Style/RedundantBegin
143
- # unless (recv_data = @ws_io.getc)
144
- # sleep(0.1)
145
- # next
146
- # end
147
- # frame << recv_data
148
- # frame << @ws_io.readuntil("\n")
149
- # frame << @ws_io.read_all
150
- frame << @ws_io.read(1)
151
- while (msg = frame.next)
152
- Log.log.debug{"#{LOG_WS_THREAD}type: #{msg.class}"}
153
- message = msg.data
154
- Log.log.debug{"#{LOG_WS_THREAD}message: [#{message}]"}
155
- if message.eql?(MSG_RECV_DATA_RECEIVED_SIGNAL)
156
- @shared_info[:mutex].synchronize do
157
- @shared_info[:count][:received_data] += 1
158
- @shared_info[:cond_var].signal
159
- end
160
- elsif message.eql?(MSG_RECV_SLICE_UPLOAD_SIGNAL)
161
- @shared_info[:mutex].synchronize do
162
- @shared_info[:count][:received_v2_slice] += 1
163
- @shared_info[:cond_var].signal
164
- end
165
- else
166
- message.chomp!
167
- error_message =
168
- if message.start_with?('"') && message.end_with?('"')
169
- JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message']
170
- elsif message.start_with?('{') && message.end_with?('}')
171
- JSON.parse(message)['message']
172
- else
173
- "unknown message from gateway: [#{message}]"
174
- end
175
- raise error_message
176
- end
177
- Log.log.debug{"#{LOG_WS_THREAD}counts: #{@shared_info[:count]}"}
178
- end # while
179
- rescue => e
180
- Log.log.debug{"#{LOG_WS_THREAD}Exception: #{e}"}
181
- @shared_info[:mutex].synchronize do
182
- @shared_info[:read_exception] = e unless e.is_a?(EOFError)
183
- @shared_info[:cond_var].signal
184
- end
185
- break
186
- end # begin
187
- end # loop
188
- Log.log.debug{"#{LOG_WS_THREAD}stopping (exc=#{@shared_info[:read_exception]},cls=#{@shared_info[:read_exception].class})"}
189
- end
190
208
  # notify progress bar
191
- notify_begin(session_id, total_bytes_to_transfer)
209
+ notify_progress(type: :session_size, session_id: session_id, info: total_bytes_to_transfer)
192
210
  # first step send transfer spec
193
- Log.dump(:ws_spec, transfer_spec)
211
+ Log.log.debug{Log.dump(:ws_spec, transfer_spec)}
194
212
  ws_snd_json(MSG_SEND_TRANSFER_SPEC, transfer_spec)
195
- wait_for_sent_msg_ack_or_exception
196
213
  # current file index
197
214
  file_index = 0
198
215
  # aggregate size sent
199
- sent_bytes = 0
200
- # last progress event
201
- last_progress_time = nil
202
-
216
+ session_sent_bytes = 0
217
+ # process each file
203
218
  transfer_spec['paths'].each do |item|
204
- # TODO: get mime type?
205
- file_mime_type = ''
206
- file_size = item['file_size']
207
- file_name = File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
208
- # compute total number of slices
209
- slice_total = ((file_size - 1) / @options[:upload_chunk_size]) + 1
210
- File.open(source_paths[file_index]) do |file|
211
- # current slice index
212
- slice_index = 0
219
+ slice_info = {
220
+ name: nil,
221
+ # TODO: get mime type?
222
+ type: 'application/octet-stream',
223
+ size: item['file_size'],
224
+ slice: 0, # current slice index
225
+ # index of last slice (i.e number of slices - 1)
226
+ last_slice: (item['file_size'] - 1) / @options[:upload_chunk_size],
227
+ fileIndex: file_index
228
+ }
229
+ file = files_to_read[file_index]
230
+ if file.is_a?(FauxFile)
231
+ slice_info[:name] = file.path
232
+ else
233
+ file = File.open(file)
234
+ slice_info[:name] = File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
235
+ end
236
+ begin
213
237
  until file.eof?
214
- file_bin_data = file.read(@options[:upload_chunk_size])
215
- slice_data = {
216
- name: file_name,
217
- type: file_mime_type,
218
- size: file_size,
219
- slice: slice_index,
220
- total_slices: slice_total,
221
- fileIndex: file_index
222
- }
223
- # Log.dump(:slice_data,slice_data) #if slice_index.eql?(0)
238
+ slice_bin_data = file.read(@options[:upload_chunk_size])
224
239
  # interrupt main thread if read thread failed
225
240
  raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
226
241
  begin
227
242
  if @options[:api_version].eql?(API_V1)
228
- slice_data[:data] = Base64.strict_encode64(file_bin_data)
229
- ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_data)
243
+ slice_info[:data] = Base64.strict_encode64(slice_bin_data)
244
+ ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_info)
230
245
  else
231
- ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_data) if slice_index.eql?(0)
232
- ws_send(file_bin_data, type: :binary)
233
- Log.log.debug{"#{LOG_WS_MAIN}sent bin buffer: #{file_index} / #{slice_index}"}
234
- ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_data) if slice_index.eql?(slice_total - 1)
246
+ # send once, before data, at beginning
247
+ ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_info) if slice_info[:slice].eql?(0)
248
+ ws_send(ws_type: :binary, data: slice_bin_data)
249
+ Log.log.debug{"#{LOG_WS_SEND}buffer: file: #{file_index}, slice: #{slice_info[:slice]}/#{slice_info[:last_slice]}"}
250
+ # send once, after data, at end
251
+ ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_info) if slice_info[:slice].eql?(slice_info[:last_slice])
235
252
  end
236
- wait_for_sent_msg_ack_or_exception
237
253
  rescue Errno::EPIPE => e
238
254
  raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
239
255
  raise e
@@ -241,26 +257,25 @@ module Aspera
241
257
  Log.log.warn{'A timeout condition using HTTPGW may signal a permission problem on destination. Check ascp logs on httpgw.'}
242
258
  raise e
243
259
  end
244
- sent_bytes += file_bin_data.length
245
- current_time = Time.now
246
- if last_progress_time.nil? || ((current_time - last_progress_time) > @options[:upload_bar_refresh_sec])
247
- notify_progress(session_id, sent_bytes)
248
- last_progress_time = current_time
249
- end
250
- slice_index += 1
260
+ session_sent_bytes += slice_bin_data.length
261
+ notify_progress(type: :transfer, session_id: session_id, info: session_sent_bytes)
262
+ slice_info[:slice] += 1
251
263
  end
264
+ ensure
265
+ file.close
252
266
  end
253
267
  file_index += 1
254
- end
255
-
256
- Log.log.debug('Finished upload, waiting for end of read thread.')
257
- ws_read_thread.join
258
- Log.log.debug{"Read thread joined, result: #{@shared_info[:count][:received_data]} / #{@shared_info[:count][:sent_other]}"}
259
- ws_send(nil, type: :close) unless @ws_io.nil?
268
+ end # loop on files
269
+ # throttling may have skipped last one
270
+ notify_progress(type: :transfer, session_id: session_id, info: session_sent_bytes)
271
+ notify_progress(type: :end, session_id: session_id)
272
+ ws_send(ws_type: :close, data: nil)
273
+ Log.log.debug("Finished upload, waiting for end of #{THR_RECV} thread.")
274
+ @ws_read_thread.join
275
+ Log.log.debug{'Read thread joined'}
276
+ # session no more used
260
277
  @ws_io = nil
261
- http_socket&.finish
262
- notify_progress(session_id, sent_bytes)
263
- notify_end(session_id)
278
+ http_session&.finish
264
279
  end
265
280
 
266
281
  def download(transfer_spec)
@@ -297,9 +312,9 @@ module Aspera
297
312
  # HTTP download only supports file list
298
313
  def start_transfer(transfer_spec, token_regenerator: nil)
299
314
  raise 'GW URL must be set' if @gw_api.nil?
300
- raise 'paths: must be Array' unless transfer_spec['paths'].is_a?(Array)
301
- raise 'only token based transfer is supported in GW' unless transfer_spec['token'].is_a?(String)
302
- Log.dump(:user_spec, transfer_spec)
315
+ assert_type(transfer_spec['paths'], Array){'paths'}
316
+ assert_type(transfer_spec['token'], String){'only token based transfer is supported in GW'}
317
+ Log.log.debug{Log.dump(:user_spec, transfer_spec)}
303
318
  transfer_spec['authentication'] ||= 'token'
304
319
  case transfer_spec['direction']
305
320
  when Fasp::TransferSpec::DIRECTION_SEND
@@ -314,43 +329,33 @@ module Aspera
314
329
  # wait for completion of all jobs started
315
330
  # @return list of :success or error message
316
331
  def wait_for_transfers_completion
332
+ # well ... transfer was done in "start"
317
333
  return [:success]
318
334
  end
319
335
 
320
- # terminates monitor thread
321
- def shutdown; end
322
-
336
+ # TODO: is that useful?
323
337
  def url=(api_url); end
324
338
 
325
339
  private
326
340
 
327
341
  def initialize(opts)
328
- Log.dump(:in_options, opts)
329
- # set default options and override if specified
330
- @options = DEFAULT_OPTIONS.dup
331
- raise "httpgw agent parameters (transfer_info): expecting Hash, but have #{opts.class}" unless opts.is_a?(Hash)
332
- opts.symbolize_keys.each do |k, v|
333
- raise "httpgw agent parameter: Unknown: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.key?(k)
334
- @options[k] = v
335
- end
336
- if @options[:url].nil?
337
- available = DEFAULT_OPTIONS.map { |k, v| "#{k}(#{v})"}.join(', ')
338
- raise "Missing mandatory parameter for HTTP GW in transfer_info: url. Allowed parameters: #{available}."
339
- end
340
- # remove /v1 from end
342
+ super(opts)
343
+ @options = AgentBase.options(default: DEFAULT_OPTIONS, options: opts)
344
+ # remove /v1 from end of user-provided GW url: we need the base url only
341
345
  @options[:url].gsub(%r{/v1/*$}, '')
342
- super()
343
346
  @gw_api = Rest.new({base_url: @options[:url]})
344
347
  @api_info = @gw_api.read('v1/info')[:data]
345
- Log.dump(:api_info, @api_info)
346
- if @options[:api_version].nil?
347
- # web socket endpoint: by default use v2 (newer gateways), without base64 encoding
348
- @options[:api_version] = API_V2
349
- # is the latest supported? else revert to old api
350
- @options[:api_version] = API_V1 unless @api_info['endpoints'].any?{|i|i.include?(@options[:api_version])}
348
+ Log.log.debug{Log.dump(:api_info, @api_info)}
349
+ # web socket endpoint: by default use v2 (newer gateways), without base64 encoding
350
+ # is the latest supported? else revert to old api
351
+ if !@options[:api_version].eql?(API_V1)
352
+ if !@api_info['endpoints'].any?{|i|i.include?(@options[:api_version])}
353
+ Log.log.warn{"API version #{@options[:api_version]} not supported, reverting to #{API_V1}"}
354
+ @options[:api_version] = API_V1
355
+ end
351
356
  end
352
357
  @options.freeze
353
- Log.dump(:final_options, @options)
358
+ Log.log.debug{Log.dump(:agent_options, @options)}
354
359
  end
355
360
  end # AgentHttpgw
356
361
  end