aspera-cli 4.17.0 → 4.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +2 -4
  3. data/CHANGELOG.md +23 -0
  4. data/CONTRIBUTING.md +15 -1
  5. data/README.md +620 -378
  6. data/bin/ascli +5 -0
  7. data/bin/asession +2 -2
  8. data/lib/aspera/agent/alpha.rb +6 -4
  9. data/lib/aspera/agent/base.rb +9 -6
  10. data/lib/aspera/agent/connect.rb +4 -4
  11. data/lib/aspera/agent/direct.rb +56 -37
  12. data/lib/aspera/agent/httpgw.rb +23 -324
  13. data/lib/aspera/agent/node.rb +19 -20
  14. data/lib/aspera/agent/trsdk.rb +19 -20
  15. data/lib/aspera/api/aoc.rb +17 -14
  16. data/lib/aspera/api/cos_node.rb +4 -4
  17. data/lib/aspera/api/httpgw.rb +339 -0
  18. data/lib/aspera/api/node.rb +34 -21
  19. data/lib/aspera/ascmd.rb +4 -3
  20. data/lib/aspera/ascp/installation.rb +15 -7
  21. data/lib/aspera/ascp/management.rb +2 -2
  22. data/lib/aspera/cli/basic_auth_plugin.rb +5 -9
  23. data/lib/aspera/cli/extended_value.rb +12 -6
  24. data/lib/aspera/cli/formatter.rb +155 -65
  25. data/lib/aspera/cli/hints.rb +18 -0
  26. data/lib/aspera/cli/main.rb +22 -29
  27. data/lib/aspera/cli/manager.rb +53 -36
  28. data/lib/aspera/cli/plugin.rb +26 -17
  29. data/lib/aspera/cli/plugin_factory.rb +31 -20
  30. data/lib/aspera/cli/plugins/alee.rb +14 -2
  31. data/lib/aspera/cli/plugins/aoc.rb +141 -131
  32. data/lib/aspera/cli/plugins/ats.rb +1 -1
  33. data/lib/aspera/cli/plugins/config.rb +52 -46
  34. data/lib/aspera/cli/plugins/console.rb +8 -5
  35. data/lib/aspera/cli/plugins/faspex.rb +27 -19
  36. data/lib/aspera/cli/plugins/faspex5.rb +222 -149
  37. data/lib/aspera/cli/plugins/faspio.rb +85 -0
  38. data/lib/aspera/cli/plugins/httpgw.rb +55 -0
  39. data/lib/aspera/cli/plugins/node.rb +86 -29
  40. data/lib/aspera/cli/plugins/orchestrator.rb +31 -29
  41. data/lib/aspera/cli/plugins/preview.rb +6 -2
  42. data/lib/aspera/cli/plugins/server.rb +5 -5
  43. data/lib/aspera/cli/plugins/shares.rb +16 -14
  44. data/lib/aspera/cli/sync_actions.rb +6 -6
  45. data/lib/aspera/cli/transfer_agent.rb +5 -4
  46. data/lib/aspera/cli/version.rb +1 -1
  47. data/lib/aspera/environment.rb +7 -6
  48. data/lib/aspera/faspex_gw.rb +5 -4
  49. data/lib/aspera/faspex_postproc.rb +2 -2
  50. data/lib/aspera/log.rb +6 -3
  51. data/lib/aspera/node_simulator.rb +2 -2
  52. data/lib/aspera/oauth/base.rb +31 -19
  53. data/lib/aspera/oauth/factory.rb +12 -13
  54. data/lib/aspera/oauth/generic.rb +1 -0
  55. data/lib/aspera/oauth/jwt.rb +18 -15
  56. data/lib/aspera/oauth/url_json.rb +8 -6
  57. data/lib/aspera/open_application.rb +5 -7
  58. data/lib/aspera/persistency_folder.rb +2 -2
  59. data/lib/aspera/preview/generator.rb +3 -3
  60. data/lib/aspera/preview/options.rb +3 -3
  61. data/lib/aspera/preview/terminal.rb +4 -4
  62. data/lib/aspera/preview/utils.rb +3 -3
  63. data/lib/aspera/proxy_auto_config.rb +5 -1
  64. data/lib/aspera/rest.rb +60 -74
  65. data/lib/aspera/rest_call_error.rb +1 -1
  66. data/lib/aspera/rest_error_analyzer.rb +2 -2
  67. data/lib/aspera/rest_errors_aspera.rb +1 -1
  68. data/lib/aspera/resumer.rb +1 -1
  69. data/lib/aspera/secret_hider.rb +2 -4
  70. data/lib/aspera/ssh.rb +1 -1
  71. data/lib/aspera/transfer/parameters.rb +39 -36
  72. data/lib/aspera/transfer/spec.rb +2 -0
  73. data/lib/aspera/transfer/sync.rb +2 -1
  74. data/lib/aspera/transfer/uri.rb +1 -1
  75. data/lib/aspera/uri_reader.rb +5 -4
  76. data/lib/aspera/web_auth.rb +1 -1
  77. data/lib/aspera/web_server_simple.rb +4 -3
  78. data.tar.gz.sig +0 -0
  79. metadata +5 -3
  80. metadata.gz.sig +0 -0
  81. data/lib/aspera/cli/plugins/bss.rb +0 -71
@@ -2,312 +2,13 @@
2
2
 
3
3
  require 'aspera/agent/base'
4
4
  require 'aspera/transfer/spec'
5
- require 'aspera/transfer/faux_file'
5
+ require 'aspera/api/httpgw'
6
6
  require 'aspera/log'
7
7
  require 'aspera/assert'
8
- require 'aspera/rest'
9
- require 'securerandom'
10
- require 'websocket'
11
- require 'base64'
12
- require 'json'
13
8
 
14
9
  module Aspera
15
10
  module Agent
16
- # Start a transfer using Aspera HTTP Gateway, using web socket secure for uploads
17
- # ref: https://api.ibm.com/explorer/catalog/aspera/product/ibm-aspera/api/http-gateway-api/doc/guides-toc
18
- # https://developer.ibm.com/apis/catalog?search=%22aspera%20http%22
19
- # HTTP GW Upload protocol:
20
- # # type Contents Ack Counter
21
- # v1
22
- # 0 JSON.transfer_spec Transfer Spec "end upload" sent_general
23
- # 1.. JSON.slice_upload File base64 chunks "end upload" sent_general
24
- # v2
25
- # 0 JSON.transfer_spec Transfer Spec "end upload" sent_general
26
- # 1 JSON.slice_upload File start "end_slice_upload" sent_v2_delimiter
27
- # 2.. Binary File binary chunks "end upload" sent_general
28
- # last JSON.slice_upload File end "end_slice_upload" sent_v2_delimiter
29
11
  class Httpgw < Base
30
- MSG_SEND_TRANSFER_SPEC = 'transfer_spec'
31
- MSG_SEND_SLICE_UPLOAD = 'slice_upload'
32
- MSG_RECV_DATA_RECEIVED_SIGNAL = 'end upload'
33
- MSG_RECV_SLICE_UPLOAD_SIGNAL = 'end_slice_upload'
34
- # upload API versions
35
- API_V1 = 'v1'
36
- API_V2 = 'v2'
37
- # options available in CLI (transfer_info)
38
- DEFAULT_OPTIONS = {
39
- url: :required,
40
- upload_chunk_size: 64_000,
41
- api_version: API_V2,
42
- synchronous: false
43
- }.freeze
44
- DEFAULT_BASE_PATH = '/aspera/http-gwy'
45
- THR_RECV = 'recv'
46
- LOG_WS_SEND = 'ws: send: '.red
47
- LOG_WS_RECV = "ws: #{THR_RECV}: ".green
48
- private_constant :DEFAULT_OPTIONS, :MSG_RECV_DATA_RECEIVED_SIGNAL, :MSG_RECV_SLICE_UPLOAD_SIGNAL, :API_V1, :API_V2
49
-
50
- # send message on http gw web socket
51
- def ws_snd_json(msg_type, payload)
52
- if msg_type.eql?(MSG_SEND_SLICE_UPLOAD) && @options[:api_version].eql?(API_V2)
53
- @shared_info[:count][:sent_v2_delimiter] += 1
54
- else
55
- @shared_info[:count][:sent_general] += 1
56
- end
57
- Log.log.debug do
58
- log_data = payload.dup
59
- log_data[:data] = "[data #{log_data[:data].length} bytes]" if log_data.key?(:data)
60
- "#{LOG_WS_SEND}json: #{msg_type}: #{JSON.generate(log_data)}"
61
- end
62
- ws_send(ws_type: :text, data: JSON.generate({msg_type => payload}))
63
- end
64
-
65
- # send data on http gw web socket
66
- def ws_send(ws_type:, data:)
67
- Log.log.debug{"#{LOG_WS_SEND}sending: #{ws_type} (#{data&.length || 0} bytes)"}
68
- @shared_info[:count][:sent_general] += 1 if ws_type.eql?(:binary)
69
- frame_generator = ::WebSocket::Frame::Outgoing::Client.new(data: data, type: ws_type, version: @ws_handshake.version)
70
- @ws_io.write(frame_generator.to_s)
71
- if @options[:synchronous]
72
- @shared_info[:mutex].synchronize do
73
- # if read thread exited, there will be no more updates
74
- # we allow for 1 of difference else it stays blocked
75
- while @ws_read_thread.alive? &&
76
- @shared_info[:read_exception].nil? &&
77
- (((@shared_info[:count][:sent_general] - @shared_info[:count][:received_general]) > 1) ||
78
- ((@shared_info[:count][:received_v2_delimiter] - @shared_info[:count][:sent_v2_delimiter]) > 1))
79
- if !@shared_info[:cond_var].wait(@shared_info[:mutex], 2.0)
80
- Log.log.debug{"#{LOG_WS_SEND}#{'timeout'.blue}: #{@shared_info[:count]}"}
81
- end
82
- end
83
- end
84
- end
85
- raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
86
- Log.log.debug{"#{LOG_WS_SEND}counts: #{@shared_info[:count]}"}
87
- end
88
-
89
- # message processing for read thread
90
- def process_received_message(message)
91
- Log.log.debug{"#{LOG_WS_RECV}message: [#{message}] (#{message.class})"}
92
- if message.eql?(MSG_RECV_DATA_RECEIVED_SIGNAL)
93
- @shared_info[:mutex].synchronize do
94
- @shared_info[:count][:received_general] += 1
95
- @shared_info[:cond_var].signal
96
- end
97
- elsif message.eql?(MSG_RECV_SLICE_UPLOAD_SIGNAL)
98
- @shared_info[:mutex].synchronize do
99
- @shared_info[:count][:received_v2_delimiter] += 1
100
- @shared_info[:cond_var].signal
101
- end
102
- else
103
- message.chomp!
104
- error_message =
105
- if message.start_with?('"') && message.end_with?('"')
106
- # remove double quotes : 1..-2
107
- JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message']
108
- elsif message.start_with?('{') && message.end_with?('}')
109
- JSON.parse(message)['message']
110
- else
111
- "unknown message from gateway: [#{message}]"
112
- end
113
- raise error_message
114
- end
115
- end
116
-
117
- # main function of read thread
118
- def process_read_thread
119
- Log.log.debug{"#{LOG_WS_RECV}read thread started"}
120
- frame_parser = ::WebSocket::Frame::Incoming::Client.new(version: @ws_handshake.version)
121
- until @ws_io.eof?
122
- begin # rubocop:disable Style/RedundantBegin
123
- # ready byte by byte until frame is ready
124
- # blocking read
125
- byte = @ws_io.read(1)
126
- Log.log.trace1{"#{LOG_WS_RECV}read: #{byte} (#{byte.class}) eof=#{@ws_io.eof?}"}
127
- frame_parser << byte
128
- frame_ok = frame_parser.next
129
- next if frame_ok.nil?
130
- process_received_message(frame_ok.data.to_s)
131
- Log.log.debug{"#{LOG_WS_RECV}counts: #{@shared_info[:count]}"}
132
- rescue => e
133
- Log.log.debug{"#{LOG_WS_RECV}Exception: #{e}"}
134
- @shared_info[:mutex].synchronize do
135
- @shared_info[:read_exception] = e
136
- @shared_info[:cond_var].signal
137
- end
138
- break
139
- end # begin/rescue
140
- end # loop
141
- Log.log.debug do
142
- "#{LOG_WS_RECV}exception: #{@shared_info[:read_exception]},cls=#{@shared_info[:read_exception].class})"
143
- end unless @shared_info[:read_exception].nil?
144
- Log.log.debug{"#{LOG_WS_RECV}read thread stopped (ws eof=#{@ws_io.eof?})"}
145
- end
146
-
147
- def upload(transfer_spec)
148
- # identify this session uniquely
149
- session_id = SecureRandom.uuid
150
- notify_progress(session_id: nil, type: :pre_start, info: 'starting')
151
- # total size of all files
152
- total_bytes_to_transfer = 0
153
- # we need to keep track of actual file path because transfer spec is modified to be sent in web socket
154
- files_to_read = []
155
- # get source root or nil
156
- source_root = transfer_spec.key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] : nil
157
- # source root is ignored by GW, used only here
158
- transfer_spec.delete('source_root')
159
- # compute total size of files to upload (for progress)
160
- # modify transfer spec to be suitable for GW
161
- transfer_spec['paths'].each do |item|
162
- # save actual file location to be able read contents later
163
- file_to_add = Transfer::FauxFile.open(item['source'])
164
- if file_to_add
165
- item['source'] = file_to_add.path
166
- item['file_size'] = file_to_add.size
167
- else
168
- file_to_add = item['source']
169
- # add source root if needed
170
- file_to_add = File.join(source_root, file_to_add) unless source_root.nil?
171
- # GW expects a simple file name in 'source' but if user wants to change the name, we take it
172
- item['source'] = File.basename(item['destination'].nil? ? item['source'] : item['destination'])
173
- item['file_size'] = File.size(file_to_add)
174
- end
175
- # save so that we can actually read the file later
176
- files_to_read.push(file_to_add)
177
- total_bytes_to_transfer += item['file_size']
178
- end
179
- # TODO: check that this is available in endpoints: @api_info['endpoints']
180
- upload_url = File.join(@gw_base_url, @options[:api_version], 'upload')
181
- notify_progress(session_id: nil, type: :pre_start, info: 'connecting wss')
182
- # open web socket to end point (equivalent to Net::HTTP.start)
183
- http_session = Rest.start_http_session(upload_url)
184
- # get the underlying socket i/o
185
- @ws_io = Rest.io_http_session(http_session)
186
- @ws_handshake = ::WebSocket::Handshake::Client.new(url: upload_url, headers: {})
187
- @ws_io.write(@ws_handshake.to_s)
188
- sleep(0.1)
189
- @ws_handshake << @ws_io.readuntil("\r\n\r\n")
190
- Aspera.assert(@ws_handshake.finished?){'Error in websocket handshake'}
191
- Log.log.debug{"#{LOG_WS_SEND}handshake success"}
192
- # start read thread after handshake
193
- @ws_read_thread = Thread.new {process_read_thread}
194
- notify_progress(session_id: session_id, type: :session_start)
195
- notify_progress(session_id: session_id, type: :session_size, info: total_bytes_to_transfer)
196
- sleep(1)
197
- # data shared between main thread and read thread
198
- @shared_info = {
199
- read_exception: nil, # error message if any in callback
200
- count: {
201
- sent_general: 0,
202
- received_general: 0,
203
- sent_v2_delimiter: 0,
204
- received_v2_delimiter: 0
205
- },
206
- mutex: Mutex.new,
207
- cond_var: ConditionVariable.new
208
- }
209
- # notify progress bar
210
- notify_progress(type: :session_size, session_id: session_id, info: total_bytes_to_transfer)
211
- # first step send transfer spec
212
- Log.log.debug{Log.dump(:ws_spec, transfer_spec)}
213
- ws_snd_json(MSG_SEND_TRANSFER_SPEC, transfer_spec)
214
- # current file index
215
- file_index = 0
216
- # aggregate size sent
217
- session_sent_bytes = 0
218
- # process each file
219
- transfer_spec['paths'].each do |item|
220
- slice_info = {
221
- name: nil,
222
- # TODO: get mime type?
223
- type: 'application/octet-stream',
224
- size: item['file_size'],
225
- slice: 0, # current slice index
226
- # index of last slice (i.e number of slices - 1)
227
- last_slice: (item['file_size'] - 1) / @options[:upload_chunk_size],
228
- fileIndex: file_index
229
- }
230
- file = files_to_read[file_index]
231
- if file.is_a?(Transfer::FauxFile)
232
- slice_info[:name] = file.path
233
- else
234
- file = File.open(file)
235
- slice_info[:name] = File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
236
- end
237
- begin
238
- until file.eof?
239
- slice_bin_data = file.read(@options[:upload_chunk_size])
240
- # interrupt main thread if read thread failed
241
- raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
242
- begin
243
- if @options[:api_version].eql?(API_V1)
244
- slice_info[:data] = Base64.strict_encode64(slice_bin_data)
245
- ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_info)
246
- else
247
- # send once, before data, at beginning
248
- ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_info) if slice_info[:slice].eql?(0)
249
- ws_send(ws_type: :binary, data: slice_bin_data)
250
- Log.log.debug{"#{LOG_WS_SEND}buffer: file: #{file_index}, slice: #{slice_info[:slice]}/#{slice_info[:last_slice]}"}
251
- # send once, after data, at end
252
- ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_info) if slice_info[:slice].eql?(slice_info[:last_slice])
253
- end
254
- rescue Errno::EPIPE => e
255
- raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
256
- raise e
257
- rescue Net::ReadTimeout => e
258
- Log.log.warn{'A timeout condition using HTTPGW may signal a permission problem on destination. Check ascp logs on httpgw.'}
259
- raise e
260
- end
261
- session_sent_bytes += slice_bin_data.length
262
- notify_progress(type: :transfer, session_id: session_id, info: session_sent_bytes)
263
- slice_info[:slice] += 1
264
- end
265
- ensure
266
- file.close
267
- end
268
- file_index += 1
269
- end # loop on files
270
- # throttling may have skipped last one
271
- notify_progress(type: :transfer, session_id: session_id, info: session_sent_bytes)
272
- notify_progress(type: :end, session_id: session_id)
273
- ws_send(ws_type: :close, data: nil)
274
- Log.log.debug("Finished upload, waiting for end of #{THR_RECV} thread.")
275
- @ws_read_thread.join
276
- Log.log.debug{'Read thread joined'}
277
- # session no more used
278
- @ws_io = nil
279
- http_session&.finish
280
- end
281
-
282
- def download(transfer_spec)
283
- transfer_spec['zip_required'] ||= false
284
- transfer_spec['source_root'] ||= '/'
285
- # is normally provided by application, like package name
286
- if !transfer_spec.key?('download_name')
287
- # by default it is the name of first file
288
- download_name = File.basename(transfer_spec['paths'].first['source'])
289
- # we remove extension
290
- download_name = download_name.gsub(/\.@gw_api.*$/, '')
291
- # ands add indication of number of files if there is more than one
292
- if transfer_spec['paths'].length > 1
293
- download_name += " #{transfer_spec['paths'].length} Files"
294
- end
295
- transfer_spec['download_name'] = download_name
296
- end
297
- creation = @gw_api.create('v1/download', {'transfer_spec' => transfer_spec})[:data]
298
- transfer_uuid = creation['url'].split('/').last
299
- file_name =
300
- if transfer_spec['zip_required'] || transfer_spec['paths'].length > 1
301
- # it is a zip file if zip is required or there is more than 1 file
302
- transfer_spec['download_name'] + '.zip'
303
- else
304
- # it is a plain file if we don't require zip and there is only one file
305
- File.basename(transfer_spec['paths'].first['source'])
306
- end
307
- file_path = File.join(transfer_spec['destination_root'], file_name)
308
- @gw_api.call(operation: 'GET', subpath: "v1/download/#{transfer_uuid}", save_to_file: file_path)
309
- end
310
-
311
12
  # start FASP transfer based on transfer spec (hash table)
312
13
  # note that it is asynchronous
313
14
  # HTTP download only supports file list
@@ -319,13 +20,13 @@ module Aspera
319
20
  transfer_spec['authentication'] ||= 'token'
320
21
  case transfer_spec['direction']
321
22
  when Transfer::Spec::DIRECTION_SEND
322
- upload(transfer_spec)
23
+ @gw_api.upload(transfer_spec)
323
24
  when Transfer::Spec::DIRECTION_RECEIVE
324
- download(transfer_spec)
25
+ @gw_api.download(transfer_spec)
325
26
  else
326
27
  raise "unexpected direction: [#{transfer_spec['direction']}]"
327
28
  end
328
- end # start_transfer
29
+ end
329
30
 
330
31
  # wait for completion of all jobs started
331
32
  # @return list of :success or error message
@@ -335,29 +36,27 @@ module Aspera
335
36
  end
336
37
 
337
38
  # TODO: is that useful?
338
- def url=(api_url); end
39
+ # def url=(api_url); end
339
40
 
340
41
  private
341
42
 
342
- def initialize(opts)
343
- super(opts)
344
- @options = Base.options(default: DEFAULT_OPTIONS, options: opts)
345
- # remove /v1 from end of user-provided GW url: we need the base url only
346
- @gw_base_url = @options[:url].gsub(%r{/v1/*$}, '')
347
- @gw_api = Rest.new(base_url: @gw_base_url)
348
- @api_info = @gw_api.read('v1/info')[:data]
349
- Log.log.debug{Log.dump(:api_info, @api_info)}
350
- # web socket endpoint: by default use v2 (newer gateways), without base64 encoding
351
- # is the latest supported? else revert to old api
352
- if !@options[:api_version].eql?(API_V1)
353
- if !@api_info['endpoints'].any?{|i|i.include?(@options[:api_version])}
354
- Log.log.warn{"API version #{@options[:api_version]} not supported, reverting to #{API_V1}"}
355
- @options[:api_version] = API_V1
356
- end
357
- end
358
- @options.freeze
359
- Log.log.debug{Log.dump(:agent_options, @options)}
360
- end
361
- end # AgentHttpgw
43
+ def initialize(
44
+ url:,
45
+ api_version: Api::Httpgw::API_V2,
46
+ upload_chunk_size: 64_000,
47
+ synchronous: false,
48
+ **base_options
49
+ )
50
+ super(**base_options)
51
+ @gw_api = Api::Httpgw.new(
52
+ # remove /v1 from end of user-provided GW url: we need the base url only
53
+ url: url.gsub(%r{/#{Api::Httpgw::API_V1}/*$}o, ''),
54
+ api_version: api_version,
55
+ upload_chunk_size: upload_chunk_size,
56
+ synchronous: synchronous,
57
+ notify_cb: ->(**event) { notify_progress(**event) }
58
+ )
59
+ end
60
+ end
362
61
  end
363
62
  end
@@ -13,36 +13,35 @@ module Aspera
13
13
  # this singleton class is used by the CLI to provide a common interface to start a transfer
14
14
  # before using it, the use must set the `node_api` member.
15
15
  class Node < Base
16
- DEFAULT_OPTIONS = {
17
- url: :required,
18
- username: :required,
19
- password: :required,
20
- root_id: nil
21
- }.freeze
22
- # option include: root_id if the node is an access key
23
- # attr_writer :options
24
-
25
- def initialize(opts)
26
- Aspera.assert_type(opts, Hash){'node agent options'}
27
- super(opts)
28
- options = Base.options(default: DEFAULT_OPTIONS, options: opts)
16
+ # @param url [String] the base url of the node api
17
+ # @param username [String] the username to use for the node api
18
+ # @param password [String] the password to use for the node api
19
+ # @param root_id [String] root file id if the node is an access key
20
+ # @param base_options [Hash] options for base class
21
+ def initialize(
22
+ url:,
23
+ username:,
24
+ password:,
25
+ root_id: nil,
26
+ **base_options
27
+ )
28
+ super(**base_options)
29
29
  # root id is required for access key
30
- @root_id = options[:root_id]
31
- rest_params = { base_url: options[:url]}
32
- if OAuth::Factory.bearer?(options[:password])
30
+ @root_id = root_id
31
+ rest_params = { base_url: url}
32
+ if OAuth::Factory.bearer?(password)
33
33
  Aspera.assert(!@root_id.nil?){'root_id not allowed for access key'}
34
- rest_params[:headers] = Api::Node.bearer_headers(options[:password], access_key: options[:username])
34
+ rest_params[:headers] = Api::Node.bearer_headers(password, access_key: username)
35
35
  else
36
36
  rest_params[:auth] = {
37
37
  type: :basic,
38
- username: options[:username],
39
- password: options[:password]
38
+ username: username,
39
+ password: password
40
40
  }
41
41
  end
42
42
  @node_api = Rest.new(**rest_params)
43
43
  # TODO: currently only supports one transfer. This is bad shortcut. but ok for CLI.
44
44
  @transfer_id = nil
45
- # Log.log.debug{Log.dump(:agent_options, @options)}
46
45
  end
47
46
 
48
47
  # used internally to ensure node api is set before using.
@@ -17,13 +17,6 @@ module Aspera
17
17
  PORT_SEP = ':'
18
18
  # port zero means select a random available high port
19
19
  AUTO_LOCAL_TCP_PORT = "#{PORT_SEP}0"
20
- DEFAULT_OPTIONS = {
21
- url: AUTO_LOCAL_TCP_PORT,
22
- external: false, # expect that an external daemon is already running
23
- keep: false # do not shutdown daemon on exit
24
- }.freeze
25
- private_constant :DEFAULT_OPTIONS
26
-
27
20
  class << self
28
21
  # Well, the port number is only in log file
29
22
  def daemon_port_from_log(log_file)
@@ -44,19 +37,25 @@ module Aspera
44
37
  end
45
38
  end
46
39
 
47
- # options come from transfer_info
48
- def initialize(user_opts={})
49
- super(user_opts)
50
- @options = Base.options(default: DEFAULT_OPTIONS, options: user_opts)
51
- is_local_auto_port = @options[:url].eql?(AUTO_LOCAL_TCP_PORT)
52
- raise 'Cannot use options `keep` or `external` with port zero' if is_local_auto_port && (@options[:keep] || @options[:external])
53
- Log.log.debug{Log.dump(:agent_options, @options)}
40
+ # @param url [String] URL of the transfer manager daemon
41
+ # @param external [Boolean] if true, expect that an external daemon is already running
42
+ # @param keep [Boolean] if true, do not shutdown daemon on exit
43
+ # @param base_options [Hash] base options
44
+ def initialize(
45
+ url: AUTO_LOCAL_TCP_PORT,
46
+ external: false,
47
+ keep: false,
48
+ **base_options
49
+ )
50
+ super(**base_options)
51
+ is_local_auto_port = @url.eql?(AUTO_LOCAL_TCP_PORT)
52
+ raise 'Cannot use options `keep` or `external` with port zero' if is_local_auto_port && (@keep || @external)
54
53
  # load SDK stub class on demand, as it's an optional gem
55
54
  $LOAD_PATH.unshift(Ascp::Installation.instance.sdk_ruby_folder)
56
55
  require 'transfer_services_pb'
57
56
  # keep PID for optional shutdown
58
57
  @daemon_pid = nil
59
- daemon_endpoint = @options[:url]
58
+ daemon_endpoint = @url
60
59
  Log.log.debug{Log.dump(:daemon_endpoint, daemon_endpoint)}
61
60
  # retry loop
62
61
  begin
@@ -67,14 +66,14 @@ module Aspera
67
66
  # Initiate actual connection
68
67
  get_info_response = @transfer_client.get_info(Transfersdk::InstanceInfoRequest.new)
69
68
  Log.log.debug{"Daemon info: #{get_info_response}"}
70
- Log.log.warn{'Attached to existing daemon'} unless @daemon_pid || @options[:external] || @options[:keep]
69
+ Log.log.warn{'Attached to existing daemon'} unless @daemon_pid || @external || @keep
71
70
  at_exit{shutdown}
72
71
  rescue GRPC::Unavailable => e
73
72
  # if transferd is external: do not start it, or other error
74
- raise if @options[:external] || !e.message.include?('failed to connect')
73
+ raise if @external || !e.message.include?('failed to connect')
75
74
  # we already tried to start a daemon, but it failed
76
75
  Aspera.assert(@daemon_pid.nil?){"Daemon started with PID #{@daemon_pid}, but connection failed to #{daemon_endpoint}}"}
77
- Log.log.warn('no daemon present, starting daemon...') if @options[:external]
76
+ Log.log.warn('no daemon present, starting daemon...') if @external
78
77
  # location of daemon binary
79
78
  sdk_folder = File.realpath(File.join(Ascp::Installation.instance.sdk_ruby_folder, '..'))
80
79
  # transferd only supports local ip and port
@@ -111,7 +110,7 @@ module Aspera
111
110
  nil
112
111
  end
113
112
  Log.log.debug{"Daemon started with pid #{@daemon_pid}"}
114
- Process.detach(@daemon_pid) if @options[:keep]
113
+ Process.detach(@daemon_pid) if @keep
115
114
  at_exit {shutdown}
116
115
  # update port for next connection attempt (if auto high port was requested)
117
116
  daemon_endpoint = "#{LOCAL_SOCKET_ADDR}#{PORT_SEP}#{self.class.daemon_port_from_log(log_stdout)}" if is_local_auto_port
@@ -171,7 +170,7 @@ module Aspera
171
170
  end
172
171
 
173
172
  def shutdown
174
- stop_daemon unless @options[:keep]
173
+ stop_daemon unless @keep
175
174
  end
176
175
 
177
176
  def stop_daemon
@@ -12,15 +12,25 @@ require 'cgi'
12
12
 
13
13
  module Aspera
14
14
  module Api
15
+ SAAS_DOMAIN_PROD = 'ibmaspera.com'
16
+ class Alee < Aspera::Rest
17
+ def initialize(entitlement_id, customer_id, api_domain: SAAS_DOMAIN_PROD, version: 'v1')
18
+ super(
19
+ base_url: "https://api.#{api_domain}/metering/#{version}",
20
+ headers: {'X-Aspera-Entitlement-Authorization' => Rest.basic_token(entitlement_id, customer_id)}
21
+ )
22
+ end
23
+ end
24
+
15
25
  class AoC < Aspera::Rest
16
26
  PRODUCT_NAME = 'Aspera on Cloud'
17
27
  DEFAULT_WORKSPACE = ''
18
28
  # Production domain of AoC
19
- PROD_DOMAIN = 'ibmaspera.com' # cspell:disable-line
29
+ SAAS_DOMAIN_PROD = 'ibmaspera.com' # cspell:disable-line
20
30
  # to avoid infinite loop in pub link redirection
21
31
  MAX_AOC_URL_REDIRECT = 10
22
32
  CLIENT_ID_PREFIX = 'aspera.'
23
- # Well-known AoC globals client apps
33
+ # Well-known AoC global client apps
24
34
  GLOBAL_CLIENT_APPS = DataRepository::ELEMENTS.select{|i|i.to_s.start_with?(CLIENT_ID_PREFIX)}.freeze
25
35
  # cookie prefix so that console can decode identity
26
36
  COOKIE_PREFIX_CONSOLE_AOC = 'aspera.aoc'
@@ -63,20 +73,13 @@ module Aspera
63
73
  end
64
74
 
65
75
  # base API url depends on domain, which could be "qa.xxx"
66
- def api_base_url(organization: 'api', api_domain: PROD_DOMAIN)
76
+ def api_base_url(organization: 'api', api_domain: SAAS_DOMAIN_PROD)
67
77
  return "https://#{organization}.#{api_domain}"
68
78
  end
69
79
 
70
- def metering_api(entitlement_id, customer_id, api_domain=PROD_DOMAIN)
71
- return Rest.new(
72
- base_url: "#{api_base_url(api_domain: api_domain)}/metering/v1",
73
- headers: {'X-Aspera-Entitlement-Authorization' => Rest.basic_token(entitlement_id, customer_id)}
74
- )
75
- end
76
-
77
80
  # split host of http://myorg.asperafiles.com in org and domain
78
81
  def url_parts(uri)
79
- raise "No host found in URL.Please check URL format: https://myorg.#{PROD_DOMAIN}" if uri.host.nil?
82
+ raise "No host found in URL.Please check URL format: https://myorg.#{SAAS_DOMAIN_PROD}" if uri.host.nil?
80
83
  parts = uri.host.split('.', 2)
81
84
  Aspera.assert(parts.length == 2){"expecting a public FQDN for #{PRODUCT_NAME}"}
82
85
  return parts
@@ -128,11 +131,11 @@ module Aspera
128
131
  organization: parts[0]
129
132
  }
130
133
  end
131
- end # static methods
134
+ end
132
135
 
133
136
  attr_reader :private_link
134
137
 
135
- def initialize(subpath: API_V1, url:, auth:, client_id: nil, client_secret: nil, scope: nil, redirect_uri: nil, private_key: nil, passphrase: nil, username: nil,
138
+ def initialize(url:, auth:, subpath: API_V1, client_id: nil, client_secret: nil, scope: nil, redirect_uri: nil, private_key: nil, passphrase: nil, username: nil,
136
139
  password: nil, workspace: nil, secret_finder: nil)
137
140
  # test here because link may set url
138
141
  raise ArgumentError, 'Missing mandatory option: url' if url.nil?
@@ -405,7 +408,7 @@ module Aspera
405
408
  end
406
409
  else # unexpected extended value, must be String or Hash
407
410
  raise "#{recipient_list_field} item must be a String (email, shared inbox) or Hash (id,type)"
408
- end # type of recipient info
411
+ end
409
412
  # add original or resolved recipient info
410
413
  resolved_list.push(short_recipient_info)
411
414
  end
@@ -52,9 +52,9 @@ module Aspera
52
52
  # read FASP connection information for bucket
53
53
  xml_result_text = s3_api.call(
54
54
  operation: 'GET',
55
- subpath: bucket,
56
- headers: {'Accept' => 'application/xml'},
57
- url_params: {'faspConnectionInfo' => nil}
55
+ subpath: bucket,
56
+ headers: {'Accept' => 'application/xml'},
57
+ query: {'faspConnectionInfo' => nil}
58
58
  )[:http].body
59
59
  ats_info = XmlSimple.xml_in(xml_result_text, {'ForceArray' => false})
60
60
  Aspera::Log.dump('ats_info', ats_info)
@@ -87,7 +87,7 @@ module Aspera
87
87
  receiver_client_ids: 'aspera_ats'
88
88
  )
89
89
  # get delegated token to be placed in rest call header and in transfer tags
90
- @storage_credentials['token'][TOKEN_FIELD] = OAuth::Factory.bearer_extract(delegated_oauth.get_authorization)
90
+ @storage_credentials['token'][TOKEN_FIELD] = OAuth::Factory.bearer_extract(delegated_oauth.token)
91
91
  @headers['X-Aspera-Storage-Credentials'] = JSON.generate(@storage_credentials)
92
92
  end
93
93
  end