aspera-cli 4.19.0 → 4.21.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +46 -0
  4. data/CONTRIBUTING.md +18 -4
  5. data/README.md +886 -510
  6. data/bin/asession +27 -20
  7. data/examples/build_exec +65 -76
  8. data/examples/build_exec_rubyc +40 -0
  9. data/examples/get_proto_file.rb +7 -0
  10. data/lib/aspera/agent/alpha.rb +18 -24
  11. data/lib/aspera/agent/base.rb +2 -18
  12. data/lib/aspera/agent/connect.rb +34 -15
  13. data/lib/aspera/agent/direct.rb +44 -54
  14. data/lib/aspera/agent/httpgw.rb +2 -3
  15. data/lib/aspera/agent/node.rb +11 -21
  16. data/lib/aspera/agent/{trsdk.rb → transferd.rb} +27 -51
  17. data/lib/aspera/api/alee.rb +15 -0
  18. data/lib/aspera/api/aoc.rb +139 -105
  19. data/lib/aspera/api/ats.rb +1 -1
  20. data/lib/aspera/api/cos_node.rb +1 -1
  21. data/lib/aspera/api/httpgw.rb +15 -10
  22. data/lib/aspera/api/node.rb +70 -32
  23. data/lib/aspera/ascmd.rb +56 -48
  24. data/lib/aspera/ascp/installation.rb +166 -70
  25. data/lib/aspera/ascp/management.rb +30 -8
  26. data/lib/aspera/assert.rb +10 -5
  27. data/lib/aspera/cli/formatter.rb +166 -162
  28. data/lib/aspera/cli/hints.rb +2 -1
  29. data/lib/aspera/cli/info.rb +12 -10
  30. data/lib/aspera/cli/main.rb +28 -13
  31. data/lib/aspera/cli/manager.rb +7 -2
  32. data/lib/aspera/cli/plugin.rb +17 -31
  33. data/lib/aspera/cli/plugins/alee.rb +3 -3
  34. data/lib/aspera/cli/plugins/aoc.rb +246 -208
  35. data/lib/aspera/cli/plugins/ats.rb +16 -14
  36. data/lib/aspera/cli/plugins/config.rb +154 -94
  37. data/lib/aspera/cli/plugins/console.rb +3 -3
  38. data/lib/aspera/cli/plugins/cos.rb +1 -0
  39. data/lib/aspera/cli/plugins/faspex.rb +15 -23
  40. data/lib/aspera/cli/plugins/faspex5.rb +64 -50
  41. data/lib/aspera/cli/plugins/faspio.rb +2 -2
  42. data/lib/aspera/cli/plugins/httpgw.rb +1 -1
  43. data/lib/aspera/cli/plugins/node.rb +174 -109
  44. data/lib/aspera/cli/plugins/orchestrator.rb +14 -13
  45. data/lib/aspera/cli/plugins/preview.rb +8 -9
  46. data/lib/aspera/cli/plugins/server.rb +5 -9
  47. data/lib/aspera/cli/plugins/shares.rb +2 -2
  48. data/lib/aspera/cli/sync_actions.rb +2 -2
  49. data/lib/aspera/cli/transfer_agent.rb +12 -14
  50. data/lib/aspera/cli/transfer_progress.rb +37 -17
  51. data/lib/aspera/cli/version.rb +1 -1
  52. data/lib/aspera/command_line_builder.rb +4 -5
  53. data/lib/aspera/coverage.rb +13 -1
  54. data/lib/aspera/environment.rb +75 -25
  55. data/lib/aspera/faspex_gw.rb +2 -2
  56. data/lib/aspera/json_rpc.rb +1 -1
  57. data/lib/aspera/keychain/macos_security.rb +7 -12
  58. data/lib/aspera/log.rb +3 -4
  59. data/lib/aspera/node_simulator.rb +230 -112
  60. data/lib/aspera/oauth/base.rb +64 -83
  61. data/lib/aspera/oauth/factory.rb +52 -6
  62. data/lib/aspera/oauth/generic.rb +4 -8
  63. data/lib/aspera/oauth/jwt.rb +6 -3
  64. data/lib/aspera/oauth/url_json.rb +1 -2
  65. data/lib/aspera/oauth/web.rb +5 -2
  66. data/lib/aspera/persistency_action_once.rb +16 -8
  67. data/lib/aspera/persistency_folder.rb +20 -2
  68. data/lib/aspera/preview/generator.rb +1 -1
  69. data/lib/aspera/preview/utils.rb +11 -17
  70. data/lib/aspera/products/alpha.rb +30 -0
  71. data/lib/aspera/products/connect.rb +48 -0
  72. data/lib/aspera/products/other.rb +82 -0
  73. data/lib/aspera/products/transferd.rb +54 -0
  74. data/lib/aspera/rest.rb +116 -87
  75. data/lib/aspera/secret_hider.rb +2 -2
  76. data/lib/aspera/ssh.rb +31 -24
  77. data/lib/aspera/transfer/faux_file.rb +4 -4
  78. data/lib/aspera/transfer/parameters.rb +16 -17
  79. data/lib/aspera/transfer/spec.rb +12 -12
  80. data/lib/aspera/transfer/spec.yaml +22 -20
  81. data/lib/aspera/transfer/sync.rb +2 -10
  82. data/lib/aspera/transfer/uri.rb +3 -3
  83. data/lib/aspera/uri_reader.rb +1 -1
  84. data/lib/aspera/web_auth.rb +166 -17
  85. data/lib/aspera/web_server_simple.rb +4 -3
  86. data/lib/transferd_pb.rb +86 -0
  87. data/lib/transferd_services_pb.rb +84 -0
  88. data.tar.gz.sig +0 -0
  89. metadata +58 -22
  90. metadata.gz.sig +0 -0
  91. data/lib/aspera/ascp/products.rb +0 -156
data/lib/aspera/rest.rb CHANGED
@@ -10,7 +10,8 @@ require 'net/http'
10
10
  require 'net/https'
11
11
  require 'json'
12
12
  require 'base64'
13
- require 'cgi'
13
+ require 'singleton'
14
+ require 'securerandom'
14
15
 
15
16
  # Cancel method for HTTP
16
17
  class Net::HTTP::Cancel < Net::HTTPRequest # rubocop:disable Style/ClassAndModuleChildren
@@ -20,22 +21,34 @@ class Net::HTTP::Cancel < Net::HTTPRequest # rubocop:disable Style/ClassAndModul
20
21
  end
21
22
 
22
23
  module Aspera
24
+ # Global settings for Rest object
25
+ # For example to remove certificate verification globally:
26
+ # `RestParameters.instance.session_cb = lambda{|http|http.verify_mode=OpenSSL::SSL::VERIFY_NONE}`
27
+ # @param user_agent [String] HTTP request header: 'User-Agent'
28
+ # @param download_partial_suffix [String] suffix for partial download
29
+ # @param session_cb [lambda] lambda called on new HTTP session. Takes the Net::HTTP as arg. Used to change parameters on creation.
30
+ # @param progress_bar [Object] progress bar object called for file transfer
31
+ class RestParameters
32
+ include Singleton
33
+
34
+ attr_accessor :user_agent, :download_partial_suffix, :retry_on_error, :retry_sleep, :session_cb, :progress_bar
35
+
36
+ private
37
+
38
+ def initialize
39
+ @user_agent = 'RubyAsperaRest'
40
+ @download_partial_suffix = '.http_partial'
41
+ @retry_on_error = 0
42
+ @retry_sleep = nil
43
+ @session_cb = nil
44
+ @progress_bar = nil
45
+ end
46
+ end
47
+
23
48
  # a simple class to make HTTP calls, equivalent to rest-client
24
49
  # rest call errors are raised as exception RestCallError
25
50
  # and error are analyzed in RestErrorAnalyzer
26
51
  class Rest
27
- # Global settings also valid for any subclass
28
- # @param user_agent [String] HTTP request header: 'User-Agent'
29
- # @param download_partial_suffix [String] suffix for partial download
30
- # @param session_cb [lambda] lambda called on new HTTP session. Takes the Net::HTTP as arg. Used to change parameters on creation.
31
- # @param progress_bar [Object] progress bar object
32
- @@global = { # rubocop:disable Style/ClassVars
33
- user_agent: 'RubyAsperaRest',
34
- download_partial_suffix: '.http_partial',
35
- session_cb: nil,
36
- progress_bar: nil
37
- }
38
-
39
52
  # flag for array parameters prefixed with []
40
53
  ARRAY_PARAMS = '[]'
41
54
 
@@ -61,37 +74,45 @@ module Aspera
61
74
  return values.first.eql?(ARRAY_PARAMS)
62
75
  end
63
76
 
64
- # Build URI from URL and parameters and check it is http or https, encode array [] parameters
65
- def build_uri(url, query_hash=nil)
77
+ # Build URI from URL and parameters and check it is http or https
78
+ # encode array [] parameters
79
+ # @param query [Hash,Array]
80
+ def build_uri(url, query=nil)
66
81
  uri = URI.parse(url)
67
82
  Aspera.assert(%w[http https].include?(uri.scheme)){"REST endpoint shall be http/s not #{uri.scheme}"}
68
- return uri if query_hash.nil?
69
- Log.log.debug{Log.dump('query', query_hash)}
70
- Aspera.assert_type(query_hash, Hash)
71
- return uri if query_hash.empty?
72
- query = []
73
- query_hash.each do |k, v|
74
- case v
75
- when Array
76
- # support array for query parameter, there is no standard. Either p[]=1&p[]=2, or p=1&p=2
77
- suffix = array_params?(v) ? v.shift : ''
78
- v.each do |e|
79
- query.push(["#{k}#{suffix}", e])
83
+ return uri if query.nil? || query.respond_to?(:empty?) && query.empty?
84
+ Log.log.debug{Log.dump('query', query)}
85
+ query_array = []
86
+ case query
87
+ when Hash
88
+ query.each do |k, v|
89
+ case v
90
+ when Array
91
+ # support array for query parameter, there is no standard. Either p[]=1&p[]=2, or p=1&p=2
92
+ suffix = array_params?(v) ? v.shift : ''
93
+ v.each do |e|
94
+ query_array.push(["#{k}#{suffix}", e])
95
+ end
96
+ else
97
+ query_array.push([k, v])
80
98
  end
81
- else
82
- query.push([k, v])
83
99
  end
100
+ when Array
101
+ Aspera.assert(query.all?{|i| i.is_a?(Array) && i.length.eql?(2)}) {'Query must be array of arrays or 2 elements'}
102
+ query_array = query
103
+ else
104
+ raise "Query must be Hash or Array, not #{query.class}"
84
105
  end
85
106
  # [] is allowed in url parameters
86
- uri.query = URI.encode_www_form(query).gsub('%5B%5D=', '[]=')
107
+ uri.query = URI.encode_www_form(query_array).gsub('%5B%5D=', '[]=')
87
108
  return uri
88
109
  end
89
110
 
90
- # decode query string as hash
111
+ # Decode query string as Hash
91
112
  # Does not support arrays in query string, no standard, e.g. PHP's way is p[]=1&p[]=2
92
- # @param query [String] query string
113
+ # @param query [String] query string as in URI.query
93
114
  # @return [Hash] decoded query
94
- def decode_query(query)
115
+ def query_to_h(query)
95
116
  URI.decode_www_form(query).each_with_object({}) do |pair, h|
96
117
  key = pair.first
97
118
  raise "Array not supported in query string: #{key}" if key.include?('[]') || h.key?(key)
@@ -108,7 +129,7 @@ module Aspera
108
129
  http_session = Net::HTTP.new(uri.host, uri.port)
109
130
  http_session.use_ssl = uri.scheme.eql?('https')
110
131
  # set http options in callback, such as timeout and cert. verification
111
- @@global[:session_cb]&.call(http_session)
132
+ RestParameters.instance.session_cb&.call(http_session)
112
133
  # manually start session for keep alive (if supported by server, else, session is closed every time)
113
134
  http_session.start
114
135
  return http_session
@@ -142,19 +163,6 @@ module Aspera
142
163
  return result
143
164
  end
144
165
 
145
- # set global parameters
146
- def set_parameters(**options)
147
- options.each do |key, value|
148
- Aspera.assert(@@global.key?(key)){"Unknown Rest option #{key}"}
149
- @@global[key] = value
150
- end
151
- end
152
-
153
- # @return [String] HTTP agent name
154
- def user_agent
155
- return @@global[:user_agent]
156
- end
157
-
158
166
  def parse_header(header)
159
167
  type, *params = header.split(/;\s*/)
160
168
  parameters = params.map do |param|
@@ -179,19 +187,23 @@ module Aspera
179
187
 
180
188
  public
181
189
 
182
- attr_reader :auth_params
183
190
  attr_reader :base_url
191
+ attr_reader :auth_params
184
192
 
193
+ # @return creation parameters
185
194
  def params
186
195
  return {
187
- base_url: @base_url,
188
- auth: @auth_params,
189
- not_auth_codes: @not_auth_codes,
190
- redirect_max: @redirect_max,
191
- headers: @headers
196
+ base_url: @base_url, # String
197
+ auth: @auth_params, # Hash
198
+ not_auth_codes: @not_auth_codes, # Array
199
+ redirect_max: @redirect_max, # Integer
200
+ headers: @headers # Hash
192
201
  }
193
202
  end
194
203
 
204
+ # Create a REST object for API calls
205
+ # HTTP sessions parameters can be modified using global parameters in RestParameters
206
+ # For example, TLS verification can be skipped.
195
207
  # @param base_url [String] base URL of REST API
196
208
  # @param auth [Hash] authentication parameters:
197
209
  # :type (:none, :basic, :url, :oauth2)
@@ -199,14 +211,15 @@ module Aspera
199
211
  # :password [:basic]
200
212
  # :url_query [:url] a hash
201
213
  # :* [:oauth2] see OAuth::Factory class
202
- # @param not_auth_codes [Array] codes that trigger a refresh/regeneration of bearer token
203
- # @param redirect_max [int] max redirection allowed
214
+ # @param not_auth_codes [Array] codes that trigger a refresh/regeneration of bearer token
215
+ # @param redirect_max [Integer] max redirection allowed
216
+ # @param headers [Hash] default headers to include in all calls
204
217
  def initialize(
205
218
  base_url:,
206
- auth: nil,
207
- not_auth_codes: nil,
219
+ auth: {type: :none},
220
+ not_auth_codes: ['401'],
208
221
  redirect_max: 0,
209
- headers: nil
222
+ headers: {}
210
223
  )
211
224
  Aspera.assert_type(base_url, String)
212
225
  # base url with no trailing slashes (note: string may be frozen)
@@ -216,19 +229,20 @@ module Aspera
216
229
  @base_url = @base_url.gsub(/:80$/, '') if @base_url.start_with?('http://')
217
230
  Log.log.debug{"Rest.new(#{@base_url})"}
218
231
  # default is no auth
219
- @auth_params = auth.nil? ? {type: :none} : auth
232
+ @auth_params = auth
220
233
  Aspera.assert_type(@auth_params, Hash)
221
234
  Aspera.assert(@auth_params.key?(:type)){'no auth type defined'}
222
- @not_auth_codes = not_auth_codes.nil? ? ['401'] : not_auth_codes
235
+ @not_auth_codes = not_auth_codes
223
236
  Aspera.assert_type(@not_auth_codes, Array)
224
237
  # persistent session
225
238
  @http_session = nil
226
- # OAuth object (created on demand)
227
- @oauth = nil
228
239
  @redirect_max = redirect_max
229
- @headers = headers.nil? ? {} : headers
240
+ Aspera.assert_type(@redirect_max, Integer)
241
+ @headers = headers
230
242
  Aspera.assert_type(@headers, Hash)
231
- @headers['User-Agent'] ||= @@global[:user_agent]
243
+ @headers['User-Agent'] ||= RestParameters.instance.user_agent
244
+ # OAuth object (created on demand)
245
+ @oauth = nil
232
246
  end
233
247
 
234
248
  # @return the OAuth object (create, or cached if already created)
@@ -263,6 +277,8 @@ module Aspera
263
277
  )
264
278
  subpath = subpath.to_s if subpath.is_a?(Symbol)
265
279
  subpath = '' if subpath.nil?
280
+ Log.log.debug{"#{operation} [#{subpath}]".red.bold.bg_green}
281
+ Log.log.debug{Log.dump(:body, body)}
266
282
  Aspera.assert_type(subpath, String)
267
283
  if headers.nil?
268
284
  headers = @headers.clone
@@ -272,7 +288,6 @@ module Aspera
272
288
  headers.merge!(h)
273
289
  end
274
290
  Aspera.assert_type(headers, Hash)
275
- Log.log.debug{"#{operation} [#{subpath}]".red.bold.bg_green}
276
291
  case @auth_params[:type]
277
292
  when :none
278
293
  # no auth
@@ -322,11 +337,13 @@ module Aspera
322
337
  end
323
338
  # :type = :basic
324
339
  req.basic_auth(@auth_params[:username], @auth_params[:password]) if @auth_params[:type].eql?(:basic)
325
- Log.log.debug{Log.dump(:req_body, req.body)}
340
+ Log.log.trace1{Log.dump(:req_body, req.body)}
326
341
  # we try the call, and will retry only if oauth, as we can, first with refresh, and then re-auth if refresh is bad
327
342
  oauth_tries ||= 2
343
+ timeout_tries ||= 5
344
+ general_tries ||= 1 + RestParameters.instance.retry_on_error
328
345
  # initialize with number of initial retries allowed, nil gives zero
329
- tries_remain_redirect = @redirect_max.to_i if tries_remain_redirect.nil?
346
+ tries_remain_redirect = @redirect_max if tries_remain_redirect.nil?
330
347
  Log.log.debug("send request (retries=#{tries_remain_redirect})")
331
348
  result_mime = nil
332
349
  file_saved = false
@@ -350,38 +367,44 @@ module Aspera
350
367
  end
351
368
  end
352
369
  # download with temp filename
353
- target_file_tmp = "#{target_file}#{@@global[:download_partial_suffix]}"
370
+ target_file_tmp = "#{target_file}#{RestParameters.instance.download_partial_suffix}"
354
371
  Log.log.debug{"saving to: #{target_file}"}
355
372
  written_size = 0
356
- @@global[:progress_bar]&.event(session_id: 1, type: :session_start)
357
- @@global[:progress_bar]&.event(session_id: 1, type: :session_size, info: total_size)
373
+ session_id = SecureRandom.uuid.freeze
374
+ RestParameters.instance.progress_bar&.event(:session_start, session_id: session_id)
375
+ RestParameters.instance.progress_bar&.event(:session_size, session_id: session_id, info: total_size)
358
376
  File.open(target_file_tmp, 'wb') do |file|
359
377
  result[:http].read_body do |fragment|
360
378
  file.write(fragment)
361
379
  written_size += fragment.length
362
- @@global[:progress_bar]&.event(session_id: 1, type: :transfer, info: written_size)
380
+ RestParameters.instance.progress_bar&.event(:transfer, session_id: session_id, info: written_size)
363
381
  end
364
382
  end
365
- @@global[:progress_bar]&.event(session_id: 1, type: :end)
383
+ RestParameters.instance.progress_bar&.event(:end, session_id: session_id)
366
384
  # rename at the end
367
385
  File.rename(target_file_tmp, target_file)
368
386
  file_saved = true
369
387
  end
370
388
  end
389
+ Log.log.debug{"result: code=#{result[:http].code} mime=#{result_mime}"}
371
390
  # sometimes there is a UTF8 char (e.g. (c) ), TODO : related to mime type encoding ?
372
391
  # result[:http].body.force_encoding('UTF-8') if result[:http].body.is_a?(String)
373
392
  # Log.log.debug{"result: body=#{result[:http].body}"}
374
- result[:data] = case result_mime
393
+ case result_mime
375
394
  when *JSON_DECODE
376
- JSON.parse(result[:http].body) rescue result[:http].body
395
+ result[:data] = JSON.parse(result[:http].body) rescue result[:http].body
396
+ Log.log.debug{Log.dump('result_data', result[:data])}
377
397
  else # when 'text/plain'
378
- result[:http].body
398
+ result[:data] = result[:http].body
379
399
  end
380
- Log.log.debug{"result: code=#{result[:http].code} mime=#{result_mime}"}
381
- Log.log.debug{Log.dump('data', result[:data])}
382
400
  RestErrorAnalyzer.instance.raise_on_error(req, result)
383
- File.write(save_to_file, result[:http].body) unless file_saved || save_to_file.nil?
401
+ File.write(save_to_file, result[:http].body, binmode: true) unless file_saved || save_to_file.nil?
384
402
  rescue RestCallError => e
403
+ do_retry = false
404
+ # AoC have some timeout , like Connect to platform.bss.asperasoft.com:443 ...
405
+ do_retry = true if e.response.body.include?('failed: connect timed out') && (timeout_tries -= 1).positive?
406
+ # possibility to retry anything if it fails
407
+ do_retry = true if (general_tries -= 1).positive?
385
408
  # not authorized: oauth token expired
386
409
  if @not_auth_codes.include?(result[:http].code.to_s) && @auth_params[:type].eql?(:oauth2)
387
410
  begin
@@ -394,7 +417,11 @@ module Aspera
394
417
  req['Authorization'] = oauth.token(refresh: true)
395
418
  end
396
419
  Log.log.debug{"using new token=#{headers['Authorization']}"}
397
- retry if (oauth_tries -= 1).nonzero?
420
+ do_retry = true if (oauth_tries -= 1).positive?
421
+ end
422
+ if do_retry
423
+ sleep(RestParameters.instance.retry_sleep) unless RestParameters.instance.retry_sleep.nil?
424
+ retry
398
425
  end
399
426
  # redirect ? (any code beginning with 3)
400
427
  if e.response.is_a?(Net::HTTPRedirection) && tries_remain_redirect.positive?
@@ -420,27 +447,28 @@ module Aspera
420
447
  end
421
448
 
422
449
  #
423
- # CRUD methods here
450
+ # CRUD simplified methods here
451
+ # If specific elements are needed, then use the full `call` method
424
452
  #
425
453
 
426
454
  def create(subpath, params)
427
- return call(operation: 'POST', subpath: subpath, headers: {'Accept' => 'application/json'}, body: params, body_type: :json)
455
+ return call(operation: 'POST', subpath: subpath, headers: {'Accept' => 'application/json'}, body: params, body_type: :json)[:data]
428
456
  end
429
457
 
430
458
  def read(subpath, query=nil)
431
- return call(operation: 'GET', subpath: subpath, headers: {'Accept' => 'application/json'}, query: query)
459
+ return call(operation: 'GET', subpath: subpath, headers: {'Accept' => 'application/json'}, query: query)[:data]
432
460
  end
433
461
 
434
462
  def update(subpath, params)
435
- return call(operation: 'PUT', subpath: subpath, headers: {'Accept' => 'application/json'}, body: params, body_type: :json)
463
+ return call(operation: 'PUT', subpath: subpath, headers: {'Accept' => 'application/json'}, body: params, body_type: :json)[:data]
436
464
  end
437
465
 
438
466
  def delete(subpath, params=nil)
439
- return call(operation: 'DELETE', subpath: subpath, headers: {'Accept' => 'application/json'}, query: params)
467
+ return call(operation: 'DELETE', subpath: subpath, headers: {'Accept' => 'application/json'}, query: params)[:data]
440
468
  end
441
469
 
442
470
  def cancel(subpath)
443
- return call(operation: 'CANCEL', subpath: subpath, headers: {'Accept' => 'application/json'})
471
+ return call(operation: 'CANCEL', subpath: subpath, headers: {'Accept' => 'application/json'})[:data]
444
472
  end
445
473
 
446
474
  # Query entity by general search (read with parameter `q`)
@@ -449,9 +477,10 @@ module Aspera
449
477
  # @param search_name name of searched entity
450
478
  # @param query additional search query parameters
451
479
  # @returns [Hash] A single entity matching the search, or an exception if not found or multiple found
452
- def lookup_by_name(subpath, search_name, query={})
480
+ def lookup_by_name(subpath, search_name, query: nil)
481
+ query = {} if query.nil?
453
482
  # returns entities matching the query (it matches against several fields in case insensitive way)
454
- matching_items = read(subpath, query.merge({'q' => search_name}))[:data]
483
+ matching_items = read(subpath, query.merge({'q' => search_name}))
455
484
  # API style: {totalcount:, ...} cspell: disable-line
456
485
  matching_items = matching_items[subpath] if matching_items.is_a?(Hash)
457
486
  Aspera.assert_type(matching_items, Array)
@@ -20,6 +20,8 @@ module Aspera
20
20
  KEY_FALSE_POSITIVES = [/^access_key$/, /^fallback_private_key$/].freeze
21
21
  # regex that define named captures :begin and :end
22
22
  REGEX_LOG_REPLACES = [
23
+ # private key values (place first)
24
+ /(?<begin>--+BEGIN [^-]+ KEY--+)[[:ascii:]]+?(?<end>--+?END [^-]+ KEY--+)\n*/,
23
25
  # CLI manager get/set options
24
26
  /(?<begin>[sg]et (?:#{KEY_SECRETS.join('|')})=).*(?<end>)/,
25
27
  # env var ascp exec
@@ -28,8 +30,6 @@ module Aspera
28
30
  /(?<begin>(?:(?<quote>["'])|:)[^"':=]*(?:#{ALL_SECRETS.join('|')})[^"':=]*\k<quote>?(?:=>|:) *")[^"]+(?<end>")/,
29
31
  # logged data
30
32
  /(?<begin>(?:#{ALL_SECRETS2.join('|')})[ =:]+).*(?<end>$)/,
31
- # private key values
32
- /(?<begin>--+BEGIN [^-]+ KEY--+)[[:ascii:]]+?(?<end>--+?END [^-]+ KEY--+)/,
33
33
  # cred in http dump
34
34
  /(?<begin>(?:#{HTTP_SECRETS.join('|')}): )[^\\]+(?<end>\\)/i
35
35
  ].freeze
data/lib/aspera/ssh.rb CHANGED
@@ -1,33 +1,41 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'net/ssh'
4
-
5
- if ENV.fetch('ASCLI_ENABLE_ED25519', 'false').eql?('false')
6
- # HACK: deactivate ed25519 and ecdsa private keys from SSH identities, as it usually causes problems
7
- old_verbose = $VERBOSE
8
- $VERBOSE = nil
9
- begin
10
- module Net; module SSH; module Authentication; class Session; private; def default_keys; %w[~/.ssh/id_dsa ~/.ssh/id_rsa ~/.ssh2/id_dsa ~/.ssh2/id_rsa]; end; end; end; end; end # rubocop:disable Layout/AccessModifierIndentation, Layout/EmptyLinesAroundAccessModifier, Layout/LineLength, Style/Semicolon
11
- rescue StandardError
12
- # ignore errors
13
- end
14
- $VERBOSE = old_verbose
15
- end
16
-
17
- if defined?(JRUBY_VERSION) && ENV.fetch('ASCLI_ENABLE_ECDSHA2', 'false').eql?('false')
18
- Net::SSH::Transport::Algorithms::ALGORITHMS.each_value { |a| a.reject! { |a| a =~ /^ecd(sa|h)-sha2/ } }
19
- Net::SSH::KnownHosts::SUPPORTED_TYPE.reject! { |t| t =~ /^ecd(sa|h)-sha2/ }
20
- end
4
+ require 'aspera/assert'
5
+ require 'aspera/log'
21
6
 
22
7
  module Aspera
23
8
  # A simple wrapper around Net::SSH
24
9
  # executes one command and get its result from stdout
25
10
  class Ssh
11
+ class << self
12
+ def disable_ed25519_keys
13
+ Log.log.debug('Disabling SSH ed25519 user keys')
14
+ old_verbose = $VERBOSE
15
+ $VERBOSE = nil
16
+ Net::SSH::Authentication::Session.class_eval do
17
+ define_method(:default_keys) do
18
+ %w[~/.ssh/id_dsa ~/.ssh/id_rsa ~/.ssh2/id_dsa ~/.ssh2/id_rsa].freeze
19
+ end
20
+ private(:default_keys)
21
+ end rescue nil
22
+ $VERBOSE = old_verbose
23
+ end
24
+
25
+ def disable_ecd_sha2_algorithms
26
+ Log.log.debug('Disabling SSH ecdsa')
27
+ Net::SSH::Transport::Algorithms::ALGORITHMS.each_value { |a| a.reject! { |a| a =~ /^ecd(sa|h)-sha2/ } }
28
+ Net::SSH::KnownHosts::SUPPORTED_TYPE.reject! { |t| t =~ /^ecd(sa|h)-sha2/ }
29
+ end
30
+ end
26
31
  # ssh_options: same as Net::SSH.start
27
32
  # see: https://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start
28
33
  def initialize(host, username, ssh_options)
29
34
  Log.log.debug{"ssh:#{username}@#{host}"}
30
35
  Log.log.debug{"ssh_options:#{ssh_options}"}
36
+ Aspera.assert_type(host, String)
37
+ Aspera.assert_type(username, String)
38
+ Aspera.assert_type(ssh_options, Hash)
31
39
  @host = host
32
40
  @username = username
33
41
  @ssh_options = ssh_options
@@ -35,10 +43,7 @@ module Aspera
35
43
  end
36
44
 
37
45
  def execute(cmd, input=nil)
38
- if cmd.is_a?(Array)
39
- # concatenate arguments, enclose in double quotes
40
- cmd = cmd.map{|v|%Q("#{v}")}.join(' ')
41
- end
46
+ Aspera.assert_type(cmd, String)
42
47
  Log.log.debug{"cmd=#{cmd}"}
43
48
  response = []
44
49
  Net::SSH.start(@host, @username, @ssh_options) do |session|
@@ -49,9 +54,7 @@ module Aspera
49
54
  channel.on_extended_data do |_chan, _type, data|
50
55
  error_message = "#{cmd}: [#{data.chomp}]"
51
56
  # Happens when windows user hasn't logged in and created home account.
52
- if data.include?('Could not chdir to home directory')
53
- error_message += "\nHint: home not created in Windows?"
54
- end
57
+ error_message += "\nHint: home not created in Windows?" if data.include?('Could not chdir to home directory')
55
58
  raise error_message
56
59
  end
57
60
  # send command to SSH channel (execute) cspell: disable-next-line
@@ -67,3 +70,7 @@ module Aspera
67
70
  end
68
71
  end
69
72
  end
73
+
74
+ # HACK: deactivate ed25519 and ecdsa private keys from SSH identities, as it usually causes problems
75
+ Aspera::Ssh.disable_ed25519_keys if ENV.fetch('ASCLI_ENABLE_ED25519', 'false').eql?('false')
76
+ Aspera::Ssh.disable_ecd_sha2_algorithms if defined?(JRUBY_VERSION) && ENV.fetch('ASCLI_ENABLE_ECDSHA2', 'false').eql?('false')
@@ -13,16 +13,16 @@ module Aspera
13
13
  # @return nil if not a faux: scheme, else a FauxFile instance
14
14
  def create(name)
15
15
  return nil unless name.start_with?(PREFIX)
16
- url_parts = name[PREFIX.length..-1].split('?')
17
- raise 'Format: #{PREFIX}<file path>?<size>' unless url_parts.length.eql?(2)
18
- raise "Format: <integer>[#{SUFFIX.join(',')}]" unless (m = url_parts[1].downcase.match(/^(\d+)([#{SUFFIX.join('')}])$/))
16
+ name_params = name[PREFIX.length..-1].split('?', 2)
17
+ raise 'Format: #{PREFIX}<file path>?<size>' unless name_params.length.eql?(2)
18
+ raise "Format: <integer>[#{SUFFIX.join(',')}]" unless (m = name_params[1].downcase.match(/^(\d+)([#{SUFFIX.join('')}])$/))
19
19
  size = m[1].to_i
20
20
  suffix = m[2]
21
21
  SUFFIX.each do |s|
22
22
  size *= 1024
23
23
  break if s.eql?(suffix)
24
24
  end
25
- return FauxFile.new(url_parts[0], size)
25
+ return FauxFile.new(name_params[0], size)
26
26
  end
27
27
  end
28
28
  attr_reader :path, :size
@@ -8,6 +8,7 @@ require 'aspera/transfer/error'
8
8
  require 'aspera/transfer/spec'
9
9
  require 'aspera/ascp/installation'
10
10
  require 'aspera/cli/formatter'
11
+ require 'aspera/agent/base'
11
12
  require 'aspera/rest'
12
13
  require 'securerandom'
13
14
  require 'base64'
@@ -20,7 +21,7 @@ module Aspera
20
21
  # translate transfer specification to ascp parameter list
21
22
  class Parameters
22
23
  # Agents shown in manual for parameters (sub list)
23
- SUPPORTED_AGENTS = %i[direct node connect trsdk httpgw].freeze
24
+ SUPPORTED_AGENTS = Agent::Base.agent_list.freeze
24
25
  FILE_LIST_OPTIONS = ['--file-list', '--file-pair-list'].freeze
25
26
  # Short names of columns in manual
26
27
  SUPPORTED_AGENTS_SHORT = SUPPORTED_AGENTS.map{|agent_sym|agent_sym.to_s[0].to_sym}
@@ -29,10 +30,21 @@ module Aspera
29
30
  private_constant :SUPPORTED_AGENTS, :FILE_LIST_OPTIONS
30
31
 
31
32
  class << self
33
+ # temp file list files are created here
34
+ def file_list_folder=(value)
35
+ @file_list_folder = value
36
+ return if @file_list_folder.nil?
37
+
38
+ FileUtils.mkdir_p(@file_list_folder)
39
+ TempFileManager.instance.cleanup_expired(@file_list_folder)
40
+ end
41
+
32
42
  # Temp folder for file lists, must contain only file lists
33
43
  # because of garbage collection takes any file there
34
- # this could be refined, as , for instance, on macos, temp folder is already user specific
35
- @file_list_folder = TempFileManager.instance.new_file_path_global('asession_filelists') # cspell:disable-line
44
+ # this could be refined, as, for example, on macos, temp folder is already user specific
45
+ def file_list_folder
46
+ @file_list_folder ||= TempFileManager.instance.new_file_path_global('asession_filelists')
47
+ end
36
48
 
37
49
  # @param formatter [Cli::Formatter] formatter to use
38
50
  # @return a table suitable to display in manual
@@ -67,9 +79,7 @@ module Aspera
67
79
  else
68
80
  param[:d].eql?(tick_yes) ? '' : 'n/a'
69
81
  end
70
- if options.key?(:enum)
71
- param[:description] += "\nAllowed values: #{options[:enum].join(', ')}"
72
- end
82
+ param[:description] += "\nAllowed values: #{options[:enum].join(', ')}" if options.key?(:enum)
73
83
  # replace "solidus" HTML entity with its text value
74
84
  param[:description] = param[:description].gsub('&sol;', '\\')
75
85
  result.push(param)
@@ -91,17 +101,6 @@ module Aspera
91
101
  def ascp_args_file_list?(ascp_args)
92
102
  ascp_args&.any?{|i|FILE_LIST_OPTIONS.include?(i)}
93
103
  end
94
-
95
- # temp file list files are created here
96
- def file_list_folder=(value)
97
- @file_list_folder = value
98
- return if @file_list_folder.nil?
99
- FileUtils.mkdir_p(@file_list_folder)
100
- TempFileManager.instance.cleanup_expired(@file_list_folder)
101
- end
102
-
103
- # static methods
104
- attr_reader :file_list_folder
105
104
  end
106
105
 
107
106
  # @param options [Hash] key: :wss: bool, :ascp_args: array of strings
@@ -10,34 +10,30 @@ module Aspera
10
10
  class Spec
11
11
  # default transfer username for access key based transfers
12
12
  ACCESS_KEY_TRANSFER_USER = 'xfer'
13
+ # default ports for SSH and UDP
13
14
  SSH_PORT = 33_001
14
15
  UDP_PORT = 33_001
16
+ # base transfer spec for access keys
15
17
  AK_TSPEC_BASE = {
16
18
  'remote_user' => ACCESS_KEY_TRANSFER_USER,
17
19
  'ssh_port' => SSH_PORT,
18
20
  'fasp_port' => UDP_PORT
19
21
  }.freeze
20
- # fields for transport
21
- TRANSPORT_FIELDS = %w[remote_host remote_user ssh_port fasp_port wss_enabled wss_port].freeze
22
+ # fields for WSS
23
+ WSS_FIELDS = %w[wss_enabled wss_port].freeze
24
+ # all fields for transport
25
+ TRANSPORT_FIELDS = %w[remote_host remote_user ssh_port fasp_port].concat(WSS_FIELDS).freeze
22
26
  # reserved tag for Aspera
23
27
  TAG_RESERVED = 'aspera'
24
28
  class << self
25
29
  # translate upload/download to send/receive
26
30
  def transfer_type_to_direction(transfer_type)
27
- case transfer_type.to_sym
28
- when :upload then DIRECTION_SEND
29
- when :download then DIRECTION_RECEIVE
30
- else Aspera.error_unexpected_value(transfer_type.to_sym)
31
- end
31
+ XFER_TYPE_TO_DIR.fetch(transfer_type)
32
32
  end
33
33
 
34
34
  # translate send/receive to upload/download
35
35
  def direction_to_transfer_type(direction)
36
- case direction
37
- when DIRECTION_SEND then :upload
38
- when DIRECTION_RECEIVE then :download
39
- else Aspera.error_unexpected_value(direction)
40
- end
36
+ XFER_DIR_TO_TYPE.fetch(direction)
41
37
  end
42
38
  end
43
39
  DESCRIPTION = CommandLineBuilder.normalize_description(YAML.load_file("#{__FILE__[0..-3]}yaml"))
@@ -49,6 +45,10 @@ module Aspera
49
45
  const_set("#{name.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/, '_')}", enum.freeze)
50
46
  end
51
47
  end
48
+ # DIRECTION_* are read from yaml
49
+ XFER_TYPE_TO_DIR = {upload: DIRECTION_SEND, download: DIRECTION_RECEIVE}.freeze
50
+ XFER_DIR_TO_TYPE = XFER_TYPE_TO_DIR.invert.freeze
51
+ private_constant :XFER_TYPE_TO_DIR, :XFER_DIR_TO_TYPE
52
52
  end
53
53
  end
54
54
  end