aspera-cli 4.21.2 → 4.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +1 -1
- data/CHANGELOG.md +34 -16
- data/CONTRIBUTING.md +6 -10
- data/README.md +805 -574
- data/examples/get_proto_file.rb +1 -1
- data/lib/aspera/agent/base.rb +9 -5
- data/lib/aspera/agent/connect.rb +30 -28
- data/lib/aspera/agent/desktop.rb +29 -25
- data/lib/aspera/agent/direct.rb +137 -125
- data/lib/aspera/agent/httpgw.rb +22 -26
- data/lib/aspera/agent/node.rb +14 -11
- data/lib/aspera/agent/transferd.rb +6 -2
- data/lib/aspera/api/aoc.rb +6 -6
- data/lib/aspera/api/cos_node.rb +1 -1
- data/lib/aspera/api/httpgw.rb +7 -3
- data/lib/aspera/api/node.rb +6 -4
- data/lib/aspera/ascmd.rb +3 -3
- data/lib/aspera/ascp/installation.rb +15 -16
- data/lib/aspera/ascp/management.rb +1 -1
- data/lib/aspera/assert.rb +11 -2
- data/lib/aspera/cli/error.rb +2 -2
- data/lib/aspera/cli/extended_value.rb +38 -19
- data/lib/aspera/cli/formatter.rb +48 -48
- data/lib/aspera/cli/hints.rb +1 -1
- data/lib/aspera/cli/main.rb +190 -168
- data/lib/aspera/cli/manager.rb +15 -15
- data/lib/aspera/cli/plugin.rb +23 -20
- data/lib/aspera/cli/plugin_factory.rb +1 -1
- data/lib/aspera/cli/plugins/alee.rb +1 -1
- data/lib/aspera/cli/plugins/aoc.rb +144 -107
- data/lib/aspera/cli/plugins/ats.rb +19 -17
- data/lib/aspera/cli/plugins/config.rb +67 -83
- data/lib/aspera/cli/plugins/console.rb +5 -3
- data/lib/aspera/cli/plugins/faspex.rb +39 -35
- data/lib/aspera/cli/plugins/faspex5.rb +104 -80
- data/lib/aspera/cli/plugins/faspio.rb +13 -1
- data/lib/aspera/cli/plugins/httpgw.rb +13 -1
- data/lib/aspera/cli/plugins/node.rb +306 -179
- data/lib/aspera/cli/plugins/orchestrator.rb +34 -40
- data/lib/aspera/cli/plugins/preview.rb +3 -3
- data/lib/aspera/cli/plugins/server.rb +6 -6
- data/lib/aspera/cli/plugins/shares.rb +5 -5
- data/lib/aspera/cli/sync_actions.rb +19 -18
- data/lib/aspera/cli/transfer_agent.rb +5 -5
- data/lib/aspera/cli/transfer_progress.rb +2 -2
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +116 -95
- data/lib/aspera/coverage.rb +4 -3
- data/lib/aspera/environment.rb +6 -6
- data/lib/aspera/faspex_gw.rb +14 -14
- data/lib/aspera/faspex_postproc.rb +7 -6
- data/lib/aspera/hash_ext.rb +2 -2
- data/lib/aspera/json_rpc.rb +1 -1
- data/lib/aspera/keychain/encrypted_hash.rb +47 -34
- data/lib/aspera/keychain/factory.rb +41 -0
- data/lib/aspera/keychain/hashicorp_vault.rb +71 -0
- data/lib/aspera/keychain/macos_security.rb +19 -11
- data/lib/aspera/log.rb +28 -34
- data/lib/aspera/nagios.rb +6 -6
- data/lib/aspera/node_simulator.rb +8 -8
- data/lib/aspera/oauth/base.rb +8 -6
- data/lib/aspera/oauth/factory.rb +5 -6
- data/lib/aspera/oauth/url_json.rb +6 -6
- data/lib/aspera/persistency_action_once.rb +6 -4
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/generator.rb +1 -1
- data/lib/aspera/preview/options.rb +16 -16
- data/lib/aspera/preview/terminal.rb +3 -3
- data/lib/aspera/preview/utils.rb +11 -13
- data/lib/aspera/products/connect.rb +1 -1
- data/lib/aspera/products/desktop.rb +1 -1
- data/lib/aspera/products/transferd.rb +1 -1
- data/lib/aspera/proxy_auto_config.rb +2 -2
- data/lib/aspera/rest.rb +52 -43
- data/lib/aspera/rest_errors_aspera.rb +1 -1
- data/lib/aspera/secret_hider.rb +5 -5
- data/lib/aspera/ssh.rb +4 -4
- data/lib/aspera/transfer/convert.rb +29 -0
- data/lib/aspera/transfer/error_info.rb +66 -66
- data/lib/aspera/transfer/parameters.rb +13 -68
- data/lib/aspera/transfer/spec.rb +5 -6
- data/lib/aspera/transfer/spec.schema.yaml +753 -0
- data/lib/aspera/transfer/spec_doc.rb +62 -0
- data/lib/aspera/transfer/sync.rb +23 -72
- data/lib/aspera/transfer/sync_instance.schema.yaml +13 -0
- data/lib/aspera/transfer/sync_session.schema.yaml +79 -0
- data/lib/aspera/transfer/uri.rb +6 -6
- data/lib/aspera/uri_reader.rb +1 -1
- data/lib/aspera/web_auth.rb +1 -1
- data/lib/aspera/web_server_simple.rb +53 -44
- data.tar.gz.sig +1 -2
- metadata +37 -4
- metadata.gz.sig +0 -0
- data/examples/build_package.sh +0 -28
- data/lib/aspera/transfer/spec.yaml +0 -718
data/lib/aspera/preview/utils.rb
CHANGED
@@ -16,14 +16,18 @@ module Aspera
|
|
16
16
|
# external binaries used
|
17
17
|
EXTERNAL_TOOLS = %i[ffmpeg ffprobe magick optipng unoconv].freeze
|
18
18
|
TEMP_FORMAT = 'img%04d.jpg'
|
19
|
+
FFMPEG_DEFAULT_PARAMS = [
|
20
|
+
'-y', # overwrite output without asking
|
21
|
+
'-loglevel', 'error' # show only errors and up
|
22
|
+
].freeze
|
19
23
|
private_constant :BASH_SPECIAL_CHARACTERS, :EXTERNAL_TOOLS, :TEMP_FORMAT
|
20
24
|
|
21
25
|
class << self
|
22
26
|
# returns string with single quotes suitable for bash if there is any bash meta-character
|
23
27
|
def shell_quote(argument)
|
24
|
-
return argument unless argument.chars.any?{|c|BASH_SPECIAL_CHARACTERS.include?(c)}
|
28
|
+
return argument unless argument.chars.any?{ |c| BASH_SPECIAL_CHARACTERS.include?(c)}
|
25
29
|
# surround with single quotes, and escape single quotes
|
26
|
-
return %Q{'#{argument.gsub("'"){|_s| %q{'"'"'}}}'}
|
30
|
+
return %Q{'#{argument.gsub("'"){ |_s| %q{'"'"'}}}'}
|
27
31
|
end
|
28
32
|
|
29
33
|
# check that external tools can be executed
|
@@ -53,17 +57,11 @@ module Aspera
|
|
53
57
|
return Environment.secure_capture(exec: command_sym.to_s, args: command_args.map(&:to_s))
|
54
58
|
end
|
55
59
|
|
56
|
-
def ffmpeg(
|
57
|
-
Aspera.assert_type(
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
'-loglevel', 'error' # show only errors and up
|
62
|
-
]
|
63
|
-
a[:in_p] ||= []
|
64
|
-
a[:out_p] ||= []
|
65
|
-
Aspera.assert(%i[gl_p in_f in_p out_f out_p].eql?(a.keys.sort)){"wrong params (#{a.keys.sort})"}
|
66
|
-
external_command(:ffmpeg, [a[:gl_p], a[:in_p], '-i', a[:in_f], a[:out_p], a[:out_f]].flatten)
|
60
|
+
def ffmpeg(gl_p: FFMPEG_DEFAULT_PARAMS, in_p: [], in_f:, out_p: [], out_f:)
|
61
|
+
Aspera.assert_type(gl_p, Array)
|
62
|
+
Aspera.assert_type(in_p, Array)
|
63
|
+
Aspera.assert_type(out_p, Array)
|
64
|
+
external_command(:ffmpeg, gl_p + in_p + ['-i', in_f] + out_p + [out_f])
|
67
65
|
end
|
68
66
|
|
69
67
|
# @return Float in seconds
|
@@ -13,7 +13,7 @@ module URI
|
|
13
13
|
def register_proxy_finder
|
14
14
|
Aspera.assert(block_given?)
|
15
15
|
# overload the method in URI : call user's provided block and fallback to original method
|
16
|
-
define_method(:find_proxy)
|
16
|
+
define_method(:find_proxy){ |env_vars=ENV| yield(to_s) || find_proxy_orig(env_vars)}
|
17
17
|
end
|
18
18
|
end
|
19
19
|
end
|
@@ -70,7 +70,7 @@ END_OF_JAVASCRIPT
|
|
70
70
|
end
|
71
71
|
|
72
72
|
def register_uri_generic
|
73
|
-
URI::Generic.register_proxy_finder{|url_str|get_proxies(url_str).first}
|
73
|
+
URI::Generic.register_proxy_finder{ |url_str| get_proxies(url_str).first}
|
74
74
|
# allow chaining
|
75
75
|
return self
|
76
76
|
end
|
data/lib/aspera/rest.rb
CHANGED
@@ -31,15 +31,18 @@ module Aspera
|
|
31
31
|
class RestParameters
|
32
32
|
include Singleton
|
33
33
|
|
34
|
-
attr_accessor :user_agent, :download_partial_suffix, :retry_on_error, :retry_sleep, :session_cb, :progress_bar
|
34
|
+
attr_accessor :user_agent, :download_partial_suffix, :retry_on_error, :retry_on_timeout, :retry_on_unavailable, :retry_max, :retry_sleep, :session_cb, :progress_bar
|
35
35
|
|
36
36
|
private
|
37
37
|
|
38
38
|
def initialize
|
39
39
|
@user_agent = 'RubyAsperaRest'
|
40
40
|
@download_partial_suffix = '.http_partial'
|
41
|
-
@retry_on_error =
|
42
|
-
@
|
41
|
+
@retry_on_error = false
|
42
|
+
@retry_on_timeout = true
|
43
|
+
@retry_on_unavailable = true
|
44
|
+
@retry_max = 1
|
45
|
+
@retry_sleep = 4
|
43
46
|
@session_cb = nil
|
44
47
|
@progress_bar = nil
|
45
48
|
end
|
@@ -57,8 +60,14 @@ module Aspera
|
|
57
60
|
# error message when entity not found (TODO: use specific exception)
|
58
61
|
ENTITY_NOT_FOUND = 'No such'
|
59
62
|
|
63
|
+
MIME_JSON = 'application/json'
|
64
|
+
MIME_WWW = 'application/x-www-form-urlencoded'
|
65
|
+
MIME_TEXT = 'text/plain'
|
66
|
+
|
60
67
|
# Content-Type that are JSON
|
61
|
-
JSON_DECODE = [
|
68
|
+
JSON_DECODE = [MIME_JSON, 'application/vnd.api+json', 'application/x-javascript'].freeze
|
69
|
+
|
70
|
+
UNAVAILABLE_CODES = ['503']
|
62
71
|
|
63
72
|
class << self
|
64
73
|
# @return [String] Basic auth token
|
@@ -98,7 +107,7 @@ module Aspera
|
|
98
107
|
end
|
99
108
|
end
|
100
109
|
when Array
|
101
|
-
Aspera.assert(query.all?{|i| i.is_a?(Array) && i.length.eql?(2)})
|
110
|
+
Aspera.assert(query.all?{ |i| i.is_a?(Array) && i.length.eql?(2)}){'Query must be array of arrays or 2 elements'}
|
102
111
|
query_array = query
|
103
112
|
else
|
104
113
|
raise "Query must be Hash or Array, not #{query.class}"
|
@@ -171,7 +180,7 @@ module Aspera
|
|
171
180
|
one[1] = one[1].gsub(/\A"|"\z/, '')
|
172
181
|
one
|
173
182
|
end.to_h
|
174
|
-
{
|
183
|
+
{type: type.downcase, parameters: parameters}
|
175
184
|
end
|
176
185
|
end
|
177
186
|
|
@@ -189,6 +198,7 @@ module Aspera
|
|
189
198
|
|
190
199
|
attr_reader :base_url
|
191
200
|
attr_reader :auth_params
|
201
|
+
attr_reader :headers
|
192
202
|
|
193
203
|
# @return creation parameters
|
194
204
|
def params
|
@@ -238,7 +248,7 @@ module Aspera
|
|
238
248
|
@http_session = nil
|
239
249
|
@redirect_max = redirect_max
|
240
250
|
Aspera.assert_type(@redirect_max, Integer)
|
241
|
-
@headers = headers
|
251
|
+
@headers = headers.clone
|
242
252
|
Aspera.assert_type(@headers, Hash)
|
243
253
|
@headers['User-Agent'] ||= RestParameters.instance.user_agent
|
244
254
|
# OAuth object (created on demand)
|
@@ -249,7 +259,7 @@ module Aspera
|
|
249
259
|
def oauth
|
250
260
|
if @oauth.nil?
|
251
261
|
Aspera.assert(@auth_params[:type].eql?(:oauth2)){'no OAuth defined'}
|
252
|
-
oauth_parameters = @auth_params.reject
|
262
|
+
oauth_parameters = @auth_params.reject{ |k, _v| k.eql?(:type)}
|
253
263
|
Log.log.debug{Log.dump('oauth parameters', oauth_parameters)}
|
254
264
|
@oauth = OAuth::Factory.instance.create(**oauth_parameters)
|
255
265
|
end
|
@@ -260,20 +270,20 @@ module Aspera
|
|
260
270
|
# @param operation [String] HTTP operation (GET, POST, PUT, DELETE)
|
261
271
|
# @param subpath [String] subpath of REST API
|
262
272
|
# @param query [Hash] URL parameters
|
273
|
+
# @param content_type [String,nil] Type of body parameters (one of MIME_*) and serialization, else use headers
|
263
274
|
# @param body [Hash, String] body parameters
|
264
|
-
# @param
|
275
|
+
# @param headers [Hash] additional headers (override Content-Type)
|
265
276
|
# @param save_to_file (filepath)
|
266
277
|
# @param return_error (bool)
|
267
|
-
# @param headers [Hash] additional headers
|
268
278
|
def call(
|
269
279
|
operation:,
|
270
280
|
subpath: nil,
|
271
281
|
query: nil,
|
282
|
+
content_type: nil,
|
272
283
|
body: nil,
|
273
|
-
|
284
|
+
headers: nil,
|
274
285
|
save_to_file: nil,
|
275
|
-
return_error: false
|
276
|
-
headers: nil
|
286
|
+
return_error: false
|
277
287
|
)
|
278
288
|
subpath = subpath.to_s if subpath.is_a?(Symbol)
|
279
289
|
subpath = '' if subpath.nil?
|
@@ -317,19 +327,18 @@ module Aspera
|
|
317
327
|
rescue NameError
|
318
328
|
raise "unsupported operation : #{operation}"
|
319
329
|
end
|
320
|
-
case
|
321
|
-
when
|
330
|
+
case content_type
|
331
|
+
when nil # ignore
|
332
|
+
when MIME_JSON
|
322
333
|
req.body = JSON.generate(body) # , ascii_only: true
|
323
|
-
req['Content-Type'] =
|
324
|
-
when
|
334
|
+
req['Content-Type'] = MIME_JSON
|
335
|
+
when MIME_WWW
|
325
336
|
req.body = URI.encode_www_form(body)
|
326
|
-
req['Content-Type'] =
|
327
|
-
when
|
337
|
+
req['Content-Type'] = MIME_WWW
|
338
|
+
when MIME_TEXT
|
328
339
|
req.body = body
|
329
|
-
req['Content-Type'] =
|
330
|
-
|
331
|
-
else
|
332
|
-
raise "unsupported body type : #{body_type}"
|
340
|
+
req['Content-Type'] = MIME_TEXT
|
341
|
+
else Aspera.error_unexpected_value(content_type){'body type'}
|
333
342
|
end
|
334
343
|
# set headers
|
335
344
|
headers.each do |key, value|
|
@@ -338,10 +347,8 @@ module Aspera
|
|
338
347
|
# :type = :basic
|
339
348
|
req.basic_auth(@auth_params[:username], @auth_params[:password]) if @auth_params[:type].eql?(:basic)
|
340
349
|
Log.log.trace1{Log.dump(:req_body, req.body)}
|
341
|
-
# we try the call, and will retry
|
342
|
-
|
343
|
-
timeout_tries ||= 5
|
344
|
-
general_tries ||= 1 + RestParameters.instance.retry_on_error
|
350
|
+
# we try the call, and will retry on some error types
|
351
|
+
error_tries ||= 1 + RestParameters.instance.retry_max
|
345
352
|
# initialize with number of initial retries allowed, nil gives zero
|
346
353
|
tries_remain_redirect = @redirect_max if tries_remain_redirect.nil?
|
347
354
|
Log.log.debug("send request (retries=#{tries_remain_redirect})")
|
@@ -350,7 +357,7 @@ module Aspera
|
|
350
357
|
# make http request (pipelined)
|
351
358
|
http_session.request(req) do |response|
|
352
359
|
result[:http] = response
|
353
|
-
result_mime = self.class.parse_header(result[:http]['Content-Type'] ||
|
360
|
+
result_mime = self.class.parse_header(result[:http]['Content-Type'] || MIME_TEXT)[:type]
|
354
361
|
# JSON data needs to be parsed, in case it contains an error code
|
355
362
|
if !save_to_file.nil? &&
|
356
363
|
result[:http].code.to_s.start_with?('2') &&
|
@@ -394,7 +401,7 @@ module Aspera
|
|
394
401
|
when *JSON_DECODE
|
395
402
|
result[:data] = JSON.parse(result[:http].body) rescue result[:http].body
|
396
403
|
Log.log.debug{Log.dump('result_data', result[:data])}
|
397
|
-
else # when
|
404
|
+
else # when MIME_TEXT
|
398
405
|
result[:data] = result[:http].body
|
399
406
|
end
|
400
407
|
RestErrorAnalyzer.instance.raise_on_error(req, result)
|
@@ -402,9 +409,11 @@ module Aspera
|
|
402
409
|
rescue RestCallError => e
|
403
410
|
do_retry = false
|
404
411
|
# AoC have some timeout , like Connect to platform.bss.asperasoft.com:443 ...
|
405
|
-
do_retry
|
412
|
+
do_retry ||= true if e.response.body.include?('failed: connect timed out') && RestParameters.instance.retry_on_timeout
|
413
|
+
# AoC sometimes not available
|
414
|
+
do_retry ||= true if RestParameters.instance.retry_on_unavailable && UNAVAILABLE_CODES.include?(result[:http].code.to_s)
|
406
415
|
# possibility to retry anything if it fails
|
407
|
-
do_retry
|
416
|
+
do_retry ||= true if RestParameters.instance.retry_on_error
|
408
417
|
# not authorized: oauth token expired
|
409
418
|
if @not_auth_codes.include?(result[:http].code.to_s) && @auth_params[:type].eql?(:oauth2)
|
410
419
|
begin
|
@@ -417,10 +426,10 @@ module Aspera
|
|
417
426
|
req['Authorization'] = oauth.authorization(cache: false)
|
418
427
|
end
|
419
428
|
Log.log.debug{"using new token=#{headers['Authorization']}"}
|
420
|
-
do_retry
|
429
|
+
do_retry ||= true
|
421
430
|
end
|
422
|
-
if do_retry
|
423
|
-
sleep(RestParameters.instance.retry_sleep) unless RestParameters.instance.retry_sleep.
|
431
|
+
if do_retry && (error_tries -= 1).positive?
|
432
|
+
sleep(RestParameters.instance.retry_sleep) unless RestParameters.instance.retry_sleep.eql?(0)
|
424
433
|
retry
|
425
434
|
end
|
426
435
|
# redirect ? (any code beginning with 3)
|
@@ -436,7 +445,7 @@ module Aspera
|
|
436
445
|
end
|
437
446
|
# forwards the request to the new location
|
438
447
|
return self.class.new(base_url: new_url, redirect_max: tries_remain_redirect).call(
|
439
|
-
operation: operation, query: query, body: body,
|
448
|
+
operation: operation, query: query, body: body, content_type: content_type,
|
440
449
|
save_to_file: save_to_file, return_error: return_error, headers: headers)
|
441
450
|
end
|
442
451
|
# raise exception if could not retry and not return error in result
|
@@ -452,23 +461,23 @@ module Aspera
|
|
452
461
|
#
|
453
462
|
|
454
463
|
def create(subpath, params)
|
455
|
-
return call(operation: 'POST', subpath: subpath, headers: {'Accept' =>
|
464
|
+
return call(operation: 'POST', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON)[:data]
|
456
465
|
end
|
457
466
|
|
458
467
|
def read(subpath, query=nil)
|
459
|
-
return call(operation: 'GET', subpath: subpath, headers: {'Accept' =>
|
468
|
+
return call(operation: 'GET', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: query)[:data]
|
460
469
|
end
|
461
470
|
|
462
471
|
def update(subpath, params)
|
463
|
-
return call(operation: 'PUT', subpath: subpath, headers: {'Accept' =>
|
472
|
+
return call(operation: 'PUT', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON)[:data]
|
464
473
|
end
|
465
474
|
|
466
475
|
def delete(subpath, params=nil)
|
467
|
-
return call(operation: 'DELETE', subpath: subpath, headers: {'Accept' =>
|
476
|
+
return call(operation: 'DELETE', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: params)[:data]
|
468
477
|
end
|
469
478
|
|
470
479
|
def cancel(subpath)
|
471
|
-
return call(operation: 'CANCEL', subpath: subpath, headers: {'Accept' =>
|
480
|
+
return call(operation: 'CANCEL', subpath: subpath, headers: {'Accept' => MIME_JSON})[:data]
|
472
481
|
end
|
473
482
|
|
474
483
|
# Query entity by general search (read with parameter `q`)
|
@@ -490,11 +499,11 @@ module Aspera
|
|
490
499
|
else
|
491
500
|
# multiple case insensitive partial matches, try case insensitive full match
|
492
501
|
# (anyway AoC does not allow creation of 2 entities with same case insensitive name)
|
493
|
-
name_matches = matching_items.select{|i|i['name'].casecmp?(search_name)}
|
502
|
+
name_matches = matching_items.select{ |i| i['name'].casecmp?(search_name)}
|
494
503
|
case name_matches.length
|
495
504
|
when 1 then return name_matches.first
|
496
|
-
when 0 then raise %Q(#{subpath}: multiple case insensitive partial match for: "#{search_name}": #{matching_items.map{|i|i['name']}} but no case insensitive full match. Please be more specific or give exact name.) # rubocop:disable Layout/LineLength
|
497
|
-
else raise "Two entities cannot have the same case insensitive name: #{name_matches.map{|i|i['name']}}"
|
505
|
+
when 0 then raise %Q(#{subpath}: multiple case insensitive partial match for: "#{search_name}": #{matching_items.map{ |i| i['name']}} but no case insensitive full match. Please be more specific or give exact name.) # rubocop:disable Layout/LineLength
|
506
|
+
else raise "Two entities cannot have the same case insensitive name: #{name_matches.map{ |i| i['name']}}"
|
498
507
|
end
|
499
508
|
end
|
500
509
|
end
|
@@ -35,7 +35,7 @@ module Aspera
|
|
35
35
|
d_t_s.each do |res|
|
36
36
|
r_err = res.dig(*%w[transfer_spec error]) || res['error']
|
37
37
|
next unless r_err.is_a?(Hash)
|
38
|
-
RestErrorAnalyzer.add_error(call_context, type,
|
38
|
+
RestErrorAnalyzer.add_error(call_context, type, r_err.values.join(': '))
|
39
39
|
end
|
40
40
|
end
|
41
41
|
RestErrorAnalyzer.instance.add_simple_handler(name: 'T9:IBM cloud IAM', path: ['errorMessage'])
|
data/lib/aspera/secret_hider.rb
CHANGED
@@ -15,8 +15,8 @@ module Aspera
|
|
15
15
|
# keys in hash that contain secrets
|
16
16
|
KEY_SECRETS = %w[password secret passphrase _key apikey crn token].freeze
|
17
17
|
HTTP_SECRETS = %w[Authorization].freeze
|
18
|
-
ALL_SECRETS =
|
19
|
-
ALL_SECRETS2 =
|
18
|
+
ALL_SECRETS = (ASCP_ENV_SECRETS + KEY_SECRETS + HTTP_SECRETS).freeze
|
19
|
+
ALL_SECRETS2 = (KEY_SECRETS + HTTP_SECRETS).freeze
|
20
20
|
KEY_FALSE_POSITIVES = [/^access_key$/, /^fallback_private_key$/].freeze
|
21
21
|
# regex that define named captures :begin and :end
|
22
22
|
REGEX_LOG_REPLACES = [
|
@@ -62,17 +62,17 @@ module Aspera
|
|
62
62
|
# only Strings can be secrets, not booleans, or hash, arrays
|
63
63
|
return false unless keyword.is_a?(String) && value.is_a?(String)
|
64
64
|
# those are not secrets
|
65
|
-
return false if KEY_FALSE_POSITIVES.any?{|f|f.match?(keyword)}
|
65
|
+
return false if KEY_FALSE_POSITIVES.any?{ |f| f.match?(keyword)}
|
66
66
|
return true if ADDITIONAL_KEYS_TO_HIDE.include?(keyword)
|
67
67
|
# check if keyword (name) contains an element that designate it as a secret
|
68
|
-
ALL_SECRETS.any?{|kw|keyword.include?(kw)}
|
68
|
+
ALL_SECRETS.any?{ |kw| keyword.include?(kw)}
|
69
69
|
end
|
70
70
|
|
71
71
|
# Hides recursively secrets in Hash or Array of Hash
|
72
72
|
def deep_remove_secret(obj)
|
73
73
|
case obj
|
74
74
|
when Array
|
75
|
-
obj.each{|i|deep_remove_secret(i)}
|
75
|
+
obj.each{ |i| deep_remove_secret(i)}
|
76
76
|
when Hash
|
77
77
|
obj.each do |k, v|
|
78
78
|
if secret?(k, v)
|
data/lib/aspera/ssh.rb
CHANGED
@@ -24,8 +24,8 @@ module Aspera
|
|
24
24
|
|
25
25
|
def disable_ecd_sha2_algorithms
|
26
26
|
Log.log.debug('Disabling SSH ecdsa')
|
27
|
-
Net::SSH::Transport::Algorithms::ALGORITHMS.each_value
|
28
|
-
Net::SSH::KnownHosts::SUPPORTED_TYPE.reject!
|
27
|
+
Net::SSH::Transport::Algorithms::ALGORITHMS.each_value{ |a| a.reject!{ |a| a =~ /^ecd(sa|h)-sha2/}}
|
28
|
+
Net::SSH::KnownHosts::SUPPORTED_TYPE.reject!{ |t| t =~ /^ecd(sa|h)-sha2/}
|
29
29
|
end
|
30
30
|
end
|
31
31
|
# ssh_options: same as Net::SSH.start
|
@@ -49,7 +49,7 @@ module Aspera
|
|
49
49
|
Net::SSH.start(@host, @username, @ssh_options) do |session|
|
50
50
|
ssh_channel = session.open_channel do |channel|
|
51
51
|
# prepare stdout processing
|
52
|
-
channel.on_data{|_chan, data|response.push(data)}
|
52
|
+
channel.on_data{ |_chan, data| response.push(data)}
|
53
53
|
# prepare stderr processing, stderr if type = 1
|
54
54
|
channel.on_extended_data do |_chan, _type, data|
|
55
55
|
error_message = "#{cmd}: [#{data.chomp}]"
|
@@ -58,7 +58,7 @@ module Aspera
|
|
58
58
|
raise error_message
|
59
59
|
end
|
60
60
|
# send command to SSH channel (execute) cspell: disable-next-line
|
61
|
-
channel.send('cexe'.reverse, cmd){|_ch, _success|channel.send_data(input) unless input.nil?}
|
61
|
+
channel.send('cexe'.reverse, cmd){ |_ch, _success| channel.send_data(input) unless input.nil?}
|
62
62
|
end
|
63
63
|
# wait for channel to finish (command exit)
|
64
64
|
ssh_channel.wait
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'aspera/assert'
|
4
|
+
module Aspera
|
5
|
+
module Transfer
|
6
|
+
# concertion class for transfert spec values to CLI values (ascp)
|
7
|
+
class Convert
|
8
|
+
class << self
|
9
|
+
# special encoding methods used in YAML (key: convert)
|
10
|
+
def remove_hyphen(value); value.tr('-', ''); end
|
11
|
+
|
12
|
+
# special encoding methods used in YAML (key: convert)
|
13
|
+
def json64(value); Base64.strict_encode64(JSON.generate(value)); end
|
14
|
+
|
15
|
+
# special encoding methods used in YAML (key: convert)
|
16
|
+
def base64(value); Base64.strict_encode64(value); end
|
17
|
+
|
18
|
+
# transform yes/no to true/false
|
19
|
+
def yes_to_true(value)
|
20
|
+
case value
|
21
|
+
when 'yes' then return true
|
22
|
+
when 'no' then return false
|
23
|
+
else Aspera.error_unexpected_value(value){'only: yes or no: '}
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -10,81 +10,81 @@ module Aspera
|
|
10
10
|
# rubocop:disable Layout/FirstHashElementLineBreak
|
11
11
|
ERROR_INFO = {
|
12
12
|
# id retry-able mnemo message additional info
|
13
|
-
1 => {
|
14
|
-
2 => {
|
15
|
-
3 => {
|
16
|
-
4 => {
|
17
|
-
5 => {
|
18
|
-
6 => {
|
19
|
-
7 => {
|
20
|
-
8 => {
|
21
|
-
9 => {
|
22
|
-
10 => {
|
23
|
-
11 => {
|
24
|
-
12 => {
|
13
|
+
1 => {r: false, c: 'FASP_PROTO', m: 'Generic fasp(tm) protocol error', a: 'fasp(tm) error'},
|
14
|
+
2 => {r: false, c: 'ASCP', m: 'Generic SCP error', a: 'ASCP error'},
|
15
|
+
3 => {r: false, c: 'AMBIGUOUS_TARGET', m: 'Target incorrectly specified', a: 'Ambiguous target'},
|
16
|
+
4 => {r: false, c: 'NO_SUCH_FILE', m: 'No such file or directory', a: 'No such file or directory'},
|
17
|
+
5 => {r: false, c: 'NO_PERMS', m: 'Insufficient permission to read or write', a: 'Insufficient permissions'},
|
18
|
+
6 => {r: false, c: 'NOT_DIR', m: 'Target is not a directory', a: 'Target must be a directory'},
|
19
|
+
7 => {r: false, c: 'IS_DIR', m: 'File is a directory - expected regular file', a: 'Expected regular file'},
|
20
|
+
8 => {r: false, c: 'USAGE', m: 'Incorrect usage of scp command', a: 'Incorrect usage of Aspera scp command'},
|
21
|
+
9 => {r: false, c: 'LIC_DUP', m: 'Duplicate license', a: 'Duplicate license'},
|
22
|
+
10 => {r: false, c: 'LIC_RATE_EXCEEDED', m: 'Rate exceeds the cap imposed by license', a: 'Rate exceeds cap imposed by license'},
|
23
|
+
11 => {r: false, c: 'INTERNAL_ERROR', m: 'Internal error (unexpected error)', a: 'Internal error'},
|
24
|
+
12 => {r: true, c: 'TRANSFER_ERROR', m: 'Error establishing control connection',
|
25
25
|
a: 'Error establishing SSH connection (check SSH port and firewall)'},
|
26
|
-
13 => {
|
26
|
+
13 => {r: true, c: 'TRANSFER_TIMEOUT', m: 'Timeout establishing control connection',
|
27
27
|
a: 'Timeout establishing SSH connection (check SSH port and firewall)'},
|
28
|
-
14 => {
|
28
|
+
14 => {r: true, c: 'CONNECTION_ERROR', m: 'Error establishing data connection',
|
29
29
|
a: 'Error establishing UDP connection (check UDP port and firewall)'},
|
30
|
-
15 => {
|
30
|
+
15 => {r: true, c: 'CONNECTION_TIMEOUT', m: 'Timeout establishing data connection',
|
31
31
|
a: 'Timeout establishing UDP connection (check UDP port and firewall)'},
|
32
|
-
16 => {
|
33
|
-
17 => {
|
34
|
-
18 => {
|
35
|
-
19 => {
|
36
|
-
20 => {
|
37
|
-
21 => {
|
38
|
-
22 => {
|
39
|
-
23 => {
|
40
|
-
24 => {
|
41
|
-
25 => {
|
42
|
-
26 => {
|
43
|
-
27 => {
|
44
|
-
28 => {
|
45
|
-
29 => {
|
46
|
-
30 => {
|
47
|
-
31 => {
|
48
|
-
32 => {
|
49
|
-
33 => {
|
50
|
-
34 => {
|
51
|
-
35 => {
|
52
|
-
36 => {
|
53
|
-
37 => {
|
54
|
-
38 => {
|
55
|
-
39 => {
|
56
|
-
40 => {
|
57
|
-
41 => {
|
58
|
-
42 => {
|
59
|
-
43 => {
|
60
|
-
44 => {
|
61
|
-
45 => {
|
62
|
-
46 => {
|
63
|
-
47 => {
|
32
|
+
16 => {r: true, c: 'CONNECTION_LOST', m: 'Connection lost', a: 'Connection lost'},
|
33
|
+
17 => {r: true, c: 'RCVR_SEND_ERROR', m: 'Receiver fails to send feedback', a: 'Network failure (receiver can\'t send feedback)'},
|
34
|
+
18 => {r: true, c: 'RCVR_RECV_ERROR', m: 'Receiver fails to receive data packets', a: 'Network failure (receiver can\'t receive UDP data)'},
|
35
|
+
19 => {r: false, c: 'AUTH', m: 'Authentication failure', a: 'Authentication failure'},
|
36
|
+
20 => {r: false, c: 'NOTHING', m: 'Nothing to transfer', a: 'Nothing to transfer'},
|
37
|
+
21 => {r: false, c: 'NOT_REGULAR', m: 'Not a regular file (special file)', a: 'Not a regular file'},
|
38
|
+
22 => {r: false, c: 'FILE_TABLE_OVR', m: 'File table overflow', a: 'File table overflow'},
|
39
|
+
23 => {r: true, c: 'TOO_MANY_FILES', m: 'Too many files open', a: 'Too many files open'},
|
40
|
+
24 => {r: false, c: 'FILE_TOO_BIG', m: 'File too big for file system', a: 'File too big for filesystem'},
|
41
|
+
25 => {r: false, c: 'NO_SPACE_LEFT', m: 'No space left on disk', a: 'No space left on disk'},
|
42
|
+
26 => {r: false, c: 'READ_ONLY_FS', m: 'Read only file system', a: 'Read only filesystem'},
|
43
|
+
27 => {r: false, c: 'SOME_FILE_ERRS', m: 'Some individual files failed', a: 'One or more files failed'},
|
44
|
+
28 => {r: false, c: 'USER_CANCEL', m: 'Cancelled by user', a: 'Cancelled by user'},
|
45
|
+
29 => {r: false, c: 'LIC_NOLIC', m: 'License not found or unable to access', a: 'Unable to access license info'},
|
46
|
+
30 => {r: false, c: 'LIC_EXPIRED', m: 'License expired', a: 'License expired'},
|
47
|
+
31 => {r: false, c: 'SOCK_SETUP', m: 'Unable to setup socket (create, bind, etc ...)', a: 'Unable to set up socket'},
|
48
|
+
32 => {r: true, c: 'OUT_OF_MEMORY', m: 'Out of memory, unable to allocate', a: 'Out of memory'},
|
49
|
+
33 => {r: true, c: 'THREAD_SPAWN', m: 'Can\'t spawn thread', a: 'Unable to spawn thread'},
|
50
|
+
34 => {r: false, c: 'UNAUTHORIZED', m: 'Unauthorized by external auth server', a: 'Unauthorized'},
|
51
|
+
35 => {r: true, c: 'DISK_READ', m: 'Error reading source file from disk', a: 'Disk read error'},
|
52
|
+
36 => {r: true, c: 'DISK_WRITE', m: 'Error writing to disk', a: 'Disk write error'},
|
53
|
+
37 => {r: true, c: 'AUTHORIZATION', m: 'Used interchangeably with ERR_UNAUTHORIZED', a: 'Authorization failure'},
|
54
|
+
38 => {r: false, c: 'LIC_ILLEGAL', m: 'Operation not permitted by license', a: 'Operation not permitted by license'},
|
55
|
+
39 => {r: true, c: 'PEER_ABORTED_SESSION', m: 'Remote peer terminated session', a: 'Peer aborted session'},
|
56
|
+
40 => {r: true, c: 'DATA_TRANSFER_TIMEOUT', m: 'Transfer stalled, timed out', a: 'Data transfer stalled, timed out'},
|
57
|
+
41 => {r: false, c: 'BAD_PATH', m: 'Path violates docroot containment', a: 'File location is outside \'docroot\' hierarchy'},
|
58
|
+
42 => {r: false, c: 'ALREADY_EXISTS', m: 'File or directory already exists', a: 'File or directory already exists'},
|
59
|
+
43 => {r: false, c: 'STAT_FAILS', m: 'Cannot stat file', a: 'Cannot collect details about file or directory'},
|
60
|
+
44 => {r: true, c: 'PMTU_BRTT_ERROR', m: 'UDP session initiation fatal error', a: 'UDP session initiation fatal error'},
|
61
|
+
45 => {r: true, c: 'BWMEAS_ERROR', m: 'Bandwidth measurement fatal error', a: 'Bandwidth measurement fatal error'},
|
62
|
+
46 => {r: false, c: 'VLINK_ERROR', m: 'Virtual link error', a: 'Virtual link error'},
|
63
|
+
47 => {r: false, c: 'CONNECTION_ERROR_HTTP', m: 'Error establishing HTTP connection',
|
64
64
|
a: 'Error establishing HTTP connection (check HTTP port and firewall)'},
|
65
|
-
48 => {
|
65
|
+
48 => {r: false, c: 'FILE_ENCRYPTION_ERROR', m: 'File encryption error, e.g. corrupt file',
|
66
66
|
a: 'File encryption/decryption error, e.g. corrupt file'},
|
67
|
-
49 => {
|
68
|
-
50 => {
|
69
|
-
51 => {
|
70
|
-
52 => {
|
71
|
-
53 => {
|
72
|
-
54 => {
|
73
|
-
55 => {
|
74
|
-
56 => {
|
67
|
+
49 => {r: false, c: 'FILE_DECRYPTION_PASS', m: 'File encryption/decryption error, e.g. corrupt file', a: 'File decryption error, bad passphrase'},
|
68
|
+
50 => {r: false, c: 'BAD_CONFIGURATION', m: 'Aspera.conf contains invalid data and was rejected', a: 'Invalid configuration'},
|
69
|
+
51 => {r: false, c: 'INSECURE_CONNECTION', m: 'Remote-host key check failure', a: 'Remote host is not who we expected'},
|
70
|
+
52 => {r: false, c: 'START_VALIDATION_FAILED', m: 'File start validation failed', a: 'File start validation failed'},
|
71
|
+
53 => {r: false, c: 'STOP_VALIDATION_FAILED', m: 'File stop validation failed', a: 'File stop validation failed'},
|
72
|
+
54 => {r: false, c: 'THRESHOLD_VALIDATION_FAILED', m: 'File threshold validation failed', a: 'File threshold validation failed'},
|
73
|
+
55 => {r: false, c: 'FILEPATH_TOO_LONG', m: 'File path/name too long for underlying file system', a: 'File path exceeds underlying file system limit'},
|
74
|
+
56 => {r: false, c: 'ILLEGAL_CHARS_IN_PATH', m: 'Windows path contains illegal characters',
|
75
75
|
a: 'Path being written to Windows file system contains illegal characters'},
|
76
|
-
57 => {
|
77
|
-
58 => {
|
78
|
-
59 => {
|
79
|
-
60 => {
|
80
|
-
61 => {
|
76
|
+
57 => {r: false, c: 'CHUNK_MUST_MATCH_ALIGNMENT', m: 'Chunk size/start must be aligned with storage', a: 'Chunk size/start must be aligned with storage'},
|
77
|
+
58 => {r: false, c: 'VALIDATION_SESSION_ABORT', m: 'Session aborted to due to validation error', a: 'Session aborted to due validation error'},
|
78
|
+
59 => {r: false, c: 'REMOTE_STORAGE_ERROR', m: 'Remote storage errored', a: 'Remote storage errored'},
|
79
|
+
60 => {r: false, c: 'LUA_SCRIPT_ABORTED_SESSION', m: 'Session aborted due to Lua script abort', a: 'Session aborted due to Lua script abort'},
|
80
|
+
61 => {r: true, c: 'SSEAR_RETRYABLE', m: 'Transfer failed because of a retryable Encryption at Rest error',
|
81
81
|
a: 'Transfer failed because of a retryable Encryption at Rest error'},
|
82
|
-
62 => {
|
82
|
+
62 => {r: false, c: 'SSEAR_FATAL', m: 'Transfer failed because of a fatal Encryption at Rest error',
|
83
83
|
a: 'Transfer failed because of a fatal Encryption at Rest error'},
|
84
|
-
63 => {
|
85
|
-
64 => {
|
86
|
-
65 => {
|
87
|
-
66 => {
|
84
|
+
63 => {r: false, c: 'LINK_LOOP', m: 'Path refers to a symbolic link loop', a: 'Path refers to a symbolic link loop'},
|
85
|
+
64 => {r: false, c: 'CANNOT_RENAME_PARTIAL_FILES', m: 'Can\'t rename a partial file', a: 'Can\'t rename a partial file.'},
|
86
|
+
65 => {r: false, c: 'CIPHER_NON_COMPAT_FIPS', m: 'Can\'t use this cipher with FIPS mode enabled', a: 'Can\'t use this cipher with FIPS mode enabled'},
|
87
|
+
66 => {r: false, c: 'PEER_REQUIRES_FIPS', m: 'Peer rejects cipher due to FIPS mode enabled on peer',
|
88
88
|
a: 'Peer rejects cipher due to FIPS mode enabled on peer'}
|
89
89
|
}.freeze
|
90
90
|
# rubocop:enable Layout/MultilineHashKeyLineBreaks
|