aspera-cli 4.14.0 → 4.15.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +54 -3
- data/CONTRIBUTING.md +7 -7
- data/README.md +1457 -880
- data/bin/ascli +18 -9
- data/bin/asession +12 -14
- data/examples/proxy.pac +1 -1
- data/lib/aspera/aoc.rb +198 -127
- data/lib/aspera/ascmd.rb +24 -14
- data/lib/aspera/cli/basic_auth_plugin.rb +9 -6
- data/lib/aspera/cli/error.rb +17 -0
- data/lib/aspera/cli/extended_value.rb +47 -12
- data/lib/aspera/cli/formatter.rb +260 -171
- data/lib/aspera/cli/hints.rb +80 -0
- data/lib/aspera/cli/main.rb +101 -147
- data/lib/aspera/cli/manager.rb +160 -124
- data/lib/aspera/cli/plugin.rb +70 -59
- data/lib/aspera/cli/plugins/alee.rb +0 -1
- data/lib/aspera/cli/plugins/aoc.rb +239 -273
- data/lib/aspera/cli/plugins/ats.rb +8 -5
- data/lib/aspera/cli/plugins/bss.rb +2 -2
- data/lib/aspera/cli/plugins/config.rb +516 -375
- data/lib/aspera/cli/plugins/console.rb +40 -0
- data/lib/aspera/cli/plugins/cos.rb +4 -5
- data/lib/aspera/cli/plugins/faspex.rb +99 -84
- data/lib/aspera/cli/plugins/faspex5.rb +179 -148
- data/lib/aspera/cli/plugins/node.rb +219 -153
- data/lib/aspera/cli/plugins/orchestrator.rb +52 -17
- data/lib/aspera/cli/plugins/preview.rb +46 -32
- data/lib/aspera/cli/plugins/server.rb +57 -17
- data/lib/aspera/cli/plugins/shares.rb +34 -12
- data/lib/aspera/cli/sync_actions.rb +68 -0
- data/lib/aspera/cli/transfer_agent.rb +45 -55
- data/lib/aspera/cli/transfer_progress.rb +74 -0
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +3 -1
- data/lib/aspera/command_line_builder.rb +14 -11
- data/lib/aspera/cos_node.rb +3 -2
- data/lib/aspera/environment.rb +17 -6
- data/lib/aspera/fasp/agent_aspera.rb +126 -0
- data/lib/aspera/fasp/agent_base.rb +31 -77
- data/lib/aspera/fasp/agent_connect.rb +21 -22
- data/lib/aspera/fasp/agent_direct.rb +88 -102
- data/lib/aspera/fasp/agent_httpgw.rb +196 -192
- data/lib/aspera/fasp/agent_node.rb +41 -34
- data/lib/aspera/fasp/agent_trsdk.rb +75 -34
- data/lib/aspera/fasp/error_info.rb +2 -2
- data/lib/aspera/fasp/faux_file.rb +52 -0
- data/lib/aspera/fasp/installation.rb +43 -184
- data/lib/aspera/fasp/management.rb +244 -0
- data/lib/aspera/fasp/parameters.rb +59 -26
- data/lib/aspera/fasp/parameters.yaml +75 -8
- data/lib/aspera/fasp/products.rb +162 -0
- data/lib/aspera/fasp/transfer_spec.rb +1 -1
- data/lib/aspera/fasp/uri.rb +4 -4
- data/lib/aspera/faspex_gw.rb +2 -2
- data/lib/aspera/faspex_postproc.rb +2 -2
- data/lib/aspera/hash_ext.rb +2 -2
- data/lib/aspera/json_rpc.rb +49 -0
- data/lib/aspera/line_logger.rb +23 -0
- data/lib/aspera/log.rb +57 -16
- data/lib/aspera/node.rb +97 -14
- data/lib/aspera/oauth.rb +36 -18
- data/lib/aspera/open_application.rb +4 -4
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/file_types.rb +4 -2
- data/lib/aspera/preview/generator.rb +22 -35
- data/lib/aspera/preview/options.rb +2 -0
- data/lib/aspera/preview/terminal.rb +24 -13
- data/lib/aspera/preview/utils.rb +19 -26
- data/lib/aspera/rest.rb +103 -72
- data/lib/aspera/rest_call_error.rb +1 -1
- data/lib/aspera/rest_error_analyzer.rb +15 -14
- data/lib/aspera/rest_errors_aspera.rb +37 -34
- data/lib/aspera/secret_hider.rb +14 -16
- data/lib/aspera/ssh.rb +4 -1
- data/lib/aspera/sync.rb +128 -122
- data/lib/aspera/temp_file_manager.rb +10 -3
- data/lib/aspera/web_auth.rb +10 -7
- data/lib/aspera/web_server_simple.rb +9 -4
- data.tar.gz.sig +0 -0
- metadata +33 -15
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/listener/line_dump.rb +0 -19
- data/lib/aspera/cli/listener/logger.rb +0 -22
- data/lib/aspera/cli/listener/progress.rb +0 -50
- data/lib/aspera/cli/listener/progress_multi.rb +0 -84
- data/lib/aspera/cli/plugins/sync.rb +0 -44
- data/lib/aspera/fasp/listener.rb +0 -13
data/lib/aspera/rest.rb
CHANGED
@@ -10,7 +10,6 @@ require 'net/https'
|
|
10
10
|
require 'json'
|
11
11
|
require 'base64'
|
12
12
|
require 'cgi'
|
13
|
-
require 'ruby-progressbar'
|
14
13
|
|
15
14
|
# add cancel method to http
|
16
15
|
class Net::HTTP::Cancel < Net::HTTPRequest # rubocop:disable Style/ClassAndModuleChildren
|
@@ -26,34 +25,25 @@ module Aspera
|
|
26
25
|
class Rest
|
27
26
|
# global settings also valid for any subclass
|
28
27
|
@@global = { # rubocop:disable Style/ClassVars
|
29
|
-
|
30
|
-
#
|
31
|
-
|
32
|
-
|
33
|
-
# a lambda which takes the Net::HTTP as arg, use this to change parameters
|
34
|
-
session_cb: nil,
|
35
|
-
proxy_user: nil,
|
36
|
-
proxy_pass: nil
|
28
|
+
user_agent: 'Ruby', # goes to HTTP request header: 'User-Agent'
|
29
|
+
download_partial_suffix: '.http_partial', # suffix for partial download
|
30
|
+
session_cb: nil, # a lambda which takes the Net::HTTP as arg, use this to change parameters
|
31
|
+
progress_bar: nil # progress bar object
|
37
32
|
}
|
38
33
|
|
34
|
+
# flag for array parameters prefixed with []
|
39
35
|
ARRAY_PARAMS = '[]'
|
40
36
|
|
41
37
|
private_constant :ARRAY_PARAMS
|
42
38
|
|
43
|
-
# error message when entity not found
|
39
|
+
# error message when entity not found (TODO: use specific exception)
|
44
40
|
ENTITY_NOT_FOUND = 'No such'
|
45
41
|
|
46
|
-
|
47
|
-
|
48
|
-
@@global.each_key do |p|
|
49
|
-
define_method(p){@@global[p]}
|
50
|
-
define_method("#{p}=") do |val|
|
51
|
-
Log.log.debug{"#{p} => #{val}".red}
|
52
|
-
@@global[p] = val
|
53
|
-
end
|
54
|
-
end
|
42
|
+
# Content-Type that are JSON
|
43
|
+
JSON_DECODE = ['application/json', 'application/vnd.api+json', 'application/x-javascript'].freeze
|
55
44
|
|
56
|
-
|
45
|
+
class << self
|
46
|
+
def basic_token(user, pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}"; end
|
57
47
|
|
58
48
|
# used to build a parameter list prefixed with "[]"
|
59
49
|
# @param values [Array] list of values
|
@@ -61,47 +51,77 @@ module Aspera
|
|
61
51
|
return [ARRAY_PARAMS].concat(values)
|
62
52
|
end
|
63
53
|
|
54
|
+
def array_params?(values)
|
55
|
+
return values.first.eql?(ARRAY_PARAMS)
|
56
|
+
end
|
57
|
+
|
64
58
|
# build URI from URL and parameters and check it is http or https
|
65
59
|
def build_uri(url, params=nil)
|
66
60
|
uri = URI.parse(url)
|
67
61
|
raise "REST endpoint shall be http/s not #{uri.scheme}" unless %w[http https].include?(uri.scheme)
|
68
|
-
if
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
end
|
80
|
-
else
|
81
|
-
params.push([k, v])
|
82
|
-
end
|
62
|
+
return uri if params.nil?
|
63
|
+
Log.log.debug{Log.dump('params', params)}
|
64
|
+
raise 'Internal Error: param must be Hash' unless params.is_a?(Hash)
|
65
|
+
query = []
|
66
|
+
params.each do |k, v|
|
67
|
+
case v
|
68
|
+
when Array
|
69
|
+
# support array url params, there is no standard. Either p[]=1&p[]=2, or p=1&p=2
|
70
|
+
suffix = array_params?(v) ? v.shift : ''
|
71
|
+
v.each do |e|
|
72
|
+
query.push(["#{k}#{suffix}", e])
|
83
73
|
end
|
74
|
+
else
|
75
|
+
query.push([k, v])
|
84
76
|
end
|
85
|
-
# CGI.unescape to transform back %5D into []
|
86
|
-
uri.query = CGI.unescape(URI.encode_www_form(params))
|
87
77
|
end
|
78
|
+
# [] is allowed in url parameters
|
79
|
+
uri.query = URI.encode_www_form(query).gsub('%5B%5D=', '[]=')
|
88
80
|
return uri
|
89
81
|
end
|
90
82
|
|
83
|
+
def decode_query(query)
|
84
|
+
URI.decode_www_form(query).each_with_object({}){|v, h|h[v.first] = v.last }
|
85
|
+
end
|
86
|
+
|
87
|
+
# start a HTTP/S session, also used for web sockets
|
88
|
+
# @param base_url [String] base url of HTTP/S session
|
89
|
+
# @return [Net::HTTP] a started HTTP session
|
91
90
|
def start_http_session(base_url)
|
92
91
|
uri = build_uri(base_url)
|
93
92
|
# this honors http_proxy env var
|
94
93
|
http_session = Net::HTTP.new(uri.host, uri.port)
|
95
|
-
http_session.proxy_user = proxy_user
|
96
|
-
http_session.proxy_pass = proxy_pass
|
97
94
|
http_session.use_ssl = uri.scheme.eql?('https')
|
98
|
-
http_session.set_debug_output($stdout) if debug
|
99
95
|
# set http options in callback, such as timeout and cert. verification
|
100
|
-
session_cb&.call(http_session)
|
96
|
+
@@global[:session_cb]&.call(http_session)
|
101
97
|
# manually start session for keep alive (if supported by server, else, session is closed every time)
|
102
98
|
http_session.start
|
103
99
|
return http_session
|
104
100
|
end
|
101
|
+
|
102
|
+
# get Net::HTTP underlying socket i/o
|
103
|
+
# little hack, handy because HTTP debug, proxy, etc... will be available
|
104
|
+
# used implement web sockets after `start_http_session`
|
105
|
+
def io_http_session(http_session)
|
106
|
+
raise "wring type #{http_session.class}" unless http_session.is_a?(Net::HTTP)
|
107
|
+
# Net::BufferedIO in net/protocol.rb
|
108
|
+
result = http_session.instance_variable_get(:@socket)
|
109
|
+
raise "no socket for #{http_session}" if result.nil?
|
110
|
+
return result
|
111
|
+
end
|
112
|
+
|
113
|
+
# set global parameters
|
114
|
+
def set_parameters(**options)
|
115
|
+
options.each do |key, value|
|
116
|
+
raise "ERROR: unknown Rest option #{key}" unless @@global.key?(key)
|
117
|
+
@@global[key] = value
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
# @return [String] HTTP agent name
|
122
|
+
def user_agent
|
123
|
+
return @@global[:user_agent]
|
124
|
+
end
|
105
125
|
end
|
106
126
|
|
107
127
|
private
|
@@ -131,7 +151,7 @@ module Aspera
|
|
131
151
|
raise 'ERROR: expecting Hash' unless a_rest_params.is_a?(Hash)
|
132
152
|
raise 'ERROR: expecting base_url' unless a_rest_params[:base_url].is_a?(String)
|
133
153
|
@params = a_rest_params.clone
|
134
|
-
Log.dump('REST params', @params)
|
154
|
+
Log.log.debug{Log.dump('REST params', @params)}
|
135
155
|
# base url without trailing slashes (note: string may be frozen)
|
136
156
|
@params[:base_url] = @params[:base_url].gsub(%r{/+$}, '')
|
137
157
|
@http_session = nil
|
@@ -139,7 +159,7 @@ module Aspera
|
|
139
159
|
@params[:auth] ||= {type: :none}
|
140
160
|
@params[:not_auth_codes] ||= ['401']
|
141
161
|
@oauth = nil
|
142
|
-
Log.dump('REST params(2)', @params)
|
162
|
+
Log.log.debug{Log.dump('REST params(2)', @params)}
|
143
163
|
end
|
144
164
|
|
145
165
|
def oauth_token(force_refresh: false)
|
@@ -159,8 +179,8 @@ module Aspera
|
|
159
179
|
raise "unsupported operation : #{call_data[:operation]}"
|
160
180
|
end
|
161
181
|
if call_data.key?(:json_params) && !call_data[:json_params].nil?
|
162
|
-
req.body = JSON.generate(call_data[:json_params])
|
163
|
-
Log.dump('body JSON data', call_data[:json_params])
|
182
|
+
req.body = JSON.generate(call_data[:json_params]) # , ascii_only: true
|
183
|
+
Log.log.debug{Log.dump('body JSON data', call_data[:json_params])}
|
164
184
|
req['Content-Type'] = 'application/json'
|
165
185
|
# call_data[:headers]['Accept']='application/json'
|
166
186
|
end
|
@@ -181,6 +201,7 @@ module Aspera
|
|
181
201
|
end
|
182
202
|
# :type = :basic
|
183
203
|
req.basic_auth(call_data[:auth][:username], call_data[:auth][:password]) if call_data[:auth][:type].eql?(:basic)
|
204
|
+
Log.log.debug{Log.dump(:req_body, req.body)}
|
184
205
|
return req
|
185
206
|
end
|
186
207
|
|
@@ -203,14 +224,14 @@ module Aspera
|
|
203
224
|
# :type (:none, :basic, :oauth2, :url)
|
204
225
|
# :username [:basic]
|
205
226
|
# :password [:basic]
|
206
|
-
# :
|
227
|
+
# :url_query [:url] a hash
|
207
228
|
# :* [:oauth2] see Oauth class
|
208
229
|
def call(call_data)
|
209
230
|
raise "Hash call parameter is required (#{call_data.class})" unless call_data.is_a?(Hash)
|
210
231
|
call_data[:subpath] = '' if call_data[:subpath].nil?
|
211
232
|
Log.log.debug{"accessing #{call_data[:subpath]}".red.bold.bg_green}
|
212
233
|
call_data[:headers] ||= {}
|
213
|
-
call_data[:headers]['User-Agent'] ||=
|
234
|
+
call_data[:headers]['User-Agent'] ||= @@global[:user_agent]
|
214
235
|
# defaults from @params are overridden by call data
|
215
236
|
call_data = @params.deep_merge(call_data)
|
216
237
|
case call_data[:auth][:type]
|
@@ -223,7 +244,7 @@ module Aspera
|
|
223
244
|
call_data[:headers]['Authorization'] = oauth_token unless call_data[:headers].key?('Authorization')
|
224
245
|
when :url
|
225
246
|
call_data[:url_params] ||= {}
|
226
|
-
call_data[:auth][:
|
247
|
+
call_data[:auth][:url_query].each do |key, value|
|
227
248
|
call_data[:url_params][key] = value
|
228
249
|
end
|
229
250
|
else raise "unsupported auth type: [#{call_data[:auth][:type]}]"
|
@@ -238,16 +259,18 @@ module Aspera
|
|
238
259
|
# initialize with number of initial retries allowed, nil gives zero
|
239
260
|
tries_remain_redirect = call_data[:redirect_max].to_i if tries_remain_redirect.nil?
|
240
261
|
Log.log.debug("send request (retries=#{tries_remain_redirect})")
|
262
|
+
result_mime = nil
|
263
|
+
file_saved = false
|
241
264
|
# make http request (pipelined)
|
242
265
|
http_session.request(req) do |response|
|
243
266
|
result[:http] = response
|
244
|
-
|
267
|
+
result_mime = (result[:http]['Content-Type'] || 'text/plain').split(';').first.downcase
|
268
|
+
# JSON data needs to be parsed, in case it contains an error code
|
269
|
+
if !call_data[:save_to_file].nil? &&
|
270
|
+
result[:http].code.to_s.start_with?('2') &&
|
271
|
+
!result[:http]['Content-Length'].nil? &&
|
272
|
+
!JSON_DECODE.include?(result_mime)
|
245
273
|
total_size = result[:http]['Content-Length'].to_i
|
246
|
-
progress = ProgressBar.create(
|
247
|
-
format: '%a %B %p%% %r KB/sec %e',
|
248
|
-
rate_scale: lambda{|rate|rate / 1024},
|
249
|
-
title: 'progress',
|
250
|
-
total: total_size)
|
251
274
|
Log.log.debug('before write file')
|
252
275
|
target_file = call_data[:save_to_file]
|
253
276
|
# override user's path to path in header
|
@@ -255,34 +278,37 @@ module Aspera
|
|
255
278
|
target_file = File.join(File.dirname(target_file), m[1])
|
256
279
|
end
|
257
280
|
# download with temp filename
|
258
|
-
target_file_tmp = "#{target_file}#{
|
281
|
+
target_file_tmp = "#{target_file}#{@@global[:download_partial_suffix]}"
|
259
282
|
Log.log.debug{"saving to: #{target_file}"}
|
283
|
+
written_size = 0
|
284
|
+
@@global[:progress_bar]&.event(session_id: 1, type: :session_start)
|
285
|
+
@@global[:progress_bar]&.event(session_id: 1, type: :session_size, info: total_size)
|
260
286
|
File.open(target_file_tmp, 'wb') do |file|
|
261
287
|
result[:http].read_body do |fragment|
|
262
288
|
file.write(fragment)
|
263
|
-
|
264
|
-
|
265
|
-
progress.progress = new_process
|
289
|
+
written_size += fragment.length
|
290
|
+
@@global[:progress_bar]&.event(session_id: 1, type: :transfer, info: written_size)
|
266
291
|
end
|
267
292
|
end
|
293
|
+
@@global[:progress_bar]&.event(session_id: 1, type: :end)
|
268
294
|
# rename at the end
|
269
295
|
File.rename(target_file_tmp, target_file)
|
270
|
-
|
296
|
+
file_saved = true
|
271
297
|
end # save_to_file
|
272
298
|
end
|
273
|
-
# sometimes there is a UTF8 char (e.g. (c) )
|
274
|
-
result[:http].body.force_encoding('UTF-8') if result[:http].body.is_a?(String)
|
275
|
-
Log.log.debug{"result: body=#{result[:http].body}"}
|
276
|
-
result_mime = (result[:http]['Content-Type'] || 'text/plain').split(';').first
|
299
|
+
# sometimes there is a UTF8 char (e.g. (c) ), TODO : related to mime type encoding ?
|
300
|
+
# result[:http].body.force_encoding('UTF-8') if result[:http].body.is_a?(String)
|
301
|
+
# Log.log.debug{"result: body=#{result[:http].body}"}
|
277
302
|
result[:data] = case result_mime
|
278
|
-
when
|
279
|
-
JSON.parse(result[:http].body) rescue
|
303
|
+
when *JSON_DECODE
|
304
|
+
JSON.parse(result[:http].body) rescue result[:http].body
|
280
305
|
else # when 'text/plain'
|
281
306
|
result[:http].body
|
282
307
|
end
|
283
|
-
Log.dump("result: parsed: #{result_mime}", result[:data])
|
308
|
+
Log.log.debug{Log.dump("result: parsed: #{result_mime}", result[:data])}
|
284
309
|
Log.log.debug{"result: code=#{result[:http].code}"}
|
285
310
|
RestErrorAnalyzer.instance.raise_on_error(req, result)
|
311
|
+
File.write(call_data[:save_to_file], result[:http].body) unless file_saved || call_data[:save_to_file].nil?
|
286
312
|
rescue RestCallError => e
|
287
313
|
# not authorized: oauth token expired
|
288
314
|
if call_data[:not_auth_codes].include?(result[:http].code.to_s) && call_data[:auth][:type].eql?(:oauth2)
|
@@ -303,7 +329,12 @@ module Aspera
|
|
303
329
|
tries_remain_redirect -= 1
|
304
330
|
current_uri = URI.parse(call_data[:base_url])
|
305
331
|
new_url = e.response['location']
|
306
|
-
|
332
|
+
# special case: relative redirect
|
333
|
+
if URI.parse(new_url).host.nil?
|
334
|
+
# we don't manage relative redirects with non-absolute path
|
335
|
+
raise "Error: redirect location is relative: #{new_url}, but does not start with /." unless new_url.start_with?('/')
|
336
|
+
new_url = current_uri.scheme + '://' + current_uri.host + new_url
|
337
|
+
end
|
307
338
|
Log.log.info{"URL is moved: #{new_url}"}
|
308
339
|
redirection_uri = URI.parse(new_url)
|
309
340
|
call_data[:base_url] = new_url
|
@@ -333,16 +364,16 @@ module Aspera
|
|
333
364
|
return call({operation: 'POST', subpath: subpath, headers: {'Accept' => 'application/json'}, encoding => params})
|
334
365
|
end
|
335
366
|
|
336
|
-
def read(subpath,
|
337
|
-
return call({operation: 'GET', subpath: subpath, headers: {'Accept' => 'application/json'}, url_params:
|
367
|
+
def read(subpath, options=nil)
|
368
|
+
return call({operation: 'GET', subpath: subpath, headers: {'Accept' => 'application/json'}, url_params: options})
|
338
369
|
end
|
339
370
|
|
340
371
|
def update(subpath, params)
|
341
372
|
return call({operation: 'PUT', subpath: subpath, headers: {'Accept' => 'application/json'}, json_params: params})
|
342
373
|
end
|
343
374
|
|
344
|
-
def delete(subpath,
|
345
|
-
return call({operation: 'DELETE', subpath: subpath, headers: {'Accept' => 'application/json'}, url_params:
|
375
|
+
def delete(subpath, params=nil)
|
376
|
+
return call({operation: 'DELETE', subpath: subpath, headers: {'Accept' => 'application/json'}, url_params: params})
|
346
377
|
end
|
347
378
|
|
348
379
|
def cancel(subpath)
|
@@ -355,7 +386,7 @@ module Aspera
|
|
355
386
|
# @param options additional search options
|
356
387
|
def lookup_by_name(subpath, search_name, options={})
|
357
388
|
# returns entities whose name contains value (case insensitive)
|
358
|
-
matching_items = read(subpath, options.merge({'q' =>
|
389
|
+
matching_items = read(subpath, options.merge({'q' => search_name}))[:data]
|
359
390
|
# API style: {totalcount:, ...} cspell: disable-line
|
360
391
|
# TODO: not generic enough ? move somewhere ? inheritance ?
|
361
392
|
matching_items = matching_items[subpath] if matching_items.is_a?(Hash)
|
@@ -27,6 +27,7 @@ module Aspera
|
|
27
27
|
# Analyzes REST call response and raises a RestCallError exception
|
28
28
|
# if HTTP result code is not 2XX
|
29
29
|
def raise_on_error(req, res)
|
30
|
+
Log.log.debug{"raise_on_error #{req.method} #{req.path} #{res[:http].code}"}
|
30
31
|
call_context = {
|
31
32
|
messages: [],
|
32
33
|
request: req,
|
@@ -44,7 +45,7 @@ module Aspera
|
|
44
45
|
Log.log.error{"ERROR in handler:\n#{e.message}\n#{e.backtrace}"}
|
45
46
|
end
|
46
47
|
end
|
47
|
-
raise RestCallError.new(call_context[:
|
48
|
+
raise RestCallError.new(call_context[:messages].join("\n"), call_context[:request], call_context[:response]) unless call_context[:messages].empty?
|
48
49
|
end
|
49
50
|
|
50
51
|
# add a new error handler (done at application initialization)
|
@@ -59,21 +60,21 @@ module Aspera
|
|
59
60
|
# add a simple error handler
|
60
61
|
# check that key exists and is string under specified path (hash)
|
61
62
|
# adds other keys as secondary information
|
62
|
-
|
63
|
+
# @param name [String] name of error handler (for logs)
|
64
|
+
# @param always [boolean] if true, always add error message, even if response code is 2XX
|
65
|
+
# @param path [Array] path to error message in response
|
66
|
+
def add_simple_handler(name:, always: false, path:)
|
67
|
+
path.freeze
|
63
68
|
add_handler(name) do |type, call_context|
|
64
|
-
# need to clone because we modify and same array is used subsequently
|
65
|
-
path = args.clone
|
66
|
-
# Log.log.debug{"path=#{path}"}
|
67
|
-
# if last in path is boolean it tells if the error is only with http error code or always
|
68
|
-
always = [true, false].include?(path.last) ? path.pop : false
|
69
69
|
if call_context[:data].is_a?(Hash) && (!call_context[:response].code.start_with?('2') || always)
|
70
|
-
|
71
|
-
# dig and find
|
72
|
-
error_struct = path.
|
73
|
-
|
74
|
-
|
70
|
+
# Log.log.debug{"simple_handler: #{type} #{path} #{path.last}"}
|
71
|
+
# dig and find hash containing error message
|
72
|
+
error_struct = path.length.eql?(1) ? call_context[:data] : call_context[:data].dig(*path[0..-2])
|
73
|
+
# Log.log.debug{"found: #{error_struct.class} #{error_struct}"}
|
74
|
+
if error_struct.is_a?(Hash) && error_struct[path.last].is_a?(String)
|
75
|
+
RestErrorAnalyzer.add_error(call_context, type, error_struct[path.last])
|
75
76
|
error_struct.each do |k, v|
|
76
|
-
next if k.eql?(
|
77
|
+
next if k.eql?(path.last)
|
77
78
|
RestErrorAnalyzer.add_error(call_context, "#{type}(sub)", "#{k}: #{v}") if [String, Integer].include?(v.class)
|
78
79
|
end
|
79
80
|
end
|
@@ -93,7 +94,7 @@ module Aspera
|
|
93
94
|
# log error for further analysis (file must exist to activate)
|
94
95
|
return if log_file.nil? || !File.exist?(log_file)
|
95
96
|
File.open(log_file, 'a+') do |f|
|
96
|
-
f.write("\n=#{type}=====\n#{call_context[:request].method} #{call_context[:request].path}\n#{call_context[:response].code}\n"\
|
97
|
+
f.write("\n=#{type}=====\n#{call_context[:request].method} #{call_context[:request].path}\n#{call_context[:response].code}\n" \
|
97
98
|
"#{JSON.generate(call_context[:data])}\n#{call_context[:messages].join("\n")}")
|
98
99
|
end
|
99
100
|
end
|
@@ -12,48 +12,51 @@ module Aspera
|
|
12
12
|
Log.log.debug('registering Aspera REST error handlers')
|
13
13
|
# Faspex 4: both user_message and internal_message, and code 200
|
14
14
|
# example: missing meta data on package creation
|
15
|
-
RestErrorAnalyzer.instance.add_simple_handler('Type 1: error:user_message',
|
16
|
-
RestErrorAnalyzer.instance.add_simple_handler('Type 2: error:description',
|
17
|
-
RestErrorAnalyzer.instance.add_simple_handler('Type 3: error:internal_message',
|
15
|
+
RestErrorAnalyzer.instance.add_simple_handler(name: 'Type 1: error:user_message', path: %w[error user_message], always: true)
|
16
|
+
RestErrorAnalyzer.instance.add_simple_handler(name: 'Type 2: error:description', path: %w[error description])
|
17
|
+
RestErrorAnalyzer.instance.add_simple_handler(name: 'Type 3: error:internal_message', path: %w[error internal_message])
|
18
18
|
# AoC Automation
|
19
|
-
RestErrorAnalyzer.instance.add_simple_handler('AoC Automation', 'error')
|
20
|
-
RestErrorAnalyzer.instance.add_simple_handler('Type 5', 'error_description')
|
21
|
-
RestErrorAnalyzer.instance.add_simple_handler('Type 6', 'message')
|
22
|
-
RestErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
end
|
19
|
+
RestErrorAnalyzer.instance.add_simple_handler(name: 'AoC Automation', path: ['error'])
|
20
|
+
RestErrorAnalyzer.instance.add_simple_handler(name: 'Type 5', path: ['error_description'])
|
21
|
+
RestErrorAnalyzer.instance.add_simple_handler(name: 'Type 6', path: ['message'])
|
22
|
+
RestErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |type, call_context|
|
23
|
+
next unless call_context[:data].is_a?(Hash) && call_context[:data]['errors'].is_a?(Hash)
|
24
|
+
call_context[:data]['errors'].each do |k, v|
|
25
|
+
RestErrorAnalyzer.add_error(call_context, type, "#{k}: #{v}")
|
27
26
|
end
|
28
27
|
end
|
29
28
|
# call to upload_setup and download_setup of node api
|
30
29
|
RestErrorAnalyzer.instance.add_handler('T8:node: *_setup') do |type, call_context|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
RestErrorAnalyzer.add_error(call_context, type, "#{r_err['code']}: #{r_err['reason']}: #{r_err['user_message']}")
|
39
|
-
end
|
40
|
-
end
|
41
|
-
end
|
30
|
+
next unless call_context[:data].is_a?(Hash)
|
31
|
+
d_t_s = call_context[:data]['transfer_specs']
|
32
|
+
next unless d_t_s.is_a?(Array)
|
33
|
+
d_t_s.each do |res|
|
34
|
+
r_err = res.dig(*%w[transfer_spec error])
|
35
|
+
next unless r_err.is_a?(Hash)
|
36
|
+
RestErrorAnalyzer.add_error(call_context, type, "#{r_err['code']}: #{r_err['reason']}: #{r_err['user_message']}")
|
42
37
|
end
|
43
38
|
end
|
44
|
-
RestErrorAnalyzer.instance.add_simple_handler('T9:IBM cloud IAM', 'errorMessage')
|
45
|
-
RestErrorAnalyzer.instance.add_simple_handler('T10:faspex v4', 'user_message')
|
39
|
+
RestErrorAnalyzer.instance.add_simple_handler(name: 'T9:IBM cloud IAM', path: ['errorMessage'])
|
40
|
+
RestErrorAnalyzer.instance.add_simple_handler(name: 'T10:faspex v4', path: ['user_message'])
|
46
41
|
RestErrorAnalyzer.instance.add_handler('bss graphql') do |type, call_context|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
42
|
+
next unless call_context[:data].is_a?(Hash)
|
43
|
+
d_t_s = call_context[:data]['errors']
|
44
|
+
next unless d_t_s.is_a?(Array)
|
45
|
+
d_t_s.each do |res|
|
46
|
+
r_err = res['message']
|
47
|
+
next unless r_err.is_a?(String)
|
48
|
+
RestErrorAnalyzer.add_error(call_context, type, r_err)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
RestErrorAnalyzer.instance.add_handler('Orchestrator') do |type, call_context|
|
52
|
+
next if call_context[:response].code.start_with?('2')
|
53
|
+
data = call_context[:data]
|
54
|
+
next unless data.is_a?(Hash)
|
55
|
+
work_order = data['work_order']
|
56
|
+
next unless work_order.is_a?(Hash)
|
57
|
+
RestErrorAnalyzer.add_error(call_context, type, work_order['statusDetails'])
|
58
|
+
data['missing_parameters']&.each do |param|
|
59
|
+
RestErrorAnalyzer.add_error(call_context, type, "missing parameter: #{param}")
|
57
60
|
end
|
58
61
|
end
|
59
62
|
end # register_handlers
|
data/lib/aspera/secret_hider.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
# cspell:ignore FILEPASS
|
3
4
|
require 'logger'
|
4
5
|
|
5
6
|
module Aspera
|
@@ -11,22 +12,25 @@ module Aspera
|
|
11
12
|
ASCP_ENV_SECRETS = %w[ASPERA_SCP_PASS ASPERA_SCP_KEY ASPERA_SCP_FILEPASS ASPERA_PROXY_PASS ASPERA_SCP_TOKEN].freeze
|
12
13
|
# keys in hash that contain secrets
|
13
14
|
KEY_SECRETS = %w[password secret passphrase _key apikey crn token].freeze
|
14
|
-
|
15
|
-
|
15
|
+
HTTP_SECRETS = %w[Authorization].freeze
|
16
|
+
ALL_SECRETS = [ASCP_ENV_SECRETS, KEY_SECRETS, HTTP_SECRETS].flatten.freeze
|
17
|
+
KEY_FALSE_POSITIVES = [/^access_key$/].freeze
|
16
18
|
# regex that define named captures :begin and :end
|
17
19
|
REGEX_LOG_REPLACES = [
|
18
20
|
# CLI manager get/set options
|
19
|
-
/(?<begin>[sg]et (
|
21
|
+
/(?<begin>[sg]et (?:#{KEY_SECRETS.join('|')})=).*(?<end>)/,
|
20
22
|
# env var ascp exec
|
21
|
-
/(?<begin> (
|
22
|
-
# rendered JSON
|
23
|
-
/(?<begin>["'
|
23
|
+
/(?<begin> (?:#{ASCP_ENV_SECRETS.join('|')})=)(\\.|[^ ])*(?<end> )/,
|
24
|
+
# rendered JSON or Ruby
|
25
|
+
/(?<begin>(?:(?<quote>["'])|:)[^"':=]*(?:#{ALL_SECRETS.join('|')})[^"':=]*\k<quote>?(?:=>|:) *")[^"]+(?<end>")/,
|
24
26
|
# option "secret"
|
25
27
|
/(?<begin>"[^"]*(secret)[^"]*"=>{)[^}]+(?<end>})/,
|
26
28
|
# option "secrets"
|
27
29
|
/(?<begin>(secrets)={)[^}]+(?<end>})/,
|
28
30
|
# private key values
|
29
|
-
/(?<begin>--+BEGIN
|
31
|
+
/(?<begin>--+BEGIN [^-]+ KEY--+)[[:ascii:]]+?(?<end>--+?END [^-]+ KEY--+)/,
|
32
|
+
# cred in http dump
|
33
|
+
/(?<begin>(?:#{HTTP_SECRETS.join('|')}): )[^\\]+(?<end>\\)/i
|
30
34
|
].freeze
|
31
35
|
private_constant :HIDDEN_PASSWORD, :ASCP_ENV_SECRETS, :KEY_SECRETS, :ALL_SECRETS, :REGEX_LOG_REPLACES
|
32
36
|
@log_secrets = false
|
@@ -51,21 +55,15 @@ module Aspera
|
|
51
55
|
# only Strings can be secrets, not booleans, or hash, arrays
|
52
56
|
return false unless keyword.is_a?(String) && value.is_a?(String)
|
53
57
|
# those are not secrets
|
54
|
-
return false if
|
58
|
+
return false if KEY_FALSE_POSITIVES.any?{|f|f.match?(keyword)}
|
55
59
|
# check if keyword (name) contains an element that designate it as a secret
|
56
60
|
ALL_SECRETS.any?{|kw|keyword.include?(kw)}
|
57
61
|
end
|
58
62
|
|
59
|
-
def deep_remove_secret(obj
|
63
|
+
def deep_remove_secret(obj)
|
60
64
|
case obj
|
61
65
|
when Array
|
62
|
-
|
63
|
-
obj.each do |i|
|
64
|
-
i['value'] = HIDDEN_PASSWORD if secret?(i['parameter'], i['value'])
|
65
|
-
end
|
66
|
-
else
|
67
|
-
obj.each{|i|deep_remove_secret(i)}
|
68
|
-
end
|
66
|
+
obj.each{|i|deep_remove_secret(i)}
|
69
67
|
when Hash
|
70
68
|
obj.each do |k, v|
|
71
69
|
if secret?(k, v)
|
data/lib/aspera/ssh.rb
CHANGED
@@ -2,12 +2,15 @@
|
|
2
2
|
|
3
3
|
require 'net/ssh'
|
4
4
|
|
5
|
-
# HACK: deactivate ed25519 and ecdsa private keys from ssh identities, as it usually
|
5
|
+
# HACK: deactivate ed25519 and ecdsa private keys from ssh identities, as it usually cause problems
|
6
|
+
old_verbose = $VERBOSE
|
7
|
+
$VERBOSE = nil
|
6
8
|
begin
|
7
9
|
module Net; module SSH; module Authentication; class Session; private; def default_keys; %w[~/.ssh/id_dsa ~/.ssh/id_rsa ~/.ssh2/id_dsa ~/.ssh2/id_rsa]; end; end; end; end; end # rubocop:disable Layout/AccessModifierIndentation, Layout/EmptyLinesAroundAccessModifier, Layout/LineLength, Style/Semicolon
|
8
10
|
rescue StandardError
|
9
11
|
# ignore errors
|
10
12
|
end
|
13
|
+
$VERBOSE = old_verbose
|
11
14
|
|
12
15
|
module Aspera
|
13
16
|
# A simple wrapper around Net::SSH
|