aspera-cli 4.13.0 → 4.15.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +81 -7
- data/CONTRIBUTING.md +22 -6
- data/README.md +2038 -1080
- data/bin/ascli +18 -9
- data/bin/asession +12 -14
- data/examples/dascli +1 -1
- data/examples/proxy.pac +1 -1
- data/examples/rubyc +24 -0
- data/lib/aspera/aoc.rb +219 -159
- data/lib/aspera/ascmd.rb +25 -14
- data/lib/aspera/cli/basic_auth_plugin.rb +12 -9
- data/lib/aspera/cli/error.rb +17 -0
- data/lib/aspera/cli/extended_value.rb +47 -12
- data/lib/aspera/cli/formatter.rb +260 -179
- data/lib/aspera/cli/hints.rb +80 -0
- data/lib/aspera/cli/main.rb +104 -156
- data/lib/aspera/cli/manager.rb +259 -209
- data/lib/aspera/cli/plugin.rb +123 -63
- data/lib/aspera/cli/plugins/alee.rb +2 -3
- data/lib/aspera/cli/plugins/aoc.rb +341 -261
- data/lib/aspera/cli/plugins/ats.rb +22 -21
- data/lib/aspera/cli/plugins/bss.rb +5 -5
- data/lib/aspera/cli/plugins/config.rb +578 -627
- data/lib/aspera/cli/plugins/console.rb +44 -6
- data/lib/aspera/cli/plugins/cos.rb +15 -17
- data/lib/aspera/cli/plugins/faspex.rb +114 -100
- data/lib/aspera/cli/plugins/faspex5.rb +411 -264
- data/lib/aspera/cli/plugins/node.rb +354 -259
- data/lib/aspera/cli/plugins/orchestrator.rb +61 -29
- data/lib/aspera/cli/plugins/preview.rb +82 -90
- data/lib/aspera/cli/plugins/server.rb +79 -32
- data/lib/aspera/cli/plugins/shares.rb +55 -42
- data/lib/aspera/cli/sync_actions.rb +68 -0
- data/lib/aspera/cli/transfer_agent.rb +66 -73
- data/lib/aspera/cli/transfer_progress.rb +74 -0
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +12 -8
- data/lib/aspera/command_line_builder.rb +14 -11
- data/lib/aspera/cos_node.rb +3 -2
- data/lib/aspera/data/6 +0 -0
- data/lib/aspera/environment.rb +24 -9
- data/lib/aspera/fasp/agent_aspera.rb +126 -0
- data/lib/aspera/fasp/agent_base.rb +31 -77
- data/lib/aspera/fasp/agent_connect.rb +25 -21
- data/lib/aspera/fasp/agent_direct.rb +89 -103
- data/lib/aspera/fasp/agent_httpgw.rb +231 -149
- data/lib/aspera/fasp/agent_node.rb +41 -34
- data/lib/aspera/fasp/agent_trsdk.rb +75 -32
- data/lib/aspera/fasp/error_info.rb +4 -2
- data/lib/aspera/fasp/faux_file.rb +52 -0
- data/lib/aspera/fasp/installation.rb +53 -195
- data/lib/aspera/fasp/management.rb +244 -0
- data/lib/aspera/fasp/parameters.rb +71 -37
- data/lib/aspera/fasp/parameters.yaml +76 -8
- data/lib/aspera/fasp/products.rb +162 -0
- data/lib/aspera/fasp/resume_policy.rb +3 -3
- data/lib/aspera/fasp/transfer_spec.rb +7 -6
- data/lib/aspera/fasp/uri.rb +26 -24
- data/lib/aspera/faspex_gw.rb +2 -2
- data/lib/aspera/faspex_postproc.rb +2 -2
- data/lib/aspera/hash_ext.rb +14 -4
- data/lib/aspera/json_rpc.rb +49 -0
- data/lib/aspera/keychain/macos_security.rb +13 -13
- data/lib/aspera/line_logger.rb +23 -0
- data/lib/aspera/log.rb +58 -16
- data/lib/aspera/node.rb +157 -92
- data/lib/aspera/oauth.rb +37 -19
- data/lib/aspera/open_application.rb +4 -4
- data/lib/aspera/persistency_action_once.rb +1 -1
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/file_types.rb +4 -2
- data/lib/aspera/preview/generator.rb +22 -35
- data/lib/aspera/preview/options.rb +2 -0
- data/lib/aspera/preview/terminal.rb +73 -16
- data/lib/aspera/preview/utils.rb +21 -28
- data/lib/aspera/proxy_auto_config.js +2 -2
- data/lib/aspera/rest.rb +136 -68
- data/lib/aspera/rest_call_error.rb +1 -1
- data/lib/aspera/rest_error_analyzer.rb +15 -14
- data/lib/aspera/rest_errors_aspera.rb +37 -34
- data/lib/aspera/secret_hider.rb +18 -15
- data/lib/aspera/ssh.rb +5 -2
- data/lib/aspera/sync.rb +127 -119
- data/lib/aspera/temp_file_manager.rb +10 -3
- data/lib/aspera/web_auth.rb +10 -7
- data/lib/aspera/web_server_simple.rb +9 -4
- data.tar.gz.sig +0 -0
- metadata +34 -17
- metadata.gz.sig +0 -0
- data/docs/test_env.conf +0 -186
- data/lib/aspera/cli/listener/line_dump.rb +0 -19
- data/lib/aspera/cli/listener/logger.rb +0 -22
- data/lib/aspera/cli/listener/progress.rb +0 -50
- data/lib/aspera/cli/listener/progress_multi.rb +0 -84
- data/lib/aspera/cli/plugins/sync.rb +0 -44
- data/lib/aspera/data/7 +0 -0
- data/lib/aspera/fasp/listener.rb +0 -13
data/lib/aspera/rest.rb
CHANGED
@@ -10,7 +10,6 @@ require 'net/https'
|
|
10
10
|
require 'json'
|
11
11
|
require 'base64'
|
12
12
|
require 'cgi'
|
13
|
-
require 'ruby-progressbar'
|
14
13
|
|
15
14
|
# add cancel method to http
|
16
15
|
class Net::HTTP::Cancel < Net::HTTPRequest # rubocop:disable Style/ClassAndModuleChildren
|
@@ -26,71 +25,103 @@ module Aspera
|
|
26
25
|
class Rest
|
27
26
|
# global settings also valid for any subclass
|
28
27
|
@@global = { # rubocop:disable Style/ClassVars
|
29
|
-
|
30
|
-
#
|
31
|
-
|
32
|
-
|
33
|
-
# a lambda which takes the Net::HTTP as arg, use this to change parameters
|
34
|
-
session_cb: nil,
|
35
|
-
proxy_user: nil,
|
36
|
-
proxy_pass: nil
|
28
|
+
user_agent: 'Ruby', # goes to HTTP request header: 'User-Agent'
|
29
|
+
download_partial_suffix: '.http_partial', # suffix for partial download
|
30
|
+
session_cb: nil, # a lambda which takes the Net::HTTP as arg, use this to change parameters
|
31
|
+
progress_bar: nil # progress bar object
|
37
32
|
}
|
38
33
|
|
34
|
+
# flag for array parameters prefixed with []
|
39
35
|
ARRAY_PARAMS = '[]'
|
40
36
|
|
37
|
+
private_constant :ARRAY_PARAMS
|
38
|
+
|
39
|
+
# error message when entity not found (TODO: use specific exception)
|
40
|
+
ENTITY_NOT_FOUND = 'No such'
|
41
|
+
|
42
|
+
# Content-Type that are JSON
|
43
|
+
JSON_DECODE = ['application/json', 'application/vnd.api+json', 'application/x-javascript'].freeze
|
44
|
+
|
41
45
|
class << self
|
42
|
-
#
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
end
|
46
|
+
def basic_token(user, pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}"; end
|
47
|
+
|
48
|
+
# used to build a parameter list prefixed with "[]"
|
49
|
+
# @param values [Array] list of values
|
50
|
+
def array_params(values)
|
51
|
+
return [ARRAY_PARAMS].concat(values)
|
49
52
|
end
|
50
53
|
|
51
|
-
def
|
54
|
+
def array_params?(values)
|
55
|
+
return values.first.eql?(ARRAY_PARAMS)
|
56
|
+
end
|
52
57
|
|
53
58
|
# build URI from URL and parameters and check it is http or https
|
54
59
|
def build_uri(url, params=nil)
|
55
60
|
uri = URI.parse(url)
|
56
61
|
raise "REST endpoint shall be http/s not #{uri.scheme}" unless %w[http https].include?(uri.scheme)
|
57
|
-
if
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
end
|
69
|
-
else
|
70
|
-
params.push([k, v])
|
71
|
-
end
|
62
|
+
return uri if params.nil?
|
63
|
+
Log.log.debug{Log.dump('params', params)}
|
64
|
+
raise 'Internal Error: param must be Hash' unless params.is_a?(Hash)
|
65
|
+
query = []
|
66
|
+
params.each do |k, v|
|
67
|
+
case v
|
68
|
+
when Array
|
69
|
+
# support array url params, there is no standard. Either p[]=1&p[]=2, or p=1&p=2
|
70
|
+
suffix = array_params?(v) ? v.shift : ''
|
71
|
+
v.each do |e|
|
72
|
+
query.push(["#{k}#{suffix}", e])
|
72
73
|
end
|
74
|
+
else
|
75
|
+
query.push([k, v])
|
73
76
|
end
|
74
|
-
# CGI.unescape to transform back %5D into []
|
75
|
-
uri.query = CGI.unescape(URI.encode_www_form(params))
|
76
77
|
end
|
78
|
+
# [] is allowed in url parameters
|
79
|
+
uri.query = URI.encode_www_form(query).gsub('%5B%5D=', '[]=')
|
77
80
|
return uri
|
78
81
|
end
|
79
82
|
|
83
|
+
def decode_query(query)
|
84
|
+
URI.decode_www_form(query).each_with_object({}){|v, h|h[v.first] = v.last }
|
85
|
+
end
|
86
|
+
|
87
|
+
# start a HTTP/S session, also used for web sockets
|
88
|
+
# @param base_url [String] base url of HTTP/S session
|
89
|
+
# @return [Net::HTTP] a started HTTP session
|
80
90
|
def start_http_session(base_url)
|
81
91
|
uri = build_uri(base_url)
|
82
92
|
# this honors http_proxy env var
|
83
93
|
http_session = Net::HTTP.new(uri.host, uri.port)
|
84
|
-
http_session.proxy_user = proxy_user
|
85
|
-
http_session.proxy_pass = proxy_pass
|
86
94
|
http_session.use_ssl = uri.scheme.eql?('https')
|
87
|
-
http_session.set_debug_output($stdout) if debug
|
88
95
|
# set http options in callback, such as timeout and cert. verification
|
89
|
-
session_cb&.call(http_session)
|
96
|
+
@@global[:session_cb]&.call(http_session)
|
90
97
|
# manually start session for keep alive (if supported by server, else, session is closed every time)
|
91
98
|
http_session.start
|
92
99
|
return http_session
|
93
100
|
end
|
101
|
+
|
102
|
+
# get Net::HTTP underlying socket i/o
|
103
|
+
# little hack, handy because HTTP debug, proxy, etc... will be available
|
104
|
+
# used implement web sockets after `start_http_session`
|
105
|
+
def io_http_session(http_session)
|
106
|
+
raise "wring type #{http_session.class}" unless http_session.is_a?(Net::HTTP)
|
107
|
+
# Net::BufferedIO in net/protocol.rb
|
108
|
+
result = http_session.instance_variable_get(:@socket)
|
109
|
+
raise "no socket for #{http_session}" if result.nil?
|
110
|
+
return result
|
111
|
+
end
|
112
|
+
|
113
|
+
# set global parameters
|
114
|
+
def set_parameters(**options)
|
115
|
+
options.each do |key, value|
|
116
|
+
raise "ERROR: unknown Rest option #{key}" unless @@global.key?(key)
|
117
|
+
@@global[key] = value
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
# @return [String] HTTP agent name
|
122
|
+
def user_agent
|
123
|
+
return @@global[:user_agent]
|
124
|
+
end
|
94
125
|
end
|
95
126
|
|
96
127
|
private
|
@@ -120,7 +151,7 @@ module Aspera
|
|
120
151
|
raise 'ERROR: expecting Hash' unless a_rest_params.is_a?(Hash)
|
121
152
|
raise 'ERROR: expecting base_url' unless a_rest_params[:base_url].is_a?(String)
|
122
153
|
@params = a_rest_params.clone
|
123
|
-
Log.dump('REST params', @params)
|
154
|
+
Log.log.debug{Log.dump('REST params', @params)}
|
124
155
|
# base url without trailing slashes (note: string may be frozen)
|
125
156
|
@params[:base_url] = @params[:base_url].gsub(%r{/+$}, '')
|
126
157
|
@http_session = nil
|
@@ -128,7 +159,7 @@ module Aspera
|
|
128
159
|
@params[:auth] ||= {type: :none}
|
129
160
|
@params[:not_auth_codes] ||= ['401']
|
130
161
|
@oauth = nil
|
131
|
-
Log.dump('REST params(2)', @params)
|
162
|
+
Log.log.debug{Log.dump('REST params(2)', @params)}
|
132
163
|
end
|
133
164
|
|
134
165
|
def oauth_token(force_refresh: false)
|
@@ -148,8 +179,8 @@ module Aspera
|
|
148
179
|
raise "unsupported operation : #{call_data[:operation]}"
|
149
180
|
end
|
150
181
|
if call_data.key?(:json_params) && !call_data[:json_params].nil?
|
151
|
-
req.body = JSON.generate(call_data[:json_params])
|
152
|
-
Log.dump('body JSON data', call_data[:json_params])
|
182
|
+
req.body = JSON.generate(call_data[:json_params]) # , ascii_only: true
|
183
|
+
Log.log.debug{Log.dump('body JSON data', call_data[:json_params])}
|
153
184
|
req['Content-Type'] = 'application/json'
|
154
185
|
# call_data[:headers]['Accept']='application/json'
|
155
186
|
end
|
@@ -170,6 +201,7 @@ module Aspera
|
|
170
201
|
end
|
171
202
|
# :type = :basic
|
172
203
|
req.basic_auth(call_data[:auth][:username], call_data[:auth][:password]) if call_data[:auth][:type].eql?(:basic)
|
204
|
+
Log.log.debug{Log.dump(:req_body, req.body)}
|
173
205
|
return req
|
174
206
|
end
|
175
207
|
|
@@ -192,14 +224,14 @@ module Aspera
|
|
192
224
|
# :type (:none, :basic, :oauth2, :url)
|
193
225
|
# :username [:basic]
|
194
226
|
# :password [:basic]
|
195
|
-
# :
|
227
|
+
# :url_query [:url] a hash
|
196
228
|
# :* [:oauth2] see Oauth class
|
197
229
|
def call(call_data)
|
198
230
|
raise "Hash call parameter is required (#{call_data.class})" unless call_data.is_a?(Hash)
|
199
231
|
call_data[:subpath] = '' if call_data[:subpath].nil?
|
200
232
|
Log.log.debug{"accessing #{call_data[:subpath]}".red.bold.bg_green}
|
201
233
|
call_data[:headers] ||= {}
|
202
|
-
call_data[:headers]['User-Agent'] ||=
|
234
|
+
call_data[:headers]['User-Agent'] ||= @@global[:user_agent]
|
203
235
|
# defaults from @params are overridden by call data
|
204
236
|
call_data = @params.deep_merge(call_data)
|
205
237
|
case call_data[:auth][:type]
|
@@ -212,7 +244,7 @@ module Aspera
|
|
212
244
|
call_data[:headers]['Authorization'] = oauth_token unless call_data[:headers].key?('Authorization')
|
213
245
|
when :url
|
214
246
|
call_data[:url_params] ||= {}
|
215
|
-
call_data[:auth][:
|
247
|
+
call_data[:auth][:url_query].each do |key, value|
|
216
248
|
call_data[:url_params][key] = value
|
217
249
|
end
|
218
250
|
else raise "unsupported auth type: [#{call_data[:auth][:type]}]"
|
@@ -227,16 +259,18 @@ module Aspera
|
|
227
259
|
# initialize with number of initial retries allowed, nil gives zero
|
228
260
|
tries_remain_redirect = call_data[:redirect_max].to_i if tries_remain_redirect.nil?
|
229
261
|
Log.log.debug("send request (retries=#{tries_remain_redirect})")
|
262
|
+
result_mime = nil
|
263
|
+
file_saved = false
|
230
264
|
# make http request (pipelined)
|
231
265
|
http_session.request(req) do |response|
|
232
266
|
result[:http] = response
|
233
|
-
|
267
|
+
result_mime = (result[:http]['Content-Type'] || 'text/plain').split(';').first.downcase
|
268
|
+
# JSON data needs to be parsed, in case it contains an error code
|
269
|
+
if !call_data[:save_to_file].nil? &&
|
270
|
+
result[:http].code.to_s.start_with?('2') &&
|
271
|
+
!result[:http]['Content-Length'].nil? &&
|
272
|
+
!JSON_DECODE.include?(result_mime)
|
234
273
|
total_size = result[:http]['Content-Length'].to_i
|
235
|
-
progress = ProgressBar.create(
|
236
|
-
format: '%a %B %p%% %r KB/sec %e',
|
237
|
-
rate_scale: lambda{|rate|rate / 1024},
|
238
|
-
title: 'progress',
|
239
|
-
total: total_size)
|
240
274
|
Log.log.debug('before write file')
|
241
275
|
target_file = call_data[:save_to_file]
|
242
276
|
# override user's path to path in header
|
@@ -244,34 +278,37 @@ module Aspera
|
|
244
278
|
target_file = File.join(File.dirname(target_file), m[1])
|
245
279
|
end
|
246
280
|
# download with temp filename
|
247
|
-
target_file_tmp = "#{target_file}#{
|
281
|
+
target_file_tmp = "#{target_file}#{@@global[:download_partial_suffix]}"
|
248
282
|
Log.log.debug{"saving to: #{target_file}"}
|
283
|
+
written_size = 0
|
284
|
+
@@global[:progress_bar]&.event(session_id: 1, type: :session_start)
|
285
|
+
@@global[:progress_bar]&.event(session_id: 1, type: :session_size, info: total_size)
|
249
286
|
File.open(target_file_tmp, 'wb') do |file|
|
250
287
|
result[:http].read_body do |fragment|
|
251
288
|
file.write(fragment)
|
252
|
-
|
253
|
-
|
254
|
-
progress.progress = new_process
|
289
|
+
written_size += fragment.length
|
290
|
+
@@global[:progress_bar]&.event(session_id: 1, type: :transfer, info: written_size)
|
255
291
|
end
|
256
292
|
end
|
293
|
+
@@global[:progress_bar]&.event(session_id: 1, type: :end)
|
257
294
|
# rename at the end
|
258
295
|
File.rename(target_file_tmp, target_file)
|
259
|
-
|
296
|
+
file_saved = true
|
260
297
|
end # save_to_file
|
261
298
|
end
|
262
|
-
# sometimes there is a UTF8 char (e.g. (c) )
|
263
|
-
result[:http].body.force_encoding('UTF-8') if result[:http].body.is_a?(String)
|
264
|
-
Log.log.debug{"result: body=#{result[:http].body}"}
|
265
|
-
result_mime = (result[:http]['Content-Type'] || 'text/plain').split(';').first
|
299
|
+
# sometimes there is a UTF8 char (e.g. (c) ), TODO : related to mime type encoding ?
|
300
|
+
# result[:http].body.force_encoding('UTF-8') if result[:http].body.is_a?(String)
|
301
|
+
# Log.log.debug{"result: body=#{result[:http].body}"}
|
266
302
|
result[:data] = case result_mime
|
267
|
-
when
|
268
|
-
JSON.parse(result[:http].body) rescue
|
303
|
+
when *JSON_DECODE
|
304
|
+
JSON.parse(result[:http].body) rescue result[:http].body
|
269
305
|
else # when 'text/plain'
|
270
306
|
result[:http].body
|
271
307
|
end
|
272
|
-
Log.dump("result: parsed: #{result_mime}", result[:data])
|
308
|
+
Log.log.debug{Log.dump("result: parsed: #{result_mime}", result[:data])}
|
273
309
|
Log.log.debug{"result: code=#{result[:http].code}"}
|
274
310
|
RestErrorAnalyzer.instance.raise_on_error(req, result)
|
311
|
+
File.write(call_data[:save_to_file], result[:http].body) unless file_saved || call_data[:save_to_file].nil?
|
275
312
|
rescue RestCallError => e
|
276
313
|
# not authorized: oauth token expired
|
277
314
|
if call_data[:not_auth_codes].include?(result[:http].code.to_s) && call_data[:auth][:type].eql?(:oauth2)
|
@@ -292,7 +329,12 @@ module Aspera
|
|
292
329
|
tries_remain_redirect -= 1
|
293
330
|
current_uri = URI.parse(call_data[:base_url])
|
294
331
|
new_url = e.response['location']
|
295
|
-
|
332
|
+
# special case: relative redirect
|
333
|
+
if URI.parse(new_url).host.nil?
|
334
|
+
# we don't manage relative redirects with non-absolute path
|
335
|
+
raise "Error: redirect location is relative: #{new_url}, but does not start with /." unless new_url.start_with?('/')
|
336
|
+
new_url = current_uri.scheme + '://' + current_uri.host + new_url
|
337
|
+
end
|
296
338
|
Log.log.info{"URL is moved: #{new_url}"}
|
297
339
|
redirection_uri = URI.parse(new_url)
|
298
340
|
call_data[:base_url] = new_url
|
@@ -322,20 +364,46 @@ module Aspera
|
|
322
364
|
return call({operation: 'POST', subpath: subpath, headers: {'Accept' => 'application/json'}, encoding => params})
|
323
365
|
end
|
324
366
|
|
325
|
-
def read(subpath,
|
326
|
-
return call({operation: 'GET', subpath: subpath, headers: {'Accept' => 'application/json'}, url_params:
|
367
|
+
def read(subpath, options=nil)
|
368
|
+
return call({operation: 'GET', subpath: subpath, headers: {'Accept' => 'application/json'}, url_params: options})
|
327
369
|
end
|
328
370
|
|
329
371
|
def update(subpath, params)
|
330
372
|
return call({operation: 'PUT', subpath: subpath, headers: {'Accept' => 'application/json'}, json_params: params})
|
331
373
|
end
|
332
374
|
|
333
|
-
def delete(subpath,
|
334
|
-
return call({operation: 'DELETE', subpath: subpath, headers: {'Accept' => 'application/json'}, url_params:
|
375
|
+
def delete(subpath, params=nil)
|
376
|
+
return call({operation: 'DELETE', subpath: subpath, headers: {'Accept' => 'application/json'}, url_params: params})
|
335
377
|
end
|
336
378
|
|
337
379
|
def cancel(subpath)
|
338
380
|
return call({operation: 'CANCEL', subpath: subpath, headers: {'Accept' => 'application/json'}})
|
339
381
|
end
|
382
|
+
|
383
|
+
# Query by name and returns a single result, else it throws an exception (no or multiple results)
|
384
|
+
# @param subpath path of entity in API
|
385
|
+
# @param search_name name of searched entity
|
386
|
+
# @param options additional search options
|
387
|
+
def lookup_by_name(subpath, search_name, options={})
|
388
|
+
# returns entities whose name contains value (case insensitive)
|
389
|
+
matching_items = read(subpath, options.merge({'q' => search_name}))[:data]
|
390
|
+
# API style: {totalcount:, ...} cspell: disable-line
|
391
|
+
# TODO: not generic enough ? move somewhere ? inheritance ?
|
392
|
+
matching_items = matching_items[subpath] if matching_items.is_a?(Hash)
|
393
|
+
raise "Internal error: expecting array, have #{matching_items.class}" unless matching_items.is_a?(Array)
|
394
|
+
case matching_items.length
|
395
|
+
when 1 then return matching_items.first
|
396
|
+
when 0 then raise %Q{#{ENTITY_NOT_FOUND} #{subpath}: "#{search_name}"}
|
397
|
+
else
|
398
|
+
# multiple case insensitive partial matches, try case insensitive full match
|
399
|
+
# (anyway AoC does not allow creation of 2 entities with same case insensitive name)
|
400
|
+
name_matches = matching_items.select{|i|i['name'].casecmp?(search_name)}
|
401
|
+
case name_matches.length
|
402
|
+
when 1 then return name_matches.first
|
403
|
+
when 0 then raise %Q(#{subpath}: multiple case insensitive partial match for: "#{search_name}": #{matching_items.map{|i|i['name']}} but no case insensitive full match. Please be more specific or give exact name.) # rubocop:disable Layout/LineLength
|
404
|
+
else raise "Two entities cannot have the same case insensitive name: #{name_matches.map{|i|i['name']}}"
|
405
|
+
end
|
406
|
+
end
|
407
|
+
end
|
340
408
|
end
|
341
409
|
end # module Aspera
|
@@ -27,6 +27,7 @@ module Aspera
|
|
27
27
|
# Analyzes REST call response and raises a RestCallError exception
|
28
28
|
# if HTTP result code is not 2XX
|
29
29
|
def raise_on_error(req, res)
|
30
|
+
Log.log.debug{"raise_on_error #{req.method} #{req.path} #{res[:http].code}"}
|
30
31
|
call_context = {
|
31
32
|
messages: [],
|
32
33
|
request: req,
|
@@ -44,7 +45,7 @@ module Aspera
|
|
44
45
|
Log.log.error{"ERROR in handler:\n#{e.message}\n#{e.backtrace}"}
|
45
46
|
end
|
46
47
|
end
|
47
|
-
raise RestCallError.new(call_context[:
|
48
|
+
raise RestCallError.new(call_context[:messages].join("\n"), call_context[:request], call_context[:response]) unless call_context[:messages].empty?
|
48
49
|
end
|
49
50
|
|
50
51
|
# add a new error handler (done at application initialization)
|
@@ -59,21 +60,21 @@ module Aspera
|
|
59
60
|
# add a simple error handler
|
60
61
|
# check that key exists and is string under specified path (hash)
|
61
62
|
# adds other keys as secondary information
|
62
|
-
|
63
|
+
# @param name [String] name of error handler (for logs)
|
64
|
+
# @param always [boolean] if true, always add error message, even if response code is 2XX
|
65
|
+
# @param path [Array] path to error message in response
|
66
|
+
def add_simple_handler(name:, always: false, path:)
|
67
|
+
path.freeze
|
63
68
|
add_handler(name) do |type, call_context|
|
64
|
-
# need to clone because we modify and same array is used subsequently
|
65
|
-
path = args.clone
|
66
|
-
# Log.log.debug{"path=#{path}"}
|
67
|
-
# if last in path is boolean it tells if the error is only with http error code or always
|
68
|
-
always = [true, false].include?(path.last) ? path.pop : false
|
69
69
|
if call_context[:data].is_a?(Hash) && (!call_context[:response].code.start_with?('2') || always)
|
70
|
-
|
71
|
-
# dig and find
|
72
|
-
error_struct = path.
|
73
|
-
|
74
|
-
|
70
|
+
# Log.log.debug{"simple_handler: #{type} #{path} #{path.last}"}
|
71
|
+
# dig and find hash containing error message
|
72
|
+
error_struct = path.length.eql?(1) ? call_context[:data] : call_context[:data].dig(*path[0..-2])
|
73
|
+
# Log.log.debug{"found: #{error_struct.class} #{error_struct}"}
|
74
|
+
if error_struct.is_a?(Hash) && error_struct[path.last].is_a?(String)
|
75
|
+
RestErrorAnalyzer.add_error(call_context, type, error_struct[path.last])
|
75
76
|
error_struct.each do |k, v|
|
76
|
-
next if k.eql?(
|
77
|
+
next if k.eql?(path.last)
|
77
78
|
RestErrorAnalyzer.add_error(call_context, "#{type}(sub)", "#{k}: #{v}") if [String, Integer].include?(v.class)
|
78
79
|
end
|
79
80
|
end
|
@@ -93,7 +94,7 @@ module Aspera
|
|
93
94
|
# log error for further analysis (file must exist to activate)
|
94
95
|
return if log_file.nil? || !File.exist?(log_file)
|
95
96
|
File.open(log_file, 'a+') do |f|
|
96
|
-
f.write("\n=#{type}=====\n#{call_context[:request].method} #{call_context[:request].path}\n#{call_context[:response].code}\n"\
|
97
|
+
f.write("\n=#{type}=====\n#{call_context[:request].method} #{call_context[:request].path}\n#{call_context[:response].code}\n" \
|
97
98
|
"#{JSON.generate(call_context[:data])}\n#{call_context[:messages].join("\n")}")
|
98
99
|
end
|
99
100
|
end
|
@@ -12,48 +12,51 @@ module Aspera
|
|
12
12
|
Log.log.debug('registering Aspera REST error handlers')
|
13
13
|
# Faspex 4: both user_message and internal_message, and code 200
|
14
14
|
# example: missing meta data on package creation
|
15
|
-
RestErrorAnalyzer.instance.add_simple_handler('Type 1: error:user_message',
|
16
|
-
RestErrorAnalyzer.instance.add_simple_handler('Type 2: error:description',
|
17
|
-
RestErrorAnalyzer.instance.add_simple_handler('Type 3: error:internal_message',
|
15
|
+
RestErrorAnalyzer.instance.add_simple_handler(name: 'Type 1: error:user_message', path: %w[error user_message], always: true)
|
16
|
+
RestErrorAnalyzer.instance.add_simple_handler(name: 'Type 2: error:description', path: %w[error description])
|
17
|
+
RestErrorAnalyzer.instance.add_simple_handler(name: 'Type 3: error:internal_message', path: %w[error internal_message])
|
18
18
|
# AoC Automation
|
19
|
-
RestErrorAnalyzer.instance.add_simple_handler('AoC Automation', 'error')
|
20
|
-
RestErrorAnalyzer.instance.add_simple_handler('Type 5', 'error_description')
|
21
|
-
RestErrorAnalyzer.instance.add_simple_handler('Type 6', 'message')
|
22
|
-
RestErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
end
|
19
|
+
RestErrorAnalyzer.instance.add_simple_handler(name: 'AoC Automation', path: ['error'])
|
20
|
+
RestErrorAnalyzer.instance.add_simple_handler(name: 'Type 5', path: ['error_description'])
|
21
|
+
RestErrorAnalyzer.instance.add_simple_handler(name: 'Type 6', path: ['message'])
|
22
|
+
RestErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |type, call_context|
|
23
|
+
next unless call_context[:data].is_a?(Hash) && call_context[:data]['errors'].is_a?(Hash)
|
24
|
+
call_context[:data]['errors'].each do |k, v|
|
25
|
+
RestErrorAnalyzer.add_error(call_context, type, "#{k}: #{v}")
|
27
26
|
end
|
28
27
|
end
|
29
28
|
# call to upload_setup and download_setup of node api
|
30
29
|
RestErrorAnalyzer.instance.add_handler('T8:node: *_setup') do |type, call_context|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
RestErrorAnalyzer.add_error(call_context, type, "#{r_err['code']}: #{r_err['reason']}: #{r_err['user_message']}")
|
39
|
-
end
|
40
|
-
end
|
41
|
-
end
|
30
|
+
next unless call_context[:data].is_a?(Hash)
|
31
|
+
d_t_s = call_context[:data]['transfer_specs']
|
32
|
+
next unless d_t_s.is_a?(Array)
|
33
|
+
d_t_s.each do |res|
|
34
|
+
r_err = res.dig(*%w[transfer_spec error])
|
35
|
+
next unless r_err.is_a?(Hash)
|
36
|
+
RestErrorAnalyzer.add_error(call_context, type, "#{r_err['code']}: #{r_err['reason']}: #{r_err['user_message']}")
|
42
37
|
end
|
43
38
|
end
|
44
|
-
RestErrorAnalyzer.instance.add_simple_handler('T9:IBM cloud IAM', 'errorMessage')
|
45
|
-
RestErrorAnalyzer.instance.add_simple_handler('T10:faspex v4', 'user_message')
|
39
|
+
RestErrorAnalyzer.instance.add_simple_handler(name: 'T9:IBM cloud IAM', path: ['errorMessage'])
|
40
|
+
RestErrorAnalyzer.instance.add_simple_handler(name: 'T10:faspex v4', path: ['user_message'])
|
46
41
|
RestErrorAnalyzer.instance.add_handler('bss graphql') do |type, call_context|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
42
|
+
next unless call_context[:data].is_a?(Hash)
|
43
|
+
d_t_s = call_context[:data]['errors']
|
44
|
+
next unless d_t_s.is_a?(Array)
|
45
|
+
d_t_s.each do |res|
|
46
|
+
r_err = res['message']
|
47
|
+
next unless r_err.is_a?(String)
|
48
|
+
RestErrorAnalyzer.add_error(call_context, type, r_err)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
RestErrorAnalyzer.instance.add_handler('Orchestrator') do |type, call_context|
|
52
|
+
next if call_context[:response].code.start_with?('2')
|
53
|
+
data = call_context[:data]
|
54
|
+
next unless data.is_a?(Hash)
|
55
|
+
work_order = data['work_order']
|
56
|
+
next unless work_order.is_a?(Hash)
|
57
|
+
RestErrorAnalyzer.add_error(call_context, type, work_order['statusDetails'])
|
58
|
+
data['missing_parameters']&.each do |param|
|
59
|
+
RestErrorAnalyzer.add_error(call_context, type, "missing parameter: #{param}")
|
57
60
|
end
|
58
61
|
end
|
59
62
|
end # register_handlers
|
data/lib/aspera/secret_hider.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
# cspell:ignore FILEPASS
|
3
4
|
require 'logger'
|
4
5
|
|
5
6
|
module Aspera
|
@@ -11,21 +12,25 @@ module Aspera
|
|
11
12
|
ASCP_ENV_SECRETS = %w[ASPERA_SCP_PASS ASPERA_SCP_KEY ASPERA_SCP_FILEPASS ASPERA_PROXY_PASS ASPERA_SCP_TOKEN].freeze
|
12
13
|
# keys in hash that contain secrets
|
13
14
|
KEY_SECRETS = %w[password secret passphrase _key apikey crn token].freeze
|
14
|
-
|
15
|
+
HTTP_SECRETS = %w[Authorization].freeze
|
16
|
+
ALL_SECRETS = [ASCP_ENV_SECRETS, KEY_SECRETS, HTTP_SECRETS].flatten.freeze
|
17
|
+
KEY_FALSE_POSITIVES = [/^access_key$/].freeze
|
15
18
|
# regex that define named captures :begin and :end
|
16
19
|
REGEX_LOG_REPLACES = [
|
17
20
|
# CLI manager get/set options
|
18
|
-
/(?<begin>[sg]et (
|
21
|
+
/(?<begin>[sg]et (?:#{KEY_SECRETS.join('|')})=).*(?<end>)/,
|
19
22
|
# env var ascp exec
|
20
|
-
/(?<begin> (
|
21
|
-
# rendered JSON
|
22
|
-
/(?<begin>["'
|
23
|
+
/(?<begin> (?:#{ASCP_ENV_SECRETS.join('|')})=)(\\.|[^ ])*(?<end> )/,
|
24
|
+
# rendered JSON or Ruby
|
25
|
+
/(?<begin>(?:(?<quote>["'])|:)[^"':=]*(?:#{ALL_SECRETS.join('|')})[^"':=]*\k<quote>?(?:=>|:) *")[^"]+(?<end>")/,
|
23
26
|
# option "secret"
|
24
27
|
/(?<begin>"[^"]*(secret)[^"]*"=>{)[^}]+(?<end>})/,
|
25
28
|
# option "secrets"
|
26
29
|
/(?<begin>(secrets)={)[^}]+(?<end>})/,
|
27
30
|
# private key values
|
28
|
-
/(?<begin>--+BEGIN
|
31
|
+
/(?<begin>--+BEGIN [^-]+ KEY--+)[[:ascii:]]+?(?<end>--+?END [^-]+ KEY--+)/,
|
32
|
+
# cred in http dump
|
33
|
+
/(?<begin>(?:#{HTTP_SECRETS.join('|')}): )[^\\]+(?<end>\\)/i
|
29
34
|
].freeze
|
30
35
|
private_constant :HIDDEN_PASSWORD, :ASCP_ENV_SECRETS, :KEY_SECRETS, :ALL_SECRETS, :REGEX_LOG_REPLACES
|
31
36
|
@log_secrets = false
|
@@ -48,19 +53,17 @@ module Aspera
|
|
48
53
|
def secret?(keyword, value)
|
49
54
|
keyword = keyword.to_s if keyword.is_a?(Symbol)
|
50
55
|
# only Strings can be secrets, not booleans, or hash, arrays
|
51
|
-
keyword.is_a?(String) &&
|
56
|
+
return false unless keyword.is_a?(String) && value.is_a?(String)
|
57
|
+
# those are not secrets
|
58
|
+
return false if KEY_FALSE_POSITIVES.any?{|f|f.match?(keyword)}
|
59
|
+
# check if keyword (name) contains an element that designate it as a secret
|
60
|
+
ALL_SECRETS.any?{|kw|keyword.include?(kw)}
|
52
61
|
end
|
53
62
|
|
54
|
-
def deep_remove_secret(obj
|
63
|
+
def deep_remove_secret(obj)
|
55
64
|
case obj
|
56
65
|
when Array
|
57
|
-
|
58
|
-
obj.each do |i|
|
59
|
-
i['value'] = HIDDEN_PASSWORD if secret?(i['parameter'], i['value'])
|
60
|
-
end
|
61
|
-
else
|
62
|
-
obj.each{|i|deep_remove_secret(i)}
|
63
|
-
end
|
66
|
+
obj.each{|i|deep_remove_secret(i)}
|
64
67
|
when Hash
|
65
68
|
obj.each do |k, v|
|
66
69
|
if secret?(k, v)
|
data/lib/aspera/ssh.rb
CHANGED
@@ -2,12 +2,15 @@
|
|
2
2
|
|
3
3
|
require 'net/ssh'
|
4
4
|
|
5
|
-
# HACK: deactivate ed25519 and ecdsa private keys from ssh identities, as it usually
|
5
|
+
# HACK: deactivate ed25519 and ecdsa private keys from ssh identities, as it usually cause problems
|
6
|
+
old_verbose = $VERBOSE
|
7
|
+
$VERBOSE = nil
|
6
8
|
begin
|
7
9
|
module Net; module SSH; module Authentication; class Session; private; def default_keys; %w[~/.ssh/id_dsa ~/.ssh/id_rsa ~/.ssh2/id_dsa ~/.ssh2/id_rsa]; end; end; end; end; end # rubocop:disable Layout/AccessModifierIndentation, Layout/EmptyLinesAroundAccessModifier, Layout/LineLength, Style/Semicolon
|
8
10
|
rescue StandardError
|
9
11
|
# ignore errors
|
10
12
|
end
|
13
|
+
$VERBOSE = old_verbose
|
11
14
|
|
12
15
|
module Aspera
|
13
16
|
# A simple wrapper around Net::SSH
|
@@ -44,7 +47,7 @@ module Aspera
|
|
44
47
|
end
|
45
48
|
raise error_message
|
46
49
|
end
|
47
|
-
# send command to SSH channel (execute)
|
50
|
+
# send command to SSH channel (execute) cspell: disable-next-line
|
48
51
|
channel.send('cexe'.reverse, cmd){|_ch, _success|channel.send_data(input) unless input.nil?}
|
49
52
|
end
|
50
53
|
# wait for channel to finish (command exit)
|