aspera-cli 4.10.0 → 4.11.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +20 -0
- data/CHANGELOG.md +509 -0
- data/CONTRIBUTING.md +118 -0
- data/README.md +621 -378
- data/bin/ascli +4 -4
- data/bin/asession +11 -11
- data/docs/test_env.conf +28 -19
- data/examples/aoc.rb +4 -4
- data/examples/dascli +11 -9
- data/examples/faspex4.rb +8 -8
- data/examples/node.rb +11 -11
- data/examples/server.rb +9 -9
- data/lib/aspera/aoc.rb +273 -266
- data/lib/aspera/ascmd.rb +56 -54
- data/lib/aspera/ats_api.rb +4 -4
- data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
- data/lib/aspera/cli/extended_value.rb +5 -5
- data/lib/aspera/cli/formater.rb +64 -64
- data/lib/aspera/cli/listener/line_dump.rb +1 -1
- data/lib/aspera/cli/listener/logger.rb +1 -1
- data/lib/aspera/cli/listener/progress.rb +5 -6
- data/lib/aspera/cli/listener/progress_multi.rb +14 -19
- data/lib/aspera/cli/main.rb +66 -67
- data/lib/aspera/cli/manager.rb +110 -110
- data/lib/aspera/cli/plugin.rb +54 -37
- data/lib/aspera/cli/plugins/alee.rb +4 -4
- data/lib/aspera/cli/plugins/aoc.rb +308 -669
- data/lib/aspera/cli/plugins/ats.rb +44 -46
- data/lib/aspera/cli/plugins/bss.rb +10 -10
- data/lib/aspera/cli/plugins/config.rb +447 -344
- data/lib/aspera/cli/plugins/console.rb +12 -12
- data/lib/aspera/cli/plugins/cos.rb +18 -20
- data/lib/aspera/cli/plugins/faspex.rb +110 -112
- data/lib/aspera/cli/plugins/faspex5.rb +67 -46
- data/lib/aspera/cli/plugins/node.rb +364 -288
- data/lib/aspera/cli/plugins/orchestrator.rb +46 -46
- data/lib/aspera/cli/plugins/preview.rb +122 -114
- data/lib/aspera/cli/plugins/server.rb +137 -83
- data/lib/aspera/cli/plugins/shares.rb +30 -29
- data/lib/aspera/cli/plugins/sync.rb +13 -33
- data/lib/aspera/cli/transfer_agent.rb +57 -57
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +3 -3
- data/lib/aspera/command_line_builder.rb +27 -27
- data/lib/aspera/cos_node.rb +22 -20
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +30 -28
- data/lib/aspera/fasp/agent_base.rb +15 -15
- data/lib/aspera/fasp/agent_connect.rb +23 -21
- data/lib/aspera/fasp/agent_direct.rb +65 -67
- data/lib/aspera/fasp/agent_httpgw.rb +72 -68
- data/lib/aspera/fasp/agent_node.rb +23 -21
- data/lib/aspera/fasp/agent_trsdk.rb +20 -20
- data/lib/aspera/fasp/error.rb +3 -2
- data/lib/aspera/fasp/error_info.rb +11 -8
- data/lib/aspera/fasp/installation.rb +78 -78
- data/lib/aspera/fasp/listener.rb +1 -1
- data/lib/aspera/fasp/parameters.rb +75 -72
- data/lib/aspera/fasp/parameters.yaml +2 -2
- data/lib/aspera/fasp/resume_policy.rb +8 -8
- data/lib/aspera/fasp/transfer_spec.rb +35 -2
- data/lib/aspera/fasp/uri.rb +7 -7
- data/lib/aspera/faspex_gw.rb +7 -5
- data/lib/aspera/hash_ext.rb +3 -3
- data/lib/aspera/id_generator.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +23 -28
- data/lib/aspera/keychain/macos_security.rb +21 -20
- data/lib/aspera/log.rb +7 -7
- data/lib/aspera/nagios.rb +19 -18
- data/lib/aspera/node.rb +209 -35
- data/lib/aspera/oauth.rb +37 -36
- data/lib/aspera/open_application.rb +19 -11
- data/lib/aspera/persistency_action_once.rb +4 -4
- data/lib/aspera/persistency_folder.rb +13 -13
- data/lib/aspera/preview/file_types.rb +8 -8
- data/lib/aspera/preview/generator.rb +67 -67
- data/lib/aspera/preview/utils.rb +27 -27
- data/lib/aspera/proxy_auto_config.js +41 -41
- data/lib/aspera/proxy_auto_config.rb +16 -16
- data/lib/aspera/rest.rb +56 -60
- data/lib/aspera/rest_call_error.rb +2 -1
- data/lib/aspera/rest_error_analyzer.rb +18 -17
- data/lib/aspera/rest_errors_aspera.rb +16 -16
- data/lib/aspera/secret_hider.rb +15 -13
- data/lib/aspera/ssh.rb +11 -10
- data/lib/aspera/sync.rb +158 -44
- data/lib/aspera/temp_file_manager.rb +2 -2
- data/lib/aspera/uri_reader.rb +4 -4
- data/lib/aspera/web_auth.rb +14 -13
- data.tar.gz.sig +0 -0
- metadata +8 -5
- metadata.gz.sig +0 -0
data/lib/aspera/rest.rb
CHANGED
@@ -13,18 +13,12 @@ require 'cgi'
|
|
13
13
|
require 'ruby-progressbar'
|
14
14
|
|
15
15
|
# add cancel method to http
|
16
|
-
class Net::HTTP::Cancel < Net::HTTPRequest
|
16
|
+
class Net::HTTP::Cancel < Net::HTTPRequest # rubocop:disable Style/ClassAndModuleChildren
|
17
17
|
METHOD = 'CANCEL'
|
18
18
|
REQUEST_HAS_BODY = false
|
19
19
|
RESPONSE_HAS_BODY = false
|
20
20
|
end
|
21
21
|
|
22
|
-
#class Net::HTTP::Delete < Net::HTTPRequest
|
23
|
-
# METHOD = 'DELETE'
|
24
|
-
# REQUEST_HAS_BODY = false
|
25
|
-
# RESPONSE_HAS_BODY = false
|
26
|
-
#end
|
27
|
-
|
28
22
|
module Aspera
|
29
23
|
# a simple class to make HTTP calls, equivalent to rest-client
|
30
24
|
# rest call errors are raised as exception RestCallError
|
@@ -44,15 +38,18 @@ module Aspera
|
|
44
38
|
|
45
39
|
class << self
|
46
40
|
# define accessors
|
47
|
-
@@global.
|
41
|
+
@@global.each_key do |p|
|
48
42
|
define_method(p){@@global[p]}
|
49
|
-
define_method("#{p}=")
|
43
|
+
define_method("#{p}=") do |val|
|
44
|
+
Log.log.debug{"#{p} => #{val}".red}
|
45
|
+
@@global[p] = val
|
46
|
+
end
|
50
47
|
end
|
51
48
|
|
52
|
-
def basic_creds(user,pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}";end
|
49
|
+
def basic_creds(user, pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}"; end
|
53
50
|
|
54
51
|
# build URI from URL and parameters and check it is http or https
|
55
|
-
def build_uri(url,params=nil)
|
52
|
+
def build_uri(url, params=nil)
|
56
53
|
uri = URI.parse(url)
|
57
54
|
raise "REST endpoint shall be http/s not #{uri.scheme}" unless %w[http https].include?(uri.scheme)
|
58
55
|
if !params.nil?
|
@@ -60,15 +57,15 @@ module Aspera
|
|
60
57
|
if params.is_a?(Hash)
|
61
58
|
orig = params
|
62
59
|
params = []
|
63
|
-
orig.each do |k,v|
|
60
|
+
orig.each do |k, v|
|
64
61
|
case v
|
65
62
|
when Array
|
66
63
|
suffix = v.first.eql?('[]') ? v.shift : ''
|
67
64
|
v.each do |e|
|
68
|
-
params.push([k + suffix,e])
|
65
|
+
params.push([k + suffix, e])
|
69
66
|
end
|
70
67
|
else
|
71
|
-
params.push([k,v])
|
68
|
+
params.push([k, v])
|
72
69
|
end
|
73
70
|
end
|
74
71
|
end
|
@@ -82,8 +79,8 @@ module Aspera
|
|
82
79
|
uri = build_uri(base_url)
|
83
80
|
# this honors http_proxy env var
|
84
81
|
http_session = Net::HTTP.new(uri.host, uri.port)
|
85
|
-
http_session.proxy_user=proxy_user
|
86
|
-
http_session.proxy_pass=proxy_pass
|
82
|
+
http_session.proxy_user = proxy_user
|
83
|
+
http_session.proxy_pass = proxy_pass
|
87
84
|
http_session.use_ssl = uri.scheme.eql?('https')
|
88
85
|
http_session.set_debug_output($stdout) if debug
|
89
86
|
# set http options in callback, such as timeout and cert. verification
|
@@ -99,7 +96,7 @@ module Aspera
|
|
99
96
|
# create and start keep alive connection on demand
|
100
97
|
def http_session
|
101
98
|
if @http_session.nil?
|
102
|
-
@http_session=self.class.start_http_session(@params[:base_url])
|
99
|
+
@http_session = self.class.start_http_session(@params[:base_url])
|
103
100
|
end
|
104
101
|
return @http_session
|
105
102
|
end
|
@@ -121,57 +118,56 @@ module Aspera
|
|
121
118
|
raise 'ERROR: expecting Hash' unless a_rest_params.is_a?(Hash)
|
122
119
|
raise 'ERROR: expecting base_url' unless a_rest_params[:base_url].is_a?(String)
|
123
120
|
@params = a_rest_params.clone
|
124
|
-
Log.dump('REST params'
|
121
|
+
Log.dump('REST params', @params)
|
125
122
|
# base url without trailing slashes (note: string may be frozen)
|
126
|
-
@params[:base_url] = @params[:base_url].gsub(
|
123
|
+
@params[:base_url] = @params[:base_url].gsub(%r{/+$}, '')
|
127
124
|
@http_session = nil
|
128
125
|
# default is no auth
|
129
126
|
@params[:auth] ||= {type: :none}
|
130
127
|
@params[:not_auth_codes] ||= ['401']
|
131
128
|
@oauth = nil
|
132
|
-
Log.dump('REST params(2)'
|
129
|
+
Log.dump('REST params(2)', @params)
|
133
130
|
end
|
134
131
|
|
135
132
|
def oauth_token(force_refresh: false)
|
136
|
-
raise "ERROR: expecting boolean, have #{force_refresh}" unless [true,false].include?(force_refresh)
|
133
|
+
raise "ERROR: expecting boolean, have #{force_refresh}" unless [true, false].include?(force_refresh)
|
137
134
|
return oauth.get_authorization(use_refresh_token: force_refresh)
|
138
135
|
end
|
139
136
|
|
140
137
|
def build_request(call_data)
|
141
138
|
# TODO: shall we percent encode subpath (spaces) test with access key delete with space in id
|
142
139
|
# URI.escape()
|
143
|
-
uri = self.class.build_uri("#{call_data[:base_url]}#{['','/'].include?(call_data[:subpath]) ? '' : '/'}#{call_data[:subpath]}",call_data[:url_params])
|
144
|
-
Log.log.debug
|
140
|
+
uri = self.class.build_uri("#{call_data[:base_url]}#{['', '/'].include?(call_data[:subpath]) ? '' : '/'}#{call_data[:subpath]}", call_data[:url_params])
|
141
|
+
Log.log.debug{"URI=#{uri}"}
|
145
142
|
begin
|
146
143
|
# instanciate request object based on string name
|
147
144
|
req = Net::HTTP.const_get(call_data[:operation].capitalize).new(uri)
|
148
145
|
rescue NameError
|
149
146
|
raise "unsupported operation : #{call_data[:operation]}"
|
150
147
|
end
|
151
|
-
if call_data.
|
148
|
+
if call_data.key?(:json_params) && !call_data[:json_params].nil?
|
152
149
|
req.body = JSON.generate(call_data[:json_params])
|
153
|
-
Log.dump('body JSON data',call_data[:json_params])
|
154
|
-
#Log.log.debug("body JSON data=#{JSON.pretty_generate(call_data[:json_params])}")
|
150
|
+
Log.dump('body JSON data', call_data[:json_params])
|
155
151
|
req['Content-Type'] = 'application/json'
|
156
|
-
#call_data[:headers]['Accept']='application/json'
|
152
|
+
# call_data[:headers]['Accept']='application/json'
|
157
153
|
end
|
158
|
-
if call_data.
|
154
|
+
if call_data.key?(:www_body_params)
|
159
155
|
req.body = URI.encode_www_form(call_data[:www_body_params])
|
160
|
-
Log.log.debug
|
156
|
+
Log.log.debug{"body www data=#{req.body.chomp}"}
|
161
157
|
req['Content-Type'] = 'application/x-www-form-urlencoded'
|
162
158
|
end
|
163
|
-
if call_data.
|
159
|
+
if call_data.key?(:text_body_params)
|
164
160
|
req.body = call_data[:text_body_params]
|
165
|
-
Log.log.debug
|
161
|
+
Log.log.debug{"body data=#{req.body.chomp}"}
|
166
162
|
end
|
167
163
|
# set headers
|
168
|
-
if call_data.
|
169
|
-
call_data[:headers].
|
164
|
+
if call_data.key?(:headers)
|
165
|
+
call_data[:headers].each_key do |key|
|
170
166
|
req[key] = call_data[:headers][key]
|
171
167
|
end
|
172
168
|
end
|
173
169
|
# :type = :basic
|
174
|
-
req.basic_auth(call_data[:auth][:username],call_data[:auth][:password]) if call_data[:auth][:type].eql?(:basic)
|
170
|
+
req.basic_auth(call_data[:auth][:username], call_data[:auth][:password]) if call_data[:auth][:type].eql?(:basic)
|
175
171
|
return req
|
176
172
|
end
|
177
173
|
|
@@ -199,7 +195,7 @@ module Aspera
|
|
199
195
|
def call(call_data)
|
200
196
|
raise "Hash call parameter is required (#{call_data.class})" unless call_data.is_a?(Hash)
|
201
197
|
call_data[:subpath] = '' if call_data[:subpath].nil?
|
202
|
-
Log.log.debug
|
198
|
+
Log.log.debug{"accessing #{call_data[:subpath]}".red.bold.bg_green}
|
203
199
|
call_data[:headers] ||= {}
|
204
200
|
call_data[:headers]['User-Agent'] ||= self.class.user_agent
|
205
201
|
# defaults from @params are overriden by call data
|
@@ -211,7 +207,7 @@ module Aspera
|
|
211
207
|
Log.log.debug('using Basic auth')
|
212
208
|
# done in build_req
|
213
209
|
when :oauth2
|
214
|
-
call_data[:headers]['Authorization'] = oauth_token unless call_data[:headers].
|
210
|
+
call_data[:headers]['Authorization'] = oauth_token unless call_data[:headers].key?('Authorization')
|
215
211
|
when :url
|
216
212
|
call_data[:url_params] ||= {}
|
217
213
|
call_data[:auth][:url_creds].each do |key, value|
|
@@ -220,7 +216,7 @@ module Aspera
|
|
220
216
|
else raise "unsupported auth type: [#{call_data[:auth][:type]}]"
|
221
217
|
end
|
222
218
|
req = build_request(call_data)
|
223
|
-
Log.log.debug
|
219
|
+
Log.log.debug{"call_data = #{call_data}"}
|
224
220
|
result = {http: nil}
|
225
221
|
# start a block to be able to retry the actual HTTP request
|
226
222
|
begin
|
@@ -242,11 +238,11 @@ module Aspera
|
|
242
238
|
target_file = call_data[:save_to_file]
|
243
239
|
# override user's path to path in header
|
244
240
|
if !response['Content-Disposition'].nil? && (m = response['Content-Disposition'].match(/filename="([^"]+)"/))
|
245
|
-
target_file = File.join(File.dirname(target_file),m[1])
|
241
|
+
target_file = File.join(File.dirname(target_file), m[1])
|
246
242
|
end
|
247
243
|
# download with temp filename
|
248
244
|
target_file_tmp = "#{target_file}#{self.class.download_partial_suffix}"
|
249
|
-
Log.log.debug
|
245
|
+
Log.log.debug{"saving to: #{target_file}"}
|
250
246
|
File.open(target_file_tmp, 'wb') do |file|
|
251
247
|
result[:http].read_body do |fragment|
|
252
248
|
file.write(fragment)
|
@@ -262,17 +258,17 @@ module Aspera
|
|
262
258
|
end
|
263
259
|
# sometimes there is a UTF8 char (e.g. (c) )
|
264
260
|
result[:http].body.force_encoding('UTF-8') if result[:http].body.is_a?(String)
|
265
|
-
Log.log.debug
|
261
|
+
Log.log.debug{"result: body=#{result[:http].body}"}
|
266
262
|
result_mime = (result[:http]['Content-Type'] || 'text/plain').split(';').first
|
267
263
|
result[:data] = case result_mime
|
268
|
-
when 'application/json','application/vnd.api+json'
|
264
|
+
when 'application/json', 'application/vnd.api+json'
|
269
265
|
JSON.parse(result[:http].body) rescue nil
|
270
|
-
else #when 'text/plain'
|
266
|
+
else # when 'text/plain'
|
271
267
|
result[:http].body
|
272
268
|
end
|
273
|
-
Log.dump("result: parsed: #{result_mime}",result[:data])
|
274
|
-
Log.log.debug
|
275
|
-
RestErrorAnalyzer.instance.raise_on_error(req,result)
|
269
|
+
Log.dump("result: parsed: #{result_mime}", result[:data])
|
270
|
+
Log.log.debug{"result: code=#{result[:http].code}"}
|
271
|
+
RestErrorAnalyzer.instance.raise_on_error(req, result)
|
276
272
|
rescue RestCallError => e
|
277
273
|
# not authorized: oauth token expired
|
278
274
|
if call_data[:not_auth_codes].include?(result[:http].code.to_s) && call_data[:auth][:type].eql?(:oauth2)
|
@@ -285,16 +281,16 @@ module Aspera
|
|
285
281
|
# regenerate a brand new token
|
286
282
|
req['Authorization'] = oauth_token(use_cache: false)
|
287
283
|
end
|
288
|
-
Log.log.debug
|
284
|
+
Log.log.debug{"using new token=#{call_data[:headers]['Authorization']}"}
|
289
285
|
retry unless (oauth_tries -= 1).zero?
|
290
286
|
end # if oauth
|
291
287
|
# moved ?
|
292
288
|
if e.response.is_a?(Net::HTTPRedirection) && tries_remain_redirect.positive?
|
293
289
|
tries_remain_redirect -= 1
|
294
290
|
current_uri = URI.parse(call_data[:base_url])
|
295
|
-
new_url=e.response['location']
|
296
|
-
new_url="#{current_uri.scheme}:#{new_url}" unless new_url.start_with?('http')
|
297
|
-
Log.log.info
|
291
|
+
new_url = e.response['location']
|
292
|
+
new_url = "#{current_uri.scheme}:#{new_url}" unless new_url.start_with?('http')
|
293
|
+
Log.log.info{"URL is moved: #{new_url}"}
|
298
294
|
redir_uri = URI.parse(new_url)
|
299
295
|
call_data[:base_url] = new_url
|
300
296
|
call_data[:subpath] = ''
|
@@ -303,14 +299,14 @@ module Aspera
|
|
303
299
|
retry
|
304
300
|
else
|
305
301
|
# change host
|
306
|
-
Log.log.info
|
302
|
+
Log.log.info{"Redirect changes host: #{current_uri.host} -> #{redir_uri.host}"}
|
307
303
|
return self.class.new(call_data).call(call_data)
|
308
304
|
end
|
309
305
|
end
|
310
306
|
# raise exception if could not retry and not return error in result
|
311
307
|
raise e unless call_data[:return_error]
|
312
308
|
end # begin request
|
313
|
-
Log.log.debug
|
309
|
+
Log.log.debug{"result=#{result}"}
|
314
310
|
return result
|
315
311
|
end
|
316
312
|
|
@@ -319,24 +315,24 @@ module Aspera
|
|
319
315
|
#
|
320
316
|
|
321
317
|
# @param encoding : one of: :json_params, :url_params
|
322
|
-
def create(subpath,params,encoding=:json_params)
|
323
|
-
return call({operation: 'POST',subpath: subpath,headers: {'Accept' => 'application/json'},encoding => params})
|
318
|
+
def create(subpath, params, encoding=:json_params)
|
319
|
+
return call({operation: 'POST', subpath: subpath, headers: {'Accept' => 'application/json'}, encoding => params})
|
324
320
|
end
|
325
321
|
|
326
|
-
def read(subpath,args=nil)
|
327
|
-
return call({operation: 'GET',subpath: subpath,headers: {'Accept' => 'application/json'},url_params: args})
|
322
|
+
def read(subpath, args=nil)
|
323
|
+
return call({operation: 'GET', subpath: subpath, headers: {'Accept' => 'application/json'}, url_params: args})
|
328
324
|
end
|
329
325
|
|
330
|
-
def update(subpath,params)
|
331
|
-
return call({operation: 'PUT',subpath: subpath,headers: {'Accept' => 'application/json'},json_params: params})
|
326
|
+
def update(subpath, params)
|
327
|
+
return call({operation: 'PUT', subpath: subpath, headers: {'Accept' => 'application/json'}, json_params: params})
|
332
328
|
end
|
333
329
|
|
334
330
|
def delete(subpath)
|
335
|
-
return call({operation: 'DELETE',subpath: subpath,headers: {'Accept' => 'application/json'}})
|
331
|
+
return call({operation: 'DELETE', subpath: subpath, headers: {'Accept' => 'application/json'}})
|
336
332
|
end
|
337
333
|
|
338
334
|
def cancel(subpath)
|
339
|
-
return call({operation: 'CANCEL',subpath: subpath,headers: {'Accept' => 'application/json'}})
|
335
|
+
return call({operation: 'CANCEL', subpath: subpath, headers: {'Accept' => 'application/json'}})
|
340
336
|
end
|
341
337
|
end
|
342
|
-
end #module Aspera
|
338
|
+
end # module Aspera
|
@@ -4,8 +4,9 @@ module Aspera
|
|
4
4
|
# raised on error after REST call
|
5
5
|
class RestCallError < StandardError
|
6
6
|
attr_accessor :request, :response
|
7
|
+
|
7
8
|
# @param http response
|
8
|
-
def initialize(req,resp,msg)
|
9
|
+
def initialize(req, resp, msg)
|
9
10
|
@request = req
|
10
11
|
@response = resp
|
11
12
|
super(msg)
|
@@ -9,15 +9,16 @@ module Aspera
|
|
9
9
|
class RestErrorAnalyzer
|
10
10
|
include Singleton
|
11
11
|
attr_accessor :log_file
|
12
|
+
|
12
13
|
# the singleton object is registered with application specific handlers
|
13
14
|
def initialize
|
14
15
|
# list of handlers
|
15
16
|
@error_handlers = []
|
16
17
|
@log_file = nil
|
17
|
-
add_handler('Type Generic') do |type,call_context|
|
18
|
+
add_handler('Type Generic') do |type, call_context|
|
18
19
|
if !call_context[:response].code.start_with?('2')
|
19
20
|
# add generic information
|
20
|
-
RestErrorAnalyzer.add_error(call_context,type,"#{call_context[:request]['host']} #{call_context[:response].code} #{call_context[:response].message}")
|
21
|
+
RestErrorAnalyzer.add_error(call_context, type, "#{call_context[:request]['host']} #{call_context[:response].code} #{call_context[:response].message}")
|
21
22
|
end
|
22
23
|
end
|
23
24
|
end
|
@@ -25,7 +26,7 @@ module Aspera
|
|
25
26
|
# Use this method to analyze a EST result and raise an exception
|
26
27
|
# Analyzes REST call response and raises a RestCallError exception
|
27
28
|
# if HTTP result code is not 2XX
|
28
|
-
def raise_on_error(req,res)
|
29
|
+
def raise_on_error(req, res)
|
29
30
|
call_context = {
|
30
31
|
messages: [],
|
31
32
|
request: req,
|
@@ -36,14 +37,14 @@ module Aspera
|
|
36
37
|
# analyze errors from provided handlers
|
37
38
|
# note that there can be an error even if code is 2XX
|
38
39
|
@error_handlers.each do |handler|
|
39
|
-
begin
|
40
|
-
#Log.log.debug
|
41
|
-
handler[:block].call(handler[:name],call_context)
|
40
|
+
begin # rubocop:disable Style/RedundantBegin
|
41
|
+
# Log.log.debug{"test exception: #{handler[:name]}"}
|
42
|
+
handler[:block].call(handler[:name], call_context)
|
42
43
|
rescue StandardError => e
|
43
|
-
Log.log.error
|
44
|
+
Log.log.error{"ERROR in handler:\n#{e.message}\n#{e.backtrace}"}
|
44
45
|
end
|
45
46
|
end
|
46
|
-
raise RestCallError.new(call_context[:request],call_context[:response],call_context[:messages].join("\n")) unless call_context[:messages].empty?
|
47
|
+
raise RestCallError.new(call_context[:request], call_context[:response], call_context[:messages].join("\n")) unless call_context[:messages].empty?
|
47
48
|
end
|
48
49
|
|
49
50
|
# add a new error handler (done at application initialisation)
|
@@ -51,18 +52,18 @@ module Aspera
|
|
51
52
|
# @param block : processing of response: takes two parameters: name, call_context
|
52
53
|
# name is the one provided here
|
53
54
|
# call_context is built in method raise_on_error
|
54
|
-
def add_handler(name
|
55
|
+
def add_handler(name, &block)
|
55
56
|
@error_handlers.unshift({name: name, block: block})
|
56
57
|
end
|
57
58
|
|
58
59
|
# add a simple error handler
|
59
60
|
# check that key exists and is string under specified path (hash)
|
60
61
|
# adds other keys as secondary information
|
61
|
-
def add_simple_handler(name
|
62
|
-
add_handler(name) do |type,call_context|
|
62
|
+
def add_simple_handler(name, *args)
|
63
|
+
add_handler(name) do |type, call_context|
|
63
64
|
# need to clone because we modify and same array is used subsequently
|
64
65
|
path = args.clone
|
65
|
-
#Log.log.debug
|
66
|
+
# Log.log.debug{"path=#{path}"}
|
66
67
|
# if last in path is boolean it tells if the error is only with http error code or always
|
67
68
|
always = [true, false].include?(path.last) ? path.pop : false
|
68
69
|
if call_context[:data].is_a?(Hash) && (!call_context[:response].code.start_with?('2') || always)
|
@@ -70,10 +71,10 @@ module Aspera
|
|
70
71
|
# dig and find sub entry corresponding to path in deep hash
|
71
72
|
error_struct = path.inject(call_context[:data]) { |subhash, key| subhash.respond_to?(:keys) ? subhash[key] : nil }
|
72
73
|
if error_struct.is_a?(Hash) && error_struct[msg_key].is_a?(String)
|
73
|
-
RestErrorAnalyzer.add_error(call_context,type,error_struct[msg_key])
|
74
|
-
error_struct.each do |k,v|
|
74
|
+
RestErrorAnalyzer.add_error(call_context, type, error_struct[msg_key])
|
75
|
+
error_struct.each do |k, v|
|
75
76
|
next if k.eql?(msg_key)
|
76
|
-
RestErrorAnalyzer.add_error(call_context,"#{type}(sub)","#{k}: #{v}") if [String,Integer].include?(v.class)
|
77
|
+
RestErrorAnalyzer.add_error(call_context, "#{type}(sub)", "#{k}: #{v}") if [String, Integer].include?(v.class)
|
77
78
|
end
|
78
79
|
end
|
79
80
|
end
|
@@ -86,12 +87,12 @@ module Aspera
|
|
86
87
|
# @param call_context a Hash containing the result call_context, provided to handler
|
87
88
|
# @param type a string describing type of exception, for logging purpose
|
88
89
|
# @param msg one error message to add to list
|
89
|
-
def add_error(call_context,type,msg)
|
90
|
+
def add_error(call_context, type, msg)
|
90
91
|
call_context[:messages].push(msg)
|
91
92
|
logfile = instance.log_file
|
92
93
|
# log error for further analysis (file must exist to activate)
|
93
94
|
return if logfile.nil? || !File.exist?(logfile)
|
94
|
-
File.open(logfile,'a+') do |f|
|
95
|
+
File.open(logfile, 'a+') do |f|
|
95
96
|
f.write("\n=#{type}=====\n#{call_context[:request].method} #{call_context[:request].path}\n#{call_context[:response].code}\n"\
|
96
97
|
"#{JSON.generate(call_context[:data])}\n#{call_context[:messages].join("\n")}")
|
97
98
|
end
|
@@ -12,45 +12,45 @@ module Aspera
|
|
12
12
|
Log.log.debug('registering Aspera REST error handlers')
|
13
13
|
# Faspex 4: both user_message and internal_message, and code 200
|
14
14
|
# example: missing meta data on package creation
|
15
|
-
RestErrorAnalyzer.instance.add_simple_handler('Type 1: error:user_message','error','user_message',true)
|
16
|
-
RestErrorAnalyzer.instance.add_simple_handler('Type 2: error:description','error','description')
|
17
|
-
RestErrorAnalyzer.instance.add_simple_handler('Type 3: error:internal_message','error','internal_message')
|
15
|
+
RestErrorAnalyzer.instance.add_simple_handler('Type 1: error:user_message', 'error', 'user_message', true)
|
16
|
+
RestErrorAnalyzer.instance.add_simple_handler('Type 2: error:description', 'error', 'description')
|
17
|
+
RestErrorAnalyzer.instance.add_simple_handler('Type 3: error:internal_message', 'error', 'internal_message')
|
18
18
|
# AoC Automation
|
19
|
-
RestErrorAnalyzer.instance.add_simple_handler('AoC Automation','error')
|
20
|
-
RestErrorAnalyzer.instance.add_simple_handler('Type 5','error_description')
|
21
|
-
RestErrorAnalyzer.instance.add_simple_handler('Type 6','message')
|
22
|
-
RestErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |name,call_context|
|
19
|
+
RestErrorAnalyzer.instance.add_simple_handler('AoC Automation', 'error')
|
20
|
+
RestErrorAnalyzer.instance.add_simple_handler('Type 5', 'error_description')
|
21
|
+
RestErrorAnalyzer.instance.add_simple_handler('Type 6', 'message')
|
22
|
+
RestErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |name, call_context|
|
23
23
|
if call_context[:data].is_a?(Hash) && call_context[:data]['errors'].is_a?(Hash)
|
24
|
-
call_context[:data]['errors'].each do |k,v|
|
25
|
-
RestErrorAnalyzer.add_error(call_context,name,"#{k}: #{v}")
|
24
|
+
call_context[:data]['errors'].each do |k, v|
|
25
|
+
RestErrorAnalyzer.add_error(call_context, name, "#{k}: #{v}")
|
26
26
|
end
|
27
27
|
end
|
28
28
|
end
|
29
29
|
# call to upload_setup and download_setup of node api
|
30
|
-
RestErrorAnalyzer.instance.add_handler('T8:node: *_setup') do |type,call_context|
|
30
|
+
RestErrorAnalyzer.instance.add_handler('T8:node: *_setup') do |type, call_context|
|
31
31
|
if call_context[:data].is_a?(Hash)
|
32
32
|
d_t_s = call_context[:data]['transfer_specs']
|
33
33
|
if d_t_s.is_a?(Array)
|
34
34
|
d_t_s.each do |res|
|
35
|
-
#r_err=res['transfer_spec']['error']
|
35
|
+
# r_err=res['transfer_spec']['error']
|
36
36
|
r_err = res['error']
|
37
37
|
if r_err.is_a?(Hash)
|
38
|
-
RestErrorAnalyzer.add_error(call_context,type,"#{r_err['code']}: #{r_err['reason']}: #{r_err['user_message']}")
|
38
|
+
RestErrorAnalyzer.add_error(call_context, type, "#{r_err['code']}: #{r_err['reason']}: #{r_err['user_message']}")
|
39
39
|
end
|
40
40
|
end
|
41
41
|
end
|
42
42
|
end
|
43
43
|
end
|
44
|
-
RestErrorAnalyzer.instance.add_simple_handler('T9:IBM cloud IAM','errorMessage')
|
45
|
-
RestErrorAnalyzer.instance.add_simple_handler('T10:faspex v4','user_message')
|
46
|
-
RestErrorAnalyzer.instance.add_handler('bss graphql') do |type,call_context|
|
44
|
+
RestErrorAnalyzer.instance.add_simple_handler('T9:IBM cloud IAM', 'errorMessage')
|
45
|
+
RestErrorAnalyzer.instance.add_simple_handler('T10:faspex v4', 'user_message')
|
46
|
+
RestErrorAnalyzer.instance.add_handler('bss graphql') do |type, call_context|
|
47
47
|
if call_context[:data].is_a?(Hash)
|
48
48
|
d_t_s = call_context[:data]['errors']
|
49
49
|
if d_t_s.is_a?(Array)
|
50
50
|
d_t_s.each do |res|
|
51
51
|
r_err = res['message']
|
52
52
|
if r_err.is_a?(String)
|
53
|
-
RestErrorAnalyzer.add_error(call_context,type,r_err)
|
53
|
+
RestErrorAnalyzer.add_error(call_context, type, r_err)
|
54
54
|
end
|
55
55
|
end
|
56
56
|
end
|
data/lib/aspera/secret_hider.rb
CHANGED
@@ -7,16 +7,17 @@ module Aspera
|
|
7
7
|
class SecretHider
|
8
8
|
# display string for hidden secrets
|
9
9
|
HIDDEN_PASSWORD = '🔑'
|
10
|
+
# env vars for ascp with secrets
|
11
|
+
ASCP_ENV_SECRETS = %w[ASPERA_SCP_PASS ASPERA_SCP_KEY ASPERA_SCP_FILEPASS ASPERA_PROXY_PASS ASPERA_SCP_TOKEN].freeze
|
10
12
|
# keys in hash that contain secrets
|
11
|
-
|
12
|
-
|
13
|
-
ALL_SECRETS =[ASCP_SECRETS,KEY_SECRETS].flatten.freeze
|
13
|
+
KEY_SECRETS = %w[password secret private_key passphrase].freeze
|
14
|
+
ALL_SECRETS = [].concat(ASCP_ENV_SECRETS, KEY_SECRETS).freeze
|
14
15
|
# regex that define namec captures :begin and :end
|
15
|
-
REGEX_LOG_REPLACES=[
|
16
|
+
REGEX_LOG_REPLACES = [
|
16
17
|
# CLI manager get/set options
|
17
18
|
/(?<begin>[sg]et (#{KEY_SECRETS.join('|')})=).*(?<end>)/,
|
18
19
|
# env var ascp exec
|
19
|
-
/(?<begin> (#{
|
20
|
+
/(?<begin> (#{ASCP_ENV_SECRETS.join('|')})=)(\\.|[^ ])*(?<end> )/,
|
20
21
|
# rendered JSON
|
21
22
|
/(?<begin>["':][^"]*(#{ALL_SECRETS.join('|')})[^"]*["']?[=>: ]+")[^"]+(?<end>")/,
|
22
23
|
# option "secret"
|
@@ -26,13 +27,14 @@ module Aspera
|
|
26
27
|
# private key values
|
27
28
|
/(?<begin>--+BEGIN .+ KEY--+)[[:ascii:]]+?(?<end>--+?END .+ KEY--+)/
|
28
29
|
].freeze
|
29
|
-
private_constant :HIDDEN_PASSWORD
|
30
|
+
private_constant :HIDDEN_PASSWORD, :ASCP_ENV_SECRETS, :KEY_SECRETS, :ALL_SECRETS, :REGEX_LOG_REPLACES
|
30
31
|
@log_secrets = false
|
31
32
|
class << self
|
32
33
|
attr_accessor :log_secrets
|
34
|
+
|
33
35
|
def log_formatter(original_formatter)
|
34
36
|
original_formatter ||= Logger::Formatter.new
|
35
|
-
#
|
37
|
+
# NOTE: that @log_secrets may be set AFTER this init is done, so it's done at runtime
|
36
38
|
return lambda do |severity, datetime, progname, msg|
|
37
39
|
if msg.is_a?(String) && !@log_secrets
|
38
40
|
REGEX_LOG_REPLACES.each do |regx|
|
@@ -43,25 +45,25 @@ module Aspera
|
|
43
45
|
end
|
44
46
|
end
|
45
47
|
|
46
|
-
def secret?(keyword,value)
|
47
|
-
keyword=keyword.to_s if keyword.is_a?(Symbol)
|
48
|
+
def secret?(keyword, value)
|
49
|
+
keyword = keyword.to_s if keyword.is_a?(Symbol)
|
48
50
|
# only Strings can be secrets, not booleans, or hash, arrays
|
49
51
|
keyword.is_a?(String) && ALL_SECRETS.any?{|kw|keyword.include?(kw)} && value.is_a?(String)
|
50
52
|
end
|
51
53
|
|
52
|
-
def deep_remove_secret(obj,is_name_value: false)
|
54
|
+
def deep_remove_secret(obj, is_name_value: false)
|
53
55
|
case obj
|
54
56
|
when Array
|
55
57
|
if is_name_value
|
56
58
|
obj.each do |i|
|
57
|
-
i['value']=HIDDEN_PASSWORD if secret?(i['parameter'],i['value'])
|
59
|
+
i['value'] = HIDDEN_PASSWORD if secret?(i['parameter'], i['value'])
|
58
60
|
end
|
59
61
|
else
|
60
62
|
obj.each{|i|deep_remove_secret(i)}
|
61
63
|
end
|
62
64
|
when Hash
|
63
|
-
obj.each do |k,v|
|
64
|
-
if secret?(k,v)
|
65
|
+
obj.each do |k, v|
|
66
|
+
if secret?(k, v)
|
65
67
|
obj[k] = HIDDEN_PASSWORD
|
66
68
|
elsif obj[k].is_a?(Hash)
|
67
69
|
deep_remove_secret(obj[k])
|
data/lib/aspera/ssh.rb
CHANGED
@@ -2,9 +2,9 @@
|
|
2
2
|
|
3
3
|
require 'net/ssh'
|
4
4
|
|
5
|
-
#
|
5
|
+
# HACK: deactivate ed25519 and ecdsa private keys from ssh identities, as it usually hurts
|
6
6
|
begin
|
7
|
-
module Net;module SSH;module Authentication;class Session;private; def default_keys; %w[~/.ssh/id_dsa ~/.ssh/id_rsa ~/.ssh2/id_dsa ~/.ssh2/id_rsa];end;end;end;end;end # rubocop:disable Layout/AccessModifierIndentation, Layout/EmptyLinesAroundAccessModifier, Layout/LineLength
|
7
|
+
module Net; module SSH; module Authentication; class Session; private; def default_keys; %w[~/.ssh/id_dsa ~/.ssh/id_rsa ~/.ssh2/id_dsa ~/.ssh2/id_rsa]; end; end; end; end; end # rubocop:disable Layout/AccessModifierIndentation, Layout/EmptyLinesAroundAccessModifier, Layout/LineLength, Style/Semicolon
|
8
8
|
rescue StandardError
|
9
9
|
# ignore errors
|
10
10
|
end
|
@@ -15,26 +15,26 @@ module Aspera
|
|
15
15
|
class Ssh
|
16
16
|
# ssh_options: same as Net::SSH.start
|
17
17
|
# see: https://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start
|
18
|
-
def initialize(host,username,ssh_options)
|
19
|
-
Log.log.debug
|
20
|
-
Log.log.debug
|
18
|
+
def initialize(host, username, ssh_options)
|
19
|
+
Log.log.debug{"ssh:#{username}@#{host}"}
|
20
|
+
Log.log.debug{"ssh_options:#{ssh_options}"}
|
21
21
|
@host = host
|
22
22
|
@username = username
|
23
23
|
@ssh_options = ssh_options
|
24
24
|
@ssh_options[:logger] = Log.log
|
25
25
|
end
|
26
26
|
|
27
|
-
def execute(cmd,input=nil)
|
27
|
+
def execute(cmd, input=nil)
|
28
28
|
if cmd.is_a?(Array)
|
29
29
|
# concatenate arguments, enclose in double quotes
|
30
30
|
cmd = cmd.map{|v|%Q("#{v}")}.join(' ')
|
31
31
|
end
|
32
|
-
Log.log.debug
|
32
|
+
Log.log.debug{"cmd=#{cmd}"}
|
33
33
|
response = []
|
34
34
|
Net::SSH.start(@host, @username, @ssh_options) do |session|
|
35
35
|
ssh_channel = session.open_channel do |channel|
|
36
36
|
# prepare stdout processing
|
37
|
-
channel.on_data{|_chan,data|response.push(data)}
|
37
|
+
channel.on_data{|_chan, data|response.push(data)}
|
38
38
|
# prepare stderr processing, stderr if type = 1
|
39
39
|
channel.on_extended_data do |_chan, _type, data|
|
40
40
|
errormsg = "#{cmd}: [#{data.chomp}]"
|
@@ -45,13 +45,14 @@ module Aspera
|
|
45
45
|
raise errormsg
|
46
46
|
end
|
47
47
|
# send commannd to SSH channel (execute)
|
48
|
-
channel.send('cexe'.reverse,cmd){|_ch,_success|channel.send_data(input) unless input.nil?}
|
48
|
+
channel.send('cexe'.reverse, cmd){|_ch, _success|channel.send_data(input) unless input.nil?}
|
49
49
|
end
|
50
50
|
# wait for channel to finish (command exit)
|
51
51
|
ssh_channel.wait
|
52
52
|
# main ssh session loop
|
53
53
|
session.loop
|
54
|
-
end #
|
54
|
+
end # start
|
55
|
+
# response as single string
|
55
56
|
return response.join
|
56
57
|
end
|
57
58
|
end
|