aspera-cli 4.10.0 → 4.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +20 -0
- data/CHANGELOG.md +509 -0
- data/CONTRIBUTING.md +118 -0
- data/README.md +621 -378
- data/bin/ascli +4 -4
- data/bin/asession +11 -11
- data/docs/test_env.conf +28 -19
- data/examples/aoc.rb +4 -4
- data/examples/dascli +11 -9
- data/examples/faspex4.rb +8 -8
- data/examples/node.rb +11 -11
- data/examples/server.rb +9 -9
- data/lib/aspera/aoc.rb +273 -266
- data/lib/aspera/ascmd.rb +56 -54
- data/lib/aspera/ats_api.rb +4 -4
- data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
- data/lib/aspera/cli/extended_value.rb +5 -5
- data/lib/aspera/cli/formater.rb +64 -64
- data/lib/aspera/cli/listener/line_dump.rb +1 -1
- data/lib/aspera/cli/listener/logger.rb +1 -1
- data/lib/aspera/cli/listener/progress.rb +5 -6
- data/lib/aspera/cli/listener/progress_multi.rb +14 -19
- data/lib/aspera/cli/main.rb +66 -67
- data/lib/aspera/cli/manager.rb +110 -110
- data/lib/aspera/cli/plugin.rb +54 -37
- data/lib/aspera/cli/plugins/alee.rb +4 -4
- data/lib/aspera/cli/plugins/aoc.rb +308 -669
- data/lib/aspera/cli/plugins/ats.rb +44 -46
- data/lib/aspera/cli/plugins/bss.rb +10 -10
- data/lib/aspera/cli/plugins/config.rb +447 -344
- data/lib/aspera/cli/plugins/console.rb +12 -12
- data/lib/aspera/cli/plugins/cos.rb +18 -20
- data/lib/aspera/cli/plugins/faspex.rb +110 -112
- data/lib/aspera/cli/plugins/faspex5.rb +67 -46
- data/lib/aspera/cli/plugins/node.rb +364 -288
- data/lib/aspera/cli/plugins/orchestrator.rb +46 -46
- data/lib/aspera/cli/plugins/preview.rb +122 -114
- data/lib/aspera/cli/plugins/server.rb +137 -83
- data/lib/aspera/cli/plugins/shares.rb +30 -29
- data/lib/aspera/cli/plugins/sync.rb +13 -33
- data/lib/aspera/cli/transfer_agent.rb +57 -57
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +3 -3
- data/lib/aspera/command_line_builder.rb +27 -27
- data/lib/aspera/cos_node.rb +22 -20
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +30 -28
- data/lib/aspera/fasp/agent_base.rb +15 -15
- data/lib/aspera/fasp/agent_connect.rb +23 -21
- data/lib/aspera/fasp/agent_direct.rb +65 -67
- data/lib/aspera/fasp/agent_httpgw.rb +72 -68
- data/lib/aspera/fasp/agent_node.rb +23 -21
- data/lib/aspera/fasp/agent_trsdk.rb +20 -20
- data/lib/aspera/fasp/error.rb +3 -2
- data/lib/aspera/fasp/error_info.rb +11 -8
- data/lib/aspera/fasp/installation.rb +78 -78
- data/lib/aspera/fasp/listener.rb +1 -1
- data/lib/aspera/fasp/parameters.rb +75 -72
- data/lib/aspera/fasp/parameters.yaml +2 -2
- data/lib/aspera/fasp/resume_policy.rb +8 -8
- data/lib/aspera/fasp/transfer_spec.rb +35 -2
- data/lib/aspera/fasp/uri.rb +7 -7
- data/lib/aspera/faspex_gw.rb +7 -5
- data/lib/aspera/hash_ext.rb +3 -3
- data/lib/aspera/id_generator.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +23 -28
- data/lib/aspera/keychain/macos_security.rb +21 -20
- data/lib/aspera/log.rb +7 -7
- data/lib/aspera/nagios.rb +19 -18
- data/lib/aspera/node.rb +209 -35
- data/lib/aspera/oauth.rb +37 -36
- data/lib/aspera/open_application.rb +19 -11
- data/lib/aspera/persistency_action_once.rb +4 -4
- data/lib/aspera/persistency_folder.rb +13 -13
- data/lib/aspera/preview/file_types.rb +8 -8
- data/lib/aspera/preview/generator.rb +67 -67
- data/lib/aspera/preview/utils.rb +27 -27
- data/lib/aspera/proxy_auto_config.js +41 -41
- data/lib/aspera/proxy_auto_config.rb +16 -16
- data/lib/aspera/rest.rb +56 -60
- data/lib/aspera/rest_call_error.rb +2 -1
- data/lib/aspera/rest_error_analyzer.rb +18 -17
- data/lib/aspera/rest_errors_aspera.rb +16 -16
- data/lib/aspera/secret_hider.rb +15 -13
- data/lib/aspera/ssh.rb +11 -10
- data/lib/aspera/sync.rb +158 -44
- data/lib/aspera/temp_file_manager.rb +2 -2
- data/lib/aspera/uri_reader.rb +4 -4
- data/lib/aspera/web_auth.rb +14 -13
- data.tar.gz.sig +0 -0
- metadata +8 -5
- metadata.gz.sig +0 -0
data/lib/aspera/rest.rb
CHANGED
|
@@ -13,18 +13,12 @@ require 'cgi'
|
|
|
13
13
|
require 'ruby-progressbar'
|
|
14
14
|
|
|
15
15
|
# add cancel method to http
|
|
16
|
-
class Net::HTTP::Cancel < Net::HTTPRequest
|
|
16
|
+
class Net::HTTP::Cancel < Net::HTTPRequest # rubocop:disable Style/ClassAndModuleChildren
|
|
17
17
|
METHOD = 'CANCEL'
|
|
18
18
|
REQUEST_HAS_BODY = false
|
|
19
19
|
RESPONSE_HAS_BODY = false
|
|
20
20
|
end
|
|
21
21
|
|
|
22
|
-
#class Net::HTTP::Delete < Net::HTTPRequest
|
|
23
|
-
# METHOD = 'DELETE'
|
|
24
|
-
# REQUEST_HAS_BODY = false
|
|
25
|
-
# RESPONSE_HAS_BODY = false
|
|
26
|
-
#end
|
|
27
|
-
|
|
28
22
|
module Aspera
|
|
29
23
|
# a simple class to make HTTP calls, equivalent to rest-client
|
|
30
24
|
# rest call errors are raised as exception RestCallError
|
|
@@ -44,15 +38,18 @@ module Aspera
|
|
|
44
38
|
|
|
45
39
|
class << self
|
|
46
40
|
# define accessors
|
|
47
|
-
@@global.
|
|
41
|
+
@@global.each_key do |p|
|
|
48
42
|
define_method(p){@@global[p]}
|
|
49
|
-
define_method("#{p}=")
|
|
43
|
+
define_method("#{p}=") do |val|
|
|
44
|
+
Log.log.debug{"#{p} => #{val}".red}
|
|
45
|
+
@@global[p] = val
|
|
46
|
+
end
|
|
50
47
|
end
|
|
51
48
|
|
|
52
|
-
def basic_creds(user,pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}";end
|
|
49
|
+
def basic_creds(user, pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}"; end
|
|
53
50
|
|
|
54
51
|
# build URI from URL and parameters and check it is http or https
|
|
55
|
-
def build_uri(url,params=nil)
|
|
52
|
+
def build_uri(url, params=nil)
|
|
56
53
|
uri = URI.parse(url)
|
|
57
54
|
raise "REST endpoint shall be http/s not #{uri.scheme}" unless %w[http https].include?(uri.scheme)
|
|
58
55
|
if !params.nil?
|
|
@@ -60,15 +57,15 @@ module Aspera
|
|
|
60
57
|
if params.is_a?(Hash)
|
|
61
58
|
orig = params
|
|
62
59
|
params = []
|
|
63
|
-
orig.each do |k,v|
|
|
60
|
+
orig.each do |k, v|
|
|
64
61
|
case v
|
|
65
62
|
when Array
|
|
66
63
|
suffix = v.first.eql?('[]') ? v.shift : ''
|
|
67
64
|
v.each do |e|
|
|
68
|
-
params.push([k + suffix,e])
|
|
65
|
+
params.push([k + suffix, e])
|
|
69
66
|
end
|
|
70
67
|
else
|
|
71
|
-
params.push([k,v])
|
|
68
|
+
params.push([k, v])
|
|
72
69
|
end
|
|
73
70
|
end
|
|
74
71
|
end
|
|
@@ -82,8 +79,8 @@ module Aspera
|
|
|
82
79
|
uri = build_uri(base_url)
|
|
83
80
|
# this honors http_proxy env var
|
|
84
81
|
http_session = Net::HTTP.new(uri.host, uri.port)
|
|
85
|
-
http_session.proxy_user=proxy_user
|
|
86
|
-
http_session.proxy_pass=proxy_pass
|
|
82
|
+
http_session.proxy_user = proxy_user
|
|
83
|
+
http_session.proxy_pass = proxy_pass
|
|
87
84
|
http_session.use_ssl = uri.scheme.eql?('https')
|
|
88
85
|
http_session.set_debug_output($stdout) if debug
|
|
89
86
|
# set http options in callback, such as timeout and cert. verification
|
|
@@ -99,7 +96,7 @@ module Aspera
|
|
|
99
96
|
# create and start keep alive connection on demand
|
|
100
97
|
def http_session
|
|
101
98
|
if @http_session.nil?
|
|
102
|
-
@http_session=self.class.start_http_session(@params[:base_url])
|
|
99
|
+
@http_session = self.class.start_http_session(@params[:base_url])
|
|
103
100
|
end
|
|
104
101
|
return @http_session
|
|
105
102
|
end
|
|
@@ -121,57 +118,56 @@ module Aspera
|
|
|
121
118
|
raise 'ERROR: expecting Hash' unless a_rest_params.is_a?(Hash)
|
|
122
119
|
raise 'ERROR: expecting base_url' unless a_rest_params[:base_url].is_a?(String)
|
|
123
120
|
@params = a_rest_params.clone
|
|
124
|
-
Log.dump('REST params'
|
|
121
|
+
Log.dump('REST params', @params)
|
|
125
122
|
# base url without trailing slashes (note: string may be frozen)
|
|
126
|
-
@params[:base_url] = @params[:base_url].gsub(
|
|
123
|
+
@params[:base_url] = @params[:base_url].gsub(%r{/+$}, '')
|
|
127
124
|
@http_session = nil
|
|
128
125
|
# default is no auth
|
|
129
126
|
@params[:auth] ||= {type: :none}
|
|
130
127
|
@params[:not_auth_codes] ||= ['401']
|
|
131
128
|
@oauth = nil
|
|
132
|
-
Log.dump('REST params(2)'
|
|
129
|
+
Log.dump('REST params(2)', @params)
|
|
133
130
|
end
|
|
134
131
|
|
|
135
132
|
def oauth_token(force_refresh: false)
|
|
136
|
-
raise "ERROR: expecting boolean, have #{force_refresh}" unless [true,false].include?(force_refresh)
|
|
133
|
+
raise "ERROR: expecting boolean, have #{force_refresh}" unless [true, false].include?(force_refresh)
|
|
137
134
|
return oauth.get_authorization(use_refresh_token: force_refresh)
|
|
138
135
|
end
|
|
139
136
|
|
|
140
137
|
def build_request(call_data)
|
|
141
138
|
# TODO: shall we percent encode subpath (spaces) test with access key delete with space in id
|
|
142
139
|
# URI.escape()
|
|
143
|
-
uri = self.class.build_uri("#{call_data[:base_url]}#{['','/'].include?(call_data[:subpath]) ? '' : '/'}#{call_data[:subpath]}",call_data[:url_params])
|
|
144
|
-
Log.log.debug
|
|
140
|
+
uri = self.class.build_uri("#{call_data[:base_url]}#{['', '/'].include?(call_data[:subpath]) ? '' : '/'}#{call_data[:subpath]}", call_data[:url_params])
|
|
141
|
+
Log.log.debug{"URI=#{uri}"}
|
|
145
142
|
begin
|
|
146
143
|
# instanciate request object based on string name
|
|
147
144
|
req = Net::HTTP.const_get(call_data[:operation].capitalize).new(uri)
|
|
148
145
|
rescue NameError
|
|
149
146
|
raise "unsupported operation : #{call_data[:operation]}"
|
|
150
147
|
end
|
|
151
|
-
if call_data.
|
|
148
|
+
if call_data.key?(:json_params) && !call_data[:json_params].nil?
|
|
152
149
|
req.body = JSON.generate(call_data[:json_params])
|
|
153
|
-
Log.dump('body JSON data',call_data[:json_params])
|
|
154
|
-
#Log.log.debug("body JSON data=#{JSON.pretty_generate(call_data[:json_params])}")
|
|
150
|
+
Log.dump('body JSON data', call_data[:json_params])
|
|
155
151
|
req['Content-Type'] = 'application/json'
|
|
156
|
-
#call_data[:headers]['Accept']='application/json'
|
|
152
|
+
# call_data[:headers]['Accept']='application/json'
|
|
157
153
|
end
|
|
158
|
-
if call_data.
|
|
154
|
+
if call_data.key?(:www_body_params)
|
|
159
155
|
req.body = URI.encode_www_form(call_data[:www_body_params])
|
|
160
|
-
Log.log.debug
|
|
156
|
+
Log.log.debug{"body www data=#{req.body.chomp}"}
|
|
161
157
|
req['Content-Type'] = 'application/x-www-form-urlencoded'
|
|
162
158
|
end
|
|
163
|
-
if call_data.
|
|
159
|
+
if call_data.key?(:text_body_params)
|
|
164
160
|
req.body = call_data[:text_body_params]
|
|
165
|
-
Log.log.debug
|
|
161
|
+
Log.log.debug{"body data=#{req.body.chomp}"}
|
|
166
162
|
end
|
|
167
163
|
# set headers
|
|
168
|
-
if call_data.
|
|
169
|
-
call_data[:headers].
|
|
164
|
+
if call_data.key?(:headers)
|
|
165
|
+
call_data[:headers].each_key do |key|
|
|
170
166
|
req[key] = call_data[:headers][key]
|
|
171
167
|
end
|
|
172
168
|
end
|
|
173
169
|
# :type = :basic
|
|
174
|
-
req.basic_auth(call_data[:auth][:username],call_data[:auth][:password]) if call_data[:auth][:type].eql?(:basic)
|
|
170
|
+
req.basic_auth(call_data[:auth][:username], call_data[:auth][:password]) if call_data[:auth][:type].eql?(:basic)
|
|
175
171
|
return req
|
|
176
172
|
end
|
|
177
173
|
|
|
@@ -199,7 +195,7 @@ module Aspera
|
|
|
199
195
|
def call(call_data)
|
|
200
196
|
raise "Hash call parameter is required (#{call_data.class})" unless call_data.is_a?(Hash)
|
|
201
197
|
call_data[:subpath] = '' if call_data[:subpath].nil?
|
|
202
|
-
Log.log.debug
|
|
198
|
+
Log.log.debug{"accessing #{call_data[:subpath]}".red.bold.bg_green}
|
|
203
199
|
call_data[:headers] ||= {}
|
|
204
200
|
call_data[:headers]['User-Agent'] ||= self.class.user_agent
|
|
205
201
|
# defaults from @params are overriden by call data
|
|
@@ -211,7 +207,7 @@ module Aspera
|
|
|
211
207
|
Log.log.debug('using Basic auth')
|
|
212
208
|
# done in build_req
|
|
213
209
|
when :oauth2
|
|
214
|
-
call_data[:headers]['Authorization'] = oauth_token unless call_data[:headers].
|
|
210
|
+
call_data[:headers]['Authorization'] = oauth_token unless call_data[:headers].key?('Authorization')
|
|
215
211
|
when :url
|
|
216
212
|
call_data[:url_params] ||= {}
|
|
217
213
|
call_data[:auth][:url_creds].each do |key, value|
|
|
@@ -220,7 +216,7 @@ module Aspera
|
|
|
220
216
|
else raise "unsupported auth type: [#{call_data[:auth][:type]}]"
|
|
221
217
|
end
|
|
222
218
|
req = build_request(call_data)
|
|
223
|
-
Log.log.debug
|
|
219
|
+
Log.log.debug{"call_data = #{call_data}"}
|
|
224
220
|
result = {http: nil}
|
|
225
221
|
# start a block to be able to retry the actual HTTP request
|
|
226
222
|
begin
|
|
@@ -242,11 +238,11 @@ module Aspera
|
|
|
242
238
|
target_file = call_data[:save_to_file]
|
|
243
239
|
# override user's path to path in header
|
|
244
240
|
if !response['Content-Disposition'].nil? && (m = response['Content-Disposition'].match(/filename="([^"]+)"/))
|
|
245
|
-
target_file = File.join(File.dirname(target_file),m[1])
|
|
241
|
+
target_file = File.join(File.dirname(target_file), m[1])
|
|
246
242
|
end
|
|
247
243
|
# download with temp filename
|
|
248
244
|
target_file_tmp = "#{target_file}#{self.class.download_partial_suffix}"
|
|
249
|
-
Log.log.debug
|
|
245
|
+
Log.log.debug{"saving to: #{target_file}"}
|
|
250
246
|
File.open(target_file_tmp, 'wb') do |file|
|
|
251
247
|
result[:http].read_body do |fragment|
|
|
252
248
|
file.write(fragment)
|
|
@@ -262,17 +258,17 @@ module Aspera
|
|
|
262
258
|
end
|
|
263
259
|
# sometimes there is a UTF8 char (e.g. (c) )
|
|
264
260
|
result[:http].body.force_encoding('UTF-8') if result[:http].body.is_a?(String)
|
|
265
|
-
Log.log.debug
|
|
261
|
+
Log.log.debug{"result: body=#{result[:http].body}"}
|
|
266
262
|
result_mime = (result[:http]['Content-Type'] || 'text/plain').split(';').first
|
|
267
263
|
result[:data] = case result_mime
|
|
268
|
-
when 'application/json','application/vnd.api+json'
|
|
264
|
+
when 'application/json', 'application/vnd.api+json'
|
|
269
265
|
JSON.parse(result[:http].body) rescue nil
|
|
270
|
-
else #when 'text/plain'
|
|
266
|
+
else # when 'text/plain'
|
|
271
267
|
result[:http].body
|
|
272
268
|
end
|
|
273
|
-
Log.dump("result: parsed: #{result_mime}",result[:data])
|
|
274
|
-
Log.log.debug
|
|
275
|
-
RestErrorAnalyzer.instance.raise_on_error(req,result)
|
|
269
|
+
Log.dump("result: parsed: #{result_mime}", result[:data])
|
|
270
|
+
Log.log.debug{"result: code=#{result[:http].code}"}
|
|
271
|
+
RestErrorAnalyzer.instance.raise_on_error(req, result)
|
|
276
272
|
rescue RestCallError => e
|
|
277
273
|
# not authorized: oauth token expired
|
|
278
274
|
if call_data[:not_auth_codes].include?(result[:http].code.to_s) && call_data[:auth][:type].eql?(:oauth2)
|
|
@@ -285,16 +281,16 @@ module Aspera
|
|
|
285
281
|
# regenerate a brand new token
|
|
286
282
|
req['Authorization'] = oauth_token(use_cache: false)
|
|
287
283
|
end
|
|
288
|
-
Log.log.debug
|
|
284
|
+
Log.log.debug{"using new token=#{call_data[:headers]['Authorization']}"}
|
|
289
285
|
retry unless (oauth_tries -= 1).zero?
|
|
290
286
|
end # if oauth
|
|
291
287
|
# moved ?
|
|
292
288
|
if e.response.is_a?(Net::HTTPRedirection) && tries_remain_redirect.positive?
|
|
293
289
|
tries_remain_redirect -= 1
|
|
294
290
|
current_uri = URI.parse(call_data[:base_url])
|
|
295
|
-
new_url=e.response['location']
|
|
296
|
-
new_url="#{current_uri.scheme}:#{new_url}" unless new_url.start_with?('http')
|
|
297
|
-
Log.log.info
|
|
291
|
+
new_url = e.response['location']
|
|
292
|
+
new_url = "#{current_uri.scheme}:#{new_url}" unless new_url.start_with?('http')
|
|
293
|
+
Log.log.info{"URL is moved: #{new_url}"}
|
|
298
294
|
redir_uri = URI.parse(new_url)
|
|
299
295
|
call_data[:base_url] = new_url
|
|
300
296
|
call_data[:subpath] = ''
|
|
@@ -303,14 +299,14 @@ module Aspera
|
|
|
303
299
|
retry
|
|
304
300
|
else
|
|
305
301
|
# change host
|
|
306
|
-
Log.log.info
|
|
302
|
+
Log.log.info{"Redirect changes host: #{current_uri.host} -> #{redir_uri.host}"}
|
|
307
303
|
return self.class.new(call_data).call(call_data)
|
|
308
304
|
end
|
|
309
305
|
end
|
|
310
306
|
# raise exception if could not retry and not return error in result
|
|
311
307
|
raise e unless call_data[:return_error]
|
|
312
308
|
end # begin request
|
|
313
|
-
Log.log.debug
|
|
309
|
+
Log.log.debug{"result=#{result}"}
|
|
314
310
|
return result
|
|
315
311
|
end
|
|
316
312
|
|
|
@@ -319,24 +315,24 @@ module Aspera
|
|
|
319
315
|
#
|
|
320
316
|
|
|
321
317
|
# @param encoding : one of: :json_params, :url_params
|
|
322
|
-
def create(subpath,params,encoding=:json_params)
|
|
323
|
-
return call({operation: 'POST',subpath: subpath,headers: {'Accept' => 'application/json'},encoding => params})
|
|
318
|
+
def create(subpath, params, encoding=:json_params)
|
|
319
|
+
return call({operation: 'POST', subpath: subpath, headers: {'Accept' => 'application/json'}, encoding => params})
|
|
324
320
|
end
|
|
325
321
|
|
|
326
|
-
def read(subpath,args=nil)
|
|
327
|
-
return call({operation: 'GET',subpath: subpath,headers: {'Accept' => 'application/json'},url_params: args})
|
|
322
|
+
def read(subpath, args=nil)
|
|
323
|
+
return call({operation: 'GET', subpath: subpath, headers: {'Accept' => 'application/json'}, url_params: args})
|
|
328
324
|
end
|
|
329
325
|
|
|
330
|
-
def update(subpath,params)
|
|
331
|
-
return call({operation: 'PUT',subpath: subpath,headers: {'Accept' => 'application/json'},json_params: params})
|
|
326
|
+
def update(subpath, params)
|
|
327
|
+
return call({operation: 'PUT', subpath: subpath, headers: {'Accept' => 'application/json'}, json_params: params})
|
|
332
328
|
end
|
|
333
329
|
|
|
334
330
|
def delete(subpath)
|
|
335
|
-
return call({operation: 'DELETE',subpath: subpath,headers: {'Accept' => 'application/json'}})
|
|
331
|
+
return call({operation: 'DELETE', subpath: subpath, headers: {'Accept' => 'application/json'}})
|
|
336
332
|
end
|
|
337
333
|
|
|
338
334
|
def cancel(subpath)
|
|
339
|
-
return call({operation: 'CANCEL',subpath: subpath,headers: {'Accept' => 'application/json'}})
|
|
335
|
+
return call({operation: 'CANCEL', subpath: subpath, headers: {'Accept' => 'application/json'}})
|
|
340
336
|
end
|
|
341
337
|
end
|
|
342
|
-
end #module Aspera
|
|
338
|
+
end # module Aspera
|
|
@@ -4,8 +4,9 @@ module Aspera
|
|
|
4
4
|
# raised on error after REST call
|
|
5
5
|
class RestCallError < StandardError
|
|
6
6
|
attr_accessor :request, :response
|
|
7
|
+
|
|
7
8
|
# @param http response
|
|
8
|
-
def initialize(req,resp,msg)
|
|
9
|
+
def initialize(req, resp, msg)
|
|
9
10
|
@request = req
|
|
10
11
|
@response = resp
|
|
11
12
|
super(msg)
|
|
@@ -9,15 +9,16 @@ module Aspera
|
|
|
9
9
|
class RestErrorAnalyzer
|
|
10
10
|
include Singleton
|
|
11
11
|
attr_accessor :log_file
|
|
12
|
+
|
|
12
13
|
# the singleton object is registered with application specific handlers
|
|
13
14
|
def initialize
|
|
14
15
|
# list of handlers
|
|
15
16
|
@error_handlers = []
|
|
16
17
|
@log_file = nil
|
|
17
|
-
add_handler('Type Generic') do |type,call_context|
|
|
18
|
+
add_handler('Type Generic') do |type, call_context|
|
|
18
19
|
if !call_context[:response].code.start_with?('2')
|
|
19
20
|
# add generic information
|
|
20
|
-
RestErrorAnalyzer.add_error(call_context,type,"#{call_context[:request]['host']} #{call_context[:response].code} #{call_context[:response].message}")
|
|
21
|
+
RestErrorAnalyzer.add_error(call_context, type, "#{call_context[:request]['host']} #{call_context[:response].code} #{call_context[:response].message}")
|
|
21
22
|
end
|
|
22
23
|
end
|
|
23
24
|
end
|
|
@@ -25,7 +26,7 @@ module Aspera
|
|
|
25
26
|
# Use this method to analyze a EST result and raise an exception
|
|
26
27
|
# Analyzes REST call response and raises a RestCallError exception
|
|
27
28
|
# if HTTP result code is not 2XX
|
|
28
|
-
def raise_on_error(req,res)
|
|
29
|
+
def raise_on_error(req, res)
|
|
29
30
|
call_context = {
|
|
30
31
|
messages: [],
|
|
31
32
|
request: req,
|
|
@@ -36,14 +37,14 @@ module Aspera
|
|
|
36
37
|
# analyze errors from provided handlers
|
|
37
38
|
# note that there can be an error even if code is 2XX
|
|
38
39
|
@error_handlers.each do |handler|
|
|
39
|
-
begin
|
|
40
|
-
#Log.log.debug
|
|
41
|
-
handler[:block].call(handler[:name],call_context)
|
|
40
|
+
begin # rubocop:disable Style/RedundantBegin
|
|
41
|
+
# Log.log.debug{"test exception: #{handler[:name]}"}
|
|
42
|
+
handler[:block].call(handler[:name], call_context)
|
|
42
43
|
rescue StandardError => e
|
|
43
|
-
Log.log.error
|
|
44
|
+
Log.log.error{"ERROR in handler:\n#{e.message}\n#{e.backtrace}"}
|
|
44
45
|
end
|
|
45
46
|
end
|
|
46
|
-
raise RestCallError.new(call_context[:request],call_context[:response],call_context[:messages].join("\n")) unless call_context[:messages].empty?
|
|
47
|
+
raise RestCallError.new(call_context[:request], call_context[:response], call_context[:messages].join("\n")) unless call_context[:messages].empty?
|
|
47
48
|
end
|
|
48
49
|
|
|
49
50
|
# add a new error handler (done at application initialisation)
|
|
@@ -51,18 +52,18 @@ module Aspera
|
|
|
51
52
|
# @param block : processing of response: takes two parameters: name, call_context
|
|
52
53
|
# name is the one provided here
|
|
53
54
|
# call_context is built in method raise_on_error
|
|
54
|
-
def add_handler(name
|
|
55
|
+
def add_handler(name, &block)
|
|
55
56
|
@error_handlers.unshift({name: name, block: block})
|
|
56
57
|
end
|
|
57
58
|
|
|
58
59
|
# add a simple error handler
|
|
59
60
|
# check that key exists and is string under specified path (hash)
|
|
60
61
|
# adds other keys as secondary information
|
|
61
|
-
def add_simple_handler(name
|
|
62
|
-
add_handler(name) do |type,call_context|
|
|
62
|
+
def add_simple_handler(name, *args)
|
|
63
|
+
add_handler(name) do |type, call_context|
|
|
63
64
|
# need to clone because we modify and same array is used subsequently
|
|
64
65
|
path = args.clone
|
|
65
|
-
#Log.log.debug
|
|
66
|
+
# Log.log.debug{"path=#{path}"}
|
|
66
67
|
# if last in path is boolean it tells if the error is only with http error code or always
|
|
67
68
|
always = [true, false].include?(path.last) ? path.pop : false
|
|
68
69
|
if call_context[:data].is_a?(Hash) && (!call_context[:response].code.start_with?('2') || always)
|
|
@@ -70,10 +71,10 @@ module Aspera
|
|
|
70
71
|
# dig and find sub entry corresponding to path in deep hash
|
|
71
72
|
error_struct = path.inject(call_context[:data]) { |subhash, key| subhash.respond_to?(:keys) ? subhash[key] : nil }
|
|
72
73
|
if error_struct.is_a?(Hash) && error_struct[msg_key].is_a?(String)
|
|
73
|
-
RestErrorAnalyzer.add_error(call_context,type,error_struct[msg_key])
|
|
74
|
-
error_struct.each do |k,v|
|
|
74
|
+
RestErrorAnalyzer.add_error(call_context, type, error_struct[msg_key])
|
|
75
|
+
error_struct.each do |k, v|
|
|
75
76
|
next if k.eql?(msg_key)
|
|
76
|
-
RestErrorAnalyzer.add_error(call_context,"#{type}(sub)","#{k}: #{v}") if [String,Integer].include?(v.class)
|
|
77
|
+
RestErrorAnalyzer.add_error(call_context, "#{type}(sub)", "#{k}: #{v}") if [String, Integer].include?(v.class)
|
|
77
78
|
end
|
|
78
79
|
end
|
|
79
80
|
end
|
|
@@ -86,12 +87,12 @@ module Aspera
|
|
|
86
87
|
# @param call_context a Hash containing the result call_context, provided to handler
|
|
87
88
|
# @param type a string describing type of exception, for logging purpose
|
|
88
89
|
# @param msg one error message to add to list
|
|
89
|
-
def add_error(call_context,type,msg)
|
|
90
|
+
def add_error(call_context, type, msg)
|
|
90
91
|
call_context[:messages].push(msg)
|
|
91
92
|
logfile = instance.log_file
|
|
92
93
|
# log error for further analysis (file must exist to activate)
|
|
93
94
|
return if logfile.nil? || !File.exist?(logfile)
|
|
94
|
-
File.open(logfile,'a+') do |f|
|
|
95
|
+
File.open(logfile, 'a+') do |f|
|
|
95
96
|
f.write("\n=#{type}=====\n#{call_context[:request].method} #{call_context[:request].path}\n#{call_context[:response].code}\n"\
|
|
96
97
|
"#{JSON.generate(call_context[:data])}\n#{call_context[:messages].join("\n")}")
|
|
97
98
|
end
|
|
@@ -12,45 +12,45 @@ module Aspera
|
|
|
12
12
|
Log.log.debug('registering Aspera REST error handlers')
|
|
13
13
|
# Faspex 4: both user_message and internal_message, and code 200
|
|
14
14
|
# example: missing meta data on package creation
|
|
15
|
-
RestErrorAnalyzer.instance.add_simple_handler('Type 1: error:user_message','error','user_message',true)
|
|
16
|
-
RestErrorAnalyzer.instance.add_simple_handler('Type 2: error:description','error','description')
|
|
17
|
-
RestErrorAnalyzer.instance.add_simple_handler('Type 3: error:internal_message','error','internal_message')
|
|
15
|
+
RestErrorAnalyzer.instance.add_simple_handler('Type 1: error:user_message', 'error', 'user_message', true)
|
|
16
|
+
RestErrorAnalyzer.instance.add_simple_handler('Type 2: error:description', 'error', 'description')
|
|
17
|
+
RestErrorAnalyzer.instance.add_simple_handler('Type 3: error:internal_message', 'error', 'internal_message')
|
|
18
18
|
# AoC Automation
|
|
19
|
-
RestErrorAnalyzer.instance.add_simple_handler('AoC Automation','error')
|
|
20
|
-
RestErrorAnalyzer.instance.add_simple_handler('Type 5','error_description')
|
|
21
|
-
RestErrorAnalyzer.instance.add_simple_handler('Type 6','message')
|
|
22
|
-
RestErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |name,call_context|
|
|
19
|
+
RestErrorAnalyzer.instance.add_simple_handler('AoC Automation', 'error')
|
|
20
|
+
RestErrorAnalyzer.instance.add_simple_handler('Type 5', 'error_description')
|
|
21
|
+
RestErrorAnalyzer.instance.add_simple_handler('Type 6', 'message')
|
|
22
|
+
RestErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |name, call_context|
|
|
23
23
|
if call_context[:data].is_a?(Hash) && call_context[:data]['errors'].is_a?(Hash)
|
|
24
|
-
call_context[:data]['errors'].each do |k,v|
|
|
25
|
-
RestErrorAnalyzer.add_error(call_context,name,"#{k}: #{v}")
|
|
24
|
+
call_context[:data]['errors'].each do |k, v|
|
|
25
|
+
RestErrorAnalyzer.add_error(call_context, name, "#{k}: #{v}")
|
|
26
26
|
end
|
|
27
27
|
end
|
|
28
28
|
end
|
|
29
29
|
# call to upload_setup and download_setup of node api
|
|
30
|
-
RestErrorAnalyzer.instance.add_handler('T8:node: *_setup') do |type,call_context|
|
|
30
|
+
RestErrorAnalyzer.instance.add_handler('T8:node: *_setup') do |type, call_context|
|
|
31
31
|
if call_context[:data].is_a?(Hash)
|
|
32
32
|
d_t_s = call_context[:data]['transfer_specs']
|
|
33
33
|
if d_t_s.is_a?(Array)
|
|
34
34
|
d_t_s.each do |res|
|
|
35
|
-
#r_err=res['transfer_spec']['error']
|
|
35
|
+
# r_err=res['transfer_spec']['error']
|
|
36
36
|
r_err = res['error']
|
|
37
37
|
if r_err.is_a?(Hash)
|
|
38
|
-
RestErrorAnalyzer.add_error(call_context,type,"#{r_err['code']}: #{r_err['reason']}: #{r_err['user_message']}")
|
|
38
|
+
RestErrorAnalyzer.add_error(call_context, type, "#{r_err['code']}: #{r_err['reason']}: #{r_err['user_message']}")
|
|
39
39
|
end
|
|
40
40
|
end
|
|
41
41
|
end
|
|
42
42
|
end
|
|
43
43
|
end
|
|
44
|
-
RestErrorAnalyzer.instance.add_simple_handler('T9:IBM cloud IAM','errorMessage')
|
|
45
|
-
RestErrorAnalyzer.instance.add_simple_handler('T10:faspex v4','user_message')
|
|
46
|
-
RestErrorAnalyzer.instance.add_handler('bss graphql') do |type,call_context|
|
|
44
|
+
RestErrorAnalyzer.instance.add_simple_handler('T9:IBM cloud IAM', 'errorMessage')
|
|
45
|
+
RestErrorAnalyzer.instance.add_simple_handler('T10:faspex v4', 'user_message')
|
|
46
|
+
RestErrorAnalyzer.instance.add_handler('bss graphql') do |type, call_context|
|
|
47
47
|
if call_context[:data].is_a?(Hash)
|
|
48
48
|
d_t_s = call_context[:data]['errors']
|
|
49
49
|
if d_t_s.is_a?(Array)
|
|
50
50
|
d_t_s.each do |res|
|
|
51
51
|
r_err = res['message']
|
|
52
52
|
if r_err.is_a?(String)
|
|
53
|
-
RestErrorAnalyzer.add_error(call_context,type,r_err)
|
|
53
|
+
RestErrorAnalyzer.add_error(call_context, type, r_err)
|
|
54
54
|
end
|
|
55
55
|
end
|
|
56
56
|
end
|
data/lib/aspera/secret_hider.rb
CHANGED
|
@@ -7,16 +7,17 @@ module Aspera
|
|
|
7
7
|
class SecretHider
|
|
8
8
|
# display string for hidden secrets
|
|
9
9
|
HIDDEN_PASSWORD = '🔑'
|
|
10
|
+
# env vars for ascp with secrets
|
|
11
|
+
ASCP_ENV_SECRETS = %w[ASPERA_SCP_PASS ASPERA_SCP_KEY ASPERA_SCP_FILEPASS ASPERA_PROXY_PASS ASPERA_SCP_TOKEN].freeze
|
|
10
12
|
# keys in hash that contain secrets
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
ALL_SECRETS =[ASCP_SECRETS,KEY_SECRETS].flatten.freeze
|
|
13
|
+
KEY_SECRETS = %w[password secret private_key passphrase].freeze
|
|
14
|
+
ALL_SECRETS = [].concat(ASCP_ENV_SECRETS, KEY_SECRETS).freeze
|
|
14
15
|
# regex that define namec captures :begin and :end
|
|
15
|
-
REGEX_LOG_REPLACES=[
|
|
16
|
+
REGEX_LOG_REPLACES = [
|
|
16
17
|
# CLI manager get/set options
|
|
17
18
|
/(?<begin>[sg]et (#{KEY_SECRETS.join('|')})=).*(?<end>)/,
|
|
18
19
|
# env var ascp exec
|
|
19
|
-
/(?<begin> (#{
|
|
20
|
+
/(?<begin> (#{ASCP_ENV_SECRETS.join('|')})=)(\\.|[^ ])*(?<end> )/,
|
|
20
21
|
# rendered JSON
|
|
21
22
|
/(?<begin>["':][^"]*(#{ALL_SECRETS.join('|')})[^"]*["']?[=>: ]+")[^"]+(?<end>")/,
|
|
22
23
|
# option "secret"
|
|
@@ -26,13 +27,14 @@ module Aspera
|
|
|
26
27
|
# private key values
|
|
27
28
|
/(?<begin>--+BEGIN .+ KEY--+)[[:ascii:]]+?(?<end>--+?END .+ KEY--+)/
|
|
28
29
|
].freeze
|
|
29
|
-
private_constant :HIDDEN_PASSWORD
|
|
30
|
+
private_constant :HIDDEN_PASSWORD, :ASCP_ENV_SECRETS, :KEY_SECRETS, :ALL_SECRETS, :REGEX_LOG_REPLACES
|
|
30
31
|
@log_secrets = false
|
|
31
32
|
class << self
|
|
32
33
|
attr_accessor :log_secrets
|
|
34
|
+
|
|
33
35
|
def log_formatter(original_formatter)
|
|
34
36
|
original_formatter ||= Logger::Formatter.new
|
|
35
|
-
#
|
|
37
|
+
# NOTE: that @log_secrets may be set AFTER this init is done, so it's done at runtime
|
|
36
38
|
return lambda do |severity, datetime, progname, msg|
|
|
37
39
|
if msg.is_a?(String) && !@log_secrets
|
|
38
40
|
REGEX_LOG_REPLACES.each do |regx|
|
|
@@ -43,25 +45,25 @@ module Aspera
|
|
|
43
45
|
end
|
|
44
46
|
end
|
|
45
47
|
|
|
46
|
-
def secret?(keyword,value)
|
|
47
|
-
keyword=keyword.to_s if keyword.is_a?(Symbol)
|
|
48
|
+
def secret?(keyword, value)
|
|
49
|
+
keyword = keyword.to_s if keyword.is_a?(Symbol)
|
|
48
50
|
# only Strings can be secrets, not booleans, or hash, arrays
|
|
49
51
|
keyword.is_a?(String) && ALL_SECRETS.any?{|kw|keyword.include?(kw)} && value.is_a?(String)
|
|
50
52
|
end
|
|
51
53
|
|
|
52
|
-
def deep_remove_secret(obj,is_name_value: false)
|
|
54
|
+
def deep_remove_secret(obj, is_name_value: false)
|
|
53
55
|
case obj
|
|
54
56
|
when Array
|
|
55
57
|
if is_name_value
|
|
56
58
|
obj.each do |i|
|
|
57
|
-
i['value']=HIDDEN_PASSWORD if secret?(i['parameter'],i['value'])
|
|
59
|
+
i['value'] = HIDDEN_PASSWORD if secret?(i['parameter'], i['value'])
|
|
58
60
|
end
|
|
59
61
|
else
|
|
60
62
|
obj.each{|i|deep_remove_secret(i)}
|
|
61
63
|
end
|
|
62
64
|
when Hash
|
|
63
|
-
obj.each do |k,v|
|
|
64
|
-
if secret?(k,v)
|
|
65
|
+
obj.each do |k, v|
|
|
66
|
+
if secret?(k, v)
|
|
65
67
|
obj[k] = HIDDEN_PASSWORD
|
|
66
68
|
elsif obj[k].is_a?(Hash)
|
|
67
69
|
deep_remove_secret(obj[k])
|
data/lib/aspera/ssh.rb
CHANGED
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
require 'net/ssh'
|
|
4
4
|
|
|
5
|
-
#
|
|
5
|
+
# HACK: deactivate ed25519 and ecdsa private keys from ssh identities, as it usually hurts
|
|
6
6
|
begin
|
|
7
|
-
module Net;module SSH;module Authentication;class Session;private; def default_keys; %w[~/.ssh/id_dsa ~/.ssh/id_rsa ~/.ssh2/id_dsa ~/.ssh2/id_rsa];end;end;end;end;end # rubocop:disable Layout/AccessModifierIndentation, Layout/EmptyLinesAroundAccessModifier, Layout/LineLength
|
|
7
|
+
module Net; module SSH; module Authentication; class Session; private; def default_keys; %w[~/.ssh/id_dsa ~/.ssh/id_rsa ~/.ssh2/id_dsa ~/.ssh2/id_rsa]; end; end; end; end; end # rubocop:disable Layout/AccessModifierIndentation, Layout/EmptyLinesAroundAccessModifier, Layout/LineLength, Style/Semicolon
|
|
8
8
|
rescue StandardError
|
|
9
9
|
# ignore errors
|
|
10
10
|
end
|
|
@@ -15,26 +15,26 @@ module Aspera
|
|
|
15
15
|
class Ssh
|
|
16
16
|
# ssh_options: same as Net::SSH.start
|
|
17
17
|
# see: https://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start
|
|
18
|
-
def initialize(host,username,ssh_options)
|
|
19
|
-
Log.log.debug
|
|
20
|
-
Log.log.debug
|
|
18
|
+
def initialize(host, username, ssh_options)
|
|
19
|
+
Log.log.debug{"ssh:#{username}@#{host}"}
|
|
20
|
+
Log.log.debug{"ssh_options:#{ssh_options}"}
|
|
21
21
|
@host = host
|
|
22
22
|
@username = username
|
|
23
23
|
@ssh_options = ssh_options
|
|
24
24
|
@ssh_options[:logger] = Log.log
|
|
25
25
|
end
|
|
26
26
|
|
|
27
|
-
def execute(cmd,input=nil)
|
|
27
|
+
def execute(cmd, input=nil)
|
|
28
28
|
if cmd.is_a?(Array)
|
|
29
29
|
# concatenate arguments, enclose in double quotes
|
|
30
30
|
cmd = cmd.map{|v|%Q("#{v}")}.join(' ')
|
|
31
31
|
end
|
|
32
|
-
Log.log.debug
|
|
32
|
+
Log.log.debug{"cmd=#{cmd}"}
|
|
33
33
|
response = []
|
|
34
34
|
Net::SSH.start(@host, @username, @ssh_options) do |session|
|
|
35
35
|
ssh_channel = session.open_channel do |channel|
|
|
36
36
|
# prepare stdout processing
|
|
37
|
-
channel.on_data{|_chan,data|response.push(data)}
|
|
37
|
+
channel.on_data{|_chan, data|response.push(data)}
|
|
38
38
|
# prepare stderr processing, stderr if type = 1
|
|
39
39
|
channel.on_extended_data do |_chan, _type, data|
|
|
40
40
|
errormsg = "#{cmd}: [#{data.chomp}]"
|
|
@@ -45,13 +45,14 @@ module Aspera
|
|
|
45
45
|
raise errormsg
|
|
46
46
|
end
|
|
47
47
|
# send commannd to SSH channel (execute)
|
|
48
|
-
channel.send('cexe'.reverse,cmd){|_ch,_success|channel.send_data(input) unless input.nil?}
|
|
48
|
+
channel.send('cexe'.reverse, cmd){|_ch, _success|channel.send_data(input) unless input.nil?}
|
|
49
49
|
end
|
|
50
50
|
# wait for channel to finish (command exit)
|
|
51
51
|
ssh_channel.wait
|
|
52
52
|
# main ssh session loop
|
|
53
53
|
session.loop
|
|
54
|
-
end #
|
|
54
|
+
end # start
|
|
55
|
+
# response as single string
|
|
55
56
|
return response.join
|
|
56
57
|
end
|
|
57
58
|
end
|