aspera-cli 4.10.0 → 4.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +20 -0
  4. data/CHANGELOG.md +509 -0
  5. data/CONTRIBUTING.md +118 -0
  6. data/README.md +621 -378
  7. data/bin/ascli +4 -4
  8. data/bin/asession +11 -11
  9. data/docs/test_env.conf +28 -19
  10. data/examples/aoc.rb +4 -4
  11. data/examples/dascli +11 -9
  12. data/examples/faspex4.rb +8 -8
  13. data/examples/node.rb +11 -11
  14. data/examples/server.rb +9 -9
  15. data/lib/aspera/aoc.rb +273 -266
  16. data/lib/aspera/ascmd.rb +56 -54
  17. data/lib/aspera/ats_api.rb +4 -4
  18. data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
  19. data/lib/aspera/cli/extended_value.rb +5 -5
  20. data/lib/aspera/cli/formater.rb +64 -64
  21. data/lib/aspera/cli/listener/line_dump.rb +1 -1
  22. data/lib/aspera/cli/listener/logger.rb +1 -1
  23. data/lib/aspera/cli/listener/progress.rb +5 -6
  24. data/lib/aspera/cli/listener/progress_multi.rb +14 -19
  25. data/lib/aspera/cli/main.rb +66 -67
  26. data/lib/aspera/cli/manager.rb +110 -110
  27. data/lib/aspera/cli/plugin.rb +54 -37
  28. data/lib/aspera/cli/plugins/alee.rb +4 -4
  29. data/lib/aspera/cli/plugins/aoc.rb +308 -669
  30. data/lib/aspera/cli/plugins/ats.rb +44 -46
  31. data/lib/aspera/cli/plugins/bss.rb +10 -10
  32. data/lib/aspera/cli/plugins/config.rb +447 -344
  33. data/lib/aspera/cli/plugins/console.rb +12 -12
  34. data/lib/aspera/cli/plugins/cos.rb +18 -20
  35. data/lib/aspera/cli/plugins/faspex.rb +110 -112
  36. data/lib/aspera/cli/plugins/faspex5.rb +67 -46
  37. data/lib/aspera/cli/plugins/node.rb +364 -288
  38. data/lib/aspera/cli/plugins/orchestrator.rb +46 -46
  39. data/lib/aspera/cli/plugins/preview.rb +122 -114
  40. data/lib/aspera/cli/plugins/server.rb +137 -83
  41. data/lib/aspera/cli/plugins/shares.rb +30 -29
  42. data/lib/aspera/cli/plugins/sync.rb +13 -33
  43. data/lib/aspera/cli/transfer_agent.rb +57 -57
  44. data/lib/aspera/cli/version.rb +1 -1
  45. data/lib/aspera/colors.rb +3 -3
  46. data/lib/aspera/command_line_builder.rb +27 -27
  47. data/lib/aspera/cos_node.rb +22 -20
  48. data/lib/aspera/data_repository.rb +1 -1
  49. data/lib/aspera/environment.rb +30 -28
  50. data/lib/aspera/fasp/agent_base.rb +15 -15
  51. data/lib/aspera/fasp/agent_connect.rb +23 -21
  52. data/lib/aspera/fasp/agent_direct.rb +65 -67
  53. data/lib/aspera/fasp/agent_httpgw.rb +72 -68
  54. data/lib/aspera/fasp/agent_node.rb +23 -21
  55. data/lib/aspera/fasp/agent_trsdk.rb +20 -20
  56. data/lib/aspera/fasp/error.rb +3 -2
  57. data/lib/aspera/fasp/error_info.rb +11 -8
  58. data/lib/aspera/fasp/installation.rb +78 -78
  59. data/lib/aspera/fasp/listener.rb +1 -1
  60. data/lib/aspera/fasp/parameters.rb +75 -72
  61. data/lib/aspera/fasp/parameters.yaml +2 -2
  62. data/lib/aspera/fasp/resume_policy.rb +8 -8
  63. data/lib/aspera/fasp/transfer_spec.rb +35 -2
  64. data/lib/aspera/fasp/uri.rb +7 -7
  65. data/lib/aspera/faspex_gw.rb +7 -5
  66. data/lib/aspera/hash_ext.rb +3 -3
  67. data/lib/aspera/id_generator.rb +5 -5
  68. data/lib/aspera/keychain/encrypted_hash.rb +23 -28
  69. data/lib/aspera/keychain/macos_security.rb +21 -20
  70. data/lib/aspera/log.rb +7 -7
  71. data/lib/aspera/nagios.rb +19 -18
  72. data/lib/aspera/node.rb +209 -35
  73. data/lib/aspera/oauth.rb +37 -36
  74. data/lib/aspera/open_application.rb +19 -11
  75. data/lib/aspera/persistency_action_once.rb +4 -4
  76. data/lib/aspera/persistency_folder.rb +13 -13
  77. data/lib/aspera/preview/file_types.rb +8 -8
  78. data/lib/aspera/preview/generator.rb +67 -67
  79. data/lib/aspera/preview/utils.rb +27 -27
  80. data/lib/aspera/proxy_auto_config.js +41 -41
  81. data/lib/aspera/proxy_auto_config.rb +16 -16
  82. data/lib/aspera/rest.rb +56 -60
  83. data/lib/aspera/rest_call_error.rb +2 -1
  84. data/lib/aspera/rest_error_analyzer.rb +18 -17
  85. data/lib/aspera/rest_errors_aspera.rb +16 -16
  86. data/lib/aspera/secret_hider.rb +15 -13
  87. data/lib/aspera/ssh.rb +11 -10
  88. data/lib/aspera/sync.rb +158 -44
  89. data/lib/aspera/temp_file_manager.rb +2 -2
  90. data/lib/aspera/uri_reader.rb +4 -4
  91. data/lib/aspera/web_auth.rb +14 -13
  92. data.tar.gz.sig +0 -0
  93. metadata +8 -5
  94. metadata.gz.sig +0 -0
data/lib/aspera/rest.rb CHANGED
@@ -13,18 +13,12 @@ require 'cgi'
13
13
  require 'ruby-progressbar'
14
14
 
15
15
  # add cancel method to http
16
- class Net::HTTP::Cancel < Net::HTTPRequest
16
+ class Net::HTTP::Cancel < Net::HTTPRequest # rubocop:disable Style/ClassAndModuleChildren
17
17
  METHOD = 'CANCEL'
18
18
  REQUEST_HAS_BODY = false
19
19
  RESPONSE_HAS_BODY = false
20
20
  end
21
21
 
22
- #class Net::HTTP::Delete < Net::HTTPRequest
23
- # METHOD = 'DELETE'
24
- # REQUEST_HAS_BODY = false
25
- # RESPONSE_HAS_BODY = false
26
- #end
27
-
28
22
  module Aspera
29
23
  # a simple class to make HTTP calls, equivalent to rest-client
30
24
  # rest call errors are raised as exception RestCallError
@@ -44,15 +38,18 @@ module Aspera
44
38
 
45
39
  class << self
46
40
  # define accessors
47
- @@global.keys.each do |p|
41
+ @@global.each_key do |p|
48
42
  define_method(p){@@global[p]}
49
- define_method("#{p}="){|val|Log.log.debug("#{p} => #{val}".red);@@global[p] = val}
43
+ define_method("#{p}=") do |val|
44
+ Log.log.debug{"#{p} => #{val}".red}
45
+ @@global[p] = val
46
+ end
50
47
  end
51
48
 
52
- def basic_creds(user,pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}";end
49
+ def basic_creds(user, pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}"; end
53
50
 
54
51
  # build URI from URL and parameters and check it is http or https
55
- def build_uri(url,params=nil)
52
+ def build_uri(url, params=nil)
56
53
  uri = URI.parse(url)
57
54
  raise "REST endpoint shall be http/s not #{uri.scheme}" unless %w[http https].include?(uri.scheme)
58
55
  if !params.nil?
@@ -60,15 +57,15 @@ module Aspera
60
57
  if params.is_a?(Hash)
61
58
  orig = params
62
59
  params = []
63
- orig.each do |k,v|
60
+ orig.each do |k, v|
64
61
  case v
65
62
  when Array
66
63
  suffix = v.first.eql?('[]') ? v.shift : ''
67
64
  v.each do |e|
68
- params.push([k + suffix,e])
65
+ params.push([k + suffix, e])
69
66
  end
70
67
  else
71
- params.push([k,v])
68
+ params.push([k, v])
72
69
  end
73
70
  end
74
71
  end
@@ -82,8 +79,8 @@ module Aspera
82
79
  uri = build_uri(base_url)
83
80
  # this honors http_proxy env var
84
81
  http_session = Net::HTTP.new(uri.host, uri.port)
85
- http_session.proxy_user=proxy_user
86
- http_session.proxy_pass=proxy_pass
82
+ http_session.proxy_user = proxy_user
83
+ http_session.proxy_pass = proxy_pass
87
84
  http_session.use_ssl = uri.scheme.eql?('https')
88
85
  http_session.set_debug_output($stdout) if debug
89
86
  # set http options in callback, such as timeout and cert. verification
@@ -99,7 +96,7 @@ module Aspera
99
96
  # create and start keep alive connection on demand
100
97
  def http_session
101
98
  if @http_session.nil?
102
- @http_session=self.class.start_http_session(@params[:base_url])
99
+ @http_session = self.class.start_http_session(@params[:base_url])
103
100
  end
104
101
  return @http_session
105
102
  end
@@ -121,57 +118,56 @@ module Aspera
121
118
  raise 'ERROR: expecting Hash' unless a_rest_params.is_a?(Hash)
122
119
  raise 'ERROR: expecting base_url' unless a_rest_params[:base_url].is_a?(String)
123
120
  @params = a_rest_params.clone
124
- Log.dump('REST params',@params)
121
+ Log.dump('REST params', @params)
125
122
  # base url without trailing slashes (note: string may be frozen)
126
- @params[:base_url] = @params[:base_url].gsub(/\/+$/,'')
123
+ @params[:base_url] = @params[:base_url].gsub(%r{/+$}, '')
127
124
  @http_session = nil
128
125
  # default is no auth
129
126
  @params[:auth] ||= {type: :none}
130
127
  @params[:not_auth_codes] ||= ['401']
131
128
  @oauth = nil
132
- Log.dump('REST params(2)',@params)
129
+ Log.dump('REST params(2)', @params)
133
130
  end
134
131
 
135
132
  def oauth_token(force_refresh: false)
136
- raise "ERROR: expecting boolean, have #{force_refresh}" unless [true,false].include?(force_refresh)
133
+ raise "ERROR: expecting boolean, have #{force_refresh}" unless [true, false].include?(force_refresh)
137
134
  return oauth.get_authorization(use_refresh_token: force_refresh)
138
135
  end
139
136
 
140
137
  def build_request(call_data)
141
138
  # TODO: shall we percent encode subpath (spaces) test with access key delete with space in id
142
139
  # URI.escape()
143
- uri = self.class.build_uri("#{call_data[:base_url]}#{['','/'].include?(call_data[:subpath]) ? '' : '/'}#{call_data[:subpath]}",call_data[:url_params])
144
- Log.log.debug("URI=#{uri}")
140
+ uri = self.class.build_uri("#{call_data[:base_url]}#{['', '/'].include?(call_data[:subpath]) ? '' : '/'}#{call_data[:subpath]}", call_data[:url_params])
141
+ Log.log.debug{"URI=#{uri}"}
145
142
  begin
146
143
  # instanciate request object based on string name
147
144
  req = Net::HTTP.const_get(call_data[:operation].capitalize).new(uri)
148
145
  rescue NameError
149
146
  raise "unsupported operation : #{call_data[:operation]}"
150
147
  end
151
- if call_data.has_key?(:json_params) && !call_data[:json_params].nil?
148
+ if call_data.key?(:json_params) && !call_data[:json_params].nil?
152
149
  req.body = JSON.generate(call_data[:json_params])
153
- Log.dump('body JSON data',call_data[:json_params])
154
- #Log.log.debug("body JSON data=#{JSON.pretty_generate(call_data[:json_params])}")
150
+ Log.dump('body JSON data', call_data[:json_params])
155
151
  req['Content-Type'] = 'application/json'
156
- #call_data[:headers]['Accept']='application/json'
152
+ # call_data[:headers]['Accept']='application/json'
157
153
  end
158
- if call_data.has_key?(:www_body_params)
154
+ if call_data.key?(:www_body_params)
159
155
  req.body = URI.encode_www_form(call_data[:www_body_params])
160
- Log.log.debug("body www data=#{req.body.chomp}")
156
+ Log.log.debug{"body www data=#{req.body.chomp}"}
161
157
  req['Content-Type'] = 'application/x-www-form-urlencoded'
162
158
  end
163
- if call_data.has_key?(:text_body_params)
159
+ if call_data.key?(:text_body_params)
164
160
  req.body = call_data[:text_body_params]
165
- Log.log.debug("body data=#{req.body.chomp}")
161
+ Log.log.debug{"body data=#{req.body.chomp}"}
166
162
  end
167
163
  # set headers
168
- if call_data.has_key?(:headers)
169
- call_data[:headers].keys.each do |key|
164
+ if call_data.key?(:headers)
165
+ call_data[:headers].each_key do |key|
170
166
  req[key] = call_data[:headers][key]
171
167
  end
172
168
  end
173
169
  # :type = :basic
174
- req.basic_auth(call_data[:auth][:username],call_data[:auth][:password]) if call_data[:auth][:type].eql?(:basic)
170
+ req.basic_auth(call_data[:auth][:username], call_data[:auth][:password]) if call_data[:auth][:type].eql?(:basic)
175
171
  return req
176
172
  end
177
173
 
@@ -199,7 +195,7 @@ module Aspera
199
195
  def call(call_data)
200
196
  raise "Hash call parameter is required (#{call_data.class})" unless call_data.is_a?(Hash)
201
197
  call_data[:subpath] = '' if call_data[:subpath].nil?
202
- Log.log.debug("accessing #{call_data[:subpath]}".red.bold.bg_green)
198
+ Log.log.debug{"accessing #{call_data[:subpath]}".red.bold.bg_green}
203
199
  call_data[:headers] ||= {}
204
200
  call_data[:headers]['User-Agent'] ||= self.class.user_agent
205
201
  # defaults from @params are overriden by call data
@@ -211,7 +207,7 @@ module Aspera
211
207
  Log.log.debug('using Basic auth')
212
208
  # done in build_req
213
209
  when :oauth2
214
- call_data[:headers]['Authorization'] = oauth_token unless call_data[:headers].has_key?('Authorization')
210
+ call_data[:headers]['Authorization'] = oauth_token unless call_data[:headers].key?('Authorization')
215
211
  when :url
216
212
  call_data[:url_params] ||= {}
217
213
  call_data[:auth][:url_creds].each do |key, value|
@@ -220,7 +216,7 @@ module Aspera
220
216
  else raise "unsupported auth type: [#{call_data[:auth][:type]}]"
221
217
  end
222
218
  req = build_request(call_data)
223
- Log.log.debug("call_data = #{call_data}")
219
+ Log.log.debug{"call_data = #{call_data}"}
224
220
  result = {http: nil}
225
221
  # start a block to be able to retry the actual HTTP request
226
222
  begin
@@ -242,11 +238,11 @@ module Aspera
242
238
  target_file = call_data[:save_to_file]
243
239
  # override user's path to path in header
244
240
  if !response['Content-Disposition'].nil? && (m = response['Content-Disposition'].match(/filename="([^"]+)"/))
245
- target_file = File.join(File.dirname(target_file),m[1])
241
+ target_file = File.join(File.dirname(target_file), m[1])
246
242
  end
247
243
  # download with temp filename
248
244
  target_file_tmp = "#{target_file}#{self.class.download_partial_suffix}"
249
- Log.log.debug("saving to: #{target_file}")
245
+ Log.log.debug{"saving to: #{target_file}"}
250
246
  File.open(target_file_tmp, 'wb') do |file|
251
247
  result[:http].read_body do |fragment|
252
248
  file.write(fragment)
@@ -262,17 +258,17 @@ module Aspera
262
258
  end
263
259
  # sometimes there is a UTF8 char (e.g. (c) )
264
260
  result[:http].body.force_encoding('UTF-8') if result[:http].body.is_a?(String)
265
- Log.log.debug("result: body=#{result[:http].body}")
261
+ Log.log.debug{"result: body=#{result[:http].body}"}
266
262
  result_mime = (result[:http]['Content-Type'] || 'text/plain').split(';').first
267
263
  result[:data] = case result_mime
268
- when 'application/json','application/vnd.api+json'
264
+ when 'application/json', 'application/vnd.api+json'
269
265
  JSON.parse(result[:http].body) rescue nil
270
- else #when 'text/plain'
266
+ else # when 'text/plain'
271
267
  result[:http].body
272
268
  end
273
- Log.dump("result: parsed: #{result_mime}",result[:data])
274
- Log.log.debug("result: code=#{result[:http].code}")
275
- RestErrorAnalyzer.instance.raise_on_error(req,result)
269
+ Log.dump("result: parsed: #{result_mime}", result[:data])
270
+ Log.log.debug{"result: code=#{result[:http].code}"}
271
+ RestErrorAnalyzer.instance.raise_on_error(req, result)
276
272
  rescue RestCallError => e
277
273
  # not authorized: oauth token expired
278
274
  if call_data[:not_auth_codes].include?(result[:http].code.to_s) && call_data[:auth][:type].eql?(:oauth2)
@@ -285,16 +281,16 @@ module Aspera
285
281
  # regenerate a brand new token
286
282
  req['Authorization'] = oauth_token(use_cache: false)
287
283
  end
288
- Log.log.debug("using new token=#{call_data[:headers]['Authorization']}")
284
+ Log.log.debug{"using new token=#{call_data[:headers]['Authorization']}"}
289
285
  retry unless (oauth_tries -= 1).zero?
290
286
  end # if oauth
291
287
  # moved ?
292
288
  if e.response.is_a?(Net::HTTPRedirection) && tries_remain_redirect.positive?
293
289
  tries_remain_redirect -= 1
294
290
  current_uri = URI.parse(call_data[:base_url])
295
- new_url=e.response['location']
296
- new_url="#{current_uri.scheme}:#{new_url}" unless new_url.start_with?('http')
297
- Log.log.info("URL is moved: #{new_url}")
291
+ new_url = e.response['location']
292
+ new_url = "#{current_uri.scheme}:#{new_url}" unless new_url.start_with?('http')
293
+ Log.log.info{"URL is moved: #{new_url}"}
298
294
  redir_uri = URI.parse(new_url)
299
295
  call_data[:base_url] = new_url
300
296
  call_data[:subpath] = ''
@@ -303,14 +299,14 @@ module Aspera
303
299
  retry
304
300
  else
305
301
  # change host
306
- Log.log.info("Redirect changes host: #{current_uri.host} -> #{redir_uri.host}")
302
+ Log.log.info{"Redirect changes host: #{current_uri.host} -> #{redir_uri.host}"}
307
303
  return self.class.new(call_data).call(call_data)
308
304
  end
309
305
  end
310
306
  # raise exception if could not retry and not return error in result
311
307
  raise e unless call_data[:return_error]
312
308
  end # begin request
313
- Log.log.debug("result=#{result}")
309
+ Log.log.debug{"result=#{result}"}
314
310
  return result
315
311
  end
316
312
 
@@ -319,24 +315,24 @@ module Aspera
319
315
  #
320
316
 
321
317
  # @param encoding : one of: :json_params, :url_params
322
- def create(subpath,params,encoding=:json_params)
323
- return call({operation: 'POST',subpath: subpath,headers: {'Accept' => 'application/json'},encoding => params})
318
+ def create(subpath, params, encoding=:json_params)
319
+ return call({operation: 'POST', subpath: subpath, headers: {'Accept' => 'application/json'}, encoding => params})
324
320
  end
325
321
 
326
- def read(subpath,args=nil)
327
- return call({operation: 'GET',subpath: subpath,headers: {'Accept' => 'application/json'},url_params: args})
322
+ def read(subpath, args=nil)
323
+ return call({operation: 'GET', subpath: subpath, headers: {'Accept' => 'application/json'}, url_params: args})
328
324
  end
329
325
 
330
- def update(subpath,params)
331
- return call({operation: 'PUT',subpath: subpath,headers: {'Accept' => 'application/json'},json_params: params})
326
+ def update(subpath, params)
327
+ return call({operation: 'PUT', subpath: subpath, headers: {'Accept' => 'application/json'}, json_params: params})
332
328
  end
333
329
 
334
330
  def delete(subpath)
335
- return call({operation: 'DELETE',subpath: subpath,headers: {'Accept' => 'application/json'}})
331
+ return call({operation: 'DELETE', subpath: subpath, headers: {'Accept' => 'application/json'}})
336
332
  end
337
333
 
338
334
  def cancel(subpath)
339
- return call({operation: 'CANCEL',subpath: subpath,headers: {'Accept' => 'application/json'}})
335
+ return call({operation: 'CANCEL', subpath: subpath, headers: {'Accept' => 'application/json'}})
340
336
  end
341
337
  end
342
- end #module Aspera
338
+ end # module Aspera
@@ -4,8 +4,9 @@ module Aspera
4
4
  # raised on error after REST call
5
5
  class RestCallError < StandardError
6
6
  attr_accessor :request, :response
7
+
7
8
  # @param http response
8
- def initialize(req,resp,msg)
9
+ def initialize(req, resp, msg)
9
10
  @request = req
10
11
  @response = resp
11
12
  super(msg)
@@ -9,15 +9,16 @@ module Aspera
9
9
  class RestErrorAnalyzer
10
10
  include Singleton
11
11
  attr_accessor :log_file
12
+
12
13
  # the singleton object is registered with application specific handlers
13
14
  def initialize
14
15
  # list of handlers
15
16
  @error_handlers = []
16
17
  @log_file = nil
17
- add_handler('Type Generic') do |type,call_context|
18
+ add_handler('Type Generic') do |type, call_context|
18
19
  if !call_context[:response].code.start_with?('2')
19
20
  # add generic information
20
- RestErrorAnalyzer.add_error(call_context,type,"#{call_context[:request]['host']} #{call_context[:response].code} #{call_context[:response].message}")
21
+ RestErrorAnalyzer.add_error(call_context, type, "#{call_context[:request]['host']} #{call_context[:response].code} #{call_context[:response].message}")
21
22
  end
22
23
  end
23
24
  end
@@ -25,7 +26,7 @@ module Aspera
25
26
  # Use this method to analyze a EST result and raise an exception
26
27
  # Analyzes REST call response and raises a RestCallError exception
27
28
  # if HTTP result code is not 2XX
28
- def raise_on_error(req,res)
29
+ def raise_on_error(req, res)
29
30
  call_context = {
30
31
  messages: [],
31
32
  request: req,
@@ -36,14 +37,14 @@ module Aspera
36
37
  # analyze errors from provided handlers
37
38
  # note that there can be an error even if code is 2XX
38
39
  @error_handlers.each do |handler|
39
- begin
40
- #Log.log.debug("test exception: #{handler[:name]}")
41
- handler[:block].call(handler[:name],call_context)
40
+ begin # rubocop:disable Style/RedundantBegin
41
+ # Log.log.debug{"test exception: #{handler[:name]}"}
42
+ handler[:block].call(handler[:name], call_context)
42
43
  rescue StandardError => e
43
- Log.log.error("ERROR in handler:\n#{e.message}\n#{e.backtrace}")
44
+ Log.log.error{"ERROR in handler:\n#{e.message}\n#{e.backtrace}"}
44
45
  end
45
46
  end
46
- raise RestCallError.new(call_context[:request],call_context[:response],call_context[:messages].join("\n")) unless call_context[:messages].empty?
47
+ raise RestCallError.new(call_context[:request], call_context[:response], call_context[:messages].join("\n")) unless call_context[:messages].empty?
47
48
  end
48
49
 
49
50
  # add a new error handler (done at application initialisation)
@@ -51,18 +52,18 @@ module Aspera
51
52
  # @param block : processing of response: takes two parameters: name, call_context
52
53
  # name is the one provided here
53
54
  # call_context is built in method raise_on_error
54
- def add_handler(name,&block)
55
+ def add_handler(name, &block)
55
56
  @error_handlers.unshift({name: name, block: block})
56
57
  end
57
58
 
58
59
  # add a simple error handler
59
60
  # check that key exists and is string under specified path (hash)
60
61
  # adds other keys as secondary information
61
- def add_simple_handler(name,*args)
62
- add_handler(name) do |type,call_context|
62
+ def add_simple_handler(name, *args)
63
+ add_handler(name) do |type, call_context|
63
64
  # need to clone because we modify and same array is used subsequently
64
65
  path = args.clone
65
- #Log.log.debug("path=#{path}")
66
+ # Log.log.debug{"path=#{path}"}
66
67
  # if last in path is boolean it tells if the error is only with http error code or always
67
68
  always = [true, false].include?(path.last) ? path.pop : false
68
69
  if call_context[:data].is_a?(Hash) && (!call_context[:response].code.start_with?('2') || always)
@@ -70,10 +71,10 @@ module Aspera
70
71
  # dig and find sub entry corresponding to path in deep hash
71
72
  error_struct = path.inject(call_context[:data]) { |subhash, key| subhash.respond_to?(:keys) ? subhash[key] : nil }
72
73
  if error_struct.is_a?(Hash) && error_struct[msg_key].is_a?(String)
73
- RestErrorAnalyzer.add_error(call_context,type,error_struct[msg_key])
74
- error_struct.each do |k,v|
74
+ RestErrorAnalyzer.add_error(call_context, type, error_struct[msg_key])
75
+ error_struct.each do |k, v|
75
76
  next if k.eql?(msg_key)
76
- RestErrorAnalyzer.add_error(call_context,"#{type}(sub)","#{k}: #{v}") if [String,Integer].include?(v.class)
77
+ RestErrorAnalyzer.add_error(call_context, "#{type}(sub)", "#{k}: #{v}") if [String, Integer].include?(v.class)
77
78
  end
78
79
  end
79
80
  end
@@ -86,12 +87,12 @@ module Aspera
86
87
  # @param call_context a Hash containing the result call_context, provided to handler
87
88
  # @param type a string describing type of exception, for logging purpose
88
89
  # @param msg one error message to add to list
89
- def add_error(call_context,type,msg)
90
+ def add_error(call_context, type, msg)
90
91
  call_context[:messages].push(msg)
91
92
  logfile = instance.log_file
92
93
  # log error for further analysis (file must exist to activate)
93
94
  return if logfile.nil? || !File.exist?(logfile)
94
- File.open(logfile,'a+') do |f|
95
+ File.open(logfile, 'a+') do |f|
95
96
  f.write("\n=#{type}=====\n#{call_context[:request].method} #{call_context[:request].path}\n#{call_context[:response].code}\n"\
96
97
  "#{JSON.generate(call_context[:data])}\n#{call_context[:messages].join("\n")}")
97
98
  end
@@ -12,45 +12,45 @@ module Aspera
12
12
  Log.log.debug('registering Aspera REST error handlers')
13
13
  # Faspex 4: both user_message and internal_message, and code 200
14
14
  # example: missing meta data on package creation
15
- RestErrorAnalyzer.instance.add_simple_handler('Type 1: error:user_message','error','user_message',true)
16
- RestErrorAnalyzer.instance.add_simple_handler('Type 2: error:description','error','description')
17
- RestErrorAnalyzer.instance.add_simple_handler('Type 3: error:internal_message','error','internal_message')
15
+ RestErrorAnalyzer.instance.add_simple_handler('Type 1: error:user_message', 'error', 'user_message', true)
16
+ RestErrorAnalyzer.instance.add_simple_handler('Type 2: error:description', 'error', 'description')
17
+ RestErrorAnalyzer.instance.add_simple_handler('Type 3: error:internal_message', 'error', 'internal_message')
18
18
  # AoC Automation
19
- RestErrorAnalyzer.instance.add_simple_handler('AoC Automation','error')
20
- RestErrorAnalyzer.instance.add_simple_handler('Type 5','error_description')
21
- RestErrorAnalyzer.instance.add_simple_handler('Type 6','message')
22
- RestErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |name,call_context|
19
+ RestErrorAnalyzer.instance.add_simple_handler('AoC Automation', 'error')
20
+ RestErrorAnalyzer.instance.add_simple_handler('Type 5', 'error_description')
21
+ RestErrorAnalyzer.instance.add_simple_handler('Type 6', 'message')
22
+ RestErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |name, call_context|
23
23
  if call_context[:data].is_a?(Hash) && call_context[:data]['errors'].is_a?(Hash)
24
- call_context[:data]['errors'].each do |k,v|
25
- RestErrorAnalyzer.add_error(call_context,name,"#{k}: #{v}")
24
+ call_context[:data]['errors'].each do |k, v|
25
+ RestErrorAnalyzer.add_error(call_context, name, "#{k}: #{v}")
26
26
  end
27
27
  end
28
28
  end
29
29
  # call to upload_setup and download_setup of node api
30
- RestErrorAnalyzer.instance.add_handler('T8:node: *_setup') do |type,call_context|
30
+ RestErrorAnalyzer.instance.add_handler('T8:node: *_setup') do |type, call_context|
31
31
  if call_context[:data].is_a?(Hash)
32
32
  d_t_s = call_context[:data]['transfer_specs']
33
33
  if d_t_s.is_a?(Array)
34
34
  d_t_s.each do |res|
35
- #r_err=res['transfer_spec']['error']
35
+ # r_err=res['transfer_spec']['error']
36
36
  r_err = res['error']
37
37
  if r_err.is_a?(Hash)
38
- RestErrorAnalyzer.add_error(call_context,type,"#{r_err['code']}: #{r_err['reason']}: #{r_err['user_message']}")
38
+ RestErrorAnalyzer.add_error(call_context, type, "#{r_err['code']}: #{r_err['reason']}: #{r_err['user_message']}")
39
39
  end
40
40
  end
41
41
  end
42
42
  end
43
43
  end
44
- RestErrorAnalyzer.instance.add_simple_handler('T9:IBM cloud IAM','errorMessage')
45
- RestErrorAnalyzer.instance.add_simple_handler('T10:faspex v4','user_message')
46
- RestErrorAnalyzer.instance.add_handler('bss graphql') do |type,call_context|
44
+ RestErrorAnalyzer.instance.add_simple_handler('T9:IBM cloud IAM', 'errorMessage')
45
+ RestErrorAnalyzer.instance.add_simple_handler('T10:faspex v4', 'user_message')
46
+ RestErrorAnalyzer.instance.add_handler('bss graphql') do |type, call_context|
47
47
  if call_context[:data].is_a?(Hash)
48
48
  d_t_s = call_context[:data]['errors']
49
49
  if d_t_s.is_a?(Array)
50
50
  d_t_s.each do |res|
51
51
  r_err = res['message']
52
52
  if r_err.is_a?(String)
53
- RestErrorAnalyzer.add_error(call_context,type,r_err)
53
+ RestErrorAnalyzer.add_error(call_context, type, r_err)
54
54
  end
55
55
  end
56
56
  end
@@ -7,16 +7,17 @@ module Aspera
7
7
  class SecretHider
8
8
  # display string for hidden secrets
9
9
  HIDDEN_PASSWORD = '🔑'
10
+ # env vars for ascp with secrets
11
+ ASCP_ENV_SECRETS = %w[ASPERA_SCP_PASS ASPERA_SCP_KEY ASPERA_SCP_FILEPASS ASPERA_PROXY_PASS ASPERA_SCP_TOKEN].freeze
10
12
  # keys in hash that contain secrets
11
- ASCP_SECRETS=%w[ASPERA_SCP_PASS ASPERA_SCP_KEY ASPERA_SCP_FILEPASS ASPERA_PROXY_PASS].freeze
12
- KEY_SECRETS =%w[password secret private_key passphrase].freeze
13
- ALL_SECRETS =[ASCP_SECRETS,KEY_SECRETS].flatten.freeze
13
+ KEY_SECRETS = %w[password secret private_key passphrase].freeze
14
+ ALL_SECRETS = [].concat(ASCP_ENV_SECRETS, KEY_SECRETS).freeze
14
15
  # regex that define namec captures :begin and :end
15
- REGEX_LOG_REPLACES=[
16
+ REGEX_LOG_REPLACES = [
16
17
  # CLI manager get/set options
17
18
  /(?<begin>[sg]et (#{KEY_SECRETS.join('|')})=).*(?<end>)/,
18
19
  # env var ascp exec
19
- /(?<begin> (#{ASCP_SECRETS.join('|')})=)[^ ]*(?<end> )/,
20
+ /(?<begin> (#{ASCP_ENV_SECRETS.join('|')})=)(\\.|[^ ])*(?<end> )/,
20
21
  # rendered JSON
21
22
  /(?<begin>["':][^"]*(#{ALL_SECRETS.join('|')})[^"]*["']?[=>: ]+")[^"]+(?<end>")/,
22
23
  # option "secret"
@@ -26,13 +27,14 @@ module Aspera
26
27
  # private key values
27
28
  /(?<begin>--+BEGIN .+ KEY--+)[[:ascii:]]+?(?<end>--+?END .+ KEY--+)/
28
29
  ].freeze
29
- private_constant :HIDDEN_PASSWORD,:ASCP_SECRETS,:KEY_SECRETS,:ALL_SECRETS,:REGEX_LOG_REPLACES
30
+ private_constant :HIDDEN_PASSWORD, :ASCP_ENV_SECRETS, :KEY_SECRETS, :ALL_SECRETS, :REGEX_LOG_REPLACES
30
31
  @log_secrets = false
31
32
  class << self
32
33
  attr_accessor :log_secrets
34
+
33
35
  def log_formatter(original_formatter)
34
36
  original_formatter ||= Logger::Formatter.new
35
- # note that @log_secrets may be set AFTER this init is done, so it's done at runtime
37
+ # NOTE: that @log_secrets may be set AFTER this init is done, so it's done at runtime
36
38
  return lambda do |severity, datetime, progname, msg|
37
39
  if msg.is_a?(String) && !@log_secrets
38
40
  REGEX_LOG_REPLACES.each do |regx|
@@ -43,25 +45,25 @@ module Aspera
43
45
  end
44
46
  end
45
47
 
46
- def secret?(keyword,value)
47
- keyword=keyword.to_s if keyword.is_a?(Symbol)
48
+ def secret?(keyword, value)
49
+ keyword = keyword.to_s if keyword.is_a?(Symbol)
48
50
  # only Strings can be secrets, not booleans, or hash, arrays
49
51
  keyword.is_a?(String) && ALL_SECRETS.any?{|kw|keyword.include?(kw)} && value.is_a?(String)
50
52
  end
51
53
 
52
- def deep_remove_secret(obj,is_name_value: false)
54
+ def deep_remove_secret(obj, is_name_value: false)
53
55
  case obj
54
56
  when Array
55
57
  if is_name_value
56
58
  obj.each do |i|
57
- i['value']=HIDDEN_PASSWORD if secret?(i['parameter'],i['value'])
59
+ i['value'] = HIDDEN_PASSWORD if secret?(i['parameter'], i['value'])
58
60
  end
59
61
  else
60
62
  obj.each{|i|deep_remove_secret(i)}
61
63
  end
62
64
  when Hash
63
- obj.each do |k,v|
64
- if secret?(k,v)
65
+ obj.each do |k, v|
66
+ if secret?(k, v)
65
67
  obj[k] = HIDDEN_PASSWORD
66
68
  elsif obj[k].is_a?(Hash)
67
69
  deep_remove_secret(obj[k])
data/lib/aspera/ssh.rb CHANGED
@@ -2,9 +2,9 @@
2
2
 
3
3
  require 'net/ssh'
4
4
 
5
- # Hack: deactivate ed25519 and ecdsa private keys from ssh identities, as it usually hurts
5
+ # HACK: deactivate ed25519 and ecdsa private keys from ssh identities, as it usually hurts
6
6
  begin
7
- module Net;module SSH;module Authentication;class Session;private; def default_keys; %w[~/.ssh/id_dsa ~/.ssh/id_rsa ~/.ssh2/id_dsa ~/.ssh2/id_rsa];end;end;end;end;end # rubocop:disable Layout/AccessModifierIndentation, Layout/EmptyLinesAroundAccessModifier, Layout/LineLength
7
+ module Net; module SSH; module Authentication; class Session; private; def default_keys; %w[~/.ssh/id_dsa ~/.ssh/id_rsa ~/.ssh2/id_dsa ~/.ssh2/id_rsa]; end; end; end; end; end # rubocop:disable Layout/AccessModifierIndentation, Layout/EmptyLinesAroundAccessModifier, Layout/LineLength, Style/Semicolon
8
8
  rescue StandardError
9
9
  # ignore errors
10
10
  end
@@ -15,26 +15,26 @@ module Aspera
15
15
  class Ssh
16
16
  # ssh_options: same as Net::SSH.start
17
17
  # see: https://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start
18
- def initialize(host,username,ssh_options)
19
- Log.log.debug("ssh:#{username}@#{host}")
20
- Log.log.debug("ssh_options:#{ssh_options}")
18
+ def initialize(host, username, ssh_options)
19
+ Log.log.debug{"ssh:#{username}@#{host}"}
20
+ Log.log.debug{"ssh_options:#{ssh_options}"}
21
21
  @host = host
22
22
  @username = username
23
23
  @ssh_options = ssh_options
24
24
  @ssh_options[:logger] = Log.log
25
25
  end
26
26
 
27
- def execute(cmd,input=nil)
27
+ def execute(cmd, input=nil)
28
28
  if cmd.is_a?(Array)
29
29
  # concatenate arguments, enclose in double quotes
30
30
  cmd = cmd.map{|v|%Q("#{v}")}.join(' ')
31
31
  end
32
- Log.log.debug("cmd=#{cmd}")
32
+ Log.log.debug{"cmd=#{cmd}"}
33
33
  response = []
34
34
  Net::SSH.start(@host, @username, @ssh_options) do |session|
35
35
  ssh_channel = session.open_channel do |channel|
36
36
  # prepare stdout processing
37
- channel.on_data{|_chan,data|response.push(data)}
37
+ channel.on_data{|_chan, data|response.push(data)}
38
38
  # prepare stderr processing, stderr if type = 1
39
39
  channel.on_extended_data do |_chan, _type, data|
40
40
  errormsg = "#{cmd}: [#{data.chomp}]"
@@ -45,13 +45,14 @@ module Aspera
45
45
  raise errormsg
46
46
  end
47
47
  # send commannd to SSH channel (execute)
48
- channel.send('cexe'.reverse,cmd){|_ch,_success|channel.send_data(input) unless input.nil?}
48
+ channel.send('cexe'.reverse, cmd){|_ch, _success|channel.send_data(input) unless input.nil?}
49
49
  end
50
50
  # wait for channel to finish (command exit)
51
51
  ssh_channel.wait
52
52
  # main ssh session loop
53
53
  session.loop
54
- end # session
54
+ end # start
55
+ # response as single string
55
56
  return response.join
56
57
  end
57
58
  end