aspera-cli 4.10.0 → 4.11.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (94) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +20 -0
  4. data/CHANGELOG.md +509 -0
  5. data/CONTRIBUTING.md +118 -0
  6. data/README.md +621 -378
  7. data/bin/ascli +4 -4
  8. data/bin/asession +11 -11
  9. data/docs/test_env.conf +28 -19
  10. data/examples/aoc.rb +4 -4
  11. data/examples/dascli +11 -9
  12. data/examples/faspex4.rb +8 -8
  13. data/examples/node.rb +11 -11
  14. data/examples/server.rb +9 -9
  15. data/lib/aspera/aoc.rb +273 -266
  16. data/lib/aspera/ascmd.rb +56 -54
  17. data/lib/aspera/ats_api.rb +4 -4
  18. data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
  19. data/lib/aspera/cli/extended_value.rb +5 -5
  20. data/lib/aspera/cli/formater.rb +64 -64
  21. data/lib/aspera/cli/listener/line_dump.rb +1 -1
  22. data/lib/aspera/cli/listener/logger.rb +1 -1
  23. data/lib/aspera/cli/listener/progress.rb +5 -6
  24. data/lib/aspera/cli/listener/progress_multi.rb +14 -19
  25. data/lib/aspera/cli/main.rb +66 -67
  26. data/lib/aspera/cli/manager.rb +110 -110
  27. data/lib/aspera/cli/plugin.rb +54 -37
  28. data/lib/aspera/cli/plugins/alee.rb +4 -4
  29. data/lib/aspera/cli/plugins/aoc.rb +308 -669
  30. data/lib/aspera/cli/plugins/ats.rb +44 -46
  31. data/lib/aspera/cli/plugins/bss.rb +10 -10
  32. data/lib/aspera/cli/plugins/config.rb +447 -344
  33. data/lib/aspera/cli/plugins/console.rb +12 -12
  34. data/lib/aspera/cli/plugins/cos.rb +18 -20
  35. data/lib/aspera/cli/plugins/faspex.rb +110 -112
  36. data/lib/aspera/cli/plugins/faspex5.rb +67 -46
  37. data/lib/aspera/cli/plugins/node.rb +364 -288
  38. data/lib/aspera/cli/plugins/orchestrator.rb +46 -46
  39. data/lib/aspera/cli/plugins/preview.rb +122 -114
  40. data/lib/aspera/cli/plugins/server.rb +137 -83
  41. data/lib/aspera/cli/plugins/shares.rb +30 -29
  42. data/lib/aspera/cli/plugins/sync.rb +13 -33
  43. data/lib/aspera/cli/transfer_agent.rb +57 -57
  44. data/lib/aspera/cli/version.rb +1 -1
  45. data/lib/aspera/colors.rb +3 -3
  46. data/lib/aspera/command_line_builder.rb +27 -27
  47. data/lib/aspera/cos_node.rb +22 -20
  48. data/lib/aspera/data_repository.rb +1 -1
  49. data/lib/aspera/environment.rb +30 -28
  50. data/lib/aspera/fasp/agent_base.rb +15 -15
  51. data/lib/aspera/fasp/agent_connect.rb +23 -21
  52. data/lib/aspera/fasp/agent_direct.rb +65 -67
  53. data/lib/aspera/fasp/agent_httpgw.rb +72 -68
  54. data/lib/aspera/fasp/agent_node.rb +23 -21
  55. data/lib/aspera/fasp/agent_trsdk.rb +20 -20
  56. data/lib/aspera/fasp/error.rb +3 -2
  57. data/lib/aspera/fasp/error_info.rb +11 -8
  58. data/lib/aspera/fasp/installation.rb +78 -78
  59. data/lib/aspera/fasp/listener.rb +1 -1
  60. data/lib/aspera/fasp/parameters.rb +75 -72
  61. data/lib/aspera/fasp/parameters.yaml +2 -2
  62. data/lib/aspera/fasp/resume_policy.rb +8 -8
  63. data/lib/aspera/fasp/transfer_spec.rb +35 -2
  64. data/lib/aspera/fasp/uri.rb +7 -7
  65. data/lib/aspera/faspex_gw.rb +7 -5
  66. data/lib/aspera/hash_ext.rb +3 -3
  67. data/lib/aspera/id_generator.rb +5 -5
  68. data/lib/aspera/keychain/encrypted_hash.rb +23 -28
  69. data/lib/aspera/keychain/macos_security.rb +21 -20
  70. data/lib/aspera/log.rb +7 -7
  71. data/lib/aspera/nagios.rb +19 -18
  72. data/lib/aspera/node.rb +209 -35
  73. data/lib/aspera/oauth.rb +37 -36
  74. data/lib/aspera/open_application.rb +19 -11
  75. data/lib/aspera/persistency_action_once.rb +4 -4
  76. data/lib/aspera/persistency_folder.rb +13 -13
  77. data/lib/aspera/preview/file_types.rb +8 -8
  78. data/lib/aspera/preview/generator.rb +67 -67
  79. data/lib/aspera/preview/utils.rb +27 -27
  80. data/lib/aspera/proxy_auto_config.js +41 -41
  81. data/lib/aspera/proxy_auto_config.rb +16 -16
  82. data/lib/aspera/rest.rb +56 -60
  83. data/lib/aspera/rest_call_error.rb +2 -1
  84. data/lib/aspera/rest_error_analyzer.rb +18 -17
  85. data/lib/aspera/rest_errors_aspera.rb +16 -16
  86. data/lib/aspera/secret_hider.rb +15 -13
  87. data/lib/aspera/ssh.rb +11 -10
  88. data/lib/aspera/sync.rb +158 -44
  89. data/lib/aspera/temp_file_manager.rb +2 -2
  90. data/lib/aspera/uri_reader.rb +4 -4
  91. data/lib/aspera/web_auth.rb +14 -13
  92. data.tar.gz.sig +0 -0
  93. metadata +8 -5
  94. metadata.gz.sig +0 -0
data/lib/aspera/rest.rb CHANGED
@@ -13,18 +13,12 @@ require 'cgi'
13
13
  require 'ruby-progressbar'
14
14
 
15
15
  # add cancel method to http
16
- class Net::HTTP::Cancel < Net::HTTPRequest
16
+ class Net::HTTP::Cancel < Net::HTTPRequest # rubocop:disable Style/ClassAndModuleChildren
17
17
  METHOD = 'CANCEL'
18
18
  REQUEST_HAS_BODY = false
19
19
  RESPONSE_HAS_BODY = false
20
20
  end
21
21
 
22
- #class Net::HTTP::Delete < Net::HTTPRequest
23
- # METHOD = 'DELETE'
24
- # REQUEST_HAS_BODY = false
25
- # RESPONSE_HAS_BODY = false
26
- #end
27
-
28
22
  module Aspera
29
23
  # a simple class to make HTTP calls, equivalent to rest-client
30
24
  # rest call errors are raised as exception RestCallError
@@ -44,15 +38,18 @@ module Aspera
44
38
 
45
39
  class << self
46
40
  # define accessors
47
- @@global.keys.each do |p|
41
+ @@global.each_key do |p|
48
42
  define_method(p){@@global[p]}
49
- define_method("#{p}="){|val|Log.log.debug("#{p} => #{val}".red);@@global[p] = val}
43
+ define_method("#{p}=") do |val|
44
+ Log.log.debug{"#{p} => #{val}".red}
45
+ @@global[p] = val
46
+ end
50
47
  end
51
48
 
52
- def basic_creds(user,pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}";end
49
+ def basic_creds(user, pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}"; end
53
50
 
54
51
  # build URI from URL and parameters and check it is http or https
55
- def build_uri(url,params=nil)
52
+ def build_uri(url, params=nil)
56
53
  uri = URI.parse(url)
57
54
  raise "REST endpoint shall be http/s not #{uri.scheme}" unless %w[http https].include?(uri.scheme)
58
55
  if !params.nil?
@@ -60,15 +57,15 @@ module Aspera
60
57
  if params.is_a?(Hash)
61
58
  orig = params
62
59
  params = []
63
- orig.each do |k,v|
60
+ orig.each do |k, v|
64
61
  case v
65
62
  when Array
66
63
  suffix = v.first.eql?('[]') ? v.shift : ''
67
64
  v.each do |e|
68
- params.push([k + suffix,e])
65
+ params.push([k + suffix, e])
69
66
  end
70
67
  else
71
- params.push([k,v])
68
+ params.push([k, v])
72
69
  end
73
70
  end
74
71
  end
@@ -82,8 +79,8 @@ module Aspera
82
79
  uri = build_uri(base_url)
83
80
  # this honors http_proxy env var
84
81
  http_session = Net::HTTP.new(uri.host, uri.port)
85
- http_session.proxy_user=proxy_user
86
- http_session.proxy_pass=proxy_pass
82
+ http_session.proxy_user = proxy_user
83
+ http_session.proxy_pass = proxy_pass
87
84
  http_session.use_ssl = uri.scheme.eql?('https')
88
85
  http_session.set_debug_output($stdout) if debug
89
86
  # set http options in callback, such as timeout and cert. verification
@@ -99,7 +96,7 @@ module Aspera
99
96
  # create and start keep alive connection on demand
100
97
  def http_session
101
98
  if @http_session.nil?
102
- @http_session=self.class.start_http_session(@params[:base_url])
99
+ @http_session = self.class.start_http_session(@params[:base_url])
103
100
  end
104
101
  return @http_session
105
102
  end
@@ -121,57 +118,56 @@ module Aspera
121
118
  raise 'ERROR: expecting Hash' unless a_rest_params.is_a?(Hash)
122
119
  raise 'ERROR: expecting base_url' unless a_rest_params[:base_url].is_a?(String)
123
120
  @params = a_rest_params.clone
124
- Log.dump('REST params',@params)
121
+ Log.dump('REST params', @params)
125
122
  # base url without trailing slashes (note: string may be frozen)
126
- @params[:base_url] = @params[:base_url].gsub(/\/+$/,'')
123
+ @params[:base_url] = @params[:base_url].gsub(%r{/+$}, '')
127
124
  @http_session = nil
128
125
  # default is no auth
129
126
  @params[:auth] ||= {type: :none}
130
127
  @params[:not_auth_codes] ||= ['401']
131
128
  @oauth = nil
132
- Log.dump('REST params(2)',@params)
129
+ Log.dump('REST params(2)', @params)
133
130
  end
134
131
 
135
132
  def oauth_token(force_refresh: false)
136
- raise "ERROR: expecting boolean, have #{force_refresh}" unless [true,false].include?(force_refresh)
133
+ raise "ERROR: expecting boolean, have #{force_refresh}" unless [true, false].include?(force_refresh)
137
134
  return oauth.get_authorization(use_refresh_token: force_refresh)
138
135
  end
139
136
 
140
137
  def build_request(call_data)
141
138
  # TODO: shall we percent encode subpath (spaces) test with access key delete with space in id
142
139
  # URI.escape()
143
- uri = self.class.build_uri("#{call_data[:base_url]}#{['','/'].include?(call_data[:subpath]) ? '' : '/'}#{call_data[:subpath]}",call_data[:url_params])
144
- Log.log.debug("URI=#{uri}")
140
+ uri = self.class.build_uri("#{call_data[:base_url]}#{['', '/'].include?(call_data[:subpath]) ? '' : '/'}#{call_data[:subpath]}", call_data[:url_params])
141
+ Log.log.debug{"URI=#{uri}"}
145
142
  begin
146
143
  # instanciate request object based on string name
147
144
  req = Net::HTTP.const_get(call_data[:operation].capitalize).new(uri)
148
145
  rescue NameError
149
146
  raise "unsupported operation : #{call_data[:operation]}"
150
147
  end
151
- if call_data.has_key?(:json_params) && !call_data[:json_params].nil?
148
+ if call_data.key?(:json_params) && !call_data[:json_params].nil?
152
149
  req.body = JSON.generate(call_data[:json_params])
153
- Log.dump('body JSON data',call_data[:json_params])
154
- #Log.log.debug("body JSON data=#{JSON.pretty_generate(call_data[:json_params])}")
150
+ Log.dump('body JSON data', call_data[:json_params])
155
151
  req['Content-Type'] = 'application/json'
156
- #call_data[:headers]['Accept']='application/json'
152
+ # call_data[:headers]['Accept']='application/json'
157
153
  end
158
- if call_data.has_key?(:www_body_params)
154
+ if call_data.key?(:www_body_params)
159
155
  req.body = URI.encode_www_form(call_data[:www_body_params])
160
- Log.log.debug("body www data=#{req.body.chomp}")
156
+ Log.log.debug{"body www data=#{req.body.chomp}"}
161
157
  req['Content-Type'] = 'application/x-www-form-urlencoded'
162
158
  end
163
- if call_data.has_key?(:text_body_params)
159
+ if call_data.key?(:text_body_params)
164
160
  req.body = call_data[:text_body_params]
165
- Log.log.debug("body data=#{req.body.chomp}")
161
+ Log.log.debug{"body data=#{req.body.chomp}"}
166
162
  end
167
163
  # set headers
168
- if call_data.has_key?(:headers)
169
- call_data[:headers].keys.each do |key|
164
+ if call_data.key?(:headers)
165
+ call_data[:headers].each_key do |key|
170
166
  req[key] = call_data[:headers][key]
171
167
  end
172
168
  end
173
169
  # :type = :basic
174
- req.basic_auth(call_data[:auth][:username],call_data[:auth][:password]) if call_data[:auth][:type].eql?(:basic)
170
+ req.basic_auth(call_data[:auth][:username], call_data[:auth][:password]) if call_data[:auth][:type].eql?(:basic)
175
171
  return req
176
172
  end
177
173
 
@@ -199,7 +195,7 @@ module Aspera
199
195
  def call(call_data)
200
196
  raise "Hash call parameter is required (#{call_data.class})" unless call_data.is_a?(Hash)
201
197
  call_data[:subpath] = '' if call_data[:subpath].nil?
202
- Log.log.debug("accessing #{call_data[:subpath]}".red.bold.bg_green)
198
+ Log.log.debug{"accessing #{call_data[:subpath]}".red.bold.bg_green}
203
199
  call_data[:headers] ||= {}
204
200
  call_data[:headers]['User-Agent'] ||= self.class.user_agent
205
201
  # defaults from @params are overriden by call data
@@ -211,7 +207,7 @@ module Aspera
211
207
  Log.log.debug('using Basic auth')
212
208
  # done in build_req
213
209
  when :oauth2
214
- call_data[:headers]['Authorization'] = oauth_token unless call_data[:headers].has_key?('Authorization')
210
+ call_data[:headers]['Authorization'] = oauth_token unless call_data[:headers].key?('Authorization')
215
211
  when :url
216
212
  call_data[:url_params] ||= {}
217
213
  call_data[:auth][:url_creds].each do |key, value|
@@ -220,7 +216,7 @@ module Aspera
220
216
  else raise "unsupported auth type: [#{call_data[:auth][:type]}]"
221
217
  end
222
218
  req = build_request(call_data)
223
- Log.log.debug("call_data = #{call_data}")
219
+ Log.log.debug{"call_data = #{call_data}"}
224
220
  result = {http: nil}
225
221
  # start a block to be able to retry the actual HTTP request
226
222
  begin
@@ -242,11 +238,11 @@ module Aspera
242
238
  target_file = call_data[:save_to_file]
243
239
  # override user's path to path in header
244
240
  if !response['Content-Disposition'].nil? && (m = response['Content-Disposition'].match(/filename="([^"]+)"/))
245
- target_file = File.join(File.dirname(target_file),m[1])
241
+ target_file = File.join(File.dirname(target_file), m[1])
246
242
  end
247
243
  # download with temp filename
248
244
  target_file_tmp = "#{target_file}#{self.class.download_partial_suffix}"
249
- Log.log.debug("saving to: #{target_file}")
245
+ Log.log.debug{"saving to: #{target_file}"}
250
246
  File.open(target_file_tmp, 'wb') do |file|
251
247
  result[:http].read_body do |fragment|
252
248
  file.write(fragment)
@@ -262,17 +258,17 @@ module Aspera
262
258
  end
263
259
  # sometimes there is a UTF8 char (e.g. (c) )
264
260
  result[:http].body.force_encoding('UTF-8') if result[:http].body.is_a?(String)
265
- Log.log.debug("result: body=#{result[:http].body}")
261
+ Log.log.debug{"result: body=#{result[:http].body}"}
266
262
  result_mime = (result[:http]['Content-Type'] || 'text/plain').split(';').first
267
263
  result[:data] = case result_mime
268
- when 'application/json','application/vnd.api+json'
264
+ when 'application/json', 'application/vnd.api+json'
269
265
  JSON.parse(result[:http].body) rescue nil
270
- else #when 'text/plain'
266
+ else # when 'text/plain'
271
267
  result[:http].body
272
268
  end
273
- Log.dump("result: parsed: #{result_mime}",result[:data])
274
- Log.log.debug("result: code=#{result[:http].code}")
275
- RestErrorAnalyzer.instance.raise_on_error(req,result)
269
+ Log.dump("result: parsed: #{result_mime}", result[:data])
270
+ Log.log.debug{"result: code=#{result[:http].code}"}
271
+ RestErrorAnalyzer.instance.raise_on_error(req, result)
276
272
  rescue RestCallError => e
277
273
  # not authorized: oauth token expired
278
274
  if call_data[:not_auth_codes].include?(result[:http].code.to_s) && call_data[:auth][:type].eql?(:oauth2)
@@ -285,16 +281,16 @@ module Aspera
285
281
  # regenerate a brand new token
286
282
  req['Authorization'] = oauth_token(use_cache: false)
287
283
  end
288
- Log.log.debug("using new token=#{call_data[:headers]['Authorization']}")
284
+ Log.log.debug{"using new token=#{call_data[:headers]['Authorization']}"}
289
285
  retry unless (oauth_tries -= 1).zero?
290
286
  end # if oauth
291
287
  # moved ?
292
288
  if e.response.is_a?(Net::HTTPRedirection) && tries_remain_redirect.positive?
293
289
  tries_remain_redirect -= 1
294
290
  current_uri = URI.parse(call_data[:base_url])
295
- new_url=e.response['location']
296
- new_url="#{current_uri.scheme}:#{new_url}" unless new_url.start_with?('http')
297
- Log.log.info("URL is moved: #{new_url}")
291
+ new_url = e.response['location']
292
+ new_url = "#{current_uri.scheme}:#{new_url}" unless new_url.start_with?('http')
293
+ Log.log.info{"URL is moved: #{new_url}"}
298
294
  redir_uri = URI.parse(new_url)
299
295
  call_data[:base_url] = new_url
300
296
  call_data[:subpath] = ''
@@ -303,14 +299,14 @@ module Aspera
303
299
  retry
304
300
  else
305
301
  # change host
306
- Log.log.info("Redirect changes host: #{current_uri.host} -> #{redir_uri.host}")
302
+ Log.log.info{"Redirect changes host: #{current_uri.host} -> #{redir_uri.host}"}
307
303
  return self.class.new(call_data).call(call_data)
308
304
  end
309
305
  end
310
306
  # raise exception if could not retry and not return error in result
311
307
  raise e unless call_data[:return_error]
312
308
  end # begin request
313
- Log.log.debug("result=#{result}")
309
+ Log.log.debug{"result=#{result}"}
314
310
  return result
315
311
  end
316
312
 
@@ -319,24 +315,24 @@ module Aspera
319
315
  #
320
316
 
321
317
  # @param encoding : one of: :json_params, :url_params
322
- def create(subpath,params,encoding=:json_params)
323
- return call({operation: 'POST',subpath: subpath,headers: {'Accept' => 'application/json'},encoding => params})
318
+ def create(subpath, params, encoding=:json_params)
319
+ return call({operation: 'POST', subpath: subpath, headers: {'Accept' => 'application/json'}, encoding => params})
324
320
  end
325
321
 
326
- def read(subpath,args=nil)
327
- return call({operation: 'GET',subpath: subpath,headers: {'Accept' => 'application/json'},url_params: args})
322
+ def read(subpath, args=nil)
323
+ return call({operation: 'GET', subpath: subpath, headers: {'Accept' => 'application/json'}, url_params: args})
328
324
  end
329
325
 
330
- def update(subpath,params)
331
- return call({operation: 'PUT',subpath: subpath,headers: {'Accept' => 'application/json'},json_params: params})
326
+ def update(subpath, params)
327
+ return call({operation: 'PUT', subpath: subpath, headers: {'Accept' => 'application/json'}, json_params: params})
332
328
  end
333
329
 
334
330
  def delete(subpath)
335
- return call({operation: 'DELETE',subpath: subpath,headers: {'Accept' => 'application/json'}})
331
+ return call({operation: 'DELETE', subpath: subpath, headers: {'Accept' => 'application/json'}})
336
332
  end
337
333
 
338
334
  def cancel(subpath)
339
- return call({operation: 'CANCEL',subpath: subpath,headers: {'Accept' => 'application/json'}})
335
+ return call({operation: 'CANCEL', subpath: subpath, headers: {'Accept' => 'application/json'}})
340
336
  end
341
337
  end
342
- end #module Aspera
338
+ end # module Aspera
@@ -4,8 +4,9 @@ module Aspera
4
4
  # raised on error after REST call
5
5
  class RestCallError < StandardError
6
6
  attr_accessor :request, :response
7
+
7
8
  # @param http response
8
- def initialize(req,resp,msg)
9
+ def initialize(req, resp, msg)
9
10
  @request = req
10
11
  @response = resp
11
12
  super(msg)
@@ -9,15 +9,16 @@ module Aspera
9
9
  class RestErrorAnalyzer
10
10
  include Singleton
11
11
  attr_accessor :log_file
12
+
12
13
  # the singleton object is registered with application specific handlers
13
14
  def initialize
14
15
  # list of handlers
15
16
  @error_handlers = []
16
17
  @log_file = nil
17
- add_handler('Type Generic') do |type,call_context|
18
+ add_handler('Type Generic') do |type, call_context|
18
19
  if !call_context[:response].code.start_with?('2')
19
20
  # add generic information
20
- RestErrorAnalyzer.add_error(call_context,type,"#{call_context[:request]['host']} #{call_context[:response].code} #{call_context[:response].message}")
21
+ RestErrorAnalyzer.add_error(call_context, type, "#{call_context[:request]['host']} #{call_context[:response].code} #{call_context[:response].message}")
21
22
  end
22
23
  end
23
24
  end
@@ -25,7 +26,7 @@ module Aspera
25
26
  # Use this method to analyze a EST result and raise an exception
26
27
  # Analyzes REST call response and raises a RestCallError exception
27
28
  # if HTTP result code is not 2XX
28
- def raise_on_error(req,res)
29
+ def raise_on_error(req, res)
29
30
  call_context = {
30
31
  messages: [],
31
32
  request: req,
@@ -36,14 +37,14 @@ module Aspera
36
37
  # analyze errors from provided handlers
37
38
  # note that there can be an error even if code is 2XX
38
39
  @error_handlers.each do |handler|
39
- begin
40
- #Log.log.debug("test exception: #{handler[:name]}")
41
- handler[:block].call(handler[:name],call_context)
40
+ begin # rubocop:disable Style/RedundantBegin
41
+ # Log.log.debug{"test exception: #{handler[:name]}"}
42
+ handler[:block].call(handler[:name], call_context)
42
43
  rescue StandardError => e
43
- Log.log.error("ERROR in handler:\n#{e.message}\n#{e.backtrace}")
44
+ Log.log.error{"ERROR in handler:\n#{e.message}\n#{e.backtrace}"}
44
45
  end
45
46
  end
46
- raise RestCallError.new(call_context[:request],call_context[:response],call_context[:messages].join("\n")) unless call_context[:messages].empty?
47
+ raise RestCallError.new(call_context[:request], call_context[:response], call_context[:messages].join("\n")) unless call_context[:messages].empty?
47
48
  end
48
49
 
49
50
  # add a new error handler (done at application initialisation)
@@ -51,18 +52,18 @@ module Aspera
51
52
  # @param block : processing of response: takes two parameters: name, call_context
52
53
  # name is the one provided here
53
54
  # call_context is built in method raise_on_error
54
- def add_handler(name,&block)
55
+ def add_handler(name, &block)
55
56
  @error_handlers.unshift({name: name, block: block})
56
57
  end
57
58
 
58
59
  # add a simple error handler
59
60
  # check that key exists and is string under specified path (hash)
60
61
  # adds other keys as secondary information
61
- def add_simple_handler(name,*args)
62
- add_handler(name) do |type,call_context|
62
+ def add_simple_handler(name, *args)
63
+ add_handler(name) do |type, call_context|
63
64
  # need to clone because we modify and same array is used subsequently
64
65
  path = args.clone
65
- #Log.log.debug("path=#{path}")
66
+ # Log.log.debug{"path=#{path}"}
66
67
  # if last in path is boolean it tells if the error is only with http error code or always
67
68
  always = [true, false].include?(path.last) ? path.pop : false
68
69
  if call_context[:data].is_a?(Hash) && (!call_context[:response].code.start_with?('2') || always)
@@ -70,10 +71,10 @@ module Aspera
70
71
  # dig and find sub entry corresponding to path in deep hash
71
72
  error_struct = path.inject(call_context[:data]) { |subhash, key| subhash.respond_to?(:keys) ? subhash[key] : nil }
72
73
  if error_struct.is_a?(Hash) && error_struct[msg_key].is_a?(String)
73
- RestErrorAnalyzer.add_error(call_context,type,error_struct[msg_key])
74
- error_struct.each do |k,v|
74
+ RestErrorAnalyzer.add_error(call_context, type, error_struct[msg_key])
75
+ error_struct.each do |k, v|
75
76
  next if k.eql?(msg_key)
76
- RestErrorAnalyzer.add_error(call_context,"#{type}(sub)","#{k}: #{v}") if [String,Integer].include?(v.class)
77
+ RestErrorAnalyzer.add_error(call_context, "#{type}(sub)", "#{k}: #{v}") if [String, Integer].include?(v.class)
77
78
  end
78
79
  end
79
80
  end
@@ -86,12 +87,12 @@ module Aspera
86
87
  # @param call_context a Hash containing the result call_context, provided to handler
87
88
  # @param type a string describing type of exception, for logging purpose
88
89
  # @param msg one error message to add to list
89
- def add_error(call_context,type,msg)
90
+ def add_error(call_context, type, msg)
90
91
  call_context[:messages].push(msg)
91
92
  logfile = instance.log_file
92
93
  # log error for further analysis (file must exist to activate)
93
94
  return if logfile.nil? || !File.exist?(logfile)
94
- File.open(logfile,'a+') do |f|
95
+ File.open(logfile, 'a+') do |f|
95
96
  f.write("\n=#{type}=====\n#{call_context[:request].method} #{call_context[:request].path}\n#{call_context[:response].code}\n"\
96
97
  "#{JSON.generate(call_context[:data])}\n#{call_context[:messages].join("\n")}")
97
98
  end
@@ -12,45 +12,45 @@ module Aspera
12
12
  Log.log.debug('registering Aspera REST error handlers')
13
13
  # Faspex 4: both user_message and internal_message, and code 200
14
14
  # example: missing meta data on package creation
15
- RestErrorAnalyzer.instance.add_simple_handler('Type 1: error:user_message','error','user_message',true)
16
- RestErrorAnalyzer.instance.add_simple_handler('Type 2: error:description','error','description')
17
- RestErrorAnalyzer.instance.add_simple_handler('Type 3: error:internal_message','error','internal_message')
15
+ RestErrorAnalyzer.instance.add_simple_handler('Type 1: error:user_message', 'error', 'user_message', true)
16
+ RestErrorAnalyzer.instance.add_simple_handler('Type 2: error:description', 'error', 'description')
17
+ RestErrorAnalyzer.instance.add_simple_handler('Type 3: error:internal_message', 'error', 'internal_message')
18
18
  # AoC Automation
19
- RestErrorAnalyzer.instance.add_simple_handler('AoC Automation','error')
20
- RestErrorAnalyzer.instance.add_simple_handler('Type 5','error_description')
21
- RestErrorAnalyzer.instance.add_simple_handler('Type 6','message')
22
- RestErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |name,call_context|
19
+ RestErrorAnalyzer.instance.add_simple_handler('AoC Automation', 'error')
20
+ RestErrorAnalyzer.instance.add_simple_handler('Type 5', 'error_description')
21
+ RestErrorAnalyzer.instance.add_simple_handler('Type 6', 'message')
22
+ RestErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |name, call_context|
23
23
  if call_context[:data].is_a?(Hash) && call_context[:data]['errors'].is_a?(Hash)
24
- call_context[:data]['errors'].each do |k,v|
25
- RestErrorAnalyzer.add_error(call_context,name,"#{k}: #{v}")
24
+ call_context[:data]['errors'].each do |k, v|
25
+ RestErrorAnalyzer.add_error(call_context, name, "#{k}: #{v}")
26
26
  end
27
27
  end
28
28
  end
29
29
  # call to upload_setup and download_setup of node api
30
- RestErrorAnalyzer.instance.add_handler('T8:node: *_setup') do |type,call_context|
30
+ RestErrorAnalyzer.instance.add_handler('T8:node: *_setup') do |type, call_context|
31
31
  if call_context[:data].is_a?(Hash)
32
32
  d_t_s = call_context[:data]['transfer_specs']
33
33
  if d_t_s.is_a?(Array)
34
34
  d_t_s.each do |res|
35
- #r_err=res['transfer_spec']['error']
35
+ # r_err=res['transfer_spec']['error']
36
36
  r_err = res['error']
37
37
  if r_err.is_a?(Hash)
38
- RestErrorAnalyzer.add_error(call_context,type,"#{r_err['code']}: #{r_err['reason']}: #{r_err['user_message']}")
38
+ RestErrorAnalyzer.add_error(call_context, type, "#{r_err['code']}: #{r_err['reason']}: #{r_err['user_message']}")
39
39
  end
40
40
  end
41
41
  end
42
42
  end
43
43
  end
44
- RestErrorAnalyzer.instance.add_simple_handler('T9:IBM cloud IAM','errorMessage')
45
- RestErrorAnalyzer.instance.add_simple_handler('T10:faspex v4','user_message')
46
- RestErrorAnalyzer.instance.add_handler('bss graphql') do |type,call_context|
44
+ RestErrorAnalyzer.instance.add_simple_handler('T9:IBM cloud IAM', 'errorMessage')
45
+ RestErrorAnalyzer.instance.add_simple_handler('T10:faspex v4', 'user_message')
46
+ RestErrorAnalyzer.instance.add_handler('bss graphql') do |type, call_context|
47
47
  if call_context[:data].is_a?(Hash)
48
48
  d_t_s = call_context[:data]['errors']
49
49
  if d_t_s.is_a?(Array)
50
50
  d_t_s.each do |res|
51
51
  r_err = res['message']
52
52
  if r_err.is_a?(String)
53
- RestErrorAnalyzer.add_error(call_context,type,r_err)
53
+ RestErrorAnalyzer.add_error(call_context, type, r_err)
54
54
  end
55
55
  end
56
56
  end
@@ -7,16 +7,17 @@ module Aspera
7
7
  class SecretHider
8
8
  # display string for hidden secrets
9
9
  HIDDEN_PASSWORD = '🔑'
10
+ # env vars for ascp with secrets
11
+ ASCP_ENV_SECRETS = %w[ASPERA_SCP_PASS ASPERA_SCP_KEY ASPERA_SCP_FILEPASS ASPERA_PROXY_PASS ASPERA_SCP_TOKEN].freeze
10
12
  # keys in hash that contain secrets
11
- ASCP_SECRETS=%w[ASPERA_SCP_PASS ASPERA_SCP_KEY ASPERA_SCP_FILEPASS ASPERA_PROXY_PASS].freeze
12
- KEY_SECRETS =%w[password secret private_key passphrase].freeze
13
- ALL_SECRETS =[ASCP_SECRETS,KEY_SECRETS].flatten.freeze
13
+ KEY_SECRETS = %w[password secret private_key passphrase].freeze
14
+ ALL_SECRETS = [].concat(ASCP_ENV_SECRETS, KEY_SECRETS).freeze
14
15
  # regex that define namec captures :begin and :end
15
- REGEX_LOG_REPLACES=[
16
+ REGEX_LOG_REPLACES = [
16
17
  # CLI manager get/set options
17
18
  /(?<begin>[sg]et (#{KEY_SECRETS.join('|')})=).*(?<end>)/,
18
19
  # env var ascp exec
19
- /(?<begin> (#{ASCP_SECRETS.join('|')})=)[^ ]*(?<end> )/,
20
+ /(?<begin> (#{ASCP_ENV_SECRETS.join('|')})=)(\\.|[^ ])*(?<end> )/,
20
21
  # rendered JSON
21
22
  /(?<begin>["':][^"]*(#{ALL_SECRETS.join('|')})[^"]*["']?[=>: ]+")[^"]+(?<end>")/,
22
23
  # option "secret"
@@ -26,13 +27,14 @@ module Aspera
26
27
  # private key values
27
28
  /(?<begin>--+BEGIN .+ KEY--+)[[:ascii:]]+?(?<end>--+?END .+ KEY--+)/
28
29
  ].freeze
29
- private_constant :HIDDEN_PASSWORD,:ASCP_SECRETS,:KEY_SECRETS,:ALL_SECRETS,:REGEX_LOG_REPLACES
30
+ private_constant :HIDDEN_PASSWORD, :ASCP_ENV_SECRETS, :KEY_SECRETS, :ALL_SECRETS, :REGEX_LOG_REPLACES
30
31
  @log_secrets = false
31
32
  class << self
32
33
  attr_accessor :log_secrets
34
+
33
35
  def log_formatter(original_formatter)
34
36
  original_formatter ||= Logger::Formatter.new
35
- # note that @log_secrets may be set AFTER this init is done, so it's done at runtime
37
+ # NOTE: that @log_secrets may be set AFTER this init is done, so it's done at runtime
36
38
  return lambda do |severity, datetime, progname, msg|
37
39
  if msg.is_a?(String) && !@log_secrets
38
40
  REGEX_LOG_REPLACES.each do |regx|
@@ -43,25 +45,25 @@ module Aspera
43
45
  end
44
46
  end
45
47
 
46
- def secret?(keyword,value)
47
- keyword=keyword.to_s if keyword.is_a?(Symbol)
48
+ def secret?(keyword, value)
49
+ keyword = keyword.to_s if keyword.is_a?(Symbol)
48
50
  # only Strings can be secrets, not booleans, or hash, arrays
49
51
  keyword.is_a?(String) && ALL_SECRETS.any?{|kw|keyword.include?(kw)} && value.is_a?(String)
50
52
  end
51
53
 
52
- def deep_remove_secret(obj,is_name_value: false)
54
+ def deep_remove_secret(obj, is_name_value: false)
53
55
  case obj
54
56
  when Array
55
57
  if is_name_value
56
58
  obj.each do |i|
57
- i['value']=HIDDEN_PASSWORD if secret?(i['parameter'],i['value'])
59
+ i['value'] = HIDDEN_PASSWORD if secret?(i['parameter'], i['value'])
58
60
  end
59
61
  else
60
62
  obj.each{|i|deep_remove_secret(i)}
61
63
  end
62
64
  when Hash
63
- obj.each do |k,v|
64
- if secret?(k,v)
65
+ obj.each do |k, v|
66
+ if secret?(k, v)
65
67
  obj[k] = HIDDEN_PASSWORD
66
68
  elsif obj[k].is_a?(Hash)
67
69
  deep_remove_secret(obj[k])
data/lib/aspera/ssh.rb CHANGED
@@ -2,9 +2,9 @@
2
2
 
3
3
  require 'net/ssh'
4
4
 
5
- # Hack: deactivate ed25519 and ecdsa private keys from ssh identities, as it usually hurts
5
+ # HACK: deactivate ed25519 and ecdsa private keys from ssh identities, as it usually hurts
6
6
  begin
7
- module Net;module SSH;module Authentication;class Session;private; def default_keys; %w[~/.ssh/id_dsa ~/.ssh/id_rsa ~/.ssh2/id_dsa ~/.ssh2/id_rsa];end;end;end;end;end # rubocop:disable Layout/AccessModifierIndentation, Layout/EmptyLinesAroundAccessModifier, Layout/LineLength
7
+ module Net; module SSH; module Authentication; class Session; private; def default_keys; %w[~/.ssh/id_dsa ~/.ssh/id_rsa ~/.ssh2/id_dsa ~/.ssh2/id_rsa]; end; end; end; end; end # rubocop:disable Layout/AccessModifierIndentation, Layout/EmptyLinesAroundAccessModifier, Layout/LineLength, Style/Semicolon
8
8
  rescue StandardError
9
9
  # ignore errors
10
10
  end
@@ -15,26 +15,26 @@ module Aspera
15
15
  class Ssh
16
16
  # ssh_options: same as Net::SSH.start
17
17
  # see: https://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start
18
- def initialize(host,username,ssh_options)
19
- Log.log.debug("ssh:#{username}@#{host}")
20
- Log.log.debug("ssh_options:#{ssh_options}")
18
+ def initialize(host, username, ssh_options)
19
+ Log.log.debug{"ssh:#{username}@#{host}"}
20
+ Log.log.debug{"ssh_options:#{ssh_options}"}
21
21
  @host = host
22
22
  @username = username
23
23
  @ssh_options = ssh_options
24
24
  @ssh_options[:logger] = Log.log
25
25
  end
26
26
 
27
- def execute(cmd,input=nil)
27
+ def execute(cmd, input=nil)
28
28
  if cmd.is_a?(Array)
29
29
  # concatenate arguments, enclose in double quotes
30
30
  cmd = cmd.map{|v|%Q("#{v}")}.join(' ')
31
31
  end
32
- Log.log.debug("cmd=#{cmd}")
32
+ Log.log.debug{"cmd=#{cmd}"}
33
33
  response = []
34
34
  Net::SSH.start(@host, @username, @ssh_options) do |session|
35
35
  ssh_channel = session.open_channel do |channel|
36
36
  # prepare stdout processing
37
- channel.on_data{|_chan,data|response.push(data)}
37
+ channel.on_data{|_chan, data|response.push(data)}
38
38
  # prepare stderr processing, stderr if type = 1
39
39
  channel.on_extended_data do |_chan, _type, data|
40
40
  errormsg = "#{cmd}: [#{data.chomp}]"
@@ -45,13 +45,14 @@ module Aspera
45
45
  raise errormsg
46
46
  end
47
47
  # send commannd to SSH channel (execute)
48
- channel.send('cexe'.reverse,cmd){|_ch,_success|channel.send_data(input) unless input.nil?}
48
+ channel.send('cexe'.reverse, cmd){|_ch, _success|channel.send_data(input) unless input.nil?}
49
49
  end
50
50
  # wait for channel to finish (command exit)
51
51
  ssh_channel.wait
52
52
  # main ssh session loop
53
53
  session.loop
54
- end # session
54
+ end # start
55
+ # response as single string
55
56
  return response.join
56
57
  end
57
58
  end