aspera-cli 4.7.0 → 4.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +1 -0
  3. data/README.md +844 -861
  4. data/bin/ascli +20 -1
  5. data/bin/asession +37 -34
  6. data/docs/test_env.conf +11 -3
  7. data/examples/aoc.rb +13 -12
  8. data/examples/dascli +26 -0
  9. data/examples/faspex4.rb +34 -29
  10. data/examples/transfer.rb +30 -29
  11. data/lib/aspera/aoc.rb +151 -143
  12. data/lib/aspera/ascmd.rb +56 -45
  13. data/lib/aspera/ats_api.rb +6 -5
  14. data/lib/aspera/cli/basic_auth_plugin.rb +18 -16
  15. data/lib/aspera/cli/extended_value.rb +32 -30
  16. data/lib/aspera/cli/formater.rb +103 -111
  17. data/lib/aspera/cli/info.rb +2 -1
  18. data/lib/aspera/cli/listener/line_dump.rb +1 -0
  19. data/lib/aspera/cli/listener/logger.rb +1 -0
  20. data/lib/aspera/cli/listener/progress.rb +13 -12
  21. data/lib/aspera/cli/listener/progress_multi.rb +21 -20
  22. data/lib/aspera/cli/main.rb +106 -89
  23. data/lib/aspera/cli/manager.rb +96 -85
  24. data/lib/aspera/cli/plugin.rb +50 -32
  25. data/lib/aspera/cli/plugins/alee.rb +6 -5
  26. data/lib/aspera/cli/plugins/aoc.rb +521 -426
  27. data/lib/aspera/cli/plugins/ats.rb +84 -83
  28. data/lib/aspera/cli/plugins/bss.rb +30 -27
  29. data/lib/aspera/cli/plugins/config.rb +483 -397
  30. data/lib/aspera/cli/plugins/console.rb +17 -15
  31. data/lib/aspera/cli/plugins/cos.rb +26 -35
  32. data/lib/aspera/cli/plugins/faspex.rb +201 -168
  33. data/lib/aspera/cli/plugins/faspex5.rb +109 -74
  34. data/lib/aspera/cli/plugins/node.rb +378 -189
  35. data/lib/aspera/cli/plugins/orchestrator.rb +71 -65
  36. data/lib/aspera/cli/plugins/preview.rb +131 -122
  37. data/lib/aspera/cli/plugins/server.rb +94 -93
  38. data/lib/aspera/cli/plugins/shares.rb +42 -28
  39. data/lib/aspera/cli/plugins/sync.rb +15 -14
  40. data/lib/aspera/cli/transfer_agent.rb +56 -52
  41. data/lib/aspera/cli/version.rb +2 -1
  42. data/lib/aspera/colors.rb +29 -28
  43. data/lib/aspera/command_line_builder.rb +50 -43
  44. data/lib/aspera/cos_node.rb +64 -38
  45. data/lib/aspera/data_repository.rb +1 -0
  46. data/lib/aspera/environment.rb +18 -8
  47. data/lib/aspera/fasp/agent_base.rb +26 -23
  48. data/lib/aspera/fasp/agent_connect.rb +35 -30
  49. data/lib/aspera/fasp/agent_direct.rb +68 -60
  50. data/lib/aspera/fasp/agent_httpgw.rb +71 -64
  51. data/lib/aspera/fasp/agent_node.rb +24 -23
  52. data/lib/aspera/fasp/agent_trsdk.rb +19 -20
  53. data/lib/aspera/fasp/error.rb +2 -1
  54. data/lib/aspera/fasp/error_info.rb +79 -68
  55. data/lib/aspera/fasp/installation.rb +122 -114
  56. data/lib/aspera/fasp/listener.rb +1 -0
  57. data/lib/aspera/fasp/parameters.rb +44 -41
  58. data/lib/aspera/fasp/resume_policy.rb +14 -11
  59. data/lib/aspera/fasp/transfer_spec.rb +6 -5
  60. data/lib/aspera/fasp/uri.rb +25 -24
  61. data/lib/aspera/faspex_gw.rb +83 -72
  62. data/lib/aspera/hash_ext.rb +10 -12
  63. data/lib/aspera/id_generator.rb +8 -7
  64. data/lib/aspera/keychain/encrypted_hash.rb +60 -45
  65. data/lib/aspera/keychain/macos_security.rb +26 -24
  66. data/lib/aspera/log.rb +34 -38
  67. data/lib/aspera/nagios.rb +14 -13
  68. data/lib/aspera/node.rb +19 -19
  69. data/lib/aspera/oauth.rb +121 -101
  70. data/lib/aspera/open_application.rb +6 -5
  71. data/lib/aspera/persistency_action_once.rb +9 -8
  72. data/lib/aspera/persistency_folder.rb +10 -9
  73. data/lib/aspera/preview/file_types.rb +261 -266
  74. data/lib/aspera/preview/generator.rb +74 -73
  75. data/lib/aspera/preview/image_error.png +0 -0
  76. data/lib/aspera/preview/options.rb +7 -6
  77. data/lib/aspera/preview/utils.rb +30 -33
  78. data/lib/aspera/preview/video_error.png +0 -0
  79. data/lib/aspera/proxy_auto_config.rb +25 -23
  80. data/lib/aspera/rest.rb +73 -74
  81. data/lib/aspera/rest_call_error.rb +1 -0
  82. data/lib/aspera/rest_error_analyzer.rb +11 -9
  83. data/lib/aspera/rest_errors_aspera.rb +5 -4
  84. data/lib/aspera/secret_hider.rb +68 -0
  85. data/lib/aspera/ssh.rb +12 -10
  86. data/lib/aspera/sync.rb +49 -47
  87. data/lib/aspera/temp_file_manager.rb +7 -5
  88. data/lib/aspera/timer_limiter.rb +9 -8
  89. data/lib/aspera/uri_reader.rb +11 -14
  90. data/lib/aspera/web_auth.rb +17 -15
  91. data.tar.gz.sig +0 -0
  92. metadata +117 -34
  93. metadata.gz.sig +2 -0
  94. data/bin/dascli +0 -13
data/lib/aspera/rest.rb CHANGED
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+
2
3
  require 'aspera/log'
3
4
  require 'aspera/oauth'
4
5
  require 'aspera/rest_error_analyzer'
@@ -30,40 +31,39 @@ module Aspera
30
31
  # and error are analyzed in RestErrorAnalyzer
31
32
  class Rest
32
33
  # global settings also valid for any subclass
33
- @@global={ # rubocop:disable Style/ClassVars
34
- debug: false,
34
+ @@global = { # rubocop:disable Style/ClassVars
35
+ debug: false,
35
36
  # true if https ignore certificate
36
- insecure: false,
37
- user_agent: 'Ruby',
37
+ user_agent: 'Ruby',
38
38
  download_partial_suffix: '.http_partial',
39
39
  # a lambda which takes the Net::HTTP as arg, use this to change parameters
40
- session_cb: nil
40
+ session_cb: nil
41
41
  }
42
42
 
43
- class<<self
43
+ class << self
44
44
  # define accessors
45
45
  @@global.keys.each do |p|
46
46
  define_method(p){@@global[p]}
47
- define_method("#{p}="){|val|Log.log.debug("#{p} => #{val}".red);@@global[p]=val}
47
+ define_method("#{p}="){|val|Log.log.debug("#{p} => #{val}".red);@@global[p] = val}
48
48
  end
49
49
 
50
50
  def basic_creds(user,pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}";end
51
51
 
52
52
  # build URI from URL and parameters and check it is http or https
53
53
  def build_uri(url,params=nil)
54
- uri=URI.parse(url)
55
- raise 'REST endpoint shall be http(s)' unless ['http','https'].include?(uri.scheme)
54
+ uri = URI.parse(url)
55
+ raise "REST endpoint shall be http/s not #{uri.scheme}" unless %w[http https].include?(uri.scheme)
56
56
  if !params.nil?
57
57
  # support array url params, there is no standard. Either p[]=1&p[]=2, or p=1&p=2
58
58
  if params.is_a?(Hash)
59
- orig=params
60
- params=[]
59
+ orig = params
60
+ params = []
61
61
  orig.each do |k,v|
62
62
  case v
63
63
  when Array
64
- suffix=v.first.eql?('[]') ? v.shift : ''
64
+ suffix = v.first.eql?('[]') ? v.shift : ''
65
65
  v.each do |e|
66
- params.push([k+suffix,e])
66
+ params.push([k + suffix,e])
67
67
  end
68
68
  else
69
69
  params.push([k,v])
@@ -71,7 +71,7 @@ module Aspera
71
71
  end
72
72
  end
73
73
  # CGI.unescape to transform back %5D into []
74
- uri.query=CGI.unescape(URI.encode_www_form(params))
74
+ uri.query = CGI.unescape(URI.encode_www_form(params))
75
75
  end
76
76
  return uri
77
77
  end
@@ -82,14 +82,13 @@ module Aspera
82
82
  # create and start keep alive connection on demand
83
83
  def http_session
84
84
  if @http_session.nil?
85
- uri=self.class.build_uri(@params[:base_url])
85
+ uri = self.class.build_uri(@params[:base_url])
86
86
  # this honors http_proxy env var
87
- @http_session=Net::HTTP.new(uri.host, uri.port)
87
+ @http_session = Net::HTTP.new(uri.host, uri.port)
88
88
  @http_session.use_ssl = uri.scheme.eql?('https')
89
- @http_session.verify_mode = OpenSSL::SSL::VERIFY_NONE if self.class.insecure
90
89
  @http_session.set_debug_output($stdout) if self.class.debug
91
90
  # set http options in callback, such as timeout and cert. verification
92
- self.class.session_cb.call(@http_session) unless self.class.session_cb.nil?
91
+ self.class.session_cb&.call(@http_session)
93
92
  # manually start session for keep alive (if supported by server, else, session is closed every time)
94
93
  @http_session.start
95
94
  end
@@ -103,7 +102,7 @@ module Aspera
103
102
  def oauth
104
103
  if @oauth.nil?
105
104
  raise 'ERROR: no OAuth defined' unless @params[:auth][:type].eql?(:oauth2)
106
- @oauth=Oauth.new(@params[:auth])
105
+ @oauth = Oauth.new(@params[:auth])
107
106
  end
108
107
  return @oauth
109
108
  end
@@ -112,15 +111,15 @@ module Aspera
112
111
  def initialize(a_rest_params)
113
112
  raise 'ERROR: expecting Hash' unless a_rest_params.is_a?(Hash)
114
113
  raise 'ERROR: expecting base_url' unless a_rest_params[:base_url].is_a?(String)
115
- @params=a_rest_params.clone
114
+ @params = a_rest_params.clone
116
115
  Log.dump('REST params',@params)
117
116
  # base url without trailing slashes (note: string may be frozen)
118
- @params[:base_url]=@params[:base_url].gsub(/\/+$/,'')
119
- @http_session=nil
117
+ @params[:base_url] = @params[:base_url].gsub(/\/+$/,'')
118
+ @http_session = nil
120
119
  # default is no auth
121
- @params[:auth]||={type: :none}
122
- @params[:not_auth_codes]||=['401']
123
- @oauth=nil
120
+ @params[:auth] ||= {type: :none}
121
+ @params[:not_auth_codes] ||= ['401']
122
+ @oauth = nil
124
123
  Log.dump('REST params(2)',@params)
125
124
  end
126
125
 
@@ -132,28 +131,28 @@ module Aspera
132
131
  def build_request(call_data)
133
132
  # TODO: shall we percent encode subpath (spaces) test with access key delete with space in id
134
133
  # URI.escape()
135
- uri=self.class.build_uri("#{call_data[:base_url]}#{['','/'].include?(call_data[:subpath]) ? '' : '/'}#{call_data[:subpath]}",call_data[:url_params])
134
+ uri = self.class.build_uri("#{call_data[:base_url]}#{['','/'].include?(call_data[:subpath]) ? '' : '/'}#{call_data[:subpath]}",call_data[:url_params])
136
135
  Log.log.debug("URI=#{uri}")
137
136
  begin
138
137
  # instanciate request object based on string name
139
- req=Net::HTTP.const_get(call_data[:operation].capitalize).new(uri)
138
+ req = Net::HTTP.const_get(call_data[:operation].capitalize).new(uri)
140
139
  rescue NameError
141
140
  raise "unsupported operation : #{call_data[:operation]}"
142
141
  end
143
142
  if call_data.has_key?(:json_params) && !call_data[:json_params].nil?
144
- req.body=JSON.generate(call_data[:json_params])
143
+ req.body = JSON.generate(call_data[:json_params])
145
144
  Log.dump('body JSON data',call_data[:json_params])
146
145
  #Log.log.debug("body JSON data=#{JSON.pretty_generate(call_data[:json_params])}")
147
146
  req['Content-Type'] = 'application/json'
148
147
  #call_data[:headers]['Accept']='application/json'
149
148
  end
150
149
  if call_data.has_key?(:www_body_params)
151
- req.body=URI.encode_www_form(call_data[:www_body_params])
150
+ req.body = URI.encode_www_form(call_data[:www_body_params])
152
151
  Log.log.debug("body www data=#{req.body.chomp}")
153
152
  req['Content-Type'] = 'application/x-www-form-urlencoded'
154
153
  end
155
154
  if call_data.has_key?(:text_body_params)
156
- req.body=call_data[:text_body_params]
155
+ req.body = call_data[:text_body_params]
157
156
  Log.log.debug("body data=#{req.body.chomp}")
158
157
  end
159
158
  # set headers
@@ -180,7 +179,7 @@ module Aspera
180
179
  # :save_to_file (filepath) default: nil
181
180
  # :return_error (bool) default: nil
182
181
  # :redirect_max (int) default: 0
183
- # :not_auth_codes (array)
182
+ # :not_auth_codes (array) codes that trigger a refresh/regeneration of bearer token
184
183
  # ----
185
184
  # authentication (:auth) :
186
185
  # :type (:none, :basic, :oauth2, :url)
@@ -190,12 +189,12 @@ module Aspera
190
189
  # :* [:oauth2] see Oauth class
191
190
  def call(call_data)
192
191
  raise "Hash call parameter is required (#{call_data.class})" unless call_data.is_a?(Hash)
193
- call_data[:subpath]='' if call_data[:subpath].nil?
192
+ call_data[:subpath] = '' if call_data[:subpath].nil?
194
193
  Log.log.debug("accessing #{call_data[:subpath]}".red.bold.bg_green)
195
- call_data[:headers]||={}
194
+ call_data[:headers] ||= {}
196
195
  call_data[:headers]['User-Agent'] ||= self.class.user_agent
197
196
  # defaults from @params are overriden by call data
198
- call_data=@params.deep_merge(call_data)
197
+ call_data = @params.deep_merge(call_data)
199
198
  case call_data[:auth][:type]
200
199
  when :none
201
200
  # no auth
@@ -203,60 +202,60 @@ module Aspera
203
202
  Log.log.debug('using Basic auth')
204
203
  # done in build_req
205
204
  when :oauth2
206
- call_data[:headers]['Authorization']=oauth_token unless call_data[:headers].has_key?('Authorization')
205
+ call_data[:headers]['Authorization'] = oauth_token unless call_data[:headers].has_key?('Authorization')
207
206
  when :url
208
- call_data[:url_params]||={}
207
+ call_data[:url_params] ||= {}
209
208
  call_data[:auth][:url_creds].each do |key, value|
210
- call_data[:url_params][key]=value
209
+ call_data[:url_params][key] = value
211
210
  end
212
211
  else raise "unsupported auth type: [#{call_data[:auth][:type]}]"
213
212
  end
214
- req=build_request(call_data)
213
+ req = build_request(call_data)
215
214
  Log.log.debug("call_data = #{call_data}")
216
- result={http: nil}
215
+ result = {http: nil}
217
216
  # start a block to be able to retry the actual HTTP request
218
217
  begin
219
218
  # we try the call, and will retry only if oauth, as we can, first with refresh, and then re-auth if refresh is bad
220
219
  oauth_tries ||= 2
221
- tries_remain_redirect||=call_data[:redirect_max].nil? ? 0 : call_data[:redirect_max].to_i
220
+ tries_remain_redirect ||= call_data[:redirect_max].nil? ? 0 : call_data[:redirect_max].to_i
222
221
  Log.log.debug('send request')
223
222
  # make http request (pipelined)
224
223
  http_session.request(req) do |response|
225
224
  result[:http] = response
226
225
  if !call_data[:save_to_file].nil? && result[:http].code.to_s.start_with?('2')
227
- total_size=result[:http]['Content-Length'].to_i
228
- progress=ProgressBar.create(
229
- format: '%a %B %p%% %r KB/sec %e',
230
- rate_scale: lambda{|rate|rate/1024},
231
- title: 'progress',
232
- total: total_size)
226
+ total_size = result[:http]['Content-Length'].to_i
227
+ progress = ProgressBar.create(
228
+ format: '%a %B %p%% %r KB/sec %e',
229
+ rate_scale: lambda{|rate|rate / 1024},
230
+ title: 'progress',
231
+ total: total_size)
233
232
  Log.log.debug('before write file')
234
- target_file=call_data[:save_to_file]
233
+ target_file = call_data[:save_to_file]
235
234
  # override user's path to path in header
236
- if !response['Content-Disposition'].nil? && (m=response['Content-Disposition'].match(/filename="([^"]+)"/))
237
- target_file=File.join(File.dirname(target_file),m[1])
235
+ if !response['Content-Disposition'].nil? && (m = response['Content-Disposition'].match(/filename="([^"]+)"/))
236
+ target_file = File.join(File.dirname(target_file),m[1])
238
237
  end
239
238
  # download with temp filename
240
- target_file_tmp="#{target_file}#{self.class.download_partial_suffix}"
239
+ target_file_tmp = "#{target_file}#{self.class.download_partial_suffix}"
241
240
  Log.log.debug("saving to: #{target_file}")
242
241
  File.open(target_file_tmp, 'wb') do |file|
243
242
  result[:http].read_body do |fragment|
244
243
  file.write(fragment)
245
- new_process=progress.progress+fragment.length
244
+ new_process = progress.progress + fragment.length
246
245
  new_process = total_size if new_process > total_size
247
- progress.progress=new_process
246
+ progress.progress = new_process
248
247
  end
249
248
  end
250
249
  # rename at the end
251
250
  File.rename(target_file_tmp, target_file)
252
- progress=nil
251
+ progress = nil
253
252
  end # save_to_file
254
253
  end
255
254
  # sometimes there is a UTF8 char (e.g. (c) )
256
255
  result[:http].body.force_encoding('UTF-8') if result[:http].body.is_a?(String)
257
256
  Log.log.debug("result: body=#{result[:http].body}")
258
- result_mime=(result[:http]['Content-Type']||'text/plain').split(';').first
259
- result[:data]=case result_mime
257
+ result_mime = (result[:http]['Content-Type'] || 'text/plain').split(';').first
258
+ result[:data] = case result_mime
260
259
  when 'application/json','application/vnd.api+json'
261
260
  JSON.parse(result[:http].body) rescue nil
262
261
  else #when 'text/plain'
@@ -270,34 +269,34 @@ module Aspera
270
269
  if call_data[:not_auth_codes].include?(result[:http].code.to_s) && call_data[:auth][:type].eql?(:oauth2)
271
270
  begin
272
271
  # try to use refresh token
273
- req['Authorization']=oauth_token(force_refresh: true)
274
- rescue RestCallError => e
272
+ req['Authorization'] = oauth_token(force_refresh: true)
273
+ rescue RestCallError => e_tok
274
+ e = e_tok
275
275
  Log.log.error('refresh failed'.bg_red)
276
276
  # regenerate a brand new token
277
- req['Authorization']=oauth_token
277
+ req['Authorization'] = oauth_token(use_cache: false)
278
278
  end
279
279
  Log.log.debug("using new token=#{call_data[:headers]['Authorization']}")
280
280
  retry unless (oauth_tries -= 1).zero?
281
281
  end # if oauth
282
282
  # moved ?
283
- raise e unless e.response.is_a?(Net::HTTPRedirection)
284
- if tries_remain_redirect.positive?
285
- tries_remain_redirect-=1
286
- Log.log.info("URL is moved: #{e.response['location']}")
287
- current_uri=URI.parse(call_data[:base_url])
288
- redir_uri=URI.parse(e.response['location'])
289
- call_data[:base_url]=e.response['location']
290
- call_data[:subpath]=''
283
+ if e.response.is_a?(Net::HTTPRedirection) && tries_remain_redirect.positive?
284
+ tries_remain_redirect -= 1
285
+ current_uri = URI.parse(call_data[:base_url])
286
+ new_url=e.response['location']
287
+ new_url="#{current_uri.scheme}:#{new_url}" unless new_url.start_with?('http')
288
+ Log.log.info("URL is moved: #{new_url}")
289
+ redir_uri = URI.parse(new_url)
290
+ call_data[:base_url] = new_url
291
+ call_data[:subpath] = ''
291
292
  if current_uri.host.eql?(redir_uri.host) && current_uri.port.eql?(redir_uri.port)
292
- req=build_request(call_data)
293
+ req = build_request(call_data)
293
294
  retry
294
295
  else
295
296
  # change host
296
297
  Log.log.info("Redirect changes host: #{current_uri.host} -> #{redir_uri.host}")
297
298
  return self.class.new(call_data).call(call_data)
298
299
  end
299
- else
300
- raise e unless call_data[:return_error]
301
300
  end
302
301
  # raise exception if could not retry and not return error in result
303
302
  raise e unless call_data[:return_error]
@@ -312,23 +311,23 @@ module Aspera
312
311
 
313
312
  # @param encoding : one of: :json_params, :url_params
314
313
  def create(subpath,params,encoding=:json_params)
315
- return call({operation: 'POST',subpath: subpath,headers: {'Accept'=>'application/json'},encoding=>params})
314
+ return call({operation: 'POST',subpath: subpath,headers: {'Accept' => 'application/json'},encoding => params})
316
315
  end
317
316
 
318
317
  def read(subpath,args=nil)
319
- return call({operation: 'GET',subpath: subpath,headers: {'Accept'=>'application/json'},url_params: args})
318
+ return call({operation: 'GET',subpath: subpath,headers: {'Accept' => 'application/json'},url_params: args})
320
319
  end
321
320
 
322
321
  def update(subpath,params)
323
- return call({operation: 'PUT',subpath: subpath,headers: {'Accept'=>'application/json'},json_params: params})
322
+ return call({operation: 'PUT',subpath: subpath,headers: {'Accept' => 'application/json'},json_params: params})
324
323
  end
325
324
 
326
325
  def delete(subpath)
327
- return call({operation: 'DELETE',subpath: subpath,headers: {'Accept'=>'application/json'}})
326
+ return call({operation: 'DELETE',subpath: subpath,headers: {'Accept' => 'application/json'}})
328
327
  end
329
328
 
330
329
  def cancel(subpath)
331
- return call({operation: 'CANCEL',subpath: subpath,headers: {'Accept'=>'application/json'}})
330
+ return call({operation: 'CANCEL',subpath: subpath,headers: {'Accept' => 'application/json'}})
332
331
  end
333
332
  end
334
333
  end #module Aspera
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+
2
3
  module Aspera
3
4
  # raised on error after REST call
4
5
  class RestCallError < StandardError
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+
2
3
  require 'aspera/log'
3
4
  require 'aspera/rest_call_error'
4
5
  require 'singleton'
@@ -11,8 +12,8 @@ module Aspera
11
12
  # the singleton object is registered with application specific handlers
12
13
  def initialize
13
14
  # list of handlers
14
- @error_handlers=[]
15
- @log_file=nil
15
+ @error_handlers = []
16
+ @log_file = nil
16
17
  add_handler('Type Generic') do |type,call_context|
17
18
  if !call_context[:response].code.start_with?('2')
18
19
  # add generic information
@@ -25,7 +26,7 @@ module Aspera
25
26
  # Analyzes REST call response and raises a RestCallError exception
26
27
  # if HTTP result code is not 2XX
27
28
  def raise_on_error(req,res)
28
- call_context={
29
+ call_context = {
29
30
  messages: [],
30
31
  request: req,
31
32
  response: res[:http],
@@ -60,14 +61,14 @@ module Aspera
60
61
  def add_simple_handler(name,*args)
61
62
  add_handler(name) do |type,call_context|
62
63
  # need to clone because we modify and same array is used subsequently
63
- path=args.clone
64
+ path = args.clone
64
65
  #Log.log.debug("path=#{path}")
65
66
  # if last in path is boolean it tells if the error is only with http error code or always
66
- always=[true, false].include?(path.last) ? path.pop : false
67
+ always = [true, false].include?(path.last) ? path.pop : false
67
68
  if call_context[:data].is_a?(Hash) && (!call_context[:response].code.start_with?('2') || always)
68
- msg_key=path.pop
69
+ msg_key = path.pop
69
70
  # dig and find sub entry corresponding to path in deep hash
70
- error_struct=path.inject(call_context[:data]) { |subhash, key| subhash.respond_to?(:keys) ? subhash[key] : nil }
71
+ error_struct = path.inject(call_context[:data]) { |subhash, key| subhash.respond_to?(:keys) ? subhash[key] : nil }
71
72
  if error_struct.is_a?(Hash) && error_struct[msg_key].is_a?(String)
72
73
  RestErrorAnalyzer.add_error(call_context,type,error_struct[msg_key])
73
74
  error_struct.each do |k,v|
@@ -86,11 +87,12 @@ module Aspera
86
87
  # @param msg one error message to add to list
87
88
  def self.add_error(call_context,type,msg)
88
89
  call_context[:messages].push(msg)
89
- logfile=instance.log_file
90
+ logfile = instance.log_file
90
91
  # log error for further analysis (file must exist to activate)
91
92
  return if logfile.nil? || !File.exist?(logfile)
92
93
  File.open(logfile,'a+') do |f|
93
- f.write("\n=#{type}=====\n#{call_context[:request].method} #{call_context[:request].path}\n#{call_context[:response].code}\n#{JSON.generate(call_context[:data])}\n#{call_context[:messages].join("\n")}")
94
+ f.write("\n=#{type}=====\n#{call_context[:request].method} #{call_context[:request].path}\n#{call_context[:response].code}\n"\
95
+ "#{JSON.generate(call_context[:data])}\n#{call_context[:messages].join("\n")}")
94
96
  end
95
97
  end
96
98
  end
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+
2
3
  require 'aspera/rest_error_analyzer'
3
4
  require 'aspera/log'
4
5
 
@@ -27,11 +28,11 @@ module Aspera
27
28
  # call to upload_setup and download_setup of node api
28
29
  RestErrorAnalyzer.instance.add_handler('T8:node: *_setup') do |type,call_context|
29
30
  if call_context[:data].is_a?(Hash)
30
- d_t_s=call_context[:data]['transfer_specs']
31
+ d_t_s = call_context[:data]['transfer_specs']
31
32
  if d_t_s.is_a?(Array)
32
33
  d_t_s.each do |res|
33
34
  #r_err=res['transfer_spec']['error']
34
- r_err=res['error']
35
+ r_err = res['error']
35
36
  if r_err.is_a?(Hash)
36
37
  RestErrorAnalyzer.add_error(call_context,type,"#{r_err['code']}: #{r_err['reason']}: #{r_err['user_message']}")
37
38
  end
@@ -43,10 +44,10 @@ module Aspera
43
44
  RestErrorAnalyzer.instance.add_simple_handler('T10:faspex v4','user_message')
44
45
  RestErrorAnalyzer.instance.add_handler('bss graphql') do |type,call_context|
45
46
  if call_context[:data].is_a?(Hash)
46
- d_t_s=call_context[:data]['errors']
47
+ d_t_s = call_context[:data]['errors']
47
48
  if d_t_s.is_a?(Array)
48
49
  d_t_s.each do |res|
49
- r_err=res['message']
50
+ r_err = res['message']
50
51
  if r_err.is_a?(String)
51
52
  RestErrorAnalyzer.add_error(call_context,type,r_err)
52
53
  end
@@ -0,0 +1,68 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'logger'
4
+
5
+ module Aspera
6
+ # remove secret from logs and output
7
+ class SecretHider
8
+ # display string for hidden secrets
9
+ HIDDEN_PASSWORD = '🔑'
10
+ # keys in hash that contain secrets
11
+ SECRET_KEYWORDS = %w[password secret private_key passphrase].freeze
12
+ # regex that define namec captures :begin and :end
13
+ REGEX_LOG_REPLACES=[
14
+ # replace values in logs with rendered JSON
15
+ /(?<begin>["':][^"]*(#{SECRET_KEYWORDS.join('|')})[^"]*["']?[=>: ]+")[^"]+(?<end>")/,
16
+ # option "secret"
17
+ /(?<begin>"[^"]*(secret)[^"]*"=>{)[^}]+(?<end>})/,
18
+ # option "secrets"
19
+ /(?<begin>(secrets)={)[^}]+(?<end>})/,
20
+ # private key values
21
+ /(?<begin>--+BEGIN .+ KEY--+)[[:ascii:]]+?(?<end>--+?END .+ KEY--+)/
22
+ ].freeze
23
+ private_constant :HIDDEN_PASSWORD,:SECRET_KEYWORDS
24
+ @log_secrets = false
25
+ class << self
26
+ attr_accessor :log_secrets
27
+ def log_formatter(original_formatter)
28
+ original_formatter ||= Logger::Formatter.new
29
+ # note that @log_secrets may be set AFTER this init is done, so it's done at runtime
30
+ return lambda do |severity, datetime, progname, msg|
31
+ if msg.is_a?(String) && !@log_secrets
32
+ REGEX_LOG_REPLACES.each do |regx|
33
+ msg = msg.gsub(regx){"#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}"}
34
+ end
35
+ end
36
+ original_formatter.call(severity, datetime, progname, msg)
37
+ end
38
+ end
39
+
40
+ def secret?(keyword,value)
41
+ keyword=keyword.to_s if keyword.is_a?(Symbol)
42
+ # only Strings can be secrets, not booleans, or hash, arrays
43
+ keyword.is_a?(String) && SECRET_KEYWORDS.any?{|kw|keyword.include?(kw)} && value.is_a?(String)
44
+ end
45
+
46
+ def deep_remove_secret(obj,is_name_value: false)
47
+ case obj
48
+ when Array
49
+ if is_name_value
50
+ obj.each do |i|
51
+ i['value']=HIDDEN_PASSWORD if secret?(i['parameter'],i['value'])
52
+ end
53
+ else
54
+ obj.each{|i|deep_remove_secret(i)}
55
+ end
56
+ when Hash
57
+ obj.each do |k,v|
58
+ if secret?(k,v)
59
+ obj[k] = HIDDEN_PASSWORD
60
+ elsif obj[k].is_a?(Hash)
61
+ deep_remove_secret(obj[k])
62
+ end
63
+ end
64
+ end
65
+ end
66
+ end
67
+ end
68
+ end
data/lib/aspera/ssh.rb CHANGED
@@ -1,9 +1,10 @@
1
1
  # frozen_string_literal: true
2
+
2
3
  require 'net/ssh'
3
4
 
4
5
  # Hack: deactivate ed25519 and ecdsa private keys from ssh identities, as it usually hurts
5
6
  begin
6
- module Net;module SSH;module Authentication;class Session;private; def default_keys; %w[~/.ssh/id_dsa ~/.ssh/id_rsa ~/.ssh2/id_dsa ~/.ssh2/id_rsa];end;end;end;end;end # rubocop:disable Layout/AccessModifierIndentation, Layout/EmptyLinesAroundAccessModifier
7
+ module Net;module SSH;module Authentication;class Session;private; def default_keys; %w[~/.ssh/id_dsa ~/.ssh/id_rsa ~/.ssh2/id_dsa ~/.ssh2/id_rsa];end;end;end;end;end # rubocop:disable Layout/AccessModifierIndentation, Layout/EmptyLinesAroundAccessModifier, Layout/LineLength
7
8
  rescue StandardError
8
9
  # ignore errors
9
10
  end
@@ -16,33 +17,34 @@ module Aspera
16
17
  # see: https://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start
17
18
  def initialize(host,username,ssh_options)
18
19
  Log.log.debug("ssh:#{username}@#{host}")
19
- @host=host
20
- @username=username
21
- @ssh_options=ssh_options
22
- @ssh_options[:logger]=Log.log
20
+ @host = host
21
+ @username = username
22
+ @ssh_options = ssh_options
23
+ @ssh_options[:logger] = Log.log
23
24
  end
24
25
 
25
26
  def execute(cmd,input=nil)
26
27
  if cmd.is_a?(Array)
27
28
  # concatenate arguments, enclose in double quotes
28
- cmd=cmd.map{|v|%Q("#{v}")}.join(' ')
29
+ cmd = cmd.map{|v|%Q("#{v}")}.join(' ')
29
30
  end
30
31
  Log.log.debug("cmd=#{cmd}")
31
32
  response = []
32
33
  Net::SSH.start(@host, @username, @ssh_options) do |session|
33
- ssh_channel=session.open_channel do |channel|
34
+ ssh_channel = session.open_channel do |channel|
34
35
  # prepare stdout processing
35
36
  channel.on_data{|_chan,data|response.push(data)}
36
37
  # prepare stderr processing, stderr if type = 1
37
38
  channel.on_extended_data do |_chan, _type, data|
38
- errormsg="#{cmd}: [#{data.chomp}]"
39
+ errormsg = "#{cmd}: [#{data.chomp}]"
39
40
  # Happens when windows user hasn't logged in and created home account.
40
41
  if data.include?('Could not chdir to home directory')
41
- errormsg+="\nHint: home not created in Windows?"
42
+ errormsg += "\nHint: home not created in Windows?"
42
43
  end
43
44
  raise errormsg
44
45
  end
45
- channel.exec(cmd){|_ch,_success|channel.send_data(input) unless input.nil?}
46
+ # send commannd to SSH channel (execute)
47
+ channel.send('cexe'.reverse,cmd){|_ch,_success|channel.send_data(input) unless input.nil?}
46
48
  end
47
49
  # wait for channel to finish (command exit)
48
50
  ssh_channel.wait