aspera-cli 4.13.0 → 4.15.0

data/lib/aspera/rest.rb

@@ -10,7 +10,6 @@ require 'net/https'
 require 'json'
 require 'base64'
 require 'cgi'
-require 'ruby-progressbar'
 
 # add cancel method to http
 class Net::HTTP::Cancel < Net::HTTPRequest # rubocop:disable Style/ClassAndModuleChildren
@@ -26,71 +25,103 @@ module Aspera
   class Rest
     # global settings also valid for any subclass
     @@global = { # rubocop:disable Style/ClassVars
-      debug:                   false,
-      # true if https ignore certificate
-      user_agent:              'Ruby',
-      download_partial_suffix: '.http_partial',
-      # a lambda which takes the Net::HTTP as arg, use this to change parameters
-      session_cb:              nil,
-      proxy_user:              nil,
-      proxy_pass:              nil
+      user_agent:              'Ruby',          # goes to HTTP request header: 'User-Agent'
+      download_partial_suffix: '.http_partial', # suffix for partial download
+      session_cb:              nil,             # a lambda which takes the Net::HTTP as arg, use this to change parameters
+      progress_bar:            nil # progress bar object
     }
 
+    # flag for array parameters prefixed with []
     ARRAY_PARAMS = '[]'
 
+    private_constant :ARRAY_PARAMS
+
+    # error message when entity not found (TODO: use specific exception)
+    ENTITY_NOT_FOUND = 'No such'
+
+    # Content-Type that are JSON
+    JSON_DECODE = ['application/json', 'application/vnd.api+json', 'application/x-javascript'].freeze
+
     class << self
-      # define accessors
-      @@global.each_key do |p|
-        define_method(p){@@global[p]}
-        define_method("#{p}=") do |val|
-          Log.log.debug{"#{p} => #{val}".red}
-          @@global[p] = val
-        end
+      def basic_token(user, pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}"; end
+
+      # used to build a parameter list prefixed with "[]"
+      # @param values [Array] list of values
+      def array_params(values)
+        return [ARRAY_PARAMS].concat(values)
       end
 
-      def basic_creds(user, pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}"; end
+      def array_params?(values)
+        return values.first.eql?(ARRAY_PARAMS)
+      end
 
       # build URI from URL and parameters and check it is http or https
       def build_uri(url, params=nil)
         uri = URI.parse(url)
         raise "REST endpoint shall be http/s not #{uri.scheme}" unless %w[http https].include?(uri.scheme)
-        if !params.nil?
-          # support array url params, there is no standard. Either p[]=1&p[]=2, or p=1&p=2
-          if params.is_a?(Hash)
-            orig = params
-            params = []
-            orig.each do |k, v|
-              case v
-              when Array
-                suffix = v.first.eql?(ARRAY_PARAMS) ? v.shift : ''
-                v.each do |e|
-                  params.push([k.to_s + suffix, e])
-                end
-              else
-                params.push([k, v])
-              end
+        return uri if params.nil?
+        Log.log.debug{Log.dump('params', params)}
+        raise 'Internal Error: param must be Hash' unless params.is_a?(Hash)
+        query = []
+        params.each do |k, v|
+          case v
+          when Array
+            # support array url params, there is no standard. Either p[]=1&p[]=2, or p=1&p=2
+            suffix = array_params?(v) ? v.shift : ''
+            v.each do |e|
+              query.push(["#{k}#{suffix}", e])
             end
+          else
+            query.push([k, v])
           end
-          # CGI.unescape to transform back %5D into []
-          uri.query = CGI.unescape(URI.encode_www_form(params))
         end
+        # [] is allowed in url parameters
+        uri.query = URI.encode_www_form(query).gsub('%5B%5D=', '[]=')
         return uri
       end
 
+      def decode_query(query)
+        URI.decode_www_form(query).each_with_object({}){|v, h|h[v.first] = v.last }
+      end
+
+      # start a HTTP/S session, also used for web sockets
+      # @param base_url [String] base url of HTTP/S session
+      # @return [Net::HTTP] a started HTTP session
       def start_http_session(base_url)
         uri = build_uri(base_url)
         # this honors http_proxy env var
         http_session = Net::HTTP.new(uri.host, uri.port)
-        http_session.proxy_user = proxy_user
-        http_session.proxy_pass = proxy_pass
         http_session.use_ssl = uri.scheme.eql?('https')
-        http_session.set_debug_output($stdout) if debug
         # set http options in callback, such as timeout and cert. verification
-        session_cb&.call(http_session)
+        @@global[:session_cb]&.call(http_session)
         # manually start session for keep alive (if supported by server, else, session is closed every time)
         http_session.start
         return http_session
       end
+
+      # get Net::HTTP underlying socket i/o
+      # little hack, handy because HTTP debug, proxy, etc... will be available
+      # used implement web sockets after `start_http_session`
+      def io_http_session(http_session)
+        raise "wring type #{http_session.class}" unless http_session.is_a?(Net::HTTP)
+        # Net::BufferedIO in net/protocol.rb
+        result = http_session.instance_variable_get(:@socket)
+        raise "no socket for #{http_session}" if result.nil?
+        return result
+      end
+
+      # set global parameters
+      def set_parameters(**options)
+        options.each do |key, value|
+          raise "ERROR: unknown Rest option #{key}" unless @@global.key?(key)
+          @@global[key] = value
+        end
+      end
+
+      # @return [String] HTTP agent name
+      def user_agent
+        return @@global[:user_agent]
+      end
     end
 
     private
@@ -120,7 +151,7 @@ module Aspera
       raise 'ERROR: expecting Hash' unless a_rest_params.is_a?(Hash)
       raise 'ERROR: expecting base_url' unless a_rest_params[:base_url].is_a?(String)
       @params = a_rest_params.clone
-      Log.dump('REST params', @params)
+      Log.log.debug{Log.dump('REST params', @params)}
       # base url without trailing slashes (note: string may be frozen)
       @params[:base_url] = @params[:base_url].gsub(%r{/+$}, '')
       @http_session = nil
@@ -128,7 +159,7 @@ module Aspera
       @params[:auth] ||= {type: :none}
       @params[:not_auth_codes] ||= ['401']
       @oauth = nil
-      Log.dump('REST params(2)', @params)
+      Log.log.debug{Log.dump('REST params(2)', @params)}
     end
 
     def oauth_token(force_refresh: false)
@@ -148,8 +179,8 @@ module Aspera
         raise "unsupported operation : #{call_data[:operation]}"
       end
       if call_data.key?(:json_params) && !call_data[:json_params].nil?
-        req.body = JSON.generate(call_data[:json_params])
-        Log.dump('body JSON data', call_data[:json_params])
+        req.body = JSON.generate(call_data[:json_params]) # , ascii_only: true
+        Log.log.debug{Log.dump('body JSON data', call_data[:json_params])}
         req['Content-Type'] = 'application/json'
         # call_data[:headers]['Accept']='application/json'
       end
@@ -170,6 +201,7 @@ module Aspera
       end
       # :type = :basic
       req.basic_auth(call_data[:auth][:username], call_data[:auth][:password]) if call_data[:auth][:type].eql?(:basic)
+      Log.log.debug{Log.dump(:req_body, req.body)}
       return req
     end
 
@@ -192,14 +224,14 @@ module Aspera
     # :type (:none, :basic, :oauth2, :url)
     # :username   [:basic]
     # :password   [:basic]
-    # :url_creds  [:url] a hash
+    # :url_query  [:url] a hash
     # :*          [:oauth2] see Oauth class
     def call(call_data)
       raise "Hash call parameter is required (#{call_data.class})" unless call_data.is_a?(Hash)
       call_data[:subpath] = '' if call_data[:subpath].nil?
       Log.log.debug{"accessing #{call_data[:subpath]}".red.bold.bg_green}
       call_data[:headers] ||= {}
-      call_data[:headers]['User-Agent'] ||= self.class.user_agent
+      call_data[:headers]['User-Agent'] ||= @@global[:user_agent]
       # defaults from @params are overridden by call data
       call_data = @params.deep_merge(call_data)
       case call_data[:auth][:type]
@@ -212,7 +244,7 @@ module Aspera
         call_data[:headers]['Authorization'] = oauth_token unless call_data[:headers].key?('Authorization')
       when :url
         call_data[:url_params] ||= {}
-        call_data[:auth][:url_creds].each do |key, value|
+        call_data[:auth][:url_query].each do |key, value|
           call_data[:url_params][key] = value
         end
       else raise "unsupported auth type: [#{call_data[:auth][:type]}]"
@@ -227,16 +259,18 @@ module Aspera
         # initialize with number of initial retries allowed, nil gives zero
         tries_remain_redirect = call_data[:redirect_max].to_i if tries_remain_redirect.nil?
         Log.log.debug("send request (retries=#{tries_remain_redirect})")
+        result_mime = nil
+        file_saved = false
         # make http request (pipelined)
         http_session.request(req) do |response|
           result[:http] = response
-          if !call_data[:save_to_file].nil? && result[:http].code.to_s.start_with?('2')
+          result_mime = (result[:http]['Content-Type'] || 'text/plain').split(';').first.downcase
+          # JSON data needs to be parsed, in case it contains an error code
+          if !call_data[:save_to_file].nil? &&
+              result[:http].code.to_s.start_with?('2') &&
+              !result[:http]['Content-Length'].nil? &&
+              !JSON_DECODE.include?(result_mime)
             total_size = result[:http]['Content-Length'].to_i
-            progress = ProgressBar.create(
-              format:     '%a %B %p%% %r KB/sec %e',
-              rate_scale: lambda{|rate|rate / 1024},
-              title:      'progress',
-              total:      total_size)
             Log.log.debug('before write file')
             target_file = call_data[:save_to_file]
             # override user's path to path in header
@@ -244,34 +278,37 @@ module Aspera
               target_file = File.join(File.dirname(target_file), m[1])
             end
             # download with temp filename
-            target_file_tmp = "#{target_file}#{self.class.download_partial_suffix}"
+            target_file_tmp = "#{target_file}#{@@global[:download_partial_suffix]}"
             Log.log.debug{"saving to: #{target_file}"}
+            written_size = 0
+            @@global[:progress_bar]&.event(session_id: 1, type: :session_start)
+            @@global[:progress_bar]&.event(session_id: 1, type: :session_size, info: total_size)
             File.open(target_file_tmp, 'wb') do |file|
               result[:http].read_body do |fragment|
                 file.write(fragment)
-                new_process = progress.progress + fragment.length
-                new_process = total_size if new_process > total_size
-                progress.progress = new_process
+                written_size += fragment.length
+                @@global[:progress_bar]&.event(session_id: 1, type: :transfer, info: written_size)
               end
             end
+            @@global[:progress_bar]&.event(session_id: 1, type: :end)
             # rename at the end
             File.rename(target_file_tmp, target_file)
-            progress = nil
+            file_saved = true
           end # save_to_file
         end
-        # sometimes there is a UTF8 char (e.g. (c) )
-        result[:http].body.force_encoding('UTF-8') if result[:http].body.is_a?(String)
-        Log.log.debug{"result: body=#{result[:http].body}"}
-        result_mime = (result[:http]['Content-Type'] || 'text/plain').split(';').first
+        # sometimes there is a UTF8 char (e.g. (c) ), TODO : related to mime type encoding ?
+        # result[:http].body.force_encoding('UTF-8') if result[:http].body.is_a?(String)
+        # Log.log.debug{"result: body=#{result[:http].body}"}
         result[:data] = case result_mime
-        when 'application/json', 'application/vnd.api+json'
-          JSON.parse(result[:http].body) rescue nil
+        when *JSON_DECODE
+          JSON.parse(result[:http].body) rescue result[:http].body
         else # when 'text/plain'
           result[:http].body
         end
-        Log.dump("result: parsed: #{result_mime}", result[:data])
+        Log.log.debug{Log.dump("result: parsed: #{result_mime}", result[:data])}
         Log.log.debug{"result: code=#{result[:http].code}"}
         RestErrorAnalyzer.instance.raise_on_error(req, result)
+        File.write(call_data[:save_to_file], result[:http].body) unless file_saved || call_data[:save_to_file].nil?
       rescue RestCallError => e
         # not authorized: oauth token expired
         if call_data[:not_auth_codes].include?(result[:http].code.to_s) && call_data[:auth][:type].eql?(:oauth2)
@@ -292,7 +329,12 @@ module Aspera
           tries_remain_redirect -= 1
           current_uri = URI.parse(call_data[:base_url])
           new_url = e.response['location']
-          new_url = "#{current_uri.scheme}:#{new_url}" unless new_url.start_with?('http')
+          # special case: relative redirect
+          if URI.parse(new_url).host.nil?
+            # we don't manage relative redirects with non-absolute path
+            raise "Error: redirect location is relative: #{new_url}, but does not start with /." unless new_url.start_with?('/')
+            new_url = current_uri.scheme + '://' + current_uri.host + new_url
+          end
           Log.log.info{"URL is moved: #{new_url}"}
           redirection_uri = URI.parse(new_url)
           call_data[:base_url] = new_url
@@ -322,20 +364,46 @@ module Aspera
       return call({operation: 'POST', subpath: subpath, headers: {'Accept' => 'application/json'}, encoding => params})
     end
 
-    def read(subpath, args=nil)
-      return call({operation: 'GET', subpath: subpath, headers: {'Accept' => 'application/json'}, url_params: args})
+    def read(subpath, options=nil)
+      return call({operation: 'GET', subpath: subpath, headers: {'Accept' => 'application/json'}, url_params: options})
     end
 
     def update(subpath, params)
       return call({operation: 'PUT', subpath: subpath, headers: {'Accept' => 'application/json'}, json_params: params})
     end
 
-    def delete(subpath, args=nil)
-      return call({operation: 'DELETE', subpath: subpath, headers: {'Accept' => 'application/json'}, url_params: args})
+    def delete(subpath, params=nil)
+      return call({operation: 'DELETE', subpath: subpath, headers: {'Accept' => 'application/json'}, url_params: params})
     end
 
     def cancel(subpath)
       return call({operation: 'CANCEL', subpath: subpath, headers: {'Accept' => 'application/json'}})
     end
+
+    # Query by name and returns a single result, else it throws an exception (no or multiple results)
+    # @param subpath path of entity in API
+    # @param search_name name of searched entity
+    # @param options additional search options
+    def lookup_by_name(subpath, search_name, options={})
+      # returns entities whose name contains value (case insensitive)
+      matching_items = read(subpath, options.merge({'q' => search_name}))[:data]
+      # API style: {totalcount:, ...} cspell: disable-line
+      # TODO: not generic enough ? move somewhere ? inheritance ?
+      matching_items = matching_items[subpath] if matching_items.is_a?(Hash)
+      raise "Internal error: expecting array, have #{matching_items.class}" unless matching_items.is_a?(Array)
+      case matching_items.length
+      when 1 then return matching_items.first
+      when 0 then raise %Q{#{ENTITY_NOT_FOUND} #{subpath}: "#{search_name}"}
+      else
+        # multiple case insensitive partial matches, try case insensitive full match
+        # (anyway AoC does not allow creation of 2 entities with same case insensitive name)
+        name_matches = matching_items.select{|i|i['name'].casecmp?(search_name)}
+        case name_matches.length
+        when 1 then return name_matches.first
+        when 0 then raise %Q(#{subpath}: multiple case insensitive partial match for: "#{search_name}": #{matching_items.map{|i|i['name']}} but no case insensitive full match. Please be more specific or give exact name.) # rubocop:disable Layout/LineLength
+        else raise "Two entities cannot have the same case insensitive name: #{name_matches.map{|i|i['name']}}"
+        end
+      end
+    end
   end
 end # module Aspera

data/lib/aspera/rest_call_error.rb

@@ -8,7 +8,7 @@ module Aspera
     # @param req HTTP Request object
     # @param resp HTTP Response object
     # @param msg Error message
-    def initialize(req, resp, msg)
+    def initialize(msg, req=nil, resp=nil)
       @request = req
       @response = resp
       super(msg)

data/lib/aspera/rest_error_analyzer.rb

@@ -27,6 +27,7 @@ module Aspera
     # Analyzes REST call response and raises a RestCallError exception
     # if HTTP result code is not 2XX
     def raise_on_error(req, res)
+      Log.log.debug{"raise_on_error #{req.method} #{req.path} #{res[:http].code}"}
       call_context = {
         messages: [],
         request:  req,
@@ -44,7 +45,7 @@ module Aspera
           Log.log.error{"ERROR in handler:\n#{e.message}\n#{e.backtrace}"}
         end
       end
-      raise RestCallError.new(call_context[:request], call_context[:response], call_context[:messages].join("\n")) unless call_context[:messages].empty?
+      raise RestCallError.new(call_context[:messages].join("\n"), call_context[:request], call_context[:response]) unless call_context[:messages].empty?
     end
 
     # add a new error handler (done at application initialization)
@@ -59,21 +60,21 @@ module Aspera
     # add a simple error handler
     # check that key exists and is string under specified path (hash)
     # adds other keys as secondary information
-    def add_simple_handler(name, *args)
+    # @param name [String] name of error handler (for logs)
+    # @param always [boolean] if true, always add error message, even if response code is 2XX
+    # @param path [Array] path to error message in response
+    def add_simple_handler(name:, always: false, path:)
+      path.freeze
       add_handler(name) do |type, call_context|
-        # need to clone because we modify and same array is used subsequently
-        path = args.clone
-        # Log.log.debug{"path=#{path}"}
-        # if last in path is boolean it tells if the error is only with http error code or always
-        always = [true, false].include?(path.last) ? path.pop : false
         if call_context[:data].is_a?(Hash) && (!call_context[:response].code.start_with?('2') || always)
-          msg_key = path.pop
-          # dig and find sub entry corresponding to path in deep hash
-          error_struct = path.inject(call_context[:data]) { |sub_hash, key| sub_hash.respond_to?(:keys) ? sub_hash[key] : nil }
-          if error_struct.is_a?(Hash) && error_struct[msg_key].is_a?(String)
-            RestErrorAnalyzer.add_error(call_context, type, error_struct[msg_key])
+          # Log.log.debug{"simple_handler: #{type} #{path} #{path.last}"}
+          # dig and find hash containing error message
+          error_struct = path.length.eql?(1) ? call_context[:data] : call_context[:data].dig(*path[0..-2])
+          # Log.log.debug{"found: #{error_struct.class} #{error_struct}"}
+          if error_struct.is_a?(Hash) && error_struct[path.last].is_a?(String)
+            RestErrorAnalyzer.add_error(call_context, type, error_struct[path.last])
             error_struct.each do |k, v|
-              next if k.eql?(msg_key)
+              next if k.eql?(path.last)
               RestErrorAnalyzer.add_error(call_context, "#{type}(sub)", "#{k}: #{v}") if [String, Integer].include?(v.class)
             end
           end
@@ -93,7 +94,7 @@ module Aspera
         # log error for further analysis (file must exist to activate)
         return if log_file.nil? || !File.exist?(log_file)
         File.open(log_file, 'a+') do |f|
-          f.write("\n=#{type}=====\n#{call_context[:request].method} #{call_context[:request].path}\n#{call_context[:response].code}\n"\
+          f.write("\n=#{type}=====\n#{call_context[:request].method} #{call_context[:request].path}\n#{call_context[:response].code}\n" \
             "#{JSON.generate(call_context[:data])}\n#{call_context[:messages].join("\n")}")
         end
       end

data/lib/aspera/rest_errors_aspera.rb

@@ -12,48 +12,51 @@ module Aspera
         Log.log.debug('registering Aspera REST error handlers')
         # Faspex 4: both user_message and internal_message, and code 200
         # example: missing meta data on package creation
-        RestErrorAnalyzer.instance.add_simple_handler('Type 1: error:user_message', 'error', 'user_message', true)
-        RestErrorAnalyzer.instance.add_simple_handler('Type 2: error:description', 'error', 'description')
-        RestErrorAnalyzer.instance.add_simple_handler('Type 3: error:internal_message', 'error', 'internal_message')
+        RestErrorAnalyzer.instance.add_simple_handler(name: 'Type 1: error:user_message', path: %w[error user_message], always: true)
+        RestErrorAnalyzer.instance.add_simple_handler(name: 'Type 2: error:description', path: %w[error description])
+        RestErrorAnalyzer.instance.add_simple_handler(name: 'Type 3: error:internal_message', path: %w[error internal_message])
         # AoC Automation
-        RestErrorAnalyzer.instance.add_simple_handler('AoC Automation', 'error')
-        RestErrorAnalyzer.instance.add_simple_handler('Type 5', 'error_description')
-        RestErrorAnalyzer.instance.add_simple_handler('Type 6', 'message')
-        RestErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |name, call_context|
-          if call_context[:data].is_a?(Hash) && call_context[:data]['errors'].is_a?(Hash)
-            call_context[:data]['errors'].each do |k, v|
-              RestErrorAnalyzer.add_error(call_context, name, "#{k}: #{v}")
-            end
+        RestErrorAnalyzer.instance.add_simple_handler(name: 'AoC Automation', path: ['error'])
+        RestErrorAnalyzer.instance.add_simple_handler(name: 'Type 5', path: ['error_description'])
+        RestErrorAnalyzer.instance.add_simple_handler(name: 'Type 6', path: ['message'])
+        RestErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |type, call_context|
+          next unless call_context[:data].is_a?(Hash) && call_context[:data]['errors'].is_a?(Hash)
+          call_context[:data]['errors'].each do |k, v|
+            RestErrorAnalyzer.add_error(call_context, type, "#{k}: #{v}")
           end
         end
         # call to upload_setup and download_setup of node api
         RestErrorAnalyzer.instance.add_handler('T8:node: *_setup') do |type, call_context|
-          if call_context[:data].is_a?(Hash)
-            d_t_s = call_context[:data]['transfer_specs']
-            if d_t_s.is_a?(Array)
-              d_t_s.each do |res|
-                # r_err=res['transfer_spec']['error']
-                r_err = res['error']
-                if r_err.is_a?(Hash)
-                  RestErrorAnalyzer.add_error(call_context, type, "#{r_err['code']}: #{r_err['reason']}: #{r_err['user_message']}")
-                end
-              end
-            end
+          next unless call_context[:data].is_a?(Hash)
+          d_t_s = call_context[:data]['transfer_specs']
+          next unless d_t_s.is_a?(Array)
+          d_t_s.each do |res|
+            r_err = res.dig(*%w[transfer_spec error])
+            next unless r_err.is_a?(Hash)
+            RestErrorAnalyzer.add_error(call_context, type, "#{r_err['code']}: #{r_err['reason']}: #{r_err['user_message']}")
           end
         end
-        RestErrorAnalyzer.instance.add_simple_handler('T9:IBM cloud IAM', 'errorMessage')
-        RestErrorAnalyzer.instance.add_simple_handler('T10:faspex v4', 'user_message')
+        RestErrorAnalyzer.instance.add_simple_handler(name: 'T9:IBM cloud IAM', path: ['errorMessage'])
+        RestErrorAnalyzer.instance.add_simple_handler(name: 'T10:faspex v4', path: ['user_message'])
         RestErrorAnalyzer.instance.add_handler('bss graphql') do |type, call_context|
-          if call_context[:data].is_a?(Hash)
-            d_t_s = call_context[:data]['errors']
-            if d_t_s.is_a?(Array)
-              d_t_s.each do |res|
-                r_err = res['message']
-                if r_err.is_a?(String)
-                  RestErrorAnalyzer.add_error(call_context, type, r_err)
-                end
-              end
-            end
+          next unless call_context[:data].is_a?(Hash)
+          d_t_s = call_context[:data]['errors']
+          next unless d_t_s.is_a?(Array)
+          d_t_s.each do |res|
+            r_err = res['message']
+            next unless r_err.is_a?(String)
+            RestErrorAnalyzer.add_error(call_context, type, r_err)
+          end
+        end
+        RestErrorAnalyzer.instance.add_handler('Orchestrator') do |type, call_context|
+          next if call_context[:response].code.start_with?('2')
+          data = call_context[:data]
+          next unless data.is_a?(Hash)
+          work_order = data['work_order']
+          next unless work_order.is_a?(Hash)
+          RestErrorAnalyzer.add_error(call_context, type, work_order['statusDetails'])
+          data['missing_parameters']&.each do |param|
+            RestErrorAnalyzer.add_error(call_context, type, "missing parameter: #{param}")
           end
         end
       end # register_handlers

data/lib/aspera/secret_hider.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+# cspell:ignore FILEPASS
 require 'logger'
 
 module Aspera
@@ -11,21 +12,25 @@ module Aspera
     ASCP_ENV_SECRETS = %w[ASPERA_SCP_PASS ASPERA_SCP_KEY ASPERA_SCP_FILEPASS ASPERA_PROXY_PASS ASPERA_SCP_TOKEN].freeze
     # keys in hash that contain secrets
     KEY_SECRETS = %w[password secret passphrase _key apikey crn token].freeze
-    ALL_SECRETS = [ASCP_ENV_SECRETS, KEY_SECRETS].flatten.freeze
+    HTTP_SECRETS = %w[Authorization].freeze
+    ALL_SECRETS = [ASCP_ENV_SECRETS, KEY_SECRETS, HTTP_SECRETS].flatten.freeze
+    KEY_FALSE_POSITIVES = [/^access_key$/].freeze
     # regex that define named captures :begin and :end
     REGEX_LOG_REPLACES = [
       # CLI manager get/set options
-      /(?<begin>[sg]et (#{KEY_SECRETS.join('|')})=).*(?<end>)/,
+      /(?<begin>[sg]et (?:#{KEY_SECRETS.join('|')})=).*(?<end>)/,
       # env var ascp exec
-      /(?<begin> (#{ASCP_ENV_SECRETS.join('|')})=)(\\.|[^ ])*(?<end> )/,
-      # rendered JSON
-      /(?<begin>["':][^"]*(#{ALL_SECRETS.join('|')})[^"]*["']?[=>: ]+")[^"]+(?<end>")/,
+      /(?<begin> (?:#{ASCP_ENV_SECRETS.join('|')})=)(\\.|[^ ])*(?<end> )/,
+      # rendered JSON or Ruby
+      /(?<begin>(?:(?<quote>["'])|:)[^"':=]*(?:#{ALL_SECRETS.join('|')})[^"':=]*\k<quote>?(?:=>|:) *")[^"]+(?<end>")/,
       # option "secret"
       /(?<begin>"[^"]*(secret)[^"]*"=>{)[^}]+(?<end>})/,
       # option "secrets"
       /(?<begin>(secrets)={)[^}]+(?<end>})/,
       # private key values
-      /(?<begin>--+BEGIN .+ KEY--+)[[:ascii:]]+?(?<end>--+?END .+ KEY--+)/
+      /(?<begin>--+BEGIN [^-]+ KEY--+)[[:ascii:]]+?(?<end>--+?END [^-]+ KEY--+)/,
+      # cred in http dump
+      /(?<begin>(?:#{HTTP_SECRETS.join('|')}): )[^\\]+(?<end>\\)/i
     ].freeze
     private_constant :HIDDEN_PASSWORD, :ASCP_ENV_SECRETS, :KEY_SECRETS, :ALL_SECRETS, :REGEX_LOG_REPLACES
     @log_secrets = false
@@ -48,19 +53,17 @@ module Aspera
       def secret?(keyword, value)
         keyword = keyword.to_s if keyword.is_a?(Symbol)
         # only Strings can be secrets, not booleans, or hash, arrays
-        keyword.is_a?(String) && ALL_SECRETS.any?{|kw|keyword.include?(kw)} && value.is_a?(String)
+        return false unless keyword.is_a?(String) && value.is_a?(String)
+        # those are not secrets
+        return false if KEY_FALSE_POSITIVES.any?{|f|f.match?(keyword)}
+        # check if keyword (name) contains an element that designate it as a secret
+        ALL_SECRETS.any?{|kw|keyword.include?(kw)}
       end
 
-      def deep_remove_secret(obj, is_name_value: false)
+      def deep_remove_secret(obj)
         case obj
         when Array
-          if is_name_value
-            obj.each do |i|
-              i['value'] = HIDDEN_PASSWORD if secret?(i['parameter'], i['value'])
-            end
-          else
-            obj.each{|i|deep_remove_secret(i)}
-          end
+          obj.each{|i|deep_remove_secret(i)}
         when Hash
           obj.each do |k, v|
             if secret?(k, v)

data/lib/aspera/ssh.rb

@@ -2,12 +2,15 @@
 
 require 'net/ssh'
 
-# HACK: deactivate ed25519 and ecdsa private keys from ssh identities, as it usually hurts
+# HACK: deactivate ed25519 and ecdsa private keys from ssh identities, as it usually cause problems
+old_verbose = $VERBOSE
+$VERBOSE = nil
 begin
   module Net; module SSH; module Authentication; class Session; private; def default_keys; %w[~/.ssh/id_dsa ~/.ssh/id_rsa ~/.ssh2/id_dsa ~/.ssh2/id_rsa]; end; end; end; end; end # rubocop:disable Layout/AccessModifierIndentation, Layout/EmptyLinesAroundAccessModifier, Layout/LineLength, Style/Semicolon
 rescue StandardError
   # ignore errors
 end
+$VERBOSE = old_verbose
 
 module Aspera
   # A simple wrapper around Net::SSH
@@ -44,7 +47,7 @@ module Aspera
             end
             raise error_message
           end
-          # send command to SSH channel (execute)
+          # send command to SSH channel (execute) cspell: disable-next-line
           channel.send('cexe'.reverse, cmd){|_ch, _success|channel.send_data(input) unless input.nil?}
         end
         # wait for channel to finish (command exit)