aspera-cli 4.7.0 → 4.9.0

data/lib/aspera/rest.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'aspera/log'
 require 'aspera/oauth'
 require 'aspera/rest_error_analyzer'
@@ -30,40 +31,39 @@ module Aspera
   # and error are analyzed in RestErrorAnalyzer
   class Rest
     # global settings also valid for any subclass
-    @@global={ # rubocop:disable Style/ClassVars
-      debug: false,
+    @@global = { # rubocop:disable Style/ClassVars
+      debug:                   false,
       # true if https ignore certificate
-      insecure: false,
-      user_agent: 'Ruby',
+      user_agent:              'Ruby',
       download_partial_suffix: '.http_partial',
       # a lambda which takes the Net::HTTP as arg, use this to change parameters
-      session_cb: nil
+      session_cb:              nil
     }
 
-    class<<self
+    class << self
       # define accessors
       @@global.keys.each do |p|
         define_method(p){@@global[p]}
-        define_method("#{p}="){|val|Log.log.debug("#{p} => #{val}".red);@@global[p]=val}
+        define_method("#{p}="){|val|Log.log.debug("#{p} => #{val}".red);@@global[p] = val}
       end
 
       def basic_creds(user,pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}";end
 
       # build URI from URL and parameters and check it is http or https
       def build_uri(url,params=nil)
-        uri=URI.parse(url)
-        raise 'REST endpoint shall be http(s)' unless ['http','https'].include?(uri.scheme)
+        uri = URI.parse(url)
+        raise "REST endpoint shall be http/s not #{uri.scheme}" unless %w[http https].include?(uri.scheme)
         if !params.nil?
           # support array url params, there is no standard. Either p[]=1&p[]=2, or p=1&p=2
           if params.is_a?(Hash)
-            orig=params
-            params=[]
+            orig = params
+            params = []
             orig.each do |k,v|
               case v
               when Array
-                suffix=v.first.eql?('[]') ? v.shift : ''
+                suffix = v.first.eql?('[]') ? v.shift : ''
                 v.each do |e|
-                  params.push([k+suffix,e])
+                  params.push([k + suffix,e])
                 end
               else
                 params.push([k,v])
@@ -71,7 +71,7 @@ module Aspera
             end
           end
           # CGI.unescape to transform back %5D into []
-          uri.query=CGI.unescape(URI.encode_www_form(params))
+          uri.query = CGI.unescape(URI.encode_www_form(params))
         end
         return uri
       end
@@ -82,14 +82,13 @@ module Aspera
     # create and start keep alive connection on demand
     def http_session
       if @http_session.nil?
-        uri=self.class.build_uri(@params[:base_url])
+        uri = self.class.build_uri(@params[:base_url])
         # this honors http_proxy env var
-        @http_session=Net::HTTP.new(uri.host, uri.port)
+        @http_session = Net::HTTP.new(uri.host, uri.port)
         @http_session.use_ssl = uri.scheme.eql?('https')
-        @http_session.verify_mode = OpenSSL::SSL::VERIFY_NONE if self.class.insecure
         @http_session.set_debug_output($stdout) if self.class.debug
         # set http options in callback, such as timeout and cert. verification
-        self.class.session_cb.call(@http_session) unless self.class.session_cb.nil?
+        self.class.session_cb&.call(@http_session)
         # manually start session for keep alive (if supported by server, else, session is closed every time)
         @http_session.start
       end
@@ -103,7 +102,7 @@ module Aspera
     def oauth
       if @oauth.nil?
         raise 'ERROR: no OAuth defined' unless @params[:auth][:type].eql?(:oauth2)
-        @oauth=Oauth.new(@params[:auth])
+        @oauth = Oauth.new(@params[:auth])
       end
       return @oauth
     end
@@ -112,15 +111,15 @@ module Aspera
     def initialize(a_rest_params)
       raise 'ERROR: expecting Hash' unless a_rest_params.is_a?(Hash)
       raise 'ERROR: expecting base_url' unless a_rest_params[:base_url].is_a?(String)
-      @params=a_rest_params.clone
+      @params = a_rest_params.clone
       Log.dump('REST params',@params)
       # base url without trailing slashes (note: string may be frozen)
-      @params[:base_url]=@params[:base_url].gsub(/\/+$/,'')
-      @http_session=nil
+      @params[:base_url] = @params[:base_url].gsub(/\/+$/,'')
+      @http_session = nil
       # default is no auth
-      @params[:auth]||={type: :none}
-      @params[:not_auth_codes]||=['401']
-      @oauth=nil
+      @params[:auth] ||= {type: :none}
+      @params[:not_auth_codes] ||= ['401']
+      @oauth = nil
       Log.dump('REST params(2)',@params)
     end
 
@@ -132,28 +131,28 @@ module Aspera
     def build_request(call_data)
       # TODO: shall we percent encode subpath (spaces) test with access key delete with space in id
       # URI.escape()
-      uri=self.class.build_uri("#{call_data[:base_url]}#{['','/'].include?(call_data[:subpath]) ? '' : '/'}#{call_data[:subpath]}",call_data[:url_params])
+      uri = self.class.build_uri("#{call_data[:base_url]}#{['','/'].include?(call_data[:subpath]) ? '' : '/'}#{call_data[:subpath]}",call_data[:url_params])
       Log.log.debug("URI=#{uri}")
       begin
         # instanciate request object based on string name
-        req=Net::HTTP.const_get(call_data[:operation].capitalize).new(uri)
+        req = Net::HTTP.const_get(call_data[:operation].capitalize).new(uri)
       rescue NameError
         raise "unsupported operation : #{call_data[:operation]}"
       end
       if call_data.has_key?(:json_params) && !call_data[:json_params].nil?
-        req.body=JSON.generate(call_data[:json_params])
+        req.body = JSON.generate(call_data[:json_params])
         Log.dump('body JSON data',call_data[:json_params])
         #Log.log.debug("body JSON data=#{JSON.pretty_generate(call_data[:json_params])}")
         req['Content-Type'] = 'application/json'
         #call_data[:headers]['Accept']='application/json'
       end
       if call_data.has_key?(:www_body_params)
-        req.body=URI.encode_www_form(call_data[:www_body_params])
+        req.body = URI.encode_www_form(call_data[:www_body_params])
         Log.log.debug("body www data=#{req.body.chomp}")
         req['Content-Type'] = 'application/x-www-form-urlencoded'
       end
       if call_data.has_key?(:text_body_params)
-        req.body=call_data[:text_body_params]
+        req.body = call_data[:text_body_params]
         Log.log.debug("body data=#{req.body.chomp}")
       end
       # set headers
@@ -180,7 +179,7 @@ module Aspera
     # :save_to_file (filepath) default: nil
     # :return_error (bool) default: nil
     # :redirect_max (int) default: 0
-    # :not_auth_codes (array)
+    # :not_auth_codes (array) codes that trigger a refresh/regeneration of bearer token
     # ----
     # authentication (:auth) :
     # :type (:none, :basic, :oauth2, :url)
@@ -190,12 +189,12 @@ module Aspera
     # :*          [:oauth2] see Oauth class
     def call(call_data)
       raise "Hash call parameter is required (#{call_data.class})" unless call_data.is_a?(Hash)
-      call_data[:subpath]='' if call_data[:subpath].nil?
+      call_data[:subpath] = '' if call_data[:subpath].nil?
       Log.log.debug("accessing #{call_data[:subpath]}".red.bold.bg_green)
-      call_data[:headers]||={}
+      call_data[:headers] ||= {}
       call_data[:headers]['User-Agent'] ||= self.class.user_agent
       # defaults from @params are overriden by call data
-      call_data=@params.deep_merge(call_data)
+      call_data = @params.deep_merge(call_data)
       case call_data[:auth][:type]
       when :none
         # no auth
@@ -203,60 +202,60 @@ module Aspera
         Log.log.debug('using Basic auth')
         # done in build_req
       when :oauth2
-        call_data[:headers]['Authorization']=oauth_token unless call_data[:headers].has_key?('Authorization')
+        call_data[:headers]['Authorization'] = oauth_token unless call_data[:headers].has_key?('Authorization')
       when :url
-        call_data[:url_params]||={}
+        call_data[:url_params] ||= {}
         call_data[:auth][:url_creds].each do |key, value|
-          call_data[:url_params][key]=value
+          call_data[:url_params][key] = value
         end
       else raise "unsupported auth type: [#{call_data[:auth][:type]}]"
       end
-      req=build_request(call_data)
+      req = build_request(call_data)
       Log.log.debug("call_data = #{call_data}")
-      result={http: nil}
+      result = {http: nil}
       # start a block to be able to retry the actual HTTP request
       begin
         # we try the call, and will retry only if oauth, as we can, first with refresh, and then re-auth if refresh is bad
         oauth_tries ||= 2
-        tries_remain_redirect||=call_data[:redirect_max].nil? ? 0 : call_data[:redirect_max].to_i
+        tries_remain_redirect ||= call_data[:redirect_max].nil? ? 0 : call_data[:redirect_max].to_i
         Log.log.debug('send request')
         # make http request (pipelined)
         http_session.request(req) do |response|
           result[:http] = response
           if !call_data[:save_to_file].nil? && result[:http].code.to_s.start_with?('2')
-            total_size=result[:http]['Content-Length'].to_i
-            progress=ProgressBar.create(
-            format:     '%a %B %p%% %r KB/sec %e',
-            rate_scale: lambda{|rate|rate/1024},
-            title:      'progress',
-            total:      total_size)
+            total_size = result[:http]['Content-Length'].to_i
+            progress = ProgressBar.create(
+              format:     '%a %B %p%% %r KB/sec %e',
+              rate_scale: lambda{|rate|rate / 1024},
+              title:      'progress',
+              total:      total_size)
             Log.log.debug('before write file')
-            target_file=call_data[:save_to_file]
+            target_file = call_data[:save_to_file]
             # override user's path to path in header
-            if !response['Content-Disposition'].nil? && (m=response['Content-Disposition'].match(/filename="([^"]+)"/))
-              target_file=File.join(File.dirname(target_file),m[1])
+            if !response['Content-Disposition'].nil? && (m = response['Content-Disposition'].match(/filename="([^"]+)"/))
+              target_file = File.join(File.dirname(target_file),m[1])
             end
             # download with temp filename
-            target_file_tmp="#{target_file}#{self.class.download_partial_suffix}"
+            target_file_tmp = "#{target_file}#{self.class.download_partial_suffix}"
             Log.log.debug("saving to: #{target_file}")
             File.open(target_file_tmp, 'wb') do |file|
               result[:http].read_body do |fragment|
                 file.write(fragment)
-                new_process=progress.progress+fragment.length
+                new_process = progress.progress + fragment.length
                 new_process = total_size if new_process > total_size
-                progress.progress=new_process
+                progress.progress = new_process
               end
             end
             # rename at the end
             File.rename(target_file_tmp, target_file)
-            progress=nil
+            progress = nil
           end # save_to_file
         end
         # sometimes there is a UTF8 char (e.g. (c) )
         result[:http].body.force_encoding('UTF-8') if result[:http].body.is_a?(String)
         Log.log.debug("result: body=#{result[:http].body}")
-        result_mime=(result[:http]['Content-Type']||'text/plain').split(';').first
-        result[:data]=case result_mime
+        result_mime = (result[:http]['Content-Type'] || 'text/plain').split(';').first
+        result[:data] = case result_mime
         when 'application/json','application/vnd.api+json'
           JSON.parse(result[:http].body) rescue nil
         else #when 'text/plain'
@@ -270,34 +269,34 @@ module Aspera
         if call_data[:not_auth_codes].include?(result[:http].code.to_s) && call_data[:auth][:type].eql?(:oauth2)
           begin
             # try to use refresh token
-            req['Authorization']=oauth_token(force_refresh: true)
-          rescue RestCallError => e
+            req['Authorization'] = oauth_token(force_refresh: true)
+          rescue RestCallError => e_tok
+            e = e_tok
             Log.log.error('refresh failed'.bg_red)
             # regenerate a brand new token
-            req['Authorization']=oauth_token
+            req['Authorization'] = oauth_token(use_cache: false)
           end
           Log.log.debug("using new token=#{call_data[:headers]['Authorization']}")
           retry unless (oauth_tries -= 1).zero?
         end # if oauth
         # moved ?
-        raise e unless e.response.is_a?(Net::HTTPRedirection)
-        if tries_remain_redirect.positive?
-          tries_remain_redirect-=1
-          Log.log.info("URL is moved: #{e.response['location']}")
-          current_uri=URI.parse(call_data[:base_url])
-          redir_uri=URI.parse(e.response['location'])
-          call_data[:base_url]=e.response['location']
-          call_data[:subpath]=''
+        if e.response.is_a?(Net::HTTPRedirection) && tries_remain_redirect.positive?
+          tries_remain_redirect -= 1
+          current_uri = URI.parse(call_data[:base_url])
+          new_url=e.response['location']
+          new_url="#{current_uri.scheme}:#{new_url}" unless new_url.start_with?('http')
+          Log.log.info("URL is moved: #{new_url}")
+          redir_uri = URI.parse(new_url)
+          call_data[:base_url] = new_url
+          call_data[:subpath] = ''
           if current_uri.host.eql?(redir_uri.host) && current_uri.port.eql?(redir_uri.port)
-            req=build_request(call_data)
+            req = build_request(call_data)
             retry
           else
             # change host
             Log.log.info("Redirect changes host: #{current_uri.host} -> #{redir_uri.host}")
             return self.class.new(call_data).call(call_data)
           end
-        else
-          raise e unless call_data[:return_error]
         end
         # raise exception if could not retry and not return error in result
         raise e unless call_data[:return_error]
@@ -312,23 +311,23 @@ module Aspera
 
     # @param encoding : one of: :json_params, :url_params
     def create(subpath,params,encoding=:json_params)
-      return call({operation: 'POST',subpath: subpath,headers: {'Accept'=>'application/json'},encoding=>params})
+      return call({operation: 'POST',subpath: subpath,headers: {'Accept' => 'application/json'},encoding => params})
     end
 
     def read(subpath,args=nil)
-      return call({operation: 'GET',subpath: subpath,headers: {'Accept'=>'application/json'},url_params: args})
+      return call({operation: 'GET',subpath: subpath,headers: {'Accept' => 'application/json'},url_params: args})
     end
 
     def update(subpath,params)
-      return call({operation: 'PUT',subpath: subpath,headers: {'Accept'=>'application/json'},json_params: params})
+      return call({operation: 'PUT',subpath: subpath,headers: {'Accept' => 'application/json'},json_params: params})
     end
 
     def delete(subpath)
-      return call({operation: 'DELETE',subpath: subpath,headers: {'Accept'=>'application/json'}})
+      return call({operation: 'DELETE',subpath: subpath,headers: {'Accept' => 'application/json'}})
     end
 
     def cancel(subpath)
-      return call({operation: 'CANCEL',subpath: subpath,headers: {'Accept'=>'application/json'}})
+      return call({operation: 'CANCEL',subpath: subpath,headers: {'Accept' => 'application/json'}})
     end
   end
 end #module Aspera

data/lib/aspera/rest_call_error.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Aspera
   # raised on error after REST call
   class RestCallError < StandardError

data/lib/aspera/rest_error_analyzer.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'aspera/log'
 require 'aspera/rest_call_error'
 require 'singleton'
@@ -11,8 +12,8 @@ module Aspera
     # the singleton object is registered with application specific handlers
     def initialize
       # list of handlers
-      @error_handlers=[]
-      @log_file=nil
+      @error_handlers = []
+      @log_file = nil
       add_handler('Type Generic') do |type,call_context|
         if !call_context[:response].code.start_with?('2')
           # add generic information
@@ -25,7 +26,7 @@ module Aspera
     # Analyzes REST call response and raises a RestCallError exception
     # if HTTP result code is not 2XX
     def raise_on_error(req,res)
-      call_context={
+      call_context = {
         messages: [],
         request:  req,
         response: res[:http],
@@ -60,14 +61,14 @@ module Aspera
     def add_simple_handler(name,*args)
       add_handler(name) do |type,call_context|
         # need to clone because we modify and same array is used subsequently
-        path=args.clone
+        path = args.clone
         #Log.log.debug("path=#{path}")
         # if last in path is boolean it tells if the error is only with http error code or always
-        always=[true, false].include?(path.last) ? path.pop : false
+        always = [true, false].include?(path.last) ? path.pop : false
         if call_context[:data].is_a?(Hash) && (!call_context[:response].code.start_with?('2') || always)
-          msg_key=path.pop
+          msg_key = path.pop
           # dig and find sub entry corresponding to path in deep hash
-          error_struct=path.inject(call_context[:data]) { |subhash, key| subhash.respond_to?(:keys) ? subhash[key] : nil }
+          error_struct = path.inject(call_context[:data]) { |subhash, key| subhash.respond_to?(:keys) ? subhash[key] : nil }
           if error_struct.is_a?(Hash) && error_struct[msg_key].is_a?(String)
             RestErrorAnalyzer.add_error(call_context,type,error_struct[msg_key])
             error_struct.each do |k,v|
@@ -79,18 +80,21 @@ module Aspera
       end
     end # add_simple_handler
 
-    # used by handler to add an error description to list of errors
-    # for logging and tracing : collect error descriptions (create file to activate)
-    # @param call_context a Hash containing the result call_context, provided to handler
-    # @param type a string describing type of exception, for logging purpose
-    # @param msg one error message  to add to list
-    def self.add_error(call_context,type,msg)
-      call_context[:messages].push(msg)
-      logfile=instance.log_file
-      # log error for further analysis (file must exist to activate)
-      return if logfile.nil? || !File.exist?(logfile)
-      File.open(logfile,'a+') do |f|
-        f.write("\n=#{type}=====\n#{call_context[:request].method} #{call_context[:request].path}\n#{call_context[:response].code}\n#{JSON.generate(call_context[:data])}\n#{call_context[:messages].join("\n")}")
+    class << self
+      # used by handler to add an error description to list of errors
+      # for logging and tracing : collect error descriptions (create file to activate)
+      # @param call_context a Hash containing the result call_context, provided to handler
+      # @param type a string describing type of exception, for logging purpose
+      # @param msg one error message  to add to list
+      def add_error(call_context,type,msg)
+        call_context[:messages].push(msg)
+        logfile = instance.log_file
+        # log error for further analysis (file must exist to activate)
+        return if logfile.nil? || !File.exist?(logfile)
+        File.open(logfile,'a+') do |f|
+          f.write("\n=#{type}=====\n#{call_context[:request].method} #{call_context[:request].path}\n#{call_context[:response].code}\n"\
+            "#{JSON.generate(call_context[:data])}\n#{call_context[:messages].join("\n")}")
+        end
       end
     end
   end

data/lib/aspera/rest_errors_aspera.rb

@@ -1,59 +1,62 @@
 # frozen_string_literal: true
+
 require 'aspera/rest_error_analyzer'
 require 'aspera/log'
 
 module Aspera
   # REST error handlers for various Aspera REST APIs
   class RestErrorsAspera
-    # handlers should probably be defined by plugins for modularity
-    def self.register_handlers
-      Log.log.debug('registering Aspera REST error handlers')
-      # Faspex 4: both user_message and internal_message, and code 200
-      # example: missing meta data on package creation
-      RestErrorAnalyzer.instance.add_simple_handler('Type 1: error:user_message','error','user_message',true)
-      RestErrorAnalyzer.instance.add_simple_handler('Type 2: error:description','error','description')
-      RestErrorAnalyzer.instance.add_simple_handler('Type 3: error:internal_message','error','internal_message')
-      # AoC Automation
-      RestErrorAnalyzer.instance.add_simple_handler('AoC Automation','error')
-      RestErrorAnalyzer.instance.add_simple_handler('Type 5','error_description')
-      RestErrorAnalyzer.instance.add_simple_handler('Type 6','message')
-      RestErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |name,call_context|
-        if call_context[:data].is_a?(Hash) && call_context[:data]['errors'].is_a?(Hash)
-          call_context[:data]['errors'].each do |k,v|
-            RestErrorAnalyzer.add_error(call_context,name,"#{k}: #{v}")
+    class << self
+      # handlers should probably be defined by plugins for modularity
+      def register_handlers
+        Log.log.debug('registering Aspera REST error handlers')
+        # Faspex 4: both user_message and internal_message, and code 200
+        # example: missing meta data on package creation
+        RestErrorAnalyzer.instance.add_simple_handler('Type 1: error:user_message','error','user_message',true)
+        RestErrorAnalyzer.instance.add_simple_handler('Type 2: error:description','error','description')
+        RestErrorAnalyzer.instance.add_simple_handler('Type 3: error:internal_message','error','internal_message')
+        # AoC Automation
+        RestErrorAnalyzer.instance.add_simple_handler('AoC Automation','error')
+        RestErrorAnalyzer.instance.add_simple_handler('Type 5','error_description')
+        RestErrorAnalyzer.instance.add_simple_handler('Type 6','message')
+        RestErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |name,call_context|
+          if call_context[:data].is_a?(Hash) && call_context[:data]['errors'].is_a?(Hash)
+            call_context[:data]['errors'].each do |k,v|
+              RestErrorAnalyzer.add_error(call_context,name,"#{k}: #{v}")
+            end
           end
         end
-      end
-      # call to upload_setup and download_setup of node api
-      RestErrorAnalyzer.instance.add_handler('T8:node: *_setup') do |type,call_context|
-        if call_context[:data].is_a?(Hash)
-          d_t_s=call_context[:data]['transfer_specs']
-          if d_t_s.is_a?(Array)
-            d_t_s.each do |res|
-              #r_err=res['transfer_spec']['error']
-              r_err=res['error']
-              if r_err.is_a?(Hash)
-                RestErrorAnalyzer.add_error(call_context,type,"#{r_err['code']}: #{r_err['reason']}: #{r_err['user_message']}")
+        # call to upload_setup and download_setup of node api
+        RestErrorAnalyzer.instance.add_handler('T8:node: *_setup') do |type,call_context|
+          if call_context[:data].is_a?(Hash)
+            d_t_s = call_context[:data]['transfer_specs']
+            if d_t_s.is_a?(Array)
+              d_t_s.each do |res|
+                #r_err=res['transfer_spec']['error']
+                r_err = res['error']
+                if r_err.is_a?(Hash)
+                  RestErrorAnalyzer.add_error(call_context,type,"#{r_err['code']}: #{r_err['reason']}: #{r_err['user_message']}")
+                end
               end
             end
           end
         end
-      end
-      RestErrorAnalyzer.instance.add_simple_handler('T9:IBM cloud IAM','errorMessage')
-      RestErrorAnalyzer.instance.add_simple_handler('T10:faspex v4','user_message')
-      RestErrorAnalyzer.instance.add_handler('bss graphql') do |type,call_context|
-        if call_context[:data].is_a?(Hash)
-          d_t_s=call_context[:data]['errors']
-          if d_t_s.is_a?(Array)
-            d_t_s.each do |res|
-              r_err=res['message']
-              if r_err.is_a?(String)
-                RestErrorAnalyzer.add_error(call_context,type,r_err)
+        RestErrorAnalyzer.instance.add_simple_handler('T9:IBM cloud IAM','errorMessage')
+        RestErrorAnalyzer.instance.add_simple_handler('T10:faspex v4','user_message')
+        RestErrorAnalyzer.instance.add_handler('bss graphql') do |type,call_context|
+          if call_context[:data].is_a?(Hash)
+            d_t_s = call_context[:data]['errors']
+            if d_t_s.is_a?(Array)
+              d_t_s.each do |res|
+                r_err = res['message']
+                if r_err.is_a?(String)
+                  RestErrorAnalyzer.add_error(call_context,type,r_err)
+                end
               end
             end
           end
         end
-      end
-    end # registerErrorTypes
+      end # register_handlers
+    end
   end
 end

data/lib/aspera/secret_hider.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require 'logger'
+
+module Aspera
+  # remove secret from logs and output
+  class SecretHider
+    # display string for hidden secrets
+    HIDDEN_PASSWORD = '🔑'
+    # keys in hash that contain secrets
+    ASCP_SECRETS=%w[ASPERA_SCP_PASS ASPERA_SCP_KEY ASPERA_SCP_FILEPASS ASPERA_PROXY_PASS].freeze
+    KEY_SECRETS =%w[password secret private_key passphrase].freeze
+    ALL_SECRETS =[ASCP_SECRETS,KEY_SECRETS].flatten.freeze
+    # regex that define namec captures :begin and :end
+    REGEX_LOG_REPLACES=[
+      # CLI manager get/set options
+      /(?<begin>[sg]et (#{KEY_SECRETS.join('|')})=).*(?<end>)/,
+      # env var ascp exec
+      /(?<begin> (#{ASCP_SECRETS.join('|')})=)[^ ]*(?<end> )/,
+      # rendered JSON
+      /(?<begin>["':][^"]*(#{ALL_SECRETS.join('|')})[^"]*["']?[=>: ]+")[^"]+(?<end>")/,
+      # option "secret"
+      /(?<begin>"[^"]*(secret)[^"]*"=>{)[^}]+(?<end>})/,
+      # option "secrets"
+      /(?<begin>(secrets)={)[^}]+(?<end>})/,
+      # private key values
+      /(?<begin>--+BEGIN .+ KEY--+)[[:ascii:]]+?(?<end>--+?END .+ KEY--+)/
+    ].freeze
+    private_constant :HIDDEN_PASSWORD,:ASCP_SECRETS,:KEY_SECRETS,:ALL_SECRETS,:REGEX_LOG_REPLACES
+    @log_secrets = false
+    class << self
+      attr_accessor :log_secrets
+      def log_formatter(original_formatter)
+        original_formatter ||= Logger::Formatter.new
+        # note that @log_secrets may be set AFTER this init is done, so it's done at runtime
+        return lambda do |severity, datetime, progname, msg|
+          if msg.is_a?(String) && !@log_secrets
+            REGEX_LOG_REPLACES.each do |regx|
+              msg = msg.gsub(regx){"#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}"}
+            end
+          end
+          original_formatter.call(severity, datetime, progname, msg)
+        end
+      end
+
+      def secret?(keyword,value)
+        keyword=keyword.to_s if keyword.is_a?(Symbol)
+        # only Strings can be secrets, not booleans, or hash, arrays
+        keyword.is_a?(String) && ALL_SECRETS.any?{|kw|keyword.include?(kw)} && value.is_a?(String)
+      end
+
+      def deep_remove_secret(obj,is_name_value: false)
+        case obj
+        when Array
+          if is_name_value
+            obj.each do |i|
+              i['value']=HIDDEN_PASSWORD if secret?(i['parameter'],i['value'])
+            end
+          else
+            obj.each{|i|deep_remove_secret(i)}
+          end
+        when Hash
+          obj.each do |k,v|
+            if secret?(k,v)
+              obj[k] = HIDDEN_PASSWORD
+            elsif obj[k].is_a?(Hash)
+              deep_remove_secret(obj[k])
+            end
+          end
+        end
+      end
+    end
+  end
+end