aspera-cli 4.7.0 → 4.9.0

data/lib/aspera/log.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
+
 require 'aspera/colors'
+require 'aspera/secret_hider'
 require 'logger'
 require 'pp'
 require 'json'
@@ -8,17 +10,14 @@ require 'singleton'
 module Aspera
   # Singleton object for logging
   class Log
-    # display string for hidden secrets
-    HIDDEN_PASSWORD='🔑'
-    private_constant :HIDDEN_PASSWORD
     include Singleton
     # class methods
     class << self
       # levels are :debug,:info,:warn,:error,fatal,:unknown
-      def levels; Logger::Severity.constants.sort{|a,b|Logger::Severity.const_get(a)<=>Logger::Severity.const_get(b)}.map{|c|c.downcase.to_sym};end
+      def levels; Logger::Severity.constants.sort{|a,b|Logger::Severity.const_get(a) <=> Logger::Severity.const_get(b)}.map{|c|c.downcase.to_sym};end
 
       # where logs are sent to
-      def logtypes; [:stderr,:stdout,:syslog];end
+      def logtypes; %i[stderr stdout syslog];end
 
       # get the logger object of singleton
       def log; instance.logger;end
@@ -27,28 +26,36 @@ module Aspera
       # @param name string or symbol
       # @param format either pp or json format
       def dump(name,object,format=:json)
-        log.debug() do
-          result=
-          case format
-          when :json
-            JSON.pretty_generate(object) rescue PP.pp(object,'')
-          when :ruby
-            PP.pp(object,'')
-          else
-            raise 'wrong parameter, expect pp or json'
-          end
+        log.debug do
+          result =
+            case format
+            when :json
+              JSON.pretty_generate(object) rescue PP.pp(object,+'')
+            when :ruby
+              PP.pp(object,+'')
+            else
+              raise 'wrong parameter, expect pp or json'
+            end
           "#{name.to_s.green} (#{format})=\n#{result}"
         end
       end
+
+      def capture_stderr
+        real_stderr = $stderr
+        $stderr = StringIO.new
+        yield
+        log.debug($stderr.string)
+      ensure
+        $stderr = real_stderr
+      end
     end # class
 
     attr_reader :logger_type, :logger
     attr_writer :program_name
-    attr_accessor :log_secrets
 
     # set log level of underlying logger given symbol level
     def level=(new_level)
-      @logger.level=Logger::Severity.const_get(new_level.to_sym.upcase)
+      @logger.level = Logger::Severity.const_get(new_level.to_sym.upcase)
     end
 
     # get symbol of debug level of underlying logger
@@ -62,9 +69,9 @@ module Aspera
 
     # change underlying logger, but keep log level
     def logger_type=(new_logtype)
-      current_severity_integer=@logger.level unless @logger.nil?
-      current_severity_integer=ENV['AS_LOG_LEVEL'] if current_severity_integer.nil? && ENV.has_key?('AS_LOG_LEVEL')
-      current_severity_integer=Logger::Severity::WARN if current_severity_integer.nil?
+      current_severity_integer = @logger.level unless @logger.nil?
+      current_severity_integer = ENV['AS_LOG_LEVEL'] if current_severity_integer.nil? && ENV.has_key?('AS_LOG_LEVEL')
+      current_severity_integer = Logger::Severity::WARN if current_severity_integer.nil?
       case new_logtype
       when :stderr
         @logger = Logger.new($stderr)
@@ -72,34 +79,23 @@ module Aspera
         @logger = Logger.new($stdout)
       when :syslog
         require 'syslog/logger'
-        @logger = Syslog::Logger.new(@program_name)
+        @logger = Syslog::Logger.new(@program_name, Syslog::LOG_LOCAL2)
       else
         raise "unknown log type: #{new_logtype.class} #{new_logtype}"
       end
-      @logger.level=current_severity_integer
-      @logger_type=new_logtype
-      original_formatter = @logger.formatter || Logger::Formatter.new
-      # update formatter with password hiding, note that @log_secrets may be set AFTER this init is done, so it's done at runtime
-      @logger.formatter=lambda do |severity, datetime, progname, msg|
-        if msg.is_a?(String) && !@log_secrets
-          msg=msg
-          .gsub(/(["':][^"]*(password|secret|private_key)[^"]*["']?[=>: ]+")([^"]+)(")/){"#{Regexp.last_match(1)}#{HIDDEN_PASSWORD}#{Regexp.last_match(4)}"}
-          .gsub(/("[^"]*(secret)[^"]*"=>{)([^}]+)(})/){"#{Regexp.last_match(1)}#{HIDDEN_PASSWORD}#{Regexp.last_match(4)}"}
-          .gsub(/((secrets)={)([^}]+)(})/){"#{Regexp.last_match(1)}#{HIDDEN_PASSWORD}#{Regexp.last_match(4)}"}
-          .gsub(/--+BEGIN[A-Z ]+KEY--+.+--+END[A-Z ]+KEY--+/m){HIDDEN_PASSWORD}
-        end
-        original_formatter.call(severity, datetime, progname, msg)
-      end
+      @logger.level = current_severity_integer
+      @logger_type = new_logtype
+      # update formatter with password hiding
+      @logger.formatter = SecretHider.log_formatter(@logger.formatter)
     end
 
     private
 
     def initialize
-      @logger=nil
-      @program_name='aspera'
-      @log_secrets=false
+      @logger = nil
+      @program_name = 'aspera'
       # this sets @logger and @logger_type (self needed to call method instead of local var)
-      self.logger_type=:stderr
+      self.logger_type = :stderr
       raise 'error logger shall be defined' if @logger.nil?
     end
   end

data/lib/aspera/nagios.rb

@@ -1,25 +1,52 @@
 # frozen_string_literal: true
+
 require 'date'
 
 module Aspera
   class Nagios
     # nagios levels
-    LEVELS=[:ok,:warning,:critical,:unknown,:dependent]
-    ADD_PREFIX='add_'
+    LEVELS = %i[ok warning critical unknown dependent].freeze
+    ADD_PREFIX = 'add_'
     # date offset levels
-    DATE_WARN_OFFSET=2
-    DATE_CRIT_OFFSET=5
+    DATE_WARN_OFFSET = 2
+    DATE_CRIT_OFFSET = 5
     private_constant :LEVELS,:ADD_PREFIX,:DATE_WARN_OFFSET,:DATE_CRIT_OFFSET
 
     # add methods to add nagios error levels, each take component name and message
     LEVELS.each_index do |code|
-      name="#{ADD_PREFIX}#{LEVELS[code]}".to_sym
+      name = "#{ADD_PREFIX}#{LEVELS[code]}".to_sym
       define_method(name){|comp,msg|@data.push({code: code,comp: comp,msg: msg})}
     end
 
+    class << self
+      # process results of a analysis and display status and exit with code
+      def process(data)
+        raise 'INTERNAL ERROR, result must be list and not empty' unless data.is_a?(Array) && !data.empty?
+        %w[status component message].each{|c|raise "INTERNAL ERROR, result must have #{c}" unless data.first.has_key?(c)}
+        res_errors = data.reject{|s|s['status'].eql?('ok')}
+        # keep only errors in case of problem, other ok are assumed so
+        data = res_errors unless res_errors.empty?
+        # first is most critical
+        data.sort!{|a,b|LEVELS.index(a['status'].to_sym) <=> LEVELS.index(b['status'].to_sym)}
+        # build message: if multiple components: concatenate
+        #message = data.map{|i|"#{i['component']}:#{i['message']}"}.join(', ').gsub("\n",' ')
+        message = data.
+          map{|i|i['component']}.
+          uniq.
+          map{|comp|comp + ':' + data.select{|d|d['component'].eql?(comp)}.map{|d|d['message']}.join(',')}.
+          join(', ').
+          tr("\n",' ')
+        status = data.first['status'].upcase
+        # display status for nagios
+        puts("#{status} - [#{message}]\n")
+        # provide exit code to nagios
+        Process.exit(LEVELS.index(data.first['status'].to_sym))
+      end
+    end
+
     attr_reader :data
     def initialize
-      @data=[]
+      @data = []
     end
 
     # comparte remote time with local time
@@ -27,9 +54,9 @@ module Aspera
       # check date if specified : 2015-10-13T07:32:01Z
       rtime = DateTime.strptime(remote_date)
       diff_time = (rtime - DateTime.now).abs
-      diff_disp=diff_time.round(-2)
+      diff_disp = diff_time.round(-2)
       Log.log.debug("DATE: #{remote_date} #{rtime} diff=#{diff_disp}")
-      msg="offset #{diff_disp} sec"
+      msg = "offset #{diff_disp} sec"
       if diff_time >= DATE_CRIT_OFFSET
         add_critical(component,msg)
       elsif diff_time >= DATE_WARN_OFFSET
@@ -47,26 +74,7 @@ module Aspera
     # translate for display
     def result
       raise 'missing result' if @data.empty?
-      {type: :object_list,data: @data.map{|i|{'status'=>LEVELS[i[:code]].to_s,'component'=>i[:comp],'message'=>i[:msg]}}}
-    end
-
-    # process results of a analysis and display status and exit with code
-    def self.process(data)
-      raise 'INTERNAL ERROR, result must be list and not empty' unless data.is_a?(Array) && !data.empty?
-      ['status','component','message'].each{|c|raise "INTERNAL ERROR, result must have #{c}" unless data.first.has_key?(c)}
-      res_errors = data.reject{|s|s['status'].eql?('ok')}
-      # keep only errors in case of problem, other ok are assumed so
-      data = res_errors unless res_errors.empty?
-      # first is most critical
-      data.sort!{|a,b|LEVELS.index(a['status'].to_sym)<=>LEVELS.index(b['status'].to_sym)}
-      # build message: if multiple components: concatenate
-      #message = data.map{|i|"#{i['component']}:#{i['message']}"}.join(', ').gsub("\n",' ')
-      message = data.map{|i|i['component']}.uniq.map{|comp|comp+':'+data.select{|d|d['component'].eql?(comp)}.map{|d|d['message']}.join(',')}.join(', ').gsub("\n",' ')
-      status=data.first['status'].upcase
-      # display status for nagios
-      puts("#{status} - [#{message}]\n")
-      # provide exit code to nagios
-      Process.exit(LEVELS.index(data.first['status'].to_sym))
+      {type: :object_list,data: @data.map{|i|{'status' => LEVELS[i[:code]].to_s,'component' => i[:comp],'message' => i[:msg]}}}
     end
   end
 end

data/lib/aspera/node.rb

@@ -1,8 +1,10 @@
 # frozen_string_literal: true
+
 require 'aspera/fasp/transfer_spec'
 require 'aspera/rest'
 require 'aspera/oauth'
 require 'aspera/log'
+require 'aspera/environment'
 require 'zlib'
 require 'base64'
 
@@ -10,21 +12,18 @@ module Aspera
   # Provides additional functions using node API.
   class Node < Rest
     # permissions
-    ACCESS_LEVELS=%w[delete list mkdir preview read rename write].freeze
+    ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
     # prefix for ruby code for filter
-    MATCH_EXEC_PREFIX='exec:'
+    MATCH_EXEC_PREFIX = 'exec:'
 
     # register node special token decoder
     Oauth.register_decoder(lambda{|token|JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition('==SIGNATURE==').first)})
 
-    class<<self
+    class << self
       def set_ak_basic_token(ts,ak,secret)
-        Log.log.warn("Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, but have #{ts['remote_user']}") unless ts['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
-        ts['token']="Basic #{Base64.strict_encode64("#{ak}:#{secret}")}"
-      end
-
-      def empty_binding
-        return Kernel.binding
+        Log.log.warn("Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, "\
+          "but have #{ts['remote_user']}") unless ts['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
+        ts['token'] = Rest.basic_creds(ak,secret)
       end
 
       # for access keys: provide expression to match entry in folder
@@ -32,9 +31,9 @@ module Aspera
       # if prefix: ruby code
       # if filder is nil, then always match
       def file_matcher(match_expression)
-        match_expression||="#{MATCH_EXEC_PREFIX}true"
+        match_expression ||= "#{MATCH_EXEC_PREFIX}true"
         if match_expression.start_with?(MATCH_EXEC_PREFIX)
-          return eval("lambda{|f|#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}", empty_binding, __FILE__, __LINE__)
+          return Environment.secure_eval("lambda{|f|#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}")
         end
         return lambda{|f|f['name'].match(/#{match_expression}/)}
       end
@@ -60,21 +59,22 @@ module Aspera
       raise "processor must have #{opt[:method]}" unless processor.respond_to?(opt[:method])
       Log.log.debug("crawl #{opt}")
       #top_info=read("files/#{opt[:top_file_id]}")[:data]
-      folders_to_explore=[{id: opt[:top_file_id], relpath: opt[:top_file_path]}]
+      folders_to_explore = [{id: opt[:top_file_id], relpath: opt[:top_file_path]}]
       Log.dump(:folders_to_explore,folders_to_explore)
       while !folders_to_explore.empty?
         current_item = folders_to_explore.shift
         Log.log.debug("searching #{current_item[:relpath]}".bg_green)
         # get folder content
-        folder_contents = begin
-          read("files/#{current_item[:id]}/files")[:data]
-        rescue StandardError => e
-          Log.log.warn("#{current_item[:relpath]}: #{e.class} #{e.message}")
-          []
-        end
+        folder_contents =
+          begin
+            read("files/#{current_item[:id]}/files")[:data]
+          rescue StandardError => e
+            Log.log.warn("#{current_item[:relpath]}: #{e.class} #{e.message}")
+            []
+          end
         Log.dump(:folder_contents,folder_contents)
         folder_contents.each do |entry|
-          relative_path=File.join(current_item[:relpath],entry['name'])
+          relative_path = File.join(current_item[:relpath],entry['name'])
           Log.log.debug("looking #{relative_path}".bg_green)
           # entry type is file, folder or link
           if processor.send(opt[:method],entry,relative_path) && entry['type'].eql?('folder')