aspera-cli 4.7.0 → 4.8.0

data/lib/aspera/keychain/macos_security.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'security'
 
 # enhance the gem to support other keychains
@@ -6,7 +7,7 @@ module Security
   class Keychain
     class << self
       def by_name(name)
-        keychains_from_output('security list-keychains').select{|kc|kc.filename.end_with?("/#{name}.keychain-db")}.first
+        keychains_from_output('security list-keychains').find{|kc|kc.filename.end_with?("/#{name}.keychain-db")}
       end
     end
   end
@@ -14,20 +15,20 @@ module Security
   class Password
     class << self
       # add some login to original method
-      alias orig_flags_for_options flags_for_options
+      alias_method :orig_flags_for_options, :flags_for_options
       def flags_for_options(options = {})
-        keychain=options.delete(:keychain)
-        url=options.delete(:url)
+        keychain = options.delete(:keychain)
+        url = options.delete(:url)
         if !url.nil?
-          uri=URI.parse(url)
+          uri = URI.parse(url)
           raise 'only https' unless uri.scheme.eql?('https')
-          options[:r]='htps'
+          options[:r] = 'htps'
           raise 'host required in URL' if uri.host.nil?
-          options[:s]=uri.host
-          options[:p]=uri.path unless ['','/'].include?(uri.path)
-          options[:P]=uri.port unless uri.port.eql?(443) && !url.include?(':443/')
+          options[:s] = uri.host
+          options[:p] = uri.path unless ['','/'].include?(uri.path)
+          options[:P] = uri.port unless uri.port.eql?(443) && !url.include?(':443/')
         end
-        flags=[orig_flags_for_options(options)]
+        flags = [orig_flags_for_options(options)]
         flags.push(keychain.filename) unless keychain.nil?
         flags.join(' ')
       end
@@ -40,35 +41,36 @@ module Aspera
     # keychain based on macOS keychain, using `security` cmmand line
     class MacosSecurity
       def initialize(name=nil)
-        @keychain=name.nil? ? Security::Keychain.default_keychain : Security::Keychain.by_name(name)
+        @keychain = name.nil? ? Security::Keychain.default_keychain : Security::Keychain.by_name(name)
         raise "no such keychain #{name}" if @keychain.nil?
       end
 
       def set(options)
         raise 'options shall be Hash' unless options.is_a?(Hash)
-        unsupported=options.keys-[:username,:url,:secret,:description]
+        unsupported = options.keys - %i[username url secret description]
         raise "unsupported options: #{unsupported}" unless unsupported.empty?
-        username=options[:username]
+        username = options[:username]
         raise 'options shall have username' if username.nil?
-        url=options[:url]
+        url = options[:url]
         raise 'options shall have url' if url.nil?
-        secret=options[:secret]
+        secret = options[:secret]
         raise 'options shall have secret' if secret.nil?
         raise 'set not implemented'
       end
 
       def get(options)
         raise 'options shall be Hash' unless options.is_a?(Hash)
-        unsupported=options.keys-[:username,:url]
+        unsupported = options.keys - %i[username url]
         raise "unsupported options: #{unsupported}" unless unsupported.empty?
-        username=options[:username]
+        username = options[:username]
         raise 'options shall have username' if username.nil?
-        url=options[:url]
+        url = options[:url]
         raise 'options shall have url' if url.nil?
-        info=Security::InternetPassword.find(keychain: @keychain, url: url, account: username)
+        info = Security::InternetPassword.find(keychain: @keychain, url: url, account: username)
         raise 'not found' if info.nil?
-        result=options.clone
-        result.merge!({secret: info.password, description: info.attributes['icmt']})
+        result = options.clone
+        result[:secret] = info.password
+        result[:description] = info.attributes['icmt']
         return result
       end
 
@@ -78,11 +80,11 @@ module Aspera
 
       def delete(options)
         raise 'options shall be Hash' unless options.is_a?(Hash)
-        unsupported=options.keys-[:username,:url]
+        unsupported = options.keys - %i[username url]
         raise "unsupported options: #{unsupported}" unless unsupported.empty?
-        username=options[:username]
+        username = options[:username]
         raise 'options shall have username' if username.nil?
-        url=options[:url]
+        url = options[:url]
         raise "delete not implemented #{url}"
       end
     end

data/lib/aspera/log.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
+
 require 'aspera/colors'
+require 'aspera/secret_hider'
 require 'logger'
 require 'pp'
 require 'json'
@@ -8,17 +10,14 @@ require 'singleton'
 module Aspera
   # Singleton object for logging
   class Log
-    # display string for hidden secrets
-    HIDDEN_PASSWORD='🔑'
-    private_constant :HIDDEN_PASSWORD
     include Singleton
     # class methods
     class << self
       # levels are :debug,:info,:warn,:error,fatal,:unknown
-      def levels; Logger::Severity.constants.sort{|a,b|Logger::Severity.const_get(a)<=>Logger::Severity.const_get(b)}.map{|c|c.downcase.to_sym};end
+      def levels; Logger::Severity.constants.sort{|a,b|Logger::Severity.const_get(a) <=> Logger::Severity.const_get(b)}.map{|c|c.downcase.to_sym};end
 
       # where logs are sent to
-      def logtypes; [:stderr,:stdout,:syslog];end
+      def logtypes; %i[stderr stdout syslog];end
 
       # get the logger object of singleton
       def log; instance.logger;end
@@ -27,28 +26,36 @@ module Aspera
       # @param name string or symbol
       # @param format either pp or json format
       def dump(name,object,format=:json)
-        log.debug() do
-          result=
-          case format
-          when :json
-            JSON.pretty_generate(object) rescue PP.pp(object,'')
-          when :ruby
-            PP.pp(object,'')
-          else
-            raise 'wrong parameter, expect pp or json'
-          end
+        log.debug do
+          result =
+            case format
+            when :json
+              JSON.pretty_generate(object) rescue PP.pp(object,+'')
+            when :ruby
+              PP.pp(object,+'')
+            else
+              raise 'wrong parameter, expect pp or json'
+            end
           "#{name.to_s.green} (#{format})=\n#{result}"
         end
       end
+
+      def capture_stderr
+        real_stderr = $stderr
+        $stderr = StringIO.new
+        yield
+        log.debug($stderr.string)
+      ensure
+        $stderr = real_stderr
+      end
     end # class
 
     attr_reader :logger_type, :logger
     attr_writer :program_name
-    attr_accessor :log_secrets
 
     # set log level of underlying logger given symbol level
     def level=(new_level)
-      @logger.level=Logger::Severity.const_get(new_level.to_sym.upcase)
+      @logger.level = Logger::Severity.const_get(new_level.to_sym.upcase)
     end
 
     # get symbol of debug level of underlying logger
@@ -62,9 +69,9 @@ module Aspera
 
     # change underlying logger, but keep log level
     def logger_type=(new_logtype)
-      current_severity_integer=@logger.level unless @logger.nil?
-      current_severity_integer=ENV['AS_LOG_LEVEL'] if current_severity_integer.nil? && ENV.has_key?('AS_LOG_LEVEL')
-      current_severity_integer=Logger::Severity::WARN if current_severity_integer.nil?
+      current_severity_integer = @logger.level unless @logger.nil?
+      current_severity_integer = ENV['AS_LOG_LEVEL'] if current_severity_integer.nil? && ENV.has_key?('AS_LOG_LEVEL')
+      current_severity_integer = Logger::Severity::WARN if current_severity_integer.nil?
       case new_logtype
       when :stderr
         @logger = Logger.new($stderr)
@@ -76,30 +83,19 @@ module Aspera
       else
         raise "unknown log type: #{new_logtype.class} #{new_logtype}"
       end
-      @logger.level=current_severity_integer
-      @logger_type=new_logtype
-      original_formatter = @logger.formatter || Logger::Formatter.new
-      # update formatter with password hiding, note that @log_secrets may be set AFTER this init is done, so it's done at runtime
-      @logger.formatter=lambda do |severity, datetime, progname, msg|
-        if msg.is_a?(String) && !@log_secrets
-          msg=msg
-          .gsub(/(["':][^"]*(password|secret|private_key)[^"]*["']?[=>: ]+")([^"]+)(")/){"#{Regexp.last_match(1)}#{HIDDEN_PASSWORD}#{Regexp.last_match(4)}"}
-          .gsub(/("[^"]*(secret)[^"]*"=>{)([^}]+)(})/){"#{Regexp.last_match(1)}#{HIDDEN_PASSWORD}#{Regexp.last_match(4)}"}
-          .gsub(/((secrets)={)([^}]+)(})/){"#{Regexp.last_match(1)}#{HIDDEN_PASSWORD}#{Regexp.last_match(4)}"}
-          .gsub(/--+BEGIN[A-Z ]+KEY--+.+--+END[A-Z ]+KEY--+/m){HIDDEN_PASSWORD}
-        end
-        original_formatter.call(severity, datetime, progname, msg)
-      end
+      @logger.level = current_severity_integer
+      @logger_type = new_logtype
+      # update formatter with password hiding
+      @logger.formatter = SecretHider.log_formatter(@logger.formatter)
     end
 
     private
 
     def initialize
-      @logger=nil
-      @program_name='aspera'
-      @log_secrets=false
+      @logger = nil
+      @program_name = 'aspera'
       # this sets @logger and @logger_type (self needed to call method instead of local var)
-      self.logger_type=:stderr
+      self.logger_type = :stderr
       raise 'error logger shall be defined' if @logger.nil?
     end
   end

data/lib/aspera/nagios.rb

@@ -1,25 +1,26 @@
 # frozen_string_literal: true
+
 require 'date'
 
 module Aspera
   class Nagios
     # nagios levels
-    LEVELS=[:ok,:warning,:critical,:unknown,:dependent]
-    ADD_PREFIX='add_'
+    LEVELS = %i[ok warning critical unknown dependent].freeze
+    ADD_PREFIX = 'add_'
     # date offset levels
-    DATE_WARN_OFFSET=2
-    DATE_CRIT_OFFSET=5
+    DATE_WARN_OFFSET = 2
+    DATE_CRIT_OFFSET = 5
     private_constant :LEVELS,:ADD_PREFIX,:DATE_WARN_OFFSET,:DATE_CRIT_OFFSET
 
     # add methods to add nagios error levels, each take component name and message
     LEVELS.each_index do |code|
-      name="#{ADD_PREFIX}#{LEVELS[code]}".to_sym
+      name = "#{ADD_PREFIX}#{LEVELS[code]}".to_sym
       define_method(name){|comp,msg|@data.push({code: code,comp: comp,msg: msg})}
     end
 
     attr_reader :data
     def initialize
-      @data=[]
+      @data = []
     end
 
     # comparte remote time with local time
@@ -27,9 +28,9 @@ module Aspera
       # check date if specified : 2015-10-13T07:32:01Z
       rtime = DateTime.strptime(remote_date)
       diff_time = (rtime - DateTime.now).abs
-      diff_disp=diff_time.round(-2)
+      diff_disp = diff_time.round(-2)
       Log.log.debug("DATE: #{remote_date} #{rtime} diff=#{diff_disp}")
-      msg="offset #{diff_disp} sec"
+      msg = "offset #{diff_disp} sec"
       if diff_time >= DATE_CRIT_OFFSET
         add_critical(component,msg)
       elsif diff_time >= DATE_WARN_OFFSET
@@ -47,22 +48,22 @@ module Aspera
     # translate for display
     def result
       raise 'missing result' if @data.empty?
-      {type: :object_list,data: @data.map{|i|{'status'=>LEVELS[i[:code]].to_s,'component'=>i[:comp],'message'=>i[:msg]}}}
+      {type: :object_list,data: @data.map{|i|{'status' => LEVELS[i[:code]].to_s,'component' => i[:comp],'message' => i[:msg]}}}
     end
 
     # process results of a analysis and display status and exit with code
     def self.process(data)
       raise 'INTERNAL ERROR, result must be list and not empty' unless data.is_a?(Array) && !data.empty?
-      ['status','component','message'].each{|c|raise "INTERNAL ERROR, result must have #{c}" unless data.first.has_key?(c)}
+      %w[status component message].each{|c|raise "INTERNAL ERROR, result must have #{c}" unless data.first.has_key?(c)}
       res_errors = data.reject{|s|s['status'].eql?('ok')}
       # keep only errors in case of problem, other ok are assumed so
       data = res_errors unless res_errors.empty?
       # first is most critical
-      data.sort!{|a,b|LEVELS.index(a['status'].to_sym)<=>LEVELS.index(b['status'].to_sym)}
+      data.sort!{|a,b|LEVELS.index(a['status'].to_sym) <=> LEVELS.index(b['status'].to_sym)}
       # build message: if multiple components: concatenate
       #message = data.map{|i|"#{i['component']}:#{i['message']}"}.join(', ').gsub("\n",' ')
-      message = data.map{|i|i['component']}.uniq.map{|comp|comp+':'+data.select{|d|d['component'].eql?(comp)}.map{|d|d['message']}.join(',')}.join(', ').gsub("\n",' ')
-      status=data.first['status'].upcase
+      message = data.map{|i|i['component']}.uniq.map{|comp|comp + ':' + data.select{|d|d['component'].eql?(comp)}.map{|d|d['message']}.join(',')}.join(', ').tr("\n",' ')
+      status = data.first['status'].upcase
       # display status for nagios
       puts("#{status} - [#{message}]\n")
       # provide exit code to nagios

data/lib/aspera/node.rb

@@ -1,8 +1,10 @@
 # frozen_string_literal: true
+
 require 'aspera/fasp/transfer_spec'
 require 'aspera/rest'
 require 'aspera/oauth'
 require 'aspera/log'
+require 'aspera/environment'
 require 'zlib'
 require 'base64'
 
@@ -10,21 +12,18 @@ module Aspera
   # Provides additional functions using node API.
   class Node < Rest
     # permissions
-    ACCESS_LEVELS=%w[delete list mkdir preview read rename write].freeze
+    ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
     # prefix for ruby code for filter
-    MATCH_EXEC_PREFIX='exec:'
+    MATCH_EXEC_PREFIX = 'exec:'
 
     # register node special token decoder
     Oauth.register_decoder(lambda{|token|JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition('==SIGNATURE==').first)})
 
-    class<<self
+    class << self
       def set_ak_basic_token(ts,ak,secret)
-        Log.log.warn("Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, but have #{ts['remote_user']}") unless ts['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
-        ts['token']="Basic #{Base64.strict_encode64("#{ak}:#{secret}")}"
-      end
-
-      def empty_binding
-        return Kernel.binding
+        Log.log.warn("Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, "\
+          "but have #{ts['remote_user']}") unless ts['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
+        ts['token'] = Rest.basic_creds(ak,secret)
       end
 
       # for access keys: provide expression to match entry in folder
@@ -32,9 +31,9 @@ module Aspera
       # if prefix: ruby code
       # if filder is nil, then always match
       def file_matcher(match_expression)
-        match_expression||="#{MATCH_EXEC_PREFIX}true"
+        match_expression ||= "#{MATCH_EXEC_PREFIX}true"
         if match_expression.start_with?(MATCH_EXEC_PREFIX)
-          return eval("lambda{|f|#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}", empty_binding, __FILE__, __LINE__)
+          return Environment.secure_eval("lambda{|f|#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}")
         end
         return lambda{|f|f['name'].match(/#{match_expression}/)}
       end
@@ -60,21 +59,22 @@ module Aspera
       raise "processor must have #{opt[:method]}" unless processor.respond_to?(opt[:method])
       Log.log.debug("crawl #{opt}")
       #top_info=read("files/#{opt[:top_file_id]}")[:data]
-      folders_to_explore=[{id: opt[:top_file_id], relpath: opt[:top_file_path]}]
+      folders_to_explore = [{id: opt[:top_file_id], relpath: opt[:top_file_path]}]
       Log.dump(:folders_to_explore,folders_to_explore)
       while !folders_to_explore.empty?
         current_item = folders_to_explore.shift
         Log.log.debug("searching #{current_item[:relpath]}".bg_green)
         # get folder content
-        folder_contents = begin
-          read("files/#{current_item[:id]}/files")[:data]
-        rescue StandardError => e
-          Log.log.warn("#{current_item[:relpath]}: #{e.class} #{e.message}")
-          []
-        end
+        folder_contents =
+          begin
+            read("files/#{current_item[:id]}/files")[:data]
+          rescue StandardError => e
+            Log.log.warn("#{current_item[:relpath]}: #{e.class} #{e.message}")
+            []
+          end
         Log.dump(:folder_contents,folder_contents)
         folder_contents.each do |entry|
-          relative_path=File.join(current_item[:relpath],entry['name'])
+          relative_path = File.join(current_item[:relpath],entry['name'])
           Log.log.debug("looking #{relative_path}".bg_green)
           # entry type is file, folder or link
           if processor.send(opt[:method],entry,relative_path) && entry['type'].eql?('folder')