aspera-cli 4.14.0 → 4.16.0

data/lib/aspera/keychain/macos_security.rb

@@ -2,6 +2,8 @@
 
 # https://github.com/fastlane-community/security
 require 'aspera/cli/info'
+require 'aspera/log'
+require 'aspera/assert'
 
 # enhance the gem to support other key chains
 module Aspera
@@ -36,7 +38,7 @@ module Aspera
             url = options&.delete(:url)
             if !url.nil?
               uri = URI.parse(url)
-              raise 'only https' unless uri.scheme.eql?('https')
+              assert(uri.scheme.eql?('https')){'only https'}
               options[:protocol] = 'htps' # cspell: disable-line
               raise 'host required in URL' if uri.host.nil?
               options[:server] = uri.host
@@ -45,7 +47,7 @@ module Aspera
             end
             cmd = ['security', command]
             options&.each do |k, v|
-              raise "unknown option: #{k}" unless supported.key?(k)
+              assert(supported.key?(k)){"unknown option: #{k}"}
               next if v.nil?
               cmd.push("-#{supported[k]}")
               cmd.push(v.shellescape) unless v.empty?
@@ -70,7 +72,7 @@ module Aspera
           end
 
           def list(options={})
-            raise ArgumentError, "Invalid domain #{options[:domain]}, expected one of: #{DOMAINS}" unless options[:domain].nil? || DOMAINS.include?(options[:domain])
+            assert_values(options[:domain], DOMAINS, exception_class: ArgumentError){'domain'} unless options[:domain].nil?
             key_chains(execute('list-key_chains', options, LIST_OPTIONS))
           end
 
@@ -89,11 +91,11 @@ module Aspera
         end
 
         def password(operation, pass_type, options)
-          raise "wrong operation: #{operation}" unless %i[add find delete].include?(operation)
-          raise "wrong pass_type: #{pass_type}" unless %i[generic internet].include?(pass_type)
-          raise 'options shall be Hash' unless options.is_a?(Hash)
+          assert_values(operation, %i[add find delete]){'operation'}
+          assert_values(pass_type, %i[generic internet]){'pass_type'}
+          assert_type(options, Hash)
           missing = (operation.eql?(:add) ? %i[account service password] : %i[label]) - options.keys
-          raise "missing options: #{missing}" unless missing.empty?
+          assert(missing.empty?){"missing options: #{missing}"}
           options[:getpass] = '' if operation.eql?(:find)
           output = self.class.execute("#{operation}-#{pass_type}-password", options, ADD_PASS_OPTIONS, @path)
           raise output.gsub(/^.*: /, '') if output.start_with?('security: ')
@@ -127,18 +129,18 @@ module Aspera
       end
 
       def set(options)
-        raise 'options shall be Hash' unless options.is_a?(Hash)
+        assert_type(options, Hash){'options'}
         unsupported = options.keys - %i[label username password url description]
-        raise "unsupported options: #{unsupported}" unless unsupported.empty?
+        assert(unsupported.empty?){"unsupported options: #{unsupported}"}
         @keychain.password(
           :add, :generic, service: options[:label],
           account: options[:username] || 'none', password: options[:password], comment: options[:description])
       end
 
       def get(options)
-        raise 'options shall be Hash' unless options.is_a?(Hash)
+        assert_type(options, Hash){'options'}
         unsupported = options.keys - %i[label]
-        raise "unsupported options: #{unsupported}" unless unsupported.empty?
+        assert(unsupported.empty?){"unsupported options: #{unsupported}"}
         info = @keychain.password(:find, :generic, label: options[:label])
         raise 'not found' if info.nil?
         result = options.clone
@@ -153,9 +155,9 @@ module Aspera
       end
 
       def delete(options)
-        raise 'options shall be Hash' unless options.is_a?(Hash)
+        assert_type(options, Hash){'options'}
         unsupported = options.keys - %i[label]
-        raise "unsupported options: #{unsupported}" unless unsupported.empty?
+        assert(unsupported.empty?){"unsupported options: #{unsupported}"}
         raise 'delete not implemented, use macos keychain app'
       end
     end

data/lib/aspera/line_logger.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'aspera/log'
+
+module Aspera
+  # used for logging http
+  class LineLogger
+    def initialize(level)
+      @level = level
+      @buffer = []
+    end
+
+    def <<(string)
+      return if string.nil? || string.empty?
+      if !string.end_with?("\n")
+        @buffer.push(string)
+        return
+      end
+      Log.log.send(@level, @buffer.join('') + string.chomp)
+      @buffer.clear
+    end
+  end
+end

data/lib/aspera/log.rb

@@ -2,17 +2,55 @@
 
 require 'aspera/colors'
 require 'aspera/secret_hider'
+require 'aspera/environment'
+require 'aspera/assert'
 require 'logger'
 require 'pp'
 require 'json'
 require 'singleton'
 
+# extend Ruby logger with trace levels
+class Logger
+  TRACE_MAX = 2
+  # add custom level to logger severity
+  module Severity
+    1.upto(TRACE_MAX).each { |level| const_set("TRACE#{level}", - level)}
+  end
+  # quick access to label
+  SEVERITY_LABEL = Severity.constants.each_with_object({}) { |name, hash| hash[Severity.const_get(name)] = name}
+  def format_severity(severity)
+    SEVERITY_LABEL[severity] || 'ANY'
+  end
+
+  # define methods for a given level
+  def self.make_methods(str_level) # rubocop:disable Style/ClassMethodsDefinitions
+    int_level = ::Logger.const_get(str_level.upcase)
+    str_level = str_level.downcase
+    Kernel.send('lave'.reverse, <<-EOM, nil, __FILE__, __LINE__ + 1)
+      def #{str_level}(message = nil, &block)
+        add(#{int_level}, message, &block)
+      end
+
+      def #{str_level}?
+        level <= #{int_level}
+      end
+
+      def #{str_level}!
+        self.level = #{int_level}
+      end
+    EOM
+  end
+  # declare methods for all levels
+  Logger::Severity.constants.each { |severity| make_methods(severity) }
+end
+
 module Aspera
   # Singleton object for logging
   class Log
     include Singleton
     # where logs are sent to
     LOG_TYPES = %i[stderr stdout syslog].freeze
+    @@format = :json # rubocop:disable Style/ClassVars
     # class methods
     class << self
       # levels are :debug,:info,:warn,:error,fatal,:unknown
@@ -21,22 +59,20 @@ module Aspera
       # get the logger object of singleton
       def log; instance.logger; end
 
-      # dump object in debug mode
+      # dump object suitable for Log.log.debug
       # @param name string or symbol
       # @param format either pp or json format
-      def dump(name, object, format=:json)
-        log.debug do
-          result =
-            case format
-            when :json
-              JSON.pretty_generate(object) rescue PP.pp(object, +'')
-            when :ruby
-              PP.pp(object, +'')
-            else
-              raise 'wrong parameter, expect pp or json'
-            end
-          "#{name.to_s.green} (#{format})=\n#{result}"
-        end
+      def dump(name, object)
+        result =
+          case @@format
+          when :json
+            JSON.pretty_generate(object) rescue PP.pp(object, +'')
+          when :ruby
+            PP.pp(object, +'')
+          else
+            raise 'wrong parameter, expect ruby or json'
+          end
+        "#{name.to_s.green} (#{@@format})=\n#{result}"
       end
 
       # Capture the output of $stderr and log it at debug level
@@ -63,26 +99,32 @@ module Aspera
       Logger::Severity.constants.each do |name|
         return name.downcase.to_sym if @logger.level.eql?(Logger::Severity.const_get(name))
       end
-      # should not happen
-      raise "INTERNAL ERROR: unexpected level #{@logger.level}"
+      error_unexpected_value(@logger.level){'log level'}
     end
 
     # change underlying logger, but keep log level
     def logger_type=(new_log_type)
       current_severity_integer = @logger.level unless @logger.nil?
-      current_severity_integer = ENV['AS_LOG_LEVEL'] if current_severity_integer.nil? && ENV.key?('AS_LOG_LEVEL')
+      current_severity_integer = ENV.fetch('AS_LOG_LEVEL', nil) if current_severity_integer.nil? && ENV.key?('AS_LOG_LEVEL')
       current_severity_integer = Logger::Severity::WARN if current_severity_integer.nil?
       case new_log_type
       when :stderr
-        # typed: Logger
         @logger = Logger.new($stderr)
       when :stdout
         @logger = Logger.new($stdout)
       when :syslog
         require 'syslog/logger'
+        # the syslog class automatically creates methods from the severity names
+        # we just need to add the mapping (but syslog lowest is DEBUG)
+        1.upto(Logger::TRACE_MAX).each do |level|
+          Syslog::Logger.const_get(:LEVEL_MAP)[Logger.const_get("TRACE#{level}")] = Syslog::LOG_DEBUG
+        end
+        Logger::Severity.constants.each do |severity|
+          Syslog::Logger.make_methods(severity.downcase)
+        end
         @logger = Syslog::Logger.new(@program_name, Syslog::LOG_LOCAL2)
       else
-        raise "unknown log type: #{new_log_type.class} #{new_log_type}"
+        raise "unknown log type: #{new_log_type}, use one of: #{LOG_TYPES.join(', ')}"
       end
       @logger.level = current_severity_integer
       @logger_type = new_log_type

data/lib/aspera/nagios.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'aspera/log'
+require 'aspera/assert'
 require 'date'
 
 module Aspera
@@ -21,8 +23,11 @@ module Aspera
     class << self
       # process results of a analysis and display status and exit with code
       def process(data)
-        raise 'INTERNAL ERROR, result must be list and not empty' unless data.is_a?(Array) && !data.empty?
-        %w[status component message].each{|c|raise "INTERNAL ERROR, result must have #{c}" unless data.first.key?(c)}
+        assert_type(data, Array)
+        assert(!data.empty?){'data is empty'}
+        %w[status component message].each do |c|
+          assert(data.first.key?(c)){"result must have #{c}"}
+        end
         res_errors = data.reject{|s|s['status'].eql?('ok')}
         # keep only errors in case of problem, other ok are assumed so
         data = res_errors unless res_errors.empty?

data/lib/aspera/node.rb

@@ -1,9 +1,11 @@
 # frozen_string_literal: true
 
+require 'aspera/cli/error'
 require 'aspera/fasp/transfer_spec'
 require 'aspera/rest'
 require 'aspera/oauth'
 require 'aspera/log'
+require 'aspera/assert'
 require 'aspera/environment'
 require 'zlib'
 require 'base64'
@@ -13,14 +15,22 @@ module Aspera
   class Node < Aspera::Rest
     # permissions
     ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
-    # prefix for ruby code for filter
+    # prefix for ruby code for filter (deprecated)
     MATCH_EXEC_PREFIX = 'exec:'
+    MATCH_TYPES = [String, Proc, Regexp, NilClass].freeze
     HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
     PATH_SEPARATOR = '/'
     TS_FIELDS_TO_COPY = %w[remote_host remote_user ssh_port fasp_port wss_enabled wss_port].freeze
+    SCOPE_USER = 'user:all'
+    SCOPE_ADMIN = 'admin:all'
+    SCOPE_PREFIX = 'node.'
+    SCOPE_SEPARATOR = ':'
+    SIGNATURE_DELIMITER = '==SIGNATURE=='
+    BEARER_TOKEN_VALIDITY_DEFAULT = 86400
+    BEARER_TOKEN_SCOPE_DEFAULT = SCOPE_USER
 
     # register node special token decoder
-    Oauth.register_decoder(lambda{|token|JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition('==SIGNATURE==').first)})
+    Oauth.register_decoder(lambda{|token|Node.decode_bearer_token(token)})
 
     # class instance variable, access with accessors on class
     @use_standard_ports = true
@@ -28,16 +38,79 @@ module Aspera
     class << self
       attr_accessor :use_standard_ports
 
-      # for access keys: provide expression to match entry in folder
-      # if no prefix: regex
-      # if prefix: ruby code
-      # if expression is nil, then always match
+      # For access keys: provide expression to match entry in folder
       def file_matcher(match_expression)
-        match_expression ||= "#{MATCH_EXEC_PREFIX}true"
-        if match_expression.start_with?(MATCH_EXEC_PREFIX)
-          return Environment.secure_eval("lambda{|f|#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}")
+        case match_expression
+        when Proc then return match_expression
+        when Regexp then return ->(f){f['name'].match?(match_expression)}
+        when String
+          if match_expression.start_with?(MATCH_EXEC_PREFIX)
+            code = "->(f){#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}"
+            Log.log.warn{"Use of prefix #{MATCH_EXEC_PREFIX} is deprecated (4.15), instead use: @ruby:'#{code}'"}
+            return Environment.secure_eval(code, __FILE__, __LINE__)
+          end
+          return lambda{|f|File.fnmatch(match_expression, f['name'], File::FNM_DOTMATCH)}
+        when NilClass then return ->(_){true}
+        else error_unexpected_value(match_expression.class.name, exception_class: Cli::BadArgument)
+        end
+      end
+
+      def file_matcher_from_argument(options)
+        return file_matcher(options.get_next_argument('filter', type: MATCH_TYPES, mandatory: false))
+      end
+
+      # node API scopes
+      def token_scope(access_key, scope)
+        return [SCOPE_PREFIX, access_key, SCOPE_SEPARATOR, scope].join('')
+      end
+
+      def decode_scope(scope)
+        items = scope.split(SCOPE_SEPARATOR, 2)
+        assert(items.length.eql?(2)){"invalid scope: #{scope}"}
+        assert(items[0].start_with?(SCOPE_PREFIX)){"invalid scope: #{scope}"}
+        return {access_key: items[0][SCOPE_PREFIX.length..-1], scope: items[1]}
+      end
+
+      # Create an Aspera Node bearer token
+      # @param payload [String] JSON payload to be included in the token
+      # @param private_key [OpenSSL::PKey::RSA] Private key to sign the token
+      def bearer_token(access_key:, payload:, private_key:)
+        assert_type(payload, Hash)
+        assert(payload.key?('user_id'))
+        assert_type(payload['user_id'], String)
+        assert(!payload['user_id'].empty?)
+        assert_type(private_key, OpenSSL::PKey::RSA)
+        # manage convenience parameters
+        expiration_sec = payload['_validity'] || BEARER_TOKEN_VALIDITY_DEFAULT
+        payload.delete('_validity')
+        scope = payload['_scope'] || BEARER_TOKEN_SCOPE_DEFAULT
+        payload.delete('_scope')
+        payload['scope'] ||= token_scope(access_key, scope)
+        payload['auth_type'] ||= 'access_key'
+        payload['expires_at'] ||= (Time.now + expiration_sec).utc.strftime('%FT%TZ')
+        payload_json = JSON.generate(payload)
+        return Base64.strict_encode64(Zlib::Deflate.deflate([
+          payload_json,
+          SIGNATURE_DELIMITER,
+          Base64.strict_encode64(private_key.sign(OpenSSL::Digest.new('sha512'), payload_json)).scan(/.{1,60}/).join("\n"),
+          ''
+        ].join("\n")))
+      end
+
+      def decode_bearer_token(token)
+        return JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition(SIGNATURE_DELIMITER).first)
+      end
+
+      def bearer_headers(bearer_auth, access_key: nil)
+        # if username is not provided, use the access key from the token
+        if access_key.nil?
+          access_key = Aspera::Node.decode_scope(Aspera::Node.decode_bearer_token(Oauth.bearer_extract(bearer_auth))['scope'])[:access_key]
+          assert(!access_key.nil?)
         end
-        return lambda{|f|f['name'].match(/#{match_expression}/)}
+        return {
+          Aspera::Node::HEADER_X_ASPERA_ACCESS_KEY => access_key,
+          'Authorization'                          => bearer_auth
+        }
       end
     end
 
@@ -52,16 +125,17 @@ module Aspera
     # @param params [Hash] Rest parameters
     # @param app_info [Hash,NilClass] special processing for AoC
     def initialize(params:, app_info: nil, add_tspec: nil)
+      # init Rest
       super(params)
       @app_info = app_info
       # this is added to transfer spec, for instance to add tags (COS)
       @add_tspec = add_tspec
       if !@app_info.nil?
         REQUIRED_APP_INFO_FIELDS.each do |field|
-          raise "INTERNAL ERROR: app_info lacks field #{field}" unless @app_info.key?(field)
+          assert(@app_info.key?(field)){"app_info lacks field #{field}"}
         end
         REQUIRED_APP_API_METHODS.each do |method|
-          raise "INTERNAL ERROR: #{@app_info[:api].class} lacks method #{method}" unless @app_info[:api].respond_to?(method)
+          assert(@app_info[:api].respond_to?(method)){"#{@app_info[:api].class} lacks method #{method}"}
         end
       end
     end
@@ -90,13 +164,13 @@ module Aspera
     # @param state [Object] state object sent to processing method
     # @param top_file_id [String] file id to start at (default = access key root file id)
     # @param top_file_path [String] path of top folder (default = /)
-    # @param block [Proc] processing method, args: entry, path, state
+    # @param block [Proc] processing method, arguments: entry, path, state
     def process_folder_tree(state:, top_file_id:, top_file_path: '/', &block)
-      raise 'INTERNAL ERROR: top_file_path not set' if top_file_path.nil?
-      raise 'INTERNAL ERROR: Missing block' unless block
+      assert(!top_file_path.nil?){'top_file_path not set'}
+      assert(block){'Missing block'}
       # start at top folder
       folders_to_explore = [{id: top_file_id, path: top_file_path}]
-      Log.dump(:folders_to_explore, folders_to_explore)
+      Log.log.debug{Log.dump(:folders_to_explore, folders_to_explore)}
       until folders_to_explore.empty?
         current_item = folders_to_explore.shift
         Log.log.debug{"searching #{current_item[:path]}".bg_green}
@@ -108,7 +182,7 @@ module Aspera
             Log.log.warn{"#{current_item[:path]}: #{e.class} #{e.message}"}
             []
           end
-        Log.dump(:folder_contents, folder_contents)
+        Log.log.debug{Log.dump(:folder_contents, folder_contents)}
         folder_contents.each do |entry|
           relative_path = File.join(current_item[:path], entry['name'])
           Log.log.debug{"process_folder_tree checking #{relative_path}"}
@@ -134,7 +208,7 @@ module Aspera
     # @param path [String]  file path
     # @return [Hash] {.api,.file_id}
     def resolve_api_fid(top_file_id, path)
-      raise 'file id shall be String' unless top_file_id.is_a?(String)
+      assert_type(top_file_id, String)
       process_last_link = path.end_with?(PATH_SEPARATOR)
       path_elements = path.split(PATH_SEPARATOR).reject(&:empty?)
       return {api: self, file_id: top_file_id} if path_elements.empty?
@@ -203,14 +277,14 @@ module Aspera
       case params[:auth][:type]
       when :basic
         ak_name = params[:auth][:username]
-        raise 'ERROR: no secret in node object' unless params[:auth][:password]
-        ak_token = Rest.basic_creds(params[:auth][:username], params[:auth][:password])
+        assert(params[:auth][:password]){'no secret in node object'}
+        ak_token = Rest.basic_token(params[:auth][:username], params[:auth][:password])
       when :oauth2
         ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
         # TODO: token_generation_lambda = lambda{|do_refresh|oauth_token(force_refresh: do_refresh)}
         # get bearer token, possibly use cache
         ak_token = oauth_token(force_refresh: false)
-      else raise "Unsupported auth method for node gen4: #{params[:auth][:type]}"
+      else error_unexpected_value(params[:auth][:type])
       end
       transfer_spec = {
         'direction' => direction,
@@ -235,9 +309,19 @@ module Aspera
         transfer_spec.merge!(Fasp::TransferSpec::AK_TSPEC_BASE)
         # by default: same address as node API
         transfer_spec['remote_host'] = URI.parse(params[:base_url]).host
+        # AoC allows specification of other url
         if !@app_info.nil? && !@app_info[:node_info]['transfer_url'].nil? && !@app_info[:node_info]['transfer_url'].empty?
           transfer_spec['remote_host'] = @app_info[:node_info]['transfer_url']
         end
+        info = read('info')[:data]
+        # get the transfer user from info on access key
+        transfer_spec['remote_user'] = info['transfer_user'] if info['transfer_user']
+        # get settings from name.value array to hash key.value
+        settings = info['settings']&.each_with_object({}){|i, h|h[i['name']] = i['value']}
+        # check WSS ports
+        %w[wss_enabled wss_port].each do |i|
+          transfer_spec[i] = settings[i] if settings.key?(i)
+        end if settings.is_a?(Hash)
       else
         # retrieve values from API (and keep a copy/cache)
         @std_t_spec_cache ||= create(