aspera-cli 4.21.2 → 4.23.0

data/lib/aspera/keychain/factory.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Aspera
+  module Keychain
+    # Manage secrets in a Hashicorp Vault
+    class Factory
+      LIST = %i[file system vault].freeze
+      class << self
+        def create(info, name, folder, password)
+          Aspera.assert_type(info, Hash)
+          Aspera.assert(info.values.all?(String)){'vault info shall have only string values'}
+          info = info.symbolize_keys
+          vault_type = info.delete(:type)
+          Aspera.assert_values(vault_type, LIST.map(&:to_s)){'vault.type'}
+          case vault_type
+          when 'file'
+            info[:file] ||= 'vault.bin'
+            info[:file] = File.join(folder, info[:file]) unless File.absolute_path?(info[:file])
+            Aspera.assert(!password.nil?){'please provide password'}
+            info[:password] = password
+            # this module requires compilation, so it is optional
+            require 'aspera/keychain/encrypted_hash'
+            @vault = Keychain::EncryptedHash.new(**info)
+          when 'system'
+            case Environment.os
+            when Environment::OS_MACOS
+              info[:name] ||= name
+              @vault = Keychain::MacosSystem.new(**info)
+            else
+              raise 'not implemented for this OS'
+            end
+          when 'vault'
+            require 'aspera/keychain/hashicorp_vault'
+            @vault = Keychain::HashicorpVault.new(**info)
+          else Aspera.error_unexpected_value(vault_type)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aspera/keychain/hashicorp_vault.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'aspera/environment'
+require 'aspera/log'
+require 'aspera/assert'
+require 'vault'
+
+module Aspera
+  module Keychain
+    # Manage secrets in a Hashicorp Vault
+    class HashicorpVault
+      SECRET_PATH = 'secret/data/'
+
+      private_constant :SECRET_PATH
+
+      def initialize(url:, token:)
+        Vault.configure do |config|
+          config.address = url
+          config.token = token
+        end
+      end
+
+      def info
+        {
+          url:      Vault.address,
+          password: Vault.auth_token
+        }
+      end
+
+      def list
+        metadata_path = SECRET_PATH.sub('/data/', '/metadata/')
+        return Vault.logical.list(metadata_path).filter_map do |label|
+          get(label: label).merge(label: label)
+        end
+      end
+
+      # Set a secret
+      # @param options [Hash] with keys :label, :username, :password, :url, :description
+      def set(options)
+        label = options.fetch(:label)
+        data = {
+          username:    options[:username],
+          password:    options[:password],
+          url:         options[:url],
+          description: options[:description]
+        }.compact
+        Vault.logical.write(path(label), data: data)
+      end
+
+      def get(label:, exception: true)
+        secret = Vault.logical.read(path(label))
+        if secret.nil?
+          raise "Secret '#{label}' not found" if exception
+          return nil
+        end
+        return secret.data[:data]
+      end
+
+      def delete(label:)
+        path = path(label)
+        Vault.logical.delete(path)
+      end
+
+      private
+
+      def path(label)
+        "#{SECRET_PATH}#{label}"
+      end
+    end
+  end
+end

data/lib/aspera/keychain/macos_security.rb

@@ -60,7 +60,7 @@ module Aspera
           end
 
           def key_chains(output)
-            output.split("\n").collect { |line| new(line.strip.gsub(/^"|"$/, '')) }
+            output.split("\n").collect{ |line| new(line.strip.gsub(/^"|"$/, ''))}
           end
 
           def default
@@ -77,7 +77,7 @@ module Aspera
           end
 
           def by_name(name)
-            list.find{|kc|kc.path.end_with?("/#{name}.keychain-db")}
+            list.find{ |kc| kc.path.end_with?("/#{name}.keychain-db")}
           end
         end
         attr_reader :path
@@ -123,15 +123,28 @@ module Aspera
     end
 
     class MacosSystem
-      def initialize(name=nil, _password=nil)
-        @keychain = name.nil? ? MacosSecurity::Keychain.default_keychain : MacosSecurity::Keychain.by_name(name)
+      OPTIONS = %i[label username password url description].freeze
+      def initialize(name: nil)
+        @keychain_name = name.nil? ? 'default keychain' : name
+        @keychain = name.nil? ? MacosSecurity::Keychain.default : MacosSecurity::Keychain.by_name(name)
         raise "no such keychain #{name}" if @keychain.nil?
       end
 
+      def info
+        return {
+          keychain: @keychain_name
+        }
+      end
+
+      def list
+        # the only way to list is `dump-keychain` which triggers security alert
+        raise 'list not implemented, use macos keychain app'
+      end
+
       def set(options)
         Aspera.assert_type(options, Hash){'options'}
-        unsupported = options.keys - %i[label username password url description]
-        Aspera.assert(unsupported.empty?){"unsupported options: #{unsupported}"}
+        unsupported = options.keys - OPTIONS
+        Aspera.assert(unsupported.empty?){"unsupported options: #{unsupported}, use #{OPTIONS.join(', ')}"}
         @keychain.password(
           :add, :generic, service: options[:label],
           account: options[:username] || 'none', password: options[:password], comment: options[:description])
@@ -149,11 +162,6 @@ module Aspera
         return result
       end
 
-      def list
-        # the only way to list is `dump-keychain` which triggers security alert
-        raise 'list not implemented, use macos keychain app'
-      end
-
       def delete(options)
         Aspera.assert_type(options, Hash){'options'}
         unsupported = options.keys - %i[label]

data/lib/aspera/log.rb

@@ -9,57 +9,53 @@ require 'json'
 require 'singleton'
 require 'stringio'
 
+# Ignore warnings
 old_verbose = $VERBOSE
 $VERBOSE = nil
 
-# extend Ruby logger with trace levels
+# Extend Ruby logger with trace levels
 class Logger
+  # Two additionnal trace levels
   TRACE_MAX = 2
-  # add custom level to logger severity
+  # Add custom level to logger severity, below debug level
   module Severity
-    1.upto(TRACE_MAX).each { |level| const_set(:"TRACE#{level}", - level)}
+    1.upto(TRACE_MAX).each{ |level| const_set(:"TRACE#{level}", - level)}
   end
-  # quick access to label
-  SEVERITY_LABEL = Severity.constants.each_with_object({}) { |name, hash| hash[Severity.const_get(name)] = name}
+  # Quick access to label
+  SEVERITY_LABEL = Severity.constants.each_with_object({}){ |name, hash| hash[Severity.const_get(name)] = name}
   def format_severity(severity)
     SEVERITY_LABEL[severity] || 'ANY'
   end
 
-  # define methods for a given level
-  def self.make_methods(str_level) # rubocop:disable Style/ClassMethodsDefinitions
-    int_level = ::Logger.const_get(str_level.upcase)
-    str_level = str_level.downcase
-    Kernel.send('lave'.reverse, <<-EOM, nil, __FILE__, __LINE__ + 1)
-      def #{str_level}(message = nil, &block)
-        add(#{int_level}, message, &block)
-      end
-
-      def #{str_level}?
-        level <= #{int_level}
-      end
-
-      def #{str_level}!
-        self.level = #{int_level}
-      end
-    EOM
+  class << self
+    # Define methods for a given log level
+    def make_methods(str_level)
+      int_level = ::Logger.const_get(str_level.upcase)
+      method_base = str_level.downcase
+      define_method(method_base, ->(message = nil, &block){add(int_level, message, &block)})
+      define_method("#{method_base}?", ->{level <= int_level})
+      define_method("#{method_base}!", ->{self.level = int_level})
+    end
   end
-  # declare methods for all levels
-  Logger::Severity.constants.each { |severity| make_methods(severity) }
+  # Declare methods for all levels
+  Logger::Severity.constants.each{ |severity| make_methods(severity)}
 end
 
+# Restore warnings
 $VERBOSE = old_verbose
 
 module Aspera
   # Singleton object for logging
   class Log
     include Singleton
-    # where logs are sent to
+
+    # Where logs are sent to
     LOG_TYPES = %i[stderr stdout syslog].freeze
     @@format = :json # rubocop:disable Style/ClassVars
-    # class methods
+    # Class methods
     class << self
       # levels are :debug,:info,:warn,:error,fatal,:unknown
-      def levels; Logger::Severity.constants.sort{|a, b|Logger::Severity.const_get(a) <=> Logger::Severity.const_get(b)}.map{|c|c.downcase.to_sym}; end
+      def levels; Logger::Severity.constants.sort{ |a, b| Logger::Severity.const_get(a) <=> Logger::Severity.const_get(b)}.map{ |c| c.downcase.to_sym}; end
 
       # get the logger object of singleton
       def log; instance.logger; end
@@ -83,7 +79,7 @@ module Aspera
       def capture_stderr
         real_stderr = $stderr
         $stderr = StringIO.new
-        yield
+        yield if block_given?
         log.debug($stderr.string)
       ensure
         $stderr = real_stderr
@@ -93,12 +89,12 @@ module Aspera
     attr_reader :logger_type, :logger
     attr_writer :program_name
 
-    # set log level of underlying logger given symbol level
+    # Set log level of underlying logger given symbol level
     def level=(new_level)
       @logger.level = Logger::Severity.const_get(new_level.to_sym.upcase)
     end
 
-    # get symbol of debug level of underlying logger
+    # Get symbol of debug level of underlying logger
     def level
       Logger::Severity.constants.each do |name|
         return name.downcase.to_sym if @logger.level.eql?(Logger::Severity.const_get(name))
@@ -106,7 +102,7 @@ module Aspera
       Aspera.error_unexpected_value(@logger.level){'log level'}
     end
 
-    # change underlying logger, but keep log level
+    # Change underlying logger, but keep log level
     def logger_type=(new_log_type)
       current_severity_integer = @logger.level unless @logger.nil?
       current_severity_integer = ENV.fetch('AS_LOG_LEVEL', nil) if current_severity_integer.nil? && ENV.key?('AS_LOG_LEVEL')
@@ -131,7 +127,7 @@ module Aspera
       end
       @logger.level = current_severity_integer
       @logger_type = new_log_type
-      # update formatter with password hiding
+      # Update formatter with password hiding
       @logger.formatter = SecretHider.log_formatter(@logger.formatter)
     end
 
@@ -140,9 +136,8 @@ module Aspera
     def initialize
       @logger = nil
       @program_name = 'aspera'
-      # this sets @logger and @logger_type (self needed to call method instead of local var)
+      # This sets @logger and @logger_type (self needed to call method instead of local var)
       self.logger_type = :stderr
-      raise 'error logger shall be defined' if @logger.nil?
     end
   end
 end

data/lib/aspera/nagios.rb

@@ -17,7 +17,7 @@ module Aspera
     # add methods to add nagios error levels, each take component name and message
     LEVELS.each_index do |code|
       name = "#{ADD_PREFIX}#{LEVELS[code]}".to_sym
-      define_method(name){|comp, msg|@data.push({code: code, comp: comp, msg: msg})}
+      define_method(name){ |comp, msg| @data.push({code: code, comp: comp, msg: msg})}
     end
 
     class << self
@@ -28,17 +28,17 @@ module Aspera
         %w[status component message].each do |c|
           Aspera.assert(data.first.key?(c)){"result must have #{c}"}
         end
-        res_errors = data.reject{|s|s['status'].eql?('ok')}
+        res_errors = data.reject{ |s| s['status'].eql?('ok')}
         # keep only errors in case of problem, other ok are assumed so
         data = res_errors unless res_errors.empty?
         # first is most critical
-        data.sort!{|a, b|LEVELS.index(a['status'].to_sym) <=> LEVELS.index(b['status'].to_sym)}
+        data.sort!{ |a, b| LEVELS.index(a['status'].to_sym) <=> LEVELS.index(b['status'].to_sym)}
         # build message: if multiple components: concatenate
         # message = data.map{|i|"#{i['component']}:#{i['message']}"}.join(', ').gsub("\n",' ')
         message = data
-          .map{|i|i['component']}
+          .map{ |i| i['component']}
           .uniq
-          .map{|comp|comp + ':' + data.select{|d|d['component'].eql?(comp)}.map{|d|d['message']}.join(',')}
+          .map{ |comp| comp + ':' + data.select{ |d| d['component'].eql?(comp)}.map{ |d| d['message']}.join(',')}
           .join(', ')
           .tr("\n", ' ')
         status = data.first['status'].upcase
@@ -80,7 +80,7 @@ module Aspera
     # translate for display
     def result
       raise 'missing result' if @data.empty?
-      {type: :object_list, data: @data.map{|i|{'status' => LEVELS[i[:code]].to_s, 'component' => i[:comp], 'message' => i[:msg]}}}
+      {type: :object_list, data: @data.map{ |i| {'status' => LEVELS[i[:code]].to_s, 'component' => i[:comp], 'message' => i[:msg]}}}
     end
   end
 end

data/lib/aspera/node_simulator.rb

@@ -18,7 +18,7 @@ module Aspera
     end
 
     def all_sessions
-      @agent.sessions.map { |session| session[:job_id] }.uniq.each.map{|job_id|job_to_transfer(job_id)}
+      @agent.sessions.map{ |session| session[:job_id]}.uniq.each.map{ |job_id| job_to_transfer(job_id)}
     end
 
     # status: ('waiting', 'partially_completed', 'unknown', 'waiting(read error)',] 'running', 'completed', 'failed'
@@ -185,9 +185,9 @@ module Aspera
           'size'        => folder_stat.size,
           'mtime'       => folder_stat.mtime.utc.iso8601,
           'permissions' => [
-            { 'name' => 'view' },
-            { 'name' => 'edit' },
-            { 'name' => 'delete' }
+            {'name' => 'view'},
+            {'name' => 'edit'},
+            {'name' => 'delete'}
           ]
         },
         'items' => []
@@ -208,9 +208,9 @@ module Aspera
           'size'        => item_stat.size,
           'mtime'       => item_stat.mtime.utc.iso8601,
           'permissions' => [
-            { 'name' => 'view' },
-            { 'name' => 'edit' },
-            { 'name' => 'delete' }
+            {'name' => 'view'},
+            {'name' => 'edit'},
+            {'name' => 'delete'}
           ]
         }
 
@@ -323,7 +323,7 @@ module Aspera
 
     def set_json_response(request, response, json, code: 200)
       response.status = code
-      response['Content-Type'] = 'application/json'
+      response['Content-Type'] = Rest::MIME_JSON
       response.body = json.to_json
       Log.log.trace1{Log.dump("response for #{request.request_method} #{request.path}", json)}
     end

data/lib/aspera/oauth/base.rb

@@ -10,14 +10,15 @@ module Aspera
     # OAuth 2 client for the REST client
     # Generate bearer token
     # Bearer tokens are cached in memory and in a file cache for later re-use
-    # https://tools.ietf.org/html/rfc6749
+    # OAuth 2.0 Authorization Framework: https://tools.ietf.org/html/rfc6749
+    # Bearer Token Usage: https://tools.ietf.org/html/rfc6750
     class Base
       # @param **             Parameters for REST
       # @param client_id      [String, nil]
       # @param client_secret  [String, nil]
       # @param scope          [String, nil]
       # @param use_query      [bool]        Provide parameters in query instead of body
-      # @param path_token     [String]      API end point to create a token
+      # @param path_token     [String]      API end point to create a token from base URL
       # @param token_field    [String]      Field in result that contains the token
       # @param cache_ids      [Array, nil]  List of unique identifiers for cache id generation
       def initialize(
@@ -53,10 +54,12 @@ module Aspera
         @token_cache_id = Factory.cache_id(@api.base_url, self.class, @base_cache_ids, @scope)
       end
 
+      attr_reader :scope
+
       # helper method to create token as per RFC
       def create_token_call(creation_params)
         Log.log.debug{'Generating a new token'.bg_green}
-        payload = { body_type: :www }
+        payload = {content_type: Rest::MIME_WWW}
         if @use_query
           payload[:query] = creation_params
         else
@@ -65,7 +68,7 @@ module Aspera
         return @api.call(
           operation: 'POST',
           subpath:   @path_token,
-          headers:   {'Accept' => 'application/json'},
+          headers:   {'Accept' => Rest::MIME_JSON},
           **payload
         )
       end
@@ -96,12 +99,13 @@ module Aspera
         unless token_info.nil?
           token_data = token_info[:data]
           # Optional optimization:
-          # check if node token is expired based on decoded content then force refresh if close enough
+          # Check if token is expired based on decoded content then force refresh if close enough
           # might help in case the transfer agent cannot refresh himself
           # `direct` agent is equipped with refresh code
           # an API was already called, but failed, we need to regenerate or refresh
           if refresh || token_info[:expired]
-            if token_data.key?('refresh_token') && token_data['refresh_token'].eql?('not_supported')
+            refresh_token = nil
+            if token_data.key?('refresh_token') && !token_data['refresh_token'].eql?('not_supported')
               # save possible refresh token, before deleting the cache
               refresh_token = token_data['refresh_token']
             end

data/lib/aspera/oauth/factory.rb

@@ -9,6 +9,7 @@ module Aspera
     # Factory to create tokens and manage their cache
     class Factory
       include Singleton
+
       # a prefix for persistency of tokens (simplify garbage collect)
       PERSIST_CATEGORY_TOKEN = 'token'
       # prefix for bearer token when in header
@@ -36,9 +37,8 @@ module Aspera
           return IdGenerator.from_list([
             PERSIST_CATEGORY_TOKEN,
             url,
-            Factory.class_to_id(creator_class),
-            *params
-          ].flatten)
+            Factory.class_to_id(creator_class)] +
+            params)
         end
 
         # @return snake version of class name
@@ -111,7 +111,7 @@ module Aspera
         token_data = JSON.parse(token_raw_string)
         Aspera.assert_type(token_data, Hash)
         decoded_token = decode_token(token_data[TOKEN_FIELD])
-        info = { data: token_data }
+        info = {data: token_data}
         Log.log.debug{Log.dump('decoded_token', decoded_token)}
         if decoded_token.is_a?(Hash)
           info[:decoded] = decoded_token
@@ -159,11 +159,11 @@ module Aspera
         Aspera.assert_type(parameters, Hash)
         id = parameters[:grant_method]
         Aspera.assert(@token_type_classes.key?(id)){"token grant method unknown: '#{id}'"}
-        create_parameters = parameters.reject { |k, _v| k.eql?(:grant_method) }
+        create_parameters = parameters.reject{ |k, _v| k.eql?(:grant_method)}
         @token_type_classes[id].new(**create_parameters)
       end
     end
     # JSON Web Signature (JWS) compact serialization: https://datatracker.ietf.org/doc/html/rfc7515
-    Factory.instance.register_decoder(lambda { |token| parts = token.split('.'); Aspera.assert(parts.length.eql?(3)){'not JWS token'}; JSON.parse(Base64.decode64(parts[1]))}) # rubocop:disable Style/Semicolon, Layout/LineLength
+    Factory.instance.register_decoder(lambda{ |token| parts = token.split('.'); Aspera.assert(parts.length.eql?(3)){'not JWS token'}; JSON.parse(Base64.decode64(parts[1]))}) # rubocop:disable Style/Semicolon, Layout/LineLength
   end
 end

data/lib/aspera/oauth/url_json.rb

@@ -20,12 +20,12 @@ module Aspera
 
       def create_token
         @api.call(
-          operation:   'POST',
-          subpath:     @path_token,
-          headers:     {'Accept' => 'application/json'},
-          query:       @query.merge(scope: @scope), # scope is here because it may change over time (node)
-          body:        @body,
-          body_type:   :json
+          operation:    'POST',
+          subpath:      @path_token,
+          query:        @query.merge(scope: @scope), # scope is here because it may change over time (node)
+          content_type: Rest::MIME_JSON,
+          body:         @body,
+          headers:      {'Accept' => Rest::MIME_JSON}
         )
       end
     end

data/lib/aspera/persistency_action_once.rb

@@ -8,9 +8,9 @@ module Aspera
   # Persist data on file system
   class PersistencyActionOnce
     DELETE_DEFAULT = lambda(&:empty?)
-    PARSE_DEFAULT = lambda {|t| JSON.parse(t)}
-    FORMAT_DEFAULT = lambda {|h| JSON.generate(h)}
-    MERGE_DEFAULT = lambda {|current, file| current.concat(file).uniq rescue current}
+    PARSE_DEFAULT = lambda{ |t| JSON.parse(t)}
+    FORMAT_DEFAULT = lambda{ |h| JSON.generate(h)}
+    MERGE_DEFAULT = lambda{ |current, file| current.concat(file).uniq rescue current}
     MANAGER_METHODS = %i[get put delete]
     private_constant :DELETE_DEFAULT, :PARSE_DEFAULT, :FORMAT_DEFAULT, :MERGE_DEFAULT, :MANAGER_METHODS
 
@@ -22,7 +22,7 @@ module Aspera
     # @param :format   Optional  dump method (default to JSON)
     # @param :merge    Optional  merge data from file to current data
     def initialize(manager:, data:, id:, delete: DELETE_DEFAULT, parse: PARSE_DEFAULT, format: FORMAT_DEFAULT, merge: MERGE_DEFAULT)
-      Aspera.assert(MANAGER_METHODS.all?{|i|manager.respond_to?(i)}){"Manager must answer to #{MANAGER_METHODS}"}
+      Aspera.assert(MANAGER_METHODS.all?{ |i| manager.respond_to?(i)}){"Manager must answer to #{MANAGER_METHODS}"}
       Aspera.assert(!data.nil?)
       Aspera.assert_type(id, String)
       Aspera.assert(!id.empty?)
@@ -39,6 +39,7 @@ module Aspera
       merge.call(@persisted_object, parse.call(value)) unless value.nil?
     end
 
+    # Save persisted object on storage
     def save
       if @delete_condition.call(@persisted_object)
         @manager.delete(@object_id)
@@ -47,6 +48,7 @@ module Aspera
       end
     end
 
+    # @return internal persisted object, in order to modify its content
     def data
       return @persisted_object
     end

data/lib/aspera/persistency_folder.rb

@@ -64,7 +64,7 @@ module Aspera
       garbage_files = current_files(persist_category)
       if !max_age_seconds.nil?
         current_time = Time.now
-        garbage_files.select! { |filepath| (current_time - File.stat(filepath).mtime).to_i > max_age_seconds}
+        garbage_files.select!{ |filepath| (current_time - File.stat(filepath).mtime).to_i > max_age_seconds}
       end
       garbage_files.each do |filepath|
         File.delete(filepath)
@@ -79,7 +79,7 @@ module Aspera
     end
 
     def current_items(persist_category)
-      current_files(persist_category).each_with_object({}) {|i, h| h[File.basename(i, FILE_SUFFIX)] = File.read(i)}
+      current_files(persist_category).each_with_object({}){ |i, h| h[File.basename(i, FILE_SUFFIX)] = File.read(i)}
     end
 
     private