aspera-cli 4.25.0.pre → 4.25.0

data/lib/aspera/dot_container.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require 'aspera/assert'
+
+module Aspera
+  # Convert dotted-path to/from nested Hash/Array container
+  class DotContainer
+    class << self
+      # Insert extended value `value` into struct `result` at `path`
+      # @param path   [String] Dotted path in container
+      # @param value  [String] Last value to insert
+      # @param result [NilClass, Hash, Array] current value
+      # @return [Hash, Array]
+      def dotted_to_container(path, value, result = nil)
+        # Typed keys
+        keys = path.split(OPTION_DOTTED_SEPARATOR).map{ |k| int_or_string(k)}
+        # Create, or re-use first level container
+        current = (result ||= new_hash_or_array_from_key(keys.first))
+        # walk the path, and create sub-containers if necessary
+        keys.each_cons(2) do |k, next_k|
+          array_requires_integer_index!(current, k)
+          current = (current[k] ||= new_hash_or_array_from_key(next_k))
+        end
+        # Assign value at last index
+        array_requires_integer_index!(current, keys.last)
+        current[keys.last] = value
+        result
+      end
+
+      private
+
+      # Convert `String` to `Integer`, or keep `String` if not `Integer`
+      def int_or_string(value)
+        Integer(value, exception: false) || value
+      end
+
+      # Assert that if `container` is an `Array`, then `index` is an `Integer`
+      # @param container [Hash, Array]
+      # @param index     [String, Integer]
+      def array_requires_integer_index!(container, index)
+        Aspera.assert(container.is_a?(Hash) || index.is_a?(Integer)){'Using String index when Integer index used previously'}
+      end
+
+      # Create a new `Hash` or `Array` depending on type of `key`
+      def new_hash_or_array_from_key(key)
+        key.is_a?(Integer) ? [] : {}
+      end
+    end
+
+    # @param [Hash,Array] Container object
+    def initialize(container)
+      Aspera.assert_type(container, Hash)
+      # tail (pop,push) contains the next element to display
+      # elements are [path, value]
+      @stack = container.empty? ? [] : [[[], container]]
+    end
+
+    # Convert nested Hash/Array container to dotted-path Hash
+    # @return [Hash] Dotted-path Hash
+    def to_dotted
+      result = {}
+      until @stack.empty?
+        path, current = @stack.pop
+        to_insert = nil
+        # empty things are left intact
+        if current.respond_to?(:empty?) && current.empty?
+          to_insert = current
+        else
+          case current
+          when Hash
+            add_elements(path, current)
+          when Array
+            # Array has no nested structures -> list of Strings
+            if current.none?{ |i| i.is_a?(Array) || i.is_a?(Hash)}
+              to_insert = current.map(&:to_s)
+            # Array of Hashes with only 'name' keys -> list of Strings
+            elsif current.all?{ |i| i.is_a?(Hash) && i.keys == ['name']}
+              to_insert = current.map{ |i| i['name']}
+            # Array of Hashes with only 'name' and 'value' keys -> Hash of key/values
+            elsif current.all?{ |i| i.is_a?(Hash) && i.keys.sort == %w[name value]}
+              add_elements(path, current.each_with_object({}){ |i, h| h[i['name']] = i['value']})
+            else
+              add_elements(path, current.each_with_index.map{ |v, i| [i, v]})
+            end
+          else
+            to_insert = current
+          end
+        end
+        result[path.map(&:to_s).join(OPTION_DOTTED_SEPARATOR)] = to_insert unless to_insert.nil?
+      end
+      result
+    end
+
+    private
+
+    # Add elements of enumerator to the @stack, in reverse order
+    def add_elements(path, enum)
+      enum.reverse_each do |key, value|
+        @stack.push([path + [key], value])
+      end
+      nil
+    end
+
+    # "."
+    OPTION_DOTTED_SEPARATOR = '.'
+    private_constant :OPTION_DOTTED_SEPARATOR
+  end
+end

data/lib/aspera/environment.rb

@@ -42,105 +42,85 @@ module Aspera
 
     RB_EXT = '.rb'
 
+    PROCESS_MODES = %i[execute background capture].freeze
+
     class << self
       def ruby_version
         return RbConfig::CONFIG['RUBY_PROGRAM_VERSION']
       end
 
-      # empty variable binding for secure eval
+      # Empty variable binding for secure eval
       def empty_binding
         return Kernel.binding
       end
 
-      # secure execution of Ruby code
+      # Secure execution of Ruby code
+      # @param code [String] Ruby code to execute
+      # @param file [String] File name for error reporting
+      # @param line [Integer] Line number for error reporting
       def secure_eval(code, file, line)
         Kernel.send('lave'.reverse, code, empty_binding, file, line)
       end
 
-      # Generate log line for external program with arguments
-      # @param exec [String]     Path to executable
-      # @param args [Array, nil] Arguments
-      # @param env [Hash, nil]   Environment variables
-      # @return [String] log line with environment, program and arguments
-      def log_spawn(exec:, args: nil, env: nil)
-        [
-          'execute:'.red,
-          env&.map{ |k, v| "#{k}=#{Shellwords.shellescape(v)}"},
-          Shellwords.shellescape(exec),
-          args&.map{ |a| Shellwords.shellescape(a)}
-        ].compact.flatten.join(' ')
-      end
-
-      # Start process in background
-      # caller can call Process.wait on returned value
-      # @param exec    [String]     Path to executable
-      # @param args    [Array, nil] Arguments for executable
-      # @param env     [Hash, nil]  Environment variables
-      # @param kwargs  [Hash]       Options for `Process.spawn`
-      # @return [String]            PID of process
-      # @raise  [Exception]         if problem
-      def secure_spawn(exec:, args: nil, env: nil, **kwargs)
-        Aspera.assert_type(exec, String)
-        Aspera.assert_type(args, Array, NilClass)
-        Aspera.assert_type(env, Hash, NilClass)
-        Log.log.debug{log_spawn(exec: exec, args: args, env: env)}
-        spawn_args = []
-        spawn_args.push(env) unless env.nil?
-        spawn_args.push([exec, exec])
-        spawn_args.concat(args) unless args.nil?
-        kwargs[:close_others] = true unless kwargs.key?(:close_others)
-        # Start separate process in background
-        pid = Process.spawn(*spawn_args, **kwargs)
-        Log.dump(:pid, pid)
-        return pid
+      # Build argv for Process.spawn / Kernel.system (no shell)
+      # @param cmd    [Array]  Command and arguments
+      # @param kwargs [Hash]   Additional arguments to `secure_execute`
+      def build_spawn_argv(cmd, kwargs)
+        env = kwargs.delete(:env)
+        argv = []
+        argv << env if env
+        argv << [cmd.first, cmd.first] # no shell, preserve argv[0]
+        argv.concat(cmd.drop(1))
+        argv
       end
 
-      # Start process (not in shell) and wait for completion.
-      # By default, sets `exception: true` in `kwargs`
-      # @param exec   [String]     Path to executable
-      # @param args   [Array, nil] Arguments
-      # @param env    [Hash, nil]  Environment variables
-      # @param kwargs [Hash]       Arguments for `Kernel.system`
-      # @return `nil`
-      # @raise [RuntimeError] if problem
-      def secure_execute(exec:, args: nil, env: nil, **kwargs)
-        Aspera.assert_type(exec, String)
-        Aspera.assert_type(args, Array, NilClass)
-        Aspera.assert_type(env, Hash, NilClass)
-        Log.log.debug{log_spawn(exec: exec, args: args, env: env)}
-        Log.dump(:kwargs, kwargs, level: :trace1)
-        spawn_args = []
-        spawn_args.push(env) unless env.nil?
-        # Ensure no shell expansion
-        spawn_args.push([exec, exec])
-        spawn_args.concat(args) unless args.nil?
-        # By default: exception on error
-        kwargs[:exception] = true unless kwargs.key?(:exception)
-        # Start in separate process
-        Kernel.system(*spawn_args, **kwargs)
-        nil
-      end
-
-      # Execute process and capture stdout
-      # @param exec   [String] path to executable
-      # @param args   [Array]  arguments to executable
-      # @param kwargs [Hash]   options to Open3.capture3
-      # @return stdout of executable or raise exception
-      def secure_capture(exec:, args: [], env: nil, exception: true, **kwargs)
-        Aspera.assert_type(exec, String)
-        Aspera.assert_type(args, Array)
-        Log.log.debug{log_spawn(exec: exec, args: args, env: env)}
-        Log.dump(:kwargs, kwargs, level: :trace2)
-        # Log.dump(:ENV, ENV.to_h, level: :trace1)
-        capture_args = []
-        capture_args.push(env) unless env.nil?
-        capture_args.push(exec)
-        capture_args.concat(args)
-        stdout, stderr, status = Open3.capture3(*capture_args, **kwargs)
-        Log.log.debug{"status=#{status}, stderr=#{stderr}"}
-        Log.log.trace1{"stdout=#{stdout}"}
-        raise "process failed: #{status.exitstatus} (#{stderr})" if !status.success? && exception
-        return stdout
+      # Execute a process securely (no shell)
+      # mode:
+      #   :execute    -> Kernel.system, return nil
+      #   :background -> Process.spawn, return pid
+      #   :capture    -> Open3.capture3, return stdout
+      # @param cmd       [Array]     Command and arguments
+      # @param env       [Hash, nil] Environment variables
+      # @param mode      [Symbol]    Execution mode (see above)
+      # @param kwargs    [Hash]      Additional arguments to underlying method, includes:
+      #                              :exception [Boolean] for :capture mode, raise error if process fails
+      #                              :close_others [Boolean] for :background mode
+      #                              :env [Hash] for :execute mode
+      def secure_execute(*cmd, mode: :execute, **kwargs)
+        cmd = cmd.map(&:to_s)
+        Aspera.assert(cmd.size.positive?, type: ArgumentError){'executable must be present'}
+        Aspera.assert_values(mode, PROCESS_MODES, type: ArgumentError){'mode'}
+        Log.log.debug do
+          parts = [mode.to_s, 'command:']
+          kwargs[:env]&.each{ |k, v| parts << "#{k}=#{Shellwords.shellescape(v.to_s)}"}
+          cmd.each{ |a| parts << Shellwords.shellescape(a)}
+          parts.join(' ')
+        end
+        case mode
+        when :execute
+          # https://docs.ruby-lang.org/en/master/Kernel.html#method-i-system
+          # https://docs.ruby-lang.org/en/master/Process.html#module-Process-label-Execution+Options
+          kwargs[:exception] = true unless kwargs.key?(:exception)
+          Kernel.system(*build_spawn_argv(cmd, kwargs), **kwargs)
+        when :background
+          # https://docs.ruby-lang.org/en/master/Process.html#method-c-spawn
+          # https://docs.ruby-lang.org/en/master/Process.html#module-Process-label-Execution+Options
+          kwargs[:close_others] = true unless kwargs.key?(:close_others)
+          pid = Process.spawn(*build_spawn_argv(cmd, kwargs), **kwargs)
+          Log.dump(:pid, pid)
+          pid
+        when :capture
+          # https://docs.ruby-lang.org/en/master/Open3.html#method-c-capture3
+          # https://docs.ruby-lang.org/en/master/Process.html#module-Process-label-Execution+Options
+          argv = [kwargs.delete(:env)].compact + cmd
+          exception = kwargs.delete(:exception){true}
+          stdout, stderr, status = Open3.capture3(*argv, **kwargs)
+          Log.log.debug{"status=#{status}, stderr=#{stderr}"}
+          Log.log.trace1{"stdout=#{stdout}"}
+          raise "Process failed: #{status.exitstatus} (#{stderr})" if exception && !status.success?
+          stdout
+        else Aspera.error_unreachable_line
+        end
       end
 
       # Write content to a file, with restricted access
@@ -273,9 +253,9 @@ module Aspera
     # @param uri [String] the URI to open
     def open_uri_graphical(uri)
       case @os
-      when Environment::OS_MACOS then return self.class.secure_execute(exec: 'open', args: [uri.to_s])
-      when Environment::OS_WINDOWS then return self.class.secure_execute(exec: 'start', args: ['explorer', %Q{"#{uri}"}])
-      when Environment::OS_LINUX   then return self.class.secure_execute(exec: 'xdg-open', args: [uri.to_s])
+      when Environment::OS_MACOS then return self.class.secure_execute('open', uri.to_s)
+      when Environment::OS_WINDOWS then return self.class.secure_execute('start', 'explorer', %Q{"#{uri}"})
+      when Environment::OS_LINUX   then return self.class.secure_execute('xdg-open', uri.to_s)
       else Assert.error_unexpected_value(os){'no graphical open method'}
       end
     end
@@ -283,9 +263,9 @@ module Aspera
     # open a file in an editor
     def open_editor(file_path)
       if ENV.key?('EDITOR')
-        self.class.secure_execute(exec: ENV['EDITOR'], args: [file_path.to_s])
+        self.class.secure_execute(ENV['EDITOR'], file_path.to_s)
       elsif @os.eql?(Environment::OS_WINDOWS)
-        self.class.secure_execute(exec: 'notepad.exe', args: [%Q{"#{file_path}"}])
+        self.class.secure_execute('notepad.exe', %Q{"#{file_path}"})
       else
         open_uri_graphical(file_path.to_s)
       end

data/lib/aspera/faspex_postproc.rb

@@ -50,7 +50,7 @@ module Aspera
         Log.dump(:webhook_parameters, webhook_parameters)
         # env expects only strings
         environment = webhook_parameters.each_with_object({}){ |(k, v), h| h[k] = v.to_s}
-        post_proc_pid = Environment.secure_spawn(env: environment, exec: script_path)
+        post_proc_pid = Environment.secure_execute(script_path, mode: :background, env: environment)
         Timeout.timeout(@parameters[:timeout_seconds]) do
           # "wait" for process to avoid zombie
           Process.wait(post_proc_pid)

data/lib/aspera/id_generator.rb

@@ -9,19 +9,16 @@ module Aspera
     class << self
       # Generate an ID from a list of object IDs
       # The generated ID is safe as file name
-      # @param object_id [Array<String>, String] the object IDs
+      # @param ids [Array] the object IDs (can be nested, will be flattened, and nils removed)
       # @return [String] the generated ID
-      def from_list(object_id)
+      def from_list(*ids)
         safe_char = Environment.instance.safe_filename_character
-        if object_id.is_a?(Array)
-          # compact: remove nils
-          object_id = object_id.flatten.compact.map do |i|
-            i.is_a?(String) && i.start_with?('https://') ? URI.parse(i).host : i.to_s
-          end.join(safe_char)
-        end
-        Aspera.assert_type(object_id, String)
+        # compact: remove nils
+        id = ids.flatten.compact.map do |i|
+          i.is_a?(String) && i.start_with?('https://') ? URI.parse(i).host : i.to_s
+        end.join(safe_char)
         # keep dot for extension only (nicer)
-        return Environment.instance.sanitized_filename(object_id.gsub('.', safe_char)).downcase
+        return Environment.instance.sanitized_filename(id.gsub('.', safe_char)).downcase
       end
     end
   end

data/lib/aspera/keychain/macos_security.rb

@@ -49,7 +49,7 @@ module Aspera
               options[:path] = uri.path unless ['', '/'].include?(uri.path)
               options[:port] = uri.port unless uri.port.eql?(443) && !url.include?(':443/')
             end
-            command_args = [command]
+            command_args = [SECURITY_UTILITY, command]
             options&.each do |k, v|
               Aspera.assert(supported.key?(k)){"unknown option: #{k}"}
               next if v.nil?
@@ -57,7 +57,7 @@ module Aspera
               command_args.push(v.shellescape) unless v.empty?
             end
             command_args.push(last_opt) unless last_opt.nil?
-            return Environment.secure_capture(exec: SECURITY_UTILITY, args: command_args)
+            return Environment.secure_execute(*command_args, mode: :capture)
           end
 
           def key_chains(output)

data/lib/aspera/log.rb

@@ -81,8 +81,9 @@ module Aspera
       # @param object [Hash, nil]       Data to dump
       # @param level  [Symbol]          Debug level
       # @param block  [Proc, nil]       Give computed object
-      def dump(name, object = nil, level: :debug)
+      def dump(name, object = nil, level: :debug, &block)
         return unless instance.logger.send(:"#{level}?")
+        Aspera.assert(object.nil? || block.nil?){'Use either object, or block, not both'}
         object = yield if block_given?
         instance.logger.send(level, obj_dump(name, object))
       end

data/lib/aspera/oauth/base.rb

@@ -14,50 +14,38 @@ module Aspera
     # Bearer Token Usage: https://tools.ietf.org/html/rfc6750
     class Base
       Aspera.require_method!(:create_token)
-      # @param **             Parameters for REST
-      # @param client_id      [String, nil]
-      # @param client_secret  [String, nil]
-      # @param scope          [String, nil]
-      # @param use_query      [bool]        Provide parameters in query instead of body
-      # @param path_token     [String]      API end point to create a token from base URL
-      # @param token_field    [String]      Field in result that contains the token
-      # @param cache_ids      [Array, nil]  List of unique identifiers for cache id generation
+      # @param params         [Hash]    Parameters for token creation (client_id, client_secret, scope, etc...)
+      # @param use_query      [Boolean] Provide parameters in query instead of body
+      # @param path_token     [String]  API end point to create a token from base URL
+      # @param token_field    [String]  Field in result that contains the token
+      # @param cache_ids      [Array]   List of unique identifiers for cache id generation
+      # @param **rest_params  [Hash]    Parameters for REST
       def initialize(
-        client_id: nil,
-        client_secret: nil,
-        scope: nil,
+        params: {},
         use_query: false,
         path_token:  'token',
         token_field: Factory::TOKEN_FIELD,
-        cache_ids: nil,
+        cache_ids: [],
         **rest_params
       )
+        Aspera.assert_type(params, Hash)
+        Aspera.assert_type(cache_ids, Array)
         # This is the OAuth API
         @api = Rest.new(**rest_params)
-        @scope = nil
-        @token_cache_id = nil
+        @params = params.dup.freeze
         @path_token = path_token
         @token_field = token_field
-        @client_id = client_id
-        @client_secret = client_secret
         @use_query = use_query
-        @base_cache_ids = cache_ids.nil? ? [] : cache_ids.clone
-        Aspera.assert_type(@base_cache_ids, Array)
-        # TODO: this shall be done in class, using cache_ids
-        @base_cache_ids.push(@api.auth_params[:username]) if @api.auth_params.key?(:username)
-        @base_cache_ids.compact!
-        @base_cache_ids.freeze
-        self.scope = scope
+        # TODO: :username and :scope shall be done in class, using cache_ids
+        @token_cache_id = Factory.cache_id(@api.base_url, self.class, cache_ids, rest_params[:username], @params[:scope])
       end
 
-      # Scope can be modified after creation, then update identifier for cache
-      def scope=(scope)
-        @scope = scope
-        # generate token unique identifier for persistency (memory/disk cache)
-        @token_cache_id = Factory.cache_id(@api.base_url, self.class, @base_cache_ids, @scope)
-      end
-
-      attr_reader :scope, :api, :path_token, :client_id
+      # The OAuth API Object
+      attr_reader :api
+      # Sub path to generate token
+      attr_reader :path_token
+      # Parameters to generate token
+      attr_reader :params
 
       # Helper method to create token as per RFC
       # @return [HTTPResponse]
@@ -68,17 +56,16 @@ module Aspera
         return @api.create(@path_token, creation_params, content_type: Rest::MIME_WWW, ret: :resp)
       end
 
+      # Create base parameters for token creation calls
       # @param add_secret [Boolean] Add secret in default call parameters
       # @return [Hash] Optional general parameters
-      def optional_scope_client_id(add_secret: false)
-        call_params = {}
-        call_params[:scope] = @scope unless @scope.nil?
-        call_params[:client_id] = @client_id unless @client_id.nil?
-        call_params[:client_secret] = @client_secret unless !add_secret || @client_id.nil? || @client_secret.nil?
+      def base_params(add_secret: false)
+        call_params = @params.dup
+        call_params.delete(:client_secret) unless add_secret
         return call_params
       end
 
-      # @return value suitable for Authorization header
+      # @return [String] value suitable for Authorization header
       def authorization(**kwargs)
         return OAuth::Factory.bearer_authorization(token(**kwargs))
       end
@@ -114,7 +101,7 @@ module Aspera
             if !refresh_token.nil?
               Log.log.info{"refresh token=[#{refresh_token}]".bg_green}
               begin
-                http = create_token_call(optional_scope_client_id(add_secret: true).merge(grant_type: 'refresh_token', refresh_token: refresh_token))
+                http = create_token_call(base_params(add_secret: true).merge(grant_type: 'refresh_token', refresh_token: refresh_token))
                 # Save only if success
                 json_data = http.body
                 token_data = JSON.parse(json_data)

data/lib/aspera/oauth/factory.rb

@@ -36,14 +36,13 @@ module Aspera
           return authorization[SPACE_BEARER_AUTH_SCHEME.length..-1]
         end
 
+        # Generate a unique cache id for a token creator
+        # @param url           [String] Base URL of the OAuth server
+        # @param creator_class [Class]  Class of the token creator
+        # @param params        [Array]  List of parameters (can be nested) to uniquely identify the token
         # @return a unique cache identifier
         def cache_id(url, creator_class, *params)
-          return IdGenerator.from_list([
-            PERSIST_CATEGORY_TOKEN,
-            url,
-            Factory.class_to_id(creator_class)
-          ] +
-            params)
+          return IdGenerator.from_list(PERSIST_CATEGORY_TOKEN, url, Factory.class_to_id(creator_class), params)
         end
 
         # @return snake version of class name

data/lib/aspera/oauth/generic.rb

@@ -23,7 +23,7 @@ module Aspera
       end
 
       def create_token
-        return create_token_call(optional_scope_client_id.merge(@create_params))
+        return create_token_call(base_params.merge(@create_params))
       end
     end
     Factory.instance.register_token_creator(Generic)

data/lib/aspera/oauth/jwt.rb

@@ -61,7 +61,7 @@ module Aspera
         Log.log.debug{"private=[#{@private_key_obj}]"}
         assertion = JWT.encode(jwt_payload, @private_key_obj, 'RS256', @headers)
         Log.log.debug{"assertion=[#{assertion}]"}
-        return create_token_call(optional_scope_client_id.merge(grant_type: GRANT_TYPE, assertion: assertion))
+        return create_token_call(base_params.merge(grant_type: GRANT_TYPE, assertion: assertion))
       end
     end
     Factory.instance.register_token_creator(Jwt)

data/lib/aspera/oauth/url_json.rb

@@ -6,8 +6,9 @@ module Aspera
   module OAuth
     # This class is used to create a token using a JSON body and a URL
     class UrlJson < Base
-      # @param url  URL to send the JSON body
-      # @param json JSON body to send
+      # @param url  [Hash] Query parameters to send
+      # @param json [Hash] Body parameters to send as JSON
+      # @param generic_params [Hash] Generic parameters for OAuth::Base
       def initialize(
         url:,
         json:,
@@ -22,7 +23,7 @@ module Aspera
         api.call(
           operation:    'POST',
           subpath:      path_token,
-          query:        @query.merge(scope: scope), # scope is here because it may change over time (node)
+          query:        @query.merge(scope: params[:scope]), # scope is here because it may change over time (node)
           content_type: Rest::MIME_JSON,
           body:         @body,
           headers:      {'Accept' => Rest::MIME_JSON},

data/lib/aspera/oauth/web.rb

@@ -32,7 +32,7 @@ module Aspera
         random_state = SecureRandom.uuid
         login_page_url = Rest.build_uri(
           "#{api.base_url}/#{@path_authorize}",
-          optional_scope_client_id.merge(response_type: 'code', redirect_uri: @redirect_uri, state: random_state)
+          base_params.merge(response_type: 'code', redirect_uri: @redirect_uri, state: random_state)
         )
         # here, we need a human to authorize on a web page
         Log.log.info{"login_page_url=#{login_page_url}".bg_red.gray}
@@ -44,7 +44,7 @@ module Aspera
         received_params = web_server.received_request
         Aspera.assert(random_state.eql?(received_params['state'])){'wrong received state'}
         # exchange code for token
-        return create_token_call(optional_scope_client_id(add_secret: true).merge(
+        return create_token_call(base_params(add_secret: true).merge(
           grant_type:   'authorization_code',
           code:         received_params['code'],
           redirect_uri: @redirect_uri

data/lib/aspera/preview/file_types.rb

@@ -53,7 +53,7 @@ module Aspera
 
       private_constant :SUPPORTED_MIME_TYPES
 
-      # @attr use_mimemagic [bool] true to use mimemagic to determine real mime type based on file content
+      # @attr use_mimemagic [Boolean] `true` to use mimemagic to determine real mime type based on file content
       attr_accessor :use_mimemagic
 
       def initialize