aspera-cli 4.24.1 → 4.25.0.pre

data/lib/aspera/cli/manager.rb

@@ -11,32 +11,128 @@ require 'optparse'
 
 module Aspera
   module Cli
-    # option is retrieved from another object using accessor
-    class AttrAccessor
-      # attr_accessor :object
-      # attr_accessor :method_name
-      def initialize(object, method_name, option_name)
-        @object = object
-        @method = method_name
-        @option_name = option_name
-        @has_writer = @object.respond_to?(writer_method)
-        Log.log.trace1{"AttrAccessor: #{@option_name}: #{@object.class}.#{@method}: writer=#{@has_writer}"}
-        Aspera.assert(@object.respond_to?(@method)){"#{object} does not respond to #{method_name}"}
+    # Constants to be used as parameter `allowed:` for `OptionValue`
+    module Allowed
+      # This option can be set to a single string or array, multiple times, and gives Array of String
+      TYPES_STRING_ARRAY = [Array, String].freeze
+      # A list of symbols with constrained values
+      TYPES_SYMBOL_ARRAY = [Array, Symbol].freeze
+      # Value will be coerced to int
+      TYPES_INTEGER = [Integer].freeze
+      TYPES_BOOLEAN = [FalseClass, TrueClass].freeze
+      # no value at all, it's a switch
+      TYPES_NONE = [].freeze
+      TYPES_ENUM = [Symbol].freeze
+      TYPES_STRING = [String].freeze
+    end
+
+    # Description of option, how to manage
+    class OptionValue
+      # [Array(Class)] List of allowed types
+      attr_reader :types, :sensitive
+      # [Array] List of allowed values (Symbols and specific values)
+      attr_accessor :values
+
+      # @param option      [Symbol] Name of option
+      # @param allowed  [see below] Allowed values
+      # @param handler       [Hash] Accessor: keys: :o(object) and :m(method)
+      # @param deprecation [String] Deprecation message
+      # `allowed`:
+      # - `nil` No validation, so just a string
+      # - `Class` The single allowed Class
+      # - `Array<Class>` Multiple allowed classes
+      # - `Array<Symbol>` List of allowed values
+      def initialize(option:, description:, allowed: Allowed::TYPES_STRING, handler: nil, deprecation: nil)
+        Log.log.trace1{"option: #{option}, allowed: #{allowed}"}
+        @option = option
+        @description = description
+        # by default passwords and secrets are sensitive, else specify when declaring the option
+        @sensitive = SecretHider.instance.secret?(@option, '')
+        # either the value, or object giving value
+        @object = handler&.[](:o)
+        @read_method = handler&.[](:m)
+        @write_method = @read_method ? "#{@read_method}=".to_sym : nil
+        @deprecation = deprecation
+        @access = if @object.nil?
+          :local
+        elsif @object.respond_to?(@write_method)
+          :write
+        else
+          :setter
+        end
+        Aspera.assert(@object.respond_to?(@read_method)){"#{@object} does not respond to #{method}"} unless @access.eql?(:local)
+        @types = nil
+        @values = nil
+        if !allowed.nil?
+          allowed = [allowed] if allowed.is_a?(Class)
+          Aspera.assert_type(allowed, Array)
+          if allowed.take(Allowed::TYPES_SYMBOL_ARRAY.length) == Allowed::TYPES_SYMBOL_ARRAY
+            # Special case: array of defined symbol values
+            @types = Allowed::TYPES_SYMBOL_ARRAY
+            @values = allowed[Allowed::TYPES_SYMBOL_ARRAY.length..-1]
+          elsif allowed.all?(Class)
+            @types = allowed
+            @values = Manager::BOOLEAN_VALUES if allowed.eql?(Allowed::TYPES_BOOLEAN)
+            # Default value for array
+            @object ||= [] if @types.first.eql?(Array) && !@types.include?(NilClass)
+            @object ||= {} if @types.first.eql?(Hash) && !@types.include?(NilClass)
+          elsif allowed.all?(Symbol)
+            @types = Allowed::TYPES_ENUM
+            @values = allowed
+          else
+            Aspera.error_unexpected_value(allowed)
+          end
+        end
+        Log.log.trace1{"declare: #{@option}: #{@access} #{@object.class}.#{@read_method}".green}
       end
 
-      def value
-        return @object.send(@method) if @has_writer
-        return @object.send(@method, @option_name, :get)
+      def clear
+        @object = nil
       end
 
-      def value=(val)
-        Log.log.trace1{"AttrAccessor: = #{@method} #{@option_name} :set #{val}, writer=#{@has_writer}"}
-        return @object.send(writer_method, val) if @has_writer
-        return @object.send(@method, @option_name, :set, val)
+      def value(log: true)
+        current_value =
+          case @access
+          when :local then @object
+          when :write then @object.send(@read_method)
+          when :setter then @object.send(@read_method, @option, :get)
+          end
+        Log.log.trace1{"#{@option} -> (#{current_value.class})#{current_value}"} if log
+        return current_value
       end
 
-      def writer_method
-        return "#{@method}="
+      # Assign value to option.
+      # Value can be a String, then evaluated with ExtendedValue, or directly a value.
+      # @param value [String, Object] Value to assign to option
+      def assign_value(value, where:)
+        Aspera.assert(!@deprecation, type: warn){"Option #{@option} is deprecated: #{@deprecation}"}
+        new_value = ExtendedValue.instance.evaluate(value, context: "option: #{@option}", allowed: @types)
+        Log.log.trace1{"#{where}: #{@option} <- (#{new_value.class})#{new_value}"}
+        new_value = Manager.enum_to_bool(new_value) if @types.eql?(Allowed::TYPES_BOOLEAN)
+        new_value = Integer(new_value) if @types.eql?(Allowed::TYPES_INTEGER)
+        new_value = [new_value] if @types.eql?(Allowed::TYPES_STRING_ARRAY) && new_value.is_a?(String)
+        # Setting a Hash to null set an empty hash
+        new_value = {} if new_value.eql?(nil) && @types&.first.eql?(Hash)
+        # Setting a Array to null set an empty hash
+        new_value = [] if new_value.eql?(nil) && @types&.first.eql?(Array)
+        if @types.eql?(Aspera::Cli::Allowed::TYPES_SYMBOL_ARRAY)
+          new_value = [new_value] if new_value.is_a?(String)
+          Aspera.assert_type(new_value, Array, type: BadArgument)
+          Aspera.assert_array_all(new_value, String, type: BadArgument)
+          new_value = new_value.map{ |v| Manager.get_from_list(v, @option, @values)}
+        end
+        Aspera.assert_type(new_value, *@types, type: BadArgument){"Option #{@option}"} if @types
+        if new_value.is_a?(Hash) || new_value.is_a?(Array)
+          current_value = value(log: false)
+          new_value = current_value.deep_merge(new_value) if new_value.is_a?(Hash) && current_value.is_a?(Hash) && !current_value.empty?
+          new_value = current_value + new_value if new_value.is_a?(Array) && current_value.is_a?(Array) && !current_value.empty?
+        end
+        case @access
+        when :local then @object = new_value
+        when :write then @object.send(@write_method, new_value)
+        when :setter then @object.send(@read_method, @option, :set, new_value)
+        end
+        Log.log.trace1{v = value(log: false); "#{@option} <- (#{v.class})#{v}"} # rubocop:disable Style/Semicolon
       end
     end
 
@@ -44,49 +140,29 @@ module Aspera
     # arguments options start with '-', others are commands
     # resolves on extended value syntax
     class Manager
-      # boolean options are set to true/false from the following values
       BOOLEAN_SIMPLE = %i[no yes].freeze
-      FALSE_VALUES = [BOOLEAN_SIMPLE.first, false].freeze
-      TRUE_VALUES = [BOOLEAN_SIMPLE.last, true].freeze
-      BOOLEAN_VALUES = (TRUE_VALUES + FALSE_VALUES).freeze
-
-      # option name separator on command line
-      OPTION_SEP_LINE = '-'
-      # option name separator in code (symbol)
-      OPTION_SEP_SYMBOL = '_'
-      OPTION_VALUE_SEPARATOR = '='
-      # an option like --a.b.c=d does: a={"b":{"c":ext_val(d)}}
-      # TODO: all Hash are additive, + way to reset Hash (e.g. --opt=@none:)
-      OPTION_HASH_SEPARATOR = '.'
-      # starts an option
-      OPTION_PREFIX = '--'
-      # when this is alone, this stops option processing
-      OPTIONS_STOP = '--'
-      SOURCE_USER = 'cmdline' # cspell:disable-line
-
-      DEFAULT_PARSER_TYPES = [Array, Hash].freeze
-
-      private_constant :FALSE_VALUES, :TRUE_VALUES, :BOOLEAN_VALUES, :OPTION_SEP_LINE, :OPTION_SEP_SYMBOL, :OPTION_VALUE_SEPARATOR, :OPTION_HASH_SEPARATOR, :OPTION_PREFIX, :OPTIONS_STOP, :SOURCE_USER, :DEFAULT_PARSER_TYPES
-
       class << self
+        # @return `true` if value is a value for `true` in BOOLEAN_VALUES
         def enum_to_bool(enum)
           Aspera.assert_values(enum, BOOLEAN_VALUES){'boolean'}
           return TRUE_VALUES.include?(enum)
         end
 
-        def time_to_string(time)
-          return time.strftime('%Y-%m-%d %H:%M:%S')
+        # @return :yes ot :no
+        def enum_to_yes_no(enum)
+          Aspera.assert_values(enum, BOOLEAN_VALUES){'boolean'}
+          return TRUE_VALUES.include?(enum) ? BOOL_YES : BOOL_NO
         end
 
-        # find shortened string value in allowed symbol list
+        # Find shortened string value in allowed symbol list
         def get_from_list(short_value, descr, allowed_values)
           Aspera.assert_type(short_value, String)
           # we accept shortcuts
           matching_exact = allowed_values.select{ |i| i.to_s.eql?(short_value)}
           return matching_exact.first if matching_exact.length == 1
           matching = allowed_values.select{ |i| i.to_s.start_with?(short_value)}
-          multi_choice_assert(!matching.empty?, "unknown value for #{descr}: #{short_value}", allowed_values)
-          multi_choice_assert(matching.length.eql?(1), "ambiguous shortcut for #{descr}: #{short_value}", matching)
+          Aspera.assert(!matching.empty?, multi_choice_assert_msg("unknown value for #{descr}: #{short_value}", allowed_values), type: BadArgument)
+          Aspera.assert(matching.length.eql?(1), multi_choice_assert_msg("ambiguous shortcut for #{descr}: #{short_value}", matching), type: BadArgument)
           return enum_to_bool(matching.first) if allowed_values.eql?(BOOLEAN_VALUES)
           return matching.first
         end
@@ -94,8 +170,8 @@ module Aspera
         # Generates error message with list of allowed values
         # @param error_msg [String] error message
         # @param accept_list [Array] list of allowed values
-        def multi_choice_assert(assertion, error_msg, accept_list)
-          raise Cli::BadArgument, [error_msg, 'Use:'].concat(accept_list.map{ |c| "- #{c}"}.sort).join("\n") unless assertion
+        def multi_choice_assert_msg(error_msg, accept_list)
+          [error_msg, 'Use:'].concat(accept_list.map{ |c| "- #{c}"}.sort).join("\n")
         end
 
         # change option name with dash to name with underscore
@@ -106,22 +182,6 @@ module Aspera
         def option_name_to_line(name)
           return "#{OPTION_PREFIX}#{name.to_s.gsub(OPTION_SEP_SYMBOL, OPTION_SEP_LINE)}"
         end
-
-        # @param what [Symbol] :option or :argument
-        # @param descr [String] description for help
-        # @param to_check [Object] value to check
-        # @param type_list [NilClass, Class, Array[Class]] accepted value type(s)
-        # @param check_array [bool] set to true if it is a list of values to check
-        def validate_type(what, descr, to_check, type_list, check_array: false)
-          return if type_list.nil?
-          Aspera.assert(type_list.is_a?(Array) && type_list.all?(Class)){'types must be a Class Array'}
-          value_list = check_array ? to_check : [to_check]
-          value_list.each do |value|
-            raise Cli::BadArgument,
-              "#{what.to_s.capitalize} #{descr} is a #{value.class} but must be #{'one of: ' if type_list.length > 1}#{type_list.map(&:name).join(', ')}" unless
-              type_list.any?{ |t| value.is_a?(t)}
-          end
-        end
       end
 
       attr_reader :parser
@@ -135,12 +195,13 @@ module Aspera
         @unprocessed_cmd_line_options = []
         # a copy of all initial options
         @initial_cli_options = []
-        # option description: key = option symbol, value=Hash, :read_write, :accessor, :value, :accepted
+        # option description: option_symbol => OptionValue
         @declared_options = {}
         # do we ask missing options and arguments to user ?
         @ask_missing_mandatory = false # STDIN.isatty
         # ask optional options if not provided and in interactive
         @ask_missing_optional = false
+        # get_option fails if a mandatory parameter is asked
         @fail_on_missing_mandatory = true
         # Array of [key(sym), value]
         # those must be set before parse
@@ -179,35 +240,94 @@ module Aspera
         Log.log.trace1{"add_cmd_line_options:commands/arguments=#{@unprocessed_cmd_line_arguments},options=#{@unprocessed_cmd_line_options}".red}
         @parser.separator('')
         @parser.separator('OPTIONS: global')
-        declare(:interactive, 'Use interactive input of missing params', values: :bool, handler: {o: self, m: :ask_missing_mandatory})
-        declare(:ask_options, 'Ask even optional options', values: :bool, handler: {o: self, m: :ask_missing_optional})
-        declare(:struct_parser, 'Default parser when expected value is a struct', values: %i[json ruby])
-        # do not parse options yet, let's wait for option `-h` to be overriden
+        declare(:interactive, 'Use interactive input of missing params', allowed: Allowed::TYPES_BOOLEAN, handler: {o: self, m: :ask_missing_mandatory})
+        declare(:ask_options, 'Ask even optional options', allowed: Allowed::TYPES_BOOLEAN, handler: {o: self, m: :ask_missing_optional})
+        # do not parse options yet, let's wait for option `-h` to be overridden
       end
 
-      # @param descr [String] description for help
-      # @param mandatory [Boolean] if true, raise error if option not set
-      # @param multiple [Boolean] if true, return remaining arguments (Array)
-      # @param accept_list [Array] list of allowed values (Symbol)
-      # @param validation [Class, Array] accepted value type(s) or list of Symbols
-      # @param aliases [Hash] map of aliases: key = alias, value = real value
-      # @param default [Object] default value
-      # @return one value, list or nil (if optional and no default)
-      def get_next_argument(descr, mandatory: true, multiple: false, accept_list: nil, validation: String, aliases: nil, default: nil)
-        Aspera.assert(accept_list.nil? || (accept_list.is_a?(Array) && accept_list.all?(Symbol)))
-        validation = Symbol if accept_list
-        Aspera.assert(validation.nil? || validation.is_a?(Class) || (validation.is_a?(Array) && validation.all?(Class))){'validation must be Class or Array of Class'}
-        Aspera.assert(aliases.nil? || (aliases.is_a?(Hash) && aliases.keys.all?(Symbol) && aliases.values.all?(Symbol))){'aliases must be Hash:Symbol: Symbol'}
-        allowed_types = validation
-        unless allowed_types.nil?
-          allowed_types = [allowed_types] unless allowed_types.is_a?(Array)
-          descr = "#{descr} (#{allowed_types.join(', ')})"
+      # Declare an option
+      # @param option_symbol [Symbol] option name
+      # @param description   [String] description for help
+      # @param short         [String] short option name
+      # @param allowed       [Object] Allowed values, see `OptionValue`
+      # @param default       [Object] default value
+      # @param handler       [Hash]   handler for option value: keys: :o(object) and :m(method)
+      # @param deprecation   [String] deprecation
+      # @param block [Proc] Block to execute when option is found
+      def declare(option_symbol, description, short: nil, allowed: nil, default: nil, handler: nil, deprecation: nil, &block)
+        Aspera.assert_type(option_symbol, Symbol)
+        Aspera.assert(!@declared_options.key?(option_symbol)){"#{option_symbol} already declared"}
+        Aspera.assert(description[-1] != '.'){"#{option_symbol} ends with dot"}
+        Aspera.assert(description[0] == description[0].upcase){"#{option_symbol} description does not start with an uppercase"}
+        Aspera.assert(!['hash', 'extended value'].any?{ |s| description.downcase.include?(s)}){"#{option_symbol} shall use :allowed"}
+        Aspera.assert_type(handler, Hash) if handler
+        Aspera.assert(handler.keys.sort.eql?(%i[m o])) if handler
+        option_attrs = @declared_options[option_symbol] = OptionValue.new(
+          option:      option_symbol,
+          description: description,
+          allowed:     allowed,
+          handler:     handler,
+          deprecation: deprecation
+        )
+        real_types = option_attrs.types&.reject{ |i| [NilClass, String, Symbol].include?(i)}
+        description = "#{description} (#{real_types.map(&:name).join(', ')})" if real_types && !real_types.empty? && !real_types.eql?(Allowed::TYPES_ENUM) && !real_types.eql?(Allowed::TYPES_BOOLEAN) && !real_types.eql?(Allowed::TYPES_STRING)
+        description = "#{description} (#{'deprecated'.blue}: #{deprecation})" if deprecation
+        set_option(option_symbol, default, where: 'default') unless default.nil?
+        on_args = [description]
+        case option_attrs.types
+        when Allowed::TYPES_ENUM, Allowed::TYPES_BOOLEAN
+          # This option value must be a symbol (or array of symbols)
+          set_option(option_symbol, Manager.enum_to_bool(default), where: 'default') if option_attrs.values.eql?(BOOLEAN_VALUES) && !default.nil?
+          value = get_option(option_symbol)
+          help_values =
+            if option_attrs.types.eql?(Allowed::TYPES_BOOLEAN)
+              highlight_current_in_list(BOOLEAN_SIMPLE, self.class.enum_to_yes_no(value))
+            else
+              highlight_current_in_list(option_attrs.values, value)
+            end
+          on_args[0] = "#{description}: #{help_values}"
+          on_args.push(symbol_to_option(option_symbol, 'ENUM'))
+          # on_args.push(option_attrs.values)
+          @parser.on(*on_args) do |v|
+            set_option(option_symbol, self.class.get_from_list(v.to_s, description, option_attrs.values), where: SOURCE_USER)
+          end
+        when Allowed::TYPES_NONE
+          Aspera.assert_type(block, Proc){"missing execution block for #{option_symbol}"}
+          on_args.push(symbol_to_option(option_symbol))
+          on_args.push("-#{short}") if short.is_a?(String)
+          @parser.on(*on_args, &block)
+        else
+          on_args.push(symbol_to_option(option_symbol, 'VALUE'))
+          on_args.push("-#{short}VALUE") unless short.nil?
+          # coerce integer
+          on_args.push(Integer) if option_attrs.types.eql?(Allowed::TYPES_INTEGER)
+          @parser.on(*on_args) do |v|
+            set_option(option_symbol, v, where: SOURCE_USER)
+          end
         end
+        Log.log.trace1{"on_args=#{on_args}"}
+      end
+
+      # @param descr       [String] description for help
+      # @param mandatory   [Boolean] if true, raise error if option not set
+      # @param multiple    [Boolean] if true, return remaining arguments (Array)
+      # @param accept_list [Array, NilClass] list of allowed values (Symbol)
+      # @param validation  [Class, Array, NilClass] Accepted value type(s) or list of Symbols
+      # @param aliases     [Hash] map of aliases: key = alias, value = real value
+      # @param default     [Object] default value
+      # @return one value, list or nil (if optional and no default)
+      def get_next_argument(descr, mandatory: true, multiple: false, accept_list: nil, validation: Allowed::TYPES_STRING, aliases: nil, default: nil)
+        Aspera.assert_array_all(accept_list, Symbol) unless accept_list.nil?
+        Aspera.assert_hash_all(aliases, Symbol, Symbol) unless aliases.nil?
+        validation = Symbol unless accept_list.nil?
+        validation = [validation] unless validation.is_a?(Array) || validation.nil?
+        Aspera.assert_array_all(validation, Class){'validation'} unless validation.nil?
+        descr = "#{descr} (#{validation.join(', ')})" unless validation.nil? || validation.eql?(Allowed::TYPES_STRING)
         result =
           if !@unprocessed_cmd_line_arguments.empty?
             how_many = multiple ? @unprocessed_cmd_line_arguments.length : 1
             values = @unprocessed_cmd_line_arguments.shift(how_many)
-            values = values.map{ |v| evaluate_extended_value(v, allowed_types)}
+            values = values.map{ |v| ExtendedValue.instance.evaluate(v, context: "argument: #{descr}", allowed: validation)}
             # if expecting list and only one arg of type array : it is the list
             values = values.first if multiple && values.length.eql?(1) && values.first.is_a?(Array)
             if accept_list
@@ -220,7 +340,7 @@ module Aspera
             # no value provided, either get value interactively, or exception
           elsif mandatory then get_interactive(descr, multiple: multiple, accept_list: accept_list)
           end
-        if result.is_a?(String) && validation.eql?(Integer)
+        if result.is_a?(String) && validation&.eql?(Allowed::TYPES_INTEGER)
           int_result = Integer(result, exception: false)
           raise Cli::BadArgument, "Invalid integer: #{result}" if int_result.nil?
           result = int_result
@@ -228,8 +348,14 @@ module Aspera
         Log.log.debug{"#{descr}=#{result}"}
         result = aliases[result] if aliases&.key?(result)
         # if value comes from JSON/YAML, it may come as Integer
-        result = result.to_s if result.is_a?(Integer) && validation.eql?(String)
-        self.class.validate_type(:argument, descr, result, allowed_types, check_array: multiple) unless result.nil? && !mandatory
+        result = result.to_s if result.is_a?(Integer) && validation&.eql?(Allowed::TYPES_STRING)
+        if validation && (mandatory || !result.nil?)
+          value_list = multiple ? result : [result]
+          value_list.each do |value|
+            raise Cli::BadArgument,
+              "Argument #{descr} is a #{value.class} but must be #{'one of: ' if validation.length > 1}#{validation.map(&:name).join(', ')}" unless validation.any?{ |t| value.is_a?(t)}
+          end
+        end
         return result
       end
 
@@ -239,146 +365,47 @@ module Aspera
       # either return value or calls handler, can return nil
       # ask interactively if requested/required
       # @param mandatory [Boolean] if true, raise error if option not set
-      def get_option(option_symbol, mandatory: false, default: nil)
+      def get_option(option_symbol, mandatory: false)
         Aspera.assert_type(option_symbol, Symbol)
-        attributes = @declared_options[option_symbol]
-        Aspera.assert(attributes){"option not declared: #{option_symbol}"}
-        result = nil
-        case attributes[:read_write]
-        when :accessor
-          result = attributes[:accessor].value
-        when :value
-          result = attributes[:value]
-        else Aspera.error_unexpected_value(attributes[:read_write]){'attribute read/write'}
-        end
-        Log.log.trace1{"(#{attributes[:read_write]}) get #{option_symbol}=#{result}"}
-        result = default if result.nil?
-        # do not fail for manual generation if option mandatory but not set
-        result = '' if result.nil? && mandatory && !@fail_on_missing_mandatory
-        # Log.log.debug{"interactive=#{@ask_missing_mandatory}"}
+        Aspera.assert(@declared_options.key?(option_symbol), type: Cli::BadArgument){"Unknown option: #{option_symbol}"}
+        option_attrs = @declared_options[option_symbol]
+        result = option_attrs.value
+        # Do not fail for manual generation if option mandatory but not set
+        return :skip_missing_mandatory if result.nil? && mandatory && !@fail_on_missing_mandatory
         if result.nil?
           if !@ask_missing_mandatory
-            raise Cli::BadArgument, "Missing mandatory option: #{option_symbol}" if mandatory
+            Aspera.assert(!mandatory, type: Cli::BadArgument){"Missing mandatory option: #{option_symbol}"}
           elsif @ask_missing_optional || mandatory
             # ask_missing_mandatory
-            accept_list = nil
-            # print "please enter: #{option_symbol.to_s}"
-            accept_list = attributes[:values] if @declared_options.key?(option_symbol) && attributes.key?(:values)
-            result = get_interactive(option_symbol.to_s, option: true, accept_list: accept_list)
+            result = get_interactive(option_symbol.to_s, check_option: true, accept_list: option_attrs.values)
             set_option(option_symbol, result, where: 'interactive')
           end
         end
-        self.class.validate_type(:option, option_symbol, result, attributes[:types]) unless result.nil? && !mandatory
         return result
       end
 
-      # set an option value by name, either store value or call handler
+      # Set an option value by name, either store value or call handler
+      # String is given to extended value
       # @param option_symbol [Symbol] option name
-      # @param value [String] value to set
-      # @param where [String] where the value comes from
-      # @param expect [Class, Array] expected value type(s)
+      # @param value [String] Value to set
+      # @param where [String] Where the value comes from
       def set_option(option_symbol, value, where: 'code override')
         Aspera.assert_type(option_symbol, Symbol)
-        raise Cli::BadArgument, "Unknown option: #{option_symbol}" unless @declared_options.key?(option_symbol)
-        attributes = @declared_options[option_symbol]
-        Log.log.warn("#{option_symbol}: Option is deprecated: #{attributes[:deprecation]}") if attributes[:deprecation]
-        value = evaluate_extended_value(value, attributes[:types])
-        value = Manager.enum_to_bool(value) if attributes[:values].eql?(BOOLEAN_VALUES)
-        Log.log.trace1{"(#{attributes[:read_write]}/#{where}) set #{option_symbol}=#{value}"}
-        self.class.validate_type(:option, option_symbol, value, attributes[:types])
-        case attributes[:read_write]
-        when :accessor
-          attributes[:accessor].value = value
-        when :value
-          attributes[:value] = value
-        else Aspera.error_unexpected_value(attributes[:read_write]){'attribute read/write'}
-        end
+        Aspera.assert(@declared_options.key?(option_symbol), type: Cli::BadArgument){"Unknown option: #{option_symbol}"}
+        @declared_options[option_symbol].assign_value(value, where: where)
       end
 
-      # declare an option
-      # @param option_symbol [Symbol] option name
-      # @param description [String] description for help
-      # @param handler [Hash] handler for option value: keys: o (object) and m (method)
-      # @param default [Object] default value
-      # @param values [nil, Array, :bool, :date, :none] list of allowed values, :bool for true/false, :date for dates, :none for on/off switch
-      # @param short [String] short option name
-      # @param coerce [Class] one of the coerce types accepted by option parser
-      # @param types [Class, Array] accepted value type(s)
-      # @param block [Proc] block to execute when option is found
-      def declare(option_symbol, description, handler: nil, default: nil, values: nil, short: nil, coerce: nil, types: nil, deprecation: nil, &block)
+      # Set option to `nil`
+      def clear_option(option_symbol)
         Aspera.assert_type(option_symbol, Symbol)
-        Aspera.assert(!@declared_options.key?(option_symbol)){"#{option_symbol} already declared"}
-        Aspera.assert(description[-1] != '.'){"#{option_symbol} ends with dot"}
-        Aspera.assert(description[0] == description[0].upcase){"#{option_symbol} description does not start with an uppercase"}
-        Aspera.assert(!['hash', 'extended value'].any?{ |s| description.downcase.include?(s)}){"#{option_symbol} shall use :types"}
-        opt = @declared_options[option_symbol] = {
-          read_write: handler.nil? ? :value : :accessor,
-          # by default passwords and secrets are sensitive, else specify when declaring the option
-          sensitive:  SecretHider.instance.secret?(option_symbol, '')
-        }
-        if !types.nil?
-          types = [types] unless types.is_a?(Array)
-          Aspera.assert(types.all?(Class)){"types must be (Array of) Class: #{types}"}
-          opt[:types] = types
-          description = "#{description} (#{types.map(&:name).join(', ')})"
-        end
-        if deprecation
-          opt[:deprecation] = deprecation
-          description = "#{description} (#{'deprecated'.blue}: #{deprecation})"
-        end
-        Log.log.trace1{"declare: #{option_symbol}: #{opt[:read_write]}".green}
-        if opt[:read_write].eql?(:accessor)
-          Aspera.assert_type(handler, Hash)
-          Aspera.assert(handler.keys.sort.eql?(%i[m o]))
-          Log.log.trace1{"set attr obj: #{option_symbol} (#{handler[:o]},#{handler[:m]})"}
-          opt[:accessor] = AttrAccessor.new(handler[:o], handler[:m], option_symbol)
-        end
-        set_option(option_symbol, default, where: 'default') unless default.nil?
-        on_args = [description]
-        case values
-        when nil
-          on_args.push(symbol_to_option(option_symbol, 'VALUE'))
-          on_args.push("-#{short}VALUE") unless short.nil?
-          on_args.push(coerce) unless coerce.nil?
-          @parser.on(*on_args){ |v| set_option(option_symbol, v, where: SOURCE_USER)}
-        when Array, :bool
-          if values.eql?(:bool)
-            values = BOOLEAN_VALUES
-            set_option(option_symbol, Manager.enum_to_bool(default), where: 'default') unless default.nil?
-          end
-          # this option value must be a symbol
-          opt[:values] = values
-          value = get_option(option_symbol)
-          help_values = values.map{ |i| i.eql?(value) ? highlight_current(i) : i}.join(', ')
-          if values.eql?(BOOLEAN_VALUES)
-            help_values = BOOLEAN_SIMPLE.map{ |i| (i.eql?(:yes) && value) || (i.eql?(:no) && !value) ? highlight_current(i) : i}.join(', ')
-          end
-          on_args[0] = "#{description}: #{help_values}"
-          on_args.push(symbol_to_option(option_symbol, 'ENUM'))
-          on_args.push(values)
-          @parser.on(*on_args){ |v| set_option(option_symbol, self.class.get_from_list(v.to_s, description, values), where: SOURCE_USER)}
-        when :date
-          on_args.push(symbol_to_option(option_symbol, 'DATE'))
-          @parser.on(*on_args) do |v|
-            time_string = case v
-            when 'now' then Manager.time_to_string(Time.now)
-            when /^-([0-9]+)h/ then Manager.time_to_string(Time.now - (Regexp.last_match(1).to_i * 3600))
-            else v
-            end
-            set_option(option_symbol, time_string, where: SOURCE_USER)
-          end
-        when :none
-          Aspera.assert(!block.nil?){"missing block for #{option_symbol}"}
-          on_args.push(symbol_to_option(option_symbol))
-          on_args.push("-#{short}") if short.is_a?(String)
-          @parser.on(*on_args, &block)
-        else Aspera.error_unexpected_value(values)
-        end
-        Log.log.trace1{"on_args=#{on_args}"}
+        Aspera.assert(@declared_options.key?(option_symbol), type: Cli::BadArgument){"Unknown option: #{option_symbol}"}
+        @declared_options[option_symbol].clear
       end
 
       # Adds each of the keys of specified hash as an option
-      # @param preset_hash [Hash] hash of options to add
+      # @param preset_hash [Hash]    Options to add
+      # @param where       [String]  Where the value comes from
+      # @param override    [Boolean] Override if already present
       def add_option_preset(preset_hash, where, override: true)
         Aspera.assert_type(preset_hash, Hash)
         Log.log.debug{"add_option_preset: #{preset_hash}, #{where}, #{override}"}
@@ -388,17 +415,17 @@ module Aspera
         end
       end
 
-      # allows a plugin to add an argument as next argument to process
+      # Allows a plugin to add an argument as next argument to process
       def unshift_next_argument(argument)
         @unprocessed_cmd_line_arguments.unshift(argument)
       end
 
-      # check if there were unprocessed values to generate error
+      # Check if there were unprocessed values to generate error
       def command_or_arg_empty?
         return @unprocessed_cmd_line_arguments.empty?
       end
 
-      # unprocessed options or arguments ?
+      # Unprocessed options or arguments ?
       def final_errors
         result = []
         result.push("unprocessed options: #{@unprocessed_cmd_line_options}") unless @unprocessed_cmd_line_options.empty?
@@ -406,7 +433,7 @@ module Aspera
         return result
       end
 
-      # get all original options on command line used to generate a config in config file
+      # Get all original options on command line used to generate a config in config file
       # @return [Hash] options as taken from config file and command line just before command execution
       def unprocessed_options_with_value
         result = {}
@@ -418,7 +445,7 @@ module Aspera
             name = Regexp.last_match(1)
             value = Regexp.last_match(2)
             name.gsub!(OPTION_SEP_LINE, OPTION_SEP_SYMBOL)
-            value = ExtendedValue.instance.evaluate(value)
+            value = ExtendedValue.instance.evaluate(value, context: "option: #{name}")
             Log.log.debug{"option #{name}=#{value}"}
             result[name] = value
             @unprocessed_cmd_line_options.delete(option_value)
@@ -436,31 +463,34 @@ module Aspera
         @declared_options.each_key do |option_symbol|
           v = get_option(option_symbol)
           result[option_symbol] = v unless only_defined && v.nil?
+        rescue => e
+          result[option_symbol] = e.to_s
         end
         return result
       end
 
-      # removes already known options from the list
+      # Removes already known options from the list
       def parse_options!
         Log.log.trace1('parse_options!'.red)
-        # first conf file, then env var
+        # First options from conf file
         consume_option_pairs(@option_pairs_batch, 'set')
+        # Then, env var (to override)
         consume_option_pairs(@option_pairs_env, 'env')
-        # command line override
+        # Then, command line override
         unknown_options = []
         begin
           # remove known options one by one, exception if unknown
           Log.log.trace1('Before parse')
+          Log.dump(:unprocessed_cmd_line_options, @unprocessed_cmd_line_options)
           @parser.parse!(@unprocessed_cmd_line_options)
           Log.log.trace1('After parse')
         rescue OptionParser::InvalidOption => e
           Log.log.trace1{"InvalidOption #{e}".red}
           if (m = e.args.first.match(/^--([a-z\-]+)\.([^=]+)=(.+)$/))
-            option, path, raw_value = m.captures
+            option, path, value = m.captures
             option_sym = self.class.option_line_to_name(option).to_sym
             if @declared_options.key?(option_sym)
-              value = path.split(OPTION_HASH_SEPARATOR).reverse.inject(smart_convert(raw_value)){ |v, k| {k => v}}
-              set_option(option_sym, value, where: 'dotted')
+              set_option(option_sym, dotted_to_extended(path, value), where: 'dotted')
               retry
             end
           end
@@ -497,21 +527,21 @@ module Aspera
       end
 
       # Prompt user for input in a list of symbols
-      # @param descr [String] description for help
-      # @param option [Boolean] true if command line option
-      # @param multiple [Boolean] true if multiple values expected
-      # @param accept_list [Array] list of expected values
-      def get_interactive(descr, option: false, multiple: false, accept_list: nil)
-        what = option ? 'option' : 'argument'
+      # @param descr        [String] description for help
+      # @param check_option [Boolean] Check attributes of option with name=descr
+      # @param multiple     [Boolean] true if multiple values expected
+      # @param accept_list  [Array] list of expected values
+      def get_interactive(descr, check_option: false, multiple: false, accept_list: nil)
+        option_attrs = @declared_options[descr.to_sym]
+        what = option_attrs ? 'option' : 'argument'
         if !@ask_missing_mandatory
           message = "missing #{what}: #{descr}"
           if accept_list.nil?
             raise Cli::BadArgument, message
           else
-            self.class.multi_choice_assert(false, message, accept_list)
+            Aspera.assert(false, self.class.multi_choice_assert_msg(message, accept_list), type: Cli::MissingArgument)
           end
         end
-        sensitive = option && @declared_options[descr.to_sym].is_a?(Hash) && @declared_options[descr.to_sym][:sensitive]
         default_prompt = "#{what}: #{descr}"
         # ask interactively
         result = []
@@ -519,9 +549,9 @@ module Aspera
         loop do
           prompt = default_prompt
           prompt = "#{accept_list.join(' ')}\n#{default_prompt}" if accept_list
-          entry = prompt_user_input(prompt, sensitive: sensitive)
+          entry = prompt_user_input(prompt, sensitive: option_attrs&.sensitive)
           break if entry.empty? && multiple
-          entry = ExtendedValue.instance.evaluate(entry)
+          entry = ExtendedValue.instance.evaluate(entry, context: 'interactive input')
           entry = self.class.get_from_list(entry, descr, accept_list) if accept_list
           return entry unless multiple
           result.push(entry)
@@ -529,24 +559,69 @@ module Aspera
         return result
       end
 
+      # Read remaining args and build an Array or Hash
+      # @param value [nil] Argument to `@:` extended value
+      def args_as_extended(value)
+        # This extended value does not take args (`@:`)
+        ExtendedValue.assert_no_value(value, :p)
+        result = nil
+        get_next_argument(:args, multiple: true).each do |arg|
+          Aspera.assert(arg.include?(OPTION_VALUE_SEPARATOR)){"Positional argument: #{arg} does not inlude #{OPTION_VALUE_SEPARATOR}"}
+          path, raw = arg.split(OPTION_VALUE_SEPARATOR, 2)
+          result = dotted_to_extended(path, raw, result)
+        end
+        result
+      end
+
+      # ======================================================
       private
 
       # Using dotted hash notation, convert value to bool, int, float or extended value
+      # @param value [String] The value to convert to appropriate type
+      # @return the converted value
       def smart_convert(value)
-        return true  if value == 'true'
-        return false if value == 'false'
-        Integer(value)
-      rescue ArgumentError
-        begin
-          Float(value)
-        rescue ArgumentError
-          evaluate_extended_value(value, nil)
+        case value
+        when 'true'  then true
+        when 'false' then false
+        else
+          Integer(value, exception: false) ||
+            Float(value, exception: false) ||
+            ExtendedValue.instance.evaluate(value, context: 'dotted expression')
         end
       end
 
-      def evaluate_extended_value(value, types)
-        return ExtendedValue.instance.evaluate_with_default(value) if DEFAULT_PARSER_TYPES.include?(types) || (types.is_a?(Array) && types.all?{ |t| DEFAULT_PARSER_TYPES.include?(t)})
-        return ExtendedValue.instance.evaluate(value)
+      # Convert `String` to `Integer`, or keep `String` if not `Integer`
+      def int_or_string(value)
+        Integer(value, exception: false) || value
+      end
+
+      def new_hash_or_array_from_key(key)
+        key.is_a?(Integer) ? [] : {}
+      end
+
+      def array_requires_integer_index!(container, index)
+        Aspera.assert(container.is_a?(Hash) || index.is_a?(Integer)){'Using String index when Integer index used previously'}
+      end
+
+      # Insert extended value `value` into struct `result` at `path`
+      # @param path   [String]
+      # @param value  [String]
+      # @param result [NilClass, Hash, Array]
+      # @return [Hash, Array]
+      def dotted_to_extended(path, value, result = nil)
+        # Typed keys
+        keys = path.split(OPTION_DOTTED_SEPARATOR).map{ |k| int_or_string(k)}
+        # Create, or re-used higher level container
+        current = (result ||= new_hash_or_array_from_key(keys.first))
+        # walk the path, and create sub-containers if necessary
+        keys.each_cons(2) do |k, next_k|
+          array_requires_integer_index!(current, k)
+          current = (current[k] ||= new_hash_or_array_from_key(next_k))
+        end
+        # Assign value at last index
+        array_requires_integer_index!(current, keys.last)
+        current[keys.last] = smart_convert(value)
+        result
       end
 
       # generate command line option from option symbol
@@ -557,11 +632,18 @@ module Aspera
       end
 
       # TODO: use formatter
-      def highlight_current(value)
-        $stdout.isatty ? value.to_s.red.bold : "[#{value}]"
+      # @return [String] comma separated list of values, with the current value highlighted
+      def highlight_current_in_list(list, current)
+        list.map do |i|
+          if i.eql?(current)
+            $stdout.isatty ? i.to_s.red.bold : "[#{i}]"
+          else
+            i
+          end
+        end.join(', ')
       end
 
-      # try to evaluate options set in batch
+      # Try to evaluate options set in batch
       # @param unprocessed_options [Array] list of options to apply (key_sym,value)
       # @param where [String] where the options come from
       def consume_option_pairs(unprocessed_options, where)
@@ -570,7 +652,7 @@ module Aspera
         unprocessed_options.each do |k, v|
           if @declared_options.key?(k)
             # constrained parameters as string are revert to symbol
-            v = self.class.get_from_list(v, "#{k} in #{where}", @declared_options[k][:values]) if @declared_options[k].key?(:values) && v.is_a?(String)
+            v = self.class.get_from_list(v, "#{k} in #{where}", @declared_options[k].values) if @declared_options[k].values && v.is_a?(String)
             options_to_set[k] = v
           else
             Log.log.trace1{"unprocessed: #{k}: #{v}"}
@@ -582,6 +664,28 @@ module Aspera
           unprocessed_options.delete(k)
         end
       end
+      # boolean options are set to true/false from the following values
+      BOOL_YES = BOOLEAN_SIMPLE.last
+      BOOL_NO = BOOLEAN_SIMPLE.first
+      FALSE_VALUES = [BOOL_NO, false].freeze
+      TRUE_VALUES = [BOOL_YES, true].freeze
+      BOOLEAN_VALUES = (TRUE_VALUES + FALSE_VALUES).freeze
+
+      # Option name separator on command line, e.g. in --option-blah, third "-"
+      OPTION_SEP_LINE = '-'
+      # Option name separator in code (symbol), e.g. in :option_blah, the "_"
+      OPTION_SEP_SYMBOL = '_'
+      # Option value separator on command line, e.g. in --option-blah=foo, the "="
+      OPTION_VALUE_SEPARATOR = '='
+      # "." : An option like --a.b.c=d does: a={"b":{"c":ext_val(d)}}
+      OPTION_DOTTED_SEPARATOR = '.'
+      # Starts an option, e.g. in --option-blah, the two first "--"
+      OPTION_PREFIX = '--'
+      # when this is alone, this stops option processing
+      OPTIONS_STOP = '--'
+      SOURCE_USER = 'cmdline' # cspell:disable-line
+
+      private_constant :BOOL_YES, :BOOL_NO, :FALSE_VALUES, :TRUE_VALUES, :OPTION_SEP_LINE, :OPTION_SEP_SYMBOL, :OPTION_VALUE_SEPARATOR, :OPTION_DOTTED_SEPARATOR, :OPTION_PREFIX, :OPTIONS_STOP, :SOURCE_USER
     end
   end
 end