aspera-cli 4.26.1 → 4.27.0

data/lib/aspera/cli/{manager.rb → parser.rb}

@@ -2,13 +2,15 @@
 
 require 'aspera/cli/extended_value'
 require 'aspera/cli/error'
+require 'aspera/cli/terminal_formatter'
 require 'aspera/colors'
 require 'aspera/secret_hider'
 require 'aspera/log'
 require 'aspera/assert'
 require 'aspera/dot_container'
+require 'aspera/schema/registry'
 require 'io/console'
-require 'optparse'
+require 'terminal-table'
 
 module Aspera
   module Cli
@@ -39,7 +41,7 @@ module Aspera
       # `false` and `true`
       TYPES = [FalseClass, TrueClass].freeze
       SYMBOLS = [NO_SYM, YES_SYM].freeze
-      # @return `true` if value is a value for `true` in ALL
+      # @return [Boolean] `true` if value is a value for `true` in ALL
       def true?(enum)
         Aspera.assert_values(enum, ALL){'boolean'}
         TRUE_VALUES.include?(enum)
@@ -51,41 +53,49 @@ module Aspera
         TRUE_VALUES.include?(enum) ? YES_SYM : NO_SYM
       end
 
-      # @return `true` if value is a value for `true` or `false` in ALL
+      # @return [Boolean] `true` if value is a value for `true` or `false` in ALL
       def symbol?(sym)
         ALL.include?(sym)
       end
       module_function :true?, :to_sym, :symbol?
     end
 
-    # Constants to be used as parameter `allowed:` for `OptionValue`
+    # Constants to be used as parameter `allowed:` for `OptionValue`.
+    # Public API: TYPES_STRING_ARRAY, TYPES_SYMBOL_ARRAY, TYPES_INTEGER, TYPES_BOOLEAN, TYPES_NONE.
+    # Internal (do not pass as `allowed:`):
+    #   TYPES_ENUM   - derived internally when `allowed:` is an Array<Symbol> (enum list)
+    #   TYPES_STRING - the implicit default; equivalent to omitting `allowed:` entirely
     module Allowed
       # This option can be set to a single string or array, multiple times, and gives Array of String
       TYPES_STRING_ARRAY = [Array, String].freeze
-      # A list of symbols with constrained values
+      # A list of symbols with constrained values; use as prefix: TYPES_SYMBOL_ARRAY + [:val1, :val2]
       TYPES_SYMBOL_ARRAY = [Array, Symbol].freeze
       # Value will be coerced to int
       TYPES_INTEGER = [Integer].freeze
       TYPES_BOOLEAN = BoolValue::TYPES
       # No value at all for the option, it's a switch, like `-N`
       TYPES_NONE = [].freeze
-      # Symbol
-      TYPES_ENUM = [Symbol].freeze
-      # String
+      # Internal: derived when allowed: is an Array<Symbol>; do not pass directly
+      TYPES_ENUM   = [Symbol].freeze
+      # Internal: implicit default (String); equivalent to omitting allowed: entirely
       TYPES_STRING = [String].freeze
     end
 
     # Description of option, how to manage
     class OptionValue
       # [Array(Class)] List of allowed types
-      attr_reader :types, :sensitive, :schema, :option
+      attr_reader :types, :sensitive, :schema, :option, :deprecation
       # [Array] List of allowed values (Symbols and specific values)
       attr_accessor :values
+      # [String] Help section group name (set by Parser#group)
+      attr_accessor :group
+      # [Proc, nil] Block to call for flag options (TYPES_NONE)
+      attr_accessor :block
 
       # @param option [Symbol] Name of option
-      # @param description [String] Description for help
+      # @param description [String, nil] Description for help; if nil, derived from schema
       # @param allowed [nil,Class,Array<Class>,Array<Symbol>] Allowed values
-      # @param handler [Hash] Accessor: keys: :o(object) and :m(method)
+      # @param handler [Hash, nil] Accessor: keys: :o(object) and :m(method); nil for local storage
       # @param deprecation [String] Deprecation message
       # @param schema [String] Declaration of schema
       # `allowed`:
@@ -93,28 +103,34 @@ module Aspera
       # - `Class` The single allowed Class
       # - `Array<Class>` Multiple allowed classes
       # - `Array<Symbol>` List of allowed values
-      def initialize(option:, description:, allowed: Allowed::TYPES_STRING, handler: nil, deprecation: nil, schema: nil)
+      def initialize(option:, description: nil, allowed: Allowed::TYPES_STRING, handler: nil, deprecation: nil, schema: nil)
         Log.log.trace1{"option: #{option}, allowed: #{allowed}"}
         @option = option
         @description = description
+        @group = nil
+        @block = nil
         # by default passwords and secrets are sensitive, else specify when declaring the option
         @sensitive = SecretHider.instance.secret?(@option, '')
-        # either the value, or object giving value
-        @object = handler&.[](:o)
-        @read_method = handler&.[](:m)
-        @write_method = @read_method ? "#{@read_method}=".to_sym : nil
         @deprecation = deprecation
         @schema = schema
-        @access = if @object.nil?
-          :local
-        elsif @object.respond_to?(@write_method)
-          :write
-        else
-          :setter
-        end
-        Aspera.assert(@object.respond_to?(@read_method)){"#{@object} does not respond to #{@read_method}"} unless @access.eql?(:local)
+        # Start with local storage; bind_handler wires the delegation if a handler is given.
+        @object = nil
+        @read_method = nil
+        @write_method = nil
+        @access = :local
+        bind_handler(handler) unless handler.nil?
         @types = nil
         @values = nil
+        # Derive allowed type from schema when not explicitly provided
+        if (allowed.nil? || allowed.eql?(Allowed::TYPES_STRING)) && schema
+          schema_node = Schema::Registry.instance.reader(schema).current rescue nil
+          if schema_node
+            case schema_node['type']
+            when 'object' then allowed = Hash
+            when 'array'  then allowed = Array
+            end
+          end
+        end
         if !allowed.nil?
           allowed = [allowed] if allowed.is_a?(Class)
           Aspera.assert_type(allowed, Array)
@@ -122,12 +138,17 @@ module Aspera
             # Special case: array of defined symbol values
             @types = Allowed::TYPES_SYMBOL_ARRAY
             @values = allowed[Allowed::TYPES_SYMBOL_ARRAY.length..]
+            # Default value for symbol array when no value has been set yet
+            assign_value([], where: 'array default', warn_deprecation: false) if value(log: false).nil?
           elsif allowed.all?(Class)
             @types = allowed
             @values = BoolValue::ALL if allowed.eql?(Allowed::TYPES_BOOLEAN)
-            # Default value for array
-            @object ||= [] if @types.first.eql?(Array) && !@types.include?(NilClass)
-            @object ||= {} if @types.first.eql?(Hash) && !@types.include?(NilClass)
+            # Default value for array/hash when no value has been set yet
+            if @types.first.eql?(Array) && !@types.include?(NilClass) && value(log: false).nil?
+              assign_value([], where: 'array default', warn_deprecation: false)
+            elsif @types.first.eql?(Hash) && !@types.include?(NilClass) && value(log: false).nil?
+              assign_value({}, where: 'hash default', warn_deprecation: false)
+            end
           elsif allowed.all?(Symbol)
             @types = Allowed::TYPES_ENUM
             @values = allowed
@@ -138,6 +159,40 @@ module Aspera
         Log.log.trace1{"declare: #{@option}: #{@access} #{@object.class}.#{@read_method}".green}
       end
 
+      # Wire (or re-wire) the getter/setter delegation for this option.
+      # Safe to call after construction - used by Parser#set_handler to bind a composed
+      # instance variable that did not exist at class-load time (Category C handlers).
+      # @param handler [Hash] Accessor hash with keys :o (object) and :m (method symbol)
+      # @return [nil]
+      def bind_handler(handler)
+        Aspera.assert_type(handler, Hash){'handler'}
+        # Capture any value already stored locally before switching to delegated storage.
+        # This transfers defaults (and any preset values already applied) to the new target.
+        pending_value = @access.eql?(:local) ? @object : nil
+        @object       = handler[:o]
+        @read_method  = handler[:m]
+        @write_method = "#{@read_method}=".to_sym
+        @access = if @object.respond_to?(@write_method)
+          :write
+        else
+          :setter
+        end
+        Aspera.assert(@object.respond_to?(@read_method)){"#{@object} does not respond to #{@read_method}"}
+        Log.log.trace1{"bind_handler: #{@option}: #{@access} #{@object.class}.#{@read_method}".green}
+        # Push the pending local value to the new target if one was stored
+        assign_value(pending_value, where: 'bind_handler', warn_deprecation: false) unless pending_value.nil?
+        nil
+      end
+
+      # @return [String] description of the option: explicit one, or first line of schema description
+      def description
+        return @description unless @description.nil?
+        return if @schema.nil?
+        schema_node = Schema::Registry.instance.reader(@schema).current
+        first_line = (schema_node['title'] || schema_node['description'].to_s).lines.first.to_s.strip
+        first_line.end_with?('.') ? first_line[0..-2] : first_line
+      end
+
       def clear
         @object = nil
       end
@@ -157,22 +212,42 @@ module Aspera
       # Value can be a `String`, then evaluated with `ExtendedValue`, or directly a value.
       # @param value [String, Object] Value to assign to option
       # @param where [String] Where the value is assigned from
+      # @param warn_deprecation [Boolean] Emit deprecation warning (false for internal transfers)
       # @return [nil]
-      def assign_value(value, where:)
-        Aspera.assert(!@deprecation, type: warn){"Option #{@option} is deprecated: #{@deprecation}"}
+      def assign_value(value, where:, warn_deprecation: true)
+        Aspera.assert(!@deprecation, type: :warn){"Option #{@option} is deprecated: #{@deprecation}"} if warn_deprecation
         new_value = ExtendedValue.instance.evaluate(value, context: "option: #{@option}", allowed: @types)
         Log.log.trace1{"#{where}: #{@option} <- (#{new_value.class})#{new_value}"}
-        new_value = BoolValue.true?(new_value) if @types.eql?(Allowed::TYPES_BOOLEAN)
-        new_value = Integer(new_value) if @types.eql?(Allowed::TYPES_INTEGER)
-        new_value = [new_value] if @types.eql?(Allowed::TYPES_STRING_ARRAY) && new_value.is_a?(String)
-        # Setting a Hash to null set an empty hash
-        new_value = {} if new_value.eql?(nil) && @types&.first.eql?(Hash)
-        # Setting a Array to null set an empty array
-        new_value = [] if new_value.eql?(nil) && @types&.first.eql?(Array)
-        if @types.eql?(Aspera::Cli::Allowed::TYPES_SYMBOL_ARRAY)
+        # Per-type coercion: String input from CLI/env/preset is normalized to the expected type.
+        # Centralized here so all sources (CLI dispatch, preset, env) go through the same path.
+        case @types
+        when Allowed::TYPES_ENUM
+          new_value = Parser.get_from_list(new_value, @option, @values) if new_value.is_a?(String)
+        when Allowed::TYPES_BOOLEAN
+          new_value = Parser.get_from_list(new_value, @option, BoolValue::ALL) if new_value.is_a?(String)
+          new_value = BoolValue.true?(new_value)
+        when Allowed::TYPES_INTEGER
+          new_value = Integer(new_value)
+        when Allowed::TYPES_STRING_ARRAY
+          new_value = [new_value] if new_value.is_a?(String)
+        when Allowed::TYPES_SYMBOL_ARRAY
           new_value = [new_value] if new_value.is_a?(String)
           Aspera.assert_array_all(new_value, String, type: BadArgument)
-          new_value = new_value.map{ |v| Manager.get_from_list(v, @option, @values)}
+          new_value = new_value.map{ |v| Parser.get_from_list(v, @option, @values)}
+        else
+          # nil (setting nil on a Hash/Array option resets to empty container)
+          new_value = {} if new_value.nil? && @types&.first.eql?(Hash)
+          new_value = [] if new_value.nil? && @types&.first.eql?(Array)
+        end
+        # Skip type validation for the special 'help' value on Hash options: store it as-is
+        # so that get_option(schema:) can raise SchemaRequest with the contextual schema later.
+        if new_value.eql?(Parser::HELP) && @types&.include?(Hash) && !@schema
+          case @access
+          when :local  then @object = new_value
+          when :write  then @object.send(@write_method, new_value)
+          when :setter then @object.send(@read_method, @option, :set, new_value)
+          end
+          return
         end
         Aspera.assert_type(new_value, *@types, type: BadArgument){"Option #{@option}"} if @types
         if new_value.is_a?(Hash) || new_value.is_a?(Array)
@@ -193,7 +268,7 @@ module Aspera
     # parse command line options
     # arguments options start with '-', others are commands
     # resolves on extended value syntax
-    class Manager
+    class Parser
       class << self
         # Find shortened string value in allowed symbol list
         def get_from_list(short_value, descr, allowed_values)
@@ -202,12 +277,21 @@ module Aspera
           matching_exact = allowed_values.select{ |i| i.to_s.eql?(short_value)}
           return matching_exact.first if matching_exact.length == 1
           matching = allowed_values.select{ |i| i.to_s.start_with?(short_value)}
+          raise BadArgument, "Identifier '#{short_value}' used where a #{descr} is expected: place the identifier after the command" if matching.empty? && short_value.match?(REGEX_LOOKUP_ID_BY_FIELD)
           Aspera.assert(!matching.empty?, multi_choice_assert_msg("unknown value for #{descr}: #{short_value}", allowed_values), type: BadArgument)
           Aspera.assert(matching.length.eql?(1), multi_choice_assert_msg("ambiguous shortcut for #{descr}: #{short_value}", matching), type: BadArgument)
           return BoolValue.true?(matching.first) if allowed_values.eql?(BoolValue::ALL)
           matching.first
         end
 
+        # Find a key in a list by exact match or unique prefix match
+        # @return [Object, nil] the matching key, or nil if none or ambiguous
+        def match_prefix(short_value, allowed_values)
+          return short_value if allowed_values.include?(short_value)
+          matches = allowed_values.select{ |k| k.to_s.start_with?(short_value.to_s)}
+          matches.length == 1 ? matches.first : nil
+        end
+
         # Generates error message with list of allowed values
         # @param error_msg [String] Error message
         # @param accept_list [Array<Symbol>] List of allowed values
@@ -216,8 +300,8 @@ module Aspera
         end
 
         # Change option name with dash to name with underscore
-        # @param name [String] option name
-        # @return [String]
+        # @param name [String] option name with dash separators
+        # @return [String] option name with underscore separators
         def option_line_to_name(name)
           name.gsub(OPTION_SEP_LINE, OPTION_SEP_SYMBOL)
         end
@@ -236,19 +320,12 @@ module Aspera
         end
       end
 
-      attr_reader :parser
-      attr_accessor :ask_missing_mandatory, :ask_missing_optional
+      attr_accessor :ask_missing_mandatory, :ask_missing_optional, :help_requested
       attr_writer :fail_on_missing_mandatory
 
       # @param program_name [String] Name of the program
       # @param argv [Array<String>, nil] Command line arguments to parse
       def initialize(program_name, argv = nil)
-        # command line values *not* starting with '-'
-        @unprocessed_cmd_line_arguments = []
-        # command line values starting with at least one '-'
-        @unprocessed_cmd_line_options = []
-        # a copy of all initial options
-        @initial_cli_options = []
         # Option descriptions: maps option symbol to its OptionValue descriptor
         # @type [Hash{Symbol => OptionValue}]
         @declared_options = {}
@@ -258,45 +335,57 @@ module Aspera
         @ask_missing_optional = false
         # get_option fails if a mandatory parameter is asked
         @fail_on_missing_mandatory = true
-        # Array of [key(sym), value]
-        # those must be set before parse
-        # parse consumes those defined only
+        # set to true when --help / -h is parsed
+        @help_requested = false
+        # options can also be provided by env vars : --param-name -> ASCLI_PARAM_NAME
         @option_pairs_batch = {}
         @option_pairs_env = {}
-        # NOTE: was initially inherited but it is preferred to have specific methods
-        @parser = OptionParser.new
-        @parser.program_name = program_name
-        # options can also be provided by env vars : --param-name -> ASCLI_PARAM_NAME
+        # Short option char -> option symbol, e.g. {'h' => :help, 'v' => :version}
+        @short_options = {}
+        # Current help section group name, set by #group
+        @current_group = 'global'
         env_prefix = program_name.upcase + OPTION_SEP_SYMBOL
         ENV.each do |k, v|
           @option_pairs_env[k.delete_prefix(env_prefix).downcase.to_sym] = v if k.start_with?(env_prefix)
         end
         Log.log.debug{"env=#{@option_pairs_env}".red}
+        # command line values starting with at least one '-'
         @unprocessed_cmd_line_options = []
+        # command line values *not* starting with '-'
         @unprocessed_cmd_line_arguments = []
+        # a copy of all initial options
+        @initial_cli_options = []
+        # For each option string: list (one entry per occurrence) of the number of positional args
+        # that appear before it in original argv. Used by `@:` in option values to skip preceding args.
+        # @type [Hash{String => Array<Integer>}]
+        @args_before_option = {}
         return if argv.nil?
         # true until `--` is found (stop options)
         process_options = true
-        until argv.empty?
-          value = argv.shift
+        arg_count = 0
+        argv.each do |value|
           if process_options && value.start_with?('-')
             Log.log.trace1{"opt: #{value}"}
             if value.eql?(OPTIONS_STOP)
               process_options = false
             else
               @unprocessed_cmd_line_options.push(value)
+              (@args_before_option[value] ||= []).push(arg_count)
             end
           else
             Log.log.trace1{"arg: #{value}"}
             @unprocessed_cmd_line_arguments.push(value)
+            arg_count += 1
           end
         end
+        # Total positional args at parse time - used in args_as_extended to compute how many to skip.
+        @arg_total_count = @unprocessed_cmd_line_arguments.length
+        # Number of original positional args before the option currently being parsed (nil = positional context).
+        @current_option_args_offset = nil
         @initial_cli_options = @unprocessed_cmd_line_options.dup.freeze
         Log.log.trace1{"add_cmd_line_options:commands/arguments=#{@unprocessed_cmd_line_arguments},options=#{@unprocessed_cmd_line_options}".red}
-        @parser.separator('')
-        @parser.separator('OPTIONS: global')
-        declare(:interactive, 'Use interactive input of missing params', allowed: Allowed::TYPES_BOOLEAN, handler: {o: self, m: :ask_missing_mandatory})
-        declare(:ask_options, 'Ask even optional options', allowed: Allowed::TYPES_BOOLEAN, handler: {o: self, m: :ask_missing_optional})
+        declare(:interactive, description: 'Use interactive input of missing params', allowed: Allowed::TYPES_BOOLEAN, handler: {o: self, m: :ask_missing_mandatory})
+        declare(:ask_options, description: 'Ask even optional options', allowed: Allowed::TYPES_BOOLEAN, handler: {o: self, m: :ask_missing_optional})
         # do not parse options yet, let's wait for option `-h` to be overridden
       end
 
@@ -310,7 +399,7 @@ module Aspera
 
       # Declare an option
       # @param option_symbol [Symbol] option name
-      # @param description   [String] description for help
+      # @param description   [String, nil] description for help; if nil, derived from schema
       # @param short         [String] short option name
       # @param allowed       [Object] Allowed values, see `OptionValue`
       # @param default       [Object] default value
@@ -318,14 +407,11 @@ module Aspera
       # @param deprecation   [String] deprecation
       # @param schema        [String] Definition of schema for Hash parameters
       # @param block [Proc] Block to execute when option is found
-      def declare(option_symbol, description, short: nil, allowed: nil, default: nil, handler: nil, deprecation: nil, schema: nil, &block)
+      def declare(option_symbol, description: nil, short: nil, allowed: nil, default: nil, handler: nil, deprecation: nil, schema: nil, &block)
         Aspera.assert_type(option_symbol, Symbol)
         Aspera.assert(!@declared_options.key?(option_symbol)){"#{option_symbol} already declared"}
-        Aspera.assert(description[-1] != '.'){"#{option_symbol} ends with dot"}
-        Aspera.assert(description[0] == description[0].upcase){"#{option_symbol} description does not start with an uppercase"}
-        Aspera.assert(!['hash', 'extended value'].any?{ |s| description.downcase.include?(s)}){"#{option_symbol} shall use :allowed instead of hash/extended value in option description"}
         Aspera.assert_type(handler, Hash) if handler
-        Aspera.assert(handler.keys.sort.eql?(%i[m o])) if handler
+        Aspera.assert(handler.keys.sort.eql?(%i[m o]), 'handler must have keys :m and :o') if handler
         option_attrs = @declared_options[option_symbol] = OptionValue.new(
           option:      option_symbol,
           description: description,
@@ -334,45 +420,44 @@ module Aspera
           deprecation: deprecation,
           schema:      schema
         )
-        real_types = option_attrs.types&.reject{ |i| [NilClass, String, Symbol].include?(i)}
-        description += add_types_info(real_types)
-        description = "#{description} (#{'deprecated'.blue}: #{deprecation})" if deprecation
+        option_attrs.group = @current_group
+        description = option_attrs.description
+        Aspera.assert(!description.nil?){"#{option_symbol}: no description and no schema to derive one from"}
+        Aspera.assert(description[-1] != '.'){"#{option_symbol} ends with dot"}
+        Aspera.assert(description[0] == description[0].upcase){"#{option_symbol} description does not start with an uppercase"}
+        Aspera.assert(!['hash', 'extended value'].any?{ |s| description.downcase.include?(s)}){"#{option_symbol} shall use :allowed instead of hash/extended value in option description"}
         set_option(option_symbol, default, where: 'default') unless default.nil?
-        on_args = [description]
         case option_attrs.types
         when Allowed::TYPES_ENUM, Allowed::TYPES_BOOLEAN
           # This option value must be a symbol (or array of symbols)
           set_option(option_symbol, BoolValue.true?(default), where: 'default') if option_attrs.values.eql?(BoolValue::ALL) && !default.nil?
-          value = get_option(option_symbol)
-          help_values =
-            if option_attrs.types.eql?(Allowed::TYPES_BOOLEAN)
-              highlight_current_in_list(BoolValue::SYMBOLS, BoolValue.to_sym(value))
-            else
-              highlight_current_in_list(option_attrs.values, value)
-            end
-          on_args[0] = "#{description}: #{help_values}"
-          on_args.push(symbol_to_option(option_symbol, 'ENUM'))
-          # on_args.push(option_attrs.values)
-          @parser.on(*on_args) do |v|
-            set_option(option_symbol, self.class.get_from_list(v.to_s, description, option_attrs.values), where: SOURCE_USER)
-          end
         when Allowed::TYPES_NONE
           Aspera.assert_type(block, Proc){"missing execution block for #{option_symbol}"}
-          on_args.push(symbol_to_option(option_symbol))
-          on_args.push("-#{short}") if short.is_a?(String)
-          @parser.on(*on_args, &block)
-        else
-          on_args.push(symbol_to_option(option_symbol, 'VALUE'))
-          on_args.push("-#{short}VALUE") unless short.nil?
-          # coerce integer
-          on_args.push(Integer) if option_attrs.types.eql?(Allowed::TYPES_INTEGER)
-          @parser.on(*on_args) do |v|
-            set_option(option_symbol, v, where: SOURCE_USER)
-          end
+          option_attrs.block = block
         end
-        Log.log.trace1{"on_args=#{on_args}"}
+        @short_options[short] = option_symbol unless short.nil?
+        Log.log.trace1{"declare: #{option_symbol}, group: #{@current_group}, short: #{short}"}
+      end
+
+      # Set the current help section group name for subsequent declarations
+      # @param name [String] group name, shown as section header in help text
+      def group(name)
+        @current_group = name
+      end
+
+      # Rename all options currently tagged with @current_group to a new name,
+      # then update @current_group. Used by add_manual_header when a plugin
+      # declares its options before its group name is known (e.g. Plugins::Config).
+      # @param name [String] new group name
+      def rename_current_group(name)
+        @declared_options.each_value{ |opt| opt.group = name if opt.group.eql?(@current_group)}
+        @current_group = name
       end
 
+      # Low-level positional argument reader.  Prefer +Base#resolve_argument+ from action methods.
+      # Direct calls from outside +Parser+ are legacy exceptions documented in ST12/ST13
+      # (mixins without DSL: sync_actions, ascp_actions; setup callbacks: aoc.rb).
+      # @api private
       # @param descr       [String] description for help
       # @param mandatory   [Boolean] `true`: raise error no more argument
       # @param multiple    [Boolean] `true`: return all remaining arguments (Array). String: until marker
@@ -380,7 +465,7 @@ module Aspera
       # @param validation  [Class, Array, NilClass] Accepted value type(s) or list of Symbols
       # @param aliases     [Hash] map of aliases: key = alias, value = real value
       # @param default     [Object] default value
-      # @return one value, list or nil (if optional and no default)
+      # @return [Object, Array, nil] one value, list or nil (if optional and no default)
       def get_next_argument(descr, mandatory: true, multiple: false, accept_list: nil, validation: Allowed::TYPES_STRING, aliases: nil, default: nil, schema: nil)
         Aspera.assert_array_all(accept_list, Symbol) unless accept_list.nil?
         Aspera.assert_hash_all(aliases, Symbol, Symbol) unless aliases.nil?
@@ -442,14 +527,14 @@ module Aspera
       #
       # @param description [String] description of the identifier
       # @param block       [Proc] block to search for identifier based on attribute value
-      # @return   [String, Array<String>] identifier or list of IDs (if `bulk` option is set)
+      # @return [String, Array<String>] identifier or list of IDs (if `bulk` option is set)
       # @yieldparam field [String] The field name from percent selector
       # @yieldparam value [String] The value from percent selector
       # @yieldreturn [String] Resolved identifier
       def instance_identifier(description: 'identifier', &block)
-        res_id = get_next_argument(description, multiple: get_option(:bulk)) if res_id.nil?
+        res_id = get_next_argument(description, multiple: get_option(:bulk))
         # Can be an Array
-        if res_id.is_a?(String) && (m = Manager.percent_selector(res_id))
+        if res_id.is_a?(String) && (m = Parser.percent_selector(res_id))
           Aspera.assert(block_given?, type: Cli::BadArgument){"Percent syntax for #{description} not supported in this context"}
           res_id = yield(m[:field], m[:value])
         end
@@ -458,8 +543,18 @@ module Aspera
 
       def get_next_command(command_list, aliases: nil); get_next_argument('command', accept_list: command_list, aliases: aliases); end
 
+      # Check whether an option has already been declared in this manager
+      # @param option_symbol [Symbol] name of the option
+      # @return [Boolean]
+      def option_declared?(option_symbol)
+        @declared_options.key?(option_symbol)
+      end
+
+      # @return [Hash{Symbol => OptionValue}] all declared options (read-only view)
+      attr_reader :declared_options
+
       # Get an option definition by name
-      # @param option_symbol [Symbol]
+      # @param option_symbol [Symbol] name of the option
       # @return [OptionValue] Option definition
       # @raise [Cli::BadArgument] if option not found
       def option_def(option_symbol)
@@ -470,12 +565,16 @@ module Aspera
       # Get an option value by name
       # either return value or calls handler, can return nil
       # ask interactively if requested/required
-      # @param option_symbol [Symbol]
+      # @param option_symbol [Symbol] name of the option to retrieve
       # @param mandatory [Boolean] if true, raise error if option not set
-      def get_option(option_symbol, mandatory: false)
+      # @param schema [String, nil] contextual schema path override; when set, raises SchemaRequest
+      #   if the option value is 'help' (used for --query whose schema depends on the current command)
+      def get_option(option_symbol, mandatory: false, schema: nil)
         Aspera.assert_type(option_symbol, Symbol)
         option_attrs = option_def(option_symbol)
         result = option_attrs.value
+        # Contextual schema: raise SchemaRequest when value is 'help'
+        raise SchemaRequest.new(:option, option_symbol.to_s, schema) if schema && result.eql?(HELP)
         # Do not fail for manual generation if option mandatory but not set
         return :skip_missing_mandatory if result.nil? && mandatory && !@fail_on_missing_mandatory
         if result.nil?
@@ -498,7 +597,10 @@ module Aspera
       def set_option(option_symbol, value, where: 'code override')
         Aspera.assert_type(option_symbol, Symbol)
         option = option_def(option_symbol)
-        raise SchemaRequest.new(:option, option.option, option.schema) if option.types&.include?(Hash) && value.eql?(HELP)
+        # Raise immediately only when the option has a static schema: the schema is known at parse time.
+        # When schema is nil (e.g. --query), 'help' is stored as-is and SchemaRequest is raised later
+        # in get_option() with the contextual schema provided by the calling command.
+        raise SchemaRequest.new(:option, option.option, option.schema) if option.types&.include?(Hash) && value.eql?(HELP) && option.schema
         option.assign_value(value, where: where)
       end
 
@@ -508,6 +610,18 @@ module Aspera
         option_def(option_symbol).clear
       end
 
+      # Bind (or re-bind) a runtime handler to an already-declared option.
+      # Called from plugin initialize() for Category C handlers whose target object
+      # (e.g. @gen_options) is created after class-load time.
+      # @param option_symbol [Symbol] name of the already-declared option
+      # @param object [Object] the target object for get/set delegation
+      # @param method [Symbol] accessor method name on object
+      # @return [nil]
+      def set_handler(option_symbol, object:, method:)
+        Aspera.assert_type(option_symbol, Symbol)
+        option_def(option_symbol).bind_handler(o: object, m: method)
+      end
+
       # Adds each of the keys of specified hash as an option
       # @param preset_hash [Hash]    Options to add
       # @param where       [String]  Where the value comes from
@@ -575,37 +689,52 @@ module Aspera
       def parse_options!
         Log.log.trace1('parse_options!'.red)
         # First options from conf file
-        consume_option_pairs(@option_pairs_batch, 'set')
+        @option_pairs_batch = consume_option_pairs(@option_pairs_batch, 'set')
         # Then, env var (to override)
-        consume_option_pairs(@option_pairs_env, 'env')
-        # Then, command line override
+        @option_pairs_env = consume_option_pairs(@option_pairs_env, 'env')
+        # Then, command line override: process one option at a time so that @current_option_args_offset
+        # can be set before each option is evaluated (used by `@:` extended value in option values).
         unknown_options = []
-        begin
-          # remove known options one by one, exception if unknown
-          Log.log.trace1('Before parse')
-          Log.dump(:unprocessed_cmd_line_options, @unprocessed_cmd_line_options, level: :trace1)
-          @parser.parse!(@unprocessed_cmd_line_options)
-          Log.log.trace1('After parse')
-        rescue OptionParser::InvalidOption => e
-          Log.log.trace1{"InvalidOption #{e}".red}
-          # An option like --a.b.c=d does: a={"b":{"c":ext_val(d)}}
-          if e.args.first.start_with?(OPTION_PREFIX)
-            name, value = e.args.first.delete_prefix(OPTION_PREFIX).split(OPTION_VALUE_SEPARATOR, 2)
-            if !value.nil?
-              path = name.split(DotContainer::SEPARATOR)
-              option_sym = self.class.option_line_to_name(path.shift).to_sym
-              if @declared_options.key?(option_sym)
-                # it's a known option, so let's process it
-                set_option(option_sym, DotContainer.dotted_to_container(path, smart_convert(value), get_option(option_sym)), where: 'dotted')
-                # resume to next
-                retry
+        Log.log.trace1('Before parse')
+        Log.dump(:unprocessed_cmd_line_options, @unprocessed_cmd_line_options, level: :trace1)
+        until @unprocessed_cmd_line_options.empty?
+          opt = @unprocessed_cmd_line_options.shift
+          # Expose args_before for this option so `args_as_extended` can skip args preceding it.
+          # Peek (first) without consuming - consumed only if this option is processed (not deferred).
+          @current_option_args_offset = @args_before_option[opt]&.first
+          if opt.start_with?(OPTION_PREFIX)
+            # Long option: --name or --name=value
+            name_raw, raw_value = opt.delete_prefix(OPTION_PREFIX).split(OPTION_VALUE_SEPARATOR, 2)
+            option_sym = self.class.option_line_to_name(name_raw).to_sym
+            resolved_sym = self.class.match_prefix(option_sym, @declared_options.keys)
+            if resolved_sym
+              dispatch_option(resolved_sym, raw_value)
+              @args_before_option[opt]&.shift # consumed: advance to next occurrence
+            else
+              # Dotted notation: --a.b.c=d does: a={"b":{"c":ext_val(d)}}
+              Log.log.trace1{"Unknown long option: #{opt}".red}
+              if !raw_value.nil?
+                path = name_raw.split(DotContainer::SEPARATOR)
+                root_sym = self.class.option_line_to_name(path.shift).to_sym
+                if @declared_options.key?(root_sym)
+                  set_option(root_sym, DotContainer.dotted_to_container(path, smart_convert(raw_value), get_option(root_sym)), where: 'dotted')
+                  @args_before_option[opt]&.shift # consumed: advance to next occurrence
+                  next
+                end
               end
+              # Unknown option: defer to next parse_options! round, do not consume the recorded offset
+              unknown_options.push(opt)
             end
+          elsif opt.start_with?('-') && (option_sym = @short_options[opt[1]])
+            # Short option: -h, -v, or -Pvalue (value glued to flag)
+            dispatch_option(option_sym, opt.length > 2 ? opt[2..] : nil)
+            @args_before_option[opt]&.shift # consumed: advance to next occurrence
+          else
+            unknown_options.push(opt)
           end
-          # Save for later processing
-          unknown_options.push(e.args.first)
-          retry
         end
+        @current_option_args_offset = nil
+        Log.log.trace1('After parse')
         Log.log.trace1{"remains: #{unknown_options}"}
         # Set unprocessed options for next time
         @unprocessed_cmd_line_options = unknown_options
@@ -647,7 +776,7 @@ module Aspera
         if !@ask_missing_mandatory
           message = "Missing #{default_prompt}"
           message = self.class.multi_choice_assert_msg(message, accept_list) if accept_list
-          message += "\nGive `#{HELP}` as argument to retrieve the schema of the missing argument." if schema
+          message += "\n#{TerminalFormatter::HINT}Give `#{HELP}` as argument to retrieve the schema of the missing argument." if schema
           raise Cli::MissingArgument, message
         end
         # ask interactively
@@ -673,21 +802,85 @@ module Aspera
         # This extended value does not take args (`@:`)
         # ExtendedValue.assert_no_value(end_marker, :p)
         end_marker = SpecialValues::EOA if end_marker.empty?
+        # When called from an option value, skip positional args that appear before the option in argv.
+        # @current_option_args_offset holds the number of original args before the option (nil = positional context).
+        # The number to actually skip = args_before_option - args_already_consumed (clamped to 0).
+        skip_count = if @current_option_args_offset
+          [@current_option_args_offset - (@arg_total_count - @unprocessed_cmd_line_arguments.length), 0].max
+        else
+          0
+        end
+        skipped = skip_count.positive? ? @unprocessed_cmd_line_arguments.shift(skip_count) : []
+        Log.log.trace1{"args_as_extended: skipping #{skipped.length} args before option: #{skipped}"} unless skipped.empty?
         result = nil
         get_next_argument('args', multiple: end_marker).each do |argument|
           Aspera.assert(argument.include?(OPTION_VALUE_SEPARATOR)){"Positional argument: #{argument} does not include #{OPTION_VALUE_SEPARATOR}"}
           path, value = argument.split(OPTION_VALUE_SEPARATOR, 2)
           result = DotContainer.dotted_to_container(path.split(DotContainer::SEPARATOR), smart_convert(value), result)
         end
+        # Restore skipped args so they remain available for command dispatching
+        @unprocessed_cmd_line_arguments.unshift(*skipped) unless skipped.empty?
         result
       end
 
+      # Generate help text for all declared options, grouped by section.
+      # @param banner [String, nil] Optional banner text to prepend
+      # @return [String] Formatted help text
+      def help_text(banner: nil)
+        rows = []
+        current_group = nil
+        @declared_options.each do |sym, opt|
+          if opt.group != current_group
+            current_group = opt.group
+            rows << [{value: "OPTIONS: #{current_group}", colspan: 2}]
+          end
+          short_char = @short_options.key(sym)
+          short_part = short_char ? "-#{short_char}, " : '    '
+          flag = "#{short_part}#{symbol_to_option(sym, option_display_value(opt))}"
+          rows << [flag, opt.description]
+        end
+        table = Terminal::Table.new(rows: rows, style: {border: HELP_BORDER, padding_left: 0, padding_right: 2})
+        banner.nil? ? table.to_s : "#{banner}\n#{table}"
+      end
+
       # ======================================================
       private
 
+      # AsciiBorder with all visible characters removed - used by help_text
+      HELP_BORDER = Terminal::Table::AsciiBorder.new.tap do |b|
+        b.top = false
+        b.bottom = false
+        b.left = false
+        b.right = false
+        b.remove_verticals
+        b.remove_horizontals
+      end.freeze
+
+      # @param opt [OptionValue] option descriptor
+      # @return [String, nil] placeholder shown in flag column: 'ENUM', 'VALUE', or nil for flag switches
+      def option_display_value(opt)
+        case opt.types
+        when Allowed::TYPES_NONE then nil
+        when Allowed::TYPES_ENUM, Allowed::TYPES_BOOLEAN then 'ENUM'
+        else 'VALUE'
+        end
+      end
+
+      # Dispatch a parsed CLI option to its handler.
+      # @param sym       [Symbol] option symbol
+      # @param raw_value [String, nil] raw string value from command line, or nil for flag switches
+      def dispatch_option(sym, raw_value)
+        opt = @declared_options[sym]
+        if opt.types.eql?(Allowed::TYPES_NONE)
+          opt.block.call
+        else
+          set_option(sym, raw_value, where: SOURCE_USER)
+        end
+      end
+
       # Using dotted hash notation, convert value to bool, int, float or extended value
       # @param value [String] The value to convert to appropriate type
-      # @return the converted value
+      # @return [Boolean, Integer, Float, String, Array, Hash] the converted value
       def smart_convert(value)
         case value
         when 'true'  then true
@@ -724,23 +917,23 @@ module Aspera
       # Try to evaluate options set in batch
       # @param unprocessed_options [Array] list of options to apply (key_sym,value)
       # @param where [String] where the options come from
-      def consume_option_pairs(unprocessed_options, where)
+      # Apply all known options from the given pairs hash and return the remaining (unknown) pairs.
+      # Pure: does not mutate the argument; the caller is responsible for storing the returned value.
+      # @param option_pairs [Hash{Symbol => Object}] candidate key/value pairs
+      # @param where [String] label used in log messages and error context
+      # @return [Hash{Symbol => Object}] pairs whose keys were not yet declared (deferred to next round)
+      def consume_option_pairs(option_pairs, where)
         Log.log.trace1{"consume_option_pairs: #{where}"}
-        options_to_set = {}
-        unprocessed_options.each do |k, v|
+        remaining = {}
+        option_pairs.each do |k, v|
           if @declared_options.key?(k)
-            # constrained parameters as string are revert to symbol
-            v = self.class.get_from_list(v, "#{k} in #{where}", @declared_options[k].values) if @declared_options[k].values && v.is_a?(String)
-            options_to_set[k] = v
+            set_option(k, v, where: where)
           else
             Log.log.trace1{"unprocessed: #{k}: #{v}"}
+            remaining[k] = v
           end
         end
-        options_to_set.each do |k, v|
-          set_option(k, v, where: where)
-          # keep only unprocessed values for next parse
-          unprocessed_options.delete(k)
-        end
+        remaining
       end
 
       # Option name separator on command line, e.g. in --option-blah, third "-"
@@ -759,7 +952,7 @@ module Aspera
       # Ask for schema of Extended value
       HELP = 'help'
 
-      private_constant :OPTION_SEP_LINE, :OPTION_SEP_SYMBOL, :OPTION_VALUE_SEPARATOR, :OPTION_PREFIX, :OPTIONS_STOP, :SOURCE_USER, :REGEX_LOOKUP_ID_BY_FIELD
+      private_constant :OPTION_SEP_LINE, :OPTION_SEP_SYMBOL, :OPTION_VALUE_SEPARATOR, :OPTION_PREFIX, :OPTIONS_STOP, :SOURCE_USER, :REGEX_LOOKUP_ID_BY_FIELD, :HELP_BORDER
     end
   end
 end