aspera-cli 4.24.2 → 4.25.0.pre

data/lib/aspera/cli/extended_value.rb

@@ -17,16 +17,8 @@ module Aspera
     class ExtendedValue
       include Singleton
 
-      MARKER_START = '@'
-      MARKER_END = ':'
-      MARKER_IN_END = '@'
-
-      # Special handlers stop processing of handlers on right
-      # :extend includes processing of other handlers in itself
-      # :val keeps the value intact
-      SPECIAL_HANDLERS = %i[extend val].freeze
-
-      private_constant :MARKER_START, :MARKER_END, :MARKER_IN_END, :SPECIAL_HANDLERS
+      # First is default
+      DEFAULT_DECODERS = %i[none json ruby yaml]
 
       class << self
         # Decode comma separated table text
@@ -45,8 +37,32 @@ module Aspera
           return hash_array
         end
 
-        def assert_no_value(value, what)
-          raise "no value allowed for extended value type: #{what}" unless value.empty?
+        # JSON Parser, with more information on error location
+        # extract a context: 10 chars before and after the error on the given line and display a pointer "^"
+        # :reek:UncommunicativeMethodName
+        def JSON_parse(value) # rubocop:disable Naming/MethodName
+          JSON.parse(value)
+        rescue JSON::ParserError => e
+          m = /at line (\d+) column (\d+)/.match(e.message)
+          raise if m.nil?
+          line = m[1].to_i - 1
+          column = m[2].to_i - 1
+          lines = value.lines
+          raise if line >= lines.size
+          error_line = lines[line].chomp
+          context_col_beg = [column - 10, 0].max
+          context_col_end = [column + 10, error_line.length].min
+          context = error_line[context_col_beg...context_col_end]
+          cursor_pos = column - context_col_beg
+          pointer = ' ' * cursor_pos + '^'.blink
+          raise BadArgument, "#{e.message}\n#{context}\n#{pointer}"
+        end
+
+        # The value must be empty
+        # @param value [String] The value as parameter
+        # @param ext_type [Symbol] The method of extended value
+        def assert_no_value(value, ext_type)
+          Aspera.assert(value.empty?, type: BadArgument){"no value allowed for extended value type: #{ext_type}"}
         end
       end
 
@@ -54,7 +70,8 @@ module Aspera
 
       def initialize
         # Base handlers
-        # Other handlers can be set using set_handler, e.g. `preset` is reader in config plugin
+        # Other handlers can be set using `on`
+        # e.g. `preset` is reader in config plugin
         @handlers = {
           val:    lambda{ |i| i},
           base64: lambda{ |i| Base64.decode64(i)},
@@ -62,7 +79,7 @@ module Aspera
           env:    lambda{ |i| ENV.fetch(i, nil)},
           file:   lambda{ |i| File.read(File.expand_path(i))},
           uri:    lambda{ |i| UriReader.read(i)},
-          json:   lambda{ |i| JSON_parse(i)},
+          json:   lambda{ |i| ExtendedValue.JSON_parse(i)},
           lines:  lambda{ |i| i.split("\n")},
           list:   lambda{ |i| i[1..-1].split(i[0])},
           none:   lambda{ |i| ExtendedValue.assert_no_value(i, :none); nil}, # rubocop:disable Style/Semicolon
@@ -74,63 +91,60 @@ module Aspera
           stdbin: lambda{ |i| ExtendedValue.assert_no_value(i, :stdbin); $stdin.binmode.read}, # rubocop:disable Style/Semicolon
           yaml:   lambda{ |i| YAML.load(i)},
           zlib:   lambda{ |i| Zlib::Inflate.inflate(i)},
-          extend: lambda{ |i| ExtendedValue.instance.evaluate_all(i)}
+          extend: lambda{ |i| ExtendedValue.instance.evaluate_extend(i)}
         }
+        @regex_single = nil
+        @regex_extend = nil
         @default_decoder = nil
+        update_regex
       end
 
-      # Regex to match an extended value
-      def handler_regex_string
-        "#{MARKER_START}(#{modifiers.join('|')})#{MARKER_END}"
-      end
-
-      # JSON Parser, with more information on error location
-      # :reek:UncommunicativeMethodName
-      def JSON_parse(value) # rubocop:disable Naming/MethodName
-        JSON.parse(value)
-      rescue JSON::ParserError => e
-        m = /at line (\d+) column (\d+)/.match(e.message)
-        raise if m.nil?
-        line = m[1].to_i - 1
-        column = m[2].to_i - 1
-        lines = value.lines
-        raise if line >= lines.size
-        error_line = lines[line].chomp
-        context_col_beg = [column - 10, 0].max
-        context_col_end = [column + 10, error_line.length].min
-        context = error_line[context_col_beg...context_col_end]
-        cursor_pos = column - context_col_beg
-        pointer = ' ' * cursor_pos + '^'.blink
-        raise BadArgument, "#{e.message}\n#{context}\n#{pointer}"
+      # Update the Regex to match an extended value based on @handlers
+      def update_regex
+        handler_regex = "#{MARKER_START}(#{modifiers.join('|')})#{MARKER_END}"
+        @regex_single = Regexp.new("^#{handler_regex}(.*)$", Regexp::MULTILINE)
+        @regex_extend = Regexp.new("^(.*)#{handler_regex}([^#{MARKER_IN_END}]*)#{MARKER_IN_END}(.*)$", Regexp::MULTILINE)
       end
 
       public
 
+      attr_reader :default_decoder
+
       def default_decoder=(value)
         Log.log.debug{"Setting default decoder to (#{value.class}) #{value}"}
-        Aspera.assert(value.nil? || @handlers.key?(value))
+        Aspera.assert_values(value, DEFAULT_DECODERS)
+        value = nil if value.eql?(:none)
         @default_decoder = value
       end
 
+      # List of Extended Value methods
       def modifiers; @handlers.keys; end
 
       # Add a new handler
-      def set_handler(name, method)
-        Log.log.debug{"setting handler for #{name}"}
+      def on(name, &block)
         Aspera.assert_type(name, Symbol){'name'}
-        @handlers[name] = method
+        Aspera.assert(block)
+        Log.log.debug{"Setting handler for #{name}"}
+        @handlers[name] = block
+        update_regex
       end
 
-      # Parse an string value to extended value, if it is a String using supported extended value modifiers
-      # Other value types are returned as is
-      # @param value [String] the value to parse
-      # @param expect [Class,Array] one or a list of expected types
-      def evaluate(value)
+      # Parses a `String` value to extended value.
+      # If it is a String using supported extended value modifiers, then evaluate them.
+      # Other value types are returned as is.
+      # @param value   [String] the value to parse
+      # @param context [String] Context in which evaluation is done
+      # @param allowed [Array<Class>,NilClass] Expected types
+      # @return [Object] Evaluated value
+      def evaluate(value, context:, allowed: nil)
         return value unless value.is_a?(String)
-        regex = Regexp.new("^#{handler_regex_string}(.*)$", Regexp::MULTILINE)
+        Aspera.assert_array_all(allowed, Class) unless allowed.nil?
+        # use default decoder if not an extended value and expect complex types
+        using_default_decoder = allowed&.all?{ |t| DEFAULT_PARSER_TYPES.include?(t)} && !@regex_single.match?(value) && !@default_decoder.nil?
+        value = [MARKER_START, @default_decoder, MARKER_END, value].join if using_default_decoder
         # First determine decoders, in reversed order
         handlers_reversed = []
-        while (m = value.match(regex))
+        while (m = value.match(@regex_single))
           handler = m[1].to_sym
           handlers_reversed.unshift(handler)
           value = m[2]
@@ -139,27 +153,37 @@ module Aspera
         Log.log.trace1{"evaluating: #{handlers_reversed}, value: #{value}"}
         handlers_reversed.each do |handler|
           value = @handlers[handler].call(value)
+        rescue => e
+          raise BadArgument, "Evaluation of #{handler} for #{context}: #{e.message}"
         end
         return value
       end
 
-      # Parse string value as extended value
-      # Use default decoder if none is specified
-      def evaluate_with_default(value)
-        value = [MARKER_START, @default_decoder, MARKER_END, value].join if value.is_a?(String) && value.match(/^#{handler_regex_string}.*$/).nil? && !@default_decoder.nil?
-        return evaluate(value)
-      end
-
       # Find inner extended values
-      def evaluate_all(value)
-        regex = Regexp.new("^(.*)#{handler_regex_string}([^#{MARKER_IN_END}]*)#{MARKER_IN_END}(.*)$", Regexp::MULTILINE)
-        while (m = value.match(regex))
+      # Only used in above lambda
+      def evaluate_extend(value)
+        while (m = value.match(@regex_extend))
           sub_value = "@#{m[2]}:#{m[3]}"
           Log.log.debug{"evaluating #{sub_value}"}
-          value = "#{m[1]}#{evaluate(sub_value)}#{m[4]}"
+          value = "#{m[1]}#{evaluate(sub_value, context: 'composite extended value')}#{m[4]}"
         end
         return value
       end
+      # marker "@"
+      MARKER_START = '@'
+      # marker ":"
+      MARKER_END = ':'
+      # marker "@"
+      MARKER_IN_END = '@'
+
+      # Special handlers stop processing of handlers on right
+      # :extend includes processing of other handlers in itself
+      # :val keeps the value intact
+      SPECIAL_HANDLERS = %i[extend val].freeze
+
+      # Array and Hash types:
+      DEFAULT_PARSER_TYPES = [Array, Hash].freeze
+      private_constant :MARKER_START, :MARKER_END, :MARKER_IN_END, :SPECIAL_HANDLERS, :DEFAULT_PARSER_TYPES
     end
   end
 end

data/lib/aspera/cli/formatter.rb

@@ -7,6 +7,7 @@ require 'aspera/secret_hider'
 require 'aspera/environment'
 require 'aspera/log'
 require 'aspera/assert'
+require 'aspera/markdown'
 require 'terminal-table'
 require 'tty-spinner'
 require 'yaml'
@@ -64,8 +65,23 @@ module Aspera
         end
 
         # used by spec_doc
-        def keyword_highlight(value)
-          value.bold
+        # @param match [MatchData,String]
+        def markdown(match)
+          if match.is_a?(String)
+            match = Markdown::FORMATS.match(match)
+            Aspera.assert(match)
+          end
+          Aspera.assert_type(match, MatchData)
+          if match[:entity]
+            Aspera.assert_values(match[:entity], 'bsol')
+            '\\'
+          elsif match[:bold]
+            match[:bold].to_s.blue
+          elsif match[:code]
+            match[:code].to_s.bold
+          else
+            Aspera.error_unexpected_value(match.to_s)
+          end
         end
 
         # replace empty values with a readable version on terminal
@@ -100,30 +116,49 @@ module Aspera
           end
         end
 
+        # Given a list of string, display that list in a single cell
+        def list_to_string(list)
+          list.join(',')
+        end
+
+        # Build new prefix
+        def add_prefix(prefix, key)
+          [prefix, key].compact.join('.')
+        end
+
+        # Add elements of enumerator to the stack, in reverse order
+        def add_elements(stack, prefix, enum)
+          enum.reverse_each do |key, value|
+            stack.push([add_prefix(prefix, key), value])
+          end
+        end
+
         # Flatten a Hash into single level hash
         def flatten_hash(input)
           Aspera.assert_type(input, Hash)
           return input if input.empty?
           flat = {}
+          # tail (pop,push) contains the next element to display
           stack = [[nil, input]]
           until stack.empty?
             prefix, current = stack.pop
+            # empty things will be displayed as such
             if current.respond_to?(:empty?) && current.empty?
               flat[prefix] = current
               next
             end
             case current
             when Hash
-              current.reverse_each{ |k, v| stack.push([[prefix, k].compact.join('.'), v])}
+              add_elements(stack, prefix, current)
             when Array
-              if current.all?(String)
-                flat[prefix] = current.join("\n")
+              if current.none?{ |i| i.is_a?(Array) || i.is_a?(Hash)}
+                flat[prefix] = list_to_string(current.map(&:to_s))
               elsif current.all?{ |i| i.is_a?(Hash) && i.keys == ['name']}
-                flat[prefix] = current.map{ |i| i['name']}.join(', ')
+                flat[prefix] = list_to_string(current.map{ |i| i['name']})
               elsif current.all?{ |i| i.is_a?(Hash) && i.keys.sort == %w[name value]}
-                stack.push([prefix, current.each_with_object({}){ |i, h| h[i['name']] = i['value']}])
+                add_elements(stack, prefix, current.each_with_object({}){ |i, h| h[i['name']] = i['value']})
               else
-                current.each_with_index.reverse_each{ |v, k| stack.push([[prefix, k].compact.join('.'), v])}
+                add_elements(stack, prefix, current.each_with_index.map{ |v, i| [i, v]})
               end
             else
               flat[prefix] = current
@@ -166,23 +201,23 @@ module Aspera
         else
           {}
         end
-        options.declare(:format, 'Output format', values: DISPLAY_FORMATS, handler: {o: self, m: :option_handler}, default: :table)
-        options.declare(:output, 'Destination for results', types: String, handler: {o: self, m: :option_handler})
-        options.declare(:display, 'Output only some information', values: DISPLAY_LEVELS, handler: {o: self, m: :option_handler}, default: :info)
+        options.declare(:format, 'Output format', allowed: DISPLAY_FORMATS, handler: {o: self, m: :option_handler}, default: :table)
+        options.declare(:output, 'Destination for results', handler: {o: self, m: :option_handler})
+        options.declare(:display, 'Output only some information', allowed: DISPLAY_LEVELS, handler: {o: self, m: :option_handler}, default: :info)
         options.declare(
           :fields, "Comma separated list of: fields, or #{SpecialValues::ALL}, or #{SpecialValues::DEF}", handler: {o: self, m: :option_handler},
-          types: [String, Array, Regexp, Proc],
+          allowed: [String, Array, Regexp, Proc],
           default: SpecialValues::DEF
         )
-        options.declare(:select, 'Select only some items in lists: column, value', types: [Hash, Proc], handler: {o: self, m: :option_handler})
-        options.declare(:table_style, '(Table) Display style', types: [Hash], handler: {o: self, m: :option_handler}, default: default_table_style)
-        options.declare(:flat_hash, '(Table) Display deep values as additional keys', values: :bool, handler: {o: self, m: :option_handler}, default: true)
+        options.declare(:select, 'Select only some items in lists: column, value', allowed: [Hash, Proc], handler: {o: self, m: :option_handler})
+        options.declare(:table_style, '(Table) Display style', allowed: [Hash], handler: {o: self, m: :option_handler}, default: default_table_style)
+        options.declare(:flat_hash, '(Table) Display deep values as additional keys', allowed: Allowed::TYPES_BOOLEAN, handler: {o: self, m: :option_handler}, default: true)
         options.declare(
-          :multi_single, '(Table) Control how object list is displayed as single table, or multiple objects', values: %i[no yes single],
+          :multi_single, '(Table) Control how object list is displayed as single table, or multiple objects', allowed: %i[no yes single],
           handler: {o: self, m: :option_handler}, default: :no
         )
-        options.declare(:show_secrets, 'Show secrets on command output', values: :bool, handler: {o: self, m: :option_handler}, default: false)
-        options.declare(:image, 'Options for image display', types: Hash, handler: {o: self, m: :option_handler}, default: {})
+        options.declare(:show_secrets, 'Show secrets on command output', allowed: Allowed::TYPES_BOOLEAN, handler: {o: self, m: :option_handler}, default: false)
+        options.declare(:image, 'Options for image display', allowed: Hash, handler: {o: self, m: :option_handler}, default: {})
       end
 
       # method accessed by option manager
@@ -326,8 +361,7 @@ module Aspera
             end
           when :object_list
             # :object_list is an Array of Hash, where key=column name
-            Aspera.assert_type(data, Array)
-            Aspera.assert(data.all?(Hash)){"expecting Array of Hash: #{data.inspect}"}
+            Aspera.assert_array_all(data, Hash){'result'}
             data = data.map{ |obj| self.class.flatten_hash(obj)} if @options[:flat_hash]
             display_table(data, compute_fields(data, fields), single: type.eql?(:single_object))
           when :value_list
@@ -406,8 +440,7 @@ module Aspera
       def filter_list_on_fields(data)
         # by default, keep all data intact
         return data if @options[:fields].eql?(SpecialValues::DEF) && @options[:select].nil?
-        Aspera.assert_type(data, Array){'Filtering fields or select requires result is an Array of Hash'}
-        Aspera.assert(data.all?(Hash)){'Filtering fields or select requires result is an Array of Hash'}
+        Aspera.assert_array_all(data, Hash){'filter or select'}
         filter_columns_on_select(data)
         return data if @options[:fields].eql?(SpecialValues::DEF)
         selected_fields = compute_fields(data, @options[:fields])

data/lib/aspera/cli/main.rb

@@ -214,6 +214,7 @@ module Aspera
         rescue Net::SSH::AuthenticationFailed => e; exception_info = {e: e, t: 'SSH', security: true}
         rescue OpenSSL::SSL::SSLError => e;         exception_info = {e: e, t: 'SSL'}
         rescue Cli::BadArgument => e;               exception_info = {e: e, t: 'Argument', usage: true}
+        rescue Cli::MissingArgument => e;           exception_info = {e: e, t: 'Missing', usage: true}
         rescue Cli::BadIdentifier => e;             exception_info = {e: e, t: 'Identifier'}
         rescue Cli::Error => e;                     exception_info = {e: e, t: 'Tool', usage: true}
         rescue Transfer::Error => e;                exception_info = {e: e, t: 'Transfer'}
@@ -228,6 +229,7 @@ module Aspera
         unless exception_info.nil?
           Log.log.warn(exception_info[:e].message) if Log.instance.logger_type.eql?(:syslog) && exception_info[:security]
           @context.formatter.display_message(:error, "#{Formatter::ERROR_FLASH} #{exception_info[:t]}: #{exception_info[:e].message}")
+          Log.log.debug{(['Backtrace:'] + exception_info[:e].backtrace).join("\n")} if exception_info[:debug]
           @context.formatter.display_message(:error, 'Use option -h to get help.') if exception_info[:usage]
           # Is that a known error condition with proposal for remediation ?
           Hints.hint_for(exception_info[:e], @context.formatter)
@@ -287,7 +289,6 @@ module Aspera
         @context.options = Manager.new(Info::CMD_NAME, @argv)
         # Formatter adds options
         @context.formatter.declare_options(@context.options)
-        ExtendedValue.instance.default_decoder = @context.options.get_option(:struct_parser)
         # Compare $0 with expected name
         current_prog_name = File.basename($PROGRAM_NAME)
         @context.formatter.display_message(
@@ -349,28 +350,34 @@ module Aspera
       # Define header for manual
       def declare_global_options
         Log.log.debug('declare_global_options')
-        @context.options.declare(:help, 'Show this message', values: :none, short: 'h'){@option_help = true}
-        @context.options.declare(:bash_comp, 'Generate bash completion for command', values: :none){@bash_completion = true}
-        @context.options.declare(:show_config, 'Display parameters used for the provided action', values: :none){@option_show_config = true}
-        @context.options.declare(:version, 'Display version', values: :none, short: 'v'){@context.formatter.display_message(:data, Cli::VERSION); Process.exit(0)} # rubocop:disable Style/Semicolon
+        @context.options.declare(:help, 'Show this message', allowed: Allowed::TYPES_NONE, short: 'h'){@option_help = true}
+        @context.options.declare(:bash_comp, 'Generate bash completion for command', allowed: Allowed::TYPES_NONE){@bash_completion = true}
+        @context.options.declare(:show_config, 'Display parameters used for the provided action', allowed: Allowed::TYPES_NONE){@option_show_config = true}
+        @context.options.declare(:version, 'Display version', allowed: Allowed::TYPES_NONE, short: 'v'){@context.formatter.display_message(:data, Cli::VERSION); Process.exit(0)} # rubocop:disable Style/Semicolon
         @context.options.declare(
           :ui, 'Method to start browser',
-          values: USER_INTERFACES,
+          allowed: USER_INTERFACES,
           handler: {o: Environment.instance, m: :url_method}
         )
         @context.options.declare(
           :invalid_characters, 'Replacement character and invalid filename characters',
           handler: {o: Environment.instance, m: :file_illegal_characters}
         )
-        @context.options.declare(:log_level, 'Log level', values: Log::LEVELS, handler: {o: Log.instance, m: :level})
-        @context.options.declare(:log_format, 'Log formatter', types: [Proc, Logger::Formatter, String], handler: {o: Log.instance, m: :formatter})
-        @context.options.declare(:logger, 'Logging method', values: Log::LOG_TYPES, handler: {o: Log.instance, m: :logger_type})
-        @context.options.declare(:lock_port, 'Prevent dual execution of a command, e.g. in cron', coerce: Integer, types: Integer)
-        @context.options.declare(:once_only, 'Process only new items (some commands)', values: :bool, default: false)
-        @context.options.declare(:log_secrets, 'Show passwords in logs', values: :bool, handler: {o: SecretHider.instance, m: :log_secrets})
-        @context.options.declare(:clean_temp, 'Cleanup temporary files on exit', values: :bool, handler: {o: TempFileManager.instance, m: :cleanup_on_exit})
+        @context.options.declare(:log_level, 'Log level', allowed: Log::LEVELS, handler: {o: Log.instance, m: :level})
+        @context.options.declare(:log_format, 'Log formatter', allowed: [Proc, Logger::Formatter, String], handler: {o: Log.instance, m: :formatter})
+        @context.options.declare(:logger, 'Logging method', allowed: Log::LOG_TYPES, handler: {o: Log.instance, m: :logger_type})
+        @context.options.declare(:lock_port, 'Prevent dual execution of a command, e.g. in cron', allowed: Allowed::TYPES_INTEGER)
+        @context.options.declare(:once_only, 'Process only new items (some commands)', allowed: Allowed::TYPES_BOOLEAN, default: false)
+        @context.options.declare(:log_secrets, 'Show passwords in logs', allowed: Allowed::TYPES_BOOLEAN, handler: {o: SecretHider.instance, m: :log_secrets})
+        @context.options.declare(:clean_temp, 'Cleanup temporary files on exit', allowed: Allowed::TYPES_BOOLEAN, handler: {o: TempFileManager.instance, m: :cleanup_on_exit})
         @context.options.declare(:temp_folder, 'Temporary folder', handler: {o: TempFileManager.instance, m: :global_temp})
-        @context.options.declare(:pid_file, 'Write process identifier to file, delete on exit', types: String)
+        @context.options.declare(:pid_file, 'Write process identifier to file, delete on exit')
+        @context.options.declare(
+          :parser, 'Default parser for structured parameters and options',
+          handler: {o: ExtendedValue.instance, m: :default_decoder},
+          allowed: ExtendedValue::DEFAULT_DECODERS,
+          default: ExtendedValue::DEFAULT_DECODERS.first
+        )
         # Parse declared options
         @context.options.parse_options!
       end