aspera-cli 4.17.0 → 4.18.0

data/lib/aspera/cli/formatter.rb

@@ -1,11 +1,13 @@
 # frozen_string_literal: true
 
 # cspell:ignore jsonpp
+require 'aspera/preview/terminal'
 require 'aspera/secret_hider'
 require 'aspera/environment'
 require 'aspera/log'
 require 'aspera/assert'
 require 'terminal-table'
+require 'tty-spinner'
 require 'yaml'
 require 'pp'
 
@@ -14,8 +16,9 @@ module Aspera
     CONF_OVERVIEW_KEYS = %w[preset parameter value].freeze
     # This class is used to transform a complex structure into a simple hash
     class Flattener
-      def initialize
+      def initialize(formatter)
         @result = nil
+        @formatter = formatter
       end
 
       # General method
@@ -48,9 +51,9 @@ module Aspera
         elsif something.is_a?(Array)
           flatten_array(something, name)
         elsif something.is_a?(String) && something.empty?
-          @result[name] = Formatter.special('empty string')
+          @result[name] = @formatter.special_format('empty string')
         elsif something.nil?
-          @result[name] = Formatter.special('null')
+          @result[name] = @formatter.special_format('null')
         # elsif something.eql?(true) || something.eql?(false)
         #  @result[name] = something
         else
@@ -63,7 +66,7 @@ module Aspera
       # @param name [String] name of englobing key
       def flatten_array(array, name)
         if array.empty?
-          @result[name] = Formatter.special('empty list')
+          @result[name] = @formatter.special_format('empty list')
         elsif array.all?(String)
           @result[name] = array.join("\n")
         elsif array.all?{|i| i.is_a?(Hash) && i.keys.eql?(%w[name])}
@@ -85,18 +88,18 @@ module Aspera
           flatten_any(v, "#{prefix}#{k}")
         end
       end
-    end # class
+    end
 
     # Take care of output
     class Formatter
+      # remove a fields from the list
       FIELDS_LESS = '-'
       CSV_RECORD_SEPARATOR = "\n"
       CSV_FIELD_SEPARATOR = ','
       # supported output formats
-      DISPLAY_FORMATS = %i[text nagios ruby json jsonpp yaml table csv].freeze
+      DISPLAY_FORMATS = %i[text nagios ruby json jsonpp yaml table csv image].freeze
       # user output levels
       DISPLAY_LEVELS = %i[info data error].freeze
-      RESULT_PARAMS = %i[type data total fields name].freeze
 
       private_constant :DISPLAY_FORMATS, :DISPLAY_LEVELS, :CSV_RECORD_SEPARATOR, :CSV_FIELD_SEPARATOR
       # prefix to display error messages in user messages (terminal)
@@ -105,19 +108,6 @@ module Aspera
       HINT_FLASH = 'HINT:'.bg_green.gray.blink.freeze
 
       class << self
-        # Highlight special values
-        def special(what, use_colors: $stdout.isatty)
-          result = $stdout.isatty ? "<#{what}>" : "&lt;#{what}&gt;"
-          if use_colors
-            result = if %w[null empty].any?{|s|what.include?(s)}
-              result.dim
-            else
-              result.reverse_color
-            end
-          end
-          return result
-        end
-
         def all_but(list)
           list = [list] unless list.is_a?(Array)
           return list.map{|i|"#{FIELDS_LESS}#{i}"}.unshift(ExtendedValue::ALL)
@@ -125,7 +115,7 @@ module Aspera
 
         def tick(yes)
           result =
-            if Environment.use_unicode?
+            if Environment.terminal_supports_unicode?
               if yes
                 "\u2713"
               else
@@ -150,24 +140,55 @@ module Aspera
           end
           return result
         end
-      end # self
+      end
 
       # initialize the formatter
       def initialize
         @options = {}
+        @spinner = nil
+      end
+
+      # Highlight special values
+      def special_format(what, use_colors: $stdout.isatty)
+        result = $stdout.isatty ? "<#{what}>" : "&lt;#{what}&gt;"
+        if use_colors
+          result = if %w[null empty].any?{|s|what.include?(s)}
+            result.dim
+          else
+            result.reverse_color
+          end
+        end
+        return result
       end
 
+      # call this after REST calls if several api calls are expected
+      def long_operation_running(title = '')
+        return unless Environment.terminal?
+        if @spinner.nil?
+          @spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
+          @spinner.start
+        end
+        @spinner.update(title: title)
+        @spinner.spin
+      end
+
+      # options are: format, output, display, fields, select, table_style, flat_hash, transpose_single
       def option_handler(option_symbol, operation, value=nil)
         Aspera.assert_values(operation, %i[set get])
         case operation
         when :set
           @options[option_symbol] = value
-          if option_symbol.eql?(:output)
+          case option_symbol
+          when :output
             $stdout = if value.eql?('-')
               STDOUT # rubocop:disable Style/GlobalStdStream
             else
               File.open(value, 'w')
             end
+          when :image
+            allowed_options = Preview::Terminal.method(:build).parameters.select{|i|i[0].eql?(:key)}.map{|i|i[1]}
+            unknown_options = value.keys.map(&:to_sym) - allowed_options
+            raise "Invalid parameter(s) for option image: #{unknown_options.join(', ')}, use #{allowed_options.join(', ')}" unless unknown_options.empty?
           end
         when :get then return @options[option_symbol]
         else Aspera.error_unreachable_line
@@ -176,6 +197,12 @@ module Aspera
       end
 
       def declare_options(options)
+        default_table_style = if Environment.terminal_supports_unicode?
+          {border: :unicode_round}
+        else
+          {}
+        end
+
         options.declare(:format, 'Output format', values: DISPLAY_FORMATS, handler: {o: self, m: :option_handler}, default: :table)
         options.declare(:output, 'Destination for results', types: String, handler: {o: self, m: :option_handler})
         options.declare(:display, 'Output only some information', values: DISPLAY_LEVELS, handler: {o: self, m: :option_handler}, default: :info)
@@ -184,10 +211,11 @@ module Aspera
           types: [String, Array, Regexp, Proc],
           default: ExtendedValue::DEF)
         options.declare(:select, 'Select only some items in lists: column, value', types: [Hash, Proc], handler: {o: self, m: :option_handler})
-        options.declare(:table_style, 'Table display style', handler: {o: self, m: :option_handler}, default: ':.:')
+        options.declare(:table_style, 'Table display style', types: [Hash], handler: {o: self, m: :option_handler}, default: default_table_style)
         options.declare(:flat_hash, 'Display deep values as additional keys', values: :bool, handler: {o: self, m: :option_handler}, default: true)
         options.declare(:transpose_single, 'Single object fields output vertically', values: :bool, handler: {o: self, m: :option_handler}, default: true)
         options.declare(:show_secrets, 'Show secrets on command output', values: :bool, handler: {o: self, m: :option_handler}, default: false)
+        options.declare(:image, 'Options for image display', types: Hash, handler: {o: self, m: :option_handler}, default: {})
       end
 
       # main output method
@@ -220,14 +248,15 @@ module Aspera
         data.each_with_object({}){|v, m|v.each_key{|c|m[c] = true}}.keys
       end
 
-      # this method computes the list of fields to display
-      # data: array of hash
-      # default: list of fields to display by default (may contain special values)
+      # @return the list of fields to display
+      # @param data [Array<Hash>] data to display
+      # @param default [Array<String>, Proc] list of fields to display by default (may contain special values)
       def compute_fields(data, default)
         Log.log.debug{"compute_fields: data:#{data.class} default:#{default.class} #{default}"}
+        # the requested list of fields, but if can contain special values
         request =
           case @options[:fields]
-          when NilClass then [ExtendedValue::DEF]
+          # when NilClass then [ExtendedValue::DEF]
           when String then @options[:fields].split(',')
           when Array then @options[:fields]
           when Regexp then return all_fields(data).select{|i|i.match(@options[:fields])}
@@ -261,20 +290,39 @@ module Aspera
         return result
       end
 
+      # filter the list of items on the fields option
+      def filter_list_on_fields(data)
+        # by default, keep all data intact
+        return data if @options[:fields].eql?(ExtendedValue::DEF) && @options[:select].nil?
+        Aspera.assert_type(data, Array){'Filtering fields or select requires result is an Array of Hash'}
+        Aspera.assert(data.all?(Hash)){'Filtering fields or select requires result is an Array of Hash'}
+        filter_columns_on_select(data)
+        return data if @options[:fields].eql?(ExtendedValue::DEF)
+        selected_fields = compute_fields(data, @options[:fields])
+        return data.map{|i|i[selected_fields.first]} if selected_fields.length == 1
+        return data.map{|i|i.select{|k, _|selected_fields.include?(k)}}
+      end
+
+      # filter the list of items on the select option
+      # @param data [Array<Hash>] list of items
+      def filter_columns_on_select(data)
+        case @options[:select]
+        when Proc
+          data.select!{|i|@options[:select].call(i)}
+        when Hash
+          @options[:select].each{|k, v|data.select!{|i|i[k].eql?(v)}}
+        end
+      end
+
       # this method displays a table
       # object_array: array of hash
       # fields: list of column names
       def display_table(object_array, fields)
         Aspera.assert(!fields.nil?){'missing fields parameter'}
-        case @options[:select]
-        when Proc
-          object_array.select!{|i|@options[:select].call(i)}
-        when Hash
-          @options[:select].each{|k, v|object_array.select!{|i|i[k].eql?(v)}}
-        end
+        filter_columns_on_select(object_array)
         if object_array.empty?
           # no  display for csv
-          display_message(:info, Formatter.special('empty')) if @options[:format].eql?(:table)
+          display_message(:info, special_format('empty')) if @options[:format].eql?(:table)
           return
         end
         if object_array.length == 1 && fields.length == 1
@@ -294,72 +342,114 @@ module Aspera
         # here : fields : list of column names
         case @options[:format]
         when :table
-          style = @options[:table_style].chars
           # display the table !
           display_message(:data, Terminal::Table.new(
             headings:  fields,
             rows:      final_table_rows,
-            border_x:  style[0],
-            border_y:  style[1],
-            border_i:  style[2]))
+            style:     @options[:table_style]&.symbolize_keys))
         when :csv
           display_message(:data, final_table_rows.map{|t| t.join(CSV_FIELD_SEPARATOR)}.join(CSV_RECORD_SEPARATOR))
         end
       end
 
+      # @return text suitable to display an image from url
+      def status_image(blob)
+        begin
+          raise URI::InvalidURIError, 'not uri' if !(blob =~ /\A#{URI::DEFAULT_PARSER.make_regexp}\z/)
+          # it's a url
+          url = blob
+          unless OpenApplication.instance.url_method.eql?(:text)
+            OpenApplication.instance.uri(url)
+            return ''
+          end
+          # remote_image = Rest.new(base_url: url).read('')
+          # mime = remote_image[:http]['content-type']
+          # blob = remote_image[:http].body
+          # Log.log.warn("Image ? #{remote_image[:http]['content-type']}") unless mime.include?('image/')
+          blob = UriReader.read(url)
+        rescue URI::InvalidURIError
+          nil
+        end
+        # try base64
+        begin
+          blob = Base64.strict_decode64(blob)
+        rescue
+          nil
+        end
+        return Preview::Terminal.build(blob, **@options[:image].symbolize_keys)
+      end
+
       # this method displays the results, especially the table format
-      def display_results(results)
-        Aspera.assert_type(results, Hash)
-        Aspera.assert((results.keys - RESULT_PARAMS).empty?){"result unsupported key: #{results.keys - RESULT_PARAMS}"}
-        Aspera.assert(results.key?(:type)){"result must have type (#{results})"}
-        Aspera.assert(results.key?(:data) || %i[empty nothing].include?(results[:type])){'result must have data'}
-        Log.log.debug{"display_results: #{results[:data].class} #{results[:type]}"}
-        display_item_count(results[:data].length, results[:total]) if results.key?(:total)
-        SecretHider.deep_remove_secret(results[:data]) unless @options[:show_secrets] || @options[:display].eql?(:data)
+      # @param type [Symbol] type of data
+      # @param data [Object] data to display
+      # @param total [Integer] total number of items
+      # @param fields [Array<String>] list of fields to display
+      # @param name [String] name of the column to display
+      def display_results(type:, data: nil, total: nil, fields: nil, name: nil)
+        Log.log.debug{"display_results: #{type} class=#{data.class} data=#{data}"}
+        Aspera.assert_type(type, Symbol){'result must have type'}
+        Aspera.assert(!data.nil? || %i[empty nothing].include?(type)){'result must have data'}
+        display_item_count(data.length, total) unless total.nil?
+        SecretHider.deep_remove_secret(data) unless @options[:show_secrets] || @options[:display].eql?(:data)
         case @options[:format]
         when :text
-          display_message(:data, results[:data].to_s)
+          display_message(:data, data.to_s)
         when :nagios
-          Nagios.process(results[:data])
+          Nagios.process(data)
         when :ruby
-          display_message(:data, PP.pp(results[:data], +''))
+          display_message(:data, PP.pp(filter_list_on_fields(data), +''))
         when :json
-          display_message(:data, JSON.generate(results[:data]))
+          display_message(:data, JSON.generate(filter_list_on_fields(data)))
         when :jsonpp
-          display_message(:data, JSON.pretty_generate(results[:data]))
+          display_message(:data, JSON.pretty_generate(filter_list_on_fields(data)))
         when :yaml
-          display_message(:data, results[:data].to_yaml)
+          display_message(:data, YAML.dump(filter_list_on_fields(data)))
+        when :image
+          # assume it is an url
+          url = data
+          case type
+          when :single_object, :object_list
+            url = [url] if type.eql?(:single_object)
+            raise 'image display requires a single result' unless url.length == 1
+            fields = compute_fields(url, fields)
+            raise 'select a field to display' unless fields.length == 1
+            url = url.first
+            raise 'no such field' unless url.key?(fields.first)
+            url = url[fields.first]
+          end
+          raise "not url: #{url.class} #{url}" unless url.is_a?(String)
+          display_message(:data, status_image(url))
         when :table, :csv
-          case results[:type]
+          case type
           when :config_over
-            display_table(Flattener.new.config_over(results[:data]), CONF_OVERVIEW_KEYS)
+            display_table(Flattener.new(self).config_over(data), CONF_OVERVIEW_KEYS)
           when :object_list, :single_object
-            obj_list = results[:data]
-            obj_list = [obj_list] if results[:type].eql?(:single_object)
+            obj_list = data
+            obj_list = [obj_list] if type.eql?(:single_object)
             Aspera.assert_type(obj_list, Array)
             Aspera.assert(obj_list.all?(Hash)){"expecting Array of Hash: #{obj_list.inspect}"}
             # :object_list is an array of hash tables, where key=colum name
-            obj_list = obj_list.map{|obj|Flattener.new.flatten(obj)} if @options[:flat_hash]
-            display_table(obj_list, compute_fields(obj_list, results[:fields]))
+            obj_list = obj_list.map{|obj|Flattener.new(self).flatten(obj)} if @options[:flat_hash]
+            display_table(obj_list, compute_fields(obj_list, fields))
           when :value_list
             # :value_list is a simple array of values, name of column provided in the :name
-            display_table(results[:data].map { |i| { results[:name] => i } }, [results[:name]])
+            display_table(data.map { |i| { name => i } }, [name])
           when :empty # no table
-            display_message(:info, Formatter.special('empty'))
+            display_message(:info, special_format('empty'))
             return
           when :nothing # no result expected
             Log.log.debug('no result expected')
           when :status # no table
             # :status displays a simple message
-            display_message(:info, results[:data])
+            display_message(:info, data)
           when :text # no table
             # :status displays a simple message
-            display_message(:data, results[:data])
+            display_message(:data, data)
           when :other_struct # no table
             # :other_struct is any other type of structure
-            display_message(:data, PP.pp(results[:data], +''))
+            display_message(:data, PP.pp(data, +''))
           else
-            raise "unknown data type: #{results[:type]}"
+            raise "unknown data type: #{type}"
           end
         end
       end

data/lib/aspera/cli/hints.rb

@@ -47,6 +47,24 @@ module Aspera
             'if you provide a path: prefix with @file:',
             'e.g. --private-key=@file:/path/to/key.pem'
           ]
+        },
+        {
+          exception:   RuntimeError,
+          match:       /unexpected last event type: INIT/,
+          remediation: [
+            'ascp exited unexpectedly',
+            'it might be due to SSH handshake failure',
+            'one can check SSH connection using ssh command'
+          ]
+        },
+        {
+          exception:   RuntimeError,
+          match:       /Wrong remote host:/,
+          remediation: [
+            'If remote node is Cloud Pak For Integration',
+            'Make sure that a LoadBalancer is active on cluster',
+            'Check the external address of Aspera tcp-proxy pod'
+          ]
         }
       ]
 

data/lib/aspera/cli/main.rb

@@ -9,7 +9,6 @@ require 'aspera/cli/transfer_agent'
 require 'aspera/cli/version'
 require 'aspera/cli/info'
 require 'aspera/cli/hints'
-require 'aspera/preview/terminal'
 require 'aspera/secret_hider'
 require 'aspera/log'
 require 'aspera/assert'
@@ -20,9 +19,10 @@ module Aspera
     class Main
       # Plugins store transfer result using this key and use result_transfer_multiple()
       STATUS_FIELD = 'status'
-      CONF_PLUGIN_SYM = :config
+      COMMAND_CONFIG = :config
+      COMMAND_HELP = :help
 
-      private_constant :CONF_PLUGIN_SYM
+      private_constant :COMMAND_CONFIG, :COMMAND_HELP
 
       class << self
         # expect some list, but nothing to display
@@ -60,14 +60,10 @@ module Aspera
           return {type: :object_list, data: status_table}
         end
 
-        def result_picture_in_terminal(options, blob)
-          terminal_options = options.get_option(:query, default: {}).symbolize_keys
-          allowed_options = Preview::Terminal.method(:build).parameters.select{|i|i[0].eql?(:key)}.map{|i|i[1]}
-          unknown_options = terminal_options.keys - allowed_options
-          raise "invalid options: #{unknown_options.join(', ')}, use #{allowed_options.join(', ')}" unless unknown_options.empty?
-          return Main.result_status(Preview::Terminal.build(blob, **terminal_options))
+        def result_image(blob, formatter:)
+          return Main.result_status(formatter.status_image(blob))
         end
-      end # self
+      end
 
       private
 
@@ -88,6 +84,8 @@ module Aspera
         @option_help = false
         @option_show_config = false
         @bash_completion = false
+        early_debug_setup
+        Log.log.trace2{Log.dump(:argv, @argv)}
       end
 
       # This can throw exception if there is a problem with the environment, needs to be caught by execute method
@@ -95,8 +93,6 @@ module Aspera
         @plug_init[:only_manual] = false
         # create formatter, in case there is an exception, it is used to display.
         @plug_init[:formatter] = Formatter.new
-        # second : manage debug level (allows debugging of option parser)
-        early_debug_setup
         @plug_init[:options] = Manager.new(PROGRAM_NAME)
         # give command line arguments to option manager
         options.parse_command_line(@argv)
@@ -164,10 +160,9 @@ module Aspera
         options.declare(:bash_comp, 'Generate bash completion for command', values: :none) { @bash_completion = true }
         options.declare(:show_config, 'Display parameters used for the provided action', values: :none) { @option_show_config = true }
         options.declare(:version, 'Display version', values: :none, short: 'v') { formatter.display_message(:data, Cli::VERSION); Process.exit(0) } # rubocop:disable Style/Semicolon, Layout/LineLength
-        options.declare(:warnings, 'Check for language warnings', values: :none, short: 'w') { $VERBOSE = true }
         options.declare(
           :ui, 'Method to start browser',
-          values: OpenApplication.user_interfaces,
+          values: OpenApplication::USER_INTERFACES,
           handler: {o: OpenApplication.instance, m: :url_method},
           default: OpenApplication.default_gui_mode)
         options.declare(:log_level, 'Log level', values: Log.levels, handler: {o: Log.instance, m: :level})
@@ -187,17 +182,15 @@ module Aspera
       def get_plugin_instance_with_options(plugin_name_sym, env=nil)
         env ||= @plug_init
         Log.log.debug{"get_plugin_instance_with_options(#{plugin_name_sym})"}
-        require PluginFactory.instance.plugins[plugin_name_sym][:require_stanza]
         # load default params only if no param already loaded before plugin instantiation
         env[:config].add_plugin_default_preset(plugin_name_sym)
         command_plugin = PluginFactory.instance.create(plugin_name_sym, **env)
-        Log.log.debug{"got #{command_plugin.class}"}
         return command_plugin
       end
 
       def generate_bash_completion
         if options.get_next_argument('', expected: :multiple, mandatory: false).nil?
-          PluginFactory.instance.plugins.each_key{|p|puts p}
+          PluginFactory.instance.plugin_list.each{|p|puts p}
         else
           Log.log.warn('only first level completion so far')
         end
@@ -210,8 +203,8 @@ module Aspera
         formatter.display_message(:error, options.parser)
         if all_plugins
           # list plugins that have a "require" field, i.e. all but main plugin
-          PluginFactory.instance.plugins.each_key do |plugin_name_sym|
-            next if plugin_name_sym.eql?(CONF_PLUGIN_SYM)
+          PluginFactory.instance.plugin_list.each do |plugin_name_sym|
+            next if plugin_name_sym.eql?(COMMAND_CONFIG)
             # override main option parser with a brand new, to avoid having global options
             plugin_env = @plug_init.clone
             plugin_env[:only_manual] = true # force declaration of all options
@@ -261,17 +254,17 @@ module Aspera
           config.periodic_check_newer_gem_version
           command_sym =
             if @option_show_config && options.command_or_arg_empty?
-              CONF_PLUGIN_SYM
+              COMMAND_CONFIG
             else
-              options.get_next_command(PluginFactory.instance.plugins.keys.dup.unshift(:help))
+              options.get_next_command(PluginFactory.instance.plugin_list.unshift(COMMAND_HELP))
             end
           # command will not be executed, but we need manual
           options.fail_on_missing_mandatory = false if @option_help || @option_show_config
           # main plugin is not dynamically instantiated
           case command_sym
-          when :help
+          when COMMAND_HELP
             exit_with_usage(true)
-          when CONF_PLUGIN_SYM
+          when COMMAND_CONFIG
             command_plugin = config
           else
             # get plugin, set options, etc
@@ -282,7 +275,7 @@ module Aspera
           # help requested for current plugin
           exit_with_usage(false) if @option_help
           if @option_show_config
-            formatter.display_results({type: :single_object, data: options.known_options(only_defined: true).stringify_keys})
+            formatter.display_results(type: :single_object, data: options.known_options(only_defined: true).stringify_keys)
             execute_command = false
           end
           # locking for single execution (only after "per plugin" option, in case lock port is there)
@@ -304,7 +297,7 @@ module Aspera
             at_exit{File.delete(pid_file)}
           end
           # execute and display (if not exclusive execution)
-          formatter.display_results(command_plugin.execute_action) if execute_command
+          formatter.display_results(**command_plugin.execute_action) if execute_command
           # save config file if command modified it
           config.save_config_file_if_needed
           # finish
@@ -347,7 +340,7 @@ module Aspera
           Process.exit(1)
         end
         return nil
-      end # process_command_line
-    end # Main
-  end # Cli
-end # Aspera
+      end
+    end
+  end
+end