aspera-cli 4.14.0 → 4.15.0

data/lib/aspera/cli/main.rb

@@ -7,32 +7,17 @@ require 'aspera/cli/extended_value'
 require 'aspera/cli/transfer_agent'
 require 'aspera/cli/version'
 require 'aspera/cli/info'
-require 'aspera/fasp/error'
-require 'aspera/open_application'
-require 'aspera/temp_file_manager'
-require 'aspera/persistency_folder'
-require 'aspera/log'
-require 'aspera/rest'
-require 'aspera/nagios'
-require 'aspera/colors'
+require 'aspera/cli/hints'
 require 'aspera/secret_hider'
-require 'net/ssh'
+require 'aspera/log'
 
 module Aspera
   module Cli
     # The main CLI class
     class Main
-      # prefix to display error messages in user messages (terminal)
-      ERROR_FLASH = 'ERROR:'.bg_red.gray.blink.freeze
-      WARNING_FLASH = 'WARNING:'.bg_red.gray.blink.freeze
-      private_constant :ERROR_FLASH, :WARNING_FLASH
-
-      # store transfer result using this key and use result_transfer_multiple
+      # Plugins store transfer result using this key and use result_transfer_multiple()
       STATUS_FIELD = 'status'
 
-      # for testing only
-      SELF_SIGNED_CERT = OpenSSL::SSL.const_get(:enon_yfirev.to_s.upcase.reverse) # cspell: disable-line
-
       class << self
         # expect some list, but nothing to display
         def result_empty; return {type: :empty, data: :nil}; end
@@ -68,85 +53,55 @@ module Aspera
           raise global_status unless global_status.eql?(:success)
           return {type: :object_list, data: status_table}
         end
-      end
+      end # self
 
       private
 
+      # shortcuts helpers like in plugins
+      %i[options transfer config formatter persistency].each do |name|
+        define_method(name){@agents[name]}
+      end
+
       # =============================================================
       # Parameter handlers
       #
-      attr_accessor :option_insecure, :option_http_options, :option_cache_tokens
-
-      def option_ui; OpenApplication.instance.url_method; end
-
-      def option_ui=(value); OpenApplication.instance.url_method = value; end
-
-      # called every time a new REST HTTP session is opened
-      # @param http [Net::HTTP] the newly created http session object
-      def http_parameters=(http)
-        if @option_insecure
-          url = http.inspect.gsub(/^[^ ]* /, 'https://').gsub(/ [^ ]*$/, '')
-          if !@ssl_warned_urls.include?(url)
-            @formatter.display_message(:error, "#{WARNING_FLASH} ignoring certificate for: #{url}. Do not deactivate certificate verification in production.")
-            @ssl_warned_urls.push(url)
-          end
-          http.verify_mode = SELF_SIGNED_CERT
-        end
-        http.set_debug_output($stdout) if @option_rest_debug
-        raise 'http_options expects Hash' unless @option_http_options.is_a?(Hash)
-
-        @option_http_options.each do |k, v|
-          method = "#{k}=".to_sym
-          # check if accessor is a method of Net::HTTP
-          # continue_timeout= read_timeout= write_timeout=
-          if http.respond_to?(method)
-            http.send(method, v)
-          else
-            Log.log.error{"no such attribute: #{k}"}
-          end
-        end
-      end
 
-      # minimum initialization
+      # minimum initialization, no exception raised
       def initialize(argv)
-        # first thing : manage debug level (allows debugging of option parser)
-        early_debug_setup(argv)
-        # compare $0 with expected name
-        current_prog_name = File.basename($PROGRAM_NAME)
-        @formatter.display_message(:error, "#{'WARNING'.bg_red.blink.gray} Please use '#{PROGRAM_NAME}' instead of '#{current_prog_name}'") \
-          unless current_prog_name.eql?(PROGRAM_NAME)
+        @argv = argv
+        # environment provided to plugin for various capabilities
+        @agents = {}
         @option_help = false
-        @bash_completion = false
         @option_show_config = false
-        @option_insecure = false
-        @option_rest_debug = false
-        @option_cache_tokens = true
-        @option_http_options = {}
-        @ssl_warned_urls = []
-        # environment provided to plugin for various capabilities
-        @plugin_env = {}
+        @bash_completion = false
+      end
+
+      # This can throw exception if there is a problem with the environment, needs to be caught by execute method
+      def init_agents_and_options
+        # first thing : manage debug level (allows debugging of option parser)
+        early_debug_setup
+        @agents[:formatter] = Formatter.new
+        @agents[:options] = Manager.new(PROGRAM_NAME)
         # give command line arguments to option manager
-        @plugin_env[:options] = @opt_mgr = Manager.new(PROGRAM_NAME, argv: argv)
+        options.parse_command_line(@argv)
         # formatter adds options
-        @formatter = @plugin_env[:formatter] = Formatter.new(@plugin_env[:options])
-        Rest.user_agent = PROGRAM_NAME
-        Rest.session_cb = lambda{|http|self.http_parameters = http}
+        formatter.declare_options(options)
+        # compare $0 with expected name
+        current_prog_name = File.basename($PROGRAM_NAME)
+        formatter.display_message(
+          :error,
+          "#{Formatter::WARNING_FLASH} Please use '#{PROGRAM_NAME}' instead of '#{current_prog_name}'") unless current_prog_name.eql?(PROGRAM_NAME)
         # declare and parse global options
-        init_global_options
+        declare_global_options
         # the Config plugin adds the @preset parser, so declare before TransferAgent which may use it
-        @plugin_env[:config] = Plugins::Config.new(@plugin_env, gem: GEM_NAME, name: PROGRAM_NAME, help: DOC_URL, version: Aspera::Cli::VERSION)
-        # the TransferAgent plugin may use the @preset parser
-        @plugin_env[:transfer] = TransferAgent.new(@plugin_env[:options], @plugin_env[:config])
+        @agents[:config] = Plugins::Config.new(@agents, gem: GEM_NAME, name: PROGRAM_NAME, help: DOC_URL, version: Aspera::Cli::VERSION)
         # data persistency
-        @plugin_env[:persistency] = PersistencyFolder.new(File.join(@plugin_env[:config].main_folder, 'persist_store'))
+        raise 'internal error: missing persistency object' unless @agents[:persistency]
+        # the TransferAgent plugin may use the @preset parser
+        @agents[:transfer] = TransferAgent.new(options, config)
         Log.log.debug('plugin env created'.red)
-        Oauth.persist_mgr = @plugin_env[:persistency] if @option_cache_tokens
-        Fasp::Parameters.file_list_folder = File.join(@plugin_env[:config].main_folder, 'filelists')
-        Aspera::RestErrorAnalyzer.instance.log_file = File.join(@plugin_env[:config].main_folder, 'rest_exceptions.log')
-        # register aspera REST call error handlers
-        Aspera::RestErrorsAspera.register_handlers
         # set banner when all environment is created so that additional extended value modifiers are known, e.g. @preset
-        @opt_mgr.parser.banner = app_banner
+        options.parser.banner = app_banner
       end
 
       def app_banner
@@ -166,7 +121,6 @@ module Aspera
           #{t}source repo: #{SRC_URL}
 
           ENVIRONMENT VARIABLES
-          #{t}#{@plugin_env[:config].conf_dir_env_var} config folder, default: $HOME/#{Plugins::Config::ASPERA_HOME_FOLDER_NAME}/#{PROGRAM_NAME}
           #{t}Any option can be set as an environment variable, refer to the manual
 
           COMMANDS
@@ -185,38 +139,36 @@ module Aspera
       end
 
       # define header for manual
-      def init_global_options
-        Log.log.debug('init_global_options')
-        @opt_mgr.declare(:help, 'Show this message', values: :none, short: 'h') { @option_help = true }
-        @opt_mgr.declare(:bash_comp, 'Generate bash completion for command', values: :none) { @bash_completion = true }
-        @opt_mgr.declare(:show_config, 'Display parameters used for the provided action', values: :none) { @option_show_config = true }
-        @opt_mgr.declare(:rest_debug, 'More debug for HTTP calls (REST)', values: :none, short: 'r') { @option_rest_debug = true }
-        @opt_mgr.declare(:version, 'Display version', values: :none, short: 'v') { @formatter.display_message(:data, Aspera::Cli::VERSION); Process.exit(0) } # rubocop:disable Style/Semicolon, Layout/LineLength
-        @opt_mgr.declare(:warnings, 'Check for language warnings', values: :none, short: 'w') { $VERBOSE = true }
-        @opt_mgr.declare(
+      def declare_global_options
+        Log.log.debug('declare_global_options')
+        options.declare(:help, 'Show this message', values: :none, short: 'h') { @option_help = true }
+        options.declare(:bash_comp, 'Generate bash completion for command', values: :none) { @bash_completion = true }
+        options.declare(:show_config, 'Display parameters used for the provided action', values: :none) { @option_show_config = true }
+        options.declare(:version, 'Display version', values: :none, short: 'v') { formatter.display_message(:data, Aspera::Cli::VERSION); Process.exit(0) } # rubocop:disable Style/Semicolon, Layout/LineLength
+        options.declare(:warnings, 'Check for language warnings', values: :none, short: 'w') { $VERBOSE = true }
+        options.declare(
           :ui, 'Method to start browser',
           values: OpenApplication.user_interfaces,
-          handler: {o: self, m: :option_ui},
+          handler: {o: OpenApplication.instance, m: :url_method},
           default: OpenApplication.default_gui_mode)
-        @opt_mgr.declare(:log_level, 'Log level', values: Log.levels, handler: {o: Log.instance, m: :level})
-        @opt_mgr.declare(:logger, 'Logging method', values: Log::LOG_TYPES, handler: {o: Log.instance, m: :logger_type})
-        @opt_mgr.declare(:lock_port, 'Prevent dual execution of a command, e.g. in cron')
-        @opt_mgr.declare(:http_options, 'Options for http socket', types: Hash, handler: {o: self, m: :option_http_options})
-        @opt_mgr.declare(:insecure, 'Do not validate HTTPS certificate', values: :bool, handler: {o: self, m: :option_insecure}, default: :no)
-        @opt_mgr.declare(:once_only, 'Process only new items (some commands)', values: :bool, default: false)
-        @opt_mgr.declare(:log_secrets, 'Show passwords in logs', values: :bool, handler: {o: SecretHider, m: :log_secrets})
-        @opt_mgr.declare(:cache_tokens, 'Save and reuse Oauth tokens', values: :bool, handler: {o: self, m: :option_cache_tokens})
+        options.declare(:log_level, 'Log level', values: Log.levels, handler: {o: Log.instance, m: :level})
+        options.declare(:logger, 'Logging method', values: Log::LOG_TYPES, handler: {o: Log.instance, m: :logger_type})
+        options.declare(:lock_port, 'Prevent dual execution of a command, e.g. in cron', coerce: Integer, types: Integer)
+        options.declare(:once_only, 'Process only new items (some commands)', values: :bool, default: false)
+        options.declare(:log_secrets, 'Show passwords in logs', values: :bool, handler: {o: SecretHider, m: :log_secrets})
+        options.declare(:clean_temp, 'Cleanup temporary files on exit', values: :bool, handler: {o: TempFileManager.instance, m: :cleanup_on_exit})
+        options.declare(:pid_file, 'Write process identifier to file, delete on exit', types: String)
         # parse declared options
-        @opt_mgr.parse_options!
+        options.parse_options!
       end
 
       # @return the plugin instance, based on name
       # also loads the plugin options, and default values from conf file
       # @param plugin_name_sym : symbol for plugin name
       def get_plugin_instance_with_options(plugin_name_sym, env=nil)
-        env ||= @plugin_env
+        env ||= @agents
         Log.log.debug{"get_plugin_instance_with_options(#{plugin_name_sym})"}
-        require @plugin_env[:config].plugins[plugin_name_sym][:require_stanza]
+        require config.plugins[plugin_name_sym][:require_stanza]
         # load default params only if no param already loaded before plugin instantiation
         env[:config].add_plugin_default_preset(plugin_name_sym)
         command_plugin = Plugins::Config.plugin_class(plugin_name_sym).new(env)
@@ -226,8 +178,8 @@ module Aspera
       end
 
       def generate_bash_completion
-        if @opt_mgr.get_next_argument('', expected: :multiple, mandatory: false).nil?
-          @plugin_env[:config].plugins.each_key{|p|puts p.to_s}
+        if options.get_next_argument('', expected: :multiple, mandatory: false).nil?
+          config.plugins.each_key{|p|puts p.to_s}
         else
           Log.log.warn('only first level completion so far')
         end
@@ -237,19 +189,19 @@ module Aspera
       def exit_with_usage(all_plugins)
         Log.log.debug('exit_with_usage'.bg_red)
         # display main plugin options
-        @formatter.display_message(:error, @opt_mgr.parser)
+        formatter.display_message(:error, options.parser)
         if all_plugins
           # list plugins that have a "require" field, i.e. all but main plugin
-          @plugin_env[:config].plugins.each_key do |plugin_name_sym|
+          config.plugins.each_key do |plugin_name_sym|
             next if plugin_name_sym.eql?(Plugins::Config::CONF_PLUGIN_SYM)
             # override main option parser with a brand new, to avoid having global options
-            plugin_env = @plugin_env.clone
-            plugin_env[:man_only] = true
+            plugin_env = @agents.clone
+            plugin_env[:all_manuals] = true # force declaration of all options
             plugin_env[:options] = Manager.new(PROGRAM_NAME)
             plugin_env[:options].parser.banner = '' # remove default banner
             get_plugin_instance_with_options(plugin_name_sym, plugin_env)
             # display generated help for plugin options
-            @formatter.display_message(:error, plugin_env[:options].parser.help)
+            formatter.display_message(:error, plugin_env[:options].parser.help)
           end
         end
         Process.exit(0)
@@ -259,9 +211,9 @@ module Aspera
 
       # early debug for parser
       # Note: does not accept shortcuts
-      def early_debug_setup(argv)
+      def early_debug_setup
         Aspera::Log.instance.program_name = PROGRAM_NAME
-        argv.each do |arg|
+        @argv.each do |arg|
           case arg
           when '--' then break
           when /^--log-level=(.*)/ then Aspera::Log.instance.level = Regexp.last_match(1).to_sym
@@ -274,69 +226,78 @@ module Aspera
 
       # this is the main function called by initial script just after constructor
       def process_command_line
-        Log.log.debug('process_command_line')
         # catch exception information , if any
         exception_info = nil
-        # false if command shall not be executed ("once_only")
+        # false if command shall not be executed (e.g. --show-config)
         execute_command = true
+        # catch exceptions
         begin
+          init_agents_and_options
           # find plugins, shall be after parse! ?
-          @plugin_env[:config].add_plugins_from_lookup_folders
+          config.add_plugins_from_lookup_folders
           # help requested without command ? (plugins must be known here)
-          exit_with_usage(true) if @option_help && @opt_mgr.command_or_arg_empty?
+          exit_with_usage(true) if @option_help && options.command_or_arg_empty?
           generate_bash_completion if @bash_completion
-          @plugin_env[:config].periodic_check_newer_gem_version
+          config.periodic_check_newer_gem_version
           command_sym =
-            if @option_show_config && @opt_mgr.command_or_arg_empty?
+            if @option_show_config && options.command_or_arg_empty?
               Plugins::Config::CONF_PLUGIN_SYM
             else
-              @opt_mgr.get_next_command(@plugin_env[:config].plugins.keys.dup.unshift(:help))
+              options.get_next_command(config.plugins.keys.dup.unshift(:help))
             end
           # command will not be executed, but we need manual
-          @opt_mgr.fail_on_missing_mandatory = false if @option_help || @option_show_config
+          options.fail_on_missing_mandatory = false if @option_help || @option_show_config
           # main plugin is not dynamically instantiated
           case command_sym
           when :help
             exit_with_usage(true)
           when Plugins::Config::CONF_PLUGIN_SYM
-            command_plugin = @plugin_env[:config]
+            command_plugin = config
           else
             # get plugin, set options, etc
             command_plugin = get_plugin_instance_with_options(command_sym)
             # parse plugin specific options
-            @opt_mgr.parse_options!
+            options.parse_options!
           end
           # help requested for current plugin
           exit_with_usage(false) if @option_help
           if @option_show_config
-            @formatter.display_results({type: :single_object, data: @opt_mgr.declared_options(only_defined: true)})
+            formatter.display_results({type: :single_object, data: options.known_options(only_defined: true).stringify_keys})
             execute_command = false
           end
           # locking for single execution (only after "per plugin" option, in case lock port is there)
-          lock_port = @opt_mgr.get_option(:lock_port)
+          lock_port = options.get_option(:lock_port)
           if !lock_port.nil?
             begin
               # no need to close later, will be freed on process exit. must save in member else it is garbage collected
-              Log.log.debug{"Opening lock port #{lock_port.to_i}"}
-              @tcp_server = TCPServer.new('127.0.0.1', lock_port.to_i)
+              Log.log.debug{"Opening lock port #{lock_port}"}
+              @tcp_server = TCPServer.new('127.0.0.1', lock_port)
             rescue StandardError => e
               execute_command = false
               Log.log.warn{"Another instance is already running (#{e.message})."}
             end
           end
+          pid_file = options.get_option(:pid_file)
+          if !pid_file.nil?
+            File.write(pid_file, Process.pid)
+            Log.log.debug{"Wrote pid #{Process.pid} to #{pid_file}"}
+            at_exit{File.delete(pid_file)}
+          end
           # execute and display (if not exclusive execution)
-          @formatter.display_results(command_plugin.execute_action) if execute_command
+          formatter.display_results(command_plugin.execute_action) if execute_command
+          # save config file if command modified it
+          config.save_config_file_if_needed
           # finish
-          @plugin_env[:transfer].shutdown
+          transfer.shutdown
         rescue Net::SSH::AuthenticationFailed => e; exception_info = {e: e, t: 'SSH', security: true}
         rescue OpenSSL::SSL::SSLError => e;         exception_info = {e: e, t: 'SSL'}
-        rescue CliBadArgument => e;                 exception_info = {e: e, t: 'Argument', usage: true}
-        rescue CliNoSuchId => e;                    exception_info = {e: e, t: 'Identifier'}
-        rescue CliError => e;                       exception_info = {e: e, t: 'Tool', usage: true}
-        rescue Fasp::Error => e;                    exception_info = {e: e, t: 'FASP(ascp)'}
+        rescue Cli::BadArgument => e;               exception_info = {e: e, t: 'Argument', usage: true}
+        rescue Cli::NoSuchIdentifier => e;          exception_info = {e: e, t: 'Identifier'}
+        rescue Cli::Error => e;                     exception_info = {e: e, t: 'Tool', usage: true}
+        rescue Fasp::Error => e;                    exception_info = {e: e, t: 'Transfer'}
         rescue Aspera::RestCallError => e;          exception_info = {e: e, t: 'Rest'}
         rescue SocketError => e;                    exception_info = {e: e, t: 'Network'}
-        rescue StandardError => e;                  exception_info = {e: e, t: 'Other', debug: true}
+        rescue StandardError => e;                  exception_info = {e: e, t: "Other(#{e.class.name})", debug: true}
         rescue Interrupt => e;                      exception_info = {e: e, t: 'Interruption', debug: true}
         end
         # cleanup file list files
@@ -344,22 +305,15 @@ module Aspera
         # 1- processing of error condition
         unless exception_info.nil?
           Log.log.warn(exception_info[:e].message) if Aspera::Log.instance.logger_type.eql?(:syslog) && exception_info[:security]
-          @formatter.display_message(:error, "#{ERROR_FLASH} #{exception_info[:t]}: #{exception_info[:e].message}")
-          @formatter.display_message(:error, 'Use option -h to get help.') if exception_info[:usage]
-          # Provide hint on FASP errors
-          if exception_info[:e].is_a?(Fasp::Error) && exception_info[:e].message.eql?('Remote host is not who we expected')
-            @formatter.display_message(:error, "For this specific error, refer to:\n"\
-              "#{SRC_URL}#error-remote-host-is-not-who-we-expected\nAdd this to arguments:\n--ts=@json:'{\"sshfp\":null}'")
-          end
-          # Provide hint on SSL errors
-          if exception_info[:e].is_a?(OpenSSL::SSL::SSLError) && ['does not match the server certificate'].any?{|m|exception_info[:e].message.include?(m)}
-            @formatter.display_message(:error, "You can ignore SSL errors with option:\n--insecure=yes")
-          end
+          formatter.display_message(:error, "#{Formatter::ERROR_FLASH} #{exception_info[:t]}: #{exception_info[:e].message}")
+          formatter.display_message(:error, 'Use option -h to get help.') if exception_info[:usage]
+          # Is that a known error condition with proposal for remediation ?
+          Hints.hint_for(exception_info[:e], formatter)
         end
         # 2- processing of command not processed (due to exception or bad command line)
         if execute_command || @option_show_config
-          @opt_mgr.final_errors.each do |msg|
-            @formatter.display_message(:error, "#{ERROR_FLASH} Argument: #{msg}")
+          options.final_errors.each do |msg|
+            formatter.display_message(:error, "#{Formatter::ERROR_FLASH} Argument: #{msg}")
             # add code as exception if there is not already an error
             exception_info = {e: Exception.new(msg), t: 'UnusedArg'} if exception_info.nil?
           end
@@ -367,9 +321,9 @@ module Aspera
         # 3- in case of error, fail the process status
         unless exception_info.nil?
           # show stack trace in debug mode
-          raise exception_info[:e] if Log.instance.level.eql?(:debug)
+          raise exception_info[:e] if Log.log.debug?
           # else give hint and exit
-          @formatter.display_message(:error, 'Use --log-level=debug to get more details.') if exception_info[:debug]
+          formatter.display_message(:error, 'Use --log-level=debug to get more details.') if exception_info[:debug]
           Process.exit(1)
         end
         return nil