aspera-cli 4.14.0 → 4.15.0

data/lib/aspera/cli/plugins/config.rb

@@ -1,25 +1,27 @@
 # frozen_string_literal: true
 
+# cspell:ignore initdemo genkey asperasoft
 require 'aspera/cli/basic_auth_plugin'
 require 'aspera/cli/extended_value'
 require 'aspera/cli/version'
 require 'aspera/cli/formatter'
 require 'aspera/cli/info'
+require 'aspera/cli/transfer_progress'
 require 'aspera/fasp/installation'
+require 'aspera/fasp/products'
 require 'aspera/fasp/parameters'
 require 'aspera/fasp/transfer_spec'
 require 'aspera/fasp/error_info'
+require 'aspera/keychain/encrypted_hash'
+require 'aspera/keychain/macos_security'
 require 'aspera/proxy_auto_config'
 require 'aspera/open_application'
 require 'aspera/persistency_action_once'
 require 'aspera/id_generator'
-require 'aspera/keychain/encrypted_hash'
-require 'aspera/keychain/macos_security'
-require 'aspera/aoc'
+require 'aspera/persistency_folder'
+require 'aspera/line_logger'
 require 'aspera/rest'
-require 'xmlsimple'
-require 'base64'
-require 'net/smtp'
+require 'aspera/log'
 require 'open3'
 require 'date'
 require 'erb'
@@ -38,23 +40,21 @@ module Aspera
         CONF_PRESET_VERSION = 'version'
         CONF_PRESET_DEFAULT = 'default'
         CONF_PRESET_GLOBAL = 'global_common_defaults'
+        GLOBAL_DEFAULT_KEYWORD = 'GLOBAL'
         CONF_PLUGIN_SYM = :config # Plugins::Config.name.split('::').last.downcase.to_sym
         CONF_GLOBAL_SYM = :config
-        # default redirect for AoC web auth
-        DEFAULT_REDIRECT = 'http://localhost:12345'
         # folder containing custom plugins in user's config folder
         ASPERA_PLUGINS_FOLDERNAME = 'plugins'
+        PERSISTENCY_FOLDER = 'persist_store'
         RUBY_FILE_EXT = '.rb'
         ASPERA = 'aspera'
-        AOC_COMMAND = 'aoc'
         SERVER_COMMAND = 'server'
         APP_NAME_SDK = 'sdk'
         CONNECT_WEB_URL = 'https://d3gcli72yxqn2z.cloudfront.net/connect'
-        CONNECT_VERSIONS = 'connectversions.js'
+        CONNECT_VERSIONS = 'connectversions.js' # cspell: disable-line
         TRANSFER_SDK_ARCHIVE_URL = 'https://ibm.biz/aspera_transfer_sdk'
         DEMO = 'demo'
-        DEMO_SERVER_PRESET = 'demoserver'
-        AOC_PATH_API_CLIENTS = 'admin/api-clients'
+        DEMO_SERVER_PRESET = 'demoserver' # cspell: disable-line
         EMAIL_TEST_TEMPLATE = <<~END_OF_TEMPLATE
           From: <%=from_name%> <<%=from_email%>>
           To: <<%=to%>>
@@ -63,103 +63,196 @@ module Aspera
           This email was sent to test #{PROGRAM_NAME}.
         END_OF_TEMPLATE
         # special extended values
-        EXTV_INCLUDE_PRESETS = :incps
-        EXTV_PRESET = :preset
-        EXTV_VAULT = :vault
+        EXTEND_PRESET = :preset
+        EXTEND_VAULT = :vault
         PRESET_DIG_SEPARATOR = '.'
         DEFAULT_CHECK_NEW_VERSION_DAYS = 7
-        DEFAULT_PRIV_KEY_FILENAME = 'aspera_aoc_key' # pragma: allowlist secret
-        DEFAULT_PRIVKEY_LENGTH = 4096
+        DEFAULT_PRIV_KEY_FILENAME = 'my_private_key.pem' # pragma: allowlist secret
+        DEFAULT_PRIV_KEY_LENGTH = 4096
         COFFEE_IMAGE = 'https://enjoyjava.com/wp-content/uploads/2018/01/How-to-make-strong-coffee.jpg'
+        WIZARD_RESULT_KEYS = %i[preset_value test_args].freeze
+        GEM_CHECK_DATE_FMT = '%Y/%m/%d'
+        # for testing only
+        SELF_SIGNED_CERT = OpenSSL::SSL.const_get(:enon_yfirev.to_s.upcase.reverse) # cspell: disable-line
         private_constant :DEFAULT_CONFIG_FILENAME,
           :CONF_PRESET_CONFIG,
           :CONF_PRESET_VERSION,
           :CONF_PRESET_DEFAULT,
           :CONF_PRESET_GLOBAL,
-          :DEFAULT_REDIRECT,
           :ASPERA_PLUGINS_FOLDERNAME,
           :RUBY_FILE_EXT,
           :ASPERA,
-          :AOC_COMMAND,
           :DEMO,
           :TRANSFER_SDK_ARCHIVE_URL,
-          :AOC_PATH_API_CLIENTS,
           :DEMO_SERVER_PRESET,
           :EMAIL_TEST_TEMPLATE,
-          :EXTV_INCLUDE_PRESETS,
-          :EXTV_PRESET,
-          :EXTV_VAULT,
+          :EXTEND_PRESET,
+          :EXTEND_VAULT,
           :DEFAULT_CHECK_NEW_VERSION_DAYS,
           :DEFAULT_PRIV_KEY_FILENAME,
           :SERVER_COMMAND,
           :PRESET_DIG_SEPARATOR,
-          :COFFEE_IMAGE
+          :COFFEE_IMAGE,
+          :WIZARD_RESULT_KEYS,
+          :SELF_SIGNED_CERT,
+          :PERSISTENCY_FOLDER,
+          :DEFAULT_PRIV_KEY_LENGTH
+
+        class << self
+          def generate_rsa_private_key(path:, length: DEFAULT_PRIV_KEY_LENGTH)
+            require 'openssl'
+            priv_key = OpenSSL::PKey::RSA.new(length)
+            File.write(path, priv_key.to_s)
+            File.write("#{path}.pub", priv_key.public_key.to_s)
+            Environment.restrict_file_access(path)
+            Environment.restrict_file_access("#{path}.pub")
+            nil
+          end
+
+          # folder containing plugins in the gem's main folder
+          def gem_plugins_folder
+            File.dirname(File.expand_path(__FILE__))
+          end
+
+          # name of englobing module
+          # @return "Aspera::Cli::Plugins"
+          def module_full_name
+            return Module.nesting[2].to_s
+          end
+
+          # @return main folder where code is, i.e. .../lib
+          # go up as many times as englobing modules (not counting class, as it is a file)
+          def gem_src_root
+            File.expand_path(module_full_name.gsub('::', '/').gsub(%r{[^/]+}, '..'), gem_plugins_folder)
+          end
+
+          # instantiate a plugin
+          # plugins must be Capitalized
+          def plugin_class(plugin_name_sym)
+            # Module.nesting[2] is Aspera::Cli::Plugins
+            return Object.const_get("#{module_full_name}::#{plugin_name_sym.to_s.capitalize}")
+          end
+
+          # deep clone hash so that it does not get modified in case of display and secret hide
+          def protect_presets(val)
+            return JSON.parse(JSON.generate(val))
+          end
+
+          # return product family folder (~/.aspera)
+          def module_family_folder
+            user_home_folder = Dir.home
+            raise Cli::Error, "Home folder does not exist: #{user_home_folder}. Check your user environment." unless Dir.exist?(user_home_folder)
+            return File.join(user_home_folder, ASPERA_HOME_FOLDER_NAME)
+          end
+
+          # return product config folder (~/.aspera/<name>)
+          def default_app_main_folder(app_name:)
+            raise 'app_name must be a non-empty String' unless app_name.is_a?(String) && !app_name.empty?
+            return File.join(module_family_folder, app_name)
+          end
+        end # self
+
         def initialize(env, params)
-          raise 'env and params must be Hash' unless env.is_a?(Hash) && params.is_a?(Hash)
-          raise 'missing param' unless %i[name help version gem].sort.eql?(params.keys.sort)
+          raise 'Internal Error: env and params must be Hash' unless env.is_a?(Hash) && params.is_a?(Hash)
+          raise 'Internal Error: missing param' unless %i[gem help name version].eql?(params.keys.sort)
+          # we need to defer parsing of options until we have the config file, so we can use @extend with @preset
           super(env)
           @info = params
-          @main_folder = default_app_main_folder
           @plugins = {}
           @plugin_lookup_folders = []
           @use_plugin_defaults = true
           @config_presets = nil
+          @config_checksum_on_disk = nil
           @connect_versions = nil
           @vault = nil
-          @conf_file_default = File.join(@main_folder, DEFAULT_CONFIG_FILENAME)
-          @option_config_file = @conf_file_default
           @pac_exec = nil
           @sdk_default_location = false
+          @option_insecure = false
+          @option_ignore_cert_host_port = []
+          @option_http_options = {}
+          @ssl_warned_urls = []
+          @option_cache_tokens = true
+          @proxy_credentials = nil
+          @main_folder = nil
+          @option_config_file = nil
+          @certificate_store = nil
+          @certificate_paths = nil
+          @progress_bar = nil
+          # option to set main folder
+          options.declare(
+            :home, 'Home folder for tool',
+            handler: {o: self, m: :main_folder},
+            types: String,
+            default: self.class.default_app_main_folder(app_name: @info[:name]))
+          options.parse_options!
           Log.log.debug{"#{@info[:name]} folder: #{@main_folder}"}
-          # set folder for FASP SDK
+          # data persistency manager
+          env[:persistency] = PersistencyFolder.new(File.join(@main_folder, PERSISTENCY_FOLDER))
+          # set folders for plugin lookup
           add_plugin_lookup_folder(self.class.gem_plugins_folder)
           add_plugin_lookup_folder(File.join(@main_folder, ASPERA_PLUGINS_FOLDERNAME))
-          # do file parameter first
-          options.declare(:config_file, "Read parameters from file in YAML format, current=#{@option_config_file}", handler: {o: self, m: :option_config_file})
+          # option to set config file
+          options.declare(
+            :config_file, 'Path to YAML file with preset configuration',
+            handler: {o: self, m: :option_config_file},
+            default: File.join(@main_folder, DEFAULT_CONFIG_FILENAME))
           options.parse_options!
-          # read correct file (set @config_presets)
+          # read config file (set @config_presets)
           read_config_file
           # add preset handler (needed for smtp)
-          ExtendedValue.instance.set_handler(EXTV_PRESET, lambda{|v|preset_by_name(v)})
-          ExtendedValue.instance.set_handler(EXTV_INCLUDE_PRESETS, lambda{|v|expanded_with_preset_includes(v)})
-          ExtendedValue.instance.set_handler(EXTV_VAULT, lambda{|v|vault_value(v)})
+          ExtendedValue.instance.set_handler(EXTEND_PRESET, lambda{|v|preset_by_name(v)})
+          ExtendedValue.instance.set_handler(EXTEND_VAULT, lambda{|v|vault_value(v)})
           # load defaults before it can be overridden
           add_plugin_default_preset(CONF_GLOBAL_SYM)
+          # vault options
+          options.declare(:secret, 'Secret for access keys')
+          options.declare(:vault, 'Vault for secrets', types: Hash)
+          options.declare(:vault_password, 'Vault password')
           options.parse_options!
+          # declare generic plugin options only after handlers are declared
+          Plugin.declare_generic_options(options)
+          # configuration options
           options.declare(:no_default, 'Do not load default configuration for plugin', values: :none, short: 'N') { @use_plugin_defaults = false }
+          options.declare(:preset, 'Load the named option preset from current config file', short: 'P', handler: {o: self, m: :option_preset})
+          options.declare(:version_check_days, 'Period in days to check new version (zero to disable)', coerce: Integer, default: DEFAULT_CHECK_NEW_VERSION_DAYS)
+          options.declare(:plugin_folder, 'Folder where to find additional plugins', handler: {o: self, m: :option_plugin_folder})
+          # wizard options
           options.declare(:override, 'Wizard: override existing value', values: :bool, default: :no)
-          options.declare(:use_generic_client, 'Wizard: AoC: use global or org specific jwt client id', values: :bool, default: true)
           options.declare(:default, 'Wizard: set as default configuration for specified plugin (also: update)', values: :bool, default: true)
           options.declare(:test_mode, 'Wizard: skip private key check step', values: :bool, default: false)
-          options.declare(:preset, 'Load the named option preset from current config file', short: 'P', handler: {o: self, m: :option_preset})
-          options.declare(:pkeypath, 'Wizard: path to private key for JWT')
+          options.declare(:key_path, 'Wizard: path to private key for JWT')
+          # Transfer SDK options
           options.declare(:ascp_path, 'Path to ascp', handler: {o: Fasp::Installation.instance, m: :ascp_path})
           options.declare(:use_product, 'Use ascp from specified product', handler: {o: self, m: :option_use_product})
-          options.declare(:smtp, 'SMTP configuration', types: Hash)
-          options.declare(:fpac, 'Proxy auto configuration script')
-          options.declare(:proxy_credentials, 'HTTP proxy credentials (Array with user and password)')
-          options.declare(:secret, 'Secret for access keys')
-          options.declare(:vault, 'Vault for secrets')
-          options.declare(:vault_password, 'Vault password')
           options.declare(:sdk_url, 'URL to get SDK', default: TRANSFER_SDK_ARCHIVE_URL)
           options.declare(:sdk_folder, 'SDK folder path', handler: {o: Fasp::Installation.instance, m: :sdk_folder})
-          options.declare(:notif_to, 'Email recipient for notification of transfers')
-          options.declare(:notif_template, 'Email ERB template for notification of transfers')
-          options.declare(:version_check_days, 'Period in days to check new version (zero to disable)', coerce: Integer, default: DEFAULT_CHECK_NEW_VERSION_DAYS)
-          options.declare(:plugin_folder, 'Folder where to find additional plugins', handler: {o: self, m: :option_plugin_folder})
+          options.declare(:progress_bar, 'Display progress bar', values: :bool, default: Environment.terminal?)
+          # email options
+          options.declare(:smtp, 'SMTP configuration', types: Hash)
+          options.declare(:notify_to, 'Email recipient for notification of transfers')
+          options.declare(:notify_template, 'Email ERB template for notification of transfers')
+          # HTTP options
+          options.declare(:insecure, 'Do not validate any HTTPS certificate', values: :bool, handler: {o: self, m: :option_insecure}, default: :no)
+          options.declare(:ignore_certificate, 'List of HTTPS url where to no validate certificate', types: Array, handler: {o: self, m: :option_ignore_cert_host_port})
+          options.declare(:cert_stores, 'List of folder with trusted certificates', types: [Array, String], handler: {o: self, m: :trusted_cert_locations})
+          options.declare(:http_options, 'Options for HTTP/S socket', types: Hash, handler: {o: self, m: :option_http_options}, default: {})
+          options.declare(:cache_tokens, 'Save and reuse Oauth tokens', values: :bool, handler: {o: self, m: :option_cache_tokens})
+          options.declare(:fpac, 'Proxy auto configuration script')
+          options.declare(:proxy_credentials, 'HTTP proxy credentials (user and password)', types: Array)
           options.parse_options!
+          @progress_bar = TransferProgress.new if options.get_option(:progress_bar)
           # Check SDK folder is set or not, for compatibility, we check in two places
           sdk_folder = Fasp::Installation.instance.sdk_folder rescue nil
           if sdk_folder.nil?
             @sdk_default_location = true
             Log.log.debug('SDK folder is not set, checking default')
             # new location
-            sdk_folder = default_app_main_folder(app_name: APP_NAME_SDK)
+            sdk_folder = self.class.default_app_main_folder(app_name: APP_NAME_SDK)
             Log.log.debug{"checking: #{sdk_folder}"}
             if !Dir.exist?(sdk_folder)
               Log.log.debug{"not exists: #{sdk_folder}"}
               # former location
-              former_sdk_folder = File.join(default_app_main_folder, APP_NAME_SDK)
+              former_sdk_folder = File.join(self.class.default_app_main_folder(app_name: @info[:name]), APP_NAME_SDK)
               Log.log.debug{"checking: #{former_sdk_folder}"}
               sdk_folder = former_sdk_folder if Dir.exist?(former_sdk_folder)
             end
@@ -169,36 +262,109 @@ module Aspera
           pac_script = options.get_option(:fpac)
           # create PAC executor
           @pac_exec = Aspera::ProxyAutoConfig.new(pac_script).register_uri_generic unless pac_script.nil?
-          proxy_creds = options.get_option(:proxy_credentials)
-          if !proxy_creds.nil?
-            raise CliBadArgument, 'proxy credentials shall be an array (#{proxy_creds.class})' unless proxy_creds.is_a?(Array)
-            raise CliBadArgument, 'proxy credentials shall have two elements (#{proxy_creds.length})' unless proxy_creds.length.eql?(2)
-            @pac_exec.proxy_user = Rest.proxy_user = proxy_creds[0]
-            @pac_exec.proxy_pass = Rest.proxy_pass = proxy_creds[1]
+          proxy_user_pass = options.get_option(:proxy_credentials)
+          if !proxy_user_pass.nil?
+            raise Cli::BadArgument, "proxy_credentials shall have two elements (#{proxy_user_pass.length})" unless proxy_user_pass.length.eql?(2)
+            @proxy_credentials = {user: proxy_user_pass[0], pass: proxy_user_pass[1]}
+            @pac_exec.proxy_user = @proxy_credentials[:user]
+            @pac_exec.proxy_pass = @proxy_credentials[:pass]
           end
+          Rest.set_parameters(
+            user_agent:  PROGRAM_NAME,
+            session_cb:  lambda{|http_session|update_http_session(http_session)},
+            progress_bar: @progress_bar)
+          Oauth.persist_mgr = persistency if @option_cache_tokens
+          Fasp::Parameters.file_list_folder = File.join(@main_folder, 'filelists') # cspell: disable-line
+          Aspera::RestErrorAnalyzer.instance.log_file = File.join(@main_folder, 'rest_exceptions.log')
+          # register aspera REST call error handlers
+          Aspera::RestErrorsAspera.register_handlers
         end
 
-        # env var name to override the app's main folder
-        # default main folder is $HOME/<vendor main app folder>/<program name>
-        def conf_dir_env_var
-          return "#{@info[:name]}_home".upcase
+        attr_accessor :main_folder, :option_cache_tokens, :option_insecure, :option_http_options
+        attr_reader :option_ignore_cert_host_port, :progress_bar
+
+        def trusted_cert_locations=(path_list)
+          path_list = [path_list] unless path_list.is_a?(Array)
+          if @certificate_store.nil?
+            Log.log.debug('Creating SSL Cert store')
+            @certificate_store = OpenSSL::X509::Store.new
+            @certificate_store.set_default_paths
+            @certificate_paths = []
+          end
+
+          path_list.each do |path|
+            raise 'Expecting a String for cert location' unless path.is_a?(String)
+            Log.log.debug("Adding cert location: #{path}")
+            if path.eql?(ExtendedValue::DEF)
+              path = OpenSSL::X509::DEFAULT_CERT_DIR
+              @certificate_store.add_path(path)
+              @certificate_paths.push(path)
+              path = OpenSSL::X509::DEFAULT_CERT_FILE
+              @certificate_store.add_file(path)
+            elsif File.file?(path)
+              @certificate_store.add_file(path)
+            elsif File.directory?(path)
+              @certificate_store.add_path(path)
+            else
+              raise "No such file or folder: #{path}"
+            end
+            @certificate_paths.push(path)
+          end
         end
 
-        # return product family folder (~/.aspera)
-        def module_family_folder
-          user_home_folder = Dir.home
-          raise CliError, "Home folder does not exist: #{user_home_folder}. Check your user environment." unless Dir.exist?(user_home_folder)
-          return File.join(user_home_folder, ASPERA_HOME_FOLDER_NAME)
+        def trusted_cert_locations(files_only: false)
+          locations = if @certificate_paths.nil?
+            [OpenSSL::X509::DEFAULT_CERT_DIR, OpenSSL::X509::DEFAULT_CERT_FILE]
+          else
+            @certificate_paths
+          end
+          locations = locations.select{|f|File.file?(f)} if files_only
+          return locations
+        end
+
+        def option_ignore_cert_host_port=(url_list)
+          url_list.each do |url|
+            uri = URI.parse(url)
+            @option_ignore_cert_host_port.push([uri.host, uri.port].freeze)
+          end
         end
 
-        # return product config folder (~/.aspera/<name>)
-        def default_app_main_folder(app_name: nil)
-          app_name = @info[:name] if app_name.nil?
-          # find out application main folder
-          app_folder = ENV[conf_dir_env_var]
-          # if env var undefined or empty
-          app_folder = File.join(module_family_folder, app_name) if app_folder.nil? || app_folder.empty?
-          return app_folder
+        def ignore_cert?(address, port)
+          endpoint = [address, port].freeze
+          Log.log.debug{"ignore cert? #{endpoint}"}
+          return false unless @option_insecure || @option_ignore_cert_host_port.any?(endpoint)
+          base_url = "https://#{address}:#{port}"
+          if !@ssl_warned_urls.include?(base_url)
+            formatter.display_message(
+              :error,
+              "#{Formatter::WARNING_FLASH} Ignoring certificate for: #{base_url}. Do not deactivate certificate verification in production.")
+            @ssl_warned_urls.push(base_url)
+          end
+          return true
+        end
+
+        # called every time a new REST HTTP session is opened to set user-provided options
+        # @param http_session [Net::HTTP] the newly created HTTP/S session object
+        def update_http_session(http_session)
+          http_session.set_debug_output(LineLogger.new(:trace2)) if Log.instance.logger.trace2?
+          # Rest.io_http_session(http_session).debug_output = Log.log
+          http_session.verify_mode = SELF_SIGNED_CERT if http_session.use_ssl? && ignore_cert?(http_session.address, http_session.port)
+          http_session.cert_store = @certificate_store if @certificate_store
+          Log.log.debug{"using cert store #{http_session.cert_store} (#{@certificate_store})"} unless http_session.cert_store.nil?
+          if @proxy_credentials
+            http_session.proxy_user = @proxy_credentials[:user]
+            http_session.proxy_pass = @proxy_credentials[:pass]
+          end
+          @option_http_options.each do |k, v|
+            method = "#{k}=".to_sym
+            # check if accessor is a method of Net::HTTP
+            # continue_timeout= read_timeout= write_timeout=
+            if http_session.respond_to?(method)
+              http_session.send(method, v)
+            else
+              Log.log.error{"no such HTTP session attribute: #{k}"}
+            end
+          end
         end
 
         def check_gem_version
@@ -211,7 +377,7 @@ module Aspera
             end
           if Gem::Version.new(Environment.ruby_version) < Gem::Version.new(RUBY_FUTURE_MINIMUM_VERSION)
             Log.log.warn do
-              "Note that a future version will require Ruby version #{RUBY_FUTURE_MINIMUM_VERSION} at minimum, "\
+              "Note that a future version will require Ruby version #{RUBY_FUTURE_MINIMUM_VERSION} at minimum, " \
                 "you are using #{Environment.ruby_version}"
             end
           end
@@ -226,7 +392,6 @@ module Aspera
         def periodic_check_newer_gem_version
           # get verification period
           delay_days = options.get_option(:version_check_days, mandatory: true)
-          Log.log.info{"check days: #{delay_days}"}
           # check only if not zero day
           return if delay_days.eql?(0)
           # get last date from persistency
@@ -237,17 +402,11 @@ module Aspera
             id:      'version_last_check')
           # get persisted date or nil
           current_date = Date.today
-          last_check_days =
-            begin
-              current_date - Date.strptime(last_check_array.first, '%Y/%m/%d')
-            rescue StandardError
-              # negative value will force check
-              -1
-            end
-          Log.log.debug{"days elapsed: #{last_check_days}"}
-          return if last_check_days < delay_days
+          last_check_days = (current_date - Date.strptime(last_check_array.first, GEM_CHECK_DATE_FMT)) rescue nil
+          Log.log.debug{"gem check new version: #{delay_days}, #{last_check_days}, #{current_date}, #{last_check_array}"}
+          return if !last_check_days.nil? && last_check_days < delay_days
           # generate timestamp
-          last_check_array[0] = current_date.strftime('%Y/%m/%d')
+          last_check_array[0] = current_date.strftime(GEM_CHECK_DATE_FMT)
           check_date_persist.save
           # compare this version and the one on internet
           check_data = check_gem_version
@@ -266,7 +425,7 @@ module Aspera
             Log.log.debug{"javascript=[\n#{connect_versions_javascript}\n]"}
             # get javascript object only
             found = connect_versions_javascript.match(/^.*? = (.*);/)
-            raise CliError, 'Problem when getting connect versions from internet' if found.nil?
+            raise Cli::Error, 'Problem when getting connect versions from internet' if found.nil?
             all_data = JSON.parse(found[1])
             @connect_versions = all_data['entries']
           end
@@ -284,66 +443,45 @@ module Aspera
           return nil
         end
 
-        private
-
-        class << self
-          def generate_rsa_private_key(path:, length: DEFAULT_PRIVKEY_LENGTH)
-            require 'openssl'
-            priv_key = OpenSSL::PKey::RSA.new(length)
-            File.write(path, priv_key.to_s)
-            File.write(path + '.pub', priv_key.public_key.to_s)
-            Environment.restrict_file_access(path)
-            Environment.restrict_file_access(path + '.pub')
-            nil
-          end
-
-          # folder containing plugins in the gem's main folder
-          def gem_plugins_folder
-            File.dirname(File.expand_path(__FILE__))
-          end
-
-          # name of englobing module
-          # @return "Aspera::Cli::Plugins"
-          def module_full_name
-            return Module.nesting[2].to_s
+        # get the default global preset, or init a new one
+        def global_default_preset
+          global_default_preset_name = get_plugin_default_config_name(CONF_GLOBAL_SYM)
+          if global_default_preset_name.nil?
+            global_default_preset_name = CONF_PRESET_GLOBAL.to_s
+            set_preset_key(CONF_PRESET_DEFAULT, CONF_GLOBAL_SYM, global_default_preset_name)
           end
+          return global_default_preset_name
+        end
 
-          # @return main folder where code is, i.e. .../lib
-          # go up as many times as englobing modules (not counting class, as it is a file)
-          def gem_src_root
-            File.expand_path(module_full_name.gsub('::', '/').gsub(%r{[^/]+}, '..'), gem_plugins_folder)
+        def set_preset_key(preset, param_name, param_value)
+          raise "Parameter name must be a String or Symbol, not #{param_name.class}" unless [String, Symbol].include?(param_name.class)
+          param_name = param_name.to_s
+          selected_preset = @config_presets[preset]
+          if selected_preset.nil?
+            Log.log.debug{"No such preset name: #{preset}, initializing"}
+            selected_preset = @config_presets[preset] = {}
           end
-
-          # instantiate a plugin
-          # plugins must be Capitalized
-          def plugin_class(plugin_name_sym)
-            # Module.nesting[2] is Aspera::Cli::Plugins
-            return Object.const_get("#{module_full_name}::#{plugin_name_sym.to_s.capitalize}")
+          raise "expecting Hash for #{preset}.#{param_name}" unless selected_preset.is_a?(Hash)
+          if selected_preset.key?(param_name)
+            if selected_preset[param_name].eql?(param_value)
+              Log.log.warn{"keeping same value for #{preset}: #{param_name}: #{param_value}"}
+              return
+            end
+            Log.log.warn{"overwriting value: #{selected_preset[param_name]}"}
           end
+          selected_preset[param_name] = param_value
+          formatter.display_status("Updated: #{preset}: #{param_name} <- #{param_value}")
+          nil
         end
 
         # set parameter and value in global config
         # creates one if none already created
         # @return preset name that contains global default
         def set_global_default(key, value)
-          # get default preset if it exists
-          global_default_preset_name = get_plugin_default_config_name(CONF_GLOBAL_SYM)
-          if global_default_preset_name.nil?
-            global_default_preset_name = CONF_PRESET_GLOBAL
-            @config_presets[CONF_PRESET_DEFAULT] ||= {}
-            @config_presets[CONF_PRESET_DEFAULT][CONF_GLOBAL_SYM.to_s] = global_default_preset_name
-          end
-          @config_presets[global_default_preset_name] ||= {}
-          @config_presets[global_default_preset_name][key.to_s] = value
-          formatter.display_status("Updated: #{global_default_preset_name}: #{key} <- #{value}")
-          save_presets_to_config_file
-          return global_default_preset_name
+          set_preset_key(global_default_preset, key, value)
         end
 
-        public
-
         # $HOME/.aspera/`program_name`
-        attr_reader :main_folder
         attr_reader :gem_url, :plugins
         attr_accessor :option_config_file
 
@@ -351,40 +489,17 @@ module Aspera
         # @param config_name name of the preset in config file
         # @param include_path used to detect and avoid include loops
         def preset_by_name(config_name, include_path=[])
-          raise CliError, 'loop in include' if include_path.include?(config_name)
+          raise Cli::Error, 'loop in include' if include_path.include?(config_name)
           include_path = include_path.clone # avoid messing up if there are multiple branches
           current = @config_presets
           config_name.split(PRESET_DIG_SEPARATOR).each do |name|
-            raise CliError, "Expecting Hash for sub key: #{include_path} (#{current.class})" unless current.is_a?(Hash)
+            raise Cli::Error, "Expecting Hash for sub key: #{include_path} (#{current.class})" unless current.is_a?(Hash)
             include_path.push(name)
             current = current[name]
-            raise CliError, "No such config preset: #{include_path}" if current.nil?
+            raise Cli::Error, "No such config preset: #{include_path}" if current.nil?
           end
-          case current
-          when Hash then return expanded_with_preset_includes(current, include_path)
-          when String then return ExtendedValue.instance.evaluate(current)
-          else return current
-          end
-        end
-
-        # @return the hash value with 'incps' keys expanded to include other presets
-        # @param hash_val
-        # @param include_path to avoid inclusion loop
-        def expanded_with_preset_includes(hash_val, include_path=[])
-          raise CliError, "#{EXTV_INCLUDE_PRESETS} requires a Hash, have #{hash_val.class}" unless hash_val.is_a?(Hash)
-          if hash_val.key?(EXTV_INCLUDE_PRESETS)
-            memory = hash_val.clone
-            includes = memory[EXTV_INCLUDE_PRESETS]
-            memory.delete(EXTV_INCLUDE_PRESETS)
-            hash_val = {}
-            raise "#{EXTV_INCLUDE_PRESETS} must be an Array" unless includes.is_a?(Array)
-            raise "#{EXTV_INCLUDE_PRESETS} must contain names" unless includes.map(&:class).uniq.eql?([String])
-            includes.each do |preset_name|
-              hash_val.merge!(preset_by_name(preset_name, include_path))
-            end
-            hash_val.merge!(memory)
-          end
-          return hash_val
+          current = self.class.protect_presets(current) unless current.is_a?(String)
+          return ExtendedValue.instance.evaluate(current)
         end
 
         def option_use_product=(value)
@@ -420,6 +535,10 @@ module Aspera
           end
         end
 
+        def config_checksum
+          JSON.generate(@config_presets).hash
+        end
+
         # read config file and validate format
         def read_config_file
           Log.log.debug{"config file is: #{@option_config_file}".red}
@@ -427,38 +546,35 @@ module Aspera
           search_files = [@option_config_file]
           # find first existing file (or nil)
           conf_file_to_load = search_files.find{|f| File.exist?(f)}
-          # require save if old version of file
-          save_required = false
           # if no file found, create default config
           if conf_file_to_load.nil?
-            Log.log.warn{"No config file found. Creating empty configuration file: #{@option_config_file}"}
-            @config_presets = {CONF_PRESET_CONFIG => {CONF_PRESET_VERSION => @info[:version]}}
+            Log.log.warn{"No config file found. New configuration file: #{@option_config_file}"}
+            @config_presets = {CONF_PRESET_CONFIG => {CONF_PRESET_VERSION => 'new file'}}
+            # @config_checksum_on_disk is nil
           else
             Log.log.debug{"loading #{@option_config_file}"}
             @config_presets = YAML.load_file(conf_file_to_load)
+            @config_checksum_on_disk = config_checksum
           end
           files_to_copy = []
-          Log.log.debug{"Available_presets: #{@config_presets}"}
+          Log.log.debug{Log.dump('Available_presets', @config_presets)}
           raise 'Expecting YAML Hash' unless @config_presets.is_a?(Hash)
           # check there is at least the config section
-          if !@config_presets.key?(CONF_PRESET_CONFIG)
-            raise "Cannot find key: #{CONF_PRESET_CONFIG}"
-          end
+          raise "Cannot find key: #{CONF_PRESET_CONFIG}" unless @config_presets.key?(CONF_PRESET_CONFIG)
           version = @config_presets[CONF_PRESET_CONFIG][CONF_PRESET_VERSION]
-          if version.nil?
-            raise 'No version found in config section.'
-          end
+          raise 'No version found in config section.' if version.nil?
           Log.log.debug{"conf version: #{version}"}
-          # if there are any conversion needed, those happen here.
-          # Place new compatibility code here
-          if save_required
-            Log.log.warn('Saving automatic conversion.')
-            @config_presets[CONF_PRESET_CONFIG][CONF_PRESET_VERSION] = @info[:version]
-            save_presets_to_config_file
+          # VVV if there are any conversion needed, those happen here.
+          # fix bug in 4.4 (creating key "true" in "default" preset)
+          @config_presets[CONF_PRESET_DEFAULT].delete(true) if @config_presets[CONF_PRESET_DEFAULT].is_a?(Hash)
+          # ^^^ Place new compatibility code before this line
+          # set version to current
+          @config_presets[CONF_PRESET_CONFIG][CONF_PRESET_VERSION] = @info[:version]
+          unless files_to_copy.empty?
             Log.log.warn('Copying referenced files')
             files_to_copy.each do |file|
               FileUtils.cp(file, @main_folder)
-              Log.log.warn{"..#{file} -> #{@main_folder}"}
+              Log.log.warn{"#{file} -> #{@main_folder}"}
             end
           end
         rescue Psych::SyntaxError => e
@@ -473,7 +589,7 @@ module Aspera
             Log.log.warn{"Renamed config file to #{new_name}."}
             Log.log.warn('Manual Conversion is required. Next time, a new empty file will be created.')
           end
-          raise CliError, e.to_s
+          raise Cli::Error, e.to_s
         end
 
         # find plugins in defined paths
@@ -505,45 +621,39 @@ module Aspera
 
         # Find a plugin, and issue the "require"
         # @return [Hash] plugin info: { product: , url:, version: }
-        def identify_plugin_for_url(url, check_only: nil)
+        def identify_plugins_for_url
+          app_url = options.get_next_argument('url', mandatory: true)
+          check_only = options.get_next_argument('plugin name', mandatory: false)
           check_only = check_only.to_sym unless check_only.nil?
+          found_apps = []
           plugins.each do |plugin_name_sym, plugin_info|
             # no detection for internal plugin
             next if plugin_name_sym.eql?(CONF_PLUGIN_SYM)
             next if check_only && !check_only.eql?(plugin_name_sym)
             # load plugin class
             require plugin_info[:require_stanza]
-            c = self.class.plugin_class(plugin_name_sym)
+            detect_plugin_class = self.class.plugin_class(plugin_name_sym)
             # requires detection method
-            next unless c.respond_to?(:detect)
-            current_url = url
+            next unless detect_plugin_class.respond_to?(:detect)
             detection_info = nil
-            # first try : direct
             begin
-              detection_info = c.detect(current_url)
+              detection_info = detect_plugin_class.detect(app_url)
             rescue OpenSSL::SSL::SSLError => e
               Log.log.warn(e.message)
               Log.log.warn('Use option --insecure=yes to allow unchecked certificate') if e.message.include?('cert')
             rescue StandardError => e
-              Log.log.debug{"Cannot detect #{plugin_name_sym} : #{e.class}/#{e.message}"}
-            end
-            # second try : is there a redirect ?
-            if detection_info.nil?
-              begin
-                # TODO: check if redirect ?
-                new_url = Rest.new(base_url: url).call(operation: 'GET', subpath: '', redirect_max: 1)[:http].uri.to_s
-                unless url.eql?(new_url)
-                  detection_info = c.detect(new_url)
-                  current_url = new_url
-                end
-              rescue StandardError => e
-                Log.log.debug{"Cannot detect #{plugin_name_sym} : #{e.message}"}
-              end
+              Log.log.debug{"detect error: #{e}"}
+              next
             end
+            next if detection_info.nil?
+            raise 'internal error' if detection_info.key?(:url) && !detection_info[:url].is_a?(String)
+            app_name = detect_plugin_class.respond_to?(:application_name) ? detect_plugin_class.application_name : detect_plugin_class.name.split('::').last
             # if there is a redirect, then the detector can override the url.
-            return {product: plugin_name_sym, url: current_url}.merge(detection_info) unless detection_info.nil?
+            found_apps.push({product: plugin_name_sym, name: app_name, url: app_url, version: 'unknown'}.merge(detection_info))
           end # loop
-          raise "No known application found at #{url}"
+          raise "No known application found at #{app_url}" if found_apps.empty?
+          raise 'Internal error' unless found_apps.all?{|a|a.keys.all?(Symbol)}
+          return found_apps
         end
 
         def execute_connect_action
@@ -551,7 +661,7 @@ module Aspera
           if %i[info version].include?(command)
             connect_id = options.get_next_argument('id or title')
             one_res = connect_versions.find{|i|i['id'].eql?(connect_id) || i['title'].eql?(connect_id)}
-            raise CliNoSuchId.new(:connect, connect_id) if one_res.nil?
+            raise Cli::NoSuchIdentifier.new(:connect, connect_id) if one_res.nil?
           end
           case command
           when :list
@@ -594,21 +704,14 @@ module Aspera
             ascp_path = options.get_next_argument('path to ascp')
             ascp_version = Fasp::Installation.instance.get_ascp_version(ascp_path)
             formatter.display_status("ascp version: #{ascp_version}")
-            preset_name = set_global_default(:ascp_path, ascp_path)
-            return Main.result_status("Saved to default global preset #{preset_name}")
+            set_global_default(:ascp_path, ascp_path)
+            return Main.result_nothing
           when :show # shows files used
             return {type: :status, data: Fasp::Installation.instance.path(:ascp)}
           when :info # shows files used
-            data = Fasp::Installation::FILES.each_with_object({}) do |v, m|
-              m[v.to_s] =
-                begin
-                  Fasp::Installation.instance.path(v)
-                rescue => e
-                  e.message
-                end
-            end
+            data = Fasp::Installation.instance.file_paths
             # read PATHs from ascp directly, and pvcl modules as well
-            Open3.popen3(Fasp::Installation.instance.path(:ascp), '-DDL-') do |_stdin, _stdout, stderr, thread|
+            Open3.popen3(data['ascp'], '-DDL-') do |_stdin, _stdout, stderr, thread|
               last_line = ''
               while (line = stderr.gets)
                 line.chomp!
@@ -616,8 +719,11 @@ module Aspera
                 case line
                 when /^DBG Path ([^ ]+) (dir|file) +: (.*)$/
                   data[Regexp.last_match(1)] = Regexp.last_match(3)
-                when /^DBG Added module group:"([^"]+)" name:"([^"]+)", version:"([^"]+)" interface:"([^"]+)"$/
-                  data[Regexp.last_match(2)] = "#{Regexp.last_match(4)} #{Regexp.last_match(1)} v#{Regexp.last_match(3)}"
+                when /^DBG Added module group:"(?<module>[^"]+)" name:"(?<scheme>[^"]+)", version:"(?<version>[^"]+)" interface:"(?<interface>[^"]+)"$/
+                  c = Regexp.last_match.named_captures.symbolize_keys
+                  data[c[:interface]] ||= {}
+                  data[c[:interface]][c[:module]] ||= []
+                  data[c[:interface]][c[:module]].push("#{c[:scheme]} v#{c[:version]}")
                 when %r{^DBG License result \(/license/(\S+)\): (.+)$}
                   data[Regexp.last_match(1)] = Regexp.last_match(2)
                 when /^LOG (.+) version ([0-9.]+)$/
@@ -631,7 +737,14 @@ module Aspera
                 raise last_line
               end
             end
-            data['keypass'] = Fasp::Installation.instance.bypass_pass
+            # ascp's openssl directory
+            ascp_file = data['ascp']
+            File.binread(ascp_file).scan(/[\x20-\x7E]{4,}/) do |match|
+              if (m = match.match(/OPENSSLDIR.*"(.*)"/))
+                data['openssldir'] = m[1]
+              end
+            end if File.file?(ascp_file)
+            data['uuid'] = Fasp::Installation.instance.ssh_cert_uuid
             # log is "-" no need to display
             data.delete('log')
             # show command line transfer spec
@@ -641,16 +754,16 @@ module Aspera
             command = options.get_next_command(%i[list use])
             case command
             when :list
-              return {type: :object_list, data: Fasp::Installation.instance.installed_products, fields: %w[name app_root]}
+              return {type: :object_list, data: Fasp::Products.installed_products, fields: %w[name app_root]}
             when :use
               default_product = options.get_next_argument('product name')
               Fasp::Installation.instance.use_ascp_from_product(default_product)
-              preset_name = set_global_default(:ascp_path, Fasp::Installation.instance.path(:ascp))
-              return Main.result_status("Saved to default global preset #{preset_name}")
+              set_global_default(:ascp_path, Fasp::Installation.instance.path(:ascp))
+              return Main.result_nothing
             end
           when :install
             # reset to default location, if older default was used
-            Fasp::Installation.instance.sdk_folder = default_app_main_folder(app_name: APP_NAME_SDK) if @sdk_default_location
+            Fasp::Installation.instance.sdk_folder = self.class.default_app_main_folder(app_name: APP_NAME_SDK) if @sdk_default_location
             v = Fasp::Installation.instance.install_sdk(options.get_option(:sdk_url, mandatory: true))
             return Main.result_status("Installed version #{v}")
           when :spec
@@ -680,24 +793,23 @@ module Aspera
         def execute_preset(action: nil, name: nil)
           action = options.get_next_command(PRESET_ALL_ACTIONS) if action.nil?
           name = instance_identifier if name.nil? && PRESET_INSTANCE_ACTIONS.include?(action)
+          name = global_default_preset if name.eql?(GLOBAL_DEFAULT_KEYWORD)
           # those operations require existing option
           raise "no such preset: #{name}" if PRESET_EXIST_ACTIONS.include?(action) && !@config_presets.key?(name)
-          selected_preset = @config_presets[name]
           case action
           when :list
             return {type: :value_list, data: @config_presets.keys, name: 'name'}
           when :overview
-            return {type: :object_list, data: Formatter.flatten_config_overview(@config_presets)}
+            # display process modifies the value (hide secrets): we do not want to save removed secrets
+            return {type: :config_over, data: self.class.protect_presets(@config_presets)}
           when :show
-            raise "no such config: #{name}" if selected_preset.nil?
-            return {type: :single_object, data: selected_preset}
+            return {type: :single_object, data: self.class.protect_presets(@config_presets[name])}
           when :delete
             @config_presets.delete(name)
-            save_presets_to_config_file
             return Main.result_status("Deleted: #{name}")
           when :get
             param_name = options.get_next_argument('parameter name')
-            value = selected_preset[param_name]
+            value = @config_presets[name][param_name]
             raise "no such option in preset #{name} : #{param_name}" if value.nil?
             case value
             when Numeric, String then return {type: :text, data: ExtendedValue.instance.evaluate(value.to_s)}
@@ -705,30 +817,20 @@ module Aspera
             return {type: :single_object, data: value}
           when :unset
             param_name = options.get_next_argument('parameter name')
-            selected_preset.delete(param_name)
-            save_presets_to_config_file
+            @config_presets[name].delete(param_name)
             return Main.result_status("Removed: #{name}: #{param_name}")
           when :set
             param_name = options.get_next_argument('parameter name')
-            param_value = options.get_next_argument('parameter value')
             param_name = Manager.option_line_to_name(param_name)
-            if !@config_presets.key?(name)
-              Log.log.debug{"no such config name: #{name}, initializing"}
-              selected_preset = @config_presets[name] = {}
-            end
-            if selected_preset.key?(param_name)
-              Log.log.warn{"overwriting value: #{selected_preset[param_name]}"}
-            end
-            selected_preset[param_name] = param_value
-            save_presets_to_config_file
-            return Main.result_status("Updated: #{name}: #{param_name} <- #{param_value}")
+            param_value = options.get_next_argument('parameter value')
+            set_preset_key(name, param_name, param_value)
+            return Main.result_nothing
           when :initialize
             config_value = options.get_next_argument('extended value', type: Hash)
             if @config_presets.key?(name)
               Log.log.warn{"configuration already exists: #{name}, overwriting"}
             end
             @config_presets[name] = config_value
-            save_presets_to_config_file
             return Main.result_status("Modified: #{@option_config_file}")
           when :update
             #  get unprocessed options
@@ -736,9 +838,6 @@ module Aspera
             Log.log.debug{"opts=#{unprocessed_options}"}
             @config_presets[name] ||= {}
             @config_presets[name].merge!(unprocessed_options)
-            # fix bug in 4.4 (creating key "true" in "default" preset)
-            @config_presets[CONF_PRESET_DEFAULT].delete(true) if @config_presets[CONF_PRESET_DEFAULT].is_a?(Hash)
-            save_presets_to_config_file
             return Main.result_status("Updated: #{name}")
           when :ask
             options.ask_missing_mandatory = :yes
@@ -747,10 +846,9 @@ module Aspera
               option_value = options.get_interactive(:option, option_name)
               @config_presets[name][option_name] = option_value
             end
-            save_presets_to_config_file
             return Main.result_status("Updated: #{name}")
           when :lookup
-            BasicAuthPlugin.register_options(@agents)
+            BasicAuthPlugin.declare_options(options)
             url = options.get_option(:url, mandatory: true)
             user = options.get_option(:username, mandatory: true)
             result = lookup_preset(url: url, username: user)
@@ -789,8 +887,9 @@ module Aspera
           open
           documentation
           genkey
+          remote_certificate
           gem
-          plugin
+          plugins
           flush_tokens
           echo
           wizard
@@ -804,9 +903,10 @@ module Aspera
           file
           check_update
           initdemo
-          vault].freeze
+          vault
+          throw].freeze
 
-        # "config" plugin
+        # Main action procedure for plugin
         def execute_action
           action = options.get_next_command(ACTIONS)
           case action
@@ -817,31 +917,45 @@ module Aspera
             return Main.result_nothing
           when :documentation
             section = options.get_next_argument('private key file path', mandatory: false)
-            section = '#' + section unless section.nil?
+            section = "##{section}" unless section.nil?
             OpenApplication.instance.uri("#{@info[:help]}#{section}")
             return Main.result_nothing
           when :genkey # generate new rsa key
             private_key_path = options.get_next_argument('private key file path')
-            private_key_length = options.get_next_argument('size in bits', mandatory: false) || DEFAULT_PRIVKEY_LENGTH
+            private_key_length = options.get_next_argument('size in bits', mandatory: false, type: Integer, default: DEFAULT_PRIV_KEY_LENGTH)
             self.class.generate_rsa_private_key(path: private_key_path, length: private_key_length)
-            return Main.result_status('Generated key: ' + private_key_path)
+            return Main.result_status("Generated #{private_key_length} bit RSA key: #{private_key_path}")
+          when :remote_certificate
+            remote_url = options.get_next_argument('remote URL')
+            @option_insecure = true
+            remote_certificate = Rest.start_http_session(remote_url).peer_cert
+            remote_certificate.subject.to_a.find { |name, _, _| name == 'CN' }[1]
+            formatter.display_status("CN=#{remote_certificate.subject.to_a.find { |name, _, _| name == 'CN' }[1] rescue ''}")
+            return Main.result_status(remote_certificate.to_pem)
           when :echo # display the content of a value given on command line
-            result = {type: :other_struct, data: options.get_next_argument('value')}
-            # special for csv
-            result[:type] = :object_list if result[:data].is_a?(Array) && result[:data].first.is_a?(Hash)
-            result[:type] = :single_object if result[:data].is_a?(Hash)
-            return result
+            return Formatter.auto_type(options.get_next_argument('value'))
           when :flush_tokens
             deleted_files = Oauth.flush_tokens
             return {type: :value_list, data: deleted_files, name: 'file'}
-          when :plugin
+          when :plugins
             case options.get_next_command(%i[list create])
             when :list
-              return {type: :object_list, data: @plugins.keys.map { |i| { 'plugin' => i.to_s, 'path' => @plugins[i][:source] } }, fields: %w[plugin path]}
+              result = []
+              @plugins.each do |name, info|
+                require info[:require_stanza]
+                plugin_class = self.class.plugin_class(name)
+                result.push({
+                  plugin: name,
+                  detect: Formatter.tick(plugin_class.respond_to?(:detect)),
+                  wizard: Formatter.tick(plugin_class.respond_to?(:wizard)),
+                  path:   info[:source]
+                })
+              end
+              return {type: :object_list, data: result, fields: %w[plugin detect wizard path]}
             when :create
               plugin_name = options.get_next_argument('name', expected: :single).downcase
-              plugin_folder = options.get_next_argument('folder', expected: :single, mandatory: false) || File.join(@main_folder, ASPERA_PLUGINS_FOLDERNAME)
-              plugin_file = File.join(plugin_folder, "#{plugin_name}.rb")
+              destination_folder = options.get_next_argument('folder', expected: :single, mandatory: false) || File.join(@main_folder, ASPERA_PLUGINS_FOLDERNAME)
+              plugin_file = File.join(destination_folder, "#{plugin_name}.rb")
               content = <<~END_OF_PLUGIN_CODE
                 require 'aspera/cli/plugin'
                 module Aspera
@@ -858,94 +972,20 @@ module Aspera
               File.write(plugin_file, content)
               return Main.result_status("Created #{plugin_file}")
             end
-          when :wizard
+          when :detect, :wizard
             # interactive mode
             options.ask_missing_mandatory = true
-            # register url option
-            BasicAuthPlugin.register_options(@agents)
-            params = {}
-            # get from option, or ask
-            params[:instance_url] = options.get_option(:url, mandatory: true)
-            # check it is a well formatted url: starts with scheme
-            if !params[:instance_url].match?(%r{^[a-z]{1,6}://})
-              new_url = "https://#{params[:instance_url]}"
-              Log.log.warn("URL #{params[:instance_url]} does not start with a scheme, using #{new_url}")
-              params[:instance_url] = new_url
-            end
-            # allow user to tell the preset name
-            params[:preset_name] = options.get_option(:id)
-            # allow user to specify type of application (symbol)
-            identification = identify_plugin_for_url(params[:instance_url], check_only: value_or_query(allowed_types: String))
-            Log.log.debug{"Detected: #{identification}"}
-            formatter.display_status("Detected: #{identification[:name]} at #{identification[:url]}".bold)
-            # we detected application (not set by user)
-            params[:plugin_sym] = identification[:product]
-            # update the url option
-            params[:instance_url] = identification[:url]
-            options.set_option(:url, params[:instance_url])
-            # instantiate plugin: command line options are known and wizard can be called
-            plugin_instance = self.class.plugin_class(params[:plugin_sym]).new(@agents.merge({skip_basic_auth_options: true}))
-            raise CliBadArgument, "Detected: #{params[:plugin_sym]}, no wizard available for this application" unless plugin_instance.respond_to?(:wizard)
-            # get default preset name if not set by user
-            params[:prepare] = true
-            plugin_instance.send(:wizard, params)
-            params[:prepare] = false
-
-            if params[:need_private_key]
-              # lets see if path to priv key is provided
-              private_key_path = options.get_option(:pkeypath)
-              # give a chance to provide
-              if private_key_path.nil?
-                formatter.display_status('Please provide path to your private RSA key, or empty to generate one:')
-                private_key_path = options.get_option(:pkeypath, mandatory: true).to_s
-                # private_key_path = File.expand_path(private_key_path)
-              end
-              # else generate path
-              if private_key_path.empty?
-                private_key_path = File.join(@main_folder, DEFAULT_PRIV_KEY_FILENAME)
-              end
-              if File.exist?(private_key_path)
-                formatter.display_status('Using existing key:')
-              else
-                formatter.display_status("Generating #{DEFAULT_PRIVKEY_LENGTH} bit RSA key...")
-                Config.generate_rsa_private_key(path: private_key_path)
-                formatter.display_status('Created key:')
-              end
-              formatter.display_status(private_key_path)
-              params[:pub_key_pem] = OpenSSL::PKey::RSA.new(File.read(private_key_path)).public_key.to_s
-              params[:private_key_path] = private_key_path
-            end
-
-            formatter.display_status("Preparing preset: #{params[:preset_name]}")
-            # init defaults if necessary
-            @config_presets[CONF_PRESET_DEFAULT] ||= {}
-            option_override = options.get_option(:override, mandatory: true)
-            raise CliError, "A default configuration already exists for plugin '#{params[:plugin_sym]}' (use --override=yes or --default=no)" \
-              if !option_override && options.get_option(:default, mandatory: true) && @config_presets[CONF_PRESET_DEFAULT].key?(params[:plugin_sym])
-            raise CliError, "Preset already exists: #{params[:preset_name]}  (use --override=yes or --id=<name>)" \
-              if !option_override && @config_presets.key?(params[:preset_name])
-            wizard_result = plugin_instance.send(:wizard, params)
-            Log.log.debug{"wizard result: #{wizard_result}"}
-            raise "Internal error: missing keys in wizard result: #{wizard_result.keys}" unless %i[preset_value test_args].eql?(wizard_result.keys.sort)
-            @config_presets[params[:preset_name]] = wizard_result[:preset_value].stringify_keys
-            params[:test_args] = wizard_result[:test_args]
-            if options.get_option(:default, mandatory: true)
-              formatter.display_status("Setting config preset as default for #{params[:plugin_sym]}")
-              @config_presets[CONF_PRESET_DEFAULT][params[:plugin_sym].to_s] = params[:preset_name]
-            else
-              params[:test_args] = "-P#{params[:preset_name]} #{params[:test_args]}"
-            end
-            formatter.display_status('Saving config file.')
-            save_presets_to_config_file
-            return Main.result_status("Done.\nYou can test with:\n#{@info[:name]} #{params[:test_args]}")
-          when :detect
-            # need url / username
-            BasicAuthPlugin.register_options(@agents)
-            return {type: :single_object, data: identify_plugin_for_url(options.get_option(:url, mandatory: true))}
+            # detect plugins by url and optional query
+            apps = identify_plugins_for_url.freeze
+            return {
+              type: :object_list,
+              data: apps
+            } if action.eql?(:detect)
+            return wizard_find(apps)
           when :coffee
             if OpenApplication.instance.url_method.eql?(:text)
               require 'aspera/preview/terminal'
-              return Main.result_status(Preview::Terminal.build(Rest.new(base_url: COFFEE_IMAGE).read('')[:http].body, reserved_lines: 3))
+              return Main.result_status(Preview::Terminal.build(Rest.new(base_url: COFFEE_IMAGE).read('')[:http].body))
             end
             OpenApplication.instance.uri(COFFEE_IMAGE)
             return Main.result_nothing
@@ -979,7 +1019,7 @@ module Aspera
             else
               Log.log.info{"Creating Demo server preset: #{DEMO_SERVER_PRESET}"}
               @config_presets[DEMO_SERVER_PRESET] = {
-                'url'                                    => 'ssh://' + DEMO + '.asperasoft.com:33001',
+                'url'                                    => "ssh://#{DEMO}.asperasoft.com:33001",
                 'username'                               => ASPERA,
                 'ssAP'.downcase.reverse + 'drow'.reverse => DEMO + ASPERA # cspell:disable-line
               }
@@ -993,16 +1033,112 @@ module Aspera
               @config_presets[CONF_PRESET_DEFAULT][SERVER_COMMAND] = DEMO_SERVER_PRESET
               Log.log.info{"Setting server default preset to : #{DEMO_SERVER_PRESET}"}
             end
-            save_presets_to_config_file
             return Main.result_status('Done')
           when :vault then execute_vault
+          when :throw
+            # :type [String]
+            options
+            exception_class_name = options.get_next_argument('exception class name', mandatory: true)
+            exception_text = options.get_next_argument('exception text', mandatory: true)
+            exception_class = Object.const_get(exception_class_name)
+            raise "#{exception_class} is not an exception: #{exception_class.class}" unless exception_class <= Exception
+            raise exception_class, exception_text
           else raise 'INTERNAL ERROR: wrong case'
           end
         end
 
-        # @return email server setting with defaults if not defined
+        def wizard_find(apps)
+          identification = if apps.length.eql?(1)
+            Log.log.debug{"Detected: #{identification}"}
+            apps.first
+          else
+            formatter.display_status('Multiple applications detected, please select from:')
+            formatter.display_results({type: :object_list, data: apps, fields: %w[product url version]})
+            answer = options.prompt_user_input_in_list('product', apps.map{|a|a[:product]})
+            apps.find{|a|a[:product].eql?(answer)}
+          end
+          wiz_url = identification[:url]
+          Log.log.debug{Log.dump(:identification, identification, :ruby)}
+          formatter.display_status("Using: #{identification[:name]} at #{wiz_url}".bold)
+          # set url for instantiation of plugin
+          options.add_option_preset({url: wiz_url})
+          # instantiate plugin: command line options will be known and wizard can be called
+          wiz_plugin_class = self.class.plugin_class(identification[:product])
+          raise Cli::BadArgument, "Detected: #{identification[:product]}, but this application has no wizard" unless wiz_plugin_class.respond_to?(:wizard)
+          # instantiate plugin: command line options will be known, e.g. private_key
+          plugin_instance = wiz_plugin_class.new(@agents)
+          wiz_params = {
+            object: plugin_instance
+          }
+          # is private key needed ?
+          if options.known_options.key?(:private_key) &&
+              (!wiz_plugin_class.respond_to?(:private_key_required?) || wiz_plugin_class.private_key_required?(wiz_url))
+            # lets see if path to priv key is provided
+            private_key_path = options.get_option(:key_path)
+            # give a chance to provide
+            if private_key_path.nil?
+              formatter.display_status('Please provide the path to your private RSA key, or nothing to generate one:')
+              private_key_path = options.get_option(:key_path, mandatory: true).to_s
+              # private_key_path = File.expand_path(private_key_path)
+            end
+            # else generate path
+            if private_key_path.empty?
+              private_key_path = File.join(@main_folder, DEFAULT_PRIV_KEY_FILENAME)
+            end
+            if File.exist?(private_key_path)
+              formatter.display_status('Using existing key:')
+            else
+              formatter.display_status("Generating #{DEFAULT_PRIV_KEY_LENGTH} bit RSA key...")
+              Config.generate_rsa_private_key(path: private_key_path)
+              formatter.display_status('Created key:')
+            end
+            formatter.display_status(private_key_path)
+            private_key_pem = File.read(private_key_path)
+            options.set_option(:private_key, private_key_pem)
+            wiz_params[:private_key_path] = private_key_path
+            wiz_params[:pub_key_pem] = OpenSSL::PKey::RSA.new(private_key_pem).public_key.to_s
+          end
+          Log.log.debug{Log.dump(:wiz_params, wiz_params)}
+          # finally, call the wizard
+          wizard_result = wiz_plugin_class.wizard(**wiz_params)
+          Log.log.debug{"wizard result: #{wizard_result}"}
+          raise "Internal error: missing or extra keys in wizard result: #{wizard_result.keys}" unless WIZARD_RESULT_KEYS.eql?(wizard_result.keys.sort)
+          # get preset name from user or default
+          wiz_preset_name = options.get_option(:id)
+          if wiz_preset_name.nil?
+            elements = [
+              identification[:product],
+              URI.parse(wiz_url).host
+            ]
+            elements.push(options.get_option(:username, mandatory: true)) unless wizard_result[:preset_value].key?(:link)
+            wiz_preset_name = elements.join('_').strip.downcase.gsub(/[^a-z0-9]/, '_').squeeze('_')
+          end
+          # test mode does not change conf file
+          return {type: :single_object, data: wizard_result} if options.get_option(:test_mode)
+          # Write configuration file
+          formatter.display_status("Preparing preset: #{wiz_preset_name}")
+          # init defaults if necessary
+          @config_presets[CONF_PRESET_DEFAULT] ||= {}
+          option_override = options.get_option(:override, mandatory: true)
+          raise Cli::Error, "A default configuration already exists for plugin '#{identification[:product]}' (use --override=yes or --default=no)" \
+            if !option_override && options.get_option(:default, mandatory: true) && @config_presets[CONF_PRESET_DEFAULT].key?(identification[:product])
+          raise Cli::Error, "Preset already exists: #{wiz_preset_name}  (use --override=yes or --id=<name>)" \
+            if !option_override && @config_presets.key?(wiz_preset_name)
+          @config_presets[wiz_preset_name] = wizard_result[:preset_value].stringify_keys
+          test_args = wizard_result[:test_args]
+          if options.get_option(:default, mandatory: true)
+            formatter.display_status("Setting config preset as default for #{identification[:product]}")
+            @config_presets[CONF_PRESET_DEFAULT][identification[:product].to_s] = wiz_preset_name
+          else
+            test_args = "-P#{wiz_preset_name} #{test_args}"
+          end
+          # TODO: actually test the command
+          return Main.result_status("You can test with:\n#{@info[:name]} #{identification[:product]} #{test_args}")
+        end
+
+        # @return [Hash] email server setting with defaults if not defined
         def email_settings
-          smtp = options.get_option(:smtp, mandatory: true, allowed_types: [Hash])
+          smtp = options.get_option(:smtp, mandatory: true)
           # change string keys into symbol keys
           smtp = smtp.symbolize_keys
           # defaults
@@ -1025,14 +1161,10 @@ module Aspera
           return smtp
         end
 
-        # create a clean binding (ruby variable environment)
-        def empty_binding
-          Kernel.binding
-        end
-
+        # send email using ERB template
         def send_email_template(email_template_default: nil, values: {})
-          values[:to] ||= options.get_option(:notif_to, mandatory: true)
-          notif_template = options.get_option(:notif_template, mandatory: email_template_default.nil?) || email_template_default
+          values[:to] ||= options.get_option(:notify_to, mandatory: true)
+          notify_template = options.get_option(:notify_template, mandatory: email_template_default.nil?) || email_template_default
           mail_conf = email_settings
           values[:from_name] ||= mail_conf[:from_name]
           values[:from_email] ||= mail_conf[:from_email]
@@ -1042,30 +1174,38 @@ module Aspera
           start_options = [mail_conf[:domain]]
           start_options.push(mail_conf[:username], mail_conf[:password], :login) if mail_conf.key?(:username) && mail_conf.key?(:password)
           # create a binding with only variables defined in values
-          template_binding = empty_binding
+          template_binding = Environment.empty_binding
           # add variables to binding
           values.each do |k, v|
             raise "key (#{k.class}) must be Symbol" unless k.is_a?(Symbol)
             template_binding.local_variable_set(k, v)
           end
           # execute template
-          msg_with_headers = ERB.new(notif_template).result(template_binding)
-          Log.dump(:msg_with_headers, msg_with_headers)
+          msg_with_headers = ERB.new(notify_template).result(template_binding)
+          Log.log.debug{Log.dump(:msg_with_headers, msg_with_headers)}
+          require 'net/smtp'
           smtp = Net::SMTP.new(mail_conf[:server], mail_conf[:port])
           smtp.enable_starttls if mail_conf[:tls]
           smtp.enable_tls if mail_conf[:ssl]
           smtp.start(*start_options) do |smtp_session|
             smtp_session.send_message(msg_with_headers, values[:from_email], values[:to])
           end
+          nil
         end
 
-        def save_presets_to_config_file
+        # Save current configuration to config file
+        # return true if file was saved
+        def save_config_file_if_needed
           raise 'no configuration loaded' if @config_presets.nil?
-          FileUtils.mkdir_p(@main_folder) unless Dir.exist?(@main_folder)
-          Log.log.debug{"Writing #{@option_config_file}"}
-          File.write(@option_config_file, @config_presets.to_yaml)
+          current_checksum = config_checksum
+          return false if @config_checksum_on_disk.eql?(current_checksum)
+          FileUtils.mkdir_p(@main_folder)
           Environment.restrict_file_access(@main_folder)
-          Environment.restrict_file_access(@option_config_file)
+          Log.log.info{"Writing #{@option_config_file}"}
+          formatter.display_status('Saving config file.')
+          Environment.write_file_restricted(@option_config_file, force: true) {@config_presets.to_yaml}
+          @config_checksum_on_disk = current_checksum
+          return true
         end
 
         # returns [String] name if config_presets has default
@@ -1081,18 +1221,19 @@ module Aspera
             default_config_name = @config_presets[CONF_PRESET_DEFAULT][plugin_name_sym.to_s]
             if !@config_presets.key?(default_config_name)
               Log.log.error do
-                "Default config name [#{default_config_name}] specified for plugin [#{plugin_name_sym}], but it does not exist in config file.\n"\
-                  'Please fix the issue: either create preset with one parameter: '\
+                "Default config name [#{default_config_name}] specified for plugin [#{plugin_name_sym}], but it does not exist in config file.\n" \
+                  'Please fix the issue: either create preset with one parameter: ' \
                   "(#{@info[:name]} config id #{default_config_name} init @json:'{}') or remove default (#{@info[:name]} config id default remove #{plugin_name_sym})."
               end
             end
-            raise CliError, "Config name [#{default_config_name}] must be a hash, check config file." if !@config_presets[default_config_name].is_a?(Hash)
+            raise Cli::Error, "Config name [#{default_config_name}] must be a hash, check config file." if !@config_presets[default_config_name].is_a?(Hash)
             return default_config_name
           end
           return nil
         end # get_plugin_default_config_name
 
-        ALLOWED_KEYS = %i[password username description].freeze
+        # TODO: delete: ALLOWED_KEYS = %i[password username description].freeze
+        # @return [Hash] result of execution of vault command
         def execute_vault
           command = options.get_next_command(%i[list show create delete password])
           case command
@@ -1120,21 +1261,22 @@ module Aspera
           end
         end
 
+        # @return [String] value from vault matching <name>.<param>
         def vault_value(name)
           m = name.match(/^(.+)\.(.+)$/)
           raise 'vault name shall match <name>.<param>' if m.nil?
+          # this raise exception if label not found:
           info = vault.get(label: m[1])
-          # raise "no such vault entry: #{m[1]}" if info.nil?
           value = info[m[2].to_sym]
           raise "no such entry value: #{m[2]}" if value.nil?
           return value
         end
 
+        # @return [Object] vault, from options or cache
         def vault
           if @vault.nil?
             vault_info = options.get_option(:vault) || {'type' => 'file', 'name' => 'vault.bin'}
             vault_password = options.get_option(:vault_password, mandatory: true)
-            raise 'vault must be Hash' unless vault_info.is_a?(Hash)
             vault_type = vault_info['type'] || 'file'
             vault_name = vault_info['name'] || (vault_type.eql?('file') ? 'vault.bin' : PROGRAM_NAME)
             case vault_type
@@ -1146,12 +1288,11 @@ module Aspera
               case Environment.os
               when Environment::OS_X
                 @vault = Keychain::MacosSystem.new(vault_name, vault_password)
-              when Environment::OS_WINDOWS, Environment::OS_LINUX, Environment::OS_AIX
-                raise 'not implemented'
-              else raise 'Error, OS not supported'
+              else
+                raise 'not implemented for this OS'
               end
             else
-              raise CliBadArgument, "Unknown vault type: #{vault_type}"
+              raise Cli::BadArgument, "Unknown vault type: #{vault_type}"
             end
           end
           raise 'No vault defined' if @vault.nil?
@@ -1170,7 +1311,7 @@ module Aspera
           @config_presets.each do |_k, v|
             next unless v.is_a?(Hash)
             conf_url = v['url'].is_a?(String) ? canonical_url(v['url']) : nil
-            return v if conf_url.eql?(url) && v['username'].eql?(username)
+            return self.class.protect_presets(v) if conf_url.eql?(url) && v['username'].eql?(username)
           end
           nil
         end