kafo 5.0.1 → 6.1.2

data/lib/kafo/hooking.rb

@@ -2,15 +2,16 @@ require 'kafo/hook_context'
 
 module Kafo
   class Hooking
-    # pre_migrations - just after kafo reads its configuration - useful for config file updates. Only in this stage it is posible to request config reload (`Kafo.request_config_reload`) to get in our changes
-    # boot - before kafo is ready to work, useful for adding new app arguments, logger won't work yet
-    # init - just after hooking is initialized and kafo is configured, parameters have no values yet
-    # pre_values - just before value from CLI is set to parameters (they already have default values)
-    # pre_validations - just after system checks and before validations are executed (and before interactive wizard is started), at this point all parameter values are already set but not yet stored in answer file
-    # pre_commit - after validations or interactive wizard have completed, all parameter values are set but not yet stored in the answer file
-    # pre - just before puppet is executed to converge system
-    # post - just after puppet is executed to converge system
-    TYPES = [:pre_migrations, :boot, :init, :pre, :post, :pre_values, :pre_validations, :pre_commit]
+    # * pre_migrations - just after kafo reads its configuration - useful for config file updates. Only in this stage it is posible to request config reload (`Kafo.request_config_reload`) to get in our changes
+    # * boot - before kafo is ready to work, useful for adding new app arguments, logger won't work yet
+    # * init - just after hooking is initialized and kafo is configured, parameters have no values yet
+    # * pre_values - just before value from CLI is set to parameters (they already have default values)
+    # * pre_validations - just after system checks and before validations are executed (and before interactive wizard is started), at this point all parameter values are already set but not yet stored in answer file
+    # * pre_commit - after validations or interactive wizard have completed, all parameter values are set but not yet stored in the answer file
+    # * pre - just before puppet is executed to converge system
+    # * post - just after puppet is executed to converge system
+    # * pre_exit - happens during exit handling, before exit is completed
+    TYPES = [:pre_migrations, :boot, :init, :pre_values, :pre_validations, :pre_commit, :pre, :post, :pre_exit]
 
     attr_accessor :hooks, :kafo
 
@@ -44,14 +45,16 @@ module Kafo
       @loaded
     end
 
-    def execute(group)
-      logger.info "Executing hooks in group #{group}"
+    def execute(group, log_stage: true)
+      logger = Logger.new(group)
+      logger.notice "Executing hooks in group #{group}" if log_stage
       self.hooks[group].keys.sort_by(&:to_s).each do |name|
         hook = self.hooks[group][name]
-        result = HookContext.execute(self.kafo, &hook)
+        result = HookContext.execute(self.kafo, logger, &hook)
         logger.debug "Hook #{name} returned #{result.inspect}"
       end
-      logger.info "All hooks in group #{group} finished"
+      logger.notice "All hooks in group #{group} finished" if log_stage
+      @group = nil
     end
 
     def register_pre_migrations(name, &block)
@@ -86,6 +89,10 @@ module Kafo
       register(:post, name, &block)
     end
 
+    def register_pre_exit(name, &block)
+      register(:pre_exit, name, &block)
+    end
+
     private
 
     def register(group, name, &block)

data/lib/kafo/kafo_configure.rb

@@ -27,20 +27,71 @@ require 'kafo/hooking'
 require 'kafo/exit_handler'
 require 'kafo/scenario_manager'
 require 'kafo/execution_environment'
+require 'kafo/logging'
+require 'kafo/app_option/declaration'
 
 module Kafo
   class KafoConfigure < Clamp::Command
     include StringHelper
 
     class << self
+      include AppOption::Declaration
+
       attr_accessor :config, :root_dir, :config_file, :gem_root,
-                    :module_dirs, :kafo_modules_dir, :verbose, :app_options, :logger,
+                    :module_dirs, :kafo_modules_dir, :verbose, :logger,
                     :check_dirs, :exit_handler, :scenario_manager, :store
       attr_writer :hooking
 
       def hooking
         @hooking ||= Hooking.new
       end
+
+      def run
+        return super
+      rescue SystemExit
+        self.exit_handler.exit(self.exit_code) # fail in initialize
+      end
+
+      def exit(code, &block)
+        exit_handler.exit(code, &block)
+      end
+
+      def exit_code
+        self.exit_handler.exit_code
+      end
+
+      def in_help_mode?
+        ARGV.include?('--help') || ARGV.include?('--full-help') || ARGV.include?('-h')
+      end
+
+      def help(*args)
+        kafo          = args.pop
+        builder_class = kafo.full_help? ? HelpBuilders::Advanced : HelpBuilders::Basic
+        args.push builder_class.new(kafo.params)
+        super(*args)
+      end
+
+      def use_colors?
+        if config
+          colors = config.app[:colors]
+        else
+          colors = ARGV.include?('--no-colors') ? false : nil
+          colors = ARGV.include?('--colors') ? true : nil if colors.nil?
+        end
+        colors
+      end
+
+      def preset_color_scheme
+        match = ARGV.join(' ').match(/--color-of-background[ =](\w+)/)
+        background = match && match[1]
+        ColorScheme.new(:background => background, :colors => use_colors?).setup
+      end
+
+      def set_color_scheme
+        ColorScheme.new(
+          :background => config.app[:color_of_background],
+          :colors => use_colors?).setup
+      end
     end
 
     def initialize(*args)
@@ -65,9 +116,9 @@ module Kafo
       request_config_reload if applied_total > 0
 
       if ARGV.include?('--migrations-only')
-        self.class.verbose = (ARGV.include?('--verbose') || ARGV.include?('-v'))
-        Logger.setup
-        self.class.logger.info('Log buffers flushed')
+        verbose = (ARGV.include?('--verbose') || ARGV.include?('-v'))
+        Logging.setup(verbose: verbose)
+        self.class.logger.notice('Log buffers flushed')
         self.class.exit(0)
       end
 
@@ -80,7 +131,7 @@ module Kafo
           prev_config.run_migrations
           self.class.config.migrate_configuration(prev_config, :skip => [:log_name])
           setup_config(self.class.config_file)
-          self.class.logger.info("Due to scenario change the configuration (#{self.class.config_file}) was updated with #{scenario_manager.previous_scenario} and reloaded.")
+          self.class.logger.notice("Due to scenario change the configuration (#{self.class.config_file}) was updated with #{scenario_manager.previous_scenario} and reloaded.")
         end
       end
 
@@ -92,7 +143,7 @@ module Kafo
       # so we limit parsing only to app config options (because of --help and later defined params)
       parse clamp_app_arguments
       parse_app_arguments # set values from ARGS to config.app
-      Logger.setup
+      Logging.setup(verbose: config.app[:verbose]) unless ARGV.any? { |option| ['--help', '--full-help'].include? option }
       self.class.set_color_scheme
 
       self.class.hooking.execute(:init)
@@ -114,19 +165,17 @@ module Kafo
 
     def run(*args)
       started_at = Time.now
-      logger.info("Running installer with args #{args.inspect}")
+      logger.debug("Running installer with args #{args.inspect}")
       super
     ensure
-      logger.info("Installer finished in #{Time.now - started_at} seconds")
+      logger.debug("Installer finished in #{Time.now - started_at} seconds")
     end
 
     def execute
       parse_cli_arguments
 
-      if (self.class.verbose = !!verbose?)
-        Logger.setup_verbose
-      else
-        @progress_bar = self.class.config.app[:colors] ? ProgressBars::Colored.new : ProgressBars::BlackWhite.new
+      if !config.app[:verbose]
+        @progress_bar = config.app[:colors] ? ProgressBars::Colored.new : ProgressBars::BlackWhite.new
       end
 
       unless skip_checks_i_know_better?
@@ -149,6 +198,7 @@ module Kafo
 
       self.class.hooking.execute(:pre_commit)
       unless dont_save_answers? || noop?
+        config.configure_application
         store_params
         self.class.scenario_manager.link_last_scenario(self.class.config_file) if self.class.scenario_manager.configured?
       end
@@ -158,46 +208,14 @@ module Kafo
       return self
     end
 
-    def self.run
-      return super
-    rescue SystemExit
-      self.exit_handler.exit(self.exit_code) # fail in initialize
-    end
-
-    def self.exit(code, &block)
-      exit_handler.exit(code, &block)
-    end
-
-    def self.exit_code
-      self.exit_handler.exit_code
-    end
-
-    def self.in_help_mode?
-      ARGV.include?('--help') || ARGV.include?('--full-help') || ARGV.include?('-h')
-    end
-
     def exit_code
       self.class.exit_code
     end
 
-
     def help
       self.class.help(invocation_path, self)
     end
 
-    def self.help(*args)
-      kafo          = args.pop
-      builder_class = kafo.full_help? ? HelpBuilders::Advanced : HelpBuilders::Basic
-      args.push builder_class.new(kafo.params)
-      super(*args)
-    end
-
-    def self.app_option(*args, &block)
-      self.app_options ||= []
-      self.app_options.push self.option(*args, &block)
-      self.app_options.last
-    end
-
     def params
       @params ||= modules.map(&:params).flatten
     rescue KafoParsers::ModuleName => e
@@ -236,7 +254,6 @@ module Kafo
       @config_reload_requested = true
     end
 
-
     private
 
     def setup_config(conf_file)
@@ -263,7 +280,7 @@ module Kafo
         scenario_manager = setup_scenario_manager
         self.class.scenario_manager = scenario_manager
         setup_config(self.class.config_file)
-        self.class.logger.info('Installer configuration was reloaded')
+        self.class.logger.notice('Installer configuration was reloaded')
         @config_reload_requested = false
       end
     end
@@ -290,56 +307,67 @@ module Kafo
       end
     end
 
+    def app_option(*args, &block)
+      self.class.app_option(*args, &block)
+    end
+
     def set_app_options
-      self.class.app_option ['--[no-]colors'], :flag, 'Use color output on STDOUT',
-                            :default => !!config.app[:colors]
-      self.class.app_option ['--color-of-background'], 'COLOR', 'Your terminal background is :bright or :dark',
-                            :default => config.app[:color_of_background]
-      self.class.app_option ['--dont-save-answers'], :flag, "Skip saving answers to '#{self.class.config.answer_file}'?",
-                            :default => !!config.app[:dont_save_answers]
-      self.class.app_option '--ignore-undocumented', :flag, 'Ignore inconsistent parameter documentation',
-                            :default => false
-      self.class.app_option ['-i', '--interactive'], :flag, 'Run in interactive mode'
-      self.class.app_option '--log-level', 'LEVEL', 'Log level for log file output',
-                            :default => config.app[:log_level]
-      self.class.app_option ['-n', '--noop'], :flag, 'Run puppet in noop mode?',
-                            :default => false
-      self.class.app_option ['-p', '--profile'], :flag, 'Run puppet in profile mode?',
-                            :default => false
-      self.class.app_option ['-s', '--skip-checks-i-know-better'], :flag, 'Skip all system checks', :default => false
-      self.class.app_option ['--skip-puppet-version-check'], :flag, 'Skip check for compatible Puppet versions', :default => false
-      self.class.app_option ['-v', '--verbose'], :flag, 'Display log on STDOUT instead of progressbar'
-      self.class.app_option ['-l', '--verbose-log-level'], 'LEVEL', 'Log level for verbose mode output',
-                            :default => 'info'
-      self.class.app_option ['-S', '--scenario'], 'SCENARIO', 'Use installation scenario'
-      self.class.app_option ['--disable-scenario'], 'SCENARIO', 'Disable installation scenario'
-      self.class.app_option ['--enable-scenario'], 'SCENARIO', 'Enable installation scenario'
-      self.class.app_option ['--list-scenarios'], :flag, 'List available installation scenarios'
-      self.class.app_option ['--force'], :flag, 'Force change of installation scenario'
-      self.class.app_option ['--compare-scenarios'], :flag, 'Show changes between last used scenario and the scenario specified with -S or --scenario argument'
-      self.class.app_option ['--migrations-only'], :flag, 'Apply migrations to a selected scenario and exit'
-      self.class.app_option ['--[no-]parser-cache'], :flag, 'Force use or bypass of Puppet module parser cache'
+      app_option ['--[no-]colors'], :flag, 'Use color output on STDOUT',
+                 :default => config.app[:colors], :advanced => true
+      app_option ['--color-of-background'], 'COLOR', 'Your terminal background is :bright or :dark',
+                 :default => config.app[:color_of_background], :advanced => true
+      app_option ['--dont-save-answers'], :flag, "Skip saving answers to '#{self.class.config.answer_file}'?",
+                 :default => config.app[:dont_save_answers], :advanced => true
+      app_option '--ignore-undocumented', :flag, 'Ignore inconsistent parameter documentation',
+                 :default => config.app[:ignore_undocumented], :advanced => true
+      app_option ['-i', '--interactive'], :flag, 'Run in interactive mode'
+      app_option '--log-level', 'LEVEL', 'Log level for log file output',
+                 :default => config.app[:log_level], :advanced => true
+      app_option ['-n', '--noop'], :flag, 'Run puppet in noop mode?',
+                 :default => false
+      app_option ['-p', '--profile'], :flag, 'Run puppet in profile mode?',
+                 :default => false, :advanced => true
+      app_option ['-s', '--skip-checks-i-know-better'], :flag, 'Skip all system checks',
+                 :default => false
+      app_option ['--skip-puppet-version-check'], :flag, 'Skip check for compatible Puppet versions',
+                 :default => false, :advanced => true
+      app_option ['-v', '--[no-]verbose'], :flag, 'Display log on STDOUT instead of progressbar',
+                 :default => config.app[:verbose]
+      app_option ['-l', '--verbose-log-level'], 'LEVEL', 'Log level for verbose mode output',
+                 :default => 'notice'
+      app_option ['-S', '--scenario'], 'SCENARIO', 'Use installation scenario'
+      app_option ['--disable-scenario'], 'SCENARIO', 'Disable installation scenario',
+                 :advanced => true
+      app_option ['--enable-scenario'], 'SCENARIO', 'Enable installation scenario',
+                 :advanced => true
+      app_option ['--list-scenarios'], :flag, 'List available installation scenarios'
+      app_option ['--force'], :flag, 'Force change of installation scenario',
+                 :advanced => true
+      app_option ['--compare-scenarios'], :flag, 'Show changes between last used scenario and the scenario specified with -S or --scenario argument',
+                 :advanced => true
+      app_option ['--migrations-only'], :flag, 'Apply migrations to a selected scenario and exit',
+                 :advanced => true
+      app_option ['--[no-]parser-cache'], :flag, 'Force use or bypass of Puppet module parser cache',
+                 :advanced => true
     end
 
     def set_options
-      self.class.option '--full-help', :flag, "print complete help" do
+      app_option '--full-help', :flag, "print complete help" do
         @full_help = true
         request_help
       end
 
       modules.each do |mod|
-        self.class.option d("--[no-]enable-#{mod.name}"),
-                          :flag,
-                          "Enable '#{mod.name}' puppet module",
-                          :default => mod.enabled?
+        app_option d("--[no-]enable-#{mod.name}"), :flag, "Enable '#{mod.name}' puppet module",
+                   :default => mod.enabled?
       end
 
       params.sort.each do |param|
         doc = param.doc.nil? ? 'UNDOCUMENTED' : param.doc.join("\n")
-        self.class.option parametrize(param), '', doc + " (current: #{param.value_to_s})",
-                          :multivalued => param.multivalued?
-        self.class.option parametrize(param, 'reset-'), :flag,
-                          "Reset #{param.name} to the default value (#{param.default_to_s})"
+        app_option parametrize(param), '', doc + " (current: #{param.value_to_s})",
+                   :multivalued => param.multivalued?
+        app_option parametrize(param, 'reset-'), :flag,
+                   "Reset #{param.name} to the default value (#{param.default_to_s})"
       end
     end
 
@@ -347,13 +375,15 @@ module Kafo
     # so we accept either allowed args or those that does not start with '-' and are right after
     # accepted argument
     def clamp_app_arguments
-      @allowed_clamp_app_arguments = self.class.app_options.map do |option|
+      @allowed_clamp_app_arguments = self.class.declared_options.map do |option|
         option.switches.map { |s| is_yes_no_flag?(s) ? build_yes_no_variants(s) : s }
       end
       @allowed_clamp_app_arguments.flatten!
 
       last_was_accepted = false
-      ARGV.select { |arg| last_was_accepted = is_allowed_attribute_name?(arg) || (last_was_accepted && is_value?(arg)) }
+      ARGV.select do |arg|
+        last_was_accepted = is_allowed_attribute_name?(arg) || (last_was_accepted && is_value?(arg))
+      end
     end
 
     def is_yes_no_flag?(s)
@@ -373,7 +403,7 @@ module Kafo
     end
 
     def parse_app_arguments
-      self.class.app_options.each do |option|
+      self.class.declared_options.each do |option|
         name                    = option.attribute_name
         value                   = send(option.flag? ? "#{name}?" : name)
         config.app[name.to_sym] = value.nil? ? option.default_value : value
@@ -410,11 +440,11 @@ module Kafo
     end
 
     def validate_all(logging = true)
-      logger.info 'Running validation checks'
+      logger.notice 'Running validation checks'
       results = enabled_params.map do |param|
         result = param.valid?
         errors = param.validation_errors.join(', ')
-        progress_log(:error, "Parameter #{with_prefix(param)} invalid: #{errors}") if logging && !result
+        progress_log(:error, "Parameter #{with_prefix(param)} invalid: #{errors}", logger) if logging && !result
         result
       end
       results.all?
@@ -429,13 +459,13 @@ module Kafo
       execution_env.store_answers
       puppetconf = execution_env.configure_puppet(
         'color'     => false,
-        'evaltrace' => !!@progress_bar,
+        'evaltrace' => true,
         'noop'      => !!noop?,
         'profile'   => !!profile?,
         'show_diff' => true,
       )
 
-      exit_code   = 0
+      self.class.exit_handler.exit_code = 0
       exit_status = nil
       options     = [
           '--verbose',
@@ -445,11 +475,29 @@ module Kafo
       begin
         command = PuppetCommand.new('include kafo_configure', options, puppetconf).command
         log_parser = PuppetLogParser.new
+        logger = Logger.new('configure')
+
+        start_message = <<-HEREDOC
+Starting system configuration.
+  The total number of configuration tasks may increase during the run.
+  Observe logs or specify --verbose-log-level to see individual configuration tasks.
+HEREDOC
+
+        logger.notice(start_message.chomp)
+
         PTY.spawn(*PuppetCommand.format_command(command)) do |stdin, stdout, pid|
           begin
             stdin.each do |line|
               line = normalize_encoding(line)
-              progress_log(*log_parser.parse(line))
+              method, message = log_parser.parse(line)
+              progress_log(method, message, logger)
+
+              if (output = line.match(%r{(.+\]): Starting to evaluate the resource( \((?<count>\d+) of (?<total>\d+)\))?}))
+                if (output[:count].to_i % 100) == 1 && output[:count].to_i != 1
+                  logger.notice("#{output[:count].to_i - 1} out of #{output[:total]} done.")
+                end
+              end
+
               @progress_bar.update(line) if @progress_bar
             end
           rescue Errno::EIO # we reach end of input
@@ -459,21 +507,22 @@ module Kafo
                 Process.wait(pid)
               rescue Errno::ECHILD # process could exit meanwhile so we rescue
               end
-              exit_code = $?.exitstatus
+              self.class.exit_handler.exit_code = $?.exitstatus
             end
           end
         end
       rescue PTY::ChildExited => e # could be raised by Process.wait on older ruby or by PTY.check
-        exit_code = e.status.exitstatus
+        self.class.exit_handler.exit_code = e.status.exitstatus
       end
+
       @progress_bar.close if @progress_bar
-      logger.info "Puppet has finished, bye!"
-      self.class.exit(exit_code) do
-        self.class.hooking.execute(:post)
-      end
+      logger.notice "System configuration has finished."
+
+      self.class.hooking.execute(:post)
+      self.class.exit(exit_code)
     end
 
-    def progress_log(method, message)
+    def progress_log(method, message, logger)
       @progress_bar.print_error(message + "\n") if method == :error && @progress_bar
       logger.send(method, message)
     end
@@ -490,30 +539,6 @@ module Kafo
       File.join(Dir.pwd, 'config', 'kafo.yaml')
     end
 
-    def self.use_colors?
-      if config
-        colors = config.app[:colors]
-      else
-        colors = ARGV.include?('--no-colors') ? false : nil
-        colors = ARGV.include?('--colors') ? true : nil if colors.nil?
-      end
-      colors
-    end
-
-    def self.preset_color_scheme
-      match = ARGV.join(' ').match(/--color-of-background[ =](\w+)/)
-      background = match && match[1]
-      ColorScheme.new(:background => background, :colors => use_colors?).setup
-    end
-
-    def self.set_color_scheme
-      ColorScheme.new(
-        :background => config.app[:color_of_background],
-        :colors => use_colors?).setup
-    end
-
-    private
-
     def normalize_encoding(line)
       if line.respond_to?(:encode) && line.respond_to?(:valid_encoding?)
         line.valid_encoding? ? line : line.encode('UTF-16be', :invalid => :replace, :replace => '?').encode('UTF-8')