cem_acpt 0.8.8 → 0.9.0

data/lib/cem_acpt/test_runner/log_formatter/goss_action_response.rb

@@ -1,53 +1,40 @@
 # frozen_string_literal: true
 
+require_relative 'base'
+
 module CemAcpt
   module TestRunner
     module LogFormatter
       # Formats the results of a Goss action
-      class GossActionResponse
-        INDENT = '  '
-
-        def initialize(config, instance_names_ips)
+      class GossActionResponse < Base
+        def initialize(config, instance_names_ips, subject: nil)
+          super(subject)
           @config = config
           @instance_names_ips = instance_names_ips
         end
 
-        def inspect
-          to_s
-        end
-
-        def to_s
-          "#<#{self.class.name}:0x#{object_id.to_s(16)}>"
-        end
-
-        def summary(response)
-          new_summary_message(response)
+        def summary(response = nil)
+          super(response)
+          new_summary_message(log_subject)
         end
 
-        def results(response)
-          new_results_message(response.results)
+        def results(response = nil)
+          super(response)
+          new_results_message(log_subject.results)
         end
 
-        def host_name(response)
-          name_from_ip(response.host)
+        def host_name(response = nil)
+          super(response)
+          name_from_ip(log_subject.host)
         end
 
-        def test_name(response)
-          test_from_ip(response.host)
+        def test_name(response = nil)
+          super(response)
+          test_from_ip(log_subject.host)
         end
 
         private
 
-        def normalize_whitespace(str)
-          raise ArgumentError, 'str must be a String' unless str.is_a?(String)
-
-          str.gsub(%r{(\n|\r|\t)}, ' ').gsub(%r{\s{2,}}, ' ').strip
-        end
-
-        def success_str(success)
-          success ? 'passed' : 'failed'
-        end
-
         def name_from_ip(ip)
           @instance_names_ips.each do |name, val|
             return name if val['ip'] == ip

data/lib/cem_acpt/test_runner/log_formatter/goss_error_formatter.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module CemAcpt
+  module TestRunner
+    module LogFormatter
+      class GossErrorFormatter < Base
+        def summary(subject = nil)
+          super(subject)
+          "Error: #{log_subject.summary}"
+        end
+
+        def results(response = nil)
+          super(response)
+          [log_subject.summary, log_subject.results.join("\n")]
+        end
+
+        def host_name(response = nil)
+          super(response)
+          "Error: #{log_subject.error.class.name}"
+        end
+
+        def test_name(response = nil)
+          super(response)
+          "Error: #{log_subject.error.class.name}"
+        end
+      end
+    end
+  end
+end

data/lib/cem_acpt/test_runner/log_formatter/standard_error_formatter.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module CemAcpt
+  module TestRunner
+    module LogFormatter
+      class StandardErrorFormatter < Base
+        def summary(subject = nil)
+          super(subject)
+          "Error: #{log_subject.detailed_message}"
+        end
+
+        def results(response = nil)
+          super(response)
+          [
+            "Error: #{log_subject.class.name}",
+            log_subject.detailed_message,
+            log_subject.backtrace.join("\n"),
+          ]
+        end
+
+        def host_name(response = nil)
+          super(response)
+          "Error: #{log_subject.class.name}"
+        end
+
+        def test_name(response = nil)
+          super(response)
+          "Error: #{log_subject.class.name}"
+        end
+      end
+    end
+  end
+end

data/lib/cem_acpt/test_runner/log_formatter.rb

@@ -1,17 +1,29 @@
 # frozen_string_literal: true
 
+require_relative 'log_formatter/bolt_summary_results_formatter'
+require_relative 'log_formatter/bolt_output_formatter'
 require_relative 'log_formatter/goss_action_response'
-require_relative 'log_formatter/error_formatter'
+require_relative 'log_formatter/goss_error_formatter'
+require_relative 'log_formatter/standard_error_formatter'
 
 module CemAcpt
   module TestRunner
     # Holds classes for formatting test runner results
     module LogFormatter
-      def self.new_formatter(result, *args, **kwargs)
-        if result.error?
-          ErrorFormatter.new
+      def self.new_formatter(result, *args, **_kwargs)
+        case result
+        when CemAcpt::Goss::Api::ActionResponse
+          if result.error?
+            GossErrorFormatter.new(result)
+          else
+            GossActionResponse.new(*args, subject: result)
+          end
+        when CemAcpt::Bolt::SummaryResults
+          BoltSummaryResultsFormatter.new(*args, subject: result)
+        when StandardError
+          StandardErrorFormatter.new(result)
         else
-          GossActionResponse.new(*args)
+          raise ArgumentError, "result must be a CemAcpt::Goss::Api::ActionResponse, CemAcpt::Bolt::SummaryResults, or StandardError, got #{result.class}"
         end
       end
     end

data/lib/cem_acpt/test_runner/test_results.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+require_relative 'log_formatter'
+
+module CemAcpt
+  module TestRunner
+    # Namespace for all test result classes and methods
+    module TestResults
+      def self.new(config = nil, instance_names_ips = nil)
+        Results.new(config, instance_names_ips)
+      end
+
+      # Class that manages test results
+      class Results
+        attr_accessor :config, :instance_names_ips
+
+        def initialize(config = nil, instance_names_ips = nil)
+          @results_queue = Queue.new
+          @config = config
+          @instance_names_ips = instance_names_ips
+        end
+
+        def combine(other)
+          new_res_queue_data = (to_a + other.to_a).compact.uniq
+          @results_queue = Queue.new(new_res_queue_data)
+
+          self
+        end
+
+        def <<(result)
+          case result
+          when CemAcpt::Goss::Api::ActionResponse, CemAcpt::Bolt::SummaryResults
+            @results_queue << TestActionResult.new(result, new_formatter(result))
+          when StandardError
+            @results_queue << TestErrorActionResult.new(result, new_formatter(result))
+          else
+            raise ArgumentError, "result must be a CemAcpt::Goss::Api::ActionResponse, CemAcpt::Bolt::SummaryResults, or StandardError, got #{result.class}"
+          end
+        end
+
+        def to_a
+          results = []
+          @results_queue.close unless @results_queue.closed?
+          while (r = @results_queue.pop)
+            results << r
+          end
+          @results_queue = Queue.new(results)
+          results
+        end
+
+        def method_missing(method_name, *args, **kwargs, &block)
+          @results_queue.send(method_name, *args, **kwargs, &block)
+        end
+
+        def respond_to_missing?(method_name, include_private = false)
+          @results_queue.respond_to?(method_name, include_private) || super
+        end
+
+        private
+
+        def new_formatter(result)
+          CemAcpt::TestRunner::LogFormatter.new_formatter(result, @config, @instance_names_ips)
+        end
+      end
+
+      # Wrapper class for the result of an action. Provides a common interface for
+      # getting reportable data from the result.
+      class TestActionResult
+        attr_reader :result, :log_formatter
+
+        def initialize(result, log_formatter = nil)
+          @result = result
+          @log_formatter = log_formatter
+        end
+
+        def error?
+          (@result.respond_to?(:error?) && @result.error?) || false
+        end
+
+        def method_missing(method_name, *args, **kwargs, &block)
+          if @result.respond_to?(method_name)
+            @result.send(method_name, *args, **kwargs, &block)
+          else
+            super
+          end
+        end
+
+        def respond_to_missing?(method_name, include_private = false)
+          @result.respond_to?(method_name, include_private) || super
+        end
+      end
+
+      # Wrapper class for an error raised during an action.
+      class TestErrorActionResult
+        attr_reader :error, :log_formatter
+
+        def initialize(error, log_formatter = nil)
+          @error = error
+          @log_formatter = log_formatter
+        end
+        alias result error
+
+        def error?
+          true
+        end
+
+        def success?
+          false
+        end
+
+        def to_s
+          "#<#{@error.class.name}:0x#{@error.object_id.to_s(16)}>"
+        end
+
+        def inspect
+          to_s
+        end
+
+        def to_h
+          {
+            class: @error.class.name,
+            message: @error.message,
+            backtrace: @error.backtrace,
+          }
+        end
+
+        def status
+          1
+        end
+        alias http_status status
+
+        def results
+          [summary, "Class: #{@error.class.name}", @error.backtrace.join("\n")]
+        end
+
+        def results?
+          true
+        end
+
+        def summary
+          @error.message
+        end
+
+        def summary?
+          true
+        end
+      end
+    end
+  end
+end

data/lib/cem_acpt/test_runner.rb

@@ -2,7 +2,8 @@
 
 require 'fileutils'
 require 'securerandom'
-require_relative 'action_result'
+require_relative 'actions'
+require_relative 'bolt'
 require_relative 'goss'
 require_relative 'logging'
 require_relative 'platform'
@@ -11,6 +12,7 @@ require_relative 'test_data'
 require_relative 'utils'
 require_relative 'version'
 require_relative 'test_runner/log_formatter'
+require_relative 'test_runner/test_results'
 
 module CemAcpt
   # Namespace for all Runner-related classes and modules
@@ -19,6 +21,8 @@ module CemAcpt
     class Runner
       include CemAcpt::Logging
 
+      SUCCESS_STATUS = [200, 0].freeze
+
       attr_reader :duration, :exit_code
       attr_accessor :run_data # This is opened up mainly for windows use.
 
@@ -27,8 +31,9 @@ module CemAcpt
         @run_data = {}
         @duration = 0
         @exit_code = 0
-        @results = []
-        @http_statuses = []
+        @bolt_test_runner = nil
+        @results = CemAcpt::TestRunner::TestResults.new
+        @statuses = []
         @provisioned = false
         @destroyed = false
       end
@@ -42,23 +47,14 @@ module CemAcpt
       end
 
       def run
-        @run_data = {}
         @start_time = Time.now
         module_dir = config.get('module_dir')
         @old_dir = Dir.pwd
         Dir.chdir(module_dir)
+        configure_actions
         logger.start_ci_group("CemAcpt v#{CemAcpt::VERSION} run started at #{@start_time}")
         logger.info('CemAcpt::TestRunner') { "Using module directory: #{module_dir}..." }
-        @run_data[:private_key], @run_data[:public_key], @run_data[:known_hosts] = new_ephemeral_ssh_keys
-        logger.info('CemAcpt::TestRunner') { 'Created ephemeral SSH key pair...' }
-        @run_data[:module_package_path] = build_module_package
-        logger.info('CemAcpt::TestRunner') { "Created module package: #{@run_data[:module_package_path]}..." }
-        @run_data[:test_data] = new_test_data
-        logger.info('CemAcpt::TestRunner') { 'Created test data...' }
-        logger.verbose('CemAcpt::TestRunner') { "Test data: #{@run_data[:test_data]}" }
-        @run_data[:nodes] = new_node_data
-        logger.info('CemAcpt::TestRunner') { 'Created node data...' }
-        logger.verbose('CemAcpt::TestRunner') { "Node data: #{@run_data[:nodes]}" }
+        pre_provision_test_nodes
         provision_test_nodes
         @instance_names_ips = provisioner_output
         logger.info('CemAcpt::TestRunner') { "Instance names and IPs class: #{@instance_names_ips.class}" }
@@ -79,18 +75,17 @@ module CemAcpt
             win_node.run
           end
         end
-        @results = run_tests(@instance_names_ips.map { |_, v| v['ip'] },
-                              config.get('actions.only'),
-                              config.get('actions.except'))
+        @hosts = @instance_names_ips.map { |_, v| v['ip'] }
+        run_tests
       rescue StandardError => e
         logger.error('CemAcpt::TestRunner') { 'Run failed due to error...' }
-        @results << ActionResult.new(e)
+        @results << e
       ensure
         logger.end_ci_group
         clean_up
         process_test_results
         Dir.chdir(@old_dir) if @old_dir
-        @results
+        @results.to_a
       end
 
       def clean_up(_trap_context = false)
@@ -117,13 +112,36 @@ module CemAcpt
 
       attr_reader :config
 
+      # Configures the actions to run based on the config
+      def configure_actions
+        logger.info('CemAcpt::TestRunner') { 'Configuring and registering actions...' }
+        goss_actions = CemAcpt::Goss::Api::ACTIONS.keys
+        CemAcpt::Actions.configure(config) do |c|
+          c.register_group(:goss, order: 0).register_action(goss_actions.first) do |context|
+            run_goss_tests(context)
+          end
+          goss_actions[1..-1].each { |a| c[:goss].register_action(a) }
+          c.register_group(:bolt, order: 1).register_action(:bolt) do |context|
+            run_bolt_tests(context)
+          end
+        end
+        logger.debug('CemAcpt::TestRunner') { "All actions #{CemAcpt::Actions.config.action_names}" }
+        logger.debug('CemAcpt::TestRunner') { "Only actions: #{CemAcpt::Actions.config.only}" }
+        logger.debug('CemAcpt::TestRunner') { "Except actions: #{CemAcpt::Actions.config.except}" }
+        logger.info('CemAcpt::TestRunner') do
+          "Configured and registered actions, will run actions: #{CemAcpt::Actions.config.action_names.join(', ')}"
+        end
+      end
+
       # @return [String] The path to the module package
       def build_module_package
-        if config.get('tests').first.include? 'windows'
-          CemAcpt::Utils.package_win_module(config.get('module_dir'))
-        else
-          CemAcpt::Utils::Puppet.build_module_package(config.get('module_dir'))
-        end
+        pkg_path = if config.get('tests').first.include? 'windows'
+                     CemAcpt::Utils.package_win_module(config.get('module_dir'))
+                   else
+                     CemAcpt::Utils::Puppet.build_module_package(config.get('module_dir'))
+                   end
+        logger.info('CemAcpt::TestRunner') { "Created module package: #{pkg_path}..." }
+        pkg_path
       end
 
       # @return [Array<String>] The paths to the ssh private key, public key, and known hosts file
@@ -131,7 +149,9 @@ module CemAcpt
         return [nil, nil, nil] if config.get('no_ephemeral_ssh_key')
 
         logger.info('CemAcpt::TestRunner') { 'Creating ephemeral SSH keys...' }
-        CemAcpt::Utils::SSH::Ephemeral.create
+        pri, pub, kh = CemAcpt::Utils::SSH::Ephemeral.create
+        logger.info('CemAcpt::TestRunner') { 'Created ephemeral SSH key pair...' }
+        [pri, pub, kh]
       end
 
       def clean_ephemeral_ssh_keys
@@ -143,12 +163,45 @@ module CemAcpt
 
       def new_test_data
         logger.debug('CemAcpt::TestRunner') { 'Creating new test data...' }
-        CemAcpt::TestData.acceptance_test_data(config)
+        tdata = CemAcpt::TestData.acceptance_test_data(config)
+        logger.info('CemAcpt::TestRunner') { 'Created test data...' }
+        logger.verbose('CemAcpt::TestRunner') { "Test data:\n#{tdata}" }
+        tdata
       end
 
       def new_node_data
         logger.debug('CemAcpt::TestRunner') { 'Creating new node data...' }
-        CemAcpt::Platform.use(config.get('platform.name'), config, @run_data)
+        ndata = CemAcpt::Platform.use(config.get('platform.name'), config, @run_data)
+        logger.info('CemAcpt::TestRunner') { 'Created node data...' }
+        logger.verbose('CemAcpt::TestRunner') { "Node data:\n#{ndata}" }
+        ndata
+      end
+
+      # Runs all methods that are needed to prep data and environment for the provision_test_nodes method
+      def pre_provision_test_nodes
+        logger.info('CemAcpt::TestRunner') { 'Pre-provisioning test nodes...' }
+        logger.info('CemAcpt::TestRunner') { 'Creating initial run data...' }
+        @run_data = {}
+        @run_data[:private_key], @run_data[:public_key], @run_data[:known_hosts] = new_ephemeral_ssh_keys
+        @run_data[:module_package_path] = build_module_package
+        @run_data[:test_data] = new_test_data
+        @run_data[:nodes] = new_node_data
+        logger.verbose('CemAcpt::TestRunner') { "Initial run data:\n#{@run_data}" }
+        logger.info('CemAcpt::TestRunner') { 'Created initial run data...' }
+        setup_bolt
+      end
+
+      def setup_bolt
+        logger.info('CemAcpt::TestRunner') { 'Setting up Bolt...' }
+        @bolt_test_runner = CemAcpt::Bolt::TestRunner.new(config, run_data: @run_data)
+        @bolt_test_runner.setup!
+        return unless @bolt_test_runner.tests.to_a.empty?
+
+        if !only_actions.empty? && only_actions.include?('bolt')
+          raise 'No Bolt tests to run and only bolt action was specified'
+        end
+
+        logger.warn('CemAcpt::TestRunner') { 'No Bolt tests to run' }
       end
 
       def provision_test_nodes
@@ -194,20 +247,60 @@ module CemAcpt
         logger.info('CemAcpt') { "Test SSH Keys:\n  Private Key: #{@run_data[:private_key]}\n  Public Key:#{@run_data[:public_key]}" }
       end
 
-      def run_tests(hosts, only_actions, except_actions)
-        logger.info('CemAcpt::TestRunner') { 'Running tests...' }
-        logger.verbose('CemAcpt::TestRunner') { "Hosts: #{hosts}" }
-        logger.verbose('CemAcpt::TestRunner') { "Only actions: #{only_actions}" }
-        logger.verbose('CemAcpt::TestRunner') { "Except actions: #{except_actions}" }
-        api_results = CemAcpt::Goss::Api.run_actions_async(hosts,
-                                                           only: only_actions || [],
-                                                           except: except_actions || [])
-        res = []
-        api_results.close unless api_results.closed?
-        while (r = api_results.pop)
-          res << ActionResult.new(r)
+      def run_tests
+        logger.info('CemAcpt::TestRunner') { 'Preparing to run tests...' }
+        logger.verbose('CemAcpt::TestRunner') { "Hosts: #{@hosts}" }
+        logger.verbose('CemAcpt::TestRunner') { "Only actions: #{CemAcpt::Actions.only}" }
+        logger.verbose('CemAcpt::TestRunner') { "Except actions: #{CemAcpt::Actions.except}" }
+        @results = CemAcpt::TestRunner::TestResults.new(config, @instance_names_ips)
+        CemAcpt::Actions.execute
+      end
+
+      def run_goss_tests(context = {})
+        logger.info('CemAcpt::TestRunner') { 'Running goss tests...' }
+        context[:hosts] = @hosts
+        context[:results] = @results
+        CemAcpt::Goss::Api.run_actions_async(context)
+      end
+
+      def run_bolt_tests(_context = {})
+        logger.info('CemAcpt::TestRunner') { 'Running Bolt tests...' }
+        # If the Bolt config has tests:only or tests:ignore lists, we need to filter the hosts
+        # based on their associated tests.
+        @bolt_test_runner.hosts = filtered_bolt_hosts
+        @bolt_test_runner.run
+        @results << @bolt_test_runner.results
+      end
+
+      def filtered_bolt_hosts
+        tests_only = config.get('bolt.tests.only')
+        tests_only_unset = tests_only.nil? || tests_only.empty?
+        logger.debug('CemAcpt::TestRunner') { "Bolt tests only: #{tests_only}" } unless tests_only_unset
+        tests_ignore = config.get('bolt.tests.ignore')
+        tests_ignore_unset = tests_ignore.nil? || tests_ignore.empty?
+        logger.debug('CemAcpt::TestRunner') { "Bolt tests ignore: #{tests_ignore}" } unless tests_ignore_unset
+        return @instance_names_ips.map { |_, v| v['ip'] } if tests_only_unset && tests_ignore_unset
+
+        logger.debug('CemAcpt::TestRunner') { 'Filtering Bolt hosts...' }
+        filtered_hosts = []
+        @instance_names_ips.each do |_, v|
+          host = v['ip']
+          test_name = v['test_name']
+          in_only = !tests_only_unset && tests_only.include?(test_name)
+          in_ignore = !tests_ignore_unset && tests_ignore.include?(test_name)
+          if in_only || !in_ignore
+            filtered_hosts << host
+            logger.debug('CemAcpt::TestRunner') { "Added host #{host} to filtered hosts" }
+          else
+            logger.debug('CemAcpt::TestRunner') do
+              "Not adding host #{host} to filtered hosts. In only? #{in_only}; In ignore? #{in_ignore}"
+            end
+          end
         end
-        res
+        filtered_hosts.compact!
+        filtered_hosts.uniq!
+        logger.debug('CemAcpt::TestRunner') { "Filtered hosts: #{filtered_hosts}" }
+        filtered_hosts
       end
 
       def process_test_results
@@ -218,14 +311,14 @@ module CemAcpt
           logger.info('CemAcpt::TestRunner') { "Processing #{@results.size} test result(s)..." }
           until @results.empty?
             result = @results.pop
-            @http_statuses << result.http_status
+            @statuses << result.status
             log_test_result(result)
           end
-          if @http_statuses.empty?
+          if @statuses.empty?
             logger.error('CemAcpt::TestRunner') { 'No test results to process' }
             @exit_code = 1
           else
-            @exit_code = (@http_statuses.any? { |s| s.to_i != 200 }) ? 1 : 0
+            @exit_code = (@statuses.any? { |s| SUCCESS_STATUS.include?(s.to_i) }) ? 1 : 0
           end
         end
         @duration = Time.now - @start_time
@@ -233,13 +326,22 @@ module CemAcpt
       end
 
       def log_test_result(result)
-        result_log_formatter = LogFormatter.new_formatter(result, config, @instance_names_ips)
-        logger.start_ci_group("Test results for #{result_log_formatter.test_name(result)}")
-        return log_error_test_result(result_log_formatter, result) if result.error?
+        logger.start_ci_group("Test results for #{result.log_formatter.test_name}")
+        case result
+        when CemAcpt::TestRunner::TestResults::TestErrorActionResult
+          log_error_test_result(result)
+        when CemAcpt::TestRunner::TestResults::TestActionResult
+          log_action_test_result(result)
+        else
+          raise ArgumentError, "result must be a CemAcpt::TestRunner::TestResults::TestActionResult or CemAcpt::TestRunner::TestResults::TestErrorActionResult, got #{result.class}"
+        end
+      ensure
+        logger.end_ci_group
+      end
 
-        logger.info(result_log_formatter.summary(result))
-        formatted_results = result_log_formatter.results(result)
-        formatted_results.each do |r|
+      def log_action_test_result(result)
+        logger.info { result.log_formatter.summary }
+        result.log_formatter.results.each do |r|
           if r.start_with?('Passed:')
             logger.verbose { r }
           elsif r.start_with?('Skipped:')
@@ -248,12 +350,10 @@ module CemAcpt
             logger.error { r }
           end
         end
-      ensure
-        logger.end_ci_group
       end
 
-      def log_error_test_result(formatter, result)
-        logger.fatal { formatter.results(result).join("\n") }
+      def log_error_test_result(result)
+        logger.fatal { result.log_formatter.results.join("\n") }
       end
 
       # Upload the cem_windows module to the bucket if we're testing the cem_windows module