cem_acpt 0.1.0 → 0.2.2

data/lib/cem_acpt/test_data.rb

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
 
+require_relative 'core_extensions'
+require_relative 'logging'
+
 module CemAcpt
   # This module provides a method and class for extracting and formatting
   # test data.
   module TestData
-    require_relative 'core_extensions'
-    require_relative 'logging'
-
     # Returns a hash of test data.
     # @param config [CemAcpt::Config] the config object
     # @return [Array<Hash>] an array of test data hashes
@@ -82,7 +82,6 @@ module CemAcpt
       def process_static_vars(test_data)
         return unless @config.has?('test_data.vars')
 
-        logger.debug('Processing static variables...')
         vars = @config.get('test_data.vars').each_with_object({}) do |(k, v), h|
           logger.debug("Static variable: #{k}")
           h[k] = v
@@ -92,7 +91,6 @@ module CemAcpt
       end
 
       def process_name_pattern_vars(test_name, test_data)
-        logger.debug('Processing test name pattern...')
         name_pattern_vars = name_pattern_matches(test_name) || {}
         test_data.merge!(name_pattern_vars)
         test_data.format!
@@ -102,20 +100,17 @@ module CemAcpt
         return test_name unless @config.has?('test_data.name_pattern_vars')
 
         pattern = @config.get('test_data.name_pattern_vars')
-        logger.debug("Test name pattern: #{pattern.inspect}")
         pattern_matches = test_name.match(pattern)&.named_captures
         unless pattern_matches
           logger.error("Test name #{test_name} does not match pattern #{pattern.inspect}")
           return
         end
-        logger.debug("Test name matches pattern: #{pattern_matches}")
         pattern_matches
       end
 
       def vars_post_processing!(test_data)
         return unless @config.has?('test_data.vars_post_processing')
 
-        logger.debug("Running test name pattern post processing with test data: #{test_data}")
         post_processing_new_vars!(test_data)
         post_processing_delete_vars!(test_data)
       end
@@ -124,7 +119,6 @@ module CemAcpt
         new_vars = @config.get('test_data.vars_post_processing.new_vars')
         return unless new_vars
 
-        logger.debug("Processing new variables: #{new_vars}")
         new_vars.each do |var|
           if var.has?('string_split')
             test_data[var.dot_dig('name')] = op_string_split(test_data, var)
@@ -135,13 +129,9 @@ module CemAcpt
       end
 
       def op_string_split(test_data, var)
-        logger.debug("Processing string split for new var #{var}")
         from_var = var.dot_dig('string_split.from')
-        logger.debug("String split from var: #{from_var}")
         from = test_data.dot_dig(from_var)
-        logger.debug("String split from value: #{from}")
         parts = from.split(var.dot_dig('string_split.using'))
-        logger.debug("String split parts: #{parts}")
         var.has?('string_split.part') ? parts[var.dot_dig('string_split.part')] : parts
       end
 
@@ -149,7 +139,6 @@ module CemAcpt
         delete_vars = @config.get('test_data.vars_post_processing.delete_vars')
         return unless delete_vars
 
-        logger.debug('Processing delete variables...')
         test_data.reject! { |k, _| delete_vars.include?(k.to_s) }
       end
     end

data/lib/cem_acpt/test_runner/run_handler.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+require 'concurrent-ruby'
+require 'open3'
+require_relative '../context'
+require_relative '../logging'
+require_relative '../puppet_helpers'
+require_relative 'runner'
+
+module CemAcpt
+  module TestRunner
+    # Raised when the RunHandler needs to be operating within the module directory and isn't
+    class RunHandlerNotInModuleDirError < StandardError; end
+
+    # Raised when the RunHandler executes a system command that fails
+    class RunHandlerSystemCommandError < StandardError; end
+
+    # RunHandler orchestrates the acceptance test suites, including
+    # creating Runner objects, handling input and output, and exception
+    # handling.
+    class RunHandler
+      include CemAcpt::Logging
+
+      attr_accessor :run_context
+
+      # @param run_context [CemAcpt::Context::Ctx] The context object for this run
+      def initialize(run_context)
+        @run_context = run_context
+        @module_pkg_path = Concurrent::IVar.new
+        @runners = Concurrent::Array.new
+        @thread_pool = Concurrent::SimpleExecutorService.new
+      end
+
+      # Gets the overall exit code for all runners
+      def exit_code
+        @runners&.map { |r| r.run_result.exit_status }&.all?(&:zero?) ? 0 : 1
+      end
+
+      # Runs the acceptance test suites.
+      def run
+        logger.info('RUN HANDLER: Creating and starting test runners...')
+        create_runners
+        logger.info("RUN HANDLER: Created #{@runners.length} runners...")
+        @runners.map { |r| @thread_pool.post { r.start } }
+        @thread_pool.shutdown
+        @thread_pool.wait_for_termination
+        @thread_pool = Concurrent::SimpleExecutorService.new
+        logger.info('Test runners have all exited...')
+      rescue StandardError => e
+        handle_fatal_error(e)
+      ensure
+        handle_test_results
+      end
+
+      def destroy_test_nodes
+        return if @runners.empty? || @runners.none?(&:node_exists)
+
+        @runners.each do |r|
+          r.send(:destroy) if r.node_exists
+        end
+      end
+
+      private
+
+      # Creates and starts Runner objects for each node in the acceptance test suites.
+      def create_runners
+        @run_context.nodes.each do |platform, nodes|
+          @runners += nodes.map { |node| create_runner(node, platform) }.compact
+        end
+      end
+
+      def create_runner(node, platform)
+        logger.info("RUN HANDLER: Creating runner for #{node.test_data[:test_name]} on node #{node.node_name}...")
+        runner = CemAcpt::TestRunner::Runner.new(node, @run_context, platform)
+        runner || runner_creation_error(node)
+      end
+
+      def runner_creation_error(node)
+        msg = [
+          "Failed to create runner object for node #{node.node_name}.",
+          "Cannot run test #{node.test_data[:test_name]}",
+        ].join(' ')
+        logger.error(msg)
+        nil
+      end
+
+      # Handles how test results are logged.
+      def handle_test_results
+        @runners.each do |runner|
+          result = runner.run_result.to_h
+          logger << "::group::{Results for #{runner.node.test_data[:test_name]}}\n"
+          if result.key?('summary_line')
+            logger.info("SUMMARY: #{result['summary_line']} for test #{runner.node.test_data[:test_name]}")
+          else
+            handle_runner_error_results(runner)
+            logger << "::endgroup::\n"
+            next
+          end
+          unless runner.test_failures?
+            logger << "::endgroup::\n"
+            next
+          end
+
+          if result.key?('examples') # Log errors outside of examples
+            if result['examples'].empty? && !result['messages'].empty?
+              logger.error(result['messages'].join("\n"))
+            else
+              failed = result['examples'].reject { |e| e['status'] == 'passed' }
+              failed.each do |e|
+                logger.error(test_error_msg(runner.node.node_name, e))
+              end
+            end
+          else
+            handle_runner_error_results(runner)
+          end
+          #debug_test_results(runner) if logger.debug?
+          logger << "::endgroup::\n"
+        end
+      end
+
+      # Handles logging the results of the runners that errored.
+      def handle_runner_error_results(runner)
+        logger.error("SUMMARY: Encountered an error with test #{runner.node.test_data[:test_name]} on node #{runner.node.node_name}")
+        runner.run_result.to_h.each do |k, v|
+          logger.error("#{k.upcase}: #{v}")
+        end
+      end
+
+      # Formats the error message for a runner that failed.
+      # @param node [Object] the node that failed
+      # @param err [Object] the error that occured
+      # @return [String] the formatted error message
+      def runner_error_msg(node, err)
+        [
+          "Error while running acceptance tests on node #{node.node_name}",
+          "Error: #{err.message}",
+          'Backtrace:',
+          err.backtrace.join("\n"),
+        ].join("\n")
+      end
+
+      # Formats a test result for tests that have failed. Is used for logging.
+      # @param node [String] the name of the node the test ran on
+      # @param result [Hash] the test result to format
+      # @return [String] the formatted test result
+      def test_error_msg(node, result)
+        [
+          "TEST FAILED: #{result['id']}",
+          "DESCRIPTION: #{result['full_description']}",
+          "STATUS: #{result['status']}",
+          "LOCATION: #{result['file_path']}:#{result['line_number']}",
+          "NODE: #{node}",
+          result['exception']['message'],
+          "\n",
+        ].join("\n")
+      end
+
+      # Logs performance data for the acceptance test suites.
+      # This is only logged if the log level is set to debug.
+      # def debug_test_results(runner)
+      #   examples_by_time = []
+      #   runner.run_result.to_h.each do |node, result|
+      #     next unless result['examples']
+
+      #     result['examples'].each do |e|
+      #       examples_by_time << [e['run_time'], e['id'], e['status'], e['line_number'], node]
+      #     end
+      #   end
+      #   if examples_by_time
+      #     logger.debug('Showing test results in order of execution time...')
+      #     examples_by_time.sort_by(&:first).reverse.each do |e|
+      #       logger.debug("RUNTIME: #{e[0]}; ID: #{e[1]}; STATUS: #{e[2]}; LINE: #{e[3]}; HOST: #{e[4]};")
+      #     end
+      #   else
+      #     logger.debug('No debug results to show')
+      #   end
+      # end
+
+      # Gracefully handles a fatal error and exits the program.
+      # @param err [StandardError, Exception] the error that caused the fatal error
+      def handle_fatal_error(err)
+        logger.fatal("RUN HANDLER: Fatal error: #{err.message}")
+        logger.debug(err.backtrace.join("\n"))
+      end
+    end
+  end
+end

data/lib/cem_acpt/test_runner/runner.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+require 'concurrent-ruby'
+require 'English'
+require_relative '../logging'
+require_relative '../rspec_utils'
+require_relative 'runner_result'
+
+module CemAcpt
+  module TestRunner
+    class RunnerError < StandardError; end
+    # Error used to wrap fatal errors raised in Runner steps
+    class RunnerStepError < StandardError
+      attr_reader :step
+
+      def initialize(step, err)
+        @step = step
+        super err
+        set_backtrace err.backtrace if err.respond_to?(:backtrace)
+      end
+    end
+    class RunnerProvisionError < RunnerStepError; end
+
+    # Runner is a class that runs a single acceptance test suite on a single node.
+    # It is responsible for managing the lifecycle of the test suite and
+    # reporting the results back to the main thread. Runner objects are created
+    # by the RunHandler and then, when started, execute their logic in a thread.
+    class Runner
+      include CemAcpt::LoggingAsync
+
+      attr_reader :node, :node_exists, :run_result
+
+      # @param node [String] the name of the node to run the acceptance test suite on
+      # @param ctx [CemAcpt::RunnerCtx] a cem_acpt Ctx (context) object
+      # @param module_pkg_path [Concurrent::IVar] the path to the module package
+      def initialize(node, context, platform)
+        @node = node
+        @context = context
+        @platform = platform
+        @debug_mode = @context.config.debug_mode?
+        @node_inventory = @context.node_inventory
+        @module_pkg_path = @context.module_package_path
+        @node_exists = false
+        @run_result = CemAcpt::TestRunner::RunnerResult.new(@node, debug: @debug_mode)
+        @completed_steps = []
+        validate!
+      end
+
+      def run_step(step_sym)
+        send(step_sym)
+        @completed_steps << step_sym
+      rescue StandardError => e
+        err = CemAcpt::TestRunner::RunnerStepError.new(step_sym, e)
+        step_error_logging(err)
+        @run_result.from_error(err)
+        destroy unless step_sym == :destroy
+      end
+
+      # Executes test suite steps
+      def start
+        async_info("Starting test suite for #{@node.node_name}", log_prefix('RUNNER'))
+        run_step(:provision)
+        run_step(:bootstrap)
+        run_step(:run_tests)
+        run_step(:destroy)
+        true
+      rescue StandardError => e
+        step_error_logging(e)
+        @run_result.from_error(e)
+        destroy
+      end
+
+      # Checks for failures in the test results.
+      # @param result [Hash] the test result to check
+      # @return [Boolean] whether or not there are test failures in result
+      def test_failures?
+        @run_result.result_errors? || @run_result.result_failures?
+      end
+
+      private
+
+      def step_error_logging(err)
+        prefix = err.respond_to?(:step) ? log_prefix(err.step.capitalize) : log_prefix('RUNNER')
+        fatal_msg = ["runner failed: #{err.message}"]
+        async_fatal(fatal_msg, prefix)
+        async_debug("Completed steps: #{@completed_steps}", prefix)
+        async_debug("Failed runner backtrace:\n#{err.backtrace.join("\n")}", prefix)
+        async_debug("Failed runner test data: #{@node.test_data}", prefix)
+      end
+
+      def log_prefix(prefix)
+        "#{prefix}: #{@node.test_data[:test_name]}:"
+      end
+
+      # Provisions the node for the acceptance test suite.
+      def provision
+        async_info("Provisioning #{@node.node_name}...", log_prefix('PROVISION'))
+        start_time = Time.now
+        @node.provision
+        @node_exists = true
+        max_retries = 60 # equals 300 seconds because we check every five seconds
+        until @node.ready?
+          if max_retries <= 0
+            async_fatal("Node #{@node.node_name} failed to provision", log_prefix('PROVISION'))
+            raise CemAcpt::TestRunner::RunnerProvisionError, "Provisioning timed out for node #{@node.node_name}"
+          end
+
+          async_info("Waiting for #{@node.node_name} to be ready for remote connections...", log_prefix('PROVISION'))
+          max_retries -= 1
+          sleep(5)
+        end
+        async_info("Node #{@node.node_name} is ready...", log_prefix('PROVISION'))
+        node_desc = {
+          test_data: @node.test_data,
+          platform: @platform,
+          local_port: @node.local_port,
+        }.merge(@node.node)
+        @node_inventory.add(@node.node_name, node_desc)
+        @node_inventory.save
+        async_info("Node #{@node.node_name} provisioned in #{Time.now - start_time} seconds", log_prefix('PROVISION'))
+      end
+
+      # Bootstraps the node for the acceptance test suite. Currently, this
+      # just uploads and installs the module package.
+      def bootstrap
+        async_info("Bootstrapping #{@node.node_name}...", log_prefix('BOOTSTRAP'))
+        until File.exist?(@module_pkg_path)
+          async_debug("Waiting for module package #{@module_pkg_path} to exist...", log_prefix('BOOTSTRAP'))
+          sleep(1)
+        end
+        async_info("Installing module package #{@module_pkg_path}...", log_prefix('BOOTSTRAP'))
+        @node.install_puppet_module_package(@module_pkg_path)
+      end
+
+      # Runs the acceptance test suite via rspec.
+      def run_tests
+        attempts = 0
+        until File.exist?(@node_inventory.save_file_path)
+          raise 'Node inventory file not found' if (attempts += 1) > 3
+
+          sleep(1)
+        end
+        async_info("Running test #{@node.test_data[:test_name]} on node #{@node.node_name}...", log_prefix('RSPEC'))
+        @node.run_tests do |cmd_env|
+          cmd_opts = rspec_opts
+          cmd_opts[:env].merge!(cmd_env) if cmd_env
+          # Documentation format gets logged in real time, JSON file is read after the fact
+          begin
+            @rspec_cmd = CemAcpt::RSpecUtils::Command.new(cmd_opts)
+            @rspec_cmd.execute(log_prefix: log_prefix('RSPEC'))
+            @run_result.from_json_file(cmd_opts[:format][:json])
+          rescue Errno::EIO => e
+            async_error("failed to run rspec: #{@node.test_data[:test_name]}: #{$ERROR_INFO}", log_prefix('RSPEC'))
+            @run_result.from_error(e)
+          rescue StandardError => e
+            async_error("failed to run rspec: #{@node.test_data[:test_name]}: #{e.message}", log_prefix('RSPEC'))
+            async_debug("Backtrace:\n#{e.backtrace}", log_prefix('RSPEC'))
+            @run_result.from_error(e)
+          end
+        end
+        async_info("Tests completed with exit code: #{@run_result.exit_status}", log_prefix('RSPEC'))
+      end
+
+      # Destroys the node for the acceptance test suite.
+      def destroy
+        kill_spec_pty_if_exists
+        if @context.config.get('no_destroy_nodes')
+          async_info("Not destroying node #{@node.node_name} because 'no_destroy_nodes' is set to true",
+                     log_prefix('DESTROY'))
+        else
+          async_info("Destroying #{@node.node_name}...", log_prefix('DESTROY'))
+          @node.destroy
+          @node_exists = false
+          async_info("Node #{@node.node_name} destroyed successfully", log_prefix('DESTROY'))
+        end
+      end
+
+      def kill_spec_pty_if_exists
+        @rspec_cmd.kill_pty
+      end
+
+      # Validates the runner configuration.
+      def validate!
+        raise 'No node provided' unless @node
+        raise 'Node does not have config' if @node.config.nil? || @node.config.empty?
+        raise 'Node does not have test data' if @node.test_data.nil? || @node.test_data.empty?
+        raise 'No node inventory provided' unless @node_inventory
+      end
+
+      # Options used with RSpec
+      def rspec_opts
+        opts = {
+          test_path: @node.test_data[:test_file],
+          use_bundler: false,
+          bundle_install: false,
+          format: {
+            json: "results_#{@node.test_data[:test_name]}.json",
+          },
+          debug: (@debug_mode && @context.config.get('verbose')),
+          quiet: @context.config.get('quiet'),
+          env: {
+            'TARGET_HOST' => @node.node_name,
+          }
+        }
+        opts[:format][:documentation] = nil unless @context.config.get('verbose')
+        opts
+      end
+    end
+  end
+end

data/lib/cem_acpt/test_runner/runner_result.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module CemAcpt
+  module TestRunner
+    require 'json'
+
+    # Error thrown when there are problems parsing the JSON output from Rspec
+    class RspecOutputJSONParserError < StandardError
+      def initialize(err = nil)
+        super err
+        set_backtrace err.backtrace unless err.nil?
+      end
+    end
+
+    # Class to process the results of a Runner
+    class RunnerResult
+      attr_reader :run_result
+
+      def initialize(node, debug: false)
+        @node = node
+        @run_result = {}
+        @debug = debug
+      end
+
+      def to_h
+        @run_result
+      end
+
+      def debug?
+        @debug
+      end
+
+      def exit_status
+        if run_result.nil?
+          99
+        elsif run_result.empty?
+          66
+        elsif result_errors? || result_failures?
+          1
+        else
+          0
+        end
+      end
+
+      def result_array
+        [run_result, exit_status]
+      end
+
+      def from_json_file(file_path)
+        res = JSON.parse(File.read(file_path))
+        new_run_result(res)
+      rescue StandardError => e
+        from_error(RspecOutputJSONParserError.new(e))
+      end
+
+      def from_error(err)
+        label = err.class.to_s.start_with?('Errno::') ? 'system_error' : 'standard_error'
+        res = {
+          label => {
+            'message' => err.message,
+            'error_class' => err.class.to_s,
+            'backtrace' => err.backtrace,
+            'cause' => err.cause,
+            'full_message' => err.full_message,
+          }
+        }
+        res[label]['errno'] = err.errno if label == 'system_error'
+        new_run_result(res)
+      end
+
+      def result_errors?
+        run_result.keys.any? { |k| k.end_with?('error') }
+      end
+
+      def result_failures?
+        if run_result['summary']
+          run_result['summary']['failure_count'].positive? ||
+            run_result['summary']['errors_outside_of_examples_count'].positive?
+        else
+          true
+        end
+      end
+
+      private
+
+      def new_run_result(results)
+        @run_result = debug? ? with_debug_data(results) : results
+      end
+
+      def with_debug_data(results)
+        results.merge(
+          {
+            'debug' => {
+              'node' => @node,
+              'config' => @config,
+              'test_data' => @test_data,
+            }
+          }
+        )
+      end
+    end
+  end
+end

data/lib/cem_acpt/test_runner.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module CemAcpt
+  # Namespace for all Runner-related classes and modules
+  module TestRunner
+    require_relative 'test_runner/run_handler'
+    require_relative 'test_runner/runner'
+    require_relative 'test_runner/runner_result'
+  end
+end