cem_acpt 0.1.0

data/lib/cem_acpt/runner.rb

@@ -0,0 +1,304 @@
+# frozen_string_literal: true
+
+require 'concurrent-ruby'
+require 'json'
+require 'open3'
+
+module CemAcpt
+  require_relative 'bootstrap'
+  require_relative 'context'
+  require_relative 'logging'
+  require_relative 'puppet_helpers'
+  require_relative 'test_data'
+  require_relative 'utils'
+
+  # RunHandler orchestrates the acceptance test suites, including
+  # creating Runner objects, handling input and output, and exception
+  # handling.
+  class RunHandler
+    include CemAcpt::Logging
+
+    attr_accessor :params, :config_file
+
+    # @param params [Hash] the parameters passed from the command line
+    def initialize(params)
+      @params = params
+      @config_file = params[:config_file] || File.expand_path('./cem_acpt_config.yaml')
+      @module_pkg_path = Concurrent::IVar.new
+      @start_time = nil
+      @runners = Concurrent::Array.new
+      @results = Concurrent::Map.new
+    end
+
+    # Runs the acceptance test suites.
+    def run
+      @start_time = Time.now
+      logger.info("Running acceptance test suite at #{@start_time}")
+      logger.debug("Params: #{@params}")
+      logger.debug("Config file: #{@config_file}")
+      logger.info("Using module directory: #{@params[:module_dir]}")
+      context_opts = {
+        config_opts: @params,
+        config_file: @config_file,
+      }
+      build_module_package
+      begin
+        keep_terminal_alive if params[:CI] || ENV['CI'] || ENV['GITHUB_ACTION']
+        create_and_start_runners(context_opts)
+      rescue StandardError, SignalException, SystemExit => e
+        handle_fatal_error(e)
+      ensure
+        @keep_terminal_alive&.exit
+      end
+      handle_test_results
+      exit_code = @runners.map(&:spec_exit_code).any? { |rc| rc != 0 } ? 1 : 0
+      exit exit_code
+    end
+
+    private
+
+    # Prints periods to the terminal in a single line to keep the terminal
+    # alive. This is used when running in CI mode.
+    def keep_terminal_alive
+      @keep_terminal_alive = Thread.new do
+        loop do
+          $stdout.print("|\r")
+          sleep(1)
+          $stdout.print("/\r")
+          sleep(1)
+          $stdout.print("-\r")
+          sleep(1)
+          $stdout.print("\\\r")
+          sleep(1)
+        end
+      end
+    end
+
+    # @return [String] the module directory that contains the acceptance tests
+    def module_dir
+      if @params.key?(:working_dir)
+        @params[:working_dir]
+      else
+        Dir.pwd
+      end
+    end
+
+    # Builds the module package in a separate thread.
+    # This thread is set to abort_on_exception so that any
+    # exceptions raised in the thread will be caught and
+    # handled by the main thread.
+    def build_module_package
+      pkg_thread = Thread.new do
+        pkg_path = CemAcpt::PuppetHelpers::Module.build_module_package(@params[:module_dir])
+        @module_pkg_path.set(pkg_path)
+      end
+      pkg_thread.abort_on_exception = true
+    end
+
+    # Creates and starts Runner objects for each node in the acceptance test suites.
+    # @param context_opts [Hash] the options to pass to the Context object
+    # @param timeout [Integer] the timeout to use for the Runner object threads in seconds
+    def create_and_start_runners(context_opts, timeout = 600)
+      CemAcpt::Context.with(**context_opts) do |nodes, conf, tdata, node_inv|
+        nodes.each do |node|
+          runner = CemAcpt::Runner.new(node, conf, tdata, node_inv, @module_pkg_path, @results)
+          runner.start
+          @runners << runner
+        end
+        @runners.each { |r| r.join(timeout) }
+      end
+    end
+
+    # Handles how test results are logged.
+    def handle_test_results
+      @results.each_pair do |node, result|
+        logger.info("SUMMARY: #{result['summary_line']} on node #{node}")
+        next unless test_failures?(result)
+
+        failed = result['examples'].reject { |e| e['status'] == 'passed' }
+        failed.each do |e|
+          logger.error(test_error_msg(node, e))
+        end
+      end
+      debug_test_results if logger.debug?
+    end
+
+    # Formats a test result for tests that have failed. Is used for logging.
+    # @param node [String] the name of the node the test ran on
+    # @param result [Hash] the test result to format
+    # @return [String] the formatted test result
+    def test_error_msg(node, result)
+      [
+        "TEST FAILED: #{result['id']}",
+        "DESCRIPTION: #{result['full_description']}",
+        "STATUS: #{result['status']}",
+        "LOCATION: #{result['file_path']}:#{result['line_number']}",
+        "NODE: #{node}",
+        result['exception']['message'],
+        "\n",
+      ].join("\n")
+    end
+
+    # Logs performance data for the acceptance test suites.
+    # This is only logged if the log level is set to debug.
+    def debug_test_results
+      examples_by_time = []
+      @results.each_pair do |node, result|
+        result['examples'].each do |e|
+          examples_by_time << [e['run_time'], e['id'], e['status'], e['line_number'], node]
+        end
+      end
+      logger.debug('Showing test results in order of execution time...')
+      examples_by_time.sort_by(&:first).reverse.each do |e|
+        logger.debug("RUNTIME: #{e[0]}; ID: #{e[1]}; STATUS: #{e[2]}; LINE: #{e[3]}; HOST: #{e[4]};")
+      end
+    end
+
+    # Checks for failures in the test results.
+    # @param result [Hash] the test result to check
+    # @return [Boolean] whether or not there are test failures in result
+    def test_failures?(result)
+      result['summary']['failure_count'].positive? || result['summary']['errors_outside_of_examples_count'].positive?
+    end
+
+    # Gracefully handles a fatal error and exits the program.
+    # @param err [StandardError, Exception] the error that caused the fatal error
+    def handle_fatal_error(err)
+      @keep_terminal_alive&.exit
+      logger.fatal("Fatal error: #{err.message}")
+      logger.debug(err.backtrace.join('; '))
+      kill_runners
+      logger.fatal("Exiting with status 1 after #{Time.now - @start_time} seconds")
+      exit(1)
+    end
+
+    # Kills all running Runner objects.
+    def kill_runners
+      @runners.each(&:kill) unless @runners.empty?
+    end
+  end
+
+  # Runner is a class that runs a single acceptance test suite on a single node.
+  # It is responsible for managing the lifecycle of the test suite and
+  # reporting the results back to the main thread. Runner objects are created
+  # by the RunHandler and then, when started, execute their logic in a thread.
+  class Runner
+    include CemAcpt::Logging
+
+    attr_reader :spec_exit_code
+
+    # @param node [String] the name of the node to run the acceptance test suite on
+    # @param conf [CemAcpt::Config] the acceptance test suite configuration
+    # @param tdata [Hash] the test data to use for the acceptance test suite
+    # @param node_inv [CemAcpt::NodeInventory] the node inventory to use for the acceptance test suite
+    # @param module_pkg_path [Concurrent::IVar] the path to the module package
+    # @param results [Concurrent::Map] the results map to use for reporting test results
+    def initialize(node, conf, tdata, node_inv, module_pkg_path, results)
+      @node = node
+      @conf = conf
+      @tdata = tdata
+      @node_inv = node_inv
+      @module_pkg_path = module_pkg_path
+      @results = results
+      @spec_exit_code = 0
+      validate!
+    end
+
+    # Starts the runner thread and runs the lifecycle stages of the
+    # acceptance test suite.
+    def start
+      logger.info("Starting test suite for #{@node.node_name}")
+      @thread = Thread.new do
+        provision
+        bootstrap
+        run_tests
+        destroy
+      end
+    end
+
+    # Joins the runner thread.
+    # @param timeout [Integer] thread timeout in seconds
+    def join(timeout = 600)
+      @thread.join(timeout)
+    end
+
+    # Runner thread exits and runner node is destroyed, if it exists.
+    def exit
+      @thread.exit
+      @node&.destroy
+    end
+
+    # Runner thread is killed and runner node is destroyed, if it exists.
+    def kill
+      @thread.kill
+      @node&.destroy
+    end
+
+    private
+
+    # Provisions the node for the acceptance test suite.
+    def provision
+      logger.info("Provisioning #{@node.node_name}...")
+      start_time = Time.now
+      @node.provision
+      sleep(1) until @node.ready?
+      logger.info("Node #{@node.node_name} is ready...")
+      node_desc = {
+        test_data: @node.test_data,
+        platform: @conf.get('platform'),
+        local_port: @node.local_port,
+      }.merge(@node.node)
+      @node_inv.add(@node.node_name, node_desc)
+      logger.info("Node #{@node.node_name} provisioned in #{Time.now - start_time} seconds")
+      @node_inv.save(File.join(@conf.get('module_dir'), 'spec', 'fixtures', 'node_inventory.yaml'))
+    end
+
+    # Bootstraps the node for the acceptance test suite. Currently, this
+    # just uploads and installs the module package.
+    def bootstrap
+      logger.info("Bootstrapping #{@node.node_name}...")
+      until File.exist?(@module_pkg_path.value)
+        logger.debug("Waiting for module package #{@module_pkg_path.value} to exist...")
+        sleep(1)
+      end
+      logger.info("Installing module package #{@module_pkg_path.value}...")
+      @node.install_puppet_module_package(@module_pkg_path.value)
+    end
+
+    # Runs the acceptance test suite via rspec.
+    def run_tests
+      require 'json'
+
+      logger.info("Running tests for #{@node.node_name}...")
+      stdout = nil
+      stderr = nil
+      # ENV['RSPEC_DEBUG'] = 'true' if @conf.get('log_level') == 'debug'
+      test_command = "cd #{@conf.get('module_dir')} && bundle exec rspec #{@node.test_data[:test_file]} --format json"
+      @node.run_tests do
+        stdout, stderr, status = Open3.capture3(test_command)
+        @spec_exit_code = status.exitstatus
+      end
+      logger.info("Tests completed with exit code: #{@spec_exit_code}")
+      @results.put_if_absent(@node.node_name, JSON.parse(stdout))
+    end
+
+    # Destroys the node for the acceptance test suite.
+    def destroy
+      if @conf.get('no_destroy_nodes')
+        logger.info("Not destroying node #{@node.node_name} because 'no_destroy_nodes' is set to true")
+        return
+      end
+      logger.info("Destroying #{@node.node_name}...")
+      @node.destroy
+      logger.info("Node #{@node.node_name} destroyed successfully")
+    end
+
+    # Validates the runner configuration.
+    def validate!
+      raise 'No node provided' unless @node
+      raise 'No config provided' unless @conf
+      raise 'No test data provided' unless @tdata
+      raise 'No node inventory provided' unless @node_inv
+    end
+  end
+end