cem_acpt 0.2.6-universal-java-17

data/lib/cem_acpt/spec_helper_acceptance.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+module CemAcpt
+  require_relative 'shared_objects'
+
+  # This methods is used in spec_helper_acceptance.rb to set up
+  # baseline configurations used in acceptance tests.
+  def self.configure_spec_helper!
+    require 'serverspec'
+    require 'yaml'
+
+    RSpec.configure do |config|
+      config.include CemAcpt::SpecHelperAcceptance
+      config.extend CemAcpt::SpecHelperAcceptance
+      config.add_setting :acpt_test_data, default: {}
+      config.add_setting :acpt_node_inventory
+      config.threadsafe = true
+      config.color_mode = :off
+      config.fail_fast = false
+    end
+
+    node_inventory = NodeInventory.new
+    node_inventory.load
+    RSpec.configuration.acpt_node_inventory = node_inventory
+  end
+
+  # This module provides methods used in accpetance tests.
+  module SpecHelperAcceptance
+    # This method must be used inside of a `describe` block as the first
+    # statement. This prepares the ServerSpec configuration for the test node
+    # and sets up the test data.
+    def initialize_test_environment!
+      test_file = caller_locations.first.path
+
+      node_name, node_data = RSpec.configuration.acpt_node_inventory.get_by_property('test_data.test_file', test_file)
+      raise "Failed to get node data for node #{node_name}" unless node_data
+      raise "Node data format is incorrect: #{node_data}" unless node_data.is_a?(Hash)
+
+      backend = nil
+      host = nil
+      ssh_options = nil
+      puppet_path = nil
+
+      # Set remote communication variables based on transport type
+      case node_data[:transport]
+      when /ssh/
+        backend = :ssh
+        host = node_data[:node_name]
+        ssh_options = node_data[:ssh_opts]
+        sudo_options = '-n -u root -i'
+        puppet_path = '/opt/puppetlabs/bin/puppet'
+      when /winrm/
+        backend = :winrm
+        puppet_path = 'C:\Program Files\Puppet Labs\Puppet\bin\puppet.bat'
+      else
+        raise "Unknown transport: #{node[:transport]}"
+      end
+
+      # Set serverspec transport options and host for remote communication.
+      set :backend, backend
+      set :host, host
+      set(:ssh_options, ssh_options) if ssh_options
+      set(:sudo_options, sudo_options) if sudo_options
+      set(:os, family: 'windows') if backend == :winrm
+
+      # Get the command provider from the node's platform
+      # We add this as a RSpec config option so that we can use it in
+      # other functions.
+      require 'cem_acpt/platform'
+
+      acpt_test_data = {
+        test_file: test_file,
+        node_name: node_name,
+        node_data: node_data,
+        backend: backend,
+        platform: CemAcpt::Platform.get(node_data[:platform]),
+        puppet_path: puppet_path,
+      }
+      acpt_test_data[:host] = host if host
+      RSpec.configuration.acpt_test_data = acpt_test_data
+    end
+
+    # This method formats Puppet Apply options
+    def puppet_apply_options(opts = {})
+      if [opts[:catch_changes], opts[:expect_changes], opts[:catch_failures], opts[:expect_failures]].compact.length > 1
+        raise ArgumentError,
+              'Please specify only one of "catch_changes", "expect_changes", "catch_failures", or "expect_failures"'
+      end
+
+      apply_opts = {}.merge(opts)
+
+      if opts[:catch_changes]
+        apply_opts[:detailed_exit_codes] = true
+        apply_opts[:acceptable_exit_codes] = [0]
+      elsif opts[:catch_failures]
+        apply_opts[:detailed_exit_codes] = true
+        apply_opts[:acceptable_exit_codes] = [0, 2]
+      elsif opts[:expect_failures]
+        apply_opts[:detailed_exit_codes] = true
+        apply_opts[:acceptable_exit_codes] = [1, 4, 6]
+      elsif opts[:expect_changes]
+        apply_opts[:detailed_exit_codes] = true
+        apply_opts[:acceptable_exit_codes] = [2]
+      else
+        apply_opts[:detailed_exit_codes] = false
+        apply_opts[:acceptable_exit_codes] = [0]
+      end
+      apply_opts
+    end
+
+    # This methods handles formatting the output from Puppet apply.
+    def handle_puppet_apply_output(result, apply_opts)
+      exit_code = result.exitstatus
+      output = result.to_s
+      if apply_opts[:catch_changes] && !apply_opts[:acceptable_exit_codes].include?(exit_code)
+        failure = <<~ERROR
+          Apply manifest expected no changes. Puppet Apply returned exit code #{exit_code}
+          ====== Start output of Puppet apply with unexpected changes ======
+          #{output}
+          ====== End output of Puppet apply with unexpected changes ======
+        ERROR
+        raise failure
+      elsif !apply_opts[:acceptable_exit_codes].include?(exit_code)
+        failure = <<~ERROR
+          Apply manifest failed with exit code #{exit_code} (expected: #{apply_opts[:acceptable_exit_codes]})
+          ====== Start output of failed Puppet apply ======
+          #{output}
+          ====== End output of failed Puppet apply ======
+        ERROR
+        raise failure
+      end
+
+      yield result if block_given?
+
+      if ENV['RSPEC_DEBUG']
+        run_result = <<~RUNRES
+          apply manifest succeded with status #{exit_code}
+          ===== Start output of successful Puppet apply ======
+          #{output}
+          ===== End output of successful Puppet apply ======
+        RUNRES
+        puts run_result
+      end
+      result
+    end
+
+    # This method runs a shell command on the test node.
+    def run_shell(cmd, opts = {})
+      cmd = cmd.join(' ') if cmd.is_a?(Array)
+
+      host = RSpec.configuration.acpt_test_data[:node_name]
+      opts[:ssh_opts] = RSpec.configuration.acpt_test_data[:node_data][:ssh_opts]
+      RSpec.configuration.acpt_test_data[:platform].run_shell(host, cmd, opts)
+    end
+
+    # This method runs puppet apply on the test node using the provided manifest.
+    def apply_manifest(manifest, opts = {})
+      host = RSpec.configuration.acpt_test_data[:node_name]
+      opts = {
+        apply: puppet_apply_options(opts),
+        ssh_opts: RSpec.configuration.acpt_test_data[:node_data][:ssh_opts],
+        puppet_path: RSpec.configuration.acpt_test_data[:puppet_path],
+      }
+      result = RSpec.configuration.acpt_test_data[:platform].apply_manifest(host, manifest, opts)
+      handle_puppet_apply_output(result, opts[:apply])
+    end
+
+    # This method runs puppet apply on the test node using the provided manifest twice and
+    # asserts that the second run has no changes.
+    def idempotent_apply(manifest, opts = {})
+      opts.reject! { |k, _| %i[catch_changes expect_changes catch_failures expect_failures].include?(k) }
+      begin
+        apply_manifest(manifest, opts.merge({ catch_failures: true }))
+      rescue StandardError => e
+        raise "Idempotent apply failed during first apply: #{e.message}\n#{e.backtrace.join("\n")}"
+      end
+      begin
+        apply_manifest(manifest, opts.merge({ catch_changes: true }))
+      rescue StandardError => e
+        raise "Idempotent apply failed during second apply: #{e.message}\n#{e.backtrace.join("\n")}"
+      end
+    end
+  end
+end

data/lib/cem_acpt/test_data.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+require_relative 'core_extensions'
+require_relative 'logging'
+
+module CemAcpt
+  # This module provides a method and class for extracting and formatting
+  # test data.
+  module TestData
+    # Returns a hash of test data.
+    # @param config [CemAcpt::Config] the config object
+    # @return [Array<Hash>] an array of test data hashes
+    def self.acceptance_test_data(config)
+      Fetcher.new(config).acceptance_test_data
+    end
+
+    # Fetcher provides the methods for extracting and formatting test data.
+    class Fetcher
+      include CemAcpt::Logging
+      using CemAcpt::CoreExtensions::ExtendedHash
+
+      attr_reader :module_dir, :acceptance_tests
+
+      # Initializes a new Fetcher object.
+      # @param config [CemAcpt::Config] the config object
+      def initialize(config)
+        @config = config
+        @module_dir = config.get('module_dir')
+        @acceptance_tests = find_acceptance_tests(config.get('module_dir'))
+      end
+
+      # Extracts, formats, and returns a test data hash.
+      # @return [Array<Hash>] an array of test data hashes
+      def acceptance_test_data
+        logger.info 'Gathering acceptance test data...'
+        acceptance_tests.each_with_object([]) do |t, a|
+          logger.debug("Processing #{t}...")
+          test_name = File.basename(t, '_spec.rb')
+          test_data = {
+            test_name: test_name,
+            test_file: File.expand_path(t),
+          }
+          next unless @config.has?('tests') && @config.get('tests').include?(test_name)
+
+          process_for_each(test_data).each do |test_data_i|
+            process_static_vars(test_data_i)
+            process_name_pattern_vars(test_name, test_data_i)
+            vars_post_processing!(test_data_i)
+            test_data_i.format!
+            a << test_data_i
+          end
+        end
+      end
+
+      private
+
+      # Locates acceptance tests in the module directory.
+      # @return [Array<String>] the list of acceptance test paths
+      def find_acceptance_tests(module_dir)
+        tests = Dir.glob(File.join(module_dir, 'spec', 'acceptance', '*_spec.rb'))
+        raise 'No acceptance tests found' if tests.empty?
+
+        logger.info "Found #{tests.size} acceptance tests"
+        tests
+      end
+
+      # Processes a for_each statement in the test data config.
+      # @param test_data [Hash] the test data hash
+      # @return [Array<Hash>] the list of test data hashes
+      def process_for_each(test_data)
+        return [test_data] unless @config.has?('test_data.for_each')
+
+        @config.get('test_data.for_each').each_with_object([]) do |(k, v), a|
+          v.each do |v_i|
+            test_data_i = test_data.dup
+            test_data_i[k] = v_i
+            a << test_data_i
+          end
+        end
+      end
+
+      def process_static_vars(test_data)
+        return unless @config.has?('test_data.vars')
+
+        vars = @config.get('test_data.vars').each_with_object({}) do |(k, v), h|
+          logger.debug("Static variable: #{k}")
+          h[k] = v
+        end
+        test_data.merge!(vars)
+        test_data.format!
+      end
+
+      def process_name_pattern_vars(test_name, test_data)
+        name_pattern_vars = name_pattern_matches(test_name) || {}
+        test_data.merge!(name_pattern_vars)
+        test_data.format!
+      end
+
+      def name_pattern_matches(test_name)
+        return test_name unless @config.has?('test_data.name_pattern_vars')
+
+        pattern = @config.get('test_data.name_pattern_vars')
+        pattern_matches = test_name.match(pattern)&.named_captures
+        unless pattern_matches
+          logger.error("Test name #{test_name} does not match pattern #{pattern.inspect}")
+          return
+        end
+        pattern_matches
+      end
+
+      def vars_post_processing!(test_data)
+        return unless @config.has?('test_data.vars_post_processing')
+
+        post_processing_new_vars!(test_data)
+        post_processing_delete_vars!(test_data)
+      end
+
+      def post_processing_new_vars!(test_data)
+        new_vars = @config.get('test_data.vars_post_processing.new_vars')
+        return unless new_vars
+
+        new_vars.each do |var|
+          if var.has?('string_split')
+            test_data[var.dot_dig('name')] = op_string_split(test_data, var)
+          else
+            logger.error("Unknown post processing operation for new var #{var}")
+          end
+        end
+      end
+
+      def op_string_split(test_data, var)
+        from_var = var.dot_dig('string_split.from')
+        from = test_data.dot_dig(from_var)
+        parts = from.split(var.dot_dig('string_split.using'))
+        var.has?('string_split.part') ? parts[var.dot_dig('string_split.part')] : parts
+      end
+
+      def post_processing_delete_vars!(test_data)
+        delete_vars = @config.get('test_data.vars_post_processing.delete_vars')
+        return unless delete_vars
+
+        test_data.reject! { |k, _| delete_vars.include?(k.to_s) }
+      end
+    end
+  end
+end

data/lib/cem_acpt/test_runner/run_handler.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+require 'concurrent-ruby'
+require 'open3'
+require_relative '../context'
+require_relative '../logging'
+require_relative '../puppet_helpers'
+require_relative 'runner'
+
+module CemAcpt
+  module TestRunner
+    # Raised when the RunHandler needs to be operating within the module directory and isn't
+    class RunHandlerNotInModuleDirError < StandardError; end
+
+    # Raised when the RunHandler executes a system command that fails
+    class RunHandlerSystemCommandError < StandardError; end
+
+    # RunHandler orchestrates the acceptance test suites, including
+    # creating Runner objects, handling input and output, and exception
+    # handling.
+    class RunHandler
+      include CemAcpt::Logging
+
+      attr_accessor :run_context
+
+      # @param run_context [CemAcpt::Context::Ctx] The context object for this run
+      def initialize(run_context)
+        @run_context = run_context
+        @module_pkg_path = Concurrent::IVar.new
+        @runners = Concurrent::Array.new
+        @thread_pool = Concurrent::SimpleExecutorService.new
+      end
+
+      # Gets the overall exit code for all runners
+      def exit_code
+        @runners&.map { |r| r.run_result.exit_status }&.all?(&:zero?) ? 0 : 1
+      end
+
+      # Runs the acceptance test suites.
+      def run
+        logger.info('RUN HANDLER: Creating and starting test runners...')
+        create_runners
+        logger.info("RUN HANDLER: Created #{@runners.length} runners...")
+        @runners.map { |r| @thread_pool.post { r.start } }
+        @thread_pool.shutdown
+        @thread_pool.wait_for_termination
+        @thread_pool = Concurrent::SimpleExecutorService.new
+        logger.info('Test runners have all exited...')
+      rescue StandardError => e
+        handle_fatal_error(e)
+      ensure
+        handle_test_results
+      end
+
+      def destroy_test_nodes
+        return if @runners.empty? || @runners.none?(&:node_exists)
+
+        @runners.each do |r|
+          r.send(:destroy) if r.node_exists
+        end
+      end
+
+      private
+
+      # Creates and starts Runner objects for each node in the acceptance test suites.
+      def create_runners
+        @run_context.nodes.each do |platform, nodes|
+          @runners += nodes.map { |node| create_runner(node, platform) }.compact
+        end
+      end
+
+      def create_runner(node, platform)
+        logger.info("RUN HANDLER: Creating runner for #{node.test_data[:test_name]} on node #{node.node_name}...")
+        runner = CemAcpt::TestRunner::Runner.new(node, @run_context, platform)
+        runner || runner_creation_error(node)
+      end
+
+      def runner_creation_error(node)
+        msg = [
+          "Failed to create runner object for node #{node.node_name}.",
+          "Cannot run test #{node.test_data[:test_name]}",
+        ].join(' ')
+        logger.error(msg)
+        nil
+      end
+
+      # Handles how test results are logged.
+      def handle_test_results
+        @runners.each do |runner|
+          result = runner.run_result.to_h
+          logger << "::group::{Results for #{runner.node.test_data[:test_name]}}\n"
+          if result.key?('summary_line')
+            logger.info("SUMMARY: #{result['summary_line']} for test #{runner.node.test_data[:test_name]}")
+          else
+            handle_runner_error_results(runner)
+            logger << "::endgroup::\n"
+            next
+          end
+          unless runner.test_failures?
+            logger << "::endgroup::\n"
+            next
+          end
+
+          if result.key?('examples') # Log errors outside of examples
+            if result['examples'].empty? && !result['messages'].empty?
+              logger.error(result['messages'].join("\n"))
+            else
+              failed = result['examples'].reject { |e| e['status'] == 'passed' }
+              failed.each do |e|
+                logger.error(test_error_msg(runner.node.node_name, e))
+              end
+            end
+          else
+            handle_runner_error_results(runner)
+          end
+          #debug_test_results(runner) if logger.debug?
+          logger << "::endgroup::\n"
+        end
+      end
+
+      # Handles logging the results of the runners that errored.
+      def handle_runner_error_results(runner)
+        logger.error("SUMMARY: Encountered an error with test #{runner.node.test_data[:test_name]} on node #{runner.node.node_name}")
+        runner.run_result.to_h.each do |k, v|
+          logger.error("#{k.upcase}: #{v}")
+        end
+      end
+
+      # Formats the error message for a runner that failed.
+      # @param node [Object] the node that failed
+      # @param err [Object] the error that occured
+      # @return [String] the formatted error message
+      def runner_error_msg(node, err)
+        [
+          "Error while running acceptance tests on node #{node.node_name}",
+          "Error: #{err.message}",
+          'Backtrace:',
+          err.backtrace.join("\n"),
+        ].join("\n")
+      end
+
+      # Formats a test result for tests that have failed. Is used for logging.
+      # @param node [String] the name of the node the test ran on
+      # @param result [Hash] the test result to format
+      # @return [String] the formatted test result
+      def test_error_msg(node, result)
+        [
+          "TEST FAILED: #{result['id']}",
+          "DESCRIPTION: #{result['full_description']}",
+          "STATUS: #{result['status']}",
+          "LOCATION: #{result['file_path']}:#{result['line_number']}",
+          "NODE: #{node}",
+          result['exception']['message'],
+          "\n",
+        ].join("\n")
+      end
+
+      # Logs performance data for the acceptance test suites.
+      # This is only logged if the log level is set to debug.
+      # def debug_test_results(runner)
+      #   examples_by_time = []
+      #   runner.run_result.to_h.each do |node, result|
+      #     next unless result['examples']
+
+      #     result['examples'].each do |e|
+      #       examples_by_time << [e['run_time'], e['id'], e['status'], e['line_number'], node]
+      #     end
+      #   end
+      #   if examples_by_time
+      #     logger.debug('Showing test results in order of execution time...')
+      #     examples_by_time.sort_by(&:first).reverse.each do |e|
+      #       logger.debug("RUNTIME: #{e[0]}; ID: #{e[1]}; STATUS: #{e[2]}; LINE: #{e[3]}; HOST: #{e[4]};")
+      #     end
+      #   else
+      #     logger.debug('No debug results to show')
+      #   end
+      # end
+
+      # Gracefully handles a fatal error and exits the program.
+      # @param err [StandardError, Exception] the error that caused the fatal error
+      def handle_fatal_error(err)
+        logger.fatal("RUN HANDLER: Fatal error: #{err.message}")
+        logger.debug(err.backtrace.join("\n"))
+      end
+    end
+  end
+end