cem_acpt 0.2.5 → 0.6.0

data/lib/cem_acpt/spec_helper_acceptance.rb

@@ -1,184 +0,0 @@
-# frozen_string_literal: true
-
-module CemAcpt
-  require_relative 'shared_objects'
-
-  # This methods is used in spec_helper_acceptance.rb to set up
-  # baseline configurations used in acceptance tests.
-  def self.configure_spec_helper!
-    require 'serverspec'
-    require 'yaml'
-
-    RSpec.configure do |config|
-      config.include CemAcpt::SpecHelperAcceptance
-      config.extend CemAcpt::SpecHelperAcceptance
-      config.add_setting :acpt_test_data, default: {}
-      config.add_setting :acpt_node_inventory
-      config.threadsafe = true
-      config.color_mode = :off
-      config.fail_fast = false
-    end
-
-    node_inventory = NodeInventory.new
-    node_inventory.load
-    RSpec.configuration.acpt_node_inventory = node_inventory
-  end
-
-  # This module provides methods used in accpetance tests.
-  module SpecHelperAcceptance
-    # This method must be used inside of a `describe` block as the first
-    # statement. This prepares the ServerSpec configuration for the test node
-    # and sets up the test data.
-    def initialize_test_environment!
-      test_file = caller_locations.first.path
-
-      node_name, node_data = RSpec.configuration.acpt_node_inventory.get_by_property('test_data.test_file', test_file)
-      raise "Failed to get node data for node #{node_name}" unless node_data
-      raise "Node data format is incorrect: #{node_data}" unless node_data.is_a?(Hash)
-
-      backend = nil
-      host = nil
-      ssh_options = nil
-      puppet_path = nil
-
-      # Set remote communication variables based on transport type
-      case node_data[:transport]
-      when /ssh/
-        backend = :ssh
-        host = node_data[:node_name]
-        ssh_options = node_data[:ssh_opts]
-        sudo_options = '-n -u root -i'
-        puppet_path = '/opt/puppetlabs/bin/puppet'
-      when /winrm/
-        backend = :winrm
-        puppet_path = 'C:\Program Files\Puppet Labs\Puppet\bin\puppet.bat'
-      else
-        raise "Unknown transport: #{node[:transport]}"
-      end
-
-      # Set serverspec transport options and host for remote communication.
-      set :backend, backend
-      set :host, host
-      set(:ssh_options, ssh_options) if ssh_options
-      set(:sudo_options, sudo_options) if sudo_options
-      set(:os, family: 'windows') if backend == :winrm
-
-      # Get the command provider from the node's platform
-      # We add this as a RSpec config option so that we can use it in
-      # other functions.
-      require 'cem_acpt/platform'
-
-      acpt_test_data = {
-        test_file: test_file,
-        node_name: node_name,
-        node_data: node_data,
-        backend: backend,
-        platform: CemAcpt::Platform.get(node_data[:platform]),
-        puppet_path: puppet_path,
-      }
-      acpt_test_data[:host] = host if host
-      RSpec.configuration.acpt_test_data = acpt_test_data
-    end
-
-    # This method formats Puppet Apply options
-    def puppet_apply_options(opts = {})
-      if [opts[:catch_changes], opts[:expect_changes], opts[:catch_failures], opts[:expect_failures]].compact.length > 1
-        raise ArgumentError,
-              'Please specify only one of "catch_changes", "expect_changes", "catch_failures", or "expect_failures"'
-      end
-
-      apply_opts = {}.merge(opts)
-
-      if opts[:catch_changes]
-        apply_opts[:detailed_exit_codes] = true
-        apply_opts[:acceptable_exit_codes] = [0]
-      elsif opts[:catch_failures]
-        apply_opts[:detailed_exit_codes] = true
-        apply_opts[:acceptable_exit_codes] = [0, 2]
-      elsif opts[:expect_failures]
-        apply_opts[:detailed_exit_codes] = true
-        apply_opts[:acceptable_exit_codes] = [1, 4, 6]
-      elsif opts[:expect_changes]
-        apply_opts[:detailed_exit_codes] = true
-        apply_opts[:acceptable_exit_codes] = [2]
-      else
-        apply_opts[:detailed_exit_codes] = false
-        apply_opts[:acceptable_exit_codes] = [0]
-      end
-      apply_opts
-    end
-
-    # This methods handles formatting the output from Puppet apply.
-    def handle_puppet_apply_output(result, apply_opts)
-      exit_code = result.exitstatus
-      output = result.to_s
-      if apply_opts[:catch_changes] && !apply_opts[:acceptable_exit_codes].include?(exit_code)
-        failure = <<~ERROR
-          Apply manifest expected no changes. Puppet Apply returned exit code #{exit_code}
-          ====== Start output of Puppet apply with unexpected changes ======
-          #{output}
-          ====== End output of Puppet apply with unexpected changes ======
-        ERROR
-        raise failure
-      elsif !apply_opts[:acceptable_exit_codes].include?(exit_code)
-        failure = <<~ERROR
-          Apply manifest failed with exit code #{exit_code} (expected: #{apply_opts[:acceptable_exit_codes]})
-          ====== Start output of failed Puppet apply ======
-          #{output}
-          ====== End output of failed Puppet apply ======
-        ERROR
-        raise failure
-      end
-
-      yield result if block_given?
-
-      if ENV['RSPEC_DEBUG']
-        run_result = <<~RUNRES
-          apply manifest succeded with status #{exit_code}
-          ===== Start output of successful Puppet apply ======
-          #{output}
-          ===== End output of successful Puppet apply ======
-        RUNRES
-        puts run_result
-      end
-      result
-    end
-
-    # This method runs a shell command on the test node.
-    def run_shell(cmd, opts = {})
-      cmd = cmd.join(' ') if cmd.is_a?(Array)
-
-      host = RSpec.configuration.acpt_test_data[:node_name]
-      opts[:ssh_opts] = RSpec.configuration.acpt_test_data[:node_data][:ssh_opts]
-      RSpec.configuration.acpt_test_data[:platform].run_shell(host, cmd, opts)
-    end
-
-    # This method runs puppet apply on the test node using the provided manifest.
-    def apply_manifest(manifest, opts = {})
-      host = RSpec.configuration.acpt_test_data[:node_name]
-      opts = {
-        apply: puppet_apply_options(opts),
-        ssh_opts: RSpec.configuration.acpt_test_data[:node_data][:ssh_opts],
-        puppet_path: RSpec.configuration.acpt_test_data[:puppet_path],
-      }
-      result = RSpec.configuration.acpt_test_data[:platform].apply_manifest(host, manifest, opts)
-      handle_puppet_apply_output(result, opts[:apply])
-    end
-
-    # This method runs puppet apply on the test node using the provided manifest twice and
-    # asserts that the second run has no changes.
-    def idempotent_apply(manifest, opts = {})
-      opts.reject! { |k, _| %i[catch_changes expect_changes catch_failures expect_failures].include?(k) }
-      begin
-        apply_manifest(manifest, opts.merge({ catch_failures: true }))
-      rescue StandardError => e
-        raise "Idempotent apply failed during first apply: #{e.message}\n#{e.backtrace.join("\n")}"
-      end
-      begin
-        apply_manifest(manifest, opts.merge({ catch_changes: true }))
-      rescue StandardError => e
-        raise "Idempotent apply failed during second apply: #{e.message}\n#{e.backtrace.join("\n")}"
-      end
-    end
-  end
-end

data/lib/cem_acpt/test_runner/run_handler.rb

@@ -1,187 +0,0 @@
-# frozen_string_literal: true
-
-require 'concurrent-ruby'
-require 'open3'
-require_relative '../context'
-require_relative '../logging'
-require_relative '../puppet_helpers'
-require_relative 'runner'
-
-module CemAcpt
-  module TestRunner
-    # Raised when the RunHandler needs to be operating within the module directory and isn't
-    class RunHandlerNotInModuleDirError < StandardError; end
-
-    # Raised when the RunHandler executes a system command that fails
-    class RunHandlerSystemCommandError < StandardError; end
-
-    # RunHandler orchestrates the acceptance test suites, including
-    # creating Runner objects, handling input and output, and exception
-    # handling.
-    class RunHandler
-      include CemAcpt::Logging
-
-      attr_accessor :run_context
-
-      # @param run_context [CemAcpt::Context::Ctx] The context object for this run
-      def initialize(run_context)
-        @run_context = run_context
-        @module_pkg_path = Concurrent::IVar.new
-        @runners = Concurrent::Array.new
-        @thread_pool = Concurrent::SimpleExecutorService.new
-      end
-
-      # Gets the overall exit code for all runners
-      def exit_code
-        @runners&.map { |r| r.run_result.exit_status }&.all?(&:zero?) ? 0 : 1
-      end
-
-      # Runs the acceptance test suites.
-      def run
-        logger.info('RUN HANDLER: Creating and starting test runners...')
-        create_runners
-        logger.info("RUN HANDLER: Created #{@runners.length} runners...")
-        @runners.map { |r| @thread_pool.post { r.start } }
-        @thread_pool.shutdown
-        @thread_pool.wait_for_termination
-        @thread_pool = Concurrent::SimpleExecutorService.new
-        logger.info('Test runners have all exited...')
-      rescue StandardError => e
-        handle_fatal_error(e)
-      ensure
-        handle_test_results
-      end
-
-      def destroy_test_nodes
-        return if @runners.empty? || @runners.none?(&:node_exists)
-
-        @runners.each do |r|
-          r.send(:destroy) if r.node_exists
-        end
-      end
-
-      private
-
-      # Creates and starts Runner objects for each node in the acceptance test suites.
-      def create_runners
-        @run_context.nodes.each do |platform, nodes|
-          @runners += nodes.map { |node| create_runner(node, platform) }.compact
-        end
-      end
-
-      def create_runner(node, platform)
-        logger.info("RUN HANDLER: Creating runner for #{node.test_data[:test_name]} on node #{node.node_name}...")
-        runner = CemAcpt::TestRunner::Runner.new(node, @run_context, platform)
-        runner || runner_creation_error(node)
-      end
-
-      def runner_creation_error(node)
-        msg = [
-          "Failed to create runner object for node #{node.node_name}.",
-          "Cannot run test #{node.test_data[:test_name]}",
-        ].join(' ')
-        logger.error(msg)
-        nil
-      end
-
-      # Handles how test results are logged.
-      def handle_test_results
-        @runners.each do |runner|
-          result = runner.run_result.to_h
-          logger << "::group::{Results for #{runner.node.test_data[:test_name]}}\n"
-          if result.key?('summary_line')
-            logger.info("SUMMARY: #{result['summary_line']} for test #{runner.node.test_data[:test_name]}")
-          else
-            handle_runner_error_results(runner)
-            logger << "::endgroup::\n"
-            next
-          end
-          unless runner.test_failures?
-            logger << "::endgroup::\n"
-            next
-          end
-
-          if result.key?('examples') # Log errors outside of examples
-            if result['examples'].empty? && !result['messages'].empty?
-              logger.error(result['messages'].join("\n"))
-            else
-              failed = result['examples'].reject { |e| e['status'] == 'passed' }
-              failed.each do |e|
-                logger.error(test_error_msg(runner.node.node_name, e))
-              end
-            end
-          else
-            handle_runner_error_results(runner)
-          end
-          #debug_test_results(runner) if logger.debug?
-          logger << "::endgroup::\n"
-        end
-      end
-
-      # Handles logging the results of the runners that errored.
-      def handle_runner_error_results(runner)
-        logger.error("SUMMARY: Encountered an error with test #{runner.node.test_data[:test_name]} on node #{runner.node.node_name}")
-        runner.run_result.to_h.each do |k, v|
-          logger.error("#{k.upcase}: #{v}")
-        end
-      end
-
-      # Formats the error message for a runner that failed.
-      # @param node [Object] the node that failed
-      # @param err [Object] the error that occured
-      # @return [String] the formatted error message
-      def runner_error_msg(node, err)
-        [
-          "Error while running acceptance tests on node #{node.node_name}",
-          "Error: #{err.message}",
-          'Backtrace:',
-          err.backtrace.join("\n"),
-        ].join("\n")
-      end
-
-      # Formats a test result for tests that have failed. Is used for logging.
-      # @param node [String] the name of the node the test ran on
-      # @param result [Hash] the test result to format
-      # @return [String] the formatted test result
-      def test_error_msg(node, result)
-        [
-          "TEST FAILED: #{result['id']}",
-          "DESCRIPTION: #{result['full_description']}",
-          "STATUS: #{result['status']}",
-          "LOCATION: #{result['file_path']}:#{result['line_number']}",
-          "NODE: #{node}",
-          result['exception']['message'],
-          "\n",
-        ].join("\n")
-      end
-
-      # Logs performance data for the acceptance test suites.
-      # This is only logged if the log level is set to debug.
-      # def debug_test_results(runner)
-      #   examples_by_time = []
-      #   runner.run_result.to_h.each do |node, result|
-      #     next unless result['examples']
-
-      #     result['examples'].each do |e|
-      #       examples_by_time << [e['run_time'], e['id'], e['status'], e['line_number'], node]
-      #     end
-      #   end
-      #   if examples_by_time
-      #     logger.debug('Showing test results in order of execution time...')
-      #     examples_by_time.sort_by(&:first).reverse.each do |e|
-      #       logger.debug("RUNTIME: #{e[0]}; ID: #{e[1]}; STATUS: #{e[2]}; LINE: #{e[3]}; HOST: #{e[4]};")
-      #     end
-      #   else
-      #     logger.debug('No debug results to show')
-      #   end
-      # end
-
-      # Gracefully handles a fatal error and exits the program.
-      # @param err [StandardError, Exception] the error that caused the fatal error
-      def handle_fatal_error(err)
-        logger.fatal("RUN HANDLER: Fatal error: #{err.message}")
-        logger.debug(err.backtrace.join("\n"))
-      end
-    end
-  end
-end

data/lib/cem_acpt/test_runner/runner.rb

@@ -1,210 +0,0 @@
-# frozen_string_literal: true
-
-require 'concurrent-ruby'
-require 'English'
-require_relative '../logging'
-require_relative '../rspec_utils'
-require_relative 'runner_result'
-
-module CemAcpt
-  module TestRunner
-    class RunnerError < StandardError; end
-    # Error used to wrap fatal errors raised in Runner steps
-    class RunnerStepError < StandardError
-      attr_reader :step
-
-      def initialize(step, err)
-        @step = step
-        super err
-        set_backtrace err.backtrace if err.respond_to?(:backtrace)
-      end
-    end
-    class RunnerProvisionError < RunnerStepError; end
-
-    # Runner is a class that runs a single acceptance test suite on a single node.
-    # It is responsible for managing the lifecycle of the test suite and
-    # reporting the results back to the main thread. Runner objects are created
-    # by the RunHandler and then, when started, execute their logic in a thread.
-    class Runner
-      include CemAcpt::LoggingAsync
-
-      attr_reader :node, :node_exists, :run_result
-
-      # @param node [String] the name of the node to run the acceptance test suite on
-      # @param ctx [CemAcpt::RunnerCtx] a cem_acpt Ctx (context) object
-      # @param module_pkg_path [Concurrent::IVar] the path to the module package
-      def initialize(node, context, platform)
-        @node = node
-        @context = context
-        @platform = platform
-        @debug_mode = @context.config.debug_mode?
-        @node_inventory = @context.node_inventory
-        @module_pkg_path = @context.module_package_path
-        @node_exists = false
-        @run_result = CemAcpt::TestRunner::RunnerResult.new(@node, debug: @debug_mode)
-        @completed_steps = []
-        validate!
-      end
-
-      def run_step(step_sym)
-        send(step_sym)
-        @completed_steps << step_sym
-      rescue StandardError => e
-        err = CemAcpt::TestRunner::RunnerStepError.new(step_sym, e)
-        step_error_logging(err)
-        @run_result.from_error(err)
-        destroy unless step_sym == :destroy
-      end
-
-      # Executes test suite steps
-      def start
-        async_info("Starting test suite for #{@node.node_name}", log_prefix('RUNNER'))
-        run_step(:provision)
-        run_step(:bootstrap)
-        run_step(:run_tests)
-        run_step(:destroy)
-        true
-      rescue StandardError => e
-        step_error_logging(e)
-        @run_result.from_error(e)
-        destroy
-      end
-
-      # Checks for failures in the test results.
-      # @param result [Hash] the test result to check
-      # @return [Boolean] whether or not there are test failures in result
-      def test_failures?
-        @run_result.result_errors? || @run_result.result_failures?
-      end
-
-      private
-
-      def step_error_logging(err)
-        prefix = err.respond_to?(:step) ? log_prefix(err.step.capitalize) : log_prefix('RUNNER')
-        fatal_msg = ["runner failed: #{err.message}"]
-        async_fatal(fatal_msg, prefix)
-        async_debug("Completed steps: #{@completed_steps}", prefix)
-        async_debug("Failed runner backtrace:\n#{err.backtrace.join("\n")}", prefix)
-        async_debug("Failed runner test data: #{@node.test_data}", prefix)
-      end
-
-      def log_prefix(prefix)
-        "#{prefix}: #{@node.test_data[:test_name]}:"
-      end
-
-      # Provisions the node for the acceptance test suite.
-      def provision
-        async_info("Provisioning #{@node.node_name}...", log_prefix('PROVISION'))
-        start_time = Time.now
-        @node.provision
-        @node_exists = true
-        max_retries = 60 # equals 300 seconds because we check every five seconds
-        until @node.ready?
-          if max_retries <= 0
-            async_fatal("Node #{@node.node_name} failed to provision", log_prefix('PROVISION'))
-            raise CemAcpt::TestRunner::RunnerProvisionError, "Provisioning timed out for node #{@node.node_name}"
-          end
-
-          async_info("Waiting for #{@node.node_name} to be ready for remote connections...", log_prefix('PROVISION'))
-          max_retries -= 1
-          sleep(5)
-        end
-        async_info("Node #{@node.node_name} is ready...", log_prefix('PROVISION'))
-        node_desc = {
-          test_data: @node.test_data,
-          platform: @platform,
-          local_port: @node.local_port,
-        }.merge(@node.node)
-        @node_inventory.add(@node.node_name, node_desc)
-        @node_inventory.save
-        async_info("Node #{@node.node_name} provisioned in #{Time.now - start_time} seconds", log_prefix('PROVISION'))
-      end
-
-      # Bootstraps the node for the acceptance test suite. Currently, this
-      # just uploads and installs the module package.
-      def bootstrap
-        async_info("Bootstrapping #{@node.node_name}...", log_prefix('BOOTSTRAP'))
-        until File.exist?(@module_pkg_path)
-          async_debug("Waiting for module package #{@module_pkg_path} to exist...", log_prefix('BOOTSTRAP'))
-          sleep(1)
-        end
-        async_info("Installing module package #{@module_pkg_path}...", log_prefix('BOOTSTRAP'))
-        @node.install_puppet_module_package(@module_pkg_path)
-      end
-
-      # Runs the acceptance test suite via rspec.
-      def run_tests
-        attempts = 0
-        until File.exist?(@node_inventory.save_file_path)
-          raise 'Node inventory file not found' if (attempts += 1) > 3
-
-          sleep(1)
-        end
-        async_info("Running test #{@node.test_data[:test_name]} on node #{@node.node_name}...", log_prefix('RSPEC'))
-        @node.run_tests do |cmd_env|
-          cmd_opts = rspec_opts
-          cmd_opts[:env].merge!(cmd_env) if cmd_env
-          # Documentation format gets logged in real time, JSON file is read after the fact
-          begin
-            @rspec_cmd = CemAcpt::RSpecUtils::Command.new(cmd_opts)
-            @rspec_cmd.execute(log_prefix: log_prefix('RSPEC'))
-            @run_result.from_json_file(cmd_opts[:format][:json])
-          rescue Errno::EIO => e
-            async_error("failed to run rspec: #{@node.test_data[:test_name]}: #{$ERROR_INFO}", log_prefix('RSPEC'))
-            @run_result.from_error(e)
-          rescue StandardError => e
-            async_error("failed to run rspec: #{@node.test_data[:test_name]}: #{e.message}", log_prefix('RSPEC'))
-            async_debug("Backtrace:\n#{e.backtrace}", log_prefix('RSPEC'))
-            @run_result.from_error(e)
-          end
-        end
-        async_info("Tests completed with exit code: #{@run_result.exit_status}", log_prefix('RSPEC'))
-      end
-
-      # Destroys the node for the acceptance test suite.
-      def destroy
-        kill_spec_pty_if_exists
-        if @context.config.get('no_destroy_nodes')
-          async_info("Not destroying node #{@node.node_name} because 'no_destroy_nodes' is set to true",
-                     log_prefix('DESTROY'))
-        else
-          async_info("Destroying #{@node.node_name}...", log_prefix('DESTROY'))
-          @node.destroy
-          @node_exists = false
-          async_info("Node #{@node.node_name} destroyed successfully", log_prefix('DESTROY'))
-        end
-      end
-
-      def kill_spec_pty_if_exists
-        @rspec_cmd.kill_pty
-      end
-
-      # Validates the runner configuration.
-      def validate!
-        raise 'No node provided' unless @node
-        raise 'Node does not have config' if @node.config.nil? || @node.config.empty?
-        raise 'Node does not have test data' if @node.test_data.nil? || @node.test_data.empty?
-        raise 'No node inventory provided' unless @node_inventory
-      end
-
-      # Options used with RSpec
-      def rspec_opts
-        opts = {
-          test_path: @node.test_data[:test_file],
-          use_bundler: false,
-          bundle_install: false,
-          format: {
-            json: "results_#{@node.test_data[:test_name]}.json",
-          },
-          debug: (@debug_mode && @context.config.get('verbose')),
-          quiet: @context.config.get('quiet'),
-          env: {
-            'TARGET_HOST' => @node.node_name,
-          }
-        }
-        opts[:format][:documentation] = nil unless @context.config.get('verbose')
-        opts
-      end
-    end
-  end
-end

data/lib/cem_acpt/test_runner/runner_result.rb

@@ -1,103 +0,0 @@
-# frozen_string_literal: true
-
-module CemAcpt
-  module TestRunner
-    require 'json'
-
-    # Error thrown when there are problems parsing the JSON output from Rspec
-    class RspecOutputJSONParserError < StandardError
-      def initialize(err = nil)
-        super err
-        set_backtrace err.backtrace unless err.nil?
-      end
-    end
-
-    # Class to process the results of a Runner
-    class RunnerResult
-      attr_reader :run_result
-
-      def initialize(node, debug: false)
-        @node = node
-        @run_result = {}
-        @debug = debug
-      end
-
-      def to_h
-        @run_result
-      end
-
-      def debug?
-        @debug
-      end
-
-      def exit_status
-        if run_result.nil?
-          99
-        elsif run_result.empty?
-          66
-        elsif result_errors? || result_failures?
-          1
-        else
-          0
-        end
-      end
-
-      def result_array
-        [run_result, exit_status]
-      end
-
-      def from_json_file(file_path)
-        res = JSON.parse(File.read(file_path))
-        new_run_result(res)
-      rescue StandardError => e
-        from_error(RspecOutputJSONParserError.new(e))
-      end
-
-      def from_error(err)
-        label = err.class.to_s.start_with?('Errno::') ? 'system_error' : 'standard_error'
-        res = {
-          label => {
-            'message' => err.message,
-            'error_class' => err.class.to_s,
-            'backtrace' => err.backtrace,
-            'cause' => err.cause,
-            'full_message' => err.full_message,
-          }
-        }
-        res[label]['errno'] = err.errno if label == 'system_error'
-        new_run_result(res)
-      end
-
-      def result_errors?
-        run_result.keys.any? { |k| k.end_with?('error') }
-      end
-
-      def result_failures?
-        if run_result['summary']
-          run_result['summary']['failure_count'].positive? ||
-            run_result['summary']['errors_outside_of_examples_count'].positive?
-        else
-          true
-        end
-      end
-
-      private
-
-      def new_run_result(results)
-        @run_result = debug? ? with_debug_data(results) : results
-      end
-
-      def with_debug_data(results)
-        results.merge(
-          {
-            'debug' => {
-              'node' => @node,
-              'config' => @config,
-              'test_data' => @test_data,
-            }
-          }
-        )
-      end
-    end
-  end
-end