cem_acpt 0.3.3-universal-java-17 → 0.3.5-universal-java-17

data/lib/cem_acpt/test_runner/runner.rb

@@ -2,9 +2,11 @@
 
 require 'concurrent-ruby'
 require 'English'
-require_relative '../logging'
 require_relative '../rspec_utils'
+require_relative 'logging'
 require_relative 'runner_result'
+require_relative 'runner_workflow_builder'
+#require_relative 'workflow'
 
 module CemAcpt
   module TestRunner
@@ -28,7 +30,7 @@ module CemAcpt
     # reporting the results back to the main thread. Runner objects are created
     # by the RunHandler and then, when started, execute their logic in a thread.
     class Runner
-      include CemAcpt::LoggingAsync
+      include CemAcpt::TestRunner::Logging
 
       attr_reader :node, :node_exists, :run_result
 
@@ -42,34 +44,34 @@ module CemAcpt
         @debug_mode = @context.config.debug_mode?
         @node_inventory = @context.node_inventory
         @module_pkg_path = @context.module_package_path
+        @provision_attempts = 0
+        @provision_start_time = nil
         @node_exists = false
         @run_result = CemAcpt::TestRunner::RunnerResult.new(@node, debug: @debug_mode)
-        @completed_steps = []
+        @logger = use_logger(nil, stage: 'Runner', node: @node.node_name, test: @node.test_data[:test_name])
         validate!
       end
 
-      def run_step(step_sym)
-        send(step_sym)
-        @completed_steps << step_sym
-      rescue StandardError => e
-        err = CemAcpt::TestRunner::RunnerStepError.new(step_sym, e)
-        step_error_logging(err)
-        @run_result.from_error(err)
-        destroy unless step_sym == :destroy
-      end
-
       # Executes test suite steps
       def start
-        async_info("Starting test suite for #{@node.node_name}", log_prefix('RUNNER'))
-        run_step(:provision)
-        run_step(:bootstrap)
-        run_step(:run_tests)
-        run_step(:destroy)
-        true
+        @logger.info('Starting test suite workflow...')
+        @workflow = new_workflow
+        @workflow.run
+        if @workflow.success?
+          @run_result = @workflow.last_result
+          @workflow.completed_steps.each do |s|
+            @logger.info("Step '#{s.name}' completed successfully")
+          end
+          true
+        else
+          @run_result = @workflow.last_error
+          step_error_logging(@workflow.last_error)
+          false
+        end
       rescue StandardError => e
-        step_error_logging(e)
+        step_error_logging(e, :fatal)
+        @run_result = CemAcpt::TestRunner::RunnerResult.new(@node, debug: @debug_mode)
         @run_result.from_error(e)
-        destroy
       end
 
       # Checks for failures in the test results.
@@ -81,104 +83,37 @@ module CemAcpt
 
       private
 
-      def step_error_logging(err)
-        prefix = err.respond_to?(:step) ? log_prefix(err.step.capitalize) : log_prefix('RUNNER')
-        fatal_msg = ["runner failed: #{err.message}"]
-        async_fatal(fatal_msg, prefix)
-        async_debug("Completed steps: #{@completed_steps}", prefix)
-        async_debug("Failed runner backtrace:\n#{err.backtrace.join("\n")}", prefix)
-        async_debug("Failed runner test data: #{@node.test_data}", prefix)
+      # Builds a new workflow for the runner
+      # @return [CemAcpt::TestRunner::Workflow::Manager] the new workflow
+      def new_workflow
+        builder = RunnerWorkflowBuilder.new(@node, @context.config, @logger)
+        builder.add_provision
+        builder.add_sleep(time: 30)
+        builder.add_wait_for_node_ssh
+        builder.add_check_dnf_automatic
+        builder.add_check_rpm_db
+        builder.add_save_node_to_inventory(node_inventory: @node_inventory, platform: @platform)
+        builder.add_check_for_module_package_path(module_pkg_path: @module_pkg_path)
+        builder.add_install_module(module_pkg_path: @module_pkg_path)
+        builder.add_check_node_inventory_file(node_inventory: @node_inventory)
+        builder.add_run_tests(rspec_opts: rspec_opts, rspec_cmd: CemAcpt::RSpecUtils::Command, run_result: @run_result)
+        builder.add_clean_up
+        builder.workflow
       end
 
-      def log_prefix(prefix)
-        "#{prefix}: #{@node.test_data[:test_name]}:"
+      def step_error_logging(err, kind = :error)
+        msg = if err.respond_to?(:step)
+                "runner failed on step '#{err.step}': #{err.message}"
+              else
+                "runner failed: #{err.message}"
+              end
+        @logger.send(kind, msg)
+        @logger.debug("failed runner backtrace:\n#{err.backtrace.join("\n")}")
+        @logger.debug("failed runner test data: #{@node.test_data}")
       end
 
-      # Provisions the node for the acceptance test suite.
-      def provision
-        async_info("Provisioning #{@node.node_name}...", log_prefix('PROVISION'))
-        start_time = Time.now
-        @node.provision
-        @node_exists = true
-        max_retries = 60 # equals 300 seconds because we check every five seconds
-        until @node.ready?
-          if max_retries <= 0
-            async_fatal("Node #{@node.node_name} failed to provision", log_prefix('PROVISION'))
-            raise CemAcpt::TestRunner::RunnerProvisionError, "Provisioning timed out for node #{@node.node_name}"
-          end
-
-          async_info("Waiting for #{@node.node_name} to be ready for remote connections...", log_prefix('PROVISION'))
-          max_retries -= 1
-          sleep(5)
-        end
-        async_info("Node #{@node.node_name} is ready...", log_prefix('PROVISION'))
-        node_desc = {
-          test_data: @node.test_data,
-          platform: @platform,
-          local_port: @node.local_port,
-        }.merge(@node.node)
-        @node_inventory.add(@node.node_name, node_desc)
-        @node_inventory.save
-        async_info("Node #{@node.node_name} provisioned in #{Time.now - start_time} seconds", log_prefix('PROVISION'))
-      end
-
-      # Bootstraps the node for the acceptance test suite. Currently, this
-      # just uploads and installs the module package.
-      def bootstrap
-        async_info("Bootstrapping #{@node.node_name}...", log_prefix('BOOTSTRAP'))
-        until File.exist?(@module_pkg_path)
-          async_debug("Waiting for module package #{@module_pkg_path} to exist...", log_prefix('BOOTSTRAP'))
-          sleep(1)
-        end
-        async_info("Installing module package #{@module_pkg_path}...", log_prefix('BOOTSTRAP'))
-        @node.install_puppet_module_package(@module_pkg_path)
-      end
-
-      # Runs the acceptance test suite via rspec.
-      def run_tests
-        attempts = 0
-        until File.exist?(@node_inventory.save_file_path)
-          raise 'Node inventory file not found' if (attempts += 1) > 3
-
-          sleep(1)
-        end
-        async_info("Running test #{@node.test_data[:test_name]} on node #{@node.node_name}...", log_prefix('RSPEC'))
-        @node.run_tests do |cmd_env|
-          cmd_opts = rspec_opts
-          cmd_opts.env = cmd_opts.env.merge(cmd_env) if cmd_env
-          # Documentation format gets logged in real time, JSON file is read after the fact
-          begin
-            @rspec_cmd = CemAcpt::RSpecUtils::Command.new(cmd_opts)
-            @rspec_cmd.execute(pty: false, log_prefix: log_prefix('RSPEC'))
-            @run_result.from_json_file(cmd_opts.format[:json])
-          rescue Errno::EIO => e
-            async_error("failed to run rspec: #{@node.test_data[:test_name]}: #{$ERROR_INFO}", log_prefix('RSPEC'))
-            @run_result.from_error(e)
-          rescue StandardError => e
-            async_error("failed to run rspec: #{@node.test_data[:test_name]}: #{e.message}", log_prefix('RSPEC'))
-            async_debug("Backtrace:\n#{e.backtrace}", log_prefix('RSPEC'))
-            @run_result.from_error(e)
-          end
-        end
-        async_info("Tests completed with exit code: #{@run_result.exit_status}", log_prefix('RSPEC'))
-      end
-
-      # Destroys the node for the acceptance test suite.
-      def destroy
-        kill_spec_pty_if_exists
-        if @context.config.get('no_destroy_nodes')
-          async_info("Not destroying node #{@node.node_name} because 'no_destroy_nodes' is set to true",
-                     log_prefix('DESTROY'))
-        else
-          async_info("Destroying #{@node.node_name}...", log_prefix('DESTROY'))
-          @node.destroy
-          @node_exists = false
-          async_info("Node #{@node.node_name} destroyed successfully", log_prefix('DESTROY'))
-        end
-      end
-
-      def kill_spec_pty_if_exists
-        @rspec_cmd&.kill_pty
+      def log_prefix(prefix)
+        "#{prefix}: #{@node.test_data[:test_name]}:"
       end
 
       # Validates the runner configuration.

data/lib/cem_acpt/test_runner/runner_workflow_builder.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+require_relative 'logging'
+require_relative 'workflow'
+
+module CemAcpt
+  module TestRunner
+    # RunnerWorkflowBuilder builds a workflow for a TestRunner
+    # @!attribute [r] workflow
+    #   @return [CemAcpt::TestRunner::Workflow::Manager] Workflow object
+    class RunnerWorkflowBuilder
+      include CemAcpt::TestRunner::Logging
+
+      attr_reader :workflow
+
+      # @param node [CemAcpt::Platform::Base] Initialized node object
+      # @param config [Hash] Context config hash
+      # @param logger [CemAcpt::TestRunner::Logging::Logger] Logger object
+      def initialize(node, config, logger = nil)
+        @node = node
+        @logger = use_logger(logger, stage: 'Workflow')
+        @workflow = CemAcpt::TestRunner::Workflow::Manager.new(workflow_manager_opts(config, logger))
+        @config = config
+      end
+
+      def add_sleep(**kwargs)
+        opts = {
+          node: @node,
+          time: kwargs[:time] || 10,
+          retryable: false,
+        }
+        @workflow.add_step(:sleep, **opts) do |s|
+          s.logger.info("sleeping for #{s.opts[:time]} seconds")
+          sleep(s.opts[:time])
+        end
+      end
+
+      def add_provision(**kwargs)
+        opts = {
+          node: @node,
+          retryable: kwargs[:retryable] || false,
+        }
+        @workflow.add_step(:provision, **opts) do |s|
+          s.opts[:node].provision
+          s.opts[:node]
+        end
+      end
+
+      def add_wait_for_node_ssh(**kwargs)
+        opts = {
+          node: @node,
+          retryable: kwargs[:retryable] || true,
+          retry_delay: kwargs[:retry_delay] || 30,
+          retry_max: kwargs[:retry_max] || 10,
+        }
+        @workflow.add_step(:wait_for_node_ssh, **opts) do |s|
+          unless s.opts[:node].ready?
+            raise "wait_for_node_ssh timed out for node #{s.opts[:node].node_name}"
+          end
+          s.opts[:node]
+        end
+      end
+
+      def add_check_dnf_automatic(**kwargs)
+        opts = {
+          node: @node,
+          retryable: kwargs[:retryable] || true,
+          retry_delay: kwargs[:retry_delay] || 30,
+          retry_workflow_on_fail: kwargs[:retry_workflow_on_fail] || true,
+        }
+        @workflow.add_step(:check_dnf_automatic, **opts) do |s|
+          unless s.opts[:node].dnf_automatic_success?
+            raise "dnf_automatic failed on node #{s.opts[:node].node_name}"
+          end
+          s.opts[:node]
+        end
+      end
+
+      def add_check_rpm_db(**kwargs)
+        opts = {
+          node: @node,
+          retryable: kwargs[:retryable] || true,
+          retry_delay: kwargs[:retry_delay] || 30,
+          retry_workflow_on_fail: kwargs[:retry_workflow_on_fail] || true,
+        }
+        @workflow.add_step(:rpm_db_check, **opts) do |s|
+          unless s.opts[:node].rpm_db_check_success?
+            raise "rpm_db_check failed on node #{s.opts[:node].node_name}"
+          end
+          s.opts[:node]
+        end
+      end
+
+      def add_save_node_to_inventory(node_inventory:, platform:, **kwargs)
+        opts = {
+          node: @node,
+          platform: platform,
+          node_inventory: node_inventory,
+          retryable: kwargs[:retryable] || false,
+        }
+        @workflow.add_step(:save_node_to_inventory, **opts) do |s|
+          node_desc = {
+            test_data: s.opts[:node].test_data,
+            platform: s.opts[:platform],
+            local_port: s.opts[:node].local_port,
+          }.merge(s.opts[:node].node)
+          s.opts[:node_inventory].add(s.opts[:node].node_name, node_desc)
+          s.opts[:node_inventory].save
+          s.logger.info("node #{s.opts[:node_name]} saved to inventory")
+          s.opts[:node_inventory]
+        end
+      end
+
+      def add_check_for_module_package_path(module_pkg_path:, **kwargs)
+        opts = {
+          module_pkg_path: module_pkg_path,
+          retryable: kwargs[:retryable] || true,
+          retry_delay: kwargs[:retry_delay] || 30,
+        }
+        @workflow.add_step(:check_for_module_package_path, **opts) do |s|
+          unless File.exist?(s.opts[:module_pkg_path])
+            raise "module package #{s.opts[:module_pkg_path]} does not exist"
+          end
+          s.opts[:module_pkg_path]
+        end
+      end
+
+      def add_install_module(module_pkg_path:, **kwargs)
+        opts = {
+          node: @node,
+          module_pkg_path: module_pkg_path,
+          retryable: kwargs[:retryable] || true,
+          retry_delay: kwargs[:retry_delay] || 60,
+        }
+        @workflow.add_step(:bootstrap, **opts) do |s|
+          s.opts[:node].install_puppet_module_package(s.opts[:module_pkg_path])
+          s.opts[:node]
+        end
+      end
+
+      def add_check_node_inventory_file(node_inventory:, **kwargs)
+        opts = {
+          ni_file_path: node_inventory.save_file_path,
+          retryable: kwargs[:retryable] || true,
+          retry_max: kwargs[:retry_max] || 60,
+          retry_delay: kwargs[:retry_delay] || 5,
+        }
+        @workflow.add_step(:check_node_inventory_file, **opts) do |s|
+          unless File.exist?(s.opts[:ni_file_path])
+            raise "node inventory file #{s.opts[:ni_file_path]} not found"
+          end
+          s.opts[:ni_file_path]
+        end
+      end
+
+      def add_run_tests(rspec_opts:, rspec_cmd:, run_result:, **kwargs)
+        opts = {
+          node: @node,
+          rspec_opts: rspec_opts,
+          rspec_cmd: rspec_cmd,
+          run_result: run_result,
+          retryable: kwargs[:retryable] || false,
+        }
+        @workflow.add_step(:run_tests, **opts) do |s|
+          s.opts[:node].run_tests do |cmd_env|
+            cmd_opts = s.opts[:rspec_opts].dup
+            cmd_opts.env = cmd_opts.env.merge(cmd_env) if cmd_env
+            rspec_cmd = s.opts[:rspec_cmd].new(cmd_opts)
+            run_result = s.opts[:run_result].dup
+            begin
+              rspec_cmd.execute(pty: false, log_prefix: "RSPEC: #{@node.test_data[:test_name]}")
+              run_result.from_json_file(cmd_opts.format[:json])
+            rescue Errno::EIO => e
+              s.logger.error("failed to run rspec: #{@node.test_data[:test_name]}: #{$ERROR_INFO}")
+              run_result.from_error(e)
+            rescue StandardError => e
+              s.logger.error("failed to run rspec: #{@node.test_data[:test_name]}: #{e.message}")
+              s.logger.debug(e.backtrace.join("\n"))
+              run_result.from_error(e)
+            end
+          end
+          run_result
+        end
+      end
+
+      def add_clean_up(force: false, **kwargs)
+        opts = {
+          node: @node,
+          config: @config,
+          force: force,
+          retryable: kwargs[:retryable] || false,
+        }
+        @workflow.add_step(:clean_up, **opts) do |s|
+          if !force && s.opts[:config].get('no_destroy_nodes')
+            s.logger.info("not destroying node #{s.opts[:node].node_name} because 'no_destroy_nodes' is set to true")
+          else
+            s.logger.info("destroying node #{s.opts[:node].node_name}")
+            s.opts[:node].destroy
+            s.logger.info("node #{s.opts[:node].node_name} destroyed")
+          end
+        end
+      end
+
+      private
+
+      def workflow_manager_opts(config, logger = nil)
+        {
+          retry_max: config.get('workflow.retry_max') || 3,
+          retry_delay: config.get('workflow.retry_delay') || 0,
+          ignore_failures: config.get('workflow.ignore_failures') || false,
+          raise_on_fail: config.get('workflow.raise_on_fail') || true,
+          logger: logger,
+        }
+      end
+    end
+  end
+end

data/lib/cem_acpt/test_runner/workflow/manager.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+require_relative '../logging'
+require_relative 'step'
+
+module CemAcpt
+  module TestRunner
+    module Workflow
+      # Manager is a class that manages how steps in a workflow are executed.
+      # @!attribute [r] completed_steps
+      #   @return [Array<Step>] The steps that have been completed
+      # @!attribute [r] last_error
+      #   @return [Exception, nil] The last error that occurred, if any
+      # @!attribute [r] last_result
+      #   @return [Any] The last result that occurred, if any
+      # @!attribute [r] retry_max
+      #   @return [Integer] The maximum number of workflow runs to attempt before failing. Default: 3
+      # @!attribute [r] retry_delay
+      #   @return [Integer] The number of seconds to wait between workflow runs. Default: 0
+      # @!attribute [r] steps
+      #   @return [Array<Step>] The steps that are part of this workflow
+      class Manager
+        include CemAcpt::TestRunner::Logging
+
+        attr_reader :completed_steps, :last_error, :last_result, :retry_max, :retry_delay, :steps
+
+        # @param [Hash] opts The options to create a new workflow manager with
+        # @option opts [Integer] :retry_max The maximum number of workflow runs to attempt before failing. Default: 3
+        # @option opts [Integer] :retry_delay The number of seconds to wait between workflow runs. Default: 0
+        # @option opts [Boolean] :ignore_failures Whether to ignore Step failures and continue to the next step. Default: false
+        # @option opts [Boolean] :raise_on_fail Whether to raise an exception when a Step fails. Default: true
+        def initialize(**opts)
+          @retry_max = opts[:retry_max] || 3
+          @retry_delay = opts[:retry_delay] || 0
+          @ignore_failures = opts[:ignore_failures] || false
+          @raise_on_fail = opts[:raise_on_fail] || true
+          @logger = use_logger(opts[:logger], stage: 'Workflow')
+          @steps = []
+          @workflow_runs = 0
+          @last_error = nil
+          @last_result = nil
+          @completed_steps = []
+        end
+
+        # Add a step to the workflow. Steps can be named anything, but if the step name is :clean_up,
+        # it will be run at the end of the workflow, regardless of the order it was added, and it's
+        # output will not be saved as the last_result. Additionally, the :clean_up step will be run
+        # even if the workflow fails.
+        # @param [Symbol] name The name of the step
+        # @param [Hash] kwargs The keyword arguments to pass to the step
+        # @param [Proc] block The block to pass to the step
+        # @yieldparam [Step] step The step that was added. This is passed to the block.
+        def add_step(name, **kwargs, &block)
+          raise ArgumentError, 'name must be a Symbol' unless name.is_a?(Symbol)
+
+          step_name = if @steps.any? { |step| step.name == name }
+                        "#{name}_#{@steps.length}".to_sym
+                      else
+                        name
+                      end
+          step = Step.new(step_name, **kwargs.merge(logger: @logger), &block)
+          @steps << StepState.new(step, @steps.length, **kwargs.merge(logger: @logger))
+        end
+
+        # Run the workflow
+        def run
+          @workflow_runs += 1
+          @completed_steps = []
+          @steps.each do |step|
+            next if step.name == :clean_up
+
+            result = step.run
+            handle_result(step, result)
+          end
+        ensure
+          clean_up
+        end
+
+        # Whether the workflow has completed successfully
+        def success?
+          (@completed_steps.length == @steps.length) && @completed_steps.none? { |step| step.failed? }
+        end
+
+        # @return [Boolean] Whether to ignore failures and continue to the next step
+        def ignore_failures?
+          @ignore_failures
+        end
+
+        # @return [Boolean] Whether to raise an exception when a step fails
+        def raise_on_fail?
+          @raise_on_fail
+        end
+
+        private
+
+        def new_logger(logger)
+          if logger.nil?
+            logger
+          else
+            @logger = logger.dup
+            @logger.add_prefix_parts(stage: 'Workflow')
+          end
+        end
+
+        def log(level, msg)
+          if @logger
+            @logger.send(level, msg)
+          else
+            send("async_#{level}".to_sym, msg)
+          end
+        end
+
+        def log_debug(msg)
+          log(:debug, msg)
+        end
+
+        def log_info(msg)
+          log(:info, msg)
+        end
+
+        def log_warn(msg)
+          log(:warn, msg)
+        end
+
+        def log_error(msg)
+          log(:error, msg)
+        end
+
+        def log_fatal(msg)
+          log(:fatal, msg)
+        end
+
+        # Handles the result of a step
+        # @param [Step] step The step that was run
+        # @param [Object] result The result of the step
+        # @raise [StandardError] If the step failed and the workflow is not configured to ignore failures
+        def handle_result(step, result)
+          @last_result = result unless step.name == :clean_up # Don't overwrite the last result with the clean_up step
+          case result
+          when :retry_workflow
+            log_warn("step '#{step.name}' failed and requested a workflow retry")
+            log_debug("step '#{step.name}' failed with error: #{step.last_error.message}")
+            log_debug("step '#{step.name}' failed with error: #{step.last_error.backtrace.join("\n")}")
+            @last_error = step.last_error
+            retry_workflow
+          when :fail
+            log_warn("step '#{step.name}' failed")
+            log_debug(step.last_error.message)
+            log_debug(step.last_error.backtrace.join("\n"))
+            @last_error = step.last_error
+            if ignore_failures?
+              log_warn("ignoring failure of step '#{step.name}'")
+              @completed_steps << step
+            else
+              log_error("failed with error: #{@last_error.message}")
+              raise @last_error
+            end
+          else
+            log_info("step '#{step.name}' succeeded")
+            log_debug("step '#{step.name}' returned: #{result}")
+            @completed_steps << step
+          end
+        end
+
+        # Retries the workflow if it is retryable
+        # @raise [StandardError] If the workflow is not retryable or has exceeded the maximum number of retries
+        def retry_workflow
+          if @workflow_runs < @retry_max
+            log_info("Retrying workflow (attempt #{@workflow_runs + 1} of #{@retry_max})")
+            sleep @retry_delay if @retry_delay > 0
+            clean_up
+            run
+          else
+            log_fatal('Workflow is not retryable or has exceeded the maximum number of retries')
+            raise @last_error
+          end
+        end
+
+        # Adds a default clean_up step if one is not defined
+        # @return [Step] The default clean_up step
+        def default_clean_up
+          add_step(:clean_up) do
+            log_info('No clean_up step defined, skipping')
+            true
+          end
+          @steps.last
+        end
+
+        # Runs the clean_up step
+        def clean_up
+          cleanup_step = @steps.find { |step| step.name == :clean_up } || default_clean_up
+          result = cleanup_step.run
+          handle_result(cleanup_step, result)
+        end
+      end
+    end
+  end
+end