cem_acpt 0.3.2-universal-java-17 → 0.3.4-universal-java-17

data/lib/cem_acpt/test_runner/runner_workflow_builder.rb

@@ -0,0 +1,238 @@
+# frozen_string_literal: true
+
+require_relative '../logging'
+require_relative 'workflow'
+
+module CemAcpt
+  module TestRunner
+    # RunnerWorkflowBuilder builds a workflow for a TestRunner
+    # @!attribute [r] workflow
+    #   @return [CemAcpt::TestRunner::Workflow::Manager] Workflow object
+    class RunnerWorkflowBuilder
+      include CemAcpt::LoggingAsync
+
+      attr_reader :workflow
+
+      # @param node [CemAcpt::Platform::Base] Initialized node object
+      # @param config [Hash] Context config hash
+      def initialize(node, config)
+        @node = node
+        @workflow = CemAcpt::TestRunner::Workflow::Manager.new(workflow_manager_opts(config))
+        @config = config
+      end
+
+      def add_sleep(**kwargs)
+        opts = {
+          node: @node,
+          time: kwargs[:time] || 10,
+          retryable: false,
+        }
+        @workflow.add_step(:sleep, **opts) do |s|
+          log_info("sleeping for #{s.opts[:time]} seconds", s)
+          sleep(s.opts[:time])
+        end
+      end
+
+      def add_provision(**kwargs)
+        opts = {
+          node: @node,
+          retryable: kwargs[:retryable] || false,
+        }
+        @workflow.add_step(:provision, **opts) do |s|
+          s.opts[:node].provision
+          s.opts[:node]
+        end
+      end
+
+      def add_wait_for_node_ssh(**kwargs)
+        opts = {
+          node: @node,
+          retryable: kwargs[:retryable] || true,
+          retry_delay: kwargs[:retry_delay] || 30,
+          retry_max: kwargs[:retry_max] || 10,
+        }
+        @workflow.add_step(:wait_for_node_ssh, **opts) do |s|
+          unless s.opts[:node].ready?
+            raise "wait_for_node_ssh timed out for node #{s.opts[:node].node_name}"
+          end
+          s.opts[:node]
+        end
+      end
+
+      def add_check_dnf_automatic(**kwargs)
+        opts = {
+          node: @node,
+          retryable: kwargs[:retryable] || true,
+          retry_delay: kwargs[:retry_delay] || 30,
+          retry_workflow_on_fail: kwargs[:retry_workflow_on_fail] || true,
+        }
+        @workflow.add_step(:check_dnf_automatic, **opts) do |s|
+          unless s.opts[:node].dnf_automatic_success?
+            raise "dnf_automatic failed on node #{s.opts[:node].node_name}"
+          end
+          s.opts[:node]
+        end
+      end
+
+      def add_check_rpm_db(**kwargs)
+        opts = {
+          node: @node,
+          retryable: kwargs[:retryable] || true,
+          retry_delay: kwargs[:retry_delay] || 30,
+          retry_workflow_on_fail: kwargs[:retry_workflow_on_fail] || true,
+        }
+        @workflow.add_step(:rpm_db_check, **opts) do |s|
+          unless s.opts[:node].rpm_db_check_success?
+            raise "rpm_db_check failed on node #{s.opts[:node].node_name}"
+          end
+          s.opts[:node]
+        end
+      end
+
+      def add_save_node_to_inventory(node_inventory:, platform:, **kwargs)
+        opts = {
+          node: @node,
+          platform: platform,
+          node_inventory: node_inventory,
+          retryable: kwargs[:retryable] || false,
+        }
+        @workflow.add_step(:save_node_to_inventory, **opts) do |s|
+          node_desc = {
+            test_data: s.opts[:node].test_data,
+            platform: s.opts[:platform],
+            local_port: s.opts[:node].local_port,
+          }.merge(s.opts[:node].node)
+          s.opts[:node_inventory].add(s.opts[:node].node_name, node_desc)
+          s.opts[:node_inventory].save
+          log_info("node #{s.opts[:node_name]} saved to inventory", s)
+          s.opts[:node_inventory]
+        end
+      end
+
+      def add_check_for_module_package_path(module_pkg_path:, **kwargs)
+        opts = {
+          module_pkg_path: module_pkg_path,
+          retryable: kwargs[:retryable] || true,
+          retry_delay: kwargs[:retry_delay] || 30,
+        }
+        @workflow.add_step(:check_for_module_package_path, **opts) do |s|
+          unless File.exist?(s.opts[:module_pkg_path])
+            raise "module package #{s.opts[:module_pkg_path]} does not exist"
+          end
+          s.opts[:module_pkg_path]
+        end
+      end
+
+      def add_install_module(module_pkg_path:, **kwargs)
+        opts = {
+          node: @node,
+          module_pkg_path: module_pkg_path,
+          retryable: kwargs[:retryable] || true,
+          retry_delay: kwargs[:retry_delay] || 60,
+        }
+        @workflow.add_step(:bootstrap, **opts) do |s|
+          s.opts[:node].install_puppet_module_package(s.opts[:module_pkg_path])
+          s.opts[:node]
+        end
+      end
+
+      def add_check_node_inventory_file(node_inventory:, **kwargs)
+        opts = {
+          ni_file_path: node_inventory.save_file_path,
+          retryable: kwargs[:retryable] || true,
+          retry_max: kwargs[:retry_max] || 60,
+          retry_delay: kwargs[:retry_delay] || 5,
+        }
+        @workflow.add_step(:check_node_inventory_file, **opts) do |s|
+          unless File.exist?(s.opts[:ni_file_path])
+            raise "node inventory file #{s.opts[:ni_file_path]} not found"
+          end
+          s.opts[:ni_file_path]
+        end
+      end
+
+      def add_run_tests(rspec_opts:, rspec_cmd:, run_result:, **kwargs)
+        opts = {
+          node: @node,
+          rspec_opts: rspec_opts,
+          rspec_cmd: rspec_cmd,
+          run_result: run_result,
+          retryable: kwargs[:retryable] || false,
+        }
+        @workflow.add_step(:run_tests, **opts) do |s|
+          s.opts[:node].run_tests do |cmd_env|
+            cmd_opts = s.opts[:rspec_opts].dup
+            cmd_opts.env = cmd_opts.env.merge(cmd_env) if cmd_env
+            rspec_cmd = s.opts[:rspec_cmd].new(cmd_opts)
+            run_result = s.opts[:run_result].dup
+            begin
+              rspec_cmd.execute(pty: false, log_prefix: "RSPEC: #{@node.test_data[:test_name]}")
+              run_result.from_json_file(cmd_opts.format[:json])
+            rescue Errno::EIO => e
+              log_error("failed to run rspec: #{@node.test_data[:test_name]}: #{$ERROR_INFO}", s)
+              run_result.from_error(e)
+            rescue StandardError => e
+              log_error("failed to run rspec: #{@node.test_data[:test_name]}: #{e.message}", s)
+              log_debug(e.backtrace.join("\n"), s)
+              run_result.from_error(e)
+            end
+          end
+          run_result
+        end
+      end
+
+      def add_clean_up(force: false, **kwargs)
+        opts = {
+          node: @node,
+          config: @config[:config],
+          force: force,
+          retryable: kwargs[:retryable] || false,
+        }
+        @workflow.add_step(:clean_up, **opts) do |s|
+          if !force && s.opts[:config].get('no_destroy_nodes')
+            log_info("not destroying node #{s.opts[:node].node_name} because 'no_destroy_nodes' is set to true", s)
+          else
+            log_info("destroying node #{s.opts[:node].node_name}", s)
+            s.opts[:node].destroy
+            log_info("node #{s.opts[:node].node_name} destroyed", s)
+          end
+        end
+      end
+
+      private
+
+      def log_msg(msg, step)
+        "Step '#{step.name}' on #{@node.node_name}: #{msg}"
+      end
+
+      def log_debug(msg, step)
+        async_debug(log_msg(msg, step))
+      end
+
+      def log_info(msg, step)
+        async_info(log_msg(msg, step))
+      end
+
+      def log_warn(msg, step)
+        async_warn(log_msg(msg, step))
+      end
+
+      def log_error(msg, step)
+        async_error(log_msg(msg, step))
+      end
+
+      def log_fatal(msg, step)
+        async_fatal(log_msg(msg, step))
+      end
+
+      def workflow_manager_opts(config)
+        {
+          retry_max: config[:workflow_retry_max] || 3,
+          retry_delay: config[:workflow_retry_delay] || 0,
+          ignore_failures: config[:workflow_ignore_failures] || false,
+          raise_on_fail: config[:workflow_raise_on_fail] || true,
+        }
+      end
+    end
+  end
+end

data/lib/cem_acpt/test_runner/workflow/manager.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+require_relative '../../logging'
+require_relative 'step'
+
+module CemAcpt
+  module TestRunner
+    module Workflow
+      # Manager is a class that manages how steps in a workflow are executed.
+      # @!attribute [r] completed_steps
+      #   @return [Array<Step>] The steps that have been completed
+      # @!attribute [r] last_error
+      #   @return [Exception, nil] The last error that occurred, if any
+      # @!attribute [r] last_result
+      #   @return [Any] The last result that occurred, if any
+      # @!attribute [r] retry_max
+      #   @return [Integer] The maximum number of workflow runs to attempt before failing. Default: 3
+      # @!attribute [r] retry_delay
+      #   @return [Integer] The number of seconds to wait between workflow runs. Default: 0
+      # @!attribute [r] steps
+      #   @return [Array<Step>] The steps that are part of this workflow
+      class Manager
+        include CemAcpt::LoggingAsync
+
+        attr_reader :completed_steps, :last_error, :last_result, :retry_max, :retry_delay, :steps
+
+        # @param [Hash] opts The options to create a new workflow manager with
+        # @option opts [Integer] :retry_max The maximum number of workflow runs to attempt before failing. Default: 3
+        # @option opts [Integer] :retry_delay The number of seconds to wait between workflow runs. Default: 0
+        # @option opts [Boolean] :ignore_failures Whether to ignore Step failures and continue to the next step. Default: false
+        # @option opts [Boolean] :raise_on_fail Whether to raise an exception when a Step fails. Default: true
+        def initialize(**opts)
+          @retry_max = opts[:retry_max] || 3
+          @retry_delay = opts[:retry_delay] || 0
+          @ignore_failures = opts[:ignore_failures] || false
+          @raise_on_fail = opts[:raise_on_fail] || true
+          @steps = []
+          @workflow_runs = 0
+          @last_error = nil
+          @last_result = nil
+          @completed_steps = []
+        end
+
+        # Add a step to the workflow. Steps can be named anything, but if the step name is :clean_up,
+        # it will be run at the end of the workflow, regardless of the order it was added, and it's
+        # output will not be saved as the last_result. Additionally, the :clean_up step will be run
+        # even if the workflow fails.
+        # @param [Symbol] name The name of the step
+        # @param [Hash] kwargs The keyword arguments to pass to the step
+        # @param [Proc] block The block to pass to the step
+        # @yieldparam [Step] step The step that was added. This is passed to the block.
+        def add_step(name, **kwargs, &block)
+          raise ArgumentError, 'name must be a Symbol' unless name.is_a?(Symbol)
+
+          step_name = if @steps.any? { |step| step.name == name }
+                        "#{name}_#{@steps.length}".to_sym
+                      else
+                        name
+                      end
+          step = Step.new(step_name, **kwargs, &block)
+          @steps << StepState.new(step, @steps.length, **kwargs)
+        end
+
+        # Run the workflow
+        def run
+          @workflow_runs += 1
+          @completed_steps = []
+          @steps.each do |step|
+            next if step.name == :clean_up
+
+            result = step.run
+            handle_result(step, result)
+          end
+        ensure
+          clean_up
+        end
+
+        # Whether the workflow has completed successfully
+        def success?
+          (@completed_steps.length == @steps.length) && @completed_steps.none? { |step| step.failed? }
+        end
+
+        # @return [Boolean] Whether to ignore failures and continue to the next step
+        def ignore_failures?
+          @ignore_failures
+        end
+
+        # @return [Boolean] Whether to raise an exception when a step fails
+        def raise_on_fail?
+          @raise_on_fail
+        end
+
+        private
+
+        # Handles the result of a step
+        # @param [Step] step The step that was run
+        # @param [Object] result The result of the step
+        # @raise [StandardError] If the step failed and the workflow is not configured to ignore failures
+        def handle_result(step, result)
+          @last_result = result unless step.name == :clean_up # Don't overwrite the last result with the clean_up step
+          case result
+          when :retry_workflow
+            async_warn("Step '#{step.name}' failed and requested a workflow retry")
+            async_debug("Step '#{step.name}' failed with error: #{step.last_error.message}")
+            async_debug("Step '#{step.name}' failed with error: #{step.last_error.backtrace.join("\n")}")
+            @last_error = step.last_error
+            retry_workflow
+          when :fail
+            async_warn("Step '#{step.name}' failed")
+            async_debug(step.last_error.message)
+            async_debug(step.last_error.backtrace.join("\n"))
+            @last_error = step.last_error
+            if ignore_failures?
+              async_warn("Ignoring failure of step '#{step.name}'")
+              @completed_steps << step
+            else
+              async_error("Workflow failed with error: #{@last_error.message}")
+              raise @last_error
+            end
+          else
+            async_info("Step '#{step.name}' succeeded")
+            async_debug("Step '#{step.name}' returned: #{result}")
+            @completed_steps << step
+          end
+        end
+
+        # Retries the workflow if it is retryable
+        # @raise [StandardError] If the workflow is not retryable or has exceeded the maximum number of retries
+        def retry_workflow
+          if @workflow_runs < @retry_max
+            async_info("Retrying workflow (attempt #{@workflow_runs + 1} of #{@retry_max})")
+            sleep @retry_delay if @retry_delay > 0
+            clean_up
+            run
+          else
+            async_fatal('Workflow is not retryable or has exceeded the maximum number of retries')
+            raise @last_error
+          end
+        end
+
+        # Adds a default clean_up step if one is not defined
+        # @return [Step] The default clean_up step
+        def default_clean_up
+          add_step(:clean_up) do
+            async_info('No clean_up step defined, skipping')
+            true
+          end
+          @steps.last
+        end
+
+        # Runs the clean_up step
+        def clean_up
+          cleanup_step = @steps.find { |step| step.name == :clean_up } || default_clean_up
+          result = cleanup_step.run
+          handle_result(cleanup_step, result)
+        end
+      end
+    end
+  end
+end

data/lib/cem_acpt/test_runner/workflow/step.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+require_relative '../../logging'
+
+module CemAcpt
+  module TestRunner
+    module Workflow
+      # Error used to wrap fatal errors raised in Runner steps
+      # @!attribute [r] step
+      #   @return [Step] The step that raised the error
+      class StepError < StandardError
+        attr_reader :step
+
+        def initialize(step, err)
+          @step = step
+          @original_error = err
+          super err
+          set_backtrace err.backtrace if err.respond_to?(:backtrace)
+        end
+      end
+
+      # Step is a class that defines a single step in a Workflow.
+      # Step objects are created by the Workflow class and are not intended to be created directly.
+      # @!attribute [r] name
+      #   @return [Symbol] The name of the step
+      # @!attribute [r] opts
+      #   @return [Hash] The options passed to the step
+      # @!attribute [r] result
+      #   @return [Object] The result of the step
+      class Step
+        include CemAcpt::LoggingAsync
+
+        attr_reader :name, :opts, :result
+
+        # @param name [Symbol] The name of the step
+        # @param opts [Hash] The options passed to the step
+        # @param block [Proc] The block to execute when the step is run
+        # @yieldparam step [Step] This step object is yielded to the block
+        def initialize(name, **opts, &block)
+          @name = name
+          @opts = opts
+          @block = block
+          @result = :not_run
+          @failed = false
+          @log_prefix = opts[:log_prefix] || "#{@name.upcase}:"
+        end
+
+        # @return [Boolean] True if the step has been run and failed
+        def failed?
+          @failed
+        end
+
+        # Run the step. This calls and executes the block passed to the constructor.
+        # @param log_prefix [String] The prefix to use when logging the step name
+        # @return [Object] The result of the step
+        def run(_log_prefix = 'Running step')
+          async_info(log_msg('Starting step'))
+          @result = @block.call(self)
+          async_debug(log_msg('SUCCESS'))
+          @result
+        rescue StandardError => e
+          async_debug(log_msg("FAILED: #{e.message}"))
+          @result = StepError.new(@name, e)
+          @failed = true
+          @result
+        ensure
+          Thread.pass # Be kind to the scheduler
+        end
+
+        private
+
+        def log_msg(msg)
+          [@log_prefix, msg].join(' ')
+        end
+      end
+
+      # StepState is a class that holds the state of a Step.
+      # StepState objects are created by the Workflow class and are not intended to be created directly.
+      # @!attribute [r] step
+      #   @return [Step] The step object
+      # @!attribute [r] position
+      #   @return [Integer] The position of the step in the workflow
+      # @!attribute [r] opts
+      #   @return [Hash] The options passed to the step
+      # @!attribute [r] run_count
+      #   @return [Integer] The number of times the step has been run
+      # @!attribute [r] results
+      #   @return [Array<Object>] The results of all runs of the step
+      # @!attribute [r] last_error
+      #   @return [StepError] The last error raised by the step
+      # @!attribute [r] last_result
+      #   @return [Object] The result of the last run of the step
+      class StepState
+        attr_reader :step, :position, :opts, :run_count, :results, :last_error, :last_result
+
+        # @param step [Step] The step object
+        # @param position [Integer] The position of the step in the workflow
+        # @param opts [Hash] The options passed to the step
+        def initialize(step, position, **opts)
+          @step = step
+          @position = position
+          @retryable = opts[:retryable] || false
+          @retry_max = opts[:retry_max] || 3
+          @retry_delay = opts[:retry_delay] || 0
+          @retry_workflow_on_fail = opts[:retry_workflow_on_fail] || false
+          @run_count = 0
+          @results = []
+          @last_error = nil
+          @last_result = nil
+        end
+
+        # Proxy any methods not defined in this class to the Step
+        def method_missing(method, *args, &block)
+          if @step.respond_to?(method)
+            @step.send(method, *args, &block)
+          else
+            super
+          end
+        end
+
+        # Proxy any methods not defined in this class to the Step
+        def respond_to_missing?(method, include_private = false)
+          @step.respond_to?(method) || super
+        end
+
+        # @return [Object] The result of the last run of the step
+        def result
+          @last_result || :not_run
+        end
+
+        # @return [Boolean] If the step is retryable
+        def retryable?
+          @retryable
+        end
+
+        # @return [Boolean] If the workflow should be retried if the step fails
+        def retry_workflow_on_fail?
+          @retry_workflow_on_fail
+        end
+
+        # @return [Boolean] True if the step has been run and failed
+        def failed?
+          @results.last.is_a?(StepError)
+        end
+
+        # Run the step. This wraps the Step#run method and handles updating the state of the step.
+        # @param log_prefix [String] The prefix to use when logging the step name
+        # @return [Object] The result of the step
+        def run(log_prefix = 'Running step')
+          @run_count += 1
+          @last_result = @step.run(log_prefix)
+          @results << @last_result
+          if @last_result.is_a?(StepError)
+            handle_error(@last_result)
+          end
+          @last_result
+        end
+
+        private
+
+        # Handle the error raised by the step
+        # @param result [StepError] The result of the step
+        def handle_error(result)
+          @last_error = result
+          if retry_workflow_on_fail?
+            @last_result = :retry_workflow
+          elsif retry?
+            retry_step
+          else
+            @last_result = :fail
+          end
+        end
+
+        # @return [Boolean] True if the step should be retried
+        def retry?
+          @retryable && @run_count < @retry_max
+        end
+
+        # Retry running the step
+        def retry_step
+          sleep @retry_delay if @retry_delay > 0
+          run("Retrying step (attempt #{@run_count} of #{@retry_max})")
+        end
+      end
+    end
+  end
+end