cem_acpt 0.3.3-universal-java-17 → 0.3.4-universal-java-17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 50d9f549fee6436b9f857dbdf313dcab94b223e988c2cbfeadd8999d86f436f0
-  data.tar.gz: 62a556754fe568b9d3bbc28b7381b3657c7aa0db872e5405bb392a1b9cef061d
+  metadata.gz: c289fcb52a1115630ffece04c465e5a8d945bdefdcc99fbf73a800398d2f2bc6
+  data.tar.gz: aca0f41efe831bf5bb8dc4399cd1a2d771f7bb72fb1ee13eeccb79d59bd45183
 SHA512:
-  metadata.gz: bf027699bcf399a68a6b845ef4191b4b912736f6eb52708e35a0393e78fce74f3c0299ee627bbfe80ec06a8ef9be0e5b030db6a1ab99a678943c8ebdf21112f6
-  data.tar.gz: eb7448394e6be3719a735cf30adedae0aefb0ea8a6144523300c2e57da61bdbb3dfbf76a394b3c2014cc78a8283a6c565f2f311481951d09f1211454f22add74
+  metadata.gz: 65dd7b8c5023304f0c5f9fd371a64cfb157567004d6f93ebdadc76bbf99f50f7483fab575f73fb9c07256cbb0db688ad4d8c10a6b644a8f7670a3441a54e866e
+  data.tar.gz: 2504bc3dc1ef1ac1a09c16d00c3cd99b08127ccd31f2b9da134f31ffb2dc19730eda580ec893def0a9b183bd4198c5aa8263ea735f50c0b2c21156a5f8f7124d

data/.github/workflows/spec.yml

@@ -0,0 +1,36 @@
+name: Unit Tests
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+    branches:
+      - main
+    tags:
+      - v.*
+
+jobs:
+  tests:
+    name: RSpec tests
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout Source
+        uses: actions/checkout@v3
+      
+      - name: Set up Java 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'adopt-hotspot'
+          java-version: '17'
+
+      - name: Set up JRuby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: jruby-9.3.3.0
+          bundler-cache: true
+
+      - name: Run RSpec
+        run: |
+          bundle exec rake spec

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    cem_acpt (0.3.3-universal-java-17)
+    cem_acpt (0.3.4-universal-java-17)
       concurrent-ruby (>= 1.1, < 2.0)
       deep_merge (>= 1.2, < 2.0)
       ed25519 (>= 1.2, < 2.0)
@@ -13,10 +13,13 @@ GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.2)
+    coderay (1.1.3)
     concurrent-ruby (1.1.10)
     deep_merge (1.2.2)
     diff-lcs (1.5.0)
     ed25519 (1.3.0-java)
+    ffi (1.15.5-java)
+    method_source (1.0.0)
     minitar (0.9)
     multi_json (1.15.0)
     net-scp (4.0.0)
@@ -27,6 +30,10 @@ GEM
     parser (3.1.2.0)
       ast (~> 2.4.1)
     pathspec (1.0.0)
+    pry (0.14.2-java)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+      spoon (~> 0.0)
     puppet-modulebuilder (0.3.0)
       minitar (~> 0.9)
       pathspec (>= 0.2.1, < 2.0.0)
@@ -73,6 +80,8 @@ GEM
       net-ssh (>= 2.7)
       net-telnet (= 0.1.1)
       sfl
+    spoon (0.0.6)
+      ffi
     unicode-display_width (2.2.0)
 
 PLATFORMS
@@ -80,9 +89,10 @@ PLATFORMS
 
 DEPENDENCIES
   cem_acpt!
+  pry
   rake (>= 12.0)
   rspec (>= 3.0)
   rubocop
 
 BUNDLED WITH
-   2.3.25
+   2.3.15

data/cem_acpt.gemspec

@@ -34,4 +34,5 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'puppet-modulebuilder', '>= 0.0.1'
   spec.add_runtime_dependency 'serverspec-cem-acpt'
   spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'pry'
 end

data/lib/cem_acpt/platform/gcp/cmd.rb

@@ -4,6 +4,7 @@ module CemAcpt::Platform::Gcp
   require 'json'
   require 'open3'
   require_relative File.join(__dir__, '..', 'base', 'cmd.rb')
+  require_relative File.join(__dir__, '..', 'utils', 'linux.rb')
 
   # This class provides methods to run gcloud commands. It allows for default values to be
   # set for the project, zone, and user and can also find these values from the local config.
@@ -59,6 +60,10 @@ module CemAcpt::Platform::Gcp
       @local_port ||= rand(49_512..65_535)
     end
 
+    def os_release
+      @os_release || :not_set
+    end
+
     def ssh_key
       return @ssh_key unless @ssh_key.nil?
 
@@ -173,12 +178,38 @@ module CemAcpt::Platform::Gcp
       logger.debug('Restarting SSH service')
       gcloud_ssh(instance_name, 'sudo systemctl restart sshd', ignore_command_errors: true)
       logger.info("SSH connection to #{instance_name} is ready")
+      logger.debug('Getting OS release')
+      osr = gcloud_ssh(instance_name, 'sudo cat /etc/os-release')
+      @os_release = CemAcpt::Platform::Utils::Linux::OSRelease.new(osr)
+      logger.debug("OS release: #{@os_release.to_h}")
       true
     rescue StandardError => e
       logger.debug("SSH connection to #{instance_name} failed: #{e}")
       false
     end
 
+    def dnf_automatic_success?(instance_name, opts: {})
+      ssh_options = ssh_opts(instance_name: instance_name, opts: opts)
+      logger.debug("Checking dnf-automatic success on #{instance_name} with options #{ssh_options}")
+      gcloud_ssh(instance_name, 'sudo systemctl restart dnf-automatic.timer')
+      true
+    rescue StandardError => e
+      logger.error("DNF automatic updates on #{instance_name} failed: #{e}")
+      false
+    end
+
+    def rpm_db_check_success?(instance_name, pkgmgr, opts: {})
+      ssh_options = ssh_opts(instance_name: instance_name, opts: opts)
+      logger.debug("Checking #{pkgmgr} rpm db on #{instance_name} with options #{ssh_options}")
+      gcloud_ssh(instance_name, "sudo #{pkgmgr} upgrade -y rpm glibc")
+      gcloud_ssh(instance_name, "sudo rm -f /var/lib/rpm/.rpm.lock")
+      gcloud_ssh(instance_name, "sudo #{pkgmgr} upgrade -y #{pkgmgr}")
+      true
+    rescue StandardError => e
+      logger.error("#{pkgmgr} rpm db check on #{instance_name} failed: #{e}")
+      false
+    end
+
     # This function spawns a background thread to run a GCP IAP tunnel, run the given
     # code block, then kill the thread. The code block will be yielded ssh_opts that
     # are used to configure SSH connections over the IAP tunnel. The IAP tunnel is

data/lib/cem_acpt/platform/gcp/compute.rb

@@ -236,6 +236,7 @@ module CemAcpt::Platform::Gcp
         node_data: data,
         transport: :ssh,
         ssh_opts: opts,
+        os_release: os_release.to_h,
       }
     end
 
@@ -260,6 +261,29 @@ module CemAcpt::Platform::Gcp
       false
     end
 
+    def dnf_automatic_success?
+      if os_release.name.match?(%r{^Red Hat.*}) && os_release.version.match?(%r{^(8|9).*})
+        logger.debug("Checking dnf-automatic on #{name} with platform #{info[:os_release][:platform]}")
+        @cmd.dnf_automatic_success?(name)
+      else
+        true
+      end
+    end
+
+    def rpm_db_check_success?
+      if os_release.name.match?(%r{^Red Hat.*})
+        logger.debug("Checking rpmdb on #{name} with platform #{info[:os_release][:platform]}")
+        pkgmgr = os_release.version.match?(%r{^(8|9).*}) ? 'dnf' : 'yum'
+        @cmd.rpm_db_check_success?(name, pkgmgr)
+      else
+        true
+      end
+    end
+
+    def os_release
+      @cmd.os_release
+    end
+
     def destroy
       @cmd.delete_instance(name)
     end

data/lib/cem_acpt/platform/gcp.rb

@@ -35,6 +35,18 @@ module Platform
     @instance.ready?
   end
 
+  # Returns true if dnf_automatic ran successfully on the GCP instance
+  def dnf_automatic_success?
+    logger.debug("Checking if dnf_automatic ran successfully on #{node_name}...")
+    @instance.dnf_automatic_success?
+  end
+
+  # Returns true if rpm_db_check ran successfully on the GCP instance
+  def rpm_db_check_success?
+    logger.debug("Checking if rpm_db_check ran successfully on #{node_name}...")
+    @instance.rpm_db_check_success?
+  end
+
   # Runs the test suite against the GCP instance. Must be given a block.
   # If necessary, can pass information into the block to be used in the test suite.
   def run_tests(&block)

data/lib/cem_acpt/platform/utils/linux.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module CemAcpt
+  module Platform
+    module Utils
+      module Linux
+        class OSRelease
+          attr_reader :properties
+
+          def initialize(os_release)
+            @os_release = parse_string(os_release)
+            @properties = []
+            define_properties(@os_release)
+          end
+
+          def to_h
+            @os_release
+          end
+
+          private
+
+          def parse_string(os_release)
+            os_release = os_release.split("\n")
+            os_release = os_release.map { |line| line.split('=', 2) }.reject { |l| l.length != 2 }
+            os_release.map! { |k, v| [k.downcase, v.delete_prefix('"').delete_suffix('"')] }
+            os_release.to_h
+          end
+
+          def define_properties(hsh)
+            hsh.each do |k, v|
+              @properties << k
+              self.class.send(:define_method, k) { v }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cem_acpt/test_runner/runner.rb

@@ -5,6 +5,8 @@ require 'English'
 require_relative '../logging'
 require_relative '../rspec_utils'
 require_relative 'runner_result'
+require_relative 'runner_workflow_builder'
+#require_relative 'workflow'
 
 module CemAcpt
   module TestRunner
@@ -23,6 +25,8 @@ module CemAcpt
 
     class RunnerProvisionError < RunnerStepError; end
 
+    
+
     # Runner is a class that runs a single acceptance test suite on a single node.
     # It is responsible for managing the lifecycle of the test suite and
     # reporting the results back to the main thread. Runner objects are created
@@ -32,6 +36,8 @@ module CemAcpt
 
       attr_reader :node, :node_exists, :run_result
 
+      MAX_PROVISION_ATTEMPTS = 3
+
       # @param node [String] the name of the node to run the acceptance test suite on
       # @param ctx [CemAcpt::RunnerCtx] a cem_acpt Ctx (context) object
       # @param module_pkg_path [Concurrent::IVar] the path to the module package
@@ -42,34 +48,33 @@ module CemAcpt
         @debug_mode = @context.config.debug_mode?
         @node_inventory = @context.node_inventory
         @module_pkg_path = @context.module_package_path
+        @provision_attempts = 0
+        @provision_start_time = nil
         @node_exists = false
         @run_result = CemAcpt::TestRunner::RunnerResult.new(@node, debug: @debug_mode)
-        @completed_steps = []
         validate!
       end
 
-      def run_step(step_sym)
-        send(step_sym)
-        @completed_steps << step_sym
-      rescue StandardError => e
-        err = CemAcpt::TestRunner::RunnerStepError.new(step_sym, e)
-        step_error_logging(err)
-        @run_result.from_error(err)
-        destroy unless step_sym == :destroy
-      end
-
       # Executes test suite steps
       def start
         async_info("Starting test suite for #{@node.node_name}", log_prefix('RUNNER'))
-        run_step(:provision)
-        run_step(:bootstrap)
-        run_step(:run_tests)
-        run_step(:destroy)
-        true
+        @workflow = new_workflow
+        @workflow.run
+        if @workflow.success?
+          @run_result = @workflow.last_result
+          @workflow.completed_steps.each do |s|
+            async_info("Step '#{s.name}' completed successfully", log_prefix('RUNNER'))
+          end
+          true
+        else
+          @run_result = @workflow.last_error
+          step_error_logging(@workflow.last_error)
+          false
+        end
       rescue StandardError => e
         step_error_logging(e)
+        @run_result = CemAcpt::TestRunner::RunnerResult.new(@node, debug: @debug_mode)
         @run_result.from_error(e)
-        destroy
       end
 
       # Checks for failures in the test results.
@@ -81,6 +86,24 @@ module CemAcpt
 
       private
 
+      # Builds a new workflow for the runner
+      # @return [CemAcpt::TestRunner::Workflow::Manager] the new workflow
+      def new_workflow
+        builder = RunnerWorkflowBuilder.new(@node, config: @context.config)
+        builder.add_provision
+        builder.add_sleep(time: 30)
+        builder.add_wait_for_node_ssh
+        builder.add_check_dnf_automatic
+        builder.add_check_rpm_db
+        builder.add_save_node_to_inventory(node_inventory: @node_inventory, platform: @platform)
+        builder.add_check_for_module_package_path(module_pkg_path: @module_pkg_path)
+        builder.add_install_module(module_pkg_path: @module_pkg_path)
+        builder.add_check_node_inventory_file(node_inventory: @node_inventory)
+        builder.add_run_tests(rspec_opts: rspec_opts, rspec_cmd: CemAcpt::RSpecUtils::Command, run_result: @run_result)
+        builder.add_clean_up
+        builder.workflow
+      end
+
       def step_error_logging(err)
         prefix = err.respond_to?(:step) ? log_prefix(err.step.capitalize) : log_prefix('RUNNER')
         fatal_msg = ["runner failed: #{err.message}"]
@@ -94,93 +117,6 @@ module CemAcpt
         "#{prefix}: #{@node.test_data[:test_name]}:"
       end
 
-      # Provisions the node for the acceptance test suite.
-      def provision
-        async_info("Provisioning #{@node.node_name}...", log_prefix('PROVISION'))
-        start_time = Time.now
-        @node.provision
-        @node_exists = true
-        max_retries = 60 # equals 300 seconds because we check every five seconds
-        until @node.ready?
-          if max_retries <= 0
-            async_fatal("Node #{@node.node_name} failed to provision", log_prefix('PROVISION'))
-            raise CemAcpt::TestRunner::RunnerProvisionError, "Provisioning timed out for node #{@node.node_name}"
-          end
-
-          async_info("Waiting for #{@node.node_name} to be ready for remote connections...", log_prefix('PROVISION'))
-          max_retries -= 1
-          sleep(5)
-        end
-        async_info("Node #{@node.node_name} is ready...", log_prefix('PROVISION'))
-        node_desc = {
-          test_data: @node.test_data,
-          platform: @platform,
-          local_port: @node.local_port,
-        }.merge(@node.node)
-        @node_inventory.add(@node.node_name, node_desc)
-        @node_inventory.save
-        async_info("Node #{@node.node_name} provisioned in #{Time.now - start_time} seconds", log_prefix('PROVISION'))
-      end
-
-      # Bootstraps the node for the acceptance test suite. Currently, this
-      # just uploads and installs the module package.
-      def bootstrap
-        async_info("Bootstrapping #{@node.node_name}...", log_prefix('BOOTSTRAP'))
-        until File.exist?(@module_pkg_path)
-          async_debug("Waiting for module package #{@module_pkg_path} to exist...", log_prefix('BOOTSTRAP'))
-          sleep(1)
-        end
-        async_info("Installing module package #{@module_pkg_path}...", log_prefix('BOOTSTRAP'))
-        @node.install_puppet_module_package(@module_pkg_path)
-      end
-
-      # Runs the acceptance test suite via rspec.
-      def run_tests
-        attempts = 0
-        until File.exist?(@node_inventory.save_file_path)
-          raise 'Node inventory file not found' if (attempts += 1) > 3
-
-          sleep(1)
-        end
-        async_info("Running test #{@node.test_data[:test_name]} on node #{@node.node_name}...", log_prefix('RSPEC'))
-        @node.run_tests do |cmd_env|
-          cmd_opts = rspec_opts
-          cmd_opts.env = cmd_opts.env.merge(cmd_env) if cmd_env
-          # Documentation format gets logged in real time, JSON file is read after the fact
-          begin
-            @rspec_cmd = CemAcpt::RSpecUtils::Command.new(cmd_opts)
-            @rspec_cmd.execute(pty: false, log_prefix: log_prefix('RSPEC'))
-            @run_result.from_json_file(cmd_opts.format[:json])
-          rescue Errno::EIO => e
-            async_error("failed to run rspec: #{@node.test_data[:test_name]}: #{$ERROR_INFO}", log_prefix('RSPEC'))
-            @run_result.from_error(e)
-          rescue StandardError => e
-            async_error("failed to run rspec: #{@node.test_data[:test_name]}: #{e.message}", log_prefix('RSPEC'))
-            async_debug("Backtrace:\n#{e.backtrace}", log_prefix('RSPEC'))
-            @run_result.from_error(e)
-          end
-        end
-        async_info("Tests completed with exit code: #{@run_result.exit_status}", log_prefix('RSPEC'))
-      end
-
-      # Destroys the node for the acceptance test suite.
-      def destroy
-        kill_spec_pty_if_exists
-        if @context.config.get('no_destroy_nodes')
-          async_info("Not destroying node #{@node.node_name} because 'no_destroy_nodes' is set to true",
-                     log_prefix('DESTROY'))
-        else
-          async_info("Destroying #{@node.node_name}...", log_prefix('DESTROY'))
-          @node.destroy
-          @node_exists = false
-          async_info("Node #{@node.node_name} destroyed successfully", log_prefix('DESTROY'))
-        end
-      end
-
-      def kill_spec_pty_if_exists
-        @rspec_cmd&.kill_pty
-      end
-
       # Validates the runner configuration.
       def validate!
         raise 'No node provided' unless @node

data/lib/cem_acpt/test_runner/runner_workflow_builder.rb

@@ -0,0 +1,238 @@
+# frozen_string_literal: true
+
+require_relative '../logging'
+require_relative 'workflow'
+
+module CemAcpt
+  module TestRunner
+    # RunnerWorkflowBuilder builds a workflow for a TestRunner
+    # @!attribute [r] workflow
+    #   @return [CemAcpt::TestRunner::Workflow::Manager] Workflow object
+    class RunnerWorkflowBuilder
+      include CemAcpt::LoggingAsync
+
+      attr_reader :workflow
+
+      # @param node [CemAcpt::Platform::Base] Initialized node object
+      # @param config [Hash] Context config hash
+      def initialize(node, config)
+        @node = node
+        @workflow = CemAcpt::TestRunner::Workflow::Manager.new(workflow_manager_opts(config))
+        @config = config
+      end
+
+      def add_sleep(**kwargs)
+        opts = {
+          node: @node,
+          time: kwargs[:time] || 10,
+          retryable: false,
+        }
+        @workflow.add_step(:sleep, **opts) do |s|
+          log_info("sleeping for #{s.opts[:time]} seconds", s)
+          sleep(s.opts[:time])
+        end
+      end
+
+      def add_provision(**kwargs)
+        opts = {
+          node: @node,
+          retryable: kwargs[:retryable] || false,
+        }
+        @workflow.add_step(:provision, **opts) do |s|
+          s.opts[:node].provision
+          s.opts[:node]
+        end
+      end
+
+      def add_wait_for_node_ssh(**kwargs)
+        opts = {
+          node: @node,
+          retryable: kwargs[:retryable] || true,
+          retry_delay: kwargs[:retry_delay] || 30,
+          retry_max: kwargs[:retry_max] || 10,
+        }
+        @workflow.add_step(:wait_for_node_ssh, **opts) do |s|
+          unless s.opts[:node].ready?
+            raise "wait_for_node_ssh timed out for node #{s.opts[:node].node_name}"
+          end
+          s.opts[:node]
+        end
+      end
+
+      def add_check_dnf_automatic(**kwargs)
+        opts = {
+          node: @node,
+          retryable: kwargs[:retryable] || true,
+          retry_delay: kwargs[:retry_delay] || 30,
+          retry_workflow_on_fail: kwargs[:retry_workflow_on_fail] || true,
+        }
+        @workflow.add_step(:check_dnf_automatic, **opts) do |s|
+          unless s.opts[:node].dnf_automatic_success?
+            raise "dnf_automatic failed on node #{s.opts[:node].node_name}"
+          end
+          s.opts[:node]
+        end
+      end
+
+      def add_check_rpm_db(**kwargs)
+        opts = {
+          node: @node,
+          retryable: kwargs[:retryable] || true,
+          retry_delay: kwargs[:retry_delay] || 30,
+          retry_workflow_on_fail: kwargs[:retry_workflow_on_fail] || true,
+        }
+        @workflow.add_step(:rpm_db_check, **opts) do |s|
+          unless s.opts[:node].rpm_db_check_success?
+            raise "rpm_db_check failed on node #{s.opts[:node].node_name}"
+          end
+          s.opts[:node]
+        end
+      end
+
+      def add_save_node_to_inventory(node_inventory:, platform:, **kwargs)
+        opts = {
+          node: @node,
+          platform: platform,
+          node_inventory: node_inventory,
+          retryable: kwargs[:retryable] || false,
+        }
+        @workflow.add_step(:save_node_to_inventory, **opts) do |s|
+          node_desc = {
+            test_data: s.opts[:node].test_data,
+            platform: s.opts[:platform],
+            local_port: s.opts[:node].local_port,
+          }.merge(s.opts[:node].node)
+          s.opts[:node_inventory].add(s.opts[:node].node_name, node_desc)
+          s.opts[:node_inventory].save
+          log_info("node #{s.opts[:node_name]} saved to inventory", s)
+          s.opts[:node_inventory]
+        end
+      end
+
+      def add_check_for_module_package_path(module_pkg_path:, **kwargs)
+        opts = {
+          module_pkg_path: module_pkg_path,
+          retryable: kwargs[:retryable] || true,
+          retry_delay: kwargs[:retry_delay] || 30,
+        }
+        @workflow.add_step(:check_for_module_package_path, **opts) do |s|
+          unless File.exist?(s.opts[:module_pkg_path])
+            raise "module package #{s.opts[:module_pkg_path]} does not exist"
+          end
+          s.opts[:module_pkg_path]
+        end
+      end
+
+      def add_install_module(module_pkg_path:, **kwargs)
+        opts = {
+          node: @node,
+          module_pkg_path: module_pkg_path,
+          retryable: kwargs[:retryable] || true,
+          retry_delay: kwargs[:retry_delay] || 60,
+        }
+        @workflow.add_step(:bootstrap, **opts) do |s|
+          s.opts[:node].install_puppet_module_package(s.opts[:module_pkg_path])
+          s.opts[:node]
+        end
+      end
+
+      def add_check_node_inventory_file(node_inventory:, **kwargs)
+        opts = {
+          ni_file_path: node_inventory.save_file_path,
+          retryable: kwargs[:retryable] || true,
+          retry_max: kwargs[:retry_max] || 60,
+          retry_delay: kwargs[:retry_delay] || 5,
+        }
+        @workflow.add_step(:check_node_inventory_file, **opts) do |s|
+          unless File.exist?(s.opts[:ni_file_path])
+            raise "node inventory file #{s.opts[:ni_file_path]} not found"
+          end
+          s.opts[:ni_file_path]
+        end
+      end
+
+      def add_run_tests(rspec_opts:, rspec_cmd:, run_result:, **kwargs)
+        opts = {
+          node: @node,
+          rspec_opts: rspec_opts,
+          rspec_cmd: rspec_cmd,
+          run_result: run_result,
+          retryable: kwargs[:retryable] || false,
+        }
+        @workflow.add_step(:run_tests, **opts) do |s|
+          s.opts[:node].run_tests do |cmd_env|
+            cmd_opts = s.opts[:rspec_opts].dup
+            cmd_opts.env = cmd_opts.env.merge(cmd_env) if cmd_env
+            rspec_cmd = s.opts[:rspec_cmd].new(cmd_opts)
+            run_result = s.opts[:run_result].dup
+            begin
+              rspec_cmd.execute(pty: false, log_prefix: "RSPEC: #{@node.test_data[:test_name]}")
+              run_result.from_json_file(cmd_opts.format[:json])
+            rescue Errno::EIO => e
+              log_error("failed to run rspec: #{@node.test_data[:test_name]}: #{$ERROR_INFO}", s)
+              run_result.from_error(e)
+            rescue StandardError => e
+              log_error("failed to run rspec: #{@node.test_data[:test_name]}: #{e.message}", s)
+              log_debug(e.backtrace.join("\n"), s)
+              run_result.from_error(e)
+            end
+          end
+          run_result
+        end
+      end
+
+      def add_clean_up(force: false, **kwargs)
+        opts = {
+          node: @node,
+          config: @config[:config],
+          force: force,
+          retryable: kwargs[:retryable] || false,
+        }
+        @workflow.add_step(:clean_up, **opts) do |s|
+          if !force && s.opts[:config].get('no_destroy_nodes')
+            log_info("not destroying node #{s.opts[:node].node_name} because 'no_destroy_nodes' is set to true", s)
+          else
+            log_info("destroying node #{s.opts[:node].node_name}", s)
+            s.opts[:node].destroy
+            log_info("node #{s.opts[:node].node_name} destroyed", s)
+          end
+        end
+      end
+
+      private
+
+      def log_msg(msg, step)
+        "Step '#{step.name}' on #{@node.node_name}: #{msg}"
+      end
+
+      def log_debug(msg, step)
+        async_debug(log_msg(msg, step))
+      end
+
+      def log_info(msg, step)
+        async_info(log_msg(msg, step))
+      end
+
+      def log_warn(msg, step)
+        async_warn(log_msg(msg, step))
+      end
+
+      def log_error(msg, step)
+        async_error(log_msg(msg, step))
+      end
+
+      def log_fatal(msg, step)
+        async_fatal(log_msg(msg, step))
+      end
+
+      def workflow_manager_opts(config)
+        {
+          retry_max: config[:workflow_retry_max] || 3,
+          retry_delay: config[:workflow_retry_delay] || 0,
+          ignore_failures: config[:workflow_ignore_failures] || false,
+          raise_on_fail: config[:workflow_raise_on_fail] || true,
+        }
+      end
+    end
+  end
+end

data/lib/cem_acpt/test_runner/workflow/manager.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+require_relative '../../logging'
+require_relative 'step'
+
+module CemAcpt
+  module TestRunner
+    module Workflow
+      # Manager is a class that manages how steps in a workflow are executed.
+      # @!attribute [r] completed_steps
+      #   @return [Array<Step>] The steps that have been completed
+      # @!attribute [r] last_error
+      #   @return [Exception, nil] The last error that occurred, if any
+      # @!attribute [r] last_result
+      #   @return [Any] The last result that occurred, if any
+      # @!attribute [r] retry_max
+      #   @return [Integer] The maximum number of workflow runs to attempt before failing. Default: 3
+      # @!attribute [r] retry_delay
+      #   @return [Integer] The number of seconds to wait between workflow runs. Default: 0
+      # @!attribute [r] steps
+      #   @return [Array<Step>] The steps that are part of this workflow
+      class Manager
+        include CemAcpt::LoggingAsync
+
+        attr_reader :completed_steps, :last_error, :last_result, :retry_max, :retry_delay, :steps
+
+        # @param [Hash] opts The options to create a new workflow manager with
+        # @option opts [Integer] :retry_max The maximum number of workflow runs to attempt before failing. Default: 3
+        # @option opts [Integer] :retry_delay The number of seconds to wait between workflow runs. Default: 0
+        # @option opts [Boolean] :ignore_failures Whether to ignore Step failures and continue to the next step. Default: false
+        # @option opts [Boolean] :raise_on_fail Whether to raise an exception when a Step fails. Default: true
+        def initialize(**opts)
+          @retry_max = opts[:retry_max] || 3
+          @retry_delay = opts[:retry_delay] || 0
+          @ignore_failures = opts[:ignore_failures] || false
+          @raise_on_fail = opts[:raise_on_fail] || true
+          @steps = []
+          @workflow_runs = 0
+          @last_error = nil
+          @last_result = nil
+          @completed_steps = []
+        end
+
+        # Add a step to the workflow. Steps can be named anything, but if the step name is :clean_up,
+        # it will be run at the end of the workflow, regardless of the order it was added, and it's
+        # output will not be saved as the last_result. Additionally, the :clean_up step will be run
+        # even if the workflow fails.
+        # @param [Symbol] name The name of the step
+        # @param [Hash] kwargs The keyword arguments to pass to the step
+        # @param [Proc] block The block to pass to the step
+        # @yieldparam [Step] step The step that was added. This is passed to the block.
+        def add_step(name, **kwargs, &block)
+          raise ArgumentError, 'name must be a Symbol' unless name.is_a?(Symbol)
+
+          step_name = if @steps.any? { |step| step.name == name }
+                        "#{name}_#{@steps.length}".to_sym
+                      else
+                        name
+                      end
+          step = Step.new(step_name, **kwargs, &block)
+          @steps << StepState.new(step, @steps.length, **kwargs)
+        end
+
+        # Run the workflow
+        def run
+          @workflow_runs += 1
+          @completed_steps = []
+          @steps.each do |step|
+            next if step.name == :clean_up
+
+            result = step.run
+            handle_result(step, result)
+          end
+        ensure
+          clean_up
+        end
+
+        # Whether the workflow has completed successfully
+        def success?
+          (@completed_steps.length == @steps.length) && @completed_steps.none? { |step| step.failed? }
+        end
+
+        # @return [Boolean] Whether to ignore failures and continue to the next step
+        def ignore_failures?
+          @ignore_failures
+        end
+
+        # @return [Boolean] Whether to raise an exception when a step fails
+        def raise_on_fail?
+          @raise_on_fail
+        end
+
+        private
+
+        # Handles the result of a step
+        # @param [Step] step The step that was run
+        # @param [Object] result The result of the step
+        # @raise [StandardError] If the step failed and the workflow is not configured to ignore failures
+        def handle_result(step, result)
+          @last_result = result unless step.name == :clean_up # Don't overwrite the last result with the clean_up step
+          case result
+          when :retry_workflow
+            async_warn("Step '#{step.name}' failed and requested a workflow retry")
+            async_debug("Step '#{step.name}' failed with error: #{step.last_error.message}")
+            async_debug("Step '#{step.name}' failed with error: #{step.last_error.backtrace.join("\n")}")
+            @last_error = step.last_error
+            retry_workflow
+          when :fail
+            async_warn("Step '#{step.name}' failed")
+            async_debug(step.last_error.message)
+            async_debug(step.last_error.backtrace.join("\n"))
+            @last_error = step.last_error
+            if ignore_failures?
+              async_warn("Ignoring failure of step '#{step.name}'")
+              @completed_steps << step
+            else
+              async_error("Workflow failed with error: #{@last_error.message}")
+              raise @last_error
+            end
+          else
+            async_info("Step '#{step.name}' succeeded")
+            async_debug("Step '#{step.name}' returned: #{result}")
+            @completed_steps << step
+          end
+        end
+
+        # Retries the workflow if it is retryable
+        # @raise [StandardError] If the workflow is not retryable or has exceeded the maximum number of retries
+        def retry_workflow
+          if @workflow_runs < @retry_max
+            async_info("Retrying workflow (attempt #{@workflow_runs + 1} of #{@retry_max})")
+            sleep @retry_delay if @retry_delay > 0
+            clean_up
+            run
+          else
+            async_fatal('Workflow is not retryable or has exceeded the maximum number of retries')
+            raise @last_error
+          end
+        end
+
+        # Adds a default clean_up step if one is not defined
+        # @return [Step] The default clean_up step
+        def default_clean_up
+          add_step(:clean_up) do
+            async_info('No clean_up step defined, skipping')
+            true
+          end
+          @steps.last
+        end
+
+        # Runs the clean_up step
+        def clean_up
+          cleanup_step = @steps.find { |step| step.name == :clean_up } || default_clean_up
+          result = cleanup_step.run
+          handle_result(cleanup_step, result)
+        end
+      end
+    end
+  end
+end

data/lib/cem_acpt/test_runner/workflow/step.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+require_relative '../../logging'
+
+module CemAcpt
+  module TestRunner
+    module Workflow
+      # Error used to wrap fatal errors raised in Runner steps
+      # @!attribute [r] step
+      #   @return [Step] The step that raised the error
+      class StepError < StandardError
+        attr_reader :step
+
+        def initialize(step, err)
+          @step = step
+          @original_error = err
+          super err
+          set_backtrace err.backtrace if err.respond_to?(:backtrace)
+        end
+      end
+
+      # Step is a class that defines a single step in a Workflow.
+      # Step objects are created by the Workflow class and are not intended to be created directly.
+      # @!attribute [r] name
+      #   @return [Symbol] The name of the step
+      # @!attribute [r] opts
+      #   @return [Hash] The options passed to the step
+      # @!attribute [r] result
+      #   @return [Object] The result of the step
+      class Step
+        include CemAcpt::LoggingAsync
+
+        attr_reader :name, :opts, :result
+
+        # @param name [Symbol] The name of the step
+        # @param opts [Hash] The options passed to the step
+        # @param block [Proc] The block to execute when the step is run
+        # @yieldparam step [Step] This step object is yielded to the block
+        def initialize(name, **opts, &block)
+          @name = name
+          @opts = opts
+          @block = block
+          @result = :not_run
+          @failed = false
+          @log_prefix = opts[:log_prefix] || "#{@name.upcase}:"
+        end
+
+        # @return [Boolean] True if the step has been run and failed
+        def failed?
+          @failed
+        end
+
+        # Run the step. This calls and executes the block passed to the constructor.
+        # @param log_prefix [String] The prefix to use when logging the step name
+        # @return [Object] The result of the step
+        def run(_log_prefix = 'Running step')
+          async_info(log_msg('Starting step'))
+          @result = @block.call(self)
+          async_debug(log_msg('SUCCESS'))
+          @result
+        rescue StandardError => e
+          async_debug(log_msg("FAILED: #{e.message}"))
+          @result = StepError.new(@name, e)
+          @failed = true
+          @result
+        ensure
+          Thread.pass # Be kind to the scheduler
+        end
+
+        private
+
+        def log_msg(msg)
+          [@log_prefix, msg].join(' ')
+        end
+      end
+
+      # StepState is a class that holds the state of a Step.
+      # StepState objects are created by the Workflow class and are not intended to be created directly.
+      # @!attribute [r] step
+      #   @return [Step] The step object
+      # @!attribute [r] position
+      #   @return [Integer] The position of the step in the workflow
+      # @!attribute [r] opts
+      #   @return [Hash] The options passed to the step
+      # @!attribute [r] run_count
+      #   @return [Integer] The number of times the step has been run
+      # @!attribute [r] results
+      #   @return [Array<Object>] The results of all runs of the step
+      # @!attribute [r] last_error
+      #   @return [StepError] The last error raised by the step
+      # @!attribute [r] last_result
+      #   @return [Object] The result of the last run of the step
+      class StepState
+        attr_reader :step, :position, :opts, :run_count, :results, :last_error, :last_result
+
+        # @param step [Step] The step object
+        # @param position [Integer] The position of the step in the workflow
+        # @param opts [Hash] The options passed to the step
+        def initialize(step, position, **opts)
+          @step = step
+          @position = position
+          @retryable = opts[:retryable] || false
+          @retry_max = opts[:retry_max] || 3
+          @retry_delay = opts[:retry_delay] || 0
+          @retry_workflow_on_fail = opts[:retry_workflow_on_fail] || false
+          @run_count = 0
+          @results = []
+          @last_error = nil
+          @last_result = nil
+        end
+
+        # Proxy any methods not defined in this class to the Step
+        def method_missing(method, *args, &block)
+          if @step.respond_to?(method)
+            @step.send(method, *args, &block)
+          else
+            super
+          end
+        end
+
+        # Proxy any methods not defined in this class to the Step
+        def respond_to_missing?(method, include_private = false)
+          @step.respond_to?(method) || super
+        end
+
+        # @return [Object] The result of the last run of the step
+        def result
+          @last_result || :not_run
+        end
+
+        # @return [Boolean] If the step is retryable
+        def retryable?
+          @retryable
+        end
+
+        # @return [Boolean] If the workflow should be retried if the step fails
+        def retry_workflow_on_fail?
+          @retry_workflow_on_fail
+        end
+
+        # @return [Boolean] True if the step has been run and failed
+        def failed?
+          @results.last.is_a?(StepError)
+        end
+
+        # Run the step. This wraps the Step#run method and handles updating the state of the step.
+        # @param log_prefix [String] The prefix to use when logging the step name
+        # @return [Object] The result of the step
+        def run(log_prefix = 'Running step')
+          @run_count += 1
+          @last_result = @step.run(log_prefix)
+          @results << @last_result
+          if @last_result.is_a?(StepError)
+            handle_error(@last_result)
+          end
+          @last_result
+        end
+
+        private
+
+        # Handle the error raised by the step
+        # @param result [StepError] The result of the step
+        def handle_error(result)
+          @last_error = result
+          if retry_workflow_on_fail?
+            @last_result = :retry_workflow
+          elsif retry?
+            retry_step
+          else
+            @last_result = :fail
+          end
+        end
+
+        # @return [Boolean] True if the step should be retried
+        def retry?
+          @retryable && @run_count < @retry_max
+        end
+
+        # Retry running the step
+        def retry_step
+          sleep @retry_delay if @retry_delay > 0
+          run("Retrying step (attempt #{@run_count} of #{@retry_max})")
+        end
+      end
+    end
+  end
+end

data/lib/cem_acpt/test_runner/workflow.rb

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+require_relative '../logging'
+require_relative 'workflow/manager'
+require_relative 'workflow/step'
+
+module CemAcpt
+  module TestRunner
+    # Namespace for workflow classes
+    module Workflow; end
+  end
+end
+
+# module CemAcpt
+#   module TestRunner
+    
+
+#     # Workflow is a class that manages how steps are executed.
+#     class Workflow
+#       include CemAcpt::LoggingAsync
+
+#       attr_reader :steps, :groups, :group_retry_max, :completed_steps
+
+#       def initialize(init_steps: [], **opts)
+#         raise ArgumentError, 'init_steps must be an Array' unless init_steps.is_a?(Array)
+#         raise ArgumentError, 'init_steps must be an Array of Step objects' unless init_steps.all? { |step| step.is_a?(Step) }
+
+#         @steps = init_steps
+#         @groups = @steps.map(&:group).compact.uniq
+#         @group_retry_max = opts[:group_retry_max] || 3
+#         @group_tries = {}
+#         @last_error = nil
+#         @completed_steps = []
+#       end
+
+#       def add_step(step, order = nil)
+#         raise ArgumentError, 'step must be a Step object' unless step.is_a?(Step)
+
+#         unless order.nil?
+#           @steps.insert(order, step)
+#         else
+#           @steps << step
+#         end
+#       end
+
+#       def add_step_with(name, order = nil, **kwargs, &block)
+#         raise ArgumentError, 'order must be an Integer' unless order.is_a?(Integer) || order.nil?
+#         raise ArgumentError, 'name must be a Symbol' unless name.is_a?(Symbol)
+
+#         unless order.nil?
+#           @steps.insert(order, Step.new(name, **kwargs, &block))
+#         else
+#           @steps << Step.new(name, **kwargs, &block)
+#         end
+#       end
+
+#       def run
+#         current_run_groups = []
+#         @steps.each do |step|
+#           next if step.name == :clean_up
+
+#           group_run_increment(step, current_run_groups)
+#           step.run
+#           @completed_steps << step
+#         end
+#         @completed_steps
+#       rescue StandardError => e
+#         async_error("Workflow failed with error: #{e}")
+#         @last_error = e
+#         raise e
+#       ensure
+#         clean_up
+#       end
+
+#       def success?
+#         (@completed_steps.length == @steps.length) && @completed_steps.none? { |step| step.result.is_a?(StepError) }
+#       end
+
+#       private
+
+#       def group_run_increment(step, current_run_groups = [])
+#         if step.grouped? && !current_run_groups.include?(step.group)
+#           current_run_groups << step.group
+#           if @group_tries[step.group].nil? || @group_tries[step.group].zero?
+#             @group_tries[step.group] = 1
+#           elsif @group_tries[step.group] >= @group_retry_max
+#             raise StepError.new(step, @last_error)
+#           else
+#             @group_tries[step.group] += 1
+#           end
+#         end
+#       end
+
+#       def run_step(step)
+#         result = step.run
+#         #handle_result(step, result)
+#         @completed_steps << step
+#         result
+#       end
+
+#       def handle_result(step, result)
+#         if result.is_a?(StepError)
+#           handle_error(step, result)
+#         else
+#           async_info("Step '#{step.name}' completed successfully")
+#         end
+#       end
+
+#       def handle_error(step, err)
+#         if step.retryable? && step.runs < @retry_max
+#           async_info("Retrying step '#{step.name}' (attempt #{step.runs + 1} of #{@retry_max})")
+#           sleep step.retry_delay if step.retry_delay > 0
+#           run_step(step)
+#         else
+#           async_debug("Step '#{step.name}' is not retryable or has exceeded the maximum number of retries")
+#           raise err if step.raise_on_fail?
+#         end
+#       end
+
+#       def default_clean_up
+#         Step.new(:clean_up, retryable: false, raise_on_fail: true) do
+#           async_info('No clean_up step defined, skipping')
+#         end
+#       end
+
+#       def clean_up
+#         cleanup_step = @steps.find { |step| step.name == :clean_up } || default_clean_up
+#         unless cleanup_step.nil?
+#           cleanup_step.retryable = false # clean_up steps should not be retried
+#           cleanup_step.raise_on_fail = true # clean_up steps should always raise on failure
+#           run_step(cleanup_step)
+#         end
+#       end
+#     end
+
+#     # Step is a class that defines a single step in a Workflow.
+#     class Step
+#       include CemAcpt::LoggingAsync
+
+#       attr_reader :group, :name, :opts, :result, :retry_delay, :retry_max, :runs
+
+#       def initialize(name, **opts, &logic)
+#         @name = name
+#         @logic = logic
+#         @group = opts[:group] || nil
+#         @retryable = opts[:retryable] || false
+#         @retry_delay = opts[:retry_delay] || 0
+#         @retry_max = opts[:retry_max] || 3
+#         @retry_group_on_fail = opts[:retry_group_on_fail] || false
+#         @raise_on_fail = opts[:raise_on_fail] || true
+#         @runs = 0
+#         @opts = opts
+#         @result = :not_run
+#       end
+
+#       def grouped?
+#         !@group.nil?
+#       end
+
+#       def retryable?
+#         @retryable
+#       end
+
+#       def retry_group_on_fail?
+#         @retry_group_on_fail
+#       end
+
+#       def raise_on_fail?
+#         @raise_on_fail
+#       end
+
+#       def retryable=(val)
+#         raise ArgumentError, 'retryable must be a Boolean' unless val.is_a?(TrueClass) || val.is_a?(FalseClass)
+
+#         @retryable = val
+#       end
+
+#       def retry_delay=(val)
+#         raise ArgumentError, 'retry_delay must be an Integer' unless val.is_a?(Integer)
+
+#         @retry_delay = val
+#       end
+
+#       def retry_group_on_fail=(val)
+#         raise ArgumentError, 'retry_group_on_fail must be a Boolean' unless val.is_a?(TrueClass) || val.is_a?(FalseClass)
+
+#         @retry_group_on_fail = val
+#       end
+
+#       def raise_on_fail=(val)
+#         raise ArgumentError, 'raise_on_fail must be a Boolean' unless val.is_a?(TrueClass) || val.is_a?(FalseClass)
+
+#         @raise_on_fail = val
+#       end
+
+#       def run
+#         begin
+#           @runs += 1
+#           async_info("Running step #{@name}...") if @runs == 1
+#           @result = @logic.call(@opts)
+#         rescue StandardError => e
+#           if retryable? && @runs < @retry_max
+#             sleep @retry_delay if @retry_delay > 0
+#             async_info("Retrying step #{@name} (attempt #{@runs + 1} of #{@retry_max})")
+#             run
+#           else
+#             @result = CemAcpt::TestRunner::StepError.new(@name, e)
+#             raise @result if raise_on_fail?
+#           end
+#         end
+#         @result
+#       end
+#     end
+#   end
+# end

data/lib/cem_acpt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CemAcpt
-  VERSION = '0.3.3'
+  VERSION = '0.3.4'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cem_acpt
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.3.4
 platform: universal-java-17
 authors:
 - puppetlabs
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-12-14 00:00:00.000000000 Z
+date: 2023-01-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -132,6 +132,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: pry
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Litmus-like library focusing on CEM Acceptance Tests
 email:
 - abide-team@puppet.com
@@ -140,6 +154,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/spec.yml"
 - ".gitignore"
 - ".rspec"
 - CODEOWNERS
@@ -167,6 +182,7 @@ files:
 - lib/cem_acpt/platform/gcp.rb
 - lib/cem_acpt/platform/gcp/cmd.rb
 - lib/cem_acpt/platform/gcp/compute.rb
+- lib/cem_acpt/platform/utils/linux.rb
 - lib/cem_acpt/platform/vmpooler.rb
 - lib/cem_acpt/puppet_helpers.rb
 - lib/cem_acpt/rspec_utils.rb
@@ -177,6 +193,10 @@ files:
 - lib/cem_acpt/test_runner/run_handler.rb
 - lib/cem_acpt/test_runner/runner.rb
 - lib/cem_acpt/test_runner/runner_result.rb
+- lib/cem_acpt/test_runner/runner_workflow_builder.rb
+- lib/cem_acpt/test_runner/workflow.rb
+- lib/cem_acpt/test_runner/workflow/manager.rb
+- lib/cem_acpt/test_runner/workflow/step.rb
 - lib/cem_acpt/utils.rb
 - lib/cem_acpt/version.rb
 - sample_config.yaml