cem_acpt 0.1.0 → 0.2.0

data/lib/cem_acpt/platform/gcp/compute.rb

@@ -240,6 +240,9 @@ module CemAcpt::Platform::Gcp
     end
 
     def create
+      # Add the test ssh key to os-login
+      logger.debug("Adding test SSH key to os-login for #{name}")
+      @cmd.local_exec("compute os-login ssh-keys add --key-file #{@cmd.ssh_key}.pub --project #{project.name} --ttl 4h")
       @cmd.local_exec(create_cmd)
     rescue StandardError => e
       raise "Failed to create VM #{name} with command #{create_cmd}: #{e}"
@@ -253,7 +256,7 @@ module CemAcpt::Platform::Gcp
 
       logger.debug("Checking instance #{name} SSH connectivity")
       @cmd.ssh_ready?(name)
-    rescue StandardError
+    rescue StandardError, Exception
       false
     end
 
@@ -263,7 +266,9 @@ module CemAcpt::Platform::Gcp
 
     def install_puppet_module_package(module_pkg_path, remote_path, puppet_path)
       @cmd.scp_upload(@name, module_pkg_path, remote_path)
+      logger.info("Uploaded module package #{module_pkg_path} to #{remote_path} on #{@name}")
       @cmd.ssh(@name, "sudo #{puppet_path} module install #{remote_path}")
+      logger.info("Installed module package #{remote_path} on #{@name}")
     end
 
     private

data/lib/cem_acpt/platform/gcp.rb

@@ -39,7 +39,7 @@ module Platform
   # If necessary, can pass information into the block to be used in the test suite.
   def run_tests(&block)
     logger.debug("Running tests for #{node_name}...")
-    block.call
+    block.call @instance.cmd.env
   end
 
   # Uploads and installs a Puppet module package on the GCP instance.
@@ -70,7 +70,7 @@ module Platform
     # @param opts [Hash] options to pass to the apply command
     # @return [String] the output of the apply command
     def apply_manifest(instance_name, manifest, opts = {})
-      CemAcpt::Platform::Gcp::Cmd.new(out_format: 'json').apply_manifest(instance_name, manifest, opts)
+      command_provider.apply_manifest(instance_name, manifest, opts)
     end
 
     # Runs a shell command on the given instance
@@ -79,7 +79,7 @@ module Platform
     # @param opts [Hash] options to pass to the run_shell command
     # @return [String] the output of the run_shell command
     def run_shell(instance_name, cmd, opts = {})
-      CemAcpt::Platform::Gcp::Cmd.new(out_format: 'json').run_shell(instance_name, cmd, opts)
+      command_provider.run_shell(instance_name, cmd, opts)
     end
   end
 end

data/lib/cem_acpt/puppet_helpers.rb

@@ -21,6 +21,7 @@ module CemAcpt
       # @return [String] Path to the built package.
       def self.build_module_package(module_dir, target_dir = nil, should_log: false)
         require 'puppet/modulebuilder'
+        require 'fileutils'
 
         builder_logger = should_log ? logger : nil
         builder = Puppet::Modulebuilder::Builder.new(File.expand_path(module_dir), target_dir, builder_logger)

data/lib/cem_acpt/rspec_utils.rb

@@ -0,0 +1,242 @@
+# frozen_string_literal: true
+
+require 'English'
+require 'open3'
+require 'pty'
+require 'shellwords'
+require_relative 'logging'
+
+module CemAcpt
+  # Module that provides methods and objects for running and managing RSpec
+  module RSpecUtils
+    class BundlerNotFoundError < StandardError; end
+    class RSpecNotFoundError < StandardError; end
+
+    # Holds and formats a RSpec command
+    class Command
+      include CemAcpt::LoggingAsync
+      attr_reader :debug, :format, :test_path, :use_bundler, :pty_pid
+
+      # @param opts [Hash] options hash for the RSpec command
+      # @option opts [String] :test_path The path (or glob path) to the test file(s) to run. If blank, runs all.
+      # @option opts [Hash] :format Format options for rspec where the key is the format (documentation, json, etc)
+      #   and the value is the out-file path. If you do not want to save the results of a format to a file, the
+      #   value should be `nil`.
+      # @option opts [Boolean] :debug True if RSpec should run in debug mode, false if not. Mutually exclusive with
+      #   `:quiet`. Default is `false`.
+      # @option opts [Boolean] :quiet True if no output should be logged from RSpec command, false if output should
+      #   be logged. Mutually exclusive with `:debug`. Default is `false`.
+      # @option opts [Boolean] :use_bundler Whether or not `bundle exec` should be used to run the RSpec command.
+      #   Default is `true`.
+      # @option opts [Boolean] :bundle_install Whether or not to run `bundle install` before the RSpec command
+      #   if `use_bundler` is `true`.
+      # @option opts [Boolean] :use_shell Whether or not to add `$SHELL` as a prefix to the command
+      # @option opts [Hash] :env Environment variables to prepend to the command
+      def initialize(opts = {})
+        @test_path = opts[:test_path]&.shellescape
+        @format = opts.fetch(:format, {})
+        @debug = opts.fetch(:debug, false)
+        @quiet = @debug ? false : opts.fetch(:quiet, false)
+        @use_bundler = opts.fetch(:use_bundler, false)
+        @bundle_install = opts.fetch(:bundle_install, false)
+        @env = opts.fetch(:env, {})
+        @pty_pid = nil
+        validate_and_set_bin_paths(opts)
+      end
+
+      # Sets debug mode to `true`
+      def set_debug
+        @debug = true
+        if @quiet
+          async_debug('Setting :quiet to false because :debug is now true.')
+          @quiet = false
+        end
+      end
+
+      # Sets debug mode to `false`
+      def unset_debug
+        @debug = false
+      end
+
+      def quiet
+        @quiet && !debug
+      end
+
+      # Adds a new format to the RSpec command
+      # @param fmt [String] The name of the format (i.e. "documentation", "json", etc.)
+      # @param out [String] If specified, saves the specified format to a file at this path
+      def with_format(fmt, out: nil)
+        @format[fmt.to_sym] = out
+      end
+
+      # Environment variables that will be used for the RSpec command
+      # @return [Hash] A Hash of environment variables with each key pair being: <var name> => <var value>
+      def env
+        @debug ? @env.merge({ 'RSPEC_DEBUG' => 'true' }) : @env
+      end
+
+      # Returns an array representation of the RSpec command
+      def to_a
+        cmd = cmd_base.dup
+        cmd << test_path if test_path
+        format.each do |fmt, out|
+          cmd += ['--format', fmt.to_s.shellescape]
+          cmd += ['--out', out.to_s.shellescape] if out
+        end
+        cmd.compact
+      end
+
+      # Returns a string representation of the RSpec command
+      def to_s
+        to_a.join(' ')
+      end
+
+      # Executes the RSpec command on the current machine
+      # @param pty [Boolean] If true, execute command in a PTY. If false, execute command directly.
+      # @param log_prefix [String] A prefix to add to log messages generated while the command is running.
+      def execute(pty: true, log_prefix: 'RSPEC')
+        if pty
+          execute_pty(log_prefix: log_prefix)
+        else
+          execute_no_pty(log_prefix: log_prefix)
+        end
+      end
+
+      # Executes the RSpec command in a psuedo-terminal (PTY). First, it spawns a process
+      # for $SHELL, sets environment variables `export_envs`, then calls the current RSpec
+      # command in the shell and exits with the last exit code `$?`. Output is read from the
+      # RSpec command in near real-time in a blocking manner unless the `:quiet` option has
+      # been specified.
+      # @param log_prefix [String] A prefix to add to the log messages that are output from
+      #   the RSpec command.
+      # @return [Integer] The exit code of the RSpec command
+      def execute_pty(log_prefix: 'RSPEC')
+        async_debug("Executing RSpec command '#{self}' in PTY...", log_prefix)
+        PTY.spawn(env, ENV['SHELL']) do |r, w, pid|
+          @pty_pid = pid
+          async_debug("Spawned RSpec PTY with PID #{@pty_pid}", log_prefix)
+          export_envs(w)
+          w.puts "#{self}; exit $?"
+          quiet ? wait_io(r) : read_io(r, log_prefix: log_prefix)
+        end
+        $CHILD_STATUS
+      end
+
+      # Executes the RSpec command using Open3.popen2e(). The output stream, which is both
+      # stderr and stdout, is read in real-time in a non-blocking manner.
+      # @param log_prefix [String] A prefix to add to the log messages that are output from
+      #   the RSpec command.
+      # @return [Integer] The exit code of the RSpec command
+      def execute_no_pty(log_prefix: 'RSPEC')
+        async_info("Executing RSpec command '#{self}' with Open3.popen2e()...", log_prefix)
+        exit_status = nil
+        Open3.popen2e(env, to_s) do |stdin, std_out_err, wait_thr|
+          stdin.close
+          quiet ? wait_io(std_out_err) : read_io(std_out_err, log_prefix: log_prefix)
+          exit_status = wait_thr.value
+        end
+        exit_status
+      end
+
+      # Kills the PTY process with `SIGKILL` if the process exists
+      def kill_pty
+        Process.kill('KILL', @pty_pid) unless @pty_pid.nil?
+      rescue Errno::ESRCH
+        true
+      end
+
+      private
+
+      # Detects if the current Ruby context is JRuby
+      def jruby?
+        File.basename(RbConfig.ruby) == 'jruby'
+      end
+
+      # The base RSpec command
+      def cmd_base
+        use_bundler ? cmd_base_bundler : cmd_base_rspec
+      end
+
+      # The base RSpec command if `:use_bundler` is `true`.
+      def cmd_base_bundler
+        base = [@bundle, 'exec', 'rspec']
+        base.unshift("#{@bundle} install;") if @bundle_install
+        base
+      end
+
+      # The base RSpec command if `:use_bundler` is `false`
+      def cmd_base_rspec
+        [@rspec]
+      end
+
+      # Puts export statements for each key-value pair in `env` to the given writer.
+      # Writer is the write pipe of a PTY session, or a similar IO object that can
+      # pass the statements to a shell.
+      # @param writer [IO] An IO object that supprts `puts` and can send statements to a shell
+      def export_envs(writer)
+        env.each do |key, val|
+          writer.puts "export #{key}=#{val}"
+        end
+      end
+
+      # Finds and sets the paths to the `bundle` and `rspec` binaries. The paths can
+      # be either passed in as options in the `opts` Hash or interrogated from the
+      # system.
+      # @param opts [Hash] The options hash
+      # @option opts [String] :bundle An absolute path on the system to the `bundle` binary.
+      # @option opts [String] :rspec An absolute path on the system to the `rspec` binary.
+      # @raise [CemAcpt::RSpecUtils::BundlerNotFoundError] if `@use_bundler` is true and
+      #   `bundle` binary is not found.
+      # @raise [CemAcpt::RSpecUtils::RSpecNotFoundError] if `rspec` binary is not found.
+      def validate_and_set_bin_paths(opts = {})
+        %i[bundle rspec].each do |bin|
+          bin_path = opts[bin] || `command -v #{bin}`.strip
+          bin_not_found(bin, bin_path) unless bin_path && File.exist?(bin_path)
+          instance_variable_set("@#{bin}", bin_path)
+        end
+      end
+
+      # Handles binary paths which are not found
+      # @param bin [Symbol] The binary that was not found, either :bundle or :rspec.
+      # @param bin_path [String] The path to the binary that was checked.
+      # @raise [CemAcpt::RSpecUtils::BundlerNotFoundError] if `@use_bundler` is true and
+      #   `bundle` binary is not found.
+      # @raise [CemAcpt::RSpecUtils::RSpecNotFoundError] if `rspec` binary is not found.
+      # @raise [RuntimeError] if `bin` is not :bundle or :rspec.
+      def bin_not_found(bin, bin_path)
+        msg_base = "#{bin} not found."
+        msg = bin_path.nil? ? "#{msg_base} Path is nil." : "#{msg_base} Path: #{bin_path}"
+        case bin
+        when :bundle
+          raise BundlerNotFoundError, msg if @use_bundler
+        when :rspec
+          raise RSpecNotFoundError, msg
+        else
+          raise "bin #{bin} not recognized!"
+        end
+      end
+
+      # Blocking wait on an IO stream. Wait stops once the IO stream has reached
+      # end of file.
+      # @param stdout [IO] An IO stream with output that can be read from.
+      def wait_io(stdout)
+        sleep(0.01) until stdout.eof?
+      end
+
+      # Reads and logs data from `stdout` in a near real-time, blocking manner.
+      # @param stdout [IO] An IO stream with output that can be read from.
+      # @param log_prefix [String] A string prefix that is added to log messages.
+      def read_io(stdout, log_prefix: 'RSPEC')
+        loop do
+          chunk = stdout.read_nonblock(4096).strip
+          async_info(chunk, log_prefix) unless chunk.nil? || chunk.empty?
+        rescue IO::WaitReadable
+          IO.select([stdout])
+          retry
+        rescue EOFError
+          break
+        end
+      end
+    end
+  end
+end

data/lib/cem_acpt/shared_objects.rb

@@ -3,7 +3,6 @@
 require 'concurrent-ruby'
 require 'deep_merge'
 require 'json'
-require 'ostruct'
 require 'yaml'
 require_relative 'core_extensions'
 require_relative 'logging'
@@ -49,7 +48,16 @@ module CemAcpt
     attr_reader :config_file
 
     def initialize
-      @opts = {}
+      @opts = {
+        thread_pool: {
+          min_threads: [2, Concurrent.processor_count - 1].max,
+          max_threads: [2, Concurrent.processor_count - 1].max,
+          max_queue: [2, Concurrent.processor_count - 1].max * 10,
+          fallback_policy: :caller_runs,
+          idletime: 1200,
+          auto_terminate: true,
+        },
+      }
     end
 
     # Returns the value of the dot-separated key.
@@ -67,18 +75,25 @@ module CemAcpt
       !!get(dot_key)
     end
 
+    # Checks to see if cem_acpt is set to run in debug mode (log level is set to debug)
+    # @return [TrueClass] If cem_acpt is running in debug mode
+    # @return [FalseClass] If cem_acpt is not runnint in debug mode
+    def debug_mode?
+      @opts.dot_dig('log_level') == 'debug'
+    end
+
     # Loads the config from specified file path. Config files must be in YAML
     # format. If 'opts' is specified, it will be merged with the config file.
     # Values in 'opts' will override values in the config file.
     # @param opts [Hash] Options to be merged with the config file
     # @param config_file [String] Path to the config file
-    def load(opts: {}, config_file: '~/.cem_acpt.yaml')
+    def load(opts: {}, config_file: './cem_acpt_config.yaml')
       raise ConfigImmutableError, 'Config is immutable, cannot load more than once' if frozen?
       raise ArgumentError, 'opts must be a Hash' unless opts.is_a?(Hash)
       raise ArgumentError, 'config_file must be a String' unless config_file.is_a?(String)
 
       @config_file = File.expand_path(config_file)
-      @opts = load_opts_from_file(File.expand_path(config_file)).deep_merge(opts)
+      @opts.deep_merge!(load_opts_from_file(File.expand_path(config_file)).deep_merge!(opts))
       @opts.format!
       @opts.freeze
       freeze
@@ -155,8 +170,13 @@ module CemAcpt
     end
   end
 
+  class NodeInventoryFileNotFoundError < StandardError; end
+  class NodeInventoryFileLoadError < StandardError; end
   class NodeClaimedError < StandardError; end
   class NodeDoesNotExistError < StandardError; end
+  class PropertyNotFoundError < StandardError; end
+  class LockWaitTimeoutError < StandardError; end
+  class LockNotRecognizedError < StandardError; end
 
   # Provides a thread-safe inventory of test nodes.
   class NodeInventory
@@ -169,7 +189,8 @@ module CemAcpt
       @lock = Concurrent::ReadWriteLock.new
       @claimed = Concurrent::Set.new
       @save_on_claim = false
-      @save_file_path = 'spec/fixtures/node_inventory.yaml'
+      @save_file_path = 'spec/fixtures/node_inventory'
+      @loaded_node_inv = nil
     end
 
     # When called, enables saving the inventory to a file on claim.
@@ -189,12 +210,51 @@ module CemAcpt
       @inventory.put_if_absent(node_name, node_data)
     end
 
+    def update(node_name, node_data)
+      @inventory.replace_if_exists(node_name, node_data)
+    end
+
     # Returns the node data for a given node.
     # @param node_name [String] The name of the node to get data for.
     def get(node_name)
       @inventory[node_name]
     end
 
+    # Returns the node_name and node_data for a given node
+    # that has a matching value for the given property.
+    # @param property_path [String] The dot-separated property path to check.
+    # @param value [Object] The value to check for.
+    # @return [Array] Two-item frozen array of node name and node data.
+    # @raise [NodeDoesNotExistError] If no nodes are found with the property equal to the value.
+    def get_by_property(property_path, value)
+      found = []
+      @inventory.each_pair do |node_name, node_data|
+        next unless node_data.dot_dig(property_path) == value
+
+        found = [node_name, node_data].freeze
+      end
+      raise NodeDoesNotExistError, "No node found with property #{property_path} == #{value}" if found.empty?
+
+      found
+    end
+
+    # Returns all property values of all nodes for the given property path
+    # @param property_path [String] The dot-separated property path to check.
+    # @return [Array] A frozen array of found property values.
+    # @raise [PropertyNotFoundError] If no values are found on any nodes for the property path.
+    def get_all_properties(property_path)
+      found = []
+      @inventory.each_pair do |_, node_data|
+        prop = node_data.dot_dig(property_path)
+        next unless prop
+
+        found << prop
+      end
+      raise PropertyNotFoundError, "No property with path #{property_path} found in nodes" if found.empty?
+
+      found.freeze
+    end
+
     # Sets a specific property on a node in the inventory.
     # @param node_name [String] The name of the node to set the property on.
     # @param property_path [String] The dot-separated property path to set.
@@ -230,7 +290,7 @@ module CemAcpt
     # @raise [NodeDoesNotExistError] If the node does not exist in the inventory.
     # @raise [NodeClaimedError] If the node is already claimed.
     def claim(node_name)
-      @lock.with_write_lock do
+      with_lock_retry(:write) do
         unless @inventory.keys.include?(node_name)
           raise NodeDoesNotExistError, "Node #{node_name} does not exist in inventory"
         end
@@ -254,16 +314,24 @@ module CemAcpt
     # @return [String] The name of the node that was claimed.
     # @raise [NodeDoesNotExistError] If no valid node is found.
     def claim_by_property(property_path, value)
-      claimed_set = @claimed.dup
+      attempts ||= 1
       claim_name = nil
       @inventory.each_pair do |node_name, node_data|
-        next if claimed_set.include?(node_name)
+        next if @claimed.include?(node_name)
 
         claim_name = node_name if node_data.dot_dig(property_path) == value
       end
-      return claim(claim_name) unless claim_name.nil?
-
-      raise NodeDoesNotExistError, "No node found with property #{property_path} == #{value}"
+      raise NodeDoesNotExistError, "No node found with property #{property_path} == #{value}" if claim_name.nil?
+
+      claim(claim_name)
+    rescue NodeDoesNotExistError => e
+      # We sleep than retry three times to help mitigate race conditions
+      if (attempts += 1) <= 3
+        sleep(1)
+        retry
+      else
+        raise e
+      end
     end
 
     # Deletes a node from the inventory.
@@ -286,7 +354,7 @@ module CemAcpt
 
     # Clears the inventory and removes claimed nodes. Thread-safe.
     def clear!
-      @lock.with_write_lock do
+      with_lock_retry(:write) do
         clear_no_lock!
       end
     end
@@ -308,6 +376,16 @@ module CemAcpt
       h
     end
 
+    # Returns a YAML string of a specific node's node data
+    def node_to_yaml(node_name)
+      get(node_name).to_h.to_yaml
+    end
+
+    # Returns a JSON string of a specific node's node data
+    def node_to_json(node_name, *args)
+      get(node_name).to_h.to_json(*args)
+    end
+
     # Returns a YAML string of the inventory.
     def to_yaml
       to_h.to_yaml
@@ -321,15 +399,21 @@ module CemAcpt
     # Saves the current node inventory to a file. DOES NOT USE LOCK.
     # If this is called outside of a lock, it will not be thread-safe.
     def save_no_lock!(file_path = @save_file_path)
-      File.open(file_path, 'w') do |file|
-        file.write(to_yaml)
+      Dir.mkdir(file_path) unless Dir.exist?(file_path)
+      @inventory.each_pair do |node_name, node_data|
+        path = File.join(file_path, "#{node_name}.yaml")
+        next if File.file?(path)
+
+        File.open(path, 'w') do |f|
+          f.write(node_data.to_h.to_yaml)
+        end
       end
     end
 
     # Saves the current node inventory to a yaml file. Thread-safe.
     # @param file_path [String] The path to the file to save to.
     def save(file_path = @save_file_path)
-      @lock.with_write_lock do
+      with_lock_retry(:write) do
         save_no_lock!(file_path)
       end
     end
@@ -338,23 +422,58 @@ module CemAcpt
     # If this is called outside of a lock, it will not be thread-safe.
     def load_no_lock!(file_path = @save_file_path)
       require 'net/ssh/proxy/command' # If ProxyCommand is used in ssh options, this is required.
-      clear_no_lock!
-      YAML.load_file(file_path).each_pair do |node_name, node_data|
-        if node_name == :claimed
-          @claimed = Set.new(node_data)
-        else
-          add(node_name, node_data)
-        end
+      attempts ||= 1
+      Dir.glob('*.yaml', base: file_path) do |file_name|
+        node_name = File.basename(file_name, '.yaml')
+        node_data = YAML.load_file(File.join(file_path, file_name))
+        add(node_name, node_data) || update(node_name, node_data)
       end
+    rescue StandardError => e
+      raise e unless (attempts += 1) <= 3
+
+      sleep(1)
+      retry
     end
 
+    def update_no_lock!; end
+
     # Loads a node inventory from a yaml file. Thread-safe.
     # @param file_path [String] The path to the file to load from.
     def load(file_path = @save_file_path)
-      @lock.with_write_lock do
+      with_lock_retry(:write) do
         load_no_lock!(file_path)
       end
     end
+
+    def clean_local_files(file_path = @save_file_path)
+      Dir.glob('*.yaml', base: file_path) do |file_name|
+        File.delete(File.expand_path(File.join(file_path, file_name)))
+      end
+    end
+
+    private
+
+    def with_lock_retry(lock_type, max_attempts: 3)
+      raise 'No block given' unless block_given?
+
+      attempts ||= 1
+      case lock_type
+      when :read
+        @lock.with_read_lock { yield }
+      when :write
+        @lock.with_write_lock { yield }
+      else
+        raise LockNotRecognizedError, "Lock type #{lock_type} not recognized! Must be :read or :write!"
+      end
+    rescue Concurrent::ResourceLimitError => e
+      if (attempts += 1) <= max_attempts
+        sleep(1)
+        logger.debug("Failed to acquire lock with error #{e.message}. Retrying...")
+        retry
+      else
+        raise LockWaitTimeoutError, 'Lock could not be aquired'
+      end
+    end
   end
 
   class LocalPortAllocationError < StandardError; end
@@ -391,9 +510,11 @@ module CemAcpt
 
         ports << port
       end
-      return ports[0] if ports.length == 1
-
-      ports
+      if ports.length == 1
+        ports[0]
+      else
+        ports
+      end
     end
 
     private

data/lib/cem_acpt/spec_helper_acceptance.rb

@@ -14,15 +14,12 @@ module CemAcpt
       config.extend CemAcpt::SpecHelperAcceptance
       config.add_setting :acpt_test_data, default: {}
       config.add_setting :acpt_node_inventory
+      config.threadsafe = true
+      config.color_mode = :off
+      config.fail_fast = false
     end
 
-    return unless RSpec.configuration.acpt_node_inventory.nil?
-
-    raise 'Node inventory file not found!' unless File.exist?('spec/fixtures/node_inventory.yaml')
-
     node_inventory = NodeInventory.new
-    node_inventory.save_on_claim
-    node_inventory.save_file_path = 'spec/fixtures/node_inventory.yaml'
     node_inventory.load
     RSpec.configuration.acpt_node_inventory = node_inventory
   end
@@ -35,8 +32,10 @@ module CemAcpt
     def initialize_test_environment!
       test_file = caller_locations.first.path
 
-      node_name = RSpec.configuration.acpt_node_inventory.claim_by_property('test_data.test_file', test_file)
-      node_data = RSpec.configuration.acpt_node_inventory.get(node_name)
+      node_name, node_data = RSpec.configuration.acpt_node_inventory.get_by_property('test_data.test_file', test_file)
+      raise "Failed to get node data for node #{node_name}" unless node_data
+      raise "Node data format is incorrect: #{node_data}" unless node_data.is_a?(Hash)
+
       backend = nil
       host = nil
       ssh_options = nil
@@ -81,13 +80,14 @@ module CemAcpt
       RSpec.configuration.acpt_test_data = acpt_test_data
     end
 
-    # This method formats Puppet Apply options 
+    # This method formats Puppet Apply options
     def puppet_apply_options(opts = {})
       if [opts[:catch_changes], opts[:expect_changes], opts[:catch_failures], opts[:expect_failures]].compact.length > 1
-        raise ArgumentError, 'Please specify only one of "catch_changes", "expect_changes", "catch_failures", or "expect_failures"'
+        raise ArgumentError,
+              'Please specify only one of "catch_changes", "expect_changes", "catch_failures", or "expect_failures"'
       end
 
-      apply_opts = { trace: true }.merge(opts)
+      apply_opts = {}.merge(opts)
 
       if opts[:catch_changes]
         apply_opts[:detailed_exit_codes] = true
@@ -169,8 +169,16 @@ module CemAcpt
     # asserts that the second run has no changes.
     def idempotent_apply(manifest, opts = {})
       opts.reject! { |k, _| %i[catch_changes expect_changes catch_failures expect_failures].include?(k) }
-      apply_manifest(manifest, opts.merge(catch_failures: true))
-      apply_manifest(manifest, opts.merge(catch_changes: true))
+      begin
+        apply_manifest(manifest, opts.merge({ catch_failures: true }))
+      rescue StandardError => e
+        raise "Idempotent apply failed during first apply: #{e.message}\n#{e.backtrace.join("\n")}"
+      end
+      begin
+        apply_manifest(manifest, opts.merge({ catch_changes: true }))
+      rescue StandardError => e
+        raise "Idempotent apply failed during second apply: #{e.message}\n#{e.backtrace.join("\n")}"
+      end
     end
   end
 end