puppet_litmus 0.17.0 → 0.18.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c81ccd0186b54db2eb71534a0cc10e5191e1b6103201e8ef44231cc017cf5faa
-  data.tar.gz: dd027dff77884a4b09b66447c966c3636a2cf237e21c1b4f0ec1d28182188937
+  metadata.gz: 60fc5eef658171b749a429d8d302e94c05e8812858206124e90c915a035a8e4e
+  data.tar.gz: 2016992ffee484e5c2ab2b92e0011d98678b21bc1c0e62dd1d75c857ad50db65
 SHA512:
-  metadata.gz: 38db1fa85c7ca655540aad80563bd095c8de46b492e618541a9dbd42b4a9305447ce2538c4ed30882f87582f4a21ebac7d0a1352288f3920d33b42af4d53fa4d
-  data.tar.gz: 849c989f388d9b1b4dde8a44a2788657c015c50bbfb8ff399255d64c3390557bca7e319939134a2bb2918f3ffc48b0c59a27c29ab7d0779d480c07f9a392a3dc
+  metadata.gz: f4bb7954ab40c5345dacb291f0fd82a745c7ac5da4e939e07323e23f6c06b7c4758303cc5f9faccf75f67134337ad409edb429ebb438ffe879682460722cacb3
+  data.tar.gz: 3367726403b6691edbb3439377617f894fdffec96f964965bb1e62aa175ade0494920ec166b49da367449c331a9ca6f8e7f87ed2ede707bb7cd7410077e63c38

data/README.md

@@ -8,27 +8,23 @@
 </div>
 
 ## Overview
-Litmus provides a simple command line tool for Puppet content creators, to enable both simple and complex test deployments against specifically configured target systems. It is available as a gem, and can be installed by running ```gem install puppet_litmus```.
 
-Litmus allows Puppet module developers to:
-* provision targets to test against,
-* install the Puppet Agent, 
-* install a module,
-* run tests, and
-* tear down the infrastructure.
+Litmus is a command line tool that allows you to run acceptance tests against Puppet modules.
 
-The tool facilitates parallel test runs, running tests in isolation, and each step is standalone, allowing other operations between test runs, such as debugging, or configuration updates on the test targets.
+Litmus allows you to:
+* Provision targets to test against
+* Install a Puppet agent
+* Install a module
+* Run tests
+* Tear down the infrastructure
 
-## Documentation
+Litmus also facilitates parallel test runs and running tests in isolation. Each step is standalone, allowing other operations between test runs, such as debugging or configuration updates on the test targets.
+
+Install Litmus as a gem by running ```gem install puppet_litmus```.
 
-All our documentation is currently available in the [Wiki](https://github.com/puppetlabs/puppet_litmus/wiki).
+## Documentation
 
-* [Overview](https://github.com/puppetlabs/puppet_litmus/wiki/Overview-of-Litmus) of the main functions
-* [Architecture](https://github.com/puppetlabs/puppet_litmus/wiki/Architecture-of-puppet-litmus) with an explanation of what's going on under the hood
-* [Step-by-step guide](https://github.com/puppetlabs/puppet_litmus/wiki/Tutorial:-use-Litmus-to-execute-acceptance-tests-with-a-sample-module-(MoTD)) of how to use Litmus with the popular and simple [MoTD Puppet module](https://forge.puppet.com/puppetlabs/motd).
-* [How to guide](https://github.com/puppetlabs/puppet_litmus/wiki/Converting-a-module-to-use-Litmus) walking through how to use Litmus in a module
-* [Helper functions](https://github.com/puppetlabs/puppet_litmus/wiki/Helper-Functions-for-Litmus) a guide to the various helper functions within Litmus.
-## Known issues
+For documentation, see our [Litmus Docs Site](https://puppetlabs.github.io/litmus/).
 
 ## Other Resources
 

data/lib/puppet_litmus/inventory_manipulation.rb

@@ -37,7 +37,6 @@ module PuppetLitmus::InventoryManipulation
               'uri' => 'litmus_localhost',
               'config' => { 'transport' => 'local' },
               'feature' => 'puppet-agent',
-              'facts' => { 'platform' => 'localhost' },
             },
           ],
         },
@@ -236,7 +235,7 @@ module PuppetLitmus::InventoryManipulation
   #
   # @param inventory_hash [Hash] hash of the inventory.yaml file
   # @param feature_name [String] feature to locate in the node
-  # node_name [String] node of nodes to limit the search for the node_name in
+  # @param node_name [String] node of nodes to limit the search for the node_name in
   # @return inventory.yaml file with feature removed from the node.
   # @return [Hash] inventory_hash with feature added to node if node_name exists in inventory hash.
   def remove_feature_from_node(inventory_hash, feature_name, node_name)
@@ -258,4 +257,13 @@ module PuppetLitmus::InventoryManipulation
   def write_to_inventory_file(inventory_hash, inventory_full_path)
     File.open(inventory_full_path, 'wb+') { |f| f.write(inventory_hash.to_yaml) }
   end
+
+  # Add the `litmus.platform` with platform information for the target
+  #
+  # @param inventory_hash [Hash] hash of the inventory.yaml file
+  # @param node_name [String] node of nodes to limit the search for the node_name in
+  def add_platform_field(inventory_hash, node_name)
+    facts = facts_from_node(inventory_hash, node_name)
+    Honeycomb.current_span.add_field('litmus.platform', facts&.dig('platform'))
+  end
 end

data/lib/puppet_litmus/puppet_helpers.rb

@@ -9,7 +9,7 @@ module PuppetLitmus::PuppetHelpers
   # @return [Boolean] The result of the 2 apply manifests.
   def idempotent_apply(manifest)
     Honeycomb.start_span(name: 'litmus.idempotent_apply') do |span|
-      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header unless ENV['HTTP_X_HONEYCOMB_TRACE']
+      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header
       manifest_file_location = create_manifest_file(manifest)
       apply_manifest(nil, expect_failures: false, manifest_file_location: manifest_file_location)
       apply_manifest(nil, catch_changes: true, manifest_file_location: manifest_file_location)
@@ -32,13 +32,14 @@ module PuppetLitmus::PuppetHelpers
   #  :manifest_file_location [Path] The place on the target system.
   #  :hiera_config [Path] The path to the hiera.yaml configuration on the runner.
   #  :prefix_command [String] prefixes the puppet apply command; eg "export LANGUAGE='ja'".
+  #  :trace [Boolean] run puppet apply with the trace flag (defaults to `true`).
   #  :debug [Boolean] run puppet apply with the debug flag.
   #  :noop [Boolean] run puppet apply with the noop flag.
   # @yieldreturn [Block] this method will yield to a block of code passed by the caller; this can be used for additional validation, etc.
   # @return [Object] A result object from the apply.
   def apply_manifest(manifest, opts = {})
     Honeycomb.start_span(name: 'litmus.apply_manifest') do |span|
-      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header unless ENV['HTTP_X_HONEYCOMB_TRACE']
+      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header
       span.add_field('litmus.manifest', manifest)
       span.add_field('litmus.opts', opts)
 
@@ -48,6 +49,8 @@ module PuppetLitmus::PuppetHelpers
       raise 'please specify only one of `catch_changes`, `expect_changes`, `catch_failures` or `expect_failures`' if
         [opts[:catch_changes], opts[:expect_changes], opts[:catch_failures], opts[:expect_failures]].compact.length > 1
 
+      opts = { trace: true }.merge(opts)
+
       if opts[:catch_changes]
         use_detailed_exit_codes = true
         acceptable_exit_codes = [0]
@@ -70,9 +73,10 @@ module PuppetLitmus::PuppetHelpers
       raise "Target '#{target_node_name}' not found in inventory.yaml" unless target_in_inventory?(inventory_hash, target_node_name)
 
       span.add_field('litmus.node_name', target_node_name)
-      span.add_field('litmus.platform', facts_from_node(inventory_hash, target_node_name)['platform'])
+      add_platform_field(inventory_hash, target_node_name)
 
       command_to_run = "#{opts[:prefix_command]} puppet apply #{manifest_file_location}"
+      command_to_run += ' --trace' if !opts[:trace].nil? && (opts[:trace] == true)
       command_to_run += " --modulepath #{Dir.pwd}/spec/fixtures/modules" if target_node_name == 'litmus_localhost'
       command_to_run += " --hiera_config='#{opts[:hiera_config]}'" unless opts[:hiera_config].nil?
       command_to_run += ' --debug' if !opts[:debug].nil? && (opts[:debug] == true)
@@ -84,9 +88,9 @@ module PuppetLitmus::PuppetHelpers
       bolt_result = run_command(command_to_run, target_node_name, config: nil, inventory: inventory_hash)
       span.add_field('litmus.bolt_result', bolt_result)
 
-      result = OpenStruct.new(exit_code: bolt_result.first['result']['exit_code'],
-                              stdout: bolt_result.first['result']['stdout'],
-                              stderr: bolt_result.first['result']['stderr'])
+      result = OpenStruct.new(exit_code: bolt_result.first['value']['exit_code'],
+                              stdout: bolt_result.first['value']['stdout'],
+                              stderr: bolt_result.first['value']['stderr'])
       span.add_field('litmus.result', result.to_h)
 
       status = result.exit_code
@@ -113,7 +117,7 @@ module PuppetLitmus::PuppetHelpers
   # @return [String] The path to the location of the manifest.
   def create_manifest_file(manifest)
     Honeycomb.start_span(name: 'litmus.create_manifest_file') do |span|
-      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header unless ENV['HTTP_X_HONEYCOMB_TRACE']
+      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header
       span.add_field('litmus.manifest', manifest)
 
       require 'tmpdir'
@@ -129,12 +133,12 @@ module PuppetLitmus::PuppetHelpers
         # transfer to TARGET_HOST
         inventory_hash = inventory_hash_from_inventory_file
         span.add_field('litmus.node_name', target_node_name)
-        span.add_field('litmus.platform', facts_from_node(inventory_hash, target_node_name)['platform'])
+        add_platform_field(inventory_hash, target_node_name)
 
-        manifest_file_location = "/tmp/#{File.basename(manifest_file)}"
+        manifest_file_location = File.basename(manifest_file)
         bolt_result = upload_file(manifest_file.path, manifest_file_location, target_node_name, options: {}, config: nil, inventory: inventory_hash)
         span.add_field('litmus.bolt_result', bolt_result)
-        raise bolt_result.first['result'].to_s unless bolt_result.first['status'] == 'success'
+        raise bolt_result.first['value'].to_s unless bolt_result.first['status'] == 'success'
       end
 
       span.add_field('litmus.manifest_file_location', manifest_file_location)
@@ -151,7 +155,7 @@ module PuppetLitmus::PuppetHelpers
   # @return [Object] A result object from the command.
   def run_shell(command_to_run, opts = {})
     Honeycomb.start_span(name: 'litmus.run_shell') do |span|
-      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header unless ENV['HTTP_X_HONEYCOMB_TRACE']
+      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header
       span.add_field('litmus.command_to_run', command_to_run)
       span.add_field('litmus.opts', opts)
 
@@ -160,19 +164,19 @@ module PuppetLitmus::PuppetHelpers
       raise "Target '#{target_node_name}' not found in inventory.yaml" unless target_in_inventory?(inventory_hash, target_node_name)
 
       span.add_field('litmus.node_name', target_node_name)
-      span.add_field('litmus.platform', facts_from_node(inventory_hash, target_node_name)['platform'])
+      add_platform_field(inventory_hash, target_node_name)
 
       bolt_result = run_command(command_to_run, target_node_name, config: nil, inventory: inventory_hash)
       span.add_field('litmus.bolt_result', bolt_result)
 
-      if bolt_result.first['result']['exit_code'] != 0 && opts[:expect_failures] != true
+      if bolt_result.first['value']['exit_code'] != 0 && opts[:expect_failures] != true
         raise "shell failed\n`#{command_to_run}`\n======\n#{bolt_result}"
       end
 
-      result = OpenStruct.new(exit_code: bolt_result.first['result']['exit_code'],
-                              exit_status: bolt_result.first['result']['exit_code'],
-                              stdout: bolt_result.first['result']['stdout'],
-                              stderr: bolt_result.first['result']['stderr'])
+      result = OpenStruct.new(exit_code: bolt_result.first['value']['exit_code'],
+                              exit_status: bolt_result.first['value']['exit_code'],
+                              stdout: bolt_result.first['value']['stdout'],
+                              stderr: bolt_result.first['value']['stderr'])
       span.add_field('litmus.result', result.to_h)
       yield result if block_given?
       result
@@ -188,7 +192,7 @@ module PuppetLitmus::PuppetHelpers
   # @return [Object] A result object from the command.
   def bolt_upload_file(source, destination, opts = {}, options = {})
     Honeycomb.start_span(name: 'litmus.bolt_upload_file') do |span|
-      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header unless ENV['HTTP_X_HONEYCOMB_TRACE']
+      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header
       span.add_field('litmus.source', source)
       span.add_field('litmus.destination', destination)
       span.add_field('litmus.opts', opts)
@@ -199,16 +203,16 @@ module PuppetLitmus::PuppetHelpers
       raise "Target '#{target_node_name}' not found in inventory.yaml" unless target_in_inventory?(inventory_hash, target_node_name)
 
       span.add_field('litmus.node_name', target_node_name)
-      span.add_field('litmus.platform', facts_from_node(inventory_hash, target_node_name)['platform'])
+      add_platform_field(inventory_hash, target_node_name)
 
       bolt_result = upload_file(source, destination, target_node_name, options: options, config: nil, inventory: inventory_hash)
       span.add_field('litmus.bolt_result', bolt_result)
 
       result_obj = {
         exit_code: 0,
-        stdout: bolt_result.first['result']['_output'],
+        stdout: bolt_result.first['value']['_output'],
         stderr: nil,
-        result: bolt_result.first['result'],
+        result: bolt_result.first['value'],
       }
 
       if bolt_result.first['status'] != 'success'
@@ -218,7 +222,7 @@ module PuppetLitmus::PuppetHelpers
         end
 
         result_obj[:exit_code] = 255
-        result_obj[:stderr]    = bolt_result.first['result']['_error']['msg']
+        result_obj[:stderr]    = bolt_result.first['value']['_error']['msg']
       end
 
       result = OpenStruct.new(exit_code: result_obj[:exit_code],
@@ -240,7 +244,7 @@ module PuppetLitmus::PuppetHelpers
   # @return [Object] A result object from the task.The values available are stdout, stderr and result.
   def run_bolt_task(task_name, params = {}, opts = {})
     Honeycomb.start_span(name: 'litmus.run_task') do |span|
-      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header unless ENV['HTTP_X_HONEYCOMB_TRACE']
+      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header
       span.add_field('litmus.task_name', task_name)
       span.add_field('litmus.params', params)
       span.add_field('litmus.opts', opts)
@@ -257,22 +261,22 @@ module PuppetLitmus::PuppetHelpers
       raise "Target '#{target_node_name}' not found in inventory.yaml" unless target_in_inventory?(inventory_hash, target_node_name)
 
       span.add_field('litmus.node_name', target_node_name)
-      span.add_field('litmus.platform', facts_from_node(inventory_hash, target_node_name)['platform'])
+      add_platform_field(inventory_hash, target_node_name)
 
       bolt_result = run_task(task_name, target_node_name, params, config: config_data, inventory: inventory_hash)
       result_obj = {
         exit_code: 0,
         stdout: nil,
         stderr: nil,
-        result: bolt_result.first['result'],
+        result: bolt_result.first['value'],
       }
 
       if bolt_result.first['status'] == 'success'
         # stdout returns unstructured data if structured data is not available
-        result_obj[:stdout] = if bolt_result.first['result']['_output'].nil?
-                                bolt_result.first['result'].to_s
+        result_obj[:stdout] = if bolt_result.first['value']['_output'].nil?
+                                bolt_result.first['value'].to_s
                               else
-                                bolt_result.first['result']['_output']
+                                bolt_result.first['value']['_output']
                               end
 
       else
@@ -281,12 +285,12 @@ module PuppetLitmus::PuppetHelpers
           raise "task failed\n`#{task_name}`\n======\n#{bolt_result}"
         end
 
-        result_obj[:exit_code] = if bolt_result.first['result']['_error']['details'].nil?
+        result_obj[:exit_code] = if bolt_result.first['value']['_error']['details'].nil?
                                    255
                                  else
-                                   bolt_result.first['result']['_error']['details'].fetch('exitcode', 255)
+                                   bolt_result.first['value']['_error']['details'].fetch('exitcode', 255)
                                  end
-        result_obj[:stderr]    = bolt_result.first['result']['_error']['msg']
+        result_obj[:stderr]    = bolt_result.first['value']['_error']['msg']
       end
 
       result = OpenStruct.new(exit_code: result_obj[:exit_code],
@@ -308,7 +312,7 @@ module PuppetLitmus::PuppetHelpers
   # @return [Object] A result object from the script run.
   def bolt_run_script(script, opts = {}, arguments: [])
     Honeycomb.start_span(name: 'litmus.bolt_run_script') do |span|
-      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header unless ENV['HTTP_X_HONEYCOMB_TRACE']
+      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header
       span.add_field('litmus.script', script)
       span.add_field('litmus.opts', opts)
       span.add_field('litmus.arguments', arguments)
@@ -318,18 +322,18 @@ module PuppetLitmus::PuppetHelpers
       raise "Target '#{target_node_name}' not found in inventory.yaml" unless target_in_inventory?(inventory_hash, target_node_name)
 
       span.add_field('litmus.node_name', target_node_name)
-      span.add_field('litmus.platform', facts_from_node(inventory_hash, target_node_name)['platform'])
+      add_platform_field(inventory_hash, target_node_name)
 
       bolt_result = run_script(script, target_node_name, arguments, options: opts, config: nil, inventory: inventory_hash)
 
-      if bolt_result.first['result']['exit_code'] != 0 && opts[:expect_failures] != true
+      if bolt_result.first['value']['exit_code'] != 0 && opts[:expect_failures] != true
         span.add_field('litmus_runscriptfailure', bolt_result)
         raise "script run failed\n`#{script}`\n======\n#{bolt_result}"
       end
 
-      result = OpenStruct.new(exit_code: bolt_result.first['result']['exit_code'],
-                              stdout: bolt_result.first['result']['stdout'],
-                              stderr: bolt_result.first['result']['stderr'])
+      result = OpenStruct.new(exit_code: bolt_result.first['value']['exit_code'],
+                              stdout: bolt_result.first['value']['stdout'],
+                              stderr: bolt_result.first['value']['stderr'])
       yield result if block_given?
       span.add_field('litmus.result', result.to_h)
       result
@@ -353,7 +357,7 @@ module PuppetLitmus::PuppetHelpers
     puppet_apply_error = <<~ERROR
       apply manifest failed
       `#{command}`
-      with exit code #{bolt_result.first['result']['exit_code']} (expected: #{acceptable_exit_codes})
+      with exit code #{bolt_result.first['value']['exit_code']} (expected: #{acceptable_exit_codes})
       ====== Start output of failed Puppet apply ======
       #{puppet_output(bolt_result)}
       ====== End output of failed Puppet apply ======
@@ -378,8 +382,8 @@ module PuppetLitmus::PuppetHelpers
 
   # Return the stdout of the puppet run
   def puppet_output(bolt_result)
-    bolt_result.dig(0, 'result', 'stderr').to_s << \
-      bolt_result.dig(0, 'result', 'stdout').to_s
+    bolt_result.dig(0, 'value', 'stderr').to_s << \
+      bolt_result.dig(0, 'value', 'stdout').to_s
   end
 
   # Checks a puppet return status and returns true if it both

data/lib/puppet_litmus/rake_helper.rb

@@ -1,13 +1,22 @@
 # frozen_string_literal: true
 
-module PuppetLitmus; end # rubocop:disable Style/Documentation
-
+require 'bolt_spec/run'
 require 'honeycomb-beeline'
+require 'puppet_litmus/version'
 Honeycomb.configure do |config|
+  # override client if no configuration is provided, so that the pesky libhoney warning about lack of configuration is not shown
+  unless ENV['HONEYCOMB_WRITEKEY'] && ENV['HONEYCOMB_DATASET']
+    config.client = Libhoney::NullClient.new
+  end
 end
-process_span = Honeycomb.start_span(name: 'Litmus Testing', serialized_trace: ENV['HTTP_X_HONEYCOMB_TRACE'])
-ENV['HTTP_X_HONEYCOMB_TRACE'] = process_span.to_trace_header unless ENV['HTTP_X_HONEYCOMB_TRACE']
+process_span = Honeycomb.start_span(name: "litmus: #{([$PROGRAM_NAME] + ($ARGV || [])).join(' ')}", serialized_trace: ENV['HTTP_X_HONEYCOMB_TRACE'])
+ENV['HTTP_X_HONEYCOMB_TRACE'] = process_span.to_trace_header
 Honeycomb.add_field_to_trace('litmus.pid', Process.pid)
+if defined? PuppetLitmus::VERSION
+  Honeycomb.add_field_to_trace('litmus.version', PuppetLitmus::VERSION)
+else
+  Honeycomb.add_field_to_trace('litmus.version', 'undefined')
+end
 if ENV['CI'] == 'true' && ENV['TRAVIS'] == 'true'
   Honeycomb.add_field_to_trace('module_name', ENV['TRAVIS_REPO_SLUG'])
   Honeycomb.add_field_to_trace('ci.provider', 'travis')
@@ -37,7 +46,8 @@ end
 
 # helper methods for the litmus rake tasks
 module PuppetLitmus::RakeHelper
-  DEFAULT_CONFIG_DATA ||= { 'modulepath' => File.join(Dir.pwd, 'spec', 'fixtures', 'modules') }.freeze
+  # DEFAULT_CONFIG_DATA should be frozen for our safety, but it needs to work around https://github.com/puppetlabs/bolt/pull/1696
+  DEFAULT_CONFIG_DATA ||= { 'modulepath' => File.join(Dir.pwd, 'spec', 'fixtures', 'modules') } # .freeze # rubocop:disable Style/MutableConstant
   SUPPORTED_PROVISIONERS ||= %w[abs docker docker_exp vagrant vmpooler].freeze
 
   # Gets a string representing the operating system and version.
@@ -86,7 +96,7 @@ module PuppetLitmus::RakeHelper
   # @return [Object] the standard out stream.
   def run_local_command(command)
     Honeycomb.start_span(name: 'litmus.run_local_command') do |span|
-      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header unless ENV['HTTP_X_HONEYCOMB_TRACE']
+      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header
       span.add_field('litmus.command', command)
 
       require 'open3'
@@ -99,32 +109,8 @@ module PuppetLitmus::RakeHelper
     end
   end
 
-  # Builds all the modules in a specified module
-  #
-  # @param source_folder [String] the folder to get the modules from
-  # @return [Array] an array of module tar's
-  def build_modules_in_folder(source_folder)
-    folder_list = Dir.entries(source_folder).reject { |f| File.directory? f }
-    module_tars = []
-    folder_list.each do |folder|
-      folder_handle = Dir.open(File.join(source_folder, folder))
-      next if File.symlink?(folder_handle)
-
-      module_dir = folder_handle.path
-      target_dir = File.join(Dir.pwd, 'pkg')
-      # remove old build folder if exists, before we build afresh
-      FileUtils.rm_rf(target_dir) if File.directory?(target_dir)
-
-      # build_module
-      module_tar = build_module(module_dir, target_dir)
-      module_tars.push(File.new(module_tar))
-    end
-    module_tars
-  end
-
   def provision(provisioner, platform, inventory_vars)
-    require 'bolt_spec/run'
-    include BoltSpec::Run
+    include ::BoltSpec::Run
     raise "the provision module was not found in #{DEFAULT_CONFIG_DATA['modulepath']}, please amend the .fixtures.yml file" unless
       File.directory?(File.join(DEFAULT_CONFIG_DATA['modulepath'], 'provision'))
 
@@ -133,14 +119,14 @@ module PuppetLitmus::RakeHelper
 
     Honeycomb.add_field_to_trace('litmus.provisioner', provisioner)
     Honeycomb.start_span(name: 'litmus.provision') do |span|
-      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header unless ENV['HTTP_X_HONEYCOMB_TRACE']
+      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header
       span.add_field('litmus.platform', platform)
       span.add_field('litmus.inventory', params['inventory'])
       span.add_field('litmus.config', DEFAULT_CONFIG_DATA)
 
       bolt_result = run_task(provisioner_task(provisioner), 'localhost', params, config: DEFAULT_CONFIG_DATA, inventory: nil)
-
-      span.add_field('litmus.node_name', bolt_result&.first&.dig('result', 'node_name'))
+      span.add_field('litmus.node_name', bolt_result&.first&.dig('value', 'node_name'))
+      raise_bolt_errors(bolt_result, "provisioning of #{platform} failed.")
 
       bolt_result
     end
@@ -162,11 +148,10 @@ module PuppetLitmus::RakeHelper
 
   def tear_down_nodes(targets, inventory_hash)
     Honeycomb.start_span(name: 'litmus.tear_down_nodes') do |span|
-      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header unless ENV['HTTP_X_HONEYCOMB_TRACE']
+      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header
       span.add_field('litmus.targets', targets)
 
-      require 'bolt_spec/run'
-      include BoltSpec::Run
+      include ::BoltSpec::Run
       config_data = { 'modulepath' => File.join(Dir.pwd, 'spec', 'fixtures', 'modules') }
       raise "the provision module was not found in #{config_data['modulepath']}, please amend the .fixtures.yml file" unless File.directory?(File.join(config_data['modulepath'], 'provision'))
 
@@ -183,26 +168,27 @@ module PuppetLitmus::RakeHelper
 
   def tear_down(node_name, inventory_hash)
     Honeycomb.start_span(name: 'litmus.tear_down') do |span|
-      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header unless ENV['HTTP_X_HONEYCOMB_TRACE']
+      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header
       # how do we know what provisioner to use
-      node_facts = facts_from_node(inventory_hash, node_name)
 
       span.add_field('litmus.node_name', node_name)
-      span.add_field('litmus.platform', node_facts['platform'])
+      add_platform_field(inventory_hash, node_name)
 
       params = { 'action' => 'tear_down', 'node_name' => node_name, 'inventory' => Dir.pwd }
-      run_task(provisioner_task(node_facts['provisioner']), 'localhost', params, config: DEFAULT_CONFIG_DATA, inventory: nil)
+      node_facts = facts_from_node(inventory_hash, node_name)
+      bolt_result = run_task(provisioner_task(node_facts['provisioner']), 'localhost', params, config: DEFAULT_CONFIG_DATA, inventory: nil)
+      raise_bolt_errors(bolt_result, "tear_down of #{node_name} failed.")
+      bolt_result
     end
   end
 
   def install_agent(collection, targets, inventory_hash)
     Honeycomb.start_span(name: 'litmus.install_agent') do |span|
-      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header unless ENV['HTTP_X_HONEYCOMB_TRACE']
+      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header
       span.add_field('litmus.collection', collection)
       span.add_field('litmus.targets', targets)
 
-      require 'bolt_spec/run'
-      include BoltSpec::Run
+      include ::BoltSpec::Run
       params = if collection.nil?
                  {}
                else
@@ -213,7 +199,9 @@ module PuppetLitmus::RakeHelper
        unless File.directory?(File.join(DEFAULT_CONFIG_DATA['modulepath'], 'puppet_agent'))
 
       # using boltspec, when the runner is called it changes the inventory_hash dropping the version field. The clone works around this
-      run_task('puppet_agent::install', targets, params, config: DEFAULT_CONFIG_DATA, inventory: inventory_hash.clone)
+      bolt_result = run_task('puppet_agent::install', targets, params, config: DEFAULT_CONFIG_DATA, inventory: inventory_hash.clone)
+      raise_bolt_errors(bolt_result, 'Installation of agent failed.')
+      bolt_result
     end
   end
 
@@ -227,40 +215,86 @@ module PuppetLitmus::RakeHelper
     results
   end
 
+  # Build the module in `module_dir` and put the resulting compressed tarball into `target_dir`.
+  #
   # @param opts Hash of options to build the module
   # @param module_dir [String] The path of the module to build. If missing defaults to Dir.pwd
-  # @param target_dir [String] The path the module will be built into. The default is <source_dir>/pkg
+  # @param target_dir [String] The path the module will be built into. The default is <module_dir>/pkg
   # @return [String] The path to the built module
   def build_module(module_dir = nil, target_dir = nil)
     require 'puppet/modulebuilder'
 
-    source_dir = module_dir || Dir.pwd
-    dest_dir = target_dir || File.join(source_dir, 'pkg')
+    module_dir ||= Dir.pwd
+    target_dir ||= File.join(source_dir, 'pkg')
+
+    puts "Building '#{module_dir}' into '#{target_dir}''"
+    builder = Puppet::Modulebuilder::Builder.new(module_dir, target_dir, nil)
 
-    builder = Puppet::Modulebuilder::Builder.new(source_dir, dest_dir, nil)
     # Force the metadata to be read. Raises if metadata could not be found
     _metadata = builder.metadata
 
     builder.build
   end
 
-  def install_module(inventory_hash, target_node_name, module_tar)
-    require 'bolt_spec/run'
-    include BoltSpec::Run
-    target_nodes = find_targets(inventory_hash, target_node_name)
-    target_string = if target_node_name.nil?
-                      'all'
-                    else
-                      target_node_name
-                    end
-    run_local_command("bundle exec bolt file upload \"#{module_tar}\" /tmp/#{File.basename(module_tar)} --nodes #{target_string} --inventoryfile inventory.yaml")
-    install_module_command = "puppet module install /tmp/#{File.basename(module_tar)}"
+  # Builds all the modules in a specified directory
+  #
+  # @param source_dir [String] the directory to get the modules from
+  # @param target_dir [String] temporary location to store tarballs before uploading. This directory will be cleaned before use. The default is <source_dir>/pkg
+  # @return [Array] an array of module tars' filenames
+  def build_modules_in_dir(source_dir, target_dir = nil)
+    target_dir ||= File.join(Dir.pwd, 'pkg')
+    # remove old build dir if exists, before we build afresh
+    FileUtils.rm_rf(target_dir) if File.directory?(target_dir)
+
+    module_tars = Dir.entries(source_dir).map do |entry|
+      next if ['.', '..'].include? entry
+
+      module_dir = File.join(source_dir, entry)
+      next unless File.directory? module_dir
+
+      build_module(module_dir, target_dir)
+    end
+    module_tars.compact
+  end
+
+  # @deprecated Use `build_modules_in_dir` instead
+  def build_modules_in_folder(source_folder)
+    build_modules_in_dir(source_folder)
+  end
+
+  # Install a specific module tarball to the specified target.
+  # This method installs dependencies using a forge repository.
+  #
+  # @param inventory_hash [Hash] the pre-loaded inventory
+  # @param target_node_name [String] the name of the target where the module should be installed
+  # @param module_tar [String] the filename of the module tarball to upload
+  # @param module_repository [String] the URL for the forge to use for downloading modules. Defaults to the public Forge API.
+  # @return a bolt result
+  def install_module(inventory_hash, target_node_name, module_tar, module_repository = nil)
     Honeycomb.start_span(name: 'install_module') do |span|
-      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header unless ENV['HTTP_X_HONEYCOMB_TRACE']
-      span.add_field('litmus.install_module_command', install_module_command)
+      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header
+      span.add_field('litmus.target_node_name', target_node_name)
+      span.add_field('litmus.module_tar', module_tar)
+
+      # make sure the module to install is not installed
+      # otherwise `puppet module install` might silently skip it
+      module_name = File.basename(module_tar, '.tar.gz').split('-', 3)[0..1].join('-')
+      uninstall_module(inventory_hash.clone, target_node_name, module_name, force: true)
+
+      include ::BoltSpec::Run
+
+      target_nodes = find_targets(inventory_hash, target_node_name)
       span.add_field('litmus.target_nodes', target_nodes)
+      bolt_result = upload_file(module_tar, File.basename(module_tar), target_nodes, options: {}, config: nil, inventory: inventory_hash.clone)
+      raise_bolt_errors(bolt_result, 'Failed to upload module.')
+
+      module_repository_opts = "--module_repository '#{module_repository}'" unless module_repository.nil?
+      install_module_command = "puppet module install #{module_repository_opts} #{File.basename(module_tar)}"
+      span.add_field('litmus.install_module_command', install_module_command)
 
-      run_command(install_module_command, target_nodes, config: nil, inventory: inventory_hash)
+      bolt_result = run_command(install_module_command, target_nodes, config: nil, inventory: inventory_hash.clone)
+      raise_bolt_errors(bolt_result, "Installation of package #{File.basename(module_tar)} failed.")
+      bolt_result
     end
   end
 
@@ -274,26 +308,33 @@ module PuppetLitmus::RakeHelper
     metadata['name']
   end
 
-  def uninstall_module(inventory_hash, target_node_name, module_to_remove = nil)
-    require 'bolt_spec/run'
-    include BoltSpec::Run
+  # Uninstall a module from a specified target
+  # @param inventory_hash [Hash] the pre-loaded inventory
+  # @param target_node_name [String] the name of the target where the module should be uninstalled
+  # @param module_to_remove [String] the name of the module to remove. Defaults to the module under test.
+  # @param opts [Hash] additional options to pass on to `puppet module uninstall`
+  def uninstall_module(inventory_hash, target_node_name, module_to_remove = nil, **opts)
+    include ::BoltSpec::Run
     module_name = module_to_remove || metadata_module_name
     target_nodes = find_targets(inventory_hash, target_node_name)
     install_module_command = "puppet module uninstall #{module_name}"
-    run_command(install_module_command, target_nodes, config: nil, inventory: inventory_hash)
+    install_module_command += ' --force' if opts[:force]
+    bolt_result = run_command(install_module_command, target_nodes, config: nil, inventory: inventory_hash)
+    # `puppet module uninstall --force` fails if the module is not installed. Ignore errors when force is set
+    raise_bolt_errors(bolt_result, "uninstalling #{module_name} failed.") unless opts[:force]
+    bolt_result
   end
 
   def check_connectivity?(inventory_hash, target_node_name)
     Honeycomb.start_span(name: 'litmus.check_connectivity') do |span|
-      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header unless ENV['HTTP_X_HONEYCOMB_TRACE']
+      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header
       # if we're only checking connectivity for a single node
       if target_node_name
         span.add_field('litmus.node_name', target_node_name)
-        span.add_field('litmus.platform', facts_from_node(inventory_hash, target_node_name)['platform'])
+        add_platform_field(inventory_hash, target_node_name)
       end
 
-      require 'bolt_spec/run'
-      include BoltSpec::Run
+      include ::BoltSpec::Run
       target_nodes = find_targets(inventory_hash, target_node_name)
       results = run_command('cd .', target_nodes, config: nil, inventory: inventory_hash)
       span.add_field('litmus.bolt_result', results)
@@ -316,4 +357,38 @@ module PuppetLitmus::RakeHelper
       provisioner.to_s
     end
   end
+
+  # Parse out errors messages in result set returned by Bolt command.
+  #
+  # @param result_set [Array] result set returned by Bolt command.
+  # @return [Hash] Errors grouped by target.
+  def check_bolt_errors(result_set)
+    errors = {}
+    # iterate through each error
+    result_set.each do |target_result|
+      status = target_result['status']
+      # jump to the next one when there is not fail
+      next if status != 'failure'
+
+      target = target_result['target']
+      # get some info from error
+      errors[target] = target_result['value']
+    end
+    errors
+  end
+
+  # Parse out errors messages in result set returned by Bolt command. If there are errors, raise them.
+  #
+  # @param result_set [Array] result set returned by Bolt command.
+  # @param error_msg [String] error message to raise when errors are detected. The actual errors will be appended.
+  def raise_bolt_errors(result_set, error_msg)
+    errors = check_bolt_errors(result_set)
+
+    unless errors.empty?
+      formatted_results = errors.map { |k, v| "  #{k}: #{v.inspect}" }.join("\n")
+      raise "#{error_msg}\nResults:\n#{formatted_results}}"
+    end
+
+    nil
+  end
 end