puppet_litmus 0.16.0 → 0.18.3

data/lib/puppet_litmus/rake_helper.rb

@@ -1,11 +1,50 @@
 # frozen_string_literal: true
 
-module PuppetLitmus; end # rubocop:disable Style/Documentation
+require 'bolt_spec/run'
+require 'honeycomb-beeline'
+require 'puppet_litmus/version'
+Honeycomb.configure do |config|
+  # override client if no configuration is provided, so that the pesky libhoney warning about lack of configuration is not shown
+  unless ENV['HONEYCOMB_WRITEKEY'] && ENV['HONEYCOMB_DATASET']
+    config.client = Libhoney::NullClient.new
+  end
+end
+process_span = Honeycomb.start_span(name: "litmus: #{([$PROGRAM_NAME] + ($ARGV || [])).join(' ')}", serialized_trace: ENV['HTTP_X_HONEYCOMB_TRACE'])
+ENV['HTTP_X_HONEYCOMB_TRACE'] = process_span.to_trace_header
+Honeycomb.add_field_to_trace('litmus.pid', Process.pid)
+Honeycomb.add_field_to_trace('litmus.version', PuppetLitmus::VERSION)
+if ENV['CI'] == 'true' && ENV['TRAVIS'] == 'true'
+  Honeycomb.add_field_to_trace('module_name', ENV['TRAVIS_REPO_SLUG'])
+  Honeycomb.add_field_to_trace('ci.provider', 'travis')
+  Honeycomb.add_field_to_trace('ci.build_id', ENV['TRAVIS_BUILD_ID'])
+  Honeycomb.add_field_to_trace('ci.build_url', ENV['TRAVIS_BUILD_WEB_URL'])
+  Honeycomb.add_field_to_trace('ci.job_url', ENV['TRAVIS_JOB_WEB_URL'])
+  Honeycomb.add_field_to_trace('ci.commit_message', ENV['TRAVIS_COMMIT_MESSAGE'])
+  Honeycomb.add_field_to_trace('ci.sha', ENV['TRAVIS_PULL_REQUEST_SHA'] || ENV['TRAVIS_COMMIT'])
+elsif ENV['CI'] == 'True' && ENV['APPVEYOR'] == 'True'
+  Honeycomb.add_field_to_trace('module_name', ENV['APPVEYOR_PROJECT_SLUG'])
+  Honeycomb.add_field_to_trace('ci.provider', 'appveyor')
+  Honeycomb.add_field_to_trace('ci.build_id', ENV['APPVEYOR_BUILD_ID'])
+  Honeycomb.add_field_to_trace('ci.build_url', "https://ci.appveyor.com/project/#{ENV['APPVEYOR_REPO_NAME']}/builds/#{ENV['APPVEYOR_BUILD_ID']}")
+  Honeycomb.add_field_to_trace('ci.job_url', "https://ci.appveyor.com/project/#{ENV['APPVEYOR_REPO_NAME']}/build/job/#{ENV['APPVEYOR_JOB_ID']}")
+  Honeycomb.add_field_to_trace('ci.commit_message', ENV['APPVEYOR_REPO_COMMIT_MESSAGE'])
+  Honeycomb.add_field_to_trace('ci.sha', ENV['APPVEYOR_PULL_REQUEST_HEAD_COMMIT'] || ENV['APPVEYOR_REPO_COMMIT'])
+elsif ENV['GITHUB_ACTIONS'] == 'true'
+  Honeycomb.add_field_to_trace('module_name', ENV['GITHUB_REPOSITORY'])
+  Honeycomb.add_field_to_trace('ci.provider', 'github')
+  Honeycomb.add_field_to_trace('ci.build_id', ENV['GITHUB_RUN_ID'])
+  Honeycomb.add_field_to_trace('ci.build_url', "https://github.com/#{ENV['GITHUB_REPOSITORY']}/actions/runs/#{ENV['GITHUB_RUN_ID']}")
+  Honeycomb.add_field_to_trace('ci.sha', ENV['GITHUB_SHA'])
+end
+at_exit do
+  process_span.send
+end
 
 # helper methods for the litmus rake tasks
 module PuppetLitmus::RakeHelper
-  DEFAULT_CONFIG_DATA ||= { 'modulepath' => File.join(Dir.pwd, 'spec', 'fixtures', 'modules') }.freeze
-  VALID_PROVISIONERS ||= %w[abs docker docker_exp vagrant vmpooler].freeze
+  # DEFAULT_CONFIG_DATA should be frozen for our safety, but it needs to work around https://github.com/puppetlabs/bolt/pull/1696
+  DEFAULT_CONFIG_DATA ||= { 'modulepath' => File.join(Dir.pwd, 'spec', 'fixtures', 'modules') } # .freeze # rubocop:disable Style/MutableConstant
+  SUPPORTED_PROVISIONERS ||= %w[abs docker docker_exp vagrant vmpooler].freeze
 
   # Gets a string representing the operating system and version.
   #
@@ -52,53 +91,41 @@ module PuppetLitmus::RakeHelper
   # @param command [String] command to execute.
   # @return [Object] the standard out stream.
   def run_local_command(command)
-    require 'open3'
-    stdout, stderr, status = Open3.capture3(command)
-    error_message = "Attempted to run\ncommand:'#{command}'\nstdout:#{stdout}\nstderr:#{stderr}"
-    raise error_message unless status.to_i.zero?
+    Honeycomb.start_span(name: 'litmus.run_local_command') do |span|
+      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header
+      span.add_field('litmus.command', command)
 
-    stdout
-  end
+      require 'open3'
+      stdout, stderr, status = Open3.capture3(command)
+      error_message = "Attempted to run\ncommand:'#{command}'\nstdout:#{stdout}\nstderr:#{stderr}"
 
-  # Builds all the modules in a specified module
-  #
-  # @param source_folder [String] the folder to get the modules from
-  # @return [Array] an array of module tar's
-  def build_modules_in_folder(source_folder)
-    folder_list = Dir.entries(source_folder).reject { |f| File.directory? f }
-    module_tars = []
-    folder_list.each do |folder|
-      folder_handle = Dir.open(File.join(source_folder, folder))
-      next if File.symlink?(folder_handle)
-
-      module_dir = folder_handle.path
-      target_dir = File.join(Dir.pwd, 'pkg')
-      # remove old build folder if exists, before we build afresh
-      FileUtils.rm_rf(target_dir) if File.directory?(target_dir)
-
-      # build_module
-      module_tar = build_module(module_dir, target_dir)
-      module_tars.push(File.new(module_tar))
+      raise error_message unless status.to_i.zero?
+
+      stdout
     end
-    module_tars
   end
 
   def provision(provisioner, platform, inventory_vars)
-    require 'bolt_spec/run'
-    include BoltSpec::Run
+    include ::BoltSpec::Run
     raise "the provision module was not found in #{DEFAULT_CONFIG_DATA['modulepath']}, please amend the .fixtures.yml file" unless
       File.directory?(File.join(DEFAULT_CONFIG_DATA['modulepath'], 'provision'))
 
-    unless VALID_PROVISIONERS.include?(provisioner)
-      raise "Unknown provisioner '#{provisioner}', try #{VALID_PROVISIONERS.join('/')}"
-    end
+    params = { 'action' => 'provision', 'platform' => platform, 'inventory' => Dir.pwd }
+    params['vars'] = inventory_vars unless inventory_vars.nil?
+
+    Honeycomb.add_field_to_trace('litmus.provisioner', provisioner)
+    Honeycomb.start_span(name: 'litmus.provision') do |span|
+      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header
+      span.add_field('litmus.platform', platform)
+      span.add_field('litmus.inventory', params['inventory'])
+      span.add_field('litmus.config', DEFAULT_CONFIG_DATA)
 
-    params = if inventory_vars.nil?
-               { 'action' => 'provision', 'platform' => platform, 'inventory' => Dir.pwd }
-             else
-               { 'action' => 'provision', 'platform' => platform, 'inventory' => Dir.pwd, 'vars' => inventory_vars }
-             end
-    run_task("provision::#{provisioner}", 'localhost', params, config: DEFAULT_CONFIG_DATA, inventory: nil)
+      bolt_result = run_task(provisioner_task(provisioner), 'localhost', params, config: DEFAULT_CONFIG_DATA, inventory: nil)
+      span.add_field('litmus.node_name', bolt_result&.first&.dig('value', 'node_name'))
+      raise_bolt_errors(bolt_result, "provisioning of #{platform} failed.")
+
+      bolt_result
+    end
   end
 
   def provision_list(provision_hash, key)
@@ -107,6 +134,8 @@ module PuppetLitmus::RakeHelper
     # Splat the params into environment variables to pass to the provision task but only in this runspace
     provision_hash[key]['params']&.each { |k, value| ENV[k.upcase] = value.to_s }
     results = []
+
+    Honeycomb.current_span.add_field('litmus.images', provision_hash[key]['images'])
     provision_hash[key]['images'].each do |image|
       results << provision(provisioner, image, inventory_vars)
     end
@@ -114,42 +143,62 @@ module PuppetLitmus::RakeHelper
   end
 
   def tear_down_nodes(targets, inventory_hash)
-    require 'bolt_spec/run'
-    include BoltSpec::Run
-    config_data = { 'modulepath' => File.join(Dir.pwd, 'spec', 'fixtures', 'modules') }
-    raise "the provision module was not found in #{config_data['modulepath']}, please amend the .fixtures.yml file" unless File.directory?(File.join(config_data['modulepath'], 'provision'))
+    Honeycomb.start_span(name: 'litmus.tear_down_nodes') do |span|
+      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header
+      span.add_field('litmus.targets', targets)
 
-    results = {}
-    targets.each do |node_name|
-      next if node_name == 'litmus_localhost'
+      include ::BoltSpec::Run
+      config_data = { 'modulepath' => File.join(Dir.pwd, 'spec', 'fixtures', 'modules') }
+      raise "the provision module was not found in #{config_data['modulepath']}, please amend the .fixtures.yml file" unless File.directory?(File.join(config_data['modulepath'], 'provision'))
 
-      result = tear_down(node_name, inventory_hash)
-      results[node_name] = result unless result == []
+      results = {}
+      targets.each do |node_name|
+        next if node_name == 'litmus_localhost'
+
+        result = tear_down(node_name, inventory_hash)
+        results[node_name] = result unless result == []
+      end
+      results
     end
-    results
   end
 
   def tear_down(node_name, inventory_hash)
-    # how do we know what provisioner to use
-    node_facts = facts_from_node(inventory_hash, node_name)
-    return [] unless VALID_PROVISIONERS.include?(node_facts['provisioner'])
-
-    params = { 'action' => 'tear_down', 'node_name' => node_name, 'inventory' => Dir.pwd }
-    run_task("provision::#{node_facts['provisioner']}", 'localhost', params, config: DEFAULT_CONFIG_DATA, inventory: nil)
+    Honeycomb.start_span(name: 'litmus.tear_down') do |span|
+      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header
+      # how do we know what provisioner to use
+
+      span.add_field('litmus.node_name', node_name)
+      add_platform_field(inventory_hash, node_name)
+
+      params = { 'action' => 'tear_down', 'node_name' => node_name, 'inventory' => Dir.pwd }
+      node_facts = facts_from_node(inventory_hash, node_name)
+      bolt_result = run_task(provisioner_task(node_facts['provisioner']), 'localhost', params, config: DEFAULT_CONFIG_DATA, inventory: nil)
+      raise_bolt_errors(bolt_result, "tear_down of #{node_name} failed.")
+      bolt_result
+    end
   end
 
   def install_agent(collection, targets, inventory_hash)
-    require 'bolt_spec/run'
-    include BoltSpec::Run
-    params = if collection.nil?
-               {}
-             else
-               { 'collection' => collection }
-             end
-    raise "puppet_agent was not found in #{DEFAULT_CONFIG_DATA['modulepath']}, please amend the .fixtures.yml file" unless File.directory?(File.join(DEFAULT_CONFIG_DATA['modulepath'], 'puppet_agent'))
-
-    # using boltspec, when the runner is called it changes the inventory_hash dropping the version field. The clone works around this
-    run_task('puppet_agent::install', targets, params, config: DEFAULT_CONFIG_DATA, inventory: inventory_hash.clone)
+    Honeycomb.start_span(name: 'litmus.install_agent') do |span|
+      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header
+      span.add_field('litmus.collection', collection)
+      span.add_field('litmus.targets', targets)
+
+      include ::BoltSpec::Run
+      params = if collection.nil?
+                 {}
+               else
+                 Honeycomb.current_span.add_field('litmus.collection', collection)
+                 { 'collection' => collection }
+               end
+      raise "puppet_agent was not found in #{DEFAULT_CONFIG_DATA['modulepath']}, please amend the .fixtures.yml file" \
+       unless File.directory?(File.join(DEFAULT_CONFIG_DATA['modulepath'], 'puppet_agent'))
+
+      # using boltspec, when the runner is called it changes the inventory_hash dropping the version field. The clone works around this
+      bolt_result = run_task('puppet_agent::install', targets, params, config: DEFAULT_CONFIG_DATA, inventory: inventory_hash.clone)
+      raise_bolt_errors(bolt_result, 'Installation of agent failed.')
+      bolt_result
+    end
   end
 
   def configure_path(inventory_hash)
@@ -162,35 +211,87 @@ module PuppetLitmus::RakeHelper
     results
   end
 
+  # Build the module in `module_dir` and put the resulting compressed tarball into `target_dir`.
+  #
   # @param opts Hash of options to build the module
   # @param module_dir [String] The path of the module to build. If missing defaults to Dir.pwd
-  # @param target_dir [String] The path the module will be built into. The default is <source_dir>/pkg
+  # @param target_dir [String] The path the module will be built into. The default is <module_dir>/pkg
   # @return [String] The path to the built module
   def build_module(module_dir = nil, target_dir = nil)
     require 'puppet/modulebuilder'
 
-    source_dir = module_dir || Dir.pwd
-    dest_dir = target_dir || File.join(source_dir, 'pkg')
+    module_dir ||= Dir.pwd
+    target_dir ||= File.join(source_dir, 'pkg')
+
+    puts "Building '#{module_dir}' into '#{target_dir}''"
+    builder = Puppet::Modulebuilder::Builder.new(module_dir, target_dir, nil)
 
-    builder = Puppet::Modulebuilder::Builder.new(source_dir, dest_dir, nil)
     # Force the metadata to be read. Raises if metadata could not be found
     _metadata = builder.metadata
 
     builder.build
   end
 
-  def install_module(inventory_hash, target_node_name, module_tar)
-    require 'bolt_spec/run'
-    include BoltSpec::Run
-    target_nodes = find_targets(inventory_hash, target_node_name)
-    target_string = if target_node_name.nil?
-                      'all'
-                    else
-                      target_node_name
-                    end
-    run_local_command("bundle exec bolt file upload \"#{module_tar}\" /tmp/#{File.basename(module_tar)} --nodes #{target_string} --inventoryfile inventory.yaml")
-    install_module_command = "puppet module install /tmp/#{File.basename(module_tar)}"
-    run_command(install_module_command, target_nodes, config: nil, inventory: inventory_hash)
+  # Builds all the modules in a specified directory
+  #
+  # @param source_dir [String] the directory to get the modules from
+  # @param target_dir [String] temporary location to store tarballs before uploading. This directory will be cleaned before use. The default is <source_dir>/pkg
+  # @return [Array] an array of module tars' filenames
+  def build_modules_in_dir(source_dir, target_dir = nil)
+    target_dir ||= File.join(Dir.pwd, 'pkg')
+    # remove old build dir if exists, before we build afresh
+    FileUtils.rm_rf(target_dir) if File.directory?(target_dir)
+
+    module_tars = Dir.entries(source_dir).map do |entry|
+      next if ['.', '..'].include? entry
+
+      module_dir = File.join(source_dir, entry)
+      next unless File.directory? module_dir
+
+      build_module(module_dir, target_dir)
+    end
+    module_tars.compact
+  end
+
+  # @deprecated Use `build_modules_in_dir` instead
+  def build_modules_in_folder(source_folder)
+    build_modules_in_dir(source_folder)
+  end
+
+  # Install a specific module tarball to the specified target.
+  # This method installs dependencies using a forge repository.
+  #
+  # @param inventory_hash [Hash] the pre-loaded inventory
+  # @param target_node_name [String] the name of the target where the module should be installed
+  # @param module_tar [String] the filename of the module tarball to upload
+  # @param module_repository [String] the URL for the forge to use for downloading modules. Defaults to the public Forge API.
+  # @return a bolt result
+  def install_module(inventory_hash, target_node_name, module_tar, module_repository = nil)
+    Honeycomb.start_span(name: 'install_module') do |span|
+      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header
+      span.add_field('litmus.target_node_name', target_node_name)
+      span.add_field('litmus.module_tar', module_tar)
+
+      # make sure the module to install is not installed
+      # otherwise `puppet module install` might silently skip it
+      module_name = File.basename(module_tar, '.tar.gz').split('-', 3)[0..1].join('-')
+      uninstall_module(inventory_hash.clone, target_node_name, module_name, force: true)
+
+      include ::BoltSpec::Run
+
+      target_nodes = find_targets(inventory_hash, target_node_name)
+      span.add_field('litmus.target_nodes', target_nodes)
+      bolt_result = upload_file(module_tar, File.basename(module_tar), target_nodes, options: {}, config: nil, inventory: inventory_hash.clone)
+      raise_bolt_errors(bolt_result, 'Failed to upload module.')
+
+      module_repository_opts = "--module_repository '#{module_repository}'" unless module_repository.nil?
+      install_module_command = "puppet module install #{module_repository_opts} #{File.basename(module_tar)}"
+      span.add_field('litmus.install_module_command', install_module_command)
+
+      bolt_result = run_command(install_module_command, target_nodes, config: nil, inventory: inventory_hash.clone)
+      raise_bolt_errors(bolt_result, "Installation of package #{File.basename(module_tar)} failed.")
+      bolt_result
+    end
   end
 
   def metadata_module_name
@@ -203,26 +304,87 @@ module PuppetLitmus::RakeHelper
     metadata['name']
   end
 
-  def uninstall_module(inventory_hash, target_node_name, module_to_remove = nil)
-    require 'bolt_spec/run'
-    include BoltSpec::Run
+  # Uninstall a module from a specified target
+  # @param inventory_hash [Hash] the pre-loaded inventory
+  # @param target_node_name [String] the name of the target where the module should be uninstalled
+  # @param module_to_remove [String] the name of the module to remove. Defaults to the module under test.
+  # @param opts [Hash] additional options to pass on to `puppet module uninstall`
+  def uninstall_module(inventory_hash, target_node_name, module_to_remove = nil, **opts)
+    include ::BoltSpec::Run
     module_name = module_to_remove || metadata_module_name
     target_nodes = find_targets(inventory_hash, target_node_name)
     install_module_command = "puppet module uninstall #{module_name}"
-    run_command(install_module_command, target_nodes, config: nil, inventory: inventory_hash)
+    install_module_command += ' --force' if opts[:force]
+    bolt_result = run_command(install_module_command, target_nodes, config: nil, inventory: inventory_hash)
+    # `puppet module uninstall --force` fails if the module is not installed. Ignore errors when force is set
+    raise_bolt_errors(bolt_result, "uninstalling #{module_name} failed.") unless opts[:force]
+    bolt_result
   end
 
   def check_connectivity?(inventory_hash, target_node_name)
-    require 'bolt_spec/run'
-    include BoltSpec::Run
-    target_nodes = find_targets(inventory_hash, target_node_name)
-    results = run_command('cd .', target_nodes, config: nil, inventory: inventory_hash)
-    failed = []
-    results.each do |result|
-      failed.push(result['target']) if result['status'] == 'failure'
+    Honeycomb.start_span(name: 'litmus.check_connectivity') do |span|
+      ENV['HTTP_X_HONEYCOMB_TRACE'] = span.to_trace_header
+      # if we're only checking connectivity for a single node
+      if target_node_name
+        span.add_field('litmus.node_name', target_node_name)
+        add_platform_field(inventory_hash, target_node_name)
+      end
+
+      include ::BoltSpec::Run
+      target_nodes = find_targets(inventory_hash, target_node_name)
+      results = run_command('cd .', target_nodes, config: nil, inventory: inventory_hash)
+      span.add_field('litmus.bolt_result', results)
+      failed = []
+      results.each do |result|
+        failed.push(result['target']) if result['status'] == 'failure'
+      end
+      span.add_field('litmus.connectivity_failed', failed)
+      raise "Connectivity has failed on: #{failed}" unless failed.length.zero?
+
+      true
+    end
+  end
+
+  def provisioner_task(provisioner)
+    if SUPPORTED_PROVISIONERS.include?(provisioner)
+      "provision::#{provisioner}"
+    else
+      warn "WARNING: Unsuported provisioner '#{provisioner}', try #{SUPPORTED_PROVISIONERS.join('/')}"
+      provisioner.to_s
+    end
+  end
+
+  # Parse out errors messages in result set returned by Bolt command.
+  #
+  # @param result_set [Array] result set returned by Bolt command.
+  # @return [Hash] Errors grouped by target.
+  def check_bolt_errors(result_set)
+    errors = {}
+    # iterate through each error
+    result_set.each do |target_result|
+      status = target_result['status']
+      # jump to the next one when there is not fail
+      next if status != 'failure'
+
+      target = target_result['target']
+      # get some info from error
+      errors[target] = target_result['value']
+    end
+    errors
+  end
+
+  # Parse out errors messages in result set returned by Bolt command. If there are errors, raise them.
+  #
+  # @param result_set [Array] result set returned by Bolt command.
+  # @param error_msg [String] error message to raise when errors are detected. The actual errors will be appended.
+  def raise_bolt_errors(result_set, error_msg)
+    errors = check_bolt_errors(result_set)
+
+    unless errors.empty?
+      formatted_results = errors.map { |k, v| "  #{k}: #{v.inspect}" }.join("\n")
+      raise "#{error_msg}\nResults:\n#{formatted_results}}"
     end
-    raise "Connectivity has failed on: #{failed}" unless failed.length.zero?
 
-    true
+    nil
   end
 end

data/lib/puppet_litmus/rake_tasks.rb

@@ -17,15 +17,15 @@ namespace :litmus do
   end
 
   # Provisions a list of OSes from provision.yaml file e.g. 'bundle exec rake litmus:provision_list[default]'.
-  # @See https://github.com/puppetlabs/puppet_litmus/wiki/Overview-of-Litmus#provisioning-via-yaml
+  # @See https://puppetlabs.github.io/litmus/Litmus-core-commands.html#provisioning-via-yaml
   #
   # @param :key [String] key that maps to a value for a provisioner and an image to be used for each OS provisioned.
-  desc "provision list of machines from provision.yaml file. 'bundle exec rake 'litmus:provision_list[default]'"
+  desc 'provision list of machines from provision.yaml file'
   task :provision_list, [:key] do |_task, args|
     raise 'Cannot find provision.yaml file' unless File.file?('./provision.yaml')
 
     provision_hash = YAML.load_file('./provision.yaml')
-    raise "No key #{args[:key]} in ./provision.yaml, see https://github.com/puppetlabs/puppet_litmus/wiki/Overview-of-Litmus#provisioning-via-yaml for examples" if provision_hash[args[:key]].nil?
+    raise "No key #{args[:key]} in ./provision.yaml, see https://puppetlabs.github.io/litmus/Litmus-core-commands.html#provisioning-via-yaml for examples" if provision_hash[args[:key]].nil?
 
     Rake::Task['spec_prep'].invoke
 
@@ -57,9 +57,9 @@ namespace :litmus do
       end
 
       if result.first['status'] != 'success'
-        failed_image_message += "=====\n#{result.first['node']}\n#{result.first['result']['_output']}\n#{result.inspect}"
+        failed_image_message += "=====\n#{result.first['target']}\n#{result.first['value']['_output']}\n#{result.inspect}"
       else
-        STDOUT.puts "#{result.first['result']['node_name']}, #{image}"
+        STDOUT.puts "#{result.first['value']['node_name']}, #{image}"
       end
       results << result
     end
@@ -71,7 +71,7 @@ namespace :litmus do
   #
   # @param :provisioner [String] provisioner to use in provisioning given platform.
   # @param :platform [String] OS platform for container or VM to use.
-  desc "provision container/VM - abs/docker/vagrant/vmpooler eg 'bundle exec rake 'litmus:provision[vmpooler, ubuntu-1604-x86_64]'"
+  desc 'provision a test system using the given provisioner and platform name. See the puppetlabs-provision module tasks for more documentation'
   task :provision, [:provisioner, :platform, :inventory_vars] do |_task, args|
     Rake::Task['spec_prep'].invoke
     if (ENV['CI'] == 'true') || !ENV['DISTELLI_BUILDNUM'].nil?
@@ -96,14 +96,14 @@ namespace :litmus do
     else
       spinner.success
     end
-    puts "#{results.first['result']['node_name']}, #{args[:platform]}"
+    puts "#{results.first['value']['node_name']}, #{args[:platform]}"
   end
 
   # Install puppet agent on a collection of nodes
   #
   # @param :collection [String] parameters to pass to the puppet agent install command.
   # @param :target_node_name [Array] nodes on which to install puppet agent.
-  desc 'install puppet agent, [:collection, :target_node_name]'
+  desc 'install a puppet agent to all or a specified set of targets'
   task :install_agent, [:collection, :target_node_name] do |_task, args|
     inventory_hash = inventory_hash_from_inventory_file
     targets = find_targets(inventory_hash, args[:target_node_name])
@@ -119,11 +119,11 @@ namespace :litmus do
     results = install_agent(args[:collection], targets, inventory_hash)
     results.each do |result|
       if result['status'] != 'success'
-        command_to_run = "bolt task run puppet_agent::install --targets #{result['node']} --inventoryfile inventory.yaml --modulepath #{DEFAULT_CONFIG_DATA['modulepath']}"
-        raise "Failed on #{result['node']}\n#{result}\ntry running '#{command_to_run}'"
+        command_to_run = "bolt task run puppet_agent::install --targets #{result['target']} --inventoryfile inventory.yaml --modulepath #{DEFAULT_CONFIG_DATA['modulepath']}"
+        raise "Failed on #{result['target']}\n#{result}\ntry running '#{command_to_run}'"
       else
         # add puppet-agent feature to successful nodes
-        inventory_hash = add_feature_to_node(inventory_hash, 'puppet-agent', result['node'])
+        inventory_hash = add_feature_to_node(inventory_hash, 'puppet-agent', result['target'])
       end
     end
     # update the inventory with the puppet-agent feature set per node
@@ -134,7 +134,7 @@ namespace :litmus do
 
     results.each do |result|
       if result['status'] != 'success'
-        puts "Failed on #{result['node']}\n#{result}"
+        puts "Failed on #{result['target']}\n#{result}"
       end
     end
   end
@@ -143,7 +143,7 @@ namespace :litmus do
   #
   # @param :target_node_name [Array] nodes on which to add the feature.
   # @param :added_feature [String] the feature which you wish to add.
-  desc 'add_feature, [:added_feature, :target_node_name]'
+  desc 'add a feature tag to a node'
   task :add_feature, [:added_feature, :target_node_name] do |_task, args|
     inventory_hash = inventory_hash_from_inventory_file
     targets = find_targets(inventory_hash, args[:target_node_name])
@@ -166,65 +166,69 @@ namespace :litmus do
     puts 'Feature added'
   end
 
-  # Install the puppet modules from a source directory to nodes. It does not install dependencies.
+  # Install the puppet module under test on a collection of nodes
   #
-  # @param :source [String] source directory to look in (ignores symlinks) defaults do './spec/fixtures/modules'.
   # @param :target_node_name [Array] nodes on which to install a puppet module for testing.
-  desc 'install_module - build and install module'
-  task :install_modules_from_directory, [:source, :target_node_name] do |_task, args|
+  desc 'build the module under test and install it onto targets'
+  task :install_module, [:target_node_name, :module_repository] do |_task, args|
+    args.with_defaults(target_node_name: nil, module_repository: nil)
     inventory_hash = inventory_hash_from_inventory_file
     target_nodes = find_targets(inventory_hash, args[:target_node_name])
     if target_nodes.empty?
       puts 'No targets found'
       exit 0
     end
-    source_folder = if args[:source].nil?
-                      './spec/fixtures/modules'
-                    else
-                      File.expand_path(args[:source])
-                    end
-    raise "Source folder doesnt exist #{source_folder}" unless File.directory?(source_folder)
-
-    module_tars = build_modules_in_folder(source_folder)
-    puts 'Building'
-    module_tars.each do |module_tar|
-      print "#{File.basename(module_tar)} "
-    end
-    require 'bolt_spec/run'
-    include BoltSpec::Run
-    puts "\nSending"
-    module_tars.each do |module_tar|
-      upload_file(module_tar.path, "/tmp/#{File.basename(module_tar)}", target_nodes, options: {}, config: nil, inventory: inventory_hash)
-      print "#{File.basename(module_tar)} "
-    end
-    puts "\nInstalling"
-    module_tars.each do |module_tar|
-      # install_module
-      install_module_command = "puppet module install --force /tmp/#{File.basename(module_tar)}"
-      run_command(install_module_command, target_nodes, config: nil, inventory: inventory_hash)
-      print "#{File.basename(module_tar)} "
-    end
+
+    module_tar = build_module
+    puts "Built '#{module_tar}'"
+
+    # module_tar = Dir.glob('pkg/*.tar.gz').max_by { |f| File.mtime(f) }
+    raise "Unable to find package in 'pkg/*.tar.gz'" if module_tar.nil?
+
+    install_module(inventory_hash, args[:target_node_name], module_tar, args[:module_repository])
+
+    puts "Installed '#{module_tar}' on #{args[:target_node_name]}"
   end
 
-  # Check that the nodes in the inventory are still contactable
+  # Install the puppet modules from a source directory to nodes. It does not install dependencies.
   #
-  # @param :target_node_name [Array] nodes on which to check connnectivity
-  desc 'check_connectivity - build and install module'
-  task :check_connectivity, [:target_node_name] do |_task, args|
+  # @param :source [String] source directory to look in (ignores symlinks) defaults do './spec/fixtures/modules'.
+  # @param :target_node_name [Array] nodes on which to install a puppet module for testing.
+  desc 'build and install all modules from a directory'
+  task :install_modules_from_directory, [:source, :target_node_name, :module_repository] do |_task, args|
+    args.with_defaults(source: nil, target_node_name: nil, module_repository: nil)
     inventory_hash = inventory_hash_from_inventory_file
     target_nodes = find_targets(inventory_hash, args[:target_node_name])
     if target_nodes.empty?
       puts 'No targets found'
       exit 0
     end
-    check_connectivity?(inventory_hash, args[:target_node_name])
+    source_dir = if args[:source].nil?
+                   './spec/fixtures/modules'
+                 else
+                   File.expand_path(args[:source])
+                 end
+    raise "Source directory doesn't exist #{source_dir}" unless File.directory?(source_dir)
+
+    puts "Building all modules in #{source_dir.inspect}"
+    module_tars = build_modules_in_dir(source_dir)
+    require 'bolt_spec/run'
+    include BoltSpec::Run
+    module_tars.each do |module_tar|
+      puts "Installing '#{module_tar}'"
+      target_nodes.each do |target_node_name|
+        install_module(inventory_hash, target_node_name, module_tar, args[:module_repository])
+        puts "Installed '#{module_tar}' on #{target_node_name}"
+      end
+    end
   end
 
-  # Install the puppet module under test on a collection of nodes
+  # Uninstall the puppet module under test on a collection of nodes
   #
   # @param :target_node_name [Array] nodes on which to install a puppet module for testing.
-  desc 'install_module - build and install module'
-  task :install_module, [:target_node_name] do |_task, args|
+  # @param :module_name [String] module name to be uninstalled
+  desc 'uninstall a specific module'
+  task :uninstall_module, [:target_node_name, :module_name] do |_task, args|
     inventory_hash = inventory_hash_from_inventory_file
     target_nodes = find_targets(inventory_hash, args[:target_node_name])
     if target_nodes.empty?
@@ -232,39 +236,58 @@ namespace :litmus do
       exit 0
     end
 
-    module_tar = build_module
-    puts 'Built'
-
-    # module_tar = Dir.glob('pkg/*.tar.gz').max_by { |f| File.mtime(f) }
-    raise "Unable to find package in 'pkg/*.tar.gz'" if module_tar.nil?
-
-    result = install_module(inventory_hash, args[:target_node_name], module_tar)
+    result = uninstall_module(inventory_hash, args[:target_node_name], args[:module_name])
 
-    raise "Failed trying to run 'puppet module install /tmp/#{File.basename(module_tar)}' against inventory." unless result.is_a?(Array)
+    raise "Failed trying to run 'puppet module uninstall #{module_name}' against inventory." unless result.is_a?(Array)
 
     result.each do |node|
-      puts "#{node['node']} failed #{node['result']}" if node['status'] != 'success'
+      puts "#{node['target']} failed #{node['value']}" if node['status'] != 'success'
     end
 
-    puts 'Installed'
+    puts 'Uninstalled'
+  end
+
+  # Reinstall the puppet module under test on a collection of nodes
+  #
+  # @param :target_node_name [Array] nodes on which to install a puppet module for testing.
+  desc 'reinstall the module under test'
+  task :reinstall_module, [:target_node_name, :module_repository] do |_task, args|
+    args.with_defaults(target_node_name: nil, module_repository: nil)
+    Rake::Task['litmus:uninstall_module'].invoke(args[:target_node_name])
+    Rake::Task['litmus:install_module'].invoke(args[:target_node_name], args[:module_repository])
+  end
+
+  # Check that the nodes in the inventory are still contactable
+  #
+  # @param :target_node_name [Array] nodes on which to check connnectivity
+  desc 'check the connectivity to all provisioned targets'
+  task :check_connectivity, [:target_node_name] do |_task, args|
+    inventory_hash = inventory_hash_from_inventory_file
+    target_nodes = find_targets(inventory_hash, args[:target_node_name])
+    if target_nodes.empty?
+      puts 'No targets found'
+      exit 0
+    end
+    check_connectivity?(inventory_hash, args[:target_node_name])
   end
 
   # Provision a list of machines, install a puppet agent, and install the puppet module under test on a collection of nodes
   #
   # @param :key [String] key that maps to a value for a provisioner and an image to be used for each OS provisioned.
   # @param :collection [String] parameters to pass to the puppet agent install command.
-  desc 'provision_install - provision a list of machines, install an agent, and the module.'
-  task :provision_install, [:key, :collection] do |_task, args|
+  desc 'provision a list of machines, install an agent, and the module.'
+  task :provision_install, [:key, :collection, :module_repository] do |_task, args|
+    args.with_defaults(module_repository: nil)
     Rake::Task['spec_prep'].invoke
     Rake::Task['litmus:provision_list'].invoke(args[:key])
     Rake::Task['litmus:install_agent'].invoke(args[:collection])
-    Rake::Task['litmus:install_module'].invoke
+    Rake::Task['litmus:install_module'].invoke(nil, args[:module_repository])
   end
 
   # Decommissions test machines.
   #
   # @param :target [Array] nodes to remove from test environemnt and decommission.
-  desc 'tear-down - decommission machines'
+  desc 'destroy provisioned targets'
   task :tear_down, [:target] do |_task, args|
     inventory_hash = inventory_hash_from_inventory_file
     targets = find_targets(inventory_hash, args[:target])
@@ -277,7 +300,7 @@ namespace :litmus do
     results = tear_down_nodes(targets, inventory_hash)
     results.each do |node, result|
       if result.first['status'] != 'success'
-        bad_results << "#{node}, #{result.first['result']['_error']['msg']}"
+        bad_results << "#{node}, #{result.first['value']['_error']['msg']}"
       else
         puts "#{node}: #{result.first['status']}"
       end
@@ -290,39 +313,6 @@ namespace :litmus do
     end
   end
 
-  # Uninstall the puppet module under test on a collection of nodes
-  #
-  # @param :target_node_name [Array] nodes on which to install a puppet module for testing.
-  # @param :module_name [String] module name to be uninstalled
-  desc 'uninstall_module - uninstall module'
-  task :uninstall_module, [:target_node_name, :module_name] do |_task, args|
-    inventory_hash = inventory_hash_from_inventory_file
-    target_nodes = find_targets(inventory_hash, args[:target_node_name])
-    if target_nodes.empty?
-      puts 'No targets found'
-      exit 0
-    end
-
-    result = uninstall_module(inventory_hash, args[:target_node_name], args[:module_name])
-
-    raise "Failed trying to run 'puppet module uninstall #{module_name}' against inventory." unless result.is_a?(Array)
-
-    result.each do |node|
-      puts "#{node['node']} failed #{node['result']}" if node['status'] != 'success'
-    end
-
-    puts 'Uninstalled'
-  end
-
-  # Reinstall the puppet module under test on a collection of nodes
-  #
-  # @param :target_node_name [Array] nodes on which to install a puppet module for testing.
-  desc 'reinstall_module - reinstall module'
-  task :reinstall_module, [:target_node_name] do |_task, args|
-    Rake::Task['litmus:uninstall_module'].invoke(args[:target_node_name])
-    Rake::Task['litmus:install_module'].invoke(args[:target_node_name])
-  end
-
   namespace :acceptance do
     require 'rspec/core/rake_task'
     if File.file?('inventory.yaml')
@@ -339,7 +329,7 @@ namespace :litmus do
         payloads = []
         # Generate list of targets to provision
         targets.each do |target|
-          test = 'bundle exec rspec ./spec/acceptance --format progress'
+          test = 'bundle exec rspec ./spec/acceptance --format progress --require rspec_honeycomb_formatter --format RSpecHoneycombFormatter'
           title = "#{target}, #{facts_from_node(inventory_hash, target)['platform']}"
           options = {
             env: {
@@ -365,7 +355,12 @@ namespace :litmus do
 
           require 'parallel'
           results = Parallel.map(payloads) do |title, test, options|
+            # avoid sending the parent process' main span in the sub-processes
+            # https://www.ruby-forum.com/t/at-exit-inherited-across-fork/122473/2
+            at_exit { exit! }
+
             env = options[:env].nil? ? {} : options[:env]
+            env['HTTP_X_HONEYCOMB_TRACE'] = Honeycomb.current_span.to_trace_header
             stdout, stderr, status = Open3.capture3(env, test)
             ["\n================\n#{title}\n", stdout, stderr, status]
           end
@@ -383,6 +378,7 @@ namespace :litmus do
           spinners = TTY::Spinner::Multi.new("[:spinner] Running against #{targets.size} targets.")
           payloads.each do |title, test, options|
             env = options[:env].nil? ? {} : options[:env]
+            env['HTTP_X_HONEYCOMB_TRACE'] = Honeycomb.current_span.to_trace_header
             spinners.register("[:spinner] #{title}") do |sp|
               stdout, stderr, status = Open3.capture3(env, test)
               if status.to_i.zero?