bolt 3.8.1 → 3.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f04dc4247f20f08ee8cfe33500b7ba9477963290ab5303b5f2c72ba60c529e1d
-  data.tar.gz: 4ebdf00f011c6d3de0e1b8e47f0763bfab04857fe4611391cf9bc800597c140e
+  metadata.gz: 1c6c3fbb5bdf8e8b53493a029b46560616f2ea310092823f863de14ad1fc836b
+  data.tar.gz: f4a3193539da70d75ff36f39a126d6f56710c91e2ff54c465a1477685b6fb19a
 SHA512:
-  metadata.gz: 5615936c79c5f157142e1703a83c9cf28e6ef2b91c090826130f57ac9dc636888847ddd87f01374053c768cf9d4e87a929f654126f6b37375ea1ae1cdc670530
-  data.tar.gz: 270c4669aff29a221c9a7711b7bac715bf6e821a01318f45866fdbadbf77e2e93424722db13b10812dc777d4dbc78c116be992f3a908365bc392820cbb6eeb0b
+  metadata.gz: e6c8837f2d6ef2e67ac2be71d150637bcb90af95207812ec22c81557841279a5f148f2348ca6e0d137e42bfd4058009c59439996a208dc6c5c72568a8842e498
+  data.tar.gz: 712b2be84c5ff24749ef1e010d439e05057c48d648394f7d321611d7576a006277d476adb38c6892fa73b380a0288726023643c8b2849a00f9ac164e06fe805f

data/bolt-modules/boltlib/lib/puppet/datatypes/future.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+# The [`background()` plan function](plan_functions.md#background) returns a
+# `Future` object, which can be passed to the [`wait()` plan
+# function](plan_functions.md#wait) to block on the result of the backgrounded
+# code block.
+#
+# @!method state
+#   Either 'running' if the Future is still executing, 'done' if the Future
+#   finished successfully, or 'error' if the Future finished with an error.
+#
+Puppet::DataTypes.create_type('Future') do
+  interface <<-PUPPET
+    attributes => {},
+    functions => {
+      state => Callable[[], Enum['running', 'done', 'error']],
+    }
+  PUPPET
+
+  load_file('bolt/plan_future')
+
+  # Needed for Puppet to recognize Bolt::Result as a Puppet object when deserializing
+  Bolt::PlanFuture.include(Puppet::Pops::Types::PuppetObject)
+  implementation_class Bolt::PlanFuture
+end

data/bolt-modules/boltlib/lib/puppet/functions/background.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+# Starts a block of code in parallel with the main plan without blocking.
+# Returns a Future object.
+#
+# > **Note:** Not available in apply block
+Puppet::Functions.create_function(:background, Puppet::Functions::InternalFunction) do
+  # Starts a block of code in parallel with the main plan without blocking.
+  # Returns a Future object.
+  # @param name An optional name for legible logs.
+  # @param block The code block to run in the background.
+  # @return A Bolt Future object
+  # @example Start a long-running process
+  #   background() || {
+  #     run_task('superlong::task', $targets)
+  #   }
+  #   run_command("echo 'Continue immediately'", $targets)
+  dispatch :background do
+    scope_param
+    optional_param 'String[1]', :name
+    block_param 'Callable[0, 0]', :block
+    return_type 'Future'
+  end
+
+  def background(scope, name = nil, &block)
+    unless Puppet[:tasks]
+      raise Puppet::ParseErrorWithIssue
+        .from_issue_and_stack(Bolt::PAL::Issues::PLAN_OPERATION_NOT_SUPPORTED_WHEN_COMPILING, action: 'background')
+    end
+
+    executor = Puppet.lookup(:bolt_executor)
+    executor.report_function_call(self.class.name)
+
+    executor.create_future(scope: scope, name: name) do |newscope|
+      # Catch 'return' calls inside the block
+      result = catch(:return) do
+        # Execute the block. Individual plan steps in the block will yield
+        # the Fiber if they haven't finished, so all this needs to do is run
+        # the block.
+        block.closure.call_by_name_with_scope(newscope, {}, true)
+      end
+
+      # If we got a return from the block, get its value
+      # Otherwise the result is the last line from the block
+      result = result.value if result.is_a?(Puppet::Pops::Evaluator::Return)
+
+      # Validate the result is a PlanResult
+      unless Puppet::Pops::Types::TypeParser.singleton.parse('Boltlib::PlanResult').instance?(result)
+        raise Bolt::InvalidParallelResult.new(result.to_s, *Puppet::Pops::PuppetStack.top_of_stack)
+      end
+
+      result
+    rescue Puppet::PreformattedError => e
+      if e.cause.is_a?(Bolt::Error)
+        e.cause
+      else
+        raise e
+      end
+    end
+  end
+end

data/bolt-modules/boltlib/lib/puppet/functions/download_file.rb

@@ -112,17 +112,13 @@ Puppet::Functions.create_function(:download_file, Puppet::Functions::InternalFun
       call_function('debug', "Simulating file download of '#{source}' - no targets given - no action taken")
       Bolt::ResultSet.new([])
     else
-      r = if executor.in_parallel
-            require 'concurrent'
-            require 'fiber'
-            future = Concurrent::Future.execute do
-              executor.download_file(targets, source, destination, options, Puppet::Pops::PuppetStack.top_of_stack)
+      file_line = Puppet::Pops::PuppetStack.top_of_stack
+      r = if executor.in_parallel?
+            executor.run_in_thread do
+              executor.download_file(targets, source, destination, options, file_line)
             end
-
-            Fiber.yield('unfinished') while future.incomplete?
-            future.value || future.reason
           else
-            executor.download_file(targets, source, destination, options, Puppet::Pops::PuppetStack.top_of_stack)
+            executor.download_file(targets, source, destination, options, file_line)
           end
 
       if !r.ok && !options[:catch_errors]

data/bolt-modules/boltlib/lib/puppet/functions/parallelize.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'bolt/yarn'
-
 # Map a code block onto an array, where each array element executes in parallel.
 # This function is experimental.
 #
@@ -35,21 +33,38 @@ Puppet::Functions.create_function(:parallelize, Puppet::Functions::InternalFunct
     executor = Puppet.lookup(:bolt_executor)
     executor.report_function_call(self.class.name)
 
-    skein = data.each_with_index.map do |object, index|
-      executor.create_yarn(scope, block, object, index)
-    end
+    futures = data.map do |object|
+      executor.create_future(scope: scope) do |newscope|
+        # Catch 'return' calls inside the block
+        result = catch(:return) do
+          # Add the object to the block parameters
+          args = { block.parameters[0][1].to_s => object }
+          # Execute the block. Individual plan steps in the block will yield
+          # the Fiber if they haven't finished, so all this needs to do is run
+          # the block.
+          block.closure.call_by_name_with_scope(newscope, args, true)
+        end
 
-    result = executor.round_robin(skein)
+        # If we got a return from the block, get its value
+        # Otherwise the result is the last line from the block
+        result = result.value if result.is_a?(Puppet::Pops::Evaluator::Return)
 
-    failed_indices = result.each_index.select do |i|
-      result[i].is_a?(Bolt::Error)
-    end
+        # Validate the result is a PlanResult
+        unless Puppet::Pops::Types::TypeParser.singleton.parse('Boltlib::PlanResult').instance?(result)
+          raise Bolt::InvalidParallelResult.new(result.to_s, *Puppet::Pops::PuppetStack.top_of_stack)
+        end
 
-    # TODO: Inner catch errors block?
-    if failed_indices.any?
-      raise Bolt::ParallelFailure.new(result, failed_indices)
+        result
+      rescue Puppet::PreformattedError => e
+        if e.cause.is_a?(Bolt::Error)
+          e.cause
+        else
+          raise e
+        end
+      end
     end
 
-    result
+    # We may eventually want parallelize to accept a timeout
+    executor.wait(futures)
   end
 end

data/bolt-modules/boltlib/lib/puppet/functions/run_command.rb

@@ -87,23 +87,13 @@ Puppet::Functions.create_function(:run_command) do
       call_function('debug', "Simulating run_command('#{command}') - no targets given - no action taken")
       Bolt::ResultSet.new([])
     else
-      r = if executor.in_parallel
-            require 'concurrent'
-            require 'fiber'
-            future = Concurrent::Future.execute do
-              executor.run_command(targets,
-                                   command,
-                                   options,
-                                   Puppet::Pops::PuppetStack.top_of_stack)
+      file_line = Puppet::Pops::PuppetStack.top_of_stack
+      r = if executor.in_parallel?
+            executor.run_in_thread do
+              executor.run_command(targets, command, options, file_line)
             end
-
-            Fiber.yield('unfinished') while future.incomplete?
-            future.value || future.reason
           else
-            executor.run_command(targets,
-                                 command,
-                                 options,
-                                 Puppet::Pops::PuppetStack.top_of_stack)
+            executor.run_command(targets, command, options, file_line)
           end
 
       if !r.ok && !options[:catch_errors]

data/bolt-modules/boltlib/lib/puppet/functions/run_script.rb

@@ -130,25 +130,13 @@ Puppet::Functions.create_function(:run_script, Puppet::Functions::InternalFuncti
     if targets.empty?
       Bolt::ResultSet.new([])
     else
-      r = if executor.in_parallel
-            require 'concurrent'
-            require 'fiber'
-            future = Concurrent::Future.execute do
-              executor.run_script(targets,
-                                  found,
-                                  arguments,
-                                  options,
-                                  Puppet::Pops::PuppetStack.top_of_stack)
+      file_line = Puppet::Pops::PuppetStack.top_of_stack
+      r = if executor.in_parallel?
+            executor.run_in_thread do
+              executor.run_script(targets, found, arguments, options, file_line)
             end
-
-            Fiber.yield('unfinished') while future.incomplete?
-            future.value || future.reason
           else
-            executor.run_script(targets,
-                                found,
-                                arguments,
-                                options,
-                                Puppet::Pops::PuppetStack.top_of_stack)
+            executor.run_script(targets, found, arguments, options, file_line)
           end
 
       if !r.ok && !options[:catch_errors]

data/bolt-modules/boltlib/lib/puppet/functions/run_task.rb

@@ -133,25 +133,13 @@ Puppet::Functions.create_function(:run_task) do
     if targets.empty?
       Bolt::ResultSet.new([])
     else
-      result = if executor.in_parallel
-                 require 'concurrent'
-                 require 'fiber'
-                 future = Concurrent::Future.execute do
-                   executor.run_task(targets,
-                                     task,
-                                     params,
-                                     options,
-                                     Puppet::Pops::PuppetStack.top_of_stack)
+      file_line = Puppet::Pops::PuppetStack.top_of_stack
+      result = if executor.in_parallel?
+                 executor.run_in_thread do
+                   executor.run_task(targets, task, params, options, file_line)
                  end
-
-                 Fiber.yield('unfinished') while future.incomplete?
-                 future.value || future.reason
                else
-                 executor.run_task(targets,
-                                   task,
-                                   params,
-                                   options,
-                                   Puppet::Pops::PuppetStack.top_of_stack)
+                 executor.run_task(targets, task, params, options, file_line)
                end
 
       if !result.ok && !options[:catch_errors]

data/bolt-modules/boltlib/lib/puppet/functions/run_task_with.rb

@@ -180,23 +180,13 @@ Puppet::Functions.create_function(:run_task_with) do
     else
       # Combine the results from the task run with any failing results that were
       # generated earlier when creating the target mapping
-      task_result = if executor.in_parallel
-                      require 'concurrent'
-                      require 'fiber'
-                      future = Concurrent::Future.execute do
-                        executor.run_task_with(target_mapping,
-                                               task,
-                                               options,
-                                               Puppet::Pops::PuppetStack.top_of_stack)
+      file_line = Puppet::Pops::PuppetStack.top_of_stack
+      task_result = if executor.in_parallel?
+                      executor.run_in_thread do
+                        executor.run_task_with(target_mapping, task, options, file_line)
                       end
-
-                      Fiber.yield('unfinished') while future.incomplete?
-                      future.value || future.reason
                     else
-                      executor.run_task_with(target_mapping,
-                                             task,
-                                             options,
-                                             Puppet::Pops::PuppetStack.top_of_stack)
+                      executor.run_task_with(target_mapping, task, options, file_line)
                     end
       result = Bolt::ResultSet.new(task_result.results + error_set)
 

data/bolt-modules/boltlib/lib/puppet/functions/upload_file.rb

@@ -87,25 +87,13 @@ Puppet::Functions.create_function(:upload_file, Puppet::Functions::InternalFunct
       call_function('debug', "Simulating file upload of '#{found}' - no targets given - no action taken")
       Bolt::ResultSet.new([])
     else
-      r = if executor.in_parallel
-            require 'concurrent'
-            require 'fiber'
-            future = Concurrent::Future.execute do
-              executor.upload_file(targets,
-                                   found,
-                                   destination,
-                                   options,
-                                   Puppet::Pops::PuppetStack.top_of_stack)
+      file_line = Puppet::Pops::PuppetStack.top_of_stack
+      r = if executor.in_parallel?
+            executor.run_in_thread do
+              executor.upload_file(targets, found, destination, options, file_line)
             end
-
-            Fiber.yield('unfinished') while future.incomplete?
-            future.value || future.reason
           else
-            executor.upload_file(targets,
-                                 found,
-                                 destination,
-                                 options,
-                                 Puppet::Pops::PuppetStack.top_of_stack)
+            executor.upload_file(targets, found, destination, options, file_line)
           end
       if !r.ok && !options[:catch_errors]
         raise Bolt::RunFailure.new(r, 'upload_file', source)

data/bolt-modules/boltlib/lib/puppet/functions/wait.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require 'bolt/logger'
+require 'bolt/target'
+
+# Wait for a Future or array of Futures to finish and return results,
+# optionally with a timeout.
+#
+# > **Note:** Not available in apply block
+Puppet::Functions.create_function(:wait, Puppet::Functions::InternalFunction) do
+  # Wait for Future(s) to finish
+  # @param futures A Bolt Future object or array of Bolt Futures to wait on.
+  # @param options A hash of additional options.
+  # @option options [Boolean] _catch_errors Whether to catch raised errors.
+  # @return A Result or Results from the Futures
+  # @example Upload a large file in the background, then wait until it's loaded
+  #   $futures = background() || {
+  #     upload_file("./very_large_file", "/opt/jfrog/artifactory/var/etc/artifactory", $targets)
+  #   }
+  #   # Run an unrelated task
+  #   run_task("deploy", $targets)
+  #   # Wait for the file upload to finish
+  #   $results = wait($futures)
+  dispatch :wait do
+    param 'Variant[Future, Array[Future]]', :futures
+    optional_param 'Hash[String[1], Any]', :options
+    return_type 'Array[Boltlib::PlanResult]'
+  end
+
+  # Wait for Future(s) to finish with timeout
+  # @param futures A Bolt Future object or array of Bolt Futures to wait on.
+  # @param timeout How long to wait for Futures to finish before raising a Timeout error.
+  # @param options A hash of additional options.
+  # @option options [Boolean] _catch_errors Whether to catch raised errors.
+  # @return A Result or Results from the Futures
+  # @example Upload a large file in the background with a 30 second timeout.
+  #   $futures = background() || {
+  #     upload_file("./very_large_file", "/opt/jfrog/artifactory/var/etc/artifactory", $targets)
+  #   }
+  #   # Run an unrelated task
+  #   run_task("deploy", $targets)
+  #   # Wait for the file upload to finish
+  #   $results = wait($futures, 30)
+  #
+  # @example Upload a large file in the background with a 30 second timeout, catching any errors.
+  #   $futures = background() || {
+  #     upload_file("./very_large_file", "/opt/jfrog/artifactory/var/etc/artifactory", $targets)
+  #   }
+  #   # Run an unrelated task
+  #   run_task("deploy", $targets)
+  #   # Wait for the file upload to finish
+  #   $results = wait($futures, 30, '_catch_errors' => true)
+  dispatch :wait_with_timeout do
+    param 'Variant[Future, Array[Future]]', :futures
+    param 'Variant[Integer[0], Float[0.0]]', :timeout
+    optional_param 'Hash[String[1], Any]', :options
+    return_type 'Array[Boltlib::PlanResult]'
+  end
+
+  def wait(futures, options = {})
+    inner_wait(futures, nil, options)
+  end
+
+  def wait_with_timeout(futures, timeout, options = {})
+    inner_wait(futures, timeout, options)
+  end
+
+  def inner_wait(futures, timeout = nil, options = {})
+    unless Puppet[:tasks]
+      raise Puppet::ParseErrorWithIssue
+        .from_issue_and_stack(Bolt::PAL::Issues::PLAN_OPERATION_NOT_SUPPORTED_WHEN_COMPILING, action: 'wait')
+    end
+
+    valid, unknown = options.partition { |k, _v| %w[_catch_errors].include?(k) }.map(&:to_h)
+    if unknown.any?
+      file, line = Puppet::Pops::PuppetStack.top_of_stack
+      msg = "The wait() function call in #{file}#L#{line} received unknown options "\
+        "#{unknown.keys}. Removing unknown options and continuing..."
+      Bolt::Logger.warn("plan_function_options", msg)
+    end
+
+    valid = valid.transform_keys { |k| k.sub(/^_/, '').to_sym }
+    valid[:timeout] = timeout if timeout
+
+    executor = Puppet.lookup(:bolt_executor)
+    executor.report_function_call(self.class.name)
+
+    futures = Array(futures)
+    executor.wait(futures, **valid)
+  end
+end

data/bolt-modules/boltlib/types/planresult.pp

@@ -13,5 +13,6 @@ type Boltlib::PlanResult = Variant[Boolean,
                                    Target,
                                    ResourceInstance,
                                    ContainerResult,
+                                   Future,
                                    Array[Boltlib::PlanResult],
                                    Hash[String, Boltlib::PlanResult]]

data/guides/inventory.txt

@@ -14,6 +14,11 @@ DESCRIPTION
     project configuration file named 'bolt-project.yaml' alongside the inventory
     file.
 
+    When Bolt loads inventory, it loads the entire inventory, not just the
+    groups and targets specified on the command line. If you've defined a
+    target in multiple groups, this might result in target configuration that
+    is different than expected.
+
 DOCUMENTATION
     https://pup.pt/bolt-inventory
     https://pup.pt/bolt-inventory-reference

data/lib/bolt/applicator.rb

@@ -306,11 +306,12 @@ module Bolt
       Puppet.lookup(:current_environment).override_with(modulepath: @plugin_dirs).modules.each do |mod|
         search_dirs = yield mod
 
-        parent = Pathname.new(mod.path).parent
+        tar_dir = Pathname.new(mod.name) # goes great with fish
+        mod_dir = Pathname.new(mod.path)
         files = Find.find(*search_dirs).select { |file| File.file?(file) }
 
         files.each do |file|
-          tar_path = Pathname.new(file).relative_path_from(parent)
+          tar_path = tar_dir + Pathname.new(file).relative_path_from(mod_dir)
           @logger.trace("Packing plugin #{file} to #{tar_path}")
           stat = File.stat(file)
           content = File.binread(file)

data/lib/bolt/config/transport/docker.rb

@@ -15,7 +15,7 @@ module Bolt
           shell-command
           tmpdir
           tty
-        ].freeze
+        ].concat(RUN_AS_OPTIONS).sort.freeze
 
         DEFAULTS = {
           'cleanup' => true
@@ -27,6 +27,10 @@ module Bolt
           if @config['interpreters']
             @config['interpreters'] = normalize_interpreters(@config['interpreters'])
           end
+
+          if Bolt::Util.windows? && @config['run-as']
+            raise Bolt::ValidationError, "run-as is not supported when using PowerShell"
+          end
         end
       end
     end

data/lib/bolt/config/transport/lxd.rb

@@ -11,7 +11,7 @@ module Bolt
           cleanup
           remote
           tmpdir
-        ].freeze
+        ].concat(RUN_AS_OPTIONS).sort.freeze
 
         DEFAULTS = {
           'cleanup' => true,

data/lib/bolt/config/transport/podman.rb

@@ -14,7 +14,7 @@ module Bolt
           shell-command
           tmpdir
           tty
-        ].freeze
+        ].concat(RUN_AS_OPTIONS).sort.freeze
 
         DEFAULTS = {
           'cleanup' => true
@@ -26,6 +26,10 @@ module Bolt
           if @config['interpreters']
             @config['interpreters'] = normalize_interpreters(@config['interpreters'])
           end
+
+          if Bolt::Util.windows? && @config['run-as']
+            raise Bolt::ValidationError, "run-as is not supported when using PowerShell"
+          end
         end
       end
     end

data/lib/bolt/error.rb

@@ -105,6 +105,16 @@ module Bolt
     end
   end
 
+  class FutureTimeoutError < Bolt::Error
+    def initialize(name, timeout)
+      details = {
+        'future' => name
+      }
+      message = "Future '#{name}' timed out after #{timeout} seconds."
+      super(message, 'bolt/future-timeout-error', details)
+    end
+  end
+
   class ParallelFailure < Bolt::Error
     def initialize(results, failed_indices)
       details = {
@@ -162,7 +172,7 @@ module Bolt
 
   class InvalidParallelResult < Error
     def initialize(result_str, file, line)
-      super("Parallel block returned an invalid result: #{result_str}",
+      super("Background block returned an invalid result: #{result_str}",
             'bolt/invalid-plan-result',
             { 'file' => file,
               'line' => line,

data/lib/bolt/executor.rb

@@ -7,10 +7,11 @@ require 'logging'
 require 'pathname'
 require 'set'
 require 'bolt/analytics'
-require 'bolt/result'
 require 'bolt/config'
-require 'bolt/result_set'
+require 'bolt/fiber_executor'
 require 'bolt/puppetdb'
+require 'bolt/result'
+require 'bolt/result_set'
 # Load transports
 require 'bolt/transport/docker'
 require 'bolt/transport/local'
@@ -20,7 +21,6 @@ require 'bolt/transport/podman'
 require 'bolt/transport/remote'
 require 'bolt/transport/ssh'
 require 'bolt/transport/winrm'
-require 'bolt/yarn'
 
 module Bolt
   TRANSPORTS = {
@@ -35,7 +35,7 @@ module Bolt
   }.freeze
 
   class Executor
-    attr_reader :noop, :transports, :in_parallel, :future
+    attr_reader :noop, :transports, :future
     attr_accessor :run_as
 
     def initialize(concurrency = 1,
@@ -66,7 +66,6 @@ module Bolt
 
       @noop = noop
       @run_as = nil
-      @in_parallel = false
       @future = future
       @pool = if concurrency > 0
                 Concurrent::ThreadPoolExecutor.new(name: 'exec', max_threads: concurrency)
@@ -77,6 +76,7 @@ module Bolt
 
       @concurrency = concurrency
       @warn_concurrency = modified_concurrency
+      @fiber_executor = Bolt::FiberExecutor.new
     end
 
     def transport(transport)
@@ -373,83 +373,62 @@ module Bolt
       plan.call_by_name_with_scope(scope, params, true)
     end
 
-    def create_yarn(scope, block, object, index)
-      fiber = Fiber.new do
-        # Create the new scope
-        newscope = Puppet::Parser::Scope.new(scope.compiler)
-        local = Puppet::Parser::Scope::LocalScope.new
-
-        # Compress the current scopes into a single vars hash to add to the new scope
-        current_scope = scope.effective_symtable(true)
-        until current_scope.nil?
-          current_scope.instance_variable_get(:@symbols)&.each_pair { |k, v| local[k] = v }
-          current_scope = current_scope.parent
-        end
-        newscope.push_ephemerals([local])
-
-        begin
-          result = catch(:return) do
-            args = { block.parameters[0][1].to_s => object }
-            block.closure.call_by_name_with_scope(newscope, args, true)
-          end
-
-          # If we got a return from the block, get it's value
-          # Otherwise the result is the last line from the block
-          result = result.value if result.is_a?(Puppet::Pops::Evaluator::Return)
+    # Call into FiberExecutor to avoid this class getting
+    # overloaded while also minimizing the Puppet lookups needed from plan
+    # functions
+    #
+    def create_future(scope: nil, name: nil, &block)
+      @fiber_executor.create_future(scope: scope, name: name, &block)
+    end
 
-          # Validate the result is a PlanResult
-          unless Puppet::Pops::Types::TypeParser.singleton.parse('Boltlib::PlanResult').instance?(result)
-            raise Bolt::InvalidParallelResult.new(result.to_s, *Puppet::Pops::PuppetStack.top_of_stack)
-          end
+    def plan_complete?
+      @fiber_executor.plan_complete?
+    end
 
-          result
-        rescue Puppet::PreformattedError => e
-          if e.cause.is_a?(Bolt::Error)
-            e.cause
-          else
-            raise e
-          end
-        end
-      end
+    def round_robin
+      @fiber_executor.round_robin
+    end
 
-      Bolt::Yarn.new(fiber, index)
+    def in_parallel?
+      @fiber_executor.in_parallel?
     end
 
-    def handle_event(event)
-      case event[:type]
-      when :node_result
-        @thread_completed = true
-      end
+    def wait(futures, **opts)
+      @fiber_executor.wait(futures, **opts)
     end
 
-    def round_robin(skein)
-      subscribe(self, [:node_result])
-      results = Array.new(skein.length)
-      @in_parallel = true
-      publish_event(type: :stop_spin)
+    def plan_futures
+      @fiber_executor.plan_futures
+    end
 
-      until skein.empty?
-        @thread_completed = false
-        r = nil
-
-        skein.each do |yarn|
-          if yarn.alive?
-            publish_event(type: :stop_spin)
-            r = yarn.resume
-          else
-            results[yarn.index] = yarn.value
-            skein.delete(yarn)
-          end
-        end
+    # Execute a plan function concurrently. This function accepts the executor
+    # function to be run and the parameters to pass to it, and returns the
+    # result of running the executor function.
+    #
+    def run_in_thread
+      require 'concurrent'
+      require 'fiber'
+      future = Concurrent::Future.execute do
+        yield
+      end
 
-        next unless r == 'unfinished'
-        sleep(0.1) until @thread_completed || skein.empty?
+      # Used to track how often we resume the same executor function
+      still_running = 0
+      # While the thread is still running
+      while future.incomplete?
+        # If the Fiber gets resumed, increment the resume tracker. This means
+        # the tracker starts at 1 since it needs to increment before yielding,
+        # since it can't yield then increment.
+        still_running += 1
+        # If the Fiber has been resumed before, still_running will be 2 or
+        # more. Yield different values for when the same Fiber is resumed
+        # multiple times and when it's resumed the first time in order to know
+        # if progress was made in the plan.
+        Fiber.yield(still_running < 2 ? :something_happened : :returned_immediately)
       end
 
-      publish_event(type: :stop_spin)
-      @in_parallel = false
-      unsubscribe(self, [:node_result])
-      results
+      # Once the thread completes, return the result.
+      future.value || future.reason
     end
 
     class TimeoutError < RuntimeError; end
@@ -519,7 +498,7 @@ module Bolt
     # coupled with the orchestrator transport since the transport behaves
     # differently when a plan is running. In order to limit how much this
     # pollutes the transport API we only handle the orchestrator transport here.
-    # Since we callt this function without resolving targets this will result
+    # Since we call this function without resolving targets this will result
     # in the orchestrator transport always being initialized during plan runs.
     # For now that's ok.
     #

data/lib/bolt/fiber_executor.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+require 'bolt/logger'
+require 'bolt/plan_future'
+
+module Bolt
+  class FiberExecutor
+    attr_reader :plan_futures
+
+    def initialize
+      @logger = Bolt::Logger.logger(self)
+      @id = 0
+      @plan_futures = []
+    end
+
+    # Whether there is more than one fiber running in parallel.
+    #
+    def in_parallel?
+      plan_futures.length > 1
+    end
+
+    # Creates a new Puppet scope from the current Plan scope so that variables
+    # can be used inside the block and won't interact with the outer scope.
+    # Then creates a new Fiber to execute the block, wraps the Fiber in a
+    # Bolt::PlanFuture, and returns the Bolt::PlanFuture.
+    #
+    def create_future(scope: nil, name: nil)
+      newscope = nil
+      if scope
+        # Save existing variables to the new scope before starting the future
+        # itself so that if the plan returns before the backgrounded block
+        # starts, we still have the variables.
+        newscope = Puppet::Parser::Scope.new(scope.compiler)
+        local = Puppet::Parser::Scope::LocalScope.new
+
+        # Compress the current scopes into a single vars hash to add to the new scope
+        scope.to_hash.each_pair { |k, v| local[k] = v }
+        newscope.push_ephemerals([local])
+      end
+
+      # Create a new Fiber that will execute the provided block.
+      future = Fiber.new do
+        # Yield the new scope - this should be ignored by the block if
+        # `newscope` is nil.
+        yield newscope
+      end
+
+      # PlanFutures are assigned an ID, which is just a global incrementing
+      # integer. The main plan should always have ID 0.
+      @id += 1
+      future = Bolt::PlanFuture.new(future, @id, name)
+      @logger.trace("Created future #{future.name}")
+
+      # Register the PlanFuture with the FiberExecutor to be executed
+      plan_futures << future
+      future
+    end
+
+    # Visit each PlanFuture registered with the FiberExecutor and resume it.
+    # Fibers will yield themselves back, either if they kicked off a
+    # long-running process or if the current long-running process hasn't
+    # completed. If the Fiber finishes after being resumed, store the result in
+    # the PlanFuture and remove the PlanFuture from the FiberExecutor.
+    #
+    def round_robin
+      plan_futures.each do |future|
+        # If the Fiber is still running and can be resumed, then resume it
+        @logger.trace("Checking future '#{future.name}'")
+        if future.alive?
+          @logger.trace("Resuming future '#{future.name}'")
+          future.resume
+        end
+
+        # Once we've restarted the Fiber, check to see if it's finished again
+        # and cleanup if it has.
+        next if future.alive?
+        @logger.trace("Cleaning up future '#{future.name}'")
+
+        # If the future errored and the main plan has already exited, log the
+        # error at warn level.
+        unless plan_futures.map(&:id).include?(0) || future.state == "done"
+          Bolt::Logger.warn('errored_futures', "Error in future '#{future.name}': #{future.value}")
+        end
+
+        # Remove the PlanFuture from the FiberExecutor.
+        plan_futures.delete(future)
+      end
+
+      # If the Fiber immediately returned or if the Fiber is blocking on a
+      # `wait` call, Bolt should pause for long enough that something can
+      # execute before checking again. This mitigates CPU
+      # thrashing.
+      return unless plan_futures.all? { |f| %i[returned_immediately unfinished].include?(f.value) }
+      @logger.trace("Nothing can be resumed. Rechecking in 0.5 seconds.")
+
+      sleep(0.5)
+    end
+
+    # Whether all PlanFutures have finished executing, indicating that the
+    # entire plan (main plan and any PlanFutures it spawned) has finished and
+    # Bolt can exit.
+    #
+    def plan_complete?
+      plan_futures.empty?
+    end
+
+    # Block until the provided PlanFuture objects have finished, or the timeout is reached.
+    #
+    def wait(futures, timeout: nil, catch_errors: false, **_kwargs)
+      if timeout.nil?
+        Fiber.yield(:unfinished) until futures.map(&:alive?).none?
+      else
+        start = Time.now
+        Fiber.yield(:unfinished) until (Time.now - start > timeout) || futures.map(&:alive?).none?
+        # Raise an error for any futures that are still alive
+        futures.each do |f|
+          if f.alive?
+            f.raise(Bolt::FutureTimeoutError.new(f.name, timeout))
+          end
+        end
+      end
+
+      results = futures.map(&:value)
+
+      failed_indices = results.each_index.select do |i|
+        results[i].is_a?(Bolt::Error)
+      end
+
+      if failed_indices.any?
+        if catch_errors
+          failed_indices.each { |i| results[i] = results[i].to_puppet_error }
+        else
+          # Do this after handling errors for simplicity and pretty printing
+          raise Bolt::ParallelFailure.new(results, failed_indices)
+        end
+      end
+
+      results
+    end
+  end
+end

data/lib/bolt/outputter/human.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'bolt/container_result'
 require 'bolt/pal'
 
 module Bolt

data/lib/bolt/pal.rb

@@ -607,10 +607,26 @@ module Bolt
       end
     end
 
-    def run_plan(plan_name, params, executor = nil, inventory = nil, pdb_client = nil, applicator = nil)
+    def run_plan(plan_name, params, executor, inventory = nil, pdb_client = nil, applicator = nil)
+      # Start the round robin inside the plan compiler, so that
+      # backgrounded tasks can finish once the main plan exits
       in_plan_compiler(executor, inventory, pdb_client, applicator) do |compiler|
-        r = compiler.call_function('run_plan', plan_name, params.merge('_bolt_api_call' => true))
-        Bolt::PlanResult.from_pcore(r, 'success')
+        # Create a Fiber for the main plan. This will be run along with any
+        # other Fibers created during the plan run in the round_robin, with the
+        # main plan always taking precedence in being resumed.
+        future = executor.create_future(name: plan_name) do |_scope|
+          r = compiler.call_function('run_plan', plan_name, params.merge('_bolt_api_call' => true))
+          Bolt::PlanResult.from_pcore(r, 'success')
+        rescue Bolt::Error => e
+          Bolt::PlanResult.new(e, 'failure')
+        end
+
+        # Round robin until all Fibers, including the main plan, have finished.
+        # This will stay alive until backgrounded tasks have finished.
+        executor.round_robin until executor.plan_complete?
+
+        # Return the result from the main plan
+        future.value
       end
     rescue Bolt::Error => e
       Bolt::PlanResult.new(e, 'failure')

data/lib/bolt/plan_future.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require 'fiber'
+
+module Bolt
+  class PlanFuture
+    attr_reader :fiber, :id
+    attr_accessor :value
+
+    def initialize(fiber, id, name = nil)
+      @fiber = fiber
+      @id    = id
+      @name  = name
+      @value = nil
+    end
+
+    def name
+      @name || @id
+    end
+
+    def to_s
+      "Future '#{name}'"
+    end
+
+    def alive?
+      fiber.alive?
+    end
+
+    def raise(exception)
+      # Make sure the value gets set
+      @value = exception
+      # This was introduced in Ruby 2.7
+      begin
+        # Raise an exception to kill the Fiber. If the Fiber has not been
+        # resumed yet, or is already terminated this will raise a FiberError.
+        # We don't especially care about the FiberError, as long as the Fiber
+        # doesn't report itself as alive.
+        fiber.raise(exception)
+      rescue FiberError
+        # If the Fiber is still alive, resume it with a block to raise the
+        # exception which will terminate it.
+        if fiber.alive?
+          fiber.resume { raise(exception) }
+        end
+      end
+    end
+
+    def resume
+      if fiber.alive?
+        @value = fiber.resume
+      else
+        @value
+      end
+    end
+
+    def state
+      if fiber.alive?
+        "running"
+      elsif value.is_a?(Exception)
+        "error"
+      else
+        "done"
+      end
+    end
+  end
+end

data/lib/bolt/transport/docker/connection.rb

@@ -27,6 +27,10 @@ module Bolt
                      end
         end
 
+        def reset_cwd?
+          true
+        end
+
         # The full ID of the target container
         #
         # @return [String] The full ID of the target container
@@ -74,7 +78,6 @@ module Bolt
           # CODEREVIEW: Is it always safe to pass --interactive?
           args += %w[--interactive]
           args += %w[--tty] if target.options['tty']
-          args += %W[--env DOCKER_HOST=#{@docker_host}] if @docker_host
           args += @env_vars if @env_vars
 
           if target.options['shell-command'] && !target.options['shell-command'].empty?
@@ -86,7 +89,7 @@ module Bolt
           docker_command = %w[docker exec] + args + [container_id] + Shellwords.split(command)
           @logger.trace { "Executing: #{docker_command.join(' ')}" }
 
-          Open3.popen3(*docker_command)
+          Open3.popen3(env_hash, *docker_command)
         rescue StandardError
           @logger.trace { "Command aborted" }
           raise

data/lib/bolt/transport/lxd/connection.rb

@@ -23,6 +23,10 @@ module Bolt
           Bolt::Shell::Bash.new(target, self)
         end
 
+        def reset_cwd?
+          true
+        end
+
         def container_id
           "#{@target.transport_config['remote']}:#{@target.host}"
         end

data/lib/bolt/transport/podman/connection.rb

@@ -30,6 +30,10 @@ module Bolt
                      end
         end
 
+        def reset_cwd?
+          true
+        end
+
         def connect
           # We don't actually have a connection, but we do need to
           # check that the container exists and is running.

data/lib/bolt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bolt
-  VERSION = '3.8.1'
+  VERSION = '3.9.0'
 end

data/lib/bolt_spec/plans/mock_executor.rb

@@ -17,13 +17,12 @@ module BoltSpec
 
     # Nothing on the executor is 'public'
     class MockExecutor
-      attr_reader :noop, :error_message, :in_parallel, :transports, :future
+      attr_reader :noop, :error_message, :transports, :future
       attr_accessor :run_as, :transport_features, :execute_any_plan
 
       def initialize(modulepath)
         @noop = false
         @run_as = nil
-        @in_parallel = false
         @future = {}
         @error_message = nil
         @allow_apply = false
@@ -38,6 +37,7 @@ module BoltSpec
         @execute_any_plan = true
         # plans that are allowed to be executed by the @executor_real
         @allowed_exec_plans = {}
+        @id = 0
       end
 
       def module_file_id(file)
@@ -257,58 +257,53 @@ module BoltSpec
       end
       # End apply_prep mocking
 
-      # Evaluates a `parallelize()` block and returns the result. Normally,
-      # Bolt's executor wraps this in a Yarn for each object passed to the
-      # `parallelize()` function, and then executes them in parallel before
-      # returning the result from the block. However, in BoltSpec the block is
-      # executed for each object sequentially, and this function returns the
-      # result itself.
-      #
-      def create_yarn(scope, block, object, _index)
-        # Create the new scope
-        newscope = Puppet::Parser::Scope.new(scope.compiler)
-        local = Puppet::Parser::Scope::LocalScope.new
-
-        # Compress the current scopes into a single vars hash to add to the new scope
-        current_scope = scope.effective_symtable(true)
-        until current_scope.nil?
-          current_scope.instance_variable_get(:@symbols)&.each_pair { |k, v| local[k] = v }
-          current_scope = current_scope.parent
-        end
-        newscope.push_ephemerals([local])
-
-        begin
-          result = catch(:return) do
-            args = { block.parameters[0][1].to_s => object }
-            block.closure.call_by_name_with_scope(newscope, args, true)
-          end
-
-          # If we got a return from the block, get it's value
-          # Otherwise the result is the last line from the block
-          result = result.value if result.is_a?(Puppet::Pops::Evaluator::Return)
+      # Parallel function mocking
+      def run_in_thread
+        yield
+      end
 
-          # Validate the result is a PlanResult
-          unless Puppet::Pops::Types::TypeParser.singleton.parse('Boltlib::PlanResult').instance?(result)
-            raise Bolt::InvalidParallelResult.new(result.to_s, *Puppet::Pops::PuppetStack.top_of_stack)
-          end
+      def in_parallel?
+        false
+      end
 
-          result
-        rescue Puppet::PreformattedError => e
-          if e.cause.is_a?(Bolt::Error)
-            e.cause
-          else
-            raise e
+      def create_future(scope: nil, name: nil)
+        newscope = nil
+        if scope
+          # Create the new scope
+          newscope = Puppet::Parser::Scope.new(scope.compiler)
+          local = Puppet::Parser::Scope::LocalScope.new
+
+          # Compress the current scopes into a single vars hash to add to the new scope
+          current_scope = scope.effective_symtable(true)
+          until current_scope.nil?
+            current_scope.instance_variable_get(:@symbols)&.each_pair { |k, v| local[k] = v }
+            current_scope = current_scope.parent
           end
+          newscope.push_ephemerals([local])
         end
+
+        # Execute "futures" serially when running in BoltSpec
+        result = yield newscope
+        @id += 1
+        future = Bolt::PlanFuture.new(nil, @id, name: name)
+        future.value = result
+        future
       end
 
-      # BoltSpec already evaluated the `parallelize()` block for each object
-      # passed to the function, so these results can be returned as-is.
-      #
-      def round_robin(results)
+      def wait(results, _timeout, **_kwargs)
         results
       end
 
+      # Since Futures are executed immediately once created, this will always
+      # be true by the time it's called.
+      def plan_complete?
+        true
+      end
+
+      def plan_futures
+        []
+      end
+
       # Public methods on Bolt::Executor that need to be mocked so there aren't
       # "undefined method" errors.
 
@@ -337,6 +332,8 @@ module BoltSpec
       def subscribe(_subscriber, _types = nil); end
 
       def unsubscribe(_subscriber, _types = nil); end
+
+      def round_robin; end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bolt
 version: !ruby/object:Gem::Version
-  version: 3.8.1
+  version: 3.9.0
 platform: ruby
 authors:
 - Puppet
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-05-17 00:00:00.000000000 Z
+date: 2021-05-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -399,6 +399,7 @@ files:
 - Puppetfile
 - bolt-modules/boltlib/lib/puppet/datatypes/applyresult.rb
 - bolt-modules/boltlib/lib/puppet/datatypes/containerresult.rb
+- bolt-modules/boltlib/lib/puppet/datatypes/future.rb
 - bolt-modules/boltlib/lib/puppet/datatypes/resourceinstance.rb
 - bolt-modules/boltlib/lib/puppet/datatypes/result.rb
 - bolt-modules/boltlib/lib/puppet/datatypes/resultset.rb
@@ -406,6 +407,7 @@ files:
 - bolt-modules/boltlib/lib/puppet/functions/add_facts.rb
 - bolt-modules/boltlib/lib/puppet/functions/add_to_group.rb
 - bolt-modules/boltlib/lib/puppet/functions/apply_prep.rb
+- bolt-modules/boltlib/lib/puppet/functions/background.rb
 - bolt-modules/boltlib/lib/puppet/functions/catch_errors.rb
 - bolt-modules/boltlib/lib/puppet/functions/download_file.rb
 - bolt-modules/boltlib/lib/puppet/functions/facts.rb
@@ -432,6 +434,7 @@ files:
 - bolt-modules/boltlib/lib/puppet/functions/set_var.rb
 - bolt-modules/boltlib/lib/puppet/functions/upload_file.rb
 - bolt-modules/boltlib/lib/puppet/functions/vars.rb
+- bolt-modules/boltlib/lib/puppet/functions/wait.rb
 - bolt-modules/boltlib/lib/puppet/functions/wait_until_available.rb
 - bolt-modules/boltlib/lib/puppet/functions/without_default_logging.rb
 - bolt-modules/boltlib/lib/puppet/functions/write_file.rb
@@ -485,6 +488,7 @@ files:
 - lib/bolt/container_result.rb
 - lib/bolt/error.rb
 - lib/bolt/executor.rb
+- lib/bolt/fiber_executor.rb
 - lib/bolt/inventory.rb
 - lib/bolt/inventory/group.rb
 - lib/bolt/inventory/inventory.rb
@@ -528,6 +532,7 @@ files:
 - lib/bolt/pal/yaml_plan/step/upload.rb
 - lib/bolt/pal/yaml_plan/transpiler.rb
 - lib/bolt/plan_creator.rb
+- lib/bolt/plan_future.rb
 - lib/bolt/plan_result.rb
 - lib/bolt/plugin.rb
 - lib/bolt/plugin/cache.rb
@@ -583,7 +588,6 @@ files:
 - lib/bolt/util/puppet_log_level.rb
 - lib/bolt/validator.rb
 - lib/bolt/version.rb
-- lib/bolt/yarn.rb
 - lib/bolt_server/acl.rb
 - lib/bolt_server/base_config.rb
 - lib/bolt_server/config.rb

data/lib/bolt/yarn.rb

@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-require 'fiber'
-
-module Bolt
-  class Yarn
-    attr_reader :fiber, :value, :index
-
-    def initialize(fiber, index)
-      @fiber = fiber
-      @index = index
-      @value = nil
-    end
-
-    def alive?
-      fiber.alive?
-    end
-
-    def resume
-      @value = fiber.resume
-    end
-  end
-end