bolt 3.13.0 → 3.16.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 80a2245a2ac92c15284c4d6206f5480c6cf72033b32b773a98303b634f853e54
-  data.tar.gz: 4a6e868eba652ea23cb8530a135e3187fae81296d63df8bf9b54d5c2117df615
+  metadata.gz: 71d383b473f9356ff41f574bb0abe4aea9a9f8c3ac5055a459acffe0da290af1
+  data.tar.gz: 00d10803d865a9d6f0cb6eb7162e4a17f619b88e6702b559e0153b3796d732d0
 SHA512:
-  metadata.gz: f7373197cc0401a971ff8e68ed70ef31f90d2087d09a5a0a5178124e40d2aae59546db768480fb5ea65f2b2993d60104abcf137df281863ddaf37c69d58365e5
-  data.tar.gz: 338c72c31cc379aaf45adae5a122a2a2e9fe6dede738aec4b46c21b5102d16a00ee5a494514dd80bc9f4fcd922ee08e62164e64d34987d40ad1d7383ff475363
+  metadata.gz: eaad9c12aab69e9d7c9f1d8b312623b8bcca4f26f983beb08b1d09c4568aafe0e430d639b88903df4e1a498a58c4c9d8add5d4b8b3930fe1dec3124bc15afad3
+  data.tar.gz: a05ec6c181fc6aa51c73ebfd928d73513d5506c1b0705bed4637bfa9490a6f1c010019c9209f7ed28c134f003a063ff6f4add490a55d658998f566d00c5ff017

data/Puppetfile

@@ -35,7 +35,7 @@ mod 'puppetlabs-stdlib', '7.1.0'
 mod 'puppetlabs-aws_inventory', '0.7.0'
 mod 'puppetlabs-azure_inventory', '0.5.0'
 mod 'puppetlabs-gcloud_inventory', '0.3.0'
-mod 'puppetlabs-http_request', '0.2.2'
+mod 'puppetlabs-http_request', '0.3.1'
 mod 'puppetlabs-pkcs7', '0.1.2'
 mod 'puppetlabs-secure_env_vars', '0.2.0'
 mod 'puppetlabs-terraform', '0.6.1'

data/bolt-modules/boltlib/lib/puppet/functions/apply_prep.rb

@@ -8,16 +8,18 @@ require 'bolt/task'
 # installed using either the configured plugin or the `task` plugin with the
 # `puppet_agent::install` task.
 #
-# Agent installation will be skipped if the target includes the `puppet-agent` feature, either as a
-# property of its transport (PCP) or by explicitly setting it as a feature in Bolt's inventory.
+# Agent installation will be skipped if the target includes the `puppet-agent`
+# feature, either as a property of its transport (PCP) or by explicitly setting
+# it as a feature in Bolt's inventory.
 #
 # > **Note:** Not available in apply block
 Puppet::Functions.create_function(:apply_prep) do
   # @param targets A pattern or array of patterns identifying a set of targets.
   # @param options Options hash.
+  # @option options [Boolean] _catch_errors Whether to catch raised errors.
   # @option options [Array] _required_modules An array of modules to sync to the target.
   # @option options [String] _run_as User to run as using privilege escalation.
-  # @return [nil]
+  # @return [Bolt::ResultSet]
   # @example Prepare targets by name.
   #   apply_prep('target1,target2')
   dispatch :apply_prep do
@@ -25,138 +27,169 @@ Puppet::Functions.create_function(:apply_prep) do
     optional_param 'Hash[String, Data]', :options
   end
 
-  def script_compiler
-    @script_compiler ||= Puppet::Pal::ScriptCompiler.new(closure_scope.compiler)
-  end
+  def apply_prep(target_spec, options = {})
+    unless Puppet[:tasks]
+      raise Puppet::ParseErrorWithIssue
+        .from_issue_and_stack(Bolt::PAL::Issues::PLAN_OPERATION_NOT_SUPPORTED_WHEN_COMPILING, action: 'apply_prep')
+    end
 
-  def inventory
-    @inventory ||= Puppet.lookup(:bolt_inventory)
-  end
+    options = options.slice(*%w[_catch_errors _required_modules _run_as])
+    targets = inventory.get_targets(target_spec)
 
-  def get_task(name, params = {})
-    tasksig = script_compiler.task_signature(name)
-    raise Bolt::Error.new("Task '#{name}' could not be found", 'bolt/apply-prep') unless tasksig
+    executor.report_function_call(self.class.name)
 
-    errors = []
-    unless tasksig.runnable_with?(params) { |msg| errors << msg }
-      # This relies on runnable with printing a partial message before the first real error
-      raise Bolt::ValidationError, "Invalid parameters for #{errors.join("\n")}"
+    executor.log_action('install puppet and gather facts', targets) do
+      executor.without_default_logging do
+        install_results = install_agents(targets, options)
+        facts_results   = get_facts(install_results.ok_set.targets, options)
+
+        Bolt::ResultSet.new(install_results.error_set.results + facts_results.results)
+      end
     end
+  end
 
-    Bolt::Task.from_task_signature(tasksig)
+  def applicator
+    @applicator ||= Puppet.lookup(:apply_executor)
   end
 
-  # rubocop:disable Naming/AccessorMethodName
-  def set_agent_feature(target)
-    inventory.set_feature(target, 'puppet-agent')
+  def executor
+    @executor ||= Puppet.lookup(:bolt_executor)
   end
-  # rubocop:enable Naming/AccessorMethodName
 
+  def inventory
+    @inventory ||= Puppet.lookup(:bolt_inventory)
+  end
+
+  # Runs a task. This method is called by the puppet_library hook.
+  #
   def run_task(targets, task, args = {}, options = {})
     executor.run_task(targets, task, args, options)
   end
 
-  # Returns true if the target has the puppet-agent feature defined, either from inventory or transport.
-  def agent?(target, executor, inventory)
+  # Returns true if the target has the puppet-agent feature defined, either from
+  # inventory or transport.
+  #
+  private def agent?(target)
     inventory.features(target).include?('puppet-agent') ||
-      executor.transport(target.transport).provided_features.include?('puppet-agent') || target.remote?
+    executor.transport(target.transport).provided_features.include?('puppet-agent') ||
+    target.remote?
   end
 
-  def executor
-    @executor ||= Puppet.lookup(:bolt_executor)
+  # Generate the plugin tarball.
+  #
+  private def build_plugin_tarball(required_modules)
+    if required_modules.any?
+      Puppet.debug("Syncing only required modules: #{required_modules.join(',')}.")
+    end
+
+    tarball = applicator.build_plugin_tarball do |mod|
+      next unless required_modules.empty? || required_modules.include?(mod.name)
+      search_dirs = []
+      search_dirs << mod.plugins if mod.plugins?
+      search_dirs << mod.pluginfacts if mod.pluginfacts?
+      search_dirs
+    end
+
+    Puppet::Pops::Types::PSensitiveType::Sensitive.new(tarball)
   end
 
-  def apply_prep(target_spec, options = {})
-    unless Puppet[:tasks]
-      raise Puppet::ParseErrorWithIssue
-        .from_issue_and_stack(Bolt::PAL::Issues::PLAN_OPERATION_NOT_SUPPORTED_WHEN_COMPILING, action: 'apply_prep')
+  # Install the puppet-agent package on targets that need it.
+  #
+  private def install_agents(targets, options)
+    results = []
+
+    agent_targets, agentless_targets = targets.partition { |target| agent?(target) }
+
+    agent_targets.each do |target|
+      Puppet.debug("Puppet Agent feature declared for #{target}")
+      results << Bolt::Result.new(target)
     end
 
-    # Unfreeze this
-    options = options.slice(*%w[_run_as _required_modules])
+    unless agentless_targets.empty?
+      hooks, errors = get_hooks(agentless_targets, options)
+      hook_results  = run_hooks(hooks)
 
-    applicator = Puppet.lookup(:apply_executor)
+      hook_results.each do |result|
+        next unless result.ok?
+        inventory.set_feature(result.target, 'puppet-agent')
+      end
 
-    executor.report_function_call(self.class.name)
+      results.concat(hook_results).concat(errors)
+    end
 
-    targets = inventory.get_targets(target_spec)
+    Bolt::ResultSet.new(results).tap do |resultset|
+      unless resultset.ok? || options['_catch_errors']
+        raise Bolt::RunFailure.new(resultset.error_set, 'apply_prep')
+      end
+    end
+  end
 
-    required_modules = options.delete('_required_modules').to_a
-    if required_modules.any?
-      Puppet.debug("Syncing only required modules: #{required_modules.join(',')}.")
+  # Retrieve facts from each target and add them to inventory.
+  #
+  private def get_facts(targets, options)
+    return Bolt::ResultSet.new([]) unless targets.any?
+
+    task    = applicator.custom_facts_task
+    args    = { 'plugins' => build_plugin_tarball(options.delete('_required_modules').to_a) }
+    results = run_task(targets, task, args, options)
+
+    unless results.ok? || options['_catch_errors']
+      raise Bolt::RunFailure.new(results, 'run_task', task.name)
     end
 
-    # Gather facts, including custom facts
-    plugins = applicator.build_plugin_tarball do |mod|
-      next unless required_modules.empty? || required_modules.include?(mod.name)
-      search_dirs = []
-      search_dirs << mod.plugins if mod.plugins?
-      search_dirs << mod.pluginfacts if mod.pluginfacts?
-      search_dirs
+    results.each do |result|
+      next unless result.ok?
+
+      if unsupported_puppet?(result['clientversion'])
+        Bolt::Logger.deprecate(
+          "unsupported_puppet",
+          "Detected unsupported Puppet agent version #{result['clientversion']} on target "\
+          "#{result.target}. Bolt supports Puppet agent 6.0.0 and higher."
+        )
+      end
+
+      inventory.add_facts(result.target, result.value)
     end
 
-    executor.log_action('install puppet and gather facts', targets) do
-      executor.without_default_logging do
-        # Skip targets that include the puppet-agent feature, as we know an agent will be available.
-        agent_targets, need_install_targets = targets.partition { |target| agent?(target, executor, inventory) }
-        agent_targets.each { |target| Puppet.debug "Puppet Agent feature declared for #{target.name}" }
-        unless need_install_targets.empty?
-          # lazy-load expensive gem code
-          require 'concurrent'
-          pool = Concurrent::ThreadPoolExecutor.new
-
-          hooks = need_install_targets.map do |t|
-            opts = t.plugin_hooks&.fetch('puppet_library').dup
-            plugin_name = opts.delete('plugin')
-            hook = inventory.plugins.get_hook(plugin_name, :puppet_library)
-            # Give plan function options precedence over inventory options
-            { 'target' => t,
-              'hook_proc' => hook.call(opts.merge(options), t, self) }
-          rescue StandardError => e
-            Bolt::Result.from_exception(t, e)
-          end
-
-          hook_errors, ok_hooks = hooks.partition { |h| h.is_a?(Bolt::Result) }
-
-          futures = ok_hooks.map do |hash|
-            Concurrent::Future.execute(executor: pool) do
-              hash['hook_proc'].call
-            end
-          end
-
-          results = futures.zip(ok_hooks).map do |f, hash|
-            f.value || Bolt::Result.from_exception(hash['target'], f.reason)
-          end
-          set = Bolt::ResultSet.new(results + hook_errors)
-          raise Bolt::RunFailure.new(set.error_set, 'apply_prep') unless set.ok
-
-          need_install_targets.each { |target| set_agent_feature(target) }
-        end
-
-        task = applicator.custom_facts_task
-        arguments = { 'plugins' => Puppet::Pops::Types::PSensitiveType::Sensitive.new(plugins) }
-        results = run_task(targets, task, arguments, options)
-
-        # TODO: Standardize RunFailure type with error above
-        raise Bolt::RunFailure.new(results, 'run_task', task.name) unless results.ok?
-
-        results.each do |result|
-          # Log a warning if the client version is < 6
-          if unsupported_puppet?(result['clientversion'])
-            Bolt::Logger.deprecate(
-              "unsupported_puppet",
-              "Detected unsupported Puppet agent version #{result['clientversion']} on target "\
-              "#{result.target}. Bolt supports Puppet agent 6.0.0 and higher."
-            )
-          end
-
-          inventory.add_facts(result.target, result.value)
-        end
+    results
+  end
+
+  # Return a list of targets and their puppet_library hooks.
+  #
+  private def get_hooks(targets, options)
+    hooks  = []
+    errors = []
+
+    targets.each do |target|
+      plugin_opts = target.plugin_hooks.fetch('puppet_library').dup
+      plugin_name = plugin_opts.delete('plugin')
+      hook        = inventory.plugins.get_hook(plugin_name, :puppet_library)
+
+      hooks << { 'target' => target,
+                 'proc'   => hook.call(plugin_opts.merge(options), target, self) }
+    rescue StandardError => e
+      errors << Bolt::Result.from_exception(target, e)
+    end
+
+    [hooks, errors]
+  end
+
+  # Runs the puppet_library hook for each target, returning the result
+  # of each.
+  #
+  private def run_hooks(hooks)
+    require 'concurrent'
+    pool = Concurrent::ThreadPoolExecutor.new
+
+    futures = hooks.map do |hook|
+      Concurrent::Future.execute(executor: pool) do
+        hook['proc'].call
       end
     end
 
-    # Return nothing
-    nil
+    futures.zip(hooks).map do |future, hook|
+      future.value || Bolt::Result.from_exception(hook['target'], future.reason)
+    end
   end
 
   # Returns true if the client's major version is < 6.

data/bolt-modules/boltlib/lib/puppet/functions/background.rb

@@ -31,7 +31,8 @@ Puppet::Functions.create_function(:background, Puppet::Functions::InternalFuncti
     executor = Puppet.lookup(:bolt_executor)
     executor.report_function_call(self.class.name)
 
-    executor.create_future(scope: scope, name: name) do |newscope|
+    plan_id = executor.get_current_plan_id(fiber: Fiber.current)
+    executor.create_future(scope: scope, name: name, plan_id: plan_id) do |newscope|
       # Catch 'return' calls inside the block
       result = catch(:return) do
         # Execute the block. Individual plan steps in the block will yield

data/bolt-modules/boltlib/lib/puppet/functions/parallelize.rb

@@ -34,7 +34,11 @@ Puppet::Functions.create_function(:parallelize, Puppet::Functions::InternalFunct
     executor.report_function_call(self.class.name)
 
     futures = data.map do |object|
-      executor.create_future(scope: scope) do |newscope|
+      # We're going to immediately wait for these futures, *and* don't want
+      # their results to be returned as part of `wait()`, so use a 'dummy'
+      # value as the plan_id. This could also be nil, though in general we want
+      # to require Futures to have a plan stack so that they don't get lost.
+      executor.create_future(scope: scope, plan_id: 'parallel') do |newscope|
         # Catch 'return' calls inside the block
         result = catch(:return) do
           # Add the object to the block parameters

data/bolt-modules/boltlib/lib/puppet/functions/run_plan.rb

@@ -125,6 +125,17 @@ Puppet::Functions.create_function(:run_plan, Puppet::Functions::InternalFunction
       params = wrap_sensitive_parameters(params, closure.parameters)
     end
 
+    # This can be anything as long as it's unique
+    plan_instance_id = SecureRandom.uuid
+
+    # Add the plan invocation ID to the plan_stack for the PlanFuture the plan is
+    # running in so that we know the PlanFuture is running in a new plan
+    # invocation. This can be nil in test cases and when `wait()` isn't
+    # supported.
+    current_future = executor.get_current_future(fiber: Fiber.current)
+    # Safe operator to make testing easier
+    current_future&.plan_stack&.unshift(plan_instance_id)
+
     # wrap plan execution in logging messages
     executor.log_plan(plan_name) do
       result = nil
@@ -150,6 +161,8 @@ Puppet::Functions.create_function(:run_plan, Puppet::Functions::InternalFunction
           raise e
         end
       ensure
+        # Pop the plan invocation ID off of the plan_id stack for the Future.
+        current_future&.plan_stack&.shift
         if run_as
           executor.run_as = old_run_as
         end

data/bolt-modules/boltlib/lib/puppet/functions/wait.rb

@@ -1,14 +1,13 @@
 # frozen_string_literal: true
 
 require 'bolt/logger'
-require 'bolt/target'
 
 # Wait for a Future or array of Futures to finish and return results,
 # optionally with a timeout.
 #
 # > **Note:** Not available in apply block
 Puppet::Functions.create_function(:wait, Puppet::Functions::InternalFunction) do
-  # Wait for Future(s) to finish
+  # Wait for Futures to finish.
   # @param futures A Bolt Future object or array of Bolt Futures to wait on.
   # @param options A hash of additional options.
   # @option options [Boolean] _catch_errors Whether to catch raised errors.
@@ -27,7 +26,37 @@ Puppet::Functions.create_function(:wait, Puppet::Functions::InternalFunction) do
     return_type 'Array[Boltlib::PlanResult]'
   end
 
-  # Wait for Future(s) to finish with timeout
+  # Wait for all Futures in the current plan to finish.
+  # @param options A hash of additional options.
+  # @option options [Boolean] _catch_errors Whether to catch raised errors.
+  # @return A Result or Results from the Futures
+  # @example Perform multiple tasks in the background, then wait for all of them to finish
+  #   background() || { upload_file("./large_file", "/opt/jfrog/...", $targets) }
+  #   background() || { run_task("db::migrate", $targets) }
+  #   # Wait for all futures in the plan to finish and return all results
+  #   $results = wait()
+  dispatch :wait_for_all do
+    optional_param 'Hash[String[1], Any]', :options
+    return_type 'Array[Boltlib::PlanResult]'
+  end
+
+  # Wait for all Futures in the current plan to finish with a timeout.
+  # @param timeout How long to wait for Futures to finish before raising a Timeout error.
+  # @param options A hash of additional options.
+  # @option options [Boolean] _catch_errors Whether to catch raised errors.
+  # @return A Result or Results from the Futures
+  # @example Perform multiple tasks in the background, then wait for all of them to finish with a timeout
+  #   background() || { upload_file("./large_file", "/opt/jfrog/...", $targets) }
+  #   background() || { run_task("db::migrate", $targets) }
+  #   # Wait for all futures in the plan to finish and return all results
+  #   $results = wait(30)
+  dispatch :wait_for_all_with_timeout do
+    param 'Variant[Integer[0], Float[0.0]]', :timeout
+    optional_param 'Hash[String[1], Any]', :options
+    return_type 'Array[Boltlib::PlanResult]'
+  end
+
+  # Wait for Futures to finish with timeout.
   # @param futures A Bolt Future object or array of Bolt Futures to wait on.
   # @param timeout How long to wait for Futures to finish before raising a Timeout error.
   # @param options A hash of additional options.
@@ -58,14 +87,22 @@ Puppet::Functions.create_function(:wait, Puppet::Functions::InternalFunction) do
   end
 
   def wait(futures, options = {})
-    inner_wait(futures, nil, options)
+    inner_wait(futures: futures, options: options)
+  end
+
+  def wait_for_all(options = {})
+    inner_wait(options: options)
+  end
+
+  def wait_for_all_with_timeout(timeout, options = {})
+    inner_wait(timeout: timeout, options: options)
   end
 
   def wait_with_timeout(futures, timeout, options = {})
-    inner_wait(futures, timeout, options)
+    inner_wait(futures: futures, timeout: timeout, options: options)
   end
 
-  def inner_wait(futures, timeout = nil, options = {})
+  def inner_wait(futures: nil, timeout: nil, options: {})
     unless Puppet[:tasks]
       raise Puppet::ParseErrorWithIssue
         .from_issue_and_stack(Bolt::PAL::Issues::PLAN_OPERATION_NOT_SUPPORTED_WHEN_COMPILING, action: 'wait')
@@ -85,7 +122,10 @@ Puppet::Functions.create_function(:wait, Puppet::Functions::InternalFunction) do
     executor = Puppet.lookup(:bolt_executor)
     executor.report_function_call(self.class.name)
 
-    futures = Array(futures)
+    # If we get a single Future, make sure it's an array. If we didn't get any
+    # futures pass that on to wait so we can continue collecting any futures
+    # that are created while waiting on existing futures.
+    futures = Array(futures) unless futures.nil?
     executor.wait(futures, **valid)
   end
 end