bolt 3.13.0 → 3.14.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 80a2245a2ac92c15284c4d6206f5480c6cf72033b32b773a98303b634f853e54
-  data.tar.gz: 4a6e868eba652ea23cb8530a135e3187fae81296d63df8bf9b54d5c2117df615
+  metadata.gz: e9dbc0b26225fd929d594a6719b55491bad8647650d257b0bbd1bf0eb2424f7c
+  data.tar.gz: ff0df82a3a02edfbf05bdbfe39814d2d5535719dbb6b80659bfff5d28651da3a
 SHA512:
-  metadata.gz: f7373197cc0401a971ff8e68ed70ef31f90d2087d09a5a0a5178124e40d2aae59546db768480fb5ea65f2b2993d60104abcf137df281863ddaf37c69d58365e5
-  data.tar.gz: 338c72c31cc379aaf45adae5a122a2a2e9fe6dede738aec4b46c21b5102d16a00ee5a494514dd80bc9f4fcd922ee08e62164e64d34987d40ad1d7383ff475363
+  metadata.gz: 61a484916431f7da5b3fa8992641bdf382f0d34eaf2f054e41638e27dc48b7118c07f7729a54246a0d6e19e2a74cedc549fdcc513ba26651f84b470f2c7b187e
+  data.tar.gz: d5bf6f496c13253e83dd7d9625058846000a5b339a13fd8d6d922b61ed57b61618125c5de76377f1e01d946e16bd220313659cc066ec793527a2c03013264bc0

data/Puppetfile

@@ -35,7 +35,7 @@ mod 'puppetlabs-stdlib', '7.1.0'
 mod 'puppetlabs-aws_inventory', '0.7.0'
 mod 'puppetlabs-azure_inventory', '0.5.0'
 mod 'puppetlabs-gcloud_inventory', '0.3.0'
-mod 'puppetlabs-http_request', '0.2.2'
+mod 'puppetlabs-http_request', '0.3.0'
 mod 'puppetlabs-pkcs7', '0.1.2'
 mod 'puppetlabs-secure_env_vars', '0.2.0'
 mod 'puppetlabs-terraform', '0.6.1'

data/bolt-modules/boltlib/lib/puppet/functions/background.rb

@@ -31,7 +31,8 @@ Puppet::Functions.create_function(:background, Puppet::Functions::InternalFuncti
     executor = Puppet.lookup(:bolt_executor)
     executor.report_function_call(self.class.name)
 
-    executor.create_future(scope: scope, name: name) do |newscope|
+    plan_id = executor.get_current_plan_id(fiber: Fiber.current)
+    executor.create_future(scope: scope, name: name, plan_id: plan_id) do |newscope|
       # Catch 'return' calls inside the block
       result = catch(:return) do
         # Execute the block. Individual plan steps in the block will yield

data/bolt-modules/boltlib/lib/puppet/functions/parallelize.rb

@@ -34,7 +34,11 @@ Puppet::Functions.create_function(:parallelize, Puppet::Functions::InternalFunct
     executor.report_function_call(self.class.name)
 
     futures = data.map do |object|
-      executor.create_future(scope: scope) do |newscope|
+      # We're going to immediately wait for these futures, *and* don't want
+      # their results to be returned as part of `wait()`, so use a 'dummy'
+      # value as the plan_id. This could also be nil, though in general we want
+      # to require Futures to have a plan stack so that they don't get lost.
+      executor.create_future(scope: scope, plan_id: 'parallel') do |newscope|
         # Catch 'return' calls inside the block
         result = catch(:return) do
           # Add the object to the block parameters

data/bolt-modules/boltlib/lib/puppet/functions/run_plan.rb

@@ -125,6 +125,17 @@ Puppet::Functions.create_function(:run_plan, Puppet::Functions::InternalFunction
       params = wrap_sensitive_parameters(params, closure.parameters)
     end
 
+    # This can be anything as long as it's unique
+    plan_instance_id = SecureRandom.uuid
+
+    # Add the plan invocation ID to the plan_stack for the PlanFuture the plan is
+    # running in so that we know the PlanFuture is running in a new plan
+    # invocation. This can be nil in test cases and when `wait()` isn't
+    # supported.
+    current_future = executor.get_current_future(fiber: Fiber.current)
+    # Safe operator to make testing easier
+    current_future&.plan_stack&.unshift(plan_instance_id)
+
     # wrap plan execution in logging messages
     executor.log_plan(plan_name) do
       result = nil
@@ -150,6 +161,8 @@ Puppet::Functions.create_function(:run_plan, Puppet::Functions::InternalFunction
           raise e
         end
       ensure
+        # Pop the plan invocation ID off of the plan_id stack for the Future.
+        current_future&.plan_stack&.shift
         if run_as
           executor.run_as = old_run_as
         end

data/bolt-modules/boltlib/lib/puppet/functions/wait.rb

@@ -1,14 +1,13 @@
 # frozen_string_literal: true
 
 require 'bolt/logger'
-require 'bolt/target'
 
 # Wait for a Future or array of Futures to finish and return results,
 # optionally with a timeout.
 #
 # > **Note:** Not available in apply block
 Puppet::Functions.create_function(:wait, Puppet::Functions::InternalFunction) do
-  # Wait for Future(s) to finish
+  # Wait for Futures to finish.
   # @param futures A Bolt Future object or array of Bolt Futures to wait on.
   # @param options A hash of additional options.
   # @option options [Boolean] _catch_errors Whether to catch raised errors.
@@ -27,7 +26,37 @@ Puppet::Functions.create_function(:wait, Puppet::Functions::InternalFunction) do
     return_type 'Array[Boltlib::PlanResult]'
   end
 
-  # Wait for Future(s) to finish with timeout
+  # Wait for all Futures in the current plan to finish.
+  # @param options A hash of additional options.
+  # @option options [Boolean] _catch_errors Whether to catch raised errors.
+  # @return A Result or Results from the Futures
+  # @example Perform multiple tasks in the background, then wait for all of them to finish
+  #   background() || { upload_file("./large_file", "/opt/jfrog/...", $targets) }
+  #   background() || { run_task("db::migrate", $targets) }
+  #   # Wait for all futures in the plan to finish and return all results
+  #   $results = wait()
+  dispatch :wait_for_all do
+    optional_param 'Hash[String[1], Any]', :options
+    return_type 'Array[Boltlib::PlanResult]'
+  end
+
+  # Wait for all Futures in the current plan to finish with a timeout.
+  # @param timeout How long to wait for Futures to finish before raising a Timeout error.
+  # @param options A hash of additional options.
+  # @option options [Boolean] _catch_errors Whether to catch raised errors.
+  # @return A Result or Results from the Futures
+  # @example Perform multiple tasks in the background, then wait for all of them to finish with a timeout
+  #   background() || { upload_file("./large_file", "/opt/jfrog/...", $targets) }
+  #   background() || { run_task("db::migrate", $targets) }
+  #   # Wait for all futures in the plan to finish and return all results
+  #   $results = wait(30)
+  dispatch :wait_for_all_with_timeout do
+    param 'Variant[Integer[0], Float[0.0]]', :timeout
+    optional_param 'Hash[String[1], Any]', :options
+    return_type 'Array[Boltlib::PlanResult]'
+  end
+
+  # Wait for Futures to finish with timeout.
   # @param futures A Bolt Future object or array of Bolt Futures to wait on.
   # @param timeout How long to wait for Futures to finish before raising a Timeout error.
   # @param options A hash of additional options.
@@ -58,14 +87,22 @@ Puppet::Functions.create_function(:wait, Puppet::Functions::InternalFunction) do
   end
 
   def wait(futures, options = {})
-    inner_wait(futures, nil, options)
+    inner_wait(futures: futures, options: options)
+  end
+
+  def wait_for_all(options = {})
+    inner_wait(options: options)
+  end
+
+  def wait_for_all_with_timeout(timeout, options = {})
+    inner_wait(timeout: timeout, options: options)
   end
 
   def wait_with_timeout(futures, timeout, options = {})
-    inner_wait(futures, timeout, options)
+    inner_wait(futures: futures, timeout: timeout, options: options)
   end
 
-  def inner_wait(futures, timeout = nil, options = {})
+  def inner_wait(futures: nil, timeout: nil, options: {})
     unless Puppet[:tasks]
       raise Puppet::ParseErrorWithIssue
         .from_issue_and_stack(Bolt::PAL::Issues::PLAN_OPERATION_NOT_SUPPORTED_WHEN_COMPILING, action: 'wait')
@@ -85,7 +122,10 @@ Puppet::Functions.create_function(:wait, Puppet::Functions::InternalFunction) do
     executor = Puppet.lookup(:bolt_executor)
     executor.report_function_call(self.class.name)
 
-    futures = Array(futures)
+    # If we get a single Future, make sure it's an array. If we didn't get any
+    # futures pass that on to wait so we can continue collecting any futures
+    # that are created while waiting on existing futures.
+    futures = Array(futures) unless futures.nil?
     executor.wait(futures, **valid)
   end
 end

data/bolt-modules/out/lib/puppet/functions/out/message.rb

@@ -5,7 +5,9 @@ require 'bolt/util/format'
 # Output a message for the user.
 #
 # This will print a message to stdout when using the human output format,
-# and print to stderr when using the json output format
+# and print to stderr when using the json output format. Messages are
+# also logged at the `info` level. For more information about logs, see
+# [Logs](logs.md).
 #
 # > **Note:** Not available in apply block
 Puppet::Functions.create_function(:'out::message') do
@@ -26,7 +28,7 @@ Puppet::Functions.create_function(:'out::message') do
 
     Puppet.lookup(:bolt_executor).tap do |executor|
       executor.report_function_call(self.class.name)
-      executor.publish_event(type: :message, message: Bolt::Util::Format.stringify(message))
+      executor.publish_event(type: :message, message: Bolt::Util::Format.stringify(message), level: :info)
     end
 
     nil

data/bolt-modules/out/lib/puppet/functions/out/verbose.rb

@@ -5,7 +5,9 @@ require 'bolt/util/format'
 # Output a message for the user when running in verbose mode.
 #
 # This will print a message to stdout when using the human output format,
-# and print to stderr when using the json output format.
+# and print to stderr when using the json output format. Messages are
+# also logged at the `debug` level. For more information about logs, see
+# [Logs](logs.md).
 #
 # > **Note:** Not available in apply block
 Puppet::Functions.create_function(:'out::verbose') do
@@ -25,7 +27,7 @@ Puppet::Functions.create_function(:'out::verbose') do
 
     Puppet.lookup(:bolt_executor).tap do |executor|
       executor.report_function_call(self.class.name)
-      executor.publish_event(type: :verbose, message: Bolt::Util::Format.stringify(message))
+      executor.publish_event(type: :verbose, message: Bolt::Util::Format.stringify(message), level: :debug)
     end
 
     nil

data/lib/bolt/analytics.rb

@@ -33,7 +33,7 @@ module Bolt
       logger = Bolt::Logger.logger(self)
       begin
         config_file = config_path
-        config = load_config(config_file)
+        config = enabled ? load_config(config_file) : {}
       rescue ArgumentError
         config = { 'disabled' => true }
       end

data/lib/bolt/bolt_option_parser.rb

@@ -507,11 +507,14 @@ module Bolt
           show
 
       #{colorize(:cyan, 'Usage')}
-          bolt module show [options]
+          bolt module show [module name] [options]
 
       #{colorize(:cyan, 'Description')}
           List modules available to the Bolt project.
 
+          Providing the name of a module will display detailed documentation for
+          the module.
+
       #{colorize(:cyan, 'Documentation')}
           To learn more about Bolt modules, run 'bolt guide module'.
     HELP

data/lib/bolt/cli.rb

@@ -494,7 +494,11 @@ module Bolt
         when 'group'
           list_groups
         when 'module'
-          list_modules
+          if options[:object]
+            show_module(options[:object])
+          else
+            list_modules
+          end
         when 'plugin'
           list_plugins
         end
@@ -852,6 +856,10 @@ module Bolt
       outputter.print_module_list(pal.list_modules)
     end
 
+    def show_module(name)
+      outputter.print_module_info(**pal.show_module(name))
+    end
+
     def list_plugins
       outputter.print_plugin_list(plugins.list_plugins, pal.user_modulepath)
     end
@@ -950,8 +958,9 @@ module Bolt
         files     = Dir.children(root_path).sort
 
         files.each_with_object({}) do |file, guides|
-          next if file !~ /\.txt\z/
-          topic = File.basename(file, '.txt')
+          next if file !~ /\.(yaml|yml)\z/
+          # The ".*" here removes any suffix
+          topic = File.basename(file, ".*")
           guides[topic] = File.join(root_path, file)
         end
       rescue SystemCallError => e
@@ -961,7 +970,7 @@ module Bolt
 
     # Display the list of available Bolt guides.
     def list_topics
-      outputter.print_topics(guides.keys - ['guide'])
+      outputter.print_topics(guides.keys)
       0
     end
 
@@ -971,12 +980,18 @@ module Bolt
         analytics.event('Guide', 'known_topic', label: topic)
 
         begin
-          guide = File.read(guides[topic])
+          guide = Bolt::Util.read_yaml_hash(guides[topic], 'guide')
         rescue SystemCallError => e
           raise Bolt::FileError("#{e.message}: unable to load guide page", filepath)
         end
 
-        outputter.print_guide(guide, topic)
+        # Make sure both topic and guide keys are defined
+        unless (%w[topic guide] - guide.keys).empty?
+          msg = "Guide file #{guides[topic]} must have a 'topic' key and 'guide' key, but has #{guide.keys} keys."
+          raise Bolt::Error.new(msg, 'bolt/invalid-guide')
+        end
+
+        outputter.print_guide(**Bolt::Util.symbolize_top_level_keys(guide))
       else
         analytics.event('Guide', 'unknown_topic', label: topic)
         outputter.print_message("Did not find guide for topic '#{topic}'.\n\n")

data/lib/bolt/config/transport/options.rb

@@ -16,6 +16,18 @@ module Bolt
             _default: false,
             _example: true
           },
+          "batch-mode" => {
+            type: [TrueClass, FalseClass],
+            description: "Whether to disable password querying. When set to `false`, SSH will fall back to "\
+                         "prompting for a password if key authentication fails. This might cause Bolt to hang. "\
+                         "To prevent Bolt from hanging, you can configure `ssh-command` to use an SSH utility "\
+                         "such as sshpass that supports providing a password non-interactively. For more "\
+                         "information, see [Providing a password non-interactively using "\
+                         "`native-ssh`](troubleshooting.md#providing-a-password-non-interactively-using-native-ssh).",
+            _plugin: true,
+            _default: true,
+            _example: false
+          },
           "bundled-ruby" => {
             description: "Whether to use the Ruby bundled with Bolt packages for local targets.",
             type: [TrueClass, FalseClass],

data/lib/bolt/config/transport/ssh.rb

@@ -34,6 +34,7 @@ module Bolt
 
         # Options available when using the native ssh transport
         NATIVE_OPTIONS = %w[
+          batch-mode
           cleanup
           copy-command
           host
@@ -49,6 +50,7 @@ module Bolt
         ].concat(RUN_AS_OPTIONS).sort.freeze
 
         DEFAULTS = {
+          "batch-mode"         => true,
           "cleanup"            => true,
           "connect-timeout"    => 10,
           "disconnect-timeout" => 5,
@@ -124,6 +126,11 @@ module Bolt
             msg = 'Cannot use native SSH transport with load-config set to false'
             raise Bolt::ValidationError, msg
           end
+
+          if !@config['batch-mode'] && !@config['ssh-command']
+            raise Bolt::ValidationError,
+                  'Must set ssh-command when batch-mode is set to false'
+          end
         end
       end
     end

data/lib/bolt/executor.rb

@@ -381,8 +381,16 @@ module Bolt
     # overloaded while also minimizing the Puppet lookups needed from plan
     # functions
     #
-    def create_future(scope: nil, name: nil, &block)
-      @fiber_executor.create_future(scope: scope, name: name, &block)
+    def create_future(plan_id:, scope: nil, name: nil, &block)
+      @fiber_executor.create_future(scope: scope, name: name, plan_id: plan_id, &block)
+    end
+
+    def get_current_future(fiber:)
+      @fiber_executor.get_current_future(fiber: fiber)
+    end
+
+    def get_current_plan_id(fiber:)
+      @fiber_executor.get_current_plan_id(fiber: fiber)
     end
 
     def plan_complete?
@@ -401,8 +409,8 @@ module Bolt
       @fiber_executor.wait(futures, **opts)
     end
 
-    def plan_futures
-      @fiber_executor.plan_futures
+    def get_futures_for_plan(plan_id:)
+      @fiber_executor.get_futures_for_plan(plan_id: plan_id)
     end
 
     # Execute a plan function concurrently. This function accepts the executor

data/lib/bolt/fiber_executor.rb

@@ -5,18 +5,19 @@ require 'bolt/plan_future'
 
 module Bolt
   class FiberExecutor
-    attr_reader :plan_futures
+    attr_reader :active_futures, :finished_futures
 
     def initialize
       @logger = Bolt::Logger.logger(self)
       @id = 0
-      @plan_futures = []
+      @active_futures = []
+      @finished_futures = []
     end
 
     # Whether there is more than one fiber running in parallel.
     #
     def in_parallel?
-      plan_futures.length > 1
+      active_futures.length > 1
     end
 
     # Creates a new Puppet scope from the current Plan scope so that variables
@@ -24,7 +25,7 @@ module Bolt
     # Then creates a new Fiber to execute the block, wraps the Fiber in a
     # Bolt::PlanFuture, and returns the Bolt::PlanFuture.
     #
-    def create_future(scope: nil, name: nil)
+    def create_future(plan_id:, scope: nil, name: nil)
       newscope = nil
       if scope
         # Save existing variables to the new scope before starting the future
@@ -46,13 +47,16 @@ module Bolt
       end
 
       # PlanFutures are assigned an ID, which is just a global incrementing
-      # integer. The main plan should always have ID 0.
+      # integer. The main plan should always have ID 0. They also have a
+      # plan_id, which identifies which plan spawned them. This is used for
+      # tracking which Futures to wait on when `wait()` is called without
+      # arguments.
       @id += 1
-      future = Bolt::PlanFuture.new(future, @id, name)
+      future = Bolt::PlanFuture.new(future, @id, name: name, plan_id: plan_id)
       @logger.trace("Created future #{future.name}")
 
       # Register the PlanFuture with the FiberExecutor to be executed
-      plan_futures << future
+      active_futures << future
       future
     end
 
@@ -63,7 +67,7 @@ module Bolt
     # the PlanFuture and remove the PlanFuture from the FiberExecutor.
     #
     def round_robin
-      plan_futures.each do |future|
+      active_futures.each do |future|
         # If the Fiber is still running and can be resumed, then resume it
         @logger.trace("Checking future '#{future.name}'")
         if future.alive?
@@ -78,19 +82,19 @@ module Bolt
 
         # If the future errored and the main plan has already exited, log the
         # error at warn level.
-        unless plan_futures.map(&:id).include?(0) || future.state == "done"
+        unless active_futures.map(&:id).include?(0) || future.state == "done"
           Bolt::Logger.warn('errored_futures', "Error in future '#{future.name}': #{future.value}")
         end
 
         # Remove the PlanFuture from the FiberExecutor.
-        plan_futures.delete(future)
+        finished_futures.push(active_futures.delete(future))
       end
 
       # If the Fiber immediately returned or if the Fiber is blocking on a
       # `wait` call, Bolt should pause for long enough that something can
       # execute before checking again. This mitigates CPU
       # thrashing.
-      return unless plan_futures.all? { |f| %i[returned_immediately unfinished].include?(f.value) }
+      return unless active_futures.all? { |f| %i[returned_immediately unfinished].include?(f.value) }
       @logger.trace("Nothing can be resumed. Rechecking in 0.5 seconds.")
 
       sleep(0.5)
@@ -101,12 +105,53 @@ module Bolt
     # Bolt can exit.
     #
     def plan_complete?
-      plan_futures.empty?
+      active_futures.empty?
+    end
+
+    def all_futures
+      active_futures + finished_futures
+    end
+
+    # Get the PlanFuture object that is currently executing
+    #
+    def get_current_future(fiber:)
+      all_futures.select { |f| f.fiber == fiber }.first
+    end
+
+    # Get the plan invocation ID for the PlanFuture that is currently executing
+    #
+    def get_current_plan_id(fiber:)
+      get_current_future(fiber: fiber).current_plan
+    end
+
+    # Get the Future objects associated with a particular plan invocation.
+    #
+    def get_futures_for_plan(plan_id:)
+      all_futures.select { |f| f.original_plan == plan_id }
     end
 
     # Block until the provided PlanFuture objects have finished, or the timeout is reached.
     #
     def wait(futures, timeout: nil, catch_errors: false, **_kwargs)
+      if futures.nil?
+        results = []
+        plan_id = get_current_plan_id(fiber: Fiber.current)
+        # Recollect the futures for this plan until all of the futures have
+        # finished. This ensures that we include futures created inside of
+        # futures being waited on.
+        until (futures = get_futures_for_plan(plan_id: plan_id)).map(&:alive?).none?
+          if futures.map(&:fiber).include?(Fiber.current)
+            msg = "The wait() function cannot be called with no arguments inside a "\
+                  "background block in the same plan."
+            raise Bolt::Error.new(msg, 'bolt/infinite-wait')
+          end
+          # Wait for all the futures we know about so far before recollecting
+          # Futures for the plan and waiting again
+          results = wait(futures, timeout: timeout, catch_errors: catch_errors)
+        end
+        return results
+      end
+
       if timeout.nil?
         Fiber.yield(:unfinished) until futures.map(&:alive?).none?
       else