bolt 3.11.0 → 3.15.0

data/guides/{modulepath.txt → modulepath.yaml}

@@ -1,7 +1,6 @@
-TOPIC
-    modulepath
-
-DESCRIPTION
+---
+topic: modulepath
+guide: |
     The modulepath is an ordered list of directories that Bolt loads modules
     from. When Bolt runs a command, it automatically loads modules from the
     modulepath.
@@ -21,5 +20,5 @@ DESCRIPTION
 
     To learn more about modules, see the 'module' guide.
 
-DOCUMENTATION
-    https://pup.pt/bolt-project-reference#modulepath
+documentation:
+  - https://pup.pt/bolt-project-reference#modulepath

data/guides/{project.txt → project.yaml}

@@ -1,7 +1,6 @@
-TOPIC
-    project
-
-DESCRIPTION
+---
+topic: project
+guide: |
     A Bolt project is a directory that serves as the launching point for Bolt
     and allows you to create a shareable orchestration application. Projects
     typically include a project configuration file, an inventory file, and any
@@ -17,6 +16,6 @@ DESCRIPTION
     and content, including inventory files, unless the data and content are part
     of a project.
 
-DOCUMENTATION
-    https://pup.pt/bolt-projects
-    https://pup.pt/bolt-project-reference
+documentation:
+  - https://pup.pt/bolt-projects
+  - https://pup.pt/bolt-project-reference

data/guides/{targets.txt → targets.yaml}

@@ -1,7 +1,6 @@
-TOPIC
-    targets
-
-DESCRIPTION
+---
+topic: targets
+guide: |
     A target is a device that Bolt connects to and runs actions on. Targets can
     be physical, such as servers, or virtual, such as containers or virtual
     machines.
@@ -25,5 +24,5 @@ DESCRIPTION
     project's inventory file. For more information about inventory files,
     see 'bolt guide inventory'.
 
-DOCUMENTATION
-    https://pup.pt/bolt-commands
+documentation:
+  - https://pup.pt/bolt-commands

data/guides/{transports.txt → transports.yaml}

@@ -1,7 +1,6 @@
-TOPIC
-    transports
-
-DESCRIPTION
+---
+topic: transports
+guide: |
     Bolt uses transports (also known as protocols) to establish a connection
     with a target in order to run actions on the target. The default transport is
     SSH, and you can see available transports along with their configuration
@@ -18,6 +17,6 @@ DESCRIPTION
     Finally, you can set the transport for a target in the inventory. For more
     information about the Bolt inventory, run 'bolt guide inventory'.
 
-DOCUMENTATION
-    https://pup.pt/bolt-commands#specify-a-transport
-    http://pup.pt/bolt-inventory#transport-configuration
+documentation:
+  - https://pup.pt/bolt-commands#specify-a-transport
+  - http://pup.pt/bolt-inventory#transport-configuration

data/lib/bolt/analytics.rb

@@ -33,7 +33,7 @@ module Bolt
       logger = Bolt::Logger.logger(self)
       begin
         config_file = config_path
-        config = load_config(config_file)
+        config = enabled ? load_config(config_file) : {}
       rescue ArgumentError
         config = { 'disabled' => true }
       end

data/lib/bolt/applicator.rb

@@ -122,7 +122,13 @@ module Bolt
         logs.each do |log|
           bolt_level = Bolt::Util::PuppetLogLevel::MAPPING[log['level'].to_sym]
           message = log['message'].chomp
-          @logger.send(bolt_level, "#{target.name}: #{message}")
+
+          case bolt_level
+          when :warn
+            handle_warning(target, message)
+          else
+            @logger.send(bolt_level, "#{target.name}: #{message}")
+          end
         end
       end
 
@@ -138,6 +144,22 @@ module Bolt
       result
     end
 
+    # Handles logging Puppet warnings, some of which are suppressable.
+    #
+    # @param target [Bolt::Target] The target the apply ran on.
+    # @param message [String] The log message.
+    #
+    private def handle_warning(target, message)
+      # Messages about exported resource declaration and collection, which are
+      # not supported in manifest blocks.
+      if message.include?(Puppet::Pops::Issues::RT_NO_STORECONFIGS_EXPORT.format) ||
+         message.include?(Puppet::Pops::Issues::RT_NO_STORECONFIGS.format)
+        Bolt::Logger.warn('exported_resources', "#{target.name}: #{message}")
+      else
+        @logger.send(:warn, "#{target.name}: #{message}")
+      end
+    end
+
     def validate_hiera_config(hiera_config)
       if File.exist?(File.path(hiera_config))
         data = File.open(File.path(hiera_config), "r:UTF-8") { |f| YAML.safe_load(f.read, [Symbol]) }

data/lib/bolt/bolt_option_parser.rb

@@ -507,11 +507,14 @@ module Bolt
           show
 
       #{colorize(:cyan, 'Usage')}
-          bolt module show [options]
+          bolt module show [module name] [options]
 
       #{colorize(:cyan, 'Description')}
           List modules available to the Bolt project.
 
+          Providing the name of a module will display detailed documentation for
+          the module.
+
       #{colorize(:cyan, 'Documentation')}
           To learn more about Bolt modules, run 'bolt guide module'.
     HELP
@@ -1088,11 +1091,11 @@ module Bolt
       end
       define('--log-level LEVEL',
              "Set the log level for the console. Available options are",
-             "trace, debug, info, warn, error, fatal, any.") do |level|
+             "trace, debug, info, warn, error, fatal.") do |level|
         @options[:log] = { 'console' => { 'level' => level } }
       end
       define('--clear-cache',
-             "Clear plugin cache before executing.") do |_|
+             "Clear plugin, plan, and task caches before executing.") do |_|
         @options[:clear_cache] = true
       end
       define('--plugin PLUGIN', 'Select the plugin to use.') do |plug|

data/lib/bolt/cli.rb

@@ -238,8 +238,10 @@ module Bolt
       config.check_path_case('modulepath', config.modulepath)
       config.project.check_deprecated_file
 
-      if options[:clear_cache] && File.exist?(config.project.plugin_cache_file)
-        FileUtils.rm(config.project.plugin_cache_file)
+      if options[:clear_cache]
+        FileUtils.rm(config.project.plugin_cache_file) if File.exist?(config.project.plugin_cache_file)
+        FileUtils.rm(config.project.task_cache_file) if File.exist?(config.project.task_cache_file)
+        FileUtils.rm(config.project.plan_cache_file) if File.exist?(config.project.plan_cache_file)
       end
 
       warn_inventory_overrides_cli(options)
@@ -494,7 +496,11 @@ module Bolt
         when 'group'
           list_groups
         when 'module'
-          list_modules
+          if options[:object]
+            show_module(options[:object])
+          else
+            list_modules
+          end
         when 'plugin'
           list_plugins
         end
@@ -787,8 +793,8 @@ module Bolt
       if %w[human rainbow].include?(options.fetch(:format, 'human'))
         executor.subscribe(outputter)
       else
-        # Only subscribe to out::message events for JSON outputter
-        executor.subscribe(outputter, [:message])
+        # Only subscribe to out module events for JSON outputter
+        executor.subscribe(outputter, %i[message verbose])
       end
 
       executor.subscribe(log_outputter)
@@ -832,13 +838,16 @@ module Bolt
 
       results = nil
       elapsed_time = Benchmark.realtime do
-        pal.in_plan_compiler(executor, inventory, puppetdb_client) do |compiler|
-          compiler.call_function('apply_prep', targets)
+        apply_prep_results = pal.in_plan_compiler(executor, inventory, puppetdb_client) do |compiler|
+          compiler.call_function('apply_prep', targets, '_catch_errors' => true)
         end
 
-        results = pal.with_bolt_executor(executor, inventory, puppetdb_client) do
-          Puppet.lookup(:apply_executor).apply_ast(ast, targets, catch_errors: true, noop: noop)
+        apply_results = pal.with_bolt_executor(executor, inventory, puppetdb_client) do
+          Puppet.lookup(:apply_executor)
+                .apply_ast(ast, apply_prep_results.ok_set.targets, catch_errors: true, noop: noop)
         end
+
+        results = Bolt::ResultSet.new(apply_prep_results.error_set.results + apply_results.results)
       end
 
       executor.shutdown
@@ -852,6 +861,10 @@ module Bolt
       outputter.print_module_list(pal.list_modules)
     end
 
+    def show_module(name)
+      outputter.print_module_info(**pal.show_module(name))
+    end
+
     def list_plugins
       outputter.print_plugin_list(plugins.list_plugins, pal.user_modulepath)
     end
@@ -950,8 +963,9 @@ module Bolt
         files     = Dir.children(root_path).sort
 
         files.each_with_object({}) do |file, guides|
-          next if file !~ /\.txt\z/
-          topic = File.basename(file, '.txt')
+          next if file !~ /\.(yaml|yml)\z/
+          # The ".*" here removes any suffix
+          topic = File.basename(file, ".*")
           guides[topic] = File.join(root_path, file)
         end
       rescue SystemCallError => e
@@ -961,7 +975,7 @@ module Bolt
 
     # Display the list of available Bolt guides.
     def list_topics
-      outputter.print_topics(guides.keys - ['guide'])
+      outputter.print_topics(guides.keys)
       0
     end
 
@@ -971,12 +985,18 @@ module Bolt
         analytics.event('Guide', 'known_topic', label: topic)
 
         begin
-          guide = File.read(guides[topic])
+          guide = Bolt::Util.read_yaml_hash(guides[topic], 'guide')
         rescue SystemCallError => e
           raise Bolt::FileError("#{e.message}: unable to load guide page", filepath)
         end
 
-        outputter.print_guide(guide, topic)
+        # Make sure both topic and guide keys are defined
+        unless (%w[topic guide] - guide.keys).empty?
+          msg = "Guide file #{guides[topic]} must have a 'topic' key and 'guide' key, but has #{guide.keys} keys."
+          raise Bolt::Error.new(msg, 'bolt/invalid-guide')
+        end
+
+        outputter.print_guide(**Bolt::Util.symbolize_top_level_keys(guide))
       else
         analytics.event('Guide', 'unknown_topic', label: topic)
         outputter.print_message("Did not find guide for topic '#{topic}'.\n\n")

data/lib/bolt/config/options.rb

@@ -202,7 +202,7 @@ module Bolt
                 "level" => {
                   description: "The type of information to log.",
                   type: String,
-                  enum: %w[trace debug error info warn fatal any],
+                  enum: %w[trace debug error info warn fatal],
                   _default: "warn"
                 }
               }
@@ -221,7 +221,7 @@ module Bolt
               "level" => {
                 description: "The type of information to log.",
                 type: String,
-                enum: %w[trace debug error info warn fatal any],
+                enum: %w[trace debug error info warn fatal],
                 _default: "warn"
               }
             }

data/lib/bolt/config/transport/options.rb

@@ -16,6 +16,18 @@ module Bolt
             _default: false,
             _example: true
           },
+          "batch-mode" => {
+            type: [TrueClass, FalseClass],
+            description: "Whether to disable password querying. When set to `false`, SSH will fall back to "\
+                         "prompting for a password if key authentication fails. This might cause Bolt to hang. "\
+                         "To prevent Bolt from hanging, you can configure `ssh-command` to use an SSH utility "\
+                         "such as sshpass that supports providing a password non-interactively. For more "\
+                         "information, see [Providing a password non-interactively using "\
+                         "`native-ssh`](troubleshooting.md#providing-a-password-non-interactively-using-native-ssh).",
+            _plugin: true,
+            _default: true,
+            _example: false
+          },
           "bundled-ruby" => {
             description: "Whether to use the Ruby bundled with Bolt packages for local targets.",
             type: [TrueClass, FalseClass],

data/lib/bolt/config/transport/ssh.rb

@@ -34,6 +34,7 @@ module Bolt
 
         # Options available when using the native ssh transport
         NATIVE_OPTIONS = %w[
+          batch-mode
           cleanup
           copy-command
           host
@@ -49,6 +50,7 @@ module Bolt
         ].concat(RUN_AS_OPTIONS).sort.freeze
 
         DEFAULTS = {
+          "batch-mode"         => true,
           "cleanup"            => true,
           "connect-timeout"    => 10,
           "disconnect-timeout" => 5,
@@ -124,6 +126,11 @@ module Bolt
             msg = 'Cannot use native SSH transport with load-config set to false'
             raise Bolt::ValidationError, msg
           end
+
+          if !@config['batch-mode'] && !@config['ssh-command']
+            raise Bolt::ValidationError,
+                  'Must set ssh-command when batch-mode is set to false'
+          end
         end
       end
     end

data/lib/bolt/error.rb

@@ -69,7 +69,7 @@ module Bolt
         'value' => result.value,
         'object' => result.object
       }
-      message = "Plan aborted: Running container '#{result.object}' failed."
+      message = "Running container '#{result.object}' failed."
       super(message, 'bolt/container-failure', details)
       @result = result
       @error_code = 2
@@ -86,7 +86,7 @@ module Bolt
         'result_set' => result_set
       }
       object_msg = " '#{object}'" if object
-      message = "Plan aborted: #{action}#{object_msg} failed on #{result_set.error_set.length} target"
+      message = "#{action}#{object_msg} failed on #{result_set.error_set.length} target"
       message += "s" unless result_set.error_set.length == 1
       super(message, 'bolt/run-failure', details)
       @result_set = result_set
@@ -122,7 +122,7 @@ module Bolt
         'failed_indices' => failed_indices,
         'results' => results
       }
-      message = "Plan aborted: parallel block failed on #{failed_indices.length} target"
+      message = "parallel block failed on #{failed_indices.length} target"
       message += "s" unless failed_indices.length == 1
       super(message, 'bolt/parallel-failure', details)
       @error_code = 2

data/lib/bolt/executor.rb

@@ -381,8 +381,16 @@ module Bolt
     # overloaded while also minimizing the Puppet lookups needed from plan
     # functions
     #
-    def create_future(scope: nil, name: nil, &block)
-      @fiber_executor.create_future(scope: scope, name: name, &block)
+    def create_future(plan_id:, scope: nil, name: nil, &block)
+      @fiber_executor.create_future(scope: scope, name: name, plan_id: plan_id, &block)
+    end
+
+    def get_current_future(fiber:)
+      @fiber_executor.get_current_future(fiber: fiber)
+    end
+
+    def get_current_plan_id(fiber:)
+      @fiber_executor.get_current_plan_id(fiber: fiber)
     end
 
     def plan_complete?
@@ -401,8 +409,8 @@ module Bolt
       @fiber_executor.wait(futures, **opts)
     end
 
-    def plan_futures
-      @fiber_executor.plan_futures
+    def get_futures_for_plan(plan_id:)
+      @fiber_executor.get_futures_for_plan(plan_id: plan_id)
     end
 
     # Execute a plan function concurrently. This function accepts the executor

data/lib/bolt/fiber_executor.rb

@@ -5,18 +5,19 @@ require 'bolt/plan_future'
 
 module Bolt
   class FiberExecutor
-    attr_reader :plan_futures
+    attr_reader :active_futures, :finished_futures
 
     def initialize
       @logger = Bolt::Logger.logger(self)
       @id = 0
-      @plan_futures = []
+      @active_futures = []
+      @finished_futures = []
     end
 
     # Whether there is more than one fiber running in parallel.
     #
     def in_parallel?
-      plan_futures.length > 1
+      active_futures.length > 1
     end
 
     # Creates a new Puppet scope from the current Plan scope so that variables
@@ -24,7 +25,7 @@ module Bolt
     # Then creates a new Fiber to execute the block, wraps the Fiber in a
     # Bolt::PlanFuture, and returns the Bolt::PlanFuture.
     #
-    def create_future(scope: nil, name: nil)
+    def create_future(plan_id:, scope: nil, name: nil)
       newscope = nil
       if scope
         # Save existing variables to the new scope before starting the future
@@ -46,13 +47,16 @@ module Bolt
       end
 
       # PlanFutures are assigned an ID, which is just a global incrementing
-      # integer. The main plan should always have ID 0.
+      # integer. The main plan should always have ID 0. They also have a
+      # plan_id, which identifies which plan spawned them. This is used for
+      # tracking which Futures to wait on when `wait()` is called without
+      # arguments.
       @id += 1
-      future = Bolt::PlanFuture.new(future, @id, name)
+      future = Bolt::PlanFuture.new(future, @id, name: name, plan_id: plan_id)
       @logger.trace("Created future #{future.name}")
 
       # Register the PlanFuture with the FiberExecutor to be executed
-      plan_futures << future
+      active_futures << future
       future
     end
 
@@ -63,7 +67,7 @@ module Bolt
     # the PlanFuture and remove the PlanFuture from the FiberExecutor.
     #
     def round_robin
-      plan_futures.each do |future|
+      active_futures.each do |future|
         # If the Fiber is still running and can be resumed, then resume it
         @logger.trace("Checking future '#{future.name}'")
         if future.alive?
@@ -78,19 +82,19 @@ module Bolt
 
         # If the future errored and the main plan has already exited, log the
         # error at warn level.
-        unless plan_futures.map(&:id).include?(0) || future.state == "done"
+        unless active_futures.map(&:id).include?(0) || future.state == "done"
           Bolt::Logger.warn('errored_futures', "Error in future '#{future.name}': #{future.value}")
         end
 
         # Remove the PlanFuture from the FiberExecutor.
-        plan_futures.delete(future)
+        finished_futures.push(active_futures.delete(future))
       end
 
       # If the Fiber immediately returned or if the Fiber is blocking on a
       # `wait` call, Bolt should pause for long enough that something can
       # execute before checking again. This mitigates CPU
       # thrashing.
-      return unless plan_futures.all? { |f| %i[returned_immediately unfinished].include?(f.value) }
+      return unless active_futures.all? { |f| %i[returned_immediately unfinished].include?(f.value) }
       @logger.trace("Nothing can be resumed. Rechecking in 0.5 seconds.")
 
       sleep(0.5)
@@ -101,12 +105,53 @@ module Bolt
     # Bolt can exit.
     #
     def plan_complete?
-      plan_futures.empty?
+      active_futures.empty?
+    end
+
+    def all_futures
+      active_futures + finished_futures
+    end
+
+    # Get the PlanFuture object that is currently executing
+    #
+    def get_current_future(fiber:)
+      all_futures.select { |f| f.fiber == fiber }.first
+    end
+
+    # Get the plan invocation ID for the PlanFuture that is currently executing
+    #
+    def get_current_plan_id(fiber:)
+      get_current_future(fiber: fiber).current_plan
+    end
+
+    # Get the Future objects associated with a particular plan invocation.
+    #
+    def get_futures_for_plan(plan_id:)
+      all_futures.select { |f| f.original_plan == plan_id }
     end
 
     # Block until the provided PlanFuture objects have finished, or the timeout is reached.
     #
     def wait(futures, timeout: nil, catch_errors: false, **_kwargs)
+      if futures.nil?
+        results = []
+        plan_id = get_current_plan_id(fiber: Fiber.current)
+        # Recollect the futures for this plan until all of the futures have
+        # finished. This ensures that we include futures created inside of
+        # futures being waited on.
+        until (futures = get_futures_for_plan(plan_id: plan_id)).map(&:alive?).none?
+          if futures.map(&:fiber).include?(Fiber.current)
+            msg = "The wait() function cannot be called with no arguments inside a "\
+                  "background block in the same plan."
+            raise Bolt::Error.new(msg, 'bolt/infinite-wait')
+          end
+          # Wait for all the futures we know about so far before recollecting
+          # Futures for the plan and waiting again
+          results = wait(futures, timeout: timeout, catch_errors: catch_errors)
+        end
+        return results
+      end
+
       if timeout.nil?
         Fiber.yield(:unfinished) until futures.map(&:alive?).none?
       else