bolt 3.8.0 → 3.10.0

data/bolt-modules/boltlib/types/planresult.pp

@@ -13,5 +13,6 @@ type Boltlib::PlanResult = Variant[Boolean,
                                    Target,
                                    ResourceInstance,
                                    ContainerResult,
+                                   Future,
                                    Array[Boltlib::PlanResult],
                                    Hash[String, Boltlib::PlanResult]]

data/bolt-modules/file/lib/puppet/functions/file/exists.rb

@@ -21,7 +21,7 @@ Puppet::Functions.create_function(:'file::exists', Puppet::Functions::InternalFu
     executor = Puppet.lookup(:bolt_executor) {}
     executor&.report_function_call(self.class.name)
 
-    future = executor&.future || Puppet.lookup(:future) || {}
+    future = executor&.future || Puppet.lookup(:future) { {} }
     fallback = future.fetch('file_paths', false)
 
     # Find the file path if it exists, otherwise return nil

data/bolt-modules/file/lib/puppet/functions/file/read.rb

@@ -20,7 +20,7 @@ Puppet::Functions.create_function(:'file::read', Puppet::Functions::InternalFunc
     executor = Puppet.lookup(:bolt_executor) {}
     executor&.report_function_call(self.class.name)
 
-    future = executor&.future || Puppet.lookup(:future) || {}
+    future = executor&.future || Puppet.lookup(:future) { {} }
     fallback = future.fetch('file_paths', false)
 
     # Find the file path if it exists, otherwise return nil

data/bolt-modules/file/lib/puppet/functions/file/readable.rb

@@ -21,7 +21,7 @@ Puppet::Functions.create_function(:'file::readable', Puppet::Functions::Internal
     executor = Puppet.lookup(:bolt_executor) {}
     executor&.report_function_call(self.class.name)
 
-    future = executor&.future || Puppet.lookup(:future) || {}
+    future = executor&.future || Puppet.lookup(:future) { {} }
     fallback = future.fetch('file_paths', false)
 
     # Find the file path if it exists, otherwise return nil

data/guides/debugging.txt

@@ -0,0 +1,28 @@
+TOPIC
+    debugging
+
+DESCRIPTION
+    When Bolt isn't behaving as expected, there are a few helpful commands and
+    logs that can help identify common issues. The first place to look is in
+    `<PROJECT>/bolt-debug.log`, which contains debug-level logs from the last Bolt
+    run. This log file includes where the Bolt project was loaded from, the
+    location of any configuration or inventory files that were loaded, and the
+    modulepath that modules were loaded from.
+
+    If you're having issues with loading targets or target configuration, you
+    can see the list of resolved Bolt target names by running `bolt inventory
+    show` on *nix systems or `Get-BoltInventory` in PowerShell. To see the
+    resolved configuration for each target, run the command with the `--detail` or
+    `-Detail` options.
+
+    Lastly, if you're having trouble loading Bolt content you can use `bolt
+    module show` on *nix systems or `Get-BoltModule` in PowerShell to see the list
+    of loaded modules, including where they were loaded from. You can also use
+    `bolt task show` or `Get-BoltTask` to list loaded tasks, and `bolt plan show`
+    or `Get-BoltPlan` to list loaded plans.
+
+    Visit the linked documentation for more in-depth troubleshooting help for
+    specific issues.
+
+DOCUMENTATION
+    https://pup.pt/bolt-troubleshooting

data/guides/inventory.txt

@@ -14,6 +14,11 @@ DESCRIPTION
     project configuration file named 'bolt-project.yaml' alongside the inventory
     file.
 
+    When Bolt loads inventory, it loads the entire inventory, not just the
+    groups and targets specified on the command line. If you've defined a
+    target in multiple groups, this might result in target configuration that
+    is different than expected.
+
 DOCUMENTATION
     https://pup.pt/bolt-inventory
     https://pup.pt/bolt-inventory-reference

data/lib/bolt/applicator.rb

@@ -306,11 +306,12 @@ module Bolt
       Puppet.lookup(:current_environment).override_with(modulepath: @plugin_dirs).modules.each do |mod|
         search_dirs = yield mod
 
-        parent = Pathname.new(mod.path).parent
+        tar_dir = Pathname.new(mod.name) # goes great with fish
+        mod_dir = Pathname.new(mod.path)
         files = Find.find(*search_dirs).select { |file| File.file?(file) }
 
         files.each do |file|
-          tar_path = Pathname.new(file).relative_path_from(parent)
+          tar_path = tar_dir + Pathname.new(file).relative_path_from(mod_dir)
           @logger.trace("Packing plugin #{file} to #{tar_path}")
           stat = File.stat(file)
           content = File.binread(file)

data/lib/bolt/bolt_option_parser.rb

@@ -67,7 +67,7 @@ module Bolt
         { flags: OPTIONS[:global] + %w[format],
           banner: GUIDE_HELP }
       when 'lookup'
-        { flags: ACTION_OPTS + %w[hiera-config],
+        { flags: ACTION_OPTS + %w[hiera-config plan-hierarchy],
           banner: LOOKUP_HELP }
       when 'module'
         case action
@@ -105,6 +105,15 @@ module Bolt
           { flags: OPTIONS[:global],
             banner: PLAN_HELP }
         end
+      when 'plugin'
+        case action
+        when 'show'
+          { flags: OPTIONS[:global] + %w[color format modulepath project],
+            banner: PLUGIN_SHOW_HELP }
+        else
+          { flags: OPTIONS[:global],
+            banner: PLUGIN_HELP }
+        end
       when 'project'
         case action
         when 'init'
@@ -192,6 +201,7 @@ module Bolt
           module            Manage Bolt project modules
           lookup            Look up a value with Hiera
           plan              Convert, create, show, and run Bolt plans
+          plugin            Show available plugins
           project           Create and migrate Bolt projects
           script            Upload a local script and run it remotely
           secret            Create encryption keys and encrypt and decrypt values
@@ -407,7 +417,7 @@ module Bolt
           lookup
 
       #{colorize(:cyan, 'Usage')}
-          bolt lookup <key> {--targets TARGETS | --query QUERY | --rerun FILTER}
+          bolt lookup <key> {--targets TARGETS | --query QUERY | --rerun FILTER | --plan-hierarchy}
             [options]
 
       #{colorize(:cyan, 'Description')}
@@ -418,6 +428,7 @@ module Bolt
 
       #{colorize(:cyan, 'Examples')}
           bolt lookup password --targets servers
+          bolt lookup password --plan-hierarchy variable=value
     HELP
 
     MODULE_HELP = <<~HELP
@@ -608,6 +619,37 @@ module Bolt
             bolt plan show aggregate::count
     HELP
 
+    PLUGIN_HELP = <<~HELP
+      #{colorize(:cyan, 'Name')}
+          plugin
+
+      #{colorize(:cyan, 'Usage')}
+          bolt plugin <action> [options]
+
+      #{colorize(:cyan, 'Description')}
+          Show available plugins.
+
+      #{colorize(:cyan, 'Documentation')}
+          Learn more about Bolt plugins at https://pup.pt/bolt-plugins.
+
+      #{colorize(:cyan, 'Actions')}
+          show          Show available plugins
+    HELP
+
+    PLUGIN_SHOW_HELP = <<~HELP
+      #{colorize(:cyan, 'Name')}
+          show
+
+      #{colorize(:cyan, 'Usage')}
+          bolt plugin show [options]
+
+      #{colorize(:cyan, 'Description')}
+          Show available plugins.
+
+      #{colorize(:cyan, 'Documentation')}
+          Learn more about Bolt plugins at https://pup.pt/bolt-plugins.
+    HELP
+
     PROJECT_HELP = <<~HELP
       #{colorize(:cyan, 'Name')}
           project
@@ -988,6 +1030,11 @@ module Bolt
         @options[:resolve] = resolve
       end
 
+      separator "\n#{self.class.colorize(:cyan, 'Lookup options')}"
+      define('--plan-hierarchy', 'Look up a value with Hiera in the context of a specific plan.') do |_|
+        @options[:plan_hierarchy] = true
+      end
+
       separator "\n#{self.class.colorize(:cyan, 'Plan options')}"
       define('--pp', 'Create a new Puppet language plan.') do |_|
         @options[:puppet] = true
@@ -996,8 +1043,8 @@ module Bolt
       separator "\n#{self.class.colorize(:cyan, 'Display options')}"
       define('--filter FILTER', 'Filter tasks and plans by a matching substring.') do |filter|
         unless /^[a-z0-9_:]+$/.match(filter)
-          msg = "Illegal characters in filter string '#{filter}'. Filters must match a legal "\
-                "task or plan name."
+          msg = "Illegal characters in filter string '#{filter}'. Filters can "\
+          "only include lowercase letters, numbers, underscores, and colons."
           raise Bolt::CLIError, msg
         end
         @options[:filter] = filter

data/lib/bolt/cli.rb

@@ -42,6 +42,7 @@ module Bolt
       'lookup'    => %w[],
       'module'    => %w[add generate-types install show],
       'plan'      => %w[show run convert new],
+      'plugin'    => %w[show],
       'project'   => %w[init migrate],
       'script'    => %w[run],
       'secret'    => %w[encrypt decrypt createkeys],
@@ -153,7 +154,6 @@ module Bolt
         options[:subcommand] = nil unless COMMANDS.include?(options[:subcommand])
 
         if Bolt::Util.first_run?
-          FileUtils.mkdir_p(Bolt::Util.first_runs_free.dirname)
           FileUtils.touch(Bolt::Util.first_runs_free)
 
           if options[:subcommand].nil? && $stdout.isatty
@@ -355,12 +355,18 @@ module Bolt
                               "the project, run '#{command} #{options[:object]}'."
       end
 
-      if options[:subcommand] != 'file' && options[:subcommand] != 'script' &&
+      if !%w[file script lookup].include?(options[:subcommand]) &&
          !options[:leftovers].empty?
         raise Bolt::CLIError,
               "Unknown argument(s) #{options[:leftovers].join(', ')}"
       end
 
+      target_opts = options.keys.select { |opt| TARGETING_OPTIONS.include?(opt) }
+      if options[:subcommand] == 'lookup' &&
+         target_opts.any? && options[:plan_hierarchy]
+        raise Bolt::CLIError, "The 'lookup' command accepts either targeting option OR --plan-hierarchy."
+      end
+
       if options[:noop] &&
          !(options[:subcommand] == 'task' && options[:action] == 'run') && options[:subcommand] != 'apply'
         raise Bolt::CLIError,
@@ -433,10 +439,11 @@ module Bolt
       end
 
       # Initialize inventory and targets. Errors here are better to catch early.
-      # options[:target_args] will contain a string/array version of the targetting options this is passed to plans
+      # options[:target_args] will contain a string/array version of the targeting options this is passed to plans
       # options[:targets] will contain a resolved set of Target objects
       unless %w[guide module project secret].include?(options[:subcommand]) ||
-             %w[convert new show].include?(options[:action])
+             %w[convert new show].include?(options[:action]) ||
+             options[:plan_hierarchy]
         update_targets(options)
       end
 
@@ -488,6 +495,8 @@ module Bolt
           list_groups
         when 'module'
           list_modules
+        when 'plugin'
+          list_plugins
         end
         return 0
       when 'convert'
@@ -517,7 +526,13 @@ module Bolt
           code = Bolt::ProjectManager.new(config, outputter, pal).migrate
         end
       when 'lookup'
-        code = lookup(options[:object], options[:targets])
+        plan_vars = Hash[options[:leftovers].map { |a| a.split('=', 2) }]
+        # Validate functions verifies one of these was passed
+        if options[:targets]
+          code = lookup(options[:object], options[:targets], plan_vars: plan_vars)
+        elsif options[:plan_hierarchy]
+          code = plan_lookup(options[:object], plan_vars: plan_vars)
+        end
       when 'plan'
         case options[:action]
         when 'new'
@@ -631,7 +646,7 @@ module Bolt
     end
 
     def list_tasks
-      tasks = filter_content(pal.list_tasks(filter_content: true), options[:filter])
+      tasks = filter_content(pal.list_tasks_with_cache(filter_content: true), options[:filter])
       outputter.print_tasks(tasks, pal.user_modulepath)
     end
 
@@ -695,10 +710,19 @@ module Bolt
       outputter.print_groups(inventory.group_names.sort, inventory.source, config.default_inventoryfile)
     end
 
+    # Looks up a value with Hiera as if in a plan outside an apply block, using
+    # provided variable values for interpolations
+    #
+    def plan_lookup(key, plan_vars: {})
+      result = pal.plan_hierarchy_lookup(key, plan_vars: plan_vars)
+      outputter.print_plan_lookup(result)
+      0
+    end
+
     # Looks up a value with Hiera, using targets as the contexts to perform the
-    # look ups in.
+    # look ups in. This should return the same value as a lookup in an apply block.
     #
-    def lookup(key, targets)
+    def lookup(key, targets, plan_vars: {})
       executor = Bolt::Executor.new(
         config.concurrency,
         analytics,
@@ -707,7 +731,7 @@ module Bolt
         config.future
       )
 
-      executor.subscribe(outputter) if options.fetch(:format, 'human') == 'human'
+      executor.subscribe(outputter) if config.format == 'human'
       executor.subscribe(log_outputter)
       executor.publish_event(type: :plan_start, plan: nil)
 
@@ -717,7 +741,7 @@ module Bolt
           targets,
           inventory,
           executor,
-          config.concurrency
+          plan_vars: plan_vars
         )
       end
 
@@ -828,6 +852,10 @@ module Bolt
       outputter.print_module_list(pal.list_modules)
     end
 
+    def list_plugins
+      outputter.print_plugin_list(plugins.list_plugins, pal.user_modulepath)
+    end
+
     def generate_types
       # generate_types will surface a nice error with helpful message if it fails
       pal.generate_types(cache: true)

data/lib/bolt/config/options.rb

@@ -279,7 +279,8 @@ module Bolt
         "modules" => {
           description: "A list of module dependencies for the project. Each dependency is a map of data specifying "\
                        "the module to install. To install the project's module dependencies, run the `bolt module "\
-                       "install` command.",
+                       "install` command. For more information about specifying modules, see [the "\
+                       "documentation](https://pup.pt/bolt-module-specs).",
           type: Array,
           items: {
             type: [Hash, String],

data/lib/bolt/config/transport/docker.rb

@@ -15,7 +15,7 @@ module Bolt
           shell-command
           tmpdir
           tty
-        ].freeze
+        ].concat(RUN_AS_OPTIONS).sort.freeze
 
         DEFAULTS = {
           'cleanup' => true

data/lib/bolt/config/transport/lxd.rb

@@ -11,7 +11,7 @@ module Bolt
           cleanup
           remote
           tmpdir
-        ].freeze
+        ].concat(RUN_AS_OPTIONS).sort.freeze
 
         DEFAULTS = {
           'cleanup' => true,

data/lib/bolt/config/transport/options.rb

@@ -235,7 +235,8 @@ module Bolt
           "private-key" => {
             type: [Hash, String],
             description: "Either the path to the private key file to use for authentication, or "\
-                         "a hash with the key `key-data` and the contents of the private key.",
+            "a hash with the key `key-data` and the contents of the private key. Note that "\
+            "the key cannot be encrypted if using the `key-data` hash.",
             required: ["key-data"],
             properties: {
               "key-data" => {

data/lib/bolt/config/transport/podman.rb

@@ -14,7 +14,7 @@ module Bolt
           shell-command
           tmpdir
           tty
-        ].freeze
+        ].concat(RUN_AS_OPTIONS).sort.freeze
 
         DEFAULTS = {
           'cleanup' => true

data/lib/bolt/error.rb

@@ -105,6 +105,16 @@ module Bolt
     end
   end
 
+  class FutureTimeoutError < Bolt::Error
+    def initialize(name, timeout)
+      details = {
+        'future' => name
+      }
+      message = "Future '#{name}' timed out after #{timeout} seconds."
+      super(message, 'bolt/future-timeout-error', details)
+    end
+  end
+
   class ParallelFailure < Bolt::Error
     def initialize(results, failed_indices)
       details = {
@@ -162,7 +172,7 @@ module Bolt
 
   class InvalidParallelResult < Error
     def initialize(result_str, file, line)
-      super("Parallel block returned an invalid result: #{result_str}",
+      super("Background block returned an invalid result: #{result_str}",
             'bolt/invalid-plan-result',
             { 'file' => file,
               'line' => line,

data/lib/bolt/executor.rb

@@ -7,10 +7,11 @@ require 'logging'
 require 'pathname'
 require 'set'
 require 'bolt/analytics'
-require 'bolt/result'
 require 'bolt/config'
-require 'bolt/result_set'
+require 'bolt/fiber_executor'
 require 'bolt/puppetdb'
+require 'bolt/result'
+require 'bolt/result_set'
 # Load transports
 require 'bolt/transport/docker'
 require 'bolt/transport/local'
@@ -20,7 +21,6 @@ require 'bolt/transport/podman'
 require 'bolt/transport/remote'
 require 'bolt/transport/ssh'
 require 'bolt/transport/winrm'
-require 'bolt/yarn'
 
 module Bolt
   TRANSPORTS = {
@@ -35,7 +35,7 @@ module Bolt
   }.freeze
 
   class Executor
-    attr_reader :noop, :transports, :in_parallel, :future
+    attr_reader :noop, :transports, :future
     attr_accessor :run_as
 
     def initialize(concurrency = 1,
@@ -66,7 +66,6 @@ module Bolt
 
       @noop = noop
       @run_as = nil
-      @in_parallel = false
       @future = future
       @pool = if concurrency > 0
                 Concurrent::ThreadPoolExecutor.new(name: 'exec', max_threads: concurrency)
@@ -77,6 +76,7 @@ module Bolt
 
       @concurrency = concurrency
       @warn_concurrency = modified_concurrency
+      @fiber_executor = Bolt::FiberExecutor.new
     end
 
     def transport(transport)
@@ -242,6 +242,10 @@ module Bolt
       @analytics&.event('Plan', plan_function, label: label)
     end
 
+    def report_noop_mode(noop)
+      @analytics&.event('Task', 'noop', label: (!!noop).to_s)
+    end
+
     def report_apply(statement_count, resource_counts)
       data = { statement_count: statement_count }
 
@@ -373,83 +377,62 @@ module Bolt
       plan.call_by_name_with_scope(scope, params, true)
     end
 
-    def create_yarn(scope, block, object, index)
-      fiber = Fiber.new do
-        # Create the new scope
-        newscope = Puppet::Parser::Scope.new(scope.compiler)
-        local = Puppet::Parser::Scope::LocalScope.new
-
-        # Compress the current scopes into a single vars hash to add to the new scope
-        current_scope = scope.effective_symtable(true)
-        until current_scope.nil?
-          current_scope.instance_variable_get(:@symbols)&.each_pair { |k, v| local[k] = v }
-          current_scope = current_scope.parent
-        end
-        newscope.push_ephemerals([local])
-
-        begin
-          result = catch(:return) do
-            args = { block.parameters[0][1].to_s => object }
-            block.closure.call_by_name_with_scope(newscope, args, true)
-          end
-
-          # If we got a return from the block, get it's value
-          # Otherwise the result is the last line from the block
-          result = result.value if result.is_a?(Puppet::Pops::Evaluator::Return)
+    # Call into FiberExecutor to avoid this class getting
+    # overloaded while also minimizing the Puppet lookups needed from plan
+    # functions
+    #
+    def create_future(scope: nil, name: nil, &block)
+      @fiber_executor.create_future(scope: scope, name: name, &block)
+    end
 
-          # Validate the result is a PlanResult
-          unless Puppet::Pops::Types::TypeParser.singleton.parse('Boltlib::PlanResult').instance?(result)
-            raise Bolt::InvalidParallelResult.new(result.to_s, *Puppet::Pops::PuppetStack.top_of_stack)
-          end
+    def plan_complete?
+      @fiber_executor.plan_complete?
+    end
 
-          result
-        rescue Puppet::PreformattedError => e
-          if e.cause.is_a?(Bolt::Error)
-            e.cause
-          else
-            raise e
-          end
-        end
-      end
+    def round_robin
+      @fiber_executor.round_robin
+    end
 
-      Bolt::Yarn.new(fiber, index)
+    def in_parallel?
+      @fiber_executor.in_parallel?
     end
 
-    def handle_event(event)
-      case event[:type]
-      when :node_result
-        @thread_completed = true
-      end
+    def wait(futures, **opts)
+      @fiber_executor.wait(futures, **opts)
     end
 
-    def round_robin(skein)
-      subscribe(self, [:node_result])
-      results = Array.new(skein.length)
-      @in_parallel = true
-      publish_event(type: :stop_spin)
+    def plan_futures
+      @fiber_executor.plan_futures
+    end
 
-      until skein.empty?
-        @thread_completed = false
-        r = nil
-
-        skein.each do |yarn|
-          if yarn.alive?
-            publish_event(type: :stop_spin)
-            r = yarn.resume
-          else
-            results[yarn.index] = yarn.value
-            skein.delete(yarn)
-          end
-        end
+    # Execute a plan function concurrently. This function accepts the executor
+    # function to be run and the parameters to pass to it, and returns the
+    # result of running the executor function.
+    #
+    def run_in_thread
+      require 'concurrent'
+      require 'fiber'
+      future = Concurrent::Future.execute do
+        yield
+      end
 
-        next unless r == 'unfinished'
-        sleep(0.1) until @thread_completed || skein.empty?
+      # Used to track how often we resume the same executor function
+      still_running = 0
+      # While the thread is still running
+      while future.incomplete?
+        # If the Fiber gets resumed, increment the resume tracker. This means
+        # the tracker starts at 1 since it needs to increment before yielding,
+        # since it can't yield then increment.
+        still_running += 1
+        # If the Fiber has been resumed before, still_running will be 2 or
+        # more. Yield different values for when the same Fiber is resumed
+        # multiple times and when it's resumed the first time in order to know
+        # if progress was made in the plan.
+        Fiber.yield(still_running < 2 ? :something_happened : :returned_immediately)
       end
 
-      publish_event(type: :stop_spin)
-      @in_parallel = false
-      unsubscribe(self, [:node_result])
-      results
+      # Once the thread completes, return the result.
+      future.value || future.reason
     end
 
     class TimeoutError < RuntimeError; end
@@ -519,7 +502,7 @@ module Bolt
     # coupled with the orchestrator transport since the transport behaves
     # differently when a plan is running. In order to limit how much this
     # pollutes the transport API we only handle the orchestrator transport here.
-    # Since we callt this function without resolving targets this will result
+    # Since we call this function without resolving targets this will result
     # in the orchestrator transport always being initialized during plan runs.
     # For now that's ok.
     #