bolt 3.6.1 → 3.9.0

data/lib/bolt/catalog.rb

@@ -65,7 +65,8 @@ module Bolt
       puppet_overrides = {
         bolt_pdb_client: pdb_client,
         bolt_inventory: inv,
-        bolt_project: bolt_project
+        bolt_project: bolt_project,
+        future: request['future']
       }
 
       # Facts will be set by the catalog compiler, so we need to ensure

data/lib/bolt/cli.rb

@@ -33,20 +33,23 @@ module Bolt
 
   class CLI
     COMMANDS = {
-      'command' => %w[run],
-      'script' => %w[run],
-      'task' => %w[show run],
-      'plan' => %w[show run convert new],
-      'file' => %w[download upload],
-      'secret' => %w[encrypt decrypt createkeys],
+      'apply'     => %w[],
+      'command'   => %w[run],
+      'file'      => %w[download upload],
+      'group'     => %w[show],
+      'guide'     => %w[],
       'inventory' => %w[show],
-      'group' => %w[show],
-      'project' => %w[init migrate],
-      'module' => %w[add generate-types install show],
-      'apply' => %w[],
-      'guide' => %w[]
+      'lookup'    => %w[],
+      'module'    => %w[add generate-types install show],
+      'plan'      => %w[show run convert new],
+      'project'   => %w[init migrate],
+      'script'    => %w[run],
+      'secret'    => %w[encrypt decrypt createkeys],
+      'task'      => %w[show run]
     }.freeze
 
+    TARGETING_OPTIONS = %i[query rerun targets].freeze
+
     attr_reader :config, :options
 
     def initialize(argv)
@@ -150,7 +153,6 @@ module Bolt
         options[:subcommand] = nil unless COMMANDS.include?(options[:subcommand])
 
         if Bolt::Util.first_run?
-          FileUtils.mkdir_p(Bolt::Util.first_runs_free.dirname)
           FileUtils.touch(Bolt::Util.first_runs_free)
 
           if options[:subcommand].nil? && $stdout.isatty
@@ -262,7 +264,7 @@ module Bolt
     end
 
     def update_targets(options)
-      target_opts = options.keys.select { |opt| %i[query rerun targets].include?(opt) }
+      target_opts = options.keys.select { |opt| TARGETING_OPTIONS.include?(opt) }
       target_string = "'--targets', '--rerun', or '--query'"
       if target_opts.length > 1
         raise Bolt::CLIError, "Only one targeting option #{target_string} can be specified"
@@ -324,6 +326,10 @@ module Bolt
         raise Bolt::CLIError, "a manifest file or --execute is required"
       end
 
+      if options[:subcommand] == 'lookup' && !options[:object]
+        raise Bolt::CLIError, "Must specify a key to look up"
+      end
+
       if options[:subcommand] == 'command' && (!options[:object] || options[:object].empty?)
         raise Bolt::CLIError, "Must specify a command to run"
       end
@@ -509,6 +515,8 @@ module Bolt
         when 'migrate'
           code = Bolt::ProjectManager.new(config, outputter, pal).migrate
         end
+      when 'lookup'
+        code = lookup(options[:object], options[:targets])
       when 'plan'
         case options[:action]
         when 'new'
@@ -561,7 +569,8 @@ module Bolt
             when 'command'
               executor.run_command(targets, options[:object], executor_opts)
             when 'script'
-              script_path = find_file(options[:object])
+              script_path = find_file(options[:object], executor.future&.fetch('file_paths', false))
+              validate_file('script', script_path)
               executor.run_script(targets, script_path, options[:leftovers], executor_opts)
             when 'task'
               pal.run_task(options[:object],
@@ -586,8 +595,9 @@ module Bolt
                 dest = File.expand_path(dest, Dir.pwd)
                 executor.download_file(targets, src, dest, executor_opts)
               when 'upload'
-                validate_file('source file', src, true)
-                executor.upload_file(targets, src, dest, executor_opts)
+                src_path = find_file(src, executor.future&.fetch('file_paths', false))
+                validate_file('source file', src_path, true)
+                executor.upload_file(targets, src_path, dest, executor_opts)
               end
             end
         end
@@ -634,13 +644,33 @@ module Bolt
     end
 
     def list_targets
-      inventoryfile = config.inventoryfile || config.default_inventoryfile
-      outputter.print_targets(group_targets_by_source, inventoryfile)
+      if options.keys.any? { |key| TARGETING_OPTIONS.include?(key) }
+        target_flag = true
+      else
+        options[:targets] = 'all'
+      end
+
+      outputter.print_targets(
+        group_targets_by_source,
+        inventory.source,
+        config.default_inventoryfile,
+        target_flag
+      )
     end
 
     def show_targets
-      inventoryfile = config.inventoryfile || config.default_inventoryfile
-      outputter.print_target_info(group_targets_by_source, inventoryfile)
+      if options.keys.any? { |key| TARGETING_OPTIONS.include?(key) }
+        target_flag = true
+      else
+        options[:targets] = 'all'
+      end
+
+      outputter.print_target_info(
+        group_targets_by_source,
+        inventory.source,
+        config.default_inventoryfile,
+        target_flag
+      )
     end
 
     # Returns a hash of targets sorted by those that are found in the
@@ -661,8 +691,39 @@ module Bolt
     end
 
     def list_groups
-      groups = inventory.group_names
-      outputter.print_groups(groups)
+      outputter.print_groups(inventory.group_names.sort, inventory.source, config.default_inventoryfile)
+    end
+
+    # Looks up a value with Hiera, using targets as the contexts to perform the
+    # look ups in.
+    #
+    def lookup(key, targets)
+      executor = Bolt::Executor.new(
+        config.concurrency,
+        analytics,
+        options[:noop],
+        config.modified_concurrency,
+        config.future
+      )
+
+      executor.subscribe(outputter) if options.fetch(:format, 'human') == 'human'
+      executor.subscribe(log_outputter)
+      executor.publish_event(type: :plan_start, plan: nil)
+
+      results = outputter.spin do
+        pal.lookup(
+          key,
+          targets,
+          inventory,
+          executor,
+          config.concurrency
+        )
+      end
+
+      executor.shutdown
+      outputter.print_result_set(results)
+
+      results.ok ? 0 : 1
     end
 
     def run_plan(plan_name, plan_arguments, nodes, options)
@@ -871,7 +932,7 @@ module Bolt
 
     # Display the list of available Bolt guides.
     def list_topics
-      outputter.print_topics(guides.keys)
+      outputter.print_topics(guides.keys - ['guide'])
       0
     end
 
@@ -908,20 +969,17 @@ module Bolt
     # the path is a Puppet file path and looks for the file in a module's files
     # directory.
     #
-    def find_file(path)
-      unless File.exist?(path) || Pathname.new(path).absolute?
-        modulepath = Bolt::Config::Modulepath.new(config.modulepath)
-        modules    = Bolt::Module.discover(modulepath.full_modulepath, config.project)
-        mod, file = path.split(File::SEPARATOR, 2)
-
-        if modules[mod]
-          @logger.debug("Did not find file at #{File.expand_path(path)}, checking in module '#{mod}'")
-          path = File.join(modules[mod].path, 'files', file)
-        end
+    def find_file(path, future_file_paths)
+      return path if File.exist?(path) || Pathname.new(path).absolute?
+      modulepath = Bolt::Config::Modulepath.new(config.modulepath)
+      modules    = Bolt::Module.discover(modulepath.full_modulepath, config.project)
+      mod, file = path.split(File::SEPARATOR, 2)
+
+      if modules[mod]
+        @logger.debug("Did not find file at #{File.expand_path(path)}, checking in module '#{mod}'")
+        found = Bolt::Util.find_file_in_module(modules[mod].path, file || "", future_file_paths)
+        path = found.nil? ? File.join(modules[mod].path, 'files', file) : found
       end
-
-      Bolt::Util.validate_file('script', path)
-
       path
     end
 

data/lib/bolt/config/options.rb

@@ -279,7 +279,8 @@ module Bolt
         "modules" => {
           description: "A list of module dependencies for the project. Each dependency is a map of data specifying "\
                        "the module to install. To install the project's module dependencies, run the `bolt module "\
-                       "install` command.",
+                       "install` command. For more information about specifying modules, see [the "\
+                       "documentation](https://pup.pt/bolt-module-specs).",
           type: Array,
           items: {
             type: [Hash, String],

data/lib/bolt/config/transport/docker.rb

@@ -15,7 +15,7 @@ module Bolt
           shell-command
           tmpdir
           tty
-        ].freeze
+        ].concat(RUN_AS_OPTIONS).sort.freeze
 
         DEFAULTS = {
           'cleanup' => true
@@ -27,6 +27,10 @@ module Bolt
           if @config['interpreters']
             @config['interpreters'] = normalize_interpreters(@config['interpreters'])
           end
+
+          if Bolt::Util.windows? && @config['run-as']
+            raise Bolt::ValidationError, "run-as is not supported when using PowerShell"
+          end
         end
       end
     end

data/lib/bolt/config/transport/lxd.rb

@@ -11,7 +11,7 @@ module Bolt
           cleanup
           remote
           tmpdir
-        ].freeze
+        ].concat(RUN_AS_OPTIONS).sort.freeze
 
         DEFAULTS = {
           'cleanup' => true,

data/lib/bolt/config/transport/options.rb

@@ -235,7 +235,8 @@ module Bolt
           "private-key" => {
             type: [Hash, String],
             description: "Either the path to the private key file to use for authentication, or "\
-                         "a hash with the key `key-data` and the contents of the private key.",
+            "a hash with the key `key-data` and the contents of the private key. Note that "\
+            "the key cannot be encrypted if using the `key-data` hash.",
             required: ["key-data"],
             properties: {
               "key-data" => {

data/lib/bolt/config/transport/podman.rb

@@ -14,7 +14,7 @@ module Bolt
           shell-command
           tmpdir
           tty
-        ].freeze
+        ].concat(RUN_AS_OPTIONS).sort.freeze
 
         DEFAULTS = {
           'cleanup' => true
@@ -26,6 +26,10 @@ module Bolt
           if @config['interpreters']
             @config['interpreters'] = normalize_interpreters(@config['interpreters'])
           end
+
+          if Bolt::Util.windows? && @config['run-as']
+            raise Bolt::ValidationError, "run-as is not supported when using PowerShell"
+          end
         end
       end
     end

data/lib/bolt/error.rb

@@ -105,6 +105,16 @@ module Bolt
     end
   end
 
+  class FutureTimeoutError < Bolt::Error
+    def initialize(name, timeout)
+      details = {
+        'future' => name
+      }
+      message = "Future '#{name}' timed out after #{timeout} seconds."
+      super(message, 'bolt/future-timeout-error', details)
+    end
+  end
+
   class ParallelFailure < Bolt::Error
     def initialize(results, failed_indices)
       details = {
@@ -162,7 +172,7 @@ module Bolt
 
   class InvalidParallelResult < Error
     def initialize(result_str, file, line)
-      super("Parallel block returned an invalid result: #{result_str}",
+      super("Background block returned an invalid result: #{result_str}",
             'bolt/invalid-plan-result',
             { 'file' => file,
               'line' => line,

data/lib/bolt/executor.rb

@@ -7,10 +7,11 @@ require 'logging'
 require 'pathname'
 require 'set'
 require 'bolt/analytics'
-require 'bolt/result'
 require 'bolt/config'
-require 'bolt/result_set'
+require 'bolt/fiber_executor'
 require 'bolt/puppetdb'
+require 'bolt/result'
+require 'bolt/result_set'
 # Load transports
 require 'bolt/transport/docker'
 require 'bolt/transport/local'
@@ -20,7 +21,6 @@ require 'bolt/transport/podman'
 require 'bolt/transport/remote'
 require 'bolt/transport/ssh'
 require 'bolt/transport/winrm'
-require 'bolt/yarn'
 
 module Bolt
   TRANSPORTS = {
@@ -35,7 +35,7 @@ module Bolt
   }.freeze
 
   class Executor
-    attr_reader :noop, :transports, :in_parallel, :future
+    attr_reader :noop, :transports, :future
     attr_accessor :run_as
 
     def initialize(concurrency = 1,
@@ -66,7 +66,6 @@ module Bolt
 
       @noop = noop
       @run_as = nil
-      @in_parallel = false
       @future = future
       @pool = if concurrency > 0
                 Concurrent::ThreadPoolExecutor.new(name: 'exec', max_threads: concurrency)
@@ -77,6 +76,7 @@ module Bolt
 
       @concurrency = concurrency
       @warn_concurrency = modified_concurrency
+      @fiber_executor = Bolt::FiberExecutor.new
     end
 
     def transport(transport)
@@ -373,83 +373,62 @@ module Bolt
       plan.call_by_name_with_scope(scope, params, true)
     end
 
-    def create_yarn(scope, block, object, index)
-      fiber = Fiber.new do
-        # Create the new scope
-        newscope = Puppet::Parser::Scope.new(scope.compiler)
-        local = Puppet::Parser::Scope::LocalScope.new
-
-        # Compress the current scopes into a single vars hash to add to the new scope
-        current_scope = scope.effective_symtable(true)
-        until current_scope.nil?
-          current_scope.instance_variable_get(:@symbols)&.each_pair { |k, v| local[k] = v }
-          current_scope = current_scope.parent
-        end
-        newscope.push_ephemerals([local])
-
-        begin
-          result = catch(:return) do
-            args = { block.parameters[0][1].to_s => object }
-            block.closure.call_by_name_with_scope(newscope, args, true)
-          end
-
-          # If we got a return from the block, get it's value
-          # Otherwise the result is the last line from the block
-          result = result.value if result.is_a?(Puppet::Pops::Evaluator::Return)
+    # Call into FiberExecutor to avoid this class getting
+    # overloaded while also minimizing the Puppet lookups needed from plan
+    # functions
+    #
+    def create_future(scope: nil, name: nil, &block)
+      @fiber_executor.create_future(scope: scope, name: name, &block)
+    end
 
-          # Validate the result is a PlanResult
-          unless Puppet::Pops::Types::TypeParser.singleton.parse('Boltlib::PlanResult').instance?(result)
-            raise Bolt::InvalidParallelResult.new(result.to_s, *Puppet::Pops::PuppetStack.top_of_stack)
-          end
+    def plan_complete?
+      @fiber_executor.plan_complete?
+    end
 
-          result
-        rescue Puppet::PreformattedError => e
-          if e.cause.is_a?(Bolt::Error)
-            e.cause
-          else
-            raise e
-          end
-        end
-      end
+    def round_robin
+      @fiber_executor.round_robin
+    end
 
-      Bolt::Yarn.new(fiber, index)
+    def in_parallel?
+      @fiber_executor.in_parallel?
     end
 
-    def handle_event(event)
-      case event[:type]
-      when :node_result
-        @thread_completed = true
-      end
+    def wait(futures, **opts)
+      @fiber_executor.wait(futures, **opts)
     end
 
-    def round_robin(skein)
-      subscribe(self, [:node_result])
-      results = Array.new(skein.length)
-      @in_parallel = true
-      publish_event(type: :stop_spin)
+    def plan_futures
+      @fiber_executor.plan_futures
+    end
 
-      until skein.empty?
-        @thread_completed = false
-        r = nil
-
-        skein.each do |yarn|
-          if yarn.alive?
-            publish_event(type: :stop_spin)
-            r = yarn.resume
-          else
-            results[yarn.index] = yarn.value
-            skein.delete(yarn)
-          end
-        end
+    # Execute a plan function concurrently. This function accepts the executor
+    # function to be run and the parameters to pass to it, and returns the
+    # result of running the executor function.
+    #
+    def run_in_thread
+      require 'concurrent'
+      require 'fiber'
+      future = Concurrent::Future.execute do
+        yield
+      end
 
-        next unless r == 'unfinished'
-        sleep(0.1) until @thread_completed || skein.empty?
+      # Used to track how often we resume the same executor function
+      still_running = 0
+      # While the thread is still running
+      while future.incomplete?
+        # If the Fiber gets resumed, increment the resume tracker. This means
+        # the tracker starts at 1 since it needs to increment before yielding,
+        # since it can't yield then increment.
+        still_running += 1
+        # If the Fiber has been resumed before, still_running will be 2 or
+        # more. Yield different values for when the same Fiber is resumed
+        # multiple times and when it's resumed the first time in order to know
+        # if progress was made in the plan.
+        Fiber.yield(still_running < 2 ? :something_happened : :returned_immediately)
       end
 
-      publish_event(type: :stop_spin)
-      @in_parallel = false
-      unsubscribe(self, [:node_result])
-      results
+      # Once the thread completes, return the result.
+      future.value || future.reason
     end
 
     class TimeoutError < RuntimeError; end
@@ -519,7 +498,7 @@ module Bolt
     # coupled with the orchestrator transport since the transport behaves
     # differently when a plan is running. In order to limit how much this
     # pollutes the transport API we only handle the orchestrator transport here.
-    # Since we callt this function without resolving targets this will result
+    # Since we call this function without resolving targets this will result
     # in the orchestrator transport always being initialized during plan runs.
     # For now that's ok.
     #