bolt 3.8.1 → 3.11.0

data/lib/bolt/fiber_executor.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+require 'bolt/logger'
+require 'bolt/plan_future'
+
+module Bolt
+  class FiberExecutor
+    attr_reader :plan_futures
+
+    def initialize
+      @logger = Bolt::Logger.logger(self)
+      @id = 0
+      @plan_futures = []
+    end
+
+    # Whether there is more than one fiber running in parallel.
+    #
+    def in_parallel?
+      plan_futures.length > 1
+    end
+
+    # Creates a new Puppet scope from the current Plan scope so that variables
+    # can be used inside the block and won't interact with the outer scope.
+    # Then creates a new Fiber to execute the block, wraps the Fiber in a
+    # Bolt::PlanFuture, and returns the Bolt::PlanFuture.
+    #
+    def create_future(scope: nil, name: nil)
+      newscope = nil
+      if scope
+        # Save existing variables to the new scope before starting the future
+        # itself so that if the plan returns before the backgrounded block
+        # starts, we still have the variables.
+        newscope = Puppet::Parser::Scope.new(scope.compiler)
+        local = Puppet::Parser::Scope::LocalScope.new
+
+        # Compress the current scopes into a single vars hash to add to the new scope
+        scope.to_hash(true, true).each_pair { |k, v| local[k] = v }
+        newscope.push_ephemerals([local])
+      end
+
+      # Create a new Fiber that will execute the provided block.
+      future = Fiber.new do
+        # Yield the new scope - this should be ignored by the block if
+        # `newscope` is nil.
+        yield newscope
+      end
+
+      # PlanFutures are assigned an ID, which is just a global incrementing
+      # integer. The main plan should always have ID 0.
+      @id += 1
+      future = Bolt::PlanFuture.new(future, @id, name)
+      @logger.trace("Created future #{future.name}")
+
+      # Register the PlanFuture with the FiberExecutor to be executed
+      plan_futures << future
+      future
+    end
+
+    # Visit each PlanFuture registered with the FiberExecutor and resume it.
+    # Fibers will yield themselves back, either if they kicked off a
+    # long-running process or if the current long-running process hasn't
+    # completed. If the Fiber finishes after being resumed, store the result in
+    # the PlanFuture and remove the PlanFuture from the FiberExecutor.
+    #
+    def round_robin
+      plan_futures.each do |future|
+        # If the Fiber is still running and can be resumed, then resume it
+        @logger.trace("Checking future '#{future.name}'")
+        if future.alive?
+          @logger.trace("Resuming future '#{future.name}'")
+          future.resume
+        end
+
+        # Once we've restarted the Fiber, check to see if it's finished again
+        # and cleanup if it has.
+        next if future.alive?
+        @logger.trace("Cleaning up future '#{future.name}'")
+
+        # If the future errored and the main plan has already exited, log the
+        # error at warn level.
+        unless plan_futures.map(&:id).include?(0) || future.state == "done"
+          Bolt::Logger.warn('errored_futures', "Error in future '#{future.name}': #{future.value}")
+        end
+
+        # Remove the PlanFuture from the FiberExecutor.
+        plan_futures.delete(future)
+      end
+
+      # If the Fiber immediately returned or if the Fiber is blocking on a
+      # `wait` call, Bolt should pause for long enough that something can
+      # execute before checking again. This mitigates CPU
+      # thrashing.
+      return unless plan_futures.all? { |f| %i[returned_immediately unfinished].include?(f.value) }
+      @logger.trace("Nothing can be resumed. Rechecking in 0.5 seconds.")
+
+      sleep(0.5)
+    end
+
+    # Whether all PlanFutures have finished executing, indicating that the
+    # entire plan (main plan and any PlanFutures it spawned) has finished and
+    # Bolt can exit.
+    #
+    def plan_complete?
+      plan_futures.empty?
+    end
+
+    # Block until the provided PlanFuture objects have finished, or the timeout is reached.
+    #
+    def wait(futures, timeout: nil, catch_errors: false, **_kwargs)
+      if timeout.nil?
+        Fiber.yield(:unfinished) until futures.map(&:alive?).none?
+      else
+        start = Time.now
+        Fiber.yield(:unfinished) until (Time.now - start > timeout) || futures.map(&:alive?).none?
+        # Raise an error for any futures that are still alive
+        futures.each do |f|
+          if f.alive?
+            f.raise(Bolt::FutureTimeoutError.new(f.name, timeout))
+          end
+        end
+      end
+
+      results = futures.map(&:value)
+
+      failed_indices = results.each_index.select do |i|
+        results[i].is_a?(Bolt::Error)
+      end
+
+      if failed_indices.any?
+        if catch_errors
+          failed_indices.each { |i| results[i] = results[i].to_puppet_error }
+        else
+          # Do this after handling errors for simplicity and pretty printing
+          raise Bolt::ParallelFailure.new(results, failed_indices)
+        end
+      end
+
+      results
+    end
+  end
+end

data/lib/bolt/module_installer/installer.rb

@@ -30,7 +30,7 @@ module Bolt
 
         settings = R10K::Settings.global_settings.evaluate(@config)
         R10K::Initializers::GlobalInitializer.new(settings).call
-        install_action = R10K::Action::Puppetfile::Install.new(r10k_opts, nil)
+        install_action = R10K::Action::Puppetfile::Install.new(r10k_opts, nil, {})
 
         # Override the r10k logger with a proxy to our own logger
         R10K::Logging.instance_variable_set(:@outputter, Bolt::R10KLogProxy.new)

data/lib/bolt/outputter/human.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'bolt/container_result'
 require 'bolt/pal'
 
 module Bolt
@@ -173,11 +174,22 @@ module Bolt
           if result.message?
             @stream.puts(remove_trail(indent(2, result.message)))
           end
-
           case result.action
           when 'command', 'script'
             safe_value = result.safe_value
-            @stream.puts(indent(2, safe_value['merged_output'])) unless safe_value['merged_output'].strip.empty?
+            if safe_value["merged_output"]
+              @stream.puts(indent(2, safe_value['merged_output'])) unless safe_value['merged_output'].strip.empty?
+
+            else # output stdout or stderr
+              unless safe_value['stdout'].nil? || safe_value['stdout'].strip.empty?
+                @stream.puts(indent(2, "STDOUT:"))
+                @stream.puts(indent(4, safe_value['stdout']))
+              end
+              unless safe_value['stderr'].nil? || safe_value['stderr'].strip.empty?
+                @stream.puts(indent(2, "STDERR:"))
+                @stream.puts(indent(4, safe_value['stderr']))
+              end
+            end
           when 'lookup'
             @stream.puts(indent(2, result['value']))
           else
@@ -459,6 +471,10 @@ module Bolt
         @stream.puts(guide)
       end
 
+      def print_plan_lookup(value)
+        @stream.puts(value)
+      end
+
       def print_module_list(module_list)
         module_list.each do |path, modules|
           if (mod = modules.find { |m| m[:internal_module_group] })
@@ -487,6 +503,34 @@ module Bolt
         end
       end
 
+      def print_plugin_list(plugin_list, modulepath)
+        info   = +''
+        length = plugin_list.values.map(&:keys).flatten.map(&:length).max + 4
+
+        plugin_list.each do |hook, plugins|
+          next if plugins.empty?
+          next if hook == :validate_resolve_reference
+
+          info << colorize(:cyan, "#{hook}\n")
+
+          plugins.each do |name, description|
+            info << indent(2, name.ljust(length))
+            info << truncate(description, 80 - length) if description
+            info << "\n"
+          end
+
+          info << "\n"
+        end
+
+        info << colorize(:cyan, "Modulepath\n")
+        info << indent(2, "#{modulepath.join(File::PATH_SEPARATOR)}\n\n")
+
+        info << colorize(:cyan, "Additional information\n")
+        info << indent(2, "For more information about using plugins see https://pup.pt/bolt-plugins")
+
+        @stream.puts info.chomp
+      end
+
       def print_targets(target_list, inventory_source, default_inventory, target_flag)
         adhoc = colorize(:yellow, "(Not found in inventory file)")
 

data/lib/bolt/outputter/json.rb

@@ -60,6 +60,11 @@ module Bolt
         print_table('tasks' => tasks, 'modulepath' => modulepath)
       end
 
+      def print_plugin_list(plugins, modulepath)
+        plugins.delete(:validate_resolve_reference)
+        print_table('plugins' => plugins, 'modulepath' => modulepath)
+      end
+
       def print_plan_info(plan)
         path = plan.delete('module')
         plan['module_dir'] = if path.start_with?(Bolt::Config::Modulepath::MODULES_PATH)
@@ -98,6 +103,10 @@ module Bolt
         }.to_json)
       end
 
+      def print_plan_lookup(value)
+        @stream.puts(value.to_json)
+      end
+
       def print_puppetfile_result(success, puppetfile, moduledir)
         @stream.puts({ success: success,
                        puppetfile: puppetfile,

data/lib/bolt/pal.rb

@@ -163,9 +163,10 @@ module Bolt
     # Runs a block in a PAL script compiler configured for Bolt.  Catches
     # exceptions thrown by the block and re-raises them ensuring they are
     # Bolt::Errors since the script compiler block will squash all exceptions.
-    def in_bolt_compiler
+    def in_bolt_compiler(compiler_params: {})
       # TODO: If we always call this inside a bolt_executor we can remove this here
       setup
+      compiler_params = compiler_params.merge(set_local_facts: false)
       r = Puppet::Pal.in_tmp_environment('bolt', modulepath: full_modulepath, facts: {}) do |pal|
         # Only load the project if it a) exists, b) has a name it can be loaded with
         Puppet.override(bolt_project: @project,
@@ -174,7 +175,7 @@ module Bolt
           # of modules, it must happen *after* we have overridden
           # bolt_project or the project will be ignored
           detect_project_conflict(@project, Puppet.lookup(:environments).get('bolt'))
-          pal.with_script_compiler(set_local_facts: false) do |compiler|
+          pal.with_script_compiler(**compiler_params) do |compiler|
             alias_types(compiler)
             register_resource_types(Puppet.lookup(:loaders)) if @resource_types
             begin
@@ -299,6 +300,49 @@ module Bolt
       end
     end
 
+    def list_tasks_with_cache(filter_content: false)
+      # Don't filter content yet, so that if users update their task filters
+      # we don't need to refresh the cache
+      task_names = list_tasks(filter_content: false).map(&:first)
+      task_cache = if @project
+                     Bolt::Util.read_optional_json_file(@project.task_cache_file, 'Task cache file')
+                   else
+                     {}
+                   end
+      updated = false
+
+      task_list = task_names.each_with_object([]) do |task_name, list|
+        data = task_cache[task_name] || get_task_info(task_name, with_mtime: true)
+
+        # Make sure all the keys are strings - if we get data from
+        # get_task_info they will be symbols
+        data = Bolt::Util.walk_keys(data, &:to_s)
+
+        # If any files in the task were updated, refresh the cache
+        if data['files']&.any?
+          # For all the files that are part of the task
+          data['files'].each do |f|
+            # If any file has been updated since we last cached, update the
+            # cache
+            next if File.mtime(f['path']).to_s == f['mtime']
+            data = get_task_info(task_name, with_mtime: true)
+            data = Bolt::Util.walk_keys(data, &:to_s)
+            # Tell Bolt to write to the cache file once we're done
+            updated = true
+            # Update the cache data
+            task_cache[task_name] = data
+          end
+        end
+        metadata = data['metadata'] || {}
+        # Don't add tasks to the list to return if they are private
+        list << [task_name, metadata['description']] unless metadata['private']
+      end
+
+      # Write the cache if any entries were updated
+      File.write(@project.task_cache_file, task_cache.to_json) if updated
+      filter_content ? filter_content(task_list, @project&.tasks) : task_list
+    end
+
     def list_tasks(filter_content: false)
       in_bolt_compiler do |compiler|
         tasks = compiler.list_tasks.map(&:name).sort.each_with_object([]) do |task_name, data|
@@ -350,14 +394,20 @@ module Bolt
       end
     end
 
-    def get_task(task_name)
+    def get_task(task_name, with_mtime: false)
       task = task_signature(task_name)
 
       if task.nil?
         raise Bolt::Error.unknown_task(task_name)
       end
 
-      Bolt::Task.from_task_signature(task)
+      task = Bolt::Task.from_task_signature(task)
+      task.add_mtimes if with_mtime
+      task
+    end
+
+    def get_task_info(task_name, with_mtime: false)
+      get_task(task_name, with_mtime: with_mtime).to_h
     end
 
     def list_plans_with_cache(filter_content: false)
@@ -372,20 +422,20 @@ module Bolt
       updated = false
 
       plan_list = plan_names.each_with_object([]) do |plan_name, list|
-        info = plan_cache[plan_name] || get_plan_info(plan_name, with_mtime: true)
+        data = plan_cache[plan_name] || get_plan_info(plan_name, with_mtime: true)
 
         # If the plan is a 'local' plan (in the project itself, or the
         # modules/ directory) then verify it hasn't been updated since we
         # cached it. If it has been updated, refresh the cache and use the
         # new data.
-        if info['file'] &&
-           (File.mtime(info.dig('file', 'path')) <=> info.dig('file', 'mtime')) != 0
-          info = get_plan_info(plan_name, with_mtime: true)
+        if data['file'] &&
+           File.mtime(data.dig('file', 'path')).to_s != data.dig('file', 'mtime')
+          data = get_plan_info(plan_name, with_mtime: true)
           updated = true
-          plan_cache[plan_name] = info
+          plan_cache[plan_name] = data
         end
 
-        list << [plan_name, info['description']] unless info['private']
+        list << [plan_name, data['description']] unless data['private']
       end
 
       File.write(@project.plan_cache_file, plan_cache.to_json) if updated
@@ -585,6 +635,7 @@ module Bolt
         inputs = generator.find_inputs(:pcore)
         FileUtils.mkdir_p(@resource_types)
         cache_plan_info if @project && cache
+        cache_task_info if @project && cache
         generator.generate(inputs, @resource_types, true)
       end
     end
@@ -600,6 +651,17 @@ module Bolt
       File.write(@project.plan_cache_file, plans_info.to_json)
     end
 
+    def cache_task_info
+      # task_name is an array here
+      tasks_info = list_tasks(filter_content: false).map do |task_name,|
+        data = get_task_info(task_name, with_mtime: true)
+        { task_name => data }
+      end.reduce({}, :merge)
+
+      FileUtils.touch(@project.task_cache_file)
+      File.write(@project.task_cache_file, tasks_info.to_json)
+    end
+
     def run_task(task_name, targets, params, executor, inventory, description = nil)
       in_task_compiler(executor, inventory) do |compiler|
         params = params.merge('_bolt_api_call' => true, '_catch_errors' => true)
@@ -607,16 +669,46 @@ module Bolt
       end
     end
 
-    def run_plan(plan_name, params, executor = nil, inventory = nil, pdb_client = nil, applicator = nil)
+    def run_plan(plan_name, params, executor, inventory = nil, pdb_client = nil, applicator = nil)
+      # Start the round robin inside the plan compiler, so that
+      # backgrounded tasks can finish once the main plan exits
       in_plan_compiler(executor, inventory, pdb_client, applicator) do |compiler|
-        r = compiler.call_function('run_plan', plan_name, params.merge('_bolt_api_call' => true))
-        Bolt::PlanResult.from_pcore(r, 'success')
+        # Create a Fiber for the main plan. This will be run along with any
+        # other Fibers created during the plan run in the round_robin, with the
+        # main plan always taking precedence in being resumed.
+        future = executor.create_future(name: plan_name) do |_scope|
+          r = compiler.call_function('run_plan', plan_name, params.merge('_bolt_api_call' => true))
+          Bolt::PlanResult.from_pcore(r, 'success')
+        rescue Bolt::Error => e
+          Bolt::PlanResult.new(e, 'failure')
+        end
+
+        # Round robin until all Fibers, including the main plan, have finished.
+        # This will stay alive until backgrounded tasks have finished.
+        executor.round_robin until executor.plan_complete?
+
+        # Return the result from the main plan
+        future.value
       end
     rescue Bolt::Error => e
       Bolt::PlanResult.new(e, 'failure')
     end
 
-    def lookup(key, targets, inventory, executor, _concurrency)
+    def plan_hierarchy_lookup(key, plan_vars: {})
+      # Do a lookup with a script compiler, which uses the 'plan_hierarchy' key in
+      # Hiera config.
+      with_puppet_settings do
+        # We want all of the setup and teardown that `in_bolt_compiler` does,
+        # but also want to pass keys to the script compiler.
+        in_bolt_compiler(compiler_params: { variables: plan_vars }) do |compiler|
+          compiler.call_function('lookup', key)
+        end
+      rescue Puppet::Error => e
+        raise PALError.from_error(e)
+      end
+    end
+
+    def lookup(key, targets, inventory, executor, plan_vars: {})
       # Install the puppet-agent package and collect facts. Facts are
       # automatically added to the targets.
       in_plan_compiler(executor, inventory, nil) do |compiler|
@@ -638,21 +730,29 @@ module Bolt
 
         trusted = Puppet::Context::TrustedInformation.local(node).to_h
 
+        # Separate environment configuration from interpolation data the same
+        # way we do when compiling Puppet catalogs.
         env_conf = {
           modulepath: @modulepath.full_modulepath,
-          facts:      target.facts,
-          variables:  target.vars
+          facts:      target.facts
+        }
+
+        interpolations = {
+          variables:        plan_vars,
+          target_variables: target.vars
         }
 
         with_puppet_settings do
           Puppet::Pal.in_tmp_environment(target.name, **env_conf) do |pal|
             Puppet.override(overrides) do
               Puppet.lookup(:pal_current_node).trusted_data = trusted
-              pal.with_catalog_compiler do |compiler|
+              pal.with_catalog_compiler(**interpolations) do |compiler|
                 Bolt::Result.for_lookup(target, key, compiler.call_function('lookup', key))
               rescue StandardError => e
                 Bolt::Result.from_exception(target, e)
               end
+            rescue Puppet::Error => e
+              raise PALError.from_error(e)
             end
           end
         end