bolt 3.6.1 → 3.9.0

data/lib/bolt/pal.rb

@@ -607,13 +607,74 @@ module Bolt
       end
     end
 
-    def run_plan(plan_name, params, executor = nil, inventory = nil, pdb_client = nil, applicator = nil)
+    def run_plan(plan_name, params, executor, inventory = nil, pdb_client = nil, applicator = nil)
+      # Start the round robin inside the plan compiler, so that
+      # backgrounded tasks can finish once the main plan exits
       in_plan_compiler(executor, inventory, pdb_client, applicator) do |compiler|
-        r = compiler.call_function('run_plan', plan_name, params.merge('_bolt_api_call' => true))
-        Bolt::PlanResult.from_pcore(r, 'success')
+        # Create a Fiber for the main plan. This will be run along with any
+        # other Fibers created during the plan run in the round_robin, with the
+        # main plan always taking precedence in being resumed.
+        future = executor.create_future(name: plan_name) do |_scope|
+          r = compiler.call_function('run_plan', plan_name, params.merge('_bolt_api_call' => true))
+          Bolt::PlanResult.from_pcore(r, 'success')
+        rescue Bolt::Error => e
+          Bolt::PlanResult.new(e, 'failure')
+        end
+
+        # Round robin until all Fibers, including the main plan, have finished.
+        # This will stay alive until backgrounded tasks have finished.
+        executor.round_robin until executor.plan_complete?
+
+        # Return the result from the main plan
+        future.value
       end
     rescue Bolt::Error => e
       Bolt::PlanResult.new(e, 'failure')
     end
+
+    def lookup(key, targets, inventory, executor, _concurrency)
+      # Install the puppet-agent package and collect facts. Facts are
+      # automatically added to the targets.
+      in_plan_compiler(executor, inventory, nil) do |compiler|
+        compiler.call_function('apply_prep', targets)
+      end
+
+      overrides = {
+        bolt_inventory: inventory,
+        bolt_project:   @project
+      }
+
+      # Do a lookup with a catalog compiler, which uses the 'hierarchy' key in
+      # Hiera config.
+      results = targets.map do |target|
+        node = Puppet::Node.from_data_hash(
+          'name'       => target.name,
+          'parameters' => { 'clientcert' => target.name }
+        )
+
+        trusted = Puppet::Context::TrustedInformation.local(node).to_h
+
+        env_conf = {
+          modulepath: @modulepath.full_modulepath,
+          facts:      target.facts,
+          variables:  target.vars
+        }
+
+        with_puppet_settings do
+          Puppet::Pal.in_tmp_environment(target.name, **env_conf) do |pal|
+            Puppet.override(overrides) do
+              Puppet.lookup(:pal_current_node).trusted_data = trusted
+              pal.with_catalog_compiler do |compiler|
+                Bolt::Result.for_lookup(target, key, compiler.call_function('lookup', key))
+              rescue StandardError => e
+                Bolt::Result.from_exception(target, e)
+              end
+            end
+          end
+        end
+      end
+
+      Bolt::ResultSet.new(results)
+    end
   end
 end

data/lib/bolt/pal/yaml_plan/step.rb

@@ -122,9 +122,11 @@ module Bolt
               raise StepError.new("Parameters key must be a hash", body['name'], step_number)
             end
 
-            metaparams = option_keys.map { |key| "_#{key}" }
+            metaparams = body['parameters'].keys
+                                           .select { |key| key.start_with?('_') }
+                                           .map { |key| key.sub(/^_/, '') }
 
-            if (dups = body['parameters'].keys & metaparams).any?
+            if (dups = body.keys & metaparams).any?
               raise StepError.new(
                 "Cannot specify metaparameters when using top-level keys with same name: #{dups.join(', ')}",
                 body['name'],

data/lib/bolt/plan_creator.rb

@@ -36,8 +36,8 @@ module Bolt
       prefix, _, basename = segment_plan_name(plan_name)
 
       unless prefix == project.name
-        message = "First segment of plan name '#{plan_name}' must match project name '#{project.name}'. "\
-                  "Did you mean '#{project.name}::#{plan_name}'?"
+        message = "Incomplete plan name: A plan name must be prefixed with the name of the "\
+          "project or module. Did you mean '#{project.name}::#{plan_name}'?"
 
         raise Bolt::ValidationError, message
       end

data/lib/bolt/plan_future.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require 'fiber'
+
+module Bolt
+  class PlanFuture
+    attr_reader :fiber, :id
+    attr_accessor :value
+
+    def initialize(fiber, id, name = nil)
+      @fiber = fiber
+      @id    = id
+      @name  = name
+      @value = nil
+    end
+
+    def name
+      @name || @id
+    end
+
+    def to_s
+      "Future '#{name}'"
+    end
+
+    def alive?
+      fiber.alive?
+    end
+
+    def raise(exception)
+      # Make sure the value gets set
+      @value = exception
+      # This was introduced in Ruby 2.7
+      begin
+        # Raise an exception to kill the Fiber. If the Fiber has not been
+        # resumed yet, or is already terminated this will raise a FiberError.
+        # We don't especially care about the FiberError, as long as the Fiber
+        # doesn't report itself as alive.
+        fiber.raise(exception)
+      rescue FiberError
+        # If the Fiber is still alive, resume it with a block to raise the
+        # exception which will terminate it.
+        if fiber.alive?
+          fiber.resume { raise(exception) }
+        end
+      end
+    end
+
+    def resume
+      if fiber.alive?
+        @value = fiber.resume
+      else
+        @value
+      end
+    end
+
+    def state
+      if fiber.alive?
+        "running"
+      elsif value.is_a?(Exception)
+        "error"
+      else
+        "done"
+      end
+    end
+  end
+end

data/lib/bolt/puppetdb/client.rb

@@ -95,6 +95,60 @@ module Bolt
         make_query(query, path)
       end
 
+      # Sends a command to PuppetDB using version 1 of the commands API.
+      # https://puppet.com/docs/puppetdb/latest/api/command/v1/commands.html
+      #
+      # @param command [String] The command to invoke.
+      # @param version [Integer] The version of the command to invoke.
+      # @param payload [Hash] The payload to send with the command.
+      # @return A UUID identifying the submitted command.
+      #
+      def send_command(command, version, payload)
+        command = command.dup.force_encoding('utf-8')
+        body    = JSON.generate(payload)
+
+        # PDB requires the following query parameters to the POST request.
+        # Error early if there's no certname, as PDB does not return a
+        # message indicating it's required.
+        unless payload['certname']
+          raise Bolt::Error.new(
+            "Payload must include 'certname', unable to invoke command.",
+            'bolt/pdb-command'
+          )
+        end
+
+        url = uri.tap do |u|
+          u.path         = 'pdb/cmd/v1'
+          u.query_values = { 'command'  => command,
+                             'version'  => version,
+                             'certname' => payload['certname'] }
+        end
+
+        # Send the command to PDB
+        begin
+          @logger.debug("Sending PuppetDB command '#{command}' to #{url}")
+          response = http_client.post(url.to_s, body: body, header: headers)
+        rescue StandardError => e
+          raise Bolt::PuppetDBFailoverError, "Failed to invoke PuppetDB command: #{e}"
+        end
+
+        @logger.debug("Got response code #{response.code} from PuppetDB")
+        if response.code != 200
+          raise Bolt::PuppetDBError, "Failed to invoke PuppetDB command: #{response.body}"
+        end
+
+        # Return the UUID string from the response body
+        begin
+          JSON.parse(response.body).fetch('uuid', nil)
+        rescue JSON::ParserError
+          raise Bolt::PuppetDBError, "Unable to parse response as JSON: #{response.body}"
+        end
+      rescue Bolt::PuppetDBFailoverError => e
+        @logger.error("Request to puppetdb at #{@current_url} failed with #{e}.")
+        reject_url
+        send_command(command, version, payload)
+      end
+
       def http_client
         return @http if @http
         # lazy-load expensive gem code

data/lib/bolt/result.rb

@@ -28,6 +28,11 @@ module Bolt
       %w[file line].zip(position).to_h.compact
     end
 
+    def self.for_lookup(target, key, value)
+      val = { 'value' => value }
+      new(target, value: val, action: 'lookup', object: key)
+    end
+
     def self.for_command(target, value, action, command, position)
       details = create_details(position)
       unless value['exit_code'] == 0

data/lib/bolt/transport/docker/connection.rb

@@ -27,6 +27,10 @@ module Bolt
                      end
         end
 
+        def reset_cwd?
+          true
+        end
+
         # The full ID of the target container
         #
         # @return [String] The full ID of the target container
@@ -46,8 +50,8 @@ module Bolt
         def connect
           # We don't actually have a connection, but we do need to
           # check that the container exists and is running.
-          output = execute_local_json_command('ps')
-          index = output.find_index { |item| item["ID"] == target.host || item["Names"] == target.host }
+          output = execute_local_json_command('ps', ['--no-trunc'])
+          index = output.find_index { |item| item["ID"].start_with?(target.host) || item["Names"] == target.host }
           raise "Could not find a container with name or ID matching '#{target.host}'" if index.nil?
           # Now find the indepth container information
           output = execute_local_json_command('inspect', [output[index]["ID"]])
@@ -74,7 +78,6 @@ module Bolt
           # CODEREVIEW: Is it always safe to pass --interactive?
           args += %w[--interactive]
           args += %w[--tty] if target.options['tty']
-          args += %W[--env DOCKER_HOST=#{@docker_host}] if @docker_host
           args += @env_vars if @env_vars
 
           if target.options['shell-command'] && !target.options['shell-command'].empty?
@@ -86,7 +89,7 @@ module Bolt
           docker_command = %w[docker exec] + args + [container_id] + Shellwords.split(command)
           @logger.trace { "Executing: #{docker_command.join(' ')}" }
 
-          Open3.popen3(*docker_command)
+          Open3.popen3(env_hash, *docker_command)
         rescue StandardError
           @logger.trace { "Command aborted" }
           raise

data/lib/bolt/transport/lxd/connection.rb

@@ -23,6 +23,10 @@ module Bolt
           Bolt::Shell::Bash.new(target, self)
         end
 
+        def reset_cwd?
+          true
+        end
+
         def container_id
           "#{@target.transport_config['remote']}:#{@target.host}"
         end

data/lib/bolt/transport/podman/connection.rb

@@ -30,6 +30,10 @@ module Bolt
                      end
         end
 
+        def reset_cwd?
+          true
+        end
+
         def connect
           # We don't actually have a connection, but we do need to
           # check that the container exists and is running.

data/lib/bolt/transport/ssh/connection.rb

@@ -39,8 +39,6 @@ module Bolt
           end
         end
 
-        PAGEANT_NAME = "Pageant\0".encode(Encoding::UTF_16LE)
-
         def connect
           options = {
             logger: @transport_logger,
@@ -115,10 +113,9 @@ module Bolt
                 options[:use_agent] = false
               end
             elsif Bolt::Util.windows?
-              require 'Win32API' # case matters in this require!
-              # https://docs.microsoft.com/en-us/windows/desktop/api/winuser/nf-winuser-findwindoww
-              @find_window ||= Win32API.new('user32', 'FindWindowW', %w[P P], 'L')
-              if @find_window.call(nil, PAGEANT_NAME).to_i == 0
+              pageant = Net::SSH::Authentication::Pageant::Win.FindWindow("Pageant", "Pageant")
+              # If pageant is not running
+              if pageant.to_i == 0
                 @logger.debug { "Disabling use_agent in net-ssh: pageant process not running" }
                 options[:use_agent] = false
               end

data/lib/bolt/util.rb

@@ -78,11 +78,83 @@ module Bolt
       end
 
       def first_runs_free
+        # If this fails, use the system path instead
+        FileUtils.mkdir_p(Bolt::Config.user_path)
         Bolt::Config.user_path + '.first_runs_free'
+      rescue StandardError
+        begin
+          # If using the system path fails, then don't bother with the welcome
+          # message
+          FileUtils.mkdir_p(Bolt::Config.system_path)
+          Bolt::Config.system_path + '.first_runs_free'
+        rescue StandardError
+          nil
+        end
       end
 
       def first_run?
-        Bolt::Config.user_path && !File.exist?(first_runs_free)
+        !first_runs_free.nil? &&
+          !File.exist?(first_runs_free)
+      end
+
+      # If Puppet is loaded, we aleady have the path to the module and should
+      # just get it. This takes the path to a file provided by the user and a
+      # Puppet Parser scope object and tries to find the file, either as an
+      # absolute path or Puppet module syntax lookup. Returns the path to the
+      # file if found, or nil.
+      #
+      def find_file_from_scope(file, scope, fallback = false)
+        # If we got an absolute path, just return that.
+        return file if Pathname.new(file).absolute?
+
+        module_name, file_pattern = Bolt::Util.split_path(file)
+        # Get the absolute path to the module root from the scope
+        mod_path = scope.compiler.environment.module(module_name)&.path
+
+        # Search the module for the file, falling back to new-style paths if enabled.
+        find_file_in_module(mod_path, file_pattern, fallback) if mod_path
+      end
+
+      # This method is used by Bolt to find files when provided a
+      # module-style path without loading Puppet. It takes the absolute path to
+      # the module root and the module-style path minus the module name and
+      # searches subdirectories in the module in order of precedence.
+      #
+      def find_file_in_module(module_path, module_file, fallback = false)
+        # If the first part of the path is 'scripts' or 'files', the path may
+        # be a new-style file location and should fall back to the new path.
+        subdir_or_file = split_path(module_file).first
+        case subdir_or_file
+          # For any subdirs that may indicate the user passed a new-style path,
+          # first look in 'mymod/files/<relative_path>' (old-style) then fall
+          # back to 'mymod/<relative_path>' (new-style) if enabled.
+        when 'scripts', 'files'
+          search_module(module_path, module_file, fallback)
+        else
+          # If the path definitely isn't new-style, only look in the 'files/'
+          # directory.
+          search_module(module_path, module_file)
+        end
+      end
+
+      # This searches a module for files under 'files/' or 'scripts/',
+      # optionally falling back to the new style of file loading. It takes the
+      # absolute path to the module root, the relative path provided by the
+      # user, and whether to fall back to the new-style script loading if the
+      # file isn't found in 'files/'.
+      #
+      private def search_module(module_path, module_file, fallback = false)
+        if File.exist?(File.join(module_path, 'files', module_file))
+          File.join(module_path, 'files', module_file)
+        elsif File.exist?(File.join(module_path, module_file)) && fallback
+          File.join(module_path, module_file)
+        end
+      end
+
+      # Copied directly from puppet/lib/puppet/parser/files.rb
+      #
+      def split_path(path)
+        path.split(File::SEPARATOR, 2)
       end
 
       # Accepts a path with either 'plans' or 'tasks' in it and determines

data/lib/bolt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bolt
-  VERSION = '3.6.1'
+  VERSION = '3.9.0'
 end

data/lib/bolt_spec/plans/mock_executor.rb

@@ -17,13 +17,12 @@ module BoltSpec
 
     # Nothing on the executor is 'public'
     class MockExecutor
-      attr_reader :noop, :error_message, :in_parallel, :transports, :future
+      attr_reader :noop, :error_message, :transports, :future
       attr_accessor :run_as, :transport_features, :execute_any_plan
 
       def initialize(modulepath)
         @noop = false
         @run_as = nil
-        @in_parallel = false
         @future = {}
         @error_message = nil
         @allow_apply = false
@@ -38,6 +37,7 @@ module BoltSpec
         @execute_any_plan = true
         # plans that are allowed to be executed by the @executor_real
         @allowed_exec_plans = {}
+        @id = 0
       end
 
       def module_file_id(file)
@@ -257,58 +257,53 @@ module BoltSpec
       end
       # End apply_prep mocking
 
-      # Evaluates a `parallelize()` block and returns the result. Normally,
-      # Bolt's executor wraps this in a Yarn for each object passed to the
-      # `parallelize()` function, and then executes them in parallel before
-      # returning the result from the block. However, in BoltSpec the block is
-      # executed for each object sequentially, and this function returns the
-      # result itself.
-      #
-      def create_yarn(scope, block, object, _index)
-        # Create the new scope
-        newscope = Puppet::Parser::Scope.new(scope.compiler)
-        local = Puppet::Parser::Scope::LocalScope.new
-
-        # Compress the current scopes into a single vars hash to add to the new scope
-        current_scope = scope.effective_symtable(true)
-        until current_scope.nil?
-          current_scope.instance_variable_get(:@symbols)&.each_pair { |k, v| local[k] = v }
-          current_scope = current_scope.parent
-        end
-        newscope.push_ephemerals([local])
-
-        begin
-          result = catch(:return) do
-            args = { block.parameters[0][1].to_s => object }
-            block.closure.call_by_name_with_scope(newscope, args, true)
-          end
-
-          # If we got a return from the block, get it's value
-          # Otherwise the result is the last line from the block
-          result = result.value if result.is_a?(Puppet::Pops::Evaluator::Return)
+      # Parallel function mocking
+      def run_in_thread
+        yield
+      end
 
-          # Validate the result is a PlanResult
-          unless Puppet::Pops::Types::TypeParser.singleton.parse('Boltlib::PlanResult').instance?(result)
-            raise Bolt::InvalidParallelResult.new(result.to_s, *Puppet::Pops::PuppetStack.top_of_stack)
-          end
+      def in_parallel?
+        false
+      end
 
-          result
-        rescue Puppet::PreformattedError => e
-          if e.cause.is_a?(Bolt::Error)
-            e.cause
-          else
-            raise e
+      def create_future(scope: nil, name: nil)
+        newscope = nil
+        if scope
+          # Create the new scope
+          newscope = Puppet::Parser::Scope.new(scope.compiler)
+          local = Puppet::Parser::Scope::LocalScope.new
+
+          # Compress the current scopes into a single vars hash to add to the new scope
+          current_scope = scope.effective_symtable(true)
+          until current_scope.nil?
+            current_scope.instance_variable_get(:@symbols)&.each_pair { |k, v| local[k] = v }
+            current_scope = current_scope.parent
           end
+          newscope.push_ephemerals([local])
         end
+
+        # Execute "futures" serially when running in BoltSpec
+        result = yield newscope
+        @id += 1
+        future = Bolt::PlanFuture.new(nil, @id, name: name)
+        future.value = result
+        future
       end
 
-      # BoltSpec already evaluated the `parallelize()` block for each object
-      # passed to the function, so these results can be returned as-is.
-      #
-      def round_robin(results)
+      def wait(results, _timeout, **_kwargs)
         results
       end
 
+      # Since Futures are executed immediately once created, this will always
+      # be true by the time it's called.
+      def plan_complete?
+        true
+      end
+
+      def plan_futures
+        []
+      end
+
       # Public methods on Bolt::Executor that need to be mocked so there aren't
       # "undefined method" errors.
 
@@ -337,6 +332,8 @@ module BoltSpec
       def subscribe(_subscriber, _types = nil); end
 
       def unsubscribe(_subscriber, _types = nil); end
+
+      def round_robin; end
     end
   end
 end