openbolt 9.0.0.pre.rc1 → 9.0.0

data/lib/bolt/fiber_executor.rb

@@ -5,6 +5,11 @@ require_relative '../bolt/plan_future'
 
 module Bolt
   class FiberExecutor
+    # Limits how many plans one future can nest, so a plan that runs itself
+    # with no condition that stops it fails instead of starting Fibers until
+    # memory runs out.
+    MAX_NESTED_PLANS = 1000
+
     attr_reader :active_futures, :finished_futures
 
     def initialize
@@ -99,6 +104,7 @@ module Bolt
       # `wait` call, Bolt should pause for long enough that something can
       # execute before checking again. This mitigates CPU
       # thrashing.
+      return if active_futures.empty?
       return unless active_futures.all? { |f| %i[returned_immediately unfinished].include?(f.value) }
 
       @logger.trace("Nothing can be resumed. Rechecking in 0.5 seconds.")
@@ -121,7 +127,45 @@ module Bolt
     # Get the PlanFuture object that is currently executing
     #
     def get_current_future(fiber:)
-      all_futures.select { |f| f.fiber == fiber }.first
+      all_futures.find { |f| f.fiber == fiber || f.nested_fibers.include?(fiber) }
+    end
+
+    # Run the block on a new Fiber, so it starts with an empty stack instead
+    # of adding to the current one. Every plan a plan runs adds dozens of
+    # frames, and a Fiber's stack is small and fixed. Values the block yields
+    # while it waits are yielded on to the scheduler, and an error raised into
+    # the current Fiber, like a wait() timeout, is raised inside the block.
+    #
+    def run_in_new_fiber(&block)
+      future = get_current_future(fiber: Fiber.current)
+      return yield unless future
+
+      if future.nested_fibers.length >= MAX_NESTED_PLANS
+        msg = "Plans cannot be nested more than #{MAX_NESTED_PLANS} levels deep. Check for a " \
+              "plan that runs itself without a condition that stops it."
+        # A plain error, not a Bolt::Error, so plans can't catch it with
+        # _catch_errors or catch_errors() and keep a runaway plan going.
+        raise msg
+      end
+
+      fiber = Fiber.new(&block)
+      future.nested_fibers.push(fiber)
+      @logger.trace("Running a nested plan on a new Fiber in future '#{future.name}'")
+      begin
+        value = fiber.resume
+        while fiber.alive?
+          begin
+            resumed_with = Fiber.yield(value)
+          rescue StandardError => e
+            value = fiber.raise(e)
+            next
+          end
+          value = fiber.resume(resumed_with)
+        end
+        value
+      ensure
+        future.nested_fibers.delete(fiber)
+      end
     end
 
     # Get the plan invocation ID for the PlanFuture that is currently executing
@@ -146,7 +190,7 @@ module Bolt
         # finished. This ensures that we include futures created inside of
         # futures being waited on.
         until (futures = get_futures_for_plan(plan_id: plan_id)).map(&:alive?).none?
-          if futures.map(&:fiber).include?(Fiber.current)
+          if futures.include?(get_current_future(fiber: Fiber.current))
             msg = "The wait() function cannot be called with no arguments inside a " \
                   "background block in the same plan."
             raise Bolt::Error.new(msg, 'bolt/infinite-wait')

data/lib/bolt/inventory/options.rb

@@ -84,7 +84,7 @@ module Bolt
         "plugin_hooks" => {
           description: "Configuration for the Puppet library plugin used to install the " \
                        "Puppet agent on the target. For more information, see " \
-                       "https://docs.openvoxproject.org/openbolt/latest/writing_plugins.html",
+                       "[plugin hooks](writing_plugins.md#plugin-hooks).",
           type: Hash,
           properties: {
             "puppet_library" => {

data/lib/bolt/plan_future.rb

@@ -4,7 +4,7 @@ require 'fiber'
 
 module Bolt
   class PlanFuture
-    attr_reader :fiber, :id, :scope
+    attr_reader :fiber, :id, :scope, :nested_fibers
     attr_accessor :value, :plan_stack
 
     def initialize(fiber, id, plan_id:, name: nil, scope: nil)
@@ -24,6 +24,10 @@ module Bolt
       # since now we're in the calling plan. These IDs are unique to each plan
       # invocation, not just plan names.
       @plan_stack = [plan_id]
+
+      # Fibers this Future started with FiberExecutor#run_in_new_fiber, which
+      # still belong to this Future
+      @nested_fibers = []
     end
 
     def original_plan

data/lib/bolt/shell/bash.rb

@@ -230,15 +230,17 @@ module Bolt
         ''
       end
 
+      # GNU sudo and sudo-rs word these differently
       def handle_sudo_errors(err)
         case err
-        when /^#{conn.user} is not in the sudoers file\./
+        when /^#{conn.user} is not in the sudoers file\./,
+             /Sorry, user #{conn.user} (may not run|is not allowed to execute)/
           @logger.trace { err }
           raise Bolt::Node::EscalateError.new(
             "User #{conn.user} does not have sudo permission on #{target}",
             'SUDO_DENIED'
           )
-        when /^Sorry, try again\./
+        when /^Sorry, try again\./, /Authentication failed, try again/
           @logger.trace { err }
           raise Bolt::Node::EscalateError.new(
             "Sudo password for user #{conn.user} not recognized on #{target}",
@@ -299,7 +301,7 @@ module Bolt
 
         result = execute(command)
         if result.exit_code != 0
-          raise Bolt::Node::FileError.new("Could not make tmpdir: #{result.stderr.string}", 'TMPDIR_ERROR')
+          raise Bolt::Node::FileError.new("Could not make tmpdir #{tmppath}: #{result.stderr.string}", 'TMPDIR_ERROR')
         end
 
         path = tmppath || result.stdout.string.chomp

data/lib/bolt/shell/powershell/snippets.rb

@@ -1,10 +1,23 @@
 # frozen_string_literal: true
 
+require 'base64'
+
 module Bolt
   class Shell
     class Powershell < Shell
       module Snippets
         class << self
+          # An expression that evaluates to the value as a string. The value
+          # is sent as base64, so no character in it can end a quoted string
+          # or here-string, whatever encoding PowerShell reads the script in.
+          # A binary string, such as a command-line argument under the C
+          # locale, goes as its bytes, which PowerShell reads as UTF-8.
+          def base64_string(value)
+            value = value.to_s
+            value = value.encode('UTF-8') unless value.encoding == Encoding::BINARY
+            "[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String('#{Base64.strict_encode64(value)}'))"
+          end
+
           def execute_process(command)
             <<~PS
             if ([Console]::InputEncoding -eq [System.Text.Encoding]::UTF8) {
@@ -46,7 +59,7 @@ module Bolt
 
           def run_script(arguments, script_path)
             build_arg_list = arguments.map do |a|
-              "$invokeArgs.ArgumentList += @'\n#{a}\n'@"
+              "$invokeArgs.ArgumentList += #{base64_string(a)}"
             end.join("\n")
             <<~PS
             $invokeArgs = @{
@@ -138,6 +151,12 @@ module Bolt
             "${boltBaseDir}\\hiera\\lib;" +
             $ENV:RUBYLIB
 
+            #{json_functions.chomp}
+            PS
+          end
+
+          def json_functions
+            <<~PS
             function ConvertFrom-PSCustomObject
             {
             PARAM([Parameter(ValueFromPipeline = $true)] $InputObject)

data/lib/bolt/shell/powershell.rb

@@ -88,12 +88,12 @@ module Bolt
 
       def env_declarations(env_vars)
         env_vars.map do |var, val|
-          "[Environment]::SetEnvironmentVariable('#{var}', @'\n#{val}\n'@)"
+          "[Environment]::SetEnvironmentVariable(#{quote_string(var.to_s)}, #{Snippets.base64_string(val)})"
         end
       end
 
       def quote_string(string)
-        "'" + string.gsub("'", "''") + "'"
+        "'" + Bolt::Util.escape_powershell_single_quotes(string) + "'"
       end
 
       def write_executable(dir, file, filename = nil)

data/lib/bolt/transport/base.rb

@@ -165,18 +165,23 @@ module Bolt
       #
       # Transports may override this method to implement their own batch processing.
       def batch_download(targets, source, destination, options = {}, position = [], &callback)
-        require 'erb'
-
         assert_batch_size_one("batch_download()", targets)
         target = targets.first
         with_events(target, callback, 'download', position) do
-          escaped_name = ERB::Util.url_encode(target.safe_name)
-          target_destination = File.expand_path(escaped_name, destination)
+          target_destination = download_dir(destination, target)
           @logger.debug { "Downloading: '#{source}' on #{target.safe_name} to #{target_destination}" }
           download(target, source, target_destination, options)
         end
       end
 
+      # The directory under the destination that receives one target's
+      # download, named after the target so that a batch keeps them apart.
+      def download_dir(destination, target)
+        require 'erb'
+
+        File.expand_path(ERB::Util.url_encode(target.safe_name), destination)
+      end
+
       def batch_connected?(targets)
         assert_batch_size_one("connected?()", targets)
         connected?(targets.first)

data/lib/bolt/transport/choria/agent_discovery.rb

@@ -3,10 +3,9 @@
 module Bolt
   module Transport
     class Choria
-      SHELL_MIN_VERSION = '1.2.1'
-
       AGENT_MIN_VERSIONS = {
-        'shell' => SHELL_MIN_VERSION
+        'shell' => '1.2.1',
+        'file_transfer' => '1.0.0'
       }.freeze
 
       # Discover agents and detect OS on targets via two batched RPC calls
@@ -14,18 +13,20 @@ module Bolt
       # Populates @agent_cache with { agents: [...], os: 'redhat' | 'windows' | ... }.
       #
       # @param targets [Array<Bolt::Target>] Targets to discover agents on
+      # @return [Hash{Bolt::Target => Hash}] What discover_agent_list returns for the uncached
+      #   targets, or an empty hash when every target was already discovered.
       def discover_agents(targets)
-        uncached = targets.reject { |target| @agent_cache.key?(choria_identity(target)) }
-        return if uncached.empty?
+        uncached = targets.reject { |target| discovered?(target) }
+        return {} if uncached.empty?
 
         logger.debug { "Discovering agents on #{target_count(uncached)}" }
-        discover_agent_list(uncached)
+        failures = discover_agent_list(uncached)
         discover_os_family(uncached)
+        failures
+      end
 
-        uncached.each do |target|
-          identity = choria_identity(target)
-          logger.warn { "No response from #{identity} during agent discovery" } unless @agent_cache.key?(identity)
-        end
+      def discovered?(target)
+        @agent_cache.key?(choria_identity(target))
       end
 
       def has_agent?(target, agent_name)
@@ -40,26 +41,37 @@ module Bolt
       # and populate @agent_cache with agent lists.
       #
       # @param targets [Array<Bolt::Target>] Targets to query for agent inventory
+      # @return [Hash{Bolt::Target => Hash}] The error output of each target that did not answer,
+      #   answered with an RPC error or answered without an agent list, or of every target when
+      #   the call failed as a whole, such as when the client could not connect. An empty hash
+      #   when discovery itself raises, which leaves those targets undiscovered.
       def discover_agent_list(targets)
         response = rpc_request('rpcutil', targets, 'rpcutil.agent_inventory') do |client|
           client.agent_inventory
         end
-        response[:errors].each { |target, output| logger.debug { "agent_inventory failed for #{target.safe_name}: #{output[:error]}" } }
+        response[:errors].each { |target, output| logger.warn { "agent_inventory failed for #{target.safe_name}: #{output[:error]}" } }
 
+        malformed = {}
         response[:responded].each do |target, data|
           sender = choria_identity(target)
-          agents = filter_agents(sender, data[:agents])
+          # A node can send data that isn't a hash, which would raise and stop
+          # discovery for every target after it
+          agents = filter_agents(sender, data[:agents]) if data.is_a?(Hash)
           unless agents
-            logger.warn { "Unexpected agent_inventory response from #{sender}. This target will be treated as unreachable." }
+            logger.warn { "Unexpected agent_inventory response from #{sender}" }
+            malformed[target] = error_output("rpcutil.agent_inventory on #{target.safe_name} returned no agent list",
+                                             'bolt/choria-missing-data')
             next
           end
           @agent_cache[sender] = { agents: agents }
           logger.debug { "Discovered agents on #{sender}: #{agents.join(', ')}" }
         end
+        response[:errors].merge(malformed)
       rescue StandardError => e
         raise if e.is_a?(Bolt::Error)
 
         logger.warn { "Agent discovery failed: #{e.class}: #{e.message}" }
+        {}
       end
 
       # Detect the OS family on targets via rpcutil.get_fact and update
@@ -68,7 +80,7 @@ module Bolt
       # @param targets [Array<Bolt::Target>] Targets to detect OS on
       def discover_os_family(targets)
         # Only fetch OS for targets that responded to agent_inventory
-        responded = targets.select { |target| @agent_cache.key?(choria_identity(target)) }
+        responded = targets.select { |target| discovered?(target) }
         return if responded.empty?
 
         response = rpc_request('rpcutil', responded, 'rpcutil.get_fact') do |client|
@@ -82,6 +94,10 @@ module Bolt
 
         response[:responded].each do |target, data|
           sender = choria_identity(target)
+          unless data.is_a?(Hash)
+            logger.warn { "Unexpected get_fact response from #{sender}. Defaulting to POSIX command syntax." }
+            next
+          end
           os_family = data[:value].to_s.downcase
           if os_family.empty?
             logger.warn { "os.family fact is empty on #{sender}. Defaulting to POSIX command syntax." }
@@ -107,6 +123,8 @@ module Bolt
         return nil unless agent_list.is_a?(Array)
 
         agent_list.filter_map do |entry|
+          next unless entry.is_a?(Hash)
+
           name = entry['agent']
           next unless name
 

data/lib/bolt/transport/choria/bolt_tasks.rb

@@ -15,13 +15,13 @@ module Bolt
       # @return [Array<Bolt::Result>] Results for all targets
       def run_task_via_bolt_tasks(targets, task, arguments, result_opts, &callback)
         logger.debug { "Running task #{task.name} via bolt_tasks agent on #{target_count(targets)}" }
-        results = []
+        groups, results = group_by_implementation(targets, task, result_opts, &callback)
 
         # Start all implementation groups. Each gets its own download +
         # run_no_wait sequence. Tasks begin executing on nodes as soon as
         # run_no_wait returns.
         started_groups = []
-        targets.group_by { |target| select_implementation(target, task) }.each do |implementation, impl_targets|
+        groups.each do |implementation, impl_targets|
           start_result = download_and_start_task(impl_targets, task, implementation,
                                                  arguments, result_opts, &callback)
           results += start_result[:failed_results]
@@ -51,35 +51,35 @@ module Bolt
       # @return [Hash] with keys:
       #   - :failed_results [Array<Bolt::Result>] Error results from setup phase
       #   - :targets [Array<Bolt::Target>] Targets that successfully started
-      #   - :task_id [String, nil] Shared task ID for polling, nil if nothing started
+      #   - :task_id [String] Shared task ID for polling
+      #   When nothing started, only :failed_results is set.
       def download_and_start_task(targets, task, implementation, arguments, result_opts, &callback)
         environment = targets.first.options['puppet-environment']
         input_method = implementation['input_method']
         impl_files = [{ 'name' => File.basename(implementation['name']), 'path' => implementation['path'] }] +
-                     (implementation['files'] || [])
+                     implementation['files']
         file_specs_json = impl_files.map { |file| task_file_spec(file, task.module_name, environment) }.to_json
 
         # The failed_results reference will get updated and if we ever end up without
         # any targets left to act on, we can return it immediately.
         failed_results = []
-        none_started_result = { failed_results: failed_results, targets: [], task_id: nil }
+        none_started_result = { failed_results: failed_results }
 
         # Download task files
         logger.debug { "Downloading task #{task.name} files via bolt_tasks to #{target_count(targets)}" }
-        response = rpc_request('bolt_tasks', targets, 'bolt_tasks.download') do |client|
+        response = rpc_request('bolt_tasks', targets, 'bolt_tasks.download', accept_aborted: true) do |client|
           client.download(task: task.name, files: file_specs_json, environment: environment)
         end
-        # The bolt_tasks agent uses reply.fail! with statuscode 1 for download
-        # failures, which rpc_request routes to :responded since statuscode 0-1
-        # means the action completed. Check rpc_statuscodes to catch these and
-        # report the download failure clearly instead of letting run_no_wait
-        # fail with a confusing "task not available" error.
+        # A download that fails reaches the client as an RPC error with
+        # statuscode 5, since the agent's download raises after its retries,
+        # and rpc_request puts it under :errors. A statuscode 1 reply stays
+        # under :responded here, since the call accepts aborted replies, so it
+        # moves to the errors too, instead of letting run_no_wait fail with a
+        # confusing "task not available" error.
         dl_errors = response[:errors]
-        response[:rpc_statuscodes].each do |target, code|
-          next if code.zero? || dl_errors.key?(target)
-
+        response[:aborted].each do |target, statusmsg|
           dl_errors[target] = error_output(
-            "bolt_tasks.download on #{target.safe_name} failed to download task files",
+            "bolt_tasks.download on #{target.safe_name} failed to download task files: #{statusmsg}",
             'bolt/choria-download-failed'
           )
         end
@@ -90,18 +90,29 @@ module Bolt
 
         # Start task execution
         logger.debug { "Starting task #{task.name} on #{target_count(remaining)}" }
-        response = rpc_request('bolt_tasks', remaining, 'bolt_tasks.run_no_wait') do |client|
+        response = rpc_request('bolt_tasks', remaining, 'bolt_tasks.run_no_wait', accept_aborted: true) do |client|
           client.run_no_wait(task: task.name, input_method: input_method,
                              files: file_specs_json, input: arguments.to_json)
         end
-        failed_results.concat(emit_results(response[:errors], **result_opts, &callback))
-        return none_started_result if response[:responded].empty?
+        # A reply with statuscode 1 lands in :responded like a download
+        # failure does, so it is moved to the errors the same way. Otherwise
+        # the target would be polled for a task that never ran, or its reply
+        # without a task_id would stand in for the whole group.
+        start_errors = response[:errors]
+        response[:aborted].each do |target, statusmsg|
+          start_errors[target] = error_output(
+            "bolt_tasks.run_no_wait on #{target.safe_name} failed to start the task: #{statusmsg}",
+            'bolt/choria-task-start-failed'
+          )
+        end
+        failed_results.concat(emit_results(start_errors, **result_opts, &callback))
+        started = response[:responded].except(*start_errors.keys)
 
         # Extract the shared task_id (all targets get the same one from
         # the single run_no_wait call that fanned out to all of them)
-        task_id = response[:responded].values.first&.dig(:task_id)
+        task_id = started.values.first&.dig(:task_id)
         unless task_id
-          no_id_errors = response[:responded].each_with_object({}) do |(target, _), errors|
+          no_id_errors = started.each_with_object({}) do |(target, _), errors|
             errors[target] = error_output(
               "bolt_tasks.run_no_wait on #{target.safe_name} succeeded but returned no task_id",
               'bolt/choria-missing-task-id'
@@ -111,8 +122,8 @@ module Bolt
           return none_started_result
         end
 
-        logger.debug { "Started task #{task.name} on #{target_count(response[:responded])}, task_id: #{task_id}" }
-        { failed_results: failed_results, targets: response[:responded].keys, task_id: task_id }
+        logger.debug { "Started task #{task.name} on #{target_count(started)}, task_id: #{task_id}" }
+        { failed_results: failed_results, targets: started.keys, task_id: task_id }
       end
 
       # Poll bolt_tasks.task_status until all targets complete or timeout.
@@ -125,12 +136,14 @@ module Bolt
         timeout = targets.first.options['task-timeout']
 
         poll_result = poll_with_retries(targets, timeout, 'bolt_tasks.task_status') do |remaining|
-          response = rpc_request('bolt_tasks', remaining, 'bolt_tasks.task_status') do |client|
+          # The agent answers statuscode 1 for a task that ran and failed,
+          # with the exit code and output in the data.
+          response = rpc_request('bolt_tasks', remaining, 'bolt_tasks.task_status', accept_aborted: true) do |client|
             client.task_status(task_id: task_id)
           end
-          next { rpc_failed: true, done: {} } if response[:rpc_failed]
+          next { rpc_failed: true } if response[:rpc_failed]
 
-          done = response[:errors].dup
+          done = response[:errors]
 
           response[:responded].each do |target, data|
             if data.nil?
@@ -149,14 +162,8 @@ module Bolt
         end
 
         remaining_errors = poll_result[:remaining].each_with_object({}) do |target, errors|
-          errors[target] =
-            if poll_result[:rpc_persistent_failure]
-              error_output("RPC requests to poll task status on #{target.safe_name} failed persistently",
-                           'bolt/choria-poll-failed')
-            else
-              error_output("Task #{task.name} timed out after #{timeout} seconds on #{target.safe_name}",
-                           'bolt/choria-task-timeout')
-            end
+          errors[target] = error_output("Task #{task.name} timed out after #{timeout} seconds on #{target.safe_name}",
+                                        'bolt/choria-task-timeout')
         end
 
         poll_result[:completed].merge(remaining_errors)
@@ -168,13 +175,14 @@ module Bolt
       # @param target [Bolt::Target] Target for logging and stdout unwrapping context
       # @return [Hash] Output hash from output() or error_output()
       def extract_task_output(data, target)
-        exitcode = exitcode_from(data, target, 'task')
+        exitcode = exitcode_from(data[:exitcode], target, 'task')
         output(stdout: unwrap_bolt_tasks_stdout(data[:stdout]),
                stderr: data[:stderr], exitcode: exitcode)
       end
 
       # Build a file spec hash for the bolt_tasks download action. Computes
-      # the Puppet Server file_content URI based on the file's module-relative path.
+      # the server's file_content URI from the file's module-relative path,
+      # the way the server does it for the task metadata it serves.
       #
       # @param file [Hash] With 'name' (module-relative path) and 'path' (local absolute path)
       # @param module_name [String] Task's module name (used for simple task files)
@@ -192,7 +200,10 @@ module Bolt
                  when 'files'
                    "/puppet/v3/file_content/modules/#{mod}/#{rest}"
                  when 'lib'
-                   "/puppet/v3/file_content/plugins/#{mod}/#{rest}"
+                   # The plugins mount serves the lib directories of every module as one tree
+                   "/puppet/v3/file_content/plugins/#{rest}"
+                 when 'scripts'
+                   "/puppet/v3/file_content/scripts/#{mod}/#{rest}"
                  else
                    "/puppet/v3/file_content/tasks/#{mod}/#{rest}"
                  end
@@ -211,7 +222,8 @@ module Bolt
         }
       end
 
-      # Fix double-encoding in the bolt_tasks agent's wrapper error path.
+      # Turn the stdout of a bolt_tasks task_status reply into what the task
+      # printed, for Result.for_task.
       #
       # Normally, create_task_stdout returns a Hash and reply_task_status
       # calls .to_json on it, producing a single JSON string like:
@@ -228,8 +240,13 @@ module Bolt
       # case, it produces a String (the inner JSON), which we return so
       # Result.for_task can parse it.
       #
+      # For a task that exits non-zero without an _error of its own,
+      # create_task_stdout adds one of kind choria.tasks/task-error. We take
+      # it out and return what the task printed, so Result.for_task reports
+      # the failure the way it does for the other transports.
+      #
       # @param agent_stdout [String, nil] JSON-encoded stdout from the bolt_tasks agent
-      # @return [String, nil] JSON string suitable for Result.for_task
+      # @return [String, nil] The text or JSON the task printed, for Result.for_task
       def unwrap_bolt_tasks_stdout(agent_stdout)
         return agent_stdout unless agent_stdout.is_a?(String)
 
@@ -239,9 +256,16 @@ module Bolt
           return agent_stdout
         end
 
-        # Normal case: parsed is a Hash, return the original JSON string.
         # Double-encoded case: parsed is a String (the inner JSON), return it.
-        parsed.is_a?(String) ? parsed : agent_stdout
+        return parsed if parsed.is_a?(String)
+
+        # A Hash without the agent's error is the normal case and returns the original JSON string
+        agent_error = parsed.is_a?(Hash) && parsed['_error'].is_a?(Hash) && parsed['_error']['kind'] == 'choria.tasks/task-error'
+        return agent_stdout unless agent_error
+
+        # The agent puts output that isn't a JSON object under _output
+        printed = parsed.except('_error')
+        printed.keys == ['_output'] && printed['_output'].is_a?(String) ? printed['_output'] : printed.to_json
       end
     end
   end