openbolt 5.6.0 → 9.0.0

data/lib/bolt/transport/choria/bolt_tasks.rb

@@ -15,13 +15,13 @@ module Bolt
       # @return [Array<Bolt::Result>] Results for all targets
       def run_task_via_bolt_tasks(targets, task, arguments, result_opts, &callback)
         logger.debug { "Running task #{task.name} via bolt_tasks agent on #{target_count(targets)}" }
-        results = []
+        groups, results = group_by_implementation(targets, task, result_opts, &callback)
 
         # Start all implementation groups. Each gets its own download +
         # run_no_wait sequence. Tasks begin executing on nodes as soon as
         # run_no_wait returns.
         started_groups = []
-        targets.group_by { |target| select_implementation(target, task) }.each do |implementation, impl_targets|
+        groups.each do |implementation, impl_targets|
           start_result = download_and_start_task(impl_targets, task, implementation,
                                                  arguments, result_opts, &callback)
           results += start_result[:failed_results]
@@ -51,35 +51,35 @@ module Bolt
       # @return [Hash] with keys:
       #   - :failed_results [Array<Bolt::Result>] Error results from setup phase
       #   - :targets [Array<Bolt::Target>] Targets that successfully started
-      #   - :task_id [String, nil] Shared task ID for polling, nil if nothing started
+      #   - :task_id [String] Shared task ID for polling
+      #   When nothing started, only :failed_results is set.
       def download_and_start_task(targets, task, implementation, arguments, result_opts, &callback)
         environment = targets.first.options['puppet-environment']
         input_method = implementation['input_method']
         impl_files = [{ 'name' => File.basename(implementation['name']), 'path' => implementation['path'] }] +
-                     (implementation['files'] || [])
+                     implementation['files']
         file_specs_json = impl_files.map { |file| task_file_spec(file, task.module_name, environment) }.to_json
 
         # The failed_results reference will get updated and if we ever end up without
         # any targets left to act on, we can return it immediately.
         failed_results = []
-        none_started_result = { failed_results: failed_results, targets: [], task_id: nil }
+        none_started_result = { failed_results: failed_results }
 
         # Download task files
         logger.debug { "Downloading task #{task.name} files via bolt_tasks to #{target_count(targets)}" }
-        response = rpc_request('bolt_tasks', targets, 'bolt_tasks.download') do |client|
+        response = rpc_request('bolt_tasks', targets, 'bolt_tasks.download', accept_aborted: true) do |client|
           client.download(task: task.name, files: file_specs_json, environment: environment)
         end
-        # The bolt_tasks agent uses reply.fail! with statuscode 1 for download
-        # failures, which rpc_request routes to :responded since statuscode 0-1
-        # means the action completed. Check rpc_statuscodes to catch these and
-        # report the download failure clearly instead of letting run_no_wait
-        # fail with a confusing "task not available" error.
+        # A download that fails reaches the client as an RPC error with
+        # statuscode 5, since the agent's download raises after its retries,
+        # and rpc_request puts it under :errors. A statuscode 1 reply stays
+        # under :responded here, since the call accepts aborted replies, so it
+        # moves to the errors too, instead of letting run_no_wait fail with a
+        # confusing "task not available" error.
         dl_errors = response[:errors]
-        response[:rpc_statuscodes].each do |target, code|
-          next if code.zero? || dl_errors.key?(target)
-
+        response[:aborted].each do |target, statusmsg|
           dl_errors[target] = error_output(
-            "bolt_tasks.download on #{target.safe_name} failed to download task files",
+            "bolt_tasks.download on #{target.safe_name} failed to download task files: #{statusmsg}",
             'bolt/choria-download-failed'
           )
         end
@@ -90,18 +90,29 @@ module Bolt
 
         # Start task execution
         logger.debug { "Starting task #{task.name} on #{target_count(remaining)}" }
-        response = rpc_request('bolt_tasks', remaining, 'bolt_tasks.run_no_wait') do |client|
+        response = rpc_request('bolt_tasks', remaining, 'bolt_tasks.run_no_wait', accept_aborted: true) do |client|
           client.run_no_wait(task: task.name, input_method: input_method,
                              files: file_specs_json, input: arguments.to_json)
         end
-        failed_results.concat(emit_results(response[:errors], **result_opts, &callback))
-        return none_started_result if response[:responded].empty?
+        # A reply with statuscode 1 lands in :responded like a download
+        # failure does, so it is moved to the errors the same way. Otherwise
+        # the target would be polled for a task that never ran, or its reply
+        # without a task_id would stand in for the whole group.
+        start_errors = response[:errors]
+        response[:aborted].each do |target, statusmsg|
+          start_errors[target] = error_output(
+            "bolt_tasks.run_no_wait on #{target.safe_name} failed to start the task: #{statusmsg}",
+            'bolt/choria-task-start-failed'
+          )
+        end
+        failed_results.concat(emit_results(start_errors, **result_opts, &callback))
+        started = response[:responded].except(*start_errors.keys)
 
         # Extract the shared task_id (all targets get the same one from
         # the single run_no_wait call that fanned out to all of them)
-        task_id = response[:responded].values.first&.dig(:task_id)
+        task_id = started.values.first&.dig(:task_id)
         unless task_id
-          no_id_errors = response[:responded].each_with_object({}) do |(target, _), errors|
+          no_id_errors = started.each_with_object({}) do |(target, _), errors|
             errors[target] = error_output(
               "bolt_tasks.run_no_wait on #{target.safe_name} succeeded but returned no task_id",
               'bolt/choria-missing-task-id'
@@ -111,8 +122,8 @@ module Bolt
           return none_started_result
         end
 
-        logger.debug { "Started task #{task.name} on #{target_count(response[:responded])}, task_id: #{task_id}" }
-        { failed_results: failed_results, targets: response[:responded].keys, task_id: task_id }
+        logger.debug { "Started task #{task.name} on #{target_count(started)}, task_id: #{task_id}" }
+        { failed_results: failed_results, targets: started.keys, task_id: task_id }
       end
 
       # Poll bolt_tasks.task_status until all targets complete or timeout.
@@ -125,12 +136,14 @@ module Bolt
         timeout = targets.first.options['task-timeout']
 
         poll_result = poll_with_retries(targets, timeout, 'bolt_tasks.task_status') do |remaining|
-          response = rpc_request('bolt_tasks', remaining, 'bolt_tasks.task_status') do |client|
+          # The agent answers statuscode 1 for a task that ran and failed,
+          # with the exit code and output in the data.
+          response = rpc_request('bolt_tasks', remaining, 'bolt_tasks.task_status', accept_aborted: true) do |client|
             client.task_status(task_id: task_id)
           end
-          next { rpc_failed: true, done: {} } if response[:rpc_failed]
+          next { rpc_failed: true } if response[:rpc_failed]
 
-          done = response[:errors].dup
+          done = response[:errors]
 
           response[:responded].each do |target, data|
             if data.nil?
@@ -149,14 +162,8 @@ module Bolt
         end
 
         remaining_errors = poll_result[:remaining].each_with_object({}) do |target, errors|
-          errors[target] =
-            if poll_result[:rpc_persistent_failure]
-              error_output("RPC requests to poll task status on #{target.safe_name} failed persistently",
-                           'bolt/choria-poll-failed')
-            else
-              error_output("Task #{task.name} timed out after #{timeout} seconds on #{target.safe_name}",
-                           'bolt/choria-task-timeout')
-            end
+          errors[target] = error_output("Task #{task.name} timed out after #{timeout} seconds on #{target.safe_name}",
+                                        'bolt/choria-task-timeout')
         end
 
         poll_result[:completed].merge(remaining_errors)
@@ -168,13 +175,14 @@ module Bolt
       # @param target [Bolt::Target] Target for logging and stdout unwrapping context
       # @return [Hash] Output hash from output() or error_output()
       def extract_task_output(data, target)
-        exitcode = exitcode_from(data, target, 'task')
+        exitcode = exitcode_from(data[:exitcode], target, 'task')
         output(stdout: unwrap_bolt_tasks_stdout(data[:stdout]),
                stderr: data[:stderr], exitcode: exitcode)
       end
 
       # Build a file spec hash for the bolt_tasks download action. Computes
-      # the Puppet Server file_content URI based on the file's module-relative path.
+      # the server's file_content URI from the file's module-relative path,
+      # the way the server does it for the task metadata it serves.
       #
       # @param file [Hash] With 'name' (module-relative path) and 'path' (local absolute path)
       # @param module_name [String] Task's module name (used for simple task files)
@@ -192,7 +200,10 @@ module Bolt
                  when 'files'
                    "/puppet/v3/file_content/modules/#{mod}/#{rest}"
                  when 'lib'
-                   "/puppet/v3/file_content/plugins/#{mod}/#{rest}"
+                   # The plugins mount serves the lib directories of every module as one tree
+                   "/puppet/v3/file_content/plugins/#{rest}"
+                 when 'scripts'
+                   "/puppet/v3/file_content/scripts/#{mod}/#{rest}"
                  else
                    "/puppet/v3/file_content/tasks/#{mod}/#{rest}"
                  end
@@ -211,7 +222,8 @@ module Bolt
         }
       end
 
-      # Fix double-encoding in the bolt_tasks agent's wrapper error path.
+      # Turn the stdout of a bolt_tasks task_status reply into what the task
+      # printed, for Result.for_task.
       #
       # Normally, create_task_stdout returns a Hash and reply_task_status
       # calls .to_json on it, producing a single JSON string like:
@@ -228,8 +240,13 @@ module Bolt
       # case, it produces a String (the inner JSON), which we return so
       # Result.for_task can parse it.
       #
+      # For a task that exits non-zero without an _error of its own,
+      # create_task_stdout adds one of kind choria.tasks/task-error. We take
+      # it out and return what the task printed, so Result.for_task reports
+      # the failure the way it does for the other transports.
+      #
       # @param agent_stdout [String, nil] JSON-encoded stdout from the bolt_tasks agent
-      # @return [String, nil] JSON string suitable for Result.for_task
+      # @return [String, nil] The text or JSON the task printed, for Result.for_task
       def unwrap_bolt_tasks_stdout(agent_stdout)
         return agent_stdout unless agent_stdout.is_a?(String)
 
@@ -239,9 +256,16 @@ module Bolt
           return agent_stdout
         end
 
-        # Normal case: parsed is a Hash, return the original JSON string.
         # Double-encoded case: parsed is a String (the inner JSON), return it.
-        parsed.is_a?(String) ? parsed : agent_stdout
+        return parsed if parsed.is_a?(String)
+
+        # A Hash without the agent's error is the normal case and returns the original JSON string
+        agent_error = parsed.is_a?(Hash) && parsed['_error'].is_a?(Hash) && parsed['_error']['kind'] == 'choria.tasks/task-error'
+        return agent_stdout unless agent_error
+
+        # The agent puts output that isn't a JSON object under _output
+        printed = parsed.except('_error')
+        printed.keys == ['_output'] && printed['_output'].is_a?(String) ? printed['_output'] : printed.to_json
       end
     end
   end

data/lib/bolt/transport/choria/client.rb

@@ -79,7 +79,7 @@ module Bolt
             logger.debug { "Choria brokers overridden: #{brokers.join(', ')}" }
           end
 
-          if opts['ssl-ca'] && opts['ssl-cert'] && opts['ssl-key']
+          if opts['ssl-ca']
             unreadable = %w[ssl-ca ssl-cert ssl-key].find { |key| !File.readable?(opts[key]) }
             if unreadable
               raise Bolt::Error.new(
@@ -101,9 +101,9 @@ module Bolt
       end
 
       # Create an MCollective::RPC::Client for one or more targets.
-      # Accepts a single target or an array. Uses MCollective's direct
-      # addressing mode (client.discover(nodes:)) to skip broadcast
-      # discovery and send requests directly to the specified nodes.
+      # Uses MCollective's direct addressing mode (client.discover(nodes:))
+      # to skip broadcast discovery and send requests directly to the
+      # specified nodes.
       #
       # Note that when the client is created, if the shell agent isn't already
       # installed on the OpenBolt controller node, then the shell DDL that we
@@ -113,24 +113,16 @@ module Bolt
       # DDL is already included in the choria-mcorpc-support gem.
       #
       # @param agent_name [String] MCollective agent name (e.g. 'shell', 'bolt_tasks')
-      # @param targets [Bolt::Target, Array<Bolt::Target>] One or more targets to address
+      # @param targets [Array<Bolt::Target>] Targets to address
       # @param timeout [Numeric] RPC call timeout in seconds
       # @return [MCollective::RPC::Client] Configured client with direct addressing enabled
       def create_rpc_client(agent_name, targets, timeout)
-        targets = Array(targets)
-        options = MCollective::Util.default_options
-        options[:timeout] = timeout
-        options[:verbose] = false
-        options[:connection_timeout] = targets.first.options['broker-timeout']
+        client = MCollective::RPC::Client.new(agent_name, options: client_options(targets))
+        # Set after the build, which replaces a timeout of exactly 5, the
+        # gem's default, with the DDL timeout plus the discovery timeout.
+        client.timeout = timeout
 
-        collective = collective_for(targets.first)
-        options[:collective] = collective if collective
-
-        client = MCollective::RPC::Client.new(agent_name, options: options)
-        client.progress = false
-
-        identities = targets.map { |target| choria_identity(target) }.uniq
-        client.discover(nodes: identities)
+        client.discover(nodes: identities_of(targets))
 
         client
       end
@@ -140,13 +132,14 @@ module Bolt
       #
       # Results are split based on MCollective RPC statuscodes:
       # - statuscode 0: action completed successfully (:responded)
-      # - statuscode 1 (RPCAborted): action completed but reported a
-      #   problem (:responded). The data is preserved rather than
-      #   discarded because some agents (notably bolt_tasks) use
-      #   statuscode 1 for application-level failures where the
-      #   response data is still valid and meaningful (e.g., a task
-      #   that ran but exited non-zero). Callers must handle this
-      #   case and not assume :responded means success.
+      # - statuscode 1 (RPCAborted): the action could not run or reported a
+      #   problem. An error (:errors) by default, since the shell and rpcutil
+      #   agents never answer 1 with data the transport can use, and a denied
+      #   action policy answers 1 as well. With accept_aborted set the reply
+      #   stays under :responded with its data and its status message goes
+      #   under :aborted, which bolt_tasks needs, since it answers 1 for a
+      #   task that ran and failed with the data still valid. Such a caller
+      #   must not assume :responded means success.
       # - statuscode 2-5: RPC infrastructure error (:errors)
       # - no response: target didn't reply (:errors)
       # - exception: total RPC failure (rpc_failed: true)
@@ -159,27 +152,27 @@ module Bolt
       # @param agent [String] MCollective agent name (e.g. 'shell', 'bolt_tasks', 'rpcutil')
       # @param targets [Bolt::Target, Array<Bolt::Target>] One or more targets to address
       # @param context [String] Human-readable label for logging (e.g. 'shell.start')
+      # @param accept_aborted [Boolean] Whether a statuscode 1 reply counts as a response, default is false
       # @yield [MCollective::RPC::Client] The configured RPC client to invoke an action on
       # @return [Hash] with keys:
-      #   - :responded [Hash] Targets where the action completed (statuscode 0-1),
-      #     mapped to their response data
+      #   - :responded [Hash] Targets where the action completed (statuscode 0, or 1 with
+      #     accept_aborted), mapped to their response data
       #   - :errors [Hash] Targets with RPC errors or no response, mapped to error output hashes
       #   - :rpc_failed [Boolean] True when the entire RPC call failed
-      #   - :rpc_statuscodes [Hash] Per-target MCollective RPC statuscodes.
-      #     Includes all targets that responded (both :responded and :errors).
-      #     Not populated when rpc_failed is true (no individual responses).
-      def rpc_request(agent, targets, context)
+      #   - :aborted [Hash] Targets in :responded whose reply had statuscode 1,
+      #     mapped to the agent's status message, so a caller can report the
+      #     agent's reason
+      def rpc_request(agent, targets, context, accept_aborted: false)
         targets = Array(targets)
         rpc_results = @rpc_mutex.synchronize do
-          rpc_timeout = targets.first.options['rpc-timeout']
-          client = create_rpc_client(agent, targets, rpc_timeout)
+          client = create_rpc_client(agent, targets, targets.first.options['rpc-timeout'])
           yield(client)
         end
         by_sender = index_results_by_sender(rpc_results, targets, context)
 
         responded = {}
         errors = {}
-        rpc_statuscodes = {}
+        aborted = {}
         targets.each do |target|
           rpc_result = by_sender[choria_identity(target)]
           if rpc_result.nil?
@@ -187,22 +180,21 @@ module Bolt
               "No response from #{target.safe_name} for #{context}",
               'bolt/choria-no-response'
             )
-          elsif rpc_result[:statuscode] > 1
-            rpc_statuscodes[target] = rpc_result[:statuscode]
+          elsif rpc_result[:statuscode] > 1 || (rpc_result[:statuscode] == 1 && !accept_aborted)
             errors[target] = error_output(
               "#{context} on #{target.safe_name} returned RPC error: " \
               "#{rpc_result[:statusmsg]} (code #{rpc_result[:statuscode]})",
               'bolt/choria-rpc-error'
             )
           else
-            rpc_statuscodes[target] = rpc_result[:statuscode]
             if rpc_result[:statuscode] == 1
               logger.warn { "#{context} on #{target.safe_name} had RPC status code #{rpc_result[:statuscode]}: #{rpc_result[:statusmsg]}" }
+              aborted[target] = rpc_result[:statusmsg]
             end
             responded[target] = rpc_result[:data]
           end
         end
-        { responded: responded, errors: errors, rpc_failed: false, rpc_statuscodes: rpc_statuscodes }
+        { responded: responded, errors: errors, rpc_failed: false, aborted: aborted }
       rescue StandardError => e
         raise if e.is_a?(Bolt::Error)
 
@@ -211,11 +203,26 @@ module Bolt
           errs[target] = error_output("#{context} failed on #{target.safe_name}: #{e.message}",
                                       'bolt/choria-rpc-failed')
         end
-        { responded: {}, errors: errors, rpc_failed: true, rpc_statuscodes: {} }
+        { responded: {}, errors: errors, rpc_failed: true, aborted: {} }
+      end
+
+      # The client options the targets' configuration gives: the broker
+      # timeout and the collective over the gem's defaults. The file
+      # transfer library builds its own clients from the same hash, under
+      # the same RPC mutex.
+      #
+      # @param targets [Array<Bolt::Target>] The targets of one batch
+      # @return [Hash]
+      def client_options(targets)
+        options = MCollective::Util.default_options
+        options[:connection_timeout] = targets.first.options['broker-timeout']
+        options[:collective] = collective_for(targets.first)
+        options
       end
 
-      # Configure the client, discover agents, partition targets by agent
-      # availability, and emit errors for incapable targets.
+      # Fire node_start for every target, configure the client, discover
+      # agents, partition targets by agent availability, and emit errors for
+      # incapable targets.
       #
       # @param targets [Array<Bolt::Target>] Targets to prepare
       # @param agent_name [String] Required agent name (e.g. 'shell', 'bolt_tasks')
@@ -225,24 +232,34 @@ module Bolt
       #   - [Array<Bolt::Target>] Targets that have the required agent
       #   - [Array<Bolt::Result>] Error results for targets that lack the agent
       def prepare_targets(targets, agent_name, result_opts, &callback)
+        targets.each { |target| callback&.call(type: :node_start, target: target) }
         configure_client(targets.first)
-        discover_agents(targets)
+        discovery_failures = discover_agents(targets)
 
         capable, incapable = targets.partition { |target| has_agent?(target, agent_name) }
 
         agent_errors = incapable.each_with_object({}) do |target, errors|
-          msg = if @agent_cache[choria_identity(target)].nil?
-                  "No agent information available for #{target.safe_name} (node did not respond to discovery)"
-                else
-                  "The '#{agent_name}' agent is not available on #{target.safe_name}."
-                end
-          errors[target] = error_output(msg, 'bolt/choria-agent-not-available')
+          errors[target] = if discovered?(target)
+                             error_output("The '#{agent_name}' agent is not available on #{target.safe_name}.", 'bolt/choria-agent-not-available')
+                           else
+                             discovery_failures[target] || undiscovered_error(target)
+                           end
         end
-        incapable_results = emit_results(agent_errors, fire_node_start: true, **result_opts, &callback)
+        incapable_results = emit_results(agent_errors, **result_opts, &callback)
 
         [capable, incapable_results]
       end
 
+      # The error output for a target that discovery has no agent list and no
+      # failure for, which happens when discovery raised before it finished.
+      #
+      # @param target [Bolt::Target]
+      # @return [Hash] An error output hash
+      def undiscovered_error(target)
+        error_output("No agent information available for #{target.safe_name} (agent discovery failed)",
+                     'bolt/choria-agent-not-available')
+      end
+
       # Index RPC results by sender, keeping only the first response per
       # sender and only from the set of expected identities. Logs and discards
       # responses from unexpected senders and duplicates.