openbolt 9.0.0.pre.rc1 → 9.0.0

data/lib/bolt/transport/choria/client.rb

@@ -79,7 +79,7 @@ module Bolt
             logger.debug { "Choria brokers overridden: #{brokers.join(', ')}" }
           end
 
-          if opts['ssl-ca'] && opts['ssl-cert'] && opts['ssl-key']
+          if opts['ssl-ca']
             unreadable = %w[ssl-ca ssl-cert ssl-key].find { |key| !File.readable?(opts[key]) }
             if unreadable
               raise Bolt::Error.new(
@@ -101,9 +101,9 @@ module Bolt
       end
 
       # Create an MCollective::RPC::Client for one or more targets.
-      # Accepts a single target or an array. Uses MCollective's direct
-      # addressing mode (client.discover(nodes:)) to skip broadcast
-      # discovery and send requests directly to the specified nodes.
+      # Uses MCollective's direct addressing mode (client.discover(nodes:))
+      # to skip broadcast discovery and send requests directly to the
+      # specified nodes.
       #
       # Note that when the client is created, if the shell agent isn't already
       # installed on the OpenBolt controller node, then the shell DDL that we
@@ -113,24 +113,16 @@ module Bolt
       # DDL is already included in the choria-mcorpc-support gem.
       #
       # @param agent_name [String] MCollective agent name (e.g. 'shell', 'bolt_tasks')
-      # @param targets [Bolt::Target, Array<Bolt::Target>] One or more targets to address
+      # @param targets [Array<Bolt::Target>] Targets to address
       # @param timeout [Numeric] RPC call timeout in seconds
       # @return [MCollective::RPC::Client] Configured client with direct addressing enabled
       def create_rpc_client(agent_name, targets, timeout)
-        targets = Array(targets)
-        options = MCollective::Util.default_options
-        options[:timeout] = timeout
-        options[:verbose] = false
-        options[:connection_timeout] = targets.first.options['broker-timeout']
+        client = MCollective::RPC::Client.new(agent_name, options: client_options(targets))
+        # Set after the build, which replaces a timeout of exactly 5, the
+        # gem's default, with the DDL timeout plus the discovery timeout.
+        client.timeout = timeout
 
-        collective = collective_for(targets.first)
-        options[:collective] = collective if collective
-
-        client = MCollective::RPC::Client.new(agent_name, options: options)
-        client.progress = false
-
-        identities = targets.map { |target| choria_identity(target) }.uniq
-        client.discover(nodes: identities)
+        client.discover(nodes: identities_of(targets))
 
         client
       end
@@ -140,13 +132,14 @@ module Bolt
       #
       # Results are split based on MCollective RPC statuscodes:
       # - statuscode 0: action completed successfully (:responded)
-      # - statuscode 1 (RPCAborted): action completed but reported a
-      #   problem (:responded). The data is preserved rather than
-      #   discarded because some agents (notably bolt_tasks) use
-      #   statuscode 1 for application-level failures where the
-      #   response data is still valid and meaningful (e.g., a task
-      #   that ran but exited non-zero). Callers must handle this
-      #   case and not assume :responded means success.
+      # - statuscode 1 (RPCAborted): the action could not run or reported a
+      #   problem. An error (:errors) by default, since the shell and rpcutil
+      #   agents never answer 1 with data the transport can use, and a denied
+      #   action policy answers 1 as well. With accept_aborted set the reply
+      #   stays under :responded with its data and its status message goes
+      #   under :aborted, which bolt_tasks needs, since it answers 1 for a
+      #   task that ran and failed with the data still valid. Such a caller
+      #   must not assume :responded means success.
       # - statuscode 2-5: RPC infrastructure error (:errors)
       # - no response: target didn't reply (:errors)
       # - exception: total RPC failure (rpc_failed: true)
@@ -159,27 +152,27 @@ module Bolt
       # @param agent [String] MCollective agent name (e.g. 'shell', 'bolt_tasks', 'rpcutil')
       # @param targets [Bolt::Target, Array<Bolt::Target>] One or more targets to address
       # @param context [String] Human-readable label for logging (e.g. 'shell.start')
+      # @param accept_aborted [Boolean] Whether a statuscode 1 reply counts as a response, default is false
       # @yield [MCollective::RPC::Client] The configured RPC client to invoke an action on
       # @return [Hash] with keys:
-      #   - :responded [Hash] Targets where the action completed (statuscode 0-1),
-      #     mapped to their response data
+      #   - :responded [Hash] Targets where the action completed (statuscode 0, or 1 with
+      #     accept_aborted), mapped to their response data
       #   - :errors [Hash] Targets with RPC errors or no response, mapped to error output hashes
       #   - :rpc_failed [Boolean] True when the entire RPC call failed
-      #   - :rpc_statuscodes [Hash] Per-target MCollective RPC statuscodes.
-      #     Includes all targets that responded (both :responded and :errors).
-      #     Not populated when rpc_failed is true (no individual responses).
-      def rpc_request(agent, targets, context)
+      #   - :aborted [Hash] Targets in :responded whose reply had statuscode 1,
+      #     mapped to the agent's status message, so a caller can report the
+      #     agent's reason
+      def rpc_request(agent, targets, context, accept_aborted: false)
         targets = Array(targets)
         rpc_results = @rpc_mutex.synchronize do
-          rpc_timeout = targets.first.options['rpc-timeout']
-          client = create_rpc_client(agent, targets, rpc_timeout)
+          client = create_rpc_client(agent, targets, targets.first.options['rpc-timeout'])
           yield(client)
         end
         by_sender = index_results_by_sender(rpc_results, targets, context)
 
         responded = {}
         errors = {}
-        rpc_statuscodes = {}
+        aborted = {}
         targets.each do |target|
           rpc_result = by_sender[choria_identity(target)]
           if rpc_result.nil?
@@ -187,22 +180,21 @@ module Bolt
               "No response from #{target.safe_name} for #{context}",
               'bolt/choria-no-response'
             )
-          elsif rpc_result[:statuscode] > 1
-            rpc_statuscodes[target] = rpc_result[:statuscode]
+          elsif rpc_result[:statuscode] > 1 || (rpc_result[:statuscode] == 1 && !accept_aborted)
             errors[target] = error_output(
               "#{context} on #{target.safe_name} returned RPC error: " \
               "#{rpc_result[:statusmsg]} (code #{rpc_result[:statuscode]})",
               'bolt/choria-rpc-error'
             )
           else
-            rpc_statuscodes[target] = rpc_result[:statuscode]
             if rpc_result[:statuscode] == 1
               logger.warn { "#{context} on #{target.safe_name} had RPC status code #{rpc_result[:statuscode]}: #{rpc_result[:statusmsg]}" }
+              aborted[target] = rpc_result[:statusmsg]
             end
             responded[target] = rpc_result[:data]
           end
         end
-        { responded: responded, errors: errors, rpc_failed: false, rpc_statuscodes: rpc_statuscodes }
+        { responded: responded, errors: errors, rpc_failed: false, aborted: aborted }
       rescue StandardError => e
         raise if e.is_a?(Bolt::Error)
 
@@ -211,11 +203,26 @@ module Bolt
           errs[target] = error_output("#{context} failed on #{target.safe_name}: #{e.message}",
                                       'bolt/choria-rpc-failed')
         end
-        { responded: {}, errors: errors, rpc_failed: true, rpc_statuscodes: {} }
+        { responded: {}, errors: errors, rpc_failed: true, aborted: {} }
+      end
+
+      # The client options the targets' configuration gives: the broker
+      # timeout and the collective over the gem's defaults. The file
+      # transfer library builds its own clients from the same hash, under
+      # the same RPC mutex.
+      #
+      # @param targets [Array<Bolt::Target>] The targets of one batch
+      # @return [Hash]
+      def client_options(targets)
+        options = MCollective::Util.default_options
+        options[:connection_timeout] = targets.first.options['broker-timeout']
+        options[:collective] = collective_for(targets.first)
+        options
       end
 
-      # Configure the client, discover agents, partition targets by agent
-      # availability, and emit errors for incapable targets.
+      # Fire node_start for every target, configure the client, discover
+      # agents, partition targets by agent availability, and emit errors for
+      # incapable targets.
       #
       # @param targets [Array<Bolt::Target>] Targets to prepare
       # @param agent_name [String] Required agent name (e.g. 'shell', 'bolt_tasks')
@@ -225,24 +232,34 @@ module Bolt
       #   - [Array<Bolt::Target>] Targets that have the required agent
       #   - [Array<Bolt::Result>] Error results for targets that lack the agent
       def prepare_targets(targets, agent_name, result_opts, &callback)
+        targets.each { |target| callback&.call(type: :node_start, target: target) }
         configure_client(targets.first)
-        discover_agents(targets)
+        discovery_failures = discover_agents(targets)
 
         capable, incapable = targets.partition { |target| has_agent?(target, agent_name) }
 
         agent_errors = incapable.each_with_object({}) do |target, errors|
-          msg = if @agent_cache[choria_identity(target)].nil?
-                  "No agent information available for #{target.safe_name} (node did not respond to discovery)"
-                else
-                  "The '#{agent_name}' agent is not available on #{target.safe_name}."
-                end
-          errors[target] = error_output(msg, 'bolt/choria-agent-not-available')
+          errors[target] = if discovered?(target)
+                             error_output("The '#{agent_name}' agent is not available on #{target.safe_name}.", 'bolt/choria-agent-not-available')
+                           else
+                             discovery_failures[target] || undiscovered_error(target)
+                           end
         end
-        incapable_results = emit_results(agent_errors, fire_node_start: true, **result_opts, &callback)
+        incapable_results = emit_results(agent_errors, **result_opts, &callback)
 
         [capable, incapable_results]
       end
 
+      # The error output for a target that discovery has no agent list and no
+      # failure for, which happens when discovery raised before it finished.
+      #
+      # @param target [Bolt::Target]
+      # @return [Hash] An error output hash
+      def undiscovered_error(target)
+        error_output("No agent information available for #{target.safe_name} (agent discovery failed)",
+                     'bolt/choria-agent-not-available')
+      end
+
       # Index RPC results by sender, keeping only the first response per
       # sender and only from the set of expected identities. Logs and discards
       # responses from unexpected senders and duplicates.

data/lib/bolt/transport/choria/command_builders.rb

@@ -23,6 +23,29 @@ module Bolt
         end
       end
 
+      # Build the command that creates a tmpdir. On POSIX it is mkdir -m 700
+      # without -p, so it fails when the directory the tmpdir goes into
+      # doesn't exist, like it does with the SSH transport. The Windows
+      # command leaves the tmpdir open only to the account the agent runs as
+      # and to the Administrators group, which the directories and files in
+      # it inherit, and it fails when it cannot set that.
+      #
+      # @param target [Bolt::Target] Used for platform detection
+      # @param tmpdir [String] Absolute path to the tmpdir on the remote node
+      # @return [String] Shell command
+      def make_tmpdir_command(target, tmpdir)
+        return "mkdir -m 700 #{Shellwords.shellescape(tmpdir)}" unless windows_target?(target)
+
+        "$ErrorActionPreference = 'Stop'; $dir = '#{ps_escape(tmpdir)}'; " \
+          '$null = New-Item -ItemType Directory -Force -Path $dir; ' \
+          '$acl = Get-Acl -LiteralPath $dir; $acl.SetAccessRuleProtection($true, $false); ' \
+          'foreach ($sid in @([Security.Principal.WindowsIdentity]::GetCurrent().User, ' \
+          "(New-Object Security.Principal.SecurityIdentifier('S-1-5-32-544')))) { " \
+          '$acl.AddAccessRule((New-Object Security.AccessControl.FileSystemAccessRule($sid, ' \
+          "'FullControl', 'ContainerInherit, ObjectInherit', 'None', 'Allow'))) }; " \
+          'Set-Acl -LiteralPath $dir -AclObject $acl'
+      end
+
       # Build a chmod +x command. Returns nil on Windows (not needed).
       #
       # @param target [Bolt::Target] Used for platform detection
@@ -43,19 +66,88 @@ module Bolt
           "rm -rf #{Shellwords.shellescape(path)}"
       end
 
+      # Build a command that removes the shell agent's job directories for
+      # the given handles, under the agent's default state path. A handle
+      # without a directory on the target is skipped, so 1 command can
+      # contain the handles of every target it goes to. On Windows a failed
+      # removal stops the command, so it exits 1 like a failed rm does.
+      #
+      # @param target [Bolt::Target] Used for platform detection
+      # @param handles [Array<String>] Job handles, each checked to be a UUID
+      # @return [String] Shell command
+      def remove_job_dirs_command(target, handles)
+        if windows_target?(target)
+          paths = handles.map { |handle| "'C:/ProgramData/mcollective-shell/#{handle}'" }.join(', ')
+          "foreach ($path in @(#{paths})) { if (Test-Path -LiteralPath $path) { " \
+            'Remove-Item -LiteralPath $path -Recurse -Force -ErrorAction Stop } }'
+        else
+          "rm -rf #{handles.map { |handle| "/var/run/mcollective-shell/#{handle}" }.join(' ')}"
+        end
+      end
+
       # Build a command that writes base64-encoded content to a file
       # after decoding the content. Requires base64 CLI on POSIX targets.
       #
       # @param target [Bolt::Target] Used for platform detection
       # @param content_b64 [String] Base64-encoded file content
       # @param dest [String] Absolute destination path on the remote node
+      # @param append [Boolean] Whether to add the content to the end of the file instead of replacing it
       # @return [String] Shell command
-      def upload_file_command(target, content_b64, dest)
-        if windows_target?(target)
+      def upload_file_command(target, content_b64, dest, append: false)
+        if windows_target?(target) && append
+          "$bytes = [Convert]::FromBase64String('#{content_b64}'); " \
+            "$stream = [IO.File]::Open('#{ps_escape(dest)}', 'Append', 'Write'); " \
+            'try { $stream.Write($bytes, 0, $bytes.Length) } finally { $stream.Close() }'
+        elsif windows_target?(target)
           "[IO.File]::WriteAllBytes('#{ps_escape(dest)}', " \
             "[Convert]::FromBase64String('#{content_b64}'))"
         else
-          "printf '%s' #{Shellwords.shellescape(content_b64)} | base64 -d > #{Shellwords.shellescape(dest)}"
+          "printf '%s' #{Shellwords.shellescape(content_b64)} | base64 -d #{append ? '>>' : '>'} #{Shellwords.shellescape(dest)}"
+        end
+      end
+
+      # The path of the run file in a tmpdir. A script or task command can
+      # contain arguments, env var values and task parameters, so it goes to
+      # the nodes in this file instead of as the shell agent's start command,
+      # which the agent keeps on disk and returns in shell.list replies.
+      #
+      # @param target [Bolt::Target] Used for platform detection
+      # @param tmpdir [String] Absolute path to the tmpdir on the remote node
+      # @return [String] Absolute path to the run file
+      def run_file_path(target, tmpdir)
+        join_path(target, tmpdir, windows_target?(target) ? 'bolt_run.ps1' : 'bolt_run.sh')
+      end
+
+      # Build the content of a run file. On POSIX it removes itself as soon
+      # as sh starts reading it, which keeps reading from the open file. A
+      # single command then replaces sh through exec, so shell.kill reaches
+      # it. A pipeline doesn't, since exec would run printf as its own
+      # process with the task input in its arguments. On Windows the content
+      # runs in a script block, which would hide a failing last command from
+      # powershell.exe, so it exits 1 then, as the command itself would.
+      #
+      # @param target [Bolt::Target] Used for platform detection
+      # @param command [String] The command the run file contains
+      # @param pipeline [Boolean] Whether the command is a pipeline, default is false
+      # @return [String] File content
+      def run_file_content(target, command, pipeline: false)
+        return "#{command}\nif (-not $?) { exit 1 }" if windows_target?(target)
+
+        "rm -f -- \"$0\"\n#{'exec ' unless pipeline}#{command}\n"
+      end
+
+      # Build the start command that runs a run file. On Windows it reads the
+      # file and removes it before running what was in it.
+      #
+      # @param target [Bolt::Target] Used for platform detection
+      # @param path [String] Absolute path to the run file
+      # @return [String] Shell command
+      def run_file_command(target, path)
+        if windows_target?(target)
+          "$boltRunFile = '#{ps_escape(path)}'; $boltRunScript = [IO.File]::ReadAllText($boltRunFile); " \
+            'Remove-Item -LiteralPath $boltRunFile -Force; & ([ScriptBlock]::Create($boltRunScript))'
+        else
+          "sh #{Shellwords.shellescape(path)}"
         end
       end
 
@@ -64,46 +156,58 @@ module Bolt
       #
       # @param target [Bolt::Target] Used for platform detection
       # @param command [String] The command to prepend env vars to
-      # @param env_vars [Hash{String => String}, nil] Variable names to values
+      # @param env_vars [Hash{String => Object}, nil] Variable names to values, which become strings
       # @param context [String] Description for error messages (e.g., 'task argument')
       # @return [String] Command with env vars prepended
       def prepend_env_vars(target, command, env_vars, context)
         return command unless env_vars&.any?
 
-        env_vars.each_key { |key| validate_env_key!(key, context) }
-
         if windows_target?(target)
-          set_stmts = env_vars.map { |key, val| "$env:#{key} = '#{ps_escape(val)}'" }
-          "#{set_stmts.join('; ')}; & #{command}"
+          "#{powershell_env_statements(env_vars, context)}; #{command}"
         else
+          env_vars.each_key { |key| validate_env_key!(key, context) }
           env_str = env_vars.map { |key, val| "#{key}=#{Shellwords.shellescape(val)}" }.join(' ')
           "/usr/bin/env #{env_str} #{command}"
         end
       end
 
+      # Build the PowerShell statements that set environment variables,
+      # after validating each name.
+      #
+      # @param env_vars [Hash{String => Object}] Variable names to values, which become strings
+      # @param context [String] Description for error messages (e.g., 'task argument')
+      # @return [String] The statements joined with '; '
+      def powershell_env_statements(env_vars, context)
+        env_vars.each_key { |key| validate_env_key!(key, context) }
+        env_vars.map { |key, val| "$env:#{key} = '#{ps_escape(val.to_s)}'" }.join('; ')
+      end
+
       # Build a command that pipes data to another command via stdin.
       #
       # @param target [Bolt::Target] Used for platform detection
       # @param data [String] Data to pipe (typically JSON task arguments)
-      # @param command [String] The command to receive stdin
+      # @param command [String] The command to receive stdin, which on Windows must
+      #   be a PowerShell command such as one from executable_command
       # @return [String] Shell command with stdin piping
       def stdin_pipe_command(target, data, command)
         if windows_target?(target)
           # Use a here-string (@'...'@) to avoid escaping issues with
-          # large JSON payloads. Content between @' and '@ is literal.
-          "@'\n#{data}\n'@ | & #{command}"
+          # large JSON payloads. Its content is literal as long as no line
+          # starts with a single quote character and @, which single-line
+          # JSON never does.
+          "@'\n#{data}\n'@ | #{command}"
         else
           "printf '%s' #{Shellwords.shellescape(data)} | #{command}"
         end
       end
 
-      # Escape a string for use as a shell argument on the target platform.
+      # Escape a value for use as a shell argument on the target platform.
       #
       # @param target [Bolt::Target] Used for platform detection
-      # @param str [String] The string to escape
+      # @param str [Object] The value to escape, turned into a string first
       # @return [String] Escaped string (single-quoted on Windows, sh-escaped on POSIX)
       def escape_arg(target, str)
-        windows_target?(target) ? "'#{ps_escape(str)}'" : Shellwords.shellescape(str)
+        windows_target?(target) ? "'#{ps_escape(str.to_s)}'" : Shellwords.shellescape(str)
       end
 
       # Join path segments using the target platform's separator.
@@ -132,9 +236,22 @@ module Bolt
       # Escape single quotes for use inside PowerShell single-quoted strings.
       #
       # @param str [String] String to escape
-      # @return [String] String with single quotes doubled
+      # @return [String] String with every single quote character doubled
       def ps_escape(str)
-        str.gsub("'", "''")
+        Bolt::Util.escape_powershell_single_quotes(str)
+      end
+
+      # Build the command that runs an executable, through the interpreter
+      # when one is given. On Windows it starts with the call operator,
+      # since PowerShell treats a quoted path as a string.
+      #
+      # @param target [Bolt::Target] Used for platform detection
+      # @param path [String] Absolute path to the executable on the remote node
+      # @param interpreter [String, Array<String>, nil] Interpreter command, or nil to run the file directly
+      # @return [String] Shell command
+      def executable_command(target, path, interpreter)
+        command = (Array(interpreter) + [path]).map { |part| escape_arg(target, part) }.join(' ')
+        windows_target?(target) ? "& #{command}" : command
       end
 
       # Build the full command string for task execution via the shell agent,
@@ -144,14 +261,19 @@ module Bolt
       # @param target [Bolt::Target] Target (used for platform detection)
       # @param remote_task_path [String] Absolute path to the task executable on the remote node
       # @param arguments [Hash] Task parameter names to values
-      # @param input_method [String] How to pass arguments: 'stdin', 'environment', or 'both'
+      # @param input_method [String] How to pass arguments: 'stdin', 'environment', 'both', or 'powershell'
       # @param interpreter_options [Hash{String => String}] File extension to interpreter path mapping
       # @return [String] The fully constructed shell command
       def build_task_command(target, remote_task_path, arguments, input_method, interpreter_options)
+        if input_method == 'powershell' && windows_target?(target) && File.extname(remote_task_path).casecmp?('.ps1')
+          # The arguments become the script's named parameters, like they
+          # do with the WinRM transport
+          return Bolt::Shell::Powershell::Snippets.json_functions +
+                 Bolt::Shell::Powershell::Snippets.ps_task(remote_task_path, arguments)
+        end
+
         interpreter = select_interpreter(remote_task_path, interpreter_options)
-        cmd = interpreter ?
-          "#{Array(interpreter).map { |part| escape_arg(target, part) }.join(' ')} #{escape_arg(target, remote_task_path)}" :
-          escape_arg(target, remote_task_path)
+        cmd = executable_command(target, remote_task_path, interpreter)
 
         needs_env = Bolt::Task::ENVIRONMENT_METHODS.include?(input_method)
         needs_stdin = Bolt::Task::STDIN_METHODS.include?(input_method)
@@ -161,11 +283,9 @@ module Bolt
           # requires a script block. Pipeline data doesn't automatically
           # flow through a script block to inner commands, so we
           # explicitly forward $input via a pipe.
-          env_params = envify_params(arguments)
-          env_params.each_key { |key| validate_env_key!(key, 'task argument') }
-          set_stmts = env_params.map { |key, val| "$env:#{key} = '#{ps_escape(val)}'" }
+          env_statements = powershell_env_statements(envify_params(arguments), 'task argument')
           cmd = stdin_pipe_command(target, arguments.to_json,
-                                   "{ #{set_stmts.join('; ')}; $input | & #{cmd} }")
+                                   "& { #{env_statements}; $input | #{cmd} }")
         else
           if needs_env
             cmd = prepend_env_vars(target, cmd, envify_params(arguments), 'task argument')

data/lib/bolt/transport/choria/file_transfer.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+module Bolt
+  module Transport
+    class Choria
+      # Upload and download through the file_transfer agent. The transfers
+      # themselves are done by the client library the agent's Puppet module
+      # installs on the controller, MCollective::Util::FileTransfer. This file
+      # maps targets, options and results onto it.
+
+      # The library's outcome kinds as this transport's error kinds. A kind
+      # a later library adds is reported as a transfer failure.
+      FILE_TRANSFER_ERROR_KINDS = {
+        no_response: 'bolt/choria-no-response',
+        rpc_error: 'bolt/choria-rpc-error',
+        rpc_failed: 'bolt/choria-rpc-failed',
+        transfer_failed: 'bolt/choria-file-transfer-failed',
+        payload_too_large: 'bolt/choria-payload-too-large',
+        checksum_mismatch: 'bolt/choria-checksum-mismatch'
+      }.freeze
+
+      # Upload one file to the capable targets through the library.
+      def batch_upload(targets, source, destination, _options = {}, position = [], &callback)
+        result_opts = { action: 'upload', name: source, position: position }
+        capable, results = prepare_targets(targets, 'file_transfer', result_opts, &callback)
+        return results if capable.empty?
+
+        logger.debug { "Uploading #{source} to #{destination} on #{target_count(capable)}" }
+        results + transfer(capable, result_opts, callback) do |client|
+          outcomes = client.upload(source, destination, identities_of(capable))
+          file_transfer_results(capable, outcomes, result_opts, callback) do |target|
+            Bolt::Result.for_upload(target, source, destination)
+          end
+        end
+      end
+
+      # Download a path from each capable target into its own directory.
+      def batch_download(targets, source, destination, _options = {}, position = [], &callback)
+        directories = targets.to_h { |target| [target, download_dir(destination, target)] }
+        download_into(directories, source, position, &callback)
+      end
+
+      # Download a path from each capable target into the directory given
+      # for it.
+      #
+      # @param directories [Hash{Bolt::Target => String}] Each target and the local directory its download goes into
+      # @param source [String] The path to download from each target
+      # @param position [Array] Positional info for result tracking
+      # @param callback [Proc] Called with :node_start and :node_result events
+      # @return [Array<Bolt::Result>]
+      def download_into(directories, source, position, &callback)
+        result_opts = { action: 'download', name: source, position: position }
+        capable, results = prepare_targets(directories.keys, 'file_transfer', result_opts, &callback)
+        return results if capable.empty?
+
+        logger.debug { "Downloading #{source} from #{target_count(capable)}" }
+        # The library takes one directory per identity, so targets behind one
+        # identity share the last one's directory.
+        destination_dirs = capable.to_h { |target| [choria_identity(target), directories.fetch(target)] }
+        results + transfer(capable, result_opts, callback) do |client|
+          outcomes = client.download(source, destination_dirs)
+          file_transfer_results(capable, outcomes, result_opts, callback) do |target, outcome|
+            Bolt::Result.for_download(target, source, destination_dirs.fetch(choria_identity(target)), outcome.path)
+          end
+        end
+      end
+
+      # Runs one transfer for the targets with a library client and returns a
+      # result per target. A library that isn't installed on this host, or an
+      # error raised for what the controller contains, such as a source that
+      # can't be read, is a result for each of the targets.
+      #
+      # @param targets [Array<Bolt::Target>] The capable targets of one batch
+      # @param result_opts [Hash] Options passed through to emit_results (:action, :name, :position)
+      # @param callback [Proc, nil] Called with :node_result events
+      # @yield [MCollective::Util::FileTransfer::Client] The client for the targets
+      # @return [Array<Bolt::Result>]
+      def transfer(targets, result_opts, callback)
+        yield file_transfer_client(targets)
+      rescue Bolt::Error => e
+        failures = targets.to_h { |target| [target, error_output(e.message, e.kind)] }
+        emit_results(failures, **result_opts, &callback)
+      rescue SystemCallError, ArgumentError => e
+        logger.warn { "The file transfer raised #{e.class} on the controller: #{e.message}" }
+        logger.debug { e.backtrace.join("\n") }
+        message = "#{e.message} (#{e.class} on the controller)"
+        failures = targets.to_h { |target| [target, error_output(message, FILE_TRANSFER_ERROR_KINDS[:transfer_failed])] }
+        emit_results(failures, **result_opts, &callback)
+      end
+
+      # A library client for the targets, with the batch's options. The
+      # library builds its connection from the options this transport's own
+      # clients get and the RPC mutex every other call in the process takes.
+      # A timeout or size the batch does not set is left to the library's
+      # default.
+      def file_transfer_client(targets)
+        load_file_transfer_library
+        options = targets.first.options
+        settings = { rpc_timeout: options['file-transfer-timeout'], message_data_size: options['message-data-size'],
+                     upload_batch_size: options['upload-batch-size'] }
+        MCollective::Util::FileTransfer::Client.new(client_options(targets), mutex: @rpc_mutex, logger: logger, **settings.compact)
+      end
+
+      # The library is a client file of the agent's Puppet module under the
+      # libdir the Choria client config names, which loading that config in
+      # configure_client put on the load path.
+      def load_file_transfer_library
+        require 'mcollective/util/file_transfer'
+      rescue LoadError => e
+        raise Bolt::Error.new(
+          'The file_transfer client library is not installed on this host. Apply the mcollective_agent_file_transfer ' \
+          "module with client => true where OpenBolt runs, which installs it under the Choria libdir (#{e.class}: #{e.message}).",
+          'bolt/choria-file-transfer-library'
+        )
+      end
+
+      # A result per target from the outcomes by identity, firing
+      # node_result for each. The block builds the result of a success.
+      def file_transfer_results(targets, outcomes, result_opts, callback)
+        targets.map do |target|
+          outcome = outcomes.fetch(choria_identity(target))
+          result = if outcome.success?
+                     yield(target, outcome)
+                   else
+                     kind = FILE_TRANSFER_ERROR_KINDS.fetch(outcome.error_kind, FILE_TRANSFER_ERROR_KINDS[:transfer_failed])
+                     build_result(target, error_output(outcome.error_message, kind), **result_opts)
+                   end
+          callback&.call(type: :node_result, result: result)
+          result
+        end
+      end
+    end
+  end
+end