bolt 2.19.0 → 2.24.0

data/lib/bolt/result.rb

@@ -79,6 +79,13 @@ module Bolt
       new(target, message: "Uploaded '#{source}' to '#{target.host}:#{destination}'", action: 'upload', object: source)
     end
 
+    def self.for_download(target, source, destination, download)
+      msg   = "Downloaded '#{target.host}:#{source}' to '#{destination}'"
+      value = { 'path' => download }
+
+      new(target, value: value, message: msg, action: 'download', object: source)
+    end
+
     # Satisfies the Puppet datatypes API
     def self.from_asserted_args(target, value)
       new(target, value: value)

data/lib/bolt/shell/bash.rb

@@ -37,7 +37,7 @@ module Bolt
           with_tmpdir do |dir|
             basename = File.basename(source)
             tmpfile = File.join(dir.to_s, basename)
-            conn.copy_file(source, tmpfile)
+            conn.upload_file(source, tmpfile)
             # pass over file ownership if we're using run-as to be a different user
             dir.chown(run_as)
             result = execute(['mv', '-f', tmpfile, destination], sudoable: true)
@@ -50,6 +50,27 @@ module Bolt
         end
       end
 
+      def download(source, destination, options = {})
+        running_as(options[:run_as]) do
+          # Target OS may be either Unix or Windows. Without knowing the target OS before-hand
+          # we can't assume whether the path separator is '/' or '\'. Assume we're connecting
+          # to a target with Unix and then check if the path exists after downloading.
+          download = File.join(destination, Bolt::Util.unix_basename(source))
+
+          conn.download_file(source, destination, download)
+
+          # If the download path doesn't exist, then the file was likely downloaded from Windows
+          # using a source path with backslashes (e.g. 'C:\Users\Administrator\foo'). The file
+          # should be saved to the expected location, so update the download path assuming a
+          # Windows basename so the result shows the correct local path.
+          unless File.exist?(download)
+            download = File.join(destination, Bolt::Util.windows_basename(source))
+          end
+
+          Bolt::Result.for_download(target, source, destination, download)
+        end
+      end
+
       def run_script(script, arguments, options = {})
         # unpack any Sensitive data
         arguments = unwrap_sensitive_args(arguments)
@@ -82,7 +103,7 @@ module Bolt
                                 " using '#{execute_options[:interpreter]}' interpreter"
                               end
           # log the arguments with sensitive data redacted, do NOT log unwrapped_arguments
-          logger.debug("Running '#{executable}' with #{arguments.to_json}#{interpreter_debug}")
+          logger.trace("Running '#{executable}' with #{arguments.to_json}#{interpreter_debug}")
           # unpack any Sensitive data
           arguments = unwrap_sensitive_args(arguments)
 
@@ -95,7 +116,7 @@ module Bolt
               task_dir = File.join(dir.to_s, task.tasks_dir)
               dir.mkdirs([task.tasks_dir] + extra_files.map { |file| File.dirname(file['name']) })
               extra_files.each do |file|
-                conn.copy_file(file['path'], File.join(dir.to_s, file['name']))
+                conn.upload_file(file['path'], File.join(dir.to_s, file['name']))
               end
             end
 
@@ -182,13 +203,13 @@ module Bolt
 
       def handle_sudo_errors(err)
         if err =~ /^#{conn.user} is not in the sudoers file\./
-          @logger.debug { err }
+          @logger.trace { err }
           raise Bolt::Node::EscalateError.new(
             "User #{conn.user} does not have sudo permission on #{target}",
             'SUDO_DENIED'
           )
         elsif err =~ /^Sorry, try again\./
-          @logger.debug { err }
+          @logger.trace { err }
           raise Bolt::Node::EscalateError.new(
             "Sudo password for user #{conn.user} not recognized on #{target}",
             'BAD_PASSWORD'
@@ -257,7 +278,7 @@ module Bolt
       def write_executable(dir, file, filename = nil)
         filename ||= File.basename(file)
         remote_path = File.join(dir.to_s, filename)
-        conn.copy_file(file, remote_path)
+        conn.upload_file(file, remote_path)
         make_executable(remote_path)
         remote_path
       end
@@ -314,7 +335,6 @@ module Bolt
           sudo_str = if use_sudo
                        sudo_exec = target.options['sudo-executable'] || "sudo"
                        sudo_flags = [sudo_exec, "-S", "-H", "-u", run_as, "-p", sudo_prompt]
-                       sudo_flags += ["-E"] if options[:environment]
                        Shellwords.shelljoin(sudo_flags)
                      else
                        Shellwords.shelljoin(@target.options['run-as-command'] + [run_as])
@@ -331,7 +351,7 @@ module Bolt
 
         command_str = [sudo_str, env_decl, command_str].compact.join(' ')
 
-        @logger.debug { "Executing: #{command_str}" }
+        @logger.trace { "Executing `#{command_str}`" }
 
         in_buffer = if !use_sudo && options[:stdin]
                       String.new(options[:stdin], encoding: 'binary')
@@ -411,16 +431,16 @@ module Bolt
         result_output.exit_code = t.value.respond_to?(:exitstatus) ? t.value.exitstatus : t.value
 
         if result_output.exit_code == 0
-          @logger.debug { "Command returned successfully" }
+          @logger.trace { "Command `#{command_str}` returned successfully" }
         else
-          @logger.info { "Command failed with exit code #{result_output.exit_code}" }
+          @logger.trace { "Command #{command_str} failed with exit code #{result_output.exit_code}" }
         end
         result_output
       rescue StandardError
         # Ensure we close stdin and kill the child process
         inp&.close
         t&.terminate if t&.alive?
-        @logger.debug { "Command aborted" }
+        @logger.trace { "Command aborted" }
         raise
       end
 

data/lib/bolt/shell/powershell.rb

@@ -85,7 +85,7 @@ module Bolt
         filename ||= File.basename(file)
         validate_extensions(File.extname(filename))
         destination = "#{dir}\\#{filename}"
-        conn.copy_file(file, destination)
+        conn.upload_file(file, destination)
         destination
       end
 
@@ -164,10 +164,16 @@ module Bolt
       end
 
       def upload(source, destination, _options = {})
-        conn.copy_file(source, destination)
+        conn.upload_file(source, destination)
         Bolt::Result.for_upload(target, source, destination)
       end
 
+      def download(source, destination, _options = {})
+        download = File.join(destination, Bolt::Util.windows_basename(source))
+        conn.download_file(source, destination, download)
+        Bolt::Result.for_download(target, source, destination, download)
+      end
+
       def run_command(command, options = {})
         command = [*env_declarations(options[:env_vars]), command].join("\r\n") if options[:env_vars]
 
@@ -220,7 +226,7 @@ module Bolt
             task_dir = File.join(dir, task.tasks_dir)
             mkdirs([task_dir] + extra_files.map { |file| File.join(dir, File.dirname(file['name'])) })
             extra_files.each do |file|
-              conn.copy_file(file['path'], File.join(dir, file['name']))
+              conn.upload_file(file['path'], File.join(dir, file['name']))
             end
           end
 
@@ -262,7 +268,7 @@ module Bolt
           return with_tmpdir do |dir|
             command += "\r\nif (!$?) { if($LASTEXITCODE) { exit $LASTEXITCODE } else { exit 1 } }"
             script_file = File.join(dir, "#{SecureRandom.uuid}_wrapper.ps1")
-            conn.copy_file(StringIO.new(command), script_file)
+            conn.upload_file(StringIO.new(command), script_file)
             args = escape_arguments([script_file])
             script_invocation = ['powershell.exe', *PS_ARGS, '-File', *args].join(' ')
             execute(script_invocation)

data/lib/bolt/transport/base.rb

@@ -167,6 +167,25 @@ module Bolt
         end
       end
 
+      # Downloads the given source file from a batch of targets to the destination location
+      # on the host.
+      #
+      # The default implementation only supports batches of size 1 and will fail otherwise.
+      #
+      # Transports may override this method to implement their own batch processing.
+      def batch_download(targets, source, destination, options = {}, &callback)
+        require 'erb'
+
+        assert_batch_size_one("batch_download()", targets)
+        target = targets.first
+        with_events(target, callback, 'download') do
+          escaped_name = ERB::Util.url_encode(target.safe_name)
+          target_destination = File.expand_path(escaped_name, destination)
+          @logger.debug { "Downloading: '#{source}' on #{target.safe_name} to #{target_destination}" }
+          download(target, source, target_destination, options)
+        end
+      end
+
       def batch_connected?(targets)
         assert_batch_size_one("connected?()", targets)
         connected?(targets.first)
@@ -201,6 +220,11 @@ module Bolt
         raise NotImplementedError, "upload() must be implemented by the transport class"
       end
 
+      # Transports should override this method with their own implementation of file download.
+      def download(*_args)
+        raise NotImplementedError, "download() must be implemented by the transport class"
+      end
+
       # Transports should override this method with their own implementation of a connection test.
       def connected?(_targets)
         raise NotImplementedError, "connected?() must be implemented by the transport class"

data/lib/bolt/transport/docker.rb

@@ -38,6 +38,14 @@ module Bolt
         end
       end
 
+      def download(target, source, destination, _options = {})
+        with_connection(target) do |conn|
+          download = File.join(destination, Bolt::Util.unix_basename(source))
+          conn.download_remote_content(source, destination)
+          Bolt::Result.for_download(target, source, destination, download)
+        end
+      end
+
       def run_command(target, command, options = {})
         execute_options = {}
         execute_options[:tty] = target.options['tty']

data/lib/bolt/transport/docker/connection.rb

@@ -12,7 +12,7 @@ module Bolt
           @target = target
           @logger = Logging.logger[target.safe_name]
           @docker_host = @target.options['service-url']
-          @logger.debug("Initializing docker connection to #{@target.safe_name}")
+          @logger.trace("Initializing docker connection to #{@target.safe_name}")
         end
 
         def connect
@@ -25,7 +25,7 @@ module Bolt
           output = execute_local_docker_json_command('inspect', [output[index]["ID"]])
           # Store the container information for later
           @container_info = output[0]
-          @logger.debug { "Opened session" }
+          @logger.trace { "Opened session" }
           true
         rescue StandardError => e
           raise Bolt::Node::ConnectError.new(
@@ -57,16 +57,16 @@ module Bolt
           command_options << container_id
           command_options.concat(command)
 
-          @logger.debug { "Executing: exec #{command_options}" }
+          @logger.trace { "Executing: exec #{command_options}" }
 
           stdout_str, stderr_str, status = execute_local_docker_command('exec', command_options, options[:stdin])
 
           # The actual result is the exitstatus not the process object
           status = status.nil? ? -32768 : status.exitstatus
           if status == 0
-            @logger.debug { "Command returned successfully" }
+            @logger.trace { "Command returned successfully" }
           else
-            @logger.info { "Command failed with exit code #{status}" }
+            @logger.trace { "Command failed with exit code #{status}" }
           end
           stdout_str.force_encoding(Encoding::UTF_8)
           stderr_str.force_encoding(Encoding::UTF_8)
@@ -75,22 +75,40 @@ module Bolt
           stderr_str.gsub!("\r\n", "\n")
           [stdout_str, stderr_str, status]
         rescue StandardError
-          @logger.debug { "Command aborted" }
+          @logger.trace { "Command aborted" }
           raise
         end
 
         def write_remote_file(source, destination)
-          @logger.debug { "Uploading #{source}, to #{destination}" }
+          @logger.trace { "Uploading #{source} to #{destination}" }
           _, stdout_str, status = execute_local_docker_command('cp', [source, "#{container_id}:#{destination}"])
-          raise "Error writing file to container #{@container_id}: #{stdout_str}" unless status.exitstatus.zero?
+          unless status.exitstatus.zero?
+            raise "Error writing file to container #{@container_id}: #{stdout_str}"
+          end
         rescue StandardError => e
           raise Bolt::Node::FileError.new(e.message, 'WRITE_ERROR')
         end
 
         def write_remote_directory(source, destination)
-          @logger.debug { "Uploading #{source}, to #{destination}" }
+          @logger.trace { "Uploading #{source} to #{destination}" }
           _, stdout_str, status = execute_local_docker_command('cp', [source, "#{container_id}:#{destination}"])
-          raise "Error writing directory to container #{@container_id}: #{stdout_str}" unless status.exitstatus.zero?
+          unless status.exitstatus.zero?
+            raise "Error writing directory to container #{@container_id}: #{stdout_str}"
+          end
+        rescue StandardError => e
+          raise Bolt::Node::FileError.new(e.message, 'WRITE_ERROR')
+        end
+
+        def download_remote_content(source, destination)
+          @logger.trace { "Downloading #{source} to #{destination}" }
+          # Create the destination directory, otherwise copying a source directory with Docker will
+          # copy the *contents* of the directory.
+          # https://docs.docker.com/engine/reference/commandline/cp/
+          FileUtils.mkdir_p(destination)
+          _, stdout_str, status = execute_local_docker_command('cp', ["#{container_id}:#{source}", destination])
+          unless status.exitstatus.zero?
+            raise "Error downloading content from container #{@container_id}: #{stdout_str}"
+          end
         rescue StandardError => e
           raise Bolt::Node::FileError.new(e.message, 'WRITE_ERROR')
         end

data/lib/bolt/transport/local/connection.rb

@@ -27,8 +27,8 @@ module Bolt
                      end
         end
 
-        def copy_file(source, dest)
-          @logger.debug { "Uploading #{source}, to #{dest}" }
+        def upload_file(source, dest)
+          @logger.trace { "Uploading #{source} to #{dest}" }
           if source.is_a?(StringIO)
             Tempfile.create(File.basename(dest)) do |f|
               f.write(source.read)
@@ -45,6 +45,19 @@ module Bolt
           raise Bolt::Node::FileError.new(message, 'COPY_ERROR')
         end
 
+        def download_file(source, dest, _download)
+          @logger.trace { "Downloading #{source} to #{dest}" }
+          # Create the destination directory for the target, or the
+          # copied file will have the target's name
+          FileUtils.mkdir_p(dest)
+          # Mimic the behavior of `cp --remove-destination`
+          # since the flag isn't supported on MacOS
+          FileUtils.cp_r(source, dest, remove_destination: true)
+        rescue StandardError => e
+          message = "Could not download file to #{dest}: #{e}"
+          raise Bolt::Node::FileError.new(message, 'DOWNLOAD_ERROR')
+        end
+
         def execute(command)
           if Bolt::Util.windows?
             # If it's already a powershell command then invoke it normally.

data/lib/bolt/transport/orch.rb

@@ -38,7 +38,7 @@ module Bolt
           @connections.each_value do |conn|
             conn.finish_plan(result)
           rescue StandardError => e
-            @logger.debug("Failed to finish plan on #{conn.key}: #{e.message}")
+            @logger.trace("Failed to finish plan on #{conn.key}: #{e.message}")
           end
         end
       end
@@ -133,7 +133,7 @@ module Bolt
           next unless File.file?(file)
 
           tar_path = Pathname.new(file).relative_path_from(Pathname.new(directory))
-          @logger.debug("Packing #{file} to #{tar_path}")
+          @logger.trace("Packing #{file} to #{tar_path}")
           stat = File.stat(file)
           content = File.binread(file)
           output.tar.add_file_simple(
@@ -146,7 +146,7 @@ module Bolt
         end
 
         duration = Time.now - start_time
-        @logger.debug("Packed upload in #{duration * 1000} ms")
+        @logger.trace("Packed upload in #{duration * 1000} ms")
 
         output.close
         io.string
@@ -184,6 +184,18 @@ module Bolt
         end
       end
 
+      def batch_download(targets, *_args)
+        error = {
+          'kind'    => 'bolt/not-supported-error',
+          'msg'     => 'pcp transport does not support downloading files',
+          'details' => {}
+        }
+
+        targets.map do |target|
+          Bolt::Result.new(target, error: error, action: 'download')
+        end
+      end
+
       def batches(targets)
         targets.group_by { |target| Connection.get_key(target.options) }.values
       end

data/lib/bolt/transport/simple.rb

@@ -32,6 +32,12 @@ module Bolt
         end
       end
 
+      def download(target, source, destination, options = {})
+        with_connection(target) do |conn|
+          conn.shell.download(source, destination, options)
+        end
+      end
+
       def run_script(target, script, arguments, options = {})
         with_connection(target) do |conn|
           conn.shell.run_script(script, arguments, options)

data/lib/bolt/transport/ssh/connection.rb

@@ -28,7 +28,7 @@ module Bolt
 
           @logger = Logging.logger[@target.safe_name]
           @transport_logger = transport_logger
-          @logger.debug("Initializing ssh connection to #{@target.safe_name}")
+          @logger.trace("Initializing ssh connection to #{@target.safe_name}")
 
           if target.options['private-key']&.instance_of?(String)
             begin
@@ -131,7 +131,7 @@ module Bolt
 
           @session = Net::SSH.start(target.host, @user, options)
           validate_ssh_version
-          @logger.debug { "Opened session" }
+          @logger.trace { "Opened session" }
         rescue Net::SSH::AuthenticationFailed => e
           raise Bolt::Node::ConnectError.new(
             e.message,
@@ -161,7 +161,7 @@ module Bolt
             rescue Timeout::Error
               @session.shutdown!
             end
-            @logger.debug { "Closed session" }
+            @logger.trace { "Closed session" }
           end
         end
 
@@ -235,14 +235,22 @@ module Bolt
           raise Bolt::Error.new(msg, 'bolt/too-many-files')
         end
 
-        def copy_file(source, destination)
+        def upload_file(source, destination)
           # Do not log wrapper script content
-          @logger.debug { "Uploading #{source}, to #{destination}" } unless source.is_a?(StringIO)
+          @logger.trace { "Uploading #{source} to #{destination}" } unless source.is_a?(StringIO)
           @session.scp.upload!(source, destination, recursive: true)
         rescue StandardError => e
           raise Bolt::Node::FileError.new(e.message, 'WRITE_ERROR')
         end
 
+        def download_file(source, destination, _download)
+          # Do not log wrapper script content
+          @logger.trace { "Downloading #{source} to #{destination}" }
+          @session.scp.download!(source, destination, recursive: true)
+        rescue StandardError => e
+          raise Bolt::Node::FileError.new(e.message, 'WRITE_ERROR')
+        end
+
         # This handles renaming Net::SSH verifiers between version 4.x and 5.x
         # of the gem
         def net_ssh_verifier(verifier)