bolt 2.19.0 → 2.20.0

data/lib/bolt/transport/base.rb

@@ -167,6 +167,25 @@ module Bolt
         end
       end
 
+      # Downloads the given source file from a batch of targets to the destination location
+      # on the host.
+      #
+      # The default implementation only supports batches of size 1 and will fail otherwise.
+      #
+      # Transports may override this method to implement their own batch processing.
+      def batch_download(targets, source, destination, options = {}, &callback)
+        require 'erb'
+
+        assert_batch_size_one("batch_download()", targets)
+        target = targets.first
+        with_events(target, callback, 'download') do
+          escaped_name = ERB::Util.url_encode(target.safe_name)
+          target_destination = File.expand_path(escaped_name, destination)
+          @logger.debug { "Downloading: '#{source}' on #{target.safe_name} to #{target_destination}" }
+          download(target, source, target_destination, options)
+        end
+      end
+
       def batch_connected?(targets)
         assert_batch_size_one("connected?()", targets)
         connected?(targets.first)
@@ -201,6 +220,11 @@ module Bolt
         raise NotImplementedError, "upload() must be implemented by the transport class"
       end
 
+      # Transports should override this method with their own implementation of file download.
+      def download(*_args)
+        raise NotImplementedError, "download() must be implemented by the transport class"
+      end
+
       # Transports should override this method with their own implementation of a connection test.
       def connected?(_targets)
         raise NotImplementedError, "connected?() must be implemented by the transport class"

data/lib/bolt/transport/docker.rb

@@ -38,6 +38,14 @@ module Bolt
         end
       end
 
+      def download(target, source, destination, _options = {})
+        with_connection(target) do |conn|
+          download = File.join(destination, Bolt::Util.unix_basename(source))
+          conn.download_remote_content(source, destination)
+          Bolt::Result.for_download(target, source, destination, download)
+        end
+      end
+
       def run_command(target, command, options = {})
         execute_options = {}
         execute_options[:tty] = target.options['tty']

data/lib/bolt/transport/docker/connection.rb

@@ -82,7 +82,9 @@ module Bolt
         def write_remote_file(source, destination)
           @logger.debug { "Uploading #{source}, to #{destination}" }
           _, stdout_str, status = execute_local_docker_command('cp', [source, "#{container_id}:#{destination}"])
-          raise "Error writing file to container #{@container_id}: #{stdout_str}" unless status.exitstatus.zero?
+          unless status.exitstatus.zero?
+            raise "Error writing file to container #{@container_id}: #{stdout_str}"
+          end
         rescue StandardError => e
           raise Bolt::Node::FileError.new(e.message, 'WRITE_ERROR')
         end
@@ -90,7 +92,23 @@ module Bolt
         def write_remote_directory(source, destination)
           @logger.debug { "Uploading #{source}, to #{destination}" }
           _, stdout_str, status = execute_local_docker_command('cp', [source, "#{container_id}:#{destination}"])
-          raise "Error writing directory to container #{@container_id}: #{stdout_str}" unless status.exitstatus.zero?
+          unless status.exitstatus.zero?
+            raise "Error writing directory to container #{@container_id}: #{stdout_str}"
+          end
+        rescue StandardError => e
+          raise Bolt::Node::FileError.new(e.message, 'WRITE_ERROR')
+        end
+
+        def download_remote_content(source, destination)
+          @logger.debug { "Downloading #{source} to #{destination}" }
+          # Create the destination directory, otherwise copying a source directory with Docker will
+          # copy the *contents* of the directory.
+          # https://docs.docker.com/engine/reference/commandline/cp/
+          FileUtils.mkdir_p(destination)
+          _, stdout_str, status = execute_local_docker_command('cp', ["#{container_id}:#{source}", destination])
+          unless status.exitstatus.zero?
+            raise "Error downloading content from container #{@container_id}: #{stdout_str}"
+          end
         rescue StandardError => e
           raise Bolt::Node::FileError.new(e.message, 'WRITE_ERROR')
         end

data/lib/bolt/transport/local/connection.rb

@@ -27,7 +27,7 @@ module Bolt
                      end
         end
 
-        def copy_file(source, dest)
+        def upload_file(source, dest)
           @logger.debug { "Uploading #{source}, to #{dest}" }
           if source.is_a?(StringIO)
             Tempfile.create(File.basename(dest)) do |f|
@@ -45,6 +45,19 @@ module Bolt
           raise Bolt::Node::FileError.new(message, 'COPY_ERROR')
         end
 
+        def download_file(source, dest, _download)
+          @logger.debug { "Downloading #{source} to #{dest}" }
+          # Create the destination directory for the target, or the
+          # copied file will have the target's name
+          FileUtils.mkdir_p(dest)
+          # Mimic the behavior of `cp --remove-destination`
+          # since the flag isn't supported on MacOS
+          FileUtils.cp_r(source, dest, remove_destination: true)
+        rescue StandardError => e
+          message = "Could not download file to #{dest}: #{e}"
+          raise Bolt::Node::FileError.new(message, 'DOWNLOAD_ERROR')
+        end
+
         def execute(command)
           if Bolt::Util.windows?
             # If it's already a powershell command then invoke it normally.

data/lib/bolt/transport/orch.rb

@@ -184,6 +184,18 @@ module Bolt
         end
       end
 
+      def batch_download(targets, *_args)
+        error = {
+          'kind'    => 'bolt/not-supported-error',
+          'msg'     => 'pcp transport does not support downloading files',
+          'details' => {}
+        }
+
+        targets.map do |target|
+          Bolt::Result.new(target, error: error, action: 'download')
+        end
+      end
+
       def batches(targets)
         targets.group_by { |target| Connection.get_key(target.options) }.values
       end

data/lib/bolt/transport/simple.rb

@@ -32,6 +32,12 @@ module Bolt
         end
       end
 
+      def download(target, source, destination, options = {})
+        with_connection(target) do |conn|
+          conn.shell.download(source, destination, options)
+        end
+      end
+
       def run_script(target, script, arguments, options = {})
         with_connection(target) do |conn|
           conn.shell.run_script(script, arguments, options)

data/lib/bolt/transport/ssh/connection.rb

@@ -235,7 +235,7 @@ module Bolt
           raise Bolt::Error.new(msg, 'bolt/too-many-files')
         end
 
-        def copy_file(source, destination)
+        def upload_file(source, destination)
           # Do not log wrapper script content
           @logger.debug { "Uploading #{source}, to #{destination}" } unless source.is_a?(StringIO)
           @session.scp.upload!(source, destination, recursive: true)
@@ -243,6 +243,14 @@ module Bolt
           raise Bolt::Node::FileError.new(e.message, 'WRITE_ERROR')
         end
 
+        def download_file(source, destination, _download)
+          # Do not log wrapper script content
+          @logger.debug { "Downloading #{source} to #{destination}" }
+          @session.scp.download!(source, destination, recursive: true)
+        rescue StandardError => e
+          raise Bolt::Node::FileError.new(e.message, 'WRITE_ERROR')
+        end
+
         # This handles renaming Net::SSH verifiers between version 4.x and 5.x
         # of the gem
         def net_ssh_verifier(verifier)

data/lib/bolt/transport/ssh/exec_connection.rb

@@ -65,7 +65,7 @@ module Bolt
           ssh_cmd << command
         end
 
-        def copy_file(source, dest)
+        def upload_file(source, dest)
           @logger.debug { "Uploading #{source}, to #{userhost}:#{dest}" } unless source.is_a?(StringIO)
 
           cp_conf = @target.transport_config['copy-command'] || ["scp", "-r"]
@@ -94,6 +94,27 @@ module Bolt
           end
         end
 
+        def download_file(source, dest, _download)
+          @logger.debug { "Downloading #{userhost}:#{source} to #{dest}" }
+
+          FileUtils.mkdir_p(dest)
+
+          cp_conf = @target.transport_config['copy-command'] || ["scp", "-r"]
+          cp_cmd = Array(cp_conf)
+          cp_cmd += ssh_opts
+          cp_cmd << "#{userhost}:#{Shellwords.escape(source)}"
+          cp_cmd << dest
+
+          _, err, stat = Open3.capture3(*cp_cmd)
+
+          if stat.success?
+            @logger.debug "Successfully downloaded #{userhost}:#{source} to #{dest}"
+          else
+            message = "Could not copy file to #{dest}: #{err}"
+            raise Bolt::Node::FileError.new(message, 'COPY_ERROR')
+          end
+        end
+
         def execute(command)
           cmd_array = build_ssh_command(command)
           Open3.popen3(*cmd_array)

data/lib/bolt/transport/winrm/connection.rb

@@ -129,23 +129,23 @@ module Bolt
           raise
         end
 
-        def copy_file(source, destination)
+        def upload_file(source, destination)
           @logger.debug { "Uploading #{source}, to #{destination}" }
           if target.options['file-protocol'] == 'smb'
-            copy_file_smb(source, destination)
+            upload_file_smb(source, destination)
           else
-            copy_file_winrm(source, destination)
+            upload_file_winrm(source, destination)
           end
         end
 
-        def copy_file_winrm(source, destination)
+        def upload_file_winrm(source, destination)
           fs = ::WinRM::FS::FileManager.new(@connection)
           fs.upload(source, destination)
         rescue StandardError => e
           raise Bolt::Node::FileError.new(e.message, 'WRITE_ERROR')
         end
 
-        def copy_file_smb(source, destination)
+        def upload_file_smb(source, destination)
           # lazy-load expensive gem code
           require 'ruby_smb'
 
@@ -165,7 +165,7 @@ module Bolt
           client = smb_client_login
           tree = client.tree_connect(path)
           begin
-            copy_file_smb_recursive(tree, source, dest)
+            upload_file_smb_recursive(tree, source, dest)
           ensure
             tree.disconnect!
           end
@@ -175,6 +175,61 @@ module Bolt
           raise Bolt::Node::FileError.new(e.message, 'WRITE_ERROR')
         end
 
+        def download_file(source, destination, download)
+          @logger.debug { "Downloading #{source} to #{destination}" }
+          if target.options['file-protocol'] == 'smb'
+            download_file_smb(source, destination)
+          else
+            download_file_winrm(source, destination, download)
+          end
+        end
+
+        def download_file_winrm(source, destination, download)
+          # The winrm gem doesn't create the destination directory if it's missing,
+          # so create it here
+          FileUtils.mkdir_p(destination)
+          fs = ::WinRM::FS::FileManager.new(@connection)
+          # params: source, destination, chunksize, first
+          # first needs to be set to false, otherwise if the source is a directory it
+          # will be nested inside a directory with the same name
+          fs.download(source, download, 1024 * 1024, false)
+        rescue StandardError => e
+          raise Bolt::Node::FileError.new(e.message, 'WRITE_ERROR')
+        end
+
+        def download_file_smb(source, destination)
+          # lazy-load expensive gem code
+          require 'ruby_smb'
+
+          win_source = source.tr('/', '\\')
+          if (md = win_source.match(/^([a-z]):\\(.*)/i))
+            # if drive, use admin share for that drive, so path is '\\host\C$'
+            path = "\\\\#{@target.host}\\#{md[1]}$"
+            src  = md[2]
+          elsif (md = win_source.match(/^(\\\\[^\\]+\\[^\\]+)\\(.*)/))
+            # if unc, path is '\\host\share'
+            path = md[1]
+            src  = md[2]
+          else
+            raise ArgumentError, "Unknown source '#{source}'"
+          end
+
+          client = smb_client_login
+          tree = client.tree_connect(path)
+
+          begin
+            # Make sure the root download directory for the target exists
+            FileUtils.mkdir_p(destination)
+            download_file_smb_recursive(tree, src, destination)
+          ensure
+            tree.disconnect!
+          end
+        rescue ::RubySMB::Error::UnexpectedStatusCode => e
+          raise Bolt::Node::FileError.new("SMB Error: #{e.message}", 'DOWNLOAD_ERROR')
+        rescue StandardError => e
+          raise Bolt::Node::FileError.new(e.message, 'DOWNLOAD_ERROR')
+        end
+
         def shell
           @shell ||= Bolt::Shell::Powershell.new(target, self)
         end
@@ -230,13 +285,13 @@ module Bolt
           )
         end
 
-        def copy_file_smb_recursive(tree, source, dest)
+        def upload_file_smb_recursive(tree, source, dest)
           if Dir.exist?(source)
             tree.open_directory(directory: dest, write: true, disposition: ::RubySMB::Dispositions::FILE_OPEN_IF)
 
             Dir.children(source).each do |child|
               child_dest = dest + '\\' + child
-              copy_file_smb_recursive(tree, File.join(source, child), child_dest)
+              upload_file_smb_recursive(tree, File.join(source, child), child_dest)
             end
             return
           end
@@ -255,6 +310,52 @@ module Bolt
             file.close
           end
         end
+
+        def download_file_smb_recursive(tree, source, destination)
+          dest = File.expand_path(Bolt::Util.windows_basename(source), destination)
+
+          # Check if the source is a directory by attempting to list its children.
+          # If the source is a directory, create the directory on the host and then
+          # recurse through the children.
+          if (children = list_directory_children_smb(tree, source))
+            FileUtils.mkdir_p(dest)
+
+            children.each do |child|
+              # File names are encoded UTF_16LE.
+              filename = child.file_name.encode(Encoding::UTF_8)
+
+              next if %w[. ..].include?(filename)
+
+              src = source + '\\' + filename
+              download_file_smb_recursive(tree, src, dest)
+            end
+          # If the source wasn't a directory and just returns 'STATUS_NOT_A_DIRECTORY, then
+          # it is a file. Write it to the host.
+          else
+            begin
+              file = tree.open_file(filename: source)
+              data = file.read
+
+              # Files may be encoded UTF_16LE
+              data = data.encode(Encoding::UTF_8) if data.encoding == Encoding::UTF_16LE
+
+              File.write(dest, data)
+            ensure
+              file.close
+            end
+          end
+        end
+
+        # Lists the children of a directory using rb_smb
+        # Returns an array of RubySMB::Fscc::FileInformation::FileIdFullDirectoryInformation objects
+        # if the source is a directory, or raises RubySMB::Error::UnexpectedStatusCode otherwise.
+        def list_directory_children_smb(tree, source)
+          tree.list(directory: source)
+        rescue RubySMB::Error::UnexpectedStatusCode => e
+          unless e.message == 'STATUS_NOT_A_DIRECTORY'
+            raise e
+          end
+        end
       end
     end
   end

data/lib/bolt/util.rb

@@ -107,12 +107,13 @@ module Bolt
       # Accepts a Data object and returns a copy with all hash keys
       # modified by block. use &:to_s to stringify keys or &:to_sym to symbolize them
       def walk_keys(data, &block)
-        if data.is_a? Hash
+        case data
+        when Hash
           data.each_with_object({}) do |(k, v), acc|
             v = walk_keys(v, &block)
             acc[yield(k)] = v
           end
-        elsif data.is_a? Array
+        when Array
           data.map { |v| walk_keys(v, &block) }
         else
           data
@@ -124,9 +125,10 @@ module Bolt
       # their descendants are.
       def walk_vals(data, skip_top = false, &block)
         data = yield(data) unless skip_top
-        if data.is_a? Hash
+        case data
+        when Hash
           data.transform_values { |v| walk_vals(v, &block) }
-        elsif data.is_a? Array
+        when Array
           data.map { |v| walk_vals(v, &block) }
         else
           data
@@ -137,9 +139,10 @@ module Bolt
       # modified by the given block. Descendants are modified before their
       # parents.
       def postwalk_vals(data, skip_top = false, &block)
-        new_data = if data.is_a? Hash
+        new_data = case data
+                   when Hash
                      data.transform_values { |v| postwalk_vals(v, &block) }
-                   elsif data.is_a? Array
+                   when Array
                      data.map { |v| postwalk_vals(v, &block) }
                    else
                      data
@@ -193,11 +196,12 @@ module Bolt
           cloned[obj.object_id] = cl
           cloned[cl.object_id] = cl
 
-          if cl.is_a? Hash
+          case cl
+          when Hash
             obj.each { |k, v| cl[k] = deep_clone(v, cloned) }
-          elsif cl.is_a? Array
+          when Array
             cl.collect! { |v| deep_clone(v, cloned) }
-          elsif cl.is_a? Struct
+          when Struct
             obj.each_pair { |k, v| cl[k] = deep_clone(v, cloned) }
           end
 
@@ -257,14 +261,25 @@ module Bolt
 
       # Recursively searches a data structure for plugin references
       def references?(input)
-        if input.is_a?(Hash)
+        case input
+        when Hash
           input.key?('_plugin') || input.values.any? { |v| references?(v) }
-        elsif input.is_a?(Array)
+        when Array
           input.any? { |v| references?(v) }
         else
           false
         end
       end
+
+      def unix_basename(path)
+        raise Bolt::ValidationError, "path must be a String, received #{path.class} #{path}" unless path.is_a?(String)
+        path.split('/').last
+      end
+
+      def windows_basename(path)
+        raise Bolt::ValidationError, "path must be a String, received #{path.class} #{path}" unless path.is_a?(String)
+        path.split(%r{[/\\]}).last
+      end
     end
   end
 end