kitchen-sshtgz 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c4e92ac86f8af2cf922d6b29205848e55cfe99e1a3ba9575d326656fdc19411e
+  data.tar.gz: 9d2909d14f74b03585c544457431708a72772781ddbc9f3602554dada8276eb4
+SHA512:
+  metadata.gz: 6f020f9ad7d934deef00c1ae0f513bd560f668406e806f2c4aa3d82e3a5e64bfb24dd5897de99262c25147f147b0d61c8a70bccd4814eae5f4e1ad7d66e20683
+  data.tar.gz: 9d3e8b4ad111528c6aa4487a0361abaa9ee2c1290d23a7fe34b6c018c70564a01c3afb7fc55f9f9671d869d5098d9d82ca326fda9af88f87ca8878ba56ba87b9

data/lib/kitchen/tgz.rb

@@ -0,0 +1,176 @@
+# -*- encoding: utf-8 -*-
+#
+# Author:: Peter Smith (<peter@petersmith.net>)
+#
+# Copyright (C) 2015, Peter Smith
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "tempfile"
+require "zlib"
+require "rubygems/package"
+
+module Kitchen
+  #
+  # Error thrown if the Tgz has an invalid format.
+  #
+  class GzipFormatError < StandardError; end
+
+  #
+  # Represents a Tar-Gzip file, allowing multiple files to be combined
+  # into a single file. This is useful for transmitting a large number
+  # of files across high-latency networks.
+  #
+  class Tgz
+    #
+    # @return [String] the file system path of the generated tar-gzip file.
+    #
+    attr_reader :path
+
+    #
+    # Create a new Tgz object, in preparation for adding files to the
+    # tar-gzip archive.
+    #
+    # @param tgz_file_name [String, nil] The output file name, in
+    #   tar-gzipped format. If not specified, a temporary file name will be generated.
+    #
+    def initialize(tgz_file_name = nil)
+      if tgz_file_name
+        # user-provided output file name
+        @tgz_file = File.open(tgz_file_name, "wb+")
+        @path = tgz_file_name
+      else
+        # auto-generated output file name
+        @tgz_file = Tempfile.new("tgz")
+        @tgz_file.binmode
+        @path = @tgz_file.path
+      end
+
+      #
+      # Intermediate file for writing the 'tar' content (which will then be
+      # gzipped into the output 'tgz' file)
+      #
+      @tar_file = Tempfile.new("tar")
+      @tar_file.binmode
+      @tar_writer = Gem::Package::TarWriter.new(@tar_file)
+    end
+
+    #
+    # Add a set of files or directories into the Tgz archive. Directories will
+    # be traversed, with all files and subdirectories being added (symlinks are ignored).
+    #
+    # For example:
+    #
+    #     tgz.add_files('/home/me/my_dir', ['fileA', 'dirA/fileB'])
+    #
+    # @param dir [String] the directory containing the files/sub-dirs to archive.
+    # @param files [Array<String>] the files/sub-directories to be added, specified
+    #   relative to the containing directory (dir).
+    #
+    def add_files(dir, files)
+      files.each do |file_name|
+        full_path = "#{dir}/#{file_name}"
+        next if File.symlink?(full_path)
+        if File.directory?(full_path)
+          add_directory_to_tar(dir, file_name, full_path)
+        else
+          add_file_to_tar(full_path, file_name)
+        end
+      end
+    end
+
+    #
+    # Close the Tgz object, generating the tgz file (from the tar file), and ensuring
+    # that it's flushed to disk.
+    #
+    def close
+      # ensure tar_writer flushes everything to our temporary 'tar' file.
+      @tar_writer.close
+
+      # proceed to convert the 'tar' file into a 'tgz' file.
+      @tar_file.rewind
+      create_tgz_file(@tar_file, @tgz_file)
+      @tar_file.close
+    end
+
+    #
+    # Class-methods
+    #
+    class << self
+      #
+      # Return the original size of the uncompressed file.
+      #
+      # @param file_name [String] name of the compressed file, in .tgz format.
+      # @return [Integer] the original (uncompressed) file's size
+      # @raise [GzipFormatError] if the file is not a valid Gzip file.
+      #
+      def original_size(file_name)
+        File.open(file_name, "r") do |file|
+          # the first three bytes of a gzip file must be 0x1f, 0x8b, 0x08
+          fail unless (file.readbyte == 0x1f) && (file.readbyte == 0x8b) &&
+              (file.readbyte == 0x08)
+          return original_file_length_field(file)
+        end
+      rescue
+        raise GzipFormatError, "Gzip file could not be opened, or has invalid format: #{file_name}"
+      end
+
+      private
+
+      #
+      # Gzip files have their original/uncompressed length in the last 4 bytes
+      # (little endian format)
+      #
+      def original_file_length_field(file)
+        file.seek(-4, IO::SEEK_END)
+        (file.readbyte) | (file.readbyte << 8) | (file.readbyte << 16) | (file.readbyte << 24)
+      end
+    end
+
+    private
+
+    #
+    # Recursively traverse/add a sub-directory to the tar-gzip file.
+    #
+    def add_directory_to_tar(dir, file_name, full_path)
+      entries = Dir.entries(full_path)
+      entries.delete(".")
+      entries.delete("..")
+      add_files(dir, entries.map { |entry| "#{file_name}/#{entry}" })
+    end
+
+    #
+    # Add a single file to the tar-gzip file.
+    #
+    def add_file_to_tar(full_path, file_name)
+      stat = File.stat(full_path)
+      @tar_writer.add_file(file_name, stat.mode) do |file|
+        File.open(full_path, "rb") do |input|
+          while (buff = input.read(4096))
+            file.write(buff)
+          end
+        end
+      end
+    end
+
+    #
+    # The temporary tar file has been fully populated, so run the compress operation
+    # to generate the final tar-gzip file.
+    #
+    def create_tgz_file(source_file, dest_file)
+      Zlib::GzipWriter.wrap(dest_file) do |gzip_writer|
+        FileUtils.copy_stream(source_file, gzip_writer)
+      end
+    end
+  end
+end

data/lib/kitchen/transport/ssh_tgz.rb

@@ -0,0 +1,139 @@
+# -*- encoding: utf-8 -*-
+#
+# Author:: Peter Smith (<peter@petersmith.net>)
+#
+# Copyright (C) 2015, Peter Smith.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "kitchen"
+require "kitchen/tgz"
+require "kitchen/transport/ssh"
+
+require "net/ssh"
+require "net/scp"
+require "pathname"
+
+module Kitchen
+  module Transport
+    #
+    # A Transport which uses the SSH protocol to execute commands and transfer files.
+    # In addition, files are tar-gzipped to improve performance over high-latency
+    # network links.
+    #
+    # Most of this class reuses functionality from the base Ssh class.
+    #
+    # @author Peter Smith <peter@petersmith.net>
+    #
+    class SshTgz < Kitchen::Transport::Ssh
+
+      #
+      # Manage a connection for the SshTgz transport mechanism. This is essentially
+      # the same as for the Ssh::Connection class, except we compress before
+      # uploading.
+      #
+      class Connection < Kitchen::Transport::Ssh::Connection
+
+        # (see Ssh::Connection#upload)
+        def upload(locals, remote)
+          # attempt tar-gzip upload, to improve performance.
+          return if Array(locals).empty? || upload_via_tgz(Array(locals), remote)
+
+          # if tgz upload fails (e.g. not supported on target platform), fall back to
+          # file-by-file upload.
+          logger.warn("Tgz upload failed. Resorting to file-by-file upload.")
+          super
+        end
+
+        private
+
+        #
+        # Upload a set of files onto the remote machine. Rather than uploading
+        # file by file, we first package all files into a tar-gzip file (.tgz)
+        # and send a single file. This avoids the slow file-copy time we'd
+        # otherwise see over a high-latency network.
+        #
+        # @param locals [Array<String>] array of path names for each of the files to
+        #   be included.
+        # @param remote [String] directory on the remote host into which the files will
+        #   be uploaded.
+        # @return [true, false] true on success, else false.
+        #
+        def upload_via_tgz(locals, remote)
+          # tar-gzip all the input files into a single .tgz file.
+          tgz = create_tgz_file(locals)
+
+          # upload the tar-gzip file to the remote server.
+          session.scp.upload!(tgz.path, "#{remote}/kitchen.tgz", {}) do |_ch, name, sent, total|
+            logger.debug("Uploaded #{name} (#{total} bytes)") if sent == total
+          end
+
+          # extract the tar-gzip file, on the remote, into individual files.
+          untar_file_on_remote(remote)
+          File.unlink(tgz.path)
+
+          # indicate success - the files extracted correctly and won't need to be
+          # uploaded individually.
+          true
+        rescue => e
+          # on any failure, return false to indicate that upload via tgz failed and we
+          # should default to copying individual files.
+          logger.debug(".tgz upload failed. Reason: #{e}")
+          false
+        end
+
+        #
+        # Create a single tar-gzipped file, containing all of the individual files.
+        #
+        # @param locals [Array<String>] array of path names for each of the files to
+        #   be included.
+        # @return [Kitchen::Tgz] the Tgz object, representing the tar-gzip file we created.
+        #
+        def create_tgz_file(locals)
+          tgz = Kitchen::Tgz.new
+          locals.each do |local|
+            pathname = Pathname.new(local)
+            tgz.add_files(pathname.dirname.to_s, [pathname.basename.to_s])
+          end
+          tgz.close
+          tgz
+        end
+
+        #
+        # Untar the tgz file on the remote.
+        #
+        # @param remote [String] file system directory on remote host into which
+        #   the tar-gzip file was uploaded uploaded.
+        # @raise [Kitchen::SSHFailed] if the untar fails for some reason.
+        #
+        def untar_file_on_remote(remote)
+          cmd = "tar -C #{remote} -xmzf #{remote}/kitchen.tgz"
+          session.exec!(cmd) do |_ch, stream, data|
+            raise SSHFailed, "Unable to untar files on remote: #{data}" if stream == :stderr
+          end
+        end
+      end
+
+      # (see Ssh#create_new_connection)
+      def create_new_connection(options, &block)
+        if @connection
+          logger.debug("[SSH] shutting previous connection #{@connection}")
+          @connection.close
+        end
+
+        @connection_options = options
+        @connection = Kitchen::Transport::SshTgz::Connection.new(options, &block)
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,161 @@
+--- !ruby/object:Gem::Specification
+name: kitchen-sshtgz
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Jeremy Jacque
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2024-03-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: test-kitchen
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.5.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.5.4
+- !ruby/object:Gem::Dependency
+  name: net-ssh
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.1.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.1.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.1'
+- !ruby/object:Gem::Dependency
+  name: net-scp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: zlib
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: tempfile
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Improved file transfers for test-kitchen using tgz archives
+email:
+- jeremyjacque@algolia.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/kitchen/tgz.rb
+- lib/kitchen/transport/ssh_tgz.rb
+homepage: https://github.com/algolia/kitchen-sshtgz
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Improved file transfers for test-kitchen
+test_files: []