kitchen-sync 1.0.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a7e74c8634b238460e2128e167a09da25bd5cd5f
-  data.tar.gz: dd830cb7438202527397586e9aea0f1239dc86a8
+  metadata.gz: f5195fe4fdfca7efd26e6cb39af8faa9886d8efd
+  data.tar.gz: 86330af015cee42ae0075b628194982aa7f22a6e
 SHA512:
-  metadata.gz: 0917e5d947d2d581170d17bded14a8ffb979334a9ac78b67e047609ab54e604df1d86dc576c19309459629579155d9e192a6aa4c71cd3f0e03564199e90ac5d1
-  data.tar.gz: 1fb8e8de36b020d04e57fad7a2dc2a73202faf1cb5af983ce2ef4b68f7c907875b5716b4efe7000d6cae39f4db6e746e8aadc356b2bdb4ea97fac95a3dfb8eec
+  metadata.gz: b519464648300eb50bcfbc2cc00edb2a3bd334dfe8149b870e5767c9ed1fb217c90ec87e10b94f0844bff15211570f8c4f3760d9f3bcc276eedeb989e48280b9
+  data.tar.gz: fbae6448d755ce2de522fc172b8db7d6355622c9e9b85075d4d003ceaf240304736583e3678180c13b4892d9a82f0c102a70ae688c9c967980916d953a8180ce

data/lib/kitchen-sync/version.rb

@@ -16,6 +16,7 @@
 # limitations under the License.
 #
 
+
 class KitchenSync
-  VERSION = '1.0.1'
+  VERSION = '1.1.0'
 end

data/lib/kitchen/transport/sftp.rb

@@ -0,0 +1,218 @@
+#
+# Copyright 2014-2015, Noah Kantrowitz
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'kitchen/transport/ssh'
+
+
+module Kitchen
+  module Transport
+    class Sftp < Ssh
+      CHECKSUMS_PATH = File.expand_path('../../../kitchen-sync/checksums.rb', __FILE__)
+      CHECKSUMS_HASH = Digest::SHA1.file(CHECKSUMS_PATH)
+      CHECKSUMS_REMOTE_PATH = "/tmp/checksums-#{CHECKSUMS_HASH}.rb" # This won't work on Windows targets
+      MAX_TRANSFERS = 64
+
+      # Copy-pasta from Ssh#create_new_connection because I need the SFTP
+      # connection class.
+      # Tracked in https://github.com/test-kitchen/test-kitchen/pull/726
+      def create_new_connection(options, &block)
+        if @connection
+          logger.debug("[SSH] shutting previous connection #{@connection}")
+          @connection.close
+        end
+
+        @connection_options = options
+        @connection = self.class::Connection.new(options, &block)
+      end
+
+      class Connection < Ssh::Connection
+        # Wrap Ssh::Connection#close to also shut down the SFTP connection.
+        def close
+          if @sftp_session
+            logger.debug("[SFTP] closing connection to #{self}")
+            sftp_session.close_channel
+          end
+        ensure
+          @sftp_session = nil
+          super
+        end
+
+        def upload(locals, remote)
+          Array(locals).each do |local|
+            full_remote = File.join(remote, File.basename(local))
+            recursive = File.directory?(local)
+            time = Benchmark.realtime do
+              sftp_upload!(local, full_remote, recursive)
+            end
+            logger.info("[SFTP] Time taken to upload #{local} to #{self}:#{full_remote}: %.2f sec" % time)
+          end
+        end
+
+        private
+
+        def sftp_upload!(local, remote, recursive)
+          # Fast path check, if the remote path doesn't exist at all we just run a direct transfer
+          unless safe_stat(remote)
+            logger.debug("[SFTP] Fast path upload from #{local} to #{remote}")
+            sftp_session.mkdir!(remote) if recursive
+            sftp_session.upload!(local, remote, requests: MAX_TRANSFERS)
+            return
+          end
+          # Get checksums for existing files on the remote side.
+          logger.debug("[SFTP] Slow path upload from #{local} to #{remote}")
+          copy_checksums_script!
+          checksum_cmd = "/opt/chef/embedded/bin/ruby #{CHECKSUMS_REMOTE_PATH} #{remote}"
+          logger.debug("[SFTP] Running #{checksum_cmd}")
+          checksums = JSON.parse(session.exec!(checksum_cmd))
+          # Sync files that have changed.
+          files_to_upload(checksums, local, recursive).each do |rel_path|
+            upload_file(checksums, local, remote, rel_path)
+          end
+          purge_files(checksums, remote)
+          # Wait until all xfers are complete.
+          sftp_loop(0)
+        end
+
+        # Bug fix for session.loop never terminating if there is an SFTP conn active
+        # since as far as it is concerned there is still active stuff.
+        # This function is Copyright Fletcher Nichol
+        # Tracked in https://github.com/test-kitchen/test-kitchen/pull/724
+        def execute_with_exit_code(command)
+          exit_code = nil
+          session.open_channel do |channel|
+
+            channel.request_pty
+
+            channel.exec(command) do |_ch, _success|
+
+              channel.on_data do |_ch, data|
+                logger << data
+              end
+
+              channel.on_extended_data do |_ch, _type, data|
+                logger << data
+              end
+
+              channel.on_request("exit-status") do |_ch, data|
+                exit_code = data.read_long
+              end
+            end
+          end
+          session.loop { exit_code.nil? } # THERE IS A CHANGE ON THIS LINE, PAY ATTENTION!!!!!!
+          exit_code
+        end
+
+        # Create the SFTP session and block until it is ready.
+        #
+        # @return [Net::SFTP::Session]
+        def sftp_session
+          @sftp_session ||= session.sftp
+        end
+
+        # Return if the path exists (because net::sftp uses exceptions for that
+        # and it makes code gross) and also raise an exception if the path is a
+        # symlink.
+        #
+        # @param path [String] Remote path to check.
+        # @return [Boolean]
+        def safe_stat(path)
+          stat = sftp_session.lstat!(path)
+          raise "#{path} is a symlink, possible security threat, bailing out" if stat.symlink?
+          true
+        rescue Net::SFTP::StatusException
+          false
+        end
+
+        # Upload the checksum script if needed.
+        #
+        # @return [void]
+        def copy_checksums_script!
+          # Fast path because upload itself is called multiple times.
+          return if @checksums_copied
+          # Only try to transfer the script if it isn't present. a stat takes about
+          # 1/3rd the time of the transfer, so worst case here is still okay.
+          sftp_session.upload!(CHECKSUMS_PATH, CHECKSUMS_REMOTE_PATH) unless safe_stat(CHECKSUMS_REMOTE_PATH)
+          @checksums_copied = true
+        end
+
+        def files_to_upload(checksums, local, recursive)
+          glob_path = if recursive
+            File.join(local, '**', '*')
+          else
+            local
+          end
+          pending = []
+          Dir.glob(glob_path, File::FNM_PATHNAME | File::FNM_DOTMATCH).each do |path|
+            next unless File.file?(path)
+            rel_path = path[local.length..-1]
+            remote_hash = checksums.delete(rel_path)
+            pending << rel_path unless remote_hash && remote_hash == Digest::SHA1.file(path).hexdigest
+          end
+          pending
+        end
+
+        def upload_file(checksums, local, remote, rel_path)
+          parts = rel_path.split('/')
+          parts.pop # Drop the filename since we are only checking dirs
+          parts_to_check = []
+          until parts.empty?
+            parts_to_check << parts.shift
+            path_to_check = parts_to_check.join('/')
+            unless checksums[path_to_check]
+              logger.debug("[SFTP] Creating directory #{remote}#{path_to_check}")
+              add_xfer(sftp_session.mkdir("#{remote}#{path_to_check}"))
+              checksums[path_to_check] = true
+            end
+          end
+          logger.debug("[SFTP] Uploading #{local}#{rel_path} to #{remote}#{rel_path}")
+          add_xfer(sftp_session.upload("#{local}#{rel_path}", "#{remote}#{rel_path}"))
+        end
+
+        def purge_files(checksums, remote)
+          checksums.each do |key, value|
+            # Special case for /tmp/kitchen/cache upload not clearing cookbooks.
+            # Tracked in https://github.com/test-kitchen/test-kitchen/issues/725
+            next if remote == '/tmp/kitchen/cache' && key.start_with?('/cookbooks')
+            # Check if the file was uploaded in #upload_file.
+            if value != true
+              logger.debug("[SFTP] Removing #{remote}#{key}")
+              add_xfer(sftp_session.remove("#{remote}#{key}"))
+            end
+          end
+        end
+
+        def sftp_xfers
+          @sftp_xfers ||= []
+        end
+
+        def add_xfer(xfer)
+          sftp_xfers << xfer
+          sftp_loop
+        end
+
+        def sftp_loop(n_xfers=MAX_TRANSFERS)
+          sftp_session.loop do
+            sftp_xfers.delete_if {|x| !(x.is_a?(Net::SFTP::Request) ? x.pending? : x.active?) } # Purge any completed operations, which has two different APIs for some reason
+            sftp_xfers.length > n_xfers # Run until we have fewer than max
+          end
+        end
+
+
+      end
+
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kitchen-sync
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.1.0
 platform: ruby
 authors:
 - Noah Kantrowitz
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-01 00:00:00.000000000 Z
+date: 2015-05-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: test-kitchen
@@ -87,6 +87,7 @@ files:
 - lib/kitchen-sync/scp.rb
 - lib/kitchen-sync/sftp.rb
 - lib/kitchen-sync/version.rb
+- lib/kitchen/transport/sftp.rb
 homepage: https://github.com/coderanger/kitchen-sync
 licenses:
 - Apache 2.0
@@ -107,9 +108,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Improved file transfers for for test-kitchen
 test_files: []
-has_rdoc: