sshkit 1.9.0 → 1.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1b6bcfd27e36ebe075f80d336e753443e1367aed
-  data.tar.gz: c41d529fd3c79e9b89506e20b95778672385f636
+  metadata.gz: 9da07f57b3af779a599eacb874eb170a75c70013
+  data.tar.gz: 547bea80a8bacc021f25188ddb2d031078077a4d
 SHA512:
-  metadata.gz: 8cc8f62186668eae9e473e7b72c073d31c008af7104ee893e5a5761bdc6c3159b9ce2ecd6063a0e4cbc4173eeb986451dffa5d6b69588e04ba577bd499cad8c7
-  data.tar.gz: be5c5e5686439d393a498d9084219eecb706192518bfc57aaeec18ae7c0b6c4803b344ef51690c0ab7ac9987725220caf2fbcd7eeac24f4ce3c5873760fda69d
+  metadata.gz: df5aaafd725f7142083d2d9a8a393561a03456230a01bdfda61d2341e3f7debc39f76a82dd373813ee67837fffeb6add8def675e48eb3139b3874061b8115543
+  data.tar.gz: f793b26a4fe0bae6c30eda6bea38484fc6faf834a7d3dc0b674f7c04c4b10bcf505fa38dda91a73ae37fa6f4f3dfca92bd1f658fee901eb387296bc44e84207e

data/CHANGELOG.md

@@ -8,6 +8,20 @@ appear at the top.
   * Add your entries below here, remember to credit yourself however you want
     to be credited!
 
+## 1.10.0 (2016-04-22)
+
+  * You can now opt-in to caching of SSH's known_hosts file for a speed boost
+    when deploying to a large fleet of servers. Refer to the
+    [README](https://github.com/capistrano/sshkit#known-hosts-caching) for
+    details. We plan to turn this on by default in a future version of SSHKit.
+    [PR #330](https://github.com/capistrano/sshkit/pull/330) @byroot
+  * SSHKit now explicitly closes its pooled SSH connections when Ruby exits;
+    this fixes `zlib(finalizer): the stream was freed prematurely` warnings
+    [PR #343](https://github.com/capistrano/sshkit/pull/343) @mattbrictson
+  * Allow command map entries (`SSHKit::CommandMap#[]`) to be Procs
+    [PR #310](https://github.com/capistrano/sshkit/pull/310)
+    @mikz
+
 ## 1.9.0
 
 **Refer to the 1.9.0.rc1 release notes for a full list of new features, fixes,

data/README.md

@@ -508,6 +508,19 @@ pooling behaviour entirely by setting the idle_timeout to zero:
 SSHKit::Backend::Netssh.pool.idle_timeout = 0 # disabled
 ```
 
+## Known hosts caching
+
+If you connect to many hosts with the `Netssh` backend, looking up `~/.ssh/known_hosts` can significantly impact performances.
+You can mitigate this by using SSHKit's lookup caching like this:
+
+```ruby
+SSHKit::Backend::Netssh.configure do |ssh|
+  ssh.ssh_options = {
+    known_hosts: SSHKit::Backend::Netssh::KnownHosts.new,
+  }
+end
+```
+
 ## Tunneling and other related SSH themes
 
 In order to do special gymnasitcs with SSH, tunneling, aliasing, complex options, etc with SSHKit it is possible to use [the underlying Net::SSH API](https://github.com/capistrano/sshkit/blob/master/EXAMPLES.md#setting-global-ssh-options) however in many cases it is preferred to use the system SSH configuration file at [`~/.ssh/config`](http://man.cx/ssh_config). This allows you to have personal configuration tied to your machine that does not have to be committed with the repository. If this is not suitable (everyone on the team needs a proxy command, or some special aliasing) a file in the same format can be placed in the project directory at `~/yourproject/.ssh/config`, this will be merged with the system settings in `~/.ssh/config`, and with any configuration specified in [`SSHKit::Backend::Netssh.config.ssh_options`](https://github.com/capistrano/sshkit/blob/master/lib/sshkit/backends/netssh.rb#L133).

data/lib/sshkit.rb

@@ -4,7 +4,7 @@ module SSHKit
 
   class << self
 
-    attr_accessor :config
+    attr_writer :config
 
     def configure
       @@config ||= Configuration.new

data/lib/sshkit/all.rb

@@ -35,5 +35,6 @@ require_relative 'backends/abstract'
 require_relative 'backends/connection_pool'
 require_relative 'backends/printer'
 require_relative 'backends/netssh'
+require_relative 'backends/netssh/known_hosts'
 require_relative 'backends/local'
 require_relative 'backends/skipper'

data/lib/sshkit/backends/connection_pool.rb

@@ -78,10 +78,12 @@ class SSHKit::Backend::ConnectionPool
     end
   end
 
-  private
+  protected
 
   attr_reader :caches, :timed_out_connections
 
+  private
+
   def cache_enabled?
     idle_timeout && idle_timeout > 0
   end

data/lib/sshkit/backends/connection_pool/cache.rb

@@ -53,10 +53,12 @@ class SSHKit::Backend::ConnectionPool::Cache
     end
   end
 
-  private
+  protected
 
   attr_reader :connections, :idle_timeout, :closer
 
+  private
+
   def fresh?(expires_at)
     expires_at > Time.now
   end

data/lib/sshkit/backends/netssh.rb

@@ -1,3 +1,6 @@
+require 'English'
+require 'strscan'
+require 'mutex_m'
 require 'net/ssh'
 require 'net/scp'
 
@@ -5,6 +8,8 @@ module Net
   module SSH
     class Config
       class << self
+        remove_method :default_files
+
         def default_files
           @@default_files + [File.join(Dir.pwd, '.ssh/config')]
         end
@@ -18,13 +23,12 @@ module SSHKit
   module Backend
 
     class Netssh < Abstract
-
       class Configuration
         attr_accessor :connection_timeout, :pty
         attr_writer :ssh_options
 
         def ssh_options
-          @ssh_options || {}
+          @ssh_options ||= {}
         end
       end
 
@@ -42,7 +46,12 @@ module SSHKit
         end
       end
 
+      # Note that this pool must be explicitly closed before Ruby exits to
+      # ensure the underlying IO objects are properly cleaned up. We register an
+      # at_exit handler to do this automatically, as long as Ruby is exiting
+      # cleanly (i.e. without an exception).
       @pool = SSHKit::Backend::ConnectionPool.new
+      at_exit { @pool.close_connections if @pool && !$ERROR_INFO }
 
       class << self
         attr_accessor :pool

data/lib/sshkit/backends/netssh/known_hosts.rb

@@ -0,0 +1,141 @@
+module SSHKit
+
+  module Backend
+
+    class Netssh < Abstract
+
+      class KnownHostsKeys
+        include Mutex_m
+
+        def initialize(path)
+          super()
+          @path = File.expand_path(path)
+          @hosts_keys = nil
+        end
+
+        def keys_for(hostlist)
+          keys, hashes = hosts_keys, hosts_hashes
+          parse_file unless keys && hashes
+          keys, hashes = hosts_keys, hosts_hashes
+
+          hostlist.split(',').each do |host|
+            key_list = keys[host]
+            return key_list if key_list
+
+            hashes.each do |(hmac, salt), hash_keys|
+              if OpenSSL::HMAC.digest(sha1, salt, host) == hmac
+                return hash_keys
+              end
+            end
+          end
+
+          []
+        end
+
+        private
+
+        attr_reader :path
+        attr_accessor :hosts_keys, :hosts_hashes
+
+        def sha1
+          @sha1 ||= OpenSSL::Digest.new('sha1')
+        end
+
+        def parse_file
+          synchronize do
+            return if hosts_keys && hosts_hashes
+
+            unless File.readable?(path)
+              self.hosts_keys = {}
+              self.hosts_hashes = []
+              return
+            end
+
+            new_keys = {}
+            new_hashes = []
+            File.open(path) do |file|
+              scanner = StringScanner.new("")
+              file.each_line do |line|
+                scanner.string = line
+                parse_line(scanner, new_keys, new_hashes)
+              end
+            end
+            self.hosts_keys = new_keys
+            self.hosts_hashes = new_hashes
+          end
+        end
+
+        def parse_line(scanner, hosts_keys, hosts_hashes)
+          return if empty_line?(scanner)
+
+          hostlist = parse_hostlist(scanner)
+          return unless supported_type?(scanner)
+          key = parse_key(scanner)
+
+          if hostlist.size == 1 && hostlist.first =~ /\A\|1(\|.+){2}\z/
+            hosts_hashes << [parse_host_hash(hostlist.first), key]
+          else
+            hostlist.each do |host|
+              (hosts_keys[host] ||= []) << key
+            end
+          end
+        end
+
+        def parse_host_hash(line)
+          _, _, salt, hmac = line.split('|')
+          [Base64.decode64(hmac), Base64.decode64(salt)]
+        end
+
+        def empty_line?(scanner)
+          scanner.skip(/\s*/)
+          scanner.match?(/$|#/)
+        end
+
+        def parse_hostlist(scanner)
+          scanner.skip(/\s*/)
+          scanner.scan(/\S+/).split(',')
+        end
+
+        def supported_type?(scanner)
+          scanner.skip(/\s*/)
+          Net::SSH::KnownHosts::SUPPORTED_TYPE.include?(scanner.scan(/\S+/))
+        end
+
+        def parse_key(scanner)
+          scanner.skip(/\s*/)
+          Net::SSH::Buffer.new(scanner.rest.unpack("m*").first).read_key
+        end
+      end
+
+      class KnownHosts
+        include Mutex_m
+
+        def initialize
+          super()
+          @files = {}
+        end
+
+        def search_for(host, options = {})
+          keys = ::Net::SSH::KnownHosts.hostfiles(options).map do |path|
+            known_hosts_file(path).keys_for(host)
+          end.flatten
+          ::Net::SSH::HostKeys.new(keys, host, self, options)
+        end
+
+        def add(*args)
+          ::Net::SSH::KnownHosts.add(*args)
+          synchronize { @files = {} }
+        end
+
+        private
+
+        def known_hosts_file(path)
+          @files[path] || synchronize { @files[path] ||= KnownHostsKeys.new(path) }
+        end
+      end
+
+    end
+
+  end
+
+end

data/lib/sshkit/command_map.rb

@@ -31,18 +31,20 @@ module SSHKit
       end
     end
 
+    TO_VALUE = ->(obj) { obj.respond_to?(:call) ? obj.call : obj }
+
     def initialize(value = nil)
       @map = CommandHash.new(value || defaults)
     end
 
     def [](command)
       if prefix[command].any?
-        prefixes = prefix[command].map{ |prefix| prefix.respond_to?(:call) ? prefix.call : prefix }
+        prefixes = prefix[command].map(&TO_VALUE)
         prefixes = prefixes.join(" ")
 
         "#{prefixes} #{command}"
       else
-        @map[command]
+        TO_VALUE.(@map[command])
       end
     end
 

data/lib/sshkit/configuration.rb

@@ -2,7 +2,7 @@ module SSHKit
 
   class Configuration
 
-    attr_accessor :umask, :output_verbosity
+    attr_accessor :umask
     attr_writer :output, :backend, :default_env, :default_runner
 
     def output

data/lib/sshkit/host.rb

@@ -99,7 +99,7 @@ module SSHKit
   class SimpleHostParser
 
     def self.suitable?(host_string)
-      !host_string.match(/[:|@]/)
+      !host_string.match(/:|@/)
     end
 
     def initialize(host_string)
@@ -127,7 +127,7 @@ module SSHKit
   class HostWithPortParser < SimpleHostParser
 
     def self.suitable?(host_string)
-      !host_string.match(/[@|\[|\]]/)
+      !host_string.match(/@|\[|\]/)
     end
 
     def port

data/lib/sshkit/version.rb

@@ -1,3 +1,3 @@
 module SSHKit
-  VERSION = "1.9.0"
+  VERSION = "1.10.0".freeze
 end

data/test/known_hosts/github

@@ -0,0 +1 @@
+github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==

data/test/known_hosts/github_hash

@@ -0,0 +1 @@
+|1|eKp+6E0rZ3lONgsIziurXEnaIik=|rcQB/rlJMUquUyFta64KugPjX4o= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==

data/test/unit/backends/test_netssh.rb

@@ -1,4 +1,5 @@
 require 'helper'
+require 'tempfile'
 
 module SSHKit
   module Backend
@@ -53,6 +54,29 @@ module SSHKit
         end
       end
 
+      if Net::SSH::Version::CURRENT >= Net::SSH::Version[3, 1, 0]
+        def test_known_hosts_for_when_all_hosts_are_recognized
+          perform_known_hosts_test("github")
+        end
+
+        def test_known_hosts_for_when_an_host_hash_is_recognized
+          perform_known_hosts_test("github_hash")
+        end
+      end
+
+      private
+
+      def perform_known_hosts_test(hostfile)
+        source = File.join(File.dirname(__FILE__), '../../known_hosts', hostfile)
+        kh = Netssh::KnownHosts.new
+        keys = kh.search_for('github.com', user_known_hosts_file: source, global_known_hosts_file: Tempfile.new('sshkit-test').path)
+
+        assert_instance_of ::Net::SSH::HostKeys, keys
+        assert_equal(1, keys.count)
+        keys.each do |key|
+          assert_equal("ssh-rsa", key.ssh_type)
+        end
+      end
     end
   end
 end

data/test/unit/test_command_map.rb

@@ -16,6 +16,15 @@ module SSHKit
       assert_equal map[:rake], "/usr/local/rbenv/shims/rake"
     end
 
+    def test_setter_procs
+      map = CommandMap.new
+      i = 0
+      map[:rake] = -> { i += 1; "/usr/local/rbenv/shims/rake#{i}" }
+
+      assert_equal map[:rake], "/usr/local/rbenv/shims/rake1"
+      assert_equal map[:rake], "/usr/local/rbenv/shims/rake2"
+    end
+
     def test_prefix
       map = CommandMap.new
       map.prefix[:rake].push("/home/vagrant/.rbenv/bin/rbenv exec")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sshkit
 version: !ruby/object:Gem::Version
-  version: 1.9.0
+  version: 1.10.0
 platform: ruby
 authors:
 - Lee Hambley
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-10 00:00:00.000000000 Z
+date: 2016-04-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: net-ssh
@@ -158,6 +158,7 @@ files:
 - lib/sshkit/backends/connection_pool/nil_cache.rb
 - lib/sshkit/backends/local.rb
 - lib/sshkit/backends/netssh.rb
+- lib/sshkit/backends/netssh/known_hosts.rb
 - lib/sshkit/backends/printer.rb
 - lib/sshkit/backends/skipper.rb
 - lib/sshkit/color.rb
@@ -189,6 +190,8 @@ files:
 - test/functional/backends/test_netssh.rb
 - test/functional/test_ssh_server_comes_up_for_functional_tests.rb
 - test/helper.rb
+- test/known_hosts/github
+- test/known_hosts/github_hash
 - test/support/vagrant_wrapper.rb
 - test/unit/backends/test_abstract.rb
 - test/unit/backends/test_connection_pool.rb
@@ -230,7 +233,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.1
+rubygems_version: 2.6.2
 signing_key: 
 specification_version: 4
 summary: SSHKit makes it easy to write structured, testable SSH commands in Ruby
@@ -240,6 +243,8 @@ test_files:
 - test/functional/backends/test_netssh.rb
 - test/functional/test_ssh_server_comes_up_for_functional_tests.rb
 - test/helper.rb
+- test/known_hosts/github
+- test/known_hosts/github_hash
 - test/support/vagrant_wrapper.rb
 - test/unit/backends/test_abstract.rb
 - test/unit/backends/test_connection_pool.rb