sshkit 1.18.0 → 1.23.0

data/lib/sshkit/command.rb

@@ -1,5 +1,6 @@
 require 'digest/sha1'
 require 'securerandom'
+require 'shellwords'
 
 # @author Lee Hambley
 module SSHKit
@@ -9,7 +10,7 @@ module SSHKit
 
     Failed = Class.new(SSHKit::StandardError)
 
-    attr_reader :command, :args, :options, :started_at, :started, :exit_status, :full_stdout, :full_stderr
+    attr_reader :command, :args, :options, :started_at, :started, :exit_status, :full_stdout, :full_stderr, :uuid
 
     # Initialize a new Command object
     #
@@ -25,6 +26,7 @@ module SSHKit
       @args    = args
       @options.symbolize_keys!
       @stdout, @stderr, @full_stdout, @full_stderr = String.new, String.new, String.new, String.new
+      @uuid = Digest::SHA1.hexdigest(SecureRandom.random_bytes(10))[0..7]
     end
 
     def complete?
@@ -41,10 +43,6 @@ module SSHKit
       @started = new_started
     end
 
-    def uuid
-      @uuid ||= Digest::SHA1.hexdigest(SecureRandom.random_bytes(10))[0..7]
-    end
-
     def success?
       exit_status.nil? ? false : exit_status.to_i == 0
     end
@@ -145,7 +143,7 @@ module SSHKit
 
     def within(&_block)
       return yield unless options[:in]
-      sprintf("cd #{options[:in]} && %s", yield)
+      "cd #{self.class.shellescape_except_tilde(options[:in])} && #{yield}"
     end
 
     def environment_hash
@@ -161,28 +159,30 @@ module SSHKit
     end
 
     def with(&_block)
-      return yield unless environment_hash.any?
-      "( export #{environment_string} ; #{yield} )"
+      env_string = environment_string
+      return yield if env_string.empty?
+      "( export #{env_string} ; #{yield} )"
     end
 
     def user(&_block)
       return yield unless options[:user]
-      "sudo -u #{options[:user]} #{environment_string + " " unless environment_string.empty?}-- sh -c '#{yield}'"
+      env_string = environment_string
+      "sudo -u #{options[:user].to_s.shellescape} #{env_string + " " unless env_string.empty?}-- sh -c #{yield.shellescape}"
     end
 
     def in_background(&_block)
       return yield unless options[:run_in_background]
-      sprintf("( nohup %s > /dev/null & )", yield)
+      "( nohup #{yield} > /dev/null & )"
     end
 
     def umask(&_block)
       return yield unless SSHKit.config.umask
-      sprintf("umask #{SSHKit.config.umask} && %s", yield)
+      "umask #{SSHKit.config.umask} && #{yield}"
     end
 
     def group(&_block)
       return yield unless options[:group]
-      %Q(sg #{options[:group]} -c "#{yield}")
+      "sg #{options[:group].to_s.shellescape} -c #{yield.shellescape}"
       # We could also use the so-called heredoc format perhaps:
       #"newgrp #{options[:group]} <<EOC \\\"%s\\\" EOC" % %Q{#{yield}}
     end
@@ -219,6 +219,11 @@ module SSHKit
       end
     end
 
+    # allow using home directory but escape everything else like spaces etc
+    def self.shellescape_except_tilde(file)
+      file.shellescape.gsub("\\~", "~")
+    end
+
     private
 
     def default_options
@@ -234,7 +239,7 @@ module SSHKit
 
     def call_interaction_handler(stream_name, data, channel)
       interaction_handler = options[:interaction_handler]
-      interaction_handler = MappingInteractionHandler.new(interaction_handler) if interaction_handler.kind_of?(Hash)
+      interaction_handler = MappingInteractionHandler.new(interaction_handler) if interaction_handler.kind_of?(Hash) or interaction_handler.kind_of?(Proc)
       interaction_handler.on_data(self, stream_name, data, channel) if interaction_handler.respond_to?(:on_data)
     end
 

data/lib/sshkit/deprecation_logger.rb

@@ -1,3 +1,5 @@
+require 'set'
+
 module SSHKit
   class DeprecationLogger
     def initialize(out)

data/lib/sshkit/host.rb

@@ -1,4 +1,5 @@
 require 'ostruct'
+require 'resolv'
 
 module SSHKit
 
@@ -7,6 +8,7 @@ module SSHKit
   class Host
 
     attr_accessor :password, :hostname, :port, :user, :ssh_options
+    attr_reader :transfer_method
 
     def key=(new_key)
       @keys = [new_key]
@@ -41,6 +43,12 @@ module SSHKit
       end
     end
 
+    def transfer_method=(method)
+      Backend::Netssh.assert_valid_transfer_method!(method) unless method.nil?
+
+      @transfer_method = method
+    end
+
     def local?
       @local
     end
@@ -115,6 +123,22 @@ module SSHKit
 
   end
 
+  # @private
+  # :nodoc:
+  class IPv6HostParser < SimpleHostParser
+    def self.suitable?(host_string)
+      host_string.match(Resolv::IPv6::Regex)
+    end
+
+    def port
+
+    end
+
+    def hostname
+      @host_string.match(Resolv::IPv6::Regex)[0]
+    end
+  end
+
   class HostWithPortParser < SimpleHostParser
 
     def self.suitable?(host_string)
@@ -151,18 +175,20 @@ module SSHKit
   # @private
   # :nodoc:
   class IPv6HostWithPortParser < SimpleHostParser
+    IPV6_REGEX = /\[([a-fA-F0-9:]+)\](?:\:(\d+))?/
 
     def self.suitable?(host_string)
-      host_string.match(/[a-fA-F0-9:]+:\d+/)
+      host_string.match(IPV6_REGEX)
     end
 
     def port
-      @host_string.split(':').last.to_i
+      prt = @host_string.match(IPV6_REGEX)[2]
+      prt = prt.to_i unless prt.nil?
+      prt
     end
 
     def hostname
-      @host_string.gsub!(/\[|\]/, '')
-      @host_string.split(':')[0..-2].join(':')
+      @host_string.match(IPV6_REGEX)[1]
     end
 
   end
@@ -183,6 +209,7 @@ module SSHKit
 
   PARSERS = [
     SimpleHostParser,
+    IPv6HostParser,
     HostWithPortParser,
     HostWithUsernameAndPortParser,
     IPv6HostWithPortParser,

data/lib/sshkit/version.rb

@@ -1,3 +1,3 @@
 module SSHKit
-  VERSION = "1.18.0".freeze
+  VERSION = "1.23.0".freeze
 end

data/sshkit.gemspec

@@ -9,6 +9,9 @@ Gem::Specification.new do |gem|
   gem.description   = %q{A comprehensive toolkit for remotely running commands in a structured manner on groups of servers.}
   gem.homepage      = "http://github.com/capistrano/sshkit"
   gem.license       = "MIT"
+  gem.metadata      = {
+    "changelog_uri" => "https://github.com/capistrano/sshkit/releases"
+  }
 
   gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.files         = `git ls-files`.split("\n")
@@ -17,13 +20,15 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = SSHKit::VERSION
 
+  gem.add_runtime_dependency('base64') if RUBY_VERSION >= "2.4"
   gem.add_runtime_dependency('net-ssh',  '>= 2.8.0')
   gem.add_runtime_dependency('net-scp',  '>= 1.1.2')
+  gem.add_runtime_dependency('net-sftp', '>= 2.1.2')
 
   gem.add_development_dependency('danger')
   gem.add_development_dependency('minitest', '>= 5.0.0')
   gem.add_development_dependency('minitest-reporters')
-  gem.add_development_dependency('rainbow', '~> 2.1.0')
+  gem.add_development_dependency('rainbow', '~> 2.2.2')
   gem.add_development_dependency('rake')
   gem.add_development_dependency('rubocop', "~> 0.49.1")
   gem.add_development_dependency('mocha')

data/test/functional/backends/netssh_transfer_tests.rb

@@ -0,0 +1,83 @@
+require 'securerandom'
+
+module SSHKit
+  module Backend
+    module NetsshTransferTests
+      def setup
+        super
+        @output = String.new
+        SSHKit.config.output_verbosity = :debug
+        SSHKit.config.output = SSHKit::Formatter::SimpleText.new(@output)
+      end
+
+      def a_host
+        DockerWrapper.host
+      end
+
+      def test_upload_and_then_capture_file_contents
+        actual_file_contents = ""
+        file_name = File.join("/tmp", SecureRandom.uuid)
+        File.open file_name, 'w+' do |f|
+          f.write "Some Content\nWith a newline and trailing spaces    \n "
+        end
+        Netssh.new(a_host) do
+          upload!(file_name, file_name)
+          actual_file_contents = capture(:cat, file_name, strip: false)
+        end.run
+        assert_equal "Some Content\nWith a newline and trailing spaces    \n ", actual_file_contents
+      end
+
+      def test_upload_within
+        file_name = SecureRandom.uuid
+        file_contents = "Some Content"
+        dir_name = SecureRandom.uuid
+        actual_file_contents = ""
+        Netssh.new(a_host) do |_host|
+          within("/tmp") do
+            execute :mkdir, "-p", dir_name
+            within(dir_name) do
+              upload!(StringIO.new(file_contents), file_name)
+            end
+          end
+          actual_file_contents = capture(:cat, "/tmp/#{dir_name}/#{file_name}", strip: false)
+        end.run
+        assert_equal file_contents, actual_file_contents
+      end
+
+      def test_upload_string_io
+        file_contents = ""
+        Netssh.new(a_host) do |_host|
+          file_name = File.join("/tmp", SecureRandom.uuid)
+          upload!(StringIO.new('example_io'), file_name)
+          file_contents = download!(file_name)
+        end.run
+        assert_equal "example_io", file_contents
+      end
+
+      def test_upload_large_file
+        size      = 25
+        fills     = SecureRandom.random_bytes(1024*1024)
+        file_name = "/tmp/file-#{size}.txt"
+        File.open(file_name, 'wb') do |f|
+          (size).times {f.write(fills) }
+        end
+        file_contents = ""
+        Netssh.new(a_host) do
+          upload!(file_name, file_name)
+          file_contents = download!(file_name)
+        end.run
+        assert_equal File.open(file_name, 'rb').read, file_contents
+      end
+
+      def test_upload_via_pathname
+        file_contents = ""
+        Netssh.new(a_host) do |_host|
+          file_name = Pathname.new(File.join("/tmp", SecureRandom.uuid))
+          upload!(StringIO.new('example_io'), file_name)
+          file_contents = download!(file_name)
+        end.run
+        assert_equal "example_io", file_contents
+      end
+    end
+  end
+end

data/test/functional/backends/test_local.rb

@@ -99,6 +99,24 @@ module SSHKit
         end.run
         assert_equal("Enter Data\nCaptured SOME DATA", captured_command_result)
       end
+
+      def test_interaction_handler_with_proc
+        captured_command_result = nil
+        Local.new do
+          command = 'echo Enter Data; read the_data; echo Captured $the_data;'
+          captured_command_result = capture(command, interaction_handler:
+            lambda { |data|
+              case data
+              when "Enter Data\n"
+                "SOME DATA\n"
+              when "Captured SOME DATA\n"
+                nil
+              end
+            }
+          )
+        end.run
+        assert_equal("Enter Data\nCaptured SOME DATA", captured_command_result)
+      end
     end
   end
 end

data/test/functional/backends/test_netssh.rb

@@ -1,5 +1,4 @@
 require 'helper'
-require 'securerandom'
 require 'benchmark'
 
 module SSHKit
@@ -16,7 +15,7 @@ module SSHKit
       end
 
       def a_host
-        VagrantWrapper.hosts['one']
+        DockerWrapper.host
       end
 
       def test_simple_netssh
@@ -38,7 +37,7 @@ module SSHKit
           "Command: /usr/bin/env ls -l\n",
           "Command: if test ! -d /tmp; then echo \"Directory does not exist '/tmp'\" 1>&2; false; fi\n",
           "Command: if ! sudo -u root whoami > /dev/null; then echo \"You cannot switch to user 'root' using sudo, please check the sudoers file\" 1>&2; false; fi\n",
-          "Command: cd /tmp && ( export RAILS_ENV=\"production\" ; sudo -u root RAILS_ENV=\"production\" -- sh -c '/usr/bin/env touch restart.txt' )\n"
+          "Command: cd /tmp && ( export RAILS_ENV=\"production\" ; sudo -u root RAILS_ENV=\"production\" -- sh -c /usr/bin/env\\ touch\\ restart.txt )\n"
         ], command_lines
       end
 
@@ -75,14 +74,14 @@ module SSHKit
 
       def test_group_netssh
         Netssh.new(a_host) do
-          as user: :root, group: :admin do
+          as user: :root, group: :root do
            execute :touch, 'restart.txt'
           end
         end.run
         command_lines = @output.lines.select { |line| line.start_with?('Command:') }
         assert_equal [
           "Command: if ! sudo -u root whoami > /dev/null; then echo \"You cannot switch to user 'root' using sudo, please check the sudoers file\" 1>&2; false; fi\n",
-          "Command: sudo -u root -- sh -c 'sg admin -c \"/usr/bin/env touch restart.txt\"'\n"
+          "Command: sudo -u root -- sh -c sg\\ root\\ -c\\ /usr/bin/env\\\\\\ touch\\\\\\ restart.txt\n"
         ], command_lines
       end
 
@@ -96,21 +95,18 @@ module SSHKit
       end
 
       def test_ssh_option_merge
-        verify_host_opt = if Net::SSH::Version::MAJOR >= 5
-                            { verify_host_key: :always }
-                          else
-                            { paranoid: true }
-                          end
-        a_host.ssh_options = verify_host_opt
+        keepalive_opt = { keepalive: true }
+        test_host = a_host.dup
+        test_host.ssh_options = keepalive_opt
         host_ssh_options = {}
         SSHKit::Backend::Netssh.config.ssh_options = { forward_agent: false }
-        Netssh.new(a_host) do |host|
+        Netssh.new(test_host) do |host|
           capture(:uname)
           host_ssh_options = host.ssh_options
         end.run
-        assert_equal [:forward_agent, *verify_host_opt.keys, :known_hosts, :logger, :password_prompt].sort, host_ssh_options.keys.sort
+        assert_equal [:forward_agent, *keepalive_opt.keys, :known_hosts, :logger, :password_prompt].sort, host_ssh_options.keys.sort
         assert_equal false, host_ssh_options[:forward_agent]
-        assert_equal verify_host_opt.values.first, host_ssh_options[verify_host_opt.keys.first]
+        assert_equal keepalive_opt.values.first, host_ssh_options[keepalive_opt.keys.first]
         assert_instance_of SSHKit::Backend::Netssh::KnownHosts, host_ssh_options[:known_hosts]
       end
 
@@ -139,71 +135,6 @@ module SSHKit
         end.run
       end
 
-      def test_upload_and_then_capture_file_contents
-        actual_file_contents = ""
-        file_name = File.join("/tmp", SecureRandom.uuid)
-        File.open file_name, 'w+' do |f|
-          f.write "Some Content\nWith a newline and trailing spaces    \n "
-        end
-        Netssh.new(a_host) do
-          upload!(file_name, file_name)
-          actual_file_contents = capture(:cat, file_name, strip: false)
-        end.run
-        assert_equal "Some Content\nWith a newline and trailing spaces    \n ", actual_file_contents
-      end
-
-      def test_upload_within
-        file_name = SecureRandom.uuid
-        file_contents = "Some Content"
-        dir_name = SecureRandom.uuid
-        actual_file_contents = ""
-        Netssh.new(a_host) do |_host|
-          within("/tmp") do
-            execute :mkdir, "-p", dir_name
-            within(dir_name) do
-              upload!(StringIO.new(file_contents), file_name)
-            end
-          end
-          actual_file_contents = capture(:cat, "/tmp/#{dir_name}/#{file_name}", strip: false)
-        end.run
-        assert_equal file_contents, actual_file_contents
-      end
-
-      def test_upload_string_io
-        file_contents = ""
-        Netssh.new(a_host) do |_host|
-          file_name = File.join("/tmp", SecureRandom.uuid)
-          upload!(StringIO.new('example_io'), file_name)
-          file_contents = download!(file_name)
-        end.run
-        assert_equal "example_io", file_contents
-      end
-
-      def test_upload_large_file
-        size      = 25
-        fills     = SecureRandom.random_bytes(1024*1024)
-        file_name = "/tmp/file-#{size}.txt"
-        File.open(file_name, 'w') do |f|
-          (size).times {f.write(fills) }
-        end
-        file_contents = ""
-        Netssh.new(a_host) do
-          upload!(file_name, file_name)
-          file_contents = download!(file_name)
-        end.run
-        assert_equal File.open(file_name).read, file_contents
-      end
-
-      def test_upload_via_pathname
-        file_contents = ""
-        Netssh.new(a_host) do |_host|
-          file_name = Pathname.new(File.join("/tmp", SecureRandom.uuid))
-          upload!(StringIO.new('example_io'), file_name)
-          file_contents = download!(file_name)
-        end.run
-        assert_equal "example_io", file_contents
-      end
-
       def test_interaction_handler
         captured_command_result = nil
         Netssh.new(a_host) do
@@ -215,6 +146,20 @@ module SSHKit
         end.run
         assert_equal("Enter Data\nCaptured SOME DATA", captured_command_result)
       end
+
+      def test_connection_pool_keepalive
+        # ensure we enable connection pool
+        SSHKit::Backend::Netssh.pool.idle_timeout = 10
+        Netssh.new(a_host) do |_host|
+          test :false
+        end.run
+        sleep 2.5
+        captured_command_result = nil
+        Netssh.new(a_host) do |_host|
+          captured_command_result = capture(:echo, 'some_value')
+        end.run
+        assert_equal "some_value", captured_command_result
+      end
     end
 
   end

data/test/functional/backends/test_netssh_scp.rb

@@ -0,0 +1,23 @@
+require 'helper'
+require_relative 'netssh_transfer_tests'
+
+module SSHKit
+  module Backend
+    class TestNetsshScp < FunctionalTest
+      include NetsshTransferTests
+
+      def setup
+        super
+        SSHKit::Backend::Netssh.configure do |ssh|
+          ssh.transfer_method = :scp
+        end
+      end
+
+      def test_scp_implementation_is_used
+        Netssh.new(a_host).send(:with_transfer, nil) do |transfer|
+          assert_instance_of Netssh::ScpTransfer, transfer
+        end
+      end
+    end
+  end
+end

data/test/functional/backends/test_netssh_sftp.rb

@@ -0,0 +1,23 @@
+require 'helper'
+require_relative 'netssh_transfer_tests'
+
+module SSHKit
+  module Backend
+    class TestNetsshSftp < FunctionalTest
+      include NetsshTransferTests
+
+      def setup
+        super
+        SSHKit::Backend::Netssh.configure do |ssh|
+          ssh.transfer_method = :sftp
+        end
+      end
+
+      def test_sftp_implementation_is_used
+        Netssh.new(a_host).send(:with_transfer, nil) do |transfer|
+          assert_instance_of Netssh::SftpTransfer, transfer
+        end
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -3,7 +3,7 @@ require 'bundler/setup'
 require 'tempfile'
 require 'minitest/autorun'
 require 'minitest/reporters'
-require 'mocha/setup'
+require 'mocha/minitest'
 require 'stringio'
 require 'json'
 
@@ -28,51 +28,13 @@ class UnitTest < Minitest::Test
 end
 
 class FunctionalTest < Minitest::Test
-
   def setup
-    unless VagrantWrapper.running?
-      warn "Vagrant VMs are not running. Please, start it manually with `vagrant up`"
-    end
-  end
-
-  private
-
-  def create_user_with_key(username, password = :secret)
-    username, password = username.to_s, password.to_s
-
-    keys = VagrantWrapper.hosts.collect do |_name, host|
-      Net::SSH.start(host.hostname, host.user, port: host.port, password: host.password) do |ssh|
-
-        # Remove the user, make it again, force-generate a key for him
-        # short keys save us a few microseconds
-        ssh.exec!("sudo userdel -rf #{username}; true") # The `rescue nil` of the shell world
-        ssh.exec!("sudo useradd -m #{username}")
-        ssh.exec!("sudo echo y | ssh-keygen -b 1024 -f #{username} -N ''")
-        ssh.exec!("sudo chown vagrant:vagrant #{username}*")
-        ssh.exec!("sudo echo #{username}:#{password} | chpasswd")
+    require_relative "support/docker_wrapper"
+    return if DockerWrapper.running?
 
-        # Make the .ssh directory, change the ownership and the
-        ssh.exec!("sudo mkdir -p /home/#{username}/.ssh")
-        ssh.exec!("sudo chown #{username}:#{username} /home/#{username}/.ssh")
-        ssh.exec!("sudo chmod 700 /home/#{username}/.ssh")
-
-        # Move the key to authorized keys and chown and chmod it
-        ssh.exec!("sudo cat #{username}.pub > /home/#{username}/.ssh/authorized_keys")
-        ssh.exec!("sudo chown #{username}:#{username} /home/#{username}/.ssh/authorized_keys")
-        ssh.exec!("sudo chmod 600 /home/#{username}/.ssh/authorized_keys")
-
-        key = ssh.exec!("cat /home/vagrant/#{username}")
-
-        # Clean Up Files
-        ssh.exec!("sudo rm #{username} #{username}.pub")
-
-        key
-      end
-    end
-
-    Hash[VagrantWrapper.hosts.collect { |n, _h| n.to_sym }.zip(keys)]
+    DockerWrapper.start
+    DockerWrapper.wait_for_ssh_server
   end
-
 end
 
 #

data/test/support/docker_wrapper.rb

@@ -0,0 +1,71 @@
+require "socket"
+
+Minitest.after_run do
+  DockerWrapper.stop if DockerWrapper.running?
+end
+
+module DockerWrapper
+  SSH_SERVER_PORT = 2122
+
+  class << self
+    def host
+      SSHKit::Host.new(
+        user: "deployer",
+        hostname: "localhost",
+        port: SSH_SERVER_PORT,
+        password: "topsecret",
+        ssh_options: host_verify_options
+      )
+    end
+
+    def running?
+      out, status = run_compose_command("ps --status running", false)
+      status.success? && out.include?("ssh_server")
+    end
+
+    def start
+      run_compose_command("up -d")
+    end
+
+    def stop
+      run_compose_command("down")
+    end
+
+    def wait_for_ssh_server(retries=3)
+      Socket.tcp("localhost", SSH_SERVER_PORT, connect_timeout: 1).close
+      sleep(1)
+    rescue Errno::ECONNREFUSED, Errno::ETIMEDOUT
+      retries -= 1
+      sleep(2) && retry if retries.positive?
+      raise
+    end
+
+    private
+
+    def run_compose_command(command, echo=true)
+      $stderr.puts "[docker compose] #{command}" if echo
+      Open3.popen2e("docker compose #{command}") do |stdin, outerr, wait_thread|
+        stdin.close
+        output = Thread.new { capture_stream(outerr, echo) }
+        [output.value, wait_thread.value]
+      end
+    end
+
+    def capture_stream(stream, echo=true)
+      buffer = String.new
+      while (line = stream.gets)
+        buffer << line
+        $stderr.puts("[docker compose] #{line}") if echo
+      end
+      buffer
+    end
+
+    def host_verify_options
+      if Net::SSH::Version::MAJOR >= 5
+        { verify_host_key: :never }
+      else
+        { paranoid: false }
+      end
+    end
+  end
+end