sshkit 1.21.4 → 1.25.0

data/lib/sshkit/backends/netssh.rb

@@ -1,8 +1,6 @@
 require 'English'
 require 'strscan'
-require 'mutex_m'
 require 'net/ssh'
-require 'net/scp'
 
 module Net
   module SSH
@@ -23,10 +21,27 @@ module SSHKit
   module Backend
 
     class Netssh < Abstract
+      def self.assert_valid_transfer_method!(method)
+        return if [:scp, :sftp].include?(method)
+
+        raise ArgumentError, "#{method.inspect} is not a valid transfer method. Supported methods are :scp, :sftp."
+      end
+
       class Configuration
         attr_accessor :connection_timeout, :pty
+        attr_reader :transfer_method
         attr_writer :ssh_options
 
+        def initialize
+          self.transfer_method = :scp
+        end
+
+        def transfer_method=(method)
+          Netssh.assert_valid_transfer_method!(method)
+
+          @transfer_method = method
+        end
+
         def ssh_options
           default_options.merge(@ssh_options ||= {})
         end
@@ -64,16 +79,16 @@ module SSHKit
       def upload!(local, remote, options = {})
         summarizer = transfer_summarizer('Uploading', options)
         remote = File.join(pwd_path, remote) unless remote.to_s.start_with?("/") || pwd_path.nil?
-        with_ssh do |ssh|
-          ssh.scp.upload!(local, remote, options, &summarizer)
+        with_transfer(summarizer) do |transfer|
+          transfer.upload!(local, remote, options)
         end
       end
 
       def download!(remote, local=nil, options = {})
         summarizer = transfer_summarizer('Downloading', options)
         remote = File.join(pwd_path, remote) unless remote.to_s.start_with?("/") || pwd_path.nil?
-        with_ssh do |ssh|
-          ssh.scp.download!(remote, local, options, &summarizer)
+        with_transfer(summarizer) do |transfer|
+          transfer.download!(remote, local, options)
         end
       end
 
@@ -105,7 +120,7 @@ module SSHKit
         last_percentage = nil
         proc do |_ch, name, transferred, total|
           percentage = (transferred.to_f * 100 / total.to_f)
-          unless percentage.nan?
+          unless percentage.nan? || percentage.infinite?
             message = "#{action} #{name} #{percentage.round(2)}%"
             percentage_r = (percentage / log_percent).truncate * log_percent
             if percentage_r > 0 && (last_name != name || last_percentage != percentage_r)
@@ -183,6 +198,20 @@ module SSHKit
         )
       end
 
+      def with_transfer(summarizer)
+        transfer_method = host.transfer_method || self.class.config.transfer_method
+        transfer_class = if transfer_method == :sftp
+                           require_relative "netssh/sftp_transfer"
+                           SftpTransfer
+                         else
+                           require_relative "netssh/scp_transfer"
+                           ScpTransfer
+                         end
+
+        with_ssh do |ssh|
+          yield(transfer_class.new(ssh, summarizer))
+        end
+      end
     end
   end
 

data/lib/sshkit/color.rb

@@ -44,8 +44,8 @@ module SSHKit
       return string unless COLOR_CODES.key?(color)
 
       result = mode == :bold ? "\e[1;" : "\e[0;"
-      result << COLOR_CODES.fetch(color).to_s
-      result << ";49m#{string}\e[0m"
+
+      "#{result}#{COLOR_CODES.fetch(color)};49m#{string}\e[0m"
     end
 
     # Returns `true` if the underlying output is a tty, or if the SSHKIT_COLOR

data/lib/sshkit/host.rb

@@ -1,4 +1,5 @@
 require 'ostruct'
+require 'resolv'
 
 module SSHKit
 
@@ -7,6 +8,7 @@ module SSHKit
   class Host
 
     attr_accessor :password, :hostname, :port, :user, :ssh_options
+    attr_reader :transfer_method
 
     def key=(new_key)
       @keys = [new_key]
@@ -41,6 +43,12 @@ module SSHKit
       end
     end
 
+    def transfer_method=(method)
+      Backend::Netssh.assert_valid_transfer_method!(method) unless method.nil?
+
+      @transfer_method = method
+    end
+
     def local?
       @local
     end
@@ -115,6 +123,22 @@ module SSHKit
 
   end
 
+  # @private
+  # :nodoc:
+  class IPv6HostParser < SimpleHostParser
+    def self.suitable?(host_string)
+      host_string.match(Resolv::IPv6::Regex)
+    end
+
+    def port
+
+    end
+
+    def hostname
+      @host_string.match(Resolv::IPv6::Regex)[0]
+    end
+  end
+
   class HostWithPortParser < SimpleHostParser
 
     def self.suitable?(host_string)
@@ -185,6 +209,7 @@ module SSHKit
 
   PARSERS = [
     SimpleHostParser,
+    IPv6HostParser,
     HostWithPortParser,
     HostWithUsernameAndPortParser,
     IPv6HostWithPortParser,

data/lib/sshkit/runners/parallel.rb

@@ -1,5 +1,3 @@
-require 'thread'
-
 module SSHKit
 
   module Runner

data/lib/sshkit/version.rb

@@ -1,3 +1,3 @@
 module SSHKit
-  VERSION = "1.21.4".freeze
+  VERSION = "1.25.0".freeze
 end

data/sshkit.gemspec

@@ -19,16 +19,20 @@ Gem::Specification.new do |gem|
   gem.name          = "sshkit"
   gem.require_paths = ["lib"]
   gem.version       = SSHKit::VERSION
+  gem.required_ruby_version = ">= 2.5"
 
+  gem.add_runtime_dependency('base64')
+  gem.add_runtime_dependency('logger')
   gem.add_runtime_dependency('net-ssh',  '>= 2.8.0')
   gem.add_runtime_dependency('net-scp',  '>= 1.1.2')
+  gem.add_runtime_dependency('net-sftp', '>= 2.1.2')
+  gem.add_runtime_dependency('ostruct')
 
   gem.add_development_dependency('danger')
   gem.add_development_dependency('minitest', '>= 5.0.0')
   gem.add_development_dependency('minitest-reporters')
-  gem.add_development_dependency('rainbow', '~> 2.2.2')
   gem.add_development_dependency('rake')
-  gem.add_development_dependency('rubocop', "~> 0.49.1")
+  gem.add_development_dependency('rubocop', "~> 0.52.0")
   gem.add_development_dependency('mocha')
 
   gem.add_development_dependency('bcrypt_pbkdf')

data/test/functional/backends/netssh_transfer_tests.rb

@@ -0,0 +1,83 @@
+require 'securerandom'
+
+module SSHKit
+  module Backend
+    module NetsshTransferTests
+      def setup
+        super
+        @output = String.new
+        SSHKit.config.output_verbosity = :debug
+        SSHKit.config.output = SSHKit::Formatter::SimpleText.new(@output)
+      end
+
+      def a_host
+        DockerWrapper.host
+      end
+
+      def test_upload_and_then_capture_file_contents
+        actual_file_contents = ""
+        file_name = File.join("/tmp", SecureRandom.uuid)
+        File.open file_name, 'w+' do |f|
+          f.write "Some Content\nWith a newline and trailing spaces    \n "
+        end
+        Netssh.new(a_host) do
+          upload!(file_name, file_name)
+          actual_file_contents = capture(:cat, file_name, strip: false)
+        end.run
+        assert_equal "Some Content\nWith a newline and trailing spaces    \n ", actual_file_contents
+      end
+
+      def test_upload_within
+        file_name = SecureRandom.uuid
+        file_contents = "Some Content"
+        dir_name = SecureRandom.uuid
+        actual_file_contents = ""
+        Netssh.new(a_host) do |_host|
+          within("/tmp") do
+            execute :mkdir, "-p", dir_name
+            within(dir_name) do
+              upload!(StringIO.new(file_contents), file_name)
+            end
+          end
+          actual_file_contents = capture(:cat, "/tmp/#{dir_name}/#{file_name}", strip: false)
+        end.run
+        assert_equal file_contents, actual_file_contents
+      end
+
+      def test_upload_string_io
+        file_contents = ""
+        Netssh.new(a_host) do |_host|
+          file_name = File.join("/tmp", SecureRandom.uuid)
+          upload!(StringIO.new('example_io'), file_name)
+          file_contents = download!(file_name)
+        end.run
+        assert_equal "example_io", file_contents
+      end
+
+      def test_upload_large_file
+        size      = 25
+        fills     = SecureRandom.random_bytes(1024*1024)
+        file_name = "/tmp/file-#{size}.txt"
+        File.open(file_name, 'wb') do |f|
+          (size).times {f.write(fills) }
+        end
+        file_contents = ""
+        Netssh.new(a_host) do
+          upload!(file_name, file_name)
+          file_contents = download!(file_name)
+        end.run
+        assert_equal File.open(file_name, 'rb').read, file_contents
+      end
+
+      def test_upload_via_pathname
+        file_contents = ""
+        Netssh.new(a_host) do |_host|
+          file_name = Pathname.new(File.join("/tmp", SecureRandom.uuid))
+          upload!(StringIO.new('example_io'), file_name)
+          file_contents = download!(file_name)
+        end.run
+        assert_equal "example_io", file_contents
+      end
+    end
+  end
+end

data/test/functional/backends/test_netssh.rb

@@ -1,5 +1,4 @@
 require 'helper'
-require 'securerandom'
 require 'benchmark'
 
 module SSHKit
@@ -16,7 +15,7 @@ module SSHKit
       end
 
       def a_host
-        VagrantWrapper.hosts['one']
+        DockerWrapper.host
       end
 
       def test_simple_netssh
@@ -75,14 +74,14 @@ module SSHKit
 
       def test_group_netssh
         Netssh.new(a_host) do
-          as user: :root, group: :admin do
+          as user: :root, group: :root do
            execute :touch, 'restart.txt'
           end
         end.run
         command_lines = @output.lines.select { |line| line.start_with?('Command:') }
         assert_equal [
           "Command: if ! sudo -u root whoami > /dev/null; then echo \"You cannot switch to user 'root' using sudo, please check the sudoers file\" 1>&2; false; fi\n",
-          "Command: sudo -u root -- sh -c sg\\ admin\\ -c\\ /usr/bin/env\\\\\\ touch\\\\\\ restart.txt\n"
+          "Command: sudo -u root -- sh -c sg\\ root\\ -c\\ /usr/bin/env\\\\\\ touch\\\\\\ restart.txt\n"
         ], command_lines
       end
 
@@ -132,75 +131,10 @@ module SSHKit
 
       def test_test_does_not_raise_on_non_zero_exit_status
         Netssh.new(a_host) do |_host|
-          test :false
+          test :false # rubocop:disable Lint/BooleanSymbol
         end.run
       end
 
-      def test_upload_and_then_capture_file_contents
-        actual_file_contents = ""
-        file_name = File.join("/tmp", SecureRandom.uuid)
-        File.open file_name, 'w+' do |f|
-          f.write "Some Content\nWith a newline and trailing spaces    \n "
-        end
-        Netssh.new(a_host) do
-          upload!(file_name, file_name)
-          actual_file_contents = capture(:cat, file_name, strip: false)
-        end.run
-        assert_equal "Some Content\nWith a newline and trailing spaces    \n ", actual_file_contents
-      end
-
-      def test_upload_within
-        file_name = SecureRandom.uuid
-        file_contents = "Some Content"
-        dir_name = SecureRandom.uuid
-        actual_file_contents = ""
-        Netssh.new(a_host) do |_host|
-          within("/tmp") do
-            execute :mkdir, "-p", dir_name
-            within(dir_name) do
-              upload!(StringIO.new(file_contents), file_name)
-            end
-          end
-          actual_file_contents = capture(:cat, "/tmp/#{dir_name}/#{file_name}", strip: false)
-        end.run
-        assert_equal file_contents, actual_file_contents
-      end
-
-      def test_upload_string_io
-        file_contents = ""
-        Netssh.new(a_host) do |_host|
-          file_name = File.join("/tmp", SecureRandom.uuid)
-          upload!(StringIO.new('example_io'), file_name)
-          file_contents = download!(file_name)
-        end.run
-        assert_equal "example_io", file_contents
-      end
-
-      def test_upload_large_file
-        size      = 25
-        fills     = SecureRandom.random_bytes(1024*1024)
-        file_name = "/tmp/file-#{size}.txt"
-        File.open(file_name, 'wb') do |f|
-          (size).times {f.write(fills) }
-        end
-        file_contents = ""
-        Netssh.new(a_host) do
-          upload!(file_name, file_name)
-          file_contents = download!(file_name)
-        end.run
-        assert_equal File.open(file_name, 'rb').read, file_contents
-      end
-
-      def test_upload_via_pathname
-        file_contents = ""
-        Netssh.new(a_host) do |_host|
-          file_name = Pathname.new(File.join("/tmp", SecureRandom.uuid))
-          upload!(StringIO.new('example_io'), file_name)
-          file_contents = download!(file_name)
-        end.run
-        assert_equal "example_io", file_contents
-      end
-
       def test_interaction_handler
         captured_command_result = nil
         Netssh.new(a_host) do
@@ -217,7 +151,7 @@ module SSHKit
         # ensure we enable connection pool
         SSHKit::Backend::Netssh.pool.idle_timeout = 10
         Netssh.new(a_host) do |_host|
-          test :false
+          test :false # rubocop:disable Lint/BooleanSymbol
         end.run
         sleep 2.5
         captured_command_result = nil

data/test/functional/backends/test_netssh_scp.rb

@@ -0,0 +1,23 @@
+require 'helper'
+require_relative 'netssh_transfer_tests'
+
+module SSHKit
+  module Backend
+    class TestNetsshScp < FunctionalTest
+      include NetsshTransferTests
+
+      def setup
+        super
+        SSHKit::Backend::Netssh.configure do |ssh|
+          ssh.transfer_method = :scp
+        end
+      end
+
+      def test_scp_implementation_is_used
+        Netssh.new(a_host).send(:with_transfer, nil) do |transfer|
+          assert_instance_of Netssh::ScpTransfer, transfer
+        end
+      end
+    end
+  end
+end

data/test/functional/backends/test_netssh_sftp.rb

@@ -0,0 +1,23 @@
+require 'helper'
+require_relative 'netssh_transfer_tests'
+
+module SSHKit
+  module Backend
+    class TestNetsshSftp < FunctionalTest
+      include NetsshTransferTests
+
+      def setup
+        super
+        SSHKit::Backend::Netssh.configure do |ssh|
+          ssh.transfer_method = :sftp
+        end
+      end
+
+      def test_sftp_implementation_is_used
+        Netssh.new(a_host).send(:with_transfer, nil) do |transfer|
+          assert_instance_of Netssh::SftpTransfer, transfer
+        end
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -28,51 +28,13 @@ class UnitTest < Minitest::Test
 end
 
 class FunctionalTest < Minitest::Test
-
   def setup
-    unless VagrantWrapper.running?
-      warn "Vagrant VMs are not running. Please, start it manually with `vagrant up`"
-    end
-  end
-
-  private
-
-  def create_user_with_key(username, password = :secret)
-    username, password = username.to_s, password.to_s
-
-    keys = VagrantWrapper.hosts.collect do |_name, host|
-      Net::SSH.start(host.hostname, host.user, port: host.port, password: host.password) do |ssh|
-
-        # Remove the user, make it again, force-generate a key for him
-        # short keys save us a few microseconds
-        ssh.exec!("sudo userdel -rf #{username}; true") # The `rescue nil` of the shell world
-        ssh.exec!("sudo useradd -m #{username}")
-        ssh.exec!("sudo echo y | ssh-keygen -b 1024 -f #{username} -N ''")
-        ssh.exec!("sudo chown vagrant:vagrant #{username}*")
-        ssh.exec!("sudo echo #{username}:#{password} | chpasswd")
+    require_relative "support/docker_wrapper"
+    return if DockerWrapper.running?
 
-        # Make the .ssh directory, change the ownership and the
-        ssh.exec!("sudo mkdir -p /home/#{username}/.ssh")
-        ssh.exec!("sudo chown #{username}:#{username} /home/#{username}/.ssh")
-        ssh.exec!("sudo chmod 700 /home/#{username}/.ssh")
-
-        # Move the key to authorized keys and chown and chmod it
-        ssh.exec!("sudo cat #{username}.pub > /home/#{username}/.ssh/authorized_keys")
-        ssh.exec!("sudo chown #{username}:#{username} /home/#{username}/.ssh/authorized_keys")
-        ssh.exec!("sudo chmod 600 /home/#{username}/.ssh/authorized_keys")
-
-        key = ssh.exec!("cat /home/vagrant/#{username}")
-
-        # Clean Up Files
-        ssh.exec!("sudo rm #{username} #{username}.pub")
-
-        key
-      end
-    end
-
-    Hash[VagrantWrapper.hosts.collect { |n, _h| n.to_sym }.zip(keys)]
+    DockerWrapper.start
+    DockerWrapper.wait_for_ssh_server
   end
-
 end
 
 #

data/test/support/docker_wrapper.rb

@@ -0,0 +1,71 @@
+require "socket"
+
+Minitest.after_run do
+  DockerWrapper.stop if DockerWrapper.running?
+end
+
+module DockerWrapper
+  SSH_SERVER_PORT = 2122
+
+  class << self
+    def host
+      SSHKit::Host.new(
+        user: "deployer",
+        hostname: "localhost",
+        port: SSH_SERVER_PORT,
+        password: "topsecret",
+        ssh_options: host_verify_options
+      )
+    end
+
+    def running?
+      out, status = run_compose_command("ps --status running", false)
+      status.success? && out.include?("ssh_server")
+    end
+
+    def start
+      run_compose_command("up -d")
+    end
+
+    def stop
+      run_compose_command("down")
+    end
+
+    def wait_for_ssh_server(retries=3)
+      Socket.tcp("localhost", SSH_SERVER_PORT, connect_timeout: 1).close
+      sleep(1)
+    rescue Errno::ECONNREFUSED, Errno::ETIMEDOUT
+      retries -= 1
+      sleep(2) && retry if retries.positive?
+      raise
+    end
+
+    private
+
+    def run_compose_command(command, echo=true)
+      $stderr.puts "[docker compose] #{command}" if echo
+      Open3.popen2e("docker compose #{command}") do |stdin, outerr, wait_thread|
+        stdin.close
+        output = Thread.new { capture_stream(outerr, echo) }
+        [output.value, wait_thread.value]
+      end
+    end
+
+    def capture_stream(stream, echo=true)
+      buffer = String.new
+      while (line = stream.gets)
+        buffer << line
+        $stderr.puts("[docker compose] #{line}") if echo
+      end
+      buffer
+    end
+
+    def host_verify_options
+      if Net::SSH::Version::MAJOR >= 5
+        { verify_host_key: :never }
+      else
+        { paranoid: false }
+      end
+    end
+  end
+end

data/test/unit/backends/test_abstract.rb

@@ -111,8 +111,63 @@ module SSHKit
         assert_equal 'cd ~/foo && /usr/bin/env cat file', backend.executed_command.to_command
       end
 
+      def test_as_properly_clears
+        backend = ExampleBackend.new do
+          as :root do
+            execute :cat, 'file', :strip => false
+          end
+
+          execute :cat, 'file', :strip => false
+        end
+
+        backend.run
+
+        assert_equal '/usr/bin/env cat file', backend.executed_command.to_command
+      end
+
+      def test_as_root
+        backend = ExampleBackend.new do
+          as :root do
+            execute :cat, 'file', :strip => false
+          end
+        end
+
+        backend.run
+
+        assert_equal 'sudo -u root -- sh -c /usr/bin/env\\ cat\\ file', backend.executed_command.to_command
+      end
+
+      def test_nested_as
+        backend = ExampleBackend.new do
+          as :root do
+            as :other_user do
+              execute :cat, 'file', :strip => false
+            end
+          end
+        end
+
+        backend.run
+
+        assert_equal 'sudo -u other_user -- sh -c /usr/bin/env\\ cat\\ file', backend.executed_command.to_command
+      end
+
+      def test_nested_as_properly_clears
+        backend = ExampleBackend.new do
+          as :root do
+            as :other_user do
+              execute :cat, 'file', :strip => false
+            end
+            execute :cat, 'file', :strip => false
+          end
+        end
+
+        backend.run
+
+        assert_equal 'sudo -u root -- sh -c /usr/bin/env\\ cat\\ file', backend.executed_command.to_command
+      end
+
       def test_background_logs_deprecation_warnings
-        deprecation_out = ''
+        deprecation_out = +''
         SSHKit.config.deprecation_output = deprecation_out
 
         ExampleBackend.new do
@@ -124,7 +179,7 @@ module SSHKit
         assert_equal 2, lines.length
 
         assert_equal("[Deprecated] The background method is deprecated. Blame badly behaved pseudo-daemons!\n", lines[0])
-        assert_match(/    \(Called from.*test_abstract.rb:\d+:in `block in test_background_logs_deprecation_warnings'\)\n/, lines[1])
+        assert_match(/    \(Called from.*test_abstract.rb:\d+:in .block in .*test_background_logs_deprecation_warnings.\)\n/, lines[1])
       end
 
       def test_calling_abstract_with_undefined_execute_command_raises_exception