net-ssh 8.0.0.rc3 → 8.0.0.rc5

data/lib/net/ssh/authentication/session.rb

@@ -69,27 +69,22 @@ module Net
 
           attempted = []
 
-          @auth_methods.each do |name|
-            next unless @allowed_auth_methods.include?(name)
-
-            attempted << name
-
-            debug { "trying #{name}" }
-            begin
-              auth_class = Methods.const_get(name.split(/\W+/).map { |p| p.capitalize }.join)
-              method = auth_class.new(self,
-                                      key_manager: key_manager, password_prompt: options[:password_prompt],
-                                      pubkey_algorithms: options[:pubkey_algorithms] || nil)
-            rescue NameError
-              debug {"Mechanism #{name} was requested, but isn't a known type.  Ignoring it."}
-              next
+          # After a partial success (sshd AuthenticationMethods) restart from the most preferred method the server now allows, as OpenSSH does.
+          loop do
+            @partial_success = false
+
+            @auth_methods.each do |name|
+              next unless @allowed_auth_methods.include?(name)
+
+              attempted << name
+              return true if try_authenticate(name, next_service, username, password, key_manager)
+              break if @partial_success
             end
 
-            return true if method.authenticate(next_service, username, password)
-          rescue Net::SSH::Authentication::DisallowedMethod
+            break unless @partial_success
           end
 
-          error { "all authorization methods failed (tried #{attempted.join(', ')})" }
+          error { "all authorization methods failed (tried #{attempted.uniq.join(', ')})" }
           return false
         ensure
           key_manager.finish if key_manager
@@ -109,6 +104,7 @@ module Net
 
             when USERAUTH_FAILURE
               @allowed_auth_methods = packet[:authentications].split(/,/)
+              @partial_success ||= packet[:partial_success]
               debug { "allowed methods: #{packet[:authentications]}" }
               return packet
 
@@ -136,6 +132,22 @@ module Net
 
         private
 
+        def try_authenticate(method_name, next_service, username, password, key_manager)
+          debug { "trying #{method_name}" }
+          begin
+            auth_class = Methods.const_get(method_name.split(/\W+/).map { |p| p.capitalize }.join)
+            method = auth_class.new(self,
+                                    key_manager: key_manager, password_prompt: options[:password_prompt])
+          rescue NameError
+            debug {"Mechanism #{method_name} was requested, but isn't a known type.  Ignoring it."}
+            return false
+          end
+
+          method.authenticate(next_service, username, password)
+        rescue Net::SSH::Authentication::DisallowedMethod
+          false
+        end
+
         # Returns an array of paths to the key files usually defined
         # by system default.
         def default_keys

data/lib/net/ssh/buffer.rb

@@ -1,7 +1,9 @@
+require 'net/ssh/errors'
 require 'net/ssh/transport/openssl'
 
 require 'net/ssh/authentication/certificate'
 require 'net/ssh/authentication/ed25519_loader'
+require 'net/ssh/authentication/security_key'
 
 module Net
   module SSH
@@ -338,8 +340,11 @@ module Net
           key = Net::SSH::Authentication::ED25519::PubKey.read_keyblob(self)
         when /^ecdsa\-sha2\-(\w*)$/
           key = OpenSSL::PKey::EC.read_keyblob($1, self)
+        when Net::SSH::Authentication::SecurityKey::SK_ECDSA_SHA2_NISTP256,
+          Net::SSH::Authentication::SecurityKey::SK_SSH_ED25519
+          key = Net::SSH::Authentication::SecurityKey::PubKey.read_keyblob(type, self)
         else
-          raise NotImplementedError, "unsupported key type `#{type}'"
+          raise Net::SSH::UnsupportedKeyType, "unsupported key type `#{type}'"
         end
 
         return key

data/lib/net/ssh/buffered_io.rb

@@ -166,6 +166,11 @@ module Net
     #    http://github.com/net-ssh/net-ssh/tree/portfwfix
     #
     module ForwardedBufferedIo
+      def self.extended(object) # :nodoc:
+        object.instance_variable_set(:@on_sent, nil)
+        object.instance_variable_set(:@shutdown_write_when_drained, false)
+      end
+
       def fill(n = 8192)
         begin
           super(n)
@@ -182,11 +187,25 @@ module Net
         end
       end
 
+      # Internal and experimental: registers a block called with the number of
+      # bytes each #send_pending wrote to the socket.
+      def on_sent(&block) # :nodoc:
+        @on_sent = block
+      end
+
       def send_pending
         begin
-          super
-        rescue Errno::ECONNRESET => e
-          debug { "connection was reset => shallowing exception:#{e}" }
+          pending = output.length
+          result = super
+          @on_sent.call(pending - output.length) if @on_sent && output.length < pending
+          shutdown_write_if_drained
+          result
+        rescue Errno::EPIPE, Errno::ECONNRESET => e
+          # Like OpenSSH's chan_write_failed: the peer is gone, so drop what it
+          # will never read and close, letting the forwarded channel close too.
+          debug { "write failed => discarding #{output.length} bytes and closing:#{e}" }
+          output.clear!
+          close
           return 0
         rescue IOError => e
           if e.message =~ /closed/ then
@@ -197,6 +216,24 @@ module Net
           end
         end
       end
+
+      # Internal: shuts down the write side once pending output has been sent,
+      # as OpenSSH does on CHANNEL_EOF (chan_rcvd_ieof, chan_obuf_empty).
+      def shutdown_write_when_drained # :nodoc:
+        @shutdown_write_when_drained = true
+        shutdown_write_if_drained
+      end
+
+      private
+
+      def shutdown_write_if_drained
+        return unless @shutdown_write_when_drained && output.empty? && !closed?
+
+        @shutdown_write_when_drained = false
+        shutdown(Socket::SHUT_WR)
+      rescue Errno::EPIPE, Errno::ENOTCONN => e
+        debug { "shutdown failed => shallowing exception:#{e}" }
+      end
     end
   end
 end

data/lib/net/ssh/config.rb

@@ -15,6 +15,8 @@ module Net
     # * Compression => :compression
     # * CompressionLevel => :compression_level
     # * ConnectTimeout => maps to the :timeout option
+    # * ControlPath => :control_path, used only when Net::SSH.start is given
+    #   control_master: :auto; ControlMaster itself is not read
     # * ForwardAgent => :forward_agent
     # * GlobalKnownHostsFile => :global_known_hosts_file
     # * HostBasedAuthentication => maps to the :auth_methods option
@@ -24,6 +26,7 @@ module Net
     # * IdentityFile => maps to the :keys option
     # * IdentityAgent => :identity_agent
     # * IdentitiesOnly => :keys_only
+    # * KexAlgorithms => maps to the :kex option
     # * CheckHostIP => :check_host_ip
     # * Macs => maps to the :hmac option
     # * PasswordAuthentication => maps to the :auth_methods option password
@@ -31,6 +34,7 @@ module Net
     # * PreferredAuthentications => maps to the :auth_methods option
     # * ProxyCommand => maps to the :proxy option
     # * ProxyJump => maps to the :proxy option
+    # * PubkeyAcceptedAlgorithms (or PubkeyAcceptedKeyTypes) => maps to :pubkey_algorithms
     # * PubKeyAuthentication => maps to the :auth_methods option
     # * RekeyLimit => :rekey_limit
     # * StrictHostKeyChecking => :verify_host_key
@@ -95,6 +99,7 @@ module Net
             next if value.nil?
 
             key.downcase!
+            value = uncomment(value)
             value = unquote(value)
 
             value = case value.strip
@@ -217,6 +222,7 @@ module Net
           bindaddress: :bind_address,
           compression: :compression,
           compressionlevel: :compression_level,
+          controlpath: :control_path,
           certificatefile: :keycerts,
           connecttimeout: :timeout,
           forwardagent: :forward_agent,
@@ -230,7 +236,7 @@ module Net
           user: :user,
           checkhostip: :check_host_ip
         }.freeze
-        def translate_config_key(hash, key, value, settings)
+        def translate_config_key(hash, key, value, settings) # rubocop:disable Metrics/AbcSize
           case key
           when :stricthostkeychecking
             hash[:verify_host_key] = translate_verify_host_key(value)
@@ -246,6 +252,8 @@ module Net
             hash[:host_key] = value.split(/,/)
           when :hostname
             hash[:host_name] = value.gsub(/%h/, settings['host'])
+          when :kexalgorithms
+            hash[:kex] = value.split(/,/)
           when :macs
             hash[:hmac] = value.split(/,/)
           when :serveralivecountmax
@@ -276,6 +284,8 @@ module Net
             if (proxy = setup_proxy(*value))
               hash[:proxy] = proxy
             end
+          when :pubkeyacceptedalgorithms, :pubkeyacceptedkeytypes
+            hash[:pubkey_algorithms] = value.split(/,/)
           when :pubkeyauthentication
             if value
               (hash[:auth_methods] << 'publickey').uniq!
@@ -399,6 +409,11 @@ module Net
           !condition_matches.empty? && condition_matches.all?
         end
 
+        # As in OpenSSH's argv_split, "#" starts a comment only at the start of a token and outside quotes.
+        def uncomment(string)
+          string[/\A(?:\s*(?!#)(?:\\.|"[^"]*"?|'[^']*'?|[^\s"'])+)*/]
+        end
+
         def unquote(string)
           string =~ /^"(.*)"$/ ? Regexp.last_match(1) : string
         end

data/lib/net/ssh/connection/channel.rb

@@ -55,6 +55,9 @@ module Net
         include Loggable
         include Constants
 
+        # OpenSSH's CHAN_SES_WINDOW_DEFAULT; some servers (e.g. Mikrotik) truncate output with smaller windows (#726).
+        DEFAULT_MAX_WIN_SIZE = 0x200000
+
         # The local id for this channel, assigned by the Net::SSH::Connection::Session instance.
         attr_reader :local_id
 
@@ -109,7 +112,7 @@ module Net
         # that time (see #do_open_confirmation).
         #
         # This also sets the default maximum packet size and maximum window size.
-        def initialize(connection, type, local_id, max_pkt_size = 0x8000, max_win_size = 0x20000, &on_confirm_open)
+        def initialize(connection, type, local_id, max_pkt_size = 0x8000, max_win_size = DEFAULT_MAX_WIN_SIZE, &on_confirm_open)
           self.logger = connection.logger
 
           @connection = connection
@@ -118,6 +121,8 @@ module Net
 
           @local_maximum_packet_size = max_pkt_size
           @local_window_size = @local_maximum_window_size = max_win_size
+          @local_consumed = 0
+          @grant_window_on_consume = false
 
           @on_confirm_open = on_confirm_open
 
@@ -254,6 +259,26 @@ module Net
           output.append(data.to_s)
         end
 
+        # Internal and experimental, used by port forwarding: from now on window
+        # is granted only for bytes reported via #consumed, so a slow consumer
+        # stalls the remote sender instead of buffering everything it sends.
+        # Must be called before any data is received.
+        def grant_window_on_consume! # :nodoc:
+          raise ArgumentError, "grant_window_on_consume! must be called before any data is received" if local_window_size != local_maximum_window_size
+
+          @grant_window_on_consume = true
+        end
+
+        # Internal and experimental: reports that +bytes+ of received data have
+        # been processed, letting the remote end send that much more.
+        def consumed(bytes) # :nodoc:
+          raise ArgumentError, "consumed requires grant_window_on_consume!" unless @grant_window_on_consume
+          raise ArgumentError, "consumed bytes must not be negative" if bytes.negative?
+
+          @local_consumed += bytes
+          adjust_window_for_consumed
+        end
+
         # Returns true if the channel exists in the channel list of the session,
         # and false otherwise. This can be used to determine whether a channel has
         # been closed or not.
@@ -598,7 +623,12 @@ module Net
         # type, and data.
         def do_extended_data(type, data)
           update_local_window_size(data.length)
-          @on_extended_data.call(self, type, data) if @on_extended_data
+          if @on_extended_data
+            @on_extended_data.call(self, type, data)
+          elsif @grant_window_on_consume
+            # Nothing will ever consume it, so don't let it shrink the window.
+            consumed(data.length)
+          end
         end
 
         # Invokes the #on_eof callback when the server indicates that no
@@ -651,6 +681,8 @@ module Net
         # server telling it that the window size has grown.
         def update_local_window_size(size)
           @local_window_size -= size
+          return adjust_window_for_consumed if @grant_window_on_consume
+
           if local_window_size < local_maximum_window_size / 2
             connection.send_message(
               Buffer.from(:byte, CHANNEL_WINDOW_ADJUST, :long, remote_id, :long, LOCAL_WINDOW_SIZE_INCREMENT)
@@ -663,6 +695,21 @@ module Net
           end
         end
 
+        # Mirrors OpenSSH's channel_check_window: once the window is below half,
+        # give back what was consumed, growing the maximum window the same way
+        # #update_local_window_size does.
+        def adjust_window_for_consumed
+          return if @local_consumed.zero? || local_window_size >= local_maximum_window_size / 2
+          return if remote_id.nil? || local_closed? || remote_closed?
+
+          growth = (GOOD_LOCAL_MAXIUMUM_WINDOW_SIZE - local_maximum_window_size).clamp(0, LOCAL_WINDOW_SIZE_INCREMENT)
+          adjust = @local_consumed + growth
+          connection.send_message(Buffer.from(:byte, CHANNEL_WINDOW_ADJUST, :long, remote_id, :long, adjust))
+          @local_window_size += adjust
+          @local_maximum_window_size += growth
+          @local_consumed = 0
+        end
+
         # Gets an +Array+ of local environment variables in the remote process'
         # environment.
         # A variable name can either be described by a +Regexp+ or +String+.

data/lib/net/ssh/connection/session.rb

@@ -24,7 +24,7 @@ module Net
       #     # 'ssh' is an instance of Net::SSH::Connection::Session
       #     ssh.exec! "/etc/init.d/some_process start"
       #   end
-      class Session
+      class Session # rubocop:disable Metrics/ClassLength
         include Loggable
         include Constants
 
@@ -74,13 +74,15 @@ module Net
           @channel_id_counter = -1
           @channels = Hash.new(NilChannel.new(self))
           @listeners = { transport.socket => nil }
+          @read_paused = {}
+          @draining = {}
           @pending_requests = []
           @channel_open_handlers = {}
           @on_global_request = {}
           @properties = (options[:properties] || {}).dup
 
           @max_pkt_size = (options.key?(:max_pkt_size) ? options[:max_pkt_size] : 0x8000)
-          @max_win_size = (options.key?(:max_win_size) ? options[:max_win_size] : 0x20000)
+          @max_win_size = (options.key?(:max_win_size) ? options[:max_win_size] : Channel::DEFAULT_MAX_WIN_SIZE)
 
           @keepalive = Keepalive.new(self)
 
@@ -120,12 +122,19 @@ module Net
         # connection.
         def close
           info { "closing remaining channels (#{channels.length} open)" }
-          channels.each { |id, channel| channel.close }
+
+          channels.each do |id, channel|
+            channel.eof!
+            channel.close
+          end
+
           begin
             loop(0.1) { channels.any? }
           rescue Net::SSH::Disconnect
             raise unless channels.empty?
           end
+          @draining.each_key(&:close)
+          @draining.clear
           transport.close
         end
 
@@ -146,10 +155,13 @@ module Net
         # a +true+ value for +include_invisible+, then those will be counted.
         #
         # This can be useful for determining whether the event loop should continue
-        # to be run.
+        # to be run. A forwarded socket still writing out data that arrived before
+        # its channel closed also counts as busy.
         #
         #   ssh.loop { ssh.busy? }
         def busy?(include_invisible = false)
+          return true if @draining.any?
+
           if include_invisible
             channels.any?
           else
@@ -255,7 +267,7 @@ module Net
         def ev_do_calculate_rw_wait(wait)
           r = listeners.keys
           w = r.select { |w2| w2.respond_to?(:pending_write?) && w2.pending_write? }
-          [r, w, io_select_wait(wait)]
+          [r.reject { |io| @read_paused.key?(io) }, w, io_select_wait(wait)]
         end
 
         # This is called internally as part of #process.
@@ -281,6 +293,7 @@ module Net
 
           Array(writers).each do |writer|
             writer.send_pending
+            close_if_drained(writer) if @draining.key?(writer)
           end
         end
 
@@ -416,7 +429,10 @@ module Net
         #
         #   matches = ssh.exec!("grep something /some/files")
         #
-        # the returned string has an exitstatus method to query its exit status
+        # With a block, output goes to the block and an empty string is returned,
+        # unless the block sets <tt>ch[:result]</tt>, which is returned instead.
+        # Either way the returned string has an exitstatus method to query the
+        # command's exit status.
         def exec!(command, status: nil, &block)
           block_or_concat = block || Proc.new do |ch, type, data|
             ch[:result] ||= String.new
@@ -427,10 +443,10 @@ module Net
           channel = exec(command, status: status, &block_or_concat)
           channel.wait
 
-          channel[:result] ||= String.new unless block
           channel[:result] &&= channel[:result].force_encoding("UTF-8") unless block
+          channel[:result] ||= String.new(encoding: Encoding::UTF_8)
 
-          StringWithExitstatus.new(channel[:result], status[:exit_code]) if channel[:result]
+          StringWithExitstatus.new(channel[:result], status[:exit_code])
         end
 
         # Enqueues a message to be sent to the server as soon as the socket is
@@ -485,9 +501,36 @@ module Net
         # Removes the given io object from the listeners collection, so that the
         # event loop will no longer monitor it.
         def stop_listening_to(io)
+          @read_paused.delete(io)
+          @draining.delete(io)
           listeners.delete(io)
         end
 
+        # Internal and experimental, used by port forwarding: stops the event
+        # loop from reading +io+ while it keeps flushing its pending output.
+        def pause_reading(io) # :nodoc:
+          raise ArgumentError, "the transport socket can't be paused" if io.equal?(transport.socket)
+
+          @read_paused[io] = true if listeners.key?(io)
+        end
+
+        def resume_reading(io) # :nodoc:
+          @read_paused.delete(io)
+        end
+
+        def reading_paused?(io) # :nodoc:
+          @read_paused.key?(io)
+        end
+
+        # Internal and experimental, used by port forwarding: stops reading +io+
+        # but keeps flushing its pending output, then closes it and stops
+        # listening, as OpenSSH's chan_rcvd_oclose does.
+        def close_when_drained(io) # :nodoc:
+          pause_reading(io)
+          @draining[io] = true
+          close_if_drained(io)
+        end
+
         # Returns a reference to the Net::SSH::Service::Forward service, which can
         # be used for forwarding ports over SSH.
         def forward
@@ -565,6 +608,13 @@ module Net
           end
         end
 
+        def close_if_drained(io)
+          return if io.pending_write? && !io.closed? && !closed? && listeners.key?(io)
+
+          io.close unless io.closed?
+          stop_listening_to(io)
+        end
+
         def channel_closed(channel)
           channel.remote_closed!
           channel.close

data/lib/net/ssh/errors.rb

@@ -19,6 +19,16 @@ module Net
     # timeouted unexpectedly.
     class Timeout < Disconnect; end
 
+    # This exception is raised when a key blob has a key type that Net::SSH
+    # cannot parse.
+    class UnsupportedKeyType < Net::SSH::Exception; end
+
+    # Raised when a session connected through an OpenSSH ControlMaster (see the
+    # :control_master option of Net::SSH.start) is asked for something the
+    # master's proxy mode cannot carry, such as remote port forwarding, agent
+    # or X11 forwarding, or other global requests.
+    class NotSupportedOverControlMaster < Net::SSH::Exception; end
+
     # This exception is primarily used internally, but if you have a channel
     # request handler (see Net::SSH::Connection::Channel#on_request) that you
     # want to fail in such a way that the server knows it failed, you can
@@ -62,19 +72,21 @@ module Net
         @data && @data[:fingerprint]
       end
 
-      # Returns the host name for the remote host, as reported by the socket.
+      # Returns the host name that was connected to.
       def host
-        @data && @data[:peer] && @data[:peer][:host]
+        session&.host
       end
 
-      # Returns the port number for the remote host, as reported by the socket.
+      # Returns the port number that was connected to.
       def port
-        @data && @data[:peer] && @data[:peer][:port]
+        session&.port
       end
 
-      # Returns the IP address of the remote host, as reported by the socket.
+      # Returns the IP address of the remote host, as reported by the socket,
+      # or nil when connected through a ProxyCommand.
       def ip
-        @data && @data[:peer] && @data[:peer][:ip]
+        peer_ip = session&.socket&.peer_ip
+        peer_ip unless peer_ip == Net::SSH::Transport::PacketStream::PROXY_COMMAND_HOST_IP
       end
 
       # Returns the key itself, as reported by the remote host.
@@ -87,6 +99,12 @@ module Net
       def remember_host!
         @callback.call
       end
+
+      private
+
+      def session
+        @data && @data[:session]
+      end
     end
 
     # Raised when the cached key for a particular host does not match the

data/lib/net/ssh/key_factory.rb

@@ -2,6 +2,7 @@ require 'net/ssh/transport/openssl'
 require 'net/ssh/prompt'
 
 require 'net/ssh/authentication/ed25519_loader'
+require 'net/ssh/authentication/security_key'
 
 module Net
   module SSH
@@ -92,7 +93,7 @@ module Net
           blob = nil
           begin
             blob = fields.shift
-          end while !blob.nil? && !/^(ssh-(rsa|dss|ed25519)|ecdsa-sha2-nistp\d+)(-cert-v01@openssh\.com)?$/.match(blob)
+          end while !blob.nil? && !public_key_type?(blob)
           blob = fields.shift
 
           raise Net::SSH::Exception, "public key at #{filename} is not valid" if blob.nil?
@@ -104,6 +105,11 @@ module Net
 
         private
 
+        def public_key_type?(type)
+          /^(ssh-(rsa|dss|ed25519)|ecdsa-sha2-nistp\d+)(-cert-v01@openssh\.com)?$/.match?(type) ||
+            Net::SSH::Authentication::SecurityKey.understands?(type)
+        end
+
         # rubocop:disable Style/Documentation, Lint/DuplicateMethods
         class KeyType
           def self.read(key_data, passphrase)

data/lib/net/ssh/known_hosts.rb

@@ -1,6 +1,5 @@
 require 'strscan'
 require 'openssl'
-require 'delegate'
 require 'net/ssh/buffer'
 require 'net/ssh/authentication/ed25519_loader'
 
@@ -8,8 +7,11 @@ module Net
   module SSH
     module HostKeyEntries
       # regular public key entry
-      class PubKey < Delegator
-        def initialize(key, comment: nil) # rubocop:disable Lint/MissingSuper
+      class PubKey
+        # The public key itself, e.g. an OpenSSL::PKey::PKey.
+        attr_reader :key
+
+        def initialize(key, comment: nil)
           @key = key
           @comment = comment
         end
@@ -26,14 +28,22 @@ module Net
           @key.to_blob
         end
 
-        def __getobj__
-          Kernel.warn("Calling Net::SSH::Buffer methods on HostKeyEntries PubKey is deprecated")
-          @key
-        end
-
         def matches_key?(server_key)
           @key.ssh_type == server_key.ssh_type && @key.to_blob == server_key.to_blob
         end
+
+        def method_missing(name, *args, &block)
+          return super unless @key.respond_to?(name)
+
+          raise NoMethodError.new(
+            "undefined method `#{name}' for #{self.class}; call it on #key, the wrapped public key",
+            name
+          )
+        end
+
+        def respond_to_missing?(_name, _include_private = false)
+          false
+        end
       end
 
       # @cert-authority entry