mongo 2.24.1 → 2.26.0

data/lib/mongo/server/connection_pool.rb

@@ -869,6 +869,22 @@ module Mongo
         end
       end
 
+      # Returns statistics about the internal state of the connection pool,
+      # primarily for testing and debugging purposes.
+      #
+      # @return [ Hash ] The state of the connection pool
+      #
+      # @api private
+      def state
+        @lock.synchronize do
+          { available_connections: @available_connections.length,
+            checked_out_connections: @checked_out_connections.length,
+            pending_connections: @pending_connections.length,
+            interrupt_connections: @interrupt_connections.length,
+            connection_requests: @connection_requests }
+        end
+      end
+
       private
 
       # Returns the next available connection, optionally with given
@@ -946,7 +962,7 @@ module Mongo
 
       # Removes and disconnects all stale available connections.
       def remove_stale_connection
-        return unless conn = @available_connections.detect { |c| connection_stale_unlocked?(c) }
+        return unless (conn = @available_connections.detect { |c| connection_stale_unlocked?(c) })
 
         conn.disconnect!(reason: :stale)
         @available_connections.delete(conn)
@@ -958,7 +974,7 @@ module Mongo
         return false if @interrupt_connections.empty?
 
         gens = Set.new
-        while conn = @interrupt_connections.pop
+        while (conn = @interrupt_connections.pop)
           if @checked_out_connections.include?(conn)
             # If the connection has been checked out, mark it as interrupted and it will
             # be disconnected on check in.
@@ -1042,7 +1058,11 @@ module Mongo
           @server.unknown!(
             generation: e.generation,
             service_id: e.service_id,
-            stop_push_monitor: true
+            stop_push_monitor: true,
+            # Only a network error makes the monitor connection suspect and
+            # requires cancelling its check; an auth failure does not.
+            network_error: e.is_a?(Mongo::Error::SocketError) ||
+              e.is_a?(Mongo::Error::SocketTimeoutError)
           )
         end
         connection.disconnect!(reason: :error)
@@ -1146,12 +1166,6 @@ module Mongo
         raise Error::ConnectionCheckOutTimeout.new(msg, address: @server.address)
       end
 
-      def raise_check_out_timeout_locked!(connection_global_id)
-        @lock.synchronize do
-          raise_check_out_timeout!(connection_global_id)
-        end
-      end
-
       def raise_if_pool_closed!
         return unless closed?
 
@@ -1246,7 +1260,7 @@ module Mongo
       # @raise [ Timeout::Error ] If the connection pool is at maximum size
       #   and remains so for longer than the wait timeout.
       def get_connection(pid, connection_global_id)
-        if connection = next_available_connection(connection_global_id)
+        if (connection = next_available_connection(connection_global_id))
           return nil unless valid_available_connection?(connection, pid, connection_global_id)
 
           # We've got a connection, so we decrement the number of connection

data/lib/mongo/server/description/features.rb

@@ -24,7 +24,6 @@ module Mongo
         # List of features and the wire protocol version they appear in.
         #
         # Wire protocol versions map to server releases as follows:
-        # -  8 => 4.2
         # -  9 => 4.4
         # - 13 => 5.0
         # - 14 => 5.1
@@ -61,7 +60,7 @@ module Mongo
         # The wire protocol versions that this version of the driver supports.
         #
         # @since 2.0.0
-        DRIVER_WIRE_VERSIONS = 8..29
+        DRIVER_WIRE_VERSIONS = 9..29
 
         # The wire protocol versions that are deprecated in this version of the
         # driver. Support for these versions will be removed in the future.

data/lib/mongo/server/description.rb

@@ -279,7 +279,7 @@ module Mongo
                   "The server at #{address.seed} did not provide a service id in handshake response"
           end
 
-          fake_service_id = if process_id = topology_version && topology_version['processId']
+          fake_service_id = if (process_id = topology_version && topology_version['processId'])
                               "process:#{process_id}"
                             else
                               "fake:#{rand((2**32) - 1) + 1}"
@@ -905,7 +905,6 @@ module Mongo
         '5.1' => 14,
         '5.0' => 12,
         '4.4' => 9,
-        '4.2' => 8,
       }.freeze
 
       # @api private

data/lib/mongo/server/monitor/connection.rb

@@ -54,9 +54,10 @@ module Mongo
         #
         # @since 2.0.0
         def initialize(address, options = {})
+          super()
           @address = address
           @options = options.dup.freeze
-          raise ArgumentError, 'App metadata is required' unless @app_metadata = options[:app_metadata]
+          raise ArgumentError, 'App metadata is required' unless (@app_metadata = options[:app_metadata])
 
           @socket = nil
           @pid = Process.pid
@@ -232,7 +233,7 @@ module Mongo
                 end
           # compressors must be set to maintain correct compression status
           # in the server description. See RUBY-2427
-          if compressors = options[:compressors]
+          if (compressors = options[:compressors])
             doc = doc.merge(compression: compressors)
           end
           doc

data/lib/mongo/server/monitor.rb

@@ -86,14 +86,23 @@ module Mongo
         @sdam_mutex = Mutex.new
         @next_earliest_scan = @next_wanted_scan = Time.now
         @update_mutex = Mutex.new
+        # Guards reads and writes of @connection so the polling connection can
+        # be cancelled from another thread. Per the Server Monitoring spec's
+        # cancelCheck pseudocode, the lock is only held long enough to copy or
+        # assign the reference - never across the blocking check.
+        @connection_lock = Mutex.new
       end
 
       # @return [ Server ] server The server that this monitor is monitoring.
       # @api private
       attr_reader :server
 
-      # @return [ Mongo::Server::Monitor::Connection ] connection The connection to use.
-      attr_reader :connection
+      # @return [ Mongo::Server::Monitor::Connection | nil ] The connection to
+      #   use, read under @connection_lock so callers never observe a stale
+      #   reference after a concurrent cancel_check! clears it.
+      def connection
+        @connection_lock.synchronize { @connection }
+      end
 
       # @return [ Hash ] options The server options.
       attr_reader :options
@@ -159,7 +168,11 @@ module Mongo
         end
       end
 
-      def create_push_monitor!(topology_version)
+      # @param [ Mongo::Server::Monitor::Connection ] connection The freshly
+      #   established monitoring connection. Passed explicitly (rather than read
+      #   from @connection) so a concurrent cancel_check! cannot nil it out from
+      #   under us between establishing it and building the PushMonitor.
+      def create_push_monitor!(topology_version, connection)
         @update_mutex.synchronize do
           @push_monitor = nil if @push_monitor && !@push_monitor.running?
 
@@ -170,7 +183,7 @@ module Mongo
             **Utils.shallow_symbolize_keys(options.merge(
                                              socket_timeout: heartbeat_interval + connection.socket_timeout,
                                              app_metadata: options[:push_monitor_app_metadata],
-                                             check_document: @connection.check_document
+                                             check_document: connection.check_document
                                            ))
           )
         end
@@ -185,6 +198,31 @@ module Mongo
         end
       end
 
+      # Cancel the in-progress check and close the monitoring connection.
+      #
+      # Called when the server is marked Unknown from a network error, per the
+      # Server Monitoring spec ("hello or legacy hello Cancellation"). Stops the
+      # streaming PushMonitor (interrupting its awaited hello read) and closes
+      # the polling connection, so the next check must establish a fresh one
+      # rather than re-validating the server over a possibly-dead socket.
+      #
+      # @api private
+      def cancel_check!
+        stop_push_monitor!
+
+        # Copy the connection reference under the lock, then interrupt and close
+        # it outside the lock. Closing the socket interrupts any in-progress
+        # read on the monitor thread; nil-ing the reference forces the next
+        # check to reconnect. The monitor thread is the only writer of a new
+        # connection, so it is safe for this thread to clear it.
+        connection = @connection_lock.synchronize do
+          conn = @connection
+          @connection = nil
+          conn
+        end
+        connection&.disconnect!
+      end
+
       # Perform a check of the server with throttling, and update
       # the server's description and average round trip time.
       #
@@ -210,28 +248,41 @@ module Mongo
         @mutex.synchronize do
           throttle_scan_frequency!
 
+          # When the streaming protocol is active the PushMonitor is the
+          # authoritative SDAM source and this scan only measures RTT on the
+          # dedicated connection. Per the Server Monitoring spec, an RTT
+          # command MUST NOT publish events or update the topology. Compute
+          # this before do_scan, which may (re)connect and change the state.
+          rtt_only = rtt_measurement_only?
+
           begin
-            result = do_scan
+            result = do_scan(publish_heartbeat: !rtt_only)
           rescue StandardError => e
-            run_sdam_flow({}, scan_error: e)
+            run_sdam_flow({}, scan_error: e, rtt_only: rtt_only)
           else
-            run_sdam_flow(result)
+            run_sdam_flow(result, rtt_only: rtt_only)
           end
         end
       end
 
-      def run_sdam_flow(result, awaited: false, scan_error: nil)
+      def run_sdam_flow(result, awaited: false, scan_error: nil, rtt_only: false)
         @sdam_mutex.synchronize do
           old_description = server.description
 
-          new_description = Description.new(
-            server.address,
-            result,
-            average_round_trip_time: server.round_trip_time_calculator.average_round_trip_time,
-            minimum_round_trip_time: server.round_trip_time_calculator.minimum_round_trip_time
-          )
-
-          server.cluster.run_sdam_flow(server.description, new_description, awaited: awaited, scan_error: scan_error)
+          # An RTT-only measurement (streaming protocol active) must not update
+          # the topology or publish SDAM events. The RTT it gathered is
+          # incorporated into the next streaming-hello description via the
+          # shared RTT calculator. The scheduling below still runs so the
+          # monitor keeps pacing its checks.
+          unless rtt_only
+            new_description = Description.new(
+              server.address,
+              result,
+              average_round_trip_time: server.round_trip_time_calculator.average_round_trip_time,
+              minimum_round_trip_time: server.round_trip_time_calculator.minimum_round_trip_time
+            )
+            server.cluster.run_sdam_flow(server.description, new_description, awaited: awaited, scan_error: scan_error)
+          end
 
           server.description.tap do |new_description|
             unless awaited
@@ -272,8 +323,12 @@ module Mongo
         server.scan_semaphore.signal
       end
 
-      def do_scan
-        monitoring.publish_heartbeat(server) do
+      def do_scan(publish_heartbeat: true)
+        if publish_heartbeat
+          monitoring.publish_heartbeat(server) do
+            check
+          end
+        else
           check
         end
       rescue StandardError => e
@@ -285,24 +340,53 @@ module Mongo
         raise e
       end
 
+      # Returns whether this scan is only an RTT measurement, which is the case
+      # when the streaming protocol is active: a dedicated connection is already
+      # established and the PushMonitor is running as the authoritative SDAM
+      # source. In the polling protocol there is no running PushMonitor, so the
+      # connection-reuse check is a real server check and not RTT-only.
+      #
+      # @return [ true | false ]
+      def rtt_measurement_only?
+        return false if connection.nil?
+
+        # Only suppress the check while the server is in a known state and the
+        # PushMonitor is the authoritative streaming source. If the server is
+        # Unknown (e.g. an operation error or a streaming failure just marked
+        # it so), the polling Monitor must run a full check to recover it
+        # rather than waiting for the next streaming response - otherwise the
+        # server can stay Unknown long enough to fail server selection.
+        return false if server.unknown?
+
+        pm = push_monitor
+        !pm.nil? && pm.running?
+      end
+
       def check
-        if @connection && @connection.pid != Process.pid
-          log_warn("Detected PID change - Mongo client should have been reconnected (old pid #{@connection.pid}, new pid #{Process.pid}")
-          @connection.disconnect!
-          @connection = nil
+        # Snapshot the connection under the lock. A concurrent cancel_check!
+        # may nil @connection from another thread; working on a local copy keeps
+        # this check consistent, and the guarded writeback below never clobbers
+        # a connection the monitor thread did not itself establish.
+        connection = self.connection
+
+        if connection && connection.pid != Process.pid
+          log_warn("Detected PID change - Mongo client should have been reconnected (old pid #{connection.pid}, new pid #{Process.pid}")
+          connection.disconnect!
+          clear_connection(connection)
+          connection = nil
         end
 
-        if @connection
+        if connection
           result = server.round_trip_time_calculator.measure do
-            doc = @connection.check_document
+            doc = connection.check_document
             cmd = Protocol::Query.new(
               Database::ADMIN, Database::COMMAND, doc, limit: -1
             )
-            message = @connection.dispatch_bytes(cmd.serialize.to_s)
+            message = connection.dispatch_bytes(cmd.serialize.to_s)
             message.documents.first
           rescue Mongo::Error
-            @connection.disconnect!
-            @connection = nil
+            connection.disconnect!
+            clear_connection(connection)
             raise
           end
         else
@@ -311,10 +395,13 @@ module Mongo
           result = server.round_trip_time_calculator.measure do
             connection.handshake!
           end
-          @connection = connection
-          if tv_doc = result['topologyVersion']
+          store_connection(connection)
+          if (tv_doc = result['topologyVersion'])
             if streaming_enabled?
-              create_push_monitor!(TopologyVersion.new(tv_doc))
+              # Run the instance we just created rather than re-reading the
+              # push_monitor getter: a concurrent cancel_check! may have nil'd
+              # @push_monitor between the two calls.
+              push_monitor = create_push_monitor!(TopologyVersion.new(tv_doc), connection)
               push_monitor.run!
             else
               stop_push_monitor!
@@ -328,6 +415,22 @@ module Mongo
         result
       end
 
+      # Store a freshly established monitoring connection.
+      def store_connection(connection)
+        @connection_lock.synchronize do
+          @connection = connection
+        end
+      end
+
+      # Clear the monitoring connection, but only if it is still the one passed
+      # in. A concurrent cancel_check! may have already cleared or replaced it,
+      # in which case we must leave the current connection alone.
+      def clear_connection(connection)
+        @connection_lock.synchronize do
+          @connection = nil if @connection.equal?(connection)
+        end
+      end
+
       # @note If the system clock is set to a time in the past, this method
       #   can sleep for a very long time.
       def throttle_scan_frequency!

data/lib/mongo/server/pending_connection.rb

@@ -23,6 +23,7 @@ module Mongo
       extend Forwardable
 
       def initialize(socket, server, monitoring, options = {})
+        super()
         @socket = socket
         @options = options
         @server = server
@@ -51,11 +52,6 @@ module Mongo
           # SCRAM-SHA-1. The use of SCRAM-SHA-256 by default is thus
           # sensible, and it is also mandated by the speculative auth spec.
           #
-          # If no mechanism was specified and we are talking to a 4.2
-          # server, we'll send speculative auth document, the server will
-          # ignore it and we'll perform authentication using explicit
-          # command.
-          #
           # If no mechanism was specified and we are talking to a 4.4+
           # server and the user account doesn't allow SCRAM-SHA-256, we will
           # authenticate in a separate command with SCRAM-SHA-1 after

data/lib/mongo/server/push_monitor.rb

@@ -72,7 +72,7 @@ module Mongo
             # Interrupt any in-progress exhausted hello reads by
             # disconnecting the connection.
             begin
-              @connection.send(:socket).close
+              @connection.interrupt_socket
             rescue StandardError
               nil
             end
@@ -118,6 +118,13 @@ module Mongo
                                 log_prefix: options[:log_prefix],
                                 bg_error_backtrace: options[:bg_error_backtrace])
 
+        # The streaming check is the authoritative SDAM source, so its failure
+        # must mark the server Unknown and clear the pool here. While streaming
+        # is active the polling Monitor only measures RTT and ignores errors
+        # (per the Server Monitoring spec), so it can no longer be relied on to
+        # notice the failure.
+        monitor.run_sdam_flow({}, scan_error: e, awaited: true)
+
         # If a request failed on a connection, stop push monitoring.
         # In case the server is dead we don't want to have two connections
         # trying to connect unsuccessfully at the same time.
@@ -142,6 +149,13 @@ module Mongo
             @server_pushing = false
             connection = PushMonitor::Connection.new(server.address, options)
             connection.connect!
+            # Identify the streaming connection to the server with a metadata
+            # handshake (carrying the appName) before streaming, mirroring the
+            # polling Monitor and the Server Monitoring spec, which establishes
+            # the monitoring connection with a handshake before issuing the
+            # awaitable hello. Without this, appName-scoped server behaviour
+            # never applies to the streaming hello.
+            connection.handshake!
             @connection = connection
           end
         end
@@ -174,7 +188,7 @@ module Mongo
       end
 
       def read_response
-        if timeout = options[:connect_timeout]
+        if (timeout = options[:connect_timeout])
           if timeout < 0
             raise Mongo::SocketTimeoutError, 'Requested to read with a negative timeout: '
           elsif timeout > 0

data/lib/mongo/server.rb

@@ -489,7 +489,8 @@ module Mongo
         unknown!(
           generation: e.generation,
           service_id: e.service_id,
-          stop_push_monitor: true
+          stop_push_monitor: true,
+          network_error: true
         )
       end
       raise
@@ -514,11 +515,13 @@ module Mongo
       raise
     rescue Mongo::Error::SocketError, Auth::Unauthorized => e
       # non-timeout network error or auth error, clear the pool and mark the
-      # topology as unknown
+      # topology as unknown. Only a network error makes the monitor connection
+      # suspect; a pure authentication failure must not close it.
       unknown!(
         generation: e.generation,
         service_id: e.service_id,
-        stop_push_monitor: true
+        stop_push_monitor: true,
+        network_error: e.is_a?(Mongo::Error::SocketError)
       )
       raise
     end
@@ -564,12 +567,18 @@ module Mongo
     # @option options [ true | false ] :keep_connection_pool Usually when the
     #   new server description is unknown, the connection pool on the
     #   respective server is cleared. Set this option to true to keep the
-    #   existing connection pool (required when handling not master errors
-    #   on 4.2+ servers).
+    #   existing connection pool (required when handling not master errors).
     # @option options [ TopologyVersion ] :topology_version Topology version
     #   of the error response that is causing the server to be marked unknown.
-    # @option options [ true | false ] :stop_push_monitor Whether to stop
-    #   the PushMonitor associated with the server, if any.
+    # @option options [ true | false ] :stop_push_monitor Set when the server
+    #   is marked Unknown from a connection-establishment error (e.g. an
+    #   authentication failure). Stops the streaming PushMonitor associated with
+    #   the server, if any.
+    # @option options [ true | false ] :network_error Set when the server is
+    #   marked Unknown from a network error. In addition to stopping the
+    #   PushMonitor, cancels the monitor's in-progress check and closes its
+    #   connection so the next check must establish a fresh one, per the Server
+    #   Monitoring spec.
     # @option options [ Object ] :service_id Discard state for the specified
     #   service id only.
     #
@@ -590,7 +599,7 @@ module Mongo
         # for example if there haven't been any successful connections yet to
         # this server, but the server can still be marked unknown if one
         # of such connections failed midway through its establishment.
-        if service_id = options[:service_id]
+        if (service_id = options[:service_id])
           pool&.disconnect!(service_id: service_id)
         end
         return
@@ -605,7 +614,17 @@ module Mongo
         return
       end
 
-      monitor&.stop_push_monitor! if options[:stop_push_monitor]
+      if options[:network_error]
+        # A network error implies every connection to the server is suspect
+        # (Server Monitoring spec, "hello or legacy hello Cancellation"): cancel
+        # the monitor's in-progress check and close its connection so the next
+        # check must establish a fresh one.
+        monitor&.cancel_check!
+      elsif options[:stop_push_monitor]
+        # Non-network errors (e.g. an authentication failure) only tear down the
+        # streaming PushMonitor; the monitor's connection is not suspect.
+        monitor&.stop_push_monitor!
+      end
 
       # SDAM flow will update description on the server without in-place
       # mutations and invoke SDAM transitions as needed.

data/lib/mongo/server_selector/base.rb

@@ -239,7 +239,7 @@ module Mongo
 
           session.unpin unless session.in_transaction?
 
-          if server = session.pinned_server
+          if (server = session.pinned_server)
             # Here we assume that a mongos stays in the topology indefinitely.
             # This will no longer be the case once SRV polling is implemented.
 
@@ -398,9 +398,7 @@ module Mongo
       # @api private
       def candidates(cluster, deprioritized = [])
         servers = cluster.servers.reject { |s| deprioritized.include?(s) }
-        if cluster.single?
-          servers
-        elsif cluster.sharded?
+        if cluster.single? || cluster.sharded?
           servers
         elsif cluster.replica_set?
           select_in_replica_set(servers)
@@ -689,8 +687,7 @@ module Mongo
         msg = ''
         dead_monitors = []
         cluster.servers_list.each do |server|
-          thread = server.monitor.instance_variable_get(:@thread)
-          dead_monitors << server if thread.nil? || !thread.alive?
+          dead_monitors << server unless server.monitor&.running?
         end
         if dead_monitors.any?
           msg += ". The following servers have dead monitor threads: #{dead_monitors.map(&:summary).join(', ')}"

data/lib/mongo/server_selector/secondary_preferred.rb

@@ -101,7 +101,10 @@ module Mongo
       #
       # @since 2.0.0
       def select_in_replica_set(candidates)
-        near_servers(secondaries(candidates)) + primary(candidates)
+        eligible = secondaries(candidates)
+        return near_servers(eligible) unless eligible.empty?
+
+        primary(candidates)
       end
 
       def max_staleness_allowed?