mongo 2.16.4 → 2.20.0

data/lib/mongo/server/description/features.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 # Copyright (C) 2014-2020 MongoDB Inc.
 #
@@ -35,15 +35,20 @@ module Mongo
         # -  8 => 4.2
         # -  9 => 4.4
         # - 13 => 5.0
+        # - 14 => 5.1
+        # - 17 => 6.0
         #
         # @since 2.0.0
         MAPPINGS = {
+          merge_out_on_secondary: 13,
+          get_more_comment: 9,
           retryable_write_error_label: 9,
           commit_quorum: 9,
           # Server versions older than 4.2 do not reliably validate options
           # provided by the client during findAndModify operations, requiring the
           # driver to raise client-side errors when those options are provided.
           find_and_modify_option_validation: 8,
+          sharded_transactions: 8,
           transactions: 7,
           scram_sha_256: 7,
           array_filters: 6,
@@ -78,7 +83,7 @@ module Mongo
         # The wire protocol versions that this version of the driver supports.
         #
         # @since 2.0.0
-        DRIVER_WIRE_VERSIONS = (2..13).freeze
+        DRIVER_WIRE_VERSIONS = (6..21).freeze
 
         # Create the methods for each mapping to tell if they are supported.
         #

data/lib/mongo/server/description/load_balancer.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 # Copyright (C) 2021 MongoDB Inc.
 #

data/lib/mongo/server/description.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 # Copyright (C) 2014-2020 MongoDB Inc.
 #
@@ -234,8 +234,7 @@ module Mongo
           # to work if the server it communicates with does not set serviceId
           # in ismaster/hello response.
           #
-          # In practice, there are currently no server version that actually
-          # sets this field.
+          # At the moment we cannot run a proper load balancer setup on evergreen
           #
           # Therefore, when connect=:load_balanced Ruby option is used instead
           # of the loadBalanced=true URI option, if serviceId is not set in
@@ -255,6 +254,8 @@ module Mongo
           # service ids are strings, to distinguish them from the real ones.
           # In particular processId is also a BSON::ObjectId, but will be
           # mapped to a string for clarity that this is a fake service id.
+          #
+          # TODO: Remove this when https://jira.mongodb.org/browse/RUBY-2881 is done.
           if ok? && !service_id
             unless force_load_balancer
               raise Error::MissingServiceId, "The server at #{address.seed} did not provide a service id in handshake response"
@@ -298,11 +299,6 @@ module Mongo
         @features
       end
 
-      # @return [ nil | Object ] The service id, if any.
-      def service_id
-        config['serviceId']
-      end
-
       # @return [ Float ] The moving average time the hello call took to complete.
       attr_reader :average_round_trip_time
 
@@ -843,6 +839,8 @@ module Mongo
         config['connectionId']
       end
 
+      # @return [ nil | Object ] The service id, if any.
+      #
       # @api experimental
       def service_id
         config['serviceId']
@@ -871,6 +869,14 @@ module Mongo
       # @api private
       def server_version_gte?(version)
         required_wv = case version
+          when '7.0'
+            21
+          when '6.0'
+            17
+          when '5.2'
+            15
+          when '5.1'
+            14
           when '5.0'
             12
           when '4.4'

data/lib/mongo/server/monitor/app_metadata.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 # Copyright (C) 2018-2020 MongoDB Inc.
 

data/lib/mongo/server/monitor/connection.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 # Copyright (C) 2015-2020 MongoDB Inc.
 #
@@ -86,6 +86,7 @@ module Mongo
           options[:connect_timeout] || Server::CONNECT_TIMEOUT
         end
 
+        # @return [ Integer ] server_connection_id The server connection id.
         attr_reader :server_connection_id
 
         # Sends a message and returns the result.

data/lib/mongo/server/monitor.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 # Copyright (C) 2014-2020 MongoDB Inc.
 #
@@ -170,6 +170,25 @@ module Mongo
         end
       end
 
+      def create_push_monitor!(topology_version)
+        @update_mutex.synchronize do
+          if @push_monitor && !@push_monitor.running?
+            @push_monitor = nil
+          end
+
+          @push_monitor ||= PushMonitor.new(
+            self,
+            topology_version,
+            monitoring,
+            **Utils.shallow_symbolize_keys(options.merge(
+              socket_timeout: heartbeat_interval + connection.socket_timeout,
+              app_metadata: options[:push_monitor_app_metadata],
+              check_document: @connection.check_document
+            )),
+          )
+        end
+      end
+
       def stop_push_monitor!
         @update_mutex.synchronize do
           if @push_monitor
@@ -204,13 +223,17 @@ module Mongo
         @mutex.synchronize do
           throttle_scan_frequency!
 
-          result = do_scan
-
-          run_sdam_flow(result)
+          begin
+            result = do_scan
+          rescue => e
+            run_sdam_flow({}, scan_error: e)
+          else
+            run_sdam_flow(result)
+          end
         end
       end
 
-      def run_sdam_flow(result, awaited: false)
+      def run_sdam_flow(result, awaited: false, scan_error: nil)
         @sdam_mutex.synchronize do
           old_description = server.description
 
@@ -218,7 +241,7 @@ module Mongo
             average_round_trip_time: server.round_trip_time_averager.average_round_trip_time
           )
 
-          server.cluster.run_sdam_flow(server.description, new_description, awaited: awaited)
+          server.cluster.run_sdam_flow(server.description, new_description, awaited: awaited, scan_error: scan_error)
 
           server.description.tap do |new_description|
             unless awaited
@@ -249,6 +272,10 @@ module Mongo
         end
       end
 
+      def to_s
+        "#<#{self.class.name}:#{object_id} #{server.address}>"
+      end
+
       private
 
       def pre_stop
@@ -267,7 +294,7 @@ module Mongo
             log_prefix: options[:log_prefix],
             bg_error_backtrace: options[:bg_error_backtrace],
           )
-          {}
+          raise exc
         end
       end
 
@@ -300,20 +327,9 @@ module Mongo
             connection.handshake!
           end
           @connection = connection
-          if result['topologyVersion']
+          if tv_doc = result['topologyVersion']
             # Successful response, server 4.4+
-            @update_mutex.synchronize do
-              @push_monitor ||= PushMonitor.new(
-                self,
-                TopologyVersion.new(result['topologyVersion']),
-                monitoring,
-                **Utils.shallow_symbolize_keys(options.merge(
-                  socket_timeout: heartbeat_interval + connection.socket_timeout,
-                  app_metadata: options[:push_monitor_app_metadata],
-                  check_document: @connection.check_document
-                )),
-              )
-            end
+            create_push_monitor!(TopologyVersion.new(tv_doc))
             push_monitor.run!
           else
             # Failed response or pre-4.4 server

data/lib/mongo/server/pending_connection.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 # Copyright (C) 2019-2020 MongoDB Inc.
 #

data/lib/mongo/server/push_monitor/connection.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 # Copyright (C) 2020 MongoDB Inc.
 #

data/lib/mongo/server/push_monitor.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 # Copyright (C) 2020 MongoDB Inc.
 #
@@ -78,7 +78,7 @@ module Mongo
           if @connection
             # Interrupt any in-progress exhausted hello reads by
             # disconnecting the connection.
-            @connection.send(:socket).close
+            @connection.send(:socket).close rescue nil
           end
         end
         super.tap do
@@ -124,8 +124,14 @@ module Mongo
           bg_error_backtrace: options[:bg_error_backtrace],
         )
 
-        # Avoid tight looping in push monitor - see RUBY-2806.
-        sleep(0.5)
+        # If a request failed on a connection, stop push monitoring.
+        # In case the server is dead we don't want to have two connections
+        # trying to connect unsuccessfully at the same time.
+        stop!
+
+        # Request an immediate check on the monitor to get reinstated as
+        # soon as possible in case the server is actually alive.
+        server.scan_semaphore.signal
       end
 
       def check
@@ -190,6 +196,10 @@ module Mongo
         end
       end
 
+      def to_s
+        "#<#{self.class.name}:#{object_id} #{server.address}>"
+      end
+
     end
   end
 end

data/lib/mongo/server/round_trip_time_averager.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 # Copyright (C) 2018-2020 MongoDB Inc.
 

data/lib/mongo/server.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 # Copyright (C) 2014-2020 MongoDB Inc.
 #
@@ -54,6 +54,15 @@ module Mongo
     #   done by this server. Note: setting this option to false will make
     #   the server non-functional. It is intended for use in tests which
     #   manually invoke SDAM state transitions.
+    # @option options [ true, false ] :populator_io For internal driver
+    #   use only. Set to false to prevent the populator threads from being
+    #   created and started in the server's connection pool. It is intended
+    #   for use in tests that also turn off monitoring_io, unless the populator
+    #   is explicitly needed. If monitoring_io is off, but the populator_io
+    #   is on, the populator needs to be manually closed at the end of the
+    #   test, since a cluster without monitoring is considered not connected,
+    #   and thus will not clean up the connection pool populator threads on
+    #   close.
     # @option options [ true | false ] :load_balancer Whether this server
     #   is a load balancer.
     # @option options [ String ] :connect The client connection mode.
@@ -267,25 +276,41 @@ module Mongo
       if monitor
         monitor.stop!
       end
-      _pool = @pool_lock.synchronize do
-        @pool
-      end
-      if _pool
-        # For backwards compatibility we disconnect/clear the pool rather
-        # than close it here. We also stop the populator which allows the
-        # the pool to continue providing connections but stops it from
-        # connecting in background on clients/servers that are in fact
-        # intended to be closed and no longer used.
-        begin
-          _pool.disconnect!(stop_populator: true)
-        rescue Error::PoolClosedError
-          # If the pool was already closed, we don't need to do anything here.
-        end
-      end
+
       @connected = false
+
+      # The current CMAP spec requires a pool to be mostly unusable
+      # if its server is unknown (or, therefore, disconnected).
+      # However any outstanding operations should continue to completion,
+      # and their connections need to be checked into the pool to be
+      # torn down. Because of this cleanup requirement we cannot just
+      # close the pool and set it to nil here, to be recreated the next
+      # time the server is discovered.
+      pool_internal&.clear
+
       true
     end
 
+    def close
+      if monitor
+        monitor.stop!
+      end
+
+      @connected = false
+
+      _pool = nil
+      @pool_lock.synchronize do
+        _pool, @pool = @pool, nil
+      end
+
+      # TODO: change this to _pool.close in RUBY-3174.
+      # Clear the pool. If the server is not unknown then the
+      # pool will stay ready. Stop the background populator thread.
+      _pool&.close(stay_ready: true)
+
+      nil
+    end
+
     # Whether the server is connected.
     #
     # @return [ true|false ] Whether the server is connected.
@@ -397,8 +422,29 @@ module Mongo
     #
     # @since 2.0.0
     def pool
+      if unknown?
+        raise Error::ServerNotUsable, address
+      end
+
       @pool_lock.synchronize do
-        @pool ||= ConnectionPool.new(self, options)
+        opts = connected? ? options : options.merge(populator_io: false)
+        @pool ||= ConnectionPool.new(self, opts).tap do |pool|
+          pool.ready
+        end
+      end
+    end
+
+    # Internal driver method to retrieve the connection pool for this server.
+    #
+    # Unlike +pool+, +pool_internal+ will not create a pool if one does not
+    # already exist.
+    #
+    # @return [ Server::ConnectionPool | nil ] The connection pool, if one exists.
+    #
+    # @api private
+    def pool_internal
+      @pool_lock.synchronize do
+        @pool
       end
     end
 
@@ -444,8 +490,8 @@ module Mongo
     # @return [ Object ] The result of the block execution.
     #
     # @since 2.3.0
-    def with_connection(service_id: nil, &block)
-      pool.with_connection(service_id: service_id, &block)
+    def with_connection(connection_global_id: nil, &block)
+      pool.with_connection(connection_global_id: connection_global_id, &block)
     end
 
     # Handle handshake failure.
@@ -480,18 +526,15 @@ module Mongo
     rescue Mongo::Error::SocketTimeoutError
       # possibly cluster is slow, do not give up on it
       raise
-    rescue Mongo::Error::SocketError => e
-      # non-timeout network error
+    rescue Mongo::Error::SocketError, Auth::Unauthorized => e
+      # non-timeout network error or auth error, clear the pool and mark the
+      # topology as unknown
       unknown!(
         generation: e.generation,
         service_id: e.service_id,
         stop_push_monitor: true,
       )
       raise
-    rescue Auth::Unauthorized
-      # auth error, keep server description and topology as they are
-      pool.disconnect!
-      raise
     end
 
     # Whether the server supports modern read retries.
@@ -511,9 +554,15 @@ module Mongo
     # @note Retryable writes are only available on server versions 3.6+ and with
     #   sharded clusters or replica sets.
     #
+    # @note Some of the conditions in this method automatically return false for
+    #       for load balanced topologies. The conditions in this method should
+    #       always be true, since load-balanced topologies are only available on
+    #       MongoDB 5.0+, and not for standalone topologies. Therefore, we can
+    #       assume that retry writes are enabled.
+    #
     # @since 2.5.0
     def retry_writes?
-      !!(features.sessions_enabled? && logical_session_timeout && !standalone?)
+      !!(features.sessions_enabled? && logical_session_timeout && !standalone?) || load_balancer?
     end
 
     # Marks server unknown and publishes the associated SDAM event
@@ -532,8 +581,6 @@ module Mongo
     #   respective server is cleared. Set this option to true to keep the
     #   existing connection pool (required when handling not master errors
     #   on 4.2+ servers).
-    # @option options [ Object ] :service_id Discard state for the specified
-    #   service id only.
     # @option options [ TopologyVersion ] :topology_version Topology version
     #   of the error response that is causing the server to be marked unknown.
     # @option options [ true | false ] :stop_push_monitor Whether to stop
@@ -543,6 +590,8 @@ module Mongo
     #
     # @since 2.4.0, SDAM events are sent as of version 2.7.0
     def unknown!(options = {})
+      pool = pool_internal
+
       if load_balancer?
         # When the client is in load-balanced topology, servers (the one and
         # only that can be) starts out as a load balancer and stays as a
@@ -551,15 +600,18 @@ module Mongo
         # However, this method also clears connection pool for the server
         # when the latter is marked unknown, and this part needs to happen
         # when the server is a load balancer.
+        #
+        # It is possible for a load balancer server to not have a service id,
+        # for example if there haven't been any successful connections yet to
+        # this server, but the server can still be marked unknown if one
+        # of such connections failed midway through its establishment.
         if service_id = options[:service_id]
-          pool.disconnect!(service_id: service_id)
-        elsif Lint.enabled?
-          raise Error::LintError, 'Load balancer was asked to be marked unknown without a service id'
+          pool&.disconnect!(service_id: service_id)
         end
         return
       end
 
-      if options[:generation] && options[:generation] < pool.generation
+      if options[:generation] && options[:generation] < pool&.generation
         return
       end
 
@@ -591,17 +643,35 @@ module Mongo
 
     # @api private
     def update_description(description)
+      pool = pool_internal
+      if pool && !description.unknown?
+        pool.ready
+      end
       @description = description
     end
 
+    # Clear the servers description so that it is considered unknown and can be
+    # safely disconnected.
+    #
+    # @api private
+    def clear_description
+      @description = Mongo::Server::Description.new(address, {})
+    end
+
     # @param [ Object ] :service_id Close connections with the specified
     #   service id only.
+    # @param [ true | false ] :interrupt_in_use_connections Whether or not the
+    #   cleared connections should be interrupted as well.
     #
     # @api private
-    def clear_connection_pool(service_id: nil)
+    def clear_connection_pool(service_id: nil, interrupt_in_use_connections: false)
       @pool_lock.synchronize do
-        if @pool
-          @pool.disconnect!(service_id: service_id)
+        # A server being marked unknown after it is closed is technically
+        # incorrect but it does not meaningfully alter any state.
+        # Because historically the driver permitted servers to be marked
+        # unknown at any time, continue doing so even if the pool is closed.
+        if @pool && !@pool.closed?
+          @pool.disconnect!(service_id: service_id, interrupt_in_use_connections: interrupt_in_use_connections)
         end
       end
     end

data/lib/mongo/server_selector/base.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 # Copyright (C) 2020 MongoDB Inc.
 #
@@ -162,6 +162,12 @@ module Mongo
       #   Deprecated and ignored.
       # @param [ Session | nil ] session Optional session to take into account
       #   for mongos pinning. Added in version 2.10.0.
+      # @param [ true | false ] write_aggregation Whether we need a server that
+      #   supports writing aggregations (e.g. with $merge/$out) on secondaries.
+      # @param [ Array<Server> ] deprioritized A list of servers that should
+      #   be selected from only if no other servers are available. This is
+      #   used to avoid selecting the same server twice in a row when
+      #   retrying a command.
       #
       # @return [ Mongo::Server ] A server matching the server preference.
       #
@@ -172,7 +178,16 @@ module Mongo
       #   lint mode is enabled.
       #
       # @since 2.0.0
-      def select_server(cluster, ping = nil, session = nil)
+      def select_server(cluster, ping = nil, session = nil, write_aggregation: false, deprioritized: [])
+        select_server_impl(cluster, ping, session, write_aggregation, deprioritized).tap do |server|
+          if Lint.enabled? && !server.pool.ready?
+            raise Error::LintError, 'Server selector returning a server with a pool which is not ready'
+          end
+        end
+      end
+
+      # Parameters and return values are the same as for select_server.
+      private def select_server_impl(cluster, ping, session, write_aggregation, deprioritized)
         if cluster.topology.is_a?(Cluster::Topology::LoadBalanced)
           return cluster.servers.first
         end
@@ -243,7 +258,19 @@ module Mongo
 =end
 
         loop do
-          server = try_select_server(cluster)
+          if Lint.enabled?
+            cluster.servers.each do |server|
+              # TODO: Add this back in RUBY-3174.
+              # if !server.unknown? && !server.connected?
+              #   raise Error::LintError, "Server #{server.summary} is known but is not connected"
+              # end
+              if !server.unknown? && !server.pool.ready?
+                raise Error::LintError, "Server #{server.summary} is known but has non-ready pool"
+              end
+            end
+          end
+
+          server = try_select_server(cluster, write_aggregation: write_aggregation, deprioritized: deprioritized)
 
           if server
             unless cluster.topology.compatible?
@@ -294,17 +321,41 @@ module Mongo
       # Tries to find a suitable server, returns the server if one is available
       # or nil if there isn't a suitable server.
       #
+      # @param [ Mongo::Cluster ] cluster The cluster from which to select
+      #   an eligible server.
+      # @param [ true | false ] write_aggregation Whether we need a server that
+      #   supports writing aggregations (e.g. with $merge/$out) on secondaries.
+      # @param [ Array<Server> ] deprioritized A list of servers that should
+      #   be selected from only if no other servers are available. This is
+      #   used to avoid selecting the same server twice in a row when
+      #   retrying a command.
+      #
       # @return [ Server | nil ] A suitable server, if one exists.
       #
       # @api private
-      def try_select_server(cluster)
-        servers = suitable_servers(cluster)
+      def try_select_server(cluster, write_aggregation: false, deprioritized: [])
+        servers = if write_aggregation && cluster.replica_set?
+          # 1. Check if ALL servers in cluster support secondary writes.
+          is_write_supported = cluster.servers.reduce(true) do |res, server|
+            res && server.features.merge_out_on_secondary_enabled?
+          end
+
+          if is_write_supported
+            # 2. If all servers support secondary writes, we respect read preference.
+            suitable_servers(cluster)
+          else
+            # 3. Otherwise we fallback to primary for replica set.
+            [cluster.servers.detect(&:primary?)]
+          end
+        else
+          suitable_servers(cluster)
+        end
 
         # This list of servers may be ordered in a specific way
         # by the selector (e.g. for secondary preferred, the first
         # server may be a secondary and the second server may be primary)
         # and we should take the first server here respecting the order
-        server = servers.first
+        server = suitable_server(servers, deprioritized)
 
         if server
           if Lint.enabled?
@@ -375,6 +426,24 @@ module Mongo
 
       private
 
+      # Returns a server from the list of servers that is suitable for
+      # executing the operation.
+      #
+      # @param [ Array<Server> ] servers The candidate servers.
+      # @param [ Array<Server> ] deprioritized A list of servers that should
+      #  be selected from only if no other servers are available.
+      #
+      # @return [ Server | nil ] The suitable server or nil if no suitable
+      #  server is available.
+      def suitable_server(servers, deprioritized)
+        preferred = servers - deprioritized
+        if preferred.empty?
+          servers.first
+        else
+          preferred.first
+        end
+      end
+
       # Convert this server preference definition into a format appropriate
       #   for sending to a MongoDB server (i.e., as a command field).
       #
@@ -582,9 +651,9 @@ module Mongo
         if cluster.server_selection_semaphore
           # Since the semaphore may have been signaled between us checking
           # the servers list earlier and the wait call below, we should not
-          # wait for the full remaining time - wait for up to 1 second, then
+          # wait for the full remaining time - wait for up to 0.5 second, then
           # recheck the state.
-          cluster.server_selection_semaphore.wait([time_remaining, 1].min)
+          cluster.server_selection_semaphore.wait([time_remaining, 0.5].min)
         else
           if Lint.enabled?
             raise Error::LintError, 'Waiting for server selection without having a server selection semaphore'

data/lib/mongo/server_selector/nearest.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 # Copyright (C) 2014-2020 MongoDB Inc.
 #