mongo 2.10.5 → 2.15.1

data/lib/mongo/server/connection_pool.rb

@@ -1,4 +1,7 @@
-# Copyright (C) 2014-2019 MongoDB, Inc.
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,6 +14,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+require 'mongo/server/connection_pool/populator'
 
 module Mongo
   class Server
@@ -35,8 +39,21 @@ module Mongo
 
       # The default timeout, in seconds, to wait for a connection.
       #
+      # This timeout applies while in flow threads are waiting for background
+      # threads to establish connections (and hence they must connect, handshake
+      # and auth in the allotted time).
+      #
+      # It is currently set to 10 seconds. The default connect timeout is
+      # 10 seconds by itself, but setting large timeouts can get applications
+      # in trouble if their requests get timed out by the reverse proxy,
+      # thus anything over 15 seconds is potentially dangerous.
+      #
       # @since 2.9.0
-      DEFAULT_WAIT_TIMEOUT = 1.freeze
+      DEFAULT_WAIT_TIMEOUT = 10.freeze
+
+      # Condition variable broadcast when the size of the pool changes
+      # to wake up the populator
+      attr_reader :populate_semaphore
 
       # Create the new connection pool.
       #
@@ -58,6 +75,9 @@ module Mongo
       #   are given, their values must be identical.
       # @option options [ Float ] :max_idle_time The time, in seconds,
       #   after which idle connections should be closed by the pool.
+      # Note: Additionally, options for connections created by this pool should
+      #   be included in the options passed here, and they will be forwarded to
+      #   any connections created by the pool.
       #
       # @since 2.0.0, API changed in 2.9.0
       def initialize(server, options = {})
@@ -99,6 +119,7 @@ module Mongo
         # or in the checked out connections set.
         @available_connections = available_connections = []
         @checked_out_connections = Set.new
+        @pending_connections = Set.new
 
         # Mutex used for synchronizing access to @available_connections and
         # @checked_out_connections. The pool object is thread-safe, thus
@@ -111,25 +132,26 @@ module Mongo
         # available connection when pool is at max size
         @available_semaphore = Semaphore.new
 
-        finalizer = proc do
-          available_connections.each do |connection|
-            connection.disconnect!(reason: :pool_closed)
-          end
-          available_connections.clear
-          # Finalizer does not close checked out connections.
-          # Those would have to be garbage collected on their own
-          # and that should close them.
-        end
-        ObjectSpace.define_finalizer(self, finalizer)
+        # Background thread reponsible for maintaining the size of
+        # the pool to at least min_size
+        @populator = Populator.new(self, options)
+        @populate_semaphore = Semaphore.new
+
+        ObjectSpace.define_finalizer(self, self.class.finalize(@available_connections, @pending_connections, @populator))
 
         publish_cmap_event(
-          Monitoring::Event::Cmap::PoolCreated.new(@server.address, options)
+          Monitoring::Event::Cmap::PoolCreated.new(@server.address, options, self)
         )
+
+        @populator.run! if min_size > 0
       end
 
       # @return [ Hash ] options The pool options.
       attr_reader :options
 
+      # @api private
+      def_delegators :@server, :address
+
       # Get the maximum size of the connection pool.
       #
       # @return [ Integer ] The maximum size of the connection pool.
@@ -194,7 +216,7 @@ module Mongo
       # already holding the lock as Ruby does not allow a thread holding a
       # lock to acquire this lock again.
       def unsynchronized_size
-        @available_connections.length + @checked_out_connections.size
+        @available_connections.length + @checked_out_connections.length + @pending_connections.length
       end
       private :unsynchronized_size
 
@@ -220,9 +242,23 @@ module Mongo
         !!@closed
       end
 
+      # @note This method is experimental and subject to change.
+      #
+      # @api experimental
+      # @since 2.11.0
+      def summary
+        @lock.synchronize do
+          "#<ConnectionPool size=#{unsynchronized_size} (#{min_size}-#{max_size}) " +
+            "used=#{@checked_out_connections.length} avail=#{@available_connections.length} pending=#{@pending_connections.length}>"
+        end
+      end
+
       # @since 2.9.0
       def_delegators :@server, :monitoring
 
+      # @api private
+      attr_reader :populator
+
       # Checks a connection out of the pool.
       #
       # If there are active connections in the pool, the most recently used
@@ -236,18 +272,30 @@ module Mongo
       # checked back in via the check_in method.
       #
       # @return [ Mongo::Server::Connection ] The checked out connection.
+      # @raise [ Error::PoolClosedError ] If the pool has been closed.
       # @raise [ Timeout::Error ] If the connection pool is at maximum size
       #   and remains so for longer than the wait timeout.
       #
       # @since 2.9.0
       def check_out
-        raise_if_closed!
+        check_invariants
 
         publish_cmap_event(
           Monitoring::Event::Cmap::ConnectionCheckOutStarted.new(@server.address)
         )
 
-        deadline = Time.now + wait_timeout
+        if closed?
+          publish_cmap_event(
+            Monitoring::Event::Cmap::ConnectionCheckOutFailed.new(
+              @server.address,
+              Monitoring::Event::Cmap::ConnectionCheckOutFailed::POOL_CLOSED
+            ),
+          )
+          raise Error::PoolClosedError.new(@server.address, self)
+        end
+
+        deadline = Utils.monotonic_time + wait_timeout
+        pid = Process.pid
         connection = nil
         # It seems that synchronize sets up its own loop, thus a simple break
         # is insufficient to break the outer loop
@@ -260,10 +308,18 @@ module Mongo
               until @available_connections.empty?
                 connection = @available_connections.pop
 
+                if connection.pid != pid
+                  log_warn("Detected PID change - Mongo client should have been reconnected (old pid #{connection.pid}, new pid #{pid}")
+                  connection.disconnect!(reason: :stale)
+                  @populate_semaphore.signal
+                  next
+                end
+
                 if connection.generation != generation
                   # Stale connections should be disconnected in the clear
                   # method, but if any don't, check again here
                   connection.disconnect!(reason: :stale)
+                  @populate_semaphore.signal
                   next
                 end
 
@@ -271,24 +327,24 @@ module Mongo
                   Time.now - connection.last_checkin > max_idle_time
                 then
                   connection.disconnect!(reason: :idle)
+                  @populate_semaphore.signal
                   next
                 end
 
+                @pending_connections << connection
                 throw(:done)
               end
 
               # Ruby does not allow a thread to lock a mutex which it already
               # holds.
               if unsynchronized_size < max_size
-                # This does not currently connect the socket and handshake,
-                # but if it did, it would be performing i/o under our lock,
-                # which is bad. Fix in the future.
                 connection = create_connection
+                @pending_connections << connection
                 throw(:done)
               end
             end
 
-            wait = deadline - Time.now
+            wait = deadline - Utils.monotonic_time
             if wait <= 0
               publish_cmap_event(
                 Monitoring::Event::Cmap::ConnectionCheckOutFailed.new(
@@ -296,17 +352,57 @@ module Mongo
                   Monitoring::Event::Cmap::ConnectionCheckOutFailed::TIMEOUT,
                 ),
               )
-              raise Error::ConnectionCheckOutTimeout.new(@server.address, wait_timeout)
+
+              msg = @lock.synchronize do
+                "Timed out attempting to check out a connection " +
+                  "from pool for #{@server.address} after #{wait_timeout} sec. " +
+                  "Connections in pool: #{@available_connections.length} available, " +
+                  "#{@checked_out_connections.length} checked out, " +
+                  "#{@pending_connections.length} pending " +
+                  "(max size: #{max_size})"
+              end
+              raise Error::ConnectionCheckOutTimeout.new(msg, address: @server.address)
             end
             @available_semaphore.wait(wait)
           end
         end
 
-        @checked_out_connections << connection
+        begin
+          connect_connection(connection)
+        rescue Exception
+          # Handshake or authentication failed
+          @lock.synchronize do
+            @pending_connections.delete(connection)
+          end
+          @populate_semaphore.signal
+
+          publish_cmap_event(
+            Monitoring::Event::Cmap::ConnectionCheckOutFailed.new(
+              @server.address,
+              Monitoring::Event::Cmap::ConnectionCheckOutFailed::CONNECTION_ERROR
+            ),
+          )
+          raise
+        end
+
+        @lock.synchronize do
+          @checked_out_connections << connection
+          @pending_connections.delete(connection)
+        end
+
         publish_cmap_event(
-          Monitoring::Event::Cmap::ConnectionCheckedOut.new(@server.address, connection.id),
+          Monitoring::Event::Cmap::ConnectionCheckedOut.new(@server.address, connection.id, self),
         )
+
+        if Lint.enabled?
+          unless connection.connected?
+            raise Error::LintError, "Connection pool for #{address} checked out a disconnected connection #{connection.generation}:#{connection.id}"
+          end
+        end
+
         connection
+      ensure
+        check_invariants
       end
 
       # Check a connection back into the pool.
@@ -317,22 +413,33 @@ module Mongo
       #
       # @since 2.9.0
       def check_in(connection)
+        check_invariants
+
         @lock.synchronize do
-          unless @checked_out_connections.include?(connection)
-            raise ArgumentError, "Trying to check in a connection which is not currently checked out by this pool: #{connection}"
+          unless connection.connection_pool == self
+            raise ArgumentError, "Trying to check in a connection which was not checked out by this pool: #{connection} checked out from pool #{connection.connection_pool} (for #{self})"
           end
 
-          @checked_out_connections.delete(connection)
+          unless @checked_out_connections.include?(connection)
+            raise ArgumentError, "Trying to check in a connection which is not currently checked out by this pool: #{connection} (for #{self})"
+          end
 
           # Note: if an event handler raises, resource will not be signaled.
           # This means threads waiting for a connection to free up when
           # the pool is at max size may time out.
           # Threads that begin waiting after this method completes (with
           # the exception) should be fine.
+
+          @checked_out_connections.delete(connection)
           publish_cmap_event(
-            Monitoring::Event::Cmap::ConnectionCheckedIn.new(@server.address, connection.id)
+            Monitoring::Event::Cmap::ConnectionCheckedIn.new(@server.address, connection.id, self)
           )
 
+          if connection.error?
+            connection.disconnect!(reason: :error)
+            return
+          end
+
           if closed?
             connection.disconnect!(reason: :pool_closed)
             return
@@ -341,8 +448,10 @@ module Mongo
           if connection.closed?
             # Connection was closed - for example, because it experienced
             # a network error. Nothing else needs to be done here.
+            @populate_semaphore.signal
           elsif connection.generation != @generation
             connection.disconnect!(reason: :stale)
+            @populate_semaphore.signal
           else
             connection.record_checkin!
             @available_connections << connection
@@ -352,6 +461,8 @@ module Mongo
             @available_semaphore.signal
           end
         end
+      ensure
+        check_invariants
       end
 
       # Closes all idle connections in the pool and schedules currently checked
@@ -362,6 +473,8 @@ module Mongo
       # @option options [ true | false ] :lazy If true, do not close any of
       #   the idle connections and instead let them be closed during a
       #   subsequent check out operation.
+      # @option options [ true | false ] :stop_populator Whether to stop
+      #   the populator background thread. For internal driver use only.
       #
       # @return [ true ] true.
       #
@@ -369,6 +482,12 @@ module Mongo
       def clear(options = nil)
         raise_if_closed!
 
+        check_invariants
+
+        if options && options[:stop_populator]
+          stop_populator
+        end
+
         @lock.synchronize do
           @generation += 1
 
@@ -380,11 +499,14 @@ module Mongo
             until @available_connections.empty?
               connection = @available_connections.pop
               connection.disconnect!(reason: :stale)
+              @populate_semaphore.signal
             end
           end
         end
 
         true
+      ensure
+        check_invariants
       end
 
       # @since 2.1.0
@@ -400,31 +522,37 @@ module Mongo
       # @option options [ true | false ] :force Also close all checked out
       #   connections.
       #
-      # @return [ true ] true.
+      # @return [ true ] Always true.
       #
       # @since 2.9.0
       def close(options = nil)
         return if closed?
 
+        options ||= {}
+
+        stop_populator
+
         @lock.synchronize do
           until @available_connections.empty?
             connection = @available_connections.pop
             connection.disconnect!(reason: :pool_closed)
           end
 
-          if options && options[:force]
+          if options[:force]
             until @checked_out_connections.empty?
               connection = @checked_out_connections.take(1).first
               connection.disconnect!(reason: :pool_closed)
               @checked_out_connections.delete(connection)
             end
           end
-        end
 
-        @closed = true
+          # mark pool as closed before releasing lock so
+          # no connections can be created, checked in, or checked out
+          @closed = true
+        end
 
         publish_cmap_event(
-          Monitoring::Event::Cmap::PoolClosed.new(@server.address)
+          Monitoring::Event::Cmap::PoolClosed.new(@server.address, self)
         )
 
         true
@@ -485,6 +613,7 @@ module Mongo
               if (Time.now - last_checkin) > max_idle_time
                 connection.disconnect!(reason: :idle)
                 @available_connections.delete_at(i)
+                @populate_semaphore.signal
                 next
               end
             end
@@ -493,25 +622,133 @@ module Mongo
         end
       end
 
-      # Creates up to the min size connections.
+      # Stop the background populator thread and clean up any connections created
+      # which have not been connected yet.
       #
-      # Used by the spec test runner.
+      # Used when closing the pool or when terminating the bg thread for testing
+      # purposes. In the latter case, this method must be called before the pool
+      # is used, to ensure no connections in pending_connections were created in-flow
+      # by the check_out method.
+      #
+      # @api private
+      def stop_populator
+        @populator.stop!
+
+        @lock.synchronize do
+          # If stop_populator is called while populate is running, there may be
+          # connections waiting to be connected, connections which have not yet
+          # been moved to available_connections, or connections moved to available_connections
+          # but not deleted from pending_connections. These should be cleaned up.
+          until @pending_connections.empty?
+            connection = @pending_connections.take(1).first
+            connection.disconnect!
+            @pending_connections.delete(connection)
+          end
+        end
+      end
+
+      # Creates and adds a connection to the pool, if the pool's size is below
+      # min_size. Retries once if a socket-related error is encountered during
+      # this process and raises if a second error or a non socket-related error occurs.
+      #
+      # Used by the pool populator background thread.
+      #
+      # @return [ true | false ] Whether this method should be called again
+      #   to create more connections.
+      # @raise [ Error::AuthError, Error ] The second socket-related error raised if a retry
+      # occured, or the non socket-related error
       #
       # @api private
       def populate
-        while size < min_size
-          @available_connections << create_connection
+        return false if closed?
+
+        begin
+          return create_and_add_connection
+        rescue Error::SocketError, Error::SocketTimeoutError => e
+          # an error was encountered while connecting the connection,
+          # ignore this first error and try again.
+          log_warn("Populator failed to connect a connection for #{address}: #{e.class}: #{e}. It will retry.")
+        end
+
+        return create_and_add_connection
+      rescue Error::AuthError, Error
+        # wake up one thread waiting for connections, since one could not
+        # be created here, and can instead be created in flow
+        @available_semaphore.signal
+        raise
+      end
+
+      # Finalize the connection pool for garbage collection.
+      #
+      # @param [ List<Mongo::Connection> ] available_connections The available connections.
+      # @param [ List<Mongo::Connection> ] pending_connections The pending connections.
+      # @param [ Populator ] populator The populator.
+      #
+      # @return [ Proc ] The Finalizer.
+      def self.finalize(available_connections, pending_connections, populator)
+        proc do
+          available_connections.each do |connection|
+            connection.disconnect!(reason: :pool_closed)
+          end
+          available_connections.clear
+
+          pending_connections.each do |connection|
+            connection.disconnect!(reason: :pool_closed)
+          end
+          pending_connections.clear
+
+          # Finalizer does not close checked out connections.
+          # Those would have to be garbage collected on their own
+          # and that should close them.
         end
       end
 
       private
 
       def create_connection
-        connection = Connection.new(@server, options.merge(generation: generation))
-        # CMAP spec requires connections to be returned from the pool
-        # fully established.
-        #connection.connect!
-        connection
+        connection = Connection.new(@server, options.merge(
+          generation: generation,
+          connection_pool: self,
+          # Do not pass app metadata - this will be retrieved by the connection
+          # based on the auth needs.
+        ))
+      end
+
+      # Create a connection, connect it, and add it to the pool.
+      #
+      # @return [ true | false ] True if a connection was created and
+      #    added to the pool, false otherwise
+      # @raise [ Mongo::Error ] An error encountered during connection connect
+      def create_and_add_connection
+        connection = nil
+
+        @lock.synchronize do
+          if !closed? && unsynchronized_size < min_size
+            connection = create_connection
+            @pending_connections << connection
+          else
+            return false
+          end
+        end
+
+        begin
+          connect_connection(connection)
+        rescue Exception
+          @lock.synchronize do
+            @pending_connections.delete(connection)
+          end
+          raise
+        end
+
+        @lock.synchronize do
+          @available_connections << connection
+          @pending_connections.delete(connection)
+
+          # wake up one thread waiting for connections, since one was created
+          @available_semaphore.signal
+        end
+
+        true
       end
 
       # Asserts that the pool has not been closed.
@@ -521,7 +758,43 @@ module Mongo
       # @since 2.9.0
       def raise_if_closed!
         if closed?
-          raise Error::PoolClosedError.new(@server.address)
+          raise Error::PoolClosedError.new(@server.address, self)
+        end
+      end
+
+      # Attempts to connect (handshake and auth) the connection. If an error is
+      # encountered, closes the connection and raises the error.
+      def connect_connection(connection)
+        begin
+          connection.connect!
+        rescue Exception
+          connection.disconnect!(reason: :error)
+          raise
+        end
+      rescue Error::SocketError, Error::SocketTimeoutError => exc
+        @server.unknown!(generation: exc.generation, stop_push_monitor: true)
+        raise
+      end
+
+      def check_invariants
+        return unless Lint.enabled?
+
+        # Server summary calls pool summary which requires pool lock -> deadlock.
+        # Obtain the server summary ahead of time.
+        server_summary = @server.summary
+
+        @lock.synchronize do
+          @available_connections.each do |connection|
+            if connection.closed?
+              raise Error::LintError, "Available connection is closed: #{connection} for #{server_summary}"
+            end
+          end
+
+          @pending_connections.each do |connection|
+            if connection.closed?
+              raise Error::LintError, "Pending connection is closed: #{connection} for #{server_summary}"
+            end
+          end
         end
       end
     end

data/lib/mongo/server/context.rb

@@ -1,4 +1,7 @@
-# Copyright (C) 2014-2019 MongoDB, Inc.
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the 'License');
 # you may not use this file except in compliance with the License.

data/lib/mongo/server/description/features.rb

@@ -1,4 +1,7 @@
-# Copyright (C) 2014-2019 MongoDB, Inc.
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the 'License');
 # you may not use this file except in compliance with the License.
@@ -20,20 +23,26 @@ module Mongo
       #
       # @since 2.0.0
       class Features
-
         # List of features and the wire protocol version they appear in.
         #
         # Wire protocol versions map to server releases as follows:
-        # - 2 => 2.6
-        # - 3 => 3.0
-        # - 4 => 3.2
-        # - 5 => 3.4
-        # - 6 => 3.6
-        # - 7 => 4.0
-        # - 8 => 4.2
+        # -  2 => 2.6
+        # -  3 => 3.0
+        # -  4 => 3.2
+        # -  5 => 3.4
+        # -  6 => 3.6
+        # -  7 => 4.0
+        # -  8 => 4.2
+        # -  9 => 4.4
+        # - 13 => 5.0
         #
         # @since 2.0.0
         MAPPINGS = {
+          :retryable_write_error_label => 9,
+          # Server versions older than 4.2 do not reliably validate options
+          # provided by the client during findAndModify operations, requiring the
+          # driver to raise client-side errors when those options are provided.
+          :find_and_modify_option_validation => 8,
           :transactions => 7,
           :scram_sha_256 => 7,
           :array_filters => 6,
@@ -41,6 +50,10 @@ module Mongo
           :sessions => 6,
           :collation => 5,
           :max_staleness => 5,
+          # Server versions older than 3.4 do not reliably validate options
+          # provided by the client during update/delete operations, requiring the
+          # driver to raise client-side errors when those options are provided.
+          :update_delete_option_validation => 5,
           :find_command => 4,
           :list_collections => 3,
           :list_indexes => 3,
@@ -64,7 +77,7 @@ module Mongo
         # The wire protocol versions that this version of the driver supports.
         #
         # @since 2.0.0
-        DRIVER_WIRE_VERSIONS = (2..8).freeze
+        DRIVER_WIRE_VERSIONS = (2..13).freeze
 
         # Create the methods for each mapping to tell if they are supported.
         #