mongo 2.10.5 → 2.11.0.rc0

data/lib/mongo/server/connection.rb

@@ -123,6 +123,14 @@ module Mongo
         options[:generation]
       end
 
+      # The connection pool from which this connection was created.
+      # May be nil.
+      #
+      # @api private
+      def connection_pool
+        options[:connection_pool]
+      end
+
       # Whether the connection was closed.
       #
       # Closed connections should no longer be used. Instead obtain a new
@@ -151,7 +159,7 @@ module Mongo
       def connect!
         if closed?
           if Lint.enabled?
-            raise Error::LintError, "Reconnecting closed connections is no longer supported"
+            raise Error::LintError, "Reconnecting closed connections is no longer supported (for #{address})"
           else
             log_warn("Reconnecting closed connections is deprecated (for #{address})")
           end
@@ -290,14 +298,14 @@ module Mongo
 
       def handshake!(socket)
         unless socket
-          raise Error::HandshakeError, "Cannot handshake because there is no usable socket"
+          raise Error::HandshakeError, "Cannot handshake because there is no usable socket (for #{address})"
         end
 
         response = average_rtt = nil
         @server.handle_handshake_failure! do
           begin
             response, exc, rtt, average_rtt =
-              @server.monitor.round_trip_time_averager.measure do
+              @server.round_trip_time_averager.measure do
                 socket.write(app_metadata.ismaster_bytes)
                 Protocol::Message.deserialize(socket, max_message_size).documents[0]
               end
@@ -349,7 +357,7 @@ module Mongo
         end
 
         new_description = Description.new(address, response, average_rtt)
-        @server.monitor.publish(Event::DESCRIPTION_CHANGED, @server.description, new_description)
+        @server.cluster.run_sdam_flow(@server.description, new_description)
       end
 
       def authenticate!(pending_connection)
@@ -376,7 +384,6 @@ module Mongo
         # Important: timeout errors are not handled here
         rescue Error::SocketError
           @server.unknown!
-          @server.pool.disconnect!
           raise
         end
       end

data/lib/mongo/server/connection_pool.rb

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+require 'mongo/server/connection_pool/populator'
 
 module Mongo
   class Server
@@ -35,8 +36,21 @@ module Mongo
 
       # The default timeout, in seconds, to wait for a connection.
       #
+      # This timeout applies while in flow threads are waiting for background
+      # threads to establish connections (and hence they must connect, handshake
+      # and auth in the allotted time).
+      #
+      # It is currently set to 10 seconds. The default connect timeout is
+      # 10 seconds by itself, but setting large timeouts can get applications
+      # in trouble if their requests get timed out by the reverse proxy,
+      # thus anything over 15 seconds is potentially dangerous.
+      #
       # @since 2.9.0
-      DEFAULT_WAIT_TIMEOUT = 1.freeze
+      DEFAULT_WAIT_TIMEOUT = 10.freeze
+
+      # Condition variable broadcast when the size of the pool changes
+      # to wake up the populator
+      attr_reader :populate_semaphore
 
       # Create the new connection pool.
       #
@@ -58,6 +72,9 @@ module Mongo
       #   are given, their values must be identical.
       # @option options [ Float ] :max_idle_time The time, in seconds,
       #   after which idle connections should be closed by the pool.
+      # Note: Additionally, options for connections created by this pool should
+      #   be included in the options passed here, and they will be forwarded to
+      #   any connections created by the pool.
       #
       # @since 2.0.0, API changed in 2.9.0
       def initialize(server, options = {})
@@ -99,6 +116,7 @@ module Mongo
         # or in the checked out connections set.
         @available_connections = available_connections = []
         @checked_out_connections = Set.new
+        @pending_connections = Set.new
 
         # Mutex used for synchronizing access to @available_connections and
         # @checked_out_connections. The pool object is thread-safe, thus
@@ -111,25 +129,26 @@ module Mongo
         # available connection when pool is at max size
         @available_semaphore = Semaphore.new
 
-        finalizer = proc do
-          available_connections.each do |connection|
-            connection.disconnect!(reason: :pool_closed)
-          end
-          available_connections.clear
-          # Finalizer does not close checked out connections.
-          # Those would have to be garbage collected on their own
-          # and that should close them.
-        end
-        ObjectSpace.define_finalizer(self, finalizer)
+        # Background thread reponsible for maintaining the size of
+        # the pool to at least min_size
+        @populator = Populator.new(self, options)
+        @populate_semaphore = Semaphore.new
+
+        ObjectSpace.define_finalizer(self, self.class.finalize(@available_connections, @pending_connections, @populator))
 
         publish_cmap_event(
-          Monitoring::Event::Cmap::PoolCreated.new(@server.address, options)
+          Monitoring::Event::Cmap::PoolCreated.new(@server.address, options, self)
         )
+
+        @populator.run! if min_size > 0
       end
 
       # @return [ Hash ] options The pool options.
       attr_reader :options
 
+      # @api private
+      def_delegators :@server, :address
+
       # Get the maximum size of the connection pool.
       #
       # @return [ Integer ] The maximum size of the connection pool.
@@ -194,7 +213,7 @@ module Mongo
       # already holding the lock as Ruby does not allow a thread holding a
       # lock to acquire this lock again.
       def unsynchronized_size
-        @available_connections.length + @checked_out_connections.size
+        @available_connections.length + @checked_out_connections.length + @pending_connections.length
       end
       private :unsynchronized_size
 
@@ -220,9 +239,23 @@ module Mongo
         !!@closed
       end
 
+      # @note This method is experimental and subject to change.
+      #
+      # @api experimental
+      # @since 2.11.0
+      def summary
+        @lock.synchronize do
+          "#<ConnectionPool size=#{unsynchronized_size} (#{min_size}-#{max_size}) " +
+            "used=#{@checked_out_connections.length} avail=#{@available_connections.length} pending=#{@pending_connections.length}>"
+        end
+      end
+
       # @since 2.9.0
       def_delegators :@server, :monitoring
 
+      # @api private
+      attr_reader :populator
+
       # Checks a connection out of the pool.
       #
       # If there are active connections in the pool, the most recently used
@@ -236,17 +269,26 @@ module Mongo
       # checked back in via the check_in method.
       #
       # @return [ Mongo::Server::Connection ] The checked out connection.
+      # @raise [ Error::PoolClosedError ] If the pool has been closed.
       # @raise [ Timeout::Error ] If the connection pool is at maximum size
       #   and remains so for longer than the wait timeout.
       #
       # @since 2.9.0
       def check_out
-        raise_if_closed!
-
         publish_cmap_event(
           Monitoring::Event::Cmap::ConnectionCheckOutStarted.new(@server.address)
         )
 
+        if closed?
+          publish_cmap_event(
+            Monitoring::Event::Cmap::ConnectionCheckOutFailed.new(
+              @server.address,
+              Monitoring::Event::Cmap::ConnectionCheckOutFailed::POOL_CLOSED
+            ),
+          )
+          raise Error::PoolClosedError.new(@server.address, self)
+        end
+
         deadline = Time.now + wait_timeout
         connection = nil
         # It seems that synchronize sets up its own loop, thus a simple break
@@ -264,6 +306,7 @@ module Mongo
                   # Stale connections should be disconnected in the clear
                   # method, but if any don't, check again here
                   connection.disconnect!(reason: :stale)
+                  @populate_semaphore.signal
                   next
                 end
 
@@ -271,19 +314,19 @@ module Mongo
                   Time.now - connection.last_checkin > max_idle_time
                 then
                   connection.disconnect!(reason: :idle)
+                  @populate_semaphore.signal
                   next
                 end
 
+                @pending_connections << connection
                 throw(:done)
               end
 
               # Ruby does not allow a thread to lock a mutex which it already
               # holds.
               if unsynchronized_size < max_size
-                # This does not currently connect the socket and handshake,
-                # but if it did, it would be performing i/o under our lock,
-                # which is bad. Fix in the future.
                 connection = create_connection
+                @pending_connections << connection
                 throw(:done)
               end
             end
@@ -296,15 +339,45 @@ module Mongo
                   Monitoring::Event::Cmap::ConnectionCheckOutFailed::TIMEOUT,
                 ),
               )
-              raise Error::ConnectionCheckOutTimeout.new(@server.address, wait_timeout)
+
+              msg = @lock.synchronize do
+                "Timed out attempting to check out a connection " +
+                  "from pool for #{@server.address} after #{wait_timeout} sec. " +
+                  "Connections in pool: #{@available_connections.length} available, " +
+                  "#{@checked_out_connections.length} checked out, " +
+                  "#{@pending_connections.length} pending"
+              end
+              raise Error::ConnectionCheckOutTimeout.new(msg, address: @server.address)
             end
             @available_semaphore.wait(wait)
           end
         end
 
-        @checked_out_connections << connection
+        begin
+          connect_connection(connection)
+        rescue Exception
+          # Handshake or authentication failed
+          @lock.synchronize do
+            @pending_connections.delete(connection)
+          end
+          @populate_semaphore.signal
+
+          publish_cmap_event(
+            Monitoring::Event::Cmap::ConnectionCheckOutFailed.new(
+              @server.address,
+              Monitoring::Event::Cmap::ConnectionCheckOutFailed::CONNECTION_ERROR
+            ),
+          )
+          raise
+        end
+
+        @lock.synchronize do
+          @checked_out_connections << connection
+          @pending_connections.delete(connection)
+        end
+
         publish_cmap_event(
-          Monitoring::Event::Cmap::ConnectionCheckedOut.new(@server.address, connection.id),
+          Monitoring::Event::Cmap::ConnectionCheckedOut.new(@server.address, connection.id, self),
         )
         connection
       end
@@ -318,19 +391,23 @@ module Mongo
       # @since 2.9.0
       def check_in(connection)
         @lock.synchronize do
-          unless @checked_out_connections.include?(connection)
-            raise ArgumentError, "Trying to check in a connection which is not currently checked out by this pool: #{connection}"
+          unless connection.connection_pool == self
+            raise ArgumentError, "Trying to check in a connection which was not checked out by this pool: #{connection} checked out from pool #{connection.connection_pool} (for #{self})"
           end
 
-          @checked_out_connections.delete(connection)
+          unless @checked_out_connections.include?(connection)
+            raise ArgumentError, "Trying to check in a connection which is not currently checked out by this pool: #{connection} (for #{self})"
+          end
 
           # Note: if an event handler raises, resource will not be signaled.
           # This means threads waiting for a connection to free up when
           # the pool is at max size may time out.
           # Threads that begin waiting after this method completes (with
           # the exception) should be fine.
+
+          @checked_out_connections.delete(connection)
           publish_cmap_event(
-            Monitoring::Event::Cmap::ConnectionCheckedIn.new(@server.address, connection.id)
+            Monitoring::Event::Cmap::ConnectionCheckedIn.new(@server.address, connection.id, self)
           )
 
           if closed?
@@ -341,8 +418,10 @@ module Mongo
           if connection.closed?
             # Connection was closed - for example, because it experienced
             # a network error. Nothing else needs to be done here.
+            @populate_semaphore.signal
           elsif connection.generation != @generation
             connection.disconnect!(reason: :stale)
+            @populate_semaphore.signal
           else
             connection.record_checkin!
             @available_connections << connection
@@ -362,6 +441,8 @@ module Mongo
       # @option options [ true | false ] :lazy If true, do not close any of
       #   the idle connections and instead let them be closed during a
       #   subsequent check out operation.
+      # @option options [ true | false ] :stop_populator Whether to stop
+      #   the populator background thread. For internal driver use only.
       #
       # @return [ true ] true.
       #
@@ -369,6 +450,10 @@ module Mongo
       def clear(options = nil)
         raise_if_closed!
 
+        if options && options[:stop_populator]
+          stop_populator
+        end
+
         @lock.synchronize do
           @generation += 1
 
@@ -380,6 +465,7 @@ module Mongo
             until @available_connections.empty?
               connection = @available_connections.pop
               connection.disconnect!(reason: :stale)
+              @populate_semaphore.signal
             end
           end
         end
@@ -400,31 +486,37 @@ module Mongo
       # @option options [ true | false ] :force Also close all checked out
       #   connections.
       #
-      # @return [ true ] true.
+      # @return [ true ] Always true.
       #
       # @since 2.9.0
       def close(options = nil)
         return if closed?
 
+        options ||= {}
+
+        stop_populator
+
         @lock.synchronize do
           until @available_connections.empty?
             connection = @available_connections.pop
             connection.disconnect!(reason: :pool_closed)
           end
 
-          if options && options[:force]
+          if options[:force]
             until @checked_out_connections.empty?
               connection = @checked_out_connections.take(1).first
               connection.disconnect!(reason: :pool_closed)
               @checked_out_connections.delete(connection)
             end
           end
-        end
 
-        @closed = true
+          # mark pool as closed before releasing lock so
+          # no connections can be created, checked in, or checked out
+          @closed = true
+        end
 
         publish_cmap_event(
-          Monitoring::Event::Cmap::PoolClosed.new(@server.address)
+          Monitoring::Event::Cmap::PoolClosed.new(@server.address, self)
         )
 
         true
@@ -485,6 +577,7 @@ module Mongo
               if (Time.now - last_checkin) > max_idle_time
                 connection.disconnect!(reason: :idle)
                 @available_connections.delete_at(i)
+                @populate_semaphore.signal
                 next
               end
             end
@@ -493,25 +586,131 @@ module Mongo
         end
       end
 
-      # Creates up to the min size connections.
+      # Stop the background populator thread and clean up any connections created
+      # which have not been connected yet.
       #
-      # Used by the spec test runner.
+      # Used when closing the pool or when terminating the bg thread for testing
+      # purposes. In the latter case, this method must be called before the pool
+      # is used, to ensure no connections in pending_connections were created in-flow
+      # by the check_out method.
+      #
+      # @api private
+      def stop_populator
+        @populator.stop!
+
+        @lock.synchronize do
+          # If stop_populator is called while populate is running, there may be
+          # connections waiting to be connected, connections which have not yet
+          # been moved to available_connections, or connections moved to available_connections
+          # but not deleted from pending_connections. These should be cleaned up.
+          until @pending_connections.empty?
+            connection = @pending_connections.take(1).first
+            connection.disconnect!
+            @pending_connections.delete(connection)
+          end
+        end
+      end
+
+      # Creates and adds a connection to the pool, if the pool's size is below
+      # min_size. Retries once if a socket-related error is encountered during
+      # this process and raises if a second error or a non socket-related error occurs.
+      #
+      # Used by the pool populator background thread.
+      #
+      # @return [ true | false ] Whether this method should be called again
+      #   to create more connections.
+      # @raise [ Error::AuthError, Error ] The second socket-related error raised if a retry
+      # occured, or the non socket-related error
       #
       # @api private
       def populate
-        while size < min_size
-          @available_connections << create_connection
+        return false if closed?
+
+        begin
+          return create_and_add_connection
+        rescue Error::SocketError, Error::SocketTimeoutError => e
+          # an error was encountered while connecting the connection,
+          # ignore this first error and try again.
+          log_warn("Populator failed to connect a connection for #{address}: #{e.class}: #{e}. It will retry.")
+        end
+
+        return create_and_add_connection
+      rescue Error::AuthError, Error
+        # wake up one thread waiting for connections, since one could not
+        # be created here, and can instead be created in flow
+        @available_semaphore.signal
+        raise
+      end
+
+      # Finalize the connection pool for garbage collection.
+      #
+      # @param [ List<Mongo::Connection> ] available_connections The available connections.
+      # @param [ List<Mongo::Connection> ] pending_connections The pending connections.
+      # @param [ Populator ] populator The populator.
+      #
+      # @return [ Proc ] The Finalizer.
+      def self.finalize(available_connections, pending_connections, populator)
+        proc do
+          populator.stop!
+
+          available_connections.each do |connection|
+            connection.disconnect!(reason: :pool_closed)
+          end
+          available_connections.clear
+
+          pending_connections.each do |connection|
+            connection.disconnect!(reason: :pool_closed)
+          end
+          pending_connections.clear
+
+          # Finalizer does not close checked out connections.
+          # Those would have to be garbage collected on their own
+          # and that should close them.
         end
       end
 
       private
 
       def create_connection
-        connection = Connection.new(@server, options.merge(generation: generation))
-        # CMAP spec requires connections to be returned from the pool
-        # fully established.
-        #connection.connect!
-        connection
+        connection = Connection.new(@server, options.merge(generation: generation,
+          connection_pool: self))
+      end
+
+      # Create a connection, connect it, and add it to the pool.
+      #
+      # @return [ true | false ] True if a connection was created and
+      #    added to the pool, false otherwise
+      # @raise [ Mongo::Error ] An error encountered during connection connect
+      def create_and_add_connection
+        connection = nil
+
+        @lock.synchronize do
+          if !closed? && unsynchronized_size < min_size
+            connection = create_connection
+            @pending_connections << connection
+          else
+            return false
+          end
+        end
+
+        begin
+          connect_connection(connection)
+        rescue Exception
+          @lock.synchronize do
+            @pending_connections.delete(connection)
+          end
+          raise
+        end
+
+        @lock.synchronize do
+          @available_connections << connection
+          @pending_connections.delete(connection)
+
+          # wake up one thread waiting for connections, since one was created
+          @available_semaphore.signal
+        end
+
+        true
       end
 
       # Asserts that the pool has not been closed.
@@ -521,7 +720,18 @@ module Mongo
       # @since 2.9.0
       def raise_if_closed!
         if closed?
-          raise Error::PoolClosedError.new(@server.address)
+          raise Error::PoolClosedError.new(@server.address, self)
+        end
+      end
+
+      # Attempts to connect (handshake and auth) the connection. If an error is
+      # encountered, closes the connection and raises the error.
+      def connect_connection(connection)
+        begin
+          connection.connect!
+        rescue Exception
+          connection.disconnect!(reason: :error)
+          raise
         end
       end
     end