mongo 2.11.2 → 2.13.0.beta1

data/lib/mongo/server/connection_base.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2019 MongoDB, Inc.
+# Copyright (C) 2019-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -22,10 +22,30 @@ module Mongo
     #   the classes which include this module is not part of the public API.
     #
     # @api semipublic
-    class ConnectionBase
+    class ConnectionBase < ConnectionCommon
       extend Forwardable
       include Monitoring::Publishable
 
+      # The maximum allowed size in bytes that a user-supplied document may
+      # take up when serialized, if the server's ismaster response does not
+      # include maxBsonObjectSize field.
+      #
+      # The commands that are sent to the server may exceed this size by
+      # MAX_BSON_COMMAND_OVERHEAD.
+      #
+      # @api private
+      DEFAULT_MAX_BSON_OBJECT_SIZE = 16777216
+
+      # The additional overhead allowed for command data (i.e. fields added
+      # to the command document by the driver, as opposed to documents
+      # provided by the user) when serializing a complete command to BSON.
+      #
+      # @api private
+      MAX_BSON_COMMAND_OVERHEAD = 16384
+
+      # @api private
+      REDUCED_MAX_BSON_SIZE = 2097152
+
       # @return [ Hash ] options The passed in options.
       attr_reader :options
 
@@ -37,15 +57,42 @@ module Mongo
       # @return [ Mongo::Address ] address The address to connect to.
       def_delegators :server, :address
 
+      # @deprecated
       def_delegators :server,
-                     :features,
-                     :max_bson_object_size,
-                     :max_message_size,
-                     :mongos?,
-                     :compressor,
                      :cluster_time,
                      :update_cluster_time
 
+      # Returns the server description for this connection, derived from
+      # the isMaster response for the handshake performed on this connection.
+      #
+      # @note A connection object that hasn't yet connected (handshaken and
+      #   authenticated, if authentication is required) does not have a
+      #   description. While handshaking and authenticating the driver must
+      #   be using global defaults, in particular not assuming that the
+      #   properties of a particular connection are the same as properties
+      #   of other connections made to the same address (since the server
+      #   on the other end could have been shut down and a different server
+      #   version could have been launched).
+      #
+      # @return [ Server::Description ] Server description for this connection.
+      # @api private
+      attr_reader :description
+
+      # @deprecated
+      def_delegators :description,
+        :features,
+        :max_bson_object_size,
+        :max_message_size,
+        :mongos?
+
+      # Connection pool generation from which this connection was created.
+      # May be nil.
+      #
+      # @return [ Integer | nil ] Connection pool generation.
+      def generation
+        options[:generation]
+      end
+
       def app_metadata
         @app_metadata ||= begin
           same = true
@@ -78,11 +125,16 @@ module Mongo
       # @param [ Array<Message> ] messages A one-element array containing
       #   the message to dispatch.
       # @param [ Integer ] operation_id The operation id to link messages.
+      # @param [ Hash ] options
+      #
+      # @option options [ Boolean ] :deserialize_as_bson Whether to deserialize
+      #   the response to this message using BSON objects in place of native
+      #   Ruby types wherever possible.
       #
       # @return [ Protocol::Message | nil ] The reply if needed.
       #
       # @since 2.0.0
-      def dispatch(messages, operation_id = nil)
+      def dispatch(messages, operation_id = nil, client = nil, options = {})
         # The monitoring code does not correctly handle multiple messages,
         # and the driver internally does not send more than one message at
         # a time ever. Thus prohibit multiple message use for now.
@@ -90,25 +142,31 @@ module Mongo
           raise ArgumentError, 'Can only dispatch one message at a time'
         end
         message = messages.first
-        deliver(message)
+        deliver(message, client, options)
       end
 
       private
 
-      def deliver(message)
-        buffer = serialize(message)
+      def deliver(message, client, options = {})
+        if Lint.enabled? && !@socket
+          raise Error::LintError, "Trying to deliver a message over a disconnected connection (to #{address})"
+        end
+        buffer = serialize(message, client)
         ensure_connected do |socket|
           operation_id = Monitoring.next_operation_id
           command_started(address, operation_id, message.payload,
-            socket_object_id: socket.object_id, connection_id: id)
+            socket_object_id: socket.object_id, connection_id: id,
+            connection_generation: generation)
           start = Time.now
           result = nil
           begin
-            socket.write(buffer.to_s)
-            result = if message.replyable?
-              Protocol::Message.deserialize(socket, max_message_size, message.request_id)
-            else
-              nil
+            result = add_server_diagnostics do
+              socket.write(buffer.to_s)
+              if message.replyable?
+                Protocol::Message.deserialize(socket, max_message_size, message.request_id, options)
+              else
+                nil
+              end
             end
           rescue Exception => e
             total_duration = Time.now - start
@@ -118,13 +176,42 @@ module Mongo
             total_duration = Time.now - start
             command_completed(result, address, operation_id, message.payload, total_duration)
           end
+          if client && result
+            result = result.maybe_decrypt(client)
+          end
           result
         end
       end
 
-      def serialize(message, buffer = BSON::ByteBuffer.new)
+      def serialize(message, client, buffer = BSON::ByteBuffer.new)
         start_size = 0
-        message.compress!(compressor, options[:zlib_compression_level]).serialize(buffer, max_bson_object_size)
+        final_message = message.maybe_compress(compressor, options[:zlib_compression_level])
+
+        # Driver specifications only mandate the fixed 16MiB limit for
+        # serialized BSON documents. However, the server returns its
+        # active serialized BSON document size limit in the ismaster response,
+        # which is +max_bson_object_size+ below. The +DEFAULT_MAX_BSON_OBJECT_SIZE+
+        # is the 16MiB value mandated by the specifications which we use
+        # only as the default if the server's ismaster did not contain
+        # maxBsonObjectSize.
+        max_bson_size = max_bson_object_size || DEFAULT_MAX_BSON_OBJECT_SIZE
+        if client && client.encrypter && client.encrypter.encrypt?
+          # The client-side encryption specification requires bulk writes to
+          # be split at a reduced maxBsonObjectSize. If this message is a bulk
+          # write and its size exceeds the reduced size limit, the serializer
+          # will raise an exception, which is caught by BulkWrite. BulkWrite
+          # will split the operation into individual writes, which will
+          # not be subject to the reduced maxBsonObjectSize.
+          if message.bulk_write?
+            # Make the new maximum size equal to the specified reduced size
+            # limit plus the 16KiB overhead allowance.
+            max_bson_size = REDUCED_MAX_BSON_SIZE + MAX_BSON_COMMAND_OVERHEAD
+          end
+        else
+          max_bson_size += MAX_BSON_COMMAND_OVERHEAD
+        end
+
+        final_message.serialize(buffer, max_bson_size)
         if max_message_size &&
           (buffer.length - start_size) > max_message_size
         then

data/lib/mongo/server/connection_common.rb

@@ -0,0 +1,135 @@
+# Copyright (C) 2020 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  class Server
+
+    # Common methods used by both monitoring and non-monitoring connections.
+    #
+    # @note Although methods of this module are part of the public API,
+    #   the fact that these methods are defined on this module and not on
+    #   the classes which include this module is not part of the public API.
+    #
+    # @api semipublic
+    class ConnectionCommon
+
+      # The compressor negotiated during the handshake for this connection,
+      # if any.
+      #
+      # This attribute is nil for connections that haven't completed the
+      # handshake yet, and for connections that negotiated no compression.
+      #
+      # @return [ String | nil ] The compressor.
+      attr_reader :compressor
+
+      # Determine if the connection is currently connected.
+      #
+      # @example Is the connection connected?
+      #   connection.connected?
+      #
+      # @return [ true, false ] If connected.
+      #
+      # @deprecated
+      def connected?
+        !!socket
+      end
+
+      # @return [ Integer ] pid The process id when the connection was created.
+      # @api private
+      attr_reader :pid
+
+      private
+
+      attr_reader :socket
+
+      def set_compressor!(reply)
+        server_compressors = reply['compression']
+
+        if options[:compressors]
+          if intersection = (server_compressors & options[:compressors])
+            @compressor = intersection.first
+          else
+            msg = if server_compressors
+              "The server at #{address} has no compression algorithms in common with those requested. " +
+                "Server algorithms: #{server_compressors.join(', ')}; " +
+                "Requested algorithms: #{options[:compressors].join(', ')}. " +
+                "Compression will not be used"
+            else
+              "The server at #{address} did not advertise compression support. " +
+                "Requested algorithms: #{options[:compressors].join(', ')}. " +
+                "Compression will not be used"
+            end
+            log_warn(msg)
+          end
+        end
+      end
+
+      # Yields to the block and, if the block raises an exception, adds a note
+      # to the exception with the address of the specified server.
+      #
+      # This method is intended to add server address information to exceptions
+      # raised during execution of operations on servers.
+      def add_server_diagnostics
+        yield
+      # Note that the exception should already have been mapped to a
+      # Mongo::Error subclass when it gets to this method.
+      rescue Error::SocketError, Error::SocketTimeoutError => e
+        # Server::Monitor::Connection does not reference its server, but
+        # knows its address. Server::Connection delegates the address to its
+        # server.
+        note = "on #{address.seed}"
+        if respond_to?(:id)
+          note << ", connection #{generation}:#{id}"
+        end
+        e.add_note(note)
+        if respond_to?(:generation)
+          # Non-monitoring connections
+          e.generation = generation
+        end
+        raise e
+      end
+
+      def ssl_options
+        @ssl_options ||= if options[:ssl]
+          options.select { |k, v| k.to_s.start_with?('ssl') }
+        else
+          # Due to the way options are propagated from the client, if we
+          # decide that we don't want to use TLS we need to have the ssl
+          # options explicitly set to false or the value provided to the
+          # connection might be overwritten by the default inherited from
+          # the client.
+          {ssl: false}
+        end.freeze
+      end
+
+      def ensure_connected
+        begin
+          unless socket
+            raise ArgumentError, "Connection #{generation}:#{id} for #{address.seed} is not connected"
+          end
+          if @error
+            raise Error::ConnectionPerished, "Connection #{generation}:#{id} for #{address.seed} is perished"
+          end
+          result = yield socket
+          success = true
+          result
+        ensure
+          unless success
+            @error = true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mongo/server/connection_pool.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2019 MongoDB, Inc.
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -275,6 +275,8 @@ module Mongo
       #
       # @since 2.9.0
       def check_out
+        check_invariants
+
         publish_cmap_event(
           Monitoring::Event::Cmap::ConnectionCheckOutStarted.new(@server.address)
         )
@@ -290,6 +292,7 @@ module Mongo
         end
 
         deadline = Time.now + wait_timeout
+        pid = Process.pid
         connection = nil
         # It seems that synchronize sets up its own loop, thus a simple break
         # is insufficient to break the outer loop
@@ -302,6 +305,13 @@ module Mongo
               until @available_connections.empty?
                 connection = @available_connections.pop
 
+                if connection.pid != pid
+                  log_warn("Detected PID change - Mongo client should have been reconnected (old pid #{connection.pid}, new pid #{pid}")
+                  connection.disconnect!(reason: :stale)
+                  @populate_semaphore.signal
+                  next
+                end
+
                 if connection.generation != generation
                   # Stale connections should be disconnected in the clear
                   # method, but if any don't, check again here
@@ -345,7 +355,8 @@ module Mongo
                   "from pool for #{@server.address} after #{wait_timeout} sec. " +
                   "Connections in pool: #{@available_connections.length} available, " +
                   "#{@checked_out_connections.length} checked out, " +
-                  "#{@pending_connections.length} pending"
+                  "#{@pending_connections.length} pending " +
+                  "(max size: #{max_size})"
               end
               raise Error::ConnectionCheckOutTimeout.new(msg, address: @server.address)
             end
@@ -379,7 +390,16 @@ module Mongo
         publish_cmap_event(
           Monitoring::Event::Cmap::ConnectionCheckedOut.new(@server.address, connection.id, self),
         )
+
+        if Lint.enabled?
+          unless connection.connected?
+            raise Error::LintError, "Connection pool for #{address} checked out a disconnected connection #{connection.generation}:#{connection.id}"
+          end
+        end
+
         connection
+      ensure
+        check_invariants
       end
 
       # Check a connection back into the pool.
@@ -390,6 +410,8 @@ module Mongo
       #
       # @since 2.9.0
       def check_in(connection)
+        check_invariants
+
         @lock.synchronize do
           unless connection.connection_pool == self
             raise ArgumentError, "Trying to check in a connection which was not checked out by this pool: #{connection} checked out from pool #{connection.connection_pool} (for #{self})"
@@ -410,6 +432,11 @@ module Mongo
             Monitoring::Event::Cmap::ConnectionCheckedIn.new(@server.address, connection.id, self)
           )
 
+          if connection.error?
+            connection.disconnect!(reason: :error)
+            return
+          end
+
           if closed?
             connection.disconnect!(reason: :pool_closed)
             return
@@ -431,6 +458,8 @@ module Mongo
             @available_semaphore.signal
           end
         end
+      ensure
+        check_invariants
       end
 
       # Closes all idle connections in the pool and schedules currently checked
@@ -450,6 +479,8 @@ module Mongo
       def clear(options = nil)
         raise_if_closed!
 
+        check_invariants
+
         if options && options[:stop_populator]
           stop_populator
         end
@@ -471,6 +502,8 @@ module Mongo
         end
 
         true
+      ensure
+        check_invariants
       end
 
       # @since 2.1.0
@@ -734,6 +767,28 @@ module Mongo
           raise
         end
       end
+
+      def check_invariants
+        return unless Lint.enabled?
+
+        # Server summary calls pool summary which requires pool lock -> deadlock.
+        # Obtain the server summary ahead of time.
+        server_summary = @server.summary
+
+        @lock.synchronize do
+          @available_connections.each do |connection|
+            if connection.closed?
+              raise Error::LintError, "Available connection is closed: #{connection} for #{server_summary}"
+            end
+          end
+
+          @pending_connections.each do |connection|
+            if connection.closed?
+              raise Error::LintError, "Pending connection is closed: #{connection} for #{server_summary}"
+            end
+          end
+        end
+      end
     end
   end
 end