mongo 2.11.4 → 2.12.3

data/lib/mongo/retryable.rb

@@ -144,7 +144,7 @@ module Mongo
     # @note This only retries read operations on socket errors.
     #
     # @param [ Hash ] options Options.
-    # @param [ Proc ] block The block to execute.
+    # @yield Calls the provided block with no arguments
     #
     # @option options [ String ] :retry_message Message to log when retrying.
     #

data/lib/mongo/server.rb

@@ -124,11 +124,16 @@ module Mongo
     # @deprecated
     def_delegators :monitor, :scan!
 
-    # The last compressor discovered by the server monitor.
+    # The compressor negotiated by the server monitor, if any.
     #
-    # The compressor state should be determined for each individual
-    # connection rather than kept per server. A future version of the
-    # driver will change how compressors are tracked and used.
+    # This attribute is nil if no server check has not yet completed, and if
+    # no compression was negatiated.
+    #
+    # @note Compression is negotiated for each connection separately.
+    #
+    # @return [ String | nil ] The negotiated compressor.
+    #
+    # @deprecated
     def compressor
       if monitor
         monitor.compressor
@@ -523,6 +528,7 @@ end
 
 require 'mongo/server/app_metadata'
 require 'mongo/server/connectable'
+require 'mongo/server/connection_common'
 require 'mongo/server/connection_base'
 require 'mongo/server/pending_connection'
 require 'mongo/server/connection'

data/lib/mongo/server/connection.rb

@@ -174,7 +174,7 @@ module Mongo
         unless @socket
           # When @socket is assigned, the socket should have handshaken and
           # authenticated and be usable.
-          @socket = do_connect
+          @socket, @description = do_connect
 
           publish_cmap_event(
             Monitoring::Event::Cmap::ConnectionReady.new(address, id)
@@ -186,13 +186,17 @@ module Mongo
       end
 
       # Separate method to permit easier mocking in the test suite.
+      #
+      # @return [ Array<Socket, Server::Description> ] Connected socket and
+      #   a server description instance from the ismaster response of the
+      #   returned socket.
       def do_connect
         socket = address.socket(socket_timeout, ssl_options, address.options)
 
         begin
-          handshake!(socket)
-          unless description.arbiter?
-            pending_connection = PendingConnection.new(socket, @server, monitoring, options.merge(id: id))
+          new_description = handshake!(socket)
+          unless new_description.arbiter?
+            pending_connection = PendingConnection.new(socket, new_description, @server, monitoring, options.merge(id: id))
             authenticate!(pending_connection)
           end
         rescue Exception
@@ -200,7 +204,7 @@ module Mongo
           raise
         end
 
-        socket
+        [socket, new_description]
       end
       private :do_connect
 
@@ -303,6 +307,8 @@ module Mongo
 
       private
 
+      # @return [ Server::Description ] The server description calculated from
+      #   ismaster response for this particular connection.
       def handshake!(socket)
         unless socket
           raise Error::HandshakeError, "Cannot handshake because there is no usable socket (for #{address})"
@@ -314,7 +320,7 @@ module Mongo
             response, exc, rtt, average_rtt =
               @server.round_trip_time_averager.measure do
                 socket.write(app_metadata.ismaster_bytes)
-                Protocol::Message.deserialize(socket, max_message_size).documents[0]
+                Protocol::Message.deserialize(socket, Protocol::Message::MAX_MESSAGE_SIZE).documents[0]
               end
 
             if exc
@@ -330,6 +336,9 @@ module Mongo
       end
 
       # This is a separate method to keep the nesting level down.
+      #
+      # @return [ Server::Description ] The server description calculated from
+      #   ismaster response for this particular connection.
       def post_handshake(response, average_rtt)
         if response["ok"] == 1
           # Auth mechanism is entirely dependent on the contents of
@@ -359,12 +368,14 @@ module Mongo
               :mongodb_cr
             end
           end
+          set_compressor!(response)
         else
           @auth_mechanism = nil
         end
 
-        @description = Description.new(address, response, average_rtt)
-        @server.cluster.run_sdam_flow(@server.description, @description)
+        Description.new(address, response, average_rtt).tap do |new_description|
+          @server.cluster.run_sdam_flow(@server.description, new_description)
+        end
       end
 
       def authenticate!(pending_connection)
@@ -389,7 +400,7 @@ module Mongo
         @auth_mechanism || (@server.features.scram_sha_1_enabled? ? :scram : :mongodb_cr)
       end
 
-      def deliver(message)
+      def deliver(message, client, options = {})
         begin
           super
         # Important: timeout errors are not handled here

data/lib/mongo/server/connection_base.rb

@@ -22,10 +22,30 @@ module Mongo
     #   the classes which include this module is not part of the public API.
     #
     # @api semipublic
-    class ConnectionBase
+    class ConnectionBase < ConnectionCommon
       extend Forwardable
       include Monitoring::Publishable
 
+      # The maximum allowed size in bytes that a user-supplied document may
+      # take up when serialized, if the server's ismaster response does not
+      # include maxBsonObjectSize field.
+      #
+      # The commands that are sent to the server may exceed this size by
+      # MAX_BSON_COMMAND_OVERHEAD.
+      #
+      # @api private
+      DEFAULT_MAX_BSON_OBJECT_SIZE = 16777216
+
+      # The additional overhead allowed for command data (i.e. fields added
+      # to the command document by the driver, as opposed to documents
+      # provided by the user) when serializing a complete command to BSON.
+      #
+      # @api private
+      MAX_BSON_COMMAND_OVERHEAD = 16384
+
+      # @api private
+      REDUCED_MAX_BSON_SIZE = 2097152
+
       # @return [ Hash ] options The passed in options.
       attr_reader :options
 
@@ -38,14 +58,31 @@ module Mongo
       def_delegators :server, :address
 
       def_delegators :server,
-                     :features,
-                     :max_bson_object_size,
-                     :max_message_size,
-                     :mongos?,
-                     :compressor,
                      :cluster_time,
                      :update_cluster_time
 
+      # Returns the server description for this connection, derived from
+      # the isMaster response for the handshake performed on this connection.
+      #
+      # @note A connection object that hasn't yet connected (handshaken and
+      #   authenticated, if authentication is required) does not have a
+      #   description. While handshaking and authenticating the driver must
+      #   be using global defaults, in particular not assuming that the
+      #   properties of a particular connection are the same as properties
+      #   of other connections made to the same address (since the server
+      #   on the other end could have been shut down and a different server
+      #   version could have been launched).
+      #
+      # @return [ Server::Description ] Server description for this connection.
+      # @api private
+      attr_reader :description
+
+      def_delegators :description,
+        :features,
+        :max_bson_object_size,
+        :max_message_size,
+        :mongos?
+
       def app_metadata
         @app_metadata ||= begin
           same = true
@@ -78,11 +115,16 @@ module Mongo
       # @param [ Array<Message> ] messages A one-element array containing
       #   the message to dispatch.
       # @param [ Integer ] operation_id The operation id to link messages.
+      # @param [ Hash ] options
+      #
+      # @option options [ Boolean ] :deserialize_as_bson Whether to deserialize
+      #   the response to this message using BSON objects in place of native
+      #   Ruby types wherever possible.
       #
       # @return [ Protocol::Message | nil ] The reply if needed.
       #
       # @since 2.0.0
-      def dispatch(messages, operation_id = nil)
+      def dispatch(messages, operation_id = nil, client = nil, options = {})
         # The monitoring code does not correctly handle multiple messages,
         # and the driver internally does not send more than one message at
         # a time ever. Thus prohibit multiple message use for now.
@@ -90,13 +132,16 @@ module Mongo
           raise ArgumentError, 'Can only dispatch one message at a time'
         end
         message = messages.first
-        deliver(message)
+        deliver(message, client, options)
       end
 
       private
 
-      def deliver(message)
-        buffer = serialize(message)
+      def deliver(message, client, options = {})
+        if Lint.enabled? && !@socket
+          raise Error::LintError, "Trying to deliver a message over a disconnected connection (to #{address})"
+        end
+        buffer = serialize(message, client)
         ensure_connected do |socket|
           operation_id = Monitoring.next_operation_id
           command_started(address, operation_id, message.payload,
@@ -106,7 +151,7 @@ module Mongo
           begin
             socket.write(buffer.to_s)
             result = if message.replyable?
-              Protocol::Message.deserialize(socket, max_message_size, message.request_id)
+              Protocol::Message.deserialize(socket, max_message_size, message.request_id, options)
             else
               nil
             end
@@ -118,18 +163,73 @@ module Mongo
             total_duration = Time.now - start
             command_completed(result, address, operation_id, message.payload, total_duration)
           end
+          if client && result
+            result = result.maybe_decrypt(client)
+          end
           result
         end
       end
 
-      def serialize(message, buffer = BSON::ByteBuffer.new)
-        start_size = 0
-        message.compress!(compressor, options[:zlib_compression_level]).serialize(buffer, max_bson_object_size)
-        if max_message_size &&
-          (buffer.length - start_size) > max_message_size
-        then
-          raise Error::MaxMessageSize.new(max_message_size)
+      def serialize(message, client, buffer = BSON::ByteBuffer.new)
+        # Driver specifications only mandate the fixed 16MiB limit for
+        # serialized BSON documents. However, the server returns its
+        # active serialized BSON document size limit in the ismaster response,
+        # which is +max_bson_object_size+ below. The +DEFAULT_MAX_BSON_OBJECT_SIZE+
+        # is the 16MiB value mandated by the specifications which we use
+        # only as the default if the server's ismaster did not contain
+        # maxBsonObjectSize.
+        max_bson_size = max_bson_object_size || DEFAULT_MAX_BSON_OBJECT_SIZE
+        if client && client.encrypter && client.encrypter.encrypt?
+          # The client-side encryption specification requires bulk writes to
+          # be split at a reduced maxBsonObjectSize. If this message is a bulk
+          # write and its size exceeds the reduced size limit, the serializer
+          # will raise an exception, which is caught by BulkWrite. BulkWrite
+          # will split the operation into individual writes, which will
+          # not be subject to the reduced maxBsonObjectSize.
+          if message.bulk_write?
+            # Make the new maximum size equal to the specified reduced size
+            # limit plus the 16KiB overhead allowance.
+            max_bson_size = REDUCED_MAX_BSON_SIZE + MAX_BSON_COMMAND_OVERHEAD
+          end
+        else
+          max_bson_size += MAX_BSON_COMMAND_OVERHEAD
+        end
+
+        # RUBY-2234: It is necessary to check that the message size does not
+        # exceed the maximum bson object size before compressing and serializing
+        # the final message.
+        #
+        # This is to avoid the case where the user performs a bulk write
+        # larger than 16MiB which, when compressed, becomes smaller than 16MiB.
+        # If the driver does not split the bulk writes prior to compression,
+        # the entire operation will be sent to the server, which will raise an
+        # error because the uncompressed operation exceeds the maximum bson size.
+        #
+        # To address this problem, we serialize the message prior to compression
+        # and raise an exception if the serialized message exceeds the maximum
+        # bson size.
+        if max_message_size
+          # Create a separate buffer that contains the un-compressed message
+          # for the purpose of checking its size. Write any pre-existing contents
+          # from the original buffer into the temporary one.
+          temp_buffer = BSON::ByteBuffer.new
+
+          # TODO: address the fact that this line mutates the buffer.
+          temp_buffer.put_bytes(buffer.get_bytes(buffer.length))
+
+          message.serialize(temp_buffer, max_bson_size)
+          if temp_buffer.length > max_message_size
+            raise Error::MaxMessageSize.new(max_message_size)
+          end
         end
+
+        # RUBY-2335: When the un-compressed message is smaller than the maximum
+        # bson size limit, the message will be serialized twice. The operations
+        # layer should be refactored to allow compression on an already-
+        # serialized message.
+        final_message = message.maybe_compress(compressor, options[:zlib_compression_level])
+        final_message.serialize(buffer, max_bson_size)
+
         buffer
       end
     end

data/lib/mongo/server/connection_common.rb

@@ -0,0 +1,61 @@
+# Copyright (C) 2020 MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  class Server
+
+    # Common methods used by both monitoring and non-monitoring connections.
+    #
+    # @note Although methods of this module are part of the public API,
+    #   the fact that these methods are defined on this module and not on
+    #   the classes which include this module is not part of the public API.
+    #
+    # @api semipublic
+    class ConnectionCommon
+
+      # The compressor negotiated during the handshake for this connection,
+      # if any.
+      #
+      # This attribute is nil for connections that haven't completed the
+      # handshake yet, and for connections that negotiated no compression.
+      #
+      # @return [ String | nil ] The compressor.
+      attr_reader :compressor
+
+      private
+
+      def set_compressor!(reply)
+        server_compressors = reply['compression']
+
+        if options[:compressors]
+          if intersection = (server_compressors & options[:compressors])
+            @compressor = intersection.first
+          else
+            msg = if server_compressors
+              "The server at #{address} has no compression algorithms in common with those requested. " +
+                "Server algorithms: #{server_compressors.join(', ')}; " +
+                "Requested algorithms: #{options[:compressors].join(', ')}. " +
+                "Compression will not be used"
+            else
+              "The server at #{address} did not advertise compression support. " +
+                "Requested algorithms: #{options[:compressors].join(', ')}. " +
+                "Compression will not be used"
+            end
+            log_warn(msg)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mongo/server/connection_pool.rb

@@ -275,6 +275,8 @@ module Mongo
       #
       # @since 2.9.0
       def check_out
+        check_invariants
+
         publish_cmap_event(
           Monitoring::Event::Cmap::ConnectionCheckOutStarted.new(@server.address)
         )
@@ -345,7 +347,8 @@ module Mongo
                   "from pool for #{@server.address} after #{wait_timeout} sec. " +
                   "Connections in pool: #{@available_connections.length} available, " +
                   "#{@checked_out_connections.length} checked out, " +
-                  "#{@pending_connections.length} pending"
+                  "#{@pending_connections.length} pending " +
+                  "(max size: #{max_size})"
               end
               raise Error::ConnectionCheckOutTimeout.new(msg, address: @server.address)
             end
@@ -379,7 +382,10 @@ module Mongo
         publish_cmap_event(
           Monitoring::Event::Cmap::ConnectionCheckedOut.new(@server.address, connection.id, self),
         )
+
         connection
+      ensure
+        check_invariants
       end
 
       # Check a connection back into the pool.
@@ -390,6 +396,8 @@ module Mongo
       #
       # @since 2.9.0
       def check_in(connection)
+        check_invariants
+
         @lock.synchronize do
           unless connection.connection_pool == self
             raise ArgumentError, "Trying to check in a connection which was not checked out by this pool: #{connection} checked out from pool #{connection.connection_pool} (for #{self})"
@@ -431,6 +439,8 @@ module Mongo
             @available_semaphore.signal
           end
         end
+      ensure
+        check_invariants
       end
 
       # Closes all idle connections in the pool and schedules currently checked
@@ -450,6 +460,8 @@ module Mongo
       def clear(options = nil)
         raise_if_closed!
 
+        check_invariants
+
         if options && options[:stop_populator]
           stop_populator
         end
@@ -471,6 +483,8 @@ module Mongo
         end
 
         true
+      ensure
+        check_invariants
       end
 
       # @since 2.1.0
@@ -734,6 +748,28 @@ module Mongo
           raise
         end
       end
+
+      def check_invariants
+        return unless Lint.enabled?
+
+        # Server summary calls pool summary which requires pool lock -> deadlock.
+        # Obtain the server summary ahead of time.
+        server_summary = @server.summary
+
+        @lock.synchronize do
+          @available_connections.each do |connection|
+            if connection.closed?
+              raise Error::LintError, "Available connection is closed: #{connection} for #{server_summary}"
+            end
+          end
+
+          @pending_connections.each do |connection|
+            if connection.closed?
+              raise Error::LintError, "Pending connection is closed: #{connection} for #{server_summary}"
+            end
+          end
+        end
+      end
     end
   end
 end