mongo 2.13.0.rc1 → 2.13.3

data/lib/mongo/operation/parallel_scan/command.rb

@@ -37,8 +37,7 @@ module Mongo
               read_concern)
           end
           sel[:maxTimeMS] = max_time_ms if max_time_ms
-          update_selector_for_read_pref(sel, connection)
-          sel
+          add_read_preference_legacy(sel, connection)
         end
 
         def message(connection)

data/lib/mongo/operation/shared/read_preference_supported.rb

@@ -36,47 +36,44 @@ module Mongo
       #
       # @since 2.0.0
       def options(connection)
-        add_slave_ok_flag_maybe(super, connection)
+        options = super
+        if add_slave_ok_flag?(connection)
+          flags = options[:flags]&.dup || []
+          flags << :slave_ok
+          options = options.merge(flags: flags)
+        end
+        options
       end
 
-      # Adds :slave_ok flag to options based on the read preference specified
-      # in the operation or implied by the topology that the connection's
-      # server is a part of.
+      # Whether to add the :slave_ok flag to the request based on the
+      # read preference specified in the operation or implied by the topology
+      # that the connection's server is a part of.
       #
-      # @param [ Hash ] options The options calculated so far.
       # @param [ Server::Connection ] connection The connection that the
       #   operation will be executed on.
       #
-      # @return [ Hash ] The new options.
-      def add_slave_ok_flag_maybe(options, connection)
-        add_flag =
-          # https://github.com/mongodb/specifications/blob/master/source/server-selection/server-selection.rst#topology-type-single
-          if connection.description.standalone?
-            # Read preference is never sent to standalones.
-            false
-          elsif connection.server.cluster.single?
-            # In Single topology the driver forces primaryPreferred read
-            # preference mode (via the slave_ok flag, in case of old servers)
-            # so that the query is satisfied.
-            true
-          else
-            # In replica sets and sharded clusters, read preference is passed
-            # to the server if one is specified by the application, and there
-            # is no default.
-            read && read.slave_ok?
-          end
-
-        if add_flag
-          options= options.dup
-          (options[:flags] ||= []) << :slave_ok
+      # @return [ true | false ] Whether the :slave_ok flag should be added.
+      def add_slave_ok_flag?(connection)
+        # https://github.com/mongodb/specifications/blob/master/source/server-selection/server-selection.rst#topology-type-single
+        if connection.description.standalone?
+          # Read preference is never sent to standalones.
+          false
+        elsif connection.server.cluster.single?
+          # In Single topology the driver forces primaryPreferred read
+          # preference mode (via the slave_ok flag, in case of old servers)
+          # so that the query is satisfied.
+          true
+        else
+          # In replica sets and sharded clusters, read preference is passed
+          # to the server if one is specified by the application, and there
+          # is no default.
+          read && read.slave_ok? || false
         end
-
-        options
       end
 
       def command(connection)
         sel = super
-        update_selector_for_read_pref(sel, connection)
+        add_read_preference_legacy(sel, connection)
       end
 
       # Adds $readPreference field to the command document.
@@ -95,14 +92,19 @@ module Mongo
       #   operation will be executed on.
       #
       # @return [ Hash ] New command document to send to the server.
-      def update_selector_for_read_pref(sel, connection)
+      def add_read_preference_legacy(sel, connection)
         if read && connection.description.mongos? && read_pref = read.to_mongos
-          Mongo::Lint.validate_camel_case_read_preference(read_pref)
-          sel = sel[:$query] ? sel : {:$query => sel}
-          sel = sel.merge(:$readPreference => read_pref)
-        else
-          sel
+          # If the read preference contains only mode and mode is secondary
+          # preferred and we are sending to a pre-OP_MSG server, this read
+          # preference is indicated by the :slave_ok wire protocol flag
+          # and $readPreference command parameter isn't sent.
+          if read_pref != {mode: 'secondaryPreferred'}
+            Mongo::Lint.validate_camel_case_read_preference(read_pref)
+            sel = sel[:$query] ? sel : {:$query => sel}
+            sel = sel.merge(:$readPreference => read_pref)
+          end
         end
+        sel
       end
     end
   end

data/lib/mongo/operation/shared/sessions_supported.rb

@@ -170,9 +170,10 @@ module Mongo
         elsif connection.description.mongos?
           # When server is a mongos:
           # - $readPreference is never sent when mode is 'primary'
-          # - When mode is 'secondaryPreferred' $readPreference is only sent
-          #   when a non-mode field (i.e. tag_sets) is present
           # - Otherwise $readPreference is sent
+          # When mode is 'secondaryPreferred' $readPreference is currently
+          # required to only be sent when a non-mode field (i.e. tag_sets)
+          # is present, but this causes wrong behavior (DRIVERS-1642).
           if read
             doc = read.to_mongos
             if doc

data/lib/mongo/protocol/message.rb

@@ -177,10 +177,19 @@ module Mongo
       #
       # @param buffer [String] buffer where the message should be inserted
       # @return [String] buffer containing the serialized message
-      def serialize(buffer = BSON::ByteBuffer.new, max_bson_size = nil)
+      def serialize(buffer = BSON::ByteBuffer.new, max_bson_size = nil, bson_overhead = nil)
+        max_size =
+          if max_bson_size && bson_overhead
+            max_bson_size + bson_overhead
+          elsif max_bson_size
+            max_bson_size
+          else
+            nil
+          end
+
         start = buffer.length
         serialize_header(buffer)
-        serialize_fields(buffer, max_bson_size)
+        serialize_fields(buffer, max_size)
         buffer.replace_int32(start, buffer.length - start)
       end
 

data/lib/mongo/protocol/msg.rb

@@ -43,8 +43,8 @@ module Mongo
       #   Msg.new([:more_to_come], {}, { ismaster: 1 },
       #           { type: 1, payload: { identifier: 'documents', sequence: [..] } })
       #
-      # @param [ Array<Symbol> ] flags The flag bits. Current supported values
-      #   are :more_to_come and :checksum_present.
+      # @param [ Array<Symbol> ] flags The flag bits. Currently supported
+      #   values are :more_to_come and :checksum_present.
       # @param [ Hash ] options The options.
       # @param [ BSON::Document, Hash ] main_document The document that will
       #   become the payload type 0 section. Can contain global args as they
@@ -146,7 +146,9 @@ module Mongo
       # @return [ BSON::ByteBuffer ] buffer containing the serialized message.
       #
       # @since 2.5.0
-      def serialize(buffer = BSON::ByteBuffer.new, max_bson_size = nil)
+      def serialize(buffer = BSON::ByteBuffer.new, max_bson_size = nil, bson_overhead = nil)
+        validate_document_size!(max_bson_size)
+
         super
         add_check_sum(buffer)
         buffer
@@ -275,6 +277,23 @@ module Mongo
 
       private
 
+      # Validate that the documents in this message are all smaller than the
+      # maxBsonObjectSize. If not, raise an exception.
+      def validate_document_size!(max_bson_size)
+        max_bson_size ||= Mongo::Server::ConnectionBase::DEFAULT_MAX_BSON_OBJECT_SIZE
+
+        contains_too_large_document = @sections.any? do |section|
+          section[:type] == 1 &&
+            section[:payload][:sequence].any? do |document|
+              document.to_bson.length > max_bson_size
+            end
+        end
+
+        if contains_too_large_document
+          raise Error::MaxBSONSize.new('The document exceeds maximum allowed BSON object size after serialization')
+        end
+      end
+
       def command
         @command ||= if @main_document
           @main_document.dup.tap do |cmd|

data/lib/mongo/protocol/query.rb

@@ -46,18 +46,18 @@ module Mongo
       # @example Find all user ids.
       #   Query.new('xgen', 'users', {}, :fields => {:id => 1})
       #
-      # @param database [String, Symbol] The database to query.
-      # @param collection [String, Symbol] The collection to query.
-      # @param selector [Hash] The query selector.
-      # @param options [Hash] The additional query options.
+      # @param [ String, Symbol ] database The database to query.
+      # @param [ String, Symbol ] collection The collection to query.
+      # @param [ Hash ] selector The query selector.
+      # @param [ Hash ] options The additional query options.
       #
-      # @option options :project [Hash] The projection.
-      # @option options :skip [Integer] The number of documents to skip.
-      # @option options :limit [Integer] The number of documents to return.
-      # @option options :flags [Array] The flags for the query message.
-      #
-      #   Supported flags: +:tailable_cursor+, +:slave_ok+, +:oplog_replay+,
-      #   +:no_cursor_timeout+, +:await_data+, +:exhaust+, +:partial+
+      # @option options [ Array<Symbol> ] :flags The flag bits.
+      #   Currently supported values are :await_data, :exhaust,
+      #   :no_cursor_timeout, :oplog_replay, :partial, :slave_ok,
+      #   :tailable_cursor.
+      # @option options [ Integer ] :limit The number of documents to return.
+      # @option options [ Hash ] :project The projection.
+      # @option options [ Integer ] :skip The number of documents to skip.
       def initialize(database, collection, selector, options = {})
         @database = database
         @namespace = "#{database}.#{collection}"
@@ -115,12 +115,48 @@ module Mongo
         compress_if_possible(selector.keys.first, compressor, zlib_compression_level)
       end
 
+      # Serializes message into bytes that can be sent on the wire.
+      #
+      # @param [ BSON::ByteBuffer ] buffer where the message should be inserted.
+      # @param [ Integer ] max_bson_size The maximum bson object size.
+      #
+      # @return [ BSON::ByteBuffer ] buffer containing the serialized message.
+      def serialize(buffer = BSON::ByteBuffer.new, max_bson_size = nil, bson_overhead = nil)
+        validate_document_size!(max_bson_size)
+
+        super
+      end
+
       protected
 
       attr_reader :upconverter
 
       private
 
+      # Validate that the documents in this message are all smaller than the
+      # maxBsonObjectSize. If not, raise an exception.
+      def validate_document_size!(max_bson_size)
+        max_bson_size ||= Mongo::Server::ConnectionBase::DEFAULT_MAX_BSON_OBJECT_SIZE
+
+        documents = if @selector.key?(:documents)
+                      @selector[:documents]
+                    elsif @selector.key?(:deletes)
+                      @selector[:deletes]
+                    elsif @selector.key?(:updates)
+                      @selector[:updates]
+                    else
+                      []
+                    end
+
+        contains_too_large_document = documents.any? do |doc|
+          doc.to_bson.length > max_bson_size
+        end
+
+        if contains_too_large_document
+          raise Error::MaxBSONSize.new('The document exceeds maximum allowed BSON object size after serialization')
+        end
+      end
+
       # The operation code required to specify a Query message.
       # @return [Fixnum] the operation code.
       #

data/lib/mongo/server/app_metadata.rb

@@ -65,12 +65,17 @@ module Mongo
       #   the metadata printed to the mongod logs upon establishing a connection
       #   in server versions >= 3.4.
       # @option options [ String ] :user The user name.
+      # @option options [ Array<Hash> ] :wrapping_libraries Information about
+      #   libraries such as ODMs that are wrapping the driver. Specify the
+      #   lower level libraries first. Allowed hash keys: :name, :version,
+      #   :platform.
       #
       # @since 2.4.0
       def initialize(options)
         @app_name = options[:app_name].to_s if options[:app_name]
         @platform = options[:platform]
         @compressors = options[:compressors] || []
+        @wrapping_libraries = options[:wrapping_libraries]
 
         if options[:user] && !options[:auth_mech]
           auth_db = options[:auth_source] || 'admin'
@@ -78,6 +83,10 @@ module Mongo
         end
       end
 
+      # @return [ Array<Hash> | nil ] Information about libraries wrapping
+      #   the driver.
+      attr_reader :wrapping_libraries
+
       # Get the bytes of the ismaster message including this metadata.
       #
       # @api private
@@ -140,9 +149,17 @@ module Mongo
       end
 
       def driver_doc
+        names = [DRIVER_NAME]
+        versions = [Mongo::VERSION]
+        if wrapping_libraries
+          wrapping_libraries.each do |library|
+            names << library[:name] || ''
+            versions << library[:version] || ''
+          end
+        end
         {
-          name: DRIVER_NAME,
-          version: Mongo::VERSION
+          name: names.join('|'),
+          version: versions.join('|'),
         }
       end
 
@@ -175,12 +192,19 @@ module Mongo
           ruby_versions = ["Ruby #{RUBY_VERSION}"]
           platforms = [RUBY_PLATFORM]
         end
-        [
+        platform = [
           @platform,
           *ruby_versions,
           *platforms,
           RbConfig::CONFIG['build'],
         ].compact.join(', ')
+        platforms = [platform]
+        if wrapping_libraries
+          wrapping_libraries.each do |library|
+            platforms << library[:platform] || ''
+          end
+        end
+        platforms.join('|')
       end
     end
   end

data/lib/mongo/server/connection_base.rb

@@ -186,9 +186,6 @@ module Mongo
       end
 
       def serialize(message, client, buffer = BSON::ByteBuffer.new)
-        start_size = 0
-        final_message = message.maybe_compress(compressor, options[:zlib_compression_level])
-
         # Driver specifications only mandate the fixed 16MiB limit for
         # serialized BSON documents. However, the server returns its
         # active serialized BSON document size limit in the ismaster response,
@@ -207,18 +204,45 @@ module Mongo
           if message.bulk_write?
             # Make the new maximum size equal to the specified reduced size
             # limit plus the 16KiB overhead allowance.
-            max_bson_size = REDUCED_MAX_BSON_SIZE + MAX_BSON_COMMAND_OVERHEAD
+            max_bson_size = REDUCED_MAX_BSON_SIZE
           end
-        else
-          max_bson_size += MAX_BSON_COMMAND_OVERHEAD
         end
 
-        final_message.serialize(buffer, max_bson_size)
-        if max_message_size &&
-          (buffer.length - start_size) > max_message_size
-        then
-          raise Error::MaxMessageSize.new(max_message_size)
+        # RUBY-2234: It is necessary to check that the message size does not
+        # exceed the maximum bson object size before compressing and serializing
+        # the final message.
+        #
+        # This is to avoid the case where the user performs a bulk write
+        # larger than 16MiB which, when compressed, becomes smaller than 16MiB.
+        # If the driver does not split the bulk writes prior to compression,
+        # the entire operation will be sent to the server, which will raise an
+        # error because the uncompressed operation exceeds the maximum bson size.
+        #
+        # To address this problem, we serialize the message prior to compression
+        # and raise an exception if the serialized message exceeds the maximum
+        # bson size.
+        if max_message_size
+          # Create a separate buffer that contains the un-compressed message
+          # for the purpose of checking its size. Write any pre-existing contents
+          # from the original buffer into the temporary one.
+          temp_buffer = BSON::ByteBuffer.new
+
+          # TODO: address the fact that this line mutates the buffer.
+          temp_buffer.put_bytes(buffer.get_bytes(buffer.length))
+
+          message.serialize(temp_buffer, max_bson_size, MAX_BSON_COMMAND_OVERHEAD)
+          if temp_buffer.length > max_message_size
+            raise Error::MaxMessageSize.new(max_message_size)
+          end
         end
+
+        # RUBY-2335: When the un-compressed message is smaller than the maximum
+        # bson size limit, the message will be serialized twice. The operations
+        # layer should be refactored to allow compression on an already-
+        # serialized message.
+        final_message = message.maybe_compress(compressor, options[:zlib_compression_level])
+        final_message.serialize(buffer, max_bson_size, MAX_BSON_COMMAND_OVERHEAD)
+
         buffer
       end
     end

data/lib/mongo/server_selector/secondary_preferred.rb

@@ -86,13 +86,8 @@ module Mongo
       #
       # @since 2.0.0
       def to_mongos
-        if tag_sets.empty? && max_staleness.nil? && hedge.nil?
-          # The server preference is not sent to mongos as part of the query
-          # selector if there are no tag sets, for maximum backwards compatibility.
-          nil
-        else
-          to_doc
-        end
+        # Always send the read preference to mongos: DRIVERS-1642.
+        to_doc
       end
 
       private

data/lib/mongo/version.rb

@@ -17,5 +17,5 @@ module Mongo
   # The current version of the driver.
   #
   # @since 2.0.0
-  VERSION = '2.13.0.rc1'.freeze
+  VERSION = '2.13.3'.freeze
 end

data/spec/integration/bson_symbol_spec.rb

@@ -25,8 +25,10 @@ describe 'Symbol encoding to BSON' do
   end
 
   it 'round-trips symbol values using the same byte buffer' do
-    if BSON::Environment.jruby?
-      pending 'https://jira.mongodb.org/browse/RUBY-2128'
+    if BSON::Environment.jruby? && (BSON::VERSION.split('.').map(&:to_i) <=> [4, 11, 0]) < 0
+      skip 'This test is only relevant to bson versions that increment ByteBuffer '\
+       'read and write positions separately in JRuby, as implemented in ' \
+       'bson version 4.11.0. For more information, see https://jira.mongodb.org/browse/RUBY-2128'
     end
 
     Hash.from_bson(hash.to_bson).should == hash

data/spec/integration/bulk_write_spec.rb

@@ -0,0 +1,19 @@
+require 'spec_helper'
+
+describe 'Bulk writes' do
+  before do
+    authorized_collection.drop
+  end
+
+  context 'when bulk write is larger than 48MB' do
+    let(:operations) do
+      [ { insert_one: { text: 'a' * 1000 * 1000 } } ] * 48
+    end
+
+    it 'succeeds' do
+      expect do
+        authorized_collection.bulk_write(operations)
+      end.not_to raise_error
+    end
+  end
+end