mongo 2.11.4 → 2.12.3

data/lib/mongo/protocol/msg.rb

@@ -35,7 +35,7 @@ module Mongo
       # moved to the end of the hash for better logging.
       #
       # @api private
-      INTERNAL_KEYS = Set.new(%w($clusterTime lsid signature txnNumber)).freeze
+      INTERNAL_KEYS = Set.new(%w($clusterTime $db lsid signature txnNumber)).freeze
 
       # Creates a new OP_MSG protocol message
       #
@@ -44,25 +44,42 @@ module Mongo
       #           { type: 1, payload: { identifier: 'documents', sequence: [..] } })
       #
       # @param [ Array<Symbol> ] flags The flag bits. Current supported values
-      # are :more_to_come and :checksum_present.
-      # @param [ Hash ] options The options. There are currently no supported
-      #   options, this is a placeholder for the future.
-      # @param [ BSON::Document, Hash ] global_args The global arguments,
-      #   becomes a section of payload type 0.
-      # @param [ BSON::Document, Hash ] sections Zero or more sections, in the format
-      #   { type: 1, payload: { identifier: <String>, sequence: <Array<BSON::Document, Hash>> } } or
-      #   { type: 0, payload: <BSON::Document, Hash> }
+      #   are :more_to_come and :checksum_present.
+      # @param [ Hash ] options The options.
+      # @param [ BSON::Document, Hash ] main_document The document that will
+      #   become the payload type 0 section. Can contain global args as they
+      #   are defined in the OP_MSG specification.
+      # @param [ Protocol::Msg::Section1 ] sequences Zero or more payload type 1
+      #   sections.
       #
-      # @option options [ true, false ] validating_keys Whether keys should be validated.
+      # @option options [ true, false ] validating_keys Whether keys should be
+      #   validated for being valid document keys (i.e. not begin with $ and
+      #   not contain dots).
       #
       # @api private
       #
       # @since 2.5.0
-      def initialize(flags, options, global_args, *sections)
+      def initialize(flags, options, main_document, *sequences)
         @flags = flags || []
         @options = options
-        @global_args = global_args
-        @sections = [ { type: 0, payload: global_args } ] + sections
+        unless main_document.is_a?(Hash)
+          raise ArgumentError, "Main document must be a Hash, given: #{main_document.class}"
+        end
+        @main_document = main_document
+        sequences.each_with_index do |section, index|
+          unless section.is_a?(Section1)
+            raise ArgumentError, "All sequences must be Section1 instances, got: #{section} at index #{index}"
+          end
+        end
+        @sequences = sequences
+        @sections = [
+          {type: 0, payload: @main_document}
+        ] + @sequences.map do |section|
+          {type: 1, payload: {
+            identifier: section.identifier,
+            sequence: section.documents,
+          }}
+        end
         @request_id = nil
         super
       end
@@ -88,11 +105,11 @@ module Mongo
       #
       # @since 2.5.0
       def payload
-        # Reorder keys in global_args for better logging - see
+        # Reorder keys in main_document for better logging - see
         # https://jira.mongodb.org/browse/RUBY-1591.
         # Note that even without the reordering, the payload is not an exact
         # match to what is sent over the wire because the command as used in
-        # the published eent combines keys from multiple sections of the
+        # the published event combines keys from multiple sections of the
         # payload sent over the wire.
         ordered_command = {}
         skipped_command = {}
@@ -107,10 +124,10 @@ module Mongo
 
         BSON::Document.new(
           command_name: ordered_command.keys.first.to_s,
-          database_name: global_args[DATABASE_IDENTIFIER],
+          database_name: @main_document[DATABASE_IDENTIFIER],
           command: ordered_command,
           request_id: request_id,
-          reply: sections[0]
+          reply: @main_document,
         )
       end
 
@@ -123,40 +140,163 @@ module Mongo
       #
       # @since 2.5.0
       def serialize(buffer = BSON::ByteBuffer.new, max_bson_size = nil)
+        validate_document_size!(max_bson_size)
+
         super
         add_check_sum(buffer)
         buffer
       end
 
-      # Compress this message.
+      # Compress the message, if the command being sent permits compression.
+      # Otherwise returns self.
       #
       # @param [ String, Symbol ] compressor The compressor to use.
       # @param [ Integer ] zlib_compression_level The zlib compression level to use.
       #
-      # @return [ Compressed, self ] A Protocol::Compressed message or self, depending on whether
-      #  this message can be compressed.
+      # @return [ Message ] A Protocol::Compressed message or self,
+      #  depending on whether this message can be compressed.
       #
       # @since 2.5.0
-      def compress!(compressor, zlib_compression_level = nil)
-        if compressor && compression_allowed?(command.keys.first)
-          Compressed.new(self, compressor, zlib_compression_level)
+      # @api private
+      def maybe_compress(compressor, zlib_compression_level = nil)
+        compress_if_possible(command.keys.first, compressor, zlib_compression_level)
+      end
+
+      # Reverse-populates the instance variables after deserialization sets
+      # the @sections instance variable to the list of documents.
+      #
+      # TODO fix deserialization so that this method is not needed.
+      #
+      # @api private
+      def fix_after_deserialization
+        if @sections.nil?
+          raise NotImplementedError, "After deserializations @sections should have been initialized"
+        end
+        if @sections.length != 1
+          raise NotImplementedError, "Deserialization must have produced exactly one section, but it produced #{sections.length} sections"
+        end
+        @main_document = @sections.first
+        @sequences = []
+        @sections = [{type: 0, payload: @main_document}]
+      end
+
+      def documents
+        [@main_document]
+      end
+
+      # Possibly encrypt this message with libmongocrypt. Message will only be
+      # encrypted if the specified client exists, that client has been given
+      # auto-encryption options, the client has not been instructed to bypass
+      # auto-encryption, and mongocryptd determines that this message is
+      # eligible for encryption. A message is eligible for encryption if it
+      # represents one of the command types whitelisted by libmongocrypt and it
+      # contains data that is required to be encrypted by a local or remote json schema.
+      #
+      # @param [ Mongo::Client | nil ] client The client used to make the original
+      #   command. This client may have auto-encryption options specified.
+      #
+      # @return [ Mongo::Protocol::Msg ] The encrypted message, or the original
+      #   message if encryption was not possible or necessary.
+      def maybe_encrypt(server, client)
+        # TODO verify compression happens later, i.e. when this method runs
+        # the message is not compressed.
+        if client && client.encrypter && client.encrypter.encrypt?
+          if server.max_wire_version < 8
+            raise Error::CryptError.new(
+              "Cannot perform encryption against a MongoDB server older than " +
+              "4.2 (wire version less than 8). Currently connected to server " +
+              "with max wire version #{server.max_wire_version}} " +
+              "(Auto-encryption requires a minimum MongoDB version of 4.2)"
+            )
+          end
+
+          db_name = @main_document[DATABASE_IDENTIFIER]
+          cmd = merge_sections
+          enc_cmd = client.encrypter.encrypt(db_name, cmd)
+          if cmd.key?('$db') && !enc_cmd.key?('$db')
+            enc_cmd['$db'] = cmd['$db']
+          end
+
+          Msg.new(@flags, @options, enc_cmd)
+        else
+          self
+        end
+      end
+
+      # Possibly decrypt this message with libmongocrypt. Message will only be
+      # decrypted if the specified client exists, that client has been given
+      # auto-encryption options, and this message is eligible for decryption.
+      # A message is eligible for decryption if it represents one of the command
+      # types whitelisted by libmongocrypt and it contains data that is required
+      # to be encrypted by a local or remote json schema.
+      #
+      # @param [ Mongo::Client | nil ] client The client used to make the original
+      #   command. This client may have auto-encryption options specified.
+      #
+      # @return [ Mongo::Protocol::Msg ] The decryption message, or the original
+      #   message if decryption was not possible or necessary.
+      def maybe_decrypt(client)
+        if client && client.encrypter
+          cmd = merge_sections
+          enc_cmd = client.encrypter.decrypt(cmd)
+          Msg.new(@flags, @options, enc_cmd)
         else
           self
         end
       end
 
+      # Whether this message represents a bulk write. A bulk write is an insert,
+      # update, or delete operation that encompasses multiple operations of
+      # the same type.
+      #
+      # @return [ Boolean ] Whether this message represents a bulk write.
+      #
+      # @note This method was written to support client-side encryption
+      #   functionality. It is not recommended that this method be used in
+      #   service of any other feature or behavior.
+      #
+      # @api private
+      def bulk_write?
+        inserts = @main_document['documents']
+        updates = @main_document['updates']
+        deletes = @main_document['deletes']
+
+        num_inserts = inserts && inserts.length || 0
+        num_updates = updates && updates.length || 0
+        num_deletes = deletes && deletes.length || 0
+
+        num_inserts > 1  || num_updates > 1 || num_deletes > 1
+      end
+
       private
 
+      # Validate that the documents in this message are all smaller than the
+      # maxBsonObjectSize. If not, raise an exception.
+      def validate_document_size!(max_bson_size)
+        max_bson_size ||= Mongo::Server::ConnectionBase::DEFAULT_MAX_BSON_OBJECT_SIZE
+
+        contains_too_large_document = @sections.any? do |section|
+          section[:type] == 1 &&
+            section[:payload][:sequence].any? do |document|
+              document.to_bson.length > max_bson_size
+            end
+        end
+
+        if contains_too_large_document
+          raise Error::MaxBSONSize.new('The document exceeds maximum allowed BSON object size after serialization')
+        end
+      end
+
       def command
-        @command ||= global_args.dup.tap do |cmd|
-          cmd.delete(DATABASE_IDENTIFIER)
-          sections.each do |section|
-            if section[:type] == 1
-              identifier = section[:payload][:identifier]
-              cmd[identifier] ||= []
-              cmd[identifier] += section[:payload][:sequence]
+        @command ||= if @main_document
+          @main_document.dup.tap do |cmd|
+            @sequences.each do |section|
+              cmd[section.identifier] ||= []
+              cmd[section.identifier] += section.documents
             end
           end
+        else
+          documents.first
         end
       end
 
@@ -166,8 +306,24 @@ module Mongo
         end
       end
 
-      def global_args
-        @global_args ||= (sections[0] || {})
+      # Encapsulates a type 1 OP_MSG section.
+      #
+      # @see https://github.com/mongodb/specifications/blob/master/source/message/OP_MSG.rst#sections
+      #
+      # @api private
+      class Section1
+        def initialize(identifier, documents)
+          @identifier, @documents = identifier, documents
+        end
+
+        attr_reader :identifier, :documents
+
+        def ==(other)
+          other.is_a?(Section1) &&
+            identifier == other.identifier && documents == other.documents
+        end
+
+        alias :eql? :==
       end
 
       # The operation code required to specify a OP_MSG message.
@@ -177,19 +333,33 @@ module Mongo
       OP_CODE = 2013
 
       # Available flags for a OP_MSG message.
-      FLAGS = Array.new(16).tap { |arr|
+      FLAGS = Array.new(16).tap do |arr|
         arr[0] = :checksum_present
         arr[1] = :more_to_come
-      }
+      end.freeze
 
       # @!attribute
       # @return [Array<Symbol>] The flags for this message.
       field :flags, BitVector.new(FLAGS)
 
-      # @!attribute
-      # @return [Hash] The sections of payload type 1 or 0.
+      # The sections that will be serialized, or the documents have been
+      # deserialized.
+      #
+      # Usually the sections contain OP_MSG-compliant sections derived
+      # from @main_document and @sequences. The information in @main_document
+      # and @sequences is duplicated in the sections.
+      #
+      # When deserializing Msg instances, sections temporarily is an array
+      # of documents returned in the type 0 section of the OP_MSG wire
+      # protocol message. #fix_after_deserialization method mutates this
+      # object to have sections, @main_document and @sequences be what
+      # they would have been had the Msg instance been constructed using
+      # the constructor (rather than having been deserialized).
+      #
+      # @return [ Array<Hash> | Array<BSON::Document> ] The sections of
+      #   payload type 1 or 0.
+      # @api private
       field :sections, Sections
-      alias :documents :sections
 
       Registry.register(OP_CODE, self)
     end

data/lib/mongo/protocol/query.rb

@@ -100,21 +100,19 @@ module Mongo
         true
       end
 
-      # Compress this message.
+      # Compress the message, if the command being sent permits compression.
+      # Otherwise returns self.
       #
       # @param [ String, Symbol ] compressor The compressor to use.
       # @param [ Integer ] zlib_compression_level The zlib compression level to use.
       #
-      # @return [ Compressed, self ] A Protocol::Compressed message or self, depending on whether
-      #  this message can be compressed.
+      # @return [ Message ] A Protocol::Compressed message or self,
+      #  depending on whether this message can be compressed.
       #
       # @since 2.5.0
-      def compress!(compressor, zlib_compression_level = nil)
-        if compressor && compression_allowed?(selector.keys.first)
-          Compressed.new(self, compressor, zlib_compression_level)
-        else
-          self
-        end
+      # @api private
+      def maybe_compress(compressor, zlib_compression_level = nil)
+        compress_if_possible(selector.keys.first, compressor, zlib_compression_level)
       end
 
       protected

data/lib/mongo/protocol/serializers.rb

@@ -57,10 +57,11 @@ module Mongo
         # Deserializes the header value from the IO stream
         #
         # @param [ String ] buffer Buffer containing the message header.
+        # @param [ Hash ] options This method currently accepts no options.
         #
         # @return [ Array<Fixnum> ] Array consisting of the deserialized
         #   length, request id, response id, and op code.
-        def self.deserialize(buffer)
+        def self.deserialize(buffer, options = {})
           buffer.get_bytes(16).unpack(HEADER_PACK)
         end
       end
@@ -123,9 +124,10 @@ module Mongo
         # Deserializes a 32-bit Fixnum from the IO stream
         #
         # @param [ String ] buffer Buffer containing the 32-bit integer
+        # @param [ Hash ] options This method currently accepts no options.
         #
         # @return [ Fixnum ] Deserialized Int32
-        def self.deserialize(buffer)
+        def self.deserialize(buffer, options = {})
           buffer.get_int32
         end
       end
@@ -156,9 +158,10 @@ module Mongo
         # Deserializes a 64-bit Fixnum from the IO stream
         #
         # @param [ String ] buffer Buffer containing the 64-bit integer.
+        # @param [ Hash ] options This method currently accepts no options.
         #
         # @return [Fixnum] Deserialized Int64.
-        def self.deserialize(buffer)
+        def self.deserialize(buffer, options = {})
           buffer.get_int64
         end
       end
@@ -198,19 +201,24 @@ module Mongo
         # Deserializes a section of an OP_MSG from the IO stream.
         #
         # @param [ BSON::ByteBuffer ] buffer Buffer containing the sections.
+        # @param [ Hash ] options
+        #
+        # @option options [ Boolean ] :deserialize_as_bson Whether to perform
+        #   section deserialization using BSON types instead of native Ruby types
+        #   wherever possible.
         #
         # @return [ Array<BSON::Document> ] Deserialized sections.
         #
         # @since 2.5.0
-        def self.deserialize(buffer)
+        def self.deserialize(buffer, options = {})
           end_length = (@flag_bits & Msg::FLAGS.index(:checksum_present)) == 1 ? 32 : 0
           sections = []
           until buffer.length == end_length
             case byte = buffer.get_byte
             when PayloadZero::TYPE_BYTE
-              sections << PayloadZero.deserialize(buffer)
+              sections << PayloadZero.deserialize(buffer, options)
             when PayloadOne::TYPE_BYTE
-              sections += PayloadOne.deserialize(buffer)
+              sections += PayloadOne.deserialize(buffer, options)
             else
               raise Error::UnknownPayloadType.new(byte)
             end
@@ -260,12 +268,18 @@ module Mongo
           # Deserializes a section of payload type 0 of an OP_MSG from the IO stream.
           #
           # @param [ BSON::ByteBuffer ] buffer Buffer containing the sections.
+          # @param [ Hash ] options
+          #
+          # @option options [ Boolean ] :deserialize_as_bson Whether to perform
+          #   section deserialization using BSON types instead of native Ruby types
+          #   wherever possible.
           #
           # @return [ Array<BSON::Document> ] Deserialized section.
           #
           # @since 2.5.0
-          def self.deserialize(buffer)
-            BSON::Document.from_bson(buffer)
+          def self.deserialize(buffer, options = {})
+            mode = options[:deserialize_as_bson] ? :bson : nil
+            BSON::Document.from_bson(buffer, **{ mode: mode })
           end
         end
 
@@ -313,6 +327,8 @@ module Mongo
           #
           # @since 2.5.0
           def self.deserialize(buffer)
+            raise NotImplementedError
+
             start_size = buffer.length
             section_size = buffer.get_int32 # get the size
             end_size = start_size - section_size
@@ -340,18 +356,26 @@ module Mongo
         def self.serialize(buffer, value, max_bson_size = nil, validating_keys = BSON::Config.validating_keys?)
           start_size = buffer.length
           value.to_bson(buffer, validating_keys)
-          if max_bson_size && buffer.length - start_size > max_bson_size
-            raise Error::MaxBSONSize.new(max_bson_size)
+          serialized_size = buffer.length - start_size
+          if max_bson_size && serialized_size > max_bson_size
+            raise Error::MaxBSONSize,
+              "The document exceeds maximum allowed BSON object size after serialization. Serialized size: #{serialized_size} bytes, maximum allowed size: #{max_bson_size} bytes"
           end
         end
 
         # Deserializes a document from the IO stream
         #
         # @param [ String ] buffer Buffer containing the BSON encoded document.
+        # @param [ Hash ] options
+        #
+        # @option options [ Boolean ] :deserialize_as_bson Whether to perform
+        #   section deserialization using BSON types instead of native Ruby types
+        #   wherever possible.
         #
         # @return [ Hash ] The decoded BSON document.
-        def self.deserialize(buffer)
-          BSON::Document.from_bson(buffer)
+        def self.deserialize(buffer, options = {})
+          mode = options[:deserialize_as_bson] ? :bson : nil
+          BSON::Document.from_bson(buffer, **{ mode: mode })
         end
 
         # Whether there can be a size limit on this type after serialization.
@@ -385,11 +409,12 @@ module Mongo
         # Deserializes a byte from the byte buffer.
         #
         # @param [ BSON::ByteBuffer ] buffer Buffer containing the value to read.
+        # @param [ Hash ] options This method currently accepts no options.
         #
         # @return [ String ] The byte.
         #
         # @since 2.5.0
-        def self.deserialize(buffer)
+        def self.deserialize(buffer, options = {})
           buffer.get_byte
         end
       end
@@ -415,12 +440,15 @@ module Mongo
         # Deserializes bytes from the byte buffer.
         #
         # @param [ BSON::ByteBuffer ] buffer Buffer containing the value to read.
-        # @param [ Integer ] num_bytes Number of bytes to read.
+        # @param [ Hash ] options The method options.
+        #
+        # @option options [ Integer ] num_bytes Number of bytes to read.
         #
         # @return [ String ] The bytes.
         #
         # @since 2.5.0
-        def self.deserialize(buffer, num_bytes = nil)
+        def self.deserialize(buffer, options = {})
+          num_bytes = options[:num_bytes]
           buffer.get_bytes(num_bytes || buffer.length)
         end
       end