mongo 2.11.6 → 2.12.0.rc0

data/lib/mongo/protocol/insert.rb

@@ -46,8 +46,10 @@ module Mongo
       # @param options [Hash] Additional options for the insertion.
       #
       # @option options :flags [Array] The flags for the insertion message.
-      #
       #   Supported flags: +:continue_on_error+
+      # @option options [ true, false ] validating_keys Whether keys should be
+      #   validated for being valid document keys (i.e. not begin with $ and
+      #   not contain dots).
       def initialize(database, collection, documents, options = {})
         @database = database
         @namespace = "#{database}.#{collection}"

data/lib/mongo/protocol/message.rb

@@ -95,27 +95,84 @@ module Mongo
         false
       end
 
-      # Compress a message.
+      # Compress the message, if supported by the wire protocol used and if
+      # the command being sent permits compression. Otherwise returns self.
       #
       # @param [ String, Symbol ] compressor The compressor to use.
       # @param [ Integer ] zlib_compression_level The zlib compression level to use.
       #
-      # @return [ self ] Always returns self. Other message types should override this method.
+      # @return [ self ] Always returns self. Other message types should
+      #   override this method.
       #
       # @since 2.5.0
-      def compress!(compressor, zlib_compression_level = nil)
+      # @api private
+      def maybe_compress(compressor, zlib_compression_level = nil)
         self
       end
 
-      # Inflate a message.
+      # Compress the message, if the command being sent permits compression.
+      # Otherwise returns self.
       #
-      # @return [ self ] Always returns self. Other message types should override this method.
+      # @param [ String ] command_name Command name extracted from the message.
+      # @param [ String | Symbol ] compressor The compressor to use.
+      # @param [ Integer ] zlib_compression_level Zlib compression level to use.
+      #
+      # @return [ Message ] A Protocol::Compressed message or self,
+      #  depending on whether this message can be compressed.
+      #
+      # @since 2.5.0
+      private def compress_if_possible(command_name, compressor, zlib_compression_level)
+        if compressor && compression_allowed?(command_name)
+          Compressed.new(self, compressor, zlib_compression_level)
+        else
+          self
+        end
+      end
+
+      # Inflate a message if it is compressed.
+      #
+      # @return [ Protocol::Message ] Always returns self. Subclasses should
+      #   override this method as necessary.
       #
       # @since 2.5.0
-      def inflate!
+      # @api private
+      def maybe_inflate
+        self
+      end
+
+      def maybe_decrypt(client)
+        # TODO determine if we should be decrypting data coming from pre-4.2
+        # servers, potentially using legacy wire protocols. If so we need
+        # to implement decryption for those wire protocols as our current
+        # encryption/decryption code is OP_MSG-specific.
         self
       end
 
+      def maybe_encrypt(server, client)
+        # Do nothing if the Message subclass has not implemented this method
+        self
+      end
+
+      private def merge_sections
+        cmd = if @sections.length > 1
+          cmd = @sections.detect { |section| section[:type] == 0 }[:payload]
+          identifier = @sections.detect { |section| section[:type] == 1}[:payload][:identifier]
+          cmd.merge(identifier.to_sym =>
+            @sections.select { |section| section[:type] == 1 }.
+              map { |section| section[:payload][:sequence] }.
+              inject([]) { |arr, documents| arr + documents }
+          )
+        elsif @sections.first[:payload]
+          @sections.first[:payload]
+        else
+          @sections.first
+        end
+        if cmd.nil?
+          raise "The command should never be nil here"
+        end
+        cmd
+      end
+
       # Serializes message into bytes that can be sent on the wire
       #
       # @param buffer [String] buffer where the message should be inserted
@@ -129,7 +186,11 @@ module Mongo
 
       alias_method :to_s, :serialize
 
-      # Deserializes messages from an IO stream
+      # Deserializes messages from an IO stream.
+      #
+      # This method returns decompressed messages (i.e. if the message on the
+      # wire was OP_COMPRESSED, this method would typically return the OP_MSG
+      # message that is the result of decompression).
       #
       # @param [ Integer ] max_message_size The max message size.
       # @param [ IO ] io Stream containing a message
@@ -160,7 +221,10 @@ module Mongo
             deserialize_field(message, buffer, field)
           end
         end
-        message.inflate!
+        if message.is_a?(Msg)
+          message.fix_after_deserialization
+        end
+        message.maybe_inflate
       end
 
       # Tests for equality between two wire protocol messages

data/lib/mongo/protocol/msg.rb

@@ -35,7 +35,7 @@ module Mongo
       # moved to the end of the hash for better logging.
       #
       # @api private
-      INTERNAL_KEYS = Set.new(%w($clusterTime lsid signature txnNumber)).freeze
+      INTERNAL_KEYS = Set.new(%w($clusterTime $db lsid signature txnNumber)).freeze
 
       # Creates a new OP_MSG protocol message
       #
@@ -44,25 +44,42 @@ module Mongo
       #           { type: 1, payload: { identifier: 'documents', sequence: [..] } })
       #
       # @param [ Array<Symbol> ] flags The flag bits. Current supported values
-      # are :more_to_come and :checksum_present.
-      # @param [ Hash ] options The options. There are currently no supported
-      #   options, this is a placeholder for the future.
-      # @param [ BSON::Document, Hash ] global_args The global arguments,
-      #   becomes a section of payload type 0.
-      # @param [ BSON::Document, Hash ] sections Zero or more sections, in the format
-      #   { type: 1, payload: { identifier: <String>, sequence: <Array<BSON::Document, Hash>> } } or
-      #   { type: 0, payload: <BSON::Document, Hash> }
+      #   are :more_to_come and :checksum_present.
+      # @param [ Hash ] options The options.
+      # @param [ BSON::Document, Hash ] main_document The document that will
+      #   become the payload type 0 section. Can contain global args as they
+      #   are defined in the OP_MSG specification.
+      # @param [ Protocol::Msg::Section1 ] sequences Zero or more payload type 1
+      #   sections.
       #
-      # @option options [ true, false ] validating_keys Whether keys should be validated.
+      # @option options [ true, false ] validating_keys Whether keys should be
+      #   validated for being valid document keys (i.e. not begin with $ and
+      #   not contain dots).
       #
       # @api private
       #
       # @since 2.5.0
-      def initialize(flags, options, global_args, *sections)
+      def initialize(flags, options, main_document, *sequences)
         @flags = flags || []
         @options = options
-        @global_args = global_args
-        @sections = [ { type: 0, payload: global_args } ] + sections
+        unless main_document.is_a?(Hash)
+          raise ArgumentError, "Main document must be a Hash, given: #{main_document.class}"
+        end
+        @main_document = main_document
+        sequences.each_with_index do |section, index|
+          unless section.is_a?(Section1)
+            raise ArgumentError, "All sequences must be Section1 instances, got: #{section} at index #{index}"
+          end
+        end
+        @sequences = sequences
+        @sections = [
+          {type: 0, payload: @main_document}
+        ] + @sequences.map do |section|
+          {type: 1, payload: {
+            identifier: section.identifier,
+            sequence: section.documents,
+          }}
+        end
         @request_id = nil
         super
       end
@@ -88,11 +105,11 @@ module Mongo
       #
       # @since 2.5.0
       def payload
-        # Reorder keys in global_args for better logging - see
+        # Reorder keys in main_document for better logging - see
         # https://jira.mongodb.org/browse/RUBY-1591.
         # Note that even without the reordering, the payload is not an exact
         # match to what is sent over the wire because the command as used in
-        # the published eent combines keys from multiple sections of the
+        # the published event combines keys from multiple sections of the
         # payload sent over the wire.
         ordered_command = {}
         skipped_command = {}
@@ -107,10 +124,10 @@ module Mongo
 
         BSON::Document.new(
           command_name: ordered_command.keys.first.to_s,
-          database_name: global_args[DATABASE_IDENTIFIER],
+          database_name: @main_document[DATABASE_IDENTIFIER],
           command: ordered_command,
           request_id: request_id,
-          reply: sections[0]
+          reply: @main_document,
         )
       end
 
@@ -128,35 +145,142 @@ module Mongo
         buffer
       end
 
-      # Compress this message.
+      # Compress the message, if the command being sent permits compression.
+      # Otherwise returns self.
       #
       # @param [ String, Symbol ] compressor The compressor to use.
       # @param [ Integer ] zlib_compression_level The zlib compression level to use.
       #
-      # @return [ Compressed, self ] A Protocol::Compressed message or self, depending on whether
-      #  this message can be compressed.
+      # @return [ Message ] A Protocol::Compressed message or self,
+      #  depending on whether this message can be compressed.
       #
       # @since 2.5.0
-      def compress!(compressor, zlib_compression_level = nil)
-        if compressor && compression_allowed?(command.keys.first)
-          Compressed.new(self, compressor, zlib_compression_level)
+      # @api private
+      def maybe_compress(compressor, zlib_compression_level = nil)
+        compress_if_possible(command.keys.first, compressor, zlib_compression_level)
+      end
+
+      # Reverse-populates the instance variables after deserialization sets
+      # @sections to the list of documents.
+      #
+      # TODO fix deserialization so that this method is not needed.
+      #
+      # @api private
+      def fix_after_deserialization
+        if @sections.nil?
+          raise NotImplementedError, "After deserializations @sections should have been initialized"
+        end
+        if @sections.length != 1
+          raise NotImplementedError, "Deserialization must have produced exactly one section, but it produced #{sections.length} sections"
+        end
+        @main_document = @sections.first
+        @sequences = []
+        @sections = [{type: 0, payload: @main_document}]
+      end
+
+      def documents
+        [@main_document]
+      end
+
+      # Possibly encrypt this message with libmongocrypt. Message will only be
+      # encrypted if the specified client exists, that client has been given
+      # auto-encryption options, the client has not been instructed to bypass
+      # auto-encryption, and mongocryptd determines that this message is
+      # eligible for encryption. A message is eligible for encryption if it
+      # represents one of the command types whitelisted by libmongocrypt and it
+      # contains data that is required to be encrypted by a local or remote json schema.
+      #
+      # @param [ Mongo::Client | nil ] client The client used to make the original
+      #   command. This client may have auto-encryption options specified.
+      #
+      # @return [ Mongo::Protocol::Msg ] The encrypted message, or the original
+      #   message if encryption was not possible or necessary.
+      def maybe_encrypt(server, client)
+        # TODO verify compression happens later, i.e. when this method runs
+        # the message is not compressed.
+        if client && client.encrypter && client.encrypter.encrypt?
+          if server.max_wire_version < 8
+            raise Error::CryptError.new(
+              "Cannot perform encryption against a MongoDB server older than " +
+              "4.2 (wire version less than 8). Currently connected to server " +
+              "with max wire version #{server.max_wire_version}} " +
+              "(Auto-encryption requires a minimum MongoDB version of 4.2)"
+            )
+          end
+
+          db_name = @main_document[DATABASE_IDENTIFIER]
+          cmd = merge_sections
+          enc_cmd = client.encrypter.encrypt(db_name, cmd)
+          if cmd.key?('$db') && !enc_cmd.key?('$db')
+            enc_cmd['$db'] = cmd['$db']
+          end
+          # This will be investigtated in RUBY-2119
+          if enc_cmd['txnNumber'].is_a?(Integer) && cmd[:txnNumber].is_a?(BSON::Int64)
+            enc_cmd['txnNumber'] = BSON::Int64.new(enc_cmd[:txnNumber])
+          end
+          Msg.new(@flags, @options, enc_cmd)
         else
           self
         end
       end
 
+      # Possibly decrypt this message with libmongocrypt. Message will only be
+      # decrypted if the specified client exists, that client has been given
+      # auto-encryption options, and this message is eligible for decryption.
+      # A message is eligible for decryption if it represents one of the command
+      # types whitelisted by libmongocrypt and it contains data that is required
+      # to be encrypted by a local or remote json schema.
+      #
+      # @param [ Mongo::Client | nil ] client The client used to make the original
+      #   command. This client may have auto-encryption options specified.
+      #
+      # @return [ Mongo::Protocol::Msg ] The decryption message, or the original
+      #   message if decryption was not possible or necessary.
+      def maybe_decrypt(client)
+        if client && client.encrypter
+          cmd = merge_sections
+          enc_cmd = client.encrypter.decrypt(cmd)
+          Msg.new(@flags, @options, enc_cmd)
+        else
+          self
+        end
+      end
+
+      # Whether this message represents a bulk write. A bulk write is an insert,
+      # update, or delete operation that encompasses multiple operations of
+      # the same type.
+      #
+      # @return [ Boolean ] Whether this message represents a bulk write.
+      #
+      # @note This method was written to support client-side encryption
+      #   functionality. It is not recommended that this method be used in
+      #   service of any other feature or behavior.
+      #
+      # @api private
+      def bulk_write?
+        inserts = @main_document['documents']
+        updates = @main_document['updates']
+        deletes = @main_document['deletes']
+
+        num_inserts = inserts && inserts.length || 0
+        num_updates = updates && updates.length || 0
+        num_deletes = deletes && deletes.length || 0
+
+        num_inserts > 1  || num_updates > 1 || num_deletes > 1
+      end
+
       private
 
       def command
-        @command ||= global_args.dup.tap do |cmd|
-          cmd.delete(DATABASE_IDENTIFIER)
-          sections.each do |section|
-            if section[:type] == 1
-              identifier = section[:payload][:identifier]
-              cmd[identifier] ||= []
-              cmd[identifier] += section[:payload][:sequence]
+        @command ||= if @main_document
+          @main_document.dup.tap do |cmd|
+            @sequences.each do |section|
+              cmd[section.identifier] ||= []
+              cmd[section.identifier] += section.documents
             end
           end
+        else
+          documents.first
         end
       end
 
@@ -166,8 +290,24 @@ module Mongo
         end
       end
 
-      def global_args
-        @global_args ||= (sections[0] || {})
+      # Encapsulates a type 1 OP_MSG section.
+      #
+      # @see https://github.com/mongodb/specifications/blob/master/source/message/OP_MSG.rst#sections
+      #
+      # @api private
+      class Section1
+        def initialize(identifier, documents)
+          @identifier, @documents = identifier, documents
+        end
+
+        attr_reader :identifier, :documents
+
+        def ==(other)
+          other.is_a?(Section1) &&
+            identifier == other.identifier && documents == other.documents
+        end
+
+        alias :eql? :==
       end
 
       # The operation code required to specify a OP_MSG message.
@@ -177,19 +317,33 @@ module Mongo
       OP_CODE = 2013
 
       # Available flags for a OP_MSG message.
-      FLAGS = Array.new(16).tap { |arr|
+      FLAGS = Array.new(16).tap do |arr|
         arr[0] = :checksum_present
         arr[1] = :more_to_come
-      }
+      end.freeze
 
       # @!attribute
       # @return [Array<Symbol>] The flags for this message.
       field :flags, BitVector.new(FLAGS)
 
-      # @!attribute
-      # @return [Hash] The sections of payload type 1 or 0.
+      # The sections that will be serialized, or the documents have been
+      # deserialized.
+      #
+      # Usually the sections contain OP_MSG-compliant sections derived
+      # from @main_document and @sequences. The information in @main_document
+      # and @sequences is duplicated in the sections.
+      #
+      # When deserializing Msg instances, sections temporarily is an array
+      # of documents returned in the type 0 section of the OP_MSG wire
+      # protocol message. #fix_after_deserialization method mutates this
+      # object to have sections, @main_document and @sequences be what
+      # they would have been had the Msg instance been constructed using
+      # the constructor (rather than having been deserialized).
+      #
+      # @return [ Array<Hash> | Array<BSON::Document> ] The sections of
+      #   payload type 1 or 0.
+      # @api private
       field :sections, Sections
-      alias :documents :sections
 
       Registry.register(OP_CODE, self)
     end

data/lib/mongo/protocol/query.rb

@@ -100,21 +100,19 @@ module Mongo
         true
       end
 
-      # Compress this message.
+      # Compress the message, if the command being sent permits compression.
+      # Otherwise returns self.
       #
       # @param [ String, Symbol ] compressor The compressor to use.
       # @param [ Integer ] zlib_compression_level The zlib compression level to use.
       #
-      # @return [ Compressed, self ] A Protocol::Compressed message or self, depending on whether
-      #  this message can be compressed.
+      # @return [ Message ] A Protocol::Compressed message or self,
+      #  depending on whether this message can be compressed.
       #
       # @since 2.5.0
-      def compress!(compressor, zlib_compression_level = nil)
-        if compressor && compression_allowed?(selector.keys.first)
-          Compressed.new(self, compressor, zlib_compression_level)
-        else
-          self
-        end
+      # @api private
+      def maybe_compress(compressor, zlib_compression_level = nil)
+        compress_if_possible(selector.keys.first, compressor, zlib_compression_level)
       end
 
       protected