mongo 2.13.0.rc1 → 2.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: db8c88ed58f349728a9361db891376926e93450f4a8ac86aae0bc67599c91a7b
-  data.tar.gz: b4f7daaf1d43b55839c1f689504d4f1c212c638e45f6d2211732bc07a41f77e5
+  metadata.gz: df3eb3a5733323f2f4440c67bc0173661ada194ecb35374e8e78345e2d072525
+  data.tar.gz: 63227bff6f14e3100f86a747b3e5d577812dd74d6da8061a069d8a313ecce09c
 SHA512:
-  metadata.gz: 22471d116179fff42716707a40fa18ebc3c53492a6313bb16be96f447db725110b8ed0b3ee8e199026098a59b8f2941290cd1493249c8e9120ec2849e0229fa1
-  data.tar.gz: 8965e04c6926343e4363110d010e14b1a8fde146eb4e239f620005f251ff24b63393bc13f5f332ac4d8aa39cc790a66f080401d9b0923c54a8b40b7d80aa25ea
+  metadata.gz: '02253019a9aab19c94dbb2a4e459a84c1fa6f1383eca9d5b1daa26bf1c2aa1a96c32f52a23267b0b7b5c58fc469144d0c1722b4f79593ae7563ac0c9deb55d95'
+  data.tar.gz: e02d93a95630d770f20a37cbe91a4dc18c98cb18e684da4b2a538a70afaaf61731e808832b2fe9aadca484dacfc8dce43cedf356ef00db019a03d77e9283cc45

data/lib/mongo/auth/aws/request.rb

@@ -61,9 +61,13 @@ module Mongo
           %i(access_key_id secret_access_key host server_nonce).each do |arg|
             value = instance_variable_get("@#{arg}")
             if value.nil? || value.empty?
-              raise ArgumentError, "Value for #{arg} is required"
+              raise Error::InvalidServerAuthResponse, "Value for '#{arg}' is required"
             end
           end
+
+          if host && host.length > 255
+              raise Error::InvalidServerAuthHost, "Value for 'host' is too long: #{@host}"
+          end
         end
 
         # @return [ String ] access_key_id The access key id.
@@ -98,8 +102,28 @@ module Mongo
 
         # @return [ String ] region The region of the host, derived from the host.
         def region
-          # TODO implement region derivation when SPEC-1646 is done.
-          'us-east-1'
+          # Common case
+          if host == 'sts.amazonaws.com'
+            return 'us-east-1'
+          end
+
+          if host.start_with?('.')
+            raise Error::InvalidServerAuthHost, "Host begins with a period: #{host}"
+          end
+          if host.end_with?('.')
+            raise Error::InvalidServerAuthHost, "Host ends with a period: #{host}"
+          end
+
+          parts = host.split('.')
+          if parts.any? { |part| part.empty? }
+            raise Error::InvalidServerAuthHost, "Host has an empty component: #{host}"
+          end
+
+          if parts.length == 1
+            'us-east-1'
+          else
+            parts[1]
+          end
         end
 
         # Returns the scope of the request, per the AWS signature V4 specification.

data/lib/mongo/client.rb

@@ -104,6 +104,7 @@ module Mongo
       :truncate_logs,
       :user,
       :wait_queue_timeout,
+      :wrapping_libraries,
       :write,
       :write_concern,
       :zlib_compression_level,
@@ -375,6 +376,10 @@ module Mongo
     # @option options [ String ] :user The user name.
     # @option options [ Float ] :wait_queue_timeout The time to wait, in
     #   seconds, in the connection pool for a connection to be checked in.
+    # @option options [ Array<Hash> ] :wrapping_libraries Information about
+    #   libraries such as ODMs that are wrapping the driver, to be added to
+    #    metadata sent to the server. Specify the lower level libraries first.
+    #    Allowed hash keys: :name, :version, :platform.
     # @option options [ Hash ] :write Deprecated. Equivalent to :write_concern
     #   option.
     # @option options [ Hash ] :write_concern The write concern options.
@@ -1160,6 +1165,36 @@ module Mongo
           raise ArgumentError, ":bg_error_backtrace option value must be true, false, nil or a positive integer: #{value}"
         end
       end
+
+      if libraries = options[:wrapping_libraries]
+        unless Array === libraries
+          raise ArgumentError, ":wrapping_libraries must be an array of hashes: #{libraries}"
+        end
+
+        libraries = libraries.map do |library|
+          Utils.shallow_symbolize_keys(library)
+        end
+
+        libraries.each do |library|
+          unless Hash === library
+            raise ArgumentError, ":wrapping_libraries element is not a hash: #{library}"
+          end
+
+          if library.empty?
+            raise ArgumentError, ":wrapping_libraries element is empty"
+          end
+
+          unless (library.keys - %i(name platform version)).empty?
+            raise ArgumentError, ":wrapping_libraries element has invalid keys (allowed keys: :name, :platform, :version): #{library}"
+          end
+
+          library.each do |key, value|
+            if value.include?('|')
+              raise ArgumentError, ":wrapping_libraries element value cannot include '|': #{value}"
+            end
+          end
+        end
+      end
     end
 
     # Validates all authentication-related options after they are set on the client

data/lib/mongo/error.rb

@@ -190,6 +190,8 @@ require 'mongo/error/invalid_application_name'
 require 'mongo/error/invalid_nonce'
 require 'mongo/error/invalid_replacement_document'
 require 'mongo/error/invalid_server_auth_response'
+# Subclass of InvalidServerAuthResponse
+require 'mongo/error/invalid_server_auth_host'
 require 'mongo/error/invalid_server_preference'
 require 'mongo/error/invalid_session'
 require 'mongo/error/invalid_signature'

data/lib/mongo/error/invalid_server_auth_host.rb

@@ -0,0 +1,22 @@
+# Copyright (C) 2020 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  class Error
+
+    # Raised when the server returned an invalid Host value in AWS auth.
+    class InvalidServerAuthHost < InvalidServerAuthResponse
+    end
+  end
+end

data/lib/mongo/index/view.rb

@@ -120,6 +120,9 @@ module Mongo
       #   a geo index.
       # @option options [ Hash ] :partial_filter_expression  Specify a filter for a partial
       #   index.
+      # @option options [ Boolean ] :hidden When :hidden is true, this index will
+      #   exist on the collection but not be used by the query planner when
+      #   executing operations.
       # @option options [ String | Integer ] :commit_quorum Specify how many
       #   data-bearing members of a replica set, including the primary, must
       #   complete the index builds successfully before the primary marks

data/lib/mongo/protocol/msg.rb

@@ -147,6 +147,8 @@ module Mongo
       #
       # @since 2.5.0
       def serialize(buffer = BSON::ByteBuffer.new, max_bson_size = nil)
+        validate_document_size!(max_bson_size)
+
         super
         add_check_sum(buffer)
         buffer
@@ -275,6 +277,23 @@ module Mongo
 
       private
 
+      # Validate that the documents in this message are all smaller than the
+      # maxBsonObjectSize. If not, raise an exception.
+      def validate_document_size!(max_bson_size)
+        max_bson_size ||= Mongo::Server::ConnectionBase::DEFAULT_MAX_BSON_OBJECT_SIZE
+
+        contains_too_large_document = @sections.any? do |section|
+          section[:type] == 1 &&
+            section[:payload][:sequence].any? do |document|
+              document.to_bson.length > max_bson_size
+            end
+        end
+
+        if contains_too_large_document
+          raise Error::MaxBSONSize.new('The document exceeds maximum allowed BSON object size after serialization')
+        end
+      end
+
       def command
         @command ||= if @main_document
           @main_document.dup.tap do |cmd|

data/lib/mongo/server/app_metadata.rb

@@ -65,12 +65,17 @@ module Mongo
       #   the metadata printed to the mongod logs upon establishing a connection
       #   in server versions >= 3.4.
       # @option options [ String ] :user The user name.
+      # @option options [ Array<Hash> ] :wrapping_libraries Information about
+      #   libraries such as ODMs that are wrapping the driver. Specify the
+      #   lower level libraries first. Allowed hash keys: :name, :version,
+      #   :platform.
       #
       # @since 2.4.0
       def initialize(options)
         @app_name = options[:app_name].to_s if options[:app_name]
         @platform = options[:platform]
         @compressors = options[:compressors] || []
+        @wrapping_libraries = options[:wrapping_libraries]
 
         if options[:user] && !options[:auth_mech]
           auth_db = options[:auth_source] || 'admin'
@@ -78,6 +83,10 @@ module Mongo
         end
       end
 
+      # @return [ Array<Hash> | nil ] Information about libraries wrapping
+      #   the driver.
+      attr_reader :wrapping_libraries
+
       # Get the bytes of the ismaster message including this metadata.
       #
       # @api private
@@ -140,9 +149,17 @@ module Mongo
       end
 
       def driver_doc
+        names = [DRIVER_NAME]
+        versions = [Mongo::VERSION]
+        if wrapping_libraries
+          wrapping_libraries.each do |library|
+            names << library[:name] || ''
+            versions << library[:version] || ''
+          end
+        end
         {
-          name: DRIVER_NAME,
-          version: Mongo::VERSION
+          name: names.join('|'),
+          version: versions.join('|'),
         }
       end
 
@@ -175,12 +192,19 @@ module Mongo
           ruby_versions = ["Ruby #{RUBY_VERSION}"]
           platforms = [RUBY_PLATFORM]
         end
-        [
+        platform = [
           @platform,
           *ruby_versions,
           *platforms,
           RbConfig::CONFIG['build'],
         ].compact.join(', ')
+        platforms = [platform]
+        if wrapping_libraries
+          wrapping_libraries.each do |library|
+            platforms << library[:platform] || ''
+          end
+        end
+        platforms.join('|')
       end
     end
   end

data/lib/mongo/server/connection_base.rb

@@ -186,9 +186,6 @@ module Mongo
       end
 
       def serialize(message, client, buffer = BSON::ByteBuffer.new)
-        start_size = 0
-        final_message = message.maybe_compress(compressor, options[:zlib_compression_level])
-
         # Driver specifications only mandate the fixed 16MiB limit for
         # serialized BSON documents. However, the server returns its
         # active serialized BSON document size limit in the ismaster response,
@@ -213,12 +210,41 @@ module Mongo
           max_bson_size += MAX_BSON_COMMAND_OVERHEAD
         end
 
-        final_message.serialize(buffer, max_bson_size)
-        if max_message_size &&
-          (buffer.length - start_size) > max_message_size
-        then
-          raise Error::MaxMessageSize.new(max_message_size)
+        # RUBY-2234: It is necessary to check that the message size does not
+        # exceed the maximum bson object size before compressing and serializing
+        # the final message.
+        #
+        # This is to avoid the case where the user performs a bulk write
+        # larger than 16MiB which, when compressed, becomes smaller than 16MiB.
+        # If the driver does not split the bulk writes prior to compression,
+        # the entire operation will be sent to the server, which will raise an
+        # error because the uncompressed operation exceeds the maximum bson size.
+        #
+        # To address this problem, we serialize the message prior to compression
+        # and raise an exception if the serialized message exceeds the maximum
+        # bson size.
+        if max_message_size
+          # Create a separate buffer that contains the un-compressed message
+          # for the purpose of checking its size. Write any pre-existing contents
+          # from the original buffer into the temporary one.
+          temp_buffer = BSON::ByteBuffer.new
+
+          # TODO: address the fact that this line mutates the buffer.
+          temp_buffer.put_bytes(buffer.get_bytes(buffer.length))
+
+          message.serialize(temp_buffer, max_bson_size)
+          if temp_buffer.length > max_message_size
+            raise Error::MaxMessageSize.new(max_message_size)
+          end
         end
+
+        # RUBY-2335: When the un-compressed message is smaller than the maximum
+        # bson size limit, the message will be serialized twice. The operations
+        # layer should be refactored to allow compression on an already-
+        # serialized message.
+        final_message = message.maybe_compress(compressor, options[:zlib_compression_level])
+        final_message.serialize(buffer, max_bson_size)
+
         buffer
       end
     end

data/lib/mongo/version.rb

@@ -17,5 +17,5 @@ module Mongo
   # The current version of the driver.
   #
   # @since 2.0.0
-  VERSION = '2.13.0.rc1'.freeze
+  VERSION = '2.13.0'.freeze
 end

data/spec/integration/bulk_write_spec.rb

@@ -0,0 +1,19 @@
+require 'spec_helper'
+
+describe 'Bulk writes' do
+  before do
+    authorized_collection.drop
+  end
+
+  context 'when bulk write is larger than 48MB' do
+    let(:operations) do
+      [ { insert_one: { text: 'a' * 1000 * 1000 } } ] * 48
+    end
+
+    it 'succeeds' do
+      expect do
+        authorized_collection.bulk_write(operations)
+      end.not_to raise_error
+    end
+  end
+end

data/spec/integration/client_side_encryption/auto_encryption_bulk_writes_spec.rb

@@ -113,7 +113,7 @@ describe 'Bulk writes with auto-encryption enabled' do
         it 'raises an exception' do
           expect do
             bulk_write.execute
-          end.to raise_error(Mongo::Error::MaxBSONSize, /maximum allowed size: 16777216 bytes/)
+          end.to raise_error(Mongo::Error::MaxBSONSize, /The document exceeds maximum allowed BSON object size after serialization/)
         end
       end
     end
@@ -255,7 +255,7 @@ describe 'Bulk writes with auto-encryption enabled' do
         it 'raises an exception' do
           expect do
             bulk_write.execute
-          end.to raise_error(Mongo::Error::MaxBSONSize, /maximum allowed size: 16777216 bytes/)
+          end.to raise_error(Mongo::Error::MaxBSONSize, /The document exceeds maximum allowed BSON object size after serialization/)
         end
       end
     end
@@ -346,7 +346,7 @@ describe 'Bulk writes with auto-encryption enabled' do
       it 'raises an exception' do
         expect do
           perform_bulk_write
-        end.to raise_error(Mongo::Error::MaxBSONSize, /maximum allowed size: 16777216 bytes/)
+        end.to raise_error(Mongo::Error::MaxBSONSize, /The document exceeds maximum allowed BSON object size after serialization/)
       end
     end
   end

data/spec/integration/size_limit_spec.rb

@@ -62,13 +62,30 @@ describe 'BSON & command size limits' do
     authorized_collection.insert_one(document)
   end
 
-  it 'fails on the server when a document larger than 16MiB is inserted' do
-    document = { key: 'a' * (max_document_size - 27), _id: 'foo' }
-    expect(document.to_bson.length).to eq(max_document_size+1)
+  context 'on server versions >= 3.6' do
+    min_server_fcv '3.6'
 
-    lambda do
-      authorized_collection.insert_one(document)
-    end.should raise_error(Mongo::Error::OperationFailure, /object to insert too large/)
+    it 'fails on the driver when a document larger than 16MiB is inserted' do
+      document = { key: 'a' * (max_document_size - 27), _id: 'foo' }
+      expect(document.to_bson.length).to eq(max_document_size+1)
+
+      lambda do
+        authorized_collection.insert_one(document)
+      end.should raise_error(Mongo::Error::MaxBSONSize, /The document exceeds maximum allowed BSON object size after serialization/)
+    end
+  end
+
+  context 'on server versions <= 3.4' do
+    max_server_fcv '3.4'
+
+    it 'fails on the server when a document larger than 16MiB is inserted' do
+      document = { key: 'a' * (max_document_size - 27), _id: 'foo' }
+      expect(document.to_bson.length).to eq(max_document_size+1)
+
+      lambda do
+        authorized_collection.insert_one(document)
+      end.should raise_error(Mongo::Error::OperationFailure, /object to insert too large/)
+    end
   end
 
   it 'fails in the driver when a document larger than 16MiB+16KiB is inserted' do
@@ -81,10 +98,6 @@ describe 'BSON & command size limits' do
   end
 
   it 'allows bulk writes of multiple documents of exactly 16 MiB each' do
-    if SpecConfig.instance.compressors
-      pending "RUBY-2234"
-    end
-
     documents = []
     1.upto(3) do |index|
       document = { key: 'a' * (max_document_size - 28), _id: "in#{index}" }

data/spec/mongo/auth/aws/request_region_spec.rb

@@ -0,0 +1,42 @@
+require 'lite_spec_helper'
+
+AWS_REGION_TEST_CASES = {
+  'sts.amazonaws.com' => 'us-east-1',
+  'sts.us-west-2.amazonaws.com' => 'us-west-2',
+  'sts.us-west-2.amazonaws.com.ch' => 'us-west-2',
+  'example.com' => 'com',
+  'localhost' => 'us-east-1',
+  'sts..com' => Mongo::Error::InvalidServerAuthHost,
+  '.amazonaws.com' => Mongo::Error::InvalidServerAuthHost,
+  'sts.amazonaws.' => Mongo::Error::InvalidServerAuthHost,
+  '' => Mongo::Error::InvalidServerAuthResponse,
+  'x' * 256 => Mongo::Error::InvalidServerAuthHost,
+}
+
+describe 'AWS auth region tests' do
+
+  AWS_REGION_TEST_CASES.each do |host, expected_region|
+    context "host '#{host}'" do
+      let(:request) do
+        Mongo::Auth::Aws::Request.new(access_key_id: 'access_key_id',
+          secret_access_key: 'secret_access_key',
+          session_token: 'session_token',
+          host: host,
+          server_nonce: 'server_nonce',
+        )
+      end
+
+      if expected_region.is_a?(String)
+        it 'derives expected region' do
+          request.region.should == expected_region
+        end
+      else
+        it 'fails with an error' do
+          lambda do
+            request.region
+          end.should raise_error(expected_region)
+        end
+      end
+    end
+  end
+end