mongo 2.18.0.beta1 → 2.18.0

data/lib/mongo/crypt/kms/kmip.rb

@@ -24,11 +24,15 @@ module Mongo
         #
         # @api private
         class Credentials
+          extend Forwardable
           include KMS::Validations
 
           # @return [ String ] KMIP KMS endpoint with optional port.
           attr_reader :endpoint
 
+          # @api private
+          def_delegator :@opts, :empty?
+
           FORMAT_HINT = "KMIP KMS provider options must be in the format: " +
                         "{ endpoint: 'ENDPOINT' }"
 
@@ -41,13 +45,17 @@ module Mongo
           # @raise [ ArgumentError ] If required options are missing or incorrectly
           #   formatted.
           def initialize(opts)
-            @endpoint = validate_param(:endpoint, opts, FORMAT_HINT)
+            @opts = opts
+            unless empty?
+              @endpoint = validate_param(:endpoint, opts, FORMAT_HINT)
+            end
           end
 
           # Convert credentials object to a BSON document in libmongocrypt format.
           #
           # @return [ BSON::Document ] Local KMS credentials in libmongocrypt format.
           def to_document
+            return BSON::Document.new({}) if empty?
             BSON::Document.new({
               endpoint: endpoint,
             })
@@ -74,17 +82,17 @@ module Mongo
           #
           # @param [ Hash ] opts A hash that contains master key options for
           #   KMIP KMS provider
-          # @option opts [ String ] :key_id KMIP Unique Identifier to
+          # @option opts [ String | nil ] :key_id KMIP Unique Identifier to
           #   a 96 byte KMIP Secret Data managed object, optional. If key_id
           #   is omitted, the driver creates a random 96 byte identifier.
-          # @option opts [ String ] :endpoint KMIP endpoint, optional.
+          # @option opts [ String | nil ] :endpoint KMIP endpoint, optional.
           #
           # @raise [ ArgumentError ] If required options are missing or incorrectly
           #   formatted.
-          def initialize(opts)
+          def initialize(opts = {})
             @key_id = validate_param(
               :key_id, opts, FORMAT_HINT, required: false
-            ) || SecureRandom.alphanumeric(96)
+            )
             @endpoint = validate_param(
               :endpoint, opts, FORMAT_HINT, required: false
             )
@@ -96,11 +104,9 @@ module Mongo
           def to_document
             BSON::Document.new({
               provider: 'kmip',
-              keyId: key_id
             }).tap do |bson|
-              unless endpoint.nil?
-                bson.update({ endpoint: endpoint })
-              end
+              bson.update({ endpoint: endpoint }) unless endpoint.nil?
+              bson.update({ keyId: key_id }) unless key_id.nil?
             end
           end
         end

data/lib/mongo/crypt/kms/local.rb

@@ -23,11 +23,15 @@ module Mongo
         #
         # @api private
         class Credentials
+          extend Forwardable
           include KMS::Validations
 
           # @return [ String ] Master key.
           attr_reader :key
 
+          # @api private
+          def_delegator :@opts, :empty?
+
           FORMAT_HINT = "Local KMS provider options must be in the format: " +
                         "{ key: 'MASTER-KEY' }"
 
@@ -40,11 +44,15 @@ module Mongo
           # @raise [ ArgumentError ] If required options are missing or incorrectly
           #   formatted.
           def initialize(opts)
-            @key = validate_param(:key, opts, FORMAT_HINT)
+            @opts = opts
+            unless empty?
+              @key = validate_param(:key, opts, FORMAT_HINT)
+            end
           end
 
           # @return [ BSON::Document ] Local KMS credentials in libmongocrypt format.
           def to_document
+            return BSON::Document.new({}) if empty?
             BSON::Document.new({
               key: BSON::Binary.new(@key, :generic),
             })

data/lib/mongo/crypt/kms/master_key_document.rb

@@ -41,7 +41,7 @@ module Mongo
           if options.nil?
             raise ArgumentError.new('Key document options must not be nil')
           end
-          master_key = options[:master_key]
+          master_key = options.fetch(:master_key, {})
           @key_document = case kms_provider.to_s
             when 'aws' then KMS::AWS::MasterKeyDocument.new(master_key)
             when 'azure' then KMS::Azure::MasterKeyDocument.new(master_key)

data/lib/mongo/crypt/rewrap_many_data_key_context.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2019-2020 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  module Crypt
+
+    # A Context object initialized specifically for the purpose of rewrapping
+    # data keys (decrypting and re-rencryting using a new KEK).
+    #
+    # @api private
+    class RewrapManyDataKeyContext < Context
+
+      # Create a new RewrapManyDataKeyContext object
+      #
+      # @param [ Mongo::Crypt::Handle ] mongocrypt a Handle that
+      #   wraps a mongocrypt_t object used to create a new mongocrypt_ctx_t
+      # @param [ Mongo::Crypt::EncryptionIO ] io An object that performs all
+      #   driver I/O on behalf of libmongocrypt
+      # @param [ Hash ] filter Filter used to find keys to be updated.
+      #   alternate names for the new data key.
+      # @param [ Mongo::Crypt::KMS::MasterKeyDocument | nil ] master_key_document The optional master
+      #   key document that contains master encryption key parameters.
+      def initialize(mongocrypt, io, filter, master_key_document)
+        super(mongocrypt, io)
+        if master_key_document
+          Binding.ctx_setopt_key_encryption_key(self, master_key_document.to_document)
+        end
+        Binding.ctx_rewrap_many_datakey_init(self, filter)
+      end
+    end
+  end
+end

data/lib/mongo/crypt/rewrap_many_data_key_result.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2019-2022 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  module Crypt
+    # Represent result of the rewrap many data ke operation.
+    #
+    # @api semiprivate
+    class RewrapManyDataKeyResult
+
+      # @returns [ BulkWrite::Result ] the result of the bulk write operation
+      #   used to update the key vault collection with rewrapped data keys.
+      attr_reader :bulk_write_result
+
+      # @param [ BulkWrite::Result | nil ] bulk_write_result The result of the
+      #   bulk write operation used to update the key vault collection
+      #   with rewrapped data keys.
+      def initialize(bulk_write_result)
+        @bulk_write_result = bulk_write_result
+      end
+    end
+  end
+end

data/lib/mongo/crypt/status.rb

@@ -118,10 +118,16 @@ module Mongo
       # information stored in this status
       #
       # Does nothing if self.ok? is true
-      def raise_crypt_error
+      #
+      # @param kms [ true | false ] Whether the operation was against the KMS.
+      #
+      # @note If kms parameter is false, the error may still have come from a
+      #   KMS. The kms parameter simply forces all errors to be treated as
+      #   KMS errors.
+      def raise_crypt_error(kms: false)
         return if ok?
 
-        if label == :error_kms
+        if kms || label == :error_kms
           error = Error::KmsError.new(message, code: code)
         else
           error = Error::CryptError.new(message, code: code)

data/lib/mongo/crypt.rb

@@ -29,6 +29,8 @@ module Mongo
     autoload(:AutoEncryptionContext, 'mongo/crypt/auto_encryption_context')
     autoload(:ExplicitDecryptionContext, 'mongo/crypt/explicit_decryption_context')
     autoload(:AutoDecryptionContext, 'mongo/crypt/auto_decryption_context')
+    autoload(:RewrapManyDataKeyContext, 'mongo/crypt/rewrap_many_data_key_context')
+    autoload(:RewrapManyDataKeyResult, 'mongo/crypt/rewrap_many_data_key_result')
     autoload(:EncryptionIO, 'mongo/crypt/encryption_io')
     autoload(:ExplicitEncrypter, 'mongo/crypt/explicit_encrypter')
     autoload(:AutoEncrypter, 'mongo/crypt/auto_encrypter')

data/lib/mongo/database.rb

@@ -225,33 +225,8 @@ module Mongo
 
       client.send(:with_session, opts) do |session|
         server = selector.select_server(cluster, nil, session)
-        # This code MUST be removed as soon as server starts accepting
-        # contention as int32.
-        _operation = operation.dup
-        if _operation['encryptedFields'] && _operation['encryptedFields'].key?('fields')
-          _operation['encryptedFields']['fields'] = _operation['encryptedFields']['fields'].map do |field|
-            if field['queries'] && field['queries'].key?('contention')
-              field['queries']['contention'] = BSON::Int64.new(field['queries']['contention'])
-            end
-            field
-          end
-        end
-        if schema = _operation.dig('encryptionInformation', 'schema')
-          _operation['encryptionInformation']['schema'] = schema.map do |coll, params|
-            if params['fields']
-              params['fields'] = params['fields'].map do |field|
-                if contention = field.dig('queries', 'contention')
-                  field['queries']['contention'] = BSON::Int64.new(contention)
-                end
-                field
-              end
-            end
-            [coll, params]
-          end.to_h
-        end
-        # End of code to be removed
         op = Operation::Command.new(
-          :selector => _operation,
+          :selector => operation,
           :db_name => name,
           :read => selector,
           :session => session
@@ -481,6 +456,11 @@ module Mongo
     #   Only recognized by server versions 4.0+.
     # @option options [ Object ] :comment A user-provided
     #   comment to attach to this command.
+    # @option options [ Boolean ] :show_expanded_events Enables the server to
+    #   send the 'expanded' list of change stream events. The list of additional
+    #   events included with this flag set are: createIndexes, dropIndexes,
+    #   modify, create, shardCollection, reshardCollection,
+    #   refineCollectionShardKey.
     #
     # @note A change stream only allows 'majority' read concern.
     # @note This helper method is preferable to running a raw aggregation with a $changeStream
@@ -490,8 +470,11 @@ module Mongo
     #
     # @since 2.6.0
     def watch(pipeline = [], options = {})
+      view_options = options.dup
+      view_options[:await_data] = true if options[:max_await_time_ms]
+
       Mongo::Collection::View::ChangeStream.new(
-        Mongo::Collection::View.new(collection("#{COMMAND}.aggregate")),
+        Mongo::Collection::View.new(collection("#{COMMAND}.aggregate"), {}, view_options),
         pipeline,
         Mongo::Collection::View::ChangeStream::DATABASE,
         options)

data/lib/mongo/error/missing_file_chunk.rb

@@ -30,11 +30,17 @@ module Mongo
       #   Mongo::Error::MissingFileChunk.new(expected_n, chunk)
       #
       # @param [ Integer ] expected_n The expected index value.
-      # @param [ Grid::File::Chunk ] chunk The chunk read from GridFS.
+      # @param [ Grid::File::Chunk | Integer ] chunk The chunk read from GridFS.
       #
       # @since 2.1.0
+      #
+      # @api private
       def initialize(expected_n, chunk)
-        super("Unexpected chunk in sequence. Expected next chunk to have index #{expected_n} but it has index #{chunk.n}")
+        if chunk.is_a?(Integer)
+          super("Missing chunk(s). Expected #{expected_n} chunks but got #{chunk}.")
+        else
+          super("Unexpected chunk in sequence. Expected next chunk to have index #{expected_n} but it has index #{chunk.n}")
+        end
       end
     end
   end

data/lib/mongo/grid/stream/read.rb

@@ -79,13 +79,19 @@ module Mongo
             ensure_readable!
             info = file_info
             num_chunks = (info.length + info.chunk_size - 1) / info.chunk_size
+            num_read = 0
             if block_given?
               view.each_with_index.reduce(0) do |length_read, (doc, index)|
                 chunk = Grid::File::Chunk.new(doc)
                 validate!(index, num_chunks, chunk, length_read)
                 data = chunk.data.data
                 yield data
+                num_read += 1
                 length_read += data.size
+              end.tap do
+                if num_read < num_chunks
+                  raise Error::MissingFileChunk.new(num_chunks, num_read)
+                end
               end
             else
               view.to_enum

data/lib/mongo/index/view.rb

@@ -71,6 +71,7 @@ module Mongo
         :weights => :weights,
         :collation => :collation,
         :comment => :comment,
+        :wildcard_projection => :wildcardProjection,
       }.freeze
 
       # Drop an index by its name.

data/lib/mongo/operation/create/op_msg.rb

@@ -34,19 +34,7 @@ module Mongo
           spec[:selector].merge(
             collation: spec[:collation],
             encryptedFields: spec[:encrypted_fields],
-          ).compact.tap do |sel|
-            # This code MUST be removed as soon as server starts accepting
-            # contention as int32.
-            if sel[:encryptedFields] && sel[:encryptedFields].key?('fields')
-              sel[:encryptedFields]['fields'] = sel[:encryptedFields]['fields'].map do |field|
-                if field['queries'] && field['queries'].key?('contention')
-                  field['queries']['contention'] = BSON::Int64.new(field['queries']['contention'])
-                end
-                field
-              end
-            end
-            # End of code to be removed
-          end
+          ).compact
         end
       end
     end

data/lib/mongo/operation/distinct/op_msg.rb

@@ -32,7 +32,10 @@ module Mongo
 
         def selector(connection)
           # Collation is always supported on 3.6+ servers that would use OP_MSG.
-          spec[:selector].merge(collation: spec[:collation]).compact
+          spec[:selector].merge(
+            collation: spec[:collation],
+            comment: spec[:comment],
+          ).compact
         end
       end
     end

data/lib/mongo/protocol/msg.rb

@@ -229,22 +229,6 @@ module Mongo
           if cmd.key?('$db') && !enc_cmd.key?('$db')
             enc_cmd['$db'] = cmd['$db']
           end
-          # This code MUST be removed as soon as server starts accepting
-          # contention as int32.
-          if schema = enc_cmd.dig('encryptionInformation', 'schema')
-            enc_cmd['encryptionInformation']['schema'] = schema.map do |coll, params|
-              if params['fields']
-                params['fields'] = params['fields'].map do |field|
-                  if contention = field.dig('queries', 'contention')
-                    field['queries']['contention'] = BSON::Int64.new(contention)
-                  end
-                  field
-                end
-              end
-              [coll, params]
-            end.to_h
-          end
-          # End of code to be removed
 
           Msg.new(@flags, @options, enc_cmd)
         else

data/lib/mongo/server/connection_pool.rb

@@ -29,7 +29,7 @@ module Mongo
       # The default max size for the connection pool.
       #
       # @since 2.9.0
-      DEFAULT_MAX_SIZE = 5.freeze
+      DEFAULT_MAX_SIZE = 20.freeze
 
       # The default min size for the connection pool.
       #
@@ -59,7 +59,8 @@ module Mongo
       # @param [ Server ] server The server which this connection pool is for.
       # @param [ Hash ] options The connection pool options.
       #
-      # @option options [ Integer ] :max_size The maximum pool size.
+      # @option options [ Integer ] :max_size The maximum pool size. Setting
+      #   this option to zero creates an unlimited connection pool.
       # @option options [ Integer ] :max_pool_size Deprecated.
       #   The maximum pool size. If max_size is also given, max_size and
       #   max_pool_size must be identical.
@@ -98,7 +99,7 @@ module Mongo
         options[:max_size] ||= options[:max_pool_size]
         options.delete(:max_pool_size)
         if options[:min_size] && options[:max_size] &&
-          options[:min_size] > options[:max_size]
+          (options[:max_size] != 0 && options[:min_size] > options[:max_size])
         then
           raise ArgumentError, "Cannot have min size #{options[:min_size]} exceed max size #{options[:max_size]}"
         end
@@ -377,7 +378,7 @@ module Mongo
                 #
                 # Ruby does not allow a thread to lock a mutex which it already
                 # holds.
-                if unsynchronized_size < max_size
+                if max_size == 0 || unsynchronized_size < max_size
                   connection = create_connection
                   @pending_connections << connection
                   throw(:done)

data/lib/mongo/server/monitor/connection.rb

@@ -228,15 +228,21 @@ module Mongo
         # @api private
         def check_document
           server_api = @app_metadata.server_api || options[:server_api]
-          if hello_ok? || server_api
-            doc = HELLO_DOC
+          doc = if hello_ok? || server_api
+            _doc = HELLO_DOC
             if server_api
-              doc = doc.merge(Utils.transform_server_api(server_api))
+              _doc = _doc.merge(Utils.transform_server_api(server_api))
             end
-            doc
+            _doc
           else
             LEGACY_HELLO_DOC
           end
+          # compressors must be set to maintain correct compression status
+          # in the server description. See RUBY-2427
+          if compressors = options[:compressors]
+            doc = doc.merge(compression: compressors)
+          end
+          doc
         end
 
         private

data/lib/mongo/server/monitor.rb

@@ -268,6 +268,10 @@ module Mongo
         end
       end
 
+      def to_s
+        "#<#{self.class.name}:#{object_id} #{server.address}>"
+      end
+
       private
 
       def pre_stop

data/lib/mongo/server/push_monitor.rb

@@ -196,6 +196,10 @@ module Mongo
         end
       end
 
+      def to_s
+        "#<#{self.class.name}:#{object_id} #{server.address}>"
+      end
+
     end
   end
 end

data/lib/mongo/version.rb

@@ -20,5 +20,5 @@ module Mongo
   # The current version of the driver.
   #
   # @since 2.0.0
-  VERSION = '2.18.0.beta1'.freeze
+  VERSION = '2.18.0'.freeze
 end

data/lib/mongo.rb

@@ -94,6 +94,8 @@ module Mongo
     # Take all the public instance methods from the Config singleton and allow
     # them to be accessed through the Mongo module directly.
     def_delegators Config, :options=
+    delegate_option Config, :broken_view_aggregate
+    delegate_option Config, :broken_view_options
     delegate_option Config, :validate_update_replace
   end
 

data/spec/README.md

@@ -713,3 +713,17 @@ To break into the debugger on JRuby, call:
 
     require 'ruby-debug'
     debugger
+
+## Testing against load balancer locally
+
+1. Install mongodb server v5.2+.
+2. Install haproxy.
+3. Install mongo-orchestration - https://github.com/10gen/mongo-orchestration/
+4. Install drivers-evergreen-tools - https://github.com/mongodb-labs/drivers-evergreen-tools. In ruby driver it is installed as git submodule under `.mod/drivers-evergreen-tools/`.
+5. Start mongo-orchestration: `mongo-orchestration start`.
+6. Start the cluster: `http PUT http://localhost:8889/v1/sharded_clusters/myCluster @.mod/drivers-evergreen-tools/.evergreen/orchestration/configs/sharded_clusters/basic-load-balancer.json` (this example uses httpie client, can be done with curl).
+7. Start load balancer: `MONGODB_URI="mongodb://localhost:27017,localhost:27018/" .mod/drivers-evergreen-tools/.evergreen/run-load-balancer.sh start`.
+8. Run tests: `TOPOLOGY=load-balanced MONGODB_URI='mongodb://127.0.0.1:8000/?loadBalanced=true' be rspec spec/`.
+9. Stop load balancer: `MONGODB_URI="mongodb://localhost:27017,localhost:27018/" .mod/drivers-evergreen-tools/.evergreen/run-load-balancer.sh stop`.
+10. Stop the cluster: `http DELETE http://localhost:8889/v1/sharded_clusters/myCluster`
+11. Stop mongo-orchestration: `mongo-orchestration stop`.

data/spec/integration/change_stream_spec.rb

@@ -4,7 +4,7 @@
 require 'spec_helper'
 
 describe 'Change stream integration' do
-  retry_test 4
+  retry_test tries: 4
   require_mri
   max_example_run_time 7
   min_server_fcv '3.6'

data/spec/integration/client_construction_spec.rb

@@ -191,16 +191,82 @@ describe 'Client construction' do
     context 'with default key vault client' do
       let(:key_vault_client) { nil }
 
-      it 'creates a working key vault client' do
-        key_vault_client = client.encrypter.key_vault_client
+      shared_examples 'creates a working key vault client' do
+        it 'creates a working key vault client' do
+          key_vault_client = client.encrypter.key_vault_client
 
-        result = key_vault_client[:test].insert_one(test: 1)
-        expect(result).to be_ok
+          result = key_vault_client[:test].insert_one(test: 1)
+          expect(result).to be_ok
+        end
       end
 
-      it 'creates a key vault client with the same cluster as the existing client' do
-        key_vault_client = client.encrypter.key_vault_client
-        expect(key_vault_client.cluster).to eq(client.cluster)
+      context 'when top-level max pool size is not 0' do
+        include_examples 'creates a working key vault client'
+
+        shared_examples 'limited connection pool' do
+          it 'creates a key vault client with a different cluster than the existing client' do
+            key_vault_client = client.encrypter.key_vault_client
+            expect(key_vault_client.cluster).not_to eq(client.cluster)
+          end
+
+          # min pool size for the key vault client can be greater than 0
+          # when the key vault client is the same as the top-level client.
+          # This is OK because we aren't making any more connections for FLE,
+          # the minimum was requested by application for its own needs.
+          it 'uses min pool size 0 for key vault client' do
+            key_vault_client = client.encrypter.key_vault_client
+            key_vault_client.options[:min_pool_size].should be 0
+          end
+        end
+
+        context 'when top-level max pool size is not specified' do
+          before do
+            client.options[:max_pool_size].should be nil
+          end
+
+          include_examples 'limited connection pool'
+
+          it 'uses unspecified max pool size for key vault client' do
+            key_vault_client = client.encrypter.key_vault_client
+            key_vault_client.options[:max_pool_size].should be nil
+          end
+        end
+
+        context 'when top-level max pool size is specified' do
+          let(:options) do
+            {
+              auto_encryption_options: auto_encryption_options,
+              max_pool_size: 42,
+            }
+          end
+
+          include_examples 'limited connection pool'
+
+          it 'uses the same max pool size for key vault client' do
+            key_vault_client = client.encrypter.key_vault_client
+            key_vault_client.options[:max_pool_size].should be 42
+          end
+        end
+      end
+
+      context 'when top-level max pool size is 0' do
+        let(:options) do
+          {
+            auto_encryption_options: auto_encryption_options,
+            max_pool_size: 0,
+          }
+        end
+
+        before do
+          client.options[:max_pool_size].should be 0
+        end
+
+        include_examples 'creates a working key vault client'
+
+        it 'creates a key vault client with the same cluster as the existing client' do
+          key_vault_client = client.encrypter.key_vault_client
+          expect(key_vault_client.cluster).to eq(client.cluster)
+        end
       end
     end
   end