mongo 2.20.0 → 2.21.0

data/spec/runners/unified/test.rb

@@ -2,6 +2,7 @@
 # rubocop:todo all
 
 require 'runners/crud/requirement'
+require 'runners/unified/ambiguous_operations'
 require 'runners/unified/client_side_encryption_operations'
 require 'runners/unified/crud_operations'
 require 'runners/unified/grid_fs_operations'
@@ -17,6 +18,7 @@ require 'support/crypt'
 module Unified
 
   class Test
+    include AmbiguousOperations
     include ClientSideEncryptionOperations
     include CrudOperations
     include GridFsOperations
@@ -42,12 +44,14 @@ module Unified
       if req = @spec['group_runOnRequirements']
         @group_reqs = req.map { |r| Mongo::CRUD::Requirement.new(r) }
       end
-      mongoses = @spec['createEntities'].select do |spec|
-        spec['client']
-      end.map do |spec|
-        spec['client']['useMultipleMongoses']
-      end.compact.uniq
-      @multiple_mongoses = mongoses.any? { |v| v }
+      if @spec['createEntities']
+        mongoses = @spec['createEntities'].select do |spec|
+          spec['client']
+        end.map do |spec|
+          spec['client']['useMultipleMongoses']
+        end.compact.uniq
+        @multiple_mongoses = mongoses.any? { |v| v }
+      end
       @test_spec.freeze
       @subscribers = {}
       @observe_sensitive = {}
@@ -85,6 +89,8 @@ module Unified
     end
 
     def generate_entities(es)
+      return if es.nil?
+
       es.each do |entity_spec|
         unless entity_spec.keys.length == 1
           raise NotImplementedError, "Entity must have exactly one key"
@@ -329,7 +335,7 @@ module Unified
         begin
           collection.create(create_options)
         rescue Mongo::Error => e
-          if Mongo::Error::OperationFailure === e && (
+          if Mongo::Error::OperationFailure::Family === e && (
               e.code == 48 || e.message =~ /collection already exists/
           )
             # Already exists
@@ -411,10 +417,16 @@ module Unified
 
             public_send(method_name, op)
           rescue Mongo::Error, bson_error, Mongo::Auth::Unauthorized, ArgumentError => e
+            if expected_error.use('isTimeoutError')
+              unless Mongo::Error::TimeoutError === e
+                raise e
+                raise Error::ErrorMismatch, %Q,Expected TimeoutError ("isTimeoutError") but got #{e},
+              end
+            end
             if expected_error.use('isClientError')
               # isClientError doesn't actually mean a client error.
               # It means anything other than OperationFailure. DRIVERS-1799
-              if Mongo::Error::OperationFailure === e
+              if Mongo::Error::OperationFailure::Family === e
                 raise Error::ErrorMismatch, %Q,Expected not OperationFailure ("isClientError") but got #{e},
               end
             end
@@ -482,7 +494,7 @@ module Unified
             if result.nil? && expected_result.keys == ["$$unsetOrMatches"]
               return
             elsif result.nil? && !expected_result.empty?
-              raise Error::ResultMismatch, "#{msg}: expected #{expected} but got nil"
+              raise Error::ResultMismatch, "expected #{expected_result} but got nil"
             elsif Array === expected_result
               assert_documents_match(result, expected_result)
             else
@@ -536,7 +548,7 @@ module Unified
         root_authorized_client.command(
           killAllSessions: [],
         )
-      rescue Mongo::Error::OperationFailure => e
+      rescue Mongo::Error::OperationFailure::Family => e
         if e.code == 11601
           # operation was interrupted, ignore. SERVER-38335
         elsif e.code == 13

data/spec/runners/unified.rb

@@ -72,7 +72,7 @@ def define_unified_spec_tests(base_path, paths, expect_failure: false)
                   test.assert_events
                 # HACK: other errors are possible and likely will need to
                 # be added here later as the tests evolve.
-                rescue Mongo::Error::OperationFailure, Unified::Error::UnsupportedOperation, UsingHash::UsingHashKeyError, Unified::Error::EntityMissing
+                rescue Mongo::Error::OperationFailure::Family, Unified::Error::UnsupportedOperation, UsingHash::UsingHashKeyError, Unified::Error::EntityMissing
                 rescue => e
                   fail "Expected to raise Mongo::Error::OperationFailure or Unified::Error::UnsupportedOperation or UsingHash::UsingHashKeyError or Unified::Error::EntityMissing, got #{e.class}: #{e}"
                 else

data/spec/solo/clean_exit_spec.rb

@@ -17,6 +17,8 @@ describe 'Clean exit' do
     it 'exits cleanly' do
       client = Mongo::Client.new(uri)
       client.database.collection_names.to_a
+    ensure
+      client.close
     end
   end
 end

data/spec/spec_tests/client_side_operations_timeout_spec.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'runners/unified'
+
+base = "#{CURRENT_PATH}/spec_tests/data/client_side_operations_timeout"
+CSOT_TESTS = Dir.glob("#{base}/**/*.yml").sort
+
+describe 'CSOT unified spec tests' do
+  if [ 1, '1', 'yes', 'true' ].include?(ENV['CSOT_SPEC_TESTS'])
+    define_unified_spec_tests(base, CSOT_TESTS)
+  else
+    skip 'CSOT spec tests are disabled. To enable them set env variable CSOT_SPEC_TESTS to 1'
+  end
+end

data/spec/spec_tests/data/change_streams_unified/change-streams-clusterTime.yml

@@ -15,7 +15,9 @@ createEntities:
 
 runOnRequirements:
   - minServerVersion: "4.0.0"
-    topologies: [ replicaset, load-balanced, sharded ]
+    # TODO(DRIVERS-2323): Run all possible tests against sharded clusters once we know the
+    # cause of unexpected command monitoring events.
+    topologies: [ replicaset ]
     serverless: forbid
 
 initialData:

data/spec/spec_tests/data/change_streams_unified/change-streams-disambiguatedPaths.yml

@@ -15,7 +15,9 @@ createEntities:
 
 runOnRequirements:
   - minServerVersion: "6.1.0"
-    topologies: [ replicaset, load-balanced, sharded ]
+    # TODO(DRIVERS-2323): Run all possible tests against sharded clusters once we know the
+    # cause of unexpected command monitoring events.
+    topologies: [ replicaset ]
     serverless: forbid
 
 initialData:

data/spec/spec_tests/data/change_streams_unified/change-streams-errors.yml

@@ -3,7 +3,9 @@ description: "change-streams-errors"
 schemaVersion: "1.7"
 
 runOnRequirements:
-  - serverless: forbid
+    # TODO(DRIVERS-2323): Run all possible tests against sharded clusters once we know the
+    # cause of unexpected command monitoring events.
+  - topologies: [ replicaset ]
 
 createEntities:
   - client:

data/spec/spec_tests/data/change_streams_unified/change-streams-pre_and_post_images.yml

@@ -4,7 +4,7 @@ schemaVersion: "1.4"
 
 runOnRequirements:
   - minServerVersion: "6.0.0"
-    topologies: [ replicaset, sharded, load-balanced ]
+    topologies: [ replicaset ]
     serverless: forbid
 
 createEntities:

data/spec/spec_tests/data/change_streams_unified/change-streams-resume-allowlist.yml

@@ -5,7 +5,7 @@ schemaVersion: "1.7"
 
 runOnRequirements:
   - minServerVersion: "3.6"
-    topologies: [ replicaset, sharded, load-balanced ]
+    topologies: [ replicaset ]
     serverless: forbid
 
 createEntities:

data/spec/spec_tests/data/change_streams_unified/change-streams-resume-errorLabels.yml

@@ -5,7 +5,7 @@ schemaVersion: "1.7"
 
 runOnRequirements:
   - minServerVersion: "4.3.1"
-    topologies: [ replicaset, sharded, load-balanced ]
+    topologies: [ replicaset ]
     serverless: forbid
 
 createEntities:

data/spec/spec_tests/data/change_streams_unified/change-streams-showExpandedEvents.yml

@@ -2,7 +2,7 @@ description: "change-streams-showExpandedEvents"
 schemaVersion: "1.7"
 runOnRequirements:
   - minServerVersion: "6.0.0"
-    topologies: [ replicaset, sharded ]
+    topologies: [ replicaset ]
     serverless: forbid
 createEntities:
   - client:

data/spec/spec_tests/data/client_side_encryption/badQueries.yml

@@ -1,5 +1,6 @@
 runOn:
   - minServerVersion: "4.1.10"
+    topology: [ "replicaset", "sharded" ]
 database_name: &database_name "default"
 collection_name: &collection_name "default"
 
@@ -533,4 +534,4 @@ tests:
           filter: {}
           fieldName: "encrypted_w_altname"
         result:
-          errorContains: "The distinct key is not allowed to be marked for encryption with a non-UUID keyId"
+          errorContains: "The distinct key is not allowed to be marked for encryption with a non-UUID keyId"

data/spec/spec_tests/data/client_side_encryption/fle2v2-BypassQueryAnalysis.yml

@@ -1,6 +1,7 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
+    maxServerVersion: "7.99.99"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

data/spec/spec_tests/data/client_side_encryption/fle2v2-Compact.yml

@@ -1,6 +1,7 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
+    maxServerVersion: "7.99.99"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

data/spec/spec_tests/data/client_side_encryption/fle2v2-CreateCollection.yml

@@ -1,6 +1,7 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
+    maxServerVersion: "7.99.99"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

data/spec/spec_tests/data/client_side_encryption/fle2v2-DecryptExistingData.yml

@@ -1,6 +1,7 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
+    maxServerVersion: "7.99.99"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

data/spec/spec_tests/data/client_side_encryption/fle2v2-Delete.yml

@@ -1,6 +1,7 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
+    maxServerVersion: "7.99.99"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

data/spec/spec_tests/data/client_side_encryption/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.yml

@@ -1,6 +1,7 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
+    maxServerVersion: "7.99.99"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

data/spec/spec_tests/data/client_side_encryption/fle2v2-EncryptedFields-vs-jsonSchema.yml

@@ -1,6 +1,7 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
+    maxServerVersion: "7.99.99"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

data/spec/spec_tests/data/client_side_encryption/fle2v2-EncryptedFieldsMap-defaults.yml

@@ -1,6 +1,7 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
+    maxServerVersion: "7.99.99"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

data/spec/spec_tests/data/client_side_encryption/fle2v2-FindOneAndUpdate.yml

@@ -1,6 +1,7 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
+    maxServerVersion: "7.99.99"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

data/spec/spec_tests/data/client_side_encryption/fle2v2-InsertFind-Indexed.yml

@@ -1,6 +1,7 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
+    maxServerVersion: "7.99.99"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

data/spec/spec_tests/data/client_side_encryption/fle2v2-InsertFind-Unindexed.yml

@@ -1,6 +1,7 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
+    maxServerVersion: "7.99.99"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

data/spec/spec_tests/data/client_side_encryption/fle2v2-MissingKey.yml

@@ -1,6 +1,7 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
+    maxServerVersion: "7.99.99"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

data/spec/spec_tests/data/client_side_encryption/fle2v2-NoEncryption.yml

@@ -1,6 +1,7 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
+    maxServerVersion: "7.99.99"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

data/spec/spec_tests/data/client_side_encryption/fle2v2-Update.yml

@@ -1,6 +1,7 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
+    maxServerVersion: "7.99.99"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

data/spec/spec_tests/data/client_side_encryption/fle2v2-validatorAndPartialFieldExpression.yml

@@ -2,6 +2,7 @@
 runOn:
     # Require server version 6.0.0 to get behavior added in SERVER-64911.
     - minServerVersion: "7.0.0"
+      maxServerVersion: "7.99.99"
       # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
       # FLE 2 Encrypted collections are not supported on standalone.
       topology: [ "replicaset", "sharded", "load-balanced" ]
@@ -166,4 +167,4 @@ tests:
               partialFilterExpression:
                 encryptedIndexed: "foo"
       result:
-        errorContains: "Comparison to encrypted fields not supported"
+        errorContains: "Comparison to encrypted fields not supported"

data/spec/spec_tests/data/client_side_encryption/timeoutMS.yml

@@ -0,0 +1,67 @@
+runOn:
+  - minServerVersion: "4.4"
+database_name: &database_name "cse-timeouts-db"
+collection_name: &collection_name "cse-timeouts-coll"
+
+data: []
+json_schema: {'properties': {'encrypted_w_altname': {'encrypt': {'keyId': '/altname', 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Random'}}, 'encrypted_string': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'random': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Random'}}, 'encrypted_string_equivalent': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}}, 'bsonType': 'object'}
+key_vault_data: [{'status': 1, '_id': {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}, 'masterKey': {'provider': 'aws', 'key': 'arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0', 'region': 'us-east-1'}, 'updateDate': {'$date': {'$numberLong': '1552949630483'}}, 'keyMaterial': {'$binary': {'base64': 'AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO', 'subType': '00'}}, 'creationDate': {'$date': {'$numberLong': '1552949630483'}}, 'keyAltNames': ['altname', 'another_altname']}]
+
+tests:
+  - description: "timeoutMS applied to listCollections to get collection schema"
+    failPoint:
+      configureFailPoint: failCommand
+      mode: { times: 1 }
+      data:
+        failCommands: ["listCollections"]
+        blockConnection: true
+        blockTimeMS: 60
+    clientOptions:
+      autoEncryptOpts:
+        kmsProviders:
+          aws: {} # Credentials filled in from environment.
+      timeoutMS: 50
+    operations:
+      - name: insertOne
+        arguments:
+          document: &doc0 { _id: 1, encrypted_string: "string0", random: "abc" }
+        result:
+          isTimeoutError: true
+    expectations:
+      # Auto encryption will request the collection info.
+      - command_started_event:
+          command:
+            listCollections: 1
+            filter:
+              name: *collection_name
+            maxTimeMS: { $$type: ["int", "long"] }
+          command_name: listCollections
+
+  # Test that timeoutMS applies to the sum of all operations done for client-side encryption. This is done by blocking
+  # listCollections and find for 30ms each and running an insertOne with timeoutMS=50. There should be one
+  # listCollections command and one "find" command, so the sum should take more than timeoutMS. A second listCollections
+  # event doesn't occur due to the internal MongoClient lacking configured auto encryption, plus libmongocrypt holds the
+  # collection schema in cache for a minute.
+  #
+  # This test does not include command monitoring expectations because the exact command sequence is dependent on the
+  # amount of time taken by mongocryptd communication. In slow runs, mongocryptd communication can breach the timeout
+  # and result in the final "find" not being sent.
+  - description: "remaining timeoutMS applied to find to get keyvault data"
+    failPoint:
+      configureFailPoint: failCommand
+      mode: { times: 2 }
+      data:
+        failCommands: ["listCollections", "find"]
+        blockConnection: true
+        blockTimeMS: 30
+    clientOptions:
+      autoEncryptOpts:
+        kmsProviders:
+          aws: {} # Credentials filled in from environment.
+      timeoutMS: 50
+    operations:
+      - name: insertOne
+        arguments:
+          document: *doc0
+        result:
+          isTimeoutError: true

data/spec/spec_tests/data/client_side_operations_timeout/bulkWrite.yml

@@ -0,0 +1,87 @@
+description: "timeoutMS behaves correctly for bulkWrite operations"
+
+schemaVersion: "1.9"
+
+runOnRequirements:
+  - minServerVersion: "4.4"
+
+createEntities:
+  - client:
+      id: &failPointClient failPointClient
+      useMultipleMongoses: false
+  - client:
+      id: &client client
+      useMultipleMongoses: false
+      observeEvents:
+        - commandStartedEvent
+      uriOptions:
+        # Used to speed up the test
+        w: 1
+  - database:
+      id: &database database
+      client: *client
+      databaseName: &databaseName test
+  - collection:
+      id: &collection collection
+      database: *database
+      collectionName: &collectionName coll 
+
+initialData:
+  - collectionName: *collectionName
+    databaseName: *databaseName
+    documents: []
+
+tests:
+  # Test that drivers do not refresh timeoutMS between commands. This is done by running a bulkWrite that will require
+  # two commands with timeoutMS=200 and blocking each command for 120ms. The server should take over 200ms total, so the
+  # bulkWrite should fail with a timeout error.
+  - description: "timeoutMS applied to entire bulkWrite, not individual commands"
+    operations:
+      # Do an operation without a timeout to ensure the servers are discovered.
+      - name: insertOne
+        object: *collection
+        arguments:
+          document: {}
+      - name: failPoint
+        object: testRunner
+        arguments:
+          client: *failPointClient
+          failPoint:
+            configureFailPoint: failCommand
+            mode: { times: 2 }
+            data:
+              failCommands: ["insert", "update"]
+              blockConnection: true
+              blockTimeMS: 120
+      - name: bulkWrite
+        object: *collection
+        arguments:
+          requests:
+            - insertOne:
+                document: { _id: 1 }
+            - replaceOne:
+                filter: { _id: 1 }
+                replacement: { x: 1 }
+          timeoutMS: 200
+        expectError:
+          isTimeoutError: true
+    expectEvents:
+      - client: *client
+        events:
+          - commandStartedEvent:
+              commandName: insert
+              databaseName: *databaseName
+              command:
+                insert: *collectionName
+          - commandStartedEvent:
+              commandName: insert
+              databaseName: *databaseName
+              command:
+                insert: *collectionName
+                maxTimeMS: { $$type: ["int", "long"] }
+          - commandStartedEvent:
+              commandName: update
+              databaseName: *databaseName
+              command:
+                update: *collectionName
+                maxTimeMS: { $$type: ["int", "long"] }