mongo 2.17.3 → 2.18.0.beta1

data/spec/spec_tests/data/client_side_encryption/gcpKMS.yml

@@ -0,0 +1,46 @@
+runOn:
+  - minServerVersion: "4.1.10"
+database_name: &database_name "default"
+collection_name: &collection_name "default"
+
+data: []
+json_schema: {'properties': {'encrypted_string_aws': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'encrypted_string_azure': {'encrypt': {'keyId': [{'$binary': {'base64': 'AZURE+AAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'encrypted_string_gcp': {'encrypt': {'keyId': [{'$binary': {'base64': 'GCP+AAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'encrypted_string_local': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'encrypted_string_kmip': {'encrypt': {'keyId': [{'$binary': {'base64': 'dBHpr8aITfeBQ15grpbLpQ==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}}, 'bsonType': 'object'}
+key_vault_data: [{'_id': {'$binary': {'base64': 'GCP+AAAAAAAAAAAAAAAAAA==', 'subType': '04'}}, 'keyMaterial': {'$binary': {'base64': 'CiQAIgLj0WyktnB4dfYHo5SLZ41K4ASQrjJUaSzl5vvVH0G12G0SiQEAjlV8XPlbnHDEDFbdTO4QIe8ER2/172U1ouLazG0ysDtFFIlSvWX5ZnZUrRMmp/R2aJkzLXEt/zf8Mn4Lfm+itnjgo5R9K4pmPNvvPKNZX5C16lrPT+aA+rd+zXFSmlMg3i5jnxvTdLHhg3G7Q/Uv1ZIJskKt95bzLoe0tUVzRWMYXLIEcohnQg==', 'subType': '00'}}, 'creationDate': {'$date': {'$numberLong': '1601574333107'}}, 'updateDate': {'$date': {'$numberLong': '1601574333107'}}, 'status': {'$numberInt': '0'}, 'masterKey': {'provider': 'gcp', 'projectId': 'devprod-drivers', 'location': 'global', 'keyRing': 'key-ring-csfle', 'keyName': 'key-name-csfle'}, 'keyAltNames': ['altname', 'gcp_altname']}]
+
+tests:
+  - description: "Insert a document with auto encryption using GCP KMS provider"
+    clientOptions:
+      autoEncryptOpts:
+        kmsProviders:
+          gcp: {}
+    operations:
+      - name: insertOne
+        arguments:
+          document: &doc0 { _id: 1, encrypted_string_gcp: "string0" }
+    expectations:
+    # Auto encryption will request the collection info.
+    - command_started_event:
+        command:
+          listCollections: 1
+          filter:
+            name: *collection_name
+        command_name: listCollections
+    # Then key is fetched from the key vault.
+    - command_started_event:
+        command:
+          find: datakeys
+          filter: { $or: [ { _id: { $in: [ {'$binary': {'base64': 'GCP+AAAAAAAAAAAAAAAAAA==', 'subType': '04'}} ] } }, { keyAltNames: { $in: [] } } ] }
+          $db: keyvault
+        command_name: find
+    - command_started_event:
+        command:
+          insert: *collection_name
+          documents:
+            - &doc0_encrypted { _id: 1, encrypted_string_gcp: {'$binary': {'base64': 'ARgj/gAAAAAAAAAAAAAAAAACwFd+Y5Ojw45GUXNvbcIpN9YkRdoHDHkR4kssdn0tIMKlDQOLFkWFY9X07IRlXsxPD8DcTiKnl6XINK28vhcGlg==', 'subType': '06'}} }
+          ordered: true
+        command_name: insert
+    outcome:
+      collection:
+        # Outcome is checked using a separate MongoClient without auto encryption.
+        data:
+          - *doc0_encrypted

data/spec/spec_tests/data/client_side_encryption/getMore.yml

@@ -38,19 +38,12 @@ tests:
             find: *collection_name
             batchSize: 2
           command_name: find
-      - command_started_event:
-          command:
-            listCollections: 1
-            filter:
-              name: "datakeys"
-            $db: admin
-          command_name: listCollections
       # Then key is fetched from the key vault.
       - command_started_event:
           command:
             find: datakeys
             filter: {"$or": [{"_id": {"$in": [ {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}} ] }}, {"keyAltNames": {"$in": []}}]}
-            $db: admin
+            $db: keyvault
             readConcern: { level: "majority" }
           command_name: find
       - command_started_event:

data/spec/spec_tests/data/client_side_encryption/insert.yml

@@ -25,19 +25,12 @@ tests:
             filter:
               name: *collection_name
           command_name: listCollections
-      - command_started_event:
-          command:
-            listCollections: 1
-            filter:
-              name: "datakeys"
-            $db: admin
-          command_name: listCollections
       # Then key is fetched from the key vault.
       - command_started_event:
           command:
             find: datakeys
             filter: {"$or": [{"_id": {"$in": [ {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}} ] }}, {"keyAltNames": {"$in": []}}]}
-            $db: admin
+            $db: keyvault
             readConcern: { level: "majority" }
           command_name: find
       - command_started_event:
@@ -71,19 +64,12 @@ tests:
             filter:
               name: *collection_name
           command_name: listCollections
-      - command_started_event:
-          command:
-            listCollections: 1
-            filter:
-              name: "datakeys"
-            $db: admin
-          command_name: listCollections
       # Then key is fetched from the key vault.
       - command_started_event:
           command:
             find: datakeys
             filter: {"$or": [{"_id": {"$in": [ {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}} ] }}, {"keyAltNames": {"$in": []}}]}
-            $db: admin
+            $db: keyvault
             readConcern: { level: "majority" }
           command_name: find
       - command_started_event:

data/spec/spec_tests/data/client_side_encryption/keyAltName.yml

@@ -25,19 +25,12 @@ tests:
             filter:
               name: *collection_name
           command_name: listCollections
-      - command_started_event:
-          command:
-            listCollections: 1
-            filter:
-              name: "datakeys"
-            $db: admin
-          command_name: listCollections
       # Then key is fetched from the key vault.
       - command_started_event:
           command:
             find: datakeys
             filter: {$or: [ { _id: { $in: [] } }, { keyAltNames: { $in: [ "altname" ] } } ] }
-            $db: admin
+            $db: keyvault
             readConcern: { level: "majority" }
           command_name: find
       - command_started_event:

data/spec/spec_tests/data/client_side_encryption/localKMS.yml

@@ -26,19 +26,12 @@ tests:
           filter:
             name: *collection_name
         command_name: listCollections
-    - command_started_event:
-        command:
-          listCollections: 1
-          filter:
-            name: "datakeys"
-          $db: admin
-        command_name: listCollections
     # Then key is fetched from the key vault.
     - command_started_event:
         command:
           find: datakeys
           filter: { $or: [ { _id: { $in: [ {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}} ] } }, { keyAltNames: { $in: [] } } ] }
-          $db: admin
+          $db: keyvault
         command_name: find
     - command_started_event:
         command:

data/spec/spec_tests/data/client_side_encryption/localSchema.yml

@@ -25,19 +25,12 @@ tests:
           filter: { _id: 1 }
         result: [*doc0]
     expectations:
-      - command_started_event:
-          command:
-            listCollections: 1
-            filter:
-              name: "datakeys"
-            $db: admin
-          command_name: listCollections
       # Then key is fetched from the key vault.
       - command_started_event:
           command:
             find: datakeys
             filter: {"$or": [{"_id": {"$in": [ {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}} ] }}, {"keyAltNames": {"$in": []}}]}
-            $db: admin
+            $db: keyvault
             readConcern: { level: "majority" }
           command_name: find
       - command_started_event:

data/spec/spec_tests/data/client_side_encryption/maxWireVersion.yml

@@ -12,6 +12,8 @@ tests:
       autoEncryptOpts:
         kmsProviders:
           aws: {} # Credentials filled in from environment.
+        extraOptions:
+          mongocryptdBypassSpawn: true # mongocryptd probably won't be on the path
     operations:
       - name: insertOne
         arguments:

data/spec/spec_tests/data/client_side_encryption/missingKey.yml

@@ -11,7 +11,7 @@ tests:
   - description: "Insert with encryption on a missing key"
     clientOptions:
       autoEncryptOpts:
-        keyVaultNamespace: "admin.different"
+        keyVaultNamespace: "keyvault.different"
         kmsProviders:
           aws: {} # Credentials filled in from environment.
     operations:
@@ -32,18 +32,11 @@ tests:
             filter:
               name: *collection_name
           command_name: listCollections
-      - command_started_event:
-          command:
-            listCollections: 1
-            filter:
-              name: "different"
-            $db: admin
-          command_name: listCollections
       # Then key is fetched from the key vault.
       - command_started_event:
           command:
             find: different
             filter: {"$or": [{"_id": {"$in": [ {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}} ] }}, {"keyAltNames": {"$in": []}}]}
-            $db: admin
+            $db: keyvault
             readConcern: { level: "majority" }
           command_name: find

data/spec/spec_tests/data/client_side_encryption/noSchema.yml

@@ -0,0 +1,39 @@
+# Test auto encryption on a collection with no jsonSchema configured.
+# This is a regression test for MONGOCRYPT-378/PYTHON-3188.
+runOn:
+  - minServerVersion: "4.1.10"
+database_name: &database_name "default"
+collection_name: &collection_name "unencrypted"
+
+data: []
+
+tests:
+  - description: "Insert on an unencrypted collection"
+    clientOptions:
+      autoEncryptOpts:
+        kmsProviders:
+          aws: {} # Credentials filled in from environment.
+    operations:
+      - name: insertOne
+        arguments:
+          document: &doc0 { _id: 1 }
+    expectations:
+      # Auto encryption will request the collection info.
+      - command_started_event:
+          command:
+            listCollections: 1
+            filter:
+              name: *collection_name
+          command_name: listCollections
+      - command_started_event:
+          command:
+            insert: *collection_name
+            documents:
+              - *doc0
+            ordered: true
+          command_name: insert
+    outcome:
+      collection:
+        # Outcome is checked using a separate MongoClient without auto encryption.
+        data:
+          - *doc0

data/spec/spec_tests/data/client_side_encryption/replaceOne.yml

@@ -31,19 +31,12 @@ tests:
             filter:
               name: *collection_name
           command_name: listCollections
-      - command_started_event:
-          command:
-            listCollections: 1
-            filter:
-              name: "datakeys"
-            $db: admin
-          command_name: listCollections
       # Then key is fetched from the key vault.
       - command_started_event:
           command:
             find: datakeys
             filter: {"$or": [{"_id": {"$in": [ {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}} ] }}, {"keyAltNames": {"$in": []}}]}
-            $db: admin
+            $db: keyvault
             readConcern: { level: "majority" }
           command_name: find
       - command_started_event:

data/spec/spec_tests/data/client_side_encryption/types.yml

@@ -27,19 +27,12 @@ tests:
           filter: { _id: 1 }
         result: *doc0
     expectations:
-      - command_started_event:
-          command:
-            listCollections: 1
-            filter:
-              name: "datakeys"
-            $db: admin
-          command_name: listCollections
       # Then key is fetched from the key vault.
       - command_started_event:
           command:
             find: datakeys
             filter: {"$or": [{"_id": {"$in": [ {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}} ] }}, {"keyAltNames": {"$in": []}}]}
-            $db: admin
+            $db: keyvault
             readConcern: { level: "majority" }
           command_name: find
       - command_started_event:
@@ -75,19 +68,12 @@ tests:
           filter: { _id: 1 }
         result: *doc1
     expectations:
-      - command_started_event:
-          command:
-            listCollections: 1
-            filter:
-              name: "datakeys"
-            $db: admin
-          command_name: listCollections
       # Then key is fetched from the key vault.
       - command_started_event:
           command:
             find: datakeys
             filter: {"$or": [{"_id": {"$in": [ {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}} ] }}, {"keyAltNames": {"$in": []}}]}
-            $db: admin
+            $db: keyvault
             readConcern: { level: "majority" }
           command_name: find
       - command_started_event:
@@ -123,19 +109,12 @@ tests:
           filter: { _id: 1 }
         result: *doc2
     expectations:
-      - command_started_event:
-          command:
-            listCollections: 1
-            filter:
-              name: "datakeys"
-            $db: admin
-          command_name: listCollections
       # Then key is fetched from the key vault.
       - command_started_event:
           command:
             find: datakeys
             filter: {"$or": [{"_id": {"$in": [ {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}} ] }}, {"keyAltNames": {"$in": []}}]}
-            $db: admin
+            $db: keyvault
             readConcern: { level: "majority" }
           command_name: find
       - command_started_event:
@@ -167,7 +146,12 @@ tests:
         arguments:
           document: &doc4 { _id: 1, encrypted_double: {"$numberDouble": "1.23"} }
         result:
-            errorContains: "Cannot use deterministic encryption for element of type: double"
+            # DRIVERS-2272: The expected error message changed in mongocryptd 6.0. Before it was:
+            # "Cannot use deterministic encryption for element of type: double"
+            # After it is:
+            # "Cannot encrypt element of type: double"
+            # Only check for the common suffix.
+            errorContains: "element of type: double"
   - description: "type=decimal"
     clientOptions:
       autoEncryptOpts:
@@ -180,7 +164,12 @@ tests:
         arguments:
           document: &doc5 { _id: 1, encrypted_decimal: {"$numberDecimal": "1.23"} }
         result:
-            errorContains: "Cannot use deterministic encryption for element of type: decimal"
+            # DRIVERS-2272: The expected error message changed in mongocryptd 6.0. Before it was:
+            # "Cannot use deterministic encryption for element of type: decimal"
+            # After it is:
+            # "Cannot encrypt element of type: decimal"
+            # Only check for the common suffix.
+            errorContains: "element of type: decimal"
   - description: "type=binData"
     clientOptions:
       autoEncryptOpts:
@@ -197,19 +186,12 @@ tests:
           filter: { _id: 1 }
         result: *doc6
     expectations:
-      - command_started_event:
-          command:
-            listCollections: 1
-            filter:
-              name: "datakeys"
-            $db: admin
-          command_name: listCollections
       # Then key is fetched from the key vault.
       - command_started_event:
           command:
             find: datakeys
             filter: {"$or": [{"_id": {"$in": [ {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}} ] }}, {"keyAltNames": {"$in": []}}]}
-            $db: admin
+            $db: keyvault
             readConcern: { level: "majority" }
           command_name: find
       - command_started_event:
@@ -245,19 +227,12 @@ tests:
           filter: { _id: 1 }
         result: *doc7
     expectations:
-      - command_started_event:
-          command:
-            listCollections: 1
-            filter:
-              name: "datakeys"
-            $db: admin
-          command_name: listCollections
       # Then key is fetched from the key vault.
       - command_started_event:
           command:
             find: datakeys
             filter: {"$or": [{"_id": {"$in": [ {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}} ] }}, {"keyAltNames": {"$in": []}}]}
-            $db: admin
+            $db: keyvault
             readConcern: { level: "majority" }
           command_name: find
       - command_started_event:
@@ -289,7 +264,12 @@ tests:
         arguments:
           document: &doc8 { _id: 1, encrypted_javascriptWithScope: {"$code": "var x = 1;", "$scope": {} } }
         result:
-          errorContains: "Cannot use deterministic encryption for element of type: javascriptWithScope"
+          # DRIVERS-2272: The expected error message changed in mongocryptd 6.0. Before it was:
+          # "Cannot use deterministic encryption for element of type: javascriptWithScope"
+          # After it is:
+          # "Cannot encrypt element of type: javascriptWithScope"
+          # Only check for the common suffix.
+          errorContains: "element of type: javascriptWithScope"
   - description: "type=object"
     clientOptions:
       autoEncryptOpts:
@@ -302,7 +282,12 @@ tests:
         arguments:
           document: &doc9 { _id: 1, encrypted_object: {} }
         result:
-          errorContains: "Cannot use deterministic encryption for element of type: object"
+          # DRIVERS-2272: The expected error message changed in mongocryptd 6.0. Before it was:
+          # "Cannot use deterministic encryption for element of type: object"
+          # After it is:
+          # "Cannot encrypt element of type: object"
+          # Only check for the common suffix.
+          errorContains: "element of type: object"
   - description: "type=timestamp"
     clientOptions:
       autoEncryptOpts:
@@ -319,19 +304,12 @@ tests:
           filter: { _id: 1 }
         result: *doc10
     expectations:
-      - command_started_event:
-          command:
-            listCollections: 1
-            filter:
-              name: "datakeys"
-            $db: admin
-          command_name: listCollections
       # Then key is fetched from the key vault.
       - command_started_event:
           command:
             find: datakeys
             filter: {"$or": [{"_id": {"$in": [ {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}} ] }}, {"keyAltNames": {"$in": []}}]}
-            $db: admin
+            $db: keyvault
             readConcern: { level: "majority" }
           command_name: find
       - command_started_event:
@@ -367,19 +345,12 @@ tests:
           filter: { _id: 1 }
         result: *doc11
     expectations:
-      - command_started_event:
-          command:
-            listCollections: 1
-            filter:
-              name: "datakeys"
-            $db: admin
-          command_name: listCollections
       # Then key is fetched from the key vault.
       - command_started_event:
           command:
             find: datakeys
             filter: {"$or": [{"_id": {"$in": [ {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}} ] }}, {"keyAltNames": {"$in": []}}]}
-            $db: admin
+            $db: keyvault
             readConcern: { level: "majority" }
           command_name: find
       - command_started_event:
@@ -415,19 +386,12 @@ tests:
           filter: { _id: 1 }
         result: *doc13
     expectations:
-      - command_started_event:
-          command:
-            listCollections: 1
-            filter:
-              name: "datakeys"
-            $db: admin
-          command_name: listCollections
       # Then key is fetched from the key vault.
       - command_started_event:
           command:
             find: datakeys
             filter: {"$or": [{"_id": {"$in": [ {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}} ] }}, {"keyAltNames": {"$in": []}}]}
-            $db: admin
+            $db: keyvault
             readConcern: { level: "majority" }
           command_name: find
       - command_started_event:
@@ -498,7 +462,12 @@ tests:
         arguments:
           document: &doc17 { _id: 1, encrypted_array: [] }
         result:
-          errorContains: "Cannot use deterministic encryption for element of type: array"
+          # DRIVERS-2272: The expected error message changed in mongocryptd 6.0. Before it was:
+          # "Cannot use deterministic encryption for element of type: array"
+          # After it is:
+          # "Cannot encrypt element of type: array"
+          # Only check for the common suffix.
+          errorContains: "element of type: array"
   - description: "type=bool"
     clientOptions:
       autoEncryptOpts:
@@ -511,7 +480,12 @@ tests:
         arguments:
           document: &doc18 { _id: 1, encrypted_bool: true }
         result:
-          errorContains: "Cannot use deterministic encryption for element of type: bool"
+          # DRIVERS-2272: The expected error message changed in mongocryptd 6.0. Before it was:
+          # "Cannot use deterministic encryption for element of type: bool"
+          # After it is:
+          # "Cannot encrypt element of type: bool"
+          # Only check for the common suffix.
+          errorContains: "element of type: bool"
   - description: "type=null"
     clientOptions:
       autoEncryptOpts:

data/spec/spec_tests/data/client_side_encryption/updateMany.yml

@@ -32,19 +32,12 @@ tests:
             filter:
               name: *collection_name
           command_name: listCollections
-      - command_started_event:
-          command:
-            listCollections: 1
-            filter:
-              name: "datakeys"
-            $db: admin
-          command_name: listCollections
       # Then key is fetched from the key vault.
       - command_started_event:
           command:
             find: datakeys
             filter: {"$or": [{"_id": {"$in": [ {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}} ] }}, {"keyAltNames": {"$in": []}}]}
-            $db: admin
+            $db: keyvault
             readConcern: { level: "majority" }
           command_name: find
       - command_started_event:

data/spec/spec_tests/data/client_side_encryption/updateOne.yml

@@ -31,19 +31,12 @@ tests:
             filter:
               name: *collection_name
           command_name: listCollections
-      - command_started_event:
-          command:
-            listCollections: 1
-            filter:
-              name: "datakeys"
-            $db: admin
-          command_name: listCollections
       # Then key is fetched from the key vault.
       - command_started_event:
           command:
             find: datakeys
             filter: {"$or": [{"_id": {"$in": [ {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}} ] }}, {"keyAltNames": {"$in": []}}]}
-            $db: admin
+            $db: keyvault
             readConcern: { level: "majority" }
           command_name: find
       - command_started_event: