mongo 2.17.4 → 2.18.0.beta1

data/spec/support/crypt/data_keys/key_document_azure.json

@@ -0,0 +1,33 @@
+{
+  "status": {
+      "$numberInt": "1"
+  },
+  "_id": {
+    "$binary": {
+        "base64": "AZUREAAAAAAAAAAAAAAAAA==",
+        "subType": "04"
+    }
+  },
+  "masterKey": {
+    "provider": "azure",
+    "keyVaultEndpoint": "key-vault-csfle.vault.azure.net",
+    "keyName": "key-name-csfle"
+  },
+  "updateDate": {
+      "$date": {
+          "$numberLong": "1557827033449"
+      }
+  },
+  "keyAltNames": ["ssn_encryption_key"],
+  "keyMaterial": {
+    "$binary": {
+        "base64": "GjKv7Q/e62E14noqXuwOWf/AI3IZJcdQ1Jcsh86MR/582kuHBQSy7hXYi1sL\n8fn8zkWe987/Ll2Oq43049djxQGEobmw8Qg3Gk2czRCzS8TMy6yASMfwROO7\nn0k+QJwiTqzLRfP+rkJVxSde1v+nPjmonup8T1L98WJywjHFDWaxI32o7X6U\nY9iTVdQ1o8RfyR9IUOg7asHWq1zbn7CwmHz264OBG79SKXN5AkG8X4QfGIQh\nu0v7H3n4r8ZpvIMa2XFrHGkoDgkCKiAmat5s5RkChT57Bu6h4Q98Fg1clIBU\n2G/BrdtbzKYktpq6CZFyvtXd48juya3w7UDLAu7h6Q==",
+        "subType": "00"
+    }
+  },
+  "creationDate": {
+      "$date": {
+          "$numberLong": "1637319584426"
+      }
+   }
+}

data/spec/support/crypt/data_keys/key_document_gcp.json

@@ -0,0 +1,37 @@
+{
+    "status": {
+        "$numberInt": "1"
+    },
+    "_id": {
+        "$binary": {
+            "base64": "GCPAAAAAAAAAAAAAAAAAAA==",
+            "subType": "04"
+        }
+    },
+    "masterKey": {
+        "provider": "gcp",
+        "projectId": "devprod-drivers",
+        "location": "global",
+        "keyRing": "key-ring-csfle",
+        "keyName": "key-name-csfle"
+    },
+    "updateDate": {
+        "$date": {
+            "$numberLong": "1557827033449"
+        }
+    },
+    "keyAltNames": [
+        "ssn_encryption_key"
+    ],
+    "keyMaterial": {
+        "$binary": {
+            "base64": "CiQAIgLj0fMxF5M7RuQbBultgQXS8zwxnJbKQPbdsHvLPvfiP1QSiQEAuvl7\nn4jiN8avA4SFq/K/Yns9jBBAiSKtA3OVxrAe4VEZ12U2lntLYHECCzp8OIP8\nBf/FqRjr3AHYKbfRDjngKgGDJBfSjqiq7SJN1OThwQxaBp2nvuvjn6UQ3t/f\noYL0FHW20+PL23+K/35rr8iSAyR4w+7spOJ6XmaQDPuhzKthLcrPaedcAQ==",
+            "subType": "00"
+        }
+    },
+    "creationDate": {
+        "$date": {
+            "$numberLong": "1557827033449"
+        }
+    }
+}

data/spec/support/crypt/data_keys/key_document_kmip.json

@@ -0,0 +1,32 @@
+{
+  "status": {
+      "$numberInt": "1"
+  },
+  "_id": {
+    "$binary": {
+        "base64": "KMIPAAAAAAAAAAAAAAAAAA==",
+        "subType": "04"
+    }
+  },
+  "masterKey": {
+      "provider": "kmip",
+      "keyId": "1"
+  },
+  "updateDate": {
+      "$date": {
+          "$numberLong": "1557827033449"
+      }
+  },
+  "keyAltNames": ["ssn_encryption_key"],
+  "keyMaterial": {
+    "$binary": {
+        "base64": "eUYDyB0HuWb+lQgUwO+6qJQyTTDTY2gp9FbemL7ZFo0pvr0x6rm6Ff9OVUTGH6HyMKipaeHdiIJU1dzsLwvqKvi7Beh+U4iaIWX/K0oEg1GOsJc0+Z/in8gNHbGUYLmycHViM3LES3kdt7FdFSUl5rEBHrM71yoNEXImz17QJWMGOuT4x6yoi2pvnaRJwfrI4DjpmnnTrDMac92jgZehbg==",
+        "subType": "00"
+    }
+  },
+  "creationDate": {
+      "$date": {
+          "$numberLong": "1557827033449"
+      }
+   }
+}

data/spec/support/crypt/encryptedFields.json

@@ -0,0 +1,33 @@
+{
+  "escCollection": "enxcol_.default.esc",
+  "eccCollection": "enxcol_.default.ecc",
+  "ecocCollection": "enxcol_.default.ecoc",
+  "fields": [
+    {
+      "keyId": {
+        "$binary": {
+          "base64": "EjRWeBI0mHYSNBI0VniQEg==",
+          "subType": "04"
+        }
+      },
+      "path": "encryptedIndexed",
+      "bsonType": "string",
+      "queries": {
+        "queryType": "equality",
+        "contention": {
+          "$numberLong": "0"
+        }
+      }
+    },
+    {
+      "keyId": {
+        "$binary": {
+          "base64": "q83vqxI0mHYSNBI0VniQEg==",
+          "subType": "04"
+        }
+      },
+      "path": "encryptedUnindexed",
+      "bsonType": "string"
+    }
+  ]
+}

data/spec/support/crypt/keys/key1-document.json

@@ -0,0 +1,30 @@
+{
+    "_id": {
+        "$binary": {
+            "base64": "EjRWeBI0mHYSNBI0VniQEg==",
+            "subType": "04"
+        }
+    },
+    "keyMaterial": {
+        "$binary": {
+            "base64": "sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==",
+            "subType": "00"
+        }
+    },
+    "creationDate": {
+        "$date": {
+            "$numberLong": "1648914851981"
+        }
+    },
+    "updateDate": {
+        "$date": {
+            "$numberLong": "1648914851981"
+        }
+    },
+    "status": {
+        "$numberInt": "0"
+    },
+    "masterKey": {
+        "provider": "local"
+    }
+}

data/spec/support/crypt/schema_maps/schema_map_azure.json

@@ -0,0 +1,17 @@
+{
+  "properties": {
+    "ssn": {
+      "encrypt": {
+        "keyId": [{
+          "$binary": {
+            "base64": "AZUREAAAAAAAAAAAAAAAAA==",
+            "subType": "04"
+          }
+        }],
+        "bsonType": "string",
+        "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
+      }
+    }
+  },
+  "bsonType": "object"
+}

data/spec/support/crypt/schema_maps/schema_map_azure_key_alt_names.json

@@ -0,0 +1,12 @@
+{
+  "properties": {
+    "ssn": {
+      "encrypt": {
+        "keyId": "/altname",
+        "bsonType": "string",
+        "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random"
+      }
+    }
+  },
+  "bsonType": "object"
+}

data/spec/support/crypt/schema_maps/schema_map_gcp.json

@@ -0,0 +1,17 @@
+{
+  "properties": {
+    "ssn": {
+      "encrypt": {
+        "keyId": [{
+          "$binary": {
+            "base64": "GCPAAAAAAAAAAAAAAAAAAA==",
+            "subType": "04"
+          }
+        }],
+        "bsonType": "string",
+        "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
+      }
+    }
+  },
+  "bsonType": "object"
+}

data/spec/support/crypt/schema_maps/schema_map_gcp_key_alt_names.json

@@ -0,0 +1,12 @@
+{
+  "properties": {
+    "ssn": {
+      "encrypt": {
+        "keyId": "/altname",
+        "bsonType": "string",
+        "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random"
+      }
+    }
+  },
+  "bsonType": "object"
+}

data/spec/support/crypt/schema_maps/schema_map_kmip.json

@@ -0,0 +1,17 @@
+{
+  "properties": {
+    "ssn": {
+      "encrypt": {
+        "keyId": [{
+          "$binary": {
+            "base64": "KMIPAAAAAAAAAAAAAAAAAA==",
+            "subType": "04"
+          }
+        }],
+        "bsonType": "string",
+        "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
+      }
+    }
+  },
+  "bsonType": "object"
+}

data/spec/support/crypt/schema_maps/schema_map_kmip_key_alt_names.json

@@ -0,0 +1,12 @@
+{
+  "properties": {
+    "ssn": {
+      "encrypt": {
+        "keyId": "/altname",
+        "bsonType": "string",
+        "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random"
+      }
+    }
+  },
+  "bsonType": "object"
+}

data/spec/support/crypt.rb

@@ -48,8 +48,36 @@ module Crypt
       }
     end
 
+    # Azure KMS provider options
+    let(:azure_kms_providers) do
+      {
+        azure: {
+          tenant_id: SpecConfig.instance.fle_azure_tenant_id,
+          client_id: SpecConfig.instance.fle_azure_client_id,
+          client_secret: SpecConfig.instance.fle_azure_client_secret,
+        }
+      }
+    end
+
+    let(:gcp_kms_providers) do
+      {
+        gcp: {
+          email: SpecConfig.instance.fle_gcp_email,
+          private_key: SpecConfig.instance.fle_gcp_private_key,
+        }
+      }
+    end
+
+    let(:kmip_kms_providers) do
+      {
+        kmip: {
+          endpoint: SpecConfig.instance.fle_kmip_endpoint,
+        }
+      }
+    end
+
     # Key vault database and collection names
-    let(:key_vault_db) { 'admin' }
+    let(:key_vault_db) { 'keyvault' }
     let(:key_vault_coll) { 'datakeys' }
     let(:key_vault_namespace) { "#{key_vault_db}.#{key_vault_coll}" }
 
@@ -69,6 +97,26 @@ module Crypt
         mongocryptd_uri: "mongodb://localhost:#{SpecConfig.instance.mongocryptd_port}",
       }
     end
+
+    let(:kms_tls_options) do
+      {}
+    end
+
+    let(:default_kms_tls_options_for_provider) do
+      {
+        ssl_ca_cert: SpecConfig.instance.fle_kmip_tls_ca_file,
+        ssl_cert: SpecConfig.instance.fle_kmip_tls_certificate_key_file,
+        ssl_key: SpecConfig.instance.fle_kmip_tls_certificate_key_file,
+      }
+    end
+
+    let(:encrypted_fields) do
+      BSON::ExtJSON.parse(File.read('spec/support/crypt/encryptedFields.json'))
+    end
+
+    let(:key1_document) do
+      BSON::ExtJSON.parse(File.read('spec/support/crypt/keys/key1-document.json'))
+    end
   end
 
   # For tests that require local KMS to be configured
@@ -107,11 +155,15 @@ module Crypt
         SpecConfig.instance.fle_aws_region &&
         SpecConfig.instance.fle_aws_arn
 
-        skip(
-          'This test requires the MONGO_RUBY_DRIVER_AWS_KEY, ' +
-          'MONGO_RUBY_DRIVER_AWS_SECRET, MONGO_RUBY_DRIVER_AWS_REGION, ' +
-          'MONGO_RUBY_DRIVER_AWS_ARN environment variables to be set information from AWS.'
-        )
+        reason = "This test requires the MONGO_RUBY_DRIVER_AWS_KEY, " +
+                "MONGO_RUBY_DRIVER_AWS_SECRET, MONGO_RUBY_DRIVER_AWS_REGION, " +
+                "MONGO_RUBY_DRIVER_AWS_ARN environment variables to be set information from AWS."
+
+        if SpecConfig.instance.fle?
+          fail(reason)
+        else
+          skip(reason)
+        end
       end
     end
 
@@ -154,4 +206,153 @@ module Crypt
       BSON::ExtJSON.parse(File.read('spec/support/crypt/schema_maps/schema_map_aws_key_alt_names.json'))
     end
   end
+
+  shared_context 'with Azure kms_providers' do
+    before do
+      unless SpecConfig.instance.fle_azure_client_id &&
+        SpecConfig.instance.fle_azure_client_secret &&
+        SpecConfig.instance.fle_azure_tenant_id &&
+        SpecConfig.instance.fle_azure_identity_platform_endpoint
+
+        reason = 'This test requires the MONGO_RUBY_DRIVER_AZURE_TENANT_ID, ' +
+        'MONGO_RUBY_DRIVER_AZURE_CLIENT_ID, MONGO_RUBY_DRIVER_AZURE_CLIENT_SECRET, ' +
+        'MONGO_RUBY_DRIVER_AZURE_IDENTITY_PLATFORM_ENDPOINT environment variables to be set information from Azure.'
+
+        if SpecConfig.instance.fle?
+          fail(reason)
+        else
+          skip(reason)
+        end
+      end
+    end
+
+    let(:kms_provider_name) { 'azure' }
+    let(:kms_providers) { azure_kms_providers }
+
+    let(:data_key) do
+      BSON::ExtJSON.parse(File.read('spec/support/crypt/data_keys/key_document_azure.json'))
+    end
+
+    let(:schema_map) do
+      BSON::ExtJSON.parse(File.read('spec/support/crypt/schema_maps/schema_map_azure.json'))
+    end
+
+    let(:data_key_options) do
+      {
+        master_key: {
+          key_vault_endpoint: SpecConfig.instance.fle_azure_key_vault_endpoint,
+          key_name: SpecConfig.instance.fle_azure_key_name,
+        }
+      }
+    end
+
+    let(:encrypted_ssn) do
+      "AQGVERAAAAAAAAAAAAAAAAACFq9wVyHGWquXjaAjjBwI3MQNuyokz/+wWSi0\n8n9iu1cKzTGI9D5uVSNs64tBulnZpywtuewBQtJIphUoEr5YpSFLglOh3bp6\nmC9hfXSyFT4="
+    end
+  end
+
+  shared_context 'with Azure kms_providers and key alt names' do
+    include_context 'with Azure kms_providers'
+
+    let(:schema_map) do
+      BSON::ExtJSON.parse(File.read('spec/support/crypt/schema_maps/schema_map_azure_key_alt_names.json'))
+    end
+  end
+
+  shared_context 'with GCP kms_providers' do
+    before do
+      unless SpecConfig.instance.fle_gcp_email &&
+        SpecConfig.instance.fle_gcp_private_key &&
+        SpecConfig.instance.fle_gcp_project_id &&
+        SpecConfig.instance.fle_gcp_location &&
+        SpecConfig.instance.fle_gcp_key_ring &&
+        SpecConfig.instance.fle_gcp_key_name
+
+        reason = 'This test requires the MONGO_RUBY_DRIVER_GCP_EMAIL, ' +
+        'MONGO_RUBY_DRIVER_GCP_PRIVATE_KEY, ' +
+        'MONGO_RUBY_DRIVER_GCP_PROJECT_ID, MONGO_RUBY_DRIVER_GCP_LOCATION, ' +
+        'MONGO_RUBY_DRIVER_GCP_KEY_RING, MONGO_RUBY_DRIVER_GCP_KEY_NAME ' +
+        'environment variables to be set information from Azure.'
+
+        if SpecConfig.instance.fle?
+          fail(reason)
+        else
+          skip(reason)
+        end
+      end
+    end
+
+    let(:kms_provider_name) { 'gcp' }
+    let(:kms_providers) { gcp_kms_providers }
+
+    let(:data_key) do
+      BSON::ExtJSON.parse(File.read('spec/support/crypt/data_keys/key_document_gcp.json'))
+    end
+
+    let(:schema_map) do
+      BSON::ExtJSON.parse(File.read('spec/support/crypt/schema_maps/schema_map_gcp.json'))
+    end
+
+    let(:data_key_options) do
+      {
+        master_key: {
+          project_id: SpecConfig.instance.fle_gcp_project_id,
+          location: SpecConfig.instance.fle_gcp_location,
+          key_ring: SpecConfig.instance.fle_gcp_key_ring,
+          key_name: SpecConfig.instance.fle_gcp_key_name,
+        }
+      }
+    end
+
+    let(:encrypted_ssn) do
+      "ARgjwAAAAAAAAAAAAAAAAAACxH7FeQ7bsdbcs8uiNn5Anj2MAU7eS5hFiQsH\nYIEMN88QVamaAgiE+EIYHiRMYGxUFaaIwD17tjzZ2wyQbDd1qMO9TctkIFzn\nqQTOP6eSajU="
+    end
+  end
+
+  shared_context 'with GCP kms_providers and key alt names' do
+    include_context 'with GCP kms_providers'
+
+    let(:schema_map) do
+      BSON::ExtJSON.parse(File.read('spec/support/crypt/schema_maps/schema_map_gcp_key_alt_names.json'))
+    end
+  end
+
+  shared_context 'with KMIP kms_providers' do
+    let(:kms_provider_name) { 'kmip' }
+    let(:kms_providers) { kmip_kms_providers }
+
+    let(:kms_tls_options) do
+      {
+        kmip: default_kms_tls_options_for_provider
+      }
+    end
+
+    let(:data_key) do
+      BSON::ExtJSON.parse(File.read('spec/support/crypt/data_keys/key_document_kmip.json'))
+    end
+
+    let(:schema_map) do
+      BSON::ExtJSON.parse(File.read('spec/support/crypt/schema_maps/schema_map_kmip.json'))
+    end
+
+    let(:data_key_options) do
+      {
+        master_key: {
+          key_id: "1"
+        }
+      }
+    end
+
+    let(:encrypted_ssn) do
+      "ASjCDwAAAAAAAAAAAAAAAAAC/ga87lE2+z1ZVpLcoP51EWKVgne7f5/vb0Jq\nt3odeB0IIuoP7xxLCqSJe+ueFm86gVA1gIiip5CKe/043PD4mquxO2ARwy8s\nCX/D4tMmvDA="
+    end
+  end
+
+  shared_context 'with KMIP kms_providers and key alt names' do
+    include_context 'with KMIP kms_providers'
+
+    let(:schema_map) do
+      BSON::ExtJSON.parse(File.read('spec/support/crypt/schema_maps/schema_map_kmip_key_alt_names.json'))
+    end
+  end
 end

data/spec/support/macros.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Mongo
+  module Macros
+
+    def config_override(key, value)
+      around do |example|
+        existing = Mongo.send(key)
+
+        Mongo.send("#{key}=", value)
+
+        example.run
+
+        Mongo.send("#{key}=", existing)
+      end
+    end
+  end
+end

data/spec/support/mongos_macros.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module MongosMacros
+
+  class << self
+    attr_accessor :distinct_ran
+  end
+  self.distinct_ran = {}
+
+  # Work around for SERVER-39704 when seeing a Mongo::Error::OperationFailure
+  # SnapshotUnavailable error -- run the distinct command on each mongos.
+  def run_mongos_distincts(db_name, collection='test')
+    MongosMacros.distinct_ran[db_name] ||= ::Utils.mongos_each_direct_client do |direct_client|
+      direct_client.use(db_name)[collection].distinct('foo').to_a
+    end
+  end
+end

data/spec/support/shared/scram_conversation.rb

@@ -5,9 +5,10 @@ shared_context 'scram conversation context' do
   let(:connection) do
     double('connection').tap do |connection|
       features = double('features')
-      allow(features).to receive(:op_msg_enabled?)
+      allow(features).to receive(:op_msg_enabled?).and_return(true)
       allow(connection).to receive(:features).and_return(features)
       allow(connection).to receive(:server)
+      allow(connection).to receive(:mongos?)
     end
   end
 end

data/spec/support/shared/session.rb

@@ -631,7 +631,8 @@ shared_examples 'an operation updating cluster time' do
   end
 
   shared_examples_for 'does not update the cluster time of the cluster' do
-    it 'does not update the cluster time of the cluster', retry: 3 do
+    retry_test
+    it 'does not update the cluster time of the cluster' do
       bct = before_cluster_time
       reply_cluster_time
       expect(client.cluster.cluster_time).to eq(before_cluster_time)
@@ -644,6 +645,7 @@ shared_examples 'an operation updating cluster time' do
       min_server_fcv '3.6'
 
       context 'when the cluster is sharded or a replica set' do
+        retry_test
         require_topology :replica_set, :sharded
 
         let(:reply_cluster_time) do
@@ -651,12 +653,12 @@ shared_examples 'an operation updating cluster time' do
           subscriber.succeeded_events[-1].reply['$clusterTime']
         end
 
-        it 'updates the cluster time of the cluster', retry: 3 do
+        it 'updates the cluster time of the cluster' do
           rct = reply_cluster_time
           expect(cluster.cluster_time).to eq(rct)
         end
 
-        it 'updates the cluster time of the session', retry: 3 do
+        it 'updates the cluster time of the session' do
           rct = reply_cluster_time
           expect(session.cluster_time).to eq(rct)
         end
@@ -676,7 +678,8 @@ shared_examples 'an operation updating cluster time' do
 
         it_behaves_like 'does not update the cluster time of the cluster'
 
-        it 'does not update the cluster time of the session', retry: 3 do
+        retry_test
+        it 'does not update the cluster time of the session' do
           reply_cluster_time
           expect(session.cluster_time).to be_nil
         end
@@ -730,7 +733,8 @@ shared_examples 'an operation updating cluster time' do
             new_cluster_time.merge(Mongo::Cluster::CLUSTER_TIME => new_timestamp)
           end
 
-          it 'includes the advanced cluster time in the second command', retry: 3 do
+          retry_test
+          it 'includes the advanced cluster time in the second command' do
             expect(second_command_cluster_time).to eq(advanced_cluster_time)
           end
         end
@@ -746,7 +750,8 @@ shared_examples 'an operation updating cluster time' do
             new_cluster_time.merge(Mongo::Cluster::CLUSTER_TIME => new_timestamp)
           end
 
-          it 'does not advance the cluster time', retry: 3 do
+          retry_test
+          it 'does not advance the cluster time' do
             expect(second_command_cluster_time).to eq(reply_cluster_time)
           end
         end
@@ -759,7 +764,8 @@ shared_examples 'an operation updating cluster time' do
           subscriber.non_auth_command_started_events[-1].command['$clusterTime']
         end
 
-        it 'includes the received cluster time in the second command', retry: 3 do
+        retry_test
+        it 'includes the received cluster time in the second command' do
           reply_cluster_time
           expect(second_command_cluster_time).to eq(reply_cluster_time)
         end