mongo 2.11.6 → 2.12.0.rc0

data/spec/integration/client_side_encryption/auto_encryption_command_monitoring_spec.rb

@@ -0,0 +1,301 @@
+require 'spec_helper'
+
+describe 'Auto Encryption' do
+  require_libmongocrypt
+  require_enterprise
+  min_server_fcv '4.2'
+
+  # Diagnostics of leaked background threads only, these tests do not
+  # actually require a clean slate. https://jira.mongodb.org/browse/RUBY-2138
+  clean_slate
+
+  include_context 'define shared FLE helpers'
+  include_context 'with local kms_providers'
+
+  let(:subscriber) { EventSubscriber.new }
+  let(:db_name) { 'auto_encryption' }
+
+  let(:encryption_client) do
+    new_local_client(
+      SpecConfig.instance.addresses,
+      SpecConfig.instance.test_options.merge(
+        auto_encryption_options: {
+          kms_providers: kms_providers,
+          key_vault_namespace: key_vault_namespace,
+          schema_map: { "auto_encryption.users" => schema_map },
+        },
+        database: db_name
+      ),
+    ).tap do |client|
+      client.subscribe(Mongo::Monitoring::COMMAND, subscriber)
+    end
+  end
+
+  before(:each) do
+    authorized_client.use(key_vault_db)[key_vault_coll].drop
+    authorized_client.use(key_vault_db)[key_vault_coll].insert_one(data_key)
+
+    encryption_client[:users].drop
+    result = encryption_client[:users].insert_one(ssn: ssn, age: 23)
+  end
+
+  let(:started_event) do
+    subscriber.started_events.find do |event|
+      event.command_name == command_name && event.database_name == db_name
+    end
+  end
+
+  let(:succeeded_event) do
+    subscriber.succeeded_events.find do |event|
+      event.command_name == command_name && event.database_name == db_name
+    end
+  end
+
+  let(:key_vault_list_collections_event) do
+    subscriber.started_events.find do |event|
+      event.command_name == 'listCollections' && event.database_name == key_vault_db
+    end
+  end
+
+  shared_examples 'it has an encrypted key_vault_client' do
+    it 'registers a listCollections event on the key vault client' do
+      expect(key_vault_list_collections_event).not_to be_nil
+      expect(key_vault_list_collections_event.command['$db']).to eq(key_vault_db)
+    end
+  end
+
+  describe '#aggregate' do
+    let(:command_name) { 'aggregate' }
+
+    before do
+      encryption_client[:users].aggregate([{ '$match' => { 'ssn' => ssn } }]).first
+    end
+
+    it 'has encrypted data in command monitoring' do
+      # Command started event occurs after ssn is encrypted
+      expect(
+        started_event.command["pipeline"].first["$match"]["ssn"]["$eq"]
+      ).to be_ciphertext
+
+      # Command succeeded event occurs before ssn is decrypted
+      expect(succeeded_event.reply["cursor"]["firstBatch"].first["ssn"]).to be_ciphertext
+    end
+
+    it_behaves_like 'it has an encrypted key_vault_client'
+  end
+
+  describe '#count' do
+    let(:command_name) { 'count' }
+
+    before do
+      encryption_client[:users].count(ssn: ssn)
+    end
+
+    it 'has encrypted data in command monitoring' do
+      # Command started event occurs after ssn is encrypted
+      # Command succeeded event does not contain any data to be decrypted
+      expect(started_event.command["query"]["ssn"]["$eq"]).to be_ciphertext
+    end
+
+    it_behaves_like 'it has an encrypted key_vault_client'
+  end
+
+  describe '#distinct' do
+    let(:command_name) { 'distinct' }
+
+    before do
+      encryption_client[:users].distinct(:ssn)
+    end
+
+    it 'has encrypted data in command monitoring' do
+      # Command started event does not contain any data to be encrypted
+      # Command succeeded event occurs before ssn is decrypted
+      expect(succeeded_event.reply["values"].first).to be_ciphertext
+    end
+
+    it_behaves_like 'it has an encrypted key_vault_client'
+  end
+
+  describe '#delete_one' do
+    let(:command_name) { 'delete' }
+
+    before do
+      encryption_client[:users].delete_one(ssn: ssn)
+    end
+
+    it 'has encrypted data in command monitoring' do
+      # Command started event occurs after ssn is encrypted
+      # Command succeeded event does not contain any data to be decrypted
+      expect(started_event.command["deletes"].first["q"]["ssn"]["$eq"]).to be_ciphertext
+    end
+
+    it_behaves_like 'it has an encrypted key_vault_client'
+  end
+
+  describe '#delete_many' do
+    let(:command_name) { 'delete' }
+
+    before do
+      encryption_client[:users].delete_many(ssn: ssn)
+    end
+
+    it 'has encrypted data in command monitoring' do
+      # Command started event occurs after ssn is encrypted
+      # Command succeeded event does not contain any data to be decrypted
+      expect(started_event.command["deletes"].first["q"]["ssn"]["$eq"]).to be_ciphertext
+    end
+
+    it_behaves_like 'it has an encrypted key_vault_client'
+  end
+
+  describe '#find' do
+    let(:command_name) { 'find' }
+
+    before do
+      encryption_client[:users].find(ssn: ssn).first
+    end
+
+    it 'has encrypted data in command monitoring' do
+      # Command started event occurs after ssn is encrypted
+      expect(started_event.command["filter"]["ssn"]["$eq"]).to be_ciphertext
+
+      # Command succeeded event occurs before ssn is decrypted
+      expect(succeeded_event.reply["cursor"]["firstBatch"].first["ssn"]).to be_ciphertext
+    end
+
+    it_behaves_like 'it has an encrypted key_vault_client'
+  end
+
+  describe '#find_one_and_delete' do
+    let(:command_name) { 'findAndModify' }
+
+    before do
+      encryption_client[:users].find_one_and_delete(ssn: ssn)
+    end
+
+    it 'has encrypted data in command monitoring' do
+      # Command started event occurs after ssn is encrypted
+      expect(started_event.command["query"]["ssn"]["$eq"]).to be_ciphertext
+
+      # Command succeeded event occurs before ssn is decrypted
+      expect(succeeded_event.reply["value"]["ssn"]).to be_ciphertext
+    end
+
+    it_behaves_like 'it has an encrypted key_vault_client'
+  end
+
+  describe '#find_one_and_replace' do
+    let(:command_name) { 'findAndModify' }
+
+    before do
+      encryption_client[:users].find_one_and_replace(
+        { ssn: ssn },
+        { ssn: '555-555-5555' }
+      )
+    end
+
+    it 'has encrypted data in command monitoring' do
+      # Command started event occurs after ssn is encrypted
+      expect(started_event.command["query"]["ssn"]["$eq"]).to be_ciphertext
+      expect(started_event.command["update"]["ssn"]).to be_ciphertext
+
+      # Command succeeded event occurs before ssn is decrypted
+      expect(succeeded_event.reply["value"]["ssn"]).to be_ciphertext
+    end
+
+    it_behaves_like 'it has an encrypted key_vault_client'
+  end
+
+  describe '#find_one_and_update' do
+    let(:command_name) { 'findAndModify' }
+
+    before do
+      encryption_client[:users].find_one_and_update(
+        { ssn: ssn },
+        { ssn: '555-555-5555' }
+      )
+    end
+
+    it 'has encrypted data in command monitoring' do
+
+      # Command started event occurs after ssn is encrypted
+      expect(started_event.command["query"]["ssn"]["$eq"]).to be_ciphertext
+      expect(started_event.command["update"]["ssn"]).to be_ciphertext
+
+      # Command succeeded event occurs before ssn is decrypted
+      expect(succeeded_event.reply["value"]["ssn"]).to be_ciphertext
+    end
+
+    it_behaves_like 'it has an encrypted key_vault_client'
+  end
+
+  describe '#insert_one' do
+    let(:command_name) { 'insert' }
+
+    before do
+      encryption_client[:users].insert_one(ssn: ssn)
+    end
+
+    it 'has encrypted data in command monitoring' do
+      # Command started event occurs after ssn is encrypted
+      # Command succeeded event does not contain any data to be decrypted
+      expect(started_event.command["documents"].first["ssn"]).to be_ciphertext
+    end
+
+    it_behaves_like 'it has an encrypted key_vault_client'
+  end
+
+  describe '#replace_one' do
+    let(:command_name) { 'update' }
+
+    before do
+      encryption_client[:users].replace_one(
+        { ssn: ssn },
+        { ssn: '555-555-5555' }
+      )
+    end
+
+    it 'has encrypted data in command monitoring' do
+      # Command started event occurs after ssn is encrypted
+      # Command succeeded event does not contain any data to be decrypted
+      expect(started_event.command["updates"].first["q"]["ssn"]["$eq"]).to be_ciphertext
+      expect(started_event.command["updates"].first["u"]["ssn"]).to be_ciphertext
+    end
+
+    it_behaves_like 'it has an encrypted key_vault_client'
+  end
+
+  describe '#update_one' do
+    let(:command_name) { 'update' }
+
+    before do
+      encryption_client[:users].update_one({ ssn: ssn }, { ssn: '555-555-5555' })
+    end
+
+    it 'has encrypted data in command monitoring' do
+      # Command started event occurs after ssn is encrypted
+      # Command succeeded event does not contain any data to be decrypted
+      expect(started_event.command["updates"].first["q"]["ssn"]["$eq"]).to be_ciphertext
+      expect(started_event.command["updates"].first["u"]["ssn"]).to be_ciphertext
+    end
+
+    it_behaves_like 'it has an encrypted key_vault_client'
+  end
+
+  describe '#update_many' do
+    let(:command_name) { 'update' }
+
+    before do
+      # update_many does not support replacement-style updates
+      encryption_client[:users].update_many({ ssn: ssn }, { "$inc" => { :age => 1 } })
+    end
+
+    it 'has encrypted data in command monitoring' do
+      # Command started event occurs after ssn is encrypted
+      # Command succeeded event does not contain any data to be decrypted
+      expect(started_event.command["updates"].first["q"]["ssn"]["$eq"]).to be_ciphertext
+    end
+
+    it_behaves_like 'it has an encrypted key_vault_client'
+  end
+end

data/spec/integration/client_side_encryption/auto_encryption_mongocryptd_spawn_spec.rb

@@ -0,0 +1,71 @@
+require 'spec_helper'
+
+describe 'Auto Encryption' do
+  require_libmongocrypt
+  min_server_fcv '4.2'
+  require_enterprise
+
+  include_context 'define shared FLE helpers'
+  include_context 'with local kms_providers'
+
+  context 'with an invalid mongocryptd spawn path' do
+    let(:client) do
+      new_local_client(
+        SpecConfig.instance.addresses,
+        SpecConfig.instance.test_options.merge(
+          auto_encryption_options: {
+            kms_providers: kms_providers,
+            key_vault_namespace: key_vault_namespace,
+            schema_map: { 'auto_encryption.users' => schema_map },
+            extra_options: {
+              mongocryptd_spawn_path: 'echo hello world',
+              mongocryptd_spawn_args: []
+            }
+          },
+          database: 'auto_encryption'
+        ),
+      )
+    end
+
+    let(:server_selector) { double("ServerSelector") }
+    let(:cluster) { double("Cluster") }
+
+    before do
+      authorized_client.use(:admin)[:datakeys].drop
+      authorized_client.use(:admin)[:datakeys].insert_one(data_key)
+
+      allow(server_selector).to receive(:name)
+      allow(server_selector).to receive(:server_selection_timeout)
+      allow(server_selector).to receive(:local_threshold)
+      allow(cluster).to receive(:summary)
+
+      # Raise a server selection error on intent-to-encrypt commands to mock
+      # what would happen if mongocryptd hadn't already been spawned. It is
+      # necessary to mock this behavior because it is likely that another test
+      # will have already spawned mongocryptd, causing this test to fail.
+      allow_any_instance_of(Mongo::Database)
+        .to receive(:command)
+        .with(
+          hash_including(
+            'insert' => 'users',
+            '$db' => 'auto_encryption',
+            'ordered' => true,
+            'lsid' => kind_of(Hash),
+            'documents' => kind_of(Array),
+            'jsonSchema' => kind_of(Hash),
+            'isRemoteSchema' => false,
+          )
+        )
+        .and_raise(Mongo::Error::NoServerAvailable.new(server_selector, cluster))
+    end
+
+    it 'raises an exception when trying to perform auto encryption' do
+      expect do
+        client[:users].insert_one(ssn: ssn)
+      end.to raise_error(
+        Mongo::Error::MongocryptdSpawnError,
+        /Failed to spawn mongocryptd at the path "echo hello world" with arguments/
+      )
+    end
+  end
+end

data/spec/integration/client_side_encryption/auto_encryption_old_wire_version_spec.rb

@@ -0,0 +1,76 @@
+require 'spec_helper'
+
+describe 'Auto Encryption' do
+  require_libmongocrypt
+  max_server_version '4.0'
+
+  # Diagnostics of leaked background threads only, these tests do not
+  # actually require a clean slate. https://jira.mongodb.org/browse/RUBY-2138
+  clean_slate
+
+  include_context 'define shared FLE helpers'
+
+  let(:encryption_client) do
+    new_local_client(
+      SpecConfig.instance.addresses,
+      SpecConfig.instance.test_options.merge(
+        auto_encryption_options: {
+          kms_providers: kms_providers,
+          key_vault_namespace: key_vault_namespace,
+          # Must use local schema map because server versions older than 4.2
+          # do not support jsonSchema collection validator.
+          schema_map: { 'auto_encryption.users' => schema_map },
+          bypass_auto_encryption: bypass_auto_encryption
+        },
+        database: 'auto_encryption'
+      ),
+    )
+  end
+
+  let(:bypass_auto_encryption) { false }
+  let(:client) { authorized_client.use(:auto_encryption) }
+
+  let(:encrypted_ssn_binary) do
+    BSON::Binary.new(Base64.decode64(encrypted_ssn), :ciphertext)
+  end
+
+  shared_examples 'it decrypts but does not encrypt on wire version < 8' do
+    before do
+      client[:users].insert_one(ssn: encrypted_ssn_binary)
+
+      authorized_client.use(:admin)[:datakeys].drop
+      authorized_client.use(:admin)[:datakeys].insert_one(data_key)
+    end
+
+    it 'raises an exception when trying to encrypt' do
+      expect do
+        encryption_client[:users].find(ssn: ssn).first
+      end.to raise_error(Mongo::Error::CryptError, /Auto-encryption requires a minimum MongoDB version of 4.2/)
+    end
+
+    context 'with bypass_auto_encryption=true' do
+      let(:bypass_auto_encryption) { true }
+
+      it 'does not raise an exception but doesn\'t encrypt' do
+        document = encryption_client[:users].find(ssn: ssn).first
+        expect(document).to be_nil
+      end
+
+      it 'still decrypts' do
+        document = encryption_client[:users].find(ssn: encrypted_ssn_binary).first
+        # ssn field is still decrypted
+        expect(document['ssn']).to eq(ssn)
+      end
+    end
+  end
+
+  context 'with AWS kms provider' do
+    include_context 'with AWS kms_providers'
+    it_behaves_like 'it decrypts but does not encrypt on wire version < 8'
+  end
+
+  context 'with local kms provider' do
+    include_context 'with local kms_providers'
+    it_behaves_like 'it decrypts but does not encrypt on wire version < 8'
+  end
+end

data/spec/integration/client_side_encryption/auto_encryption_reconnect_spec.rb

@@ -0,0 +1,216 @@
+require 'spec_helper'
+
+describe 'Client with auto encryption #reconnect' do
+  require_libmongocrypt
+  min_server_fcv '4.2'
+  require_enterprise
+
+  # Diagnostics of leaked background threads only, these tests do not
+  # actually require a clean slate. https://jira.mongodb.org/browse/RUBY-2138
+  clean_slate
+
+  include_context 'define shared FLE helpers'
+
+  let(:client) do
+    new_local_client(
+      SpecConfig.instance.addresses,
+      SpecConfig.instance.test_options.merge(
+        {
+          auto_encryption_options: {
+            kms_providers: kms_providers,
+            key_vault_namespace: key_vault_namespace,
+            key_vault_client: key_vault_client_option,
+            schema_map: { 'auto_encryption.users': schema_map }
+          },
+          database: 'auto_encryption'
+        }
+      )
+    )
+  end
+
+  let(:unencrypted_client) { authorized_client.use(:auto_encryption) }
+
+  let(:mongocryptd_client) { client.encrypter.mongocryptd_client }
+  let(:key_vault_client) { client.encrypter.key_vault_client }
+  let(:data_key_id) { data_key['_id'] }
+
+  shared_examples 'a functioning client' do
+    it 'can perform an encrypted find command' do
+      doc = client[:users].find(ssn: ssn).first
+      expect(doc).not_to be_nil
+      expect(doc['ssn']).to eq(ssn)
+    end
+  end
+
+  shared_examples 'a functioning mongocryptd client' do
+    it 'can perform a schemaRequiresEncryption command' do
+      # A schemaRequiresEncryption command; mongocryptd should respond that
+      # this command requires encryption.
+      response = mongocryptd_client.database.command(
+        insert: 'users',
+        ordered: true,
+        lsid: { id: BSON::Binary.new("\x00" * 16, :uuid) },
+        documents: [{
+          ssn: '123-456-7890',
+          _id: BSON::ObjectId.new,
+        }],
+        jsonSchema: schema_map,
+        isRemoteSchema: false
+      )
+
+      expect(response).to be_ok
+      expect(response.documents.first['schemaRequiresEncryption']).to be true
+    end
+  end
+
+  shared_examples 'a functioning key vault client' do
+    it 'can perform a find command' do
+      doc = key_vault_client.use(key_vault_db)[key_vault_coll].find(_id: data_key_id).first
+      expect(doc).not_to be_nil
+      expect(doc['_id']).to eq(data_key_id)
+    end
+  end
+
+  shared_examples 'an auto-encryption client that reconnects properly' do
+    before do
+      authorized_client.use(:admin)[:datakeys].drop
+      authorized_client.use(:admin)[:datakeys].insert_one(data_key)
+
+      unencrypted_client[:users].drop
+      # Use a client without auto_encryption_options to insert an
+      # encrypted document into the collection; this ensures that the
+      # client with auto_encryption_options must perform decryption
+      # to properly read the document.
+      unencrypted_client[:users].insert_one(
+        ssn: BSON::Binary.new(Base64.decode64(encrypted_ssn), :ciphertext)
+      )
+    end
+
+    context 'after reconnecting without closing main client' do
+      before do
+        client.reconnect
+      end
+
+      it_behaves_like 'a functioning client'
+      it_behaves_like 'a functioning mongocryptd client'
+      it_behaves_like 'a functioning key vault client'
+    end
+
+    context 'after closing and reconnecting main client' do
+      before do
+        client.close
+        client.reconnect
+      end
+
+      it_behaves_like 'a functioning client'
+      it_behaves_like 'a functioning mongocryptd client'
+      it_behaves_like 'a functioning key vault client'
+    end
+
+    context 'after killing client monitor thread' do
+      before do
+        thread = client.cluster.servers.first.monitor.instance_variable_get('@thread')
+        expect(thread).to be_alive
+
+        thread.kill
+
+        sleep 0.1
+        expect(thread).not_to be_alive
+
+        client.reconnect
+      end
+
+      it_behaves_like 'a functioning client'
+      it_behaves_like 'a functioning mongocryptd client'
+      it_behaves_like 'a functioning key vault client'
+    end
+
+    context 'after closing mongocryptd client and reconnecting' do
+      before do
+        mongocryptd_client.close
+        client.reconnect
+      end
+
+      it_behaves_like 'a functioning client'
+      it_behaves_like 'a functioning mongocryptd client'
+      it_behaves_like 'a functioning key vault client'
+    end
+
+    context 'after killing mongocryptd client monitor thread and reconnecting' do
+      before do
+        thread = mongocryptd_client.cluster.servers.first.monitor.instance_variable_get('@thread')
+        expect(thread).to be_alive
+
+        thread.kill
+
+        sleep 0.1
+        expect(thread).not_to be_alive
+
+        client.reconnect
+      end
+
+      it_behaves_like 'a functioning client'
+      it_behaves_like 'a functioning mongocryptd client'
+      it_behaves_like 'a functioning key vault client'
+    end
+
+    context 'after closing key_vault_client and reconnecting' do
+      before do
+        key_vault_client.close
+        client.reconnect
+      end
+
+      it_behaves_like 'a functioning client'
+      it_behaves_like 'a functioning mongocryptd client'
+      it_behaves_like 'a functioning key vault client'
+    end
+
+    context 'after killing key_vault_client monitor thread and reconnecting' do
+      before do
+        thread = key_vault_client.cluster.servers.first.monitor.instance_variable_get('@thread')
+        expect(thread).to be_alive
+
+        thread.kill
+
+        sleep 0.1
+        expect(thread).not_to be_alive
+
+        client.reconnect
+      end
+
+      it_behaves_like 'a functioning client'
+      it_behaves_like 'a functioning mongocryptd client'
+      it_behaves_like 'a functioning key vault client'
+    end
+  end
+
+  context 'with default key vault client option' do
+    let(:key_vault_client_option) { nil }
+
+    context 'with AWS KMS providers' do
+      include_context 'with AWS kms_providers'
+      it_behaves_like 'an auto-encryption client that reconnects properly'
+    end
+
+    context 'with local KMS providers' do
+      include_context 'with local kms_providers'
+      it_behaves_like 'an auto-encryption client that reconnects properly'
+    end
+  end
+
+  context 'with custom key vault client option' do
+    let(:key_vault_client_option) do
+      Mongo::Client.new(SpecConfig.instance.addresses).use(:test)
+    end
+
+    context 'with AWS KMS providers' do
+      include_context 'with AWS kms_providers'
+      it_behaves_like 'an auto-encryption client that reconnects properly'
+    end
+
+    context 'with local KMS providers' do
+      include_context 'with local kms_providers'
+      it_behaves_like 'an auto-encryption client that reconnects properly'
+    end
+  end
+end