mongo 2.4.1 → 2.14.0

data/spec/integration/client_side_encryption/auto_encryption_bulk_writes_spec.rb

@@ -0,0 +1,357 @@
+require 'spec_helper'
+
+describe 'Bulk writes with auto-encryption enabled' do
+  require_libmongocrypt
+  require_enterprise
+  min_server_fcv '4.2'
+
+  include_context 'define shared FLE helpers'
+  include_context 'with local kms_providers'
+
+  let(:subscriber) { EventSubscriber.new }
+
+  let(:client) do
+    new_local_client(
+      SpecConfig.instance.addresses,
+      SpecConfig.instance.test_options.merge(
+        auto_encryption_options: {
+          kms_providers: kms_providers,
+          key_vault_namespace: key_vault_namespace,
+          schema_map: { "auto_encryption.users" => schema_map },
+          # Spawn mongocryptd on non-default port for sharded cluster tests
+          extra_options: extra_options,
+        },
+        database: 'auto_encryption'
+      ),
+    ).tap do |client|
+      client.subscribe(Mongo::Monitoring::COMMAND, subscriber)
+    end
+  end
+
+  let(:size_limit) { Mongo::Server::ConnectionBase::REDUCED_MAX_BSON_SIZE }
+
+  before do
+    authorized_client.use('auto_encryption')['users'].drop
+
+    key_vault_collection.drop
+    key_vault_collection.insert_one(data_key)
+  end
+
+  let(:command_succeeded_events) do
+    subscriber.succeeded_events.select do |event|
+      event.command_name == command_name
+    end
+  end
+
+  shared_examples 'a functioning encrypted bulk write' do |options={}|
+    num_writes = options[:num_writes]
+
+    before do
+      perform_bulk_write
+    end
+
+    it 'executes an encrypted bulk write' do
+      documents = authorized_client.use('auto_encryption')['users'].find
+      ssns = documents.map { |doc| doc['ssn'] }
+      expect(ssns).to all(be_ciphertext)
+    end
+
+    it 'executes the correct number of writes' do
+      expect(command_succeeded_events.length).to eq(num_writes)
+    end
+  end
+
+  context 'using BulkWrite' do
+    let(:collection) { client['users'] }
+    let(:bulk_write) { Mongo::BulkWrite.new(collection, requests, {}) }
+    let(:perform_bulk_write) { bulk_write.execute }
+
+    context 'with insert operations' do
+      let(:command_name) { 'insert' }
+
+      context 'when total request size does not exceed 2MiB' do
+        let(:requests) do
+          [
+            { insert_one: { ssn: 'a' * (size_limit/2) } },
+            { insert_one: { ssn: 'a' * (size_limit/2) } }
+          ]
+        end
+
+        it_behaves_like 'a functioning encrypted bulk write', num_writes: 1
+      end
+
+      context 'when each operation is smaller than 2MiB, but the total request size is greater than 2MiB' do
+        let(:requests) do
+          [
+            { insert_one: { ssn: 'a' * (size_limit - 2000) } },
+            { insert_one: { ssn: 'a' * (size_limit - 2000) } }
+          ]
+        end
+
+        it_behaves_like 'a functioning encrypted bulk write', num_writes: 2
+      end
+
+      context 'when each operation is larger than 2MiB' do
+        let(:requests) do
+          [
+            { insert_one: { ssn: 'a' * (size_limit * 2) } },
+            { insert_one: { ssn: 'a' * (size_limit * 2) } }
+          ]
+        end
+
+        it_behaves_like 'a functioning encrypted bulk write', num_writes: 2
+      end
+
+      context 'when one operation is larger than 16MiB' do
+        let(:requests) do
+          [
+            { insert_one: { ssn: 'a' * (Mongo::Server::ConnectionBase::DEFAULT_MAX_BSON_OBJECT_SIZE + 1000) } },
+            { insert_one: { ssn: 'a' * size_limit } }
+          ]
+        end
+
+        it 'raises an exception' do
+          expect do
+            bulk_write.execute
+          end.to raise_error(Mongo::Error::MaxBSONSize, /The document exceeds maximum allowed BSON object size after serialization/)
+        end
+      end
+    end
+
+    context 'with update operations' do
+      let(:command_name) { 'update' }
+
+      before do
+        client['users'].insert_one(_id: 1)
+        client['users'].insert_one(_id: 2)
+      end
+
+      context 'when total request size does not exceed 2MiB' do
+        let(:requests) do
+          [
+            { update_one: { filter: { _id: 1 }, update: { ssn: 'a' * (size_limit/2) } } },
+            { update_one: { filter: { _id: 2 }, update: { ssn: 'a' * (size_limit/2) } } },
+          ]
+        end
+
+        it_behaves_like 'a functioning encrypted bulk write', num_writes: 1
+      end
+
+      context 'when each operation is smaller than 2MiB, but the total request size is greater than 2MiB' do
+        let(:requests) do
+          [
+            { update_one: { filter: { _id: 1 }, update: { ssn: 'a' * (size_limit - 2000) } } },
+            { update_one: { filter: { _id: 2 }, update: { ssn: 'a' * (size_limit - 2000) } } },
+          ]
+        end
+
+        it_behaves_like 'a functioning encrypted bulk write', num_writes: 2
+      end
+
+      context 'when each operation is larger than 2MiB' do
+        let(:requests) do
+          [
+            { update_one: { filter: { _id: 1 }, update: { ssn: 'a' * (size_limit * 2) } } },
+            { update_one: { filter: { _id: 2 }, update: { ssn: 'a' * (size_limit * 2) } } },
+          ]
+        end
+
+        it_behaves_like 'a functioning encrypted bulk write', num_writes: 2
+      end
+
+      context 'when one operation is larger than 16MiB' do
+        let(:requests) do
+          [
+            { update_one: { filter: { _id: 1 }, update: { ssn: 'a' * (Mongo::Server::ConnectionBase::DEFAULT_MAX_BSON_OBJECT_SIZE) } } },
+            { update_one: { filter: { _id: 2 }, update: { ssn: 'a' * size_limit } } },
+          ]
+        end
+
+        before do
+          expect(requests.first.to_bson.length).to be > Mongo::Server::ConnectionBase::DEFAULT_MAX_BSON_OBJECT_SIZE
+        end
+
+        it 'raises an exception' do
+          expect do
+            bulk_write.execute
+          end.to raise_error(Mongo::Error::MaxBSONSize, /The document exceeds maximum allowed BSON object size after serialization/)
+        end
+      end
+    end
+
+    context 'with delete operations' do
+      let(:command_name) { 'delete' }
+
+      context 'when total request size does not exceed 2MiB' do
+        before do
+          client['users'].insert_one(ssn: 'a' * (size_limit/2))
+          client['users'].insert_one(ssn: 'b' * (size_limit/2))
+        end
+
+        let(:requests) do
+          [
+            { delete_one: { filter: { ssn: 'a' * (size_limit/2) } } },
+            { delete_one: { filter: { ssn: 'b' * (size_limit/2) } } }
+          ]
+        end
+
+        it 'performs one delete' do
+          bulk_write.execute
+
+          documents = authorized_client.use('auto_encryption')['users'].find.to_a
+          expect(documents.length).to eq(0)
+          expect(command_succeeded_events.length).to eq(1)
+        end
+      end
+
+      context 'when each operation is smaller than 2MiB, but the total request size is greater than 2MiB' do
+        before do
+          client['users'].insert_one(ssn: 'a' * (size_limit - 2000))
+          client['users'].insert_one(ssn: 'b' * (size_limit - 2000))
+        end
+
+        let(:requests) do
+          [
+            { delete_one: { filter: { ssn: 'a' * (size_limit - 2000) } } },
+            { delete_one: { filter: { ssn: 'b' * (size_limit - 2000) } } }
+          ]
+        end
+
+        it 'performs two deletes' do
+          bulk_write.execute
+
+          documents = authorized_client.use('auto_encryption')['users'].find.to_a
+          expect(documents.length).to eq(0)
+          expect(command_succeeded_events.length).to eq(2)
+        end
+      end
+
+      context 'when each operation is larger than 2MiB' do
+        before do
+          client['users'].insert_one(ssn: 'a' * (size_limit * 2))
+          client['users'].insert_one(ssn: 'b' * (size_limit * 2))
+        end
+
+        let(:requests) do
+          [
+            { delete_one: { filter: { ssn: 'a' * (size_limit * 2) } } },
+            { delete_one: { filter: { ssn: 'b' * (size_limit * 2) } } }
+          ]
+        end
+
+        it 'performs two deletes' do
+          bulk_write.execute
+
+          documents = authorized_client.use('auto_encryption')['users'].find.to_a
+          expect(documents.length).to eq(0)
+          expect(command_succeeded_events.length).to eq(2)
+        end
+      end
+
+      context 'when one operation is larger than 16MiB' do
+        let(:requests) do
+          [
+            { delete_one: { filter: { ssn: 'a' * (Mongo::Server::ConnectionBase::DEFAULT_MAX_BSON_OBJECT_SIZE + 1000) } } },
+            { delete_one: { filter: { ssn: 'b' * (size_limit * 2) } } }
+          ]
+        end
+
+        it 'raises an exception' do
+          expect do
+            bulk_write.execute
+          end.to raise_error(Mongo::Error::MaxBSONSize, /The document exceeds maximum allowed BSON object size after serialization/)
+        end
+      end
+    end
+
+    context 'with insert, update, and delete operations' do
+      context 'when total request size does not exceed 2MiB' do
+        let(:requests) do
+          [
+            { insert_one: { _id: 1, ssn: 'a' * (size_limit/3) } },
+            { update_one: { filter: { _id: 1 }, update: { ssn: 'b' * (size_limit/3) } } },
+            { delete_one: { filter: { ssn: 'b' * (size_limit/3) } } }
+          ]
+        end
+
+        it 'successfully performs the bulk write' do
+          bulk_write.execute
+
+          documents = authorized_client.use('auto_encryption')['users'].find.to_a
+          expect(documents.length).to eq(0)
+        end
+
+        # Bulk writes with different types of operations should
+        it 'performs 1 insert, 1 update, and 1 delete' do
+          bulk_write.execute
+
+          command_succeeded_events = subscriber.succeeded_events
+
+          inserts = command_succeeded_events.select { |event| event.command_name == 'insert' }
+          updates = command_succeeded_events.select { |event| event.command_name == 'update' }
+          deletes = command_succeeded_events.select { |event| event.command_name == 'delete' }
+
+          expect(inserts.length).to eq(1)
+          expect(updates.length).to eq(1)
+          expect(deletes.length).to eq(1)
+        end
+      end
+    end
+  end
+
+  context '#insert_many' do
+    let(:perform_bulk_write) do
+      client['users'].insert_many(documents)
+    end
+
+    let(:command_name) { 'insert' }
+
+    context 'when total request size does not exceed 2MiB' do
+      let(:documents) do
+        [
+          { ssn: 'a' * (size_limit/2) },
+          { ssn: 'a' * (size_limit/2) },
+        ]
+      end
+
+      it_behaves_like 'a functioning encrypted bulk write', num_writes: 1
+    end
+
+    context 'when each operation is smaller than 2MiB, but the total request size is greater than 2MiB' do
+      let(:documents) do
+        [
+          { ssn: 'a' * (size_limit - 2000) },
+          { ssn: 'a' * (size_limit - 2000) },
+        ]
+      end
+
+      it_behaves_like 'a functioning encrypted bulk write', num_writes: 2
+    end
+
+    context 'when each operation is larger than 2MiB' do
+      let(:documents) do
+        [
+          { ssn: 'a' * (size_limit * 2) },
+          { ssn: 'a' * (size_limit * 2) },
+        ]
+      end
+
+      it_behaves_like 'a functioning encrypted bulk write', num_writes: 2
+    end
+
+    context 'when one operation is larger than 16MiB' do
+      let(:documents) do
+        [
+          { ssn: 'a' * (Mongo::Server::ConnectionBase::DEFAULT_MAX_BSON_OBJECT_SIZE + 1000) },
+          { ssn: 'a' * size_limit },
+        ]
+      end
+
+      it 'raises an exception' do
+        expect do
+          perform_bulk_write
+        end.to raise_error(Mongo::Error::MaxBSONSize, /The document exceeds maximum allowed BSON object size after serialization/)
+      end
+    end
+  end
+end

data/spec/integration/client_side_encryption/auto_encryption_command_monitoring_spec.rb

@@ -0,0 +1,303 @@
+require 'spec_helper'
+
+describe 'Auto Encryption' do
+  require_libmongocrypt
+  require_enterprise
+  min_server_fcv '4.2'
+
+  # Diagnostics of leaked background threads only, these tests do not
+  # actually require a clean slate. https://jira.mongodb.org/browse/RUBY-2138
+  clean_slate
+
+  include_context 'define shared FLE helpers'
+  include_context 'with local kms_providers'
+
+  let(:subscriber) { EventSubscriber.new }
+  let(:db_name) { 'auto_encryption' }
+
+  let(:encryption_client) do
+    new_local_client(
+      SpecConfig.instance.addresses,
+      SpecConfig.instance.test_options.merge(
+        auto_encryption_options: {
+          kms_providers: kms_providers,
+          key_vault_namespace: key_vault_namespace,
+          schema_map: { "auto_encryption.users" => schema_map },
+          # Spawn mongocryptd on non-default port for sharded cluster tests
+          extra_options: extra_options,
+        },
+        database: db_name
+      ),
+    ).tap do |client|
+      client.subscribe(Mongo::Monitoring::COMMAND, subscriber)
+    end
+  end
+
+  before(:each) do
+    key_vault_collection.drop
+    key_vault_collection.insert_one(data_key)
+
+    encryption_client['users'].drop
+    result = encryption_client['users'].insert_one(ssn: ssn, age: 23)
+  end
+
+  let(:started_event) do
+    subscriber.started_events.find do |event|
+      event.command_name == command_name && event.database_name == db_name
+    end
+  end
+
+  let(:succeeded_event) do
+    subscriber.succeeded_events.find do |event|
+      event.command_name == command_name && event.database_name == db_name
+    end
+  end
+
+  let(:key_vault_list_collections_event) do
+    subscriber.started_events.find do |event|
+      event.command_name == 'listCollections' && event.database_name == key_vault_db
+    end
+  end
+
+  shared_examples 'it has an encrypted key_vault_client' do
+    it 'registers a listCollections event on the key vault client' do
+      expect(key_vault_list_collections_event).not_to be_nil
+      expect(key_vault_list_collections_event.command['$db']).to eq(key_vault_db)
+    end
+  end
+
+  describe '#aggregate' do
+    let(:command_name) { 'aggregate' }
+
+    before do
+      encryption_client['users'].aggregate([{ '$match' => { 'ssn' => ssn } }]).first
+    end
+
+    it 'has encrypted data in command monitoring' do
+      # Command started event occurs after ssn is encrypted
+      expect(
+        started_event.command["pipeline"].first["$match"]["ssn"]["$eq"]
+      ).to be_ciphertext
+
+      # Command succeeded event occurs before ssn is decrypted
+      expect(succeeded_event.reply["cursor"]["firstBatch"].first["ssn"]).to be_ciphertext
+    end
+
+    it_behaves_like 'it has an encrypted key_vault_client'
+  end
+
+  describe '#count' do
+    let(:command_name) { 'count' }
+
+    before do
+      encryption_client['users'].count(ssn: ssn)
+    end
+
+    it 'has encrypted data in command monitoring' do
+      # Command started event occurs after ssn is encrypted
+      # Command succeeded event does not contain any data to be decrypted
+      expect(started_event.command["query"]["ssn"]["$eq"]).to be_ciphertext
+    end
+
+    it_behaves_like 'it has an encrypted key_vault_client'
+  end
+
+  describe '#distinct' do
+    let(:command_name) { 'distinct' }
+
+    before do
+      encryption_client['users'].distinct(:ssn)
+    end
+
+    it 'has encrypted data in command monitoring' do
+      # Command started event does not contain any data to be encrypted
+      # Command succeeded event occurs before ssn is decrypted
+      expect(succeeded_event.reply["values"].first).to be_ciphertext
+    end
+
+    it_behaves_like 'it has an encrypted key_vault_client'
+  end
+
+  describe '#delete_one' do
+    let(:command_name) { 'delete' }
+
+    before do
+      encryption_client['users'].delete_one(ssn: ssn)
+    end
+
+    it 'has encrypted data in command monitoring' do
+      # Command started event occurs after ssn is encrypted
+      # Command succeeded event does not contain any data to be decrypted
+      expect(started_event.command["deletes"].first["q"]["ssn"]["$eq"]).to be_ciphertext
+    end
+
+    it_behaves_like 'it has an encrypted key_vault_client'
+  end
+
+  describe '#delete_many' do
+    let(:command_name) { 'delete' }
+
+    before do
+      encryption_client['users'].delete_many(ssn: ssn)
+    end
+
+    it 'has encrypted data in command monitoring' do
+      # Command started event occurs after ssn is encrypted
+      # Command succeeded event does not contain any data to be decrypted
+      expect(started_event.command["deletes"].first["q"]["ssn"]["$eq"]).to be_ciphertext
+    end
+
+    it_behaves_like 'it has an encrypted key_vault_client'
+  end
+
+  describe '#find' do
+    let(:command_name) { 'find' }
+
+    before do
+      encryption_client['users'].find(ssn: ssn).first
+    end
+
+    it 'has encrypted data in command monitoring' do
+      # Command started event occurs after ssn is encrypted
+      expect(started_event.command["filter"]["ssn"]["$eq"]).to be_ciphertext
+
+      # Command succeeded event occurs before ssn is decrypted
+      expect(succeeded_event.reply["cursor"]["firstBatch"].first["ssn"]).to be_ciphertext
+    end
+
+    it_behaves_like 'it has an encrypted key_vault_client'
+  end
+
+  describe '#find_one_and_delete' do
+    let(:command_name) { 'findAndModify' }
+
+    before do
+      encryption_client['users'].find_one_and_delete(ssn: ssn)
+    end
+
+    it 'has encrypted data in command monitoring' do
+      # Command started event occurs after ssn is encrypted
+      expect(started_event.command["query"]["ssn"]["$eq"]).to be_ciphertext
+
+      # Command succeeded event occurs before ssn is decrypted
+      expect(succeeded_event.reply["value"]["ssn"]).to be_ciphertext
+    end
+
+    it_behaves_like 'it has an encrypted key_vault_client'
+  end
+
+  describe '#find_one_and_replace' do
+    let(:command_name) { 'findAndModify' }
+
+    before do
+      encryption_client['users'].find_one_and_replace(
+        { ssn: ssn },
+        { ssn: '555-555-5555' }
+      )
+    end
+
+    it 'has encrypted data in command monitoring' do
+      # Command started event occurs after ssn is encrypted
+      expect(started_event.command["query"]["ssn"]["$eq"]).to be_ciphertext
+      expect(started_event.command["update"]["ssn"]).to be_ciphertext
+
+      # Command succeeded event occurs before ssn is decrypted
+      expect(succeeded_event.reply["value"]["ssn"]).to be_ciphertext
+    end
+
+    it_behaves_like 'it has an encrypted key_vault_client'
+  end
+
+  describe '#find_one_and_update' do
+    let(:command_name) { 'findAndModify' }
+
+    before do
+      encryption_client['users'].find_one_and_update(
+        { ssn: ssn },
+        { ssn: '555-555-5555' }
+      )
+    end
+
+    it 'has encrypted data in command monitoring' do
+
+      # Command started event occurs after ssn is encrypted
+      expect(started_event.command["query"]["ssn"]["$eq"]).to be_ciphertext
+      expect(started_event.command["update"]["ssn"]).to be_ciphertext
+
+      # Command succeeded event occurs before ssn is decrypted
+      expect(succeeded_event.reply["value"]["ssn"]).to be_ciphertext
+    end
+
+    it_behaves_like 'it has an encrypted key_vault_client'
+  end
+
+  describe '#insert_one' do
+    let(:command_name) { 'insert' }
+
+    before do
+      encryption_client['users'].insert_one(ssn: ssn)
+    end
+
+    it 'has encrypted data in command monitoring' do
+      # Command started event occurs after ssn is encrypted
+      # Command succeeded event does not contain any data to be decrypted
+      expect(started_event.command["documents"].first["ssn"]).to be_ciphertext
+    end
+
+    it_behaves_like 'it has an encrypted key_vault_client'
+  end
+
+  describe '#replace_one' do
+    let(:command_name) { 'update' }
+
+    before do
+      encryption_client['users'].replace_one(
+        { ssn: ssn },
+        { ssn: '555-555-5555' }
+      )
+    end
+
+    it 'has encrypted data in command monitoring' do
+      # Command started event occurs after ssn is encrypted
+      # Command succeeded event does not contain any data to be decrypted
+      expect(started_event.command["updates"].first["q"]["ssn"]["$eq"]).to be_ciphertext
+      expect(started_event.command["updates"].first["u"]["ssn"]).to be_ciphertext
+    end
+
+    it_behaves_like 'it has an encrypted key_vault_client'
+  end
+
+  describe '#update_one' do
+    let(:command_name) { 'update' }
+
+    before do
+      encryption_client['users'].update_one({ ssn: ssn }, { ssn: '555-555-5555' })
+    end
+
+    it 'has encrypted data in command monitoring' do
+      # Command started event occurs after ssn is encrypted
+      # Command succeeded event does not contain any data to be decrypted
+      expect(started_event.command["updates"].first["q"]["ssn"]["$eq"]).to be_ciphertext
+      expect(started_event.command["updates"].first["u"]["ssn"]).to be_ciphertext
+    end
+
+    it_behaves_like 'it has an encrypted key_vault_client'
+  end
+
+  describe '#update_many' do
+    let(:command_name) { 'update' }
+
+    before do
+      # update_many does not support replacement-style updates
+      encryption_client['users'].update_many({ ssn: ssn }, { "$inc" => { :age => 1 } })
+    end
+
+    it 'has encrypted data in command monitoring' do
+      # Command started event occurs after ssn is encrypted
+      # Command succeeded event does not contain any data to be decrypted
+      expect(started_event.command["updates"].first["q"]["ssn"]["$eq"]).to be_ciphertext
+    end
+
+    it_behaves_like 'it has an encrypted key_vault_client'
+  end
+end