mongo 2.11.2 → 2.13.0.beta1

data/spec/integration/client_side_encryption/bson_size_limit_spec.rb

@@ -0,0 +1,187 @@
+require 'spec_helper'
+
+describe 'Client-Side Encryption' do
+  describe 'Prose tests: BSON size limits and batch splitting' do
+    require_libmongocrypt
+    require_enterprise
+    min_server_fcv '4.2'
+
+    include_context 'define shared FLE helpers'
+
+    let(:subscriber) { EventSubscriber.new }
+
+    let(:client) do
+      authorized_client.use('db')
+    end
+
+    let(:json_schema) do
+      BSON::ExtJSON.parse(File.read('spec/support/crypt/limits/limits-schema.json'))
+    end
+
+    let(:limits_doc) do
+      BSON::ExtJSON.parse(File.read('spec/support/crypt/limits/limits-doc.json'))
+    end
+
+    let(:client_encrypted) do
+      new_local_client(
+        SpecConfig.instance.addresses,
+        SpecConfig.instance.test_options.merge(
+          auto_encryption_options: {
+            kms_providers: {
+              local: { key: local_master_key },
+            },
+            key_vault_namespace: 'admin.datakeys',
+            # Spawn mongocryptd on non-default port for sharded cluster tests
+            extra_options: extra_options,
+          },
+          database: 'db',
+        )
+      ).tap do |client|
+        client.subscribe(Mongo::Monitoring::COMMAND, subscriber)
+      end
+    end
+
+    before do
+      client['coll'].drop
+      client['coll',
+        {
+          'validator' => { '$jsonSchema' => json_schema }
+        }
+      ].create
+
+      key_vault_collection = client.use('admin')['datakeys', write_concern: { w: :majority }]
+
+      key_vault_collection.drop
+      key_vault_collection.insert_one(
+        BSON::ExtJSON.parse(File.read('spec/support/crypt/limits/limits-key.json'))
+      )
+    end
+
+    let(:_2mib) { 2097152 }
+    let(:_16mib) { 16777216 }
+
+    context 'when a single, unencrypted document is larger than 2MiB' do
+      it 'can perform insert_one using the encrypted client' do
+        document = {
+          _id: "over_2mib_under_16mib",
+          unencrypted: 'a' * _2mib
+        }
+
+        result = client_encrypted['coll'].insert_one(document)
+
+        expect(result).to be_ok
+      end
+    end
+
+    context 'when a single encrypted document is larger than 2MiB' do
+      it 'can perform insert_one using the encrypted client' do
+        result = client_encrypted['coll'].insert_one(
+          limits_doc.merge(
+            _id: "encryption_exceeds_2mi",
+            unencrypted: 'a' * (_2mib - 2000)
+          )
+        )
+
+        expect(result).to be_ok
+      end
+    end
+
+    context 'when bulk inserting two unencrypted documents under 2MiB' do
+      it 'can perform bulk insert using the encrypted client' do
+        bulk_write = Mongo::BulkWrite.new(
+          client_encrypted['coll'],
+          [
+            { insert_one: { _id: 'over_2mib_1', unencrypted: 'a' * _2mib } },
+            { insert_one: { _id: 'over_2mib_2', unencrypted: 'a' * _2mib } },
+          ]
+        )
+
+        result = bulk_write.execute
+        expect(result.inserted_count).to eq(2)
+
+        command_succeeded_events = subscriber.succeeded_events.select do |event|
+          event.command_name == 'insert'
+        end
+
+        expect(command_succeeded_events.length).to eq(2)
+      end
+    end
+
+    context 'when bulk deletes two unencrypted documents under 2MiB' do
+      it 'can perform bulk delete using the encrypted client' do
+        # Insert documents that we can match and delete later
+        bulk_write = Mongo::BulkWrite.new(
+          client_encrypted['coll'],
+          [
+            { insert_one: { _id: 'over_2mib_1', unencrypted: 'a' * _2mib } },
+            { insert_one: { _id: 'over_2mib_2', unencrypted: 'a' * _2mib } },
+          ]
+        )
+
+        result = bulk_write.execute
+        expect(result.inserted_count).to eq(2)
+
+        command_succeeded_events = subscriber.succeeded_events.select do |event|
+          event.command_name == 'insert'
+        end
+
+        expect(command_succeeded_events.length).to eq(2)
+      end
+    end
+
+    context 'when bulk inserting two encrypted documents under 2MiB' do
+      it 'can perform bulk_insert using the encrypted client' do
+        bulk_write = Mongo::BulkWrite.new(
+          client_encrypted['coll'],
+          [
+            {
+              insert_one: limits_doc.merge(
+                _id: "encryption_exceeds_2mib_1",
+                unencrypted: 'a' * (_2mib - 2000)
+              )
+            },
+            {
+              insert_one: limits_doc.merge(
+                _id: 'encryption_exceeds_2mib_2',
+                unencrypted: 'a' * (_2mib - 2000)
+              )
+            },
+          ]
+        )
+
+        result = bulk_write.execute
+        expect(result.inserted_count).to eq(2)
+
+        command_succeeded_events = subscriber.succeeded_events.select do |event|
+          event.command_name == 'insert'
+        end
+
+        expect(command_succeeded_events.length).to eq(2)
+      end
+    end
+
+    context 'when a single document is just smaller than 16MiB' do
+      it 'can perform insert_one using the encrypted client' do
+        result = client_encrypted['coll'].insert_one(
+          _id: "under_16mib",
+          unencrypted: "a" * (_16mib - 2000)
+        )
+
+        expect(result).to be_ok
+      end
+    end
+
+    context 'when an encrypted document is greater than the 16MiB limit' do
+      it 'raises an exception when attempting to insert the document' do
+        expect do
+          client_encrypted['coll'].insert_one(
+            limits_doc.merge(
+              _id: "encryption_exceeds_16mib",
+              unencrypted: "a" * (16*1024*1024 + 500*1024),
+            )
+          )
+        end.to raise_error(Mongo::Error::MaxBSONSize, /The document exceeds maximum allowed BSON object size after serialization/)
+      end
+    end
+  end
+end

data/spec/integration/client_side_encryption/bypass_mongocryptd_spawn_spec.rb

@@ -0,0 +1,78 @@
+require 'spec_helper'
+
+describe 'Client-Side Encryption' do
+  describe 'Prose tests: Bypass mongocryptd spawn' do
+    require_libmongocrypt
+    require_enterprise
+    min_server_fcv '4.2'
+
+    include_context 'define shared FLE helpers'
+
+    # Choose a different port for mongocryptd than the one used by all the other
+    # tests to avoid failures caused by other tests spawning mongocryptd.
+    let(:mongocryptd_port) { 27091 }
+
+    context 'via mongocryptdBypassSpawn' do
+      let(:test_schema_map) do
+        BSON::ExtJSON.parse(File.read('spec/support/crypt/external/external-schema.json'))
+      end
+
+      let(:client) do
+        new_local_client(
+          SpecConfig.instance.addresses,
+          SpecConfig.instance.test_options.merge(
+            auto_encryption_options: {
+              kms_providers: local_kms_providers,
+              key_vault_namespace: 'admin.datakeys',
+              schema_map: { 'db.coll' => test_schema_map },
+              extra_options: {
+                mongocryptd_bypass_spawn: true,
+                mongocryptd_uri: "mongodb://localhost:#{mongocryptd_port}/db?serverSelectionTimeoutMS=1000",
+                mongocryptd_spawn_args: [ "--pidfilepath=bypass-spawning-mongocryptd.pid", "--port=#{mongocryptd_port}"],
+              },
+            },
+            database: 'db'
+          ),
+        )
+      end
+
+      it 'does not spawn' do
+        lambda do
+          client['coll'].insert_one(encrypted: 'test')
+        end.should raise_error(Mongo::Error::NoServerAvailable, /Server address=localhost:#{Regexp.quote(mongocryptd_port.to_s)} UNKNOWN/)
+      end
+    end
+
+    context 'via bypassAutoEncryption' do
+      let(:client) do
+        new_local_client(
+          SpecConfig.instance.addresses,
+          SpecConfig.instance.test_options.merge(
+            auto_encryption_options: {
+              kms_providers: local_kms_providers,
+              key_vault_namespace: 'admin.datakeys',
+              bypass_auto_encryption: true,
+              extra_options: {
+                mongocryptd_spawn_args: [ "--pidfilepath=bypass-spawning-mongocryptd.pid", "--port=#{mongocryptd_port}"],
+              },
+            },
+            database: 'db'
+          ),
+        )
+      end
+
+      let(:mongocryptd_client) do
+        new_local_client(["localhost:#{mongocryptd_port}"], server_selection_timeout: 1)
+      end
+
+      it 'does not spawn' do
+        lambda do
+          client['coll'].insert_one(encrypted: 'test')
+        end.should_not raise_error
+        lambda do
+          mongocryptd_client.database.command(ismaster: 1)
+        end.should raise_error(Mongo::Error::NoServerAvailable)
+      end
+    end
+  end
+end

data/spec/integration/client_side_encryption/client_close_spec.rb

@@ -0,0 +1,63 @@
+require 'spec_helper'
+
+describe 'Auto encryption client' do
+  require_libmongocrypt
+  require_enterprise
+  min_server_fcv '4.2'
+
+  context 'after client is disconnected' do
+    include_context 'define shared FLE helpers'
+    include_context 'with local kms_providers'
+
+    let(:client) do
+      new_local_client(
+        SpecConfig.instance.addresses,
+        SpecConfig.instance.test_options.merge(
+          auto_encryption_options: {
+            kms_providers: kms_providers,
+            key_vault_namespace: 'admin.datakeys',
+            schema_map: { 'auto_encryption.users' => schema_map },
+            # Spawn mongocryptd on non-default port for sharded cluster tests
+            extra_options: extra_options,
+          },
+          database: 'auto_encryption',
+        )
+      )
+    end
+
+    shared_examples 'a functioning auto-encrypter' do
+      it 'can still perform encryption' do
+        result = client['users'].insert_one(ssn: '000-000-0000')
+        expect(result).to be_ok
+
+        encrypted_document = authorized_client
+          .use('auto_encryption')['users']
+          .find(_id: result.inserted_ids.first)
+          .first
+
+        expect(encrypted_document['ssn']).to be_ciphertext
+      end
+    end
+
+    context 'after performing operation with auto encryption' do
+      before do
+        key_vault_collection.drop
+        key_vault_collection.insert_one(data_key)
+
+        client['users'].insert_one(ssn: ssn)
+        client.close
+      end
+
+      it_behaves_like 'a functioning auto-encrypter'
+    end
+
+    context 'after performing operation without auto encryption' do
+      before do
+        client['users'].insert_one(age: 23)
+        client.close
+      end
+
+      it_behaves_like 'a functioning auto-encrypter'
+    end
+  end
+end

data/spec/integration/client_side_encryption/corpus_spec.rb

@@ -0,0 +1,233 @@
+require 'spec_helper'
+
+describe 'Client-Side Encryption' do
+  describe 'Prose tests: Corpus Test' do
+    require_libmongocrypt
+    require_enterprise
+    min_server_fcv '4.2'
+
+    include_context 'define shared FLE helpers'
+
+    let(:client) { authorized_client }
+
+    let(:key_vault_client) do
+      client.with(
+        database: 'admin',
+        write_concern: { w: :majority }
+      )['datakeys']
+    end
+
+    let(:test_schema_map) { BSON::ExtJSON.parse(File.read('spec/support/crypt/corpus/corpus-schema.json')) }
+    let(:local_data_key) { BSON::ExtJSON.parse(File.read('spec/support/crypt/corpus/corpus-key-local.json')) }
+    let(:aws_data_key) { BSON::ExtJSON.parse(File.read('spec/support/crypt/corpus/corpus-key-aws.json')) }
+
+    let(:client_encrypted) do
+      new_local_client(
+        SpecConfig.instance.addresses,
+        SpecConfig.instance.test_options.merge(
+          auto_encryption_options: {
+            kms_providers: {
+              local: { key: local_master_key },
+              aws: {
+                access_key_id: SpecConfig.instance.fle_aws_key,
+                secret_access_key: SpecConfig.instance.fle_aws_secret,
+              },
+            },
+            key_vault_namespace: 'admin.datakeys',
+            schema_map: local_schema_map,
+            # Spawn mongocryptd on non-default port for sharded cluster tests
+            extra_options: extra_options,
+          },
+          database: 'db',
+        )
+      )
+    end
+
+    let(:client_encryption) do
+      Mongo::ClientEncryption.new(
+        client,
+        {
+          kms_providers: {
+            local: { key: local_master_key },
+            aws: {
+              access_key_id: SpecConfig.instance.fle_aws_key,
+              secret_access_key: SpecConfig.instance.fle_aws_secret,
+            }
+          },
+          key_vault_namespace: 'admin.datakeys',
+        },
+      )
+    end
+
+    let(:corpus) do
+      BSON::ExtJSON.parse(File.read('spec/support/crypt/corpus/corpus.json'), mode: :bson)
+    end
+
+    let(:corpus_encrypted_expected) do
+      BSON::ExtJSON.parse(File.read('spec/support/crypt/corpus/corpus_encrypted.json'))
+    end
+
+    let(:corpus_copied) do
+      # As per the instructions of the prose spec, corpus_copied is a copy of
+      # the corpus BSON::Document that encrypts all fields that are meant to
+      # be explicitly encrypted. corpus is a document containing many
+      # sub-documents, each with a value to encrypt and information about how
+      # to encrypt that value.
+      corpus_copied = BSON::Document.new
+      corpus.each do |key, doc|
+        if ['_id', 'altname_aws', 'altname_local'].include?(key)
+          corpus_copied[key] = doc
+          next
+        end
+
+        if doc['method'] == 'auto'
+          corpus_copied[key] = doc
+        elsif doc['method'] == 'explicit'
+          options = if doc['identifier'] == 'id'
+            key_id = if doc['kms'] == 'local'
+              'LOCALAAAAAAAAAAAAAAAAA=='
+            else
+              'AWSAAAAAAAAAAAAAAAAAAA=='
+            end
+
+            { key_id: BSON::Binary.new(Base64.decode64(key_id), :uuid) }
+          elsif doc['identifier'] == 'altname'
+            { key_alt_name: doc['kms'] }
+          end
+
+          algorithm = if doc['algo'] == 'rand'
+            'AEAD_AES_256_CBC_HMAC_SHA_512-Random'
+          else
+            'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'
+          end
+
+          begin
+            encrypted_value = client_encryption.encrypt(
+              doc['value'],
+              options.merge({ algorithm: algorithm })
+            )
+
+            corpus_copied[key] = doc.merge('value' => encrypted_value)
+          rescue => e
+            # If doc['allowed'] is true, it means that this field should have
+            # been encrypted without error, and thus that this error is unexpected.
+            # If doc['allowed'] is false, this error was expected and the value
+            # should be copied over without being encrypted.
+            if doc['allowed']
+              raise "Unexpected error occured in client-side encryption " +
+                "corpus tests: #{e.class}: #{e.message}"
+            end
+
+            corpus_copied[key] = doc
+          end
+        end
+      end
+
+      corpus_copied
+    end
+
+    before do
+      client.use('db')['coll'].drop
+
+      key_vault_collection = client.use('admin')['datakeys', write_concern: { w: :majority }]
+      key_vault_collection.drop
+      key_vault_collection.insert_one(local_data_key)
+      key_vault_collection.insert_one(aws_data_key)
+    end
+
+    shared_context 'with jsonSchema collection validator' do
+      let(:local_schema_map) { nil }
+
+      before do
+        client.use('db')['coll',
+          {
+            'validator' => { '$jsonSchema' => test_schema_map }
+          }
+        ].create
+      end
+    end
+
+    shared_context 'with local schema map' do
+      let(:local_schema_map) { { 'db.coll' => test_schema_map } }
+    end
+
+    shared_examples 'a functioning encrypter' do
+      it 'properly encrypts and decrypts a document' do
+        corpus_encrypted_id = client_encrypted['coll']
+          .insert_one(corpus_copied)
+          .inserted_id
+
+        corpus_decrypted = client_encrypted['coll']
+          .find(_id: corpus_encrypted_id)
+          .first
+
+        # Ensure that corpus_decrypted is the same as the original corpus
+        # document by checking that they have the same set of keys, and that
+        # they have the same values at those keys (improved diagnostics).
+        expect(corpus_decrypted.keys).to eq(corpus.keys)
+
+        corpus_decrypted.each do |key, doc|
+          expect(key => doc).to eq(key => corpus[key])
+        end
+
+        corpus_encrypted_actual = client
+          .use('db')['coll']
+          .find(_id: corpus_encrypted_id)
+          .first
+
+        # Check that the actual encrypted document matches the expected
+        # encrypted document.
+        expect(corpus_encrypted_actual.keys).to eq(corpus_encrypted_expected.keys)
+
+        corpus_encrypted_actual.each do |key, value|
+          # If it was deterministically encrypted, test the encrypted values
+          # for equality.
+          if value['algo'] == 'det'
+            expect(value['value']).to eq(corpus_encrypted_expected[key]['value'])
+          else
+            # If the document was randomly encrypted, the two encrypted values
+            # will not be equal. Ensure that they are equal when decrypted.
+            if value['allowed']
+              actual_decrypted_value = client_encryption.decrypt(value['value'])
+              expected_decrypted_value = client_encryption.decrypt(corpus_encrypted_expected[key]['value'])
+
+              expect(actual_decrypted_value).to eq(expected_decrypted_value)
+            else
+              # If 'allowed' was false, the value was never encrypted; ensure
+              # that it is equal to the original, unencrypted value.
+              expect(value['value']).to eq(corpus[key]['value'])
+            end
+          end
+        end
+      end
+    end
+
+    context 'with local KMS provider' do
+      include_context 'with local kms_providers'
+
+      context 'with collection validator' do
+        include_context 'with jsonSchema collection validator'
+        it_behaves_like 'a functioning encrypter'
+      end
+
+      context 'with schema map' do
+        include_context 'with local schema map'
+        it_behaves_like 'a functioning encrypter'
+      end
+    end
+
+    context 'with AWS KMS provider' do
+      include_context 'with AWS kms_providers'
+
+      context 'with collection validator' do
+        include_context 'with jsonSchema collection validator'
+        it_behaves_like 'a functioning encrypter'
+      end
+
+      context 'with schema map' do
+        include_context 'with local schema map'
+        it_behaves_like 'a functioning encrypter'
+      end
+    end
+  end
+end