mongo 2.11.2 → 2.13.0.beta1

data/spec/support/cluster_config.rb

@@ -19,6 +19,11 @@ class ClusterConfig
     @server_version
   end
 
+  def enterprise?
+    determine_cluster_config
+    @enterprise
+  end
+
   def short_server_version
     server_version.split('.')[0..1].join('.')
   end
@@ -44,6 +49,7 @@ class ClusterConfig
     end
   end
 
+  # @return [ Mongo::Address ] The address of the primary in the deployment.
   def primary_address
     determine_cluster_config
     @primary_address
@@ -178,7 +184,10 @@ class ClusterConfig
 
     @single_server = client.cluster.servers_list.length == 1
 
-    @server_version = client.database.command(buildInfo: 1).first['version']
+    build_info = client.database.command(buildInfo: 1).first
+
+    @server_version = build_info['version']
+    @enterprise = build_info['modules'] && build_info['modules'].include?('enterprise')
 
     if @topology != :sharded && short_server_version >= '3.4'
       rv = client.use(:admin).command(getParameter: 1, featureCompatibilityVersion: 1).first['featureCompatibilityVersion']

data/spec/support/cluster_tools.rb

@@ -58,7 +58,7 @@ class ClusterTools
       raise ArgumentError, 'Value must be true or false'
     end
 
-    direct_client_for_each_server do |client|
+    direct_client_for_each_data_bearing_server do |client|
       client.use(:admin).database.command(setParameter: 1, enableElectionHandoff: value)
     end
   end
@@ -147,7 +147,7 @@ class ClusterTools
     existing_primary_address = existing_primary.address
 
     target = admin_client.cluster.servers_list.detect do |server|
-      server.address != existing_primary_address
+      !server.arbiter? && server.address != existing_primary_address
     end
 
     cfg = get_rs_config
@@ -279,6 +279,7 @@ class ClusterTools
 
   def unfreeze_all
     admin_client.cluster.servers_list.each do |server|
+      next if server.arbiter?
       client = direct_client(server.address)
       # Primary refuses to be unfrozen with this message:
       # cannot freeze node when primary or running for election. state: Primary (95)
@@ -303,7 +304,12 @@ class ClusterTools
   def set_rs_config(config)
     config = config.dup
     config['version'] += 1
-    result = admin_client.database.command(replSetReconfig: config)
+    cmd = {replSetReconfig: config}
+    if ClusterConfig.instance.fcv_ish >= '4.4'
+      # Workaround for https://jira.mongodb.org/browse/SERVER-46894
+      cmd[:force] = true
+    end
+    result = admin_client.database.command(cmd)
     doc = result.reply.documents.first
     if doc['ok'] != 1
       raise 'Failed to reconfigure RS'
@@ -353,8 +359,9 @@ class ClusterTools
     admin_client.cluster.servers_list.each(&block)
   end
 
-  def direct_client_for_each_server(&block)
+  def direct_client_for_each_data_bearing_server(&block)
     each_server do |server|
+      next if server.arbiter?
       yield direct_client(server.address)
     end
   end

data/spec/support/common_shortcuts.rb

@@ -30,6 +30,18 @@ module CommonShortcuts
       end
     end
 
+    # If only the lite spec helper was loaded, this method does nothing.
+    # If the full spec helper was loaded, this method performs the same function
+    # as clean_state_for_all.
+    def clean_slate_for_all_if_possible
+      before(:all) do
+        if defined?(ClusterTools)
+          ClientRegistry.instance.close_all_clients
+          BackgroundThreadRegistry.instance.verify_empty!
+        end
+      end
+    end
+
     # For some reason, there are tests which fail on evergreen either
     # intermittently or reliably that always succeed locally.
     # Debugging of tests in evergreen is difficult/impossible,
@@ -41,6 +53,49 @@ module CommonShortcuts
         end
       end
     end
+
+    # Applies environment variable overrides in +env+ to the global environment
+    # (+ENV+) for the duration of each test.
+    #
+    # If a key's value in +env+ is nil, this key is removed from +ENV+.
+    #
+    # When the test finishes, the values in original +ENV+ that were overridden
+    # by +env+ are restored. If a key was not in original +ENV+ and was
+    # overridden by +env+, this key is removed from +ENV+ after the test.
+    #
+    # If the environment variables are not known at test definition time
+    # but are determined at test execution time, pass a block instead of
+    # the +env+ parameter and return the desired environment variables as
+    # a Hash from the block.
+    def local_env(env = nil, &block)
+      around do |example|
+        env ||= block.call
+
+        # This duplicates ENV.
+        # Note that ENV.dup produces an Object which does not behave like
+        # the original ENV, and hence is not usable.
+        saved_env = ENV.to_h
+        env.each do |k, v|
+          if v.nil?
+            ENV.delete(k)
+          else
+            ENV[k] = v
+          end
+        end
+
+        begin
+          example.run
+        ensure
+          env.each do |k, v|
+            if saved_env.key?(k)
+              ENV[k] = saved_env[k]
+            else
+              ENV.delete(k)
+            end
+          end
+        end
+      end
+    end
   end
 
   module InstanceMethods

data/spec/support/constraints.rb

@@ -72,6 +72,19 @@ module Constraints
     require_topology :replica_set
   end
 
+  # Fail command fail point was added to mongod in 4.0 and to mongos in 4.2.
+  def require_fail_command
+    min_server_fcv '4.0'
+
+    before(:all) do
+      if ClusterConfig.instance.topology == :sharded
+        unless ClusterConfig.instance.short_server_version >= '4.2'
+          skip 'Test requires failCommand fail point which was added to mongos in 4.2'
+        end
+      end
+    end
+  end
+
   def require_tls
     before(:all) do
       unless SpecConfig.instance.ssl?
@@ -132,26 +145,41 @@ module Constraints
     end
   end
 
-  def require_auth
+  def require_auth(*values)
     before(:all) do
-      unless ENV['AUTH'] == 'auth' || SpecConfig.instance.user || ClusterConfig.instance.auth_enabled?
-        skip "Auth required"
+      if values.any?
+        unless values.include?(ENV['AUTH'])
+          msg = values.map { |v| "AUTH=#{v}" }.join(' or ')
+          skip "This test requires #{msg}"
+        end
+      else
+        unless ENV['AUTH'] == 'auth' || SpecConfig.instance.user || ClusterConfig.instance.auth_enabled?
+          skip "Auth required"
+        end
       end
     end
   end
 
   def require_no_auth
     before(:all) do
-      if ENV['AUTH'] == 'auth' || SpecConfig.instance.user || ClusterConfig.instance.auth_enabled?
+      if (ENV['AUTH'] && ENV['AUTH'] != 'noauth') || SpecConfig.instance.user || ClusterConfig.instance.auth_enabled?
         skip "Auth not allowed"
       end
     end
   end
 
-  def require_no_x509_auth
+  def require_x509_auth
     before(:all) do
-      if SpecConfig.instance.x509_auth?
-        skip "X.509 auth not allowed"
+      unless SpecConfig.instance.x509_auth?
+        skip "X.509 auth required"
+      end
+    end
+  end
+
+  def require_no_external_user
+    before(:all) do
+      if SpecConfig.instance.external_user?
+        skip "External user configurations are not compatible with this test"
       end
     end
   end
@@ -208,6 +236,14 @@ module Constraints
     end
   end
 
+  def require_enterprise
+    before(:all) do
+      unless ClusterConfig.instance.enterprise?
+        skip 'Test requires enterprise build of MongoDB'
+      end
+    end
+  end
+
   # Integration tests for SRV polling require internet connectivity to
   # look up SRV records and a sharded cluster configured on default port on
   # localhost (localhost:27017, localhost:27018).

data/spec/support/crypt.rb

@@ -0,0 +1,154 @@
+# Copyright (C) 2009-2020 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Crypt
+  # For all FLE-related tests
+  shared_context 'define shared FLE helpers' do
+    # 96-byte binary string, base64-encoded local master key
+    let(:local_master_key_b64) do
+      "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3" +
+        "YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk"
+    end
+
+    let(:local_master_key) { Base64.decode64(local_master_key_b64) }
+
+    # Data key id as a binary string
+    let(:key_id) { data_key['_id'] }
+
+    # Data key alternate name
+    let(:key_alt_name) { 'ssn_encryption_key' }
+
+    # Deterministic encryption algorithm
+    let(:algorithm) { 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic' }
+
+    # Local KMS provider options
+    let(:local_kms_providers) { { local: { key: local_master_key } } }
+
+    # AWS KMS provider options
+    let(:aws_kms_providers) do
+      {
+        aws: {
+          access_key_id: SpecConfig.instance.fle_aws_key,
+          secret_access_key: SpecConfig.instance.fle_aws_secret,
+        }
+      }
+    end
+
+    # Key vault database and collection names
+    let(:key_vault_db) { 'admin' }
+    let(:key_vault_coll) { 'datakeys' }
+    let(:key_vault_namespace) { "#{key_vault_db}.#{key_vault_coll}" }
+
+    # Example value to encrypt
+    let(:ssn) { '123-456-7890' }
+
+    let(:key_vault_collection) do
+      authorized_client.with(
+        database: key_vault_db,
+        write_concern: { w: :majority }
+      )[key_vault_coll]
+    end
+
+    let(:extra_options) do
+      {
+        mongocryptd_spawn_args: ["--port=#{SpecConfig.instance.mongocryptd_port}"],
+        mongocryptd_uri: "mongodb://localhost:#{SpecConfig.instance.mongocryptd_port}",
+      }
+    end
+  end
+
+  # For tests that require local KMS to be configured
+  shared_context 'with local kms_providers' do
+    let(:kms_provider_name) { 'local' }
+    let(:kms_providers) { local_kms_providers }
+
+    let(:data_key) do
+      BSON::ExtJSON.parse(File.read('spec/support/crypt/data_keys/key_document_local.json'))
+    end
+
+    let(:schema_map) do
+      BSON::ExtJSON.parse(File.read('spec/support/crypt/schema_maps/schema_map_local.json'))
+    end
+    let(:data_key_options) { {} }
+
+    let(:encrypted_ssn) do
+      "ASzggCwAAAAAAAAAAAAAAAAC/OvUvE0N5eZ5vhjcILtGKZlxovGhYJduEfsR\n7NiH68Ft" +
+      "tXzHYqT0DKgvn3QjjTbS/4SPfBEYrMIS10Uzf9R1Ky4D5a19mYCp\nmv76Z8Rzdmo=\n"
+    end
+  end
+
+  shared_context 'with local kms_providers and key alt names' do
+    include_context 'with local kms_providers'
+
+    let(:schema_map) do
+      BSON::ExtJSON.parse(File.read('spec/support/crypt/schema_maps/schema_map_local_key_alt_names.json'))
+    end
+  end
+
+  # For tests that require AWS KMS to be configured
+  shared_context 'with AWS kms_providers' do
+    before do
+      unless SpecConfig.instance.fle_aws_key &&
+        SpecConfig.instance.fle_aws_secret &&
+        SpecConfig.instance.fle_aws_region &&
+        SpecConfig.instance.fle_aws_arn
+
+        skip(
+          'This test requires the MONGO_RUBY_DRIVER_AWS_KEY, ' +
+          'MONGO_RUBY_DRIVER_AWS_SECRET, MONGO_RUBY_DRIVER_AWS_REGION, ' +
+          'MONGO_RUBY_DRIVER_AWS_ARN environment variables to be set information from AWS.'
+        )
+      end
+    end
+
+    let(:kms_provider_name) { 'aws' }
+    let(:kms_providers) { aws_kms_providers }
+
+    let(:data_key) do
+      BSON::ExtJSON.parse(File.read('spec/support/crypt/data_keys/key_document_aws.json'))
+    end
+
+    let(:schema_map) do
+      BSON::ExtJSON.parse(File.read('spec/support/crypt/schema_maps/schema_map_aws.json'))
+    end
+
+    let(:data_key_options) do
+      {
+        master_key: {
+          region: aws_region,
+          key: aws_arn,
+          endpoint: "#{aws_endpoint_host}:#{aws_endpoint_port}"
+        }
+      }
+    end
+
+    let(:aws_region) { SpecConfig.instance.fle_aws_region }
+    let(:aws_arn) { SpecConfig.instance.fle_aws_arn }
+    let(:aws_endpoint_host) { "kms.#{aws_region}.amazonaws.com" }
+    let(:aws_endpoint_port) { 443 }
+
+    let(:encrypted_ssn) do
+      "AQFkgAAAAAAAAAAAAAAAAAACX/YG2ZOHWU54kARE17zDdeZzKgpZffOXNaoB\njmvdVa/" +
+      "yTifOikvxEov16KxtQrnaKWdxQL03TVgpoLt4Jb28pqYKlgBj3XMp\nuItZpQeFQB4=\n"
+    end
+  end
+
+  shared_context 'with AWS kms_providers and key alt names' do
+    include_context 'with AWS kms_providers'
+
+    let(:schema_map) do
+      BSON::ExtJSON.parse(File.read('spec/support/crypt/schema_maps/schema_map_aws_key_alt_names.json'))
+    end
+  end
+end

data/spec/support/crypt/corpus/corpus-key-aws.json

@@ -0,0 +1,33 @@
+{
+  "status": {
+      "$numberInt": "1"
+  },
+  "_id": {
+      "$binary": {
+          "base64": "AWSAAAAAAAAAAAAAAAAAAA==",
+          "subType": "04"
+      }
+  },
+  "masterKey": {
+      "region": "us-east-1",
+      "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0",
+      "provider": "aws"
+  },
+  "updateDate": {
+      "$date": {
+          "$numberLong": "1557827033449"
+      }
+  },
+  "keyMaterial": {
+      "$binary": {
+          "base64": "AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO",
+          "subType": "00"
+      }
+  },
+  "creationDate": {
+      "$date": {
+          "$numberLong": "1557827033449"
+      }
+  },
+  "keyAltNames": ["aws"]
+}

data/spec/support/crypt/corpus/corpus-key-local.json

@@ -0,0 +1,31 @@
+{
+  "status": {
+      "$numberInt": "1"
+  },
+  "_id": {
+      "$binary": {
+          "base64": "LOCALAAAAAAAAAAAAAAAAA==",
+          "subType": "04"
+      }
+  },
+  "masterKey": {
+      "provider": "local"
+  },
+  "updateDate": {
+      "$date": {
+          "$numberLong": "1557827033449"
+      }
+  },
+  "keyMaterial": {
+      "$binary": {
+          "base64": "Ce9HSz/HKKGkIt4uyy+jDuKGA+rLC2cycykMo6vc8jXxqa1UVDYHWq1r+vZKbnnSRBfB981akzRKZCFpC05CTyFqDhXv6OnMjpG97OZEREGIsHEYiJkBW0jJJvfLLgeLsEpBzsro9FztGGXASxyxFRZFhXvHxyiLOKrdWfs7X1O/iK3pEoHMx6uSNSfUOgbebLfIqW7TO++iQS5g1xovXA==",
+          "subType": "00"
+      }
+  },
+  "creationDate": {
+      "$date": {
+          "$numberLong": "1557827033449"
+      }
+  },
+  "keyAltNames": [ "local" ]
+}