mongo 2.11.6 → 2.12.0.rc0

data/spec/{support → runners}/crud.rb

@@ -12,15 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require 'support/gridfs'
-require 'support/crud/requirement'
-require 'support/crud/spec'
-require 'support/crud/test_base'
-require 'support/crud/test'
-require 'support/crud/outcome'
-require 'support/crud/context'
-require 'support/crud/operation'
-require 'support/crud/verifier'
+require 'runners/gridfs'
+require 'runners/crud/requirement'
+require 'runners/crud/spec'
+require 'runners/crud/test_base'
+require 'runners/crud/test'
+require 'runners/crud/outcome'
+require 'runners/crud/context'
+require 'runners/crud/operation'
+require 'runners/crud/verifier'
 
 def collection_data(collection)
   collection.find.to_a

data/spec/{support → runners}/crud/operation.rb

@@ -28,7 +28,11 @@ module Mongo
         @crud_test = crud_test
         @spec = IceNine.deep_freeze(spec)
         @name = spec['name']
-        @arguments = spec['arguments'] || {}
+        if spec['arguments']
+          @arguments = BSON::ExtJSON.parse_obj(spec['arguments'])
+        else
+          @arguments = {}
+        end
         @outcome = Outcome.new(outcome_spec || spec)
       end
 
@@ -179,7 +183,7 @@ module Mongo
       end
 
       def download(fs_bucket, context)
-        stream = fs_bucket.open_download_stream(BSON::ObjectId.from_string(arguments['id']['$oid']))
+        stream = fs_bucket.open_download_stream(arguments['id'])
         stream.read
       end
 
@@ -190,7 +194,7 @@ module Mongo
 
       def map_reduce(collection, context)
         view = Mongo::Collection::View.new(collection)
-        mr = Mongo::Collection::View::MapReduce.new(view, arguments['map']['$code'], arguments['reduce']['$code'])
+        mr = Mongo::Collection::View::MapReduce.new(view, arguments['map'].javascript, arguments['reduce'].javascript)
         mr.to_a
       end
 

data/spec/{support → runners}/crud/requirement.rb

@@ -42,7 +42,7 @@ module Mongo
           ok &&= cc.fcv_ish >= short_min_server_version
         end
         if max_server_version
-          ok &&= cc.server_version <= max_server_version
+          ok &&= cc.short_server_version <= max_server_version
         end
         if topologies
           ok &&= topologies.include?(cc.topology)

data/spec/{support → runners}/crud/spec.rb

@@ -20,9 +20,13 @@ module Mongo
 
         @spec = YAML.load(contents)
         @description = File.basename(test_path)
-        @data = @spec['data']
+        @data = BSON::ExtJSON.parse_obj(@spec['data'])
         @tests = @spec['tests']
 
+        # Introduced with Client-Side Encryption tests
+        @json_schema = BSON::ExtJSON.parse_obj(@spec['json_schema'])
+        @key_vault_data = BSON::ExtJSON.parse_obj(@spec['key_vault_data'])
+
         @requirements = if run_on = @spec['runOn']
           run_on.map do |spec|
             Requirement.new(spec)
@@ -41,6 +45,13 @@ module Mongo
 
       attr_reader :requirements
 
+      # @return [ Hash ] The jsonSchema collection validator.
+      attr_reader :json_schema
+
+      # @return [ Array<Hash> ] Data to insert into the key vault before
+      #   running each test.
+      attr_reader :key_vault_data
+
       def collection_name
         # Older spec tests do not specify a collection name, thus
         # we provide a default here

data/spec/{support → runners}/crud/verifier.rb

@@ -34,12 +34,7 @@ module Mongo
           expect(actual_collection_data).to be_empty
         else
           expect(actual_collection_data).not_to be nil
-          expected_collection_data.each do |doc|
-            expect(actual_collection_data).to include(doc)
-          end
-          actual_collection_data.each do |doc|
-            expect(expected_collection_data).to include(doc)
-          end
+          expect(actual_collection_data).to match_with_type(expected_collection_data)
         end
       end
 
@@ -105,15 +100,14 @@ EOT
         expected_absence = expected_command.select { |k, v| v.nil? }
 
         expected_presence.each do |k, v|
-          expect(k => actual_command[k]).to eq(k => v)
+          expect(actual_command[k]).to match_with_type(v)
         end
 
         expected_absence.each do |k, v|
           expect(actual_command).not_to have_key(k)
         end
 
-        # this compares remaining fields in events after command is removed
-        expect(actual_event).to eq(expected_event)
+        expect(actual_event).to match_with_type(expected_event)
       end
 
       private
@@ -181,9 +175,13 @@ EOT
           return
         end
 
-        # This should produce a meaningful error message,
-        # even though we do not actually require that expected[k] == actual[k]
-        expect({k => expected[k]}).to eq({k => actual[k]})
+        if expected[k].is_a?(Time)
+          expect({k => expected[k].utc.to_s}).to eq({k => actual[k].utc.to_s})
+        else
+          # This should produce a meaningful error message,
+          # even though we do not actually require that expected[k] == actual[k]
+          expect({k => expected[k]}).to eq({k => actual[k]})
+        end
       end
     end
   end

data/spec/{support → runners}/transactions.rb

@@ -12,10 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require 'support/transactions/context'
-require 'support/transactions/operation'
-require 'support/transactions/spec'
-require 'support/transactions/test'
+require 'runners/transactions/context'
+require 'runners/transactions/operation'
+require 'runners/transactions/spec'
+require 'runners/transactions/test'
 
 def define_transactions_spec_tests(test_paths)
 

data/spec/{support → runners}/transactions/test.rb

@@ -70,11 +70,15 @@ module Mongo
           Operation.new(self, op)
         end
 
-        @expectations = test['expectations']
+        @expectations = BSON::ExtJSON.parse_obj(test['expectations'], mode: :bson)
+
         if test['outcome']
-          @outcome = Mongo::CRUD::Outcome.new(test['outcome'])
+          @outcome = Mongo::CRUD::Outcome.new(BSON::ExtJSON.parse_obj(test['outcome'], mode: :bson))
         end
+
         @expected_results = @operations.map do |o|
+          o = BSON::ExtJSON.parse_obj(o)
+
           # We check both o.key('error') and o['error'] to provide a better
           # error message in case error: false is ever needed in the tests
           if o.key?('error')
@@ -176,8 +180,8 @@ module Mongo
 
         @results = {
           results: results,
-          contents: @collection.with(
-          read: {mode: 'primary'},
+          contents: @result_collection.with(
+            read: {mode: 'primary'},
             read_concern: { level: 'local' },
           ).find.to_a,
           events: actual_events,
@@ -194,9 +198,33 @@ module Mongo
           direct_client.command(configureFailPoint: 'failCommand', mode: 'off')
         end
 
+        # Insert data into the key vault collection if required to do so by
+        # the tests.
+        if @spec.key_vault_data && !@spec.key_vault_data.empty?
+          key_vault_coll = support_client
+            .use(:admin)[:datakeys]
+            .with(write: { w: :majority })
+
+          key_vault_coll.drop
+          key_vault_coll.insert_many(@spec.key_vault_data)
+        end
+
         coll = support_client[@spec.collection_name].with(write: { w: :majority })
         coll.drop
-        support_client.command(create: @spec.collection_name, writeConcern: { w: :majority })
+
+        # Place a jsonSchema validator on the collection if required to do so
+        # by the tests.
+        collection_validator = if @spec.json_schema
+          { '$jsonSchema' => @spec.json_schema }
+        else
+          {}
+        end
+
+        support_client.command(
+          create: @spec.collection_name,
+          validator: collection_validator,
+          writeConcern: { w: :majority }
+        )
 
         coll.insert_many(@data) unless @data.empty?
 
@@ -210,6 +238,10 @@ module Mongo
 
         @collection = test_client[@spec.collection_name]
 
+        # Client-side encryption tests require the use of a separate client
+        # without auto_encryption_options for querying results.
+        @result_collection = support_client.use(@spec.database_name)[@spec.collection_name]
+
         @session0 = test_client.start_session(@session_options[:session0] || {})
         @session1 = test_client.start_session(@session_options[:session1] || {})
       end

data/spec/spec_helper.rb

@@ -1,10 +1,5 @@
 require 'lite_spec_helper'
 
-# Replica set name can be overridden via replicaSet parameter in MONGODB_URI
-# environment variable or by specifying RS_NAME environment variable when
-# not using MONGODB_URI.
-TEST_SET = 'ruby-driver-rs'
-
 require 'support/authorization'
 require 'support/primary_socket'
 require 'support/constraints'

data/spec/spec_tests/auth_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'lite_spec_helper'
 
 describe 'Auth' do
   include Mongo::Auth
@@ -15,9 +15,9 @@ describe 'Auth' do
 
           context 'when the auth configuration is invalid', unless: test.valid? do
             it 'raises an error' do
-              expect {
+              expect do
                 test.client
-            }.to raise_error(Mongo::Auth::InvalidConfiguration)
+              end.to raise_error(Mongo::Auth::InvalidConfiguration)
             end
           end
 

data/spec/spec_tests/client_side_encryption_spec.rb

@@ -0,0 +1,13 @@
+require 'spec_helper'
+
+describe 'Client-Side Encryption' do
+  require_libmongocrypt
+  require_enterprise
+
+  before do
+    unless ClusterConfig.instance.single_server?
+      skip "SPEC-2147: tests only work on single server"
+    end
+  end
+  define_transactions_spec_tests(CLIENT_SIDE_ENCRYPTION_TESTS)
+end

data/spec/spec_tests/connection_string_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'lite_spec_helper'
 
 describe 'Connection String' do
   define_connection_string_spec_tests(CONNECTION_STRING_TESTS)

data/spec/spec_tests/data/auth/connection-string.yml

@@ -85,6 +85,14 @@ tests:
             mechanism: "GSSAPI"
             mechanism_properties:
                 SERVICE_NAME: "mongodb"
+    -
+        description: "must raise an error when the authSource is empty"
+        uri: "mongodb://user:password@localhost/foo?authSource="
+        valid: false
+    -
+        description: "must raise an error when the authSource is empty without credentials"
+        uri: "mongodb://localhost/admin?authSource="
+        valid: false
     -
         description: "should throw an exception if authSource is invalid (GSSAPI)"
         uri: "mongodb://user%40DOMAIN.COM@localhost/?authMechanism=GSSAPI&authSource=foo"
@@ -287,6 +295,11 @@ tests:
         uri: "mongodb://localhost/foo"
         valid: true
         credential: ~
+    -
+        description: "authSource without username doesn't create credential"
+        uri: "mongodb://localhost/?authSource=foo"
+        valid: true
+        credential: ~
     -
         description: "should throw an exception if no username provided (userinfo implies default mechanism)"
         uri: "mongodb://@localhost.com/"

data/spec/spec_tests/data/client_side_encryption/aggregate.yml

@@ -0,0 +1,134 @@
+runOn:
+  - minServerVersion: "4.1.10"
+database_name: &database_name "default"
+collection_name: &collection_name "default"
+
+data:
+  - &doc0_encrypted { _id: 1, encrypted_string: {'$binary': {'base64': 'AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==', 'subType': '06'}} }
+json_schema: {'properties': {'encrypted_w_altname': {'encrypt': {'keyId': '/altname', 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Random'}}, 'encrypted_string': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'random': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Random'}}, 'encrypted_string_equivalent': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}}, 'bsonType': 'object'}
+key_vault_data: [{'status': 1, '_id': {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}, 'masterKey': {'provider': 'aws', 'key': 'arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0', 'region': 'us-east-1'}, 'updateDate': {'$date': {'$numberLong': '1552949630483'}}, 'keyMaterial': {'$binary': {'base64': 'AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO', 'subType': '00'}}, 'creationDate': {'$date': {'$numberLong': '1552949630483'}}, 'keyAltNames': ['altname', 'another_altname']}]
+
+tests:
+  - description: "Aggregate with deterministic encryption"
+    skipReason: "SERVER-39395"
+    clientOptions:
+      autoEncryptOpts:
+        kmsProviders:
+          aws: {} # Credentials filled in from environment.
+    operations:
+      - name: aggregate
+        arguments:
+          pipeline:
+            - { $match: { encrypted_string: "457-55-5642" } }
+        result:
+          - &doc0 { _id: 1, encrypted_string: "string0" }
+    expectations:
+      # Auto encryption will request the collection info.
+      - command_started_event:
+          command:
+            listCollections: 1
+            filter:
+              name: *collection_name
+          command_name: listCollections
+      - command_started_event:
+          command:
+            listCollections: 1
+            filter:
+              name: "datakeys"
+            $db: admin
+          command_name: listCollections
+      # Then key is fetched from the key vault.
+      - command_started_event:
+          command:
+            find: datakeys
+            filter: {"$or": [{"_id": {"$in": [ {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}} ] }}, {"keyAltNames": {"$in": []}}]}
+            $db: admin
+            readConcern: { level: "majority" }
+          command_name: find
+      - command_started_event:
+          command:
+            aggregate: *collection_name
+            pipeline:
+              - { $match: { encrypted_string: "457-55-5642" } }
+          command_name: aggregate
+    outcome:
+      collection:
+        # Outcome is checked using a separate MongoClient without auto encryption.
+        data:
+          - *doc0_encrypted
+  - description: "Aggregate with empty pipeline"
+    skipReason: "SERVER-40829 hides agg support behind enableTestCommands flag."
+    clientOptions:
+      autoEncryptOpts:
+        kmsProviders:
+          aws: {} # Credentials filled in from environment.
+    operations:
+      - name: aggregate
+        arguments:
+          pipeline: []
+        result:
+          - { _id: 1, encrypted_string: "string0" }
+    expectations:
+      # Auto encryption will request the collection info.
+      - command_started_event:
+          command:
+            listCollections: 1
+            filter:
+              name: *collection_name
+          command_name: listCollections
+      - command_started_event:
+          command:
+            aggregate: *collection_name
+            pipeline: []
+            cursor: {}
+          command_name: aggregate
+      # Needs to fetch key when decrypting results
+      - command_started_event:
+          command:
+            listCollections: 1
+            filter:
+              name: "datakeys"
+            $db: admin
+          command_name: listCollections
+      # Then key is fetched from the key vault.
+      - command_started_event:
+          command:
+            find: datakeys
+            filter: {"$or": [{"_id": {"$in": [ {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}} ] }}, {"keyAltNames": {"$in": []}}]}
+            $db: admin
+            readConcern: { level: "majority" }
+          command_name: find
+    outcome:
+      collection:
+        # Outcome is checked using a separate MongoClient without auto encryption.
+        data:
+          - *doc0_encrypted
+  - description: "Aggregate should fail with random encryption"
+    skipReason: "SERVER-39395"
+    clientOptions:
+      autoEncryptOpts:
+        kmsProviders:
+          aws: {} # Credentials filled in from environment.
+    operations:
+      - name: aggregate
+        arguments:
+          pipeline:
+            - { $match: { random: "abc" } }
+        result:
+          errorContains: "Cannot query on fields encrypted with the randomized encryption"
+  - description: "Database aggregate should fail"
+    clientOptions:
+      autoEncryptOpts:
+        kmsProviders:
+          aws: {} # Credentials filled in from environment.
+    operations:
+      - name: aggregate
+        object: database
+        arguments:
+          pipeline:
+            - $currentOp: { allUsers: false, idleConnections: false, localOps: true }
+            - $match: { command.aggregate: { $eq: 1 } }
+            - $project: { command: 1 }
+            - $project: { command.lsid: 0 }
+        result:
+          errorContains: "non-collection command not supported for auto encryption: aggregate"