mongo 2.17.4 → 2.18.0.beta1

data/spec/runners/crud/operation.rb

@@ -274,7 +274,14 @@ module Mongo
 
       def create_collection(database, context)
         opts = transformed_options(context)
-        database[arguments.fetch('collection')].create(session: opts[:session])
+        database[arguments.fetch('collection')]
+          .create(
+            {
+              session: opts[:session],
+              encrypted_fields: opts[:encrypted_fields],
+              validator: opts[:validator],
+            }.compact
+          )
       end
 
       def rename(collection, context)
@@ -285,11 +292,13 @@ module Mongo
       end
 
       def drop(collection, context)
-        collection.drop
+        opts = transformed_options(context)
+        collection.drop(encrypted_fields: opts[:encrypted_fields])
       end
 
       def drop_collection(database, context)
-        database[arguments.fetch('collection')].drop
+        opts = transformed_options(context)
+        database[arguments.fetch('collection')].drop(encrypted_fields: opts[:encrypted_fields])
       end
 
       def create_index(collection, context)

data/spec/runners/crud/requirement.rb

@@ -75,11 +75,11 @@ module Mongo
       def satisfied?
         cc = ClusterConfig.instance
         ok = true
-        if short_min_server_version
-          ok &&= cc.fcv_ish >= short_min_server_version
+        if min_server_version
+          ok &&= Gem::Version.new(cc.fcv_ish) >= Gem::Version.new(min_server_version)
         end
         if max_server_version
-          ok &&= cc.short_server_version <= max_server_version
+          ok &&= Gem::Version.new(cc.server_version) <= Gem::Version.new(max_server_version)
         end
         if topologies
           ok &&= topologies.include?(cc.topology)

data/spec/runners/crud/spec.rb

@@ -20,6 +20,7 @@ module Mongo
         # Introduced with Client-Side Encryption tests
         @json_schema = BSON::ExtJSON.parse_obj(@spec['json_schema'])
         @key_vault_data = BSON::ExtJSON.parse_obj(@spec['key_vault_data'])
+        @encrypted_fields = BSON::ExtJSON.parse_obj(@spec['encrypted_fields'])
 
         @requirements = if run_on = @spec['runOn']
           run_on.map do |spec|
@@ -46,6 +47,10 @@ module Mongo
       #   running each test.
       attr_reader :key_vault_data
 
+      # @return [ Hash ]  An encryptedFields option that should be set on the
+      #   collection (using createCollection) before each test run.
+      attr_reader :encrypted_fields
+
       def collection_name
         # Older spec tests do not specify a collection name, thus
         # we provide a default here

data/spec/runners/crud/verifier.rb

@@ -159,6 +159,12 @@ EOT
               verify_hash_items_equal(expected, actual, k)
             end
           end
+        when Array
+          expect(actual).to be_a(Array)
+          expect(actual.size).to eq(expected.size)
+          expected.zip(actual).each do |pair|
+            verify_result(pair.first, pair.last)
+          end
         else
           expect(actual).to eq(expected)
         end

data/spec/runners/transactions/test.rb

@@ -22,6 +22,7 @@ module Mongo
     #
     # @since 2.6.0
     class TransactionsTest < CRUD::CRUDTestBase
+      include MongosMacros
 
       attr_reader :expected_results
       attr_reader :skip_reason
@@ -42,7 +43,7 @@ module Mongo
       def initialize(crud_spec, data, test)
         test = IceNine.deep_freeze(test)
         @spec = crud_spec
-        @data = data
+        @data = data || []
         @description = test['description']
         @client_options = {
           # Disable legacy read & write retries, so that when spec tests
@@ -232,19 +233,37 @@ module Mongo
           end
         end
 
+        key_vault_coll = support_client
+        .use(:keyvault)[:datakeys]
+        .with(write: { w: :majority })
+
+        key_vault_coll.drop
         # Insert data into the key vault collection if required to do so by
         # the tests.
         if @spec.key_vault_data && !@spec.key_vault_data.empty?
-          key_vault_coll = support_client
-            .use(:admin)[:datakeys]
-            .with(write: { w: :majority })
-
-          key_vault_coll.drop
           key_vault_coll.insert_many(@spec.key_vault_data)
         end
 
+        if @spec.encrypted_fields
+          encrypted_fields = @spec.encrypted_fields.dup
+          # This code MUST be removed as soon as server starts accepting
+          # contention as int32.
+          if encrypted_fields.key?('fields')
+            encrypted_fields['fields'] = encrypted_fields['fields'].dup.map do |field|
+              if field['queries'] && field['queries'].key?('contention')
+                new_field = field.dup
+                new_field['queries'] = field['queries'].dup
+                new_field['queries']['contention'] = BSON::Int64.new(field['queries']['contention'])
+                new_field
+              else
+                field
+              end
+            end
+          end
+          # End of code to be removed
+        end
         coll = support_client[@spec.collection_name].with(write: { w: :majority })
-        coll.drop
+        coll.drop(encrypted_fields: encrypted_fields)
 
         # Place a jsonSchema validator on the collection if required to do so
         # by the tests.
@@ -254,19 +273,19 @@ module Mongo
           {}
         end
 
-        support_client.command(
+        create_collection_spec = {
           create: @spec.collection_name,
           validator: collection_validator,
           writeConcern: { w: 'majority' }
-        )
+        }
+
+        create_collection_spec[:encryptedFields] = encrypted_fields if encrypted_fields
+        support_client.command(create_collection_spec)
 
         coll.insert_many(@data) unless @data.empty?
 
-        $distinct_ran ||= {}
-        $distinct_ran[@spec.database_name] ||= if description =~ /distinct/ || @operations.any? { |op| op.name == 'distinct' }
-          ::Utils.mongos_each_direct_client do |direct_client|
-            direct_client.use(@spec.database_name)['test'].distinct('foo').to_a
-          end
+        if description =~ /distinct/ || @operations.any? { |op| op.name == 'distinct' }
+          run_mongos_distincts(@spec.database_name, 'test')
         end
 
         admin_support_client.command(@fail_point_command) if @fail_point_command

data/spec/runners/transactions.rb

@@ -20,6 +20,7 @@ require 'runners/transactions/spec'
 require 'runners/transactions/test'
 
 def define_transactions_spec_tests(test_paths)
+  config_override :validate_update_replace, true
 
   test_paths.each do |file|
 
@@ -51,16 +52,18 @@ def define_transactions_spec_tests(test_paths)
               end
             end
 
-            before(:all) do
-              if req.satisfied?
-                test.setup_test
+            unless req.satisfied?
+              before(:all) do
+                skip "Requirements not satisfied"
               end
             end
 
+            before(:all) do
+              test.setup_test
+            end
+
             after(:all) do
-              if req.satisfied?
-                test.teardown_test
-              end
+              test.teardown_test
             end
 
             let(:results) do

data/spec/runners/unified/assertions.rb

@@ -51,6 +51,31 @@ module Unified
             end
           end
 
+          %w(acknowledged).each do |k|
+            expected_v = expected.use(k)
+            next unless expected_v
+            actual_v = case actual
+            when Mongo::BulkWrite::Result, Mongo::Operation::Result
+              if Hash === expected_v && expected_v.keys == %w($$unsetOrMatches)
+                expected_v = expected_v.values.first
+              end
+              actual.send("#{k}?")
+            else
+              actual[k]
+            end
+            if expected_v
+              if expected_v.empty?
+                if actual_v && !actual_v.empty?
+                  raise Error::ResultMismatch, "Actual not empty"
+                end
+              else
+                if actual_v != expected_v
+                  raise Error::ResultMismatch, "Mismatch: actual #{actual_v}, expected #{expected_v}"
+                end
+              end
+            end
+          end
+
           assert_matches(actual, expected, 'result')
           expected.clear
         end
@@ -99,6 +124,12 @@ module Unified
         client = entities.get(:client, client_id)
         subscriber = @subscribers.fetch(client)
         expected_events = spec.use!('events')
+        ignore_extra_events = if ignore = spec.use('ignoreExtraEvents')
+          # Ruby treats 0 as truthy, whereas the spec tests use it as falsy.
+          ignore == 0 ? false : ignore
+        else
+          false
+        end
         actual_events = subscriber.wanted_events(@observe_sensitive)
         case spec.use('eventType')
         when nil, 'command'
@@ -107,10 +138,12 @@ module Unified
           end
         when 'cmap'
           actual_events.select! do |event|
-            event.class.name.sub(/.*::/, '') =~ /^Pool/
+            event.class.name.sub(/.*::/, '') =~ /^(?:Pool|Connection)/
           end
         end
-        unless actual_events.length == expected_events.length
+
+        if (!ignore_extra_events && actual_events.length != expected_events.length) ||
+           (ignore_extra_events && actual_events.length < expected_events.length)
           raise Error::ResultMismatch, "Event count mismatch: expected #{expected_events.length}, actual #{actual_events.length}\nExpected: #{expected_events}\nActual: #{actual_events}"
         end
         expected_events.each_with_index do |event, i|
@@ -131,6 +164,9 @@ module Unified
       if spec.use('hasServiceId')
         actual.service_id.should_not be nil
       end
+      if spec.use('hasServerConnectionId')
+        actual.server_connection_id.should_not be nil
+      end
       if db_name = spec.use('databaseName')
         assert_eq(actual.database_name, db_name, 'Database names differ')
       end
@@ -155,8 +191,12 @@ module Unified
     end
 
     def assert_matches(actual, expected, msg)
-      if actual.nil? && !expected.nil?
-        raise Error::ResultMismatch, "#{msg}: expected #{expected} but got nil"
+      if actual.nil?
+        if expected&.keys == ["$$unsetOrMatches"]
+          return
+        elsif !expected.nil?
+          raise Error::ResultMismatch, "#{msg}: expected #{expected} but got nil"
+        end
       end
 
       case expected
@@ -208,10 +248,18 @@ module Unified
     end
 
     def assert_type(object, type)
+      ok = [*type].reduce(false) { |acc, x| acc || type_matches?(object, x) }
+
+      unless ok
+        raise Error::ResultMismatch, "Object #{object} is not of type #{type}"
+      end
+    end
+
+    def type_matches?(object, type)
       ok = case type
       when 'object'
         Hash === object
-      when %w(int long)
+      when 'int', 'long'
         Integer === object || BSON::Int32 === object || BSON::Int64 === object
       when 'objectId'
         BSON::ObjectId === object
@@ -219,12 +267,11 @@ module Unified
         Time === object
       when 'double'
         Float === object
+      when 'string'
+        String === object
       else
         raise NotImplementedError, "Unhandled type #{type}"
       end
-      unless ok
-        raise Error::ResultMismatch, "Object #{object} is not of type #{type}"
-      end
     end
 
     def assert_value_matches(actual, expected, msg)
@@ -235,8 +282,10 @@ module Unified
         case operator
         when '$$unsetOrMatches'
           if actual
-            unless actual == expected_v
-              raise Error::ResultMismatch, "Mismatch for #{msg}: expected #{expected}, have #{actual}"
+            if Mongo::BulkWrite::Result === actual || Mongo::Operation::Result === actual
+              assert_result_matches(actual, UsingHash[expected_v])
+            else
+              assert_matches(actual, expected_v, msg)
             end
           end
         when '$$matchesHexBytes'

data/spec/runners/unified/change_stream_operations.rb

@@ -14,6 +14,15 @@ module Unified
         if batch_size = args.use('batchSize')
           opts[:batch_size] = batch_size
         end
+        if comment = args.use('comment')
+          opts[:comment] = comment
+        end
+        if full_document = args.use('fullDocument')
+          opts[:full_document] = full_document
+        end
+        if full_document_before_change = args.use('fullDocumentBeforeChange')
+          opts[:full_document_before_change] = full_document_before_change
+        end
         cs = object.watch(pipeline, **opts)
         name = op.use!('saveResultAsEntity')
         entities.set(:change_stream, name, cs)

data/spec/runners/unified/crud_operations.rb

@@ -10,6 +10,8 @@ module Unified
       use_arguments(op) do |args|
         opts = {
           let: args.use('let'),
+          comment: args.use('comment'),
+          allow_disk_use: args.use('allowDiskUse'),
         }
         if session = args.use('session')
           opts[:session] = entities.get(:session, session)
@@ -35,6 +37,9 @@ module Unified
         if session = args.use('session')
           opts[:session] = entities.get(:session, session)
         end
+        if comment = args.use('comment')
+          opts[:comment] = comment
+        end
         collection.find(args.use!('filter')).count_documents(**opts)
       end
     end
@@ -46,6 +51,9 @@ module Unified
         if max_time_ms = args.use('maxTimeMS')
           opts[:max_time_ms] = max_time_ms
         end
+        if comment = args.use('comment')
+          opts[:comment] = comment
+        end
         collection.estimated_document_count(**opts)
       end
     end
@@ -69,6 +77,8 @@ module Unified
         update = args.use!('update')
         opts = {
           let: args.use('let'),
+          comment: args.use('comment'),
+          hint: args.use('hint'),
         }
         if return_document = args.use('returnDocument')
           opts[:return_document] = return_document.downcase.to_sym
@@ -87,6 +97,8 @@ module Unified
         update = args.use!('replacement')
         opts = {
           let: args.use('let'),
+          comment: args.use('comment'),
+          hint: args.use('hint'),
         }
         if session = args.use('session')
           opts[:session] = entities.get(:session, session)
@@ -101,6 +113,8 @@ module Unified
         filter = args.use!('filter')
         opts = {
           let: args.use('let'),
+          comment: args.use('comment'),
+          hint: args.use('hint'),
         }
         if session = args.use('session')
           opts[:session] = entities.get(:session, session)
@@ -112,7 +126,9 @@ module Unified
     def insert_one(op)
       collection = entities.get(:collection, op.use!('object'))
       use_arguments(op) do |args|
-        opts = {}
+        opts = {
+          comment: args.use('comment')
+        }
         if session = args.use('session')
           opts[:session] = entities.get(:session, session)
         end
@@ -123,7 +139,9 @@ module Unified
     def insert_many(op)
       collection = entities.get(:collection, op.use!('object'))
       use_arguments(op) do |args|
-        opts = {}
+        opts = {
+          comment: args.use('comment')
+        }
         unless (ordered = args.use('ordered')).nil?
           opts[:ordered] = ordered
         end
@@ -139,6 +157,8 @@ module Unified
       use_arguments(op) do |args|
         opts = {
           let: args.use('let'),
+          comment: args.use('comment'),
+          hint: args.use('hint'),
         }
         if session = args.use('session')
           opts[:session] = entities.get(:session, session)
@@ -152,6 +172,8 @@ module Unified
       use_arguments(op) do |args|
         opts = {
           let: args.use('let'),
+          comment: args.use('comment'),
+          hint: args.use('hint'),
         }
         collection.update_many(args.use!('filter'), args.use!('update'), **opts)
       end
@@ -163,7 +185,10 @@ module Unified
         collection.replace_one(
           args.use!('filter'),
           args.use!('replacement'),
+          comment: args.use('comment'),
           upsert: args.use('upsert'),
+          let: args.use('let'),
+          hint: args.use('hint')
         )
       end
     end
@@ -173,6 +198,8 @@ module Unified
       use_arguments(op) do |args|
         opts = {
           let: args.use('let'),
+          comment: args.use('comment'),
+          hint: args.use('hint'),
         }
         if session = args.use('session')
           opts[:session] = entities.get(:session, session)
@@ -186,6 +213,8 @@ module Unified
       use_arguments(op) do |args|
         opts = {
           let: args.use('let'),
+          comment: args.use('comment'),
+          hint: args.use('hint'),
         }
         collection.delete_many(args.use!('filter'), **opts)
       end
@@ -201,6 +230,12 @@ module Unified
         if ordered = args.use('ordered')
           opts[:ordered] = true
         end
+        if comment = args.use('comment')
+          opts[:comment] = comment
+        end
+        if let = args.use('let')
+          opts[:let] = let
+        end
         collection.bulk_write(requests, **opts)
       end
     end
@@ -215,6 +250,15 @@ module Unified
       if session = args.use('session')
         opts[:session] = entities.get(:session, session)
       end
+      if comment = args.use('comment')
+        opts[:comment] = comment
+      end
+      if batch_size = args.use('batchSize')
+        opts[:batch_size] = batch_size
+      end
+      if args.key?('allowDiskUse')
+        opts[:allow_disk_use] = args.use('allowDiskUse')
+      end
       unless args.empty?
         raise NotImplementedError, "Unhandled spec keys: #{args} in #{test_spec}"
       end
@@ -237,16 +281,20 @@ module Unified
           filter: spec.use('filter'),
           update: spec.use('update'),
           upsert: spec.use('upsert'),
+          array_filters: spec.use('arrayFilters'),
+          hint: spec.use('hint'),
         }
       when 'replaceOne'
         {
           filter: spec.use('filter'),
           replacement: spec.use('replacement'),
           upsert: spec.use('upsert'),
+          hint: spec.use('hint'),
         }
       when 'deleteOne', 'deleteMany'
         {
           filter: spec.use('filter'),
+          hint: spec.use('hint'),
         }
       else
         raise NotImplementedError, "Unknown operation #{op}"

data/spec/runners/unified/ddl_operations.rb

@@ -30,6 +30,12 @@ module Unified
         if expire_after_seconds = args.use('expireAfterSeconds')
           collection_opts[:expire_after] = expire_after_seconds
         end
+        if clustered_index = args.use('clusteredIndex')
+          collection_opts[:clustered_index] = clustered_index
+        end
+        if change_stream_pre_and_post_images = args.use('changeStreamPreAndPostImages')
+          collection_opts[:change_stream_pre_and_post_images] = change_stream_pre_and_post_images
+        end
         database[args.use!('collection'), collection_opts].create(**opts)
       end
     end
@@ -41,6 +47,9 @@ module Unified
         if session = args.use('session')
           opts[:session] = entities.get(:session, session)
         end
+        if filter = args.use('filter')
+          opts[:filter] = filter
+        end
         database.list_collections(**opts)
       end
     end
@@ -53,6 +62,17 @@ module Unified
       end
     end
 
+    def rename(op)
+      collection = entities.get(:collection, op.use!('object'))
+      use_arguments(op) do |args|
+        to = args.use!('to')
+        collection.client.use(:admin).command({
+          renameCollection: "#{collection.database.name}.#{collection.name}",
+          to: "#{collection.database.name}.#{to}"
+        })
+      end
+    end
+
     def assert_collection_exists(op, state = true)
       consume_test_runner(op)
       use_arguments(op) do |args|

data/spec/runners/unified/error.rb

@@ -23,6 +23,7 @@ module Unified
     class InvalidTest < Error
     end
 
+    class UnsupportedOperation < Error
+    end
   end
-
 end

data/spec/runners/unified/support_operations.rb

@@ -59,8 +59,11 @@ module Unified
 
     def assert_session_dirty(op)
       consume_test_runner(op)
-      # https://jira.mongodb.org/browse/RUBY-1813
-      true
+      use_arguments(op) do |args|
+        session = entities.get(:session, args.use!('session'))
+        # https://jira.mongodb.org/browse/RUBY-1813
+        true
+      end
     end
 
     def assert_session_not_dirty(op)

data/spec/runners/unified/test.rb

@@ -19,6 +19,7 @@ module Unified
     include ChangeStreamOperations
     include SupportOperations
     include Assertions
+    include RSpec::Core::Pending
 
     def initialize(spec, **opts)
       @spec = spec
@@ -132,11 +133,11 @@ module Unified
 
           create_client(**opts).tap do |client|
             @observe_sensitive = spec.use('observeSensitiveCommands')
+            subscriber = (@subscribers[client] ||= EventSubscriber.new)
             if oe = spec.use('observeEvents')
               oe.each do |event|
                 case event
                 when 'commandStartedEvent', 'commandSucceededEvent', 'commandFailedEvent'
-                  subscriber = (@subscribers[client] ||= EventSubscriber.new)
                   unless client.send(:monitoring).subscribers[Mongo::Monitoring::COMMAND].include?(subscriber)
                     client.subscribe(Mongo::Monitoring::COMMAND, subscriber)
                   end
@@ -146,7 +147,6 @@ module Unified
                     subscriber.ignore_commands(ignore_events)
                   end
                 when /\A(?:pool|connection)/
-                  subscriber = (@subscribers[client] ||= EventSubscriber.new)
                   unless client.send(:monitoring).subscribers[Mongo::Monitoring::CONNECTION_POOL]&.include?(subscriber)
                     client.subscribe(Mongo::Monitoring::CONNECTION_POOL, subscriber)
                   end
@@ -278,8 +278,17 @@ module Unified
         if name.to_s == 'loop'
           method_name = "_#{name}"
         end
+
+        if ["modify_collection"].include?(name.to_s)
+          skip "Mongo Ruby Driver does not support #{name.to_s}"
+        end
+
         if expected_error = op.use('expectError')
           begin
+            unless respond_to?(method_name)
+              raise Error::UnsupportedOperation, "Mongo Ruby Driver does not support #{name.to_s}"
+            end
+
             public_send(method_name, op)
           rescue Mongo::Error, BSON::String::IllegalKey => e
             if expected_error.use('isClientError')
@@ -332,10 +341,16 @@ module Unified
             raise Error::ErrorMismatch, "Expected exception but none was raised"
           end
         else
+          unless respond_to?(method_name, true)
+            raise Error::UnsupportedOperation, "Mongo Ruby Driver does not support #{name.to_s}"
+          end
+
           result = send(method_name, op)
           if expected_result = op.use('expectResult')
-            if result.nil? && !expected_result.empty?
-              raise Error::ResultMismatch, "Actual result nil but expected result #{expected_result}"
+            if result.nil? && expected_result.keys == ["$$unsetOrMatches"]
+              return
+            elsif result.nil? && !expected_result.empty?
+              raise Error::ResultMismatch, "#{msg}: expected #{expected} but got nil"
             elsif Array === expected_result
               assert_documents_match(result, expected_result)
             else

data/spec/runners/unified.rb

@@ -9,6 +9,8 @@ require 'runners/unified/test'
 require 'runners/unified/test_group'
 
 def define_unified_spec_tests(base_path, paths, expect_failure: false)
+  config_override :validate_update_replace, true
+
   paths.each do |path|
     basename = path[base_path.length+1...path.length]
     context basename do
@@ -61,13 +63,18 @@ def define_unified_spec_tests(base_path, paths, expect_failure: false)
               begin
                 test.create_entities
                 test.set_initial_data
-                lambda do
+                begin
                   test.run
                   test.assert_outcome
                   test.assert_events
                 # HACK: other errors are possible and likely will need to
                 # be added here later as the tests evolve.
-                end.should raise_error(Mongo::Error::OperationFailure)
+                rescue Mongo::Error::OperationFailure, Unified::Error::UnsupportedOperation
+                rescue => e
+                  fail "Expected to raise Mongo::Error::OperationFailure or Unified::Error::UnsupportedOperation, got #{e}"
+                else
+                  fail "Expected to raise Mongo::Error::OperationFailure or Unified::Error::UnsupportedOperation, but no error was raised"
+                end
               ensure
                 test.cleanup
               end