mongo 2.18.3 → 2.19.0

data/spec/runners/unified/support_operations.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 module Unified
 
@@ -17,6 +17,9 @@ module Unified
         if session = args.use('session')
           opts[:session] = entities.get(:session, session)
         end
+        if read_preference = args.use('readPreference')
+          opts[:read] = ::Utils.snakeize_hash(read_preference)
+        end
 
         database.command(cmd, **opts)
       end
@@ -26,7 +29,13 @@ module Unified
       consume_test_runner(op)
       use_arguments(op) do |args|
         client = entities.get(:client, args.use!('client'))
-        client.command(args.use('failPoint'))
+        client.command(fp = args.use('failPoint'))
+
+        $disable_fail_points ||= []
+        $disable_fail_points << [
+          fp,
+          ClusterConfig.instance.primary_address,
+        ]
       end
     end
 
@@ -247,6 +256,61 @@ module Unified
       end
     end
 
+    def assert_event_count(op)
+      consume_test_runner(op)
+      use_arguments(op) do |args|
+        client = entities.get(:client, args.use!('client'))
+        subscriber = @subscribers.fetch(client)
+        event = args.use!('event')
+        assert_eq(event.keys.length, 1, "Expected event must have one key: #{event}")
+        count = args.use!('count')
+
+        events = select_events(subscriber, event)
+        if %w(serverDescriptionChangedEvent poolClearedEvent).include?(event.keys.first)
+          # We publish SDAM events from both regular and push monitors.
+          # This means sometimes there are two ServerMarkedUnknownEvent
+          # events published for the same server transition.
+          # Allow actual event count to be at least the expected event count
+          # in case there are multiple transitions in a single test.
+          assert_gte(events.length, count, "Expected event #{event} to occur #{count} times but received it #{events.length} times.")
+        else
+          assert_eq(events.length, count, "Expected event #{event} to occur #{count} times but received it #{events.length} times.")
+        end
+      end
+    end
+
+    def select_events(subscriber, event)
+      expected_name, opts = event.first
+      expected_name = expected_name.sub(/Event$/, '').sub(/^(.)/) { $1.upcase }
+      subscriber.wanted_events.select do |wevent|
+        if wevent.class.name.sub(/.*::/, '') == expected_name
+          spec = UsingHash[opts]
+          result = true
+          if new_desc = spec.use('newDescription')
+            if type = new_desc.use('type')
+              result &&= wevent.new_description.server_type == type.downcase.to_sym
+            end
+          end
+          unless spec.empty?
+            raise NotImplementedError, "Unhandled keys: #{spec}"
+          end
+          result
+        end
+      end
+    end
+
+    def assert_number_connections_checked_out(op)
+      consume_test_runner(op)
+      use_arguments(op) do |args|
+        client = entities.get(:client, args.use!('client'))
+        connections = args.use!('connections')
+        actual_c = client.cluster.servers.map(&:pool_internal).compact.sum do |p|
+          p.instance_variable_get(:@checked_out_connections).length
+        end
+        assert_eq(actual_c, connections, "Expected client #{client} to have #{connections} checked out connections but there are #{actual_c}.")
+      end
+    end
+
     private
 
     def assert_no_arguments(op)

data/spec/runners/unified/test.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 require 'runners/crud/requirement'
 require 'runners/unified/client_side_encryption_operations'
@@ -8,6 +8,7 @@ require 'runners/unified/grid_fs_operations'
 require 'runners/unified/ddl_operations'
 require 'runners/unified/change_stream_operations'
 require 'runners/unified/support_operations'
+require 'runners/unified/thread_operations'
 require 'runners/unified/assertions'
 require 'support/utils'
 require 'support/crypt'
@@ -21,6 +22,7 @@ module Unified
     include DdlOperations
     include ChangeStreamOperations
     include SupportOperations
+    include ThreadOperations
     include Assertions
     include RSpec::Core::Pending
 
@@ -46,6 +48,7 @@ module Unified
       @multiple_mongoses = mongoses.any? { |v| v }
       @test_spec.freeze
       @subscribers = {}
+      @observe_sensitive = {}
       @options = opts
     end
 
@@ -56,6 +59,10 @@ module Unified
     attr_reader :reqs, :group_reqs
     attr_reader :options
 
+    def retry?
+      @description =~ /KMS/i
+    end
+
     def skip?
       !!@skip_reason
     end
@@ -70,8 +77,13 @@ module Unified
 
     attr_reader :entities
 
-    def create_entities
-      @spec['createEntities'].each do |entity_spec|
+    def create_spec_entities
+      return if @entities_created
+      generate_entities(@spec['createEntities'])
+    end
+
+    def generate_entities(es)
+      es.each do |entity_spec|
         unless entity_spec.keys.length == 1
           raise NotImplementedError, "Entity must have exactly one key"
         end
@@ -88,6 +100,13 @@ module Unified
             opts = {}
           end
 
+          # max_pool_size gets automatically set to 3 if not explicitly set by
+          # the test, therefore, if min_pool_size is set, make sure to set the
+          # max_pool_size as well to something greater.
+          if !opts.key?('max_pool_size') && min_pool_size = opts[:min_pool_size]
+            opts[:max_pool_size] = min_pool_size + 3
+          end
+
           if spec.use('useMultipleMongoses')
             if ClusterConfig.instance.topology == :sharded
               unless SpecConfig.instance.addresses.length > 1
@@ -109,8 +128,6 @@ module Unified
             store_events.each do |spec|
               entity_name = spec['id']
               event_names = spec['events']
-              #event_name = event_name.gsub(/Event$/, '').gsub(/[A-Z]/) { |m| "_#{m}" }.upcase
-              #event_name = event_name.gsub(/Event$/, '').sub(/./) { |m| m.upcase }
               event_names.each do |event_name|
                 store_event_names[event_name] = entity_name
               end
@@ -134,10 +151,12 @@ module Unified
             opts[:server_api] = server_api
           end
 
-          create_client(**opts).tap do |client|
-            @observe_sensitive = spec.use('observeSensitiveCommands')
-            subscriber = (@subscribers[client] ||= EventSubscriber.new)
-            if oe = spec.use('observeEvents')
+          observe_events = spec.use('observeEvents')
+          subscriber = EventSubscriber.new
+          current_proc = opts[:sdam_proc]
+          opts[:sdam_proc] = lambda do |client|
+            current_proc.call(client) if current_proc
+            if oe = observe_events
               oe.each do |event|
                 case event
                 when 'commandStartedEvent', 'commandSucceededEvent', 'commandFailedEvent'
@@ -155,12 +174,23 @@ module Unified
                   end
                   kind = event.sub('Event', '').gsub(/([A-Z])/) { "_#{$1}" }.sub('pool', 'Pool').downcase.to_sym
                   subscriber.add_wanted_events(kind)
+                when 'serverDescriptionChangedEvent'
+                  unless client.send(:monitoring).subscribers[Mongo::Monitoring::SERVER_DESCRIPTION_CHANGED]&.include?(subscriber)
+                    client.subscribe(Mongo::Monitoring::SERVER_DESCRIPTION_CHANGED, subscriber)
+                  end
+                  kind = event.sub('Event', '').gsub(/([A-Z])/) { "_#{$1}" }.downcase.to_sym
+                  subscriber.add_wanted_events(kind)
                 else
                   raise NotImplementedError, "Unknown event #{event}"
                 end
               end
             end
           end
+
+          create_client(**opts).tap do |client|
+            @observe_sensitive[id] = spec.use('observeSensitiveCommands')
+            @subscribers[client] ||= subscriber
+          end
         when 'database'
           client = entities.get(:client, spec.use!('client'))
           opts = Utils.snakeize_hash(spec.use('databaseOptions') || {})
@@ -251,6 +281,28 @@ module Unified
             key_vault_client,
             opts
           )
+        when 'thread'
+          thread_context = ThreadContext.new
+          thread = Thread.new do
+            loop do
+              begin
+                op_spec = thread_context.operations.pop(true)
+                execute_operation(op_spec)
+              rescue ThreadError
+                # Queue is empty
+              end
+              if thread_context.stop?
+                break
+              else
+                sleep 0.1
+              end
+            end
+          end
+          class << thread
+            attr_accessor :context
+          end
+          thread.context = thread_context
+          thread
         else
           raise NotImplementedError, "Unknown type #{type}"
         end
@@ -259,6 +311,7 @@ module Unified
         end
         entities.set(type.to_sym, id, entity)
       end
+      @entities_created = true
     end
 
     def set_initial_data
@@ -337,7 +390,7 @@ module Unified
           method_name = "_#{name}"
         end
 
-        if ["modify_collection"].include?(name.to_s)
+        if ["modify_collection", "list_index_names"].include?(name.to_s)
           skip "Mongo Ruby Driver does not support #{name.to_s}"
         end
 
@@ -348,7 +401,7 @@ module Unified
             end
 
             public_send(method_name, op)
-          rescue Mongo::Error, BSON::String::IllegalKey, ArgumentError => e
+          rescue Mongo::Error, bson_error, Mongo::Auth::Unauthorized, ArgumentError => e
             if expected_error.use('isClientError')
               # isClientError doesn't actually mean a client error.
               # It means anything other than OperationFailure. DRIVERS-1799
@@ -385,9 +438,11 @@ module Unified
                 end
               end
             end
+            if error_response = expected_error.use("errorResponse")
+              assert_result_matches(e.document, error_response)
+            end
             if expected_result = expected_error.use('expectResult')
               assert_result_matches(e.result, expected_result)
-              #expected_result.clear
             # Important: this must be the last branch.
             elsif expected_error.use('isError')
               # Nothing but we consume the key.
@@ -398,6 +453,16 @@ module Unified
           else
             raise Error::ErrorMismatch, "Expected exception but none was raised"
           end
+        elsif op.use('ignoreResultAndError')
+          unless respond_to?(method_name)
+            raise Error::UnsupportedOperation, "Mongo Ruby Driver does not support #{name.to_s}"
+          end
+
+          begin
+            send(method_name, op)
+          # We can possibly rescue more errors here, add as needed.
+          rescue Mongo::Error
+          end
         else
           unless respond_to?(method_name, true)
             raise Error::UnsupportedOperation, "Mongo Ruby Driver does not support #{name.to_s}"
@@ -507,5 +572,14 @@ module Unified
       ).update(opts)
       Mongo::Client.new(*args)
     end
+
+    # The error to rescue BSON tests for. If we still define
+    # BSON::String::IllegalKey then we should rescue that particular error,
+    # otherwise, rescue an arbitrary BSON::Error
+    def bson_error
+      BSON::String.const_defined?(:IllegalKey) ?
+        BSON::String.const_get(:IllegalKey) :
+        BSON::Error
+    end
   end
 end

data/spec/runners/unified/test_group.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 module Unified
 

data/spec/runners/unified/thread_operations.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+# rubocop:todo all
+
+module Unified
+
+  module ThreadOperations
+
+    class ThreadContext
+      def initialize
+        @operations = Queue.new
+      end
+
+      def stop?
+        !!@stop
+      end
+
+      def signal_stop
+        @stop = true
+      end
+
+      attr_reader :operations
+    end
+
+    def wait(op)
+      consume_test_runner(op)
+      use_arguments(op) do |args|
+        sleep args.use!('ms') / 1000.0
+      end
+    end
+
+    def wait_for_event(op)
+      consume_test_runner(op)
+      use_arguments(op) do |args|
+        client = entities.get(:client, args.use!('client'))
+        subscriber = @subscribers.fetch(client)
+        event = args.use!('event')
+        assert_eq(event.keys.length, 1, "Expected event must have one key: #{event}")
+        count = args.use!('count')
+
+        deadline = Mongo::Utils.monotonic_time + 10
+        loop do
+          events = select_events(subscriber, event)
+          if events.length >= count
+            break
+          end
+          if Mongo::Utils.monotonic_time >= deadline
+            raise "Did not receive an event matching #{event} in 10 seconds; received #{events.length} but expected #{count} events"
+          else
+            sleep 0.1
+          end
+        end
+      end
+    end
+
+    def run_on_thread(op)
+      consume_test_runner(op)
+      use_arguments(op) do |args|
+        thread = entities.get(:thread, args.use!('thread'))
+        operation = args.use!('operation')
+        thread.context.operations << operation
+      end
+    end
+
+    def wait_for_thread(op)
+      consume_test_runner(op)
+      use_arguments(op) do |args|
+        thread = entities.get(:thread, args.use!('thread'))
+        thread.context.signal_stop
+        thread.join
+      end
+    end
+  end
+end

data/spec/runners/unified.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 require 'support/using_hash'
 require 'runners/unified/error'
@@ -48,6 +48,9 @@ def define_unified_spec_tests(base_path, paths, expect_failure: false)
             end
           end
 
+          if test.retry?
+            retry_test tries: 3
+          end
           if expect_failure
             it 'fails as expected' do
               if test.group_reqs
@@ -61,7 +64,7 @@ def define_unified_spec_tests(base_path, paths, expect_failure: false)
                 end
               end
               begin
-                test.create_entities
+                test.create_spec_entities
                 test.set_initial_data
                 begin
                   test.run
@@ -69,11 +72,11 @@ def define_unified_spec_tests(base_path, paths, expect_failure: false)
                   test.assert_events
                 # HACK: other errors are possible and likely will need to
                 # be added here later as the tests evolve.
-                rescue Mongo::Error::OperationFailure, Unified::Error::UnsupportedOperation
+                rescue Mongo::Error::OperationFailure, Unified::Error::UnsupportedOperation, UsingHash::UsingHashKeyError, Unified::Error::EntityMissing
                 rescue => e
-                  fail "Expected to raise Mongo::Error::OperationFailure or Unified::Error::UnsupportedOperation, got #{e}"
+                  fail "Expected to raise Mongo::Error::OperationFailure or Unified::Error::UnsupportedOperation or UsingHash::UsingHashKeyError or Unified::Error::EntityMissing, got #{e.class}: #{e}"
                 else
-                  fail "Expected to raise Mongo::Error::OperationFailure or Unified::Error::UnsupportedOperation, but no error was raised"
+                  fail "Expected to raise Mongo::Error::OperationFailure or Unified::Error::UnsupportedOperation or UsingHash::UsingHashKeyError or Unified::Error::EntityMissing, but no error was raised"
                 end
               ensure
                 test.cleanup
@@ -91,7 +94,7 @@ def define_unified_spec_tests(base_path, paths, expect_failure: false)
                   skip "Requirements not satisfied"
                 end
               end
-              test.create_entities
+              test.create_spec_entities
               test.set_initial_data
               test.run
               test.assert_outcome

data/spec/shared/lib/mrss/lite_constraints.rb

@@ -98,8 +98,8 @@ module Mrss
     def min_libmongocrypt_version(version)
       require_libmongocrypt
       before(:all) do
-        actual_version = Gem::Version.new(Mongo::Crypt::Binding.mongocrypt_version(nil))
-        min_version = Gem::Version.new(version)
+        actual_version = Utils.parse_version(Mongo::Crypt::Binding.mongocrypt_version(nil))
+        min_version = Utils.parse_version(version)
         unless actual_version >= min_version
           skip "libmongocrypt version #{min_version} required, but version #{actual_version} is available"
         end

data/spec/shared/lib/mrss/utils.rb

@@ -3,13 +3,35 @@
 
 module Mrss
   module Utils
+    extend self
 
-    module_function def print_backtrace(dest=STDERR)
-      begin
-        hello world
-      rescue => e
-        dest.puts e.backtrace.join("\n")
-      end
+    def print_backtrace(dest=STDERR)
+      raise
+    rescue => e
+      dest.puts e.backtrace.join("\n")
+    end
+
+    # Parses the given version string, accounting for suffix information that
+    # Gem::Version cannot successfully parse.
+    #
+    # @param [ String ] version the version to parse
+    #
+    # @return [ Gem::Version ] the parsed version
+    #
+    # @raise [ ArgumentError ] if the string cannot be parsed.
+    def parse_version(version)
+      Gem::Version.new(version)
+    rescue ArgumentError
+      match = version.match(/\A(?<major>\d+)\.(?<minor>\d+)\.(?<patch>\d+)?(-[A-Za-z\+\d]+)?\z/)
+      raise ArgumentError.new("Malformed version number string #{version}") if match.nil?
+
+      Gem::Version.new(
+        [
+          match[:major],
+          match[:minor],
+          match[:patch]
+        ].join('.')
+      )
     end
   end
 end

data/spec/shared/shlib/server.sh

@@ -197,7 +197,7 @@ calculate_server_args() {
   fi
 
   if test $mongo_version = latest; then
-    mongo_version=60
+    mongo_version=70
   fi
 
   local args="--setParameter enableTestCommands=1"

data/spec/solo/clean_exit_spec.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 require 'mongo'
 require 'lite_spec_helper'

data/spec/spec_helper.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 require 'lite_spec_helper'
 

data/spec/spec_tests/auth_spec.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 require 'lite_spec_helper'
 

data/spec/spec_tests/change_streams_unified_spec.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 require 'spec_helper'
 
@@ -9,5 +9,6 @@ base = "#{CURRENT_PATH}/spec_tests/data/change_streams_unified"
 CHANGE_STREAM_UNIFIED_TESTS = Dir.glob("#{base}/**/*.yml").sort
 
 describe 'Change stream unified spec tests' do
+  require_no_multi_mongos
   define_unified_spec_tests(base, CHANGE_STREAM_UNIFIED_TESTS)
 end

data/spec/spec_tests/client_side_encryption_spec.rb

@@ -1,14 +1,37 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 require 'spec_helper'
 
 require 'runners/crud'
 require 'runners/transactions'
 
+SPECS_IGNORING_BSON_TYPES = %w[ fle2v2-CreateCollection.yml ]
+
+# expect bson types for all specs EXCEPT those mentioned in
+# SPECS_IGNORING_BSON_TYPES
+EXPECTATIONS_BSON_TYPES = -> (test) { !SPECS_IGNORING_BSON_TYPES.include?(test.spec.description) }
+
 describe 'Client-Side Encryption' do
   require_libmongocrypt
   require_enterprise
+  min_libmongocrypt_version '1.8.0'
+
+  context 'with mongocryptd' do
+    SpecConfig.instance.without_crypt_shared_lib_path do
+      define_transactions_spec_tests(CLIENT_SIDE_ENCRYPTION_TESTS, expectations_bson_types: EXPECTATIONS_BSON_TYPES)
+    end
+  end
+
+  context 'with crypt_shared' do
+    # Under JRuby+Evergreen, these specs complain about the crypt_shared
+    # library not loading; however, crypt_shared appears to load for other
+    # specs that require it (see the client_side_encryption_unified_spec and
+    # mongocryptd_prose_spec tests).
+    fails_on_jruby
 
-  define_transactions_spec_tests(CLIENT_SIDE_ENCRYPTION_TESTS, expectations_bson_types: false)
+    SpecConfig.instance.require_crypt_shared do
+      define_transactions_spec_tests(CLIENT_SIDE_ENCRYPTION_TESTS, expectations_bson_types: EXPECTATIONS_BSON_TYPES)
+    end
+  end
 end

data/spec/spec_tests/client_side_encryption_unified_spec.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 require 'spec_helper'
 
@@ -12,5 +12,15 @@ describe 'Client side encryption spec tests - unified' do
   require_libmongocrypt
   require_enterprise
 
-  define_unified_spec_tests(base, CLIENT_SIDE_ENCRYPTION_UNIFIED_TESTS)
+  context 'with mongocryptd' do
+    SpecConfig.instance.without_crypt_shared_lib_path do
+      define_unified_spec_tests(base, CLIENT_SIDE_ENCRYPTION_UNIFIED_TESTS)
+    end
+  end
+
+  context 'with crypt_shared' do
+    SpecConfig.instance.require_crypt_shared do
+      define_unified_spec_tests(base, CLIENT_SIDE_ENCRYPTION_UNIFIED_TESTS)
+    end
+  end
 end

data/spec/spec_tests/cmap_spec.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 require 'spec_helper'
 
@@ -27,7 +27,7 @@ describe 'Cmap' do
   let(:options) do
     Mongo::Utils.shallow_symbolize_keys(Mongo::Client.canonicalize_ruby_options(
       SpecConfig.instance.all_test_options,
-    )).update(monitoring_io: false).tap do |options|
+    )).update(monitoring_io: false, populator_io: true).tap do |options|
       # We have a wait queue timeout set in the test suite options, but having
       # this option set interferes with assertions in the cmap spec tests.
       options.delete(:wait_queue_timeout)
@@ -38,6 +38,11 @@ describe 'Cmap' do
     spec = Mongo::Cmap::Spec.new(file)
 
     context("#{spec.description} (#{file.sub(%r'.*/data/cmap/', '')})") do
+      unless spec.satisfied?
+        before(:all) do
+          skip "Requirements not satisfied"
+        end
+      end
 
       before do
         subscriber = Mrss::EventSubscriber.new
@@ -54,28 +59,37 @@ describe 'Cmap' do
             options.merge(spec.pool_options)
           ).tap do |server|
             allow(server).to receive(:description).and_return(ClusterConfig.instance.primary_description)
+            # Since we use a mock for the cluster, run_sdam_flow does not clear
+            # the pool or mark the server unknown. Manually clear the pool and
+            # mock the server as unknown.
+            allow(server).to receive(:unknown!).and_wrap_original do |m, *args|
+              m.call(*args)
+              RSpec::Mocks.with_temporary_scope do
+                allow(server).to receive(:unknown?).and_return(true)
+                server.pool_internal&.clear(lazy: true)
+              end
+            end
           end
         )
-        spec.setup(@server, subscriber)
+
+        if app_name = spec.pool_options[:app_name]
+          allow(cluster).to receive(:app_metadata).and_return(Mongo::Server::AppMetadata.new({ app_name: app_name }))
+        end
+
+        @client = ClusterTools.instance.direct_client(ClusterConfig.instance.primary_address,
+          database: 'admin')
+        spec.setup(@server, @client, subscriber)
       end
 
       after do
-        if @server && (pool = @server.instance_variable_get('@pool'))
-          begin
-            pool.disconnect!
-          rescue Mongo::Error::PoolClosedError
-          end
+        if pool = @server&.pool_internal
+          pool.disconnect!
         end
+
+        spec.pool&.close
       end
 
       let!(:result) do
-        socket = double('fake socket')
-        allow(socket).to receive(:close)
-        # When linting, connection pool ensures the connection returned
-        # is connected.
-        allow(socket).to receive(:alive?).and_return(true)
-
-        allow_any_instance_of(Mongo::Server::Connection).to receive(:do_connect).and_return(socket)
         if @server.load_balancer?
           allow_any_instance_of(Mongo::Server::Connection).to receive(:service_id).and_return('very fake')
         end