mongo 2.18.2 → 2.19.3

data/spec/runners/unified/support_operations.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 module Unified
 
@@ -17,6 +17,9 @@ module Unified
         if session = args.use('session')
           opts[:session] = entities.get(:session, session)
         end
+        if read_preference = args.use('readPreference')
+          opts[:read] = ::Utils.snakeize_hash(read_preference)
+        end
 
         database.command(cmd, **opts)
       end
@@ -26,7 +29,13 @@ module Unified
       consume_test_runner(op)
       use_arguments(op) do |args|
         client = entities.get(:client, args.use!('client'))
-        client.command(args.use('failPoint'))
+        client.command(fp = args.use('failPoint'))
+
+        $disable_fail_points ||= []
+        $disable_fail_points << [
+          fp,
+          ClusterConfig.instance.primary_address,
+        ]
       end
     end
 
@@ -247,6 +256,61 @@ module Unified
       end
     end
 
+    def assert_event_count(op)
+      consume_test_runner(op)
+      use_arguments(op) do |args|
+        client = entities.get(:client, args.use!('client'))
+        subscriber = @subscribers.fetch(client)
+        event = args.use!('event')
+        assert_eq(event.keys.length, 1, "Expected event must have one key: #{event}")
+        count = args.use!('count')
+
+        events = select_events(subscriber, event)
+        if %w(serverDescriptionChangedEvent poolClearedEvent).include?(event.keys.first)
+          # We publish SDAM events from both regular and push monitors.
+          # This means sometimes there are two ServerMarkedUnknownEvent
+          # events published for the same server transition.
+          # Allow actual event count to be at least the expected event count
+          # in case there are multiple transitions in a single test.
+          assert_gte(events.length, count, "Expected event #{event} to occur #{count} times but received it #{events.length} times.")
+        else
+          assert_eq(events.length, count, "Expected event #{event} to occur #{count} times but received it #{events.length} times.")
+        end
+      end
+    end
+
+    def select_events(subscriber, event)
+      expected_name, opts = event.first
+      expected_name = expected_name.sub(/Event$/, '').sub(/^(.)/) { $1.upcase }
+      subscriber.wanted_events.select do |wevent|
+        if wevent.class.name.sub(/.*::/, '') == expected_name
+          spec = UsingHash[opts]
+          result = true
+          if new_desc = spec.use('newDescription')
+            if type = new_desc.use('type')
+              result &&= wevent.new_description.server_type == type.downcase.to_sym
+            end
+          end
+          unless spec.empty?
+            raise NotImplementedError, "Unhandled keys: #{spec}"
+          end
+          result
+        end
+      end
+    end
+
+    def assert_number_connections_checked_out(op)
+      consume_test_runner(op)
+      use_arguments(op) do |args|
+        client = entities.get(:client, args.use!('client'))
+        connections = args.use!('connections')
+        actual_c = client.cluster.servers.map(&:pool_internal).compact.sum do |p|
+          p.instance_variable_get(:@checked_out_connections).length
+        end
+        assert_eq(actual_c, connections, "Expected client #{client} to have #{connections} checked out connections but there are #{actual_c}.")
+      end
+    end
+
     private
 
     def assert_no_arguments(op)

data/spec/runners/unified/test.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 require 'runners/crud/requirement'
 require 'runners/unified/client_side_encryption_operations'
@@ -8,6 +8,8 @@ require 'runners/unified/grid_fs_operations'
 require 'runners/unified/ddl_operations'
 require 'runners/unified/change_stream_operations'
 require 'runners/unified/support_operations'
+require 'runners/unified/thread_operations'
+require 'runners/unified/search_index_operations'
 require 'runners/unified/assertions'
 require 'support/utils'
 require 'support/crypt'
@@ -21,6 +23,8 @@ module Unified
     include DdlOperations
     include ChangeStreamOperations
     include SupportOperations
+    include ThreadOperations
+    include SearchIndexOperations
     include Assertions
     include RSpec::Core::Pending
 
@@ -46,6 +50,7 @@ module Unified
       @multiple_mongoses = mongoses.any? { |v| v }
       @test_spec.freeze
       @subscribers = {}
+      @observe_sensitive = {}
       @options = opts
     end
 
@@ -56,6 +61,10 @@ module Unified
     attr_reader :reqs, :group_reqs
     attr_reader :options
 
+    def retry?
+      @description =~ /KMS/i
+    end
+
     def skip?
       !!@skip_reason
     end
@@ -70,8 +79,13 @@ module Unified
 
     attr_reader :entities
 
-    def create_entities
-      @spec['createEntities'].each do |entity_spec|
+    def create_spec_entities
+      return if @entities_created
+      generate_entities(@spec['createEntities'])
+    end
+
+    def generate_entities(es)
+      es.each do |entity_spec|
         unless entity_spec.keys.length == 1
           raise NotImplementedError, "Entity must have exactly one key"
         end
@@ -88,6 +102,13 @@ module Unified
             opts = {}
           end
 
+          # max_pool_size gets automatically set to 3 if not explicitly set by
+          # the test, therefore, if min_pool_size is set, make sure to set the
+          # max_pool_size as well to something greater.
+          if !opts.key?('max_pool_size') && min_pool_size = opts[:min_pool_size]
+            opts[:max_pool_size] = min_pool_size + 3
+          end
+
           if spec.use('useMultipleMongoses')
             if ClusterConfig.instance.topology == :sharded
               unless SpecConfig.instance.addresses.length > 1
@@ -101,7 +122,7 @@ module Unified
             # the other set members, in standalone deployments because
             # there is only one server, but changes behavior in
             # sharded clusters compared to how the test suite is configured.
-            opts[:single_address] = true
+            options[:single_address] = true
           end
 
           if store_events = spec.use('storeEventsAsEntities')
@@ -109,8 +130,6 @@ module Unified
             store_events.each do |spec|
               entity_name = spec['id']
               event_names = spec['events']
-              #event_name = event_name.gsub(/Event$/, '').gsub(/[A-Z]/) { |m| "_#{m}" }.upcase
-              #event_name = event_name.gsub(/Event$/, '').sub(/./) { |m| m.upcase }
               event_names.each do |event_name|
                 store_event_names[event_name] = entity_name
               end
@@ -134,10 +153,12 @@ module Unified
             opts[:server_api] = server_api
           end
 
-          create_client(**opts).tap do |client|
-            @observe_sensitive = spec.use('observeSensitiveCommands')
-            subscriber = (@subscribers[client] ||= EventSubscriber.new)
-            if oe = spec.use('observeEvents')
+          observe_events = spec.use('observeEvents')
+          subscriber = EventSubscriber.new
+          current_proc = opts[:sdam_proc]
+          opts[:sdam_proc] = lambda do |client|
+            current_proc.call(client) if current_proc
+            if oe = observe_events
               oe.each do |event|
                 case event
                 when 'commandStartedEvent', 'commandSucceededEvent', 'commandFailedEvent'
@@ -155,12 +176,23 @@ module Unified
                   end
                   kind = event.sub('Event', '').gsub(/([A-Z])/) { "_#{$1}" }.sub('pool', 'Pool').downcase.to_sym
                   subscriber.add_wanted_events(kind)
+                when 'serverDescriptionChangedEvent'
+                  unless client.send(:monitoring).subscribers[Mongo::Monitoring::SERVER_DESCRIPTION_CHANGED]&.include?(subscriber)
+                    client.subscribe(Mongo::Monitoring::SERVER_DESCRIPTION_CHANGED, subscriber)
+                  end
+                  kind = event.sub('Event', '').gsub(/([A-Z])/) { "_#{$1}" }.downcase.to_sym
+                  subscriber.add_wanted_events(kind)
                 else
                   raise NotImplementedError, "Unknown event #{event}"
                 end
               end
             end
           end
+
+          create_client(**opts).tap do |client|
+            @observe_sensitive[id] = spec.use('observeSensitiveCommands')
+            @subscribers[client] ||= subscriber
+          end
         when 'database'
           client = entities.get(:client, spec.use!('client'))
           opts = Utils.snakeize_hash(spec.use('databaseOptions') || {})
@@ -251,6 +283,28 @@ module Unified
             key_vault_client,
             opts
           )
+        when 'thread'
+          thread_context = ThreadContext.new
+          thread = Thread.new do
+            loop do
+              begin
+                op_spec = thread_context.operations.pop(true)
+                execute_operation(op_spec)
+              rescue ThreadError
+                # Queue is empty
+              end
+              if thread_context.stop?
+                break
+              else
+                sleep 0.1
+              end
+            end
+          end
+          class << thread
+            attr_accessor :context
+          end
+          thread.context = thread_context
+          thread
         else
           raise NotImplementedError, "Unknown type #{type}"
         end
@@ -259,6 +313,7 @@ module Unified
         end
         entities.set(type.to_sym, id, entity)
       end
+      @entities_created = true
     end
 
     def set_initial_data
@@ -267,22 +322,22 @@ module Unified
         collection = root_authorized_client.with(write_concern: {w: :majority}).
           use(spec.use!('databaseName'))[spec.use!('collectionName')]
         collection.drop
+        create_options = spec.use('createOptions') || {}
         docs = spec.use!('documents')
-        if docs.any?
-          collection.insert_many(docs)
-        else
-          begin
-            collection.create
-          rescue Mongo::Error => e
-            if Mongo::Error::OperationFailure === e && (
+        begin
+          collection.create(create_options)
+        rescue Mongo::Error => e
+          if Mongo::Error::OperationFailure === e && (
               e.code == 48 || e.message =~ /collection already exists/
-            )
-              # Already exists
-            else
-              raise
-            end
+          )
+            # Already exists
+          else
+            raise
           end
         end
+        if docs.any?
+          collection.insert_many(docs)
+        end
         unless spec.empty?
           raise NotImplementedError, "Unhandled spec keys: #{spec}"
         end
@@ -337,7 +392,7 @@ module Unified
           method_name = "_#{name}"
         end
 
-        if ["modify_collection"].include?(name.to_s)
+        if ["modify_collection", "list_index_names"].include?(name.to_s)
           skip "Mongo Ruby Driver does not support #{name.to_s}"
         end
 
@@ -348,7 +403,7 @@ module Unified
             end
 
             public_send(method_name, op)
-          rescue Mongo::Error, BSON::String::IllegalKey, ArgumentError => e
+          rescue Mongo::Error, bson_error, Mongo::Auth::Unauthorized, ArgumentError => e
             if expected_error.use('isClientError')
               # isClientError doesn't actually mean a client error.
               # It means anything other than OperationFailure. DRIVERS-1799
@@ -385,9 +440,11 @@ module Unified
                 end
               end
             end
+            if error_response = expected_error.use("errorResponse")
+              assert_result_matches(e.document, error_response)
+            end
             if expected_result = expected_error.use('expectResult')
               assert_result_matches(e.result, expected_result)
-              #expected_result.clear
             # Important: this must be the last branch.
             elsif expected_error.use('isError')
               # Nothing but we consume the key.
@@ -398,6 +455,16 @@ module Unified
           else
             raise Error::ErrorMismatch, "Expected exception but none was raised"
           end
+        elsif op.use('ignoreResultAndError')
+          unless respond_to?(method_name)
+            raise Error::UnsupportedOperation, "Mongo Ruby Driver does not support #{name.to_s}"
+          end
+
+          begin
+            send(method_name, op)
+          # We can possibly rescue more errors here, add as needed.
+          rescue Mongo::Error
+          end
         else
           unless respond_to?(method_name, true)
             raise Error::UnsupportedOperation, "Mongo Ruby Driver does not support #{name.to_s}"
@@ -507,5 +574,14 @@ module Unified
       ).update(opts)
       Mongo::Client.new(*args)
     end
+
+    # The error to rescue BSON tests for. If we still define
+    # BSON::String::IllegalKey then we should rescue that particular error,
+    # otherwise, rescue an arbitrary BSON::Error
+    def bson_error
+      BSON::String.const_defined?(:IllegalKey) ?
+        BSON::String.const_get(:IllegalKey) :
+        BSON::Error
+    end
   end
 end

data/spec/runners/unified/test_group.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 module Unified
 

data/spec/runners/unified/thread_operations.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+# rubocop:todo all
+
+module Unified
+
+  module ThreadOperations
+
+    class ThreadContext
+      def initialize
+        @operations = Queue.new
+      end
+
+      def stop?
+        !!@stop
+      end
+
+      def signal_stop
+        @stop = true
+      end
+
+      attr_reader :operations
+    end
+
+    def wait(op)
+      consume_test_runner(op)
+      use_arguments(op) do |args|
+        sleep args.use!('ms') / 1000.0
+      end
+    end
+
+    def wait_for_event(op)
+      consume_test_runner(op)
+      use_arguments(op) do |args|
+        client = entities.get(:client, args.use!('client'))
+        subscriber = @subscribers.fetch(client)
+        event = args.use!('event')
+        assert_eq(event.keys.length, 1, "Expected event must have one key: #{event}")
+        count = args.use!('count')
+
+        deadline = Mongo::Utils.monotonic_time + 10
+        loop do
+          events = select_events(subscriber, event)
+          if events.length >= count
+            break
+          end
+          if Mongo::Utils.monotonic_time >= deadline
+            raise "Did not receive an event matching #{event} in 10 seconds; received #{events.length} but expected #{count} events"
+          else
+            sleep 0.1
+          end
+        end
+      end
+    end
+
+    def run_on_thread(op)
+      consume_test_runner(op)
+      use_arguments(op) do |args|
+        thread = entities.get(:thread, args.use!('thread'))
+        operation = args.use!('operation')
+        thread.context.operations << operation
+      end
+    end
+
+    def wait_for_thread(op)
+      consume_test_runner(op)
+      use_arguments(op) do |args|
+        thread = entities.get(:thread, args.use!('thread'))
+        thread.context.signal_stop
+        thread.join
+      end
+    end
+  end
+end

data/spec/runners/unified.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 require 'support/using_hash'
 require 'runners/unified/error'
@@ -48,6 +48,9 @@ def define_unified_spec_tests(base_path, paths, expect_failure: false)
             end
           end
 
+          if test.retry?
+            retry_test tries: 3
+          end
           if expect_failure
             it 'fails as expected' do
               if test.group_reqs
@@ -61,7 +64,7 @@ def define_unified_spec_tests(base_path, paths, expect_failure: false)
                 end
               end
               begin
-                test.create_entities
+                test.create_spec_entities
                 test.set_initial_data
                 begin
                   test.run
@@ -69,11 +72,11 @@ def define_unified_spec_tests(base_path, paths, expect_failure: false)
                   test.assert_events
                 # HACK: other errors are possible and likely will need to
                 # be added here later as the tests evolve.
-                rescue Mongo::Error::OperationFailure, Unified::Error::UnsupportedOperation
+                rescue Mongo::Error::OperationFailure, Unified::Error::UnsupportedOperation, UsingHash::UsingHashKeyError, Unified::Error::EntityMissing
                 rescue => e
-                  fail "Expected to raise Mongo::Error::OperationFailure or Unified::Error::UnsupportedOperation, got #{e}"
+                  fail "Expected to raise Mongo::Error::OperationFailure or Unified::Error::UnsupportedOperation or UsingHash::UsingHashKeyError or Unified::Error::EntityMissing, got #{e.class}: #{e}"
                 else
-                  fail "Expected to raise Mongo::Error::OperationFailure or Unified::Error::UnsupportedOperation, but no error was raised"
+                  fail "Expected to raise Mongo::Error::OperationFailure or Unified::Error::UnsupportedOperation or UsingHash::UsingHashKeyError or Unified::Error::EntityMissing, but no error was raised"
                 end
               ensure
                 test.cleanup
@@ -91,7 +94,7 @@ def define_unified_spec_tests(base_path, paths, expect_failure: false)
                   skip "Requirements not satisfied"
                 end
               end
-              test.create_entities
+              test.create_spec_entities
               test.set_initial_data
               test.run
               test.assert_outcome

data/spec/shared/lib/mrss/docker_runner.rb

@@ -231,6 +231,10 @@ module Mrss
       distro =~ /debian|ubuntu/
     end
 
+    def ubuntu?
+      distro=~ /ubuntu/
+    end
+
     def preload?
       !!@options[:preload]
     end

data/spec/shared/lib/mrss/lite_constraints.rb

@@ -98,8 +98,8 @@ module Mrss
     def min_libmongocrypt_version(version)
       require_libmongocrypt
       before(:all) do
-        actual_version = Gem::Version.new(Mongo::Crypt::Binding.mongocrypt_version(nil))
-        min_version = Gem::Version.new(version)
+        actual_version = Utils.parse_version(Mongo::Crypt::Binding.mongocrypt_version(nil))
+        min_version = Utils.parse_version(version)
         unless actual_version >= min_version
           skip "libmongocrypt version #{min_version} required, but version #{actual_version} is available"
         end
@@ -209,6 +209,14 @@ module Mrss
       end
     end
 
+    def require_no_fallbacks
+      before(:all) do
+        if %w(yes true 1).include?((ENV['TEST_I18N_FALLBACKS'] || '').downcase)
+          skip 'Set TEST_I18N_FALLBACKS=0 environment variable to run these tests'
+        end
+      end
+    end
+
     # This is a macro for retrying flaky tests on CI that occasionally fail.
     # Note that the tests will only be retried on CI.
     #

data/spec/shared/lib/mrss/server_version_registry.rb

@@ -24,6 +24,21 @@ module Mrss
 
     attr_reader :desired_version, :arch
 
+    def target_arch
+      # can't use RbConfig::CONFIG["arch"] because JRuby doesn't
+      # return anything meaningful there.
+      #
+      # also, need to use `uname -a` instead of (e.g.) `uname -p`
+      # because debian (at least) does not return anything meaningful
+      # for `uname -p`.
+      uname = `uname -a`.strip
+      @target_arch ||= case uname
+        when /aarch/ then "aarch64"
+        when /x86/   then "x86_64"
+        else raise "unsupported architecture #{uname.inspect}"
+        end
+    end
+
     def download_url
       @download_url ||= begin
         version, version_ok = detect_version(current_catalog)
@@ -40,35 +55,13 @@ module Mrss
         end
         dl = version['downloads'].detect do |dl|
           dl['archive']['url'].index("enterprise-#{arch}") &&
-          dl['arch'] == 'x86_64'
+          dl['arch'] == target_arch
         end
         unless dl
           raise MissingDownloadUrl, "No download for #{arch} for #{version['version']}"
         end
         url = dl['archive']['url']
       end
-    rescue MissingDownloadUrl
-      if %w(2.6 3.0).include?(desired_version) && arch == 'ubuntu1604'
-        # 2.6 and 3.0 are only available for ubuntu1204 and ubuntu1404.
-        # Those ubuntus have ancient Pythons that don't work due to not
-        # implementing recent TLS protocols.
-        # Because of this we test on ubuntu1604 which has a newer Python.
-        # But we still need to retrieve ubuntu1404-targeting builds.
-        url = self.class.new('3.2', arch).download_url
-        unless url.include?('3.2.')
-          raise 'URL not in expected format'
-        end
-        url = case desired_version
-        when '2.6'
-          url.sub(/\b3\.2\.\d+/, '2.6.12')
-        when '3.0'
-          url.sub(/\b3\.2\.\d+/, '3.0.15')
-        else
-          raise NotImplementedError
-        end.sub('ubuntu1604', 'ubuntu1404')
-      else
-        raise
-      end
     end
 
     private

data/spec/shared/lib/mrss/utils.rb

@@ -3,13 +3,35 @@
 
 module Mrss
   module Utils
+    extend self
 
-    module_function def print_backtrace(dest=STDERR)
-      begin
-        hello world
-      rescue => e
-        dest.puts e.backtrace.join("\n")
-      end
+    def print_backtrace(dest=STDERR)
+      raise
+    rescue => e
+      dest.puts e.backtrace.join("\n")
+    end
+
+    # Parses the given version string, accounting for suffix information that
+    # Gem::Version cannot successfully parse.
+    #
+    # @param [ String ] version the version to parse
+    #
+    # @return [ Gem::Version ] the parsed version
+    #
+    # @raise [ ArgumentError ] if the string cannot be parsed.
+    def parse_version(version)
+      Gem::Version.new(version)
+    rescue ArgumentError
+      match = version.match(/\A(?<major>\d+)\.(?<minor>\d+)\.(?<patch>\d+)?(-[A-Za-z\+\d]+)?\z/)
+      raise ArgumentError.new("Malformed version number string #{version}") if match.nil?
+
+      Gem::Version.new(
+        [
+          match[:major],
+          match[:minor],
+          match[:patch]
+        ].join('.')
+      )
     end
   end
 end