mongo 2.12.4 → 2.14.0.rc1

data/spec/runners/transactions.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2019 MongoDB, Inc.
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require 'runners/transactions/context'
 require 'runners/transactions/operation'
 require 'runners/transactions/spec'
 require 'runners/transactions/test'
@@ -24,24 +23,24 @@ def define_transactions_spec_tests(test_paths)
     spec = Mongo::Transactions::Spec.new(file)
 
     context(spec.description) do
-      require_wired_tiger
 
       define_spec_tests_with_requirements(spec) do |req|
+
         spec.tests.each do |test|
 
-          before do
-            if ClusterConfig.instance.topology == :sharded
-              if test.multiple_mongoses? && SpecConfig.instance.addresses.length == 1
-                skip "Test requires multiple mongoses"
-              elsif !test.multiple_mongoses? && SpecConfig.instance.addresses.length > 1
-                # Many transaction spec tests that do not specifically deal with
-                # sharded transactions fail when run against a multi-mongos cluster
-                skip "Test does not specify multiple mongoses"
+          context(test.description) do
+
+            before(:all) do
+              if ClusterConfig.instance.topology == :sharded
+                if test.multiple_mongoses? && SpecConfig.instance.addresses.length == 1
+                  skip "Test requires multiple mongoses"
+                elsif !test.multiple_mongoses? && SpecConfig.instance.addresses.length > 1
+                  # Many transaction spec tests that do not specifically deal with
+                  # sharded transactions fail when run against a multi-mongos cluster
+                  skip "Test does not specify multiple mongoses"
+                end
               end
             end
-          end
-
-          context(test.description) do
 
             if test.skip_reason
               before(:all) do

data/spec/runners/transactions/operation.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2019 MongoDB, Inc.
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,23 +17,12 @@ module Mongo
     class Operation < Mongo::CRUD::Operation
       include RSpec::Matchers
 
-      def execute(target, session0, session1, active_session=nil)
-        session = case arguments && arguments['session']
-        when 'session0'
-          session0
-        when 'session1'
-          session1
-        else
-          # active session could be nil
-          active_session
-        end
-
-        context = Context.new(
-          session0,
-          session1,
-          session)
+      def needs_session?
+        arguments && arguments['session'] || object =~ /session/
+      end
 
-        op_name = Utils.underscore(name).to_sym
+      def execute(target, context)
+        op_name = ::Utils.underscore(name).to_sym
         if op_name == :with_transaction
           args = [target]
         else
@@ -49,9 +38,14 @@ module Mongo
             result['error'] = false
           end
         end
+
         result
       rescue Mongo::Error::OperationFailure => e
-        err_doc = e.instance_variable_get(:@result).send(:first_document)
+        result = e.instance_variable_get(:@result)
+        if result.nil?
+          raise "OperationFailure had nil result: #{e}"
+        end
+        err_doc = result.send(:first_document)
         error_code_name = err_doc['codeName'] || err_doc['writeConcernError'] && err_doc['writeConcernError']['codeName']
         if error_code_name.nil?
           # Sometimes the server does not return the error code name,
@@ -98,13 +92,13 @@ module Mongo
         command_value = cmd.delete(command_name)
         cmd = { command_name.to_sym => command_value }.merge(cmd)
 
-        opts = Utils.snakeize_hash(context.transform_arguments(options)).dup
+        opts = ::Utils.snakeize_hash(transformed_options(context)).dup
         opts[:read] = opts.delete(:read_preference)
         database.command(cmd, opts).documents.first
       end
 
       def start_transaction(session, context)
-        session.start_transaction(Utils.convert_operation_options(arguments['options']))
+        session.start_transaction(::Utils.convert_operation_options(arguments['options']))
         nil
       end
 
@@ -124,7 +118,7 @@ module Mongo
         end
 
         if arguments['options']
-          options = Utils.snakeize_hash(arguments['options'])
+          options = ::Utils.snakeize_hash(arguments['options'])
         else
           options = nil
         end
@@ -132,7 +126,7 @@ module Mongo
           callback['operations'].each do |op_spec|
             op = Operation.new(@crud_test, op_spec)
             target = @crud_test.resolve_target(@crud_test.test_client, op)
-            rv = op.execute(target, context.session0, context.session1, session)
+            rv = op.execute(target, context)
             if rv && rv['exception']
               raise rv['exception']
             end
@@ -147,7 +141,7 @@ module Mongo
       end
 
       def targeted_fail_point(collection, context)
-        args = context.transform_arguments(options)
+        args = transformed_options(context)
         session = args[:session]
         unless session.pinned_server
           raise ArgumentError, 'Targeted fail point requires session to be pinned to a server'
@@ -165,7 +159,7 @@ module Mongo
       end
 
       def assert_session_pinned(collection, context)
-        args = context.transform_arguments(options)
+        args = transformed_options(context)
         session = args[:session]
         unless session.pinned_server
           raise ArgumentError, 'Expected session to be pinned'
@@ -173,12 +167,148 @@ module Mongo
       end
 
       def assert_session_unpinned(collection, context)
-        args = context.transform_arguments(options)
+        args = transformed_options(context)
         session = args[:session]
         if session.pinned_server
           raise ArgumentError, 'Expected session to not be pinned'
         end
       end
+
+      def wait_for_event(client, context)
+        deadline = Time.now + 5
+        loop do
+          events = _select_events(context)
+          if events.length >= arguments['count']
+            break
+          end
+          if Time.now >= deadline
+            raise "Did not receive an event matching #{arguments} in 5 seconds; received #{events.length} but expected #{arguments['count']} events"
+          else
+            sleep 0.1
+          end
+        end
+      end
+
+      def assert_event_count(client, context)
+        events = _select_events(context)
+        unless events.length == arguments['count']
+          raise "Exppected #{arguments['count']} #{arguments['event']} events, but have #{events.length}"
+        end
+      end
+
+      def _select_events(context)
+        case arguments['event']
+        when 'ServerMarkedUnknownEvent'
+          context.sdam_subscriber.all_events.select do |event|
+            event.is_a?(Mongo::Monitoring::Event::ServerDescriptionChanged) &&
+            event.new_description.unknown?
+          end
+        else
+          context.sdam_subscriber.all_events.select do |event|
+            event.class.name.sub(/.*::/, '') == arguments['event'].sub(/Event$/, '')
+          end
+        end
+      end
+
+      class ThreadContext
+        def initialize
+          @operations = Queue.new
+          @unexpected_operation_results = []
+        end
+
+        def stop?
+          !!@stop
+        end
+
+        def signal_stop
+          @stop = true
+        end
+
+        attr_reader :operations
+        attr_reader :unexpected_operation_results
+      end
+
+      def start_thread(client, context)
+        thread_context = ThreadContext.new
+        thread = Thread.new do
+          loop do
+            begin
+              op_spec = thread_context.operations.pop(true)
+              op = Operation.new(@crud_test, op_spec)
+              target = @crud_test.resolve_target(@crud_test.test_client, op)
+              result = op.execute(target, context)
+              if op_spec['error']
+                unless result['error']
+                  thread_context.unexpected_operation_results << result
+                end
+              else
+                if result['error']
+                  thread_context.unexpected_operation_results << result
+                end
+              end
+            rescue ThreadError
+              # Queue is empty
+            end
+            if thread_context.stop?
+              break
+            else
+              sleep 1
+            end
+          end
+        end
+        class << thread
+          attr_accessor :context
+        end
+        thread.context = thread_context
+        unless context.threads
+          context.threads ||= {}
+        end
+        context.threads[arguments['name']] = thread
+      end
+
+      def run_on_thread(client, context)
+        thread = context.threads.fetch(arguments['name'])
+        thread.context.operations << arguments['operation']
+      end
+
+      def wait_for_thread(client, context)
+        thread = context.threads.fetch(arguments['name'])
+        thread.context.signal_stop
+        thread.join
+        unless thread.context.unexpected_operation_results.empty?
+          raise "Thread #{arguments['name']} had #{thread.context.unexpected_operation_results}.length unexpected operation results"
+        end
+      end
+
+      def wait(client, context)
+        sleep arguments['ms'] / 1000.0
+      end
+
+      def record_primary(client, context)
+        context.primary_address = client.cluster.next_primary.address
+      end
+
+      def run_admin_command(support_client, context)
+        support_client.use('admin').database.command(arguments['command'])
+      end
+
+      def wait_for_primary_change(client, context)
+        timeout = if arguments['timeoutMS']
+          arguments['timeoutMS'] / 1000.0
+        else
+          10
+        end
+        deadline = Time.now + timeout
+        loop do
+          client.cluster.scan!
+          if client.cluster.next_primary.address != context.primary_address
+            break
+          end
+          if Time.now >= deadline
+            raise "Failed to change primary in #{timeout} seconds"
+          end
+        end
+      end
     end
   end
 end

data/spec/runners/transactions/spec.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2019 MongoDB, Inc.
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/spec/runners/transactions/test.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2019 MongoDB, Inc.
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,18 +20,6 @@ module Mongo
     # @since 2.6.0
     class TransactionsTest < CRUD::CRUDTestBase
 
-      # The test description.
-      #
-      # @return [ String ] description The test description.
-      #
-      # @since 2.6.0
-      attr_reader :description
-
-      # The expected command monitoring events
-      #
-      # @since 2.6.0
-      attr_reader :expectations
-
       attr_reader :expected_results
       attr_reader :skip_reason
 
@@ -53,20 +41,29 @@ module Mongo
         @spec = crud_spec
         @data = data
         @description = test['description']
-        @client_options = Utils.convert_client_options(test['clientOptions'] || {})
+        @client_options = {
+          # Disable legacy read & write retries, so that when spec tests
+          # disable modern retries we do not retry at all instead of using
+          # legacy retries which is contrary to what the tests want.
+          max_read_retries: 0,
+          max_write_retries: 0,
+          app_name: 'Tx spec - test client',
+        }.update(::Utils.convert_client_options(test['clientOptions'] || {}))
+
+        @fail_point_command = test['failPoint']
+
         @session_options = if opts = test['sessionOptions']
           Hash[opts.map do |session_name, options|
-            [session_name.to_sym, Utils.convert_operation_options(options)]
+            [session_name.to_sym, ::Utils.convert_operation_options(options)]
           end]
         else
           {}
         end
-        @fail_point = test['failPoint']
         @skip_reason = test['skipReason']
         @multiple_mongoses = test['useMultipleMongoses']
 
-        @operations = test['operations']
-        @ops = @operations.map do |op|
+        operations = test['operations']
+        @operations = operations.map do |op|
           Operation.new(self, op)
         end
 
@@ -76,7 +73,7 @@ module Mongo
           @outcome = Mongo::CRUD::Outcome.new(BSON::ExtJSON.parse_obj(test['outcome'], mode: :bson))
         end
 
-        @expected_results = @operations.map do |o|
+        @expected_results = operations.map do |o|
           o = BSON::ExtJSON.parse_obj(o)
 
           # We check both o.key('error') and o['error'] to provide a better
@@ -117,30 +114,35 @@ module Mongo
       end
 
       def test_client
-        @test_client ||= ClientRegistry.instance.global_client(
-          'authorized_without_retry_writes'
-        ).with(@client_options.merge(
-          database: @spec.database_name,
-        ))
+        @test_client ||= begin
+          sdam_proc = lambda do |test_client|
+            test_client.subscribe(Mongo::Monitoring::COMMAND, command_subscriber)
+
+            test_client.subscribe(Mongo::Monitoring::TOPOLOGY_OPENING, sdam_subscriber)
+            test_client.subscribe(Mongo::Monitoring::SERVER_OPENING, sdam_subscriber)
+            test_client.subscribe(Mongo::Monitoring::SERVER_DESCRIPTION_CHANGED, sdam_subscriber)
+            test_client.subscribe(Mongo::Monitoring::TOPOLOGY_CHANGED, sdam_subscriber)
+            test_client.subscribe(Mongo::Monitoring::SERVER_CLOSED, sdam_subscriber)
+            test_client.subscribe(Mongo::Monitoring::TOPOLOGY_CLOSED, sdam_subscriber)
+            test_client.subscribe(Mongo::Monitoring::CONNECTION_POOL, sdam_subscriber)
+          end
+
+          ClientRegistry.instance.new_local_client(
+            SpecConfig.instance.addresses,
+            SpecConfig.instance.authorized_test_options.merge(
+              database: @spec.database_name,
+              auth_source: SpecConfig.instance.auth_options[:auth_source] || 'admin',
+              sdam_proc: sdam_proc,
+            ).merge(@client_options))
+        end
       end
 
-      def event_subscriber
-        @event_subscriber ||= EventSubscriber.new
+      def command_subscriber
+        @command_subscriber ||= EventSubscriber.new
       end
 
-      def mongos_each_direct_client
-        if ClusterConfig.instance.topology == :sharded
-          client = ClientRegistry.instance.global_client('basic')
-          client.cluster.next_primary
-          client.cluster.servers.each do |server|
-            direct_client = ClientRegistry.instance.new_local_client(
-              [server.address.to_s],
-              SpecConfig.instance.test_options.merge(
-                connect: :sharded
-              ).merge(SpecConfig.instance.auth_options))
-            yield direct_client
-          end
-        end
+      def sdam_subscriber
+        @sdam_subscriber ||= EventSubscriber.new(name: 'sdam subscriber')
       end
 
       # Run the test.
@@ -152,20 +154,43 @@ module Mongo
       #
       # @since 2.6.0
       def run
-        test_client.subscribe(Mongo::Monitoring::COMMAND, event_subscriber)
+        @threads = {}
 
-        results = @ops.map do |op|
+        results = @operations.map do |op|
           target = resolve_target(test_client, op)
-          op.execute(target, @session0, @session1)
+          if op.needs_session?
+            context = CRUD::Context.new(
+              session0: session0,
+              session1: session1,
+              sdam_subscriber: sdam_subscriber,
+              threads: @threads,
+              primary_address: @primary_address,
+            )
+          else
+            # Hack to support write concern operations tests, which are
+            # defined to use transactions format but target pre-3.6 servers
+            # that do not support sessions
+            target ||= support_client
+            context = CRUD::Context.new(
+              sdam_subscriber: sdam_subscriber,
+              threads: @threads,
+              primary_address: @primary_address,
+            )
+          end
+
+          op.execute(target, context).tap do
+            @threads = context.threads
+            @primary_address = context.primary_address
+          end
         end
 
-        session0_id = @session0.session_id
-        session1_id = @session1.session_id
+        session0_id = @session0&.session_id
+        session1_id = @session1&.session_id
 
-        @session0.end_session
-        @session1.end_session
+        @session0&.end_session
+        @session1&.end_session
 
-        actual_events = Utils.yamlify_command_events(event_subscriber.started_events)
+        actual_events = ::Utils.yamlify_command_events(command_subscriber.started_events)
         actual_events = actual_events.reject do |event|
           event['command_started_event']['command']['endSessions']
         end
@@ -173,8 +198,12 @@ module Mongo
 
           # Replace the session id placeholders with the actual session ids.
           payload = e['command_started_event']
-          payload['command']['lsid'] = 'session0' if payload['command']['lsid'] == session0_id
-          payload['command']['lsid'] = 'session1' if payload['command']['lsid'] == session1_id
+          if @session0
+            payload['command']['lsid'] = 'session0' if payload['command']['lsid'] == session0_id
+          end
+          if @session1
+            payload['command']['lsid'] = 'session1' if payload['command']['lsid'] == session1_id
+          end
 
         end
 
@@ -183,7 +212,7 @@ module Mongo
           contents: @result_collection.with(
             read: {mode: 'primary'},
             read_concern: { level: 'local' },
-          ).find.to_a,
+          ).find.sort(_id: 1).to_a,
           events: actual_events,
         }
       end
@@ -194,8 +223,10 @@ module Mongo
         rescue Mongo::Error
         end
 
-        mongos_each_direct_client do |direct_client|
-          direct_client.command(configureFailPoint: 'failCommand', mode: 'off')
+        if ClusterConfig.instance.fcv_ish >= '4.2'
+          mongos_each_direct_client do |direct_client|
+            direct_client.command(configureFailPoint: 'failCommand', mode: 'off')
+          end
         end
 
         # Insert data into the key vault collection if required to do so by
@@ -223,40 +254,39 @@ module Mongo
         support_client.command(
           create: @spec.collection_name,
           validator: collection_validator,
-          writeConcern: { w: :majority }
+          writeConcern: { w: 'majority' }
         )
 
         coll.insert_many(@data) unless @data.empty?
 
-        $distinct_ran ||= if description =~ /distinct/ || @ops.any? { |op| op.name == 'distinct' }
+        $distinct_ran ||= {}
+        $distinct_ran[@spec.database_name] ||= if description =~ /distinct/ || @operations.any? { |op| op.name == 'distinct' }
           mongos_each_direct_client do |direct_client|
-            direct_client['test'].distinct('foo').to_a
+            direct_client.use(@spec.database_name)['test'].distinct('foo').to_a
           end
         end
 
-        admin_support_client.command(@fail_point) if @fail_point
+        admin_support_client.command(@fail_point_command) if @fail_point_command
 
         @collection = test_client[@spec.collection_name]
 
         # Client-side encryption tests require the use of a separate client
         # without auto_encryption_options for querying results.
-        @result_collection = support_client.use(@spec.database_name)[@spec.collection_name]
-
-        @session0 = test_client.start_session(@session_options[:session0] || {})
-        @session1 = test_client.start_session(@session_options[:session1] || {})
+        result_collection_name = outcome&.collection_name || @spec.collection_name
+        @result_collection = support_client.use(@spec.database_name)[result_collection_name]
       end
 
       def teardown_test
 
-        if @fail_point
+        if @fail_point_command
           admin_support_client.command(configureFailPoint: 'failCommand', mode: 'off')
         end
 
         if $disable_fail_points
-          $disable_fail_points.each do |(fail_point, address)|
+          $disable_fail_points.each do |(fail_point_command, address)|
             client = ClusterTools.instance.direct_client(address,
               database: 'admin')
-            client.command(configureFailPoint: fail_point['configureFailPoint'],
+            client.command(configureFailPoint: fail_point_command['configureFailPoint'],
               mode: 'off')
           end
         end
@@ -269,9 +299,9 @@ module Mongo
       def resolve_target(client, operation)
         case operation.object
         when 'session0'
-          @session0
+          session0
         when 'session1'
-          @session1
+          session1
         when 'testRunner'
           # We don't actually use this target in any way.
           nil
@@ -279,6 +309,14 @@ module Mongo
           super
         end
       end
+
+      def session0
+        @session0 ||= test_client.start_session(@session_options[:session0] || {})
+      end
+
+      def session1
+        @session1 ||= test_client.start_session(@session_options[:session1] || {})
+      end
     end
   end
 end