mongo 2.13.0.beta1 → 2.13.0.rc1

data/spec/runners/transactions.rb

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require 'runners/transactions/context'
 require 'runners/transactions/operation'
 require 'runners/transactions/spec'
 require 'runners/transactions/test'
@@ -26,21 +25,22 @@ def define_transactions_spec_tests(test_paths)
     context(spec.description) do
 
       define_spec_tests_with_requirements(spec) do |req|
+
         spec.tests.each do |test|
 
-          before do
-            if ClusterConfig.instance.topology == :sharded
-              if test.multiple_mongoses? && SpecConfig.instance.addresses.length == 1
-                skip "Test requires multiple mongoses"
-              elsif !test.multiple_mongoses? && SpecConfig.instance.addresses.length > 1
-                # Many transaction spec tests that do not specifically deal with
-                # sharded transactions fail when run against a multi-mongos cluster
-                skip "Test does not specify multiple mongoses"
+          context(test.description) do
+
+            before(:all) do
+              if ClusterConfig.instance.topology == :sharded
+                if test.multiple_mongoses? && SpecConfig.instance.addresses.length == 1
+                  skip "Test requires multiple mongoses"
+                elsif !test.multiple_mongoses? && SpecConfig.instance.addresses.length > 1
+                  # Many transaction spec tests that do not specifically deal with
+                  # sharded transactions fail when run against a multi-mongos cluster
+                  skip "Test does not specify multiple mongoses"
+                end
               end
             end
-          end
-
-          context(test.description) do
 
             if test.skip_reason
               before(:all) do

data/spec/runners/transactions/operation.rb

@@ -21,23 +21,8 @@ module Mongo
         arguments && arguments['session'] || object =~ /session/
       end
 
-      def execute(target, session0, session1, active_session=nil)
-        session = case arguments && arguments['session']
-        when 'session0'
-          session0
-        when 'session1'
-          session1
-        else
-          # active session could be nil
-          active_session
-        end
-
-        context = Context.new(
-          session0,
-          session1,
-          session)
-
-        op_name = Utils.underscore(name).to_sym
+      def execute(target, context)
+        op_name = ::Utils.underscore(name).to_sym
         if op_name == :with_transaction
           args = [target]
         else
@@ -53,9 +38,14 @@ module Mongo
             result['error'] = false
           end
         end
+
         result
       rescue Mongo::Error::OperationFailure => e
-        err_doc = e.instance_variable_get(:@result).send(:first_document)
+        result = e.instance_variable_get(:@result)
+        if result.nil?
+          raise "OperationFailure had nil result: #{e}"
+        end
+        err_doc = result.send(:first_document)
         error_code_name = err_doc['codeName'] || err_doc['writeConcernError'] && err_doc['writeConcernError']['codeName']
         if error_code_name.nil?
           # Sometimes the server does not return the error code name,
@@ -102,13 +92,13 @@ module Mongo
         command_value = cmd.delete(command_name)
         cmd = { command_name.to_sym => command_value }.merge(cmd)
 
-        opts = Utils.snakeize_hash(context.transform_arguments(options)).dup
+        opts = ::Utils.snakeize_hash(transformed_options(context)).dup
         opts[:read] = opts.delete(:read_preference)
         database.command(cmd, opts).documents.first
       end
 
       def start_transaction(session, context)
-        session.start_transaction(Utils.convert_operation_options(arguments['options']))
+        session.start_transaction(::Utils.convert_operation_options(arguments['options']))
         nil
       end
 
@@ -128,7 +118,7 @@ module Mongo
         end
 
         if arguments['options']
-          options = Utils.snakeize_hash(arguments['options'])
+          options = ::Utils.snakeize_hash(arguments['options'])
         else
           options = nil
         end
@@ -136,7 +126,7 @@ module Mongo
           callback['operations'].each do |op_spec|
             op = Operation.new(@crud_test, op_spec)
             target = @crud_test.resolve_target(@crud_test.test_client, op)
-            rv = op.execute(target, context.session0, context.session1, session)
+            rv = op.execute(target, context)
             if rv && rv['exception']
               raise rv['exception']
             end
@@ -151,7 +141,7 @@ module Mongo
       end
 
       def targeted_fail_point(collection, context)
-        args = context.transform_arguments(options)
+        args = transformed_options(context)
         session = args[:session]
         unless session.pinned_server
           raise ArgumentError, 'Targeted fail point requires session to be pinned to a server'
@@ -169,7 +159,7 @@ module Mongo
       end
 
       def assert_session_pinned(collection, context)
-        args = context.transform_arguments(options)
+        args = transformed_options(context)
         session = args[:session]
         unless session.pinned_server
           raise ArgumentError, 'Expected session to be pinned'
@@ -177,12 +167,148 @@ module Mongo
       end
 
       def assert_session_unpinned(collection, context)
-        args = context.transform_arguments(options)
+        args = transformed_options(context)
         session = args[:session]
         if session.pinned_server
           raise ArgumentError, 'Expected session to not be pinned'
         end
       end
+
+      def wait_for_event(client, context)
+        deadline = Time.now + 5
+        loop do
+          events = _select_events(context)
+          if events.length >= arguments['count']
+            break
+          end
+          if Time.now >= deadline
+            raise "Did not receive an event matching #{arguments} in 5 seconds; received #{events.length} but expected #{arguments['count']} events"
+          else
+            sleep 0.1
+          end
+        end
+      end
+
+      def assert_event_count(client, context)
+        events = _select_events(context)
+        unless events.length == arguments['count']
+          raise "Exppected #{arguments['count']} #{arguments['event']} events, but have #{events.length}"
+        end
+      end
+
+      def _select_events(context)
+        case arguments['event']
+        when 'ServerMarkedUnknownEvent'
+          context.sdam_subscriber.all_events.select do |event|
+            event.is_a?(Mongo::Monitoring::Event::ServerDescriptionChanged) &&
+            event.new_description.unknown?
+          end
+        else
+          context.sdam_subscriber.all_events.select do |event|
+            event.class.name.sub(/.*::/, '') == arguments['event'].sub(/Event$/, '')
+          end
+        end
+      end
+
+      class ThreadContext
+        def initialize
+          @operations = Queue.new
+          @unexpected_operation_results = []
+        end
+
+        def stop?
+          !!@stop
+        end
+
+        def signal_stop
+          @stop = true
+        end
+
+        attr_reader :operations
+        attr_reader :unexpected_operation_results
+      end
+
+      def start_thread(client, context)
+        thread_context = ThreadContext.new
+        thread = Thread.new do
+          loop do
+            begin
+              op_spec = thread_context.operations.pop(true)
+              op = Operation.new(@crud_test, op_spec)
+              target = @crud_test.resolve_target(@crud_test.test_client, op)
+              result = op.execute(target, context)
+              if op_spec['error']
+                unless result['error']
+                  thread_context.unexpected_operation_results << result
+                end
+              else
+                if result['error']
+                  thread_context.unexpected_operation_results << result
+                end
+              end
+            rescue ThreadError
+              # Queue is empty
+            end
+            if thread_context.stop?
+              break
+            else
+              sleep 1
+            end
+          end
+        end
+        class << thread
+          attr_accessor :context
+        end
+        thread.context = thread_context
+        unless context.threads
+          context.threads ||= {}
+        end
+        context.threads[arguments['name']] = thread
+      end
+
+      def run_on_thread(client, context)
+        thread = context.threads.fetch(arguments['name'])
+        thread.context.operations << arguments['operation']
+      end
+
+      def wait_for_thread(client, context)
+        thread = context.threads.fetch(arguments['name'])
+        thread.context.signal_stop
+        thread.join
+        unless thread.context.unexpected_operation_results.empty?
+          raise "Thread #{arguments['name']} had #{thread.context.unexpected_operation_results}.length unexpected operation results"
+        end
+      end
+
+      def wait(client, context)
+        sleep arguments['ms'] / 1000.0
+      end
+
+      def record_primary(client, context)
+        context.primary_address = client.cluster.next_primary.address
+      end
+
+      def run_admin_command(support_client, context)
+        support_client.use('admin').database.command(arguments['command'])
+      end
+
+      def wait_for_primary_change(client, context)
+        timeout = if arguments['timeoutMS']
+          arguments['timeoutMS'] / 1000.0
+        else
+          10
+        end
+        deadline = Time.now + timeout
+        loop do
+          client.cluster.scan!
+          if client.cluster.next_primary.address != context.primary_address
+            break
+          end
+          if Time.now >= deadline
+            raise "Failed to change primary in #{timeout} seconds"
+          end
+        end
+      end
     end
   end
 end

data/spec/runners/transactions/test.rb

@@ -41,13 +41,20 @@ module Mongo
         @spec = crud_spec
         @data = data
         @description = test['description']
-        @client_options = Utils.convert_client_options(test['clientOptions'] || {})
+        @client_options = {
+          # Disable legacy read & write retries, so that when spec tests
+          # disable modern retries we do not retry at all instead of using
+          # legacy retries which is contrary to what the tests want.
+          max_read_retries: 0,
+          max_write_retries: 0,
+          app_name: 'Tx spec - test client',
+        }.update(::Utils.convert_client_options(test['clientOptions'] || {}))
 
         @fail_point_command = test['failPoint']
 
         @session_options = if opts = test['sessionOptions']
           Hash[opts.map do |session_name, options|
-            [session_name.to_sym, Utils.convert_operation_options(options)]
+            [session_name.to_sym, ::Utils.convert_operation_options(options)]
           end]
         else
           {}
@@ -107,15 +114,35 @@ module Mongo
       end
 
       def test_client
-        @test_client ||= ClientRegistry.instance.global_client(
-          'authorized_without_retry_writes'
-        ).with(@client_options.merge(
-          database: @spec.database_name,
-        ))
+        @test_client ||= begin
+          sdam_proc = lambda do |test_client|
+            test_client.subscribe(Mongo::Monitoring::COMMAND, command_subscriber)
+
+            test_client.subscribe(Mongo::Monitoring::TOPOLOGY_OPENING, sdam_subscriber)
+            test_client.subscribe(Mongo::Monitoring::SERVER_OPENING, sdam_subscriber)
+            test_client.subscribe(Mongo::Monitoring::SERVER_DESCRIPTION_CHANGED, sdam_subscriber)
+            test_client.subscribe(Mongo::Monitoring::TOPOLOGY_CHANGED, sdam_subscriber)
+            test_client.subscribe(Mongo::Monitoring::SERVER_CLOSED, sdam_subscriber)
+            test_client.subscribe(Mongo::Monitoring::TOPOLOGY_CLOSED, sdam_subscriber)
+            test_client.subscribe(Mongo::Monitoring::CONNECTION_POOL, sdam_subscriber)
+          end
+
+          ClientRegistry.instance.new_local_client(
+            SpecConfig.instance.addresses,
+            SpecConfig.instance.authorized_test_options.merge(
+              database: @spec.database_name,
+              auth_source: SpecConfig.instance.auth_options[:auth_source] || 'admin',
+              sdam_proc: sdam_proc,
+            ).merge(@client_options))
+        end
       end
 
-      def event_subscriber
-        @event_subscriber ||= EventSubscriber.new
+      def command_subscriber
+        @command_subscriber ||= EventSubscriber.new
+      end
+
+      def sdam_subscriber
+        @sdam_subscriber ||= EventSubscriber.new(name: 'sdam subscriber')
       end
 
       # Run the test.
@@ -127,17 +154,33 @@ module Mongo
       #
       # @since 2.6.0
       def run
-        test_client.subscribe(Mongo::Monitoring::COMMAND, event_subscriber)
+        @threads = {}
 
         results = @operations.map do |op|
           target = resolve_target(test_client, op)
           if op.needs_session?
-            op.execute(target, session0, session1)
+            context = CRUD::Context.new(
+              session0: session0,
+              session1: session1,
+              sdam_subscriber: sdam_subscriber,
+              threads: @threads,
+              primary_address: @primary_address,
+            )
           else
             # Hack to support write concern operations tests, which are
             # defined to use transactions format but target pre-3.6 servers
             # that do not support sessions
-            op.execute(target || support_client, nil, nil)
+            target ||= support_client
+            context = CRUD::Context.new(
+              sdam_subscriber: sdam_subscriber,
+              threads: @threads,
+              primary_address: @primary_address,
+            )
+          end
+
+          op.execute(target, context).tap do
+            @threads = context.threads
+            @primary_address = context.primary_address
           end
         end
 
@@ -147,7 +190,7 @@ module Mongo
         @session0&.end_session
         @session1&.end_session
 
-        actual_events = Utils.yamlify_command_events(event_subscriber.started_events)
+        actual_events = ::Utils.yamlify_command_events(command_subscriber.started_events)
         actual_events = actual_events.reject do |event|
           event['command_started_event']['command']['endSessions']
         end
@@ -169,7 +212,7 @@ module Mongo
           contents: @result_collection.with(
             read: {mode: 'primary'},
             read_concern: { level: 'local' },
-          ).find.to_a,
+          ).find.sort(_id: 1).to_a,
           events: actual_events,
         }
       end
@@ -216,9 +259,10 @@ module Mongo
 
         coll.insert_many(@data) unless @data.empty?
 
-        $distinct_ran ||= if description =~ /distinct/ || @operations.any? { |op| op.name == 'distinct' }
+        $distinct_ran ||= {}
+        $distinct_ran[@spec.database_name] ||= if description =~ /distinct/ || @operations.any? { |op| op.name == 'distinct' }
           mongos_each_direct_client do |direct_client|
-            direct_client['test'].distinct('foo').to_a
+            direct_client.use(@spec.database_name)['test'].distinct('foo').to_a
           end
         end
 

data/spec/spec_tests/command_monitoring_spec.rb

@@ -20,7 +20,7 @@ describe 'Command Monitoring Events' do
         end
 
         let(:subscriber) do
-          Mongo::CommandMonitoring::TestSubscriber.new
+          EventSubscriber.new
         end
 
         let(:monitoring) do
@@ -32,25 +32,35 @@ describe 'Command Monitoring Events' do
           authorized_client.subscribe(Mongo::Monitoring::COMMAND, subscriber)
         end
 
-        after do
-          monitoring.subscribers[Mongo::Monitoring::COMMAND].delete(subscriber)
-          authorized_collection.find.delete_many
-        end
-
-        test.expectations.each do |expectation|
+        test.expectations.each_with_index do |expectation, index|
 
           it "generates a #{expectation.event_name} for #{expectation.command_name}" do
             begin
-              test.run(authorized_collection)
-              event = subscriber.send(expectation.event_type)[expectation.command_name]
-              expect(event).to send(expectation.matcher, expectation)
+              test.run(authorized_collection, subscriber)
+              check_event(subscriber, index, expectation)
             rescue Mongo::Error::OperationFailure, Mongo::Error::BulkWriteError
-              event = subscriber.send(expectation.event_type)[expectation.command_name]
-              expect(event).to send(expectation.matcher, expectation)
+              check_event(subscriber, index, expectation)
             end
           end
         end
       end
     end
   end
+
+  def check_event(subscriber, index, expectation)
+    subscriber.all_events.length.should > index
+    # TODO move this filtering into EventSubscriber
+    events = subscriber.all_events.reject do |event|
+      (
+        event.is_a?(Mongo::Monitoring::Event::CommandStarted) ||
+        event.is_a?(Mongo::Monitoring::Event::CommandSucceeded) ||
+        event.is_a?(Mongo::Monitoring::Event::CommandFailed)
+      ) &&
+      %w(authenticate getnonce saslStart saslContinue).include?(event.command_name)
+    end
+    actual_event = events[index]
+    expected_event_type = expectation.event_type.sub(/_event$/, '')
+    Utils.underscore(actual_event.class.name.sub(/.*::/, '')).to_s.should == expected_event_type
+    expect(actual_event).to send(expectation.matcher, expectation)
+  end
 end