mongo 2.4.1 → 2.14.0

data/spec/runners/transactions/operation.rb

@@ -0,0 +1,314 @@
+# Copyright (C) 2014-2020 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  module Transactions
+    class Operation < Mongo::CRUD::Operation
+      include RSpec::Matchers
+
+      def needs_session?
+        arguments && arguments['session'] || object =~ /session/
+      end
+
+      def execute(target, context)
+        op_name = ::Utils.underscore(name).to_sym
+        if op_name == :with_transaction
+          args = [target]
+        else
+          args = []
+        end
+        if op_name.nil?
+          raise "Unknown operation #{name}"
+        end
+        result = send(op_name, target, context, *args)
+        if result
+          if result.is_a?(Hash)
+            result = result.dup
+            result['error'] = false
+          end
+        end
+
+        result
+      rescue Mongo::Error::OperationFailure => e
+        result = e.instance_variable_get(:@result)
+        if result.nil?
+          raise "OperationFailure had nil result: #{e}"
+        end
+        err_doc = result.send(:first_document)
+        error_code_name = err_doc['codeName'] || err_doc['writeConcernError'] && err_doc['writeConcernError']['codeName']
+        if error_code_name.nil?
+          # Sometimes the server does not return the error code name,
+          # but does return the error code (or we can parse the error code
+          # out of the message).
+          # https://jira.mongodb.org/browse/SERVER-39706
+          warn "Error without error code name: #{e.code}"
+        end
+
+        {
+          'errorCode' => e.code,
+          'errorCodeName' => e.code_name,
+          'errorContains' => e.message,
+          'errorLabels' => e.labels,
+          'exception' => e,
+          'error' => true,
+        }
+      rescue Mongo::Error => e
+        {
+          'errorContains' => e.message,
+          'errorLabels' => e.labels,
+          'exception' => e,
+          'error' => true,
+        }
+      # We do not have a base class for client side BSON-related errors.
+      # See https://jira.mongodb.org/browse/RUBY-1806.
+      # Rescue this particular exception for the time being.
+      rescue BSON::String::IllegalKey => e
+        {
+          'exception' => e,
+          'clientError' => true,
+          'error' => true,
+        }
+      end
+
+      private
+
+      # operations
+
+      def run_command(database, context)
+        # Convert the first key (i.e. the command name) to a symbol.
+        cmd = arguments['command'].dup
+        command_name = cmd.first.first
+        command_value = cmd.delete(command_name)
+        cmd = { command_name.to_sym => command_value }.merge(cmd)
+
+        opts = ::Utils.snakeize_hash(transformed_options(context)).dup
+        opts[:read] = opts.delete(:read_preference)
+        database.command(cmd, opts).documents.first
+      end
+
+      def start_transaction(session, context)
+        session.start_transaction(::Utils.convert_operation_options(arguments['options']))
+        nil
+      end
+
+      def commit_transaction(session, context)
+        session.commit_transaction
+        nil
+      end
+
+      def abort_transaction(session, context)
+        session.abort_transaction
+        nil
+      end
+
+      def with_transaction(session, context, collection)
+        unless callback = arguments['callback']
+          raise ArgumentError, 'with_transaction requires a callback to be present'
+        end
+
+        if arguments['options']
+          options = ::Utils.snakeize_hash(arguments['options'])
+        else
+          options = nil
+        end
+        session.with_transaction(options) do
+          callback['operations'].each do |op_spec|
+            op = Operation.new(@crud_test, op_spec)
+            target = @crud_test.resolve_target(@crud_test.test_client, op)
+            rv = op.execute(target, context)
+            if rv && rv['exception']
+              raise rv['exception']
+            end
+          end
+        end
+      end
+
+      def assert_session_transaction_state(collection, context)
+        session = context.send(arguments['session'])
+        actual_state = session.instance_variable_get('@state').to_s.sub(/^transaction_|_transaction$/, '').sub(/^no$/, 'none')
+        expect(actual_state).to eq(arguments['state'])
+      end
+
+      def targeted_fail_point(collection, context)
+        args = transformed_options(context)
+        session = args[:session]
+        unless session.pinned_server
+          raise ArgumentError, 'Targeted fail point requires session to be pinned to a server'
+        end
+
+        client = ClusterTools.instance.direct_client(session.pinned_server.address,
+          database: 'admin')
+        client.command(arguments['failPoint'])
+
+        $disable_fail_points ||= []
+        $disable_fail_points << [
+          arguments['failPoint'],
+          session.pinned_server.address,
+        ]
+      end
+
+      def assert_session_pinned(collection, context)
+        args = transformed_options(context)
+        session = args[:session]
+        unless session.pinned_server
+          raise ArgumentError, 'Expected session to be pinned'
+        end
+      end
+
+      def assert_session_unpinned(collection, context)
+        args = transformed_options(context)
+        session = args[:session]
+        if session.pinned_server
+          raise ArgumentError, 'Expected session to not be pinned'
+        end
+      end
+
+      def wait_for_event(client, context)
+        deadline = Time.now + 5
+        loop do
+          events = _select_events(context)
+          if events.length >= arguments['count']
+            break
+          end
+          if Time.now >= deadline
+            raise "Did not receive an event matching #{arguments} in 5 seconds; received #{events.length} but expected #{arguments['count']} events"
+          else
+            sleep 0.1
+          end
+        end
+      end
+
+      def assert_event_count(client, context)
+        events = _select_events(context)
+        unless events.length == arguments['count']
+          raise "Exppected #{arguments['count']} #{arguments['event']} events, but have #{events.length}"
+        end
+      end
+
+      def _select_events(context)
+        case arguments['event']
+        when 'ServerMarkedUnknownEvent'
+          context.sdam_subscriber.all_events.select do |event|
+            event.is_a?(Mongo::Monitoring::Event::ServerDescriptionChanged) &&
+            event.new_description.unknown?
+          end
+        else
+          context.sdam_subscriber.all_events.select do |event|
+            event.class.name.sub(/.*::/, '') == arguments['event'].sub(/Event$/, '')
+          end
+        end
+      end
+
+      class ThreadContext
+        def initialize
+          @operations = Queue.new
+          @unexpected_operation_results = []
+        end
+
+        def stop?
+          !!@stop
+        end
+
+        def signal_stop
+          @stop = true
+        end
+
+        attr_reader :operations
+        attr_reader :unexpected_operation_results
+      end
+
+      def start_thread(client, context)
+        thread_context = ThreadContext.new
+        thread = Thread.new do
+          loop do
+            begin
+              op_spec = thread_context.operations.pop(true)
+              op = Operation.new(@crud_test, op_spec)
+              target = @crud_test.resolve_target(@crud_test.test_client, op)
+              result = op.execute(target, context)
+              if op_spec['error']
+                unless result['error']
+                  thread_context.unexpected_operation_results << result
+                end
+              else
+                if result['error']
+                  thread_context.unexpected_operation_results << result
+                end
+              end
+            rescue ThreadError
+              # Queue is empty
+            end
+            if thread_context.stop?
+              break
+            else
+              sleep 1
+            end
+          end
+        end
+        class << thread
+          attr_accessor :context
+        end
+        thread.context = thread_context
+        unless context.threads
+          context.threads ||= {}
+        end
+        context.threads[arguments['name']] = thread
+      end
+
+      def run_on_thread(client, context)
+        thread = context.threads.fetch(arguments['name'])
+        thread.context.operations << arguments['operation']
+      end
+
+      def wait_for_thread(client, context)
+        thread = context.threads.fetch(arguments['name'])
+        thread.context.signal_stop
+        thread.join
+        unless thread.context.unexpected_operation_results.empty?
+          raise "Thread #{arguments['name']} had #{thread.context.unexpected_operation_results}.length unexpected operation results"
+        end
+      end
+
+      def wait(client, context)
+        sleep arguments['ms'] / 1000.0
+      end
+
+      def record_primary(client, context)
+        context.primary_address = client.cluster.next_primary.address
+      end
+
+      def run_admin_command(support_client, context)
+        support_client.use('admin').database.command(arguments['command'])
+      end
+
+      def wait_for_primary_change(client, context)
+        timeout = if arguments['timeoutMS']
+          arguments['timeoutMS'] / 1000.0
+        else
+          10
+        end
+        deadline = Time.now + timeout
+        loop do
+          client.cluster.scan!
+          if client.cluster.next_primary.address != context.primary_address
+            break
+          end
+          if Time.now >= deadline
+            raise "Failed to change primary in #{timeout} seconds"
+          end
+        end
+      end
+    end
+  end
+end

data/spec/runners/transactions/spec.rb

@@ -0,0 +1,26 @@
+# Copyright (C) 2014-2020 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  module Transactions
+
+    class Spec < Mongo::CRUD::Spec
+      def tests
+        @tests.map do |test|
+          Mongo::Transactions::TransactionsTest.new(self, @data, test)
+        end
+      end
+    end
+  end
+end

data/spec/runners/transactions/test.rb

@@ -0,0 +1,322 @@
+# Copyright (C) 2014-2020 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  module Transactions
+
+    # Represents a single transaction test.
+    #
+    # @since 2.6.0
+    class TransactionsTest < CRUD::CRUDTestBase
+
+      attr_reader :expected_results
+      attr_reader :skip_reason
+
+      attr_reader :results
+
+      # Instantiate the new CRUDTest.
+      #
+      # @example Create the test.
+      #   TransactionTest.new(data, test)
+      #
+      # @param [ Crud::Spec ] crud_spec The top level YAML specification object.
+      # @param [ Array<Hash> ] data The documents the collection
+      # must have before the test runs.
+      # @param [ Hash ] test The test specification.
+      #
+      # @since 2.6.0
+      def initialize(crud_spec, data, test)
+        test = IceNine.deep_freeze(test)
+        @spec = crud_spec
+        @data = data
+        @description = test['description']
+        @client_options = {
+          # Disable legacy read & write retries, so that when spec tests
+          # disable modern retries we do not retry at all instead of using
+          # legacy retries which is contrary to what the tests want.
+          max_read_retries: 0,
+          max_write_retries: 0,
+          app_name: 'Tx spec - test client',
+        }.update(::Utils.convert_client_options(test['clientOptions'] || {}))
+
+        @fail_point_command = test['failPoint']
+
+        @session_options = if opts = test['sessionOptions']
+          Hash[opts.map do |session_name, options|
+            [session_name.to_sym, ::Utils.convert_operation_options(options)]
+          end]
+        else
+          {}
+        end
+        @skip_reason = test['skipReason']
+        @multiple_mongoses = test['useMultipleMongoses']
+
+        operations = test['operations']
+        @operations = operations.map do |op|
+          Operation.new(self, op)
+        end
+
+        @expectations = BSON::ExtJSON.parse_obj(test['expectations'], mode: :bson)
+
+        if test['outcome']
+          @outcome = Mongo::CRUD::Outcome.new(BSON::ExtJSON.parse_obj(test['outcome'], mode: :bson))
+        end
+
+        @expected_results = operations.map do |o|
+          o = BSON::ExtJSON.parse_obj(o)
+
+          # We check both o.key('error') and o['error'] to provide a better
+          # error message in case error: false is ever needed in the tests
+          if o.key?('error')
+            if o['error']
+              {'error' => true}
+            else
+              raise "Unsupported error value #{o['error']}"
+            end
+          else
+            result = o['result']
+            next result unless result.class == Hash
+
+            # Change maps of result ids to arrays of ids
+            result.dup.tap do |r|
+              r.each do |k, v|
+                next unless ['insertedIds', 'upsertedIds'].include?(k)
+                r[k] = v.to_a.sort_by(&:first).map(&:last)
+              end
+            end
+          end
+        end
+      end
+
+      attr_reader :outcome
+
+      def multiple_mongoses?
+        @multiple_mongoses
+      end
+
+      def support_client
+        @support_client ||= ClientRegistry.instance.global_client('root_authorized').use(@spec.database_name)
+      end
+
+      def admin_support_client
+        @admin_support_client ||= support_client.use('admin')
+      end
+
+      def test_client
+        @test_client ||= begin
+          sdam_proc = lambda do |test_client|
+            test_client.subscribe(Mongo::Monitoring::COMMAND, command_subscriber)
+
+            test_client.subscribe(Mongo::Monitoring::TOPOLOGY_OPENING, sdam_subscriber)
+            test_client.subscribe(Mongo::Monitoring::SERVER_OPENING, sdam_subscriber)
+            test_client.subscribe(Mongo::Monitoring::SERVER_DESCRIPTION_CHANGED, sdam_subscriber)
+            test_client.subscribe(Mongo::Monitoring::TOPOLOGY_CHANGED, sdam_subscriber)
+            test_client.subscribe(Mongo::Monitoring::SERVER_CLOSED, sdam_subscriber)
+            test_client.subscribe(Mongo::Monitoring::TOPOLOGY_CLOSED, sdam_subscriber)
+            test_client.subscribe(Mongo::Monitoring::CONNECTION_POOL, sdam_subscriber)
+          end
+
+          ClientRegistry.instance.new_local_client(
+            SpecConfig.instance.addresses,
+            SpecConfig.instance.authorized_test_options.merge(
+              database: @spec.database_name,
+              auth_source: SpecConfig.instance.auth_options[:auth_source] || 'admin',
+              sdam_proc: sdam_proc,
+            ).merge(@client_options))
+        end
+      end
+
+      def command_subscriber
+        @command_subscriber ||= EventSubscriber.new
+      end
+
+      def sdam_subscriber
+        @sdam_subscriber ||= EventSubscriber.new(name: 'sdam subscriber')
+      end
+
+      # Run the test.
+      #
+      # @example Run the test.
+      #   test.run
+      #
+      # @return [ Result ] The result of running the test.
+      #
+      # @since 2.6.0
+      def run
+        @threads = {}
+
+        results = @operations.map do |op|
+          target = resolve_target(test_client, op)
+          if op.needs_session?
+            context = CRUD::Context.new(
+              session0: session0,
+              session1: session1,
+              sdam_subscriber: sdam_subscriber,
+              threads: @threads,
+              primary_address: @primary_address,
+            )
+          else
+            # Hack to support write concern operations tests, which are
+            # defined to use transactions format but target pre-3.6 servers
+            # that do not support sessions
+            target ||= support_client
+            context = CRUD::Context.new(
+              sdam_subscriber: sdam_subscriber,
+              threads: @threads,
+              primary_address: @primary_address,
+            )
+          end
+
+          op.execute(target, context).tap do
+            @threads = context.threads
+            @primary_address = context.primary_address
+          end
+        end
+
+        session0_id = @session0&.session_id
+        session1_id = @session1&.session_id
+
+        @session0&.end_session
+        @session1&.end_session
+
+        actual_events = ::Utils.yamlify_command_events(command_subscriber.started_events)
+        actual_events = actual_events.reject do |event|
+          event['command_started_event']['command']['endSessions']
+        end
+        actual_events.each do |e|
+
+          # Replace the session id placeholders with the actual session ids.
+          payload = e['command_started_event']
+          if @session0
+            payload['command']['lsid'] = 'session0' if payload['command']['lsid'] == session0_id
+          end
+          if @session1
+            payload['command']['lsid'] = 'session1' if payload['command']['lsid'] == session1_id
+          end
+
+        end
+
+        @results = {
+          results: results,
+          contents: @result_collection.with(
+            read: {mode: 'primary'},
+            read_concern: { level: 'local' },
+          ).find.sort(_id: 1).to_a,
+          events: actual_events,
+        }
+      end
+
+      def setup_test
+        begin
+          admin_support_client.command(killAllSessions: [])
+        rescue Mongo::Error
+        end
+
+        if ClusterConfig.instance.fcv_ish >= '4.2'
+          ::Utils.mongos_each_direct_client do |direct_client|
+            direct_client.command(configureFailPoint: 'failCommand', mode: 'off')
+          end
+        end
+
+        # Insert data into the key vault collection if required to do so by
+        # the tests.
+        if @spec.key_vault_data && !@spec.key_vault_data.empty?
+          key_vault_coll = support_client
+            .use(:admin)[:datakeys]
+            .with(write: { w: :majority })
+
+          key_vault_coll.drop
+          key_vault_coll.insert_many(@spec.key_vault_data)
+        end
+
+        coll = support_client[@spec.collection_name].with(write: { w: :majority })
+        coll.drop
+
+        # Place a jsonSchema validator on the collection if required to do so
+        # by the tests.
+        collection_validator = if @spec.json_schema
+          { '$jsonSchema' => @spec.json_schema }
+        else
+          {}
+        end
+
+        support_client.command(
+          create: @spec.collection_name,
+          validator: collection_validator,
+          writeConcern: { w: 'majority' }
+        )
+
+        coll.insert_many(@data) unless @data.empty?
+
+        $distinct_ran ||= {}
+        $distinct_ran[@spec.database_name] ||= if description =~ /distinct/ || @operations.any? { |op| op.name == 'distinct' }
+          ::Utils.mongos_each_direct_client do |direct_client|
+            direct_client.use(@spec.database_name)['test'].distinct('foo').to_a
+          end
+        end
+
+        admin_support_client.command(@fail_point_command) if @fail_point_command
+
+        @collection = test_client[@spec.collection_name]
+
+        # Client-side encryption tests require the use of a separate client
+        # without auto_encryption_options for querying results.
+        result_collection_name = outcome&.collection_name || @spec.collection_name
+        @result_collection = support_client.use(@spec.database_name)[result_collection_name]
+      end
+
+      def teardown_test
+
+        if @fail_point_command
+          admin_support_client.command(configureFailPoint: 'failCommand', mode: 'off')
+        end
+
+        if $disable_fail_points
+          $disable_fail_points.each do |(fail_point_command, address)|
+            client = ClusterTools.instance.direct_client(address,
+              database: 'admin')
+            client.command(configureFailPoint: fail_point_command['configureFailPoint'],
+              mode: 'off')
+          end
+        end
+
+        if @test_client
+          @test_client.cluster.session_pool.end_sessions
+        end
+      end
+
+      def resolve_target(client, operation)
+        case operation.object
+        when 'session0'
+          session0
+        when 'session1'
+          session1
+        when 'testRunner'
+          # We don't actually use this target in any way.
+          nil
+        else
+          super
+        end
+      end
+
+      def session0
+        @session0 ||= test_client.start_session(@session_options[:session0] || {})
+      end
+
+      def session1
+        @session1 ||= test_client.start_session(@session_options[:session1] || {})
+      end
+    end
+  end
+end