mongo 2.12.1 → 2.13.0

data/spec/runners/crud/test_base.rb

@@ -3,7 +3,35 @@ module Mongo
 
     class CRUDTestBase
 
+      # The test description.
+      #
+      # @return [ String ] description The test description.
+      attr_reader :description
+
+      # The expected command monitoring events.
+      attr_reader :expectations
+
+      def setup_fail_point(client)
+        if @fail_point_command
+          client.use(:admin).command(@fail_point_command)
+        end
+      end
+
+      def clear_fail_point(client)
+        if @fail_point_command
+          client.use(:admin).command(BSON::Document.new(@fail_point_command).merge(mode: "off"))
+        end
+      end
+
+      private
+
       def resolve_target(client, operation)
+        if operation.database_options
+          # Some CRUD spec tests specify "database options". In Ruby there is
+          # no facility to specify options on a database, hence these are
+          # lifted to the client.
+          client = client.with(operation.database_options)
+        end
         case operation.object
         when 'collection'
           client[@spec.collection_name].with(operation.collection_options)
@@ -17,6 +45,25 @@ module Mongo
           raise "Unknown target #{operation.object}"
         end
       end
+
+      # If the deployment is a sharded cluster, creates a direct client
+      # to each of the mongos nodes and yields each in turn to the
+      # provided block. Does nothing in other topologies.
+      def mongos_each_direct_client
+        if ClusterConfig.instance.topology == :sharded
+          client = ClientRegistry.instance.global_client('basic')
+          client.cluster.next_primary
+          client.cluster.servers.each do |server|
+            direct_client = ClientRegistry.instance.new_local_client(
+              [server.address.to_s],
+              SpecConfig.instance.test_options.merge(
+                connect: :sharded
+              ).merge(SpecConfig.instance.auth_options))
+            yield direct_client
+            direct_client.close
+          end
+        end
+      end
     end
   end
 end

data/spec/runners/crud/verifier.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2019 MongoDB, Inc.
+# Copyright (C) 2019-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -71,6 +71,18 @@ EOT
         end
       end
 
+      # This variant used by change stream tests which provide the first N
+      # events rather than all of them.
+      def verify_command_started_event_min_count(expected_events, actual_events)
+        if actual_events.length < expected_events.length
+          raise RSpec::Expectations::ExpectationNotMetError.new, <<-EOT
+Expected at least #{expected_events.length} events, got #{actual_events.length} events.
+Expected events: #{expected_events.pretty_inspect}
+Actual events: #{actual_events.pretty_inspect}
+EOT
+        end
+      end
+
       def verify_command_started_event(expected_events, actual_events, i)
         expect(expected_events.length).to be > i
         expect(actual_events.length).to be > i
@@ -116,6 +128,8 @@ EOT
         case expected
         when nil
           expect(actual).to be nil
+        when 42, '42'
+          expect(actual).not_to be nil
         when Hash
           if actual.is_a?(Hash) && actual['error'] &&
             !expected.keys.any? { |key| key.start_with?('error') }
@@ -155,6 +169,10 @@ EOT
           return
         end
 
+        if [42, '42'].include?(expected[k]) && actual[k]
+          return
+        end
+
         if %w(deletedCount matchedCount modifiedCount upsertedCount).include?(k)
           # Some tests assert that some of these counts are zero.
           # The driver may omit the respective key, which is fine.
@@ -176,11 +194,11 @@ EOT
         end
 
         if expected[k].is_a?(Time)
-          expect({k => expected[k].utc.to_s}).to eq({k => actual[k].utc.to_s})
+          expect(k => actual[k].utc.to_s).to eq(k => expected[k].utc.to_s)
         else
           # This should produce a meaningful error message,
           # even though we do not actually require that expected[k] == actual[k]
-          expect({k => expected[k]}).to eq({k => actual[k]})
+          expect(k => actual[k]).to eq(k => expected[k])
         end
       end
     end

data/spec/runners/gridfs.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2019 MongoDB, Inc.
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/spec/runners/{server_discovery_and_monitoring.rb → sdam.rb}

@@ -50,15 +50,6 @@ module Mongo
       client.cluster.servers_list.detect{ |s| s.address.to_s == address_str }
     end
 
-    # Helper to convert an extended JSON ObjectId electionId to BSON::ObjectId.
-    #
-    # @since 2.1.0
-    def self.convert_election_ids(docs)
-      docs.each do |doc |
-        doc['electionId'] = BSON::ObjectId.from_string(doc['electionId']['$oid']) if doc['electionId']
-      end
-    end
-
     # Represents a specification.
     #
     # @since 2.0.0
@@ -102,6 +93,8 @@ module Mongo
       #   the phase.
       attr_reader :responses
 
+      attr_reader :application_errors
+
       # Create the new phase.
       #
       # @example Create the new phase.
@@ -113,8 +106,9 @@ module Mongo
       # @since 2.0.0
       def initialize(phase, uri)
         @phase = phase
-        @responses = @phase['responses'].map{ |response| Response.new(SDAM::convert_election_ids(response), uri) }
-        @outcome = Outcome.new(@phase['outcome'])
+        @responses = @phase['responses']&.map{ |response| Response.new(response, uri) }
+        @application_errors = @phase['applicationErrors']&.map{ |error_spec| ApplicationError.new(error_spec) }
+        @outcome = Outcome.new(BSON::ExtJSON.parse_obj(@phase['outcome']))
       end
     end
 
@@ -141,7 +135,38 @@ module Mongo
       def initialize(response, uri)
         @uri = uri
         @address = response[0]
-        @ismaster = response[1]
+        @ismaster = BSON::ExtJSON.parse_obj(response[1])
+      end
+    end
+
+    class ApplicationError
+      def initialize(spec)
+        @spec = spec
+      end
+
+      def address_str
+        @spec.fetch('address')
+      end
+
+      def when
+        ::Utils.underscore(@spec.fetch('when'))
+      end
+
+      def max_wire_version
+        @spec['max_wire_version']
+      end
+
+      def generation
+        @spec['generation']
+      end
+
+      def type
+        ::Utils.underscore(@spec.fetch('type'))
+      end
+
+      def result
+        msg = Mongo::Protocol::Msg.new([], {}, BSON::ExtJSON.parse_obj(@spec['response']))
+        Mongo::Operation::Result.new([msg])
       end
     end
 
@@ -179,14 +204,14 @@ module Mongo
       #
       # @since 2.0.0
       def initialize(outcome)
-        @servers = process_servers(outcome['servers']) if outcome['servers']
+        @servers = outcome['servers'] if outcome['servers']
         @set_name = outcome['setName']
         @topology_type = outcome['topologyType']
         @logical_session_timeout = outcome['logicalSessionTimeoutMinutes']
-        @events = process_events(outcome['events']) if outcome['events']
+        @events = map_events(outcome['events']) if outcome['events']
         @compatible = outcome['compatible']
         if outcome['maxElectionId']
-          @max_election_id = BSON::ObjectId.from_string(outcome['maxElectionId']['$oid'])
+          @max_election_id = outcome['maxElectionId']
         end
         @max_set_version = outcome['maxSetVersion']
       end
@@ -209,17 +234,11 @@ module Mongo
 
       private
 
-      def process_events(events)
+      def map_events(events)
         events.map do |event|
           Event.new(event.keys.first, event.values.first)
         end
       end
-
-      def process_servers(servers)
-        servers.each do |s|
-          SDAM::convert_election_ids([ s[1] ])
-        end
-      end
     end
 
     class Event

data/spec/runners/sdam/verifier.rb

@@ -56,8 +56,8 @@ module Sdam
       verify_description_matches(expected.data['newDescription'], actual.new_description)
     end
 
-    def verify_description_matches(expected, actual)
-      case expected['type']
+    def verify_description_matches(server_spec, actual)
+      case server_spec['type']
       when 'Standalone'
         expect(actual).to be_standalone
       when 'RSPrimary'
@@ -76,12 +76,30 @@ module Sdam
         expect(actual).to be_other
       end
 
-      expect(actual.address.to_s).to eq(expected['address'])
-      expect(actual.arbiters).to eq(expected['arbiters'])
-      expect(actual.hosts).to eq(expected['hosts'])
-      expect(actual.passives).to eq(expected['passives'])
-      expect(actual.primary_host).to eq(expected['primary'])
-      expect(actual.replica_set_name).to eq(expected['setName'])
+      if server_spec['arbiters']
+        expect(actual.arbiters).to eq(server_spec['arbiters'])
+      end
+      if server_spec['hosts']
+        expect(actual.hosts).to eq(server_spec['hosts'])
+      end
+      if server_spec['passives']
+        expect(actual.passives).to eq(server_spec['passives'])
+      end
+      if server_spec['primary']
+        expect(actual.primary_host).to eq(server_spec['primary'])
+      end
+      expect(actual.replica_set_name).to eq(server_spec['setName'])
+
+      if server_spec['topologyVersion']
+        # In the Ruby TopologyVersion object, the counter is a
+        # Ruby integer. It would serialize to BSON int.
+        # The expected topology version specifies counter as a
+        # BSON long.
+        # Parse expected value as extended json and compare
+        # Ruby objects.
+        expected_tv = server_spec['topologyVersion']
+        expect(actual.topology_version).to eq(expected_tv)
+      end
     end
 
     def verify_server_closed_event(expected, actual)

data/spec/runners/server_selection.rb

@@ -73,19 +73,6 @@ module Mongo
           @type = Mongo::Cluster::Topology.const_get(@test['topology_description']['type'])
         end
 
-        # Whether this spec describes a replica set.
-        #
-        # @example Determine if the spec describes a replica set.
-        #   spec.replica_set?
-        #
-        # @return [true, false] If the spec describes a replica set.
-        #
-        # @since 2.0.0
-        def replica_set?
-          type == Mongo::Cluster::Topology::ReplicaSetNoPrimary ||
-          type == Mongo::Cluster::Topology::ReplicaSetWithPrimary
-        end
-
         # Does this spec expect a server to be found.
         #
         # @example Will a server be found with this spec.
@@ -98,15 +85,10 @@ module Mongo
           !in_latency_window.empty?
         end
 
-        # Is the max staleness setting invalid.
+        # Whether the test requires an error to be raised during server selection.
         #
-        # @example Will the max staleness setting be valid with other options.
-        #   spec.invalid_max_staleness?
-        #
-        # @return [ true, false ] If an error will be raised by the max staleness setting.
-        #
-        # @since 2.4.0
-        def invalid_max_staleness?
+        # @return [ true, false ] Whether the test expects an error.
+        def error?
           @test['error']
         end
 
@@ -122,9 +104,6 @@ module Mongo
         #
         # @since 2.0.0
         def in_latency_window
-          if read_preference['mode'] == :secondary_preferred && primary
-            return @in_latency_window.push(primary).uniq
-          end
           @in_latency_window
         end
 
@@ -134,13 +113,248 @@ module Mongo
         #
         # @since 2.0.0
         def candidate_servers
-          @candidate_servers.select { |s| !['Unknown', 'PossiblePrimary'].include?(s['type']) }
+          @candidate_servers
+        end
+      end
+    end
+  end
+end
+
+def define_server_selection_spec_tests(test_paths)
+  # Linter insists that a server selection semaphore is present when
+  # performing server selection.
+  skip_if_linting
+
+  test_paths.each do |file|
+
+    spec = Mongo::ServerSelection::Read::Spec.new(file)
+
+    context(spec.description) do
+      # Cluster needs a topology and topology needs a cluster...
+      # This temporary cluster is used for topology construction.
+      let(:temp_cluster) do
+        double('temp cluster').tap do |cluster|
+          allow(cluster).to receive(:servers_list).and_return([])
+        end
+      end
+
+      let(:topology) do
+        options = if spec.type <= Mongo::Cluster::Topology::ReplicaSetNoPrimary
+          {replica_set_name: 'foo'}
+        else
+          {}
+        end
+        spec.type.new(options, monitoring, temp_cluster)
+      end
+
+      let(:monitoring) do
+        Mongo::Monitoring.new(monitoring: false)
+      end
+
+      let(:listeners) do
+        Mongo::Event::Listeners.new
+      end
+
+      let(:options) do
+        if spec.heartbeat_frequency
+          {server_selection_timeout: 0.1, heartbeat_frequency: spec.heartbeat_frequency}
+        else
+          {server_selection_timeout: 0.1}
+        end
+      end
+
+      let(:cluster) do
+        double('cluster').tap do |c|
+          allow(c).to receive(:server_selection_semaphore)
+          allow(c).to receive(:connected?).and_return(true)
+          allow(c).to receive(:summary)
+          allow(c).to receive(:topology).and_return(topology)
+          allow(c).to receive(:single?).and_return(topology.single?)
+          allow(c).to receive(:sharded?).and_return(topology.sharded?)
+          allow(c).to receive(:replica_set?).and_return(topology.replica_set?)
+          allow(c).to receive(:unknown?).and_return(topology.unknown?)
+          allow(c).to receive(:options).and_return(options)
+          allow(c).to receive(:scan!).and_return(true)
+          allow(c).to receive(:app_metadata).and_return(app_metadata)
+          allow(c).to receive(:heartbeat_interval).and_return(
+            spec.heartbeat_frequency || Mongo::Server::Monitor::DEFAULT_HEARTBEAT_INTERVAL)
         end
+      end
+
+      # One of the spec test assertions is on the set of servers that are
+      # eligible for selection without taking latency into account.
+      # In the driver, latency is taken into account at various points during
+      # server selection, hence there isn't a method that can be called to
+      # retrieve the list of servers without accounting for latency.
+      # Work around this by executing server selection with all servers set
+      # to zero latency, when evaluating the candidate server set.
+      let(:ignore_latency) { false }
+
+      let(:candidate_servers) do
+        spec.candidate_servers.collect do |server|
+          features = double('features').tap do |feat|
+            allow(feat).to receive(:max_staleness_enabled?).and_return(server['maxWireVersion'] && server['maxWireVersion'] >= 5)
+            allow(feat).to receive(:check_driver_support!).and_return(true)
+          end
+          address = Mongo::Address.new(server['address'])
+          Mongo::Server.new(address, cluster, monitoring, listeners,
+            {monitoring_io: false}.update(options)
+          ).tap do |s|
+            allow(s).to receive(:average_round_trip_time) do
+              if ignore_latency
+                0
+              elsif server['avg_rtt_ms']
+                server['avg_rtt_ms'] / 1000.0
+              end
+            end
+            allow(s).to receive(:tags).and_return(server['tags'])
+            allow(s).to receive(:secondary?).and_return(server['type'] == 'RSSecondary')
+            allow(s).to receive(:primary?).and_return(server['type'] == 'RSPrimary')
+            allow(s).to receive(:mongos?).and_return(server['type'] == 'Mongos')
+            allow(s).to receive(:standalone?).and_return(server['type'] == 'Standalone')
+            allow(s).to receive(:unknown?).and_return(server['type'] == 'Unknown')
+            allow(s).to receive(:connectable?).and_return(true)
+            allow(s).to receive(:last_write_date).and_return(
+              Time.at(server['lastWrite']['lastWriteDate']['$numberLong'].to_f / 1000)) if server['lastWrite']
+            allow(s).to receive(:last_scan).and_return(
+              Time.at(server['lastUpdateTime'].to_f / 1000))
+            allow(s).to receive(:features).and_return(features)
+            allow(s).to receive(:replica_set_name).and_return('foo')
+          end
+        end
+      end
+
+      let(:suitable_servers) do
+        spec.suitable_servers.collect do |server|
+          Mongo::Server.new(Mongo::Address.new(server['address']), cluster, monitoring, listeners,
+            options.merge(monitoring_io: false))
+        end
+      end
 
-        private
+      let(:in_latency_window) do
+        spec.in_latency_window.collect do |server|
+          Mongo::Server.new(Mongo::Address.new(server['address']), cluster, monitoring, listeners,
+            options.merge(monitoring_io: false))
+        end
+      end
+
+      let(:server_selector_definition) do
+        { mode: spec.read_preference['mode'] }.tap do |definition|
+          definition[:tag_sets] = spec.read_preference['tag_sets']
+          definition[:max_staleness] = spec.max_staleness if spec.max_staleness
+        end
+      end
+
+      let(:server_selector) do
+        Mongo::ServerSelector.get(server_selector_definition)
+      end
+
+      let(:app_metadata) do
+        Mongo::Server::AppMetadata.new({})
+      end
+
+      before do
+        allow(cluster).to receive(:servers_list).and_return(candidate_servers)
+        allow(cluster).to receive(:servers) do
+          # Copy Cluster#servers definition because clusters is a double
+          cluster.topology.servers(cluster.servers_list)
+        end
+        allow(cluster).to receive(:addresses).and_return(candidate_servers.map(&:address))
+      end
+
+      if spec.error?
+
+        it 'Raises an InvalidServerPreference exception' do
+
+          expect do
+            server_selector.select_server(cluster)
+          end.to raise_exception(Mongo::Error::InvalidServerPreference)
+        end
+
+      else
+
+        if spec.server_available?
+
+          it 'has non-empty suitable servers' do
+            spec.suitable_servers.should be_a(Array)
+            spec.suitable_servers.should_not be_empty
+          end
+
+          if spec.in_latency_window.length == 1
+
+            it 'selects the expected server' do
+              [server_selector.select_server(cluster)].should == in_latency_window
+            end
+
+          else
+
+            it 'selects a server in the suitable list' do
+              in_latency_window.should include(server_selector.select_server(cluster))
+            end
+
+            let(:expected_addresses) do
+              in_latency_window.map(&:address).map(&:seed).sort
+            end
+
+            let(:actual_addresses) do
+              server_selector.suitable_servers(cluster).map(&:address).map(&:seed).sort
+            end
+
+            it 'identifies expected suitable servers' do
+              actual_addresses.should == expected_addresses
+            end
+
+          end
+
+          context 'candidate servers without taking latency into account' do
+            let(:ignore_latency) { true }
+
+            let(:expected_addresses) do
+              suitable_servers.map(&:address).map(&:seed).sort
+            end
+
+            let(:actual_addresses) do
+              servers = server_selector.send(:suitable_servers, cluster)
+
+              # The tests expect that only secondaries are "suitable" for
+              # server selection with secondary preferred read preference.
+              # In actuality, primaries are also suitable, and the driver
+              # returns the primaries also. Remove primaries from the
+              # actual set when read preference is secondary preferred.
+              # HOWEVER, if a test ends up selecting a primary, then it
+              # includes that primary into its suitable servers. Therefore
+              # only remove primaries when the number of suitable servers
+              # is greater than 1.
+              servers.delete_if do |server|
+                server_selector.is_a?(Mongo::ServerSelector::SecondaryPreferred) &&
+                  server.primary? &&
+                  servers.length > 1
+              end
+
+              # Since we remove the latency requirement, the servers
+              # may be returned in arbitrary order.
+              servers.map(&:address).map(&:seed).sort
+            end
+
+            it 'identifies expected suitable servers' do
+              actual_addresses.should == expected_addresses
+            end
+          end
+
+        else
+
+          # Runner does not handle non-empty suitable servers with
+          # no servers in latency window.
+          it 'has empty suitable servers' do
+            expect(spec.suitable_servers).to eq([])
+          end
+
+          it 'Raises a NoServerAvailable Exception' do
+            expect do
+              server_selector.select_server(cluster)
+            end.to raise_exception(Mongo::Error::NoServerAvailable)
+          end
 
-        def primary
-          @candidate_servers.find { |s| s['type'] == 'RSPrimary' }
         end
       end
     end