mongo 2.11.4 → 2.18.1

data/spec/integration/sdam_error_handling_spec.rb

@@ -1,17 +1,47 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 require 'spec_helper'
 
 describe 'SDAM error handling' do
-  clean_slate_for_all
+  require_topology :single, :replica_set, :sharded
+
+  clean_slate
+
+  after do
+    # Close all clients after every test to avoid leaking expectations into
+    # subsequent tests because we set global assertions on sockets.
+    ClientRegistry.instance.close_all_clients
+  end
 
   # These tests operate on specific servers, and don't work in a multi
   # shard cluster where multiple servers are equally eligible
-  require_no_multi_shard
-
-  let(:client) { authorized_client_without_any_retries }
+  require_no_multi_mongos
+
+  let(:diagnostic_subscriber) { Mrss::VerboseEventSubscriber.new }
+
+  let(:client) do
+    new_local_client(SpecConfig.instance.addresses,
+      SpecConfig.instance.all_test_options.merge(
+        socket_timeout: 3, connect_timeout: 3,
+        heartbeat_frequency: 100,
+        # Uncomment to print all events to stdout:
+        #sdam_proc: Utils.subscribe_all_sdam_proc(diagnostic_subscriber),
+        **Utils.disable_retries_client_options)
+    )
+  end
 
   let(:server) { client.cluster.next_primary }
 
   shared_examples_for 'marks server unknown' do
+    before do
+      server.monitor.stop!
+    end
+
+    after do
+      client.close
+    end
+
     it 'marks server unknown' do
       expect(server).not_to be_unknown
       RSpec::Mocks.with_temporary_scope do
@@ -22,6 +52,14 @@ describe 'SDAM error handling' do
   end
 
   shared_examples_for 'does not mark server unknown' do
+    before do
+      server.monitor.stop!
+    end
+
+    after do
+      client.close
+    end
+
     it 'does not mark server unknown' do
       expect(server).not_to be_unknown
       RSpec::Mocks.with_temporary_scope do
@@ -75,7 +113,6 @@ describe 'SDAM error handling' do
 
     before do
       client.cluster.next_primary
-      stop_monitoring(client)
       # we also need a connection to the primary so that our error
       # expectations do not get triggered during handshakes which
       # have different behavior from non-handshake errors
@@ -149,6 +186,9 @@ describe 'SDAM error handling' do
     end
 
     context 'network error' do
+      # With 4.4 servers we set up two monitoring connections, hence global
+      # socket expectations get hit twice.
+      max_server_version '4.2'
 
       let(:operation) do
         expect_any_instance_of(Mongo::Socket).to receive(:read).and_raise(exception)
@@ -179,38 +219,208 @@ describe 'SDAM error handling' do
     end
   end
 
-  # These tests fail intermittently in Evergreen
-  describe 'when there is an error on monitoring connection', retry: 3 do
-    let(:client) do
-      authorized_client_without_any_retries.with(
-        connect_timeout: 1, socket_timeout: 1)
+  describe 'when there is an error during connection establishment' do
+    require_topology :single
+
+    # The push monitor creates sockets unpredictably and interferes with this
+    # test.
+    max_server_version '4.2'
+
+    # When TLS is used there are two socket classes and we can't simply
+    # mock the base Socket class.
+    require_no_tls
+
+    {
+      SystemCallError => Mongo::Error::SocketError,
+      Errno::ETIMEDOUT => Mongo::Error::SocketTimeoutError,
+    }.each do |raw_error_cls, mapped_error_cls|
+      context raw_error_cls.name do
+        let(:socket) do
+          double('mock socket').tap do |socket|
+            allow(socket).to receive(:set_encoding)
+            allow(socket).to receive(:setsockopt)
+            allow(socket).to receive(:getsockopt)
+            allow(socket).to receive(:connect)
+            allow(socket).to receive(:close)
+            socket.should receive(:write).and_raise(raw_error_cls, 'mocked failure')
+          end
+        end
+
+        it 'marks server unknown' do
+          server = client.cluster.next_primary
+          client.cluster.servers.map(&:disconnect!)
+
+          RSpec::Mocks.with_temporary_scope do
+
+            Socket.should receive(:new).with(any_args).ordered.once.and_return(socket)
+
+            lambda do
+              client.command(ping: 1)
+            end.should raise_error(mapped_error_cls, /mocked failure/)
+
+            server.should be_unknown
+          end
+        end
+
+        it 'recovers' do
+          server = client.cluster.next_primary
+          # If we do not kill the monitor, the client will recover automatically.
+
+          RSpec::Mocks.with_temporary_scope do
+
+            Socket.should receive(:new).with(any_args).ordered.once.and_return(socket)
+            Socket.should receive(:new).with(any_args).ordered.once.and_call_original
+
+            lambda do
+              client.command(ping: 1)
+            end.should raise_error(mapped_error_cls, /mocked failure/)
+
+            client.command(ping: 1)
+          end
+        end
+      end
+    end
+
+    after do
+      # Since we stopped monitoring on the client, close it.
+      ClientRegistry.instance.close_all_clients
+    end
+  end
+
+  describe 'when there is an error on monitoring connection' do
+    clean_slate_for_all
+
+    let(:subscriber) { Mrss::EventSubscriber.new }
+
+    let(:set_subscribers) do
+      client.subscribe(Mongo::Monitoring::SERVER_DESCRIPTION_CHANGED, subscriber)
+      client.subscribe(Mongo::Monitoring::CONNECTION_POOL, subscriber)
     end
 
     let(:operation) do
       expect(server.monitor.connection).not_to be nil
-      expect(server.monitor.connection).to receive(:ismaster).at_least(:once).and_raise(exception)
-      server.scan_semaphore.broadcast
-      6.times do
-        sleep 0.5
-        if server.unknown?
-          break
+      set_subscribers
+      RSpec::Mocks.with_temporary_scope do
+        expect(server.monitor).to receive(:check).and_raise(exception)
+        server.monitor.scan!
+      end
+      expect_server_state_change
+    end
+
+    shared_examples_for 'marks server unknown - sdam event' do
+      it 'marks server unknown' do
+        expect(server).not_to be_unknown
+
+        #subscriber.clear_events!
+        events = subscriber.select_succeeded_events(Mongo::Monitoring::Event::ServerDescriptionChanged)
+        events.should be_empty
+
+        RSpec::Mocks.with_temporary_scope do
+          operation
+
+          events = subscriber.select_succeeded_events(Mongo::Monitoring::Event::ServerDescriptionChanged)
+          events.should_not be_empty
+          event = events.detect do |event|
+            event.new_description.address == server.address &&
+            event.new_description.unknown?
+          end
+          event.should_not be_nil
         end
       end
-      expect(server).to be_unknown
     end
 
-    context 'non-timeout network error' do
-      let(:exception) { Mongo::Error::SocketError }
+    shared_examples_for 'clears connection pool - cmap event' do
+      it 'clears connection pool' do
+        #subscriber.clear_events!
+        events = subscriber.select_published_events(Mongo::Monitoring::Event::Cmap::PoolCleared)
+        events.should be_empty
+
+        RSpec::Mocks.with_temporary_scope do
+          operation
+
+          events = subscriber.select_published_events(Mongo::Monitoring::Event::Cmap::PoolCleared)
+          events.should_not be_empty
+          event = events.detect do |event|
+            event.address == server.address
+          end
+          event.should_not be_nil
+        end
+      end
+    end
 
-      it_behaves_like 'marks server unknown'
-      it_behaves_like 'clears connection pool'
+    shared_examples_for 'marks server unknown and clears connection pool' do
+=begin These tests are not reliable
+      context 'via object inspection' do
+        let(:expect_server_state_change) do
+          server.summary.should =~ /unknown/i
+          expect(server).to be_unknown
+        end
+
+        it_behaves_like 'marks server unknown'
+        it_behaves_like 'clears connection pool'
+      end
+=end
+
+      context 'via events' do
+        # When we use events we do not need to examine object state, therefore
+        # it does not matter whether the server stays unknown or gets
+        # successfully checked.
+        let(:expect_server_state_change) do
+          # nothing
+        end
+
+        it_behaves_like 'marks server unknown - sdam event'
+        it_behaves_like 'clears connection pool - cmap event'
+      end
     end
 
-    context 'network timeout' do
-      let(:exception) { Mongo::Error::SocketTimeoutError }
+    context 'via stubs' do
+      # With 4.4 servers we set up two monitoring connections, hence global
+      # socket expectations get hit twice.
+      max_server_version '4.2'
 
-      it_behaves_like 'marks server unknown'
-      it_behaves_like 'clears connection pool'
+      context 'network timeout' do
+        let(:exception) { Mongo::Error::SocketTimeoutError }
+
+        it_behaves_like 'marks server unknown and clears connection pool'
+      end
+
+      context 'non-timeout network error' do
+        let(:exception) { Mongo::Error::SocketError }
+
+        it_behaves_like 'marks server unknown and clears connection pool'
+      end
+    end
+
+    context 'non-timeout network error via fail point' do
+      require_fail_command
+
+      let(:admin_client) { client.use(:admin) }
+
+      let(:set_fail_point) do
+        admin_client.command(
+          configureFailPoint: 'failCommand',
+          mode: {times: 2},
+          data: {
+            failCommands: %w(isMaster hello),
+            closeConnection: true,
+          },
+        )
+      end
+
+      let(:operation) do
+        expect(server.monitor.connection).not_to be nil
+        set_subscribers
+        set_fail_point
+        server.monitor.scan!
+        expect_server_state_change
+      end
+
+      it_behaves_like 'marks server unknown and clears connection pool'
+
+      after do
+        admin_client.command(configureFailPoint: 'failCommand', mode: 'off')
+      end
     end
   end
 end

data/spec/integration/sdam_events_spec.rb

@@ -1,7 +1,10 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 require 'spec_helper'
 
 describe 'SDAM events' do
-  let(:subscriber) { Mongo::SDAMMonitoring::TestSubscriber.new }
+  let(:subscriber) { Mrss::EventSubscriber.new }
 
   describe 'server closed event' do
     it 'is published when client is closed' do
@@ -10,12 +13,12 @@ describe 'SDAM events' do
       client.subscribe(Mongo::Monitoring::SERVER_CLOSED, subscriber)
 
       # get the client connected
-      client.database.command(ismaster: 1)
-      expect(subscriber.events).to be_empty
+      client.database.command(ping: 1)
+      expect(subscriber.succeeded_events).to be_empty
 
       client.close
 
-      expect(subscriber.events).not_to be_empty
+      expect(subscriber.succeeded_events).not_to be_empty
       event = subscriber.first_event('server_closed_event')
       expect(event).not_to be_nil
     end
@@ -28,16 +31,148 @@ describe 'SDAM events' do
       client.subscribe(Mongo::Monitoring::TOPOLOGY_CLOSED, subscriber)
 
       # get the client connected
-      client.database.command(ismaster: 1)
-      expect(subscriber.events).to be_empty
+      client.database.command(ping: 1)
+      expect(subscriber.succeeded_events).to be_empty
 
       client.close
 
-      expect(subscriber.events).not_to be_empty
+      expect(subscriber.succeeded_events).not_to be_empty
       event = subscriber.first_event('topology_closed_event')
       expect(event).not_to be_nil
 
       expect(event.topology).to eql(client.cluster.topology)
     end
   end
+
+  describe 'heartbeat event' do
+    require_topology :single
+
+    context 'pre-4.4 servers' do
+      max_server_version '4.2'
+
+      let(:sdam_proc) do
+        Proc.new do |client|
+          client.subscribe(Mongo::Monitoring::SERVER_HEARTBEAT, subscriber)
+        end
+      end
+
+      let(:client) do
+        new_local_client(SpecConfig.instance.addresses,
+          # Heartbeat interval is bound by 500 ms
+          SpecConfig.instance.test_options.merge(
+            heartbeat_frequency: 0.5,
+            sdam_proc: sdam_proc
+          ),
+        )
+      end
+
+      it 'is published every heartbeat interval' do
+        client
+        sleep 4
+        client.close
+
+        started_events = subscriber.select_started_events(Mongo::Monitoring::Event::ServerHeartbeatStarted)
+        # Expect about 8 events, maybe 9 or 7
+        started_events.length.should >= 6
+        started_events.length.should <= 10
+
+        succeeded_events = subscriber.select_succeeded_events(Mongo::Monitoring::Event::ServerHeartbeatSucceeded)
+        started_events.length.should > 1
+        (succeeded_events.length..succeeded_events.length+1).should include(started_events.length)
+      end
+    end
+
+    context '4.4+ servers' do
+      min_server_fcv '4.4'
+
+      let(:sdam_proc) do
+        Proc.new do |client|
+          client.subscribe(Mongo::Monitoring::SERVER_HEARTBEAT, subscriber)
+        end
+      end
+
+      let(:client) do
+        new_local_client(SpecConfig.instance.addresses,
+          # Heartbeat interval is bound by 500 ms
+          SpecConfig.instance.test_options.merge(
+            heartbeat_frequency: 0.5,
+            sdam_proc: sdam_proc
+          ),
+        )
+      end
+
+      it 'is published up to twice every heartbeat interval' do
+        client
+        sleep 3
+        client.close
+
+        started_events = subscriber.select_started_events(
+          Mongo::Monitoring::Event::ServerHeartbeatStarted
+        )
+        # We could have up to 16 events and should have no fewer than 8 events.
+        # Whenever an awaited hello succeeds while the regular monitor is
+        # waiting, the regular monitor's next scan is pushed forward.
+        started_events.length.should >= 6
+        started_events.length.should <= 18
+        (started_awaited = started_events.select(&:awaited?)).should_not be_empty
+        (started_regular = started_events.reject(&:awaited?)).should_not be_empty
+
+        completed_events = subscriber.select_completed_events(
+          Mongo::Monitoring::Event::ServerHeartbeatSucceeded,
+          Mongo::Monitoring::Event::ServerHeartbeatFailed,
+        )
+        completed_events.length.should >= 6
+        completed_events.length.should <= 18
+        (succeeded_awaited = completed_events.select(&:awaited?)).should_not be_empty
+        (succeeded_regular = completed_events.reject(&:awaited?)).should_not be_empty
+
+        # There may be in-flight hellos that don't complete, both
+        # regular and awaited.
+        started_awaited.length.should > 1
+        (succeeded_awaited.length..succeeded_awaited.length+1).should include(started_awaited.length)
+        started_regular.length.should > 1
+        (succeeded_regular.length..succeeded_regular.length+1).should include(started_regular.length)
+      end
+    end
+  end
+
+  describe 'server description changed' do
+    require_topology :single
+
+    let(:sdam_proc) do
+      Proc.new do |client|
+        client.subscribe(Mongo::Monitoring::SERVER_DESCRIPTION_CHANGED, subscriber)
+      end
+    end
+
+    let(:client) do
+      new_local_client(SpecConfig.instance.addresses,
+        # Heartbeat interval is bound by 500 ms
+        SpecConfig.instance.test_options.merge(client_options).merge(
+          heartbeat_frequency: 0.5,
+          sdam_proc: sdam_proc,
+        ),
+      )
+    end
+
+    let(:client_options) do
+      {}
+    end
+
+    it 'is not published when there are no changes in server state' do
+      client
+      sleep 6
+      client.close
+
+      events = subscriber.select_succeeded_events(Mongo::Monitoring::Event::ServerDescriptionChanged)
+
+      # In 6 seconds we should have about 10 or 12 heartbeats.
+      # We expect 1 or 2 description changes:
+      # The first one from unknown to known,
+      # The second one because server changes the fields it returns based on
+      # driver server check payload (e.g. ismaster/isWritablePrimary).
+      events.length.should >= 1
+      events.length.should <= 2
+    end
+  end
 end

data/spec/integration/sdam_prose_spec.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+require 'spec_helper'
+
+describe 'SDAM prose tests' do
+  # The "streaming protocol tests" are covered by the tests in
+  # sdam_events_spec.rb.
+
+  describe 'RTT tests' do
+    min_server_fcv '4.4'
+    require_topology :single
+
+    let(:subscriber) { Mrss::EventSubscriber.new }
+
+    let(:client) do
+      new_local_client(SpecConfig.instance.addresses,
+        # Heartbeat interval is bound by 500 ms
+        SpecConfig.instance.test_options.merge(
+          heartbeat_frequency: 0.5,
+          app_name: 'streamingRttTest',
+        ),
+      ).tap do |client|
+        client.subscribe(Mongo::Monitoring::SERVER_HEARTBEAT, subscriber)
+      end
+    end
+
+    it 'updates RTT' do
+      server = client.cluster.next_primary
+
+      sleep 2
+
+      events = subscriber.select_succeeded_events(Mongo::Monitoring::Event::ServerHeartbeatSucceeded)
+      events.each do |event|
+        event.round_trip_time.should be_a(Numeric)
+        event.round_trip_time.should > 0
+      end
+
+      root_authorized_client.use('admin').database.command(
+        configureFailPoint: 'failCommand',
+        mode: {times: 1000},
+        data: {
+          failCommands: %w(isMaster hello),
+          blockConnection: true,
+          blockTimeMS: 500,
+          appName: "streamingRttTest",
+        },
+      )
+
+      deadline = Time.now + 10
+      loop do
+        if server.average_round_trip_time > 0.25
+          break
+        end
+        if Time.now >= deadline
+          raise "Failed to witness RTT growing to >= 250 ms in 10 seconds"
+        end
+        sleep 0.2
+      end
+    end
+
+    after do
+      root_authorized_client.use('admin').database.command(
+        configureFailPoint: 'failCommand', mode: 'off')
+    end
+  end
+end

data/spec/integration/secondary_reads_spec.rb

@@ -0,0 +1,102 @@
+require 'spec_helper'
+
+describe 'Secondary reads' do
+  before do
+    root_authorized_client.use('sr')['secondary_reads'].drop
+    root_authorized_client.use('sr')['secondary_reads'].insert_one(test: 1)
+  end
+
+  shared_examples 'performs reads as per read preference' do
+
+    %i(primary primary_preferred).each do |mode|
+
+      context mode.inspect do
+
+        let(:client) do
+          root_authorized_client.with(read: {mode: mode}).use('sr')
+        end
+
+        it 'reads from primary' do
+          start_stats = get_read_counters
+
+          30.times do
+            client['secondary_reads'].find.to_a
+          end
+
+          end_stats = get_read_counters
+
+          end_stats[:secondary].should be_within(10).of(start_stats[:secondary])
+          end_stats[:primary].should >= start_stats[:primary] + 30
+        end
+      end
+    end
+
+    %i(secondary secondary_preferred).each do |mode|
+
+      context mode.inspect do
+        let(:client) do
+          root_authorized_client.with(read: {mode: mode}).use('sr')
+        end
+
+        it 'reads from secondaries' do
+          start_stats = get_read_counters
+
+          30.times do
+            client['secondary_reads'].find.to_a
+          end
+
+          end_stats = get_read_counters
+
+          end_stats[:primary].should be_within(10).of(start_stats[:primary])
+          end_stats[:secondary].should >= start_stats[:secondary] + 30
+        end
+      end
+    end
+  end
+
+  context 'replica set' do
+    require_topology :replica_set
+
+    include_examples 'performs reads as per read preference'
+  end
+
+  context 'sharded cluster' do
+    require_topology :sharded
+
+    include_examples 'performs reads as per read preference'
+  end
+
+  def get_read_counters
+    client = ClientRegistry.instance.global_client('root_authorized')
+    addresses = []
+    if client.cluster.sharded?
+      doc = client.use('admin').command(listShards: 1).documents.first
+      doc['shards'].each do |shard|
+        addresses += shard['host'].split('/').last.split(',')
+      end
+    else
+      client.cluster.servers.each do |server|
+        next unless server.primary? || server.secondary?
+        addresses << server.address.seed
+      end
+    end
+    stats = Hash.new(0)
+    addresses.each do |address|
+      ClientRegistry.instance.new_local_client(
+        [address],
+        SpecConfig.instance.all_test_options.merge(connect: :direct),
+      ) do |c|
+        server = c.cluster.servers.first
+        next unless server.primary? || server.secondary?
+        stat = c.command(serverStatus: 1).documents.first
+        queries = stat['opcounters']['query']
+        if server.primary?
+          stats[:primary] += queries
+        else
+          stats[:secondary] += queries
+        end
+      end
+    end
+    stats
+  end
+end

data/spec/integration/server_description_spec.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 require 'spec_helper'
 
 describe 'Server description' do