mongo 2.10.5 → 2.11.0.rc0

data/spec/stress/connection_pool_timing_spec.rb

@@ -0,0 +1,181 @@
+require 'spec_helper'
+
+describe 'Connection pool timing test' do
+  before(:all) do
+    if !SpecConfig.instance.stress_spec?
+      skip 'Stress spec not enabled'
+    end
+
+    ClientRegistry.instance.close_all_clients
+
+    # This set up is taken from the step_down_spec file. In a future PR, ClusterTools
+    # may be modified so this set up is no longer necessary.
+    if ClusterConfig.instance.fcv_ish >= '4.2' && ClusterConfig.instance.topology == :replica_set
+      ClusterTools.instance.set_election_timeout(5)
+      ClusterTools.instance.set_election_handoff(false)
+    end
+  end
+
+  after(:all) do
+    if ClusterConfig.instance.fcv_ish >= '4.2' && ClusterConfig.instance.topology == :replica_set
+      ClusterTools.instance.set_election_timeout(10)
+      ClusterTools.instance.set_election_handoff(true)
+      ClusterTools.instance.reset_priorities
+    end
+  end
+
+  let(:client) do
+    authorized_client.with(options.merge(monitoring: true))
+  end
+
+  let!(:collection) do
+    client[authorized_collection.name].tap do |collection|
+      collection.drop
+      collection.insert_many(documents)
+    end
+  end
+
+  let(:documents) do
+    [].tap do |documents|
+      10000.times do |i|
+        documents << { a: i}
+      end
+    end
+  end
+
+  let(:operation_threads) do
+    [].tap do |threads|
+      thread_count.times do |i|
+        threads << Thread.new do
+          100.times do |j|
+            collection.find(a: i+j).to_a
+            sleep 0.01
+            collection.find(a: i+j).to_a
+          end
+        end
+      end
+    end
+  end
+
+  let(:thread_count) { 5 }
+
+  context 'when there is no max idle time' do
+    let(:options) do
+      { max_pool_size: 10, min_pool_size: 5 }
+    end
+
+    let(:threads) { operation_threads }
+
+    it 'does not error' do
+      start = Time.now
+      expect {
+        threads.collect { |t| t.join }
+      }.not_to raise_error
+      puts "[Connection Pool Timing] Duration with no max idle time: #{Time.now - start}"
+    end
+  end
+
+  context 'when there is a low max idle time' do
+    let(:options) do
+      { max_pool_size: 10, min_pool_size: 5, max_idle_time: 0.1 }
+    end
+
+    let(:threads) { operation_threads }
+
+    it 'does not error' do
+      start = Time.now
+      expect {
+        threads.collect { |t| t.join }
+      }.not_to raise_error
+      puts "[Connection Pool Timing] Duration with low max idle time: #{Time.now - start}"
+    end
+  end
+
+  context 'when clear is called periodically' do
+    let(:options) do
+      { max_pool_size: 10, min_pool_size: 5 }
+    end
+
+    let(:threads) do
+      threads = operation_threads
+      threads << Thread.new do
+        10.times do
+          sleep 0.1
+          client.cluster.next_primary.pool.clear
+        end
+      end
+      threads
+    end
+
+    it 'does not error' do
+      start = Time.now
+      expect {
+        threads.collect { |t| t.join }
+      }.not_to raise_error
+      puts "[Connection Pool Timing] Duration when clear is called periodically: #{Time.now - start}"
+    end
+  end
+
+  context 'when primary is changed, then more operations are performed' do
+    min_server_fcv '4.2'
+    require_topology :replica_set
+
+    let(:options) do
+      { max_pool_size: 10, min_pool_size: 5 }
+    end
+
+    let(:more_threads) do
+      PossiblyConcurrentArray.new.tap do |more_threads|
+        5.times do |i|
+          more_threads << Thread.new do
+            10.times do |j|
+              collection.find(a: i+j).to_a
+              sleep 0.01
+              collection.find(a: i+j).to_a
+            end
+          end
+        end
+      end
+    end
+
+    let(:threads) do
+      threads = PossiblyConcurrentArray.new
+
+      5.times do |i|
+        threads << Thread.new do
+          10.times do |j|
+            collection.find(a: i+j).to_a
+            sleep 0.01
+            collection.find(a: i+j).to_a
+          end
+        end
+      end
+
+      threads << Thread.new do
+        # Wait for other threads to terminate first, otherwise we get an error
+        # when trying to perform operations during primary change
+        sleep 1
+
+        @primary_chane_start = Time.now
+        ClusterTools.instance.change_primary
+        @primary_change_end = Time.now
+
+        # Primary change is complete; execute more operations
+        more_threads.collect { |t| t.join }
+      end
+      threads
+    end
+
+    it 'does not error' do
+      threads
+      start = Time.now
+      expect do
+        threads.each do |t|
+          t.join
+        end
+      end.not_to raise_error
+      puts "[Connection Pool Timing] Duration before primary change: #{@primary_chane_start - start}. "\
+        "Duration after primary change: #{Time.now - @primary_change_end}"
+    end
+  end
+end

data/spec/support/auth.rb

@@ -0,0 +1,113 @@
+# Copyright (C) 2014-2019 MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+RSpec::Matchers.define :have_blank_credentials do 
+  match do |client|
+    %i(auth_mech auth_mech_properties auth_source password user).all? do |key|
+      client.options[key].nil?
+    end
+  end
+
+  failure_message do |client|
+    "Expected client to have blank credentials, but got the following credentials: \n\n" +
+      client.options.inspect
+  end
+end
+
+module Mongo
+  module Auth
+    class Spec
+
+      attr_reader :description
+      attr_reader :tests
+
+      def initialize(file)
+        file = File.new(file)
+        @spec = YAML.load(ERB.new(file.read).result)
+        file.close
+        @description = File.basename(file)
+      end
+
+      def tests
+        @tests ||= @spec['tests'].collect do |spec|
+          Test.new(spec)
+        end
+      end
+    end
+
+    class Test
+      attr_reader :description
+      attr_reader :uri_string
+
+      def initialize(spec)
+        @spec = spec
+        @description = @spec['description']
+        @uri_string = @spec['uri']
+      end
+
+      def valid?
+        @spec['valid']
+      end
+
+      def credential
+        @spec['credential']
+      end
+
+      def client
+        @client ||= ClientRegistry.instance.new_local_client(@spec['uri'], monitoring_io: false)
+      end
+
+      def expected_credential
+        expected_credential = {
+          'auth_source' => expected_auth_source,
+        }
+
+        if credential['username']
+          expected_credential['user'] = credential['username']
+          expected_credential['password'] = credential['password']
+        end
+
+        if credential['mechanism']
+          expected_credential['auth_mech'] = expected_auth_mech
+        end
+
+        if credential['mechanism_properties']
+          expected_credential['auth_mech_properties'] = expected_auth_mech_properties
+        end
+
+        expected_credential
+      end
+
+      def received_credential
+        client.options.select do |k, _|
+          %w(auth_mech auth_mech_properties auth_source password user).include?(k)
+        end
+      end
+
+      private
+
+      def expected_auth_mech
+        Mongo::URI::AUTH_MECH_MAP[credential['mechanism']]
+      end
+
+      def expected_auth_mech_properties
+        credential['mechanism_properties'].keys.map(&:downcase)
+      end
+
+      def expected_auth_source
+        return :external if credential['source'] == '$external'
+        credential['source']
+      end
+    end
+  end
+end

data/spec/support/background_thread_registry.rb

@@ -0,0 +1,63 @@
+require 'singleton'
+require 'ostruct'
+
+module Mongo
+  module BackgroundThread
+
+    alias :start_without_tracking! :start!
+
+    def start!
+      start_without_tracking!.tap do |thread|
+        BackgroundThreadRegistry.instance.register(self, thread)
+      end
+    end
+  end
+end
+
+class BackgroundThreadRegistry
+  include Singleton
+
+  def initialize
+    @lock = Mutex.new
+    @records = []
+  end
+
+  def register(object, thread)
+    @lock.synchronize do
+      @records << OpenStruct.new(
+        thread: thread,
+        object: object,
+        example: $current_example,
+      )
+    end
+  end
+
+  def verify_empty!
+    @lock.synchronize do
+      alive_thread_records = @records.select { |record| record.thread.alive? }
+      if alive_thread_records.any?
+        msg = "Live background threads after closing all clients:"
+        alive_thread_records.each do |record|
+          msg << "\n  #{record.object}"
+          if record.object.respond_to?(:options)
+            msg << "\n  #{record.object.options}"
+          end
+          msg << "\n  in #{record.example.id} #{record.example.full_description}"
+        end
+        raise msg
+      end
+      @records.clear
+    end
+  end
+end
+
+RSpec.configure do |config|
+  config.around do |example|
+    $current_example = example
+    begin
+      example.run
+    ensure
+      $current_example = nil
+    end
+  end
+end

data/spec/support/client_registry.rb

@@ -209,7 +209,16 @@ class ClientRegistry
 
   def close_local_clients
     @lock.synchronize do
-      @local_clients.map(&:close)
+      @local_clients.each do |client|
+        cluster = client.cluster
+        # Disconnect this cluster if and only if it is not shared with
+        # any of the global clients we know about.
+        if @global_clients.none? { |name, global_client|
+          cluster.object_id == global_client.cluster.object_id
+        }
+          cluster.disconnect!
+        end
+      end
       @local_clients = []
     end
   end
@@ -219,7 +228,7 @@ class ClientRegistry
     close_local_clients
     @lock.synchronize do
       @global_clients.each do |name, client|
-        client.close(true)
+        client.close
       end
     end
   end

data/spec/support/cluster_config.rb

@@ -3,36 +3,19 @@ require 'singleton'
 class ClusterConfig
   include Singleton
 
-  def basic_client
-    # Do not cache the result here so that if the client gets closed,
-    # client registry reconnects it in subsequent tests
-    ClientRegistry.instance.global_client('basic')
-  end
-
   def single_server?
-    basic_client.cluster.servers.length == 1
-  end
-
-  def mongos?
-    if @mongos.nil?
-      basic_client.cluster.next_primary
-      @mongos = basic_client.cluster.topology.is_a?(Mongo::Cluster::Topology::Sharded)
-    end
-    @mongos
+    determine_cluster_config
+    @single_server
   end
 
   def replica_set_name
-    @replica_set_name ||= begin
-      basic_client.cluster.next_primary
-      basic_client.cluster.topology.replica_set_name
-    end
+    determine_cluster_config
+    @replica_set_name
   end
 
   def server_version
-    @server_version ||= begin
-      client = ClientRegistry.instance.global_client('authorized')
-      client.database.command(buildInfo: 1).first['version']
-    end
+    determine_cluster_config
+    @server_version
   end
 
   def short_server_version
@@ -40,11 +23,8 @@ class ClusterConfig
   end
 
   def fcv
-    @fcv ||= begin
-      client = ClientRegistry.instance.global_client('root_authorized')
-      rv = client.use(:admin).command(getParameter: 1, featureCompatibilityVersion: 1).first['featureCompatibilityVersion']
-      rv['version'] || rv
-    end
+    determine_cluster_config
+    @fcv
   end
 
   # Per https://jira.mongodb.org/browse/SERVER-39052, working with FCV
@@ -52,7 +32,7 @@ class ClusterConfig
   # less than 3.4. This method returns FCV on 3.4+ servers when in single
   # or RS topologies, and otherwise returns the major.minor server version.
   def fcv_ish
-    if server_version >= '3.4' && !mongos?
+    if server_version >= '3.4' && topology != :sharded
       fcv
     else
       if short_server_version == '4.1'
@@ -64,18 +44,13 @@ class ClusterConfig
   end
 
   def primary_address
-    @primary_address ||= begin
-      client = ClientRegistry.instance.global_client('authorized')
-      if client.cluster.topology.is_a?(Mongo::Cluster::Topology::ReplicaSetWithPrimary)
-        client.cluster.servers.detect { |server| server.primary? }.address
-      else
-        client.cluster.servers.first.address
-      end.seed
-    end
+    determine_cluster_config
+    @primary_address
   end
 
   def primary_address_str
-    primary_address
+    determine_cluster_config
+    @primary_address.seed
   end
 
   def primary_address_host
@@ -88,6 +63,11 @@ class ClusterConfig
     both.split(':')[1] || 27017
   end
 
+  def primary_description
+    determine_cluster_config
+    @primary_description
+  end
+
   # Try running a command on the admin database to see if the mongod was
   # started with auth.
   def auth_enabled?
@@ -102,14 +82,8 @@ class ClusterConfig
   end
 
   def topology
-    @topology ||= begin
-      topology = basic_client.cluster.topology.class.name.sub(/.*::/, '')
-      topology = topology.gsub(/([A-Z])/) { |match| '_' + match.downcase }.sub(/^_/, '')
-      if topology =~ /^replica_set/
-        topology = 'replica_set'
-      end
-      topology.to_sym
-    end
+    determine_cluster_config
+    @topology
   end
 
   def storage_engine
@@ -121,6 +95,9 @@ class ClusterConfig
         client = ClientRegistry.instance.global_client('root_authorized')
         if topology == :sharded
           shards = client.use(:admin).command(listShards: 1).first
+          if shards['shards'].empty?
+            raise 'Shards are empty'
+          end
           shard = shards['shards'].first
           address_str = shard['host'].sub(/^.*\//, '').sub(/,.*/, '')
           client = ClusterTools.instance.direct_client(address_str,
@@ -136,4 +113,46 @@ class ClusterConfig
       end
     end
   end
+
+  private
+
+  def determine_cluster_config
+    return if @primary_address
+
+    # Run all commands to figure out the cluster configuration from the same
+    # client. This is somewhat wasteful when running a single test, but reduces
+    # test runtime for the suite overall because all commands are sent on the
+    # same connection rather than each command connecting to the cluster by
+    # itself.
+    client = ClientRegistry.instance.global_client('root_authorized')
+
+    primary = client.cluster.next_primary
+    @primary_address = primary.address
+    @primary_description = primary.description
+    @replica_set_name = client.cluster.topology.replica_set_name
+
+    @topology ||= begin
+      topology = client.cluster.topology.class.name.sub(/.*::/, '')
+      topology = topology.gsub(/([A-Z])/) { |match| '_' + match.downcase }.sub(/^_/, '')
+      if topology =~ /^replica_set/
+        topology = 'replica_set'
+      end
+      topology.to_sym
+    end
+
+    @single_server = client.cluster.servers_list.length == 1
+
+    @server_version = client.database.command(buildInfo: 1).first['version']
+
+    if @topology != :sharded && short_server_version >= '3.4'
+      rv = client.use(:admin).command(getParameter: 1, featureCompatibilityVersion: 1).first['featureCompatibilityVersion']
+      @fcv = rv['version'] || rv
+    end
+  end
+
+  def basic_client
+    # Do not cache the result here so that if the client gets closed,
+    # client registry reconnects it in subsequent tests
+    ClientRegistry.instance.global_client('basic')
+  end
 end