mongo 2.10.5 → 2.15.1

data/spec/stress/connection_pool_timing_spec.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+require 'spec_helper'
+
+describe 'Connection pool timing test' do
+  require_stress
+  clean_slate_for_all
+
+  before(:all) do
+    # This set up is taken from the step_down_spec file. In a future PR, ClusterTools
+    # may be modified so this set up is no longer necessary.
+    if ClusterConfig.instance.fcv_ish >= '4.2' && ClusterConfig.instance.topology == :replica_set
+      ClusterTools.instance.set_election_timeout(5)
+      ClusterTools.instance.set_election_handoff(false)
+    end
+  end
+
+  after(:all) do
+    if ClusterConfig.instance.fcv_ish >= '4.2' && ClusterConfig.instance.topology == :replica_set
+      ClusterTools.instance.set_election_timeout(10)
+      ClusterTools.instance.set_election_handoff(true)
+      ClusterTools.instance.reset_priorities
+    end
+  end
+
+  let(:client) do
+    authorized_client.with(options.merge(monitoring: true))
+  end
+
+  let!(:collection) do
+    client[authorized_collection.name].tap do |collection|
+      collection.drop
+      collection.insert_many(documents)
+    end
+  end
+
+  let(:documents) do
+    [].tap do |documents|
+      10000.times do |i|
+        documents << { a: i}
+      end
+    end
+  end
+
+  let(:operation_threads) do
+    [].tap do |threads|
+      thread_count.times do |i|
+        threads << Thread.new do
+          100.times do |j|
+            collection.find(a: i+j).to_a
+            sleep 0.01
+            collection.find(a: i+j).to_a
+          end
+        end
+      end
+    end
+  end
+
+  let(:thread_count) { 5 }
+
+  context 'when there is no max idle time' do
+    let(:options) do
+      { max_pool_size: 10, min_pool_size: 5 }
+    end
+
+    let(:threads) { operation_threads }
+
+    it 'does not error' do
+      start = Time.now
+      expect {
+        threads.collect { |t| t.join }
+      }.not_to raise_error
+      puts "[Connection Pool Timing] Duration with no max idle time: #{Time.now - start}"
+    end
+  end
+
+  context 'when there is a low max idle time' do
+    let(:options) do
+      { max_pool_size: 10, min_pool_size: 5, max_idle_time: 0.1 }
+    end
+
+    let(:threads) { operation_threads }
+
+    it 'does not error' do
+      start = Time.now
+      expect {
+        threads.collect { |t| t.join }
+      }.not_to raise_error
+      puts "[Connection Pool Timing] Duration with low max idle time: #{Time.now - start}"
+    end
+  end
+
+  context 'when clear is called periodically' do
+    let(:options) do
+      { max_pool_size: 10, min_pool_size: 5 }
+    end
+
+    let(:threads) do
+      threads = operation_threads
+      threads << Thread.new do
+        10.times do
+          sleep 0.1
+          client.cluster.next_primary.pool.clear
+        end
+      end
+      threads
+    end
+
+    it 'does not error' do
+      start = Time.now
+      expect {
+        threads.collect { |t| t.join }
+      }.not_to raise_error
+      puts "[Connection Pool Timing] Duration when clear is called periodically: #{Time.now - start}"
+    end
+  end
+
+  context 'when primary is changed, then more operations are performed' do
+    min_server_fcv '4.2'
+    require_topology :replica_set
+
+    let(:options) do
+      { max_pool_size: 10, min_pool_size: 5 }
+    end
+
+    let(:more_threads) do
+      PossiblyConcurrentArray.new.tap do |more_threads|
+        5.times do |i|
+          more_threads << Thread.new do
+            10.times do |j|
+              collection.find(a: i+j).to_a
+              sleep 0.01
+              collection.find(a: i+j).to_a
+            end
+          end
+        end
+      end
+    end
+
+    let(:threads) do
+      threads = PossiblyConcurrentArray.new
+
+      5.times do |i|
+        threads << Thread.new do
+          10.times do |j|
+            collection.find(a: i+j).to_a
+            sleep 0.01
+            collection.find(a: i+j).to_a
+          end
+        end
+      end
+
+      threads << Thread.new do
+        # Wait for other threads to terminate first, otherwise we get an error
+        # when trying to perform operations during primary change
+        sleep 1
+
+        @primary_change_start = Time.now
+        ClusterTools.instance.change_primary
+        @primary_change_end = Time.now
+
+        # Primary change is complete; execute more operations
+        more_threads.collect { |t| t.join }
+      end
+      threads
+    end
+
+    # On JRuby, sometimes the following error is produced indicating
+    # possible data corruption or an interpreter bug:
+    # RSpec::Expectations::ExpectationNotMetError: expected no Exception, got #<Mongo::Error::OperationFailure: Invalid ns [ruby-driver.] (73) (on localhost:27018, modern retry, attempt 1) (on localhost:27018, modern retry, attempt 1)>
+    it 'does not error', retry: (BSON::Environment.jruby? ? 3 : 1) do
+      threads
+      start = Time.now
+      expect do
+        threads.each do |t|
+          t.join
+        end
+      end.not_to raise_error
+      puts "[Connection Pool Timing] Duration before primary change: #{@primary_change_start - start}. "\
+        "Duration after primary change: #{Time.now - @primary_change_end}"
+    end
+  end
+end

data/spec/stress/fork_reconnect_stress_spec.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+require 'spec_helper'
+
+describe 'fork reconnect' do
+  require_fork
+  require_mri
+
+  # On multi-shard sharded clusters a succeeding write request does not
+  # guarantee that the next operation will succeed (since it could be sent to
+  # another shard with a dead connection).
+  require_no_multi_shard
+
+  # On Ruby 2.3 $?.exitstatus is sometimes nil after Process.wait returns which
+  # is not supposed to happen.
+  ruby_version_gte '2.4'
+
+  require_stress
+
+  let(:client) { authorized_client }
+
+  describe 'client' do
+    it 'works after fork' do
+      # Perform a write so that we discover the current primary.
+      # Previous test may have stepped down the server that authorized client
+      # considers the primary.
+      # In standalone deployments there are no retries, hence execute the
+      # operation twice manually.
+      client['foo'].insert_one(test: 1) rescue nil
+      client['foo'].insert_one(test: 1)
+
+      pids = []
+      deadline = Time.now + 5
+      1.upto(10) do
+        if pid = fork
+          pids << pid
+        else
+          Utils.wrap_forked_child do
+            while Time.now < deadline
+              client.database.command(hello: 1).should be_a(Mongo::Operation::Result)
+            end
+          end
+        end
+      end
+
+      while Time.now < deadline
+        # Use a read which is retried in case of an error
+        client['foo'].find(test: 1).to_a
+      end
+
+      pids.each do |pid|
+        pid, status = Process.wait2(pid)
+        status.exitstatus.should == 0
+      end
+    end
+
+    context 'when parent is operating on client during the fork', retry: 3 do
+      # This test intermittently fails in evergreen with pool size of 5,
+      # with a number of pending connections in the pool.
+      # The reason could be that handshaking is slow or operations are slow
+      # post handshakes.
+      # Sometimes it seems the monitoring connection experiences network
+      # errors (despite being a loopback connection) which causes the test
+      # to fail as then server selection fails.
+      # The retry: 3 is to deal with network errors on monitoring connection.
+
+      let(:client) { authorized_client.with(max_pool_size: 10,
+        wait_queue_timeout: 10, socket_timeout: 2, connect_timeout: 2) }
+
+      it 'works' do
+        client.database.command(hello: 1).should be_a(Mongo::Operation::Result)
+
+        threads = []
+        5.times do
+          threads << Thread.new do
+            loop do
+              client['foo'].find(test: 1).to_a
+            end
+          end
+        end
+
+        pids = []
+        deadline = Time.now + 5
+        10.times do
+          if pid = fork
+            pids << pid
+          else
+            Utils.wrap_forked_child do
+              while Time.now < deadline
+                client.database.command(hello: 1).should be_a(Mongo::Operation::Result)
+              end
+            end
+          end
+        end
+
+        while Time.now < deadline
+          sleep 0.1
+        end
+
+        threads.map(&:kill)
+        threads.map(&:join)
+
+        pids.each do |pid|
+          pid, status = Process.wait2(pid)
+          status.exitstatus.should == 0
+        end
+      end
+
+    end
+  end
+end

data/spec/stress/push_monitor_close_spec.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+require 'spec_helper'
+
+# This test repeatedly creates and closes clients across several threads.
+# Its goal is to ensure that the push monitor connections specifically get
+# closed without any errors or warnings being reported to applications.
+#
+# Although the test is specifically meant to test 4.4+ servers (that utilize
+# the push monitor) in non-LB connections, run it everywhere for good measure.
+describe 'Push monitor close test' do
+  require_stress
+
+  let(:options) do
+    SpecConfig.instance.all_test_options
+  end
+
+  before(:all) do
+    # load if necessary
+    ClusterConfig.instance.primary_address
+    ClientRegistry.instance.close_all_clients
+  end
+
+  it 'does not warn/error on cleanup' do
+    Mongo::Logger.logger.should_not receive(:warn)
+
+    threads = 10.times.map do
+      Thread.new do
+        10.times do
+          client = new_local_client([ClusterConfig.instance.primary_address.seed], options)
+          if rand > 0.33
+            client.command(ping: 1)
+            sleep(rand * 3)
+          end
+          client.close
+          STDOUT << '.'
+        end
+      end
+    end
+    threads.each(&:join)
+    puts
+  end
+end

data/spec/support/authorization.rb

@@ -1,4 +1,7 @@
-# Copyright (C) 2009-2019 MongoDB, Inc.
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2009-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -65,12 +68,10 @@ module Authorization
     #
     # @since 2.5.1
     context.let(:authorized_client_with_retry_writes) do
-      EventSubscriber.clear_events!
       ClientRegistry.instance.global_client('authorized_with_retry_writes')
     end
 
     context.let(:authorized_client_without_retry_writes) do
-      EventSubscriber.clear_events!
       ClientRegistry.instance.global_client('authorized_without_retry_writes')
     end
 
@@ -86,14 +87,6 @@ module Authorization
       ClientRegistry.instance.global_client('authorized_without_any_retries')
     end
 
-    # Provides an authorized mongo client that has a Command subscriber.
-    #
-    # @since 2.5.1
-    context.let(:subscribed_client) do
-      EventSubscriber.clear_events!
-      ClientRegistry.instance.global_client('subscribed')
-    end
-
     # Provides an unauthorized mongo client on the default test database.
     #
     # @since 2.0.0

data/spec/support/aws_utils/base.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+module AwsUtils
+  class Base
+
+    def initialize(access_key_id: nil, secret_access_key: nil, region: nil, **options)
+      @access_key_id = access_key_id || ENV['AWS_ACCESS_KEY_ID']
+      @secret_access_key = secret_access_key || ENV['AWS_SECRET_ACCESS_KEY']
+      @region = region || ENV['AWS_REGION']
+      @options = options
+    end
+
+    attr_reader :access_key_id, :secret_access_key, :region, :options
+
+    private
+
+    def detect_object(resp, resp_attr, object_attr, value)
+      resp.each do |batch|
+        batch.send(resp_attr).each do |object|
+          if object.send(object_attr) == value
+            return object
+          end
+        end
+      end
+      nil
+    end
+
+    def ssh_security_group_id
+      begin
+        sg = ec2_client.describe_security_groups(
+          group_names: [AWS_AUTH_SECURITY_GROUP_NAME],
+        ).security_groups.first
+        sg&.group_id
+      rescue Aws::EC2::Errors::InvalidGroupNotFound
+        # Unlike almost all other describe calls, this one raises an exception
+        # if there isn't a security group matching the criteria.
+        nil
+      end
+    end
+
+    def ssh_security_group_id!
+      ssh_security_group_id.tap do |security_group_id|
+        if security_group_id.nil?
+          raise 'Security group does not exist, please provision'
+        end
+      end
+    end
+
+    def ssh_vpc_security_group_id
+      begin
+        # If the top-level group_name parameter is used, only non-VPC
+        # security groups are returned which does not find the VPC group
+        # we are looking for here.
+        sg = ec2_client.describe_security_groups(
+          filters: [{
+            name: 'group-name',
+            values: [AWS_AUTH_VPC_SECURITY_GROUP_NAME],
+          }],
+        ).security_groups.first
+        sg&.group_id
+      rescue Aws::EC2::Errors::InvalidGroupNotFound
+        # Unlike almost all other describe calls, this one raises an exception
+        # if there isn't a security group matching the criteria.
+        nil
+      end
+    end
+
+    def ssh_vpc_security_group_id!
+      ssh_vpc_security_group_id.tap do |security_group_id|
+        if security_group_id.nil?
+          raise 'Security group does not exist, please provision'
+        end
+      end
+    end
+
+    def subnet_id
+      # This directly queries the subnets for the one with the expected
+      # CIDR block, to save on the number of requests made to AWS.
+      ec2_client.describe_subnets(
+        filters: [{
+          name: 'cidr-block',
+          values: [AWS_AUTH_VPC_CIDR],
+        }],
+      ).subnets.first&.subnet_id
+    end
+
+    def subnet_id!
+      subnet_id.tap do |subnet_id|
+        if subnet_id.nil?
+          raise 'Subnet does not exist, please provision'
+        end
+      end
+    end
+
+    def credentials
+      Aws::Credentials.new(access_key_id, secret_access_key)
+    end
+
+    public
+
+    def ec2_client
+      @ec2_client ||= Aws::EC2::Client.new(
+        region: region,
+        credentials: credentials,
+      )
+    end
+
+    def iam_client
+      iam_client = Aws::IAM::Client.new(
+        region: region,
+        credentials: credentials,
+      )
+    end
+
+    def ecs_client
+      @ecs_client ||= Aws::ECS::Client.new(
+        region: region,
+        credentials: credentials,
+      )
+    end
+
+    def logs_client
+      @logs_client ||= Aws::CloudWatchLogs::Client.new(
+        region: region,
+        credentials: credentials,
+      )
+    end
+
+    def sts_client
+      @sts_client ||= Aws::STS::Client.new(
+        region: region,
+        credentials: credentials,
+      )
+    end
+  end
+end