mongo 2.12.4 → 2.14.0.rc1

data/spec/stress/cleanup_spec.rb

@@ -0,0 +1,58 @@
+require 'spec_helper'
+
+describe 'Cleanup stress test' do
+  require_stress
+
+  let(:options) do
+    SpecConfig.instance.all_test_options
+  end
+
+  before(:all) do
+    # load if necessary
+    ClusterConfig.instance.primary_address
+    ClientRegistry.instance.close_all_clients
+  end
+
+  context 'single client disconnect/reconnect' do
+    let(:client) do
+      new_local_client([ClusterConfig.instance.primary_address.seed], options)
+    end
+
+    it 'cleans up' do
+      client
+      client.cluster.servers_list.map(&:scan!)
+
+      sleep 1
+      GC.start
+
+      start_resources = resources
+
+      500.times do
+        client.close
+        client.reconnect
+      end
+
+      sleep 1
+      GC.start
+
+      end_resources = resources
+
+      # There seem to be a temporary file descriptor leak in CI,
+      # where we start with 75 fds and end with 77 fds.
+      # Allow a few to be leaked, run more iterations to ensure the leak
+      # is not a real one.
+      # Sometimes we end with fewer fds than we started with also...
+      end_resources[:open_file_count].should >= start_resources[:open_file_count] - 3
+      end_resources[:open_file_count].should <= start_resources[:open_file_count] + 3
+
+      end_resources[:running_thread_count].should == start_resources[:running_thread_count]
+    end
+  end
+
+  def resources
+    {
+      open_file_count: Dir["/proc/#{Process.pid}/fd/*"].count,
+      running_thread_count: Thread.list.select { |thread| thread.status == 'run' }.count,
+    }
+  end
+end

data/spec/stress/connection_pool_stress_spec.rb

@@ -1,11 +1,7 @@
 require 'spec_helper'
 
 describe 'Connection pool stress test' do
-  before(:all) do
-    if !SpecConfig.instance.stress_spec?
-      skip 'Stress spec not enabled'
-    end
-  end
+  require_stress
 
   let(:options) do
     { max_pool_size: 5, min_pool_size: 3 }
@@ -25,9 +21,9 @@ describe 'Connection pool stress test' do
     [].tap do |threads|
       thread_count.times do |i|
         threads << Thread.new do
-          10.times do |j|
+          100.times do |j|
             collection.find(a: i+j).to_a
-            sleep 0.5
+            sleep 0.1
             collection.find(a: i+j).to_a
           end
         end
@@ -136,7 +132,7 @@ describe 'Connection pool stress test' do
     end
 
     context '25 threads, max pool size 5' do
-      let(:thread_count) { 20 }
+      let(:thread_count) { 25 }
 
       it_behaves_like 'does not raise error'
     end
@@ -187,17 +183,19 @@ describe 'Connection pool stress test' do
       # The populator calls create_and_add_connection, which calls connection.connect!
       # and raises if an error occurs (eg, an auth error), so in this test we
       # randomly raise errors on this method.
-      expect(server.pool).to receive(:create_and_add_connection).at_least(:once).and_wrap_original { |m, *args|
-        if rand < 0.05
+      expect(server.pool).to receive(:create_and_add_connection).at_least(:once).and_wrap_original do |m, *args|
+        if rand < 0.1
           raise Mongo::Error::SocketError
         else
           m.call(*args)
         end
-      }
+      end
 
-      expect {
+      server.pool.clear(stop_populator: false)
+
+      expect do
         threads.collect { |t| t.join }
-      }.not_to raise_error
+      end.not_to raise_error
     end
   end
 end

data/spec/stress/connection_pool_timing_spec.rb

@@ -1,13 +1,10 @@
 require 'spec_helper'
 
 describe 'Connection pool timing test' do
-  before(:all) do
-    if !SpecConfig.instance.stress_spec?
-      skip 'Stress spec not enabled'
-    end
-
-    ClientRegistry.instance.close_all_clients
+  require_stress
+  clean_slate_for_all
 
+  before(:all) do
     # This set up is taken from the step_down_spec file. In a future PR, ClusterTools
     # may be modified so this set up is no longer necessary.
     if ClusterConfig.instance.fcv_ish >= '4.2' && ClusterConfig.instance.topology == :replica_set

data/spec/stress/fork_reconnect_stress_spec.rb

@@ -0,0 +1,109 @@
+require 'spec_helper'
+
+describe 'fork reconnect' do
+  require_fork
+  only_mri
+
+  # On multi-shard sharded clusters a succeeding write request does not
+  # guarantee that the next operation will succeed (since it could be sent to
+  # another shard with a dead connection).
+  require_no_multi_shard
+
+  # On Ruby 2.3 $?.exitstatus is sometimes nil after Process.wait returns which
+  # is not supposed to happen.
+  ruby_version_gte '2.4'
+
+  require_stress
+
+  let(:client) { authorized_client }
+
+  describe 'client' do
+    it 'works after fork' do
+      # Perform a write so that we discover the current primary.
+      # Previous test may have stepped down the server that authorized client
+      # considers the primary.
+      # In standalone deployments there are no retries, hence execute the
+      # operation twice manually.
+      client['foo'].insert_one(test: 1) rescue nil
+      client['foo'].insert_one(test: 1)
+
+      pids = []
+      deadline = Time.now + 5
+      1.upto(10) do
+        if pid = fork
+          pids << pid
+        else
+          Utils.wrap_forked_child do
+            while Time.now < deadline
+              client.database.command(ismaster: 1).should be_a(Mongo::Operation::Result)
+            end
+          end
+        end
+      end
+
+      while Time.now < deadline
+        # Use a read which is retried in case of an error
+        client['foo'].find(test: 1).to_a
+      end
+
+      pids.each do |pid|
+        pid, status = Process.wait2(pid)
+        status.exitstatus.should == 0
+      end
+    end
+
+    context 'when parent is operating on client during the fork', retry: 3 do
+      # This test intermittently fails in evergreen with pool size of 5,
+      # with a number of pending connections in the pool.
+      # The reason could be that handshaking is slow or operations are slow
+      # post handshakes.
+      # Sometimes it seems the monitoring connection experiences network
+      # errors (despite being a loopback connection) which causes the test
+      # to fail as then server selection fails.
+      # The retry: 3 is to deal with network errors on monitoring connection.
+
+      let(:client) { authorized_client.with(max_pool_size: 10,
+        wait_queue_timeout: 10, socket_timeout: 2, connect_timeout: 2) }
+
+      it 'works' do
+        client.database.command(ismaster: 1).should be_a(Mongo::Operation::Result)
+
+        threads = []
+        5.times do
+          threads << Thread.new do
+            loop do
+              client['foo'].find(test: 1).to_a
+            end
+          end
+        end
+
+        pids = []
+        deadline = Time.now + 5
+        10.times do
+          if pid = fork
+            pids << pid
+          else
+            Utils.wrap_forked_child do
+              while Time.now < deadline
+                client.database.command(ismaster: 1).should be_a(Mongo::Operation::Result)
+              end
+            end
+          end
+        end
+
+        while Time.now < deadline
+          sleep 0.1
+        end
+
+        threads.map(&:kill)
+        threads.map(&:join)
+
+        pids.each do |pid|
+          pid, status = Process.wait2(pid)
+          status.exitstatus.should == 0
+        end
+      end
+
+    end
+  end
+end

data/spec/support/authorization.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2009-2019 MongoDB, Inc.
+# Copyright (C) 2009-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -65,12 +65,10 @@ module Authorization
     #
     # @since 2.5.1
     context.let(:authorized_client_with_retry_writes) do
-      EventSubscriber.clear_events!
       ClientRegistry.instance.global_client('authorized_with_retry_writes')
     end
 
     context.let(:authorized_client_without_retry_writes) do
-      EventSubscriber.clear_events!
       ClientRegistry.instance.global_client('authorized_without_retry_writes')
     end
 
@@ -86,14 +84,6 @@ module Authorization
       ClientRegistry.instance.global_client('authorized_without_any_retries')
     end
 
-    # Provides an authorized mongo client that has a Command subscriber.
-    #
-    # @since 2.5.1
-    context.let(:subscribed_client) do
-      EventSubscriber.clear_events!
-      ClientRegistry.instance.global_client('subscribed')
-    end
-
     # Provides an unauthorized mongo client on the default test database.
     #
     # @since 2.0.0

data/spec/support/aws_utils.rb

@@ -0,0 +1,62 @@
+autoload :Byebug, 'byebug'
+autoload :Paint, 'paint'
+
+module Aws
+  autoload :CloudWatchLogs, 'aws-sdk-cloudwatchlogs'
+  autoload :Credentials, 'aws-sdk-core'
+  autoload :EC2, 'aws-sdk-ec2'
+  autoload :ECS, 'aws-sdk-ecs'
+  autoload :IAM, 'aws-sdk-iam'
+  autoload :STS, 'aws-sdk-core'
+end
+
+module AwsUtils
+  NAMESPACE = 'mdb-ruby'.freeze
+
+  AWS_AUTH_REGULAR_USER_NAME = "#{NAMESPACE}.aws-auth-regular".freeze
+
+  AWS_AUTH_ASSUME_ROLE_NAME = "#{NAMESPACE}.assume-role".freeze
+
+  AWS_AUTH_SECURITY_GROUP_NAME = "#{NAMESPACE}.ssh".freeze
+
+  AWS_AUTH_VPC_GATEWAY_NAME = NAMESPACE
+
+  AWS_AUTH_VPC_SECURITY_GROUP_NAME = "#{NAMESPACE}.vpc-ssh".freeze
+
+  AWS_AUTH_VPC_CIDR = "10.42.142.64/28".freeze
+
+  AWS_AUTH_EC2_AMI_NAMES = {
+    # https://wiki.debian.org/Cloud/AmazonEC2Image/Buster
+    'debian10' => 'debian-10-amd64-20200210-166',
+    'ubuntu1604' => 'ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-server-20200317',
+  }.freeze
+
+  AWS_AUTH_EC2_INSTANCE_NAME = "#{NAMESPACE}.aws-auth)".freeze
+
+  AWS_AUTH_INSTANCE_PROFILE_NAME = "#{NAMESPACE}.ip".freeze
+
+  AWS_AUTH_ASSUME_ROLE_USER_POLICY_NAME = "#{NAMESPACE}.assume-role-user-policy".freeze
+
+  AWS_AUTH_EC2_ROLE_NAME = "#{NAMESPACE}.ec2-role".freeze
+
+  AWS_AUTH_ECS_CLUSTER_NAME = "#{NAMESPACE}_aws-auth".freeze
+
+  AWS_AUTH_ECS_TASK_FAMILY = "#{NAMESPACE}_aws-auth".freeze
+
+  AWS_AUTH_ECS_SERVICE_NAME = "#{NAMESPACE}_aws-auth".freeze
+
+  AWS_AUTH_ECS_LOG_GROUP = "/ecs/#{NAMESPACE}/aws-auth-ecs".freeze
+
+  AWS_AUTH_ECS_LOG_STREAM_PREFIX = "task".freeze
+
+  # This role allows ECS tasks access to output logs to CloudWatch.
+  AWS_AUTH_ECS_EXECUTION_ROLE_NAME = "#{NAMESPACE}.ecs-execution-role".freeze
+
+  # This role is assumed by ECS tasks.
+  AWS_AUTH_ECS_TASK_ROLE_NAME = "#{NAMESPACE}.ecs-task-role".freeze
+
+  autoload :Base, 'support/aws_utils/base'
+  autoload :Inspector, 'support/aws_utils/inspector'
+  autoload :Orchestrator, 'support/aws_utils/orchestrator'
+  autoload :Provisioner, 'support/aws_utils/provisioner'
+end

data/spec/support/aws_utils/base.rb

@@ -0,0 +1,134 @@
+module AwsUtils
+  class Base
+
+    def initialize(access_key_id: nil, secret_access_key: nil, region: nil, **options)
+      @access_key_id = access_key_id || ENV['AWS_ACCESS_KEY_ID']
+      @secret_access_key = secret_access_key || ENV['AWS_SECRET_ACCESS_KEY']
+      @region = region || ENV['AWS_REGION']
+      @options = options
+    end
+
+    attr_reader :access_key_id, :secret_access_key, :region, :options
+
+    private
+
+    def detect_object(resp, resp_attr, object_attr, value)
+      resp.each do |batch|
+        batch.send(resp_attr).each do |object|
+          if object.send(object_attr) == value
+            return object
+          end
+        end
+      end
+      nil
+    end
+
+    def ssh_security_group_id
+      begin
+        sg = ec2_client.describe_security_groups(
+          group_names: [AWS_AUTH_SECURITY_GROUP_NAME],
+        ).security_groups.first
+        sg&.group_id
+      rescue Aws::EC2::Errors::InvalidGroupNotFound
+        # Unlike almost all other describe calls, this one raises an exception
+        # if there isn't a security group matching the criteria.
+        nil
+      end
+    end
+
+    def ssh_security_group_id!
+      ssh_security_group_id.tap do |security_group_id|
+        if security_group_id.nil?
+          raise 'Security group does not exist, please provision'
+        end
+      end
+    end
+
+    def ssh_vpc_security_group_id
+      begin
+        # If the top-level group_name parameter is used, only non-VPC
+        # security groups are returned which does not find the VPC group
+        # we are looking for here.
+        sg = ec2_client.describe_security_groups(
+          filters: [{
+            name: 'group-name',
+            values: [AWS_AUTH_VPC_SECURITY_GROUP_NAME],
+          }],
+        ).security_groups.first
+        sg&.group_id
+      rescue Aws::EC2::Errors::InvalidGroupNotFound
+        # Unlike almost all other describe calls, this one raises an exception
+        # if there isn't a security group matching the criteria.
+        nil
+      end
+    end
+
+    def ssh_vpc_security_group_id!
+      ssh_vpc_security_group_id.tap do |security_group_id|
+        if security_group_id.nil?
+          raise 'Security group does not exist, please provision'
+        end
+      end
+    end
+
+    def subnet_id
+      # This directly queries the subnets for the one with the expected
+      # CIDR block, to save on the number of requests made to AWS.
+      ec2_client.describe_subnets(
+        filters: [{
+          name: 'cidr-block',
+          values: [AWS_AUTH_VPC_CIDR],
+        }],
+      ).subnets.first&.subnet_id
+    end
+
+    def subnet_id!
+      subnet_id.tap do |subnet_id|
+        if subnet_id.nil?
+          raise 'Subnet does not exist, please provision'
+        end
+      end
+    end
+
+    def credentials
+      Aws::Credentials.new(access_key_id, secret_access_key)
+    end
+
+    public
+
+    def ec2_client
+      @ec2_client ||= Aws::EC2::Client.new(
+        region: region,
+        credentials: credentials,
+      )
+    end
+
+    def iam_client
+      iam_client = Aws::IAM::Client.new(
+        region: region,
+        credentials: credentials,
+      )
+    end
+
+    def ecs_client
+      @ecs_client ||= Aws::ECS::Client.new(
+        region: region,
+        credentials: credentials,
+      )
+    end
+
+    def logs_client
+      @logs_client ||= Aws::CloudWatchLogs::Client.new(
+        region: region,
+        credentials: credentials,
+      )
+    end
+
+    def sts_client
+      @sts_client ||= Aws::STS::Client.new(
+        region: region,
+        credentials: credentials,
+      )
+    end
+  end
+end