mongo 2.11.2 → 2.13.0.beta1

data/spec/spec_tests/server_selection_spec.rb

@@ -1,5 +1,7 @@
 require 'spec_helper'
 
+require 'runners/server_selection'
+
 describe 'Server Selection' do
 
   include Mongo::ServerSelection::Read

data/spec/spec_tests/transactions_api_spec.rb

@@ -1,5 +1,10 @@
 require 'spec_helper'
 
+require 'runners/crud'
+require 'runners/transactions'
+
 describe 'Transactions API' do
+  require_wired_tiger
+
   define_transactions_spec_tests(TRANSACTIONS_API_TESTS)
 end

data/spec/spec_tests/transactions_spec.rb

@@ -1,5 +1,10 @@
 require 'spec_helper'
 
+require 'runners/crud'
+require 'runners/transactions'
+
 describe 'Transactions' do
+  require_wired_tiger
+
   define_transactions_spec_tests(TRANSACTIONS_TESTS)
 end

data/spec/spec_tests/uri_options_spec.rb

@@ -1,21 +1,21 @@
-require 'spec_helper'
+require 'lite_spec_helper'
+
+require 'runners/connection_string'
 
 describe 'Uri Options' do
   include Mongo::ConnectionString
 
+  # Since the tests issue global assertions on Mongo::Logger,
+  # we need to close all clients/stop monitoring to avoid monitoring
+  # threads warning and interfering with these assertions
+  clean_slate_for_all_if_possible
+
   URI_OPTIONS_TESTS.each do |file|
 
     spec = Mongo::ConnectionString::Spec.new(file)
 
     context(spec.description) do
 
-      before(:all) do
-        # Since the tests issue global assertions on Mongo::Logger,
-        # we need to close all clients/stop monitoring to avoid monitoring
-        # threads warning and interfering with these assertions
-        ClientRegistry.instance.close_all_clients
-      end
-
       spec.tests.each do |test|
         context "#{test.description}" do
           if test.description.downcase.include?("gssapi")

data/spec/stress/cleanup_spec.rb

@@ -0,0 +1,43 @@
+require 'spec_helper'
+
+describe 'Cleanup stress test' do
+  require_stress
+
+  let(:options) do
+    SpecConfig.instance.all_test_options
+  end
+
+  before(:all) do
+    # load if necessary
+    ClusterConfig.instance.primary_address
+    ClientRegistry.instance.close_all_clients
+  end
+
+  context 'single client disconnect/reconnect' do
+    let(:client) do
+      new_local_client([ClusterConfig.instance.primary_address.seed], options)
+    end
+
+    it 'cleans up' do
+      client
+
+      start_resources = resources
+
+      100.times do
+        client.close
+        client.reconnect
+      end
+
+      end_resources = resources
+
+      end_resources.should == start_resources
+    end
+  end
+
+  def resources
+    {
+      open_file_count: Dir["/proc/#{Process.pid}/fd/*"].count,
+      running_thread_count: Thread.list.select { |thread| thread.status == 'run' }.count,
+    }
+  end
+end

data/spec/stress/connection_pool_stress_spec.rb

@@ -1,11 +1,7 @@
 require 'spec_helper'
 
 describe 'Connection pool stress test' do
-  before(:all) do
-    if !SpecConfig.instance.stress_spec?
-      skip 'Stress spec not enabled'
-    end
-  end
+  require_stress
 
   let(:options) do
     { max_pool_size: 5, min_pool_size: 3 }

data/spec/stress/connection_pool_timing_spec.rb

@@ -1,13 +1,10 @@
 require 'spec_helper'
 
 describe 'Connection pool timing test' do
-  before(:all) do
-    if !SpecConfig.instance.stress_spec?
-      skip 'Stress spec not enabled'
-    end
-
-    ClientRegistry.instance.close_all_clients
+  require_stress
+  clean_slate_for_all
 
+  before(:all) do
     # This set up is taken from the step_down_spec file. In a future PR, ClusterTools
     # may be modified so this set up is no longer necessary.
     if ClusterConfig.instance.fcv_ish >= '4.2' && ClusterConfig.instance.topology == :replica_set
@@ -156,7 +153,7 @@ describe 'Connection pool timing test' do
         # when trying to perform operations during primary change
         sleep 1
 
-        @primary_chane_start = Time.now
+        @primary_change_start = Time.now
         ClusterTools.instance.change_primary
         @primary_change_end = Time.now
 
@@ -166,7 +163,10 @@ describe 'Connection pool timing test' do
       threads
     end
 
-    it 'does not error' do
+    # On JRuby, sometimes the following error is produced indicating
+    # possible data corruption or an interpreter bug:
+    # RSpec::Expectations::ExpectationNotMetError: expected no Exception, got #<Mongo::Error::OperationFailure: Invalid ns [ruby-driver.] (73) (on localhost:27018, modern retry, attempt 1) (on localhost:27018, modern retry, attempt 1)>
+    it 'does not error', retry: (BSON::Environment.jruby? ? 3 : 1) do
       threads
       start = Time.now
       expect do
@@ -174,7 +174,7 @@ describe 'Connection pool timing test' do
           t.join
         end
       end.not_to raise_error
-      puts "[Connection Pool Timing] Duration before primary change: #{@primary_chane_start - start}. "\
+      puts "[Connection Pool Timing] Duration before primary change: #{@primary_change_start - start}. "\
         "Duration after primary change: #{Time.now - @primary_change_end}"
     end
   end

data/spec/stress/fork_reconnect_stress_spec.rb

@@ -0,0 +1,109 @@
+require 'spec_helper'
+
+describe 'fork reconnect' do
+  require_fork
+  only_mri
+
+  # On multi-shard sharded clusters a succeeding write request does not
+  # guarantee that the next operation will succeed (since it could be sent to
+  # another shard with a dead connection).
+  require_no_multi_shard
+
+  # On Ruby 2.3 $?.exitstatus is sometimes nil after Process.wait returns which
+  # is not supposed to happen.
+  ruby_version_gte '2.4'
+
+  require_stress
+
+  let(:client) { authorized_client }
+
+  describe 'client' do
+    it 'works after fork' do
+      # Perform a write so that we discover the current primary.
+      # Previous test may have stepped down the server that authorized client
+      # considers the primary.
+      # In standalone deployments there are no retries, hence execute the
+      # operation twice manually.
+      client['foo'].insert_one(test: 1) rescue nil
+      client['foo'].insert_one(test: 1)
+
+      pids = []
+      deadline = Time.now + 5
+      1.upto(10) do
+        if pid = fork
+          pids << pid
+        else
+          Utils.wrap_forked_child do
+            while Time.now < deadline
+              client.database.command(ismaster: 1).should be_a(Mongo::Operation::Result)
+            end
+          end
+        end
+      end
+
+      while Time.now < deadline
+        # Use a read which is retried in case of an error
+        client['foo'].find(test: 1).to_a
+      end
+
+      pids.each do |pid|
+        pid, status = Process.wait2(pid)
+        status.exitstatus.should == 0
+      end
+    end
+
+    context 'when parent is operating on client during the fork', retry: 3 do
+      # This test intermittently fails in evergreen with pool size of 5,
+      # with a number of pending connections in the pool.
+      # The reason could be that handshaking is slow or operations are slow
+      # post handshakes.
+      # Sometimes it seems the monitoring connection experiences network
+      # errors (despite being a loopback connection) which causes the test
+      # to fail as then server selection fails.
+      # The retry: 3 is to deal with network errors on monitoring connection.
+
+      let(:client) { authorized_client.with(max_pool_size: 10,
+        wait_queue_timeout: 10, socket_timeout: 2, connect_timeout: 2) }
+
+      it 'works' do
+        client.database.command(ismaster: 1).should be_a(Mongo::Operation::Result)
+
+        threads = []
+        5.times do
+          threads << Thread.new do
+            loop do
+              client['foo'].find(test: 1).to_a
+            end
+          end
+        end
+
+        pids = []
+        deadline = Time.now + 5
+        10.times do
+          if pid = fork
+            pids << pid
+          else
+            Utils.wrap_forked_child do
+              while Time.now < deadline
+                client.database.command(ismaster: 1).should be_a(Mongo::Operation::Result)
+              end
+            end
+          end
+        end
+
+        while Time.now < deadline
+          sleep 0.1
+        end
+
+        threads.map(&:kill)
+        threads.map(&:join)
+
+        pids.each do |pid|
+          pid, status = Process.wait2(pid)
+          status.exitstatus.should == 0
+        end
+      end
+
+    end
+  end
+end

data/spec/support/authorization.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2009-2019 MongoDB, Inc.
+# Copyright (C) 2009-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -65,12 +65,10 @@ module Authorization
     #
     # @since 2.5.1
     context.let(:authorized_client_with_retry_writes) do
-      EventSubscriber.clear_events!
       ClientRegistry.instance.global_client('authorized_with_retry_writes')
     end
 
     context.let(:authorized_client_without_retry_writes) do
-      EventSubscriber.clear_events!
       ClientRegistry.instance.global_client('authorized_without_retry_writes')
     end
 
@@ -86,14 +84,6 @@ module Authorization
       ClientRegistry.instance.global_client('authorized_without_any_retries')
     end
 
-    # Provides an authorized mongo client that has a Command subscriber.
-    #
-    # @since 2.5.1
-    context.let(:subscribed_client) do
-      EventSubscriber.clear_events!
-      ClientRegistry.instance.global_client('subscribed')
-    end
-
     # Provides an unauthorized mongo client on the default test database.
     #
     # @since 2.0.0

data/spec/support/aws_utils.rb

@@ -0,0 +1,62 @@
+autoload :Byebug, 'byebug'
+autoload :Paint, 'paint'
+
+module Aws
+  autoload :CloudWatchLogs, 'aws-sdk-cloudwatchlogs'
+  autoload :Credentials, 'aws-sdk-core'
+  autoload :EC2, 'aws-sdk-ec2'
+  autoload :ECS, 'aws-sdk-ecs'
+  autoload :IAM, 'aws-sdk-iam'
+  autoload :STS, 'aws-sdk-core'
+end
+
+module AwsUtils
+  NAMESPACE = 'mdb-ruby'.freeze
+
+  AWS_AUTH_REGULAR_USER_NAME = "#{NAMESPACE}.aws-auth-regular".freeze
+
+  AWS_AUTH_ASSUME_ROLE_NAME = "#{NAMESPACE}.assume-role".freeze
+
+  AWS_AUTH_SECURITY_GROUP_NAME = "#{NAMESPACE}.ssh".freeze
+
+  AWS_AUTH_VPC_GATEWAY_NAME = NAMESPACE
+
+  AWS_AUTH_VPC_SECURITY_GROUP_NAME = "#{NAMESPACE}.vpc-ssh".freeze
+
+  AWS_AUTH_VPC_CIDR = "10.42.142.64/28".freeze
+
+  AWS_AUTH_EC2_AMI_NAMES = {
+    # https://wiki.debian.org/Cloud/AmazonEC2Image/Buster
+    'debian10' => 'debian-10-amd64-20200210-166',
+    'ubuntu1604' => 'ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-server-20200317',
+  }.freeze
+
+  AWS_AUTH_EC2_INSTANCE_NAME = "#{NAMESPACE}.aws-auth)".freeze
+
+  AWS_AUTH_INSTANCE_PROFILE_NAME = "#{NAMESPACE}.ip".freeze
+
+  AWS_AUTH_ASSUME_ROLE_USER_POLICY_NAME = "#{NAMESPACE}.assume-role-user-policy".freeze
+
+  AWS_AUTH_EC2_ROLE_NAME = "#{NAMESPACE}.ec2-role".freeze
+
+  AWS_AUTH_ECS_CLUSTER_NAME = "#{NAMESPACE}_aws-auth".freeze
+
+  AWS_AUTH_ECS_TASK_FAMILY = "#{NAMESPACE}_aws-auth".freeze
+
+  AWS_AUTH_ECS_SERVICE_NAME = "#{NAMESPACE}_aws-auth".freeze
+
+  AWS_AUTH_ECS_LOG_GROUP = "/ecs/#{NAMESPACE}/aws-auth-ecs".freeze
+
+  AWS_AUTH_ECS_LOG_STREAM_PREFIX = "task".freeze
+
+  # This role allows ECS tasks access to output logs to CloudWatch.
+  AWS_AUTH_ECS_EXECUTION_ROLE_NAME = "#{NAMESPACE}.ecs-execution-role".freeze
+
+  # This role is assumed by ECS tasks.
+  AWS_AUTH_ECS_TASK_ROLE_NAME = "#{NAMESPACE}.ecs-task-role".freeze
+
+  autoload :Base, 'support/aws_utils/base'
+  autoload :Inspector, 'support/aws_utils/inspector'
+  autoload :Orchestrator, 'support/aws_utils/orchestrator'
+  autoload :Provisioner, 'support/aws_utils/provisioner'
+end

data/spec/support/aws_utils/base.rb

@@ -0,0 +1,134 @@
+module AwsUtils
+  class Base
+
+    def initialize(access_key_id: nil, secret_access_key: nil, region: nil, **options)
+      @access_key_id = access_key_id || ENV['AWS_ACCESS_KEY_ID']
+      @secret_access_key = secret_access_key || ENV['AWS_SECRET_ACCESS_KEY']
+      @region = region || ENV['AWS_REGION']
+      @options = options
+    end
+
+    attr_reader :access_key_id, :secret_access_key, :region, :options
+
+    private
+
+    def detect_object(resp, resp_attr, object_attr, value)
+      resp.each do |batch|
+        batch.send(resp_attr).each do |object|
+          if object.send(object_attr) == value
+            return object
+          end
+        end
+      end
+      nil
+    end
+
+    def ssh_security_group_id
+      begin
+        sg = ec2_client.describe_security_groups(
+          group_names: [AWS_AUTH_SECURITY_GROUP_NAME],
+        ).security_groups.first
+        sg&.group_id
+      rescue Aws::EC2::Errors::InvalidGroupNotFound
+        # Unlike almost all other describe calls, this one raises an exception
+        # if there isn't a security group matching the criteria.
+        nil
+      end
+    end
+
+    def ssh_security_group_id!
+      ssh_security_group_id.tap do |security_group_id|
+        if security_group_id.nil?
+          raise 'Security group does not exist, please provision'
+        end
+      end
+    end
+
+    def ssh_vpc_security_group_id
+      begin
+        # If the top-level group_name parameter is used, only non-VPC
+        # security groups are returned which does not find the VPC group
+        # we are looking for here.
+        sg = ec2_client.describe_security_groups(
+          filters: [{
+            name: 'group-name',
+            values: [AWS_AUTH_VPC_SECURITY_GROUP_NAME],
+          }],
+        ).security_groups.first
+        sg&.group_id
+      rescue Aws::EC2::Errors::InvalidGroupNotFound
+        # Unlike almost all other describe calls, this one raises an exception
+        # if there isn't a security group matching the criteria.
+        nil
+      end
+    end
+
+    def ssh_vpc_security_group_id!
+      ssh_vpc_security_group_id.tap do |security_group_id|
+        if security_group_id.nil?
+          raise 'Security group does not exist, please provision'
+        end
+      end
+    end
+
+    def subnet_id
+      # This directly queries the subnets for the one with the expected
+      # CIDR block, to save on the number of requests made to AWS.
+      ec2_client.describe_subnets(
+        filters: [{
+          name: 'cidr-block',
+          values: [AWS_AUTH_VPC_CIDR],
+        }],
+      ).subnets.first&.subnet_id
+    end
+
+    def subnet_id!
+      subnet_id.tap do |subnet_id|
+        if subnet_id.nil?
+          raise 'Subnet does not exist, please provision'
+        end
+      end
+    end
+
+    def credentials
+      Aws::Credentials.new(access_key_id, secret_access_key)
+    end
+
+    public
+
+    def ec2_client
+      @ec2_client ||= Aws::EC2::Client.new(
+        region: region,
+        credentials: credentials,
+      )
+    end
+
+    def iam_client
+      iam_client = Aws::IAM::Client.new(
+        region: region,
+        credentials: credentials,
+      )
+    end
+
+    def ecs_client
+      @ecs_client ||= Aws::ECS::Client.new(
+        region: region,
+        credentials: credentials,
+      )
+    end
+
+    def logs_client
+      @logs_client ||= Aws::CloudWatchLogs::Client.new(
+        region: region,
+        credentials: credentials,
+      )
+    end
+
+    def sts_client
+      @sts_client ||= Aws::STS::Client.new(
+        region: region,
+        credentials: credentials,
+      )
+    end
+  end
+end