mongo 2.12.4 → 2.13.0

data/spec/spec_tests/server_selection_rtt_spec.rb

@@ -1,5 +1,7 @@
 require 'spec_helper'
 
+require 'runners/server_selection_rtt'
+
 describe 'Server Selection moving average round trip time calculation' do
 
   include Mongo::ServerSelection::RTT

data/spec/spec_tests/server_selection_spec.rb

@@ -1,119 +1,9 @@
-require 'spec_helper'
+require 'lite_spec_helper'
 
-describe 'Server Selection' do
+require 'runners/server_selection'
 
-  include Mongo::ServerSelection::Read
+SERVER_SELECTION_TESTS = Dir.glob("#{CURRENT_PATH}/spec_tests/data/server_selection/**/*.yml").sort
 
-  SERVER_SELECTION_TESTS.each do |file|
-
-    spec = Mongo::ServerSelection::Read::Spec.new(file)
-
-    context(spec.description) do
-
-      let(:monitoring) do
-        Mongo::Monitoring.new(monitoring: false)
-      end
-
-      # Cluster needs a topology and topology needs a cluster...
-      # This temporary cluster is used for topology construction.
-      let(:temp_cluster) do
-        double('temp cluster').tap do |cluster|
-          allow(cluster).to receive(:servers_list).and_return([])
-        end
-      end
-
-      let(:topology) do
-        options = if spec.type <= Mongo::Cluster::Topology::ReplicaSetNoPrimary
-          {replica_set_name: 'foo'}
-        else
-          {}
-        end
-        spec.type.new(options, monitoring, temp_cluster)
-      end
-
-      let(:listeners) do
-        Mongo::Event::Listeners.new
-      end
-
-      let(:cluster) do
-        double('cluster').tap do |c|
-          allow(c).to receive(:connected?).and_return(true)
-          allow(c).to receive(:summary)
-          allow(c).to receive(:topology).and_return(topology)
-          allow(c).to receive(:single?).and_return(topology.single?)
-          allow(c).to receive(:sharded?).and_return(topology.sharded?)
-          allow(c).to receive(:replica_set?).and_return(topology.replica_set?)
-          allow(c).to receive(:unknown?).and_return(topology.unknown?)
-          allow(c).to receive(:app_metadata).and_return(app_metadata)
-          allow(c).to receive(:options).and_return({})
-          allow(c).to receive(:server_selection_semaphore).and_return(nil)
-        end
-      end
-
-      let(:candidate_servers) do
-        spec.candidate_servers.collect do |server|
-          address = Mongo::Address.new(server['address'])
-          Mongo::Server.new(address, cluster, monitoring, listeners,
-            {monitoring_io: false}.update(SpecConfig.instance.test_options)
-          ).tap do |s|
-            allow(s).to receive(:average_round_trip_time).and_return(server['avg_rtt_ms'] / 1000.0)
-            allow(s).to receive(:tags).and_return(server['tags'])
-            allow(s).to receive(:secondary?).and_return(server['type'] == 'RSSecondary')
-            allow(s).to receive(:primary?).and_return(server['type'] == 'RSPrimary')
-            allow(s).to receive(:connectable?).and_return(true)
-            allow(s).to receive(:check_driver_support!).and_return(true)
-          end
-        end
-      end
-
-      let(:in_latency_window) do
-        spec.in_latency_window.collect do |server|
-          address = Mongo::Address.new(server['address'])
-          Mongo::Server.new(address, cluster, monitoring, listeners,
-            {monitoring_io: false}.update(SpecConfig.instance.test_options)
-          ).tap do |s|
-            allow(s).to receive(:average_round_trip_time).and_return(server['avg_rtt_ms'] / 1000.0)
-            allow(s).to receive(:tags).and_return(server['tags'])
-            allow(s).to receive(:connectable?).and_return(true)
-            allow(s).to receive(:check_driver_support!).and_return(true)
-          end
-        end
-      end
-
-      let(:server_selector) do
-        Mongo::ServerSelector.get(:mode => spec.read_preference['mode'],
-                                  :tag_sets => spec.read_preference['tag_sets'])
-      end
-
-      before do
-        allow(cluster).to receive(:servers).and_return(candidate_servers)
-        allow(cluster).to receive(:servers_list).and_return(candidate_servers)
-        allow(cluster).to receive(:addresses).and_return(candidate_servers.map(&:address))
-        allow(cluster).to receive(:options).and_return(server_selection_timeout: 0.2)
-        allow(cluster).to receive(:scan!).and_return(true)
-        allow(cluster).to receive(:app_metadata).and_return(app_metadata)
-      end
-
-      context 'Valid read preference and matching server available', if: spec.server_available? do
-
-        it 'Finds all suitable servers in the latency window', if: spec.replica_set? do
-          expect(server_selector.send(:select, cluster.servers)).to match_array(in_latency_window)
-        end
-
-        it 'Finds the most suitable server in the latency window' do
-          expect(in_latency_window).to include(server_selector.select_server(cluster))
-        end
-      end
-
-      context 'No matching server available', if: !spec.server_available? do
-        skip_if_linting
-
-        it 'Raises exception' do
-          expect do
-            server_selector.select_server(cluster)
-          end.to raise_exception(Mongo::Error::NoServerAvailable)
-        end
-      end
-    end
-  end
+describe 'Server selection spec tests' do
+  define_server_selection_spec_tests(SERVER_SELECTION_TESTS)
 end

data/spec/spec_tests/transactions_api_spec.rb

@@ -1,5 +1,10 @@
 require 'spec_helper'
 
+require 'runners/crud'
+require 'runners/transactions'
+
 describe 'Transactions API' do
+  require_wired_tiger
+
   define_transactions_spec_tests(TRANSACTIONS_API_TESTS)
 end

data/spec/spec_tests/transactions_spec.rb

@@ -1,5 +1,10 @@
 require 'spec_helper'
 
+require 'runners/crud'
+require 'runners/transactions'
+
 describe 'Transactions' do
+  require_wired_tiger
+
   define_transactions_spec_tests(TRANSACTIONS_TESTS)
 end

data/spec/spec_tests/uri_options_spec.rb

@@ -1,5 +1,7 @@
 require 'lite_spec_helper'
 
+require 'runners/connection_string'
+
 describe 'Uri Options' do
   include Mongo::ConnectionString
 

data/spec/stress/cleanup_spec.rb

@@ -0,0 +1,58 @@
+require 'spec_helper'
+
+describe 'Cleanup stress test' do
+  require_stress
+
+  let(:options) do
+    SpecConfig.instance.all_test_options
+  end
+
+  before(:all) do
+    # load if necessary
+    ClusterConfig.instance.primary_address
+    ClientRegistry.instance.close_all_clients
+  end
+
+  context 'single client disconnect/reconnect' do
+    let(:client) do
+      new_local_client([ClusterConfig.instance.primary_address.seed], options)
+    end
+
+    it 'cleans up' do
+      client
+      client.cluster.servers_list.map(&:scan!)
+
+      sleep 1
+      GC.start
+
+      start_resources = resources
+
+      500.times do
+        client.close
+        client.reconnect
+      end
+
+      sleep 1
+      GC.start
+
+      end_resources = resources
+
+      # There seem to be a temporary file descriptor leak in CI,
+      # where we start with 75 fds and end with 77 fds.
+      # Allow a few to be leaked, run more iterations to ensure the leak
+      # is not a real one.
+      # Sometimes we end with fewer fds than we started with also...
+      end_resources[:open_file_count].should >= start_resources[:open_file_count] - 3
+      end_resources[:open_file_count].should <= start_resources[:open_file_count] + 3
+
+      end_resources[:running_thread_count].should == start_resources[:running_thread_count]
+    end
+  end
+
+  def resources
+    {
+      open_file_count: Dir["/proc/#{Process.pid}/fd/*"].count,
+      running_thread_count: Thread.list.select { |thread| thread.status == 'run' }.count,
+    }
+  end
+end

data/spec/stress/connection_pool_stress_spec.rb

@@ -1,11 +1,7 @@
 require 'spec_helper'
 
 describe 'Connection pool stress test' do
-  before(:all) do
-    if !SpecConfig.instance.stress_spec?
-      skip 'Stress spec not enabled'
-    end
-  end
+  require_stress
 
   let(:options) do
     { max_pool_size: 5, min_pool_size: 3 }
@@ -25,9 +21,9 @@ describe 'Connection pool stress test' do
     [].tap do |threads|
       thread_count.times do |i|
         threads << Thread.new do
-          10.times do |j|
+          100.times do |j|
             collection.find(a: i+j).to_a
-            sleep 0.5
+            sleep 0.1
             collection.find(a: i+j).to_a
           end
         end
@@ -136,7 +132,7 @@ describe 'Connection pool stress test' do
     end
 
     context '25 threads, max pool size 5' do
-      let(:thread_count) { 20 }
+      let(:thread_count) { 25 }
 
       it_behaves_like 'does not raise error'
     end
@@ -187,17 +183,19 @@ describe 'Connection pool stress test' do
       # The populator calls create_and_add_connection, which calls connection.connect!
       # and raises if an error occurs (eg, an auth error), so in this test we
       # randomly raise errors on this method.
-      expect(server.pool).to receive(:create_and_add_connection).at_least(:once).and_wrap_original { |m, *args|
-        if rand < 0.05
+      expect(server.pool).to receive(:create_and_add_connection).at_least(:once).and_wrap_original do |m, *args|
+        if rand < 0.1
           raise Mongo::Error::SocketError
         else
           m.call(*args)
         end
-      }
+      end
 
-      expect {
+      server.pool.clear(stop_populator: false)
+
+      expect do
         threads.collect { |t| t.join }
-      }.not_to raise_error
+      end.not_to raise_error
     end
   end
 end

data/spec/stress/connection_pool_timing_spec.rb

@@ -1,13 +1,10 @@
 require 'spec_helper'
 
 describe 'Connection pool timing test' do
-  before(:all) do
-    if !SpecConfig.instance.stress_spec?
-      skip 'Stress spec not enabled'
-    end
-
-    ClientRegistry.instance.close_all_clients
+  require_stress
+  clean_slate_for_all
 
+  before(:all) do
     # This set up is taken from the step_down_spec file. In a future PR, ClusterTools
     # may be modified so this set up is no longer necessary.
     if ClusterConfig.instance.fcv_ish >= '4.2' && ClusterConfig.instance.topology == :replica_set

data/spec/stress/fork_reconnect_stress_spec.rb

@@ -0,0 +1,109 @@
+require 'spec_helper'
+
+describe 'fork reconnect' do
+  require_fork
+  only_mri
+
+  # On multi-shard sharded clusters a succeeding write request does not
+  # guarantee that the next operation will succeed (since it could be sent to
+  # another shard with a dead connection).
+  require_no_multi_shard
+
+  # On Ruby 2.3 $?.exitstatus is sometimes nil after Process.wait returns which
+  # is not supposed to happen.
+  ruby_version_gte '2.4'
+
+  require_stress
+
+  let(:client) { authorized_client }
+
+  describe 'client' do
+    it 'works after fork' do
+      # Perform a write so that we discover the current primary.
+      # Previous test may have stepped down the server that authorized client
+      # considers the primary.
+      # In standalone deployments there are no retries, hence execute the
+      # operation twice manually.
+      client['foo'].insert_one(test: 1) rescue nil
+      client['foo'].insert_one(test: 1)
+
+      pids = []
+      deadline = Time.now + 5
+      1.upto(10) do
+        if pid = fork
+          pids << pid
+        else
+          Utils.wrap_forked_child do
+            while Time.now < deadline
+              client.database.command(ismaster: 1).should be_a(Mongo::Operation::Result)
+            end
+          end
+        end
+      end
+
+      while Time.now < deadline
+        # Use a read which is retried in case of an error
+        client['foo'].find(test: 1).to_a
+      end
+
+      pids.each do |pid|
+        pid, status = Process.wait2(pid)
+        status.exitstatus.should == 0
+      end
+    end
+
+    context 'when parent is operating on client during the fork', retry: 3 do
+      # This test intermittently fails in evergreen with pool size of 5,
+      # with a number of pending connections in the pool.
+      # The reason could be that handshaking is slow or operations are slow
+      # post handshakes.
+      # Sometimes it seems the monitoring connection experiences network
+      # errors (despite being a loopback connection) which causes the test
+      # to fail as then server selection fails.
+      # The retry: 3 is to deal with network errors on monitoring connection.
+
+      let(:client) { authorized_client.with(max_pool_size: 10,
+        wait_queue_timeout: 10, socket_timeout: 2, connect_timeout: 2) }
+
+      it 'works' do
+        client.database.command(ismaster: 1).should be_a(Mongo::Operation::Result)
+
+        threads = []
+        5.times do
+          threads << Thread.new do
+            loop do
+              client['foo'].find(test: 1).to_a
+            end
+          end
+        end
+
+        pids = []
+        deadline = Time.now + 5
+        10.times do
+          if pid = fork
+            pids << pid
+          else
+            Utils.wrap_forked_child do
+              while Time.now < deadline
+                client.database.command(ismaster: 1).should be_a(Mongo::Operation::Result)
+              end
+            end
+          end
+        end
+
+        while Time.now < deadline
+          sleep 0.1
+        end
+
+        threads.map(&:kill)
+        threads.map(&:join)
+
+        pids.each do |pid|
+          pid, status = Process.wait2(pid)
+          status.exitstatus.should == 0
+        end
+      end
+
+    end
+  end
+end

data/spec/support/authorization.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2009-2019 MongoDB, Inc.
+# Copyright (C) 2009-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -65,12 +65,10 @@ module Authorization
     #
     # @since 2.5.1
     context.let(:authorized_client_with_retry_writes) do
-      EventSubscriber.clear_events!
       ClientRegistry.instance.global_client('authorized_with_retry_writes')
     end
 
     context.let(:authorized_client_without_retry_writes) do
-      EventSubscriber.clear_events!
       ClientRegistry.instance.global_client('authorized_without_retry_writes')
     end
 
@@ -86,14 +84,6 @@ module Authorization
       ClientRegistry.instance.global_client('authorized_without_any_retries')
     end
 
-    # Provides an authorized mongo client that has a Command subscriber.
-    #
-    # @since 2.5.1
-    context.let(:subscribed_client) do
-      EventSubscriber.clear_events!
-      ClientRegistry.instance.global_client('subscribed')
-    end
-
     # Provides an unauthorized mongo client on the default test database.
     #
     # @since 2.0.0

data/spec/support/aws_utils.rb

@@ -0,0 +1,62 @@
+autoload :Byebug, 'byebug'
+autoload :Paint, 'paint'
+
+module Aws
+  autoload :CloudWatchLogs, 'aws-sdk-cloudwatchlogs'
+  autoload :Credentials, 'aws-sdk-core'
+  autoload :EC2, 'aws-sdk-ec2'
+  autoload :ECS, 'aws-sdk-ecs'
+  autoload :IAM, 'aws-sdk-iam'
+  autoload :STS, 'aws-sdk-core'
+end
+
+module AwsUtils
+  NAMESPACE = 'mdb-ruby'.freeze
+
+  AWS_AUTH_REGULAR_USER_NAME = "#{NAMESPACE}.aws-auth-regular".freeze
+
+  AWS_AUTH_ASSUME_ROLE_NAME = "#{NAMESPACE}.assume-role".freeze
+
+  AWS_AUTH_SECURITY_GROUP_NAME = "#{NAMESPACE}.ssh".freeze
+
+  AWS_AUTH_VPC_GATEWAY_NAME = NAMESPACE
+
+  AWS_AUTH_VPC_SECURITY_GROUP_NAME = "#{NAMESPACE}.vpc-ssh".freeze
+
+  AWS_AUTH_VPC_CIDR = "10.42.142.64/28".freeze
+
+  AWS_AUTH_EC2_AMI_NAMES = {
+    # https://wiki.debian.org/Cloud/AmazonEC2Image/Buster
+    'debian10' => 'debian-10-amd64-20200210-166',
+    'ubuntu1604' => 'ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-server-20200317',
+  }.freeze
+
+  AWS_AUTH_EC2_INSTANCE_NAME = "#{NAMESPACE}.aws-auth)".freeze
+
+  AWS_AUTH_INSTANCE_PROFILE_NAME = "#{NAMESPACE}.ip".freeze
+
+  AWS_AUTH_ASSUME_ROLE_USER_POLICY_NAME = "#{NAMESPACE}.assume-role-user-policy".freeze
+
+  AWS_AUTH_EC2_ROLE_NAME = "#{NAMESPACE}.ec2-role".freeze
+
+  AWS_AUTH_ECS_CLUSTER_NAME = "#{NAMESPACE}_aws-auth".freeze
+
+  AWS_AUTH_ECS_TASK_FAMILY = "#{NAMESPACE}_aws-auth".freeze
+
+  AWS_AUTH_ECS_SERVICE_NAME = "#{NAMESPACE}_aws-auth".freeze
+
+  AWS_AUTH_ECS_LOG_GROUP = "/ecs/#{NAMESPACE}/aws-auth-ecs".freeze
+
+  AWS_AUTH_ECS_LOG_STREAM_PREFIX = "task".freeze
+
+  # This role allows ECS tasks access to output logs to CloudWatch.
+  AWS_AUTH_ECS_EXECUTION_ROLE_NAME = "#{NAMESPACE}.ecs-execution-role".freeze
+
+  # This role is assumed by ECS tasks.
+  AWS_AUTH_ECS_TASK_ROLE_NAME = "#{NAMESPACE}.ecs-task-role".freeze
+
+  autoload :Base, 'support/aws_utils/base'
+  autoload :Inspector, 'support/aws_utils/inspector'
+  autoload :Orchestrator, 'support/aws_utils/orchestrator'
+  autoload :Provisioner, 'support/aws_utils/provisioner'
+end