mongo 2.10.5 → 2.15.1

data/spec/support/cluster_tools.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 require 'singleton'
 
 # There is an undocumented {replSetStepUp: 1} command which can be used to
@@ -58,7 +61,7 @@ class ClusterTools
       raise ArgumentError, 'Value must be true or false'
     end
 
-    direct_client_for_each_server do |client|
+    direct_client_for_each_data_bearing_server do |client|
       client.use(:admin).database.command(setParameter: 1, enableElectionHandoff: value)
     end
   end
@@ -118,7 +121,7 @@ class ClusterTools
       rescue Mongo::Error::OperationFailure => e
         # Election failed. (125)
         if e.code == 125
-          # Possible reason is the node we are trying to elect has blacklisted
+          # Possible reason is the node we are trying to elect has deny-listed
           # itself. This is where {replSetFreeze: 0} should make it eligible
           # for election again but this seems to not always work.
         else
@@ -147,7 +150,7 @@ class ClusterTools
     existing_primary_address = existing_primary.address
 
     target = admin_client.cluster.servers_list.detect do |server|
-      server.address != existing_primary_address
+      !server.arbiter? && server.address != existing_primary_address
     end
 
     cfg = get_rs_config
@@ -279,6 +282,7 @@ class ClusterTools
 
   def unfreeze_all
     admin_client.cluster.servers_list.each do |server|
+      next if server.arbiter?
       client = direct_client(server.address)
       # Primary refuses to be unfrozen with this message:
       # cannot freeze node when primary or running for election. state: Primary (95)
@@ -303,7 +307,12 @@ class ClusterTools
   def set_rs_config(config)
     config = config.dup
     config['version'] += 1
-    result = admin_client.database.command(replSetReconfig: config)
+    cmd = {replSetReconfig: config}
+    if ClusterConfig.instance.fcv_ish >= '4.4'
+      # Workaround for https://jira.mongodb.org/browse/SERVER-46894
+      cmd[:force] = true
+    end
+    result = admin_client.database.command(cmd)
     doc = result.reply.documents.first
     if doc['ok'] != 1
       raise 'Failed to reconfigure RS'
@@ -353,8 +362,9 @@ class ClusterTools
     admin_client.cluster.servers_list.each(&block)
   end
 
-  def direct_client_for_each_server(&block)
+  def direct_client_for_each_data_bearing_server(&block)
     each_server do |server|
+      next if server.arbiter?
       yield direct_client(server.address)
     end
   end

data/spec/support/common_shortcuts.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 module CommonShortcuts
   module ClassMethods
     # Declares a topology double, which is configured to accept summary
@@ -16,6 +19,7 @@ module CommonShortcuts
     def clean_slate
       before do
         ClientRegistry.instance.close_all_clients
+        BackgroundThreadRegistry.instance.verify_empty!
       end
     end
 
@@ -25,6 +29,19 @@ module CommonShortcuts
     def clean_slate_for_all
       before(:all) do
         ClientRegistry.instance.close_all_clients
+        BackgroundThreadRegistry.instance.verify_empty!
+      end
+    end
+
+    # If only the lite spec helper was loaded, this method does nothing.
+    # If the full spec helper was loaded, this method performs the same function
+    # as clean_state_for_all.
+    def clean_slate_for_all_if_possible
+      before(:all) do
+        if defined?(ClusterTools)
+          ClientRegistry.instance.close_all_clients
+          BackgroundThreadRegistry.instance.verify_empty!
+        end
       end
     end
 
@@ -39,6 +56,118 @@ module CommonShortcuts
         end
       end
     end
+
+    # Applies environment variable overrides in +env+ to the global environment
+    # (+ENV+) for the duration of each test.
+    #
+    # If a key's value in +env+ is nil, this key is removed from +ENV+.
+    #
+    # When the test finishes, the values in original +ENV+ that were overridden
+    # by +env+ are restored. If a key was not in original +ENV+ and was
+    # overridden by +env+, this key is removed from +ENV+ after the test.
+    #
+    # If the environment variables are not known at test definition time
+    # but are determined at test execution time, pass a block instead of
+    # the +env+ parameter and return the desired environment variables as
+    # a Hash from the block.
+    def local_env(env = nil, &block)
+      around do |example|
+        env ||= block.call
+
+        # This duplicates ENV.
+        # Note that ENV.dup produces an Object which does not behave like
+        # the original ENV, and hence is not usable.
+        saved_env = ENV.to_h
+        env.each do |k, v|
+          if v.nil?
+            ENV.delete(k)
+          else
+            ENV[k] = v
+          end
+        end
+
+        begin
+          example.run
+        ensure
+          env.each do |k, v|
+            if saved_env.key?(k)
+              ENV[k] = saved_env[k]
+            else
+              ENV.delete(k)
+            end
+          end
+        end
+      end
+    end
+
+    def clear_ocsp_cache
+      before do
+        Mongo.clear_ocsp_cache
+      end
+    end
+
+    def with_ocsp_mock(ca_file_path, responder_cert_path, responder_key_path,
+      fault: nil, port: 8100
+    )
+      clear_ocsp_cache
+
+      around do |example|
+        args = [
+          SpecConfig.instance.ocsp_files_dir.join('ocsp_mock.py').to_s,
+          '--ca_file', ca_file_path.to_s,
+          '--ocsp_responder_cert', responder_cert_path.to_s,
+          '--ocsp_responder_key', responder_key_path.to_s,
+          '-p', port.to_s,
+        ]
+        if SpecConfig.instance.client_debug?
+          # Use when debugging - tests run faster without -v.
+          args << '-v'
+        end
+        if fault
+          args += ['--fault', fault]
+        end
+        process = ChildProcess.new(*args)
+
+        process.io.inherit!
+
+        retried = false
+        begin
+          process.start
+        rescue
+          if retried
+            raise
+          else
+            sleep 1
+            retried = true
+            retry
+          end
+        end
+
+        begin
+          sleep 0.4
+          example.run
+        ensure
+          if process.exited?
+            raise "Spawned process exited before we stopped it"
+          end
+
+          process.stop
+          process.wait
+        end
+      end
+    end
+
+    def with_openssl_debug
+      around do |example|
+        v = OpenSSL.debug
+        OpenSSL.debug = true
+        begin
+          example.run
+        ensure
+          OpenSSL.debug = v
+        end
+      end
+    end
   end
 
   module InstanceMethods
@@ -81,10 +210,10 @@ module CommonShortcuts
       end
 
       if mode == :unknown
-        ismaster = {}
+        config = {}
       else
-        ismaster = {
-          'ismaster' => mode == :primary,
+        config = {
+          'isWritablePrimary' => mode == :primary,
           'secondary' => mode == :secondary,
           'arbiterOnly' => mode == :arbiter,
           'isreplicaset' => mode == :ghost,
@@ -95,7 +224,7 @@ module CommonShortcuts
           'minWireVersion' => 2, 'maxWireVersion' => 8,
         }
         if [:primary, :secondary, :arbiter, :other].include?(mode)
-          ismaster['setName'] = 'mongodb_set'
+          config['setName'] = 'mongodb_set'
         end
       end
 
@@ -107,8 +236,20 @@ module CommonShortcuts
       allow(cluster).to receive(:topology).and_return(topology)
       allow(cluster).to receive(:app_metadata)
       allow(cluster).to receive(:options).and_return({})
-      server = Mongo::Server.new(address, cluster, monitoring, listeners, SpecConfig.instance.test_options)
-      description = Mongo::Server::Description.new(address, ismaster, average_round_trip_time)
+      allow(cluster).to receive(:run_sdam_flow)
+      allow(cluster).to receive(:monitor_app_metadata)
+      allow(cluster).to receive(:push_monitor_app_metadata)
+      allow(cluster).to receive(:heartbeat_interval).and_return(10)
+      server = Mongo::Server.new(address, cluster, monitoring, listeners,
+        monitoring_io: false)
+      # Since the server references a double for the cluster, the server
+      # must be closed in the scope of the example.
+      register_server(server)
+      description = Mongo::Server::Description.new(
+        address,
+        config,
+        average_round_trip_time,
+      )
       server.tap do |s|
         allow(s).to receive(:description).and_return(description)
       end
@@ -138,5 +279,97 @@ module CommonShortcuts
         'ok' => 0, 'code' => 91, 'errmsg' => 'shutdown in progress'
       )
     end
+
+    def register_cluster(cluster)
+      finalizer = lambda do |cluster|
+        cluster.disconnect!
+      end
+      LocalResourceRegistry.instance.register(cluster, finalizer)
+    end
+
+    def register_server(server)
+      finalizer = lambda do |server|
+        if server.connected?
+          server.disconnect!
+        end
+      end
+      LocalResourceRegistry.instance.register(server, finalizer)
+    end
+
+    def register_background_thread_object(bgt_object)
+      finalizer = lambda do |bgt_object|
+        bgt_object.stop!
+      end
+      LocalResourceRegistry.instance.register(bgt_object, finalizer)
+    end
+
+    def register_pool(pool)
+      finalizer = lambda do |pool|
+        if !pool.closed?
+          pool.close(wait: true)
+        end
+      end
+      LocalResourceRegistry.instance.register(pool, finalizer)
+    end
+
+    # Stop monitoring threads on the specified clients, after ensuring
+    # each client has a writable server. Used for tests which assert on
+    # global side effects like log messages being generated, to prevent
+    # background threads from interfering with assertions.
+    def stop_monitoring(*clients)
+      clients.each do |client|
+        client.cluster.next_primary
+        client.cluster.disconnect!
+      end
+    end
+
+    DNS_INTERFACES = [
+      [:udp, "0.0.0.0", 5300],
+      [:tcp, "0.0.0.0", 5300],
+    ]
+
+    def mock_dns(config)
+      semaphore = Mongo::Semaphore.new
+
+      thread = Thread.new do
+        RubyDNS::run_server(DNS_INTERFACES) do
+          config.each do |(query, type, *answers)|
+
+            resource_cls = Resolv::DNS::Resource::IN.const_get(type.to_s.upcase)
+            resources = answers.map do |answer|
+              resource_cls.new(*answer)
+            end
+            match(query, resource_cls) do |req|
+              req.add(resources)
+            end
+          end
+
+          semaphore.signal
+        end
+      end
+
+      semaphore.wait
+
+      begin
+        yield
+      ensure
+        10.times do
+          if $last_async_task
+            break
+          end
+          sleep 0.5
+        end
+
+        # Hack to stop the server - https://github.com/socketry/rubydns/issues/75
+        if $last_async_task.nil?
+          STDERR.puts "No async task - server never started?"
+        else
+          $last_async_task.stop
+        end
+
+        thread.kill
+        thread.join
+      end
+    end
   end
 end

data/spec/support/constraints.rb

@@ -1,193 +1,18 @@
-module Constraints
-  def min_server_version(version)
-    unless version =~ /^\d+\.\d+$/
-      raise ArgumentError, "Version can only be major.minor: #{version}"
-    end
-
-    before do
-      if version > ClusterConfig.instance.server_version
-        skip "Server version #{version} or higher required, we have #{ClusterConfig.instance.server_version}"
-      end
-    end
-  end
-
-  def max_server_version(version)
-    unless version =~ /^\d+\.\d+$/
-      raise ArgumentError, "Version can only be major.minor: #{version}"
-    end
-
-    before do
-      if version < ClusterConfig.instance.short_server_version
-        skip "Server version #{version} or lower required, we have #{ClusterConfig.instance.server_version}"
-      end
-    end
-  end
+# frozen_string_literal: true
+# encoding: utf-8
 
-  def min_server_fcv(version)
-    unless version =~ /^\d+\.\d+$/
-      raise ArgumentError, "FCV can only be major.minor: #{version}"
-    end
-
-    before do
-      unless ClusterConfig.instance.fcv_ish >= version
-        skip "FCV #{version} or higher required, we have #{ClusterConfig.instance.fcv_ish} (server #{ClusterConfig.instance.server_version})"
-      end
-    end
-  end
-
-  def max_server_fcv(version)
-    unless version =~ /^\d+\.\d+$/
-      raise ArgumentError, "Version can only be major.minor: #{version}"
-    end
-
-    before do
-      if version < ClusterConfig.instance.fcv_ish
-        skip "FCV #{version} or lower required, we have #{ClusterConfig.instance.fcv_ish} (server #{ClusterConfig.instance.server_version})"
-      end
-    end
-  end
-
-  def require_topology(*topologies)
-    invalid_topologies = topologies - [:single, :replica_set, :sharded]
-    unless invalid_topologies.empty?
-      raise ArgumentError, "Invalid topologies requested: #{invalid_topologies.join(', ')}"
-    end
-    before do
-      unless topologies.include?(topology = ClusterConfig.instance.topology)
-        skip "Topology #{topologies.join(' or ')} required, we have #{topology}"
-      end
-    end
-  end
-
-  def max_example_run_time(timeout)
-    around do |example|
-      TimeoutInterrupt.timeout(timeout, TimeoutInterrupt::Error.new("Test execution terminated after #{timeout} seconds")) do
-        example.run
-      end
-    end
-  end
-
-  def require_transaction_support
-    min_server_fcv '4.0'
-    require_topology :replica_set
-  end
-
-  def require_tls
-    before do
-      unless SpecConfig.instance.ssl?
-        skip "SSL not enabled"
-      end
-    end
-  end
-
-  def require_no_tls
-    before do
-      if SpecConfig.instance.ssl?
-        skip "SSL enabled"
-      end
-    end
-  end
+module Constraints
 
+  # Some tests hardcode the TLS certificates shipped with the driver's
+  # test suite, and will fail when using TLS connections that use other
+  # certificates.
   def require_local_tls
     require_tls
-  end
 
-  def require_no_retry_writes
-    before do
-      if SpecConfig.instance.retry_writes?
-        skip "Retry writes is enabled"
-      end
-    end
-  end
-
-  def require_compression
-    before do
-      if SpecConfig.instance.compressors.nil?
-        skip "Compression is not enabled"
-      end
-    end
-  end
-
-  def require_no_compression
-    before do
-      if SpecConfig.instance.compressors
-        skip "Compression is enabled"
-      end
-    end
-  end
-
-  def ruby_version_gte(version)
-    before do
-      if RUBY_VERSION < version
-        skip "Ruby version #{version} or higher required"
-      end
-    end
-  end
-
-  def ruby_version_lt(version)
-    before do
-      if RUBY_VERSION >= version
-        skip "Ruby version less than #{version} required"
-      end
-    end
-  end
-
-  def require_auth
-    before do
-      unless ENV['AUTH'] == 'auth' || SpecConfig.instance.user || ClusterConfig.instance.auth_enabled?
-        skip "Auth required"
-      end
-    end
-  end
-
-  def require_no_auth
-    before do
-      if ENV['AUTH'] == 'auth' || SpecConfig.instance.user || ClusterConfig.instance.auth_enabled?
-        skip "Auth not allowed"
-      end
-    end
-  end
-
-  # Can the driver specify a write concern that won't be overridden?
-  # (mongos 4.0+ overrides the write concern)
-  def require_set_write_concern
-    before do
-      if ClusterConfig.instance.topology == :sharded && ClusterConfig.instance.short_server_version >= '4.0'
-        skip "mongos 4.0+ overrides write concern"
-      end
-    end
-  end
-
-  def require_no_multi_shard
-    before do
-      if ClusterConfig.instance.topology == :sharded && SpecConfig.instance.addresses.length > 1
-        skip 'Test requires a single shard if run in sharded topology'
-      end
-    end
-  end
-
-  def require_wired_tiger
     before(:all) do
-      if ClusterConfig.instance.storage_engine != :wired_tiger
-        skip 'Test requires WiredTiger storage engine'
-      end
-    end
-  end
-
-  def require_wired_tiger_on_36
-    before(:all) do
-      if ClusterConfig.instance.short_server_version >= '3.6'
-        if ClusterConfig.instance.storage_engine != :wired_tiger
-          skip 'Test requires WiredTiger storage engine on 3.6+ servers'
-        end
-      end
-    end
-  end
-
-  def require_mmapv1
-    before do
-      if ClusterConfig.instance.storage_engine != :mmapv1
-        skip 'Test requires MMAPv1 storage engine'
+      # TODO This isn't actually the foolproof check
+      if ENV['OCSP_ALGORITHM']
+        skip 'Driver TLS certificate required, OCSP certificates are not acceptable'
       end
     end
   end

data/spec/support/crypt/corpus/corpus-key-aws.json

@@ -0,0 +1,33 @@
+{
+  "status": {
+      "$numberInt": "1"
+  },
+  "_id": {
+      "$binary": {
+          "base64": "AWSAAAAAAAAAAAAAAAAAAA==",
+          "subType": "04"
+      }
+  },
+  "masterKey": {
+      "region": "us-east-1",
+      "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0",
+      "provider": "aws"
+  },
+  "updateDate": {
+      "$date": {
+          "$numberLong": "1557827033449"
+      }
+  },
+  "keyMaterial": {
+      "$binary": {
+          "base64": "AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO",
+          "subType": "00"
+      }
+  },
+  "creationDate": {
+      "$date": {
+          "$numberLong": "1557827033449"
+      }
+  },
+  "keyAltNames": ["aws"]
+}

data/spec/support/crypt/corpus/corpus-key-local.json

@@ -0,0 +1,31 @@
+{
+  "status": {
+      "$numberInt": "1"
+  },
+  "_id": {
+      "$binary": {
+          "base64": "LOCALAAAAAAAAAAAAAAAAA==",
+          "subType": "04"
+      }
+  },
+  "masterKey": {
+      "provider": "local"
+  },
+  "updateDate": {
+      "$date": {
+          "$numberLong": "1557827033449"
+      }
+  },
+  "keyMaterial": {
+      "$binary": {
+          "base64": "Ce9HSz/HKKGkIt4uyy+jDuKGA+rLC2cycykMo6vc8jXxqa1UVDYHWq1r+vZKbnnSRBfB981akzRKZCFpC05CTyFqDhXv6OnMjpG97OZEREGIsHEYiJkBW0jJJvfLLgeLsEpBzsro9FztGGXASxyxFRZFhXvHxyiLOKrdWfs7X1O/iK3pEoHMx6uSNSfUOgbebLfIqW7TO++iQS5g1xovXA==",
+          "subType": "00"
+      }
+  },
+  "creationDate": {
+      "$date": {
+          "$numberLong": "1557827033449"
+      }
+  },
+  "keyAltNames": [ "local" ]
+}