mongo 2.17.1 → 2.18.1

data/spec/runners/unified/test.rb

@@ -2,6 +2,7 @@
 # encoding: utf-8
 
 require 'runners/crud/requirement'
+require 'runners/unified/client_side_encryption_operations'
 require 'runners/unified/crud_operations'
 require 'runners/unified/grid_fs_operations'
 require 'runners/unified/ddl_operations'
@@ -9,16 +10,19 @@ require 'runners/unified/change_stream_operations'
 require 'runners/unified/support_operations'
 require 'runners/unified/assertions'
 require 'support/utils'
+require 'support/crypt'
 
 module Unified
 
   class Test
+    include ClientSideEncryptionOperations
     include CrudOperations
     include GridFsOperations
     include DdlOperations
     include ChangeStreamOperations
     include SupportOperations
     include Assertions
+    include RSpec::Core::Pending
 
     def initialize(spec, **opts)
       @spec = spec
@@ -132,11 +136,11 @@ module Unified
 
           create_client(**opts).tap do |client|
             @observe_sensitive = spec.use('observeSensitiveCommands')
+            subscriber = (@subscribers[client] ||= EventSubscriber.new)
             if oe = spec.use('observeEvents')
               oe.each do |event|
                 case event
                 when 'commandStartedEvent', 'commandSucceededEvent', 'commandFailedEvent'
-                  subscriber = (@subscribers[client] ||= EventSubscriber.new)
                   unless client.send(:monitoring).subscribers[Mongo::Monitoring::COMMAND].include?(subscriber)
                     client.subscribe(Mongo::Monitoring::COMMAND, subscriber)
                   end
@@ -146,7 +150,6 @@ module Unified
                     subscriber.ignore_commands(ignore_events)
                   end
                 when /\A(?:pool|connection)/
-                  subscriber = (@subscribers[client] ||= EventSubscriber.new)
                   unless client.send(:monitoring).subscribers[Mongo::Monitoring::CONNECTION_POOL]&.include?(subscriber)
                     client.subscribe(Mongo::Monitoring::CONNECTION_POOL, subscriber)
                   end
@@ -193,6 +196,61 @@ module Unified
           end
 
           client.start_session(**opts)
+        when 'clientEncryption'
+          client_encryption_opts = spec.use!('clientEncryptionOpts')
+          key_vault_client = entities.get(:client, client_encryption_opts['keyVaultClient'])
+          opts = {
+            key_vault_namespace: client_encryption_opts['keyVaultNamespace'],
+            kms_providers: Utils.snakeize_hash(client_encryption_opts['kmsProviders']),
+            kms_tls_options: {
+              kmip: {
+                ssl_cert: SpecConfig.instance.fle_kmip_tls_certificate_key_file,
+                ssl_key: SpecConfig.instance.fle_kmip_tls_certificate_key_file,
+                ssl_ca_cert: SpecConfig.instance.fle_kmip_tls_ca_file
+              }
+            }
+          }
+          opts[:kms_providers] = opts[:kms_providers].map do |provider, options|
+            converted_options = options.map do |key, value|
+              converted_value = if value == { '$$placeholder'.to_sym => 1 }
+                case provider
+                when :aws
+                  case key
+                  when :access_key_id then SpecConfig.instance.fle_aws_key
+                  when :secret_access_key then SpecConfig.instance.fle_aws_secret
+                  end
+                when :azure
+                  case key
+                  when :tenant_id then SpecConfig.instance.fle_azure_tenant_id
+                  when :client_id then SpecConfig.instance.fle_azure_client_id
+                  when :client_secret then SpecConfig.instance.fle_azure_client_secret
+                  end
+                when :gcp
+                  case key
+                  when :email then SpecConfig.instance.fle_gcp_email
+                  when :private_key then SpecConfig.instance.fle_gcp_private_key
+                  end
+                when :kmip
+                  case key
+                  when :endpoint then SpecConfig.instance.fle_kmip_endpoint
+                  end
+                when :local
+                  case key
+                  when :key then Crypt::LOCAL_MASTER_KEY
+                  end
+                end
+              else
+                value
+              end
+              [key, converted_value]
+            end.to_h
+            [provider, converted_options]
+          end.to_h
+
+          Mongo::ClientEncryption.new(
+            key_vault_client,
+            opts
+          )
         else
           raise NotImplementedError, "Unknown type #{type}"
         end
@@ -278,10 +336,19 @@ module Unified
         if name.to_s == 'loop'
           method_name = "_#{name}"
         end
+
+        if ["modify_collection"].include?(name.to_s)
+          skip "Mongo Ruby Driver does not support #{name.to_s}"
+        end
+
         if expected_error = op.use('expectError')
           begin
+            unless respond_to?(method_name)
+              raise Error::UnsupportedOperation, "Mongo Ruby Driver does not support #{name.to_s}"
+            end
+
             public_send(method_name, op)
-          rescue Mongo::Error, BSON::String::IllegalKey => e
+          rescue Mongo::Error, BSON::String::IllegalKey, ArgumentError => e
             if expected_error.use('isClientError')
               # isClientError doesn't actually mean a client error.
               # It means anything other than OperationFailure. DRIVERS-1799
@@ -332,10 +399,16 @@ module Unified
             raise Error::ErrorMismatch, "Expected exception but none was raised"
           end
         else
+          unless respond_to?(method_name, true)
+            raise Error::UnsupportedOperation, "Mongo Ruby Driver does not support #{name.to_s}"
+          end
+
           result = send(method_name, op)
           if expected_result = op.use('expectResult')
-            if result.nil? && !expected_result.empty?
-              raise Error::ResultMismatch, "Actual result nil but expected result #{expected_result}"
+            if result.nil? && expected_result.keys == ["$$unsetOrMatches"]
+              return
+            elsif result.nil? && !expected_result.empty?
+              raise Error::ResultMismatch, "#{msg}: expected #{expected} but got nil"
             elsif Array === expected_result
               assert_documents_match(result, expected_result)
             else

data/spec/runners/unified.rb

@@ -9,6 +9,8 @@ require 'runners/unified/test'
 require 'runners/unified/test_group'
 
 def define_unified_spec_tests(base_path, paths, expect_failure: false)
+  config_override :validate_update_replace, true
+
   paths.each do |path|
     basename = path[base_path.length+1...path.length]
     context basename do
@@ -61,13 +63,18 @@ def define_unified_spec_tests(base_path, paths, expect_failure: false)
               begin
                 test.create_entities
                 test.set_initial_data
-                lambda do
+                begin
                   test.run
                   test.assert_outcome
                   test.assert_events
                 # HACK: other errors are possible and likely will need to
                 # be added here later as the tests evolve.
-                end.should raise_error(Mongo::Error::OperationFailure)
+                rescue Mongo::Error::OperationFailure, Unified::Error::UnsupportedOperation
+                rescue => e
+                  fail "Expected to raise Mongo::Error::OperationFailure or Unified::Error::UnsupportedOperation, got #{e}"
+                else
+                  fail "Expected to raise Mongo::Error::OperationFailure or Unified::Error::UnsupportedOperation, but no error was raised"
+                end
               ensure
                 test.cleanup
               end

data/spec/shared/lib/mrss/constraints.rb

@@ -4,48 +4,40 @@
 module Mrss
   module Constraints
     def min_server_version(version)
-      unless version =~ /\A\d+\.\d+\z/
-        raise ArgumentError, "Version can only be major.minor: #{version}"
-      end
+      parsed_version = Gem::Version.new(version)
 
       before(:all) do
-        if version > ClusterConfig.instance.server_version
+        if parsed_version > Gem::Version.new(ClusterConfig.instance.server_version)
           skip "Server version #{version} or higher required, we have #{ClusterConfig.instance.server_version}"
         end
       end
     end
 
     def max_server_version(version)
-      unless version =~ /\A\d+\.\d+\z/
-        raise ArgumentError, "Version can only be major.minor: #{version}"
-      end
+      parsed_version = Gem::Version.new(version)
 
       before(:all) do
-        if version < ClusterConfig.instance.short_server_version
+        if parsed_version < Gem::Version.new(ClusterConfig.instance.server_version)
           skip "Server version #{version} or lower required, we have #{ClusterConfig.instance.server_version}"
         end
       end
     end
 
     def min_server_fcv(version)
-      unless version =~ /\A\d+\.\d+\z/
-        raise ArgumentError, "FCV can only be major.minor: #{version}"
-      end
+      parsed_version = Gem::Version.new(version)
 
       before(:all) do
-        unless ClusterConfig.instance.fcv_ish >= version
+        unless Gem::Version.new(ClusterConfig.instance.fcv_ish) >= parsed_version
           skip "FCV #{version} or higher required, we have #{ClusterConfig.instance.fcv_ish} (server #{ClusterConfig.instance.server_version})"
         end
       end
     end
 
     def max_server_fcv(version)
-      unless version =~ /\A\d+\.\d+\z/
-        raise ArgumentError, "Version can only be major.minor: #{version}"
-      end
+      parsed_version = Gem::Version.new(version)
 
       before(:all) do
-        if version < ClusterConfig.instance.fcv_ish
+        if parsed_version < Gem::Version.new(ClusterConfig.instance.fcv_ish)
           skip "FCV #{version} or lower required, we have #{ClusterConfig.instance.fcv_ish} (server #{ClusterConfig.instance.server_version})"
         end
       end
@@ -223,7 +215,8 @@ module Mrss
 
     def require_no_auth
       before(:all) do
-        if (ENV['AUTH'] && ENV['AUTH'] != 'noauth') || SpecConfig.instance.user || ClusterConfig.instance.auth_enabled?
+        auth = ENV.fetch('AUTH', '')
+        if (!auth.empty? && auth != 'noauth') || SpecConfig.instance.user || ClusterConfig.instance.auth_enabled?
           skip "Auth not allowed"
         end
       end

data/spec/shared/lib/mrss/docker_runner.rb

@@ -16,7 +16,7 @@ module Mrss
       opts.fetch(:default_script)
       opts.fetch(:project_lib_subdir)
 
-      @options = opts
+      @options = opts.merge(preload: true)
     end
 
     attr_reader :options
@@ -63,10 +63,14 @@ module Mrss
           @options[:mongo_only] = v.to_i
         end
 
-        opts.on('-p', '--preload', 'Preload Ruby toolchain and server binaries in docker') do |v|
+        opts.on('-p', '--preload', 'Preload Ruby toolchain and server binaries in docker (default)') do |v|
           @options[:preload] = v
         end
 
+        opts.on('-P', '--no-preload', 'Do not preload Ruby toolchain and server binaries in docker') do
+          @options[:preload] = false
+        end
+
         opts.on('-s', '--script=SCRIPT', 'Test script to invoke') do |v|
           @options[:script] = v
         end
@@ -173,7 +177,18 @@ module Mrss
     end
 
     def distro
-      @options[:distro] || 'ubuntu1604'
+      @options[:distro] || if app_tests?
+        'ubuntu2004'
+        else
+          case server_version
+          when '3.6'
+            'debian9'
+          when '4.0', '4.2'
+            'ubuntu1804'
+          else
+            'ubuntu2004'
+          end
+        end
     end
 
     BASE_IMAGES = {
@@ -253,6 +268,11 @@ module Mrss
       %w(1 true yes).include?(@env['FLE']&.downcase)
     end
 
+    # Mongoid
+    def app_tests?
+      %w(1 true yes).include?(@env['APP_TESTS']&.downcase)
+    end
+
     def num_exposed_ports
       case @env['TOPOLOGY'] || 'standalone'
       when 'standalone'

data/spec/shared/lib/mrss/eg_config_utils.rb

@@ -0,0 +1,51 @@
+autoload :YAML, 'yaml'
+require 'erubi'
+require 'erubi/capture_end'
+require 'tilt'
+
+module Mrss
+  module EgConfigUtils
+
+    DEBIAN_FOR_RUBY = {
+      'ruby-2.3' => 'debian92',
+      'ruby-2.4' => 'debian92',
+      'ruby-2.5' => 'debian10',
+      'ruby-2.6' => 'debian10',
+      'ruby-2.7' => 'debian10',
+      'ruby-3.0' => 'debian10',
+    }
+
+    def standard_debian_rubies(rubies, key: nil, &block)
+      rubies.flatten!
+      text = block.call
+      contents = YAML.load(text)
+      out = rubies.map do |ruby|
+        contents.merge(
+          'matrix_name' => "#{contents['matrix_name']} - #{ruby}",
+          'matrix_spec' => contents['matrix_spec'].merge(
+            'ruby' => ruby,
+            key || 'os' => DEBIAN_FOR_RUBY.fetch(ruby),
+          ),
+        )
+      end.to_yaml
+      text =~ /\A\n?(\s+)/
+      unless text
+        raise "Couldn't figure out indentation level"
+      end
+      indent = ' ' * ($1.length - 2)
+      "\n" + out.sub(/\A---.*\n/, indent).gsub("\n", "\n#{indent}")
+    end
+
+    def transform_config(template_path, context)
+      Tilt.new(template_path, engine_class: Erubi::CaptureEndEngine).render(context)
+    end
+
+    def generated_file_warning
+      <<-EOT
+# GENERATED FILE - DO NOT EDIT.
+# Run ./.evergreen/update-evergreen-configs to regenerate this file.
+
+EOT
+    end
+  end
+end

data/spec/shared/lib/mrss/event_subscriber.rb

@@ -84,22 +84,32 @@ module Mrss
 
     # Locates command stated events for the specified command name,
     # asserts that there is exactly one such event, and returns it.
-    def single_command_started_event(command_name, include_auth: false)
+    def single_command_started_event(command_name, include_auth: false, database_name: nil)
       events = if include_auth
                  started_events
                else
                  non_auth_command_started_events
                end
-      events.select! do |event|
-        event.command[command_name]
+      get_one_event(events, command_name, 'started', database_name: database_name)
+    end
+
+    # Locates command succeeded events for the specified command name,
+    # asserts that there is exactly one such event, and returns it.
+    def single_command_succeeded_event(command_name, database_name: nil)
+      get_one_event(succeeded_events, command_name, 'succeeded', database_name: database_name)
+    end
+
+    def get_one_event(events, command_name, kind, database_name: nil)
+      events = events.select do |event|
+        event.command_name == command_name and
+        database_name.nil? || database_name == event.database_name
       end
       if events.length != 1
-        raise "Expected a single #{command_name} event but we have #{events.length}"
+        raise "Expected a single '#{command_name}' #{kind} event#{database_name ? " for '#{database_name}'" : ''} but we have #{events.length}"
       end
       events.first
     end
 
-
     # Get the first succeeded event published for the name, and then delete it.
     #
     # @param [ String ] name The event name.

data/spec/shared/lib/mrss/lite_constraints.rb

@@ -87,12 +87,25 @@ module Mrss
 
     def require_libmongocrypt
       before(:all) do
-        unless ENV['LIBMONGOCRYPT_PATH']
+        # If FLE is set in environment, the entire test run is supposed to
+        # include FLE therefore run the FLE tests.
+        if (ENV['LIBMONGOCRYPT_PATH'] || '').empty? && (ENV['FLE'] || '').empty?
           skip 'Test requires path to libmongocrypt to be specified in LIBMONGOCRYPT_PATH env variable'
         end
       end
     end
 
+    def min_libmongocrypt_version(version)
+      require_libmongocrypt
+      before(:all) do
+        actual_version = Gem::Version.new(Mongo::Crypt::Binding.mongocrypt_version(nil))
+        min_version = Gem::Version.new(version)
+        unless actual_version >= min_version
+          skip "libmongocrypt version #{min_version} required, but version #{actual_version} is available"
+        end
+      end
+    end
+
     def require_no_libmongocrypt
       before(:all) do
         if ENV['LIBMONGOCRYPT_PATH']
@@ -187,5 +200,31 @@ module Mrss
         end
       end
     end
+
+    def require_fallbacks
+      before(:all) do
+        unless %w(yes true 1).include?((ENV['TEST_I18N_FALLBACKS'] || '').downcase)
+          skip 'Set TEST_I18N_FALLBACKS=1 environment variable to run these tests'
+        end
+      end
+    end
+
+    # This is a macro for retrying flaky tests on CI that occasionally fail.
+    # Note that the tests will only be retried on CI.
+    #
+    # @param [ Integer ] :tries The number of times to retry.
+    # @param [ Integer ] :sleep The number of seconds to sleep in between retries.
+    #   If nothing, or nil, is passed, we won't wait in between retries.
+    def retry_test(tries: 3, sleep: nil)
+      if %w(1 yes true).include?(ENV['CI'])
+        around do |example|
+          if sleep
+            example.run_with_retry retry: tries, retry_wait: sleep
+          else
+            example.run_with_retry retry: tries
+          end
+        end
+      end
+    end
   end
 end

data/spec/shared/lib/mrss/session_registry.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+require 'singleton'
+
+module Mrss
+
+  def self.patch_mongo_for_session_registry
+
+    Mongo::Client.class_eval do
+      alias :get_session_without_tracking :get_session
+
+      def get_session(options = {})
+        get_session_without_tracking(options).tap do |session|
+          SessionRegistry.instance.register(session) if session&.materialized?
+        end
+      end
+    end
+
+    Mongo::Session.class_eval do
+      alias :end_session_without_tracking :end_session
+
+      def end_session
+        SessionRegistry.instance.unregister(self)
+        end_session_without_tracking
+      end
+
+      alias :materialize_if_needed_without_tracking :materialize_if_needed
+
+      def materialize_if_needed
+        materialize_if_needed_without_tracking.tap do
+          SessionRegistry.instance.register(self)
+        end
+      end
+    end
+  end
+end
+
+module Mrss
+  class SessionRegistry
+    include Singleton
+
+    def initialize
+      @registry = {}
+    end
+
+    def register(session)
+      @registry[session.session_id] = session if session
+    end
+
+    def unregister(session)
+      return if session.ended? || !session.materialized?
+      @registry.delete(session.session_id)
+    end
+
+    def verify_sessions_ended!
+      @registry.delete_if { |_, session| session.ended? }
+
+      unless @registry.empty?
+        sessions = @registry.map { |_, session| session }
+        raise "Session registry contains live sessions: #{sessions.join(', ')}"
+      end
+    end
+
+    def clear_registry
+      @registry = {}
+    end
+  end
+end

data/spec/shared/lib/mrss/session_registry_legacy.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+require 'singleton'
+
+module Mrss
+
+  def self.patch_mongo_for_session_registry
+
+    Mongo::Client.class_eval do
+      alias :get_session_without_tracking :get_session
+
+      def get_session(options = {})
+        get_session_without_tracking(options).tap do |session|
+          SessionRegistry.instance.register(session)
+        end
+      end
+    end
+
+    Mongo::Session.class_eval do
+      alias :end_session_without_tracking :end_session
+
+      def end_session
+        SessionRegistry.instance.unregister(self)
+        end_session_without_tracking
+      end
+    end
+  end
+end
+
+module Mrss
+  class SessionRegistry
+    include Singleton
+
+    def initialize
+      @registry = {}
+    end
+
+    def register(session)
+      @registry[session.session_id] = session if session
+    end
+
+    def unregister(session)
+      @registry.delete(session.session_id) unless session.ended?
+    end
+
+    def verify_sessions_ended!
+      @registry.delete_if { |_, session| session.ended? }
+
+      unless @registry.empty?
+        sessions = @registry.map { |_, session| session }
+        raise "Session registry contains live sessions: #{sessions.join(', ')}"
+      end
+    end
+
+    def clear_registry
+      @registry = {}
+    end
+  end
+end