mongo 2.15.0 → 2.18.1

data/spec/runners/unified/test.rb

@@ -2,6 +2,7 @@
 # encoding: utf-8
 
 require 'runners/crud/requirement'
+require 'runners/unified/client_side_encryption_operations'
 require 'runners/unified/crud_operations'
 require 'runners/unified/grid_fs_operations'
 require 'runners/unified/ddl_operations'
@@ -9,16 +10,19 @@ require 'runners/unified/change_stream_operations'
 require 'runners/unified/support_operations'
 require 'runners/unified/assertions'
 require 'support/utils'
+require 'support/crypt'
 
 module Unified
 
   class Test
+    include ClientSideEncryptionOperations
     include CrudOperations
     include GridFsOperations
     include DdlOperations
     include ChangeStreamOperations
     include SupportOperations
     include Assertions
+    include RSpec::Core::Pending
 
     def initialize(spec, **opts)
       @spec = spec
@@ -132,11 +136,11 @@ module Unified
 
           create_client(**opts).tap do |client|
             @observe_sensitive = spec.use('observeSensitiveCommands')
+            subscriber = (@subscribers[client] ||= EventSubscriber.new)
             if oe = spec.use('observeEvents')
               oe.each do |event|
                 case event
                 when 'commandStartedEvent', 'commandSucceededEvent', 'commandFailedEvent'
-                  subscriber = (@subscribers[client] ||= EventSubscriber.new)
                   unless client.send(:monitoring).subscribers[Mongo::Monitoring::COMMAND].include?(subscriber)
                     client.subscribe(Mongo::Monitoring::COMMAND, subscriber)
                   end
@@ -146,7 +150,6 @@ module Unified
                     subscriber.ignore_commands(ignore_events)
                   end
                 when /\A(?:pool|connection)/
-                  subscriber = (@subscribers[client] ||= EventSubscriber.new)
                   unless client.send(:monitoring).subscribers[Mongo::Monitoring::CONNECTION_POOL]&.include?(subscriber)
                     client.subscribe(Mongo::Monitoring::CONNECTION_POOL, subscriber)
                   end
@@ -160,11 +163,25 @@ module Unified
           end
         when 'database'
           client = entities.get(:client, spec.use!('client'))
-          client.use(spec.use!('databaseName')).database
+          opts = Utils.snakeize_hash(spec.use('databaseOptions') || {})
+            .merge(database: spec.use!('databaseName'))
+          if opts.key?(:read_preference)
+            opts[:read] = opts.delete(:read_preference)
+            if opts[:read].key?(:max_staleness_seconds)
+              opts[:read][:max_staleness] = opts[:read].delete(:max_staleness_seconds)
+            end
+          end
+          client.with(opts).database
         when 'collection'
           database = entities.get(:database, spec.use!('database'))
           # TODO verify
           opts = Utils.snakeize_hash(spec.use('collectionOptions') || {})
+          if opts.key?(:read_preference)
+            opts[:read] = opts.delete(:read_preference)
+            if opts[:read].key?(:max_staleness_seconds)
+              opts[:read][:max_staleness] = opts[:read].delete(:max_staleness_seconds)
+            end
+          end
           database[spec.use!('collectionName'), opts]
         when 'bucket'
           database = entities.get(:database, spec.use!('database'))
@@ -179,6 +196,61 @@ module Unified
           end
 
           client.start_session(**opts)
+        when 'clientEncryption'
+          client_encryption_opts = spec.use!('clientEncryptionOpts')
+          key_vault_client = entities.get(:client, client_encryption_opts['keyVaultClient'])
+          opts = {
+            key_vault_namespace: client_encryption_opts['keyVaultNamespace'],
+            kms_providers: Utils.snakeize_hash(client_encryption_opts['kmsProviders']),
+            kms_tls_options: {
+              kmip: {
+                ssl_cert: SpecConfig.instance.fle_kmip_tls_certificate_key_file,
+                ssl_key: SpecConfig.instance.fle_kmip_tls_certificate_key_file,
+                ssl_ca_cert: SpecConfig.instance.fle_kmip_tls_ca_file
+              }
+            }
+          }
+          opts[:kms_providers] = opts[:kms_providers].map do |provider, options|
+            converted_options = options.map do |key, value|
+              converted_value = if value == { '$$placeholder'.to_sym => 1 }
+                case provider
+                when :aws
+                  case key
+                  when :access_key_id then SpecConfig.instance.fle_aws_key
+                  when :secret_access_key then SpecConfig.instance.fle_aws_secret
+                  end
+                when :azure
+                  case key
+                  when :tenant_id then SpecConfig.instance.fle_azure_tenant_id
+                  when :client_id then SpecConfig.instance.fle_azure_client_id
+                  when :client_secret then SpecConfig.instance.fle_azure_client_secret
+                  end
+                when :gcp
+                  case key
+                  when :email then SpecConfig.instance.fle_gcp_email
+                  when :private_key then SpecConfig.instance.fle_gcp_private_key
+                  end
+                when :kmip
+                  case key
+                  when :endpoint then SpecConfig.instance.fle_kmip_endpoint
+                  end
+                when :local
+                  case key
+                  when :key then Crypt::LOCAL_MASTER_KEY
+                  end
+                end
+              else
+                value
+              end
+              [key, converted_value]
+            end.to_h
+            [provider, converted_options]
+          end.to_h
+
+          Mongo::ClientEncryption.new(
+            key_vault_client,
+            opts
+          )
         else
           raise NotImplementedError, "Unknown type #{type}"
         end
@@ -192,7 +264,8 @@ module Unified
     def set_initial_data
       @spec['initialData']&.each do |entity_spec|
         spec = UsingHash[entity_spec]
-        collection = root_authorized_client.use(spec.use!('databaseName'))[spec.use!('collectionName')]
+        collection = root_authorized_client.with(write_concern: {w: :majority}).
+          use(spec.use!('databaseName'))[spec.use!('collectionName')]
         collection.drop
         docs = spec.use!('documents')
         if docs.any?
@@ -263,10 +336,19 @@ module Unified
         if name.to_s == 'loop'
           method_name = "_#{name}"
         end
+
+        if ["modify_collection"].include?(name.to_s)
+          skip "Mongo Ruby Driver does not support #{name.to_s}"
+        end
+
         if expected_error = op.use('expectError')
           begin
-            send(method_name, op)
-          rescue Mongo::Error, BSON::String::IllegalKey => e
+            unless respond_to?(method_name)
+              raise Error::UnsupportedOperation, "Mongo Ruby Driver does not support #{name.to_s}"
+            end
+
+            public_send(method_name, op)
+          rescue Mongo::Error, BSON::String::IllegalKey, ArgumentError => e
             if expected_error.use('isClientError')
               # isClientError doesn't actually mean a client error.
               # It means anything other than OperationFailure. DRIVERS-1799
@@ -317,10 +399,16 @@ module Unified
             raise Error::ErrorMismatch, "Expected exception but none was raised"
           end
         else
+          unless respond_to?(method_name, true)
+            raise Error::UnsupportedOperation, "Mongo Ruby Driver does not support #{name.to_s}"
+          end
+
           result = send(method_name, op)
           if expected_result = op.use('expectResult')
-            if result.nil? && !expected_result.empty?
-              raise Error::ResultMismatch, "Actual result nil but expected result #{expected_result}"
+            if result.nil? && expected_result.keys == ["$$unsetOrMatches"]
+              return
+            elsif result.nil? && !expected_result.empty?
+              raise Error::ResultMismatch, "#{msg}: expected #{expected} but got nil"
             elsif Array === expected_result
               assert_documents_match(result, expected_result)
             else

data/spec/runners/unified/test_group.rb

@@ -6,7 +6,7 @@ module Unified
   class TestGroup
     def initialize(path, **opts)
       if String === path
-        data = YAML.load(File.read(path))
+        data = ::Utils.load_spec_yaml_file(path)
       else
         data = path
       end

data/spec/runners/unified.rb

@@ -1,14 +1,16 @@
 # frozen_string_literal: true
 # encoding: utf-8
 
+require 'support/using_hash'
 require 'runners/unified/error'
 require 'runners/unified/entity_map'
 require 'runners/unified/event_subscriber'
 require 'runners/unified/test'
 require 'runners/unified/test_group'
-require 'runners/unified/using_hash'
 
 def define_unified_spec_tests(base_path, paths, expect_failure: false)
+  config_override :validate_update_replace, true
+
   paths.each do |path|
     basename = path[base_path.length+1...path.length]
     context basename do
@@ -61,13 +63,18 @@ def define_unified_spec_tests(base_path, paths, expect_failure: false)
               begin
                 test.create_entities
                 test.set_initial_data
-                lambda do
+                begin
                   test.run
                   test.assert_outcome
                   test.assert_events
                 # HACK: other errors are possible and likely will need to
                 # be added here later as the tests evolve.
-                end.should raise_error(Mongo::Error::OperationFailure)
+                rescue Mongo::Error::OperationFailure, Unified::Error::UnsupportedOperation
+                rescue => e
+                  fail "Expected to raise Mongo::Error::OperationFailure or Unified::Error::UnsupportedOperation, got #{e}"
+                else
+                  fail "Expected to raise Mongo::Error::OperationFailure or Unified::Error::UnsupportedOperation, but no error was raised"
+                end
               ensure
                 test.cleanup
               end

data/spec/shared/lib/mrss/cluster_config.rb

@@ -209,7 +209,12 @@ module Mrss
       @server_version = build_info['version']
       @enterprise = build_info['modules'] && build_info['modules'].include?('enterprise')
 
-      @server_parameters = client.use(:admin).command(getParameter: '*').first
+      @server_parameters = begin
+        client.use(:admin).command(getParameter: '*').first
+      rescue => e
+        STDERR.puts("WARNING: Failed to obtain server parameters: #{e.class}: #{e.message}")
+        {}
+      end
 
       if !sharded_ish? && short_server_version >= '3.4'
         rv = @server_parameters['featureCompatibilityVersion']

data/spec/shared/lib/mrss/constraints.rb

@@ -4,48 +4,40 @@
 module Mrss
   module Constraints
     def min_server_version(version)
-      unless version =~ /\A\d+\.\d+\z/
-        raise ArgumentError, "Version can only be major.minor: #{version}"
-      end
+      parsed_version = Gem::Version.new(version)
 
       before(:all) do
-        if version > ClusterConfig.instance.server_version
+        if parsed_version > Gem::Version.new(ClusterConfig.instance.server_version)
           skip "Server version #{version} or higher required, we have #{ClusterConfig.instance.server_version}"
         end
       end
     end
 
     def max_server_version(version)
-      unless version =~ /\A\d+\.\d+\z/
-        raise ArgumentError, "Version can only be major.minor: #{version}"
-      end
+      parsed_version = Gem::Version.new(version)
 
       before(:all) do
-        if version < ClusterConfig.instance.short_server_version
+        if parsed_version < Gem::Version.new(ClusterConfig.instance.server_version)
           skip "Server version #{version} or lower required, we have #{ClusterConfig.instance.server_version}"
         end
       end
     end
 
     def min_server_fcv(version)
-      unless version =~ /\A\d+\.\d+\z/
-        raise ArgumentError, "FCV can only be major.minor: #{version}"
-      end
+      parsed_version = Gem::Version.new(version)
 
       before(:all) do
-        unless ClusterConfig.instance.fcv_ish >= version
+        unless Gem::Version.new(ClusterConfig.instance.fcv_ish) >= parsed_version
           skip "FCV #{version} or higher required, we have #{ClusterConfig.instance.fcv_ish} (server #{ClusterConfig.instance.server_version})"
         end
       end
     end
 
     def max_server_fcv(version)
-      unless version =~ /\A\d+\.\d+\z/
-        raise ArgumentError, "Version can only be major.minor: #{version}"
-      end
+      parsed_version = Gem::Version.new(version)
 
       before(:all) do
-        if version < ClusterConfig.instance.fcv_ish
+        if parsed_version < Gem::Version.new(ClusterConfig.instance.fcv_ish)
           skip "FCV #{version} or lower required, we have #{ClusterConfig.instance.fcv_ish} (server #{ClusterConfig.instance.server_version})"
         end
       end
@@ -223,7 +215,8 @@ module Mrss
 
     def require_no_auth
       before(:all) do
-        if (ENV['AUTH'] && ENV['AUTH'] != 'noauth') || SpecConfig.instance.user || ClusterConfig.instance.auth_enabled?
+        auth = ENV.fetch('AUTH', '')
+        if (!auth.empty? && auth != 'noauth') || SpecConfig.instance.user || ClusterConfig.instance.auth_enabled?
           skip "Auth not allowed"
         end
       end
@@ -257,10 +250,14 @@ module Mrss
       end
     end
 
-    def require_multi_shard
+    def require_multi_mongos
       before(:all) do
         if ClusterConfig.instance.topology == :sharded && SpecConfig.instance.addresses.length == 1
-          skip 'Test requires a minimum of two shards if run in sharded topology'
+          skip 'Test requires a minimum of two mongoses if run in sharded topology'
+        end
+
+        if ClusterConfig.instance.topology == :load_balanced && SpecConfig.instance.single_mongos?
+          skip 'Test requires a minimum of two mongoses if run in load-balanced topology'
         end
       end
     end
@@ -273,17 +270,19 @@ module Mrss
     #
     # In load-balanced topology, the same problem can happen when there is
     # more than one mongos behind the load balancer.
-    def require_no_multi_shard
+    def require_no_multi_mongos
       before(:all) do
         if ClusterConfig.instance.topology == :sharded && SpecConfig.instance.addresses.length > 1
-          skip 'Test requires a single shard if run in sharded topology'
+          skip 'Test requires a single mongos if run in sharded topology'
         end
         if ClusterConfig.instance.topology == :load_balanced && !SpecConfig.instance.single_mongos?
-          skip 'Test requires a minimum of two shards if run in sharded topology'
+          skip 'Test requires a single mongos, as indicated by SINGLE_MONGOS=1 environment variable, if run in load-balanced topology'
         end
       end
     end
 
+    alias :require_no_multi_shard :require_no_multi_mongos
+
     def require_wired_tiger
       before(:all) do
         # Storage detection fails for serverless instances. However, it is safe to

data/spec/shared/lib/mrss/docker_runner.rb

@@ -16,7 +16,7 @@ module Mrss
       opts.fetch(:default_script)
       opts.fetch(:project_lib_subdir)
 
-      @options = opts
+      @options = opts.merge(preload: true)
     end
 
     attr_reader :options
@@ -63,10 +63,14 @@ module Mrss
           @options[:mongo_only] = v.to_i
         end
 
-        opts.on('-p', '--preload', 'Preload Ruby toolchain and server binaries in docker') do |v|
+        opts.on('-p', '--preload', 'Preload Ruby toolchain and server binaries in docker (default)') do |v|
           @options[:preload] = v
         end
 
+        opts.on('-P', '--no-preload', 'Do not preload Ruby toolchain and server binaries in docker') do
+          @options[:preload] = false
+        end
+
         opts.on('-s', '--script=SCRIPT', 'Test script to invoke') do |v|
           @options[:script] = v
         end
@@ -173,7 +177,18 @@ module Mrss
     end
 
     def distro
-      @options[:distro] || 'ubuntu1604'
+      @options[:distro] || if app_tests?
+        'ubuntu2004'
+        else
+          case server_version
+          when '3.6'
+            'debian9'
+          when '4.0', '4.2'
+            'ubuntu1804'
+          else
+            'ubuntu2004'
+          end
+        end
     end
 
     BASE_IMAGES = {
@@ -253,6 +268,11 @@ module Mrss
       %w(1 true yes).include?(@env['FLE']&.downcase)
     end
 
+    # Mongoid
+    def app_tests?
+      %w(1 true yes).include?(@env['APP_TESTS']&.downcase)
+    end
+
     def num_exposed_ports
       case @env['TOPOLOGY'] || 'standalone'
       when 'standalone'

data/spec/shared/lib/mrss/eg_config_utils.rb

@@ -0,0 +1,51 @@
+autoload :YAML, 'yaml'
+require 'erubi'
+require 'erubi/capture_end'
+require 'tilt'
+
+module Mrss
+  module EgConfigUtils
+
+    DEBIAN_FOR_RUBY = {
+      'ruby-2.3' => 'debian92',
+      'ruby-2.4' => 'debian92',
+      'ruby-2.5' => 'debian10',
+      'ruby-2.6' => 'debian10',
+      'ruby-2.7' => 'debian10',
+      'ruby-3.0' => 'debian10',
+    }
+
+    def standard_debian_rubies(rubies, key: nil, &block)
+      rubies.flatten!
+      text = block.call
+      contents = YAML.load(text)
+      out = rubies.map do |ruby|
+        contents.merge(
+          'matrix_name' => "#{contents['matrix_name']} - #{ruby}",
+          'matrix_spec' => contents['matrix_spec'].merge(
+            'ruby' => ruby,
+            key || 'os' => DEBIAN_FOR_RUBY.fetch(ruby),
+          ),
+        )
+      end.to_yaml
+      text =~ /\A\n?(\s+)/
+      unless text
+        raise "Couldn't figure out indentation level"
+      end
+      indent = ' ' * ($1.length - 2)
+      "\n" + out.sub(/\A---.*\n/, indent).gsub("\n", "\n#{indent}")
+    end
+
+    def transform_config(template_path, context)
+      Tilt.new(template_path, engine_class: Erubi::CaptureEndEngine).render(context)
+    end
+
+    def generated_file_warning
+      <<-EOT
+# GENERATED FILE - DO NOT EDIT.
+# Run ./.evergreen/update-evergreen-configs to regenerate this file.
+
+EOT
+    end
+  end
+end

data/spec/shared/lib/mrss/event_subscriber.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+module Mrss
+  # Test event subscriber.
+  class EventSubscriber
+
+    # The mappings of event names to types.
+    MAPPINGS = {
+      'topology_opening_event' => Mongo::Monitoring::Event::TopologyOpening,
+      'topology_description_changed_event' => Mongo::Monitoring::Event::TopologyChanged,
+      'topology_closed_event' => Mongo::Monitoring::Event::TopologyClosed,
+      'server_opening_event' => Mongo::Monitoring::Event::ServerOpening,
+      'server_description_changed_event' => Mongo::Monitoring::Event::ServerDescriptionChanged,
+      'server_closed_event' => Mongo::Monitoring::Event::ServerClosed
+    }.freeze
+
+    attr_reader :all_events
+
+    attr_reader :started_events
+
+    attr_reader :succeeded_events
+
+    attr_reader :failed_events
+
+    attr_reader :published_events
+
+    # @param [ String ] name Optional name for the event subscriber.
+    def initialize(name: nil)
+      @mutex = Mutex.new
+      clear_events!
+      @name = name
+    end
+
+    def to_s
+      %Q`#<EventSubscriber:#{@name ? "\"#{@name}\"" : '%x' % object_id} \
+  started=#{started_events.length} \
+  succeeded=#{succeeded_events.length} \
+  failed=#{failed_events.length} \
+  published=#{published_events.length}>`
+    end
+
+    alias :inspect :to_s
+
+    # Event retrieval
+
+    def select_started_events(cls)
+      started_events.select do |event|
+        event.is_a?(cls)
+      end
+    end
+
+    def select_succeeded_events(cls)
+      succeeded_events.select do |event|
+        event.is_a?(cls)
+      end
+    end
+
+    def select_completed_events(*classes)
+      (succeeded_events + failed_events).select do |event|
+        classes.any? { |c| c === event }
+      end
+    end
+
+    def select_published_events(cls)
+      published_events.select do |event|
+        event.is_a?(cls)
+      end
+    end
+
+    # Filters command started events for the specified command name.
+    def command_started_events(command_name)
+      started_events.select do |event|
+        event.command[command_name]
+      end
+    end
+
+    def non_auth_command_started_events
+      started_events.reject do |event|
+        %w(authenticate getnonce saslSstart saslContinue).any? do |cmd|
+          event.command[cmd]
+        end
+      end
+    end
+
+    # Locates command stated events for the specified command name,
+    # asserts that there is exactly one such event, and returns it.
+    def single_command_started_event(command_name, include_auth: false, database_name: nil)
+      events = if include_auth
+                 started_events
+               else
+                 non_auth_command_started_events
+               end
+      get_one_event(events, command_name, 'started', database_name: database_name)
+    end
+
+    # Locates command succeeded events for the specified command name,
+    # asserts that there is exactly one such event, and returns it.
+    def single_command_succeeded_event(command_name, database_name: nil)
+      get_one_event(succeeded_events, command_name, 'succeeded', database_name: database_name)
+    end
+
+    def get_one_event(events, command_name, kind, database_name: nil)
+      events = events.select do |event|
+        event.command_name == command_name and
+        database_name.nil? || database_name == event.database_name
+      end
+      if events.length != 1
+        raise "Expected a single '#{command_name}' #{kind} event#{database_name ? " for '#{database_name}'" : ''} but we have #{events.length}"
+      end
+      events.first
+    end
+
+    # Get the first succeeded event published for the name, and then delete it.
+    #
+    # @param [ String ] name The event name.
+    #
+    # @return [ Event ] The matching event.
+    def first_event(name)
+      cls = MAPPINGS[name]
+      if cls.nil?
+        raise ArgumentError, "Bogus event name #{name}"
+      end
+      matching = succeeded_events.find do |event|
+        cls === event
+      end
+      succeeded_events.delete(matching)
+      matching
+    end
+
+    # Event recording
+
+    # Cache the started event.
+    #
+    # @param [ Event ] event The event.
+    def started(event)
+      @mutex.synchronize do
+        started_events << event
+        all_events << event
+      end
+    end
+
+    # Cache the succeeded event.
+    #
+    # @param [ Event ] event The event.
+    def succeeded(event)
+      @mutex.synchronize do
+        succeeded_events << event
+        all_events << event
+      end
+    end
+
+    # Cache the failed event.
+    #
+    # @param [ Event ] event The event.
+    def failed(event)
+      @mutex.synchronize do
+        failed_events << event
+        all_events << event
+      end
+    end
+
+    def published(event)
+      @mutex.synchronize do
+        published_events << event
+        all_events << event
+      end
+    end
+
+    # Clear all cached events.
+    def clear_events!
+      @all_events = []
+      @started_events = []
+      @succeeded_events = []
+      @failed_events = []
+      @published_events = []
+      self
+    end
+  end
+  # Only handles succeeded events correctly.
+  class PhasedEventSubscriber < EventSubscriber
+    def initialize
+      super
+      @phase_events = {}
+    end
+
+    def phase_finished(phase_index)
+      @phase_events[phase_index] = succeeded_events
+      @succeeded_events = []
+    end
+
+    def phase_events(phase_index)
+      @phase_events[phase_index]
+    end
+
+    def event_count
+      @phase_events.inject(0) do |sum, event|
+        sum + event.length
+      end
+    end
+  end
+
+  class VerboseEventSubscriber < EventSubscriber
+    %w(started succeeded failed published).each do |meth|
+      define_method(meth) do |event|
+        puts event.summary
+        super(event)
+      end
+    end
+  end
+end