mongoid 7.4.0 → 7.4.1

data/spec/mongoid/clients/factory_spec.rb

@@ -36,6 +36,7 @@ describe Mongoid::Clients::Factory do
       context "when the configuration exists" do
 
         context "when the configuration is standard" do
+          restore_config_clients
 
           let(:config) do
             {
@@ -46,7 +47,6 @@ describe Mongoid::Clients::Factory do
 
           before do
             Mongoid::Config.send(:clients=, config)
-            # TODO: We should restore overwritten configuration in after block
           end
 
           after do
@@ -120,6 +120,7 @@ describe Mongoid::Clients::Factory do
         end
 
         context "when the configuration has no ports" do
+          restore_config_clients
 
           let(:config) do
             {
@@ -130,7 +131,6 @@ describe Mongoid::Clients::Factory do
 
           before do
             Mongoid::Config.send(:clients=, config)
-            # TODO: We should restore overwritten configuration in after block
           end
 
           after do
@@ -165,6 +165,7 @@ describe Mongoid::Clients::Factory do
         context "when configured via a uri" do
 
           context "when the uri has a single host:port" do
+            restore_config_clients
 
             let(:config) do
               {
@@ -175,7 +176,6 @@ describe Mongoid::Clients::Factory do
 
             before do
               Mongoid::Config.send(:clients=, config)
-              # TODO: We should restore overwritten configuration in after block
             end
 
             after do
@@ -204,6 +204,7 @@ describe Mongoid::Clients::Factory do
           end
 
           context "when the uri has multiple host:port pairs" do
+            restore_config_clients
 
             let(:config) do
               {
@@ -214,7 +215,6 @@ describe Mongoid::Clients::Factory do
 
             before do
               Mongoid::Config.send(:clients=, config)
-              # TODO: We should restore overwritten configuration in after block
             end
 
             after do
@@ -255,6 +255,7 @@ describe Mongoid::Clients::Factory do
     end
 
     context "when no name is provided" do
+      restore_config_clients
 
       let(:config) do
         { default: { hosts: SpecConfig.instance.addresses, database: database_id }}
@@ -262,7 +263,6 @@ describe Mongoid::Clients::Factory do
 
       before do
         Mongoid::Config.send(:clients=, config)
-        # TODO: We should restore overwritten configuration in after block
       end
 
       after do
@@ -289,12 +289,12 @@ describe Mongoid::Clients::Factory do
     end
 
     context "when nil is provided and no default config" do
+      restore_config_clients
 
       let(:config) { nil }
 
       before do
         Mongoid.clients[:default] = nil
-        # TODO: We should restore overwritten configuration in after block
       end
 
       it "raises NoClientsConfig error" do
@@ -304,6 +304,7 @@ describe Mongoid::Clients::Factory do
   end
 
   describe ".default" do
+    restore_config_clients
 
     let(:config) do
       { default: { hosts: SpecConfig.instance.addresses, database: database_id }}
@@ -311,7 +312,6 @@ describe Mongoid::Clients::Factory do
 
     before do
       Mongoid::Config.send(:clients=, config)
-      # TODO: We should restore overwritten configuration in after block
     end
 
     after do
@@ -338,6 +338,7 @@ describe Mongoid::Clients::Factory do
   end
 
   context "when options are provided with string keys" do
+    restore_config_clients
 
     let(:config) do
       {
@@ -354,7 +355,6 @@ describe Mongoid::Clients::Factory do
 
     before do
       Mongoid::Config.send(:clients=, config)
-      # TODO: We should restore overwritten configuration in after block
     end
 
     after do
@@ -393,6 +393,8 @@ describe Mongoid::Clients::Factory do
   end
 
   context "unexpected config options" do
+    restore_config_clients
+
     let(:unknown_opts) do
       {
         bad_one: 1,
@@ -410,11 +412,8 @@ describe Mongoid::Clients::Factory do
       }
     end
 
-    around(:each) do |example|
-      old_config = Mongoid::Config.clients
+    before do
       Mongoid::Config.send(:clients=, config)
-      example.run
-      Mongoid::Config.send(:clients=, old_config)
     end
 
     [:bad_one, :bad_two].each do |env|

data/spec/mongoid/config/environment_spec.rb

@@ -106,7 +106,7 @@ describe Mongoid::Config::Environment do
 
     context 'when file found' do
       before do
-        allow(File).to receive(:new).with('mongoid.yml').and_return(StringIO.new(file_contents))
+        allow(File).to receive(:read).with('mongoid.yml').and_return(file_contents)
       end
 
       let(:file_contents) do
@@ -162,5 +162,43 @@ describe Mongoid::Config::Environment do
         it { is_expected.to be_nil }
       end
     end
+
+    context 'when configuration includes schema map' do
+      paths = Dir.glob(File.join(File.dirname(__FILE__), '../../support/schema_maps/*.json'))
+
+      if paths.empty?
+        raise "Expected to find some schema maps"
+      end
+
+      before do
+        allow(File).to receive(:read).with('mongoid.yml').and_return(file_contents)
+      end
+
+      let(:file_contents) do
+        <<~FILE
+          test:
+            clients:
+              default:
+                database: mongoid_test
+                hosts: [localhost]
+                options:
+                  auto_encryption_options:
+                    schema_map: #{schema_map.to_yaml.sub(/\A---/, '').gsub(/\n/, "\n" + ' '*100)}
+        FILE
+      end
+
+      paths.each do |path|
+        context File.basename(path) do
+          let(:schema_map) do
+            BSON::ExtJSON.parse(File.read(path))
+          end
+
+          it 'loads successfully' do
+            subject.should be_a(Hash)
+            subject.fetch('clients').fetch('default').fetch('options').fetch('auto_encryption_options').fetch('schema_map').should be_a(Hash)
+          end
+        end
+      end
+    end
   end
 end

data/spec/mongoid/config_spec.rb

@@ -531,6 +531,46 @@ describe Mongoid::Config do
         end
       end
     end
+
+    context 'when schema map is provided with uuid' do
+      let(:file) do
+        File.join(File.dirname(__FILE__), "..", "config", "mongoid_with_schema_map_uuid.yml")
+      end
+
+      before do
+        described_class.load!(file, :test)
+      end
+
+      let(:client) { Mongoid.default_client }
+
+      # Wrapping libraries are only recognized by driver 2.13.0+.
+      min_driver_version '2.13'
+
+      it 'passes uuid to driver' do
+        Mongo::Client.should receive(:new).with(SpecConfig.instance.addresses,
+          auto_encryption_options: {
+            'key_vault_namespace' => 'admin.datakeys',
+            'kms_providers' => {'local' => {'key' => 'z7iYiYKLuYymEWtk4kfny1ESBwwFdA58qMqff96A8ghiOcIK75lJGPUIocku8LOFjQuEgeIP4xlln3s7r93FV9J5sAE7zg8U'}},
+            'schema_map' => {'blog_development.comments' => {
+              'bsonType' => 'object',
+              'properties' => {
+                'message' => {'encrypt' => {
+                  'algorithm' => 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic',
+                  'bsonType' => 'string',
+                  'keyId' => [BSON::Binary.new("G\xF0 5\xCC@HX\xA2%b\x97\xA9a\xA8\xE7", :uuid)],
+                }},
+              },
+            }}},
+          database: 'mongoid_test',
+          platform: "mongoid-#{Mongoid::VERSION}",
+          wrapping_libraries: [
+            {'name' => 'Mongoid', 'version' => Mongoid::VERSION},
+          ],
+        )
+
+        client
+      end
+    end
   end
 
   describe "#options=" do

data/spec/mongoid/errors/mongoid_error_spec.rb

@@ -9,7 +9,7 @@ describe Mongoid::Errors::MongoidError do
   let(:options) { {} }
 
   before do
-    if RUBY_VERSION.start_with?('3.', '2.7')
+    if RUBY_VERSION >= '2.7'
       {"message_title" => "message", "summary_title" => "summary", "resolution_title" => "resolution"}.each do |key, name|
         expect(::I18n).to receive(:translate).with("mongoid.errors.messages.#{key}", **{}).and_return(name)
       end

data/spec/shared/lib/mrss/constraints.rb

@@ -4,48 +4,40 @@
 module Mrss
   module Constraints
     def min_server_version(version)
-      unless version =~ /\A\d+\.\d+\z/
-        raise ArgumentError, "Version can only be major.minor: #{version}"
-      end
+      parsed_version = Gem::Version.new(version)
 
       before(:all) do
-        if version > ClusterConfig.instance.server_version
+        if parsed_version > Gem::Version.new(ClusterConfig.instance.server_version)
           skip "Server version #{version} or higher required, we have #{ClusterConfig.instance.server_version}"
         end
       end
     end
 
     def max_server_version(version)
-      unless version =~ /\A\d+\.\d+\z/
-        raise ArgumentError, "Version can only be major.minor: #{version}"
-      end
+      parsed_version = Gem::Version.new(version)
 
       before(:all) do
-        if version < ClusterConfig.instance.short_server_version
+        if parsed_version < Gem::Version.new(ClusterConfig.instance.server_version)
           skip "Server version #{version} or lower required, we have #{ClusterConfig.instance.server_version}"
         end
       end
     end
 
     def min_server_fcv(version)
-      unless version =~ /\A\d+\.\d+\z/
-        raise ArgumentError, "FCV can only be major.minor: #{version}"
-      end
+      parsed_version = Gem::Version.new(version)
 
       before(:all) do
-        unless ClusterConfig.instance.fcv_ish >= version
+        unless Gem::Version.new(ClusterConfig.instance.fcv_ish) >= parsed_version
           skip "FCV #{version} or higher required, we have #{ClusterConfig.instance.fcv_ish} (server #{ClusterConfig.instance.server_version})"
         end
       end
     end
 
     def max_server_fcv(version)
-      unless version =~ /\A\d+\.\d+\z/
-        raise ArgumentError, "Version can only be major.minor: #{version}"
-      end
+      parsed_version = Gem::Version.new(version)
 
       before(:all) do
-        if version < ClusterConfig.instance.fcv_ish
+        if parsed_version < Gem::Version.new(ClusterConfig.instance.fcv_ish)
           skip "FCV #{version} or lower required, we have #{ClusterConfig.instance.fcv_ish} (server #{ClusterConfig.instance.server_version})"
         end
       end

data/spec/shared/lib/mrss/docker_runner.rb

@@ -16,7 +16,7 @@ module Mrss
       opts.fetch(:default_script)
       opts.fetch(:project_lib_subdir)
 
-      @options = opts
+      @options = opts.merge(preload: true)
     end
 
     attr_reader :options
@@ -63,10 +63,14 @@ module Mrss
           @options[:mongo_only] = v.to_i
         end
 
-        opts.on('-p', '--preload', 'Preload Ruby toolchain and server binaries in docker') do |v|
+        opts.on('-p', '--preload', 'Preload Ruby toolchain and server binaries in docker (default)') do |v|
           @options[:preload] = v
         end
 
+        opts.on('-P', '--no-preload', 'Do not preload Ruby toolchain and server binaries in docker') do
+          @options[:preload] = false
+        end
+
         opts.on('-s', '--script=SCRIPT', 'Test script to invoke') do |v|
           @options[:script] = v
         end
@@ -173,7 +177,18 @@ module Mrss
     end
 
     def distro
-      @options[:distro] || 'ubuntu1804'
+      @options[:distro] || if app_tests?
+        'ubuntu2004'
+        else
+          case server_version
+          when '3.6'
+            'debian9'
+          when '4.0', '4.2'
+            'ubuntu1804'
+          else
+            'ubuntu2004'
+          end
+        end
     end
 
     BASE_IMAGES = {
@@ -253,6 +268,11 @@ module Mrss
       %w(1 true yes).include?(@env['FLE']&.downcase)
     end
 
+    # Mongoid
+    def app_tests?
+      %w(1 true yes).include?(@env['APP_TESTS']&.downcase)
+    end
+
     def num_exposed_ports
       case @env['TOPOLOGY'] || 'standalone'
       when 'standalone'

data/spec/shared/lib/mrss/eg_config_utils.rb

@@ -0,0 +1,51 @@
+autoload :YAML, 'yaml'
+require 'erubi'
+require 'erubi/capture_end'
+require 'tilt'
+
+module Mrss
+  module EgConfigUtils
+
+    DEBIAN_FOR_RUBY = {
+      'ruby-2.3' => 'debian92',
+      'ruby-2.4' => 'debian92',
+      'ruby-2.5' => 'debian10',
+      'ruby-2.6' => 'debian10',
+      'ruby-2.7' => 'debian10',
+      'ruby-3.0' => 'debian10',
+    }
+
+    def standard_debian_rubies(rubies, key: nil, &block)
+      rubies.flatten!
+      text = block.call
+      contents = YAML.load(text)
+      out = rubies.map do |ruby|
+        contents.merge(
+          'matrix_name' => "#{contents['matrix_name']} - #{ruby}",
+          'matrix_spec' => contents['matrix_spec'].merge(
+            'ruby' => ruby,
+            key || 'os' => DEBIAN_FOR_RUBY.fetch(ruby),
+          ),
+        )
+      end.to_yaml
+      text =~ /\A\n?(\s+)/
+      unless text
+        raise "Couldn't figure out indentation level"
+      end
+      indent = ' ' * ($1.length - 2)
+      "\n" + out.sub(/\A---.*\n/, indent).gsub("\n", "\n#{indent}")
+    end
+
+    def transform_config(template_path, context)
+      Tilt.new(template_path, engine_class: Erubi::CaptureEndEngine).render(context)
+    end
+
+    def generated_file_warning
+      <<-EOT
+# GENERATED FILE - DO NOT EDIT.
+# Run ./.evergreen/update-evergreen-configs to regenerate this file.
+
+EOT
+    end
+  end
+end

data/spec/shared/lib/mrss/event_subscriber.rb

@@ -84,22 +84,32 @@ module Mrss
 
     # Locates command stated events for the specified command name,
     # asserts that there is exactly one such event, and returns it.
-    def single_command_started_event(command_name, include_auth: false)
+    def single_command_started_event(command_name, include_auth: false, database_name: nil)
       events = if include_auth
                  started_events
                else
                  non_auth_command_started_events
                end
-      events.select! do |event|
-        event.command[command_name]
+      get_one_event(events, command_name, 'started', database_name: database_name)
+    end
+
+    # Locates command succeeded events for the specified command name,
+    # asserts that there is exactly one such event, and returns it.
+    def single_command_succeeded_event(command_name, database_name: nil)
+      get_one_event(succeeded_events, command_name, 'succeeded', database_name: database_name)
+    end
+
+    def get_one_event(events, command_name, kind, database_name: nil)
+      events = events.select do |event|
+        event.command_name == command_name and
+        database_name.nil? || database_name == event.database_name
       end
       if events.length != 1
-        raise "Expected a single #{command_name} event but we have #{events.length}"
+        raise "Expected a single '#{command_name}' #{kind} event#{database_name ? " for '#{database_name}'" : ''} but we have #{events.length}"
       end
       events.first
     end
 
-
     # Get the first succeeded event published for the name, and then delete it.
     #
     # @param [ String ] name The event name.

data/spec/shared/lib/mrss/lite_constraints.rb

@@ -87,12 +87,25 @@ module Mrss
 
     def require_libmongocrypt
       before(:all) do
-        unless ENV['LIBMONGOCRYPT_PATH']
+        # If FLE is set in environment, the entire test run is supposed to
+        # include FLE therefore run the FLE tests.
+        if (ENV['LIBMONGOCRYPT_PATH'] || '').empty? && (ENV['FLE'] || '').empty?
           skip 'Test requires path to libmongocrypt to be specified in LIBMONGOCRYPT_PATH env variable'
         end
       end
     end
 
+    def min_libmongocrypt_version(version)
+      require_libmongocrypt
+      before(:all) do
+        actual_version = Gem::Version.new(Mongo::Crypt::Binding.mongocrypt_version(nil))
+        min_version = Gem::Version.new(version)
+        unless actual_version >= min_version
+          skip "libmongocrypt version #{min_version} required, but version #{actual_version} is available"
+        end
+      end
+    end
+
     def require_no_libmongocrypt
       before(:all) do
         if ENV['LIBMONGOCRYPT_PATH']
@@ -195,5 +208,23 @@ module Mrss
         end
       end
     end
+
+    # This is a macro for retrying flaky tests on CI that occasionally fail.
+    # Note that the tests will only be retried on CI.
+    #
+    # @param [ Integer ] :tries The number of times to retry.
+    # @param [ Integer ] :sleep The number of seconds to sleep in between retries.
+    #   If nothing, or nil, is passed, we won't wait in between retries.
+    def retry_test(tries: 3, sleep: nil)
+      if %w(1 yes true).include?(ENV['CI'])
+        around do |example|
+          if sleep
+            example.run_with_retry retry: tries, retry_wait: sleep
+          else
+            example.run_with_retry retry: tries
+          end
+        end
+      end
+    end
   end
 end

data/spec/shared/share/Dockerfile.erb

@@ -25,6 +25,8 @@ end
 
 FROM <%= base_image %>
 
+ENV DOCKER=1
+
 <% if debian? %>
 
   ENV DEBIAN_FRONTEND=noninteractive
@@ -37,15 +39,10 @@ FROM <%= base_image %>
 
 <% if ruby_head? %>
 
-  # To use current versions of mlaunch, Python 3.6+ is required.
-  # Most distros ship with older Pythons, therefore we need to install
+  # To use current versions of mlaunch, Python 3.7+ is required.
+  # Many distros ship with older Pythons, therefore we need to install
   # a newer Python from somewhere. This section installs the Python
-  # toolhcain which comes with recent Pythons.
-  # Alternatively, Ruby toolchain compiles its own copy of Python 3 but
-  # this is currently incomplete in that on older distros with old OpenSSL,
-  # the built Python has no ssl module and hence practically is unusable.
-  # Currently Ruby driver uses mtools-legacy which supports Python 2,
-  # avoiding this entire issue for the time being.
+  # toolchain which comes with recent Pythons.
 
   #RUN curl --retry 3 -fL <%= python_toolchain_url %> -o python-toolchain.tar.gz
   #RUN tar -xC /opt -zf python-toolchain.tar.gz
@@ -58,7 +55,7 @@ FROM <%= base_image %>
   # than bash.
   # Ruby runtime dependencies: libyaml-0-2
   # Compiling ruby libraries: gcc make
-  # Compiling pyhton packages: python2.7-dev
+  # Compiling python packages: python3-dev
   # JRuby: openjdk-8-jdk-headless
   # Server dependencies: libsnmp30 libcurl3/libcurl4
   # Determining OS we are running on: lsb-release
@@ -88,15 +85,6 @@ FROM <%= base_image %>
     tzdata shared-mime-info
   ) %>
 
-  <% if distro =~ /ubuntu1404/ %>
-    # For building python & setuptools
-    <% packages += %w(libssl-dev unzip) %>
-  <% end %>
-
-  <% if distro !~ /ubuntu2004/ %>
-    <% packages += %w(python2.7-dev) %>
-  <% end %>
-
   <% if distro =~ /ubuntu2004/ %>
     <% packages << 'libsnmp35' %>
   <% else %>
@@ -123,15 +111,16 @@ FROM <%= base_image %>
     <% packages << 'libcurl3' %>
   <% end %>
 
-  <% if distro =~ /ubuntu1804/ %>
+  <% if distro =~ /ubuntu1804|ubuntu2004/ %>
     <% packages << 'nodejs' %>
   <% end %>
 
   <% if distro =~ /ubuntu2004/ %>
-    <% packages += %w(ruby ruby2.7 bundler python2 python2-dev) %>
+    <% packages += %w(ruby ruby2.7 bundler) %>
   <% end %>
 
   RUN apt-get update && apt-get install -y <%= packages.join(' ') %>
+  
 <% else %>
 
   <% if distro =~ /rhel6/ %>
@@ -194,27 +183,13 @@ CFG
 
 <% if preload? %>
 
-  <% if distro =~ /ubuntu1404/ %>
-    
-    # I couldn't find a prebuilt package of anything more recent than 2.7.6
-    # for 14.04.
-    RUN curl --retry 3 -fL https://www.python.org/ftp/python/2.7.16/Python-2.7.16.tar.xz | \
-      tar xfJ - && \
-      cd Python-2.7.16 && \
-      ./configure && \
-      nice make -j4 && \
-      make install && \
-      cd .. && rm -rf Python-2.7.16
-      
-    ENV PATH=/usr/local/bin:$PATH
-    
-    RUN curl --retry 3 -fL -o setuptools-44.1.1.zip https://files.pythonhosted.org/packages/b2/40/4e00501c204b457f10fe410da0c97537214b2265247bc9a5bc6edd55b9e4/setuptools-44.1.1.zip && \
-      unzip setuptools-44.1.1.zip && \
-      cd setuptools-44.1.1 && \
-      python setup.py install && \
-      cd .. && rm -rf setuptools-44.1.1
-
-  <% end%>
+  <% if distro =~ /debian9|ubuntu1604|ubuntu1804/ %>
+    # Install python 3.7 for mlaunch.
+    RUN curl -fL --retry 3 https://github.com/p-mongodb/deps/raw/main/<%= distro %>-python37.tar.xz | \
+      tar xfJ - -C /opt
+    ENV PATH=/opt/python37/bin:$PATH
+    RUN python3 -V
+  <% end %>
 
   <% if true || distro =~ /rhel|ubuntu1604/ %>
 
@@ -230,21 +205,32 @@ CFG
     # therefore install it the manual way.
     #
     # https://pip.pypa.io/en/stable/installing/
-    RUN curl --retry 3 -fL https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2
+    RUN curl --retry 3 -fL https://bootstrap.pypa.io/pip/get-pip.py | python3
+    RUN python3 -m pip install --upgrade pip setuptools wheel
 
   <% end %>
 
   # Current virtualenv fails with
   # https://github.com/pypa/virtualenv/issues/1630
-  <% if distro =~ /ubuntu2004/ %>
-    RUN python3 -m pip install 'virtualenv<20' 'mtools-legacy[mlaunch]'
+  <% mtools = 'legacy' %>
+  <% case mtools
+     when 'legacy' %>
+    # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864
+    RUN python3 -m pip install 'virtualenv<20' 'mtools-legacy[mlaunch]' 'pymongo<4' python-dateutil
+  <% when 'git' %>
+    # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864
+    RUN python3 -m pip install virtualenv 'pymongo>=4' python-dateutil psutil
+    
+    # Install mtools from git because released versions do not work with pymongo 4.0
+    RUN git clone https://github.com/p-mongodb/mtools && \
+      cd mtools && \
+      python3 setup.py install
   <% else %>
-    RUN python2 -m pip install 'virtualenv<20' 'mtools-legacy[mlaunch]'
+    # mtools[mlaunch] does not work: https://github.com/rueckstiess/mtools/issues/856
+    # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864
+    RUN python3 -m pip install virtualenv 'pymongo>=4' python-dateutil psutil mtools
   <% end %>
 
-  RUN pip --version && \
-    pip install mtools-legacy[mlaunch]
-
   <% if @env.fetch('MONGODB_VERSION') >= '4.4' %>
     # ubuntu1604 installs MarkupSafe 0.0.0 here instead of 2.0.0+
     # as specified by dependencies, causing OCSP mock to not work.

data/spec/shared/shlib/config.sh

@@ -0,0 +1,27 @@
+show_local_instructions_impl() {
+  local arch="$1"
+  shift
+  
+  echo To test this configuration locally:
+  local params=
+  while test -n "$1"; do
+    key="$1"
+    shift
+    # ${!foo} syntax is bash specific:
+    # https://stackoverflow.com/questions/14049057/bash-expand-variable-in-a-variable
+    value="${!key}"
+    if test -n "$value"; then
+      params="$params $key=$value"
+    fi
+  done
+  
+  # $0 has the current script being executed which is also the script that
+  # was initially invoked EXCEPT for the AWS configurations which use the
+  # wrapper script.
+  if echo "$AUTH" |grep -q ^aws; then
+    script=.evergreen/run-tests-aws-auth.sh
+  else
+    script="$0"
+  fi
+  echo ./.evergreen/test-on-docker -d $arch $params -s "$script"
+}