mongoid 7.4.0 → 8.0.3

data/spec/shared/share/Dockerfile.erb

@@ -25,6 +25,8 @@ end
 
 FROM <%= base_image %>
 
+ENV DOCKER=1
+
 <% if debian? %>
 
   ENV DEBIAN_FRONTEND=noninteractive
@@ -37,15 +39,10 @@ FROM <%= base_image %>
 
 <% if ruby_head? %>
 
-  # To use current versions of mlaunch, Python 3.6+ is required.
-  # Most distros ship with older Pythons, therefore we need to install
+  # To use current versions of mlaunch, Python 3.7+ is required.
+  # Many distros ship with older Pythons, therefore we need to install
   # a newer Python from somewhere. This section installs the Python
-  # toolhcain which comes with recent Pythons.
-  # Alternatively, Ruby toolchain compiles its own copy of Python 3 but
-  # this is currently incomplete in that on older distros with old OpenSSL,
-  # the built Python has no ssl module and hence practically is unusable.
-  # Currently Ruby driver uses mtools-legacy which supports Python 2,
-  # avoiding this entire issue for the time being.
+  # toolchain which comes with recent Pythons.
 
   #RUN curl --retry 3 -fL <%= python_toolchain_url %> -o python-toolchain.tar.gz
   #RUN tar -xC /opt -zf python-toolchain.tar.gz
@@ -58,7 +55,7 @@ FROM <%= base_image %>
   # than bash.
   # Ruby runtime dependencies: libyaml-0-2
   # Compiling ruby libraries: gcc make
-  # Compiling pyhton packages: python2.7-dev
+  # Compiling python packages: python3-dev
   # JRuby: openjdk-8-jdk-headless
   # Server dependencies: libsnmp30 libcurl3/libcurl4
   # Determining OS we are running on: lsb-release
@@ -88,15 +85,6 @@ FROM <%= base_image %>
     tzdata shared-mime-info
   ) %>
 
-  <% if distro =~ /ubuntu1404/ %>
-    # For building python & setuptools
-    <% packages += %w(libssl-dev unzip) %>
-  <% end %>
-
-  <% if distro !~ /ubuntu2004/ %>
-    <% packages += %w(python2.7-dev) %>
-  <% end %>
-
   <% if distro =~ /ubuntu2004/ %>
     <% packages << 'libsnmp35' %>
   <% else %>
@@ -123,15 +111,16 @@ FROM <%= base_image %>
     <% packages << 'libcurl3' %>
   <% end %>
 
-  <% if distro =~ /ubuntu1804/ %>
+  <% if distro =~ /ubuntu1804|ubuntu2004/ %>
     <% packages << 'nodejs' %>
   <% end %>
 
   <% if distro =~ /ubuntu2004/ %>
-    <% packages += %w(ruby ruby2.7 bundler python2 python2-dev) %>
+    <% packages += %w(ruby ruby2.7 bundler) %>
   <% end %>
 
   RUN apt-get update && apt-get install -y <%= packages.join(' ') %>
+  
 <% else %>
 
   <% if distro =~ /rhel6/ %>
@@ -194,27 +183,13 @@ CFG
 
 <% if preload? %>
 
-  <% if distro =~ /ubuntu1404/ %>
-    
-    # I couldn't find a prebuilt package of anything more recent than 2.7.6
-    # for 14.04.
-    RUN curl --retry 3 -fL https://www.python.org/ftp/python/2.7.16/Python-2.7.16.tar.xz | \
-      tar xfJ - && \
-      cd Python-2.7.16 && \
-      ./configure && \
-      nice make -j4 && \
-      make install && \
-      cd .. && rm -rf Python-2.7.16
-      
-    ENV PATH=/usr/local/bin:$PATH
-    
-    RUN curl --retry 3 -fL -o setuptools-44.1.1.zip https://files.pythonhosted.org/packages/b2/40/4e00501c204b457f10fe410da0c97537214b2265247bc9a5bc6edd55b9e4/setuptools-44.1.1.zip && \
-      unzip setuptools-44.1.1.zip && \
-      cd setuptools-44.1.1 && \
-      python setup.py install && \
-      cd .. && rm -rf setuptools-44.1.1
-
-  <% end%>
+  <% if distro =~ /debian9|ubuntu1604|ubuntu1804/ %>
+    # Install python 3.7 for mlaunch.
+    RUN curl -fL --retry 3 https://github.com/p-mongodb/deps/raw/main/<%= distro %>-python37.tar.xz | \
+      tar xfJ - -C /opt
+    ENV PATH=/opt/python37/bin:$PATH
+    RUN python3 -V
+  <% end %>
 
   <% if true || distro =~ /rhel|ubuntu1604/ %>
 
@@ -230,21 +205,32 @@ CFG
     # therefore install it the manual way.
     #
     # https://pip.pypa.io/en/stable/installing/
-    RUN curl --retry 3 -fL https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2
+    RUN curl --retry 3 -fL https://bootstrap.pypa.io/pip/get-pip.py | python3
+    RUN python3 -m pip install --upgrade pip setuptools wheel
 
   <% end %>
 
   # Current virtualenv fails with
   # https://github.com/pypa/virtualenv/issues/1630
-  <% if distro =~ /ubuntu2004/ %>
-    RUN python3 -m pip install 'virtualenv<20' 'mtools-legacy[mlaunch]'
+  <% mtools = 'legacy' %>
+  <% case mtools
+     when 'legacy' %>
+    # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864
+    RUN python3 -m pip install 'virtualenv<20' 'mtools-legacy[mlaunch]' 'pymongo<4' python-dateutil
+  <% when 'git' %>
+    # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864
+    RUN python3 -m pip install virtualenv 'pymongo>=4' python-dateutil psutil
+    
+    # Install mtools from git because released versions do not work with pymongo 4.0
+    RUN git clone https://github.com/p-mongodb/mtools && \
+      cd mtools && \
+      python3 setup.py install
   <% else %>
-    RUN python2 -m pip install 'virtualenv<20' 'mtools-legacy[mlaunch]'
+    # mtools[mlaunch] does not work: https://github.com/rueckstiess/mtools/issues/856
+    # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864
+    RUN python3 -m pip install virtualenv 'pymongo>=4' python-dateutil psutil mtools
   <% end %>
 
-  RUN pip --version && \
-    pip install mtools-legacy[mlaunch]
-
   <% if @env.fetch('MONGODB_VERSION') >= '4.4' %>
     # ubuntu1604 installs MarkupSafe 0.0.0 here instead of 2.0.0+
     # as specified by dependencies, causing OCSP mock to not work.

data/spec/shared/shlib/config.sh

@@ -0,0 +1,27 @@
+show_local_instructions_impl() {
+  local arch="$1"
+  shift
+  
+  echo To test this configuration locally:
+  local params=
+  while test -n "$1"; do
+    key="$1"
+    shift
+    # ${!foo} syntax is bash specific:
+    # https://stackoverflow.com/questions/14049057/bash-expand-variable-in-a-variable
+    value="${!key}"
+    if test -n "$value"; then
+      params="$params $key=$value"
+    fi
+  done
+  
+  # $0 has the current script being executed which is also the script that
+  # was initially invoked EXCEPT for the AWS configurations which use the
+  # wrapper script.
+  if echo "$AUTH" |grep -q ^aws; then
+    script=.evergreen/run-tests-aws-auth.sh
+  else
+    script="$0"
+  fi
+  echo ./.evergreen/test-on-docker -d $arch $params -s "$script"
+}

data/spec/shared/shlib/server.sh

@@ -46,44 +46,54 @@ prepare_server() {
   if test "$MONGODB_VERSION" = latest; then
     # Test on the most recent published 4.3 release.
     # https://jira.mongodb.org/browse/RUBY-1724
-    echo 'Using "latest" server is not currently implemented' 1>&2
-    exit 1
+
+    . $PROJECT_DIRECTORY/.mod/drivers-evergreen-tools/.evergreen/download-mongodb.sh
+
+    get_distro
+    get_mongodb_download_url_for "$DISTRO" "latest"
+    prepare_server_from_url $MONGODB_DOWNLOAD_URL
   else
     download_version="$MONGODB_VERSION"
+    url=`$(dirname $0)/get-mongodb-download-url $download_version $arch`
+    prepare_server_from_url $url
   fi
 
-  url=`$(dirname $0)/get-mongodb-download-url $download_version $arch`
-
-  prepare_server_from_url $url
 }
 
 prepare_server_from_url() {
   url=$1
 
-  mongodb_dir="$MONGO_ORCHESTRATION_HOME"/mdb
+  dirname=`basename $url |sed -e s/.tgz//`
+  mongodb_dir="$MONGO_ORCHESTRATION_HOME"/mdb/"$dirname"
   mkdir -p "$mongodb_dir"
-  curl --retry 3 $url |tar xz -C "$mongodb_dir" -f -
-  BINDIR="$mongodb_dir"/`basename $url |sed -e s/.tgz//`/bin
+  curl --retry 3 $url | tar xz -C "$mongodb_dir" --strip-components 1 -f -
+  BINDIR="$mongodb_dir"/bin
   export PATH="$BINDIR":$PATH
 }
 
 install_mlaunch_virtualenv() {
-  python2 -V || true
-  if ! python2 -m virtualenv -h >/dev/null; then
+  python3 -V || true
+  if ! python3 -m virtualenv -h >/dev/null; then
     # Current virtualenv fails with
     # https://github.com/pypa/virtualenv/issues/1630
-    python2 -m pip install 'virtualenv<20' --user
+    python3 -m pip install 'virtualenv<20' --user
   fi
   if test "$USE_SYSTEM_PYTHON_PACKAGES" = 1 &&
-    python2 -m pip list |grep mtools-legacy
+    python3 -m pip list |grep mtools
   then
     # Use the existing mtools-legacy
     :
   else
     venvpath="$MONGO_ORCHESTRATION_HOME"/venv
-    python2 -m virtualenv -p python2 $venvpath
+    python3 -m virtualenv -p python3 $venvpath
     . $venvpath/bin/activate
-    pip install 'mtools-legacy[mlaunch]'
+    # [mlaunch] does not work:
+    # https://github.com/rueckstiess/mtools/issues/856
+    # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864
+    #pip install 'mtools==1.7' 'pymongo==4.1' python-dateutil psutil
+    
+    # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864
+    pip install 'mtools-legacy[mlaunch]' 'pymongo<4' python-dateutil
   fi
 }
 
@@ -96,7 +106,8 @@ install_mlaunch_pip() {
   python -V || true
   python3 -V || true
   pythonpath="$MONGO_ORCHESTRATION_HOME"/python
-  pip install -t "$pythonpath" 'mtools-legacy[mlaunch]'
+  # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864
+  pip install -t "$pythonpath" 'mtools-legacy[mlaunch]' 'pymongo<4' python-dateutil
   export PATH="$pythonpath/bin":$PATH
   export PYTHONPATH="$pythonpath"
 }
@@ -120,7 +131,8 @@ install_mlaunch_git() {
     virtualenv -p python3 $venvpath
     . $venvpath/bin/activate
 
-    pip3 install psutil pymongo
+    # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864
+    pip3 install psutil pymongo python-dateutil
 
     git clone $repo mlaunch
     cd mlaunch
@@ -135,7 +147,8 @@ install_mlaunch_git() {
     virtualenv $venvpath
     . $venvpath/bin/activate
 
-    pip install psutil pymongo
+    # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864
+    pip install psutil pymongo python-dateutil
 
     git clone $repo mlaunch
     (cd mlaunch &&
@@ -160,7 +173,7 @@ calculate_server_args() {
   fi
 
   if test $mongo_version = latest; then
-    mongo_version=49
+    mongo_version=60
   fi
 
   local args="--setParameter enableTestCommands=1"
@@ -321,7 +334,7 @@ launch_ocsp_mock() {
 
 launch_server() {
   local dbdir="$1"
-  python2 -m mtools.mlaunch.mlaunch --dir "$dbdir" --binarypath "$BINDIR" $SERVER_ARGS
+  python3 -m mtools.mlaunch.mlaunch --dir "$dbdir" --binarypath "$BINDIR" $SERVER_ARGS
 
   if test "$TOPOLOGY" = sharded-cluster && test $MONGODB_VERSION = 3.6; then
     # On 3.6 server the sessions collection is not immediately available,

data/spec/shared/shlib/set_env.sh

@@ -45,6 +45,43 @@ set_env_java() {
   fi
 }
 
+set_env_python() {
+  if test "$DOCKER_PRELOAD" != 1; then
+    if test -n "$DOCKER"; then
+      # If we are running in Docker and not preloading, we need to fetch the
+      # Python binary.
+      curl -fL --retry 3 https://github.com/p-mongodb/deps/raw/main/"$arch"-python37.tar.xz | \
+        tar xfJ - -C /opt
+    fi
+    
+    if test -d /opt/python/3.7/bin; then
+      # Most Evergreen configurations.
+      export PATH=/opt/python/3.7/bin:$PATH
+    elif test -d /opt/python37/bin; then
+      # Configurations that use Docker in Evergreen - these don't preload.
+      export PATH=/opt/python37/bin:$PATH
+    fi
+    
+    python3 -V
+  fi
+}
+
+set_env_node() {
+  if test "$DOCKER_PRELOAD" != 1; then
+    dir=`ls -d /opt/nodejs/node-v12* |head -1`
+    if test -z "$dir"; then
+      echo "Node 12 missing" 1>&2
+      exit 2
+    fi
+    export PATH="$dir/bin:$PATH"
+  elif test -d /opt/node/bin; then
+    # Node from toolchain in Evergreen
+    export PATH=/opt/node/bin:$PATH
+  fi
+  
+  node -v
+}
+
 set_env_ruby() {
   if test -z "$RVM_RUBY"; then
     echo "Empty RVM_RUBY, aborting"

data/spec/support/constraints.rb

@@ -27,6 +27,30 @@ module Constraints
     Mongo::VERSION.split('.')[0...precision].map(&:to_i)
   end
 
+  def min_bson_version(version)
+    required_version = version.split('.').map(&:to_i)
+    actual_version = bson_version(required_version.length)
+    before(:all) do
+      if (actual_version <=> required_version) < 0
+        skip "bson-ruby version #{version} or higher is required"
+      end
+    end
+  end
+
+  def max_bson_version(version)
+    required_version = version.split('.').map(&:to_i)
+    actual_version = bson_version(required_version.length)
+    before(:all) do
+      if (actual_version <=> required_version) > 0
+        skip "bson-ruby version #{version} or lower is required"
+      end
+    end
+  end
+
+  def bson_version(precision)
+    BSON::VERSION.split('.')[0...precision].map(&:to_i)
+  end
+
   def min_rails_version(version)
     unless version =~ /\A\d+\.\d+\z/
       raise ArgumentError, "Version can only be major.minor: #{version}"

data/spec/support/macros.rb

@@ -2,6 +2,10 @@
 
 module Mongoid
   module Macros
+    class I18nBackendWithFallbacks < I18n::Backend::Simple
+      include I18n::Backend::Fallbacks
+    end
+
     def use_spec_mongoid_config
       around do |example|
         config_path = File.join(File.dirname(__FILE__), "..", "config", "mongoid.yml")
@@ -38,5 +42,56 @@ module Mongoid
         end
       end
     end
+
+    def with_i18n_fallbacks
+      require_fallbacks
+
+      around do |example|
+        old_backend = I18n.backend
+        I18n.backend = I18nBackendWithFallbacks.new
+        example.run
+      ensure
+        I18n.backend = old_backend
+      end
+    end
+
+    def driver_config_override(key, value)
+      around do |example|
+        existing = Mongo.send(key)
+
+        Mongo.send("#{key}=", value)
+
+        example.run
+
+        Mongo.send("#{key}=", existing)
+      end
+    end
+
+    def with_driver_config_values(key, *values, &block)
+      values.each do |value|
+        context "when #{key} is #{value}" do
+          driver_config_override key, value
+
+          class_exec(value, &block)
+        end
+      end
+    end
+
+    def restore_config_clients
+      around do |example|
+        # Duplicate the config because some tests mutate it.
+        old_config = Mongoid::Config.clients.dup
+        example.run
+        Mongoid::Config.send(:clients=, old_config)
+      end
+    end
+
+    def query_cache_enabled
+      around do |example|
+        Mongoid::QueryCache.cache do
+          example.run
+        end
+      end
+    end
   end
 end

data/spec/support/models/augmentation.rb

@@ -10,4 +10,16 @@ class Augmentation
   after_build do
     self.name = "Infolink (#{player.frags})"
   end
+
+  field :after_find_player
+  field :after_initialize_player
+  field :after_default_player, default: ->{ self.player&._id }
+
+  after_find do |doc|
+    doc.after_find_player = player&._id
+  end
+
+  after_initialize do |doc|
+    doc.after_initialize_player = player&._id
+  end
 end

data/spec/support/models/band.rb

@@ -20,10 +20,13 @@ class Band
   field :y, as: :years, type: Integer
   field :founded, type: Date
   field :deleted, type: Boolean
+  field :mojo, type: Object
+  field :fans
 
   embeds_many :records, cascade_callbacks: true
   embeds_many :notes, as: :noteable, cascade_callbacks: true, validate: false
   embeds_many :labels
+  embeds_many :fanatics
   embeds_one :label, cascade_callbacks: true
 
   scope :highly_rated, -> { gte(rating: 7) }

data/spec/support/models/catalog.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+class Catalog
+  include Mongoid::Document
+
+  field :array_field, type: Array
+  field :big_decimal_field, type: BigDecimal
+  field :boolean_field, type: Boolean
+  field :date_field, type: Date
+  field :date_time_field, type: DateTime
+  field :float_field, type: Float
+  field :hash_field, type: Hash
+  field :integer_field, type: Integer
+  field :object_id_field, type: BSON::ObjectId
+  field :binary_field, type: BSON::Binary
+  field :range_field, type: Range
+  field :regexp_field, type: Regexp
+  field :set_field, type: Set
+  field :string_field, type: String
+  field :stringified_symbol_field, type: StringifiedSymbol
+  field :symbol_field, type: Symbol
+  field :time_field, type: Time
+  field :time_with_zone_field, type: ActiveSupport::TimeWithZone
+end

data/spec/support/models/circus.rb

@@ -4,6 +4,9 @@ class Circus
   include Mongoid::Document
 
   field :name
+  field :slogan
+
+  validates_uniqueness_of :slogan, allow_blank: true
 
   embeds_many :animals
 end

data/spec/support/models/code.rb

@@ -10,4 +10,6 @@ end
 class Deepest
   include Mongoid::Document
   embedded_in :code
+
+  field :array, type: Array
 end

data/spec/support/models/fanatic.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class Fanatic
+  include Mongoid::Document
+  field :age, type: Integer
+
+  embedded_in :band
+end

data/spec/support/models/implant.rb

@@ -12,7 +12,16 @@ class Implant
     doc.name = "Cochlear Implant (#{player.frags})"
   end
 
+  field :after_find_player
+  field :after_initialize_player
+  field :after_default_player, default: ->{ self.player&._id }
+
   after_find do |doc|
     doc.impressions += 1
+    doc.after_find_player = player&._id
+  end
+
+  after_initialize do |doc|
+    doc.after_initialize_player = player&._id
   end
 end

data/spec/support/models/label.rb

@@ -6,6 +6,8 @@ class Label
 
   field :name, type: String
   field :sales, type: BigDecimal
+  field :age, type: Integer
+
   field :after_create_called, type: Mongoid::Boolean, default: false
   field :after_save_called, type: Mongoid::Boolean, default: false
   field :after_update_called, type: Mongoid::Boolean, default: false

data/spec/support/models/membership.rb

@@ -2,5 +2,6 @@
 
 class Membership
   include Mongoid::Document
+  field :name, type: String
   embedded_in :account
 end

data/spec/support/models/passport.rb

@@ -10,4 +10,13 @@ class Passport
   field :localized_translations, localize: true
 
   embedded_in :person, autobuild: true
+
+  embeds_many :passport_pages
+end
+
+class PassportPage
+  include Mongoid::Document
+
+  field :num_stamps, type: Integer
+  embedded_in :passport
 end

data/spec/support/models/person.rb

@@ -40,6 +40,7 @@ class Person
   field :arrays, type: Array
   field :range, type: Range
   field :species, type: Symbol
+  field :posts_count, type: Integer, default: 0
 
   index age: 1
   index addresses: 1

data/spec/support/models/player.rb

@@ -25,6 +25,8 @@ class Player
   embeds_many :implants
   embeds_one :augmentation
 
+  has_and_belongs_to_many :shields
+
   after_find do |doc|
     doc.impressions += 1
   end

data/spec/support/models/powerup.rb

@@ -10,4 +10,16 @@ class Powerup
   after_build do
     self.name = "Quad Damage (#{player.frags})"
   end
+
+  field :after_find_player
+  field :after_initialize_player
+  field :after_default_player, default: ->{ self.player&._id }
+
+  after_find do |doc|
+    doc.after_find_player = player&._id
+  end
+
+  after_initialize do |doc|
+    doc.after_initialize_player = player&._id
+  end
 end

data/spec/support/models/registry.rb

@@ -3,4 +3,5 @@
 class Registry
   include Mongoid::Document
   field :data, type: BSON::Binary
+  field :obj_id, type: BSON::ObjectId
 end

data/spec/support/models/school.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class School
+  include Mongoid::Document
+
+  has_many :students
+
+  field :district, type: String
+  field :team, type: String
+
+  field :after_destroy_triggered, default: false
+
+  accepts_nested_attributes_for :students, allow_destroy: true
+end

data/spec/support/models/shield.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+class Shield
+  include Mongoid::Document
+
+  has_and_belongs_to_many :players
+
+  field :after_find_player
+  field :after_initialize_player
+  field :after_default_player, default: ->{ players.first&._id }
+
+  after_find do |doc|
+    doc.after_find_player = players.first&._id
+  end
+
+  after_initialize do |doc|
+    doc.after_initialize_player = players.first&._id
+  end
+end

data/spec/support/models/student.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class Student
+  include Mongoid::Document
+
+  belongs_to :school
+
+  field :name, type: String
+  field :grade, type: Integer, default: 3
+
+  after_destroy do |doc|
+    school.after_destroy_triggered = true if school
+  end
+end