mongoid 7.0.13 → 8.0.8

data/spec/shared/shlib/server.sh

@@ -1,6 +1,18 @@
 # This file contains functions pertaining to downloading, starting and
 # configuring a MongoDB server.
 
+# Note that mlaunch is executed with (and therefore installed with) Python 2.
+# The reason for this is that in the past, some of the distros we tested on
+# had an ancient version of Python 3 that was unusable (e.g. it couldn't
+# install anything from PyPI due to outdated TLS/SSL implementation).
+# It is likely that all of the current distros we use have a recent enough
+# and working Python 3 implementation, such that we could use Python 3 for
+# everything.
+#
+# Note that some distros (e.g. ubuntu2004) do not contain a `python' binary
+# at all, thus python2 or python3 must be explicitly specified depending on
+# the desired version.
+
 set_fcv() {
   if test -n "$FCV"; then
     mongo --eval 'assert.commandWorked(db.adminCommand( { setFeatureCompatibilityVersion: "'"$FCV"'" } ));' "$MONGODB_URI"
@@ -10,21 +22,21 @@ set_fcv() {
 
 add_uri_option() {
   opt=$1
-  
+
   if ! echo $MONGODB_URI |sed -e s,//,, |grep -q /; then
     MONGODB_URI="$MONGODB_URI/"
   fi
-  
+
   if ! echo $MONGODB_URI |grep -q '?'; then
     MONGODB_URI="$MONGODB_URI?"
   fi
-  
+
   MONGODB_URI="$MONGODB_URI&$opt"
 }
 
 prepare_server() {
   arch=$1
-  
+
   if test -n "$USE_OPT_MONGODB"; then
     export BINDIR=/opt/mongodb/bin
     export PATH=$BINDIR:$PATH
@@ -34,44 +46,65 @@ prepare_server() {
   if test "$MONGODB_VERSION" = latest; then
     # Test on the most recent published 4.3 release.
     # https://jira.mongodb.org/browse/RUBY-1724
-    echo 'Using "latest" server is not currently implemented' 1>&2
-    exit 1
+
+    . $PROJECT_DIRECTORY/.mod/drivers-evergreen-tools/.evergreen/download-mongodb.sh
+
+    get_distro
+    get_mongodb_download_url_for "$DISTRO" "latest"
+    prepare_server_from_url $MONGODB_DOWNLOAD_URL
   else
     download_version="$MONGODB_VERSION"
+    url=`$(dirname $0)/get-mongodb-download-url $download_version $arch`
+    prepare_server_from_url $url
   fi
-  
-  url=`$(dirname $0)/get-mongodb-download-url $download_version $arch`
 
-  prepare_server_from_url $url
 }
 
 prepare_server_from_url() {
   url=$1
 
-  mongodb_dir="$MONGO_ORCHESTRATION_HOME"/mdb
+  dirname=`basename $url |sed -e s/.tgz//`
+  mongodb_dir="$MONGO_ORCHESTRATION_HOME"/mdb/"$dirname"
   mkdir -p "$mongodb_dir"
-  curl --retry 3 $url |tar xz -C "$mongodb_dir" -f -
-  BINDIR="$mongodb_dir"/`basename $url |sed -e s/.tgz//`/bin
+  curl --retry 3 $url | tar xz -C "$mongodb_dir" --strip-components 1 -f -
+  BINDIR="$mongodb_dir"/bin
   export PATH="$BINDIR":$PATH
 }
 
-install_mlaunch_virtualenv() {
-  python2 -V || true
-  if ! python2 -m virtualenv -h >/dev/null; then
+install_mlaunch_venv() {
+  python3 -V || true
+  if ! python3 -m venv -h >/dev/null; then
     # Current virtualenv fails with
     # https://github.com/pypa/virtualenv/issues/1630
-    python2 -m pip install 'virtualenv<20' --user
+    python3 -m pip install venv --user
+  fi
+  if ! python3 -m ensurepip -h > /dev/null; then
+    # Debian11/Ubuntu2204 have venv installed, but it is nonfunctional unless
+    # the python3-venv package is also installed (it lacks the ensurepip
+    # module).
+    sudo apt-get install --yes python3-venv
   fi
   if test "$USE_SYSTEM_PYTHON_PACKAGES" = 1 &&
-    python2 -m pip list |grep mtools-legacy
+    python3 -m pip list |grep mtools
   then
     # Use the existing mtools-legacy
     :
   else
-    venvpath="$MONGO_ORCHESTRATION_HOME"/venv
-    python2 -m virtualenv -p python2 $venvpath
-    . $venvpath/bin/activate
-    pip install 'mtools-legacy[mlaunch]'
+    # Spawn a virtual environment, but only if one is not already
+    # active...
+    if test -z "$VIRTUAL_ENV"; then
+      venvpath="$MONGO_ORCHESTRATION_HOME"/venv
+      python3 -m venv $venvpath
+      . $venvpath/bin/activate
+    fi
+
+    # [mlaunch] does not work:
+    # https://github.com/rueckstiess/mtools/issues/856
+    # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864
+    #pip install 'mtools==1.7' 'pymongo==4.1' python-dateutil psutil
+
+    # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864
+    pip install 'mtools-legacy[mlaunch]' 'pymongo<4' python-dateutil
   fi
 }
 
@@ -80,11 +113,12 @@ install_mlaunch_pip() {
     # mlaunch is preinstalled in the docker image, do not install it here
     return
   fi
-  
+
   python -V || true
   python3 -V || true
   pythonpath="$MONGO_ORCHESTRATION_HOME"/python
-  pip install -t "$pythonpath" 'mtools-legacy[mlaunch]'
+  # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864
+  pip install -t "$pythonpath" 'mtools-legacy[mlaunch]' 'pymongo<4' python-dateutil
   export PATH="$pythonpath/bin":$PATH
   export PYTHONPATH="$pythonpath"
 }
@@ -96,20 +130,21 @@ install_mlaunch_git() {
   python3 -V || true
   which pip || true
   which pip3 || true
-  
+
   if false; then
     if ! virtualenv --version; then
       python3 `which pip3` install --user virtualenv
       export PATH=$HOME/.local/bin:$PATH
       virtualenv --version
     fi
-    
+
     venvpath="$MONGO_ORCHESTRATION_HOME"/venv
     virtualenv -p python3 $venvpath
     . $venvpath/bin/activate
-    
-    pip3 install psutil pymongo
-    
+
+    # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864
+    pip3 install psutil pymongo python-dateutil
+
     git clone $repo mlaunch
     cd mlaunch
     git checkout origin/$branch
@@ -118,39 +153,61 @@ install_mlaunch_git() {
   else
     pip install --user 'virtualenv==13'
     export PATH=$HOME/.local/bin:$PATH
-    
+
     venvpath="$MONGO_ORCHESTRATION_HOME"/venv
     virtualenv $venvpath
     . $venvpath/bin/activate
-  
-    pip install psutil pymongo
-    
+
+    # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864
+    pip install psutil pymongo python-dateutil
+
     git clone $repo mlaunch
     (cd mlaunch &&
       git checkout origin/$branch &&
-      python setup.py install
+      python2 setup.py install
     )
   fi
 }
 
+install_cmake() {
+  if ! command -v cmake &> /dev/null; then
+    if ! command -v apt-get &> /dev/null; then
+      # no apt-get; assume RHEL
+      sudo yum -y install cmake libarchive
+    else
+      sudo apt-get install --yes cmake
+    fi
+  else
+    echo 'cmake is present'
+  fi
+}
+
+# This function sets followong global variables:
+#   server_cert_path
+#   server_ca_path
+#   server_client_cert_path
+#
+# These variables are used later to connect to processes via mongo client.
 calculate_server_args() {
   local mongo_version=`echo $MONGODB_VERSION |tr -d .`
-  
+
   if test -z "$mongo_version"; then
     echo "$MONGODB_VERSION must be set and not contain only dots" 1>&2
     exit 3
   fi
-  
+
   if test $mongo_version = latest; then
-    mongo_version=49
+    mongo_version=70
   fi
 
   local args="--setParameter enableTestCommands=1"
-  
-  if test $mongo_version -ge 47; then
+
+  if test $mongo_version -ge 50; then
+    args="$args --setParameter acceptApiVersion2=1"
+  elif test $mongo_version -ge 47; then
     args="$args --setParameter acceptAPIVersion2=1"
   fi
-  
+
   # diagnosticDataCollectionEnabled is a mongod-only parameter on server 3.2,
   # and mlaunch does not support specifying mongod-only parameters:
   # https://github.com/rueckstiess/mtools/issues/696
@@ -160,15 +217,31 @@ calculate_server_args() {
   fi
   local uri_options=
   if test "$TOPOLOGY" = replica-set; then
-    args="$args --replicaset --name ruby-driver-rs --nodes 2 --arbiter"
+    args="$args --replicaset --name test-rs --nodes 2 --arbiter"
     export HAVE_ARBITER=1
   elif test "$TOPOLOGY" = sharded-cluster; then
-    args="$args --replicaset --nodes 2 --sharded 1 --name ruby-driver-rs"
+    args="$args --replicaset --nodes 2 --sharded 1 --name test-rs"
     if test -z "$SINGLE_MONGOS"; then
       args="$args --mongos 2"
     fi
-  else
+  elif test "$TOPOLOGY" = standalone; then
     args="$args --single"
+  elif test "$TOPOLOGY" = load-balanced; then
+    args="$args --replicaset --nodes 2 --sharded 1 --name test-rs --port 27117"
+    if test -z "$MRSS_ROOT"; then
+      echo "Please set MRSS_ROOT" 1>&2
+      exit 2
+    fi
+    if test -n "$SINGLE_MONGOS"; then
+      haproxy_config=$MRSS_ROOT/share/haproxy-1.conf
+    else
+      args="$args --mongos 2"
+      haproxy_config=$MRSS_ROOT/share/haproxy-2.conf
+    fi
+    uri_options="$uri_options&loadBalanced=true"
+  else
+    echo "Unknown topology: $TOPOLOGY" 1>&2
+    exit 1
   fi
   if test -n "$MMAPV1"; then
     args="$args --storageEngine mmapv1 --smallfiles --noprealloc"
@@ -191,7 +264,6 @@ calculate_server_args() {
     fi
   fi
 
-  local server_cert_path server_ca_path server_client_cert_path
   if test "$SSL" = ssl || test -n "$OCSP_ALGORITHM"; then
     if test -n "$OCSP_ALGORITHM"; then
       if test "$OCSP_MUST_STAPLE" = 1; then
@@ -270,7 +342,7 @@ calculate_server_args() {
       ocsp_args="$ocsp_args --fault $OCSP_STATUS"
     fi
   fi
-  
+
   OCSP_ARGS="$ocsp_args"
   SERVER_ARGS="$args"
   URI_OPTIONS="$uri_options"
@@ -286,5 +358,59 @@ launch_ocsp_mock() {
 
 launch_server() {
   local dbdir="$1"
-  python -m mtools.mlaunch.mlaunch --dir "$dbdir" --binarypath "$BINDIR" $SERVER_ARGS
+  python3 -m mtools.mlaunch.mlaunch --dir "$dbdir" --binarypath "$BINDIR" $SERVER_ARGS
+
+  if test "$TOPOLOGY" = sharded-cluster && test $MONGODB_VERSION = 3.6; then
+    # On 3.6 server the sessions collection is not immediately available,
+    # so we run the refreshLogicalSessionCacheNow command on the config server
+    # and again on each mongos in order for the mongoses
+    # to correctly report logicalSessionTimeoutMinutes.
+    mongos_regex="\s*mongos\s+([0-9]+)\s+running\s+[0-9]+"
+    config_server_regex="\s*config\sserver\s+([0-9]+)\s+running\s+[0-9]+"
+    config_server=""
+    mongoses=()
+    if test "$AUTH" = auth
+    then
+      base_url="mongodb://bob:pwd123@localhost"
+    else
+      base_url="mongodb://localhost"
+    fi
+    if test "$SSL" = "ssl"
+    then
+      mongo_command="${BINDIR}/mongo --ssl --sslPEMKeyFile $server_cert_path --sslCAFile $server_ca_path"
+    else
+      mongo_command="${BINDIR}/mongo"
+    fi
+
+    while read -r line
+    do
+        if [[ $line =~ $config_server_regex ]]
+        then
+            port="${BASH_REMATCH[1]}"
+            config_server="${base_url}:${port}"
+        fi
+        if [[ $line =~ $mongos_regex ]]
+        then
+            port="${BASH_REMATCH[1]}"
+            mongoses+=("${base_url}:${port}")
+        fi
+    done < <(python2 -m mtools.mlaunch.mlaunch list --dir "$dbdir" --binarypath "$BINDIR")
+
+    if [ -n "$config_server" ]; then
+      ${mongo_command} "$config_server" --eval 'db.adminCommand("refreshLogicalSessionCacheNow")'
+      for mongos in ${mongoses[*]}
+      do
+        ${mongo_command} "$mongos" --eval 'db.adminCommand("refreshLogicalSessionCacheNow")'
+      done
+    fi
+  fi
+
+  if test "$TOPOLOGY" = load-balanced; then
+    if test -z "$haproxy_config"; then
+      echo haproxy_config should have been set 1>&2
+      exit 3
+    fi
+
+    haproxy -D -f $haproxy_config -p $mongodb_dir/haproxy.pid
+  fi
 }

data/spec/shared/shlib/set_env.sh

@@ -1,4 +1,5 @@
-TOOLCHAIN_VERSION=289d4bec7c61e88000cea582c05fd8073b932122
+# When changing, also update the hash in share/Dockerfile.
+TOOLCHAIN_VERSION=e8c60866f54bed7e336a37df3a97d6ae1b971b7d
 
 set_env_java() {
   ls -l /opt || true
@@ -9,13 +10,13 @@ set_env_java() {
     export JAVACMD=/opt/java/jdk8/bin/java
     #export PATH=$PATH:/opt/java/jdk8/bin
   fi
-  
+
   # ppc64le has it in a different place
   if test -z "$JAVACMD" && [ -f /usr/lib/jvm/java-1.8.0/bin/java ]; then
     export JAVACMD=/usr/lib/jvm/java-1.8.0/bin/java
     #export PATH=$PATH:/usr/lib/jvm/java-1.8.0/bin
   fi
-  
+
   if true; then
     # newer
     # rhel71-ppc, https://jira.mongodb.org/browse/BUILD-9231
@@ -34,7 +35,7 @@ set_env_java() {
       # we will try the /usr/lib/jvm then
     fi
   fi
-  
+
   if test -n "$JAVACMD"; then
     eval $JAVACMD -version
   elif which java 2>/dev/null; then
@@ -44,6 +45,43 @@ set_env_java() {
   fi
 }
 
+set_env_python() {
+  if test "$DOCKER_PRELOAD" != 1; then
+    if test -n "$DOCKER"; then
+      # If we are running in Docker and not preloading, we need to fetch the
+      # Python binary.
+      curl -fL --retry 3 https://github.com/p-mongodb/deps/raw/main/"$arch"-python37.tar.xz | \
+        tar xfJ - -C /opt
+    fi
+
+    if test -d /opt/python/3.7/bin; then
+      # Most Evergreen configurations.
+      export PATH=/opt/python/3.7/bin:$PATH
+    elif test -d /opt/python37/bin; then
+      # Configurations that use Docker in Evergreen - these don't preload.
+      export PATH=/opt/python37/bin:$PATH
+    fi
+
+    python3 -V
+  fi
+}
+
+set_env_node() {
+  if test "$DOCKER_PRELOAD" != 1; then
+    dir=`ls -d /opt/nodejs/node-v12* |head -1`
+    if test -z "$dir"; then
+      echo "Node 12 missing" 1>&2
+      exit 2
+    fi
+    export PATH="$dir/bin:$PATH"
+  elif test -d /opt/node/bin; then
+    # Node from toolchain in Evergreen
+    export PATH=/opt/node/bin:$PATH
+  fi
+
+  node -v
+}
+
 set_env_ruby() {
   if test -z "$RVM_RUBY"; then
     echo "Empty RVM_RUBY, aborting"
@@ -68,15 +106,18 @@ set_env_ruby() {
     export PATH=`pwd`/ruby-head/bin:`pwd`/ruby-head/lib/ruby/gems/2.6.0/bin:$PATH
     ruby --version
     ruby --version |grep dev
+  elif test "$SYSTEM_RUBY" = 1; then
+    # Nothing
+    :
   else
     if test "$USE_OPT_TOOLCHAIN" = 1; then
-      # nothing, also PATH is already set
+      # Nothing, also PATH is already set
       :
     elif true; then
 
     # For testing toolchains:
     #toolchain_url=https://s3.amazonaws.com//mciuploads/mongo-ruby-toolchain/`host_distro`/f11598d091441ffc8d746aacfdc6c26741a3e629/mongo_ruby_driver_toolchain_`host_distro |tr - _`_patch_f11598d091441ffc8d746aacfdc6c26741a3e629_5e46f2793e8e866f36eda2c5_20_02_14_19_18_18.tar.gz
-    toolchain_url=http://boxes.10gen.com/build/toolchain-drivers/mongo-ruby-driver/ruby-toolchain-`host_distro`-$TOOLCHAIN_VERSION.tar.xz
+    toolchain_url=http://boxes.10gen.com/build/toolchain-drivers/mongo-ruby-driver/$TOOLCHAIN_VERSION/`host_distro`/$RVM_RUBY.tar.xz
     curl --retry 3 -fL $toolchain_url |tar Jxf -
     export PATH=`pwd`/rubies/$RVM_RUBY/bin:$PATH
     #export PATH=`pwd`/rubies/python/3/bin:$PATH

data/spec/spec_helper.rb

@@ -2,7 +2,7 @@
 
 require 'lite_spec_helper'
 
-MODELS = File.join(File.dirname(__FILE__), "app/models")
+MODELS = File.join(File.dirname(__FILE__), "support/models")
 $LOAD_PATH.unshift(MODELS)
 
 require "action_controller"
@@ -17,9 +17,9 @@ else
 end
 
 # When testing locally we use the database named mongoid_test. However when
-# tests are running in parallel on Travis we need to use different database
-# names for each process running since we do not have transactions and want a
-# clean slate before each spec run.
+# tests are running in parallel in a CI environment we need to use different
+# database names for each process running since we do not have transactions
+# and want a clean slate before each spec run.
 def database_id
   "mongoid_test"
 end
@@ -28,20 +28,34 @@ def database_id_alt
   "mongoid_test_alt"
 end
 
+begin
+  require 'mrss/cluster_config'
+  require 'support/client_registry'
+  require 'mrss/constraints'
+  require 'mrss/event_subscriber'
+rescue LoadError => exc
+  raise LoadError.new <<~MSG.strip
+    The test suite requires shared tooling to be installed.
+    Please refer to spec/README.md for instructions.
+    #{exc.class}: #{exc}
+  MSG
+end
+
+ClusterConfig = Mrss::ClusterConfig
+
 require 'support/authorization'
 require 'support/expectations'
+require 'support/helpers'
 require 'support/macros'
-require 'support/cluster_config'
 require 'support/constraints'
-require 'mrss/constraints'
 
-# Give MongoDB time to start up on the travis ci environment.
-if (ENV['CI'] == 'travis' || ENV['CI'] == 'evergreen')
+# Give MongoDB servers time to start up in CI environments
+if SpecConfig.instance.ci?
   starting = true
   client = Mongo::Client.new(SpecConfig.instance.addresses)
   while starting
     begin
-      client.command(Mongo::Server::Monitor::Connection::ISMASTER)
+      client.command(ping: 1)
       break
     rescue Mongo::Error::OperationFailure => e
       sleep(2)
@@ -81,7 +95,7 @@ Mongoid.configure do |config|
   config.load_configuration(CONFIG)
 end
 
-# Autoload every model for the test suite that sits in spec/app/models.
+# Autoload every model for the test suite that sits in spec/support/models.
 Dir[ File.join(MODELS, "*.rb") ].sort.each do |file|
   name = File.basename(file, ".rb")
   autoload name.camelize.to_sym, name
@@ -110,22 +124,28 @@ end
 
 I18n.config.enforce_available_locales = false
 
+# The user must be created before any of the tests are loaded, until
+# https://jira.mongodb.org/browse/MONGOID-4827 is implemented.
+client = Mongo::Client.new(SpecConfig.instance.addresses, server_selection_timeout: 3.03)
+begin
+  # Create the root user administrator as the first user to be added to the
+  # database. This user will need to be authenticated in order to add any
+  # more users to any other databases.
+  client.database.users.create(MONGOID_ROOT_USER)
+rescue Mongo::Error::OperationFailure => e
+ensure
+  client.close
+end
+
 RSpec.configure do |config|
   config.raise_errors_for_deprecations!
+  config.include(Helpers)
   config.include(Mongoid::Expectations)
   config.extend(Mrss::Constraints)
   config.extend(Constraints)
   config.extend(Mongoid::Macros)
 
   config.before(:suite) do
-    client = Mongo::Client.new(SpecConfig.instance.addresses, server_selection_timeout: 3.03)
-    begin
-      # Create the root user administrator as the first user to be added to the
-      # database. This user will need to be authenticated in order to add any
-      # more users to any other databases.
-      client.database.users.create(MONGOID_ROOT_USER)
-    rescue Exception => e
-    end
     Mongoid.purge!
   end
 
@@ -143,31 +163,21 @@ RSpec.configure do |config|
 end
 
 # A subscriber to be used with the Ruby driver for testing.
-#
-# @since 6.4.0
 class EventSubscriber
 
   # The started events.
-  #
-  # @since 6.4.0
   attr_reader :started_events
 
   # The succeeded events.
-  #
-  # @since 6.4.0
   attr_reader :succeeded_events
 
   # The failed events.
-  #
-  # @since 6.4.0
   attr_reader :failed_events
 
   # Create the test event subscriber.
   #
   # @example Create the subscriber
   #   EventSubscriber.new
-  #
-  # @since 6.4.0
   def initialize
     @started_events = []
     @succeeded_events = []
@@ -177,8 +187,6 @@ class EventSubscriber
   # Cache the succeeded event.
   #
   # @param [ Event ] event The event.
-  #
-  # @since 6.4.0
   def succeeded(event)
     @succeeded_events.push(event)
   end
@@ -186,8 +194,6 @@ class EventSubscriber
   # Cache the started event.
   #
   # @param [ Event ] event The event.
-  #
-  # @since 6.4.0
   def started(event)
     @started_events.push(event)
   end
@@ -195,15 +201,11 @@ class EventSubscriber
   # Cache the failed event.
   #
   # @param [ Event ] event The event.
-  #
-  # @since 6.4.0
   def failed(event)
     @failed_events.push(event)
   end
 
   # Clear all cached events.
-  #
-  # @since 6.4.0
   def clear_events!
     @started_events = []
     @succeeded_events = []

data/spec/support/authorization.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Set up a root user so we can set up authentication on a database level.
 MONGOID_ROOT_USER = Mongo::Auth::User.new(
   database: Mongo::Database::ADMIN,

data/spec/support/client_registry.rb

@@ -0,0 +1,9 @@
+require 'singleton'
+
+class ClientRegistry
+  include Singleton
+
+  def global_client(name)
+    Mongoid.default_client
+  end
+end