mongoid 7.2.2 → 7.2.6

data/spec/shared/share/Dockerfile.erb

@@ -55,9 +55,10 @@ FROM <%= base_image %>
   # Ruby runtime dependencies: libyaml-0-2
   # Compiling ruby libraries: gcc make
   # Compiling pyhton packages: python2.7-dev
-  # JRuby: openjdk-8-jre-headless
+  # JRuby: openjdk-8-jdk-headless
   # Server dependencies: libsnmp30 libcurl3/libcurl4
   # Determining OS we are running on: lsb-release
+  # Load balancer testing: haproxy
   # Kerberos testing: krb5-user
   # Local Kerberos server: krb5-kdc krb5-admin-server
   # Installing mlaunch from git: git
@@ -65,8 +66,9 @@ FROM <%= base_image %>
   # nio4r on JRuby: libgmp-dev
   # Snappy compression: libsnappy-dev
   # nokogiri: zlib1g-dev
-  # Mongoid testing: tzdata
+  # Mongoid testing: tzdata shared-mime-info
   # Mongoid application testing: nodejs (8.x or newer)
+  # Test suite: procps for ps (to kill JRubies)
   #
   # We currently use Python 2-compatible version of mtools, which
   # is installable via pip (which uses Python 2). All of the MongoDB
@@ -74,22 +76,44 @@ FROM <%= base_image %>
   # therefore install python-pip in all configurations here.
 
   <% packages = %w(
-    lsb-release bzip2 curl zsh
+    procps lsb-release bzip2 curl zsh
     git make gcc libyaml-0-2 libgmp-dev zlib1g-dev libsnappy-dev
-    libsnmp30
     krb5-user krb5-kdc krb5-admin-server libsasl2-dev libsasl2-modules-gssapi-mit
-    python-pip python2.7-dev python3-pip
-    tzdata
+    haproxy
+    python3-pip
+    tzdata shared-mime-info
   ) %>
 
-  # ubuntu1404 only has openjdk-7-jre-headless
-  <% if distro !~ /ubuntu1404/ %>
-    <% packages << 'openjdk-8-jre-headless' %>
+  <% if distro =~ /ubuntu1404/ %>
+    # For building python & setuptools
+    <% packages += %w(libssl-dev unzip) %>
+  <% end %>
+
+  <% if distro !~ /ubuntu2004/ %>
+    <% packages += %w(python2.7-dev) %>
+  <% end %>
+
+  <% if distro =~ /ubuntu2004/ %>
+    <% packages << 'libsnmp35' %>
+  <% else %>
+    <% packages << 'libsnmp30' %>
+  <% end %>
+
+  <% if distro !~ /ubuntu2004/ %>
+    <% packages << 'python-pip' %>
+  <% end %>
+
+  <% if distro =~ /debian10/ %>
+    <% packages << 'openjdk-11-jdk-headless' %>
+  <% elsif distro =~ /ubuntu1404/ %>
+    # Ubuntu 14.04 only has openjdk 7, this is too old to be useful
+  <% else %>
+    <% packages << 'openjdk-8-jdk-headless' %>
   <% end %>
 
   # ubuntu1404, ubuntu1604: libcurl3
-  # ubuntu1804: libcurl4
-  <% if distro =~ /ubuntu1804/ %>
+  # ubuntu1804, ubuntu2004, debian10: libcurl4
+  <% if distro =~ /ubuntu1804|ubuntu2004|debian10/ %>
     <% packages << 'libcurl4' %>
   <% else %>
     <% packages << 'libcurl3' %>
@@ -99,9 +123,36 @@ FROM <%= base_image %>
     <% packages << 'nodejs' %>
   <% end %>
 
+  <% if distro =~ /ubuntu2004/ %>
+    <% packages += %w(ruby ruby2.7 bundler python2 python2-dev) %>
+  <% end %>
+
   RUN apt-get update && apt-get install -y <%= packages.join(' ') %>
 <% else %>
 
+  <% if distro =~ /rhel6/ %>
+  
+    # CentOS 6 is dead - to use it retrieve the packages from vault:
+    # https://stackoverflow.com/questions/53562691/error-cannot-retrieve-repository-metadata-repomd-xml-for-repository-base-pl
+  
+    <%
+      
+      cfg = <<-CFG
+[base]
+name=CentOS-$releasever - Base
+#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=os
+#baseurl=http://mirror.centos.org/centos/$releasever/os/$basearch/
+baseurl=http://vault.centos.org/6.10/os/x86_64/
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
+CFG
+
+    %>
+  
+    RUN printf "<%= cfg.gsub("\n", "\\n") %>" >/etc/yum.repos.d/CentOS-Base.repo
+  
+  <% end %>
+  
   # Enterprise server: net-snmp
   # lsb_release: redhat-lsb-core
   # our runner scripts: which
@@ -118,7 +169,7 @@ FROM <%= base_image %>
 
   RUN yum install -y redhat-lsb-core which git gcc libyaml krb5-server \
     krb5-workstation cyrus-sasl-devel cyrus-sasl-gssapi java-1.8.0-openjdk \
-    net-snmp
+    net-snmp python3
 
   <% if distro =~ /rhel6/ %>
 
@@ -134,23 +185,69 @@ FROM <%= base_image %>
     RUN yum install -y python-devel
 
   <% end %>
-
+  
 <% end %>
 
 <% if preload? %>
 
+  <% if distro =~ /ubuntu1404/ %>
+    
+    # I couldn't find a prebuilt package of anything more recent than 2.7.6
+    # for 14.04.
+    RUN curl --retry 3 -fL https://www.python.org/ftp/python/2.7.16/Python-2.7.16.tar.xz | \
+      tar xfJ - && \
+      cd Python-2.7.16 && \
+      ./configure && \
+      nice make -j4 && \
+      make install && \
+      cd .. && rm -rf Python-2.7.16
+      
+    ENV PATH=/usr/local/bin:$PATH
+    
+    RUN curl --retry 3 -fL -o setuptools-44.1.1.zip https://files.pythonhosted.org/packages/b2/40/4e00501c204b457f10fe410da0c97537214b2265247bc9a5bc6edd55b9e4/setuptools-44.1.1.zip && \
+      unzip setuptools-44.1.1.zip && \
+      cd setuptools-44.1.1 && \
+      python setup.py install && \
+      cd .. && rm -rf setuptools-44.1.1
+
+  <% end%>
+
+  <% if true || distro =~ /rhel|ubuntu1604/ %>
+
+    # Ubuntu 12.04 ships pip 1.0 which is ancient and does not work.
+    #
+    # Ubuntu 16.04 apparently also ships a pip that does not work:
+    # https://stackoverflow.com/questions/37495375/python-pip-install-throws-typeerror-unsupported-operand-types-for-retry
+    # Potentially this only affects environments with less than ideal
+    # connectivity (or, perhaps, when python package registry is experiencing
+    # availability issues) when pip must retry to install packages.
+    #
+    # rhel apparently does not package pip at all in core repoitories,
+    # therefore install it the manual way.
+    #
+    # https://pip.pypa.io/en/stable/installing/
+    RUN curl --retry 3 -fL https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2
+
+  <% end %>
+
   # Current virtualenv fails with
   # https://github.com/pypa/virtualenv/issues/1630
-  RUN python2 -m pip install 'virtualenv<20' 'mtools-legacy[mlaunch]'
+  <% if distro =~ /ubuntu2004/ %>
+    RUN python3 -m pip install 'virtualenv<20' 'mtools-legacy[mlaunch]'
+  <% else %>
+    RUN python2 -m pip install 'virtualenv<20' 'mtools-legacy[mlaunch]'
+  <% end %>
 
   RUN pip --version && \
     pip install mtools-legacy[mlaunch]
 
   <% if @env.fetch('MONGODB_VERSION') >= '4.4' %>
-    RUN python3 -mpip install asn1crypto oscrypto flask
+    # ubuntu1604 installs MarkupSafe 0.0.0 here instead of 2.0.0+
+    # as specified by dependencies, causing OCSP mock to not work.
+    RUN python3 -mpip install asn1crypto oscrypto flask --upgrade
   <% end %>
 
-  <% unless ruby_head? %>
+  <% unless ruby_head? || system_ruby? %>
 
     RUN curl --retry 3 -fL <%= ruby_toolchain_url %> |tar -xC /opt -Jf -
     ENV PATH=/opt/rubies/<%= ruby %>/bin:$PATH \
@@ -159,30 +256,20 @@ FROM <%= base_image %>
 
   <% end %>
 
-  <% if distro =~ /rhel|ubuntu1604/ %>
-
-    # Ubuntu 12.04 ships pip 1.0 which is ancient and does not work.
-    #
-    # Ubuntu 16.04 apparently also ships a pip that does not work:
-    # https://stackoverflow.com/questions/37495375/python-pip-install-throws-typeerror-unsupported-operand-types-for-retry
-    # Potentially this only affects environments with less than ideal
-    # connectivity (or, perhaps, when python package registry is experiencing
-    # availability issues) when pip must retry to install packages.
-    #
-    # rhel apparently does not package pip at all in core repoitories,
-    # therefore install it the manual way.
-    #
-    # https://pip.pypa.io/en/stable/installing/
-    RUN curl --retry 3 -fL https://bootstrap.pypa.io/pip/2.7/get-pip.py | python
-
-  <% end %>
-
   RUN curl --retry 3 -fL <%= server_download_url %> |tar xzf - && \
     mv mongo*/ /opt/mongodb
   ENV USE_OPT_MONGODB=1 USE_SYSTEM_PYTHON_PACKAGES=1
 
 <% end %>
 
+<% if distro =~ /debian|ubuntu/ %>
+  # mkdir was moved from /usr/bin to /bin and MongoDB's distros
+  # apparently keep using the old location.
+  # This definitely affects debian10.
+  # https://stackoverflow.com/questions/64653051/make-usr-bin-mkdir-command-not-found-during-gem-install-nokogiri-in-ubuntu
+  RUN test -f /usr/bin/mkdir || ln -s /bin/mkdir /usr/bin/mkdir
+<% end %>
+
 WORKDIR /app
 
 <% if preload? && !ruby_head? %>
@@ -193,7 +280,12 @@ WORKDIR /app
   COPY lib/<%= project_lib_subdir %>/version.rb lib/<%= project_lib_subdir %>/version.rb
   RUN bundle install
   COPY .evergreen/patch-debuggers .evergreen/patch-debuggers
-  RUN .evergreen/patch-debuggers /opt/rubies
+  <% if system_ruby? %>
+    # Running under docker with root access
+    RUN .evergreen/patch-debuggers /var/lib/gems
+  <% else %>
+    RUN .evergreen/patch-debuggers /opt/rubies
+  <% end %>
 
 <% end %>
 

data/spec/shared/share/haproxy-1.conf

@@ -0,0 +1,16 @@
+# Modeled after
+# https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-load-balancer.sh
+
+defaults
+  mode tcp
+  timeout connect 7s
+  timeout client 55s
+  timeout server 55s
+
+frontend mongos_frontend
+  bind *:27017
+  use_backend mongos_backend
+
+backend mongos_backend
+  mode tcp
+  server mongos_one 127.0.0.1:27117 check

data/spec/shared/share/haproxy-2.conf

@@ -0,0 +1,17 @@
+# Modeled after
+# https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-load-balancer.sh
+
+defaults
+  mode tcp
+  timeout connect 7s
+  timeout client 55s
+  timeout server 55s
+
+frontend mongos_frontend
+  bind *:27017
+  use_backend mongos_backend
+
+backend mongos_backend
+  mode tcp
+  server mongos_one 127.0.0.1:27117 check
+  server mongos_two 127.0.0.1:27118 check

data/spec/shared/shlib/server.sh

@@ -10,21 +10,21 @@ set_fcv() {
 
 add_uri_option() {
   opt=$1
-  
+
   if ! echo $MONGODB_URI |sed -e s,//,, |grep -q /; then
     MONGODB_URI="$MONGODB_URI/"
   fi
-  
+
   if ! echo $MONGODB_URI |grep -q '?'; then
     MONGODB_URI="$MONGODB_URI?"
   fi
-  
+
   MONGODB_URI="$MONGODB_URI&$opt"
 }
 
 prepare_server() {
   arch=$1
-  
+
   if test -n "$USE_OPT_MONGODB"; then
     export BINDIR=/opt/mongodb/bin
     export PATH=$BINDIR:$PATH
@@ -39,7 +39,7 @@ prepare_server() {
   else
     download_version="$MONGODB_VERSION"
   fi
-  
+
   url=`$(dirname $0)/get-mongodb-download-url $download_version $arch`
 
   prepare_server_from_url $url
@@ -80,7 +80,7 @@ install_mlaunch_pip() {
     # mlaunch is preinstalled in the docker image, do not install it here
     return
   fi
-  
+
   python -V || true
   python3 -V || true
   pythonpath="$MONGO_ORCHESTRATION_HOME"/python
@@ -96,20 +96,20 @@ install_mlaunch_git() {
   python3 -V || true
   which pip || true
   which pip3 || true
-  
+
   if false; then
     if ! virtualenv --version; then
       python3 `which pip3` install --user virtualenv
       export PATH=$HOME/.local/bin:$PATH
       virtualenv --version
     fi
-    
+
     venvpath="$MONGO_ORCHESTRATION_HOME"/venv
     virtualenv -p python3 $venvpath
     . $venvpath/bin/activate
-    
+
     pip3 install psutil pymongo
-    
+
     git clone $repo mlaunch
     cd mlaunch
     git checkout origin/$branch
@@ -118,13 +118,13 @@ install_mlaunch_git() {
   else
     pip install --user 'virtualenv==13'
     export PATH=$HOME/.local/bin:$PATH
-    
+
     venvpath="$MONGO_ORCHESTRATION_HOME"/venv
     virtualenv $venvpath
     . $venvpath/bin/activate
-  
+
     pip install psutil pymongo
-    
+
     git clone $repo mlaunch
     (cd mlaunch &&
       git checkout origin/$branch &&
@@ -133,24 +133,32 @@ install_mlaunch_git() {
   fi
 }
 
+# This function sets followong global variables:
+#   server_cert_path
+#   server_ca_path
+#   server_client_cert_path
+#
+# These variables are used later to connect to processes via mongo client.
 calculate_server_args() {
   local mongo_version=`echo $MONGODB_VERSION |tr -d .`
-  
+
   if test -z "$mongo_version"; then
     echo "$MONGODB_VERSION must be set and not contain only dots" 1>&2
     exit 3
   fi
-  
+
   if test $mongo_version = latest; then
     mongo_version=49
   fi
 
   local args="--setParameter enableTestCommands=1"
-  
-  if test $mongo_version -ge 47; then
+
+  if test $mongo_version -ge 50; then
+    args="$args --setParameter acceptApiVersion2=1"
+  elif test $mongo_version -ge 47; then
     args="$args --setParameter acceptAPIVersion2=1"
   fi
-  
+
   # diagnosticDataCollectionEnabled is a mongod-only parameter on server 3.2,
   # and mlaunch does not support specifying mongod-only parameters:
   # https://github.com/rueckstiess/mtools/issues/696
@@ -160,15 +168,31 @@ calculate_server_args() {
   fi
   local uri_options=
   if test "$TOPOLOGY" = replica-set; then
-    args="$args --replicaset --name ruby-driver-rs --nodes 2 --arbiter"
+    args="$args --replicaset --name test-rs --nodes 2 --arbiter"
     export HAVE_ARBITER=1
   elif test "$TOPOLOGY" = sharded-cluster; then
-    args="$args --replicaset --nodes 2 --sharded 1 --name ruby-driver-rs"
+    args="$args --replicaset --nodes 2 --sharded 1 --name test-rs"
     if test -z "$SINGLE_MONGOS"; then
       args="$args --mongos 2"
     fi
-  else
+  elif test "$TOPOLOGY" = standalone; then
     args="$args --single"
+  elif test "$TOPOLOGY" = load-balanced; then
+    args="$args --replicaset --nodes 2 --sharded 1 --name test-rs --port 27117"
+    if test -z "$MRSS_ROOT"; then
+      echo "Please set MRSS_ROOT" 1>&2
+      exit 2
+    fi
+    if test -n "$SINGLE_MONGOS"; then
+      haproxy_config=$MRSS_ROOT/share/haproxy-1.conf
+    else
+      args="$args --mongos 2"
+      haproxy_config=$MRSS_ROOT/share/haproxy-2.conf
+    fi
+    uri_options="$uri_options&loadBalanced=true"
+  else
+    echo "Unknown topology: $TOPOLOGY" 1>&2
+    exit 1
   fi
   if test -n "$MMAPV1"; then
     args="$args --storageEngine mmapv1 --smallfiles --noprealloc"
@@ -191,7 +215,6 @@ calculate_server_args() {
     fi
   fi
 
-  local server_cert_path server_ca_path server_client_cert_path
   if test "$SSL" = ssl || test -n "$OCSP_ALGORITHM"; then
     if test -n "$OCSP_ALGORITHM"; then
       if test "$OCSP_MUST_STAPLE" = 1; then
@@ -270,7 +293,7 @@ calculate_server_args() {
       ocsp_args="$ocsp_args --fault $OCSP_STATUS"
     fi
   fi
-  
+
   OCSP_ARGS="$ocsp_args"
   SERVER_ARGS="$args"
   URI_OPTIONS="$uri_options"
@@ -287,4 +310,58 @@ launch_ocsp_mock() {
 launch_server() {
   local dbdir="$1"
   python -m mtools.mlaunch.mlaunch --dir "$dbdir" --binarypath "$BINDIR" $SERVER_ARGS
+
+  if test "$TOPOLOGY" = sharded-cluster && test $MONGODB_VERSION = 3.6; then
+    # On 3.6 server the sessions collection is not immediately available,
+    # so we run the refreshLogicalSessionCacheNow command on the config server
+    # and again on each mongos in order for the mongoses
+    # to correctly report logicalSessionTimeoutMinutes.
+    mongos_regex="\s*mongos\s+([0-9]+)\s+running\s+[0-9]+"
+    config_server_regex="\s*config\sserver\s+([0-9]+)\s+running\s+[0-9]+"
+    config_server=""
+    mongoses=()
+    if test "$AUTH" = auth
+    then
+      base_url="mongodb://bob:pwd123@localhost"
+    else
+      base_url="mongodb://localhost"
+    fi
+    if test "$SSL" = "ssl"
+    then
+      mongo_command="${BINDIR}/mongo --ssl --sslPEMKeyFile $server_cert_path --sslCAFile $server_ca_path"
+    else
+      mongo_command="${BINDIR}/mongo"
+    fi
+
+    while read -r line
+    do
+        if [[ $line =~ $config_server_regex ]]
+        then
+            port="${BASH_REMATCH[1]}"
+            config_server="${base_url}:${port}"
+        fi
+        if [[ $line =~ $mongos_regex ]]
+        then
+            port="${BASH_REMATCH[1]}"
+            mongoses+=("${base_url}:${port}")
+        fi
+    done < <(python -m mtools.mlaunch.mlaunch list --dir "$dbdir" --binarypath "$BINDIR")
+
+    if [ -n "$config_server" ]; then
+      ${mongo_command} "$config_server" --eval 'db.adminCommand("refreshLogicalSessionCacheNow")'
+      for mongos in ${mongoses[*]}
+      do
+        ${mongo_command} "$mongos" --eval 'db.adminCommand("refreshLogicalSessionCacheNow")'
+      done
+    fi
+  fi
+
+  if test "$TOPOLOGY" = load-balanced; then
+    if test -z "$haproxy_config"; then
+      echo haproxy_config should have been set 1>&2
+      exit 3
+    fi
+
+    haproxy -D -f $haproxy_config -p $mongodb_dir/haproxy.pid
+  fi
 }