mongo 2.13.2 → 2.13.3

data/spec/shared/lib/mrss/spec_organizer.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 autoload :JSON, 'json'
 autoload :FileUtils, 'fileutils'
 autoload :Find, 'find'

data/spec/shared/lib/mrss/utils.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+module Mrss
+  module Utils
+
+    module_function def print_backtrace(dest=STDERR)
+      begin
+        hello world
+      rescue => e
+        dest.puts e.backtrace.join("\n")
+      end
+    end
+  end
+end

data/spec/shared/share/Dockerfile.erb

@@ -0,0 +1,231 @@
+# Python toolchain as of this writing is available on rhel62, debian92 and
+# ubuntu1604.
+#
+# To run rhel62 in docker, host system must be configured to emulate syscalls:
+# https://github.com/CentOS/sig-cloud-instance-images/issues/103
+
+<%
+
+python_toolchain_url = "https://s3.amazonaws.com//mciuploads/mongo-python-driver-toolchain/#{distro}/ba92de2700c04ee2d4f82c3ffdfc33105140cb04/mongo_python_driver_toolchain_#{distro.gsub('-', '_')}_ba92de2700c04ee2d4f82c3ffdfc33105140cb04_19_11_14_15_33_33.tar.gz"
+server_version = '4.3.3'
+server_url = "http://downloads.10gen.com/linux/mongodb-linux-x86_64-enterprise-#{distro}-#{server_version}.tgz"
+server_archive_basename = File.basename(server_url)
+server_extracted_dir = server_archive_basename.sub(/\.(tar\.gz|tgz)$/, '')
+
+toolchain_upper='291ba4a4e8297f142796e70eee71b99f333e35e1'
+
+ruby_toolchain_url = "http://boxes.10gen.com/build/toolchain-drivers/mongo-ruby-driver/ruby-toolchain-#{distro}-#{toolchain_upper}.tar.xz"
+#ruby_toolchain_url = "https://s3.amazonaws.com//mciuploads/mongo-ruby-toolchain/#{distro}/#{toolchain_upper}/mongo_ruby_driver_toolchain_#{distro.gsub('-', '_')}_patch_#{toolchain_upper}_#{toolchain_lower}.tar.gz"
+
+%>
+
+FROM <%= base_image %>
+
+<% if debian? %>
+
+  ENV DEBIAN_FRONTEND=noninteractive
+
+<% else %>
+
+  RUN echo assumeyes=1 |tee -a /etc/yum.conf
+
+<% end %>
+
+<% if ruby_head? %>
+
+  # To use current versions of mlaunch, Python 3.6+ is required.
+  # Most distros ship with older Pythons, therefore we need to install
+  # a newer Python from somewhere. This section installs the Python
+  # toolhcain which comes with recent Pythons.
+  # Alternatively, Ruby toolchain compiles its own copy of Python 3 but
+  # this is currently incomplete in that on older distros with old OpenSSL,
+  # the built Python has no ssl module and hence practically is unusable.
+  # Currently Ruby driver uses mtools-legacy which supports Python 2,
+  # avoiding this entire issue for the time being.
+
+  #RUN curl --retry 3 -fL <%= python_toolchain_url %> -o python-toolchain.tar.gz
+  #RUN tar -xC /opt -zf python-toolchain.tar.gz
+
+<% end %>
+
+<% if debian? %>
+
+  # zsh is not required for any scripts but it is a better interactive shell
+  # than bash.
+  # Ruby runtime dependencies: libyaml-0-2
+  # Compiling ruby libraries: gcc make
+  # Compiling pyhton packages: python2.7-dev
+  # JRuby: openjdk-8-jre-headless
+  # Server dependencies: libsnmp30 libcurl3/libcurl4
+  # Determining OS we are running on: lsb-release
+  # Kerberos testing: krb5-user
+  # Local Kerberos server: krb5-kdc krb5-admin-server
+  # Installing mlaunch from git: git
+  # ruby-head archive: bzip2
+  # nio4r on JRuby: libgmp-dev
+  # Snappy compression: libsnappy-dev
+  # nokogiri: zlib1g-dev
+  # Mongoid testing: tzdata
+  # Mongoid application testing: nodejs (8.x or newer)
+  #
+  # We currently use Python 2-compatible version of mtools, which
+  # is installable via pip (which uses Python 2). All of the MongoDB
+  # distros have pip installed (but none as of this writing have pip3)
+  # therefore install python-pip in all configurations here.
+
+  <% packages = %w(
+    lsb-release bzip2 curl zsh
+    git make gcc libyaml-0-2 libgmp-dev zlib1g-dev libsnappy-dev
+    libsnmp30
+    krb5-user krb5-kdc krb5-admin-server libsasl2-dev libsasl2-modules-gssapi-mit
+    python-pip python2.7-dev python3-pip
+    tzdata
+  ) %>
+
+  # ubuntu1404 only has openjdk-7-jre-headless
+  <% if distro !~ /ubuntu1404/ %>
+    <% packages << 'openjdk-8-jre-headless' %>
+  <% end %>
+
+  # ubuntu1404, ubuntu1604: libcurl3
+  # ubuntu1804: libcurl4
+  <% if distro =~ /ubuntu1804/ %>
+    <% packages << 'libcurl4' %>
+  <% else %>
+    <% packages << 'libcurl3' %>
+  <% end %>
+
+  <% if distro =~ /ubuntu1804/ %>
+    <% packages << 'nodejs' %>
+  <% end %>
+
+  RUN apt-get update && apt-get install -y <%= packages.join(' ') %>
+<% else %>
+
+  # Enterprise server: net-snmp
+  # lsb_release: redhat-lsb-core
+  # our runner scripts: which
+  # Ruby dependency: libyaml
+  # compiling python packages: gcc python-devel
+  # Kerberos tests: krb5-workstation + cyrus-sasl-devel to build the
+  # mongo_kerberos gem + cyrus-sasl-gssapi for authentication to work
+  # Local Kerberos server: krb5-server
+  # JRuby: java-1.8.0-openjdk
+  #
+  # Note: lacking cyrus-sasl-gssapi produces a cryptic message
+  # "SASL(-4): no mechanism available: No worthy mechs found"
+  # https://github.com/farorm/python-ad/issues/10
+
+  RUN yum install -y redhat-lsb-core which git gcc libyaml krb5-server \
+    krb5-workstation cyrus-sasl-devel cyrus-sasl-gssapi java-1.8.0-openjdk \
+    net-snmp
+
+  <% if distro =~ /rhel6/ %>
+
+    # RHEL 6 ships with Python 2.6.
+
+    RUN yum install -y centos-release-scl && \
+      yum install -y python27-python python27-python-devel
+    ENV PATH=/opt/rh/python27/root/usr/bin:$PATH \
+      LD_LIBRARY_PATH=/opt/rh/python27/root/usr/lib64
+
+  <% else %>
+
+    RUN yum install -y python-devel
+
+  <% end %>
+
+<% end %>
+
+<% if preload? %>
+
+  # Current virtualenv fails with
+  # https://github.com/pypa/virtualenv/issues/1630
+  RUN python2 -m pip install 'virtualenv<20' 'mtools-legacy[mlaunch]'
+
+  RUN pip --version && \
+    pip install mtools-legacy[mlaunch]
+
+  <% if @env.fetch('MONGODB_VERSION') >= '4.4' %>
+    RUN python3 -mpip install asn1crypto oscrypto flask
+  <% end %>
+
+  <% unless ruby_head? %>
+
+    RUN curl --retry 3 -fL <%= ruby_toolchain_url %> |tar -xC /opt -Jf -
+    ENV PATH=/opt/rubies/<%= ruby %>/bin:$PATH \
+      USE_OPT_TOOLCHAIN=1
+    #ENV PATH=/opt/rubies/python/3/bin:$PATH
+
+  <% end %>
+
+  <% if distro =~ /rhel|ubuntu1604/ %>
+
+    # Ubuntu 12.04 ships pip 1.0 which is ancient and does not work.
+    #
+    # Ubuntu 16.04 apparently also ships a pip that does not work:
+    # https://stackoverflow.com/questions/37495375/python-pip-install-throws-typeerror-unsupported-operand-types-for-retry
+    # Potentially this only affects environments with less than ideal
+    # connectivity (or, perhaps, when python package registry is experiencing
+    # availability issues) when pip must retry to install packages.
+    #
+    # rhel apparently does not package pip at all in core repoitories,
+    # therefore install it the manual way.
+    #
+    # https://pip.pypa.io/en/stable/installing/
+    RUN curl --retry 3 -fL https://bootstrap.pypa.io/pip/2.7/get-pip.py | python
+
+  <% end %>
+
+  RUN curl --retry 3 -fL <%= server_download_url %> |tar xzf - && \
+    mv mongo*/ /opt/mongodb
+  ENV USE_OPT_MONGODB=1 USE_SYSTEM_PYTHON_PACKAGES=1
+
+<% end %>
+
+WORKDIR /app
+
+<% if preload? && !ruby_head? %>
+
+  COPY Gemfile .
+  COPY gemfiles gemfiles
+  COPY *.gemspec .
+  COPY lib/<%= project_lib_subdir %>/version.rb lib/<%= project_lib_subdir %>/version.rb
+  RUN bundle install
+  COPY .evergreen/patch-debuggers .evergreen/patch-debuggers
+  RUN .evergreen/patch-debuggers /opt/rubies
+
+<% end %>
+
+<% if fle? %>
+  RUN curl --retry 3 -fLo libmongocrypt-all.tar.gz "https://s3.amazonaws.com/mciuploads/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz"
+  RUN tar xf libmongocrypt-all.tar.gz
+
+  <%= "ENV LIBMONGOCRYPT_PATH #{libmongocrypt_path}" %>
+<% end %>
+
+ENV MONGO_ORCHESTRATION_HOME=/tmpfs \
+  PROJECT_DIRECTORY=/app \
+  <%= @env.map { |k, v| %Q`#{k}="#{v.gsub('$', "\\$").gsub('"', "\\\"")}"` }.join(" \\\n  ") %>
+
+<% if interactive? %>
+  ENV INTERACTIVE=1
+<% end %>
+
+COPY . .
+
+<% if expose? %>
+
+  <% ports = [] %>
+
+  <% 0.upto(num_exposed_ports-1) do |i| %>
+    <% ports << 27017 + i %>
+  <% end %>
+
+  <% if @env['OCSP_ALGORITHM'] %>
+    <% ports << 8100 %>
+  <% end %>
+
+  EXPOSE <%= ports.map(&:to_s).join(' ') %>
+
+<% end %>

data/spec/shared/shlib/distro.sh

@@ -0,0 +1,73 @@
+detected_distro=
+
+host_distro() {
+  if test -z "$detected_distro"; then
+    detected_distro=`_detect_distro`
+  fi
+  echo "$detected_distro"
+}
+
+_detect_distro() {
+  local distro
+  distro=
+  if test -f /etc/debian_version; then
+    # Debian or Ubuntu
+    if test "`uname -m`" = aarch64; then
+      distro=ubuntu1604-arm
+    elif lsb_release -is |grep -q Debian; then
+      release=`lsb_release -rs |tr -d .`
+      # In docker, release is something like 9.11.
+      # In evergreen, release is 9.2.
+      release=`echo $release |sed -e 's/^9.*/92/'`
+      distro="debian$release"
+    elif lsb_release -is |grep -q Ubuntu; then
+      if test "`uname -m`" = ppc64le; then
+        release=`lsb_release -rs |tr -d .`
+        distro="ubuntu$release-ppc"
+      else
+        release=`lsb_release -rs |tr -d .`
+        distro="ubuntu$release"
+      fi
+    else
+      echo 'Unknown Debian flavor' 1>&2
+      exit 1
+    fi
+  elif lsb_release -is |grep -qi suse; then
+    if test "`uname -m`" = s390x; then
+      release=`lsb_release -rs |sed -e 's/\..*//'`
+      distro="suse$release-s390x"
+    else
+      echo 'Unknown Suse arch' 1>&2
+      exit 1
+    fi
+  elif test -f /etc/redhat-release; then
+    # RHEL or CentOS
+    if test "`uname -m`" = s390x; then
+      distro=rhel72-s390x
+    elif test "`uname -m`" = ppc64le; then
+      distro=rhel71-ppc
+    elif lsb_release >/dev/null 2>&1; then
+      if lsb_release -is |grep -q RedHat; then
+        release=`lsb_release -rs |tr -d .`
+        distro="rhel$release"
+      elif lsb_release -is |grep -q CentOS; then
+        release=`lsb_release -rs |cut -c 1 |sed -e s/7/70/ -e s/6/62/ -e s/8/80/`
+        distro="rhel$release"
+      else
+        echo 'Unknown RHEL flavor' 1>&2
+        exit 1
+      fi
+    else
+      echo lsb_release missing, using /etc/redhat-release 1>&2
+      release=`grep -o 'release [0-9]' /etc/redhat-release |awk '{print $2}'`
+      release=`echo $release |sed -e s/7/70/ -e s/6/62/ -e s/8/80/`
+      distro=rhel$release
+    fi
+  else
+    lsb_release -a
+    echo 'Unknown distro' 1>&2
+    exit 1
+  fi
+  echo "Detected distro: $distro" 1>&2
+  echo $distro
+}

data/spec/shared/shlib/server.sh

@@ -0,0 +1,290 @@
+# This file contains functions pertaining to downloading, starting and
+# configuring a MongoDB server.
+
+set_fcv() {
+  if test -n "$FCV"; then
+    mongo --eval 'assert.commandWorked(db.adminCommand( { setFeatureCompatibilityVersion: "'"$FCV"'" } ));' "$MONGODB_URI"
+    mongo --quiet --eval 'db.adminCommand( { getParameter: 1, featureCompatibilityVersion: 1 } )' |grep  "version.*$FCV"
+  fi
+}
+
+add_uri_option() {
+  opt=$1
+  
+  if ! echo $MONGODB_URI |sed -e s,//,, |grep -q /; then
+    MONGODB_URI="$MONGODB_URI/"
+  fi
+  
+  if ! echo $MONGODB_URI |grep -q '?'; then
+    MONGODB_URI="$MONGODB_URI?"
+  fi
+  
+  MONGODB_URI="$MONGODB_URI&$opt"
+}
+
+prepare_server() {
+  arch=$1
+  
+  if test -n "$USE_OPT_MONGODB"; then
+    export BINDIR=/opt/mongodb/bin
+    export PATH=$BINDIR:$PATH
+    return
+  fi
+
+  if test "$MONGODB_VERSION" = latest; then
+    # Test on the most recent published 4.3 release.
+    # https://jira.mongodb.org/browse/RUBY-1724
+    echo 'Using "latest" server is not currently implemented' 1>&2
+    exit 1
+  else
+    download_version="$MONGODB_VERSION"
+  fi
+  
+  url=`$(dirname $0)/get-mongodb-download-url $download_version $arch`
+
+  prepare_server_from_url $url
+}
+
+prepare_server_from_url() {
+  url=$1
+
+  mongodb_dir="$MONGO_ORCHESTRATION_HOME"/mdb
+  mkdir -p "$mongodb_dir"
+  curl --retry 3 $url |tar xz -C "$mongodb_dir" -f -
+  BINDIR="$mongodb_dir"/`basename $url |sed -e s/.tgz//`/bin
+  export PATH="$BINDIR":$PATH
+}
+
+install_mlaunch_virtualenv() {
+  python2 -V || true
+  if ! python2 -m virtualenv -h >/dev/null; then
+    # Current virtualenv fails with
+    # https://github.com/pypa/virtualenv/issues/1630
+    python2 -m pip install 'virtualenv<20' --user
+  fi
+  if test "$USE_SYSTEM_PYTHON_PACKAGES" = 1 &&
+    python2 -m pip list |grep mtools-legacy
+  then
+    # Use the existing mtools-legacy
+    :
+  else
+    venvpath="$MONGO_ORCHESTRATION_HOME"/venv
+    python2 -m virtualenv -p python2 $venvpath
+    . $venvpath/bin/activate
+    pip install 'mtools-legacy[mlaunch]'
+  fi
+}
+
+install_mlaunch_pip() {
+  if test -n "$USE_OPT_MONGODB" && which mlaunch >/dev/null 2>&1; then
+    # mlaunch is preinstalled in the docker image, do not install it here
+    return
+  fi
+  
+  python -V || true
+  python3 -V || true
+  pythonpath="$MONGO_ORCHESTRATION_HOME"/python
+  pip install -t "$pythonpath" 'mtools-legacy[mlaunch]'
+  export PATH="$pythonpath/bin":$PATH
+  export PYTHONPATH="$pythonpath"
+}
+
+install_mlaunch_git() {
+  repo=$1
+  branch=$2
+  python -V || true
+  python3 -V || true
+  which pip || true
+  which pip3 || true
+  
+  if false; then
+    if ! virtualenv --version; then
+      python3 `which pip3` install --user virtualenv
+      export PATH=$HOME/.local/bin:$PATH
+      virtualenv --version
+    fi
+    
+    venvpath="$MONGO_ORCHESTRATION_HOME"/venv
+    virtualenv -p python3 $venvpath
+    . $venvpath/bin/activate
+    
+    pip3 install psutil pymongo
+    
+    git clone $repo mlaunch
+    cd mlaunch
+    git checkout origin/$branch
+    python3 setup.py install
+    cd ..
+  else
+    pip install --user 'virtualenv==13'
+    export PATH=$HOME/.local/bin:$PATH
+    
+    venvpath="$MONGO_ORCHESTRATION_HOME"/venv
+    virtualenv $venvpath
+    . $venvpath/bin/activate
+  
+    pip install psutil pymongo
+    
+    git clone $repo mlaunch
+    (cd mlaunch &&
+      git checkout origin/$branch &&
+      python setup.py install
+    )
+  fi
+}
+
+calculate_server_args() {
+  local mongo_version=`echo $MONGODB_VERSION |tr -d .`
+  
+  if test -z "$mongo_version"; then
+    echo "$MONGODB_VERSION must be set and not contain only dots" 1>&2
+    exit 3
+  fi
+  
+  if test $mongo_version = latest; then
+    mongo_version=49
+  fi
+
+  local args="--setParameter enableTestCommands=1"
+  
+  if test $mongo_version -ge 47; then
+    args="$args --setParameter acceptAPIVersion2=1"
+  fi
+  
+  # diagnosticDataCollectionEnabled is a mongod-only parameter on server 3.2,
+  # and mlaunch does not support specifying mongod-only parameters:
+  # https://github.com/rueckstiess/mtools/issues/696
+  # Pass it to 3.4 and newer servers where it is accepted by all daemons.
+  if test $mongo_version -ge 34; then
+    args="$args --setParameter diagnosticDataCollectionEnabled=false"
+  fi
+  local uri_options=
+  if test "$TOPOLOGY" = replica-set; then
+    args="$args --replicaset --name ruby-driver-rs --nodes 2 --arbiter"
+    export HAVE_ARBITER=1
+  elif test "$TOPOLOGY" = sharded-cluster; then
+    args="$args --replicaset --nodes 2 --sharded 1 --name ruby-driver-rs"
+    if test -z "$SINGLE_MONGOS"; then
+      args="$args --mongos 2"
+    fi
+  else
+    args="$args --single"
+  fi
+  if test -n "$MMAPV1"; then
+    args="$args --storageEngine mmapv1 --smallfiles --noprealloc"
+    uri_options="$uri_options&retryReads=false&retryWrites=false"
+  fi
+  if test "$AUTH" = auth; then
+    args="$args --auth --username bob --password pwd123"
+  elif test "$AUTH" = x509; then
+    args="$args --auth --username bootstrap --password bootstrap"
+  elif echo "$AUTH" |grep -q ^aws; then
+    args="$args --auth --username bootstrap --password bootstrap"
+    args="$args --setParameter authenticationMechanisms=MONGODB-AWS,SCRAM-SHA-1,SCRAM-SHA-256"
+    uri_options="$uri_options&authMechanism=MONGODB-AWS&authSource=\$external"
+  fi
+
+  if test -n "$OCSP"; then
+    if test -z "$OCSP_ALGORITHM"; then
+      echo "OCSP_ALGORITHM must be set if OCSP is set" 1>&2
+      exit 1
+    fi
+  fi
+
+  local server_cert_path server_ca_path server_client_cert_path
+  if test "$SSL" = ssl || test -n "$OCSP_ALGORITHM"; then
+    if test -n "$OCSP_ALGORITHM"; then
+      if test "$OCSP_MUST_STAPLE" = 1; then
+        server_cert_path=spec/support/ocsp/$OCSP_ALGORITHM/server-mustStaple.pem
+      else
+        server_cert_path=spec/support/ocsp/$OCSP_ALGORITHM/server.pem
+      fi
+      server_ca_path=spec/support/ocsp/$OCSP_ALGORITHM/ca.crt
+      server_client_cert_path=spec/support/ocsp/$OCSP_ALGORITHM/server.pem
+    else
+      server_cert_path=spec/support/certificates/server-second-level-bundle.pem
+      server_ca_path=spec/support/certificates/ca.crt
+      server_client_cert_path=spec/support/certificates/client.pem
+    fi
+
+    if test -n "$OCSP_ALGORITHM"; then
+      client_cert_path=spec/support/ocsp/$OCSP_ALGORITHM/server.pem
+    elif test "$AUTH" = x509; then
+      client_cert_path=spec/support/certificates/client-x509.pem
+
+      uri_options="$uri_options&authMechanism=MONGODB-X509"
+    elif echo $RVM_RUBY |grep -q jruby; then
+      # JRuby does not grok chained certificate bundles -
+      # https://github.com/jruby/jruby-openssl/issues/181
+      client_cert_path=spec/support/certificates/client.pem
+    else
+      client_cert_path=spec/support/certificates/client-second-level-bundle.pem
+    fi
+
+    uri_options="$uri_options&tls=true&"\
+"tlsCAFile=$server_ca_path&"\
+"tlsCertificateKeyFile=$client_cert_path"
+
+    args="$args --sslMode requireSSL"\
+" --sslPEMKeyFile $server_cert_path"\
+" --sslCAFile $server_ca_path"\
+" --sslClientCertificate $server_client_cert_path"
+  fi
+
+  # Docker forwards ports to the external interface, not to the loopback.
+  # Hence we must bind to all interfaces here.
+  if test -n "$BIND_ALL"; then
+    args="$args --bind_ip_all"
+  fi
+
+  # MongoDB servers pre-4.2 do not enable zlib compression by default
+  if test "$COMPRESSOR" = snappy; then
+    args="$args --networkMessageCompressors snappy"
+  elif test "$COMPRESSOR" = zlib; then
+    args="$args --networkMessageCompressors zlib"
+  fi
+
+  if test -n "$OCSP_ALGORITHM" || test -n "$OCSP_VERIFIER"; then
+    python3 -m pip install asn1crypto oscrypto flask
+  fi
+
+  local ocsp_args=
+  if test -n "$OCSP_ALGORITHM"; then
+    if test -z "$server_ca_path"; then
+      echo "server_ca_path must have been set" 1>&2
+      exit 1
+    fi
+    ocsp_args="--ca_file $server_ca_path"
+    if test "$OCSP_DELEGATE" = 1; then
+      ocsp_args="$ocsp_args \
+  --ocsp_responder_cert spec/support/ocsp/$OCSP_ALGORITHM/ocsp-responder.crt \
+  --ocsp_responder_key spec/support/ocsp/$OCSP_ALGORITHM/ocsp-responder.key \
+  "
+    else
+      ocsp_args="$ocsp_args \
+  --ocsp_responder_cert spec/support/ocsp/$OCSP_ALGORITHM/ca.crt \
+  --ocsp_responder_key spec/support/ocsp/$OCSP_ALGORITHM/ca.key \
+  "
+    fi
+    if test -n "$OCSP_STATUS"; then
+      ocsp_args="$ocsp_args --fault $OCSP_STATUS"
+    fi
+  fi
+  
+  OCSP_ARGS="$ocsp_args"
+  SERVER_ARGS="$args"
+  URI_OPTIONS="$uri_options"
+}
+
+launch_ocsp_mock() {
+  if test -n "$OCSP_ARGS"; then
+    # Bind to 0.0.0.0 for Docker
+    python3 spec/support/ocsp/ocsp_mock.py $OCSP_ARGS -b 0.0.0.0 -p 8100 &
+    OCSP_MOCK_PID=$!
+  fi
+}
+
+launch_server() {
+  local dbdir="$1"
+  python -m mtools.mlaunch.mlaunch --dir "$dbdir" --binarypath "$BINDIR" $SERVER_ARGS
+}