bson 5.0.2 → 5.1.0

data/spec/shared/share/Dockerfile.erb

@@ -0,0 +1,251 @@
+<%
+
+def ruby_toolchain_url(ruby)
+  "http://boxes.10gen.com/build/toolchain-drivers/mongo-ruby-toolchain/library/#{distro}/#{ruby}.tar.xz"
+end
+
+%>
+
+FROM <%= base_image %>
+
+ENV DOCKER=1
+
+<% if debian? %>
+
+  ENV DEBIAN_FRONTEND=noninteractive
+
+<% else %>
+
+  RUN echo assumeyes=1 |tee -a /etc/yum.conf
+
+<% end %>
+
+<% if debian? %>
+
+  # zsh is not required for any scripts but it is a better interactive shell
+  # than bash.
+  # Ruby runtime dependencies: libyaml-0-2
+  # Compiling ruby libraries: gcc make
+  # Compiling python packages: python3-dev
+  # JRuby: openjdk-17-jdk-headless
+  # Server dependencies: libsnmp30 libcurl3/libcurl4
+  # Determining OS we are running on: lsb-release
+  # Load balancer testing: haproxy
+  # Kerberos testing: krb5-user
+  # Local Kerberos server: krb5-kdc krb5-admin-server
+  # Installing mlaunch from git: git
+  # ruby-head archive: bzip2
+  # nio4r on JRuby: libgmp-dev
+  # Snappy compression: libsnappy-dev
+  # nokogiri: zlib1g-dev
+  # Mongoid testing: tzdata shared-mime-info
+  # Mongoid application testing: nodejs (8.x or newer)
+  # Test suite: procps for ps (to kill JRubies)
+  #
+  # We currently use Python 2-compatible version of mtools, which
+  # is installable via pip (which uses Python 2). All of the MongoDB
+  # distros have pip installed (but none as of this writing have pip3)
+  # therefore install python-pip in all configurations here.
+
+  <% packages = %w(
+    procps lsb-release bzip2 curl wget gpg zsh
+    git make gcc g++ libyaml-dev libgmp-dev zlib1g-dev libsnappy-dev
+    krb5-user krb5-kdc krb5-admin-server libsasl2-dev libsasl2-modules-gssapi-mit
+    haproxy libcurl4
+    tzdata shared-mime-info software-properties-common xz-utils nodejs npm
+    openjdk-17-jdk-headless
+  ) %>
+
+  <% if distro =~ /ubuntu2004/ %>
+    <% packages << 'libsnmp35' %>
+  <% elsif distro =~ /ubuntu2204|debian11/ %>
+    <% packages << 'libsnmp40' %>
+  <% else %>
+    <% packages << 'libsnmp30' %>
+  <% end %>
+
+  <% if distro !~ /ubuntu2004|ubuntu2204|debian11/ %>
+    <% packages << 'python-pip' %>
+  <% end %>
+
+  <% if distro =~ /ubuntu2204|debian11/ %>
+    <% packages << 'python3-venv' %>
+  <% end %>
+
+  <% if distro =~ /ubuntu2004|ubuntu2204/ %>
+    <% packages += %w(ruby bundler) %>
+  <% end %>
+
+  RUN apt-get update && apt-get install -y <%= packages.join(' ') %>
+
+  <% if ubuntu? %>
+    RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
+    RUN echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null
+  <% end %>
+  RUN apt-add-repository ppa:deadsnakes/ppa -y
+  RUN apt-get update && apt-get install -y cmake python3.10 python3.10-dev python3.10-venv
+  RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1
+<% else %>
+
+  # Enterprise server: net-snmp
+  # lsb_release: redhat-lsb-core
+  # our runner scripts: which
+  # Ruby dependency: libyaml
+  # compiling python packages: gcc python-devel
+  # Kerberos tests: krb5-workstation + cyrus-sasl-devel to build the
+  # mongo_kerberos gem + cyrus-sasl-gssapi for authentication to work
+  # Local Kerberos server: krb5-server
+  # JRuby: java-17-openjdk
+  #
+  # Note: lacking cyrus-sasl-gssapi produces a cryptic message
+  # "SASL(-4): no mechanism available: No worthy mechs found"
+  # https://github.com/farorm/python-ad/issues/10
+
+  RUN yum --enablerepo=powertools install -y redhat-lsb-core which git gcc gcc-c++ libyaml-devel krb5-server \
+    krb5-workstation cyrus-sasl-devel cyrus-sasl-gssapi java-17-openjdk \
+    net-snmp python38 python38-devel cmake nodejs npm xz
+
+<% end %>
+
+<% if preload? %>
+
+  <% if distro =~ /debian9|ubuntu1604|ubuntu1804/ %>
+    # Install python 3.7 for mlaunch.
+    RUN curl -fL --retry 3 https://github.com/p-mongodb/deps/raw/main/<%= distro %>-python37.tar.xz | \
+      tar xfJ - -C /opt
+    ENV PATH=/opt/python37/bin:$PATH
+    RUN python3 -V
+  <% end %>
+
+  <% if true || distro =~ /rhel|ubuntu1604/ %>
+
+    # Ubuntu 12.04 ships pip 1.0 which is ancient and does not work.
+    #
+    # Ubuntu 16.04 apparently also ships a pip that does not work:
+    # https://stackoverflow.com/questions/37495375/python-pip-install-throws-typeerror-unsupported-operand-types-for-retry
+    # Potentially this only affects environments with less than ideal
+    # connectivity (or, perhaps, when python package registry is experiencing
+    # availability issues) when pip must retry to install packages.
+    #
+    # rhel apparently does not package pip at all in core repoitories,
+    # therefore install it the manual way.
+    #
+    # https://pip.pypa.io/en/stable/installing/
+    RUN curl --retry 3 -fL https://bootstrap.pypa.io/pip/get-pip.py | python3
+    RUN python3 -m pip install --upgrade pip setuptools wheel
+
+  <% end %>
+
+  # Current virtualenv fails with
+  # https://github.com/pypa/virtualenv/issues/1630
+  <% mtools = 'legacy' %>
+  <% case mtools
+     when 'legacy' %>
+    # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864
+    RUN python3 -m pip install 'virtualenv<20' 'mtools-legacy[mlaunch]' 'pymongo<4' python-dateutil
+  <% when 'git' %>
+    # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864
+    RUN python3 -m pip install virtualenv 'pymongo>=4' python-dateutil psutil
+
+    # Install mtools from git because released versions do not work with pymongo 4.0
+    RUN git clone https://github.com/p-mongodb/mtools && \
+      cd mtools && \
+      python3 setup.py install
+  <% else %>
+    # mtools[mlaunch] does not work: https://github.com/rueckstiess/mtools/issues/856
+    # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864
+    RUN python3 -m pip install virtualenv 'pymongo>=4' python-dateutil psutil mtools
+  <% end %>
+
+  <% if @env.fetch('MONGODB_VERSION') >= '4.4' %>
+    # ubuntu1604 installs MarkupSafe 0.0.0 here instead of 2.0.0+
+    # as specified by dependencies, causing OCSP mock to not work.
+    RUN python3 -mpip install asn1crypto oscrypto flask --upgrade --ignore-installed
+  <% end %>
+
+  # FLE is tested against 4.0+ servers.
+  <% if @env.fetch('MONGODB_VERSION') >= '4.0' %>
+    # Requirements in drivers-evergreen-tools:
+    # boto3~=1.19 cryptography~=3.4.8 pykmip~=0.10.0
+    # cryptography does not install due to lacking setuptools_rust
+    # (either that version or anything that isn't part of system packages)
+    RUN python3 -mpip install boto3~=1.19 cryptography pykmip~=0.10.0 'sqlalchemy<2.0.0'
+  <% end %>
+
+  <% unless ruby_head? || system_ruby? %>
+
+    RUN curl --retry 3 -fL <%= ruby_toolchain_url(ruby) %> |tar -xC /opt -Jf -
+    ENV PATH=/opt/rubies/<%= ruby %>/bin:$PATH \
+      USE_OPT_TOOLCHAIN=1
+
+  <% end %>
+
+<% end %>
+
+<% if distro =~ /debian|ubuntu/ %>
+  # mkdir was moved from /usr/bin to /bin and MongoDB's distros
+  # apparently keep using the old location.
+  # This definitely affects debian10.
+  # https://stackoverflow.com/questions/64653051/make-usr-bin-mkdir-command-not-found-during-gem-install-nokogiri-in-ubuntu
+  RUN test -f /usr/bin/mkdir || ln -s /bin/mkdir /usr/bin/mkdir
+<% end %>
+
+WORKDIR /app
+
+<% if preload? && !ruby_head? %>
+
+  COPY Gemfile .
+  COPY gemfiles gemfiles
+  COPY *.gemspec .
+  COPY lib/<%= project_lib_subdir %>/version.rb lib/<%= project_lib_subdir %>/version.rb
+  RUN bundle install
+  COPY .evergreen/patch-debuggers .evergreen/patch-debuggers
+  <% if system_ruby? %>
+    # Running under docker with root access
+    RUN .evergreen/patch-debuggers /var/lib/gems
+  <% else %>
+    RUN .evergreen/patch-debuggers /opt/rubies
+  <% end %>
+
+<% end %>
+
+<% if fle? %>
+  RUN curl --retry 3 -fL "https://s3.amazonaws.com/mciuploads/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz" |tar zxf -
+
+  <%= "ENV LIBMONGOCRYPT_PATH #{libmongocrypt_path}" %>
+<% end %>
+
+<% if preload? %>
+  ENV DOCKER_PRELOAD=1
+<% end %>
+
+RUN npm install --global yarn
+
+ENV MONGO_ORCHESTRATION_HOME=/tmpfs \
+  PROJECT_DIRECTORY=/app \
+  <%= @env.map { |k, v| %Q`#{k}="#{v.gsub('$', "\\$").gsub('"', "\\\"")}"` }.join(" \\\n  ") %>
+
+<% if interactive? %>
+  ENV INTERACTIVE=1
+<% end %>
+
+COPY . .
+
+RUN bash -c '. .evergreen/download-mongodb.sh && get_distro && get_mongodb_download_url_for "$DISTRO" "<%= server_version %>" && curl --retry 3 -fL $MONGODB_DOWNLOAD_URL |tar xzf - && mv mongo*/ /opt/mongodb'
+ENV USE_OPT_MONGODB=1 USE_SYSTEM_PYTHON_PACKAGES=1
+
+<% if expose? %>
+
+  <% ports = [] %>
+
+  <% 0.upto(num_exposed_ports-1) do |i| %>
+    <% ports << 27017 + i %>
+  <% end %>
+
+  <% if @env['OCSP_ALGORITHM'] %>
+    <% ports << 8100 %>
+  <% end %>
+
+  EXPOSE <%= ports.map(&:to_s).join(' ') %>
+
+<% end %>

data/spec/shared/share/haproxy-1.conf

@@ -0,0 +1,16 @@
+# Modeled after
+# https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-load-balancer.sh
+
+defaults
+  mode tcp
+  timeout connect 7s
+  timeout client 55s
+  timeout server 55s
+
+frontend mongos_frontend
+  bind *:27017
+  use_backend mongos_backend
+
+backend mongos_backend
+  mode tcp
+  server mongos_one 127.0.0.1:27117 check

data/spec/shared/share/haproxy-2.conf

@@ -0,0 +1,17 @@
+# Modeled after
+# https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-load-balancer.sh
+
+defaults
+  mode tcp
+  timeout connect 7s
+  timeout client 55s
+  timeout server 55s
+
+frontend mongos_frontend
+  bind *:27017
+  use_backend mongos_backend
+
+backend mongos_backend
+  mode tcp
+  server mongos_one 127.0.0.1:27117 check
+  server mongos_two 127.0.0.1:27118 check

data/spec/shared/shlib/config.sh

@@ -0,0 +1,27 @@
+show_local_instructions_impl() {
+  local arch="$1"
+  shift
+  
+  echo To test this configuration locally:
+  local params=
+  while test -n "$1"; do
+    key="$1"
+    shift
+    # ${!foo} syntax is bash specific:
+    # https://stackoverflow.com/questions/14049057/bash-expand-variable-in-a-variable
+    value="${!key}"
+    if test -n "$value"; then
+      params="$params $key=$value"
+    fi
+  done
+  
+  # $0 has the current script being executed which is also the script that
+  # was initially invoked EXCEPT for the AWS configurations which use the
+  # wrapper script.
+  if echo "$AUTH" |grep -q ^aws; then
+    script=.evergreen/run-tests-aws-auth.sh
+  else
+    script="$0"
+  fi
+  echo ./.evergreen/test-on-docker -d $arch $params -s "$script"
+}

data/spec/shared/shlib/distro.sh

@@ -0,0 +1,84 @@
+detected_distro=
+
+host_distro() {
+  if test -z "$detected_distro"; then
+    detected_distro=`_detect_distro`
+  fi
+  echo "$detected_distro"
+}
+
+_detect_distro() {
+  local distro
+  distro=
+  if test -f /etc/debian_version; then
+    # Debian or Ubuntu
+    if test "`uname -m`" = aarch64; then
+      release=`lsb_release -rs |tr -d .`
+      distro="ubuntu$release"-arm
+    elif lsb_release -is |grep -q Debian; then
+      release=`lsb_release -rs |tr -d .`
+      # In docker, release is something like 9.11.
+      # In evergreen, release is 9.2.
+      release=`echo $release |sed -e 's/^9.*/92/'`
+      distro="debian$release"
+    elif lsb_release -is |grep -q Ubuntu; then
+      if test "`uname -m`" = ppc64le; then
+        release=`lsb_release -rs |tr -d .`
+        distro="ubuntu$release-ppc"
+      else
+        release=`lsb_release -rs |tr -d .`
+        distro="ubuntu$release"
+      fi
+    else
+      echo 'Unknown Debian flavor' 1>&2
+      exit 1
+    fi
+  elif lsb_release -is |grep -qi suse; then
+    if test "`uname -m`" = s390x; then
+      release=`lsb_release -rs |sed -e 's/\..*//'`
+      distro="suse$release-s390x"
+    else
+      echo 'Unknown Suse arch' 1>&2
+      exit 1
+    fi
+  elif test -f /etc/redhat-release; then
+    # RHEL or CentOS
+    if test "`uname -m`" = s390x; then
+      distro=rhel72-s390x
+    elif test "`uname -m`" = ppc64le; then
+      distro=rhel71-ppc
+    elif lsb_release >/dev/null 2>&1; then
+      if lsb_release -is |grep -q RedHat; then
+        release=`lsb_release -rs |tr -d .`
+        distro="rhel$release"
+      elif lsb_release -is |grep -q CentOS; then
+        release=`lsb_release -rs |cut -c 1 |sed -e s/7/70/ -e s/6/62/ -e s/8/80/`
+        distro="rhel$release"
+      else
+        echo 'Unknown RHEL flavor' 1>&2
+        exit 1
+      fi
+    else
+      echo lsb_release missing, using /etc/redhat-release 1>&2
+      release=`grep -o 'release [0-9]' /etc/redhat-release |awk '{print $2}'`
+      release=`echo $release |sed -e s/7/70/ -e s/6/62/ -e s/8/80/`
+      distro=rhel$release
+    fi
+  elif test -f /etc/os-release; then
+    name=`grep -o '^NAME=.*' /etc/os-release | awk -F '"' '{ print $2 }'`
+    version=`grep -o '^VERSION=.*' /etc/os-release | awk -F '"' '{ print $2 }'`
+    if test "$name" = "Amazon Linux"; then
+      distro=amazon$version
+    else
+      cat /etc/os-release
+      echo 'Unknown distro' 1>&2
+      exit 1
+    fi
+  else
+    lsb_release -a
+    echo 'Unknown distro' 1>&2
+    exit 1
+  fi
+  echo "Detected distro: $distro" 1>&2
+  echo $distro
+}