bson 4.14.1 → 5.0.0

data/spec/shared/lib/mrss/server_version_registry.rb

@@ -24,6 +24,21 @@ module Mrss
 
     attr_reader :desired_version, :arch
 
+    def target_arch
+      # can't use RbConfig::CONFIG["arch"] because JRuby doesn't
+      # return anything meaningful there.
+      #
+      # also, need to use `uname -a` instead of (e.g.) `uname -p`
+      # because debian (at least) does not return anything meaningful
+      # for `uname -p`.
+      uname = `uname -a`.strip
+      @target_arch ||= case uname
+        when /aarch/ then "aarch64"
+        when /x86/   then "x86_64"
+        else raise "unsupported architecture #{uname.inspect}"
+        end
+    end
+
     def download_url
       @download_url ||= begin
         version, version_ok = detect_version(current_catalog)
@@ -40,35 +55,13 @@ module Mrss
         end
         dl = version['downloads'].detect do |dl|
           dl['archive']['url'].index("enterprise-#{arch}") &&
-          dl['arch'] == 'x86_64'
+          dl['arch'] == target_arch
         end
         unless dl
           raise MissingDownloadUrl, "No download for #{arch} for #{version['version']}"
         end
         url = dl['archive']['url']
       end
-    rescue MissingDownloadUrl
-      if %w(2.6 3.0).include?(desired_version) && arch == 'ubuntu1604'
-        # 2.6 and 3.0 are only available for ubuntu1204 and ubuntu1404.
-        # Those ubuntus have ancient Pythons that don't work due to not
-        # implementing recent TLS protocols.
-        # Because of this we test on ubuntu1604 which has a newer Python.
-        # But we still need to retrieve ubuntu1404-targeting builds.
-        url = self.class.new('3.2', arch).download_url
-        unless url.include?('3.2.')
-          raise 'URL not in expected format'
-        end
-        url = case desired_version
-        when '2.6'
-          url.sub(/\b3\.2\.\d+/, '2.6.12')
-        when '3.0'
-          url.sub(/\b3\.2\.\d+/, '3.0.15')
-        else
-          raise NotImplementedError
-        end.sub('ubuntu1604', 'ubuntu1404')
-      else
-        raise
-      end
     end
 
     private

data/spec/shared/lib/mrss/session_registry.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+require 'singleton'
+
+module Mrss
+
+  def self.patch_mongo_for_session_registry
+
+    Mongo::Client.class_eval do
+      alias :get_session_without_tracking :get_session
+
+      def get_session(options = {})
+        get_session_without_tracking(options).tap do |session|
+          SessionRegistry.instance.register(session) if session&.materialized?
+        end
+      end
+    end
+
+    Mongo::Session.class_eval do
+      alias :end_session_without_tracking :end_session
+
+      def end_session
+        SessionRegistry.instance.unregister(self)
+        end_session_without_tracking
+      end
+
+      alias :materialize_if_needed_without_tracking :materialize_if_needed
+
+      def materialize_if_needed
+        materialize_if_needed_without_tracking.tap do
+          SessionRegistry.instance.register(self)
+        end
+      end
+    end
+  end
+end
+
+module Mrss
+  class SessionRegistry
+    include Singleton
+
+    def initialize
+      @registry = {}
+    end
+
+    def register(session)
+      @registry[session.session_id] = session if session
+    end
+
+    def unregister(session)
+      return if session.ended? || !session.materialized?
+      @registry.delete(session.session_id)
+    end
+
+    def verify_sessions_ended!
+      @registry.delete_if { |_, session| session.ended? }
+
+      unless @registry.empty?
+        sessions = @registry.map { |_, session| session }
+        raise "Session registry contains live sessions: #{sessions.join(', ')}"
+      end
+    end
+
+    def clear_registry
+      @registry = {}
+    end
+  end
+end

data/spec/shared/lib/mrss/session_registry_legacy.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+require 'singleton'
+
+module Mrss
+
+  def self.patch_mongo_for_session_registry
+
+    Mongo::Client.class_eval do
+      alias :get_session_without_tracking :get_session
+
+      def get_session(options = {})
+        get_session_without_tracking(options).tap do |session|
+          SessionRegistry.instance.register(session)
+        end
+      end
+    end
+
+    Mongo::Session.class_eval do
+      alias :end_session_without_tracking :end_session
+
+      def end_session
+        SessionRegistry.instance.unregister(self)
+        end_session_without_tracking
+      end
+    end
+  end
+end
+
+module Mrss
+  class SessionRegistry
+    include Singleton
+
+    def initialize
+      @registry = {}
+    end
+
+    def register(session)
+      @registry[session.session_id] = session if session
+    end
+
+    def unregister(session)
+      @registry.delete(session.session_id) unless session.ended?
+    end
+
+    def verify_sessions_ended!
+      @registry.delete_if { |_, session| session.ended? }
+
+      unless @registry.empty?
+        sessions = @registry.map { |_, session| session }
+        raise "Session registry contains live sessions: #{sessions.join(', ')}"
+      end
+    end
+
+    def clear_registry
+      @registry = {}
+    end
+  end
+end

data/spec/shared/lib/mrss/utils.rb

@@ -3,13 +3,35 @@
 
 module Mrss
   module Utils
+    extend self
 
-    module_function def print_backtrace(dest=STDERR)
-      begin
-        hello world
-      rescue => e
-        dest.puts e.backtrace.join("\n")
-      end
+    def print_backtrace(dest=STDERR)
+      raise
+    rescue => e
+      dest.puts e.backtrace.join("\n")
+    end
+
+    # Parses the given version string, accounting for suffix information that
+    # Gem::Version cannot successfully parse.
+    #
+    # @param [ String ] version the version to parse
+    #
+    # @return [ Gem::Version ] the parsed version
+    #
+    # @raise [ ArgumentError ] if the string cannot be parsed.
+    def parse_version(version)
+      Gem::Version.new(version)
+    rescue ArgumentError
+      match = version.match(/\A(?<major>\d+)\.(?<minor>\d+)\.(?<patch>\d+)?(-[A-Za-z\+\d]+)?\z/)
+      raise ArgumentError.new("Malformed version number string #{version}") if match.nil?
+
+      Gem::Version.new(
+        [
+          match[:major],
+          match[:minor],
+          match[:patch]
+        ].join('.')
+      )
     end
   end
 end

data/spec/shared/share/Dockerfile.erb

@@ -7,23 +7,26 @@
 <%
 
 python_toolchain_url = "https://s3.amazonaws.com//mciuploads/mongo-python-driver-toolchain/#{distro}/ba92de2700c04ee2d4f82c3ffdfc33105140cb04/mongo_python_driver_toolchain_#{distro.gsub('-', '_')}_ba92de2700c04ee2d4f82c3ffdfc33105140cb04_19_11_14_15_33_33.tar.gz"
-server_version = '4.3.3'
+# server_version = '4.3.3'
 server_url = "http://downloads.10gen.com/linux/mongodb-linux-x86_64-enterprise-#{distro}-#{server_version}.tgz"
 server_archive_basename = File.basename(server_url)
 server_extracted_dir = server_archive_basename.sub(/\.(tar\.gz|tgz)$/, '')
 
-TOOLCHAIN_VERSION='35fb9fb797a941094bf4ee8678284da88849435c'
+# When changing, also update the hash in shlib/set_env.sh.
+TOOLCHAIN_VERSION='e8c60866f54bed7e336a37df3a97d6ae1b971b7d'
 
 def ruby_toolchain_url(ruby)
   "http://boxes.10gen.com/build/toolchain-drivers/mongo-ruby-driver/#{TOOLCHAIN_VERSION}/#{distro}/#{ruby}.tar.xz"
 end
 
-#ruby_toolchain_url = "https://s3.amazonaws.com//mciuploads/mongo-ruby-toolchain/#{distro}/#{toolchain_upper}/mongo_ruby_driver_toolchain_#{distro.gsub('-', '_')}_patch_#{toolchain_upper}_#{toolchain_lower}.tar.gz"
+#ruby_toolchain_url = "https://s3.amazonaws.com//mciuploads/mongo-ruby-toolchain/#{distro}/#{TOOLCHAIN_VERSION}/mongo_ruby_driver_toolchain_#{distro.gsub('-', '_')}_patch_#{TOOLCHAIN_VERSION}_#{toolchain_lower}.tar.gz"
 
 %>
 
 FROM <%= base_image %>
 
+ENV DOCKER=1
+
 <% if debian? %>
 
   ENV DEBIAN_FRONTEND=noninteractive
@@ -36,15 +39,10 @@ FROM <%= base_image %>
 
 <% if ruby_head? %>
 
-  # To use current versions of mlaunch, Python 3.6+ is required.
-  # Most distros ship with older Pythons, therefore we need to install
+  # To use current versions of mlaunch, Python 3.7+ is required.
+  # Many distros ship with older Pythons, therefore we need to install
   # a newer Python from somewhere. This section installs the Python
-  # toolhcain which comes with recent Pythons.
-  # Alternatively, Ruby toolchain compiles its own copy of Python 3 but
-  # this is currently incomplete in that on older distros with old OpenSSL,
-  # the built Python has no ssl module and hence practically is unusable.
-  # Currently Ruby driver uses mtools-legacy which supports Python 2,
-  # avoiding this entire issue for the time being.
+  # toolchain which comes with recent Pythons.
 
   #RUN curl --retry 3 -fL <%= python_toolchain_url %> -o python-toolchain.tar.gz
   #RUN tar -xC /opt -zf python-toolchain.tar.gz
@@ -57,7 +55,7 @@ FROM <%= base_image %>
   # than bash.
   # Ruby runtime dependencies: libyaml-0-2
   # Compiling ruby libraries: gcc make
-  # Compiling pyhton packages: python2.7-dev
+  # Compiling python packages: python3-dev
   # JRuby: openjdk-8-jdk-headless
   # Server dependencies: libsnmp30 libcurl3/libcurl4
   # Determining OS we are running on: lsb-release
@@ -79,34 +77,31 @@ FROM <%= base_image %>
   # therefore install python-pip in all configurations here.
 
   <% packages = %w(
-    procps lsb-release bzip2 curl zsh
+    procps lsb-release bzip2 curl wget gpg zsh
     git make gcc libyaml-0-2 libgmp-dev zlib1g-dev libsnappy-dev
     krb5-user krb5-kdc krb5-admin-server libsasl2-dev libsasl2-modules-gssapi-mit
     haproxy
     python3-pip
-    tzdata shared-mime-info
+    tzdata shared-mime-info software-properties-common
   ) %>
 
-  <% if distro =~ /ubuntu1404/ %>
-    # For building python & setuptools
-    <% packages += %w(libssl-dev unzip) %>
-  <% end %>
-
-  <% if distro !~ /ubuntu2004/ %>
-    <% packages += %w(python2.7-dev) %>
-  <% end %>
-
   <% if distro =~ /ubuntu2004/ %>
     <% packages << 'libsnmp35' %>
+  <% elsif distro =~ /ubuntu2204|debian11/ %>
+    <% packages << 'libsnmp40' %>
   <% else %>
     <% packages << 'libsnmp30' %>
   <% end %>
 
-  <% if distro !~ /ubuntu2004/ %>
+  <% if distro !~ /ubuntu2004|ubuntu2204|debian11/ %>
     <% packages << 'python-pip' %>
   <% end %>
 
-  <% if distro =~ /debian10/ %>
+  <% if distro =~ /ubuntu2204|debian11/ %>
+    <% packages << 'python3-venv' %>
+  <% end %>
+
+  <% if distro =~ /debian10|ubuntu2204|debian11/ %>
     <% packages << 'openjdk-11-jdk-headless' %>
   <% elsif distro =~ /ubuntu1404/ %>
     # Ubuntu 14.04 only has openjdk 7, this is too old to be useful
@@ -116,30 +111,37 @@ FROM <%= base_image %>
 
   # ubuntu1404, ubuntu1604: libcurl3
   # ubuntu1804, ubuntu2004, debian10: libcurl4
-  <% if distro =~ /ubuntu1804|ubuntu2004|debian10/ %>
+  <% if distro =~ /ubuntu1804|ubuntu2004|ubuntu2204|debian10|debian11/ %>
     <% packages << 'libcurl4' %>
   <% else %>
     <% packages << 'libcurl3' %>
   <% end %>
 
-  <% if distro =~ /ubuntu1804/ %>
+  <% if distro =~ /ubuntu1804|ubuntu2004|ubuntu2204/ %>
     <% packages << 'nodejs' %>
   <% end %>
 
-  <% if distro =~ /ubuntu2004/ %>
-    <% packages += %w(ruby ruby2.7 bundler python2 python2-dev) %>
+  <% if distro =~ /ubuntu2004|ubuntu2204/ %>
+    <% packages += %w(ruby bundler) %>
   <% end %>
 
   RUN apt-get update && apt-get install -y <%= packages.join(' ') %>
+
+  <% if ubuntu? %>
+    RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
+    RUN echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null
+  <% end %>
+  RUN apt-get update && apt-get install -y cmake
+
 <% else %>
 
   <% if distro =~ /rhel6/ %>
-  
+
     # CentOS 6 is dead - to use it retrieve the packages from vault:
     # https://stackoverflow.com/questions/53562691/error-cannot-retrieve-repository-metadata-repomd-xml-for-repository-base-pl
-  
+
     <%
-      
+
       cfg = <<-CFG
 [base]
 name=CentOS-$releasever - Base
@@ -151,11 +153,11 @@ gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
 CFG
 
     %>
-  
+
     RUN printf "<%= cfg.gsub("\n", "\\n") %>" >/etc/yum.repos.d/CentOS-Base.repo
-  
+
   <% end %>
-  
+
   # Enterprise server: net-snmp
   # lsb_release: redhat-lsb-core
   # our runner scripts: which
@@ -172,48 +174,19 @@ CFG
 
   RUN yum install -y redhat-lsb-core which git gcc libyaml krb5-server \
     krb5-workstation cyrus-sasl-devel cyrus-sasl-gssapi java-1.8.0-openjdk \
-    net-snmp python3
-
-  <% if distro =~ /rhel6/ %>
-
-    # RHEL 6 ships with Python 2.6.
-
-    RUN yum install -y centos-release-scl && \
-      yum install -y python27-python python27-python-devel
-    ENV PATH=/opt/rh/python27/root/usr/bin:$PATH \
-      LD_LIBRARY_PATH=/opt/rh/python27/root/usr/lib64
-
-  <% else %>
+    net-snmp python38 python38-devel cmake nodejs
 
-    RUN yum install -y python-devel
-
-  <% end %>
-  
 <% end %>
 
 <% if preload? %>
 
-  <% if distro =~ /ubuntu1404/ %>
-    
-    # I couldn't find a prebuilt package of anything more recent than 2.7.6
-    # for 14.04.
-    RUN curl --retry 3 -fL https://www.python.org/ftp/python/2.7.16/Python-2.7.16.tar.xz | \
-      tar xfJ - && \
-      cd Python-2.7.16 && \
-      ./configure && \
-      nice make -j4 && \
-      make install && \
-      cd .. && rm -rf Python-2.7.16
-      
-    ENV PATH=/usr/local/bin:$PATH
-    
-    RUN curl --retry 3 -fL -o setuptools-44.1.1.zip https://files.pythonhosted.org/packages/b2/40/4e00501c204b457f10fe410da0c97537214b2265247bc9a5bc6edd55b9e4/setuptools-44.1.1.zip && \
-      unzip setuptools-44.1.1.zip && \
-      cd setuptools-44.1.1 && \
-      python setup.py install && \
-      cd .. && rm -rf setuptools-44.1.1
-
-  <% end%>
+  <% if distro =~ /debian9|ubuntu1604|ubuntu1804/ %>
+    # Install python 3.7 for mlaunch.
+    RUN curl -fL --retry 3 https://github.com/p-mongodb/deps/raw/main/<%= distro %>-python37.tar.xz | \
+      tar xfJ - -C /opt
+    ENV PATH=/opt/python37/bin:$PATH
+    RUN python3 -V
+  <% end %>
 
   <% if true || distro =~ /rhel|ubuntu1604/ %>
 
@@ -229,25 +202,36 @@ CFG
     # therefore install it the manual way.
     #
     # https://pip.pypa.io/en/stable/installing/
-    RUN curl --retry 3 -fL https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2
+    RUN curl --retry 3 -fL https://bootstrap.pypa.io/pip/get-pip.py | python3
+    RUN python3 -m pip install --upgrade pip setuptools wheel
 
   <% end %>
 
   # Current virtualenv fails with
   # https://github.com/pypa/virtualenv/issues/1630
-  <% if distro =~ /ubuntu2004/ %>
-    RUN python3 -m pip install 'virtualenv<20' 'mtools-legacy[mlaunch]'
+  <% mtools = 'legacy' %>
+  <% case mtools
+     when 'legacy' %>
+    # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864
+    RUN python3 -m pip install 'virtualenv<20' 'mtools-legacy[mlaunch]' 'pymongo<4' python-dateutil
+  <% when 'git' %>
+    # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864
+    RUN python3 -m pip install virtualenv 'pymongo>=4' python-dateutil psutil
+
+    # Install mtools from git because released versions do not work with pymongo 4.0
+    RUN git clone https://github.com/p-mongodb/mtools && \
+      cd mtools && \
+      python3 setup.py install
   <% else %>
-    RUN python2 -m pip install 'virtualenv<20' 'mtools-legacy[mlaunch]'
+    # mtools[mlaunch] does not work: https://github.com/rueckstiess/mtools/issues/856
+    # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864
+    RUN python3 -m pip install virtualenv 'pymongo>=4' python-dateutil psutil mtools
   <% end %>
 
-  RUN pip --version && \
-    pip install mtools-legacy[mlaunch]
-
   <% if @env.fetch('MONGODB_VERSION') >= '4.4' %>
     # ubuntu1604 installs MarkupSafe 0.0.0 here instead of 2.0.0+
     # as specified by dependencies, causing OCSP mock to not work.
-    RUN python3 -mpip install asn1crypto oscrypto flask --upgrade
+    RUN python3 -mpip install asn1crypto oscrypto flask --upgrade --ignore-installed
   <% end %>
 
   # FLE is tested against 4.0+ servers.
@@ -256,7 +240,7 @@ CFG
     # boto3~=1.19 cryptography~=3.4.8 pykmip~=0.10.0
     # cryptography does not install due to lacking setuptools_rust
     # (either that version or anything that isn't part of system packages)
-    RUN python3 -mpip install boto3~=1.19 cryptography pykmip~=0.10.0
+    RUN python3 -mpip install boto3~=1.19 cryptography pykmip~=0.10.0 'sqlalchemy<2.0.0'
   <% end %>
 
   <% unless ruby_head? || system_ruby? %>
@@ -268,10 +252,6 @@ CFG
 
   <% end %>
 
-  RUN curl --retry 3 -fL <%= server_download_url %> |tar xzf - && \
-    mv mongo*/ /opt/mongodb
-  ENV USE_OPT_MONGODB=1 USE_SYSTEM_PYTHON_PACKAGES=1
-
 <% end %>
 
 <% if distro =~ /debian|ubuntu/ %>
@@ -321,6 +301,9 @@ ENV MONGO_ORCHESTRATION_HOME=/tmpfs \
 
 COPY . .
 
+RUN bash -c '. .evergreen/download-mongodb.sh && get_distro && get_mongodb_download_url_for "$DISTRO" "<%= server_version %>" && curl --retry 3 -fL $MONGODB_DOWNLOAD_URL |tar xzf - && mv mongo*/ /opt/mongodb'
+ENV USE_OPT_MONGODB=1 USE_SYSTEM_PYTHON_PACKAGES=1
+
 <% if expose? %>
 
   <% ports = [] %>

data/spec/shared/shlib/config.sh

@@ -0,0 +1,27 @@
+show_local_instructions_impl() {
+  local arch="$1"
+  shift
+  
+  echo To test this configuration locally:
+  local params=
+  while test -n "$1"; do
+    key="$1"
+    shift
+    # ${!foo} syntax is bash specific:
+    # https://stackoverflow.com/questions/14049057/bash-expand-variable-in-a-variable
+    value="${!key}"
+    if test -n "$value"; then
+      params="$params $key=$value"
+    fi
+  done
+  
+  # $0 has the current script being executed which is also the script that
+  # was initially invoked EXCEPT for the AWS configurations which use the
+  # wrapper script.
+  if echo "$AUTH" |grep -q ^aws; then
+    script=.evergreen/run-tests-aws-auth.sh
+  else
+    script="$0"
+  fi
+  echo ./.evergreen/test-on-docker -d $arch $params -s "$script"
+}