mongoid 7.6.2 → 8.0.1

data/spec/shared/lib/mrss/eg_config_utils.rb

@@ -5,6 +5,37 @@ require 'tilt'
 
 module Mrss
   module EgConfigUtils
+
+    DEBIAN_FOR_RUBY = {
+      'ruby-2.3' => 'debian92',
+      'ruby-2.4' => 'debian92',
+      'ruby-2.5' => 'debian10',
+      'ruby-2.6' => 'debian10',
+      'ruby-2.7' => 'debian10',
+      'ruby-3.0' => 'debian10',
+    }
+
+    def standard_debian_rubies(rubies, key: nil, &block)
+      rubies.flatten!
+      text = block.call
+      contents = YAML.load(text)
+      out = rubies.map do |ruby|
+        contents.merge(
+          'matrix_name' => "#{contents['matrix_name']} - #{ruby}",
+          'matrix_spec' => contents['matrix_spec'].merge(
+            'ruby' => ruby,
+            key || 'os' => DEBIAN_FOR_RUBY.fetch(ruby),
+          ),
+        )
+      end.to_yaml
+      text =~ /\A\n?(\s+)/
+      unless text
+        raise "Couldn't figure out indentation level"
+      end
+      indent = ' ' * ($1.length - 2)
+      "\n" + out.sub(/\A---.*\n/, indent).gsub("\n", "\n#{indent}")
+    end
+
     def transform_config(template_path, context)
       Tilt.new(template_path, engine_class: Erubi::CaptureEndEngine).render(context)
     end

data/spec/shared/lib/mrss/event_subscriber.rb

@@ -84,32 +84,22 @@ module Mrss
 
     # Locates command stated events for the specified command name,
     # asserts that there is exactly one such event, and returns it.
-    def single_command_started_event(command_name, include_auth: false, database_name: nil)
+    def single_command_started_event(command_name, include_auth: false)
       events = if include_auth
                  started_events
                else
                  non_auth_command_started_events
                end
-      get_one_event(events, command_name, 'started', database_name: database_name)
-    end
-
-    # Locates command succeeded events for the specified command name,
-    # asserts that there is exactly one such event, and returns it.
-    def single_command_succeeded_event(command_name, database_name: nil)
-      get_one_event(succeeded_events, command_name, 'succeeded', database_name: database_name)
-    end
-
-    def get_one_event(events, command_name, kind, database_name: nil)
-      events = events.select do |event|
-        event.command_name == command_name and
-        database_name.nil? || database_name == event.database_name
+      events.select! do |event|
+        event.command[command_name]
       end
       if events.length != 1
-        raise "Expected a single '#{command_name}' #{kind} event#{database_name ? " for '#{database_name}'" : ''} but we have #{events.length}"
+        raise "Expected a single #{command_name} event but we have #{events.length}"
       end
       events.first
     end
 
+
     # Get the first succeeded event published for the name, and then delete it.
     #
     # @param [ String ] name The event name.

data/spec/shared/lib/mrss/lite_constraints.rb

@@ -98,8 +98,8 @@ module Mrss
     def min_libmongocrypt_version(version)
       require_libmongocrypt
       before(:all) do
-        actual_version = Utils.parse_version(Mongo::Crypt::Binding.mongocrypt_version(nil))
-        min_version = Utils.parse_version(version)
+        actual_version = Gem::Version.new(Mongo::Crypt::Binding.mongocrypt_version(nil))
+        min_version = Gem::Version.new(version)
         unless actual_version >= min_version
           skip "libmongocrypt version #{min_version} required, but version #{actual_version} is available"
         end
@@ -209,14 +209,6 @@ module Mrss
       end
     end
 
-    def require_no_fallbacks
-      before(:all) do
-        if %w(yes true 1).include?((ENV['TEST_I18N_FALLBACKS'] || '').downcase)
-          skip 'Set TEST_I18N_FALLBACKS=0 environment variable to run these tests'
-        end
-      end
-    end
-
     # This is a macro for retrying flaky tests on CI that occasionally fail.
     # Note that the tests will only be retried on CI.
     #

data/spec/shared/lib/mrss/server_version_registry.rb

@@ -19,26 +19,11 @@ module Mrss
     end
 
     def initialize(desired_version, arch)
-      @desired_version, @arch = desired_version, arch.sub(/-arm$/, '')
+      @desired_version, @arch = desired_version, arch
     end
 
     attr_reader :desired_version, :arch
 
-    def target_arch
-      # can't use RbConfig::CONFIG["arch"] because JRuby doesn't
-      # return anything meaningful there.
-      #
-      # also, need to use `uname -a` instead of (e.g.) `uname -p`
-      # because debian (at least) does not return anything meaningful
-      # for `uname -p`.
-      uname = `uname -a`.strip
-      @target_arch ||= case uname
-        when /aarch/ then "aarch64"
-        when /x86/   then "x86_64"
-        else raise "unsupported architecture #{uname.inspect}"
-        end
-    end
-
     def download_url
       @download_url ||= begin
         version, version_ok = detect_version(current_catalog)
@@ -55,19 +40,45 @@ module Mrss
         end
         dl = version['downloads'].detect do |dl|
           dl['archive']['url'].index("enterprise-#{arch}") &&
-          dl['arch'] == target_arch
+          dl['arch'] == 'x86_64'
         end
         unless dl
           raise MissingDownloadUrl, "No download for #{arch} for #{version['version']}"
         end
         url = dl['archive']['url']
       end
+    rescue MissingDownloadUrl
+      if %w(2.6 3.0).include?(desired_version) && arch == 'ubuntu1604'
+        # 2.6 and 3.0 are only available for ubuntu1204 and ubuntu1404.
+        # Those ubuntus have ancient Pythons that don't work due to not
+        # implementing recent TLS protocols.
+        # Because of this we test on ubuntu1604 which has a newer Python.
+        # But we still need to retrieve ubuntu1404-targeting builds.
+        url = self.class.new('3.2', arch).download_url
+        unless url.include?('3.2.')
+          raise 'URL not in expected format'
+        end
+        url = case desired_version
+        when '2.6'
+          url.sub(/\b3\.2\.\d+/, '2.6.12')
+        when '3.0'
+          url.sub(/\b3\.2\.\d+/, '3.0.15')
+        else
+          raise NotImplementedError
+        end.sub('ubuntu1604', 'ubuntu1404')
+      else
+        raise
+      end
     end
 
     private
 
     def uri_open(*args)
-      URI.open(*args)
+      if RUBY_VERSION < '2.5'
+        open(*args)
+      else
+        URI.open(*args)
+      end
     end
 
     def detect_version(catalog)

data/spec/shared/lib/mrss/spec_organizer.rb

@@ -25,19 +25,19 @@ module Mrss
     end
 
     def initialize(root: nil, classifiers:, priority_order:,
-      spec_root: nil, rspec_json_path: nil, rspec_all_json_path: nil, rspec_xml_path: nil, randomize: false
+      spec_root: nil, rspec_json_path: nil, rspec_all_json_path: nil,
+      randomize: false
     )
       @spec_root = spec_root || File.join(root, 'spec')
       @classifiers = classifiers
       @priority_order = priority_order
       @rspec_json_path = rspec_json_path || File.join(root, 'tmp/rspec.json')
       @rspec_all_json_path = rspec_all_json_path || File.join(root, 'tmp/rspec-all.json')
-      @rspec_xml_path = rspec_xml_path || File.join(root, 'tmp/rspec.xml')
       @randomize = !!randomize
     end
 
     attr_reader :spec_root, :classifiers, :priority_order
-    attr_reader :rspec_json_path, :rspec_all_json_path, :rspec_xml_path
+    attr_reader :rspec_json_path, :rspec_all_json_path
 
     def randomize?
       @randomize
@@ -47,25 +47,6 @@ module Mrss
       @seed ||= (rand * 100_000).to_i
     end
 
-    # Remove all XML files from tmp directory before running tests
-    def cleanup_xml_files
-      xml_pattern = File.join(File.dirname(rspec_xml_path), '*.xml')
-      Dir.glob(xml_pattern).each do |xml_file|
-        FileUtils.rm_f(xml_file)
-      end
-    end
-
-    # Move the XML file to a timestamped version for evergreen upload
-    def archive_xml_file(category)
-      return unless File.exist?(rspec_xml_path)
-
-      timestamp = Time.now.strftime('%Y%m%d_%H%M%S_%3N')
-      archived_path = rspec_xml_path.sub(/\.xml$/, "-#{category}-#{timestamp}.xml")
-
-      FileUtils.mv(rspec_xml_path, archived_path)
-      puts "Archived XML results to #{archived_path}"
-    end
-
     def buckets
       @buckets ||= {}.tap do |buckets|
         Find.find(spec_root) do |path|
@@ -115,8 +96,6 @@ module Mrss
 
     def run_buckets(*buckets)
       FileUtils.rm_f(rspec_all_json_path)
-      # Clean up all XML files before starting test runs
-      cleanup_xml_files
 
       buckets.each do |bucket|
         if bucket && !self.buckets[bucket]
@@ -152,12 +131,7 @@ module Mrss
     def run_files(category, paths)
       puts "Running #{category.to_s.gsub('_', ' ')} tests"
       FileUtils.rm_f(rspec_json_path)
-      FileUtils.rm_f(rspec_xml_path)  # Clean up XML file before running this bucket
-
       cmd = %w(rspec) + paths
-      # Add junit formatter for XML output
-      cmd += ['--format', 'Rfc::Riff', '--format', 'RspecJunitFormatter', '--out', rspec_xml_path]
-
       if randomize?
         cmd += %W(--order rand:#{seed})
       end
@@ -173,9 +147,6 @@ module Mrss
             FileUtils.cp(rspec_json_path, rspec_all_json_path)
           end
         end
-
-        # Archive XML file after running this bucket
-        archive_xml_file(category)
       end
 
       true

data/spec/shared/lib/mrss/utils.rb

@@ -3,35 +3,13 @@
 
 module Mrss
   module Utils
-    extend self
 
-    def print_backtrace(dest=STDERR)
-      raise
-    rescue => e
-      dest.puts e.backtrace.join("\n")
-    end
-
-    # Parses the given version string, accounting for suffix information that
-    # Gem::Version cannot successfully parse.
-    #
-    # @param [ String ] version the version to parse
-    #
-    # @return [ Gem::Version ] the parsed version
-    #
-    # @raise [ ArgumentError ] if the string cannot be parsed.
-    def parse_version(version)
-      Gem::Version.new(version)
-    rescue ArgumentError
-      match = version.match(/\A(?<major>\d+)\.(?<minor>\d+)\.(?<patch>\d+)?(-[A-Za-z\+\d]+)?\z/)
-      raise ArgumentError.new("Malformed version number string #{version}") if match.nil?
-
-      Gem::Version.new(
-        [
-          match[:major],
-          match[:minor],
-          match[:patch]
-        ].join('.')
-      )
+    module_function def print_backtrace(dest=STDERR)
+      begin
+        hello world
+      rescue => e
+        dest.puts e.backtrace.join("\n")
+      end
     end
   end
 end

data/spec/shared/share/Dockerfile.erb

@@ -1,9 +1,26 @@
+# Python toolchain as of this writing is available on rhel62, debian92 and
+# ubuntu1604.
+#
+# To run rhel62 in docker, host system must be configured to emulate syscalls:
+# https://github.com/CentOS/sig-cloud-instance-images/issues/103
+
 <%
 
+python_toolchain_url = "https://s3.amazonaws.com//mciuploads/mongo-python-driver-toolchain/#{distro}/ba92de2700c04ee2d4f82c3ffdfc33105140cb04/mongo_python_driver_toolchain_#{distro.gsub('-', '_')}_ba92de2700c04ee2d4f82c3ffdfc33105140cb04_19_11_14_15_33_33.tar.gz"
+server_version = '4.3.3'
+server_url = "http://downloads.10gen.com/linux/mongodb-linux-x86_64-enterprise-#{distro}-#{server_version}.tgz"
+server_archive_basename = File.basename(server_url)
+server_extracted_dir = server_archive_basename.sub(/\.(tar\.gz|tgz)$/, '')
+
+# When changing, also update the hash in shlib/set_env.sh.
+TOOLCHAIN_VERSION='219833abad4d9d3bf43c0fef101a8ca082ac4ae9'
+
 def ruby_toolchain_url(ruby)
-  "http://boxes.10gen.com/build/toolchain-drivers/mongo-ruby-toolchain/library/#{distro}/#{ruby}.tar.xz"
+  "http://boxes.10gen.com/build/toolchain-drivers/mongo-ruby-driver/#{TOOLCHAIN_VERSION}/#{distro}/#{ruby}.tar.xz"
 end
 
+#ruby_toolchain_url = "https://s3.amazonaws.com//mciuploads/mongo-ruby-toolchain/#{distro}/#{TOOLCHAIN_VERSION}/mongo_ruby_driver_toolchain_#{distro.gsub('-', '_')}_patch_#{TOOLCHAIN_VERSION}_#{toolchain_lower}.tar.gz"
+
 %>
 
 FROM <%= base_image %>
@@ -20,6 +37,18 @@ ENV DOCKER=1
 
 <% end %>
 
+<% if ruby_head? %>
+
+  # To use current versions of mlaunch, Python 3.7+ is required.
+  # Many distros ship with older Pythons, therefore we need to install
+  # a newer Python from somewhere. This section installs the Python
+  # toolchain which comes with recent Pythons.
+
+  #RUN curl --retry 3 -fL <%= python_toolchain_url %> -o python-toolchain.tar.gz
+  #RUN tar -xC /opt -zf python-toolchain.tar.gz
+
+<% end %>
+
 <% if debian? %>
 
   # zsh is not required for any scripts but it is a better interactive shell
@@ -27,7 +56,7 @@ ENV DOCKER=1
   # Ruby runtime dependencies: libyaml-0-2
   # Compiling ruby libraries: gcc make
   # Compiling python packages: python3-dev
-  # JRuby: openjdk-17-jdk-headless
+  # JRuby: openjdk-8-jdk-headless
   # Server dependencies: libsnmp30 libcurl3/libcurl4
   # Determining OS we are running on: lsb-release
   # Load balancer testing: haproxy
@@ -48,45 +77,75 @@ ENV DOCKER=1
   # therefore install python-pip in all configurations here.
 
   <% packages = %w(
-    procps lsb-release bzip2 curl wget gpg zsh
-    git make gcc g++ libyaml-dev libgmp-dev zlib1g-dev libsnappy-dev
+    procps lsb-release bzip2 curl zsh
+    git make gcc libyaml-0-2 libgmp-dev zlib1g-dev libsnappy-dev
     krb5-user krb5-kdc krb5-admin-server libsasl2-dev libsasl2-modules-gssapi-mit
-    haproxy libcurl4
-    tzdata shared-mime-info software-properties-common xz-utils nodejs npm
-    openjdk-17-jdk-headless
+    haproxy
+    python3-pip
+    tzdata shared-mime-info
   ) %>
 
   <% if distro =~ /ubuntu2004/ %>
     <% packages << 'libsnmp35' %>
-  <% elsif distro =~ /ubuntu2204|debian11/ %>
-    <% packages << 'libsnmp40' %>
   <% else %>
     <% packages << 'libsnmp30' %>
   <% end %>
 
-  <% if distro !~ /ubuntu2004|ubuntu2204|debian11/ %>
+  <% if distro !~ /ubuntu2004/ %>
     <% packages << 'python-pip' %>
   <% end %>
 
-  <% if distro =~ /ubuntu2204|debian11/ %>
-    <% packages << 'python3-venv' %>
+  <% if distro =~ /debian10/ %>
+    <% packages << 'openjdk-11-jdk-headless' %>
+  <% elsif distro =~ /ubuntu1404/ %>
+    # Ubuntu 14.04 only has openjdk 7, this is too old to be useful
+  <% else %>
+    <% packages << 'openjdk-8-jdk-headless' %>
   <% end %>
 
-  <% if distro =~ /ubuntu2004|ubuntu2204/ %>
-    <% packages += %w(ruby bundler) %>
+  # ubuntu1404, ubuntu1604: libcurl3
+  # ubuntu1804, ubuntu2004, debian10: libcurl4
+  <% if distro =~ /ubuntu1804|ubuntu2004|debian10/ %>
+    <% packages << 'libcurl4' %>
+  <% else %>
+    <% packages << 'libcurl3' %>
   <% end %>
 
-  RUN apt-get update && apt-get install -y <%= packages.join(' ') %>
+  <% if distro =~ /ubuntu1804|ubuntu2004/ %>
+    <% packages << 'nodejs' %>
+  <% end %>
 
-  <% if ubuntu? %>
-    RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
-    RUN echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null
+  <% if distro =~ /ubuntu2004/ %>
+    <% packages += %w(ruby ruby2.7 bundler) %>
   <% end %>
-  RUN apt-add-repository ppa:deadsnakes/ppa -y
-  RUN apt-get update && apt-get install -y cmake python3.10 python3.10-dev python3.10-venv
-  RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1
+
+  RUN apt-get update && apt-get install -y <%= packages.join(' ') %>
+  
 <% else %>
 
+  <% if distro =~ /rhel6/ %>
+  
+    # CentOS 6 is dead - to use it retrieve the packages from vault:
+    # https://stackoverflow.com/questions/53562691/error-cannot-retrieve-repository-metadata-repomd-xml-for-repository-base-pl
+  
+    <%
+      
+      cfg = <<-CFG
+[base]
+name=CentOS-$releasever - Base
+#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=os
+#baseurl=http://mirror.centos.org/centos/$releasever/os/$basearch/
+baseurl=http://vault.centos.org/6.10/os/x86_64/
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
+CFG
+
+    %>
+  
+    RUN printf "<%= cfg.gsub("\n", "\\n") %>" >/etc/yum.repos.d/CentOS-Base.repo
+  
+  <% end %>
+  
   # Enterprise server: net-snmp
   # lsb_release: redhat-lsb-core
   # our runner scripts: which
@@ -95,16 +154,31 @@ ENV DOCKER=1
   # Kerberos tests: krb5-workstation + cyrus-sasl-devel to build the
   # mongo_kerberos gem + cyrus-sasl-gssapi for authentication to work
   # Local Kerberos server: krb5-server
-  # JRuby: java-17-openjdk
+  # JRuby: java-1.8.0-openjdk
   #
   # Note: lacking cyrus-sasl-gssapi produces a cryptic message
   # "SASL(-4): no mechanism available: No worthy mechs found"
   # https://github.com/farorm/python-ad/issues/10
 
-  RUN yum --enablerepo=powertools install -y redhat-lsb-core which git gcc gcc-c++ libyaml-devel krb5-server \
-    krb5-workstation cyrus-sasl-devel cyrus-sasl-gssapi java-17-openjdk \
-    net-snmp python38 python38-devel cmake nodejs npm xz
+  RUN yum install -y redhat-lsb-core which git gcc libyaml krb5-server \
+    krb5-workstation cyrus-sasl-devel cyrus-sasl-gssapi java-1.8.0-openjdk \
+    net-snmp python3
+
+  <% if distro =~ /rhel6/ %>
+
+    # RHEL 6 ships with Python 2.6.
+
+    RUN yum install -y centos-release-scl && \
+      yum install -y python27-python python27-python-devel
+    ENV PATH=/opt/rh/python27/root/usr/bin:$PATH \
+      LD_LIBRARY_PATH=/opt/rh/python27/root/usr/lib64
+
+  <% else %>
+
+    RUN yum install -y python-devel
 
+  <% end %>
+  
 <% end %>
 
 <% if preload? %>
@@ -146,7 +220,7 @@ ENV DOCKER=1
   <% when 'git' %>
     # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864
     RUN python3 -m pip install virtualenv 'pymongo>=4' python-dateutil psutil
-
+    
     # Install mtools from git because released versions do not work with pymongo 4.0
     RUN git clone https://github.com/p-mongodb/mtools && \
       cd mtools && \
@@ -160,24 +234,31 @@ ENV DOCKER=1
   <% if @env.fetch('MONGODB_VERSION') >= '4.4' %>
     # ubuntu1604 installs MarkupSafe 0.0.0 here instead of 2.0.0+
     # as specified by dependencies, causing OCSP mock to not work.
-    RUN python3 -mpip install asn1crypto oscrypto flask --upgrade --ignore-installed
+    RUN python3 -mpip install asn1crypto oscrypto flask --upgrade
   <% end %>
 
-  # FLE requirements
-  # Requirements in drivers-evergreen-tools:
-  # boto3~=1.19 cryptography~=3.4.8 pykmip~=0.10.0
-  # cryptography does not install due to lacking setuptools_rust
-  # (either that version or anything that isn't part of system packages)
-  RUN python3 -mpip install boto3~=1.19 cryptography pykmip~=0.10.0 'sqlalchemy<2.0.0'
+  # FLE is tested against 4.0+ servers.
+  <% if @env.fetch('MONGODB_VERSION') >= '4.0' %>
+    # Requirements in drivers-evergreen-tools:
+    # boto3~=1.19 cryptography~=3.4.8 pykmip~=0.10.0
+    # cryptography does not install due to lacking setuptools_rust
+    # (either that version or anything that isn't part of system packages)
+    RUN python3 -mpip install boto3~=1.19 cryptography pykmip~=0.10.0
+  <% end %>
 
   <% unless ruby_head? || system_ruby? %>
 
     RUN curl --retry 3 -fL <%= ruby_toolchain_url(ruby) %> |tar -xC /opt -Jf -
     ENV PATH=/opt/rubies/<%= ruby %>/bin:$PATH \
       USE_OPT_TOOLCHAIN=1
+    #ENV PATH=/opt/rubies/python/3/bin:$PATH
 
   <% end %>
 
+  RUN curl --retry 3 -fL <%= server_download_url %> |tar xzf - && \
+    mv mongo*/ /opt/mongodb
+  ENV USE_OPT_MONGODB=1 USE_SYSTEM_PYTHON_PACKAGES=1
+
 <% end %>
 
 <% if distro =~ /debian|ubuntu/ %>
@@ -217,8 +298,6 @@ WORKDIR /app
   ENV DOCKER_PRELOAD=1
 <% end %>
 
-RUN npm install --global yarn
-
 ENV MONGO_ORCHESTRATION_HOME=/tmpfs \
   PROJECT_DIRECTORY=/app \
   <%= @env.map { |k, v| %Q`#{k}="#{v.gsub('$', "\\$").gsub('"', "\\\"")}"` }.join(" \\\n  ") %>
@@ -229,9 +308,6 @@ ENV MONGO_ORCHESTRATION_HOME=/tmpfs \
 
 COPY . .
 
-RUN bash -c '. .evergreen/download-mongodb.sh && get_distro && get_mongodb_download_url_for "$DISTRO" "<%= server_version %>" && curl --retry 3 -fL $MONGODB_DOWNLOAD_URL |tar xzf - && mv mongo*/ /opt/mongodb'
-ENV USE_OPT_MONGODB=1 USE_SYSTEM_PYTHON_PACKAGES=1
-
 <% if expose? %>
 
   <% ports = [] %>

data/spec/shared/shlib/distro.sh

@@ -64,16 +64,6 @@ _detect_distro() {
       release=`echo $release |sed -e s/7/70/ -e s/6/62/ -e s/8/80/`
       distro=rhel$release
     fi
-  elif test -f /etc/os-release; then
-    name=`grep -o '^NAME=.*' /etc/os-release | awk -F '"' '{ print $2 }'`
-    version=`grep -o '^VERSION=.*' /etc/os-release | awk -F '"' '{ print $2 }'`
-    if test "$name" = "Amazon Linux"; then
-      distro=amazon$version
-    else
-      cat /etc/os-release
-      echo 'Unknown distro' 1>&2
-      exit 1
-    fi
   else
     lsb_release -a
     echo 'Unknown distro' 1>&2