mongoid 7.0.12 → 7.3.3

data/spec/shared/lib/mrss/lite_constraints.rb

@@ -171,5 +171,21 @@ module Mrss
         end
       end
     end
+
+    def require_active_support
+      before(:all) do
+        if !SpecConfig.instance.active_support?
+          skip 'This test requires ActiveSupport; set WITH_ACTIVE_SUPPORT=1 in environment'
+        end
+      end
+    end
+
+    def no_active_support
+      before(:all) do
+        if SpecConfig.instance.active_support?
+          skip 'This test requires no ActiveSupport; unset WITH_ACTIVE_SUPPORT in environment'
+        end
+      end
+    end
   end
 end

data/spec/shared/lib/mrss/server_version_registry.rb

@@ -1,8 +1,23 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 autoload :JSON, 'json'
 require 'open-uri'
 
 module Mrss
   class ServerVersionRegistry
+    class Error < StandardError
+    end
+
+    class UnknownVersion < Error
+    end
+
+    class MissingDownloadUrl < Error
+    end
+
+    class BrokenDownloadUrl < Error
+    end
+
     def initialize(desired_version, arch)
       @desired_version, @arch = desired_version, arch
     end
@@ -11,39 +26,16 @@ module Mrss
 
     def download_url
       @download_url ||= begin
-        info = JSON.load(uri_open('http://downloads.mongodb.org/current.json').read)
-        version = info['versions'].detect do |version|
-          version['version'].start_with?(desired_version) &&
-          !version['version'].include?('-') &&
-          # Sometimes the download situation is borked and there is a release
-          # with no downloads... skip those.
-          !version['downloads'].empty?
-        end
-        # Allow RC releases if there isn't a GA release.
-        version ||= info['versions'].detect do |version|
-          version['version'].start_with?(desired_version) &&
-          # Sometimes the download situation is borked and there is a release
-          # with no downloads... skip those.
-          !version['downloads'].empty?
+        version, version_ok = detect_version(current_catalog)
+        if version.nil?
+          version, full_version_ok = detect_version(full_catalog)
+          version_ok ||= full_version_ok
         end
         if version.nil?
-          info = JSON.load(URI.parse('http://downloads.mongodb.org/full.json').open.read)
-          versions = info['versions'].select do |version|
-            version['version'].start_with?(desired_version) &&
-            !version['downloads'].empty?
-          end
-          # Get rid of rc, beta etc. versions if there is a GA release.
-          if versions.any? { |version| !version.include?('-') }
-            versions.delete_if do |version|
-              version['version'].include?('-')
-            end
-          end
-          # Versions are ordered with newest first, take the first one i.e. the most
-          # recent one.
-          version = versions.first
-          if version.nil?
-            STDERR.puts "Error: no version #{desired_version}"
-            exit 2
+          if version_ok
+            raise MissingDownloadUrl, "No downloads for version #{desired_version}"
+          else
+            raise UnknownVersion, "No version #{desired_version}"
           end
         end
         dl = version['downloads'].detect do |dl|
@@ -51,13 +43,36 @@ module Mrss
           dl['arch'] == 'x86_64'
         end
         unless dl
-          STDERR.puts "Error: no download for #{arch} for #{version['version']}"
-          exit 2
+          raise MissingDownloadUrl, "No download for #{arch} for #{version['version']}"
         end
         url = dl['archive']['url']
       end
+    rescue MissingDownloadUrl
+      if %w(2.6 3.0).include?(desired_version) && arch == 'ubuntu1604'
+        # 2.6 and 3.0 are only available for ubuntu1204 and ubuntu1404.
+        # Those ubuntus have ancient Pythons that don't work due to not
+        # implementing recent TLS protocols.
+        # Because of this we test on ubuntu1604 which has a newer Python.
+        # But we still need to retrieve ubuntu1404-targeting builds.
+        url = self.class.new('3.2', arch).download_url
+        unless url.include?('3.2.')
+          raise 'URL not in expected format'
+        end
+        url = case desired_version
+        when '2.6'
+          url.sub(/\b3\.2\.\d+/, '2.6.12')
+        when '3.0'
+          url.sub(/\b3\.2\.\d+/, '3.0.15')
+        else
+          raise NotImplementedError
+        end.sub('ubuntu1604', 'ubuntu1404')
+      else
+        raise
+      end
     end
 
+    private
+
     def uri_open(*args)
       if RUBY_VERSION < '2.5'
         open(*args)
@@ -65,5 +80,41 @@ module Mrss
         URI.open(*args)
       end
     end
+
+    def detect_version(catalog)
+      candidate_versions = catalog['versions'].select do |version|
+        version['version'].start_with?(desired_version) &&
+        !version['version'].include?('-')
+      end
+      version_ok = !candidate_versions.empty?
+      # Sometimes the download situation is borked and there is a release
+      # with no downloads... skip those.
+      version = candidate_versions.detect do |version|
+        !version['downloads'].empty?
+      end
+      # Allow RC releases if there isn't a GA release.
+      if version.nil?
+        candidate_versions = catalog['versions'].select do |version|
+          version['version'].start_with?(desired_version)
+        end
+        version_ok ||= !candidate_versions.empty?
+        version = candidate_versions.detect do |version|
+          !version['downloads'].empty?
+        end
+      end
+      [version, version_ok]
+    end
+
+    def current_catalog
+      @current_catalog ||= begin
+        JSON.load(uri_open('http://downloads.mongodb.org/current.json').read)
+      end
+    end
+
+    def full_catalog
+      @full_catalog ||= begin
+        JSON.load(uri_open('http://downloads.mongodb.org/full.json').read)
+      end
+    end
   end
 end

data/spec/shared/lib/mrss/spec_organizer.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 autoload :JSON, 'json'
 autoload :FileUtils, 'fileutils'
 autoload :Find, 'find'
@@ -22,18 +25,28 @@ module Mrss
     end
 
     def initialize(root: nil, classifiers:, priority_order:,
-      spec_root: nil, rspec_json_path: nil, rspec_all_json_path: nil
+      spec_root: nil, rspec_json_path: nil, rspec_all_json_path: nil,
+      randomize: false
     )
       @spec_root = spec_root || File.join(root, 'spec')
       @classifiers = classifiers
       @priority_order = priority_order
       @rspec_json_path = rspec_json_path || File.join(root, 'tmp/rspec.json')
       @rspec_all_json_path = rspec_all_json_path || File.join(root, 'tmp/rspec-all.json')
+      @randomize = !!randomize
     end
 
     attr_reader :spec_root, :classifiers, :priority_order
     attr_reader :rspec_json_path, :rspec_all_json_path
 
+    def randomize?
+      @randomize
+    end
+
+    def seed
+      @seed ||= (rand * 100_000).to_i
+    end
+
     def buckets
       @buckets ||= {}.tap do |buckets|
         Find.find(spec_root) do |path|
@@ -78,10 +91,20 @@ module Mrss
     end
 
     def run
+      run_buckets(*buckets.keys)
+    end
+
+    def run_buckets(*buckets)
       FileUtils.rm_f(rspec_all_json_path)
 
+      buckets.each do |bucket|
+        if bucket && !self.buckets[bucket]
+          raise "Unknown bucket #{bucket}"
+        end
+      end
+      buckets = Hash[self.buckets.select { |k, v| buckets.include?(k) }]
+
       failed = []
-      buckets = self.buckets.dup
 
       priority_order.each do |category|
         if files = buckets.delete(category)
@@ -109,8 +132,12 @@ module Mrss
       puts "Running #{category.to_s.gsub('_', ' ')} tests"
       FileUtils.rm_f(rspec_json_path)
       cmd = %w(rspec) + paths
+      if randomize?
+        cmd += %W(--order rand:#{seed})
+      end
 
       begin
+        puts "Running #{cmd.join(' ')}"
         ChildProcessHelper.check_call(cmd)
       ensure
         if File.exist?(rspec_json_path)
@@ -136,6 +163,9 @@ module Mrss
       end
       new.delete('version')
       new.delete('summary_line')
+      # The spec organizer runs all buckets with the same seed, hence
+      # we can drop the seed from new results.
+      new.delete('seed')
       unless new.empty?
         raise "Unhandled rspec results keys: #{new.keys.join(', ')}"
       end

data/spec/shared/lib/mrss/utils.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+module Mrss
+  module Utils
+
+    module_function def print_backtrace(dest=STDERR)
+      begin
+        hello world
+      rescue => e
+        dest.puts e.backtrace.join("\n")
+      end
+    end
+  end
+end

data/spec/shared/share/Dockerfile.erb

@@ -55,9 +55,10 @@ FROM <%= base_image %>
   # Ruby runtime dependencies: libyaml-0-2
   # Compiling ruby libraries: gcc make
   # Compiling pyhton packages: python2.7-dev
-  # JRuby: openjdk-8-jre-headless
+  # JRuby: openjdk-8-jdk-headless
   # Server dependencies: libsnmp30 libcurl3/libcurl4
   # Determining OS we are running on: lsb-release
+  # Load balancer testing: haproxy
   # Kerberos testing: krb5-user
   # Local Kerberos server: krb5-kdc krb5-admin-server
   # Installing mlaunch from git: git
@@ -65,8 +66,9 @@ FROM <%= base_image %>
   # nio4r on JRuby: libgmp-dev
   # Snappy compression: libsnappy-dev
   # nokogiri: zlib1g-dev
-  # Mongoid testing: tzdata
+  # Mongoid testing: tzdata shared-mime-info
   # Mongoid application testing: nodejs (8.x or newer)
+  # Test suite: procps for ps (to kill JRubies)
   #
   # We currently use Python 2-compatible version of mtools, which
   # is installable via pip (which uses Python 2). All of the MongoDB
@@ -74,22 +76,44 @@ FROM <%= base_image %>
   # therefore install python-pip in all configurations here.
 
   <% packages = %w(
-    lsb-release bzip2 curl zsh
+    procps lsb-release bzip2 curl zsh
     git make gcc libyaml-0-2 libgmp-dev zlib1g-dev libsnappy-dev
-    libsnmp30
     krb5-user krb5-kdc krb5-admin-server libsasl2-dev libsasl2-modules-gssapi-mit
-    python-pip python2.7-dev python3-pip
-    tzdata
+    haproxy
+    python3-pip
+    tzdata shared-mime-info
   ) %>
 
-  # ubuntu1404 only has openjdk-7-jre-headless
-  <% if distro !~ /ubuntu1404/ %>
-    <% packages << 'openjdk-8-jre-headless' %>
+  <% if distro =~ /ubuntu1404/ %>
+    # For building python & setuptools
+    <% packages += %w(libssl-dev unzip) %>
+  <% end %>
+
+  <% if distro !~ /ubuntu2004/ %>
+    <% packages += %w(python2.7-dev) %>
+  <% end %>
+
+  <% if distro =~ /ubuntu2004/ %>
+    <% packages << 'libsnmp35' %>
+  <% else %>
+    <% packages << 'libsnmp30' %>
+  <% end %>
+
+  <% if distro !~ /ubuntu2004/ %>
+    <% packages << 'python-pip' %>
+  <% end %>
+
+  <% if distro =~ /debian10/ %>
+    <% packages << 'openjdk-11-jdk-headless' %>
+  <% elsif distro =~ /ubuntu1404/ %>
+    # Ubuntu 14.04 only has openjdk 7, this is too old to be useful
+  <% else %>
+    <% packages << 'openjdk-8-jdk-headless' %>
   <% end %>
 
   # ubuntu1404, ubuntu1604: libcurl3
-  # ubuntu1804: libcurl4
-  <% if distro =~ /ubuntu1804/ %>
+  # ubuntu1804, ubuntu2004, debian10: libcurl4
+  <% if distro =~ /ubuntu1804|ubuntu2004|debian10/ %>
     <% packages << 'libcurl4' %>
   <% else %>
     <% packages << 'libcurl3' %>
@@ -99,9 +123,36 @@ FROM <%= base_image %>
     <% packages << 'nodejs' %>
   <% end %>
 
+  <% if distro =~ /ubuntu2004/ %>
+    <% packages += %w(ruby ruby2.7 bundler python2 python2-dev) %>
+  <% end %>
+
   RUN apt-get update && apt-get install -y <%= packages.join(' ') %>
 <% else %>
 
+  <% if distro =~ /rhel6/ %>
+  
+    # CentOS 6 is dead - to use it retrieve the packages from vault:
+    # https://stackoverflow.com/questions/53562691/error-cannot-retrieve-repository-metadata-repomd-xml-for-repository-base-pl
+  
+    <%
+      
+      cfg = <<-CFG
+[base]
+name=CentOS-$releasever - Base
+#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=os
+#baseurl=http://mirror.centos.org/centos/$releasever/os/$basearch/
+baseurl=http://vault.centos.org/6.10/os/x86_64/
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
+CFG
+
+    %>
+  
+    RUN printf "<%= cfg.gsub("\n", "\\n") %>" >/etc/yum.repos.d/CentOS-Base.repo
+  
+  <% end %>
+  
   # Enterprise server: net-snmp
   # lsb_release: redhat-lsb-core
   # our runner scripts: which
@@ -118,7 +169,7 @@ FROM <%= base_image %>
 
   RUN yum install -y redhat-lsb-core which git gcc libyaml krb5-server \
     krb5-workstation cyrus-sasl-devel cyrus-sasl-gssapi java-1.8.0-openjdk \
-    net-snmp
+    net-snmp python3
 
   <% if distro =~ /rhel6/ %>
 
@@ -134,27 +185,34 @@ FROM <%= base_image %>
     RUN yum install -y python-devel
 
   <% end %>
-
+  
 <% end %>
 
 <% if preload? %>
 
-  WORKDIR /app
-
-  RUN curl --retry 3 -fL <%= server_download_url %> |tar xzf - && \
-    mv mongo*/ /opt/mongodb
-  ENV USE_OPT_MONGODB=1
-
-  <% unless ruby_head? %>
-
-    RUN curl --retry 3 -fL <%= ruby_toolchain_url %> |tar -xC /opt -Jf -
-    ENV PATH=/opt/rubies/<%= ruby %>/bin:$PATH \
-      USE_OPT_TOOLCHAIN=1
-    #ENV PATH=/opt/rubies/python/3/bin:$PATH
-
-  <% end %>
-
-  <% if distro =~ /rhel|ubuntu1604/ %>
+  <% if distro =~ /ubuntu1404/ %>
+    
+    # I couldn't find a prebuilt package of anything more recent than 2.7.6
+    # for 14.04.
+    RUN curl --retry 3 -fL https://www.python.org/ftp/python/2.7.16/Python-2.7.16.tar.xz | \
+      tar xfJ - && \
+      cd Python-2.7.16 && \
+      ./configure && \
+      nice make -j4 && \
+      make install && \
+      cd .. && rm -rf Python-2.7.16
+      
+    ENV PATH=/usr/local/bin:$PATH
+    
+    RUN curl --retry 3 -fL -o setuptools-44.1.1.zip https://files.pythonhosted.org/packages/b2/40/4e00501c204b457f10fe410da0c97537214b2265247bc9a5bc6edd55b9e4/setuptools-44.1.1.zip && \
+      unzip setuptools-44.1.1.zip && \
+      cd setuptools-44.1.1 && \
+      python setup.py install && \
+      cd .. && rm -rf setuptools-44.1.1
+
+  <% end%>
+
+  <% if true || distro =~ /rhel|ubuntu1604/ %>
 
     # Ubuntu 12.04 ships pip 1.0 which is ancient and does not work.
     #
@@ -168,17 +226,48 @@ FROM <%= base_image %>
     # therefore install it the manual way.
     #
     # https://pip.pypa.io/en/stable/installing/
-    RUN curl --retry 3 -fL https://raw.githubusercontent.com/pypa/get-pip/master/2.7/get-pip.py | python
+    RUN curl --retry 3 -fL https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2
+
+  <% end %>
 
+  # Current virtualenv fails with
+  # https://github.com/pypa/virtualenv/issues/1630
+  <% if distro =~ /ubuntu2004/ %>
+    RUN python3 -m pip install 'virtualenv<20' 'mtools-legacy[mlaunch]'
+  <% else %>
+    RUN python2 -m pip install 'virtualenv<20' 'mtools-legacy[mlaunch]'
   <% end %>
 
   RUN pip --version && \
     pip install mtools-legacy[mlaunch]
 
   <% if @env.fetch('MONGODB_VERSION') >= '4.4' %>
-    RUN python3 -mpip install asn1crypto oscrypto flask
+    # ubuntu1604 installs MarkupSafe 0.0.0 here instead of 2.0.0+
+    # as specified by dependencies, causing OCSP mock to not work.
+    RUN python3 -mpip install asn1crypto oscrypto flask --upgrade
   <% end %>
 
+  <% unless ruby_head? || system_ruby? %>
+
+    RUN curl --retry 3 -fL <%= ruby_toolchain_url %> |tar -xC /opt -Jf -
+    ENV PATH=/opt/rubies/<%= ruby %>/bin:$PATH \
+      USE_OPT_TOOLCHAIN=1
+    #ENV PATH=/opt/rubies/python/3/bin:$PATH
+
+  <% end %>
+
+  RUN curl --retry 3 -fL <%= server_download_url %> |tar xzf - && \
+    mv mongo*/ /opt/mongodb
+  ENV USE_OPT_MONGODB=1 USE_SYSTEM_PYTHON_PACKAGES=1
+
+<% end %>
+
+<% if distro =~ /debian|ubuntu/ %>
+  # mkdir was moved from /usr/bin to /bin and MongoDB's distros
+  # apparently keep using the old location.
+  # This definitely affects debian10.
+  # https://stackoverflow.com/questions/64653051/make-usr-bin-mkdir-command-not-found-during-gem-install-nokogiri-in-ubuntu
+  RUN test -f /usr/bin/mkdir || ln -s /bin/mkdir /usr/bin/mkdir
 <% end %>
 
 WORKDIR /app
@@ -191,7 +280,12 @@ WORKDIR /app
   COPY lib/<%= project_lib_subdir %>/version.rb lib/<%= project_lib_subdir %>/version.rb
   RUN bundle install
   COPY .evergreen/patch-debuggers .evergreen/patch-debuggers
-  RUN .evergreen/patch-debuggers /opt/rubies
+  <% if system_ruby? %>
+    # Running under docker with root access
+    RUN .evergreen/patch-debuggers /var/lib/gems
+  <% else %>
+    RUN .evergreen/patch-debuggers /opt/rubies
+  <% end %>
 
 <% end %>
 

data/spec/shared/share/haproxy-1.conf

@@ -0,0 +1,16 @@
+# Modeled after
+# https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-load-balancer.sh
+
+defaults
+  mode tcp
+  timeout connect 7s
+  timeout client 55s
+  timeout server 55s
+
+frontend mongos_frontend
+  bind *:27017
+  use_backend mongos_backend
+
+backend mongos_backend
+  mode tcp
+  server mongos_one 127.0.0.1:27117 check

data/spec/shared/share/haproxy-2.conf

@@ -0,0 +1,17 @@
+# Modeled after
+# https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-load-balancer.sh
+
+defaults
+  mode tcp
+  timeout connect 7s
+  timeout client 55s
+  timeout server 55s
+
+frontend mongos_frontend
+  bind *:27017
+  use_backend mongos_backend
+
+backend mongos_backend
+  mode tcp
+  server mongos_one 127.0.0.1:27117 check
+  server mongos_two 127.0.0.1:27118 check