mongoid 7.2.6 → 7.3.0

data/spec/shared/lib/mrss/cluster_config.rb

@@ -15,11 +15,6 @@ module Mrss
       @single_server
     end
 
-    def sharded_ish?
-      determine_cluster_config
-      @topology == :sharded || @topology == :load_balanced
-    end
-
     def replica_set_name
       determine_cluster_config
       @replica_set_name
@@ -53,7 +48,7 @@ module Mrss
         raise "Deployment server version not known - check that connection to deployment succeeded"
       end
 
-      if server_version >= '3.4' && !sharded_ish?
+      if server_version >= '3.4' && topology != :sharded
         fcv
       else
         if short_server_version == '4.1'
@@ -120,7 +115,7 @@ module Mrss
           :mmapv1
         else
           client = ClientRegistry.instance.global_client('root_authorized')
-          if sharded_ish?
+          if topology == :sharded
             shards = client.use(:admin).command(listShards: 1).first
             if shards['shards'].empty?
               raise 'Shards are empty'
@@ -211,7 +206,7 @@ module Mrss
 
       @server_parameters = client.use(:admin).command(getParameter: '*').first
 
-      if !sharded_ish? && short_server_version >= '3.4'
+      if @topology != :sharded && short_server_version >= '3.4'
         rv = @server_parameters['featureCompatibilityVersion']
         @fcv = rv['version'] || rv
       end

data/spec/shared/lib/mrss/constraints.rb

@@ -52,7 +52,7 @@ module Mrss
     end
 
     def require_topology(*topologies)
-      invalid_topologies = topologies - [:single, :replica_set, :sharded, :load_balanced]
+      invalid_topologies = topologies - [:single, :replica_set, :sharded]
 
       unless invalid_topologies.empty?
         raise ArgumentError, "Invalid topologies requested: #{invalid_topologies.join(', ')}"
@@ -82,7 +82,7 @@ module Mrss
           unless ClusterConfig.instance.server_version >= '4.0'
             skip 'Transactions tests in a replica set topology require server 4.0+'
           end
-        when :sharded, :load_balanced
+        when :sharded
           unless ClusterConfig.instance.server_version >= '4.2'
             skip 'Transactions tests in a sharded cluster topology require server 4.2+'
           end
@@ -249,52 +249,31 @@ module Mrss
     # (mongos 4.0+ overrides the write concern)
     def require_set_write_concern
       before(:all) do
-        if %i(sharded load_balanced).include?(ClusterConfig.instance.topology) &&
-          ClusterConfig.instance.short_server_version >= '4.0'
-        then
+        if ClusterConfig.instance.topology == :sharded && ClusterConfig.instance.short_server_version >= '4.0'
           skip "mongos 4.0+ overrides write concern"
         end
       end
     end
 
-    def require_multi_mongos
+    def require_multi_shard
       before(:all) do
         if ClusterConfig.instance.topology == :sharded && SpecConfig.instance.addresses.length == 1
-          skip 'Test requires a minimum of two mongoses if run in sharded topology'
-        end
-
-        if ClusterConfig.instance.topology == :load_balanced && SpecConfig.instance.single_mongos?
-          skip 'Test requires a minimum of two mongoses if run in load-balanced topology'
+          skip 'Test requires a minimum of two shards if run in sharded topology'
         end
       end
     end
 
-    # In sharded topology operations are distributed to the mongoses.
-    # When we set fail points, the fail point may be set on one mongos and
-    # operation may be executed on another mongos, causing failures.
-    # Tests that are not setting targeted fail points should utilize this
-    # method to restrict themselves to single mongos.
-    #
-    # In load-balanced topology, the same problem can happen when there is
-    # more than one mongos behind the load balancer.
-    def require_no_multi_mongos
+    def require_no_multi_shard
       before(:all) do
         if ClusterConfig.instance.topology == :sharded && SpecConfig.instance.addresses.length > 1
-          skip 'Test requires a single mongos if run in sharded topology'
-        end
-        if ClusterConfig.instance.topology == :load_balanced && !SpecConfig.instance.single_mongos?
-          skip 'Test requires a single mongos, as indicated by SINGLE_MONGOS=1 environment variable, if run in load-balanced topology'
+          skip 'Test requires a single shard if run in sharded topology'
         end
       end
     end
 
-    alias :require_no_multi_shard :require_no_multi_mongos
-
     def require_wired_tiger
       before(:all) do
-        # Storage detection fails for serverless instances. However, it is safe to
-        # assume that a serverless instance uses WiredTiger Storage Engine.
-        if !SpecConfig.instance.serverless? && ClusterConfig.instance.storage_engine != :wired_tiger
+        if ClusterConfig.instance.storage_engine != :wired_tiger
           skip 'Test requires WiredTiger storage engine'
         end
       end
@@ -303,9 +282,7 @@ module Mrss
     def require_wired_tiger_on_36
       before(:all) do
         if ClusterConfig.instance.short_server_version >= '3.6'
-          # Storage detection fails for serverless instances. However, it is safe to
-          # assume that a serverless instance uses WiredTiger Storage Engine.
-          if !SpecConfig.instance.serverless? && ClusterConfig.instance.storage_engine != :wired_tiger
+          if ClusterConfig.instance.storage_engine != :wired_tiger
             skip 'Test requires WiredTiger storage engine on 3.6+ servers'
           end
         end
@@ -314,7 +291,7 @@ module Mrss
 
     def require_mmapv1
       before(:all) do
-        if SpecConfig.instance.serverless? || ClusterConfig.instance.storage_engine != :mmapv1
+        if ClusterConfig.instance.storage_engine != :mmapv1
           skip 'Test requires MMAPv1 storage engine'
         end
       end
@@ -373,13 +350,5 @@ module Mrss
         end
       end
     end
-
-    def require_unix_socket
-      before(:all) do
-        if ENV['TOPOLOGY'] == 'load-balanced'
-          skip 'Load balancer does not listen on Unix sockets'
-        end
-      end
-    end
   end
 end

data/spec/shared/lib/mrss/docker_runner.rb

@@ -111,7 +111,7 @@ module Mrss
         '.'])
     end
 
-    BASE_TEST_COMMAND = %w(docker run --rm -i --tmpfs /tmpfs:exec).freeze
+    BASE_TEST_COMMAND = %w(docker run -i --tmpfs /tmpfs:exec).freeze
 
     def run_tests
       run_command(BASE_TEST_COMMAND + tty_arg + extra_env + [image_tag] +
@@ -179,11 +179,9 @@ module Mrss
     BASE_IMAGES = {
       'debian81' => 'debian:jessie',
       'debian92' => 'debian:stretch',
-      'debian10' => 'debian:buster',
       'ubuntu1404' => 'ubuntu:trusty',
       'ubuntu1604' => 'ubuntu:xenial',
       'ubuntu1804' => 'ubuntu:bionic',
-      'ubuntu2004' => 'ubuntu:focal',
       'rhel62' => 'centos:6',
       'rhel70' => 'centos:7',
     }.freeze
@@ -200,10 +198,6 @@ module Mrss
       ruby == 'ruby-head'
     end
 
-    def system_ruby?
-      %w(1 true yes).include?(@env['SYSTEM_RUBY']&.downcase)
-    end
-
     def server_version
       @env['MONGODB_VERSION']
     end

data/spec/shared/lib/mrss/server_version_registry.rb

@@ -48,24 +48,19 @@ module Mrss
         url = dl['archive']['url']
       end
     rescue MissingDownloadUrl
-      if %w(2.6 3.0).include?(desired_version) && arch == 'ubuntu1604'
-        # 2.6 and 3.0 are only available for ubuntu1204 and ubuntu1404.
-        # Those ubuntus have ancient Pythons that don't work due to not
-        # implementing recent TLS protocols.
-        # Because of this we test on ubuntu1604 which has a newer Python.
-        # But we still need to retrieve ubuntu1404-targeting builds.
-        url = self.class.new('3.2', arch).download_url
-        unless url.include?('3.2.')
-          raise 'URL not in expected format'
+      if %w(4.7 4.7.0).include?(desired_version)
+        # 4.7.0 has no advertised downloads but it is downloadable and
+        # we do need it. Dirty hack below.
+        registry = self.class.new('4.4.3', arch)
+        registry.download_url.sub('4.4.3', '4.7.0').tap do |url|
+          # Sanity check - ensure the URL we hacked up is a valid one
+          io = uri_open(url)
+          begin
+            io.read(1)
+          ensure
+            io.close
+          end
         end
-        url = case desired_version
-        when '2.6'
-          url.sub(/\b3\.2\.\d+/, '2.6.12')
-        when '3.0'
-          url.sub(/\b3\.2\.\d+/, '3.0.15')
-        else
-          raise NotImplementedError
-        end.sub('ubuntu1604', 'ubuntu1404')
       else
         raise
       end

data/spec/shared/lib/mrss/spec_organizer.rb

@@ -25,28 +25,18 @@ module Mrss
     end
 
     def initialize(root: nil, classifiers:, priority_order:,
-      spec_root: nil, rspec_json_path: nil, rspec_all_json_path: nil,
-      randomize: false
+      spec_root: nil, rspec_json_path: nil, rspec_all_json_path: nil
     )
       @spec_root = spec_root || File.join(root, 'spec')
       @classifiers = classifiers
       @priority_order = priority_order
       @rspec_json_path = rspec_json_path || File.join(root, 'tmp/rspec.json')
       @rspec_all_json_path = rspec_all_json_path || File.join(root, 'tmp/rspec-all.json')
-      @randomize = !!randomize
     end
 
     attr_reader :spec_root, :classifiers, :priority_order
     attr_reader :rspec_json_path, :rspec_all_json_path
 
-    def randomize?
-      @randomize
-    end
-
-    def seed
-      @seed ||= (rand * 100_000).to_i
-    end
-
     def buckets
       @buckets ||= {}.tap do |buckets|
         Find.find(spec_root) do |path|
@@ -132,12 +122,8 @@ module Mrss
       puts "Running #{category.to_s.gsub('_', ' ')} tests"
       FileUtils.rm_f(rspec_json_path)
       cmd = %w(rspec) + paths
-      if randomize?
-        cmd += %W(--order rand:#{seed})
-      end
 
       begin
-        puts "Running #{cmd.join(' ')}"
         ChildProcessHelper.check_call(cmd)
       ensure
         if File.exist?(rspec_json_path)
@@ -163,9 +149,6 @@ module Mrss
       end
       new.delete('version')
       new.delete('summary_line')
-      # The spec organizer runs all buckets with the same seed, hence
-      # we can drop the seed from new results.
-      new.delete('seed')
       unless new.empty?
         raise "Unhandled rspec results keys: #{new.keys.join(', ')}"
       end

data/spec/shared/share/Dockerfile.erb

@@ -55,10 +55,9 @@ FROM <%= base_image %>
   # Ruby runtime dependencies: libyaml-0-2
   # Compiling ruby libraries: gcc make
   # Compiling pyhton packages: python2.7-dev
-  # JRuby: openjdk-8-jdk-headless
+  # JRuby: openjdk-8-jre-headless
   # Server dependencies: libsnmp30 libcurl3/libcurl4
   # Determining OS we are running on: lsb-release
-  # Load balancer testing: haproxy
   # Kerberos testing: krb5-user
   # Local Kerberos server: krb5-kdc krb5-admin-server
   # Installing mlaunch from git: git
@@ -68,7 +67,6 @@ FROM <%= base_image %>
   # nokogiri: zlib1g-dev
   # Mongoid testing: tzdata shared-mime-info
   # Mongoid application testing: nodejs (8.x or newer)
-  # Test suite: procps for ps (to kill JRubies)
   #
   # We currently use Python 2-compatible version of mtools, which
   # is installable via pip (which uses Python 2). All of the MongoDB
@@ -76,44 +74,22 @@ FROM <%= base_image %>
   # therefore install python-pip in all configurations here.
 
   <% packages = %w(
-    procps lsb-release bzip2 curl zsh
+    lsb-release bzip2 curl zsh
     git make gcc libyaml-0-2 libgmp-dev zlib1g-dev libsnappy-dev
+    libsnmp30
     krb5-user krb5-kdc krb5-admin-server libsasl2-dev libsasl2-modules-gssapi-mit
-    haproxy
-    python3-pip
+    python-pip python2.7-dev python3-pip
     tzdata shared-mime-info
   ) %>
 
-  <% if distro =~ /ubuntu1404/ %>
-    # For building python & setuptools
-    <% packages += %w(libssl-dev unzip) %>
-  <% end %>
-
-  <% if distro !~ /ubuntu2004/ %>
-    <% packages += %w(python2.7-dev) %>
-  <% end %>
-
-  <% if distro =~ /ubuntu2004/ %>
-    <% packages << 'libsnmp35' %>
-  <% else %>
-    <% packages << 'libsnmp30' %>
-  <% end %>
-
-  <% if distro !~ /ubuntu2004/ %>
-    <% packages << 'python-pip' %>
-  <% end %>
-
-  <% if distro =~ /debian10/ %>
-    <% packages << 'openjdk-11-jdk-headless' %>
-  <% elsif distro =~ /ubuntu1404/ %>
-    # Ubuntu 14.04 only has openjdk 7, this is too old to be useful
-  <% else %>
-    <% packages << 'openjdk-8-jdk-headless' %>
+  # ubuntu1404 only has openjdk-7-jre-headless
+  <% if distro !~ /ubuntu1404/ %>
+    <% packages << 'openjdk-8-jre-headless' %>
   <% end %>
 
   # ubuntu1404, ubuntu1604: libcurl3
-  # ubuntu1804, ubuntu2004, debian10: libcurl4
-  <% if distro =~ /ubuntu1804|ubuntu2004|debian10/ %>
+  # ubuntu1804: libcurl4
+  <% if distro =~ /ubuntu1804/ %>
     <% packages << 'libcurl4' %>
   <% else %>
     <% packages << 'libcurl3' %>
@@ -123,36 +99,9 @@ FROM <%= base_image %>
     <% packages << 'nodejs' %>
   <% end %>
 
-  <% if distro =~ /ubuntu2004/ %>
-    <% packages += %w(ruby ruby2.7 bundler python2 python2-dev) %>
-  <% end %>
-
   RUN apt-get update && apt-get install -y <%= packages.join(' ') %>
 <% else %>
 
-  <% if distro =~ /rhel6/ %>
-  
-    # CentOS 6 is dead - to use it retrieve the packages from vault:
-    # https://stackoverflow.com/questions/53562691/error-cannot-retrieve-repository-metadata-repomd-xml-for-repository-base-pl
-  
-    <%
-      
-      cfg = <<-CFG
-[base]
-name=CentOS-$releasever - Base
-#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=os
-#baseurl=http://mirror.centos.org/centos/$releasever/os/$basearch/
-baseurl=http://vault.centos.org/6.10/os/x86_64/
-gpgcheck=1
-gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
-CFG
-
-    %>
-  
-    RUN printf "<%= cfg.gsub("\n", "\\n") %>" >/etc/yum.repos.d/CentOS-Base.repo
-  
-  <% end %>
-  
   # Enterprise server: net-snmp
   # lsb_release: redhat-lsb-core
   # our runner scripts: which
@@ -169,7 +118,7 @@ CFG
 
   RUN yum install -y redhat-lsb-core which git gcc libyaml krb5-server \
     krb5-workstation cyrus-sasl-devel cyrus-sasl-gssapi java-1.8.0-openjdk \
-    net-snmp python3
+    net-snmp
 
   <% if distro =~ /rhel6/ %>
 
@@ -185,69 +134,23 @@ CFG
     RUN yum install -y python-devel
 
   <% end %>
-  
+
 <% end %>
 
 <% if preload? %>
 
-  <% if distro =~ /ubuntu1404/ %>
-    
-    # I couldn't find a prebuilt package of anything more recent than 2.7.6
-    # for 14.04.
-    RUN curl --retry 3 -fL https://www.python.org/ftp/python/2.7.16/Python-2.7.16.tar.xz | \
-      tar xfJ - && \
-      cd Python-2.7.16 && \
-      ./configure && \
-      nice make -j4 && \
-      make install && \
-      cd .. && rm -rf Python-2.7.16
-      
-    ENV PATH=/usr/local/bin:$PATH
-    
-    RUN curl --retry 3 -fL -o setuptools-44.1.1.zip https://files.pythonhosted.org/packages/b2/40/4e00501c204b457f10fe410da0c97537214b2265247bc9a5bc6edd55b9e4/setuptools-44.1.1.zip && \
-      unzip setuptools-44.1.1.zip && \
-      cd setuptools-44.1.1 && \
-      python setup.py install && \
-      cd .. && rm -rf setuptools-44.1.1
-
-  <% end%>
-
-  <% if true || distro =~ /rhel|ubuntu1604/ %>
-
-    # Ubuntu 12.04 ships pip 1.0 which is ancient and does not work.
-    #
-    # Ubuntu 16.04 apparently also ships a pip that does not work:
-    # https://stackoverflow.com/questions/37495375/python-pip-install-throws-typeerror-unsupported-operand-types-for-retry
-    # Potentially this only affects environments with less than ideal
-    # connectivity (or, perhaps, when python package registry is experiencing
-    # availability issues) when pip must retry to install packages.
-    #
-    # rhel apparently does not package pip at all in core repoitories,
-    # therefore install it the manual way.
-    #
-    # https://pip.pypa.io/en/stable/installing/
-    RUN curl --retry 3 -fL https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2
-
-  <% end %>
-
   # Current virtualenv fails with
   # https://github.com/pypa/virtualenv/issues/1630
-  <% if distro =~ /ubuntu2004/ %>
-    RUN python3 -m pip install 'virtualenv<20' 'mtools-legacy[mlaunch]'
-  <% else %>
-    RUN python2 -m pip install 'virtualenv<20' 'mtools-legacy[mlaunch]'
-  <% end %>
+  RUN python2 -m pip install 'virtualenv<20' 'mtools-legacy[mlaunch]'
 
   RUN pip --version && \
     pip install mtools-legacy[mlaunch]
 
   <% if @env.fetch('MONGODB_VERSION') >= '4.4' %>
-    # ubuntu1604 installs MarkupSafe 0.0.0 here instead of 2.0.0+
-    # as specified by dependencies, causing OCSP mock to not work.
-    RUN python3 -mpip install asn1crypto oscrypto flask --upgrade
+    RUN python3 -mpip install asn1crypto oscrypto flask
   <% end %>
 
-  <% unless ruby_head? || system_ruby? %>
+  <% unless ruby_head? %>
 
     RUN curl --retry 3 -fL <%= ruby_toolchain_url %> |tar -xC /opt -Jf -
     ENV PATH=/opt/rubies/<%= ruby %>/bin:$PATH \
@@ -256,20 +159,30 @@ CFG
 
   <% end %>
 
+  <% if distro =~ /rhel|ubuntu1604/ %>
+
+    # Ubuntu 12.04 ships pip 1.0 which is ancient and does not work.
+    #
+    # Ubuntu 16.04 apparently also ships a pip that does not work:
+    # https://stackoverflow.com/questions/37495375/python-pip-install-throws-typeerror-unsupported-operand-types-for-retry
+    # Potentially this only affects environments with less than ideal
+    # connectivity (or, perhaps, when python package registry is experiencing
+    # availability issues) when pip must retry to install packages.
+    #
+    # rhel apparently does not package pip at all in core repoitories,
+    # therefore install it the manual way.
+    #
+    # https://pip.pypa.io/en/stable/installing/
+    RUN curl --retry 3 -fL https://bootstrap.pypa.io/pip/2.7/get-pip.py | python
+
+  <% end %>
+
   RUN curl --retry 3 -fL <%= server_download_url %> |tar xzf - && \
     mv mongo*/ /opt/mongodb
   ENV USE_OPT_MONGODB=1 USE_SYSTEM_PYTHON_PACKAGES=1
 
 <% end %>
 
-<% if distro =~ /debian|ubuntu/ %>
-  # mkdir was moved from /usr/bin to /bin and MongoDB's distros
-  # apparently keep using the old location.
-  # This definitely affects debian10.
-  # https://stackoverflow.com/questions/64653051/make-usr-bin-mkdir-command-not-found-during-gem-install-nokogiri-in-ubuntu
-  RUN test -f /usr/bin/mkdir || ln -s /bin/mkdir /usr/bin/mkdir
-<% end %>
-
 WORKDIR /app
 
 <% if preload? && !ruby_head? %>
@@ -280,12 +193,7 @@ WORKDIR /app
   COPY lib/<%= project_lib_subdir %>/version.rb lib/<%= project_lib_subdir %>/version.rb
   RUN bundle install
   COPY .evergreen/patch-debuggers .evergreen/patch-debuggers
-  <% if system_ruby? %>
-    # Running under docker with root access
-    RUN .evergreen/patch-debuggers /var/lib/gems
-  <% else %>
-    RUN .evergreen/patch-debuggers /opt/rubies
-  <% end %>
+  RUN .evergreen/patch-debuggers /opt/rubies
 
 <% end %>