mongoid 7.2.0 → 7.2.4

data/spec/shared/lib/mrss/docker_runner.rb

@@ -0,0 +1,271 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+require 'optparse'
+require 'erb'
+autoload :Dotenv, 'dotenv'
+
+module Mrss
+  autoload :ServerVersionRegistry, 'mrss/server_version_registry'
+
+  class DockerRunner
+    def initialize(**opts)
+      # These options are required:
+      opts.fetch(:image_tag)
+      opts.fetch(:dockerfile_path)
+      opts.fetch(:default_script)
+      opts.fetch(:project_lib_subdir)
+
+      @options = opts
+    end
+
+    attr_reader :options
+
+    def run
+      process_arguments
+      unless @options[:exec_only]
+        create_dockerfile
+        create_image
+      end
+      if @options[:mongo_only]
+        run_deployment
+      else
+        run_tests
+      end
+    end
+
+    private
+
+    def process_arguments
+      #@options = {}
+      OptionParser.new do |opts|
+        opts.banner = "Usage: test-on-docker [-d distro] [evergreen_key=value ...]"
+
+        opts.on("-a", "--add-env=PATH", "Load environment variables from PATH in .env format") do |path|
+          @options[:extra_env] ||= {}
+          unless File.exist?(path)
+            raise "-a option references nonexistent file #{path}"
+          end
+          Dotenv.parse(path).each do |k, v|
+            @options[:extra_env][k] = v
+          end
+        end
+
+        opts.on("-d", "--distro=DISTRO", "Distro to use") do |v|
+          @options[:distro] = v
+        end
+
+        opts.on('-e', '--exec-only', 'Execute tests using existing Dockerfile (for offline user)') do |v|
+          @options[:exec_only] = v
+        end
+
+        opts.on('-m', '--mongo-only=PORT', 'Start the MongoDB deployment and expose it to host on ports starting with PORT') do |v|
+          @options[:mongo_only] = v.to_i
+        end
+
+        opts.on('-p', '--preload', 'Preload Ruby toolchain and server binaries in docker') do |v|
+          @options[:preload] = v
+        end
+
+        opts.on('-s', '--script=SCRIPT', 'Test script to invoke') do |v|
+          @options[:script] = v
+        end
+
+        opts.on('-i', '--interactive', 'Interactive mode - disable per-test timeouts') do |v|
+          @options[:interactive] = v
+        end
+      end.parse!
+
+      @env = Hash[ARGV.map do |arg|
+        arg.split('=', 2)
+      end]
+
+      @env['RVM_RUBY'] ||= 'ruby-2.7'
+      unless ruby =~ /^j?ruby-/
+        raise "RVM_RUBY option is not in expected format: #{ruby}"
+      end
+
+      @env['MONGODB_VERSION'] ||= '4.4'
+    end
+
+    def create_dockerfile
+      template_path = File.join(File.dirname(__FILE__), '../../share/Dockerfile.erb')
+      result = ERB.new(File.read(template_path)).result(binding)
+      File.open(dockerfile_path, 'w') do |f|
+        f << result
+      end
+    end
+
+    def image_tag
+      options.fetch(:image_tag)
+    end
+
+    def dockerfile_path
+      options.fetch(:dockerfile_path)
+    end
+
+    def create_image
+      run_command(['docker', 'build',
+        '-t', image_tag,
+        '-f', dockerfile_path,
+        '.'])
+    end
+
+    BASE_TEST_COMMAND = %w(docker run --rm -i --tmpfs /tmpfs:exec).freeze
+
+    def run_tests
+      run_command(BASE_TEST_COMMAND + tty_arg + extra_env + [image_tag] +
+        script.split(/\s+/))
+    end
+
+    def run_deployment
+      run_command(BASE_TEST_COMMAND + tty_arg + extra_env + [
+        '-e', %q`TEST_CMD=watch -x bash -c "ps awwxu |egrep 'mongo|ocsp'"`,
+        '-e', 'BIND_ALL=true',
+      ] + port_forwards + [image_tag] + script.split(/\s+/))
+    end
+
+    def tty_arg
+      tty = File.open('/dev/stdin') do |f|
+        f.isatty
+      end
+      if tty
+        %w(-t --init)
+      else
+        []
+      end
+    end
+
+    def extra_env
+      if @options[:extra_env]
+        @options[:extra_env].map do |k, v|
+          # Here the value must not be escaped
+          ['-e', "#{k}=#{v}"]
+        end.flatten
+      else
+        []
+      end
+    end
+
+    def port_forwards
+      args = (0...num_exposed_ports).map do |i|
+        host_port = @options[:mongo_only] + i
+        container_port = 27017 + i
+        ['-p', "#{host_port}:#{container_port}"]
+      end.flatten
+
+      if @env['OCSP_ALGORITHM'] && !@env['OCSP_VERIFIER']
+        args += %w(-p 8100:8100)
+      end
+
+      args
+    end
+
+    def run_command(cmd)
+      if pid = fork
+        Process.wait(pid)
+        unless $?.exitstatus == 0
+          raise "Process exited with code #{$?.exitstatus}"
+        end
+      else
+        exec(*cmd)
+      end
+    end
+
+    def distro
+      @options[:distro] || 'ubuntu1604'
+    end
+
+    BASE_IMAGES = {
+      'debian81' => 'debian:jessie',
+      'debian92' => 'debian:stretch',
+      'debian10' => 'debian:buster',
+      'ubuntu1404' => 'ubuntu:trusty',
+      'ubuntu1604' => 'ubuntu:xenial',
+      'ubuntu1804' => 'ubuntu:bionic',
+      'ubuntu2004' => 'ubuntu:focal',
+      'rhel62' => 'centos:6',
+      'rhel70' => 'centos:7',
+    }.freeze
+
+    def base_image
+      BASE_IMAGES[distro] or raise "Unknown distro: #{distro}"
+    end
+
+    def ruby
+      @env['RVM_RUBY']
+    end
+
+    def ruby_head?
+      ruby == 'ruby-head'
+    end
+
+    def system_ruby?
+      %w(1 true yes).include?(@env['SYSTEM_RUBY']&.downcase)
+    end
+
+    def server_version
+      @env['MONGODB_VERSION']
+    end
+
+    def script
+      @options[:script] || options.fetch(:default_script)
+    end
+
+    def debian?
+      distro =~ /debian|ubuntu/
+    end
+
+    def preload?
+      !!@options[:preload]
+    end
+
+    def interactive?
+      !!@options[:interactive]
+    end
+
+    def project_lib_subdir
+      options.fetch(:project_lib_subdir)
+    end
+
+    def server_download_url
+      @server_download_url ||= ServerVersionRegistry.new(server_version, distro).download_url
+    end
+
+    def libmongocrypt_path
+      case distro
+      when /ubuntu1604/
+        "./ubuntu1604/nocrypto/lib64/libmongocrypt.so"
+      when /ubuntu1804/
+        "./ubuntu1804-64/nocrypto/lib64/libmongocrypt.so"
+      when /debian92/
+        "./debian92/nocrypto/lib64/libmongocrypt.so"
+      else
+        raise "This script does not support running FLE tests on #{distro}. Use ubuntu1604, ubuntu1804 or debian92 instead"
+      end
+    end
+
+    def expose?
+      !!@options[:mongo_only]
+    end
+
+    def fle?
+      %w(1 true yes).include?(@env['FLE']&.downcase)
+    end
+
+    def num_exposed_ports
+      case @env['TOPOLOGY'] || 'standalone'
+      when 'standalone'
+        1
+      when 'replica-set'
+        3
+      when 'sharded-cluster'
+        if @env['SINGLE_MONGOS']
+          1
+        else
+          2
+        end
+      end
+    end
+  end
+end

data/spec/shared/lib/mrss/lite_constraints.rb

@@ -171,5 +171,21 @@ module Mrss
         end
       end
     end
+
+    def require_active_support
+      before(:all) do
+        if !SpecConfig.instance.active_support?
+          skip 'This test requires ActiveSupport; set WITH_ACTIVE_SUPPORT=1 in environment'
+        end
+      end
+    end
+
+    def no_active_support
+      before(:all) do
+        if SpecConfig.instance.active_support?
+          skip 'This test requires no ActiveSupport; unset WITH_ACTIVE_SUPPORT in environment'
+        end
+      end
+    end
   end
 end

data/spec/shared/lib/mrss/server_version_registry.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+autoload :JSON, 'json'
+require 'open-uri'
+
+module Mrss
+  class ServerVersionRegistry
+    class Error < StandardError
+    end
+
+    class UnknownVersion < Error
+    end
+
+    class MissingDownloadUrl < Error
+    end
+
+    class BrokenDownloadUrl < Error
+    end
+
+    def initialize(desired_version, arch)
+      @desired_version, @arch = desired_version, arch
+    end
+
+    attr_reader :desired_version, :arch
+
+    def download_url
+      @download_url ||= begin
+        version, version_ok = detect_version(current_catalog)
+        if version.nil?
+          version, full_version_ok = detect_version(full_catalog)
+          version_ok ||= full_version_ok
+        end
+        if version.nil?
+          if version_ok
+            raise MissingDownloadUrl, "No downloads for version #{desired_version}"
+          else
+            raise UnknownVersion, "No version #{desired_version}"
+          end
+        end
+        dl = version['downloads'].detect do |dl|
+          dl['archive']['url'].index("enterprise-#{arch}") &&
+          dl['arch'] == 'x86_64'
+        end
+        unless dl
+          raise MissingDownloadUrl, "No download for #{arch} for #{version['version']}"
+        end
+        url = dl['archive']['url']
+      end
+    rescue MissingDownloadUrl
+      if %w(4.7 4.7.0).include?(desired_version)
+        # 4.7.0 has no advertised downloads but it is downloadable and
+        # we do need it. Dirty hack below.
+        registry = self.class.new('4.4.3', arch)
+        registry.download_url.sub('4.4.3', '4.7.0').tap do |url|
+          # Sanity check - ensure the URL we hacked up is a valid one
+          io = uri_open(url)
+          begin
+            io.read(1)
+          ensure
+            io.close
+          end
+        end
+      else
+        raise
+      end
+    end
+
+    private
+
+    def uri_open(*args)
+      if RUBY_VERSION < '2.5'
+        open(*args)
+      else
+        URI.open(*args)
+      end
+    end
+
+    def detect_version(catalog)
+      candidate_versions = catalog['versions'].select do |version|
+        version['version'].start_with?(desired_version) &&
+        !version['version'].include?('-')
+      end
+      version_ok = !candidate_versions.empty?
+      # Sometimes the download situation is borked and there is a release
+      # with no downloads... skip those.
+      version = candidate_versions.detect do |version|
+        !version['downloads'].empty?
+      end
+      # Allow RC releases if there isn't a GA release.
+      if version.nil?
+        candidate_versions = catalog['versions'].select do |version|
+          version['version'].start_with?(desired_version)
+        end
+        version_ok ||= !candidate_versions.empty?
+        version = candidate_versions.detect do |version|
+          !version['downloads'].empty?
+        end
+      end
+      [version, version_ok]
+    end
+
+    def current_catalog
+      @current_catalog ||= begin
+        JSON.load(uri_open('http://downloads.mongodb.org/current.json').read)
+      end
+    end
+
+    def full_catalog
+      @full_catalog ||= begin
+        JSON.load(uri_open('http://downloads.mongodb.org/full.json').read)
+      end
+    end
+  end
+end

data/spec/shared/lib/mrss/spec_organizer.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 autoload :JSON, 'json'
 autoload :FileUtils, 'fileutils'
 autoload :Find, 'find'
@@ -22,18 +25,28 @@ module Mrss
     end
 
     def initialize(root: nil, classifiers:, priority_order:,
-      spec_root: nil, rspec_json_path: nil, rspec_all_json_path: nil
+      spec_root: nil, rspec_json_path: nil, rspec_all_json_path: nil,
+      randomize: false
     )
       @spec_root = spec_root || File.join(root, 'spec')
       @classifiers = classifiers
       @priority_order = priority_order
       @rspec_json_path = rspec_json_path || File.join(root, 'tmp/rspec.json')
       @rspec_all_json_path = rspec_all_json_path || File.join(root, 'tmp/rspec-all.json')
+      @randomize = !!randomize
     end
 
     attr_reader :spec_root, :classifiers, :priority_order
     attr_reader :rspec_json_path, :rspec_all_json_path
 
+    def randomize?
+      @randomize
+    end
+
+    def seed
+      @seed ||= (rand * 100_000).to_i
+    end
+
     def buckets
       @buckets ||= {}.tap do |buckets|
         Find.find(spec_root) do |path|
@@ -78,10 +91,20 @@ module Mrss
     end
 
     def run
+      run_buckets(*buckets.keys)
+    end
+
+    def run_buckets(*buckets)
       FileUtils.rm_f(rspec_all_json_path)
 
+      buckets.each do |bucket|
+        if bucket && !self.buckets[bucket]
+          raise "Unknown bucket #{bucket}"
+        end
+      end
+      buckets = Hash[self.buckets.select { |k, v| buckets.include?(k) }]
+
       failed = []
-      buckets = self.buckets.dup
 
       priority_order.each do |category|
         if files = buckets.delete(category)
@@ -109,8 +132,12 @@ module Mrss
       puts "Running #{category.to_s.gsub('_', ' ')} tests"
       FileUtils.rm_f(rspec_json_path)
       cmd = %w(rspec) + paths
+      if randomize?
+        cmd += %W(--order rand:#{seed})
+      end
 
       begin
+        puts "Running #{cmd.join(' ')}"
         ChildProcessHelper.check_call(cmd)
       ensure
         if File.exist?(rspec_json_path)
@@ -136,6 +163,9 @@ module Mrss
       end
       new.delete('version')
       new.delete('summary_line')
+      # The spec organizer runs all buckets with the same seed, hence
+      # we can drop the seed from new results.
+      new.delete('seed')
       unless new.empty?
         raise "Unhandled rspec results keys: #{new.keys.join(', ')}"
       end