mongo 2.14.0 → 2.18.1

data/spec/shared/lib/mrss/docker_runner.rb

@@ -0,0 +1,291 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+require 'optparse'
+require 'erb'
+autoload :Dotenv, 'dotenv'
+
+module Mrss
+  autoload :ServerVersionRegistry, 'mrss/server_version_registry'
+
+  class DockerRunner
+    def initialize(**opts)
+      # These options are required:
+      opts.fetch(:image_tag)
+      opts.fetch(:dockerfile_path)
+      opts.fetch(:default_script)
+      opts.fetch(:project_lib_subdir)
+
+      @options = opts.merge(preload: true)
+    end
+
+    attr_reader :options
+
+    def run
+      process_arguments
+      unless @options[:exec_only]
+        create_dockerfile
+        create_image
+      end
+      if @options[:mongo_only]
+        run_deployment
+      else
+        run_tests
+      end
+    end
+
+    private
+
+    def process_arguments
+      #@options = {}
+      OptionParser.new do |opts|
+        opts.banner = "Usage: test-on-docker [-d distro] [evergreen_key=value ...]"
+
+        opts.on("-a", "--add-env=PATH", "Load environment variables from PATH in .env format") do |path|
+          @options[:extra_env] ||= {}
+          unless File.exist?(path)
+            raise "-a option references nonexistent file #{path}"
+          end
+          Dotenv.parse(path).each do |k, v|
+            @options[:extra_env][k] = v
+          end
+        end
+
+        opts.on("-d", "--distro=DISTRO", "Distro to use") do |v|
+          @options[:distro] = v
+        end
+
+        opts.on('-e', '--exec-only', 'Execute tests using existing Dockerfile (for offline user)') do |v|
+          @options[:exec_only] = v
+        end
+
+        opts.on('-m', '--mongo-only=PORT', 'Start the MongoDB deployment and expose it to host on ports starting with PORT') do |v|
+          @options[:mongo_only] = v.to_i
+        end
+
+        opts.on('-p', '--preload', 'Preload Ruby toolchain and server binaries in docker (default)') do |v|
+          @options[:preload] = v
+        end
+
+        opts.on('-P', '--no-preload', 'Do not preload Ruby toolchain and server binaries in docker') do
+          @options[:preload] = false
+        end
+
+        opts.on('-s', '--script=SCRIPT', 'Test script to invoke') do |v|
+          @options[:script] = v
+        end
+
+        opts.on('-i', '--interactive', 'Interactive mode - disable per-test timeouts') do |v|
+          @options[:interactive] = v
+        end
+      end.parse!
+
+      @env = Hash[ARGV.map do |arg|
+        arg.split('=', 2)
+      end]
+
+      @env['RVM_RUBY'] ||= 'ruby-2.7'
+      unless ruby =~ /^j?ruby-/
+        raise "RVM_RUBY option is not in expected format: #{ruby}"
+      end
+
+      @env['MONGODB_VERSION'] ||= '4.4'
+    end
+
+    def create_dockerfile
+      template_path = File.join(File.dirname(__FILE__), '../../share/Dockerfile.erb')
+      result = ERB.new(File.read(template_path)).result(binding)
+      File.open(dockerfile_path, 'w') do |f|
+        f << result
+      end
+    end
+
+    def image_tag
+      options.fetch(:image_tag)
+    end
+
+    def dockerfile_path
+      options.fetch(:dockerfile_path)
+    end
+
+    def create_image
+      run_command(['docker', 'build',
+        '-t', image_tag,
+        '-f', dockerfile_path,
+        '.'])
+    end
+
+    BASE_TEST_COMMAND = %w(docker run --rm -i --tmpfs /tmpfs:exec).freeze
+
+    def run_tests
+      run_command(BASE_TEST_COMMAND + tty_arg + extra_env + [image_tag] +
+        script.split(/\s+/))
+    end
+
+    def run_deployment
+      run_command(BASE_TEST_COMMAND + tty_arg + extra_env + [
+        '-e', %q`TEST_CMD=watch -x bash -c "ps awwxu |egrep 'mongo|ocsp'"`,
+        '-e', 'BIND_ALL=true',
+      ] + port_forwards + [image_tag] + script.split(/\s+/))
+    end
+
+    def tty_arg
+      tty = File.open('/dev/stdin') do |f|
+        f.isatty
+      end
+      if tty
+        %w(-t --init)
+      else
+        []
+      end
+    end
+
+    def extra_env
+      if @options[:extra_env]
+        @options[:extra_env].map do |k, v|
+          # Here the value must not be escaped
+          ['-e', "#{k}=#{v}"]
+        end.flatten
+      else
+        []
+      end
+    end
+
+    def port_forwards
+      args = (0...num_exposed_ports).map do |i|
+        host_port = @options[:mongo_only] + i
+        container_port = 27017 + i
+        ['-p', "#{host_port}:#{container_port}"]
+      end.flatten
+
+      if @env['OCSP_ALGORITHM'] && !@env['OCSP_VERIFIER']
+        args += %w(-p 8100:8100)
+      end
+
+      args
+    end
+
+    def run_command(cmd)
+      if pid = fork
+        Process.wait(pid)
+        unless $?.exitstatus == 0
+          raise "Process exited with code #{$?.exitstatus}"
+        end
+      else
+        exec(*cmd)
+      end
+    end
+
+    def distro
+      @options[:distro] || if app_tests?
+        'ubuntu2004'
+        else
+          case server_version
+          when '3.6'
+            'debian9'
+          when '4.0', '4.2'
+            'ubuntu1804'
+          else
+            'ubuntu2004'
+          end
+        end
+    end
+
+    BASE_IMAGES = {
+      'debian81' => 'debian:jessie',
+      'debian92' => 'debian:stretch',
+      'debian10' => 'debian:buster',
+      'ubuntu1404' => 'ubuntu:trusty',
+      'ubuntu1604' => 'ubuntu:xenial',
+      'ubuntu1804' => 'ubuntu:bionic',
+      'ubuntu2004' => 'ubuntu:focal',
+      'rhel62' => 'centos:6',
+      'rhel70' => 'centos:7',
+    }.freeze
+
+    def base_image
+      BASE_IMAGES[distro] or raise "Unknown distro: #{distro}"
+    end
+
+    def ruby
+      @env['RVM_RUBY']
+    end
+
+    def ruby_head?
+      ruby == 'ruby-head'
+    end
+
+    def system_ruby?
+      %w(1 true yes).include?(@env['SYSTEM_RUBY']&.downcase)
+    end
+
+    def server_version
+      @env['MONGODB_VERSION']
+    end
+
+    def script
+      @options[:script] || options.fetch(:default_script)
+    end
+
+    def debian?
+      distro =~ /debian|ubuntu/
+    end
+
+    def preload?
+      !!@options[:preload]
+    end
+
+    def interactive?
+      !!@options[:interactive]
+    end
+
+    def project_lib_subdir
+      options.fetch(:project_lib_subdir)
+    end
+
+    def server_download_url
+      @server_download_url ||= ServerVersionRegistry.new(server_version, distro).download_url
+    end
+
+    def libmongocrypt_path
+      case distro
+      when /ubuntu1604/
+        "./ubuntu1604/nocrypto/lib64/libmongocrypt.so"
+      when /ubuntu1804/
+        "./ubuntu1804-64/nocrypto/lib64/libmongocrypt.so"
+      when /debian92/
+        "./debian92/nocrypto/lib64/libmongocrypt.so"
+      else
+        raise "This script does not support running FLE tests on #{distro}. Use ubuntu1604, ubuntu1804 or debian92 instead"
+      end
+    end
+
+    def expose?
+      !!@options[:mongo_only]
+    end
+
+    def fle?
+      %w(1 true yes).include?(@env['FLE']&.downcase)
+    end
+
+    # Mongoid
+    def app_tests?
+      %w(1 true yes).include?(@env['APP_TESTS']&.downcase)
+    end
+
+    def num_exposed_ports
+      case @env['TOPOLOGY'] || 'standalone'
+      when 'standalone'
+        1
+      when 'replica-set'
+        3
+      when 'sharded-cluster'
+        if @env['SINGLE_MONGOS']
+          1
+        else
+          2
+        end
+      end
+    end
+  end
+end

data/spec/shared/lib/mrss/eg_config_utils.rb

@@ -0,0 +1,51 @@
+autoload :YAML, 'yaml'
+require 'erubi'
+require 'erubi/capture_end'
+require 'tilt'
+
+module Mrss
+  module EgConfigUtils
+
+    DEBIAN_FOR_RUBY = {
+      'ruby-2.3' => 'debian92',
+      'ruby-2.4' => 'debian92',
+      'ruby-2.5' => 'debian10',
+      'ruby-2.6' => 'debian10',
+      'ruby-2.7' => 'debian10',
+      'ruby-3.0' => 'debian10',
+    }
+
+    def standard_debian_rubies(rubies, key: nil, &block)
+      rubies.flatten!
+      text = block.call
+      contents = YAML.load(text)
+      out = rubies.map do |ruby|
+        contents.merge(
+          'matrix_name' => "#{contents['matrix_name']} - #{ruby}",
+          'matrix_spec' => contents['matrix_spec'].merge(
+            'ruby' => ruby,
+            key || 'os' => DEBIAN_FOR_RUBY.fetch(ruby),
+          ),
+        )
+      end.to_yaml
+      text =~ /\A\n?(\s+)/
+      unless text
+        raise "Couldn't figure out indentation level"
+      end
+      indent = ' ' * ($1.length - 2)
+      "\n" + out.sub(/\A---.*\n/, indent).gsub("\n", "\n#{indent}")
+    end
+
+    def transform_config(template_path, context)
+      Tilt.new(template_path, engine_class: Erubi::CaptureEndEngine).render(context)
+    end
+
+    def generated_file_warning
+      <<-EOT
+# GENERATED FILE - DO NOT EDIT.
+# Run ./.evergreen/update-evergreen-configs to regenerate this file.
+
+EOT
+    end
+  end
+end

data/spec/shared/lib/mrss/event_subscriber.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+module Mrss
+  # Test event subscriber.
+  class EventSubscriber
+
+    # The mappings of event names to types.
+    MAPPINGS = {
+      'topology_opening_event' => Mongo::Monitoring::Event::TopologyOpening,
+      'topology_description_changed_event' => Mongo::Monitoring::Event::TopologyChanged,
+      'topology_closed_event' => Mongo::Monitoring::Event::TopologyClosed,
+      'server_opening_event' => Mongo::Monitoring::Event::ServerOpening,
+      'server_description_changed_event' => Mongo::Monitoring::Event::ServerDescriptionChanged,
+      'server_closed_event' => Mongo::Monitoring::Event::ServerClosed
+    }.freeze
+
+    attr_reader :all_events
+
+    attr_reader :started_events
+
+    attr_reader :succeeded_events
+
+    attr_reader :failed_events
+
+    attr_reader :published_events
+
+    # @param [ String ] name Optional name for the event subscriber.
+    def initialize(name: nil)
+      @mutex = Mutex.new
+      clear_events!
+      @name = name
+    end
+
+    def to_s
+      %Q`#<EventSubscriber:#{@name ? "\"#{@name}\"" : '%x' % object_id} \
+  started=#{started_events.length} \
+  succeeded=#{succeeded_events.length} \
+  failed=#{failed_events.length} \
+  published=#{published_events.length}>`
+    end
+
+    alias :inspect :to_s
+
+    # Event retrieval
+
+    def select_started_events(cls)
+      started_events.select do |event|
+        event.is_a?(cls)
+      end
+    end
+
+    def select_succeeded_events(cls)
+      succeeded_events.select do |event|
+        event.is_a?(cls)
+      end
+    end
+
+    def select_completed_events(*classes)
+      (succeeded_events + failed_events).select do |event|
+        classes.any? { |c| c === event }
+      end
+    end
+
+    def select_published_events(cls)
+      published_events.select do |event|
+        event.is_a?(cls)
+      end
+    end
+
+    # Filters command started events for the specified command name.
+    def command_started_events(command_name)
+      started_events.select do |event|
+        event.command[command_name]
+      end
+    end
+
+    def non_auth_command_started_events
+      started_events.reject do |event|
+        %w(authenticate getnonce saslSstart saslContinue).any? do |cmd|
+          event.command[cmd]
+        end
+      end
+    end
+
+    # Locates command stated events for the specified command name,
+    # asserts that there is exactly one such event, and returns it.
+    def single_command_started_event(command_name, include_auth: false, database_name: nil)
+      events = if include_auth
+                 started_events
+               else
+                 non_auth_command_started_events
+               end
+      get_one_event(events, command_name, 'started', database_name: database_name)
+    end
+
+    # Locates command succeeded events for the specified command name,
+    # asserts that there is exactly one such event, and returns it.
+    def single_command_succeeded_event(command_name, database_name: nil)
+      get_one_event(succeeded_events, command_name, 'succeeded', database_name: database_name)
+    end
+
+    def get_one_event(events, command_name, kind, database_name: nil)
+      events = events.select do |event|
+        event.command_name == command_name and
+        database_name.nil? || database_name == event.database_name
+      end
+      if events.length != 1
+        raise "Expected a single '#{command_name}' #{kind} event#{database_name ? " for '#{database_name}'" : ''} but we have #{events.length}"
+      end
+      events.first
+    end
+
+    # Get the first succeeded event published for the name, and then delete it.
+    #
+    # @param [ String ] name The event name.
+    #
+    # @return [ Event ] The matching event.
+    def first_event(name)
+      cls = MAPPINGS[name]
+      if cls.nil?
+        raise ArgumentError, "Bogus event name #{name}"
+      end
+      matching = succeeded_events.find do |event|
+        cls === event
+      end
+      succeeded_events.delete(matching)
+      matching
+    end
+
+    # Event recording
+
+    # Cache the started event.
+    #
+    # @param [ Event ] event The event.
+    def started(event)
+      @mutex.synchronize do
+        started_events << event
+        all_events << event
+      end
+    end
+
+    # Cache the succeeded event.
+    #
+    # @param [ Event ] event The event.
+    def succeeded(event)
+      @mutex.synchronize do
+        succeeded_events << event
+        all_events << event
+      end
+    end
+
+    # Cache the failed event.
+    #
+    # @param [ Event ] event The event.
+    def failed(event)
+      @mutex.synchronize do
+        failed_events << event
+        all_events << event
+      end
+    end
+
+    def published(event)
+      @mutex.synchronize do
+        published_events << event
+        all_events << event
+      end
+    end
+
+    # Clear all cached events.
+    def clear_events!
+      @all_events = []
+      @started_events = []
+      @succeeded_events = []
+      @failed_events = []
+      @published_events = []
+      self
+    end
+  end
+  # Only handles succeeded events correctly.
+  class PhasedEventSubscriber < EventSubscriber
+    def initialize
+      super
+      @phase_events = {}
+    end
+
+    def phase_finished(phase_index)
+      @phase_events[phase_index] = succeeded_events
+      @succeeded_events = []
+    end
+
+    def phase_events(phase_index)
+      @phase_events[phase_index]
+    end
+
+    def event_count
+      @phase_events.inject(0) do |sum, event|
+        sum + event.length
+      end
+    end
+  end
+
+  class VerboseEventSubscriber < EventSubscriber
+    %w(started succeeded failed published).each do |meth|
+      define_method(meth) do |event|
+        puts event.summary
+        super(event)
+      end
+    end
+  end
+end

data/spec/shared/lib/mrss/lite_constraints.rb

@@ -87,12 +87,25 @@ module Mrss
 
     def require_libmongocrypt
       before(:all) do
-        unless ENV['LIBMONGOCRYPT_PATH']
+        # If FLE is set in environment, the entire test run is supposed to
+        # include FLE therefore run the FLE tests.
+        if (ENV['LIBMONGOCRYPT_PATH'] || '').empty? && (ENV['FLE'] || '').empty?
           skip 'Test requires path to libmongocrypt to be specified in LIBMONGOCRYPT_PATH env variable'
         end
       end
     end
 
+    def min_libmongocrypt_version(version)
+      require_libmongocrypt
+      before(:all) do
+        actual_version = Gem::Version.new(Mongo::Crypt::Binding.mongocrypt_version(nil))
+        min_version = Gem::Version.new(version)
+        unless actual_version >= min_version
+          skip "libmongocrypt version #{min_version} required, but version #{actual_version} is available"
+        end
+      end
+    end
+
     def require_no_libmongocrypt
       before(:all) do
         if ENV['LIBMONGOCRYPT_PATH']
@@ -171,5 +184,47 @@ module Mrss
         end
       end
     end
+
+    def require_active_support
+      before(:all) do
+        if !SpecConfig.instance.active_support?
+          skip 'This test requires ActiveSupport; set WITH_ACTIVE_SUPPORT=1 in environment'
+        end
+      end
+    end
+
+    def no_active_support
+      before(:all) do
+        if SpecConfig.instance.active_support?
+          skip 'This test requires no ActiveSupport; unset WITH_ACTIVE_SUPPORT in environment'
+        end
+      end
+    end
+
+    def require_fallbacks
+      before(:all) do
+        unless %w(yes true 1).include?((ENV['TEST_I18N_FALLBACKS'] || '').downcase)
+          skip 'Set TEST_I18N_FALLBACKS=1 environment variable to run these tests'
+        end
+      end
+    end
+
+    # This is a macro for retrying flaky tests on CI that occasionally fail.
+    # Note that the tests will only be retried on CI.
+    #
+    # @param [ Integer ] :tries The number of times to retry.
+    # @param [ Integer ] :sleep The number of seconds to sleep in between retries.
+    #   If nothing, or nil, is passed, we won't wait in between retries.
+    def retry_test(tries: 3, sleep: nil)
+      if %w(1 yes true).include?(ENV['CI'])
+        around do |example|
+          if sleep
+            example.run_with_retry retry: tries, retry_wait: sleep
+          else
+            example.run_with_retry retry: tries
+          end
+        end
+      end
+    end
   end
 end