mongo 2.14.0 → 2.16.0

data/spec/shared/lib/mrss/cluster_config.rb

@@ -0,0 +1,231 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+# ClusterConfig requires ClientRegistry class provided by the host project.
+
+require 'singleton'
+
+module Mrss
+  class ClusterConfig
+    include Singleton
+    include RSpec::Core::Pending
+
+    def single_server?
+      determine_cluster_config
+      @single_server
+    end
+
+    def sharded_ish?
+      determine_cluster_config
+      @topology == :sharded || @topology == :load_balanced
+    end
+
+    def replica_set_name
+      determine_cluster_config
+      @replica_set_name
+    end
+
+    def server_version
+      determine_cluster_config
+      @server_version
+    end
+
+    def enterprise?
+      determine_cluster_config
+      @enterprise
+    end
+
+    def short_server_version
+      server_version.split('.')[0..1].join('.')
+    end
+
+    def fcv
+      determine_cluster_config
+      @fcv
+    end
+
+    # Per https://jira.mongodb.org/browse/SERVER-39052, working with FCV
+    # in sharded topologies is annoying. Also, FCV doesn't exist in servers
+    # less than 3.4. This method returns FCV on 3.4+ servers when in single
+    # or RS topologies, and otherwise returns the major.minor server version.
+    def fcv_ish
+      if server_version.nil?
+        raise "Deployment server version not known - check that connection to deployment succeeded"
+      end
+
+      if server_version >= '3.4' && !sharded_ish?
+        fcv
+      else
+        if short_server_version == '4.1'
+          '4.2'
+        else
+          short_server_version
+        end
+      end
+    end
+
+    # @return [ Mongo::Address ] The address of the primary in the deployment.
+    def primary_address
+      determine_cluster_config
+      @primary_address
+    end
+
+    def primary_address_str
+      determine_cluster_config
+      @primary_address.seed
+    end
+
+    def primary_address_host
+      both = primary_address_str
+      both.split(':').first
+    end
+
+    def primary_address_port
+      both = primary_address_str
+      both.split(':')[1] || 27017
+    end
+
+    def primary_description
+      determine_cluster_config
+      @primary_description
+    end
+
+    def server_parameters
+      determine_cluster_config
+      @server_parameters
+    end
+
+    # Try running a command on the admin database to see if the mongod was
+    # started with auth.
+    def auth_enabled?
+      if @auth_enabled.nil?
+        @auth_enabled = begin
+          basic_client.use(:admin).command(getCmdLineOpts: 1).first["argv"].include?("--auth")
+        rescue => e
+          e.message =~ /(not authorized)|(unauthorized)|(no users authenticated)|(requires authentication)/
+        end
+      end
+      @auth_enabled
+    end
+
+    def topology
+      determine_cluster_config
+      @topology
+    end
+
+    def storage_engine
+      @storage_engine ||= begin
+        # 2.6 does not have wired tiger
+        if short_server_version == '2.6'
+          :mmapv1
+        else
+          client = ClientRegistry.instance.global_client('root_authorized')
+          if sharded_ish?
+            shards = client.use(:admin).command(listShards: 1).first
+            if shards['shards'].empty?
+              raise 'Shards are empty'
+            end
+            shard = shards['shards'].first
+            address_str = shard['host'].sub(/^.*\//, '').sub(/,.*/, '')
+            client = ClusterTools.instance.direct_client(address_str,
+              SpecConfig.instance.test_options.merge(SpecConfig.instance.auth_options).merge(connect: :direct))
+          end
+          rv = client.use(:admin).command(serverStatus: 1).first
+          rv = rv['storageEngine']['name']
+          rv_map = {
+            'wiredTiger' => :wired_tiger,
+            'mmapv1' => :mmapv1,
+          }
+          rv_map[rv] || rv
+        end
+      end
+    end
+
+    # This method returns an alternate address for connecting to the configured
+    # deployment. For example, if the replica set is configured with nodes at
+    # of localhost:27017 and so on, this method will return 127.0.0.:27017.
+    #
+    # Note that the "alternate" refers to replica set configuration, not the
+    # addresses specified in test suite configuration. If the deployment topology
+    # is not a replica set, "alternate" refers to test suite configuration as
+    # this is the only configuration available.
+    def alternate_address
+      @alternate_address ||= begin
+        address = primary_address_host
+        str = case address
+        when '127.0.0.1'
+          'localhost'
+        when /^(\d+\.){3}\d+$/
+          skip 'This test requires a hostname or 127.0.0.1 as address'
+        else
+          # We don't know if mongod is listening on ipv4 or ipv6, in principle.
+          # Our tests use ipv4, so hardcode that for now.
+          # To support both we need to try both addresses which will make this
+          # test more complicated.
+          #
+          # JRuby chokes on primary_address_port as the port (e.g. 27017).
+          # Since the port does not actually matter, use a common port like 80.
+          resolved_address = Addrinfo.getaddrinfo(address, 80, Socket::PF_INET).first.ip_address
+          if resolved_address.include?(':')
+            "[#{resolved_address}]"
+          else
+            resolved_address
+          end
+        end + ":#{primary_address_port}"
+        Mongo::Address.new(str)
+      end
+    end
+
+    private
+
+    def determine_cluster_config
+      return if @primary_address
+
+      # Run all commands to figure out the cluster configuration from the same
+      # client. This is somewhat wasteful when running a single test, but reduces
+      # test runtime for the suite overall because all commands are sent on the
+      # same connection rather than each command connecting to the cluster by
+      # itself.
+      client = ClientRegistry.instance.global_client('root_authorized')
+
+      primary = client.cluster.next_primary
+      @primary_address = primary.address
+      @primary_description = primary.description
+      @replica_set_name = client.cluster.topology.replica_set_name
+
+      @topology ||= begin
+        topology = client.cluster.topology.class.name.sub(/.*::/, '')
+        topology = topology.gsub(/([A-Z])/) { |match| '_' + match.downcase }.sub(/^_/, '')
+        if topology =~ /^replica_set/
+          topology = 'replica_set'
+        end
+        topology.to_sym
+      end
+
+      @single_server = client.cluster.servers_list.length == 1
+
+      build_info = client.database.command(buildInfo: 1).first
+
+      @server_version = build_info['version']
+      @enterprise = build_info['modules'] && build_info['modules'].include?('enterprise')
+
+      @server_parameters = begin
+        client.use(:admin).command(getParameter: '*').first
+      rescue => e
+        STDERR.puts("WARNING: Failed to obtain server parameters: #{e.class}: #{e.message}")
+        {}
+      end
+
+      if !sharded_ish? && short_server_version >= '3.4'
+        rv = @server_parameters['featureCompatibilityVersion']
+        @fcv = rv['version'] || rv
+      end
+    end
+
+    def basic_client
+      # Do not cache the result here so that if the client gets closed,
+      # client registry reconnects it in subsequent tests
+      ClientRegistry.instance.global_client('basic')
+    end
+  end
+end

data/spec/shared/lib/mrss/constraints.rb

@@ -52,7 +52,7 @@ module Mrss
     end
 
     def require_topology(*topologies)
-      invalid_topologies = topologies - [:single, :replica_set, :sharded]
+      invalid_topologies = topologies - [:single, :replica_set, :sharded, :load_balanced]
 
       unless invalid_topologies.empty?
         raise ArgumentError, "Invalid topologies requested: #{invalid_topologies.join(', ')}"
@@ -82,7 +82,7 @@ module Mrss
           unless ClusterConfig.instance.server_version >= '4.0'
             skip 'Transactions tests in a replica set topology require server 4.0+'
           end
-        when :sharded
+        when :sharded, :load_balanced
           unless ClusterConfig.instance.server_version >= '4.2'
             skip 'Transactions tests in a sharded cluster topology require server 4.2+'
           end
@@ -113,6 +113,14 @@ module Mrss
       end
     end
 
+    def require_retry_writes
+      before(:all) do
+        unless SpecConfig.instance.retry_writes?
+          skip "Retry writes is disabled"
+        end
+      end
+    end
+
     def require_no_retry_writes
       before(:all) do
         if SpecConfig.instance.retry_writes?
@@ -147,6 +155,33 @@ module Mrss
       end
     end
 
+    def require_no_snappy_compression
+      before(:all) do
+        compressors = SpecConfig.instance.compressors
+        if compressors && compressors.include?('snappy')
+          skip "Snappy compression is enabled"
+        end
+      end
+    end
+
+    def require_zstd_compression
+      before(:all) do
+        compressors = SpecConfig.instance.compressors
+        unless compressors && compressors.include?('zstd')
+          skip "Zstd compression is not enabled"
+        end
+      end
+    end
+
+    def require_no_zstd_compression
+      before(:all) do
+        compressors = SpecConfig.instance.compressors
+        if compressors && compressors.include?('zstd')
+          skip "Zstd compression is enabled"
+        end
+      end
+    end
+
     def require_no_compression
       before(:all) do
         if SpecConfig.instance.compressors
@@ -214,31 +249,52 @@ module Mrss
     # (mongos 4.0+ overrides the write concern)
     def require_set_write_concern
       before(:all) do
-        if ClusterConfig.instance.topology == :sharded && ClusterConfig.instance.short_server_version >= '4.0'
+        if %i(sharded load_balanced).include?(ClusterConfig.instance.topology) &&
+          ClusterConfig.instance.short_server_version >= '4.0'
+        then
           skip "mongos 4.0+ overrides write concern"
         end
       end
     end
 
-    def require_multi_shard
+    def require_multi_mongos
       before(:all) do
         if ClusterConfig.instance.topology == :sharded && SpecConfig.instance.addresses.length == 1
-          skip 'Test requires a minimum of two shards if run in sharded topology'
+          skip 'Test requires a minimum of two mongoses if run in sharded topology'
+        end
+
+        if ClusterConfig.instance.topology == :load_balanced && SpecConfig.instance.single_mongos?
+          skip 'Test requires a minimum of two mongoses if run in load-balanced topology'
         end
       end
     end
 
-    def require_no_multi_shard
+    # In sharded topology operations are distributed to the mongoses.
+    # When we set fail points, the fail point may be set on one mongos and
+    # operation may be executed on another mongos, causing failures.
+    # Tests that are not setting targeted fail points should utilize this
+    # method to restrict themselves to single mongos.
+    #
+    # In load-balanced topology, the same problem can happen when there is
+    # more than one mongos behind the load balancer.
+    def require_no_multi_mongos
       before(:all) do
         if ClusterConfig.instance.topology == :sharded && SpecConfig.instance.addresses.length > 1
-          skip 'Test requires a single shard if run in sharded topology'
+          skip 'Test requires a single mongos if run in sharded topology'
+        end
+        if ClusterConfig.instance.topology == :load_balanced && !SpecConfig.instance.single_mongos?
+          skip 'Test requires a single mongos, as indicated by SINGLE_MONGOS=1 environment variable, if run in load-balanced topology'
         end
       end
     end
 
+    alias :require_no_multi_shard :require_no_multi_mongos
+
     def require_wired_tiger
       before(:all) do
-        if ClusterConfig.instance.storage_engine != :wired_tiger
+        # Storage detection fails for serverless instances. However, it is safe to
+        # assume that a serverless instance uses WiredTiger Storage Engine.
+        if !SpecConfig.instance.serverless? && ClusterConfig.instance.storage_engine != :wired_tiger
           skip 'Test requires WiredTiger storage engine'
         end
       end
@@ -247,7 +303,9 @@ module Mrss
     def require_wired_tiger_on_36
       before(:all) do
         if ClusterConfig.instance.short_server_version >= '3.6'
-          if ClusterConfig.instance.storage_engine != :wired_tiger
+          # Storage detection fails for serverless instances. However, it is safe to
+          # assume that a serverless instance uses WiredTiger Storage Engine.
+          if !SpecConfig.instance.serverless? && ClusterConfig.instance.storage_engine != :wired_tiger
             skip 'Test requires WiredTiger storage engine on 3.6+ servers'
           end
         end
@@ -256,7 +314,7 @@ module Mrss
 
     def require_mmapv1
       before(:all) do
-        if ClusterConfig.instance.storage_engine != :mmapv1
+        if SpecConfig.instance.serverless? || ClusterConfig.instance.storage_engine != :mmapv1
           skip 'Test requires MMAPv1 storage engine'
         end
       end
@@ -299,5 +357,29 @@ module Mrss
         end
       end
     end
+
+    def require_required_api_version
+      before(:all) do
+        unless ENV['API_VERSION_REQUIRED'] == '1'
+          skip 'Set API_VERSION_REQUIRED=1 to run this test'
+        end
+      end
+    end
+
+    def require_no_required_api_version
+      before(:all) do
+        if ENV['API_VERSION_REQUIRED'] == '1'
+          skip 'Cannot have API_VERSION_REQUIRED=1 to run this test'
+        end
+      end
+    end
+
+    def require_unix_socket
+      before(:all) do
+        if ENV['TOPOLOGY'] == 'load-balanced'
+          skip 'Load balancer does not listen on Unix sockets'
+        end
+      end
+    end
   end
 end

data/spec/shared/lib/mrss/docker_runner.rb

@@ -0,0 +1,271 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+require 'optparse'
+require 'erb'
+autoload :Dotenv, 'dotenv'
+
+module Mrss
+  autoload :ServerVersionRegistry, 'mrss/server_version_registry'
+
+  class DockerRunner
+    def initialize(**opts)
+      # These options are required:
+      opts.fetch(:image_tag)
+      opts.fetch(:dockerfile_path)
+      opts.fetch(:default_script)
+      opts.fetch(:project_lib_subdir)
+
+      @options = opts
+    end
+
+    attr_reader :options
+
+    def run
+      process_arguments
+      unless @options[:exec_only]
+        create_dockerfile
+        create_image
+      end
+      if @options[:mongo_only]
+        run_deployment
+      else
+        run_tests
+      end
+    end
+
+    private
+
+    def process_arguments
+      #@options = {}
+      OptionParser.new do |opts|
+        opts.banner = "Usage: test-on-docker [-d distro] [evergreen_key=value ...]"
+
+        opts.on("-a", "--add-env=PATH", "Load environment variables from PATH in .env format") do |path|
+          @options[:extra_env] ||= {}
+          unless File.exist?(path)
+            raise "-a option references nonexistent file #{path}"
+          end
+          Dotenv.parse(path).each do |k, v|
+            @options[:extra_env][k] = v
+          end
+        end
+
+        opts.on("-d", "--distro=DISTRO", "Distro to use") do |v|
+          @options[:distro] = v
+        end
+
+        opts.on('-e', '--exec-only', 'Execute tests using existing Dockerfile (for offline user)') do |v|
+          @options[:exec_only] = v
+        end
+
+        opts.on('-m', '--mongo-only=PORT', 'Start the MongoDB deployment and expose it to host on ports starting with PORT') do |v|
+          @options[:mongo_only] = v.to_i
+        end
+
+        opts.on('-p', '--preload', 'Preload Ruby toolchain and server binaries in docker') do |v|
+          @options[:preload] = v
+        end
+
+        opts.on('-s', '--script=SCRIPT', 'Test script to invoke') do |v|
+          @options[:script] = v
+        end
+
+        opts.on('-i', '--interactive', 'Interactive mode - disable per-test timeouts') do |v|
+          @options[:interactive] = v
+        end
+      end.parse!
+
+      @env = Hash[ARGV.map do |arg|
+        arg.split('=', 2)
+      end]
+
+      @env['RVM_RUBY'] ||= 'ruby-2.7'
+      unless ruby =~ /^j?ruby-/
+        raise "RVM_RUBY option is not in expected format: #{ruby}"
+      end
+
+      @env['MONGODB_VERSION'] ||= '4.4'
+    end
+
+    def create_dockerfile
+      template_path = File.join(File.dirname(__FILE__), '../../share/Dockerfile.erb')
+      result = ERB.new(File.read(template_path)).result(binding)
+      File.open(dockerfile_path, 'w') do |f|
+        f << result
+      end
+    end
+
+    def image_tag
+      options.fetch(:image_tag)
+    end
+
+    def dockerfile_path
+      options.fetch(:dockerfile_path)
+    end
+
+    def create_image
+      run_command(['docker', 'build',
+        '-t', image_tag,
+        '-f', dockerfile_path,
+        '.'])
+    end
+
+    BASE_TEST_COMMAND = %w(docker run --rm -i --tmpfs /tmpfs:exec).freeze
+
+    def run_tests
+      run_command(BASE_TEST_COMMAND + tty_arg + extra_env + [image_tag] +
+        script.split(/\s+/))
+    end
+
+    def run_deployment
+      run_command(BASE_TEST_COMMAND + tty_arg + extra_env + [
+        '-e', %q`TEST_CMD=watch -x bash -c "ps awwxu |egrep 'mongo|ocsp'"`,
+        '-e', 'BIND_ALL=true',
+      ] + port_forwards + [image_tag] + script.split(/\s+/))
+    end
+
+    def tty_arg
+      tty = File.open('/dev/stdin') do |f|
+        f.isatty
+      end
+      if tty
+        %w(-t --init)
+      else
+        []
+      end
+    end
+
+    def extra_env
+      if @options[:extra_env]
+        @options[:extra_env].map do |k, v|
+          # Here the value must not be escaped
+          ['-e', "#{k}=#{v}"]
+        end.flatten
+      else
+        []
+      end
+    end
+
+    def port_forwards
+      args = (0...num_exposed_ports).map do |i|
+        host_port = @options[:mongo_only] + i
+        container_port = 27017 + i
+        ['-p', "#{host_port}:#{container_port}"]
+      end.flatten
+
+      if @env['OCSP_ALGORITHM'] && !@env['OCSP_VERIFIER']
+        args += %w(-p 8100:8100)
+      end
+
+      args
+    end
+
+    def run_command(cmd)
+      if pid = fork
+        Process.wait(pid)
+        unless $?.exitstatus == 0
+          raise "Process exited with code #{$?.exitstatus}"
+        end
+      else
+        exec(*cmd)
+      end
+    end
+
+    def distro
+      @options[:distro] || 'ubuntu1604'
+    end
+
+    BASE_IMAGES = {
+      'debian81' => 'debian:jessie',
+      'debian92' => 'debian:stretch',
+      'debian10' => 'debian:buster',
+      'ubuntu1404' => 'ubuntu:trusty',
+      'ubuntu1604' => 'ubuntu:xenial',
+      'ubuntu1804' => 'ubuntu:bionic',
+      'ubuntu2004' => 'ubuntu:focal',
+      'rhel62' => 'centos:6',
+      'rhel70' => 'centos:7',
+    }.freeze
+
+    def base_image
+      BASE_IMAGES[distro] or raise "Unknown distro: #{distro}"
+    end
+
+    def ruby
+      @env['RVM_RUBY']
+    end
+
+    def ruby_head?
+      ruby == 'ruby-head'
+    end
+
+    def system_ruby?
+      %w(1 true yes).include?(@env['SYSTEM_RUBY']&.downcase)
+    end
+
+    def server_version
+      @env['MONGODB_VERSION']
+    end
+
+    def script
+      @options[:script] || options.fetch(:default_script)
+    end
+
+    def debian?
+      distro =~ /debian|ubuntu/
+    end
+
+    def preload?
+      !!@options[:preload]
+    end
+
+    def interactive?
+      !!@options[:interactive]
+    end
+
+    def project_lib_subdir
+      options.fetch(:project_lib_subdir)
+    end
+
+    def server_download_url
+      @server_download_url ||= ServerVersionRegistry.new(server_version, distro).download_url
+    end
+
+    def libmongocrypt_path
+      case distro
+      when /ubuntu1604/
+        "./ubuntu1604/nocrypto/lib64/libmongocrypt.so"
+      when /ubuntu1804/
+        "./ubuntu1804-64/nocrypto/lib64/libmongocrypt.so"
+      when /debian92/
+        "./debian92/nocrypto/lib64/libmongocrypt.so"
+      else
+        raise "This script does not support running FLE tests on #{distro}. Use ubuntu1604, ubuntu1804 or debian92 instead"
+      end
+    end
+
+    def expose?
+      !!@options[:mongo_only]
+    end
+
+    def fle?
+      %w(1 true yes).include?(@env['FLE']&.downcase)
+    end
+
+    def num_exposed_ports
+      case @env['TOPOLOGY'] || 'standalone'
+      when 'standalone'
+        1
+      when 'replica-set'
+        3
+      when 'sharded-cluster'
+        if @env['SINGLE_MONGOS']
+          1
+        else
+          2
+        end
+      end
+    end
+  end
+end