bson 4.11.1 → 4.14.0

data/spec/shared/lib/mrss/constraints.rb

@@ -0,0 +1,386 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+module Mrss
+  module Constraints
+    def min_server_version(version)
+      unless version =~ /\A\d+\.\d+\z/
+        raise ArgumentError, "Version can only be major.minor: #{version}"
+      end
+
+      before(:all) do
+        if version > ClusterConfig.instance.server_version
+          skip "Server version #{version} or higher required, we have #{ClusterConfig.instance.server_version}"
+        end
+      end
+    end
+
+    def max_server_version(version)
+      unless version =~ /\A\d+\.\d+\z/
+        raise ArgumentError, "Version can only be major.minor: #{version}"
+      end
+
+      before(:all) do
+        if version < ClusterConfig.instance.short_server_version
+          skip "Server version #{version} or lower required, we have #{ClusterConfig.instance.server_version}"
+        end
+      end
+    end
+
+    def min_server_fcv(version)
+      unless version =~ /\A\d+\.\d+\z/
+        raise ArgumentError, "FCV can only be major.minor: #{version}"
+      end
+
+      before(:all) do
+        unless ClusterConfig.instance.fcv_ish >= version
+          skip "FCV #{version} or higher required, we have #{ClusterConfig.instance.fcv_ish} (server #{ClusterConfig.instance.server_version})"
+        end
+      end
+    end
+
+    def max_server_fcv(version)
+      unless version =~ /\A\d+\.\d+\z/
+        raise ArgumentError, "Version can only be major.minor: #{version}"
+      end
+
+      before(:all) do
+        if version < ClusterConfig.instance.fcv_ish
+          skip "FCV #{version} or lower required, we have #{ClusterConfig.instance.fcv_ish} (server #{ClusterConfig.instance.server_version})"
+        end
+      end
+    end
+
+    def require_topology(*topologies)
+      invalid_topologies = topologies - [:single, :replica_set, :sharded, :load_balanced]
+
+      unless invalid_topologies.empty?
+        raise ArgumentError, "Invalid topologies requested: #{invalid_topologies.join(', ')}"
+      end
+
+      before(:all) do
+        unless topologies.include?(topology = ClusterConfig.instance.topology)
+          skip "Topology #{topologies.join(' or ')} required, we have #{topology}"
+        end
+      end
+    end
+
+    def max_example_run_time(timeout)
+      around do |example|
+        TimeoutInterrupt.timeout(timeout, TimeoutInterrupt::Error.new("Test execution terminated after #{timeout} seconds")) do
+          example.run
+        end
+      end
+    end
+
+    def require_transaction_support
+      before(:all) do
+        case ClusterConfig.instance.topology
+        when :single
+          skip 'Transactions tests require a replica set (4.0+) or a sharded cluster (4.2+)'
+        when :replica_set
+          unless ClusterConfig.instance.server_version >= '4.0'
+            skip 'Transactions tests in a replica set topology require server 4.0+'
+          end
+        when :sharded, :load_balanced
+          unless ClusterConfig.instance.server_version >= '4.2'
+            skip 'Transactions tests in a sharded cluster topology require server 4.2+'
+          end
+        else
+          raise NotImplementedError
+        end
+      end
+    end
+
+    # Fail command fail point was added to mongod in 4.0 and to mongos in 4.2.
+    def require_fail_command
+      require_transaction_support
+    end
+
+    def require_tls
+      before(:all) do
+        unless SpecConfig.instance.ssl?
+          skip "SSL not enabled"
+        end
+      end
+    end
+
+    def require_no_tls
+      before(:all) do
+        if SpecConfig.instance.ssl?
+          skip "SSL enabled"
+        end
+      end
+    end
+
+    def require_retry_writes
+      before(:all) do
+        unless SpecConfig.instance.retry_writes?
+          skip "Retry writes is disabled"
+        end
+      end
+    end
+
+    def require_no_retry_writes
+      before(:all) do
+        if SpecConfig.instance.retry_writes?
+          skip "Retry writes is enabled"
+        end
+      end
+    end
+
+    def require_compression
+      before(:all) do
+        if SpecConfig.instance.compressors.nil?
+          skip "Compression is not enabled"
+        end
+      end
+    end
+
+    def require_zlib_compression
+      before(:all) do
+        compressors = SpecConfig.instance.compressors
+        unless compressors && compressors.include?('zlib')
+          skip "Zlib compression is not enabled"
+        end
+      end
+    end
+
+    def require_snappy_compression
+      before(:all) do
+        compressors = SpecConfig.instance.compressors
+        unless compressors && compressors.include?('snappy')
+          skip "Snappy compression is not enabled"
+        end
+      end
+    end
+
+    def require_no_snappy_compression
+      before(:all) do
+        compressors = SpecConfig.instance.compressors
+        if compressors && compressors.include?('snappy')
+          skip "Snappy compression is enabled"
+        end
+      end
+    end
+
+    def require_zstd_compression
+      before(:all) do
+        compressors = SpecConfig.instance.compressors
+        unless compressors && compressors.include?('zstd')
+          skip "Zstd compression is not enabled"
+        end
+      end
+    end
+
+    def require_no_zstd_compression
+      before(:all) do
+        compressors = SpecConfig.instance.compressors
+        if compressors && compressors.include?('zstd')
+          skip "Zstd compression is enabled"
+        end
+      end
+    end
+
+    def require_no_compression
+      before(:all) do
+        if SpecConfig.instance.compressors
+          skip "Compression is enabled"
+        end
+      end
+    end
+
+    def ruby_version_gte(version)
+      before(:all) do
+        if RUBY_VERSION < version
+          skip "Ruby version #{version} or higher required"
+        end
+      end
+    end
+
+    def ruby_version_lt(version)
+      before(:all) do
+        if RUBY_VERSION >= version
+          skip "Ruby version less than #{version} required"
+        end
+      end
+    end
+
+    def require_auth(*values)
+      before(:all) do
+        if values.any?
+          unless values.include?(ENV['AUTH'])
+            msg = values.map { |v| "AUTH=#{v}" }.join(' or ')
+            skip "This test requires #{msg}"
+          end
+        else
+          unless ENV['AUTH'] == 'auth' || SpecConfig.instance.user || ClusterConfig.instance.auth_enabled?
+            skip "Auth required"
+          end
+        end
+      end
+    end
+
+    def require_no_auth
+      before(:all) do
+        auth = ENV.fetch('AUTH', '')
+        if (!auth.empty? && auth != 'noauth') || SpecConfig.instance.user || ClusterConfig.instance.auth_enabled?
+          skip "Auth not allowed"
+        end
+      end
+    end
+
+    def require_x509_auth
+      before(:all) do
+        unless SpecConfig.instance.x509_auth?
+          skip "X.509 auth required"
+        end
+      end
+    end
+
+    def require_no_external_user
+      before(:all) do
+        if SpecConfig.instance.external_user?
+          skip "External user configurations are not compatible with this test"
+        end
+      end
+    end
+
+    # Can the driver specify a write concern that won't be overridden?
+    # (mongos 4.0+ overrides the write concern)
+    def require_set_write_concern
+      before(:all) do
+        if %i(sharded load_balanced).include?(ClusterConfig.instance.topology) &&
+          ClusterConfig.instance.short_server_version >= '4.0'
+        then
+          skip "mongos 4.0+ overrides write concern"
+        end
+      end
+    end
+
+    def require_multi_mongos
+      before(:all) do
+        if ClusterConfig.instance.topology == :sharded && SpecConfig.instance.addresses.length == 1
+          skip 'Test requires a minimum of two mongoses if run in sharded topology'
+        end
+
+        if ClusterConfig.instance.topology == :load_balanced && SpecConfig.instance.single_mongos?
+          skip 'Test requires a minimum of two mongoses if run in load-balanced topology'
+        end
+      end
+    end
+
+    # In sharded topology operations are distributed to the mongoses.
+    # When we set fail points, the fail point may be set on one mongos and
+    # operation may be executed on another mongos, causing failures.
+    # Tests that are not setting targeted fail points should utilize this
+    # method to restrict themselves to single mongos.
+    #
+    # In load-balanced topology, the same problem can happen when there is
+    # more than one mongos behind the load balancer.
+    def require_no_multi_mongos
+      before(:all) do
+        if ClusterConfig.instance.topology == :sharded && SpecConfig.instance.addresses.length > 1
+          skip 'Test requires a single mongos if run in sharded topology'
+        end
+        if ClusterConfig.instance.topology == :load_balanced && !SpecConfig.instance.single_mongos?
+          skip 'Test requires a single mongos, as indicated by SINGLE_MONGOS=1 environment variable, if run in load-balanced topology'
+        end
+      end
+    end
+
+    alias :require_no_multi_shard :require_no_multi_mongos
+
+    def require_wired_tiger
+      before(:all) do
+        # Storage detection fails for serverless instances. However, it is safe to
+        # assume that a serverless instance uses WiredTiger Storage Engine.
+        if !SpecConfig.instance.serverless? && ClusterConfig.instance.storage_engine != :wired_tiger
+          skip 'Test requires WiredTiger storage engine'
+        end
+      end
+    end
+
+    def require_wired_tiger_on_36
+      before(:all) do
+        if ClusterConfig.instance.short_server_version >= '3.6'
+          # Storage detection fails for serverless instances. However, it is safe to
+          # assume that a serverless instance uses WiredTiger Storage Engine.
+          if !SpecConfig.instance.serverless? && ClusterConfig.instance.storage_engine != :wired_tiger
+            skip 'Test requires WiredTiger storage engine on 3.6+ servers'
+          end
+        end
+      end
+    end
+
+    def require_mmapv1
+      before(:all) do
+        if SpecConfig.instance.serverless? || ClusterConfig.instance.storage_engine != :mmapv1
+          skip 'Test requires MMAPv1 storage engine'
+        end
+      end
+    end
+
+    def require_enterprise
+      before(:all) do
+        unless ClusterConfig.instance.enterprise?
+          skip 'Test requires enterprise build of MongoDB'
+        end
+      end
+    end
+
+    # Integration tests for SRV polling require internet connectivity to
+    # look up SRV records and a sharded cluster configured on default port on
+    # localhost (localhost:27017, localhost:27018).
+    def require_default_port_deployment
+      # Because the DNS records at test1.test.build.10gen.cc point to
+      # localhost:27017 & localhost:27018, the test suite must have been
+      # configured to use these addresses
+      before(:all) do
+        have_default_port = SpecConfig.instance.addresses.any? do |address|
+          %w(127.0.0.1 127.0.0.1:27017 localhost localhost:27017).include?(address)
+        end
+        unless have_default_port
+          skip 'This test requires the test suite to be configured for localhost:27017'
+        end
+      end
+    end
+
+    # Some tests perform assertions on what the driver is logging.
+    # Some test configurations, for example OCSP with unknown response,
+    # produce warnings due to optional checks failing.
+    # This constraint skips tests that issue logging assertions on configurations
+    # that may produce non-test-originated log entries.
+    def require_warning_clean
+      before(:all) do
+        if ENV['OCSP_STATUS'] == 'unknown'
+          skip 'Unknown OCSP status is not global warning-clean'
+        end
+      end
+    end
+
+    def require_required_api_version
+      before(:all) do
+        unless ENV['API_VERSION_REQUIRED'] == '1'
+          skip 'Set API_VERSION_REQUIRED=1 to run this test'
+        end
+      end
+    end
+
+    def require_no_required_api_version
+      before(:all) do
+        if ENV['API_VERSION_REQUIRED'] == '1'
+          skip 'Cannot have API_VERSION_REQUIRED=1 to run this test'
+        end
+      end
+    end
+
+    def require_unix_socket
+      before(:all) do
+        if ENV['TOPOLOGY'] == 'load-balanced'
+          skip 'Load balancer does not listen on Unix sockets'
+        end
+      end
+    end
+  end
+end

data/spec/shared/lib/mrss/docker_runner.rb

@@ -0,0 +1,271 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+require 'optparse'
+require 'erb'
+autoload :Dotenv, 'dotenv'
+
+module Mrss
+  autoload :ServerVersionRegistry, 'mrss/server_version_registry'
+
+  class DockerRunner
+    def initialize(**opts)
+      # These options are required:
+      opts.fetch(:image_tag)
+      opts.fetch(:dockerfile_path)
+      opts.fetch(:default_script)
+      opts.fetch(:project_lib_subdir)
+
+      @options = opts
+    end
+
+    attr_reader :options
+
+    def run
+      process_arguments
+      unless @options[:exec_only]
+        create_dockerfile
+        create_image
+      end
+      if @options[:mongo_only]
+        run_deployment
+      else
+        run_tests
+      end
+    end
+
+    private
+
+    def process_arguments
+      #@options = {}
+      OptionParser.new do |opts|
+        opts.banner = "Usage: test-on-docker [-d distro] [evergreen_key=value ...]"
+
+        opts.on("-a", "--add-env=PATH", "Load environment variables from PATH in .env format") do |path|
+          @options[:extra_env] ||= {}
+          unless File.exist?(path)
+            raise "-a option references nonexistent file #{path}"
+          end
+          Dotenv.parse(path).each do |k, v|
+            @options[:extra_env][k] = v
+          end
+        end
+
+        opts.on("-d", "--distro=DISTRO", "Distro to use") do |v|
+          @options[:distro] = v
+        end
+
+        opts.on('-e', '--exec-only', 'Execute tests using existing Dockerfile (for offline user)') do |v|
+          @options[:exec_only] = v
+        end
+
+        opts.on('-m', '--mongo-only=PORT', 'Start the MongoDB deployment and expose it to host on ports starting with PORT') do |v|
+          @options[:mongo_only] = v.to_i
+        end
+
+        opts.on('-p', '--preload', 'Preload Ruby toolchain and server binaries in docker') do |v|
+          @options[:preload] = v
+        end
+
+        opts.on('-s', '--script=SCRIPT', 'Test script to invoke') do |v|
+          @options[:script] = v
+        end
+
+        opts.on('-i', '--interactive', 'Interactive mode - disable per-test timeouts') do |v|
+          @options[:interactive] = v
+        end
+      end.parse!
+
+      @env = Hash[ARGV.map do |arg|
+        arg.split('=', 2)
+      end]
+
+      @env['RVM_RUBY'] ||= 'ruby-2.7'
+      unless ruby =~ /^j?ruby-/
+        raise "RVM_RUBY option is not in expected format: #{ruby}"
+      end
+
+      @env['MONGODB_VERSION'] ||= '4.4'
+    end
+
+    def create_dockerfile
+      template_path = File.join(File.dirname(__FILE__), '../../share/Dockerfile.erb')
+      result = ERB.new(File.read(template_path)).result(binding)
+      File.open(dockerfile_path, 'w') do |f|
+        f << result
+      end
+    end
+
+    def image_tag
+      options.fetch(:image_tag)
+    end
+
+    def dockerfile_path
+      options.fetch(:dockerfile_path)
+    end
+
+    def create_image
+      run_command(['docker', 'build',
+        '-t', image_tag,
+        '-f', dockerfile_path,
+        '.'])
+    end
+
+    BASE_TEST_COMMAND = %w(docker run --rm -i --tmpfs /tmpfs:exec).freeze
+
+    def run_tests
+      run_command(BASE_TEST_COMMAND + tty_arg + extra_env + [image_tag] +
+        script.split(/\s+/))
+    end
+
+    def run_deployment
+      run_command(BASE_TEST_COMMAND + tty_arg + extra_env + [
+        '-e', %q`TEST_CMD=watch -x bash -c "ps awwxu |egrep 'mongo|ocsp'"`,
+        '-e', 'BIND_ALL=true',
+      ] + port_forwards + [image_tag] + script.split(/\s+/))
+    end
+
+    def tty_arg
+      tty = File.open('/dev/stdin') do |f|
+        f.isatty
+      end
+      if tty
+        %w(-t --init)
+      else
+        []
+      end
+    end
+
+    def extra_env
+      if @options[:extra_env]
+        @options[:extra_env].map do |k, v|
+          # Here the value must not be escaped
+          ['-e', "#{k}=#{v}"]
+        end.flatten
+      else
+        []
+      end
+    end
+
+    def port_forwards
+      args = (0...num_exposed_ports).map do |i|
+        host_port = @options[:mongo_only] + i
+        container_port = 27017 + i
+        ['-p', "#{host_port}:#{container_port}"]
+      end.flatten
+
+      if @env['OCSP_ALGORITHM'] && !@env['OCSP_VERIFIER']
+        args += %w(-p 8100:8100)
+      end
+
+      args
+    end
+
+    def run_command(cmd)
+      if pid = fork
+        Process.wait(pid)
+        unless $?.exitstatus == 0
+          raise "Process exited with code #{$?.exitstatus}"
+        end
+      else
+        exec(*cmd)
+      end
+    end
+
+    def distro
+      @options[:distro] || 'ubuntu1804'
+    end
+
+    BASE_IMAGES = {
+      'debian81' => 'debian:jessie',
+      'debian92' => 'debian:stretch',
+      'debian10' => 'debian:buster',
+      'ubuntu1404' => 'ubuntu:trusty',
+      'ubuntu1604' => 'ubuntu:xenial',
+      'ubuntu1804' => 'ubuntu:bionic',
+      'ubuntu2004' => 'ubuntu:focal',
+      'rhel62' => 'centos:6',
+      'rhel70' => 'centos:7',
+    }.freeze
+
+    def base_image
+      BASE_IMAGES[distro] or raise "Unknown distro: #{distro}"
+    end
+
+    def ruby
+      @env['RVM_RUBY']
+    end
+
+    def ruby_head?
+      ruby == 'ruby-head'
+    end
+
+    def system_ruby?
+      %w(1 true yes).include?(@env['SYSTEM_RUBY']&.downcase)
+    end
+
+    def server_version
+      @env['MONGODB_VERSION']
+    end
+
+    def script
+      @options[:script] || options.fetch(:default_script)
+    end
+
+    def debian?
+      distro =~ /debian|ubuntu/
+    end
+
+    def preload?
+      !!@options[:preload]
+    end
+
+    def interactive?
+      !!@options[:interactive]
+    end
+
+    def project_lib_subdir
+      options.fetch(:project_lib_subdir)
+    end
+
+    def server_download_url
+      @server_download_url ||= ServerVersionRegistry.new(server_version, distro).download_url
+    end
+
+    def libmongocrypt_path
+      case distro
+      when /ubuntu1604/
+        "./ubuntu1604/nocrypto/lib64/libmongocrypt.so"
+      when /ubuntu1804/
+        "./ubuntu1804-64/nocrypto/lib64/libmongocrypt.so"
+      when /debian92/
+        "./debian92/nocrypto/lib64/libmongocrypt.so"
+      else
+        raise "This script does not support running FLE tests on #{distro}. Use ubuntu1604, ubuntu1804 or debian92 instead"
+      end
+    end
+
+    def expose?
+      !!@options[:mongo_only]
+    end
+
+    def fle?
+      %w(1 true yes).include?(@env['FLE']&.downcase)
+    end
+
+    def num_exposed_ports
+      case @env['TOPOLOGY'] || 'standalone'
+      when 'standalone'
+        1
+      when 'replica-set'
+        3
+      when 'sharded-cluster'
+        if @env['SINGLE_MONGOS']
+          1
+        else
+          2
+        end
+      end
+    end
+  end
+end