mongo 2.14.0 → 2.15.1

data/spec/shared/bin/s3-upload

@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'aws-sdk-s3'
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Usage: s3-upload options"
+
+  opts.on("-r", "--region=REGION", "AWS region to use (default us-east-1)") do |v|
+    options[:region] = v
+  end
+
+  opts.on("-p", "--param=KEY=VALUE", "Specify parameter for S3 upload") do |v|
+    options[:params] ||= {}
+    k, v = v.split('=', 2)
+    options[:params][k.to_sym] = v
+  end
+
+  opts.on("-f", "--file=PATH", "Path to the file to upload, - to upload standard input") do |v|
+    options[:file] = v
+  end
+
+  opts.on("-w", "--write=BUCKET:PATH", "Bucket name and key (or path) to upload to") do |v|
+    options[:write] = v
+  end
+
+  opts.on("-c", "--copy=BUCKET:PATH", "Bucket name and key (or path) to copy to (may be specified more than once)") do |v|
+    options[:copy] ||= []
+    options[:copy] << v
+  end
+end.parse!
+
+ENV['AWS_REGION'] ||= options[:region] || 'us-east-1'
+
+def upload(f, options)
+  s3 = Aws::S3::Client.new
+  write = options.fetch(:write)
+  STDERR.puts "Writing #{write}"
+  bucket, key = write.split(':', 2)
+  s3.put_object(
+    body: f.read,
+    bucket: bucket,
+    key: key,
+    **options[:params] || {},
+  )
+  if copy = options[:copy]
+    copy.each do |dest|
+      STDERR.puts "Copying to #{dest}"
+      dbucket, dkey = dest.split(':', 2)
+      s3.copy_object(
+        bucket: dbucket,
+        key: dkey,
+        copy_source: "/#{bucket}/#{key}",
+        **options[:params] || {},
+      )
+    end
+  end
+end
+
+if options[:file] == '-'
+  upload(STDIN, options)
+elsif options[:file]
+  File.open(options[:file]) do |f|
+    upload(f, options)
+  end
+else
+  upload(STDIN, options)
+end

data/spec/shared/lib/mrss/cluster_config.rb

@@ -0,0 +1,226 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+# ClusterConfig requires ClientRegistry class provided by the host project.
+
+require 'singleton'
+
+module Mrss
+  class ClusterConfig
+    include Singleton
+    include RSpec::Core::Pending
+
+    def single_server?
+      determine_cluster_config
+      @single_server
+    end
+
+    def sharded_ish?
+      determine_cluster_config
+      @topology == :sharded || @topology == :load_balanced
+    end
+
+    def replica_set_name
+      determine_cluster_config
+      @replica_set_name
+    end
+
+    def server_version
+      determine_cluster_config
+      @server_version
+    end
+
+    def enterprise?
+      determine_cluster_config
+      @enterprise
+    end
+
+    def short_server_version
+      server_version.split('.')[0..1].join('.')
+    end
+
+    def fcv
+      determine_cluster_config
+      @fcv
+    end
+
+    # Per https://jira.mongodb.org/browse/SERVER-39052, working with FCV
+    # in sharded topologies is annoying. Also, FCV doesn't exist in servers
+    # less than 3.4. This method returns FCV on 3.4+ servers when in single
+    # or RS topologies, and otherwise returns the major.minor server version.
+    def fcv_ish
+      if server_version.nil?
+        raise "Deployment server version not known - check that connection to deployment succeeded"
+      end
+
+      if server_version >= '3.4' && !sharded_ish?
+        fcv
+      else
+        if short_server_version == '4.1'
+          '4.2'
+        else
+          short_server_version
+        end
+      end
+    end
+
+    # @return [ Mongo::Address ] The address of the primary in the deployment.
+    def primary_address
+      determine_cluster_config
+      @primary_address
+    end
+
+    def primary_address_str
+      determine_cluster_config
+      @primary_address.seed
+    end
+
+    def primary_address_host
+      both = primary_address_str
+      both.split(':').first
+    end
+
+    def primary_address_port
+      both = primary_address_str
+      both.split(':')[1] || 27017
+    end
+
+    def primary_description
+      determine_cluster_config
+      @primary_description
+    end
+
+    def server_parameters
+      determine_cluster_config
+      @server_parameters
+    end
+
+    # Try running a command on the admin database to see if the mongod was
+    # started with auth.
+    def auth_enabled?
+      if @auth_enabled.nil?
+        @auth_enabled = begin
+          basic_client.use(:admin).command(getCmdLineOpts: 1).first["argv"].include?("--auth")
+        rescue => e
+          e.message =~ /(not authorized)|(unauthorized)|(no users authenticated)|(requires authentication)/
+        end
+      end
+      @auth_enabled
+    end
+
+    def topology
+      determine_cluster_config
+      @topology
+    end
+
+    def storage_engine
+      @storage_engine ||= begin
+        # 2.6 does not have wired tiger
+        if short_server_version == '2.6'
+          :mmapv1
+        else
+          client = ClientRegistry.instance.global_client('root_authorized')
+          if sharded_ish?
+            shards = client.use(:admin).command(listShards: 1).first
+            if shards['shards'].empty?
+              raise 'Shards are empty'
+            end
+            shard = shards['shards'].first
+            address_str = shard['host'].sub(/^.*\//, '').sub(/,.*/, '')
+            client = ClusterTools.instance.direct_client(address_str,
+              SpecConfig.instance.test_options.merge(SpecConfig.instance.auth_options).merge(connect: :direct))
+          end
+          rv = client.use(:admin).command(serverStatus: 1).first
+          rv = rv['storageEngine']['name']
+          rv_map = {
+            'wiredTiger' => :wired_tiger,
+            'mmapv1' => :mmapv1,
+          }
+          rv_map[rv] || rv
+        end
+      end
+    end
+
+    # This method returns an alternate address for connecting to the configured
+    # deployment. For example, if the replica set is configured with nodes at
+    # of localhost:27017 and so on, this method will return 127.0.0.:27017.
+    #
+    # Note that the "alternate" refers to replica set configuration, not the
+    # addresses specified in test suite configuration. If the deployment topology
+    # is not a replica set, "alternate" refers to test suite configuration as
+    # this is the only configuration available.
+    def alternate_address
+      @alternate_address ||= begin
+        address = primary_address_host
+        str = case address
+        when '127.0.0.1'
+          'localhost'
+        when /^(\d+\.){3}\d+$/
+          skip 'This test requires a hostname or 127.0.0.1 as address'
+        else
+          # We don't know if mongod is listening on ipv4 or ipv6, in principle.
+          # Our tests use ipv4, so hardcode that for now.
+          # To support both we need to try both addresses which will make this
+          # test more complicated.
+          #
+          # JRuby chokes on primary_address_port as the port (e.g. 27017).
+          # Since the port does not actually matter, use a common port like 80.
+          resolved_address = Addrinfo.getaddrinfo(address, 80, Socket::PF_INET).first.ip_address
+          if resolved_address.include?(':')
+            "[#{resolved_address}]"
+          else
+            resolved_address
+          end
+        end + ":#{primary_address_port}"
+        Mongo::Address.new(str)
+      end
+    end
+
+    private
+
+    def determine_cluster_config
+      return if @primary_address
+
+      # Run all commands to figure out the cluster configuration from the same
+      # client. This is somewhat wasteful when running a single test, but reduces
+      # test runtime for the suite overall because all commands are sent on the
+      # same connection rather than each command connecting to the cluster by
+      # itself.
+      client = ClientRegistry.instance.global_client('root_authorized')
+
+      primary = client.cluster.next_primary
+      @primary_address = primary.address
+      @primary_description = primary.description
+      @replica_set_name = client.cluster.topology.replica_set_name
+
+      @topology ||= begin
+        topology = client.cluster.topology.class.name.sub(/.*::/, '')
+        topology = topology.gsub(/([A-Z])/) { |match| '_' + match.downcase }.sub(/^_/, '')
+        if topology =~ /^replica_set/
+          topology = 'replica_set'
+        end
+        topology.to_sym
+      end
+
+      @single_server = client.cluster.servers_list.length == 1
+
+      build_info = client.database.command(buildInfo: 1).first
+
+      @server_version = build_info['version']
+      @enterprise = build_info['modules'] && build_info['modules'].include?('enterprise')
+
+      @server_parameters = client.use(:admin).command(getParameter: '*').first
+
+      if !sharded_ish? && short_server_version >= '3.4'
+        rv = @server_parameters['featureCompatibilityVersion']
+        @fcv = rv['version'] || rv
+      end
+    end
+
+    def basic_client
+      # Do not cache the result here so that if the client gets closed,
+      # client registry reconnects it in subsequent tests
+      ClientRegistry.instance.global_client('basic')
+    end
+  end
+end

data/spec/shared/lib/mrss/constraints.rb

@@ -52,7 +52,7 @@ module Mrss
     end
 
     def require_topology(*topologies)
-      invalid_topologies = topologies - [:single, :replica_set, :sharded]
+      invalid_topologies = topologies - [:single, :replica_set, :sharded, :load_balanced]
 
       unless invalid_topologies.empty?
         raise ArgumentError, "Invalid topologies requested: #{invalid_topologies.join(', ')}"
@@ -82,7 +82,7 @@ module Mrss
           unless ClusterConfig.instance.server_version >= '4.0'
             skip 'Transactions tests in a replica set topology require server 4.0+'
           end
-        when :sharded
+        when :sharded, :load_balanced
           unless ClusterConfig.instance.server_version >= '4.2'
             skip 'Transactions tests in a sharded cluster topology require server 4.2+'
           end
@@ -113,6 +113,14 @@ module Mrss
       end
     end
 
+    def require_retry_writes
+      before(:all) do
+        unless SpecConfig.instance.retry_writes?
+          skip "Retry writes is disabled"
+        end
+      end
+    end
+
     def require_no_retry_writes
       before(:all) do
         if SpecConfig.instance.retry_writes?
@@ -147,6 +155,33 @@ module Mrss
       end
     end
 
+    def require_no_snappy_compression
+      before(:all) do
+        compressors = SpecConfig.instance.compressors
+        if compressors && compressors.include?('snappy')
+          skip "Snappy compression is enabled"
+        end
+      end
+    end
+
+    def require_zstd_compression
+      before(:all) do
+        compressors = SpecConfig.instance.compressors
+        unless compressors && compressors.include?('zstd')
+          skip "Zstd compression is not enabled"
+        end
+      end
+    end
+
+    def require_no_zstd_compression
+      before(:all) do
+        compressors = SpecConfig.instance.compressors
+        if compressors && compressors.include?('zstd')
+          skip "Zstd compression is enabled"
+        end
+      end
+    end
+
     def require_no_compression
       before(:all) do
         if SpecConfig.instance.compressors
@@ -214,7 +249,9 @@ module Mrss
     # (mongos 4.0+ overrides the write concern)
     def require_set_write_concern
       before(:all) do
-        if ClusterConfig.instance.topology == :sharded && ClusterConfig.instance.short_server_version >= '4.0'
+        if %i(sharded load_balanced).include?(ClusterConfig.instance.topology) &&
+          ClusterConfig.instance.short_server_version >= '4.0'
+        then
           skip "mongos 4.0+ overrides write concern"
         end
       end
@@ -228,17 +265,30 @@ module Mrss
       end
     end
 
+    # In sharded topology operations are distributed to the mongoses.
+    # When we set fail points, the fail point may be set on one mongos and
+    # operation may be executed on another mongos, causing failures.
+    # Tests that are not setting targeted fail points should utilize this
+    # method to restrict themselves to single mongos.
+    #
+    # In load-balanced topology, the same problem can happen when there is
+    # more than one mongos behind the load balancer.
     def require_no_multi_shard
       before(:all) do
         if ClusterConfig.instance.topology == :sharded && SpecConfig.instance.addresses.length > 1
           skip 'Test requires a single shard if run in sharded topology'
         end
+        if ClusterConfig.instance.topology == :load_balanced && !SpecConfig.instance.single_mongos?
+          skip 'Test requires a minimum of two shards if run in sharded topology'
+        end
       end
     end
 
     def require_wired_tiger
       before(:all) do
-        if ClusterConfig.instance.storage_engine != :wired_tiger
+        # Storage detection fails for serverless instances. However, it is safe to
+        # assume that a serverless instance uses WiredTiger Storage Engine.
+        if !SpecConfig.instance.serverless? && ClusterConfig.instance.storage_engine != :wired_tiger
           skip 'Test requires WiredTiger storage engine'
         end
       end
@@ -247,7 +297,9 @@ module Mrss
     def require_wired_tiger_on_36
       before(:all) do
         if ClusterConfig.instance.short_server_version >= '3.6'
-          if ClusterConfig.instance.storage_engine != :wired_tiger
+          # Storage detection fails for serverless instances. However, it is safe to
+          # assume that a serverless instance uses WiredTiger Storage Engine.
+          if !SpecConfig.instance.serverless? && ClusterConfig.instance.storage_engine != :wired_tiger
             skip 'Test requires WiredTiger storage engine on 3.6+ servers'
           end
         end
@@ -256,7 +308,7 @@ module Mrss
 
     def require_mmapv1
       before(:all) do
-        if ClusterConfig.instance.storage_engine != :mmapv1
+        if SpecConfig.instance.serverless? || ClusterConfig.instance.storage_engine != :mmapv1
           skip 'Test requires MMAPv1 storage engine'
         end
       end
@@ -299,5 +351,29 @@ module Mrss
         end
       end
     end
+
+    def require_required_api_version
+      before(:all) do
+        unless ENV['API_VERSION_REQUIRED'] == '1'
+          skip 'Set API_VERSION_REQUIRED=1 to run this test'
+        end
+      end
+    end
+
+    def require_no_required_api_version
+      before(:all) do
+        if ENV['API_VERSION_REQUIRED'] == '1'
+          skip 'Cannot have API_VERSION_REQUIRED=1 to run this test'
+        end
+      end
+    end
+
+    def require_unix_socket
+      before(:all) do
+        if ENV['TOPOLOGY'] == 'load-balanced'
+          skip 'Load balancer does not listen on Unix sockets'
+        end
+      end
+    end
   end
 end