mongoid 7.1.7 → 7.1.11

data/spec/shared/bin/get-mongodb-download-url

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+desired_version, arch = ARGV
+if arch.nil?
+  STDERR.puts "Usage: get-mongodb-download-url desired-version arch"
+  exit 1
+end
+
+$: << File.join(File.dirname(__FILE__), '../lib')
+require 'mrss/server_version_registry'
+
+begin
+  puts Mrss::ServerVersionRegistry.new(desired_version, arch).download_url
+rescue Mrss::ServerVersionRegistry::Error => exc
+  STDERR.puts "Error: #{exc}"
+  exit 2
+end

data/spec/shared/bin/s3-copy

@@ -0,0 +1,45 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'aws-sdk-s3'
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Usage: s3-copy options"
+
+  opts.on("-r", "--region=REGION", "AWS region to use (default us-east-1)") do |v|
+    options[:region] = v
+  end
+
+  opts.on("-p", "--param=KEY=VALUE", "Specify parameter for new files") do |v|
+    options[:params] ||= {}
+    k, v = v.split('=', 2)
+    options[:params][k.to_sym] = v
+  end
+
+  opts.on("-f", "--from=BUCKET:PATH", "Bucket name and key (or path) to copy from") do |v|
+    options[:from] = v
+  end
+
+  opts.on("-t", "--to=BUCKET:PATH", "Bucket name and key (or path) to write to (may be specified more than once)") do |v|
+    options[:to] ||= []
+    options[:to] << v
+  end
+end.parse!
+
+ENV['AWS_REGION'] ||= options[:region] || 'us-east-1'
+
+bucket, key = options.fetch(:from).split(':', 2)
+
+s3 = Aws::S3::Client.new
+
+options.fetch(:to).each do |dest|
+  STDERR.puts "Copying to #{dest}"
+  dbucket, dkey = dest.split(':', 2)
+  s3.copy_object(
+    bucket: dbucket,
+    key: dkey,
+    copy_source: "/#{bucket}/#{key}",
+    **options[:params] || {},
+  )
+end

data/spec/shared/bin/s3-upload

@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'aws-sdk-s3'
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Usage: s3-upload options"
+
+  opts.on("-r", "--region=REGION", "AWS region to use (default us-east-1)") do |v|
+    options[:region] = v
+  end
+
+  opts.on("-p", "--param=KEY=VALUE", "Specify parameter for S3 upload") do |v|
+    options[:params] ||= {}
+    k, v = v.split('=', 2)
+    options[:params][k.to_sym] = v
+  end
+
+  opts.on("-f", "--file=PATH", "Path to the file to upload, - to upload standard input") do |v|
+    options[:file] = v
+  end
+
+  opts.on("-w", "--write=BUCKET:PATH", "Bucket name and key (or path) to upload to") do |v|
+    options[:write] = v
+  end
+
+  opts.on("-c", "--copy=BUCKET:PATH", "Bucket name and key (or path) to copy to (may be specified more than once)") do |v|
+    options[:copy] ||= []
+    options[:copy] << v
+  end
+end.parse!
+
+ENV['AWS_REGION'] ||= options[:region] || 'us-east-1'
+
+def upload(f, options)
+  s3 = Aws::S3::Client.new
+  write = options.fetch(:write)
+  STDERR.puts "Writing #{write}"
+  bucket, key = write.split(':', 2)
+  s3.put_object(
+    body: f.read,
+    bucket: bucket,
+    key: key,
+    **options[:params] || {},
+  )
+  if copy = options[:copy]
+    copy.each do |dest|
+      STDERR.puts "Copying to #{dest}"
+      dbucket, dkey = dest.split(':', 2)
+      s3.copy_object(
+        bucket: dbucket,
+        key: dkey,
+        copy_source: "/#{bucket}/#{key}",
+        **options[:params] || {},
+      )
+    end
+  end
+end
+
+if options[:file] == '-'
+  upload(STDIN, options)
+elsif options[:file]
+  File.open(options[:file]) do |f|
+    upload(f, options)
+  end
+else
+  upload(STDIN, options)
+end

data/spec/shared/lib/mrss/cluster_config.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # ClusterConfig requires ClientRegistry class provided by the host project.
 
 require 'singleton'
@@ -12,6 +15,11 @@ module Mrss
       @single_server
     end
 
+    def sharded_ish?
+      determine_cluster_config
+      @topology == :sharded || @topology == :load_balanced
+    end
+
     def replica_set_name
       determine_cluster_config
       @replica_set_name
@@ -45,7 +53,7 @@ module Mrss
         raise "Deployment server version not known - check that connection to deployment succeeded"
       end
 
-      if server_version >= '3.4' && topology != :sharded
+      if server_version >= '3.4' && !sharded_ish?
         fcv
       else
         if short_server_version == '4.1'
@@ -82,6 +90,11 @@ module Mrss
       @primary_description
     end
 
+    def server_parameters
+      determine_cluster_config
+      @server_parameters
+    end
+
     # Try running a command on the admin database to see if the mongod was
     # started with auth.
     def auth_enabled?
@@ -107,7 +120,7 @@ module Mrss
           :mmapv1
         else
           client = ClientRegistry.instance.global_client('root_authorized')
-          if topology == :sharded
+          if sharded_ish?
             shards = client.use(:admin).command(listShards: 1).first
             if shards['shards'].empty?
               raise 'Shards are empty'
@@ -196,8 +209,10 @@ module Mrss
       @server_version = build_info['version']
       @enterprise = build_info['modules'] && build_info['modules'].include?('enterprise')
 
-      if @topology != :sharded && short_server_version >= '3.4'
-        rv = client.use(:admin).command(getParameter: 1, featureCompatibilityVersion: 1).first['featureCompatibilityVersion']
+      @server_parameters = client.use(:admin).command(getParameter: '*').first
+
+      if !sharded_ish? && short_server_version >= '3.4'
+        rv = @server_parameters['featureCompatibilityVersion']
         @fcv = rv['version'] || rv
       end
     end

data/spec/shared/lib/mrss/constraints.rb

@@ -52,7 +52,7 @@ module Mrss
     end
 
     def require_topology(*topologies)
-      invalid_topologies = topologies - [:single, :replica_set, :sharded]
+      invalid_topologies = topologies - [:single, :replica_set, :sharded, :load_balanced]
 
       unless invalid_topologies.empty?
         raise ArgumentError, "Invalid topologies requested: #{invalid_topologies.join(', ')}"
@@ -82,7 +82,7 @@ module Mrss
           unless ClusterConfig.instance.server_version >= '4.0'
             skip 'Transactions tests in a replica set topology require server 4.0+'
           end
-        when :sharded
+        when :sharded, :load_balanced
           unless ClusterConfig.instance.server_version >= '4.2'
             skip 'Transactions tests in a sharded cluster topology require server 4.2+'
           end
@@ -113,6 +113,14 @@ module Mrss
       end
     end
 
+    def require_retry_writes
+      before(:all) do
+        unless SpecConfig.instance.retry_writes?
+          skip "Retry writes is disabled"
+        end
+      end
+    end
+
     def require_no_retry_writes
       before(:all) do
         if SpecConfig.instance.retry_writes?
@@ -172,8 +180,8 @@ module Mrss
           skip "Zstd compression is enabled"
         end
       end
-    end    
-    
+    end
+
     def require_no_compression
       before(:all) do
         if SpecConfig.instance.compressors
@@ -241,31 +249,52 @@ module Mrss
     # (mongos 4.0+ overrides the write concern)
     def require_set_write_concern
       before(:all) do
-        if ClusterConfig.instance.topology == :sharded && ClusterConfig.instance.short_server_version >= '4.0'
+        if %i(sharded load_balanced).include?(ClusterConfig.instance.topology) &&
+          ClusterConfig.instance.short_server_version >= '4.0'
+        then
           skip "mongos 4.0+ overrides write concern"
         end
       end
     end
 
-    def require_multi_shard
+    def require_multi_mongos
       before(:all) do
         if ClusterConfig.instance.topology == :sharded && SpecConfig.instance.addresses.length == 1
-          skip 'Test requires a minimum of two shards if run in sharded topology'
+          skip 'Test requires a minimum of two mongoses if run in sharded topology'
+        end
+
+        if ClusterConfig.instance.topology == :load_balanced && SpecConfig.instance.single_mongos?
+          skip 'Test requires a minimum of two mongoses if run in load-balanced topology'
         end
       end
     end
 
-    def require_no_multi_shard
+    # In sharded topology operations are distributed to the mongoses.
+    # When we set fail points, the fail point may be set on one mongos and
+    # operation may be executed on another mongos, causing failures.
+    # Tests that are not setting targeted fail points should utilize this
+    # method to restrict themselves to single mongos.
+    #
+    # In load-balanced topology, the same problem can happen when there is
+    # more than one mongos behind the load balancer.
+    def require_no_multi_mongos
       before(:all) do
         if ClusterConfig.instance.topology == :sharded && SpecConfig.instance.addresses.length > 1
-          skip 'Test requires a single shard if run in sharded topology'
+          skip 'Test requires a single mongos if run in sharded topology'
+        end
+        if ClusterConfig.instance.topology == :load_balanced && !SpecConfig.instance.single_mongos?
+          skip 'Test requires a single mongos, as indicated by SINGLE_MONGOS=1 environment variable, if run in load-balanced topology'
         end
       end
     end
 
+    alias :require_no_multi_shard :require_no_multi_mongos
+
     def require_wired_tiger
       before(:all) do
-        if ClusterConfig.instance.storage_engine != :wired_tiger
+        # Storage detection fails for serverless instances. However, it is safe to
+        # assume that a serverless instance uses WiredTiger Storage Engine.
+        if !SpecConfig.instance.serverless? && ClusterConfig.instance.storage_engine != :wired_tiger
           skip 'Test requires WiredTiger storage engine'
         end
       end
@@ -274,7 +303,9 @@ module Mrss
     def require_wired_tiger_on_36
       before(:all) do
         if ClusterConfig.instance.short_server_version >= '3.6'
-          if ClusterConfig.instance.storage_engine != :wired_tiger
+          # Storage detection fails for serverless instances. However, it is safe to
+          # assume that a serverless instance uses WiredTiger Storage Engine.
+          if !SpecConfig.instance.serverless? && ClusterConfig.instance.storage_engine != :wired_tiger
             skip 'Test requires WiredTiger storage engine on 3.6+ servers'
           end
         end
@@ -283,7 +314,7 @@ module Mrss
 
     def require_mmapv1
       before(:all) do
-        if ClusterConfig.instance.storage_engine != :mmapv1
+        if SpecConfig.instance.serverless? || ClusterConfig.instance.storage_engine != :mmapv1
           skip 'Test requires MMAPv1 storage engine'
         end
       end
@@ -326,5 +357,29 @@ module Mrss
         end
       end
     end
+
+    def require_required_api_version
+      before(:all) do
+        unless ENV['API_VERSION_REQUIRED'] == '1'
+          skip 'Set API_VERSION_REQUIRED=1 to run this test'
+        end
+      end
+    end
+
+    def require_no_required_api_version
+      before(:all) do
+        if ENV['API_VERSION_REQUIRED'] == '1'
+          skip 'Cannot have API_VERSION_REQUIRED=1 to run this test'
+        end
+      end
+    end
+
+    def require_unix_socket
+      before(:all) do
+        if ENV['TOPOLOGY'] == 'load-balanced'
+          skip 'Load balancer does not listen on Unix sockets'
+        end
+      end
+    end
   end
 end

data/spec/shared/lib/mrss/docker_runner.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 require 'optparse'
 require 'erb'
 autoload :Dotenv, 'dotenv'
@@ -108,7 +111,7 @@ module Mrss
         '.'])
     end
 
-    BASE_TEST_COMMAND = %w(docker run -i --tmpfs /tmpfs:exec).freeze
+    BASE_TEST_COMMAND = %w(docker run --rm -i --tmpfs /tmpfs:exec).freeze
 
     def run_tests
       run_command(BASE_TEST_COMMAND + tty_arg + extra_env + [image_tag] +
@@ -176,9 +179,11 @@ module Mrss
     BASE_IMAGES = {
       'debian81' => 'debian:jessie',
       'debian92' => 'debian:stretch',
+      'debian10' => 'debian:buster',
       'ubuntu1404' => 'ubuntu:trusty',
       'ubuntu1604' => 'ubuntu:xenial',
       'ubuntu1804' => 'ubuntu:bionic',
+      'ubuntu2004' => 'ubuntu:focal',
       'rhel62' => 'centos:6',
       'rhel70' => 'centos:7',
     }.freeze
@@ -195,6 +200,10 @@ module Mrss
       ruby == 'ruby-head'
     end
 
+    def system_ruby?
+      %w(1 true yes).include?(@env['SYSTEM_RUBY']&.downcase)
+    end
+
     def server_version
       @env['MONGODB_VERSION']
     end

data/spec/shared/lib/mrss/event_subscriber.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+module Mrss
+  # Test event subscriber.
+  class EventSubscriber
+
+    # The mappings of event names to types.
+    MAPPINGS = {
+      'topology_opening_event' => Mongo::Monitoring::Event::TopologyOpening,
+      'topology_description_changed_event' => Mongo::Monitoring::Event::TopologyChanged,
+      'topology_closed_event' => Mongo::Monitoring::Event::TopologyClosed,
+      'server_opening_event' => Mongo::Monitoring::Event::ServerOpening,
+      'server_description_changed_event' => Mongo::Monitoring::Event::ServerDescriptionChanged,
+      'server_closed_event' => Mongo::Monitoring::Event::ServerClosed
+    }.freeze
+
+    attr_reader :all_events
+
+    attr_reader :started_events
+
+    attr_reader :succeeded_events
+
+    attr_reader :failed_events
+
+    attr_reader :published_events
+
+    # @param [ String ] name Optional name for the event subscriber.
+    def initialize(name: nil)
+      @mutex = Mutex.new
+      clear_events!
+      @name = name
+    end
+
+    def to_s
+      %Q`#<EventSubscriber:#{@name ? "\"#{@name}\"" : '%x' % object_id} \
+  started=#{started_events.length} \
+  succeeded=#{succeeded_events.length} \
+  failed=#{failed_events.length} \
+  published=#{published_events.length}>`
+    end
+
+    alias :inspect :to_s
+
+    # Event retrieval
+
+    def select_started_events(cls)
+      started_events.select do |event|
+        event.is_a?(cls)
+      end
+    end
+
+    def select_succeeded_events(cls)
+      succeeded_events.select do |event|
+        event.is_a?(cls)
+      end
+    end
+
+    def select_completed_events(*classes)
+      (succeeded_events + failed_events).select do |event|
+        classes.any? { |c| c === event }
+      end
+    end
+
+    def select_published_events(cls)
+      published_events.select do |event|
+        event.is_a?(cls)
+      end
+    end
+
+    # Filters command started events for the specified command name.
+    def command_started_events(command_name)
+      started_events.select do |event|
+        event.command[command_name]
+      end
+    end
+
+    def non_auth_command_started_events
+      started_events.reject do |event|
+        %w(authenticate getnonce saslSstart saslContinue).any? do |cmd|
+          event.command[cmd]
+        end
+      end
+    end
+
+    # Locates command stated events for the specified command name,
+    # asserts that there is exactly one such event, and returns it.
+    def single_command_started_event(command_name, include_auth: false)
+      events = if include_auth
+                 started_events
+               else
+                 non_auth_command_started_events
+               end
+      events.select! do |event|
+        event.command[command_name]
+      end
+      if events.length != 1
+        raise "Expected a single #{command_name} event but we have #{events.length}"
+      end
+      events.first
+    end
+
+
+    # Get the first succeeded event published for the name, and then delete it.
+    #
+    # @param [ String ] name The event name.
+    #
+    # @return [ Event ] The matching event.
+    def first_event(name)
+      cls = MAPPINGS[name]
+      if cls.nil?
+        raise ArgumentError, "Bogus event name #{name}"
+      end
+      matching = succeeded_events.find do |event|
+        cls === event
+      end
+      succeeded_events.delete(matching)
+      matching
+    end
+
+    # Event recording
+
+    # Cache the started event.
+    #
+    # @param [ Event ] event The event.
+    def started(event)
+      @mutex.synchronize do
+        started_events << event
+        all_events << event
+      end
+    end
+
+    # Cache the succeeded event.
+    #
+    # @param [ Event ] event The event.
+    def succeeded(event)
+      @mutex.synchronize do
+        succeeded_events << event
+        all_events << event
+      end
+    end
+
+    # Cache the failed event.
+    #
+    # @param [ Event ] event The event.
+    def failed(event)
+      @mutex.synchronize do
+        failed_events << event
+        all_events << event
+      end
+    end
+
+    def published(event)
+      @mutex.synchronize do
+        published_events << event
+        all_events << event
+      end
+    end
+
+    # Clear all cached events.
+    def clear_events!
+      @all_events = []
+      @started_events = []
+      @succeeded_events = []
+      @failed_events = []
+      @published_events = []
+      self
+    end
+  end
+  # Only handles succeeded events correctly.
+  class PhasedEventSubscriber < EventSubscriber
+    def initialize
+      super
+      @phase_events = {}
+    end
+
+    def phase_finished(phase_index)
+      @phase_events[phase_index] = succeeded_events
+      @succeeded_events = []
+    end
+
+    def phase_events(phase_index)
+      @phase_events[phase_index]
+    end
+
+    def event_count
+      @phase_events.inject(0) do |sum, event|
+        sum + event.length
+      end
+    end
+  end
+
+  class VerboseEventSubscriber < EventSubscriber
+    %w(started succeeded failed published).each do |meth|
+      define_method(meth) do |event|
+        puts event.summary
+        super(event)
+      end
+    end
+  end
+end