mongoid 7.1.6 → 7.1.10

data/spec/mongoid/factory_spec.rb

@@ -109,11 +109,11 @@ describe Mongoid::Factory do
     context "when the attributes are nil" do
 
       let(:document) do
-        described_class.from_db(Address, nil)
+        described_class.from_db(Animal, nil)
       end
 
       it "generates based on the provided class" do
-        expect(document).to be_a(Address)
+        expect(document).to be_a(Animal)
       end
 
       it "sets the attributes to empty" do

data/spec/mongoid/persistable/savable_spec.rb

@@ -169,8 +169,8 @@ describe Mongoid::Persistable::Savable do
           Person.create(title: "Blah")
         end
 
-        let!(:address) do
-          person.addresses.build(street: "Bond St")
+        let!(:symptom) do
+          person.symptoms.build(name: "Headache")
         end
 
         let!(:name) do
@@ -193,7 +193,7 @@ describe Mongoid::Persistable::Savable do
               "name.first_name" => "Ryan"
             },
             "$push"=> {
-              "addresses" => { '$each' => [ { "_id" => address.id, "street" => "Bond St" } ] }
+              "symptoms" => { '$each' => [ { "_id" => symptom.id, "name" => "Headache" } ] }
             }
           })
         end
@@ -205,7 +205,7 @@ describe Mongoid::Persistable::Savable do
           end
 
           it "saves embedded many relations" do
-            expect(person.addresses.first.street).to eq("Bond St")
+            expect(person.symptoms.first.name).to eq("Headache")
           end
 
           it "saves embedded one relations" do

data/spec/mongoid/persistable/settable_spec.rb

@@ -512,4 +512,34 @@ describe Mongoid::Persistable::Settable do
       end
     end
   end
+
+  context "when the field being set was projected out" do
+    let(:full_agent) do
+      Agent.create!(title: "Double-Oh Eight")
+    end
+
+    let(:agent) do
+      Agent.where(_id: full_agent.id).only(:dob).first
+    end
+
+    context 'field exists in database' do
+      it "raises MissingAttributeError" do
+        lambda do
+          agent.set(title: '008')
+        end.should raise_error(ActiveModel::MissingAttributeError)
+
+        expect(agent.reload.title).to eq 'Double-Oh Eight'
+      end
+    end
+
+    context 'field does not exist in database' do
+      it "raises MissingAttributeError" do
+        lambda do
+          agent.set(number: '008')
+        end.should raise_error(ActiveModel::MissingAttributeError)
+
+        expect(agent.reload.read_attribute(:number)).to be nil
+      end
+    end
+  end
 end

data/spec/mongoid/persistable_spec.rb

@@ -178,8 +178,8 @@ describe Mongoid::Persistable do
 
         before do
           class Band
-            def my_updates(*args)
-              atomically(*args) do |d|
+            def my_updates(**args)
+              atomically(**args) do |d|
                 d.set(name: "Placebo")
                 d.unset(:origin)
               end

data/spec/shared/bin/get-mongodb-download-url

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+desired_version, arch = ARGV
+if arch.nil?
+  STDERR.puts "Usage: get-mongodb-download-url desired-version arch"
+  exit 1
+end
+
+$: << File.join(File.dirname(__FILE__), '../lib')
+require 'mrss/server_version_registry'
+
+begin
+  puts Mrss::ServerVersionRegistry.new(desired_version, arch).download_url
+rescue Mrss::ServerVersionRegistry::Error => exc
+  STDERR.puts "Error: #{exc}"
+  exit 2
+end

data/spec/shared/bin/s3-copy

@@ -0,0 +1,45 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'aws-sdk-s3'
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Usage: s3-copy options"
+
+  opts.on("-r", "--region=REGION", "AWS region to use (default us-east-1)") do |v|
+    options[:region] = v
+  end
+
+  opts.on("-p", "--param=KEY=VALUE", "Specify parameter for new files") do |v|
+    options[:params] ||= {}
+    k, v = v.split('=', 2)
+    options[:params][k.to_sym] = v
+  end
+
+  opts.on("-f", "--from=BUCKET:PATH", "Bucket name and key (or path) to copy from") do |v|
+    options[:from] = v
+  end
+
+  opts.on("-t", "--to=BUCKET:PATH", "Bucket name and key (or path) to write to (may be specified more than once)") do |v|
+    options[:to] ||= []
+    options[:to] << v
+  end
+end.parse!
+
+ENV['AWS_REGION'] ||= options[:region] || 'us-east-1'
+
+bucket, key = options.fetch(:from).split(':', 2)
+
+s3 = Aws::S3::Client.new
+
+options.fetch(:to).each do |dest|
+  STDERR.puts "Copying to #{dest}"
+  dbucket, dkey = dest.split(':', 2)
+  s3.copy_object(
+    bucket: dbucket,
+    key: dkey,
+    copy_source: "/#{bucket}/#{key}",
+    **options[:params] || {},
+  )
+end

data/spec/shared/bin/s3-upload

@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'aws-sdk-s3'
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Usage: s3-upload options"
+
+  opts.on("-r", "--region=REGION", "AWS region to use (default us-east-1)") do |v|
+    options[:region] = v
+  end
+
+  opts.on("-p", "--param=KEY=VALUE", "Specify parameter for S3 upload") do |v|
+    options[:params] ||= {}
+    k, v = v.split('=', 2)
+    options[:params][k.to_sym] = v
+  end
+
+  opts.on("-f", "--file=PATH", "Path to the file to upload, - to upload standard input") do |v|
+    options[:file] = v
+  end
+
+  opts.on("-w", "--write=BUCKET:PATH", "Bucket name and key (or path) to upload to") do |v|
+    options[:write] = v
+  end
+
+  opts.on("-c", "--copy=BUCKET:PATH", "Bucket name and key (or path) to copy to (may be specified more than once)") do |v|
+    options[:copy] ||= []
+    options[:copy] << v
+  end
+end.parse!
+
+ENV['AWS_REGION'] ||= options[:region] || 'us-east-1'
+
+def upload(f, options)
+  s3 = Aws::S3::Client.new
+  write = options.fetch(:write)
+  STDERR.puts "Writing #{write}"
+  bucket, key = write.split(':', 2)
+  s3.put_object(
+    body: f.read,
+    bucket: bucket,
+    key: key,
+    **options[:params] || {},
+  )
+  if copy = options[:copy]
+    copy.each do |dest|
+      STDERR.puts "Copying to #{dest}"
+      dbucket, dkey = dest.split(':', 2)
+      s3.copy_object(
+        bucket: dbucket,
+        key: dkey,
+        copy_source: "/#{bucket}/#{key}",
+        **options[:params] || {},
+      )
+    end
+  end
+end
+
+if options[:file] == '-'
+  upload(STDIN, options)
+elsif options[:file]
+  File.open(options[:file]) do |f|
+    upload(f, options)
+  end
+else
+  upload(STDIN, options)
+end

data/spec/shared/lib/mrss/cluster_config.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # ClusterConfig requires ClientRegistry class provided by the host project.
 
 require 'singleton'
@@ -12,6 +15,11 @@ module Mrss
       @single_server
     end
 
+    def sharded_ish?
+      determine_cluster_config
+      @topology == :sharded || @topology == :load_balanced
+    end
+
     def replica_set_name
       determine_cluster_config
       @replica_set_name
@@ -45,7 +53,7 @@ module Mrss
         raise "Deployment server version not known - check that connection to deployment succeeded"
       end
 
-      if server_version >= '3.4' && topology != :sharded
+      if server_version >= '3.4' && !sharded_ish?
         fcv
       else
         if short_server_version == '4.1'
@@ -82,6 +90,11 @@ module Mrss
       @primary_description
     end
 
+    def server_parameters
+      determine_cluster_config
+      @server_parameters
+    end
+
     # Try running a command on the admin database to see if the mongod was
     # started with auth.
     def auth_enabled?
@@ -107,7 +120,7 @@ module Mrss
           :mmapv1
         else
           client = ClientRegistry.instance.global_client('root_authorized')
-          if topology == :sharded
+          if sharded_ish?
             shards = client.use(:admin).command(listShards: 1).first
             if shards['shards'].empty?
               raise 'Shards are empty'
@@ -196,8 +209,10 @@ module Mrss
       @server_version = build_info['version']
       @enterprise = build_info['modules'] && build_info['modules'].include?('enterprise')
 
-      if @topology != :sharded && short_server_version >= '3.4'
-        rv = client.use(:admin).command(getParameter: 1, featureCompatibilityVersion: 1).first['featureCompatibilityVersion']
+      @server_parameters = client.use(:admin).command(getParameter: '*').first
+
+      if !sharded_ish? && short_server_version >= '3.4'
+        rv = @server_parameters['featureCompatibilityVersion']
         @fcv = rv['version'] || rv
       end
     end

data/spec/shared/lib/mrss/constraints.rb

@@ -52,7 +52,7 @@ module Mrss
     end
 
     def require_topology(*topologies)
-      invalid_topologies = topologies - [:single, :replica_set, :sharded]
+      invalid_topologies = topologies - [:single, :replica_set, :sharded, :load_balanced]
 
       unless invalid_topologies.empty?
         raise ArgumentError, "Invalid topologies requested: #{invalid_topologies.join(', ')}"
@@ -82,7 +82,7 @@ module Mrss
           unless ClusterConfig.instance.server_version >= '4.0'
             skip 'Transactions tests in a replica set topology require server 4.0+'
           end
-        when :sharded
+        when :sharded, :load_balanced
           unless ClusterConfig.instance.server_version >= '4.2'
             skip 'Transactions tests in a sharded cluster topology require server 4.2+'
           end
@@ -113,6 +113,14 @@ module Mrss
       end
     end
 
+    def require_retry_writes
+      before(:all) do
+        unless SpecConfig.instance.retry_writes?
+          skip "Retry writes is disabled"
+        end
+      end
+    end
+
     def require_no_retry_writes
       before(:all) do
         if SpecConfig.instance.retry_writes?
@@ -156,6 +164,24 @@ module Mrss
       end
     end
 
+    def require_zstd_compression
+      before(:all) do
+        compressors = SpecConfig.instance.compressors
+        unless compressors && compressors.include?('zstd')
+          skip "Zstd compression is not enabled"
+        end
+      end
+    end
+
+    def require_no_zstd_compression
+      before(:all) do
+        compressors = SpecConfig.instance.compressors
+        if compressors && compressors.include?('zstd')
+          skip "Zstd compression is enabled"
+        end
+      end
+    end
+
     def require_no_compression
       before(:all) do
         if SpecConfig.instance.compressors
@@ -223,7 +249,9 @@ module Mrss
     # (mongos 4.0+ overrides the write concern)
     def require_set_write_concern
       before(:all) do
-        if ClusterConfig.instance.topology == :sharded && ClusterConfig.instance.short_server_version >= '4.0'
+        if %i(sharded load_balanced).include?(ClusterConfig.instance.topology) &&
+          ClusterConfig.instance.short_server_version >= '4.0'
+        then
           skip "mongos 4.0+ overrides write concern"
         end
       end
@@ -247,7 +275,9 @@ module Mrss
 
     def require_wired_tiger
       before(:all) do
-        if ClusterConfig.instance.storage_engine != :wired_tiger
+        # Storage detection fails for serverless instances. However, it is safe to
+        # assume that a serverless instance uses WiredTiger Storage Engine.
+        if !SpecConfig.instance.serverless? && ClusterConfig.instance.storage_engine != :wired_tiger
           skip 'Test requires WiredTiger storage engine'
         end
       end
@@ -256,7 +286,9 @@ module Mrss
     def require_wired_tiger_on_36
       before(:all) do
         if ClusterConfig.instance.short_server_version >= '3.6'
-          if ClusterConfig.instance.storage_engine != :wired_tiger
+          # Storage detection fails for serverless instances. However, it is safe to
+          # assume that a serverless instance uses WiredTiger Storage Engine.
+          if !SpecConfig.instance.serverless? && ClusterConfig.instance.storage_engine != :wired_tiger
             skip 'Test requires WiredTiger storage engine on 3.6+ servers'
           end
         end
@@ -265,7 +297,7 @@ module Mrss
 
     def require_mmapv1
       before(:all) do
-        if ClusterConfig.instance.storage_engine != :mmapv1
+        if SpecConfig.instance.serverless? || ClusterConfig.instance.storage_engine != :mmapv1
           skip 'Test requires MMAPv1 storage engine'
         end
       end
@@ -308,5 +340,29 @@ module Mrss
         end
       end
     end
+
+    def require_required_api_version
+      before(:all) do
+        unless ENV['API_VERSION_REQUIRED'] == '1'
+          skip 'Set API_VERSION_REQUIRED=1 to run this test'
+        end
+      end
+    end
+
+    def require_no_required_api_version
+      before(:all) do
+        if ENV['API_VERSION_REQUIRED'] == '1'
+          skip 'Cannot have API_VERSION_REQUIRED=1 to run this test'
+        end
+      end
+    end
+
+    def require_unix_socket
+      before(:all) do
+        if ENV['TOPOLOGY'] == 'load-balanced'
+          skip 'Load balancer does not listen on Unix sockets'
+        end
+      end
+    end
   end
 end

data/spec/shared/lib/mrss/docker_runner.rb

@@ -0,0 +1,271 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+require 'optparse'
+require 'erb'
+autoload :Dotenv, 'dotenv'
+
+module Mrss
+  autoload :ServerVersionRegistry, 'mrss/server_version_registry'
+
+  class DockerRunner
+    def initialize(**opts)
+      # These options are required:
+      opts.fetch(:image_tag)
+      opts.fetch(:dockerfile_path)
+      opts.fetch(:default_script)
+      opts.fetch(:project_lib_subdir)
+
+      @options = opts
+    end
+
+    attr_reader :options
+
+    def run
+      process_arguments
+      unless @options[:exec_only]
+        create_dockerfile
+        create_image
+      end
+      if @options[:mongo_only]
+        run_deployment
+      else
+        run_tests
+      end
+    end
+
+    private
+
+    def process_arguments
+      #@options = {}
+      OptionParser.new do |opts|
+        opts.banner = "Usage: test-on-docker [-d distro] [evergreen_key=value ...]"
+
+        opts.on("-a", "--add-env=PATH", "Load environment variables from PATH in .env format") do |path|
+          @options[:extra_env] ||= {}
+          unless File.exist?(path)
+            raise "-a option references nonexistent file #{path}"
+          end
+          Dotenv.parse(path).each do |k, v|
+            @options[:extra_env][k] = v
+          end
+        end
+
+        opts.on("-d", "--distro=DISTRO", "Distro to use") do |v|
+          @options[:distro] = v
+        end
+
+        opts.on('-e', '--exec-only', 'Execute tests using existing Dockerfile (for offline user)') do |v|
+          @options[:exec_only] = v
+        end
+
+        opts.on('-m', '--mongo-only=PORT', 'Start the MongoDB deployment and expose it to host on ports starting with PORT') do |v|
+          @options[:mongo_only] = v.to_i
+        end
+
+        opts.on('-p', '--preload', 'Preload Ruby toolchain and server binaries in docker') do |v|
+          @options[:preload] = v
+        end
+
+        opts.on('-s', '--script=SCRIPT', 'Test script to invoke') do |v|
+          @options[:script] = v
+        end
+
+        opts.on('-i', '--interactive', 'Interactive mode - disable per-test timeouts') do |v|
+          @options[:interactive] = v
+        end
+      end.parse!
+
+      @env = Hash[ARGV.map do |arg|
+        arg.split('=', 2)
+      end]
+
+      @env['RVM_RUBY'] ||= 'ruby-2.7'
+      unless ruby =~ /^j?ruby-/
+        raise "RVM_RUBY option is not in expected format: #{ruby}"
+      end
+
+      @env['MONGODB_VERSION'] ||= '4.4'
+    end
+
+    def create_dockerfile
+      template_path = File.join(File.dirname(__FILE__), '../../share/Dockerfile.erb')
+      result = ERB.new(File.read(template_path)).result(binding)
+      File.open(dockerfile_path, 'w') do |f|
+        f << result
+      end
+    end
+
+    def image_tag
+      options.fetch(:image_tag)
+    end
+
+    def dockerfile_path
+      options.fetch(:dockerfile_path)
+    end
+
+    def create_image
+      run_command(['docker', 'build',
+        '-t', image_tag,
+        '-f', dockerfile_path,
+        '.'])
+    end
+
+    BASE_TEST_COMMAND = %w(docker run --rm -i --tmpfs /tmpfs:exec).freeze
+
+    def run_tests
+      run_command(BASE_TEST_COMMAND + tty_arg + extra_env + [image_tag] +
+        script.split(/\s+/))
+    end
+
+    def run_deployment
+      run_command(BASE_TEST_COMMAND + tty_arg + extra_env + [
+        '-e', %q`TEST_CMD=watch -x bash -c "ps awwxu |egrep 'mongo|ocsp'"`,
+        '-e', 'BIND_ALL=true',
+      ] + port_forwards + [image_tag] + script.split(/\s+/))
+    end
+
+    def tty_arg
+      tty = File.open('/dev/stdin') do |f|
+        f.isatty
+      end
+      if tty
+        %w(-t --init)
+      else
+        []
+      end
+    end
+
+    def extra_env
+      if @options[:extra_env]
+        @options[:extra_env].map do |k, v|
+          # Here the value must not be escaped
+          ['-e', "#{k}=#{v}"]
+        end.flatten
+      else
+        []
+      end
+    end
+
+    def port_forwards
+      args = (0...num_exposed_ports).map do |i|
+        host_port = @options[:mongo_only] + i
+        container_port = 27017 + i
+        ['-p', "#{host_port}:#{container_port}"]
+      end.flatten
+
+      if @env['OCSP_ALGORITHM'] && !@env['OCSP_VERIFIER']
+        args += %w(-p 8100:8100)
+      end
+
+      args
+    end
+
+    def run_command(cmd)
+      if pid = fork
+        Process.wait(pid)
+        unless $?.exitstatus == 0
+          raise "Process exited with code #{$?.exitstatus}"
+        end
+      else
+        exec(*cmd)
+      end
+    end
+
+    def distro
+      @options[:distro] || 'ubuntu1604'
+    end
+
+    BASE_IMAGES = {
+      'debian81' => 'debian:jessie',
+      'debian92' => 'debian:stretch',
+      'debian10' => 'debian:buster',
+      'ubuntu1404' => 'ubuntu:trusty',
+      'ubuntu1604' => 'ubuntu:xenial',
+      'ubuntu1804' => 'ubuntu:bionic',
+      'ubuntu2004' => 'ubuntu:focal',
+      'rhel62' => 'centos:6',
+      'rhel70' => 'centos:7',
+    }.freeze
+
+    def base_image
+      BASE_IMAGES[distro] or raise "Unknown distro: #{distro}"
+    end
+
+    def ruby
+      @env['RVM_RUBY']
+    end
+
+    def ruby_head?
+      ruby == 'ruby-head'
+    end
+
+    def system_ruby?
+      %w(1 true yes).include?(@env['SYSTEM_RUBY']&.downcase)
+    end
+
+    def server_version
+      @env['MONGODB_VERSION']
+    end
+
+    def script
+      @options[:script] || options.fetch(:default_script)
+    end
+
+    def debian?
+      distro =~ /debian|ubuntu/
+    end
+
+    def preload?
+      !!@options[:preload]
+    end
+
+    def interactive?
+      !!@options[:interactive]
+    end
+
+    def project_lib_subdir
+      options.fetch(:project_lib_subdir)
+    end
+
+    def server_download_url
+      @server_download_url ||= ServerVersionRegistry.new(server_version, distro).download_url
+    end
+
+    def libmongocrypt_path
+      case distro
+      when /ubuntu1604/
+        "./ubuntu1604/nocrypto/lib64/libmongocrypt.so"
+      when /ubuntu1804/
+        "./ubuntu1804-64/nocrypto/lib64/libmongocrypt.so"
+      when /debian92/
+        "./debian92/nocrypto/lib64/libmongocrypt.so"
+      else
+        raise "This script does not support running FLE tests on #{distro}. Use ubuntu1604, ubuntu1804 or debian92 instead"
+      end
+    end
+
+    def expose?
+      !!@options[:mongo_only]
+    end
+
+    def fle?
+      %w(1 true yes).include?(@env['FLE']&.downcase)
+    end
+
+    def num_exposed_ports
+      case @env['TOPOLOGY'] || 'standalone'
+      when 'standalone'
+        1
+      when 'replica-set'
+        3
+      when 'sharded-cluster'
+        if @env['SINGLE_MONGOS']
+          1
+        else
+          2
+        end
+      end
+    end
+  end
+end