mongo 2.17.2 → 2.18.1

data/spec/runners/unified/crud_operations.rb

@@ -10,6 +10,16 @@ module Unified
       use_arguments(op) do |args|
         opts = {
           let: args.use('let'),
+          comment: args.use('comment'),
+          allow_disk_use: args.use('allowDiskUse'),
+          show_disk_loc: args.use('showRecordId'),
+          return_key: args.use('returnKey'),
+          projection: args.use('projection'),
+          skip: args.use('skip'),
+          hint: args.use('hint'),
+          max_value: args.use('max'),
+          max_time_ms: args.use('maxTimeMS'),
+          min_value: args.use('min'),
         }
         if session = args.use('session')
           opts[:session] = entities.get(:session, session)
@@ -24,6 +34,9 @@ module Unified
         if limit = args.use('limit')
           req = req.limit(limit)
         end
+        if projection = args.use('projection')
+          req = req.projection(projection)
+        end
         result = req.to_a
       end
     end
@@ -35,6 +48,9 @@ module Unified
         if session = args.use('session')
           opts[:session] = entities.get(:session, session)
         end
+        if comment = args.use('comment')
+          opts[:comment] = comment
+        end
         collection.find(args.use!('filter')).count_documents(**opts)
       end
     end
@@ -46,6 +62,9 @@ module Unified
         if max_time_ms = args.use('maxTimeMS')
           opts[:max_time_ms] = max_time_ms
         end
+        if comment = args.use('comment')
+          opts[:comment] = comment
+        end
         collection.estimated_document_count(**opts)
       end
     end
@@ -57,6 +76,9 @@ module Unified
         if session = args.use('session')
           opts[:session] = entities.get(:session, session)
         end
+        if comment = args.use('comment')
+          opts[:comment] = comment
+        end
         req = collection.find(args.use!('filter'), **opts).distinct(args.use!('fieldName'), **opts)
         result = req.to_a
       end
@@ -69,6 +91,8 @@ module Unified
         update = args.use!('update')
         opts = {
           let: args.use('let'),
+          comment: args.use('comment'),
+          hint: args.use('hint'),
         }
         if return_document = args.use('returnDocument')
           opts[:return_document] = return_document.downcase.to_sym
@@ -87,6 +111,8 @@ module Unified
         update = args.use!('replacement')
         opts = {
           let: args.use('let'),
+          comment: args.use('comment'),
+          hint: args.use('hint'),
         }
         if session = args.use('session')
           opts[:session] = entities.get(:session, session)
@@ -101,6 +127,8 @@ module Unified
         filter = args.use!('filter')
         opts = {
           let: args.use('let'),
+          comment: args.use('comment'),
+          hint: args.use('hint'),
         }
         if session = args.use('session')
           opts[:session] = entities.get(:session, session)
@@ -112,7 +140,9 @@ module Unified
     def insert_one(op)
       collection = entities.get(:collection, op.use!('object'))
       use_arguments(op) do |args|
-        opts = {}
+        opts = {
+          comment: args.use('comment')
+        }
         if session = args.use('session')
           opts[:session] = entities.get(:session, session)
         end
@@ -123,7 +153,9 @@ module Unified
     def insert_many(op)
       collection = entities.get(:collection, op.use!('object'))
       use_arguments(op) do |args|
-        opts = {}
+        opts = {
+          comment: args.use('comment')
+        }
         unless (ordered = args.use('ordered')).nil?
           opts[:ordered] = ordered
         end
@@ -139,6 +171,9 @@ module Unified
       use_arguments(op) do |args|
         opts = {
           let: args.use('let'),
+          comment: args.use('comment'),
+          hint: args.use('hint'),
+          upsert: args.use('upsert'),
         }
         if session = args.use('session')
           opts[:session] = entities.get(:session, session)
@@ -152,6 +187,8 @@ module Unified
       use_arguments(op) do |args|
         opts = {
           let: args.use('let'),
+          comment: args.use('comment'),
+          hint: args.use('hint'),
         }
         collection.update_many(args.use!('filter'), args.use!('update'), **opts)
       end
@@ -163,7 +200,10 @@ module Unified
         collection.replace_one(
           args.use!('filter'),
           args.use!('replacement'),
+          comment: args.use('comment'),
           upsert: args.use('upsert'),
+          let: args.use('let'),
+          hint: args.use('hint')
         )
       end
     end
@@ -173,6 +213,8 @@ module Unified
       use_arguments(op) do |args|
         opts = {
           let: args.use('let'),
+          comment: args.use('comment'),
+          hint: args.use('hint'),
         }
         if session = args.use('session')
           opts[:session] = entities.get(:session, session)
@@ -186,6 +228,8 @@ module Unified
       use_arguments(op) do |args|
         opts = {
           let: args.use('let'),
+          comment: args.use('comment'),
+          hint: args.use('hint'),
         }
         collection.delete_many(args.use!('filter'), **opts)
       end
@@ -198,8 +242,14 @@ module Unified
           convert_bulk_write_spec(req)
         end
         opts = {}
-        if ordered = args.use('ordered')
-          opts[:ordered] = true
+        if args.key?('ordered')
+          opts[:ordered] = args.use!('ordered')
+        end
+        if comment = args.use('comment')
+          opts[:comment] = comment
+        end
+        if let = args.use('let')
+          opts[:let] = let
         end
         collection.bulk_write(requests, **opts)
       end
@@ -215,6 +265,15 @@ module Unified
       if session = args.use('session')
         opts[:session] = entities.get(:session, session)
       end
+      if comment = args.use('comment')
+        opts[:comment] = comment
+      end
+      if batch_size = args.use('batchSize')
+        opts[:batch_size] = batch_size
+      end
+      if args.key?('allowDiskUse')
+        opts[:allow_disk_use] = args.use('allowDiskUse')
+      end
       unless args.empty?
         raise NotImplementedError, "Unhandled spec keys: #{args} in #{test_spec}"
       end
@@ -237,16 +296,20 @@ module Unified
           filter: spec.use('filter'),
           update: spec.use('update'),
           upsert: spec.use('upsert'),
+          array_filters: spec.use('arrayFilters'),
+          hint: spec.use('hint'),
         }
       when 'replaceOne'
         {
           filter: spec.use('filter'),
           replacement: spec.use('replacement'),
           upsert: spec.use('upsert'),
+          hint: spec.use('hint'),
         }
       when 'deleteOne', 'deleteMany'
         {
           filter: spec.use('filter'),
+          hint: spec.use('hint'),
         }
       else
         raise NotImplementedError, "Unknown operation #{op}"

data/spec/runners/unified/ddl_operations.rb

@@ -30,6 +30,15 @@ module Unified
         if expire_after_seconds = args.use('expireAfterSeconds')
           collection_opts[:expire_after] = expire_after_seconds
         end
+        if clustered_index = args.use('clusteredIndex')
+          collection_opts[:clustered_index] = clustered_index
+        end
+        if change_stream_pre_and_post_images = args.use('changeStreamPreAndPostImages')
+          collection_opts[:change_stream_pre_and_post_images] = change_stream_pre_and_post_images
+        end
+        if view_on = args.use('viewOn')
+          collection_opts[:view_on] = view_on
+        end
         database[args.use!('collection'), collection_opts].create(**opts)
       end
     end
@@ -41,6 +50,9 @@ module Unified
         if session = args.use('session')
           opts[:session] = entities.get(:session, session)
         end
+        if filter = args.use('filter')
+          opts[:filter] = filter
+        end
         database.list_collections(**opts)
       end
     end
@@ -53,6 +65,23 @@ module Unified
       end
     end
 
+    def rename(op)
+      collection = entities.get(:collection, op.use!('object'))
+      use_arguments(op) do |args|
+        to = args.use!('to')
+        cmd = {
+          renameCollection: "#{collection.database.name}.#{collection.name}",
+          to: "#{collection.database.name}.#{to}"
+        }
+
+        if args.key?("dropTarget")
+          cmd[:dropTarget] = args.use("dropTarget")
+        end
+
+        collection.client.use(:admin).command(**cmd)
+      end
+    end
+
     def assert_collection_exists(op, state = true)
       consume_test_runner(op)
       use_arguments(op) do |args|
@@ -102,6 +131,22 @@ module Unified
       end
     end
 
+    def drop_index(op)
+      collection = entities.get(:collection, op.use!('object'))
+      use_arguments(op) do |args|
+        opts = {}
+        if session = args.use('session')
+          opts[:session] = entities.get(:session, session)
+        end
+
+        collection.indexes.drop_one(
+          args.use!('name'),
+          **opts,
+        )
+      end
+    end
+
+
     def assert_index_exists(op)
       consume_test_runner(op)
       use_arguments(op) do |args|

data/spec/runners/unified/error.rb

@@ -23,6 +23,7 @@ module Unified
     class InvalidTest < Error
     end
 
+    class UnsupportedOperation < Error
+    end
   end
-
 end

data/spec/runners/unified/grid_fs_operations.rb

@@ -20,6 +20,18 @@ module Unified
       end
     end
 
+    def download_by_name(op)
+      bucket = entities.get(:bucket, op.use!('object'))
+      use_arguments(op) do |args|
+        opts = {}
+        if revision = args.use('revision')
+          opts[:revision] = revision
+        end
+        stream = bucket.open_download_stream_by_name(args.use!('filename'), opts)
+        stream.read
+      end
+    end
+
     def upload(op)
       bucket = entities.get(:bucket, op.use!('object'))
       use_arguments(op) do |args|
@@ -27,6 +39,15 @@ module Unified
         if chunk_size = args.use('chunkSizeBytes')
           opts[:chunk_size] = chunk_size
         end
+        if metadata = args.use('metadata')
+          opts[:metadata] = metadata
+        end
+        if content_type = args.use('contentType')
+          opts[:content_type] = content_type
+        end
+        if disable_md5 = args.use('disableMD5')
+          opts[:disable_md5] = disable_md5
+        end
         contents = transform_contents(args.use!('source'))
         file_id = nil
         bucket.open_upload_stream(args.use!('filename'), **opts) do |stream|

data/spec/runners/unified/support_operations.rb

@@ -59,8 +59,11 @@ module Unified
 
     def assert_session_dirty(op)
       consume_test_runner(op)
-      # https://jira.mongodb.org/browse/RUBY-1813
-      true
+      use_arguments(op) do |args|
+        session = entities.get(:session, args.use!('session'))
+        # https://jira.mongodb.org/browse/RUBY-1813
+        true
+      end
     end
 
     def assert_session_not_dirty(op)

data/spec/runners/unified/test.rb

@@ -2,6 +2,7 @@
 # encoding: utf-8
 
 require 'runners/crud/requirement'
+require 'runners/unified/client_side_encryption_operations'
 require 'runners/unified/crud_operations'
 require 'runners/unified/grid_fs_operations'
 require 'runners/unified/ddl_operations'
@@ -9,16 +10,19 @@ require 'runners/unified/change_stream_operations'
 require 'runners/unified/support_operations'
 require 'runners/unified/assertions'
 require 'support/utils'
+require 'support/crypt'
 
 module Unified
 
   class Test
+    include ClientSideEncryptionOperations
     include CrudOperations
     include GridFsOperations
     include DdlOperations
     include ChangeStreamOperations
     include SupportOperations
     include Assertions
+    include RSpec::Core::Pending
 
     def initialize(spec, **opts)
       @spec = spec
@@ -132,11 +136,11 @@ module Unified
 
           create_client(**opts).tap do |client|
             @observe_sensitive = spec.use('observeSensitiveCommands')
+            subscriber = (@subscribers[client] ||= EventSubscriber.new)
             if oe = spec.use('observeEvents')
               oe.each do |event|
                 case event
                 when 'commandStartedEvent', 'commandSucceededEvent', 'commandFailedEvent'
-                  subscriber = (@subscribers[client] ||= EventSubscriber.new)
                   unless client.send(:monitoring).subscribers[Mongo::Monitoring::COMMAND].include?(subscriber)
                     client.subscribe(Mongo::Monitoring::COMMAND, subscriber)
                   end
@@ -146,7 +150,6 @@ module Unified
                     subscriber.ignore_commands(ignore_events)
                   end
                 when /\A(?:pool|connection)/
-                  subscriber = (@subscribers[client] ||= EventSubscriber.new)
                   unless client.send(:monitoring).subscribers[Mongo::Monitoring::CONNECTION_POOL]&.include?(subscriber)
                     client.subscribe(Mongo::Monitoring::CONNECTION_POOL, subscriber)
                   end
@@ -193,6 +196,61 @@ module Unified
           end
 
           client.start_session(**opts)
+        when 'clientEncryption'
+          client_encryption_opts = spec.use!('clientEncryptionOpts')
+          key_vault_client = entities.get(:client, client_encryption_opts['keyVaultClient'])
+          opts = {
+            key_vault_namespace: client_encryption_opts['keyVaultNamespace'],
+            kms_providers: Utils.snakeize_hash(client_encryption_opts['kmsProviders']),
+            kms_tls_options: {
+              kmip: {
+                ssl_cert: SpecConfig.instance.fle_kmip_tls_certificate_key_file,
+                ssl_key: SpecConfig.instance.fle_kmip_tls_certificate_key_file,
+                ssl_ca_cert: SpecConfig.instance.fle_kmip_tls_ca_file
+              }
+            }
+          }
+          opts[:kms_providers] = opts[:kms_providers].map do |provider, options|
+            converted_options = options.map do |key, value|
+              converted_value = if value == { '$$placeholder'.to_sym => 1 }
+                case provider
+                when :aws
+                  case key
+                  when :access_key_id then SpecConfig.instance.fle_aws_key
+                  when :secret_access_key then SpecConfig.instance.fle_aws_secret
+                  end
+                when :azure
+                  case key
+                  when :tenant_id then SpecConfig.instance.fle_azure_tenant_id
+                  when :client_id then SpecConfig.instance.fle_azure_client_id
+                  when :client_secret then SpecConfig.instance.fle_azure_client_secret
+                  end
+                when :gcp
+                  case key
+                  when :email then SpecConfig.instance.fle_gcp_email
+                  when :private_key then SpecConfig.instance.fle_gcp_private_key
+                  end
+                when :kmip
+                  case key
+                  when :endpoint then SpecConfig.instance.fle_kmip_endpoint
+                  end
+                when :local
+                  case key
+                  when :key then Crypt::LOCAL_MASTER_KEY
+                  end
+                end
+              else
+                value
+              end
+              [key, converted_value]
+            end.to_h
+            [provider, converted_options]
+          end.to_h
+
+          Mongo::ClientEncryption.new(
+            key_vault_client,
+            opts
+          )
         else
           raise NotImplementedError, "Unknown type #{type}"
         end
@@ -278,10 +336,19 @@ module Unified
         if name.to_s == 'loop'
           method_name = "_#{name}"
         end
+
+        if ["modify_collection"].include?(name.to_s)
+          skip "Mongo Ruby Driver does not support #{name.to_s}"
+        end
+
         if expected_error = op.use('expectError')
           begin
+            unless respond_to?(method_name)
+              raise Error::UnsupportedOperation, "Mongo Ruby Driver does not support #{name.to_s}"
+            end
+
             public_send(method_name, op)
-          rescue Mongo::Error, BSON::String::IllegalKey => e
+          rescue Mongo::Error, BSON::String::IllegalKey, ArgumentError => e
             if expected_error.use('isClientError')
               # isClientError doesn't actually mean a client error.
               # It means anything other than OperationFailure. DRIVERS-1799
@@ -332,10 +399,16 @@ module Unified
             raise Error::ErrorMismatch, "Expected exception but none was raised"
           end
         else
+          unless respond_to?(method_name, true)
+            raise Error::UnsupportedOperation, "Mongo Ruby Driver does not support #{name.to_s}"
+          end
+
           result = send(method_name, op)
           if expected_result = op.use('expectResult')
-            if result.nil? && !expected_result.empty?
-              raise Error::ResultMismatch, "Actual result nil but expected result #{expected_result}"
+            if result.nil? && expected_result.keys == ["$$unsetOrMatches"]
+              return
+            elsif result.nil? && !expected_result.empty?
+              raise Error::ResultMismatch, "#{msg}: expected #{expected} but got nil"
             elsif Array === expected_result
               assert_documents_match(result, expected_result)
             else

data/spec/runners/unified.rb

@@ -9,6 +9,8 @@ require 'runners/unified/test'
 require 'runners/unified/test_group'
 
 def define_unified_spec_tests(base_path, paths, expect_failure: false)
+  config_override :validate_update_replace, true
+
   paths.each do |path|
     basename = path[base_path.length+1...path.length]
     context basename do
@@ -61,13 +63,18 @@ def define_unified_spec_tests(base_path, paths, expect_failure: false)
               begin
                 test.create_entities
                 test.set_initial_data
-                lambda do
+                begin
                   test.run
                   test.assert_outcome
                   test.assert_events
                 # HACK: other errors are possible and likely will need to
                 # be added here later as the tests evolve.
-                end.should raise_error(Mongo::Error::OperationFailure)
+                rescue Mongo::Error::OperationFailure, Unified::Error::UnsupportedOperation
+                rescue => e
+                  fail "Expected to raise Mongo::Error::OperationFailure or Unified::Error::UnsupportedOperation, got #{e}"
+                else
+                  fail "Expected to raise Mongo::Error::OperationFailure or Unified::Error::UnsupportedOperation, but no error was raised"
+                end
               ensure
                 test.cleanup
               end

data/spec/shared/lib/mrss/constraints.rb

@@ -4,48 +4,40 @@
 module Mrss
   module Constraints
     def min_server_version(version)
-      unless version =~ /\A\d+\.\d+\z/
-        raise ArgumentError, "Version can only be major.minor: #{version}"
-      end
+      parsed_version = Gem::Version.new(version)
 
       before(:all) do
-        if version > ClusterConfig.instance.server_version
+        if parsed_version > Gem::Version.new(ClusterConfig.instance.server_version)
           skip "Server version #{version} or higher required, we have #{ClusterConfig.instance.server_version}"
         end
       end
     end
 
     def max_server_version(version)
-      unless version =~ /\A\d+\.\d+\z/
-        raise ArgumentError, "Version can only be major.minor: #{version}"
-      end
+      parsed_version = Gem::Version.new(version)
 
       before(:all) do
-        if version < ClusterConfig.instance.short_server_version
+        if parsed_version < Gem::Version.new(ClusterConfig.instance.server_version)
           skip "Server version #{version} or lower required, we have #{ClusterConfig.instance.server_version}"
         end
       end
     end
 
     def min_server_fcv(version)
-      unless version =~ /\A\d+\.\d+\z/
-        raise ArgumentError, "FCV can only be major.minor: #{version}"
-      end
+      parsed_version = Gem::Version.new(version)
 
       before(:all) do
-        unless ClusterConfig.instance.fcv_ish >= version
+        unless Gem::Version.new(ClusterConfig.instance.fcv_ish) >= parsed_version
           skip "FCV #{version} or higher required, we have #{ClusterConfig.instance.fcv_ish} (server #{ClusterConfig.instance.server_version})"
         end
       end
     end
 
     def max_server_fcv(version)
-      unless version =~ /\A\d+\.\d+\z/
-        raise ArgumentError, "Version can only be major.minor: #{version}"
-      end
+      parsed_version = Gem::Version.new(version)
 
       before(:all) do
-        if version < ClusterConfig.instance.fcv_ish
+        if parsed_version < Gem::Version.new(ClusterConfig.instance.fcv_ish)
           skip "FCV #{version} or lower required, we have #{ClusterConfig.instance.fcv_ish} (server #{ClusterConfig.instance.server_version})"
         end
       end
@@ -223,7 +215,8 @@ module Mrss
 
     def require_no_auth
       before(:all) do
-        if (ENV['AUTH'] && ENV['AUTH'] != 'noauth') || SpecConfig.instance.user || ClusterConfig.instance.auth_enabled?
+        auth = ENV.fetch('AUTH', '')
+        if (!auth.empty? && auth != 'noauth') || SpecConfig.instance.user || ClusterConfig.instance.auth_enabled?
           skip "Auth not allowed"
         end
       end

data/spec/shared/lib/mrss/docker_runner.rb

@@ -16,7 +16,7 @@ module Mrss
       opts.fetch(:default_script)
       opts.fetch(:project_lib_subdir)
 
-      @options = opts
+      @options = opts.merge(preload: true)
     end
 
     attr_reader :options
@@ -63,10 +63,14 @@ module Mrss
           @options[:mongo_only] = v.to_i
         end
 
-        opts.on('-p', '--preload', 'Preload Ruby toolchain and server binaries in docker') do |v|
+        opts.on('-p', '--preload', 'Preload Ruby toolchain and server binaries in docker (default)') do |v|
           @options[:preload] = v
         end
 
+        opts.on('-P', '--no-preload', 'Do not preload Ruby toolchain and server binaries in docker') do
+          @options[:preload] = false
+        end
+
         opts.on('-s', '--script=SCRIPT', 'Test script to invoke') do |v|
           @options[:script] = v
         end
@@ -173,7 +177,18 @@ module Mrss
     end
 
     def distro
-      @options[:distro] || 'ubuntu1604'
+      @options[:distro] || if app_tests?
+        'ubuntu2004'
+        else
+          case server_version
+          when '3.6'
+            'debian9'
+          when '4.0', '4.2'
+            'ubuntu1804'
+          else
+            'ubuntu2004'
+          end
+        end
     end
 
     BASE_IMAGES = {
@@ -253,6 +268,11 @@ module Mrss
       %w(1 true yes).include?(@env['FLE']&.downcase)
     end
 
+    # Mongoid
+    def app_tests?
+      %w(1 true yes).include?(@env['APP_TESTS']&.downcase)
+    end
+
     def num_exposed_ports
       case @env['TOPOLOGY'] || 'standalone'
       when 'standalone'

data/spec/shared/lib/mrss/eg_config_utils.rb

@@ -0,0 +1,51 @@
+autoload :YAML, 'yaml'
+require 'erubi'
+require 'erubi/capture_end'
+require 'tilt'
+
+module Mrss
+  module EgConfigUtils
+
+    DEBIAN_FOR_RUBY = {
+      'ruby-2.3' => 'debian92',
+      'ruby-2.4' => 'debian92',
+      'ruby-2.5' => 'debian10',
+      'ruby-2.6' => 'debian10',
+      'ruby-2.7' => 'debian10',
+      'ruby-3.0' => 'debian10',
+    }
+
+    def standard_debian_rubies(rubies, key: nil, &block)
+      rubies.flatten!
+      text = block.call
+      contents = YAML.load(text)
+      out = rubies.map do |ruby|
+        contents.merge(
+          'matrix_name' => "#{contents['matrix_name']} - #{ruby}",
+          'matrix_spec' => contents['matrix_spec'].merge(
+            'ruby' => ruby,
+            key || 'os' => DEBIAN_FOR_RUBY.fetch(ruby),
+          ),
+        )
+      end.to_yaml
+      text =~ /\A\n?(\s+)/
+      unless text
+        raise "Couldn't figure out indentation level"
+      end
+      indent = ' ' * ($1.length - 2)
+      "\n" + out.sub(/\A---.*\n/, indent).gsub("\n", "\n#{indent}")
+    end
+
+    def transform_config(template_path, context)
+      Tilt.new(template_path, engine_class: Erubi::CaptureEndEngine).render(context)
+    end
+
+    def generated_file_warning
+      <<-EOT
+# GENERATED FILE - DO NOT EDIT.
+# Run ./.evergreen/update-evergreen-configs to regenerate this file.
+
+EOT
+    end
+  end
+end