mongo 2.18.0.beta1 → 2.18.0

data/spec/runners/transactions/test.rb

@@ -40,7 +40,7 @@ module Mongo
       # @param [ Hash ] test The test specification.
       #
       # @since 2.6.0
-      def initialize(crud_spec, data, test)
+      def initialize(crud_spec, data, test, expectations_bson_types: true)
         test = IceNine.deep_freeze(test)
         @spec = crud_spec
         @data = data || []
@@ -71,7 +71,8 @@ module Mongo
           Operation.new(self, op)
         end
 
-        @expectations = BSON::ExtJSON.parse_obj(test['expectations'], mode: :bson)
+        mode = if expectations_bson_types then :bson else nil end
+        @expectations = BSON::ExtJSON.parse_obj(test['expectations'], mode: mode)
 
         if test['outcome']
           @outcome = Mongo::CRUD::Outcome.new(BSON::ExtJSON.parse_obj(test['outcome'], mode: :bson))
@@ -244,24 +245,7 @@ module Mongo
           key_vault_coll.insert_many(@spec.key_vault_data)
         end
 
-        if @spec.encrypted_fields
-          encrypted_fields = @spec.encrypted_fields.dup
-          # This code MUST be removed as soon as server starts accepting
-          # contention as int32.
-          if encrypted_fields.key?('fields')
-            encrypted_fields['fields'] = encrypted_fields['fields'].dup.map do |field|
-              if field['queries'] && field['queries'].key?('contention')
-                new_field = field.dup
-                new_field['queries'] = field['queries'].dup
-                new_field['queries']['contention'] = BSON::Int64.new(field['queries']['contention'])
-                new_field
-              else
-                field
-              end
-            end
-          end
-          # End of code to be removed
-        end
+        encrypted_fields = @spec.encrypted_fields if @spec.encrypted_fields
         coll = support_client[@spec.collection_name].with(write: { w: :majority })
         coll.drop(encrypted_fields: encrypted_fields)
 

data/spec/runners/transactions.rb

@@ -19,7 +19,7 @@ require 'runners/transactions/operation'
 require 'runners/transactions/spec'
 require 'runners/transactions/test'
 
-def define_transactions_spec_tests(test_paths)
+def define_transactions_spec_tests(test_paths, expectations_bson_types: true)
   config_override :validate_update_replace, true
 
   test_paths.each do |file|
@@ -30,7 +30,7 @@ def define_transactions_spec_tests(test_paths)
 
       define_spec_tests_with_requirements(spec) do |req|
 
-        spec.tests.each do |test|
+        spec.tests(expectations_bson_types: expectations_bson_types).each do |test|
 
           context(test.description) do
 

data/spec/runners/unified/assertions.rb

@@ -76,6 +76,32 @@ module Unified
             end
           end
 
+          %w(bulkWriteResult).each do |k|
+            expected_v = expected.use(k)
+            next unless expected_v
+            actual_v = case actual
+            when Mongo::Crypt::RewrapManyDataKeyResult
+              actual.send(Utils.underscore(k))
+            else
+              raise Error::ResultMismatch, "Mismatch: actual #{actual_v}, expected #{expected_v}"
+            end
+            if expected_v
+              if expected_v.empty?
+                if actual_v && !actual_v.empty?
+                  raise Error::ResultMismatch, "Actual not empty"
+                end
+              else
+                %w(deleted inserted matched modified upserted).each do |k|
+                  if count = expected_v.use("#{k}Count")
+                    if Hash === count || count > 0
+                      actual_count = actual_v.send("#{k}_count")
+                      assert_value_matches(actual_count, count, "#{k} count")
+                    end
+                  end
+                end
+              end
+            end
+          end
           assert_matches(actual, expected, 'result')
           expected.clear
         end
@@ -192,7 +218,7 @@ module Unified
 
     def assert_matches(actual, expected, msg)
       if actual.nil?
-        if expected&.keys == ["$$unsetOrMatches"]
+        if expected.is_a?(Hash) && expected.keys == ["$$unsetOrMatches"]
           return
         elsif !expected.nil?
           raise Error::ResultMismatch, "#{msg}: expected #{expected} but got nil"
@@ -219,7 +245,7 @@ module Unified
           if expected.empty?
             # This needs to be a match assertion. Check type only
             # and allow BulkWriteResult and generic operation result.
-            unless Hash === actual || Mongo::BulkWrite::Result === actual || Mongo::Operation::Result === actual
+            unless Hash === actual || Mongo::BulkWrite::Result === actual || Mongo::Operation::Result === actual || Mongo::Crypt::RewrapManyDataKeyResult === actual
               raise Error::ResultMismatch, "#{msg}: expected #{expected}, actual #{actual}"
             end
           else
@@ -269,6 +295,10 @@ module Unified
         Float === object
       when 'string'
         String === object
+      when 'binData'
+        BSON::Binary === object
+      when 'array'
+        Array === object
       else
         raise NotImplementedError, "Unhandled type #{type}"
       end

data/spec/runners/unified/change_stream_operations.rb

@@ -23,6 +23,9 @@ module Unified
         if full_document_before_change = args.use('fullDocumentBeforeChange')
           opts[:full_document_before_change] = full_document_before_change
         end
+        if args.key?('showExpandedEvents')
+          opts[:show_expanded_events] = args.use!('showExpandedEvents')
+        end
         cs = object.watch(pipeline, **opts)
         name = op.use!('saveResultAsEntity')
         entities.set(:change_stream, name, cs)

data/spec/runners/unified/client_side_encryption_operations.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+module Unified
+  module ClientSideEncryptionOperations
+    def create_data_key(op)
+      client_encryption = entities.get(:clientEncryption, op.use!('object'))
+      use_arguments(op) do |args|
+        opts = Utils.shallow_snakeize_hash(args.use('opts')) || {}
+        opts[:master_key] = Utils.shallow_snakeize_hash(opts[:master_key]) if opts[:master_key]
+        opts[:key_material] = opts[:key_material].data if opts[:key_material]
+        client_encryption.create_data_key(
+          args.use!('kmsProvider'),
+          opts,
+        )
+      end
+    end
+
+    def add_key_alt_name(op)
+      client_encryption = entities.get(:clientEncryption, op.use!('object'))
+      use_arguments(op) do |args|
+        client_encryption.add_key_alt_name(
+          args.use!('id'),
+          args.use!('keyAltName')
+        )
+      end
+    end
+
+    def delete_key(op)
+      client_encryption = entities.get(:clientEncryption, op.use!('object'))
+      use_arguments(op) do |args|
+        client_encryption.delete_key(
+          args.use!('id')
+        )
+      end
+    end
+
+    def get_key(op)
+      client_encryption = entities.get(:clientEncryption, op.use!('object'))
+      use_arguments(op) do |args|
+        client_encryption.get_key(
+          args.use!('id')
+        )
+      end
+    end
+
+    def get_key_by_alt_name(op)
+      client_encryption = entities.get(:clientEncryption, op.use!('object'))
+      use_arguments(op) do |args|
+        client_encryption.get_key_by_alt_name(
+          args.use!('keyAltName')
+        )
+      end
+    end
+
+    def get_keys(op)
+      client_encryption = entities.get(:clientEncryption, op.use!('object'))
+      client_encryption.get_keys.to_a
+    end
+
+    def remove_key_alt_name(op)
+      client_encryption = entities.get(:clientEncryption, op.use!('object'))
+      use_arguments(op) do |args|
+        client_encryption.remove_key_alt_name(
+          args.use!('id'),
+          args.use!('keyAltName')
+        )
+      end
+    end
+
+    def rewrap_many_data_key(op)
+      client_encryption = entities.get(:clientEncryption, op.use!('object'))
+      use_arguments(op) do |args|
+        opts = Utils.shallow_snakeize_hash(args.use('opts')) || {}
+        opts[:master_key] = Utils.shallow_snakeize_hash(opts[:master_key]) if opts[:master_key]
+        client_encryption.rewrap_many_data_key(
+          args.use!('filter'),
+          opts
+        )
+      end
+    end
+  end
+end

data/spec/runners/unified/crud_operations.rb

@@ -12,6 +12,14 @@ module Unified
           let: args.use('let'),
           comment: args.use('comment'),
           allow_disk_use: args.use('allowDiskUse'),
+          show_disk_loc: args.use('showRecordId'),
+          return_key: args.use('returnKey'),
+          projection: args.use('projection'),
+          skip: args.use('skip'),
+          hint: args.use('hint'),
+          max_value: args.use('max'),
+          max_time_ms: args.use('maxTimeMS'),
+          min_value: args.use('min'),
         }
         if session = args.use('session')
           opts[:session] = entities.get(:session, session)
@@ -26,6 +34,9 @@ module Unified
         if limit = args.use('limit')
           req = req.limit(limit)
         end
+        if projection = args.use('projection')
+          req = req.projection(projection)
+        end
         result = req.to_a
       end
     end
@@ -65,6 +76,9 @@ module Unified
         if session = args.use('session')
           opts[:session] = entities.get(:session, session)
         end
+        if comment = args.use('comment')
+          opts[:comment] = comment
+        end
         req = collection.find(args.use!('filter'), **opts).distinct(args.use!('fieldName'), **opts)
         result = req.to_a
       end
@@ -159,6 +173,7 @@ module Unified
           let: args.use('let'),
           comment: args.use('comment'),
           hint: args.use('hint'),
+          upsert: args.use('upsert'),
         }
         if session = args.use('session')
           opts[:session] = entities.get(:session, session)
@@ -227,8 +242,8 @@ module Unified
           convert_bulk_write_spec(req)
         end
         opts = {}
-        if ordered = args.use('ordered')
-          opts[:ordered] = true
+        if args.key?('ordered')
+          opts[:ordered] = args.use!('ordered')
         end
         if comment = args.use('comment')
           opts[:comment] = comment

data/spec/runners/unified/ddl_operations.rb

@@ -36,6 +36,9 @@ module Unified
         if change_stream_pre_and_post_images = args.use('changeStreamPreAndPostImages')
           collection_opts[:change_stream_pre_and_post_images] = change_stream_pre_and_post_images
         end
+        if view_on = args.use('viewOn')
+          collection_opts[:view_on] = view_on
+        end
         database[args.use!('collection'), collection_opts].create(**opts)
       end
     end
@@ -66,10 +69,16 @@ module Unified
       collection = entities.get(:collection, op.use!('object'))
       use_arguments(op) do |args|
         to = args.use!('to')
-        collection.client.use(:admin).command({
+        cmd = {
           renameCollection: "#{collection.database.name}.#{collection.name}",
           to: "#{collection.database.name}.#{to}"
-        })
+        }
+
+        if args.key?("dropTarget")
+          cmd[:dropTarget] = args.use("dropTarget")
+        end
+
+        collection.client.use(:admin).command(**cmd)
       end
     end
 
@@ -122,6 +131,22 @@ module Unified
       end
     end
 
+    def drop_index(op)
+      collection = entities.get(:collection, op.use!('object'))
+      use_arguments(op) do |args|
+        opts = {}
+        if session = args.use('session')
+          opts[:session] = entities.get(:session, session)
+        end
+
+        collection.indexes.drop_one(
+          args.use!('name'),
+          **opts,
+        )
+      end
+    end
+
+
     def assert_index_exists(op)
       consume_test_runner(op)
       use_arguments(op) do |args|

data/spec/runners/unified/grid_fs_operations.rb

@@ -20,6 +20,18 @@ module Unified
       end
     end
 
+    def download_by_name(op)
+      bucket = entities.get(:bucket, op.use!('object'))
+      use_arguments(op) do |args|
+        opts = {}
+        if revision = args.use('revision')
+          opts[:revision] = revision
+        end
+        stream = bucket.open_download_stream_by_name(args.use!('filename'), opts)
+        stream.read
+      end
+    end
+
     def upload(op)
       bucket = entities.get(:bucket, op.use!('object'))
       use_arguments(op) do |args|
@@ -27,6 +39,15 @@ module Unified
         if chunk_size = args.use('chunkSizeBytes')
           opts[:chunk_size] = chunk_size
         end
+        if metadata = args.use('metadata')
+          opts[:metadata] = metadata
+        end
+        if content_type = args.use('contentType')
+          opts[:content_type] = content_type
+        end
+        if disable_md5 = args.use('disableMD5')
+          opts[:disable_md5] = disable_md5
+        end
         contents = transform_contents(args.use!('source'))
         file_id = nil
         bucket.open_upload_stream(args.use!('filename'), **opts) do |stream|

data/spec/runners/unified/test.rb

@@ -2,6 +2,7 @@
 # encoding: utf-8
 
 require 'runners/crud/requirement'
+require 'runners/unified/client_side_encryption_operations'
 require 'runners/unified/crud_operations'
 require 'runners/unified/grid_fs_operations'
 require 'runners/unified/ddl_operations'
@@ -9,10 +10,12 @@ require 'runners/unified/change_stream_operations'
 require 'runners/unified/support_operations'
 require 'runners/unified/assertions'
 require 'support/utils'
+require 'support/crypt'
 
 module Unified
 
   class Test
+    include ClientSideEncryptionOperations
     include CrudOperations
     include GridFsOperations
     include DdlOperations
@@ -193,6 +196,61 @@ module Unified
           end
 
           client.start_session(**opts)
+        when 'clientEncryption'
+          client_encryption_opts = spec.use!('clientEncryptionOpts')
+          key_vault_client = entities.get(:client, client_encryption_opts['keyVaultClient'])
+          opts = {
+            key_vault_namespace: client_encryption_opts['keyVaultNamespace'],
+            kms_providers: Utils.snakeize_hash(client_encryption_opts['kmsProviders']),
+            kms_tls_options: {
+              kmip: {
+                ssl_cert: SpecConfig.instance.fle_kmip_tls_certificate_key_file,
+                ssl_key: SpecConfig.instance.fle_kmip_tls_certificate_key_file,
+                ssl_ca_cert: SpecConfig.instance.fle_kmip_tls_ca_file
+              }
+            }
+          }
+          opts[:kms_providers] = opts[:kms_providers].map do |provider, options|
+            converted_options = options.map do |key, value|
+              converted_value = if value == { '$$placeholder'.to_sym => 1 }
+                case provider
+                when :aws
+                  case key
+                  when :access_key_id then SpecConfig.instance.fle_aws_key
+                  when :secret_access_key then SpecConfig.instance.fle_aws_secret
+                  end
+                when :azure
+                  case key
+                  when :tenant_id then SpecConfig.instance.fle_azure_tenant_id
+                  when :client_id then SpecConfig.instance.fle_azure_client_id
+                  when :client_secret then SpecConfig.instance.fle_azure_client_secret
+                  end
+                when :gcp
+                  case key
+                  when :email then SpecConfig.instance.fle_gcp_email
+                  when :private_key then SpecConfig.instance.fle_gcp_private_key
+                  end
+                when :kmip
+                  case key
+                  when :endpoint then SpecConfig.instance.fle_kmip_endpoint
+                  end
+                when :local
+                  case key
+                  when :key then Crypt::LOCAL_MASTER_KEY
+                  end
+                end
+              else
+                value
+              end
+              [key, converted_value]
+            end.to_h
+            [provider, converted_options]
+          end.to_h
+
+          Mongo::ClientEncryption.new(
+            key_vault_client,
+            opts
+          )
         else
           raise NotImplementedError, "Unknown type #{type}"
         end
@@ -290,7 +348,7 @@ module Unified
             end
 
             public_send(method_name, op)
-          rescue Mongo::Error, BSON::String::IllegalKey => e
+          rescue Mongo::Error, BSON::String::IllegalKey, ArgumentError => e
             if expected_error.use('isClientError')
               # isClientError doesn't actually mean a client error.
               # It means anything other than OperationFailure. DRIVERS-1799

data/spec/shared/lib/mrss/docker_runner.rb

@@ -183,6 +183,8 @@ module Mrss
           case server_version
           when '3.6'
             'debian9'
+          when '4.0', '4.2'
+            'ubuntu1804'
           else
             'ubuntu2004'
           end

data/spec/shared/lib/mrss/eg_config_utils.rb

@@ -0,0 +1,51 @@
+autoload :YAML, 'yaml'
+require 'erubi'
+require 'erubi/capture_end'
+require 'tilt'
+
+module Mrss
+  module EgConfigUtils
+
+    DEBIAN_FOR_RUBY = {
+      'ruby-2.3' => 'debian92',
+      'ruby-2.4' => 'debian92',
+      'ruby-2.5' => 'debian10',
+      'ruby-2.6' => 'debian10',
+      'ruby-2.7' => 'debian10',
+      'ruby-3.0' => 'debian10',
+    }
+
+    def standard_debian_rubies(rubies, key: nil, &block)
+      rubies.flatten!
+      text = block.call
+      contents = YAML.load(text)
+      out = rubies.map do |ruby|
+        contents.merge(
+          'matrix_name' => "#{contents['matrix_name']} - #{ruby}",
+          'matrix_spec' => contents['matrix_spec'].merge(
+            'ruby' => ruby,
+            key || 'os' => DEBIAN_FOR_RUBY.fetch(ruby),
+          ),
+        )
+      end.to_yaml
+      text =~ /\A\n?(\s+)/
+      unless text
+        raise "Couldn't figure out indentation level"
+      end
+      indent = ' ' * ($1.length - 2)
+      "\n" + out.sub(/\A---.*\n/, indent).gsub("\n", "\n#{indent}")
+    end
+
+    def transform_config(template_path, context)
+      Tilt.new(template_path, engine_class: Erubi::CaptureEndEngine).render(context)
+    end
+
+    def generated_file_warning
+      <<-EOT
+# GENERATED FILE - DO NOT EDIT.
+# Run ./.evergreen/update-evergreen-configs to regenerate this file.
+
+EOT
+    end
+  end
+end

data/spec/shared/lib/mrss/lite_constraints.rb

@@ -211,10 +211,18 @@ module Mrss
 
     # This is a macro for retrying flaky tests on CI that occasionally fail.
     # Note that the tests will only be retried on CI.
-    def retry_test(n = 3)
+    #
+    # @param [ Integer ] :tries The number of times to retry.
+    # @param [ Integer ] :sleep The number of seconds to sleep in between retries.
+    #   If nothing, or nil, is passed, we won't wait in between retries.
+    def retry_test(tries: 3, sleep: nil)
       if %w(1 yes true).include?(ENV['CI'])
         around do |example|
-          example.run_with_retry retry: n
+          if sleep
+            example.run_with_retry retry: tries, retry_wait: sleep
+          else
+            example.run_with_retry retry: tries
+          end
         end
       end
     end

data/spec/shared/shlib/set_env.sh

@@ -74,6 +74,9 @@ set_env_node() {
       exit 2
     fi
     export PATH="$dir/bin:$PATH"
+  elif test -d /opt/node/bin; then
+    # Node from toolchain in Evergreen
+    export PATH=/opt/node/bin:$PATH
   fi
   
   node -v

data/spec/solo/clean_exit_spec.rb

@@ -9,9 +9,14 @@ describe 'Clean exit' do
     unless %w(1 true yes).include?(ENV['SOLO'])
       skip 'Set SOLO=1 in environment to run solo tests'
     end
+
+    if %w(1 true yes).include?(ENV['EXTERNAL_DISABLED'])
+      skip "Test requires external connectivity"
+    end
   end
 
   context 'with SRV URI' do
+
     let(:uri) do
       'mongodb+srv://test1.test.build.10gen.cc/?tls=false'
     end

data/spec/spec_tests/client_side_encryption_spec.rb

@@ -10,5 +10,5 @@ describe 'Client-Side Encryption' do
   require_libmongocrypt
   require_enterprise
 
-  define_transactions_spec_tests(CLIENT_SIDE_ENCRYPTION_TESTS)
+  define_transactions_spec_tests(CLIENT_SIDE_ENCRYPTION_TESTS, expectations_bson_types: false)
 end

data/spec/spec_tests/client_side_encryption_unified_spec.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+require 'spec_helper'
+
+require 'runners/unified'
+
+base = "#{CURRENT_PATH}/spec_tests/data/client_side_encryption"
+CLIENT_SIDE_ENCRYPTION_UNIFIED_TESTS = Dir.glob("#{base}/unified/**/*.yml").sort
+
+describe 'Client side encryption spec tests - unified' do
+  require_libmongocrypt
+  require_enterprise
+
+  define_unified_spec_tests(base, CLIENT_SIDE_ENCRYPTION_UNIFIED_TESTS)
+end