mongo 2.15.0 → 2.17.0

data/spec/runners/unified/crud_operations.rb

@@ -8,7 +8,9 @@ module Unified
     def find(op)
       collection = entities.get(:collection, op.use!('object'))
       use_arguments(op) do |args|
-        opts = {}
+        opts = {
+          let: args.use('let'),
+        }
         if session = args.use('session')
           opts[:session] = entities.get(:session, session)
         end
@@ -29,7 +31,11 @@ module Unified
     def count_documents(op)
       collection = entities.get(:collection, op.use!('object'))
       use_arguments(op) do |args|
-        collection.find(args.use!('filter')).count_documents
+        opts = {}
+        if session = args.use('session')
+          opts[:session] = entities.get(:session, session)
+        end
+        collection.find(args.use!('filter')).count_documents(**opts)
       end
     end
 
@@ -47,7 +53,11 @@ module Unified
     def distinct(op)
       collection = entities.get(:collection, op.use!('object'))
       use_arguments(op) do |args|
-        req = collection.find(args.use!('filter')).distinct(args.use!('fieldName'))
+        opts = {}
+        if session = args.use('session')
+          opts[:session] = entities.get(:session, session)
+        end
+        req = collection.find(args.use!('filter'), **opts).distinct(args.use!('fieldName'), **opts)
         result = req.to_a
       end
     end
@@ -57,10 +67,15 @@ module Unified
       use_arguments(op) do |args|
         filter = args.use!('filter')
         update = args.use!('update')
-        opts = {}
+        opts = {
+          let: args.use('let'),
+        }
         if return_document = args.use('returnDocument')
           opts[:return_document] = return_document.downcase.to_sym
         end
+        if session = args.use('session')
+          opts[:session] = entities.get(:session, session)
+        end
         collection.find_one_and_update(filter, update, **opts)
       end
     end
@@ -70,7 +85,13 @@ module Unified
       use_arguments(op) do |args|
         filter = args.use!('filter')
         update = args.use!('replacement')
-        collection.find_one_and_replace(filter, update)
+        opts = {
+          let: args.use('let'),
+        }
+        if session = args.use('session')
+          opts[:session] = entities.get(:session, session)
+        end
+        collection.find_one_and_replace(filter, update, **opts)
       end
     end
 
@@ -78,7 +99,13 @@ module Unified
       collection = entities.get(:collection, op.use!('object'))
       use_arguments(op) do |args|
         filter = args.use!('filter')
-        collection.find_one_and_delete(filter)
+        opts = {
+          let: args.use('let'),
+        }
+        if session = args.use('session')
+          opts[:session] = entities.get(:session, session)
+        end
+        collection.find_one_and_delete(filter, **opts)
       end
     end
 
@@ -96,25 +123,37 @@ module Unified
     def insert_many(op)
       collection = entities.get(:collection, op.use!('object'))
       use_arguments(op) do |args|
-        options = {}
+        opts = {}
         unless (ordered = args.use('ordered')).nil?
-          options[:ordered] = ordered
+          opts[:ordered] = ordered
+        end
+        if session = args.use('session')
+          opts[:session] = entities.get(:session, session)
         end
-        collection.insert_many(args.use!('documents'), **options)
+        collection.insert_many(args.use!('documents'), **opts)
       end
     end
 
     def update_one(op)
       collection = entities.get(:collection, op.use!('object'))
       use_arguments(op) do |args|
-        collection.update_one(args.use!('filter'), args.use!('update'))
+        opts = {
+          let: args.use('let'),
+        }
+        if session = args.use('session')
+          opts[:session] = entities.get(:session, session)
+        end
+        collection.update_one(args.use!('filter'), args.use!('update'), **opts)
       end
     end
 
     def update_many(op)
       collection = entities.get(:collection, op.use!('object'))
       use_arguments(op) do |args|
-        collection.update_many(args.use!('filter'), args.use!('update'))
+        opts = {
+          let: args.use('let'),
+        }
+        collection.update_many(args.use!('filter'), args.use!('update'), **opts)
       end
     end
 
@@ -132,14 +171,23 @@ module Unified
     def delete_one(op)
       collection = entities.get(:collection, op.use!('object'))
       use_arguments(op) do |args|
-        collection.delete_one(args.use!('filter'))
+        opts = {
+          let: args.use('let'),
+        }
+        if session = args.use('session')
+          opts[:session] = entities.get(:session, session)
+        end
+        collection.delete_one(args.use!('filter'), **opts)
       end
     end
 
     def delete_many(op)
       collection = entities.get(:collection, op.use!('object'))
       use_arguments(op) do |args|
-        collection.delete_many(args.use!('filter'))
+        opts = {
+          let: args.use('let'),
+        }
+        collection.delete_many(args.use!('filter'), **opts)
       end
     end
 
@@ -157,6 +205,22 @@ module Unified
       end
     end
 
+    def aggregate(op)
+      obj = entities.get_any(op.use!('object'))
+      args = op.use!('arguments')
+      pipeline = args.use!('pipeline')
+      opts = {
+        let: args.use('let'),
+      }
+      if session = args.use('session')
+        opts[:session] = entities.get(:session, session)
+      end
+      unless args.empty?
+        raise NotImplementedError, "Unhandled spec keys: #{args} in #{test_spec}"
+      end
+      obj.aggregate(pipeline, **opts).to_a
+    end
+
     private
 
     def convert_bulk_write_spec(spec)
@@ -192,15 +256,5 @@ module Unified
       end
       {Utils.underscore(op) =>out}
     end
-
-    def aggregate(op)
-      obj = entities.get_any(op.use!('object'))
-      args = op.use!('arguments')
-      pipeline = args.use!('pipeline')
-      unless args.empty?
-        raise NotImplementedError, "Unhandled spec keys: #{test_spec}"
-      end
-      obj.aggregate(pipeline).to_a
-    end
   end
 end

data/spec/runners/unified/ddl_operations.rb

@@ -7,7 +7,13 @@ module Unified
 
     def list_databases(op)
       client = entities.get(:client, op.use!('object'))
-      client.list_databases
+      use_arguments(op) do |args|
+        opts = {}
+        if session = args.use('session')
+          opts[:session] = entities.get(:session, session)
+        end
+        client.list_databases({}, false, **opts)
+      end
     end
 
     def create_collection(op)
@@ -28,6 +34,17 @@ module Unified
       end
     end
 
+    def list_collections(op)
+      database = entities.get(:database, op.use!('object'))
+      use_arguments(op) do |args|
+        opts = {}
+        if session = args.use('session')
+          opts[:session] = entities.get(:session, session)
+        end
+        database.list_collections(**opts)
+      end
+    end
+
     def drop_collection(op)
       database = entities.get(:database, op.use!('object'))
       use_arguments(op) do |args|
@@ -58,6 +75,17 @@ module Unified
       assert_collection_exists(op, false)
     end
 
+    def list_indexes(op)
+      collection = entities.get(:collection, op.use!('object'))
+      use_arguments(op) do |args|
+        opts = {}
+        if session = args.use('session')
+          opts[:session] = entities.get(:session, session)
+        end
+        collection.indexes(**opts).to_a
+      end
+    end
+
     def create_index(op)
       collection = entities.get(:collection, op.use!('object'))
       use_arguments(op) do |args|

data/spec/runners/unified/entity_map.rb

@@ -3,6 +3,8 @@
 
 module Unified
   class EntityMap
+    extend Forwardable
+
     def initialize
       @map = {}
     end
@@ -35,8 +37,6 @@ module Unified
       raise Error::EntityMissing, "There is no #{id} known"
     end
 
-    def [](type)
-      @map[type]
-    end
+    def_delegators :@map, :[], :fetch
   end
 end

data/spec/runners/unified/event_subscriber.rb

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 # encoding: utf-8
 
-require 'support/event_subscriber'
+require 'mrss/event_subscriber'
 
 module Unified
 
-  class EventSubscriber < ::EventSubscriber
+  class EventSubscriber < Mrss::EventSubscriber
     def ignore_commands(command_names)
       @ignore_commands = command_names
     end

data/spec/runners/unified/support_operations.rb

@@ -13,7 +13,12 @@ module Unified
 
         cmd = args.use!('command')
 
-        database.command(cmd)
+        opts = {}
+        if session = args.use('session')
+          opts[:session] = entities.get(:session, session)
+        end
+
+        database.command(cmd, **opts)
       end
     end
 
@@ -208,12 +213,20 @@ module Unified
             else
               raise
             end
+          rescue Interrupt
+            raise
           rescue => e
-            if store_errors
+            if store_failures
+              STDERR.puts "Error: #{e.class}: #{e} (reporting as failure)"
+              entities.get(:failure_list, store_failures) << {
+                error: "#{e.class}: #{e}",
+                time: Time.now.to_f,
+              }
+            elsif store_errors
               STDERR.puts "Error: #{e.class}: #{e}"
               entities.get(:error_list, store_errors) << {
                 error: "#{e.class}: #{e}",
-                observedAt: Time.now.to_f,
+                time: Time.now.to_f,
               }
             else
               raise

data/spec/runners/unified/test.rb

@@ -160,11 +160,25 @@ module Unified
           end
         when 'database'
           client = entities.get(:client, spec.use!('client'))
-          client.use(spec.use!('databaseName')).database
+          opts = Utils.snakeize_hash(spec.use('databaseOptions') || {})
+            .merge(database: spec.use!('databaseName'))
+          if opts.key?(:read_preference)
+            opts[:read] = opts.delete(:read_preference)
+            if opts[:read].key?(:max_staleness_seconds)
+              opts[:read][:max_staleness] = opts[:read].delete(:max_staleness_seconds)
+            end
+          end
+          client.with(opts).database
         when 'collection'
           database = entities.get(:database, spec.use!('database'))
           # TODO verify
           opts = Utils.snakeize_hash(spec.use('collectionOptions') || {})
+          if opts.key?(:read_preference)
+            opts[:read] = opts.delete(:read_preference)
+            if opts[:read].key?(:max_staleness_seconds)
+              opts[:read][:max_staleness] = opts[:read].delete(:max_staleness_seconds)
+            end
+          end
           database[spec.use!('collectionName'), opts]
         when 'bucket'
           database = entities.get(:database, spec.use!('database'))
@@ -192,7 +206,8 @@ module Unified
     def set_initial_data
       @spec['initialData']&.each do |entity_spec|
         spec = UsingHash[entity_spec]
-        collection = root_authorized_client.use(spec.use!('databaseName'))[spec.use!('collectionName')]
+        collection = root_authorized_client.with(write_concern: {w: :majority}).
+          use(spec.use!('databaseName'))[spec.use!('collectionName')]
         collection.drop
         docs = spec.use!('documents')
         if docs.any?
@@ -265,7 +280,7 @@ module Unified
         end
         if expected_error = op.use('expectError')
           begin
-            send(method_name, op)
+            public_send(method_name, op)
           rescue Mongo::Error, BSON::String::IllegalKey => e
             if expected_error.use('isClientError')
               # isClientError doesn't actually mean a client error.

data/spec/runners/unified.rb

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
 # encoding: utf-8
 
+require 'support/using_hash'
 require 'runners/unified/error'
 require 'runners/unified/entity_map'
 require 'runners/unified/event_subscriber'
 require 'runners/unified/test'
 require 'runners/unified/test_group'
-require 'runners/unified/using_hash'
 
 def define_unified_spec_tests(base_path, paths, expect_failure: false)
   paths.each do |path|

data/spec/shared/lib/mrss/cluster_config.rb

@@ -209,7 +209,12 @@ module Mrss
       @server_version = build_info['version']
       @enterprise = build_info['modules'] && build_info['modules'].include?('enterprise')
 
-      @server_parameters = client.use(:admin).command(getParameter: '*').first
+      @server_parameters = begin
+        client.use(:admin).command(getParameter: '*').first
+      rescue => e
+        STDERR.puts("WARNING: Failed to obtain server parameters: #{e.class}: #{e.message}")
+        {}
+      end
 
       if !sharded_ish? && short_server_version >= '3.4'
         rv = @server_parameters['featureCompatibilityVersion']

data/spec/shared/lib/mrss/constraints.rb

@@ -257,10 +257,14 @@ module Mrss
       end
     end
 
-    def require_multi_shard
+    def require_multi_mongos
       before(:all) do
         if ClusterConfig.instance.topology == :sharded && SpecConfig.instance.addresses.length == 1
-          skip 'Test requires a minimum of two shards if run in sharded topology'
+          skip 'Test requires a minimum of two mongoses if run in sharded topology'
+        end
+
+        if ClusterConfig.instance.topology == :load_balanced && SpecConfig.instance.single_mongos?
+          skip 'Test requires a minimum of two mongoses if run in load-balanced topology'
         end
       end
     end
@@ -273,17 +277,19 @@ module Mrss
     #
     # In load-balanced topology, the same problem can happen when there is
     # more than one mongos behind the load balancer.
-    def require_no_multi_shard
+    def require_no_multi_mongos
       before(:all) do
         if ClusterConfig.instance.topology == :sharded && SpecConfig.instance.addresses.length > 1
-          skip 'Test requires a single shard if run in sharded topology'
+          skip 'Test requires a single mongos if run in sharded topology'
         end
         if ClusterConfig.instance.topology == :load_balanced && !SpecConfig.instance.single_mongos?
-          skip 'Test requires a minimum of two shards if run in sharded topology'
+          skip 'Test requires a single mongos, as indicated by SINGLE_MONGOS=1 environment variable, if run in load-balanced topology'
         end
       end
     end
 
+    alias :require_no_multi_shard :require_no_multi_mongos
+
     def require_wired_tiger
       before(:all) do
         # Storage detection fails for serverless instances. However, it is safe to

data/spec/shared/lib/mrss/event_subscriber.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+module Mrss
+  # Test event subscriber.
+  class EventSubscriber
+
+    # The mappings of event names to types.
+    MAPPINGS = {
+      'topology_opening_event' => Mongo::Monitoring::Event::TopologyOpening,
+      'topology_description_changed_event' => Mongo::Monitoring::Event::TopologyChanged,
+      'topology_closed_event' => Mongo::Monitoring::Event::TopologyClosed,
+      'server_opening_event' => Mongo::Monitoring::Event::ServerOpening,
+      'server_description_changed_event' => Mongo::Monitoring::Event::ServerDescriptionChanged,
+      'server_closed_event' => Mongo::Monitoring::Event::ServerClosed
+    }.freeze
+
+    attr_reader :all_events
+
+    attr_reader :started_events
+
+    attr_reader :succeeded_events
+
+    attr_reader :failed_events
+
+    attr_reader :published_events
+
+    # @param [ String ] name Optional name for the event subscriber.
+    def initialize(name: nil)
+      @mutex = Mutex.new
+      clear_events!
+      @name = name
+    end
+
+    def to_s
+      %Q`#<EventSubscriber:#{@name ? "\"#{@name}\"" : '%x' % object_id} \
+  started=#{started_events.length} \
+  succeeded=#{succeeded_events.length} \
+  failed=#{failed_events.length} \
+  published=#{published_events.length}>`
+    end
+
+    alias :inspect :to_s
+
+    # Event retrieval
+
+    def select_started_events(cls)
+      started_events.select do |event|
+        event.is_a?(cls)
+      end
+    end
+
+    def select_succeeded_events(cls)
+      succeeded_events.select do |event|
+        event.is_a?(cls)
+      end
+    end
+
+    def select_completed_events(*classes)
+      (succeeded_events + failed_events).select do |event|
+        classes.any? { |c| c === event }
+      end
+    end
+
+    def select_published_events(cls)
+      published_events.select do |event|
+        event.is_a?(cls)
+      end
+    end
+
+    # Filters command started events for the specified command name.
+    def command_started_events(command_name)
+      started_events.select do |event|
+        event.command[command_name]
+      end
+    end
+
+    def non_auth_command_started_events
+      started_events.reject do |event|
+        %w(authenticate getnonce saslSstart saslContinue).any? do |cmd|
+          event.command[cmd]
+        end
+      end
+    end
+
+    # Locates command stated events for the specified command name,
+    # asserts that there is exactly one such event, and returns it.
+    def single_command_started_event(command_name, include_auth: false)
+      events = if include_auth
+                 started_events
+               else
+                 non_auth_command_started_events
+               end
+      events.select! do |event|
+        event.command[command_name]
+      end
+      if events.length != 1
+        raise "Expected a single #{command_name} event but we have #{events.length}"
+      end
+      events.first
+    end
+
+
+    # Get the first succeeded event published for the name, and then delete it.
+    #
+    # @param [ String ] name The event name.
+    #
+    # @return [ Event ] The matching event.
+    def first_event(name)
+      cls = MAPPINGS[name]
+      if cls.nil?
+        raise ArgumentError, "Bogus event name #{name}"
+      end
+      matching = succeeded_events.find do |event|
+        cls === event
+      end
+      succeeded_events.delete(matching)
+      matching
+    end
+
+    # Event recording
+
+    # Cache the started event.
+    #
+    # @param [ Event ] event The event.
+    def started(event)
+      @mutex.synchronize do
+        started_events << event
+        all_events << event
+      end
+    end
+
+    # Cache the succeeded event.
+    #
+    # @param [ Event ] event The event.
+    def succeeded(event)
+      @mutex.synchronize do
+        succeeded_events << event
+        all_events << event
+      end
+    end
+
+    # Cache the failed event.
+    #
+    # @param [ Event ] event The event.
+    def failed(event)
+      @mutex.synchronize do
+        failed_events << event
+        all_events << event
+      end
+    end
+
+    def published(event)
+      @mutex.synchronize do
+        published_events << event
+        all_events << event
+      end
+    end
+
+    # Clear all cached events.
+    def clear_events!
+      @all_events = []
+      @started_events = []
+      @succeeded_events = []
+      @failed_events = []
+      @published_events = []
+      self
+    end
+  end
+  # Only handles succeeded events correctly.
+  class PhasedEventSubscriber < EventSubscriber
+    def initialize
+      super
+      @phase_events = {}
+    end
+
+    def phase_finished(phase_index)
+      @phase_events[phase_index] = succeeded_events
+      @succeeded_events = []
+    end
+
+    def phase_events(phase_index)
+      @phase_events[phase_index]
+    end
+
+    def event_count
+      @phase_events.inject(0) do |sum, event|
+        sum + event.length
+      end
+    end
+  end
+
+  class VerboseEventSubscriber < EventSubscriber
+    %w(started succeeded failed published).each do |meth|
+      define_method(meth) do |event|
+        puts event.summary
+        super(event)
+      end
+    end
+  end
+end

data/spec/shared/lib/mrss/server_version_registry.rb

@@ -48,19 +48,24 @@ module Mrss
         url = dl['archive']['url']
       end
     rescue MissingDownloadUrl
-      if %w(4.7 4.7.0).include?(desired_version)
-        # 4.7.0 has no advertised downloads but it is downloadable and
-        # we do need it. Dirty hack below.
-        registry = self.class.new('4.4.3', arch)
-        registry.download_url.sub('4.4.3', '4.7.0').tap do |url|
-          # Sanity check - ensure the URL we hacked up is a valid one
-          io = uri_open(url)
-          begin
-            io.read(1)
-          ensure
-            io.close
-          end
+      if %w(2.6 3.0).include?(desired_version) && arch == 'ubuntu1604'
+        # 2.6 and 3.0 are only available for ubuntu1204 and ubuntu1404.
+        # Those ubuntus have ancient Pythons that don't work due to not
+        # implementing recent TLS protocols.
+        # Because of this we test on ubuntu1604 which has a newer Python.
+        # But we still need to retrieve ubuntu1404-targeting builds.
+        url = self.class.new('3.2', arch).download_url
+        unless url.include?('3.2.')
+          raise 'URL not in expected format'
         end
+        url = case desired_version
+        when '2.6'
+          url.sub(/\b3\.2\.\d+/, '2.6.12')
+        when '3.0'
+          url.sub(/\b3\.2\.\d+/, '3.0.15')
+        else
+          raise NotImplementedError
+        end.sub('ubuntu1604', 'ubuntu1404')
       else
         raise
       end