mongo 2.9.2 → 2.10.0.rc0

data/spec/support/cluster_config.rb

@@ -74,20 +74,6 @@ class ClusterConfig
     end
   end
 
-  def primary_address_str
-    primary_address
-  end
-
-  def primary_address_host
-    both = primary_address_str
-    both.split(':').first
-  end
-
-  def primary_address_port
-    both = primary_address_str
-    both.split(':')[1] || 27017
-  end
-
   # Try running a command on the admin database to see if the mongod was
   # started with auth.
   def auth_enabled?
@@ -111,4 +97,29 @@ class ClusterConfig
       topology.to_sym
     end
   end
+
+  def storage_engine
+    @storage_engine ||= begin
+      # 2.6 does not have wired tiger
+      if short_server_version == '2.6'
+        :mmapv1
+      else
+        client = ClientRegistry.instance.global_client('root_authorized')
+        if topology == :sharded
+          shards = client.use(:admin).command(listShards: 1).first
+          shard = shards['shards'].first
+          address_str = shard['host'].sub(/^.*\//, '').sub(/,.*/, '')
+          client = ClusterTools.instance.direct_client(address_str,
+            SpecConfig.instance.test_options.merge(SpecConfig.instance.auth_options).merge(connect: :direct))
+        end
+        rv = client.use(:admin).command(serverStatus: 1).first
+        rv = rv['storageEngine']['name']
+        rv_map = {
+          'wiredTiger' => :wired_tiger,
+          'mmapv1' => :mmapv1,
+        }
+        rv_map[rv] || rv
+      end
+    end
+  end
 end

data/spec/support/cluster_tools.rb

@@ -309,25 +309,47 @@ class ClusterTools
     end
   end
 
-  private
-
   def admin_client
     # Since we are triggering elections, we need to have a higher server
     # selection timeout applied. The default timeout for tests assumes a
     # stable deployment.
-    @admin_client ||= ClientRegistry.instance.global_client('root_authorized_admin').
-      with(server_selection_timeout: 15)
+    (
+      @admin_client ||= ClientRegistry.instance.global_client('root_authorized').
+        with(server_selection_timeout: 15).use(:admin)
+    ).tap do |client|
+      ClientRegistry.reconnect_client_if_perished(client)
+    end
   end
 
-  def direct_client(address)
+  def direct_client(address, options = {})
     @direct_clients ||= {}
-    @direct_clients[address] ||= ClientRegistry.instance.new_local_client(
-      [address.to_s],
-      SpecConfig.instance.test_options.merge(
-        SpecConfig.instance.auth_options).merge(
-        connect: :direct, server_selection_timeout: 10))
+    cache_key = {address: address}.update(options)
+    (
+      @direct_clients[cache_key] ||= ClientRegistry.instance.new_local_client(
+        [address.to_s],
+        SpecConfig.instance.test_options.merge(
+          SpecConfig.instance.auth_options).merge(
+          connect: :direct, server_selection_timeout: 10).merge(options))
+    ).tap do |client|
+      ClientRegistry.reconnect_client_if_perished(client)
+    end
   end
 
+  def close_clients
+    if @admin_client
+      @admin_client.close
+      @admin_client = nil
+    end
+    if @direct_clients
+      @direct_clients.each do |cache_key, client|
+        client.close
+      end
+      @direct_clients = nil
+    end
+  end
+
+  private
+
   def each_server(&block)
     admin_client.cluster.servers_list.each(&block)
   end

data/spec/support/command_monitoring.rb

@@ -248,7 +248,7 @@ module Mongo
         @description = test['description']
         @max_server_version = test['ignore_if_server_version_greater_than']
         @min_server_fcv = test['ignore_if_server_version_less_than']
-        @operation = Mongo::CRUD::Operation.get(test['operation'])
+        @operation = Mongo::CRUD::Operation.new(self, test['operation'])
         @expectations = test['expectations'].map{ |e| Expectation.new(e) }
       end
 

data/spec/support/common_shortcuts.rb

@@ -9,6 +9,36 @@ module CommonShortcuts
         end
       end
     end
+
+    # For tests which require clients to connect, clean slate asks all
+    # existing clients to be closed prior to the test execution.
+    # Note that clean_slate closes all clients for each test in the scope.
+    def clean_slate
+      before do
+        ClientRegistry.instance.close_all_clients
+      end
+    end
+
+    # Similar to clean slate but closes clients once before all tests in
+    # the scope. Use when the tests do not create new clients but do not
+    # want any background output from previously existing clients.
+    def clean_slate_for_all
+      before(:all) do
+        ClientRegistry.instance.close_all_clients
+      end
+    end
+
+    # For some reason, there are tests which fail on evergreen either
+    # intermittently or reliably that always succeed locally.
+    # Debugging of tests in evergreen is difficult/impossible,
+    # thus this workaround.
+    def clean_slate_on_evergreen
+      before(:all) do
+        if SpecConfig.instance.ci?
+          ClientRegistry.instance.close_all_clients
+        end
+      end
+    end
   end
 
   module InstanceMethods

data/spec/support/connection_string.rb

@@ -123,6 +123,7 @@ module Mongo
     end
 
     class Test
+      include RSpec::Core::Pending
 
       attr_reader :description
       attr_reader :uri_string
@@ -153,6 +154,10 @@ module Mongo
 
       def client
         @client ||= ClientRegistry.instance.new_local_client(@spec['uri'], monitoring_io: false)
+      rescue Mongo::Error::LintError => e
+        if e.message =~ /arbitraryButStillValid/
+          skip 'Test uses a read concern that fails linter'
+        end
       end
 
       def uri
@@ -223,9 +228,9 @@ module Mongo
         'maxidletimems' => :max_idle_time,
 
          # Write  Options
-        'journal' => [:write, 'j'],
-        'w' => [:write, 'w'],
-        'wtimeoutms' => [:write, 'wtimeout'],
+        'journal' => [:write_concern, 'j'],
+        'w' => [:write_concern, 'w'],
+        'wtimeoutms' => [:write_concern, 'wtimeout'],
 
         # Read Options
         'readpreference' => ['read', 'mode'],

data/spec/support/constraints.rb

@@ -157,4 +157,38 @@ module Constraints
       end
     end
   end
+
+  def require_no_multi_shard
+    before do
+      if ClusterConfig.instance.topology == :sharded && SpecConfig.instance.addresses.length > 1
+        skip 'Test requires a single shard if run in sharded topology'
+      end
+    end
+  end
+
+  def require_wired_tiger
+    before(:all) do
+      if ClusterConfig.instance.storage_engine != :wired_tiger
+        skip 'Test requires WiredTiger storage engine'
+      end
+    end
+  end
+
+  def require_wired_tiger_on_36
+    before(:all) do
+      if ClusterConfig.instance.short_server_version >= '3.6'
+        if ClusterConfig.instance.storage_engine != :wired_tiger
+          skip 'Test requires WiredTiger storage engine on 3.6+ servers'
+        end
+      end
+    end
+  end
+
+  def require_mmapv1
+    before do
+      if ClusterConfig.instance.storage_engine != :mmapv1
+        skip 'Test requires MMAPv1 storage engine'
+      end
+    end
+  end
 end

data/spec/support/crud.rb

@@ -15,13 +15,17 @@
 require 'support/gridfs'
 require 'support/crud/requirement'
 require 'support/crud/spec'
+require 'support/crud/test_base'
 require 'support/crud/test'
 require 'support/crud/outcome'
+require 'support/crud/context'
 require 'support/crud/operation'
-require 'support/crud/read'
-require 'support/crud/write'
 require 'support/crud/verifier'
 
+def collection_data(collection)
+  collection.find.to_a
+end
+
 def crud_execute_operations(spec, test, num_ops, event_subscriber, expect_error,
   client
 )
@@ -42,7 +46,7 @@ def crud_execute_operations(spec, test, num_ops, event_subscriber, expect_error,
     result = if expect_error.nil?
       res = nil
       begin
-        res = test.run(spec, client, num_ops)
+        res = test.run(client, num_ops)
       rescue => e
         res = e
       end
@@ -50,13 +54,13 @@ def crud_execute_operations(spec, test, num_ops, event_subscriber, expect_error,
     elsif expect_error
       error = nil
       begin
-        test.run(spec, client, num_ops)
+        test.run(client, num_ops)
       rescue => e
         error = e
       end
       error
     else
-      test.run(spec, client, num_ops)
+      test.run(client, num_ops)
     end
 
     $crud_event_cache ||= {}
@@ -68,7 +72,7 @@ def crud_execute_operations(spec, test, num_ops, event_subscriber, expect_error,
     if last_op.outcome && last_op.outcome.collection_data?
       verify_collection = client[last_op.verify_collection_name]
       $crud_collection_data_cache ||= {}
-      $crud_collection_data_cache[cache_key] = verify_collection.find.to_a
+      $crud_collection_data_cache[cache_key] = collection_data(verify_collection)
     end
 
     result
@@ -136,7 +140,7 @@ def define_crud_spec_test_examples(spec, req = nil, &block)
                 result
                 verifier.verify_collection_data(
                   operation.outcome.collection_data,
-                  verify_collection.find.to_a)
+                  collection_data(verify_collection))
               end
             end
 
@@ -173,6 +177,20 @@ def define_crud_spec_test_examples(spec, req = nil, &block)
           end
         end
       end
+
+      if test.outcome && test.outcome.collection_data?
+        let(:result) do
+          crud_execute_operations(spec, test, test.operations.length,
+            event_subscriber, nil, client)
+        end
+
+        it 'has the correct data in the collection' do
+          result
+          verifier.verify_collection_data(
+            test.outcome.collection_data,
+            collection_data(client[test.outcome.collection_name || spec.collection_name]))
+        end
+      end
     end
   end
 end
@@ -203,17 +221,14 @@ def define_spec_tests_with_requirements(spec, &block)
   end
 end
 
-def define_crud_spec_tests(description, test_paths, &block)
-  describe(description) do
-
-    test_paths.each do |path|
+def define_crud_spec_tests(test_paths, spec_cls = Mongo::CRUD::Spec, &block)
+  test_paths.each do |path|
 
-      spec = Mongo::CRUD::Spec.new(path)
+    spec = spec_cls.new(path)
 
-      context(spec.description) do
-        define_spec_tests_with_requirements(spec) do |req|
-          define_crud_spec_test_examples(spec, req, &block)
-        end
+    context(spec.description) do
+      define_spec_tests_with_requirements(spec) do |req|
+        define_crud_spec_test_examples(spec, req, &block)
       end
     end
   end

data/spec/support/crud/context.rb

@@ -0,0 +1,23 @@
+# Copyright (C) 2019 MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  module CRUD
+    class Context
+      def transform_arguments(arguments)
+        arguments
+      end
+    end
+  end
+end

data/spec/support/crud/operation.rb

@@ -15,29 +15,326 @@
 module Mongo
   module CRUD
 
-    # Helper module for instantiating either a Read or Write test operation.
-    #
-    # @since 2.0.0
-    module Operation
-      extend self
+    class Operation
 
-      # Get a new Operation.
-      #
-      # @example Get the operation.
-      #   Operation.get(spec)
+      # Instantiate the operation.
       #
       # @param [ Hash ] spec The operation specification.
+      # @param [ Hash ] outcome_spec The outcome specification.
+      #   If not provided, outcome is taken out of operation specification.
+      #
+      # @since 2.0.0
+      def initialize(crud_test, spec, outcome_spec = nil)
+        @crud_test = crud_test
+        @spec = IceNine.deep_freeze(spec)
+        @name = spec['name']
+        @arguments = spec['arguments'] || {}
+        @outcome = Outcome.new(outcome_spec || spec)
+      end
+
+      # The operation name.
       #
-      # @return [ Operation::Write, Operation::Read ] The Operation object.
+      # @return [ String ] name The operation name.
       #
       # @since 2.0.0
-      def get(spec, outcome_spec = nil)
-        if Write::OPERATIONS.keys.include?(spec['name'])
-          Write.new(spec, outcome_spec)
+      attr_reader :name
+
+      attr_reader :arguments
+
+      attr_reader :outcome
+
+      def object
+        @spec['object'] || 'collection'
+      end
+
+      # Which collection to verify results from.
+      # Returns the collection name specified on the operation, or
+      # the collection name for the entire spec file.
+      def verify_collection_name
+        if outcome && outcome.collection_name
+          outcome.collection_name
         else
-          Read.new(spec, outcome_spec)
+          @spec['collection_name'] || 'crud_spec_test'
+        end
+      end
+
+      # Whether the operation is expected to have results.
+      #
+      # @example Whether the operation is expected to have results.
+      #   operation.has_results?
+      #
+      # @return [ true, false ] If the operation is expected to have results.
+      #
+      # @since 2.0.0
+      def has_results?
+        !(name == 'aggregate' &&
+            pipeline.find {|op| op.keys.include?('$out') })
+      end
+
+      # Execute the operation.
+      #
+      # @example Execute the operation.
+      #   operation.execute
+      #
+      # @param [ Collection ] collection The collection to execute the operation on.
+      #
+      # @return [ Result, Array<Hash> ] The result of executing the operation.
+      #
+      # @since 2.0.0
+      def execute(target)
+        op_name = Utils.underscore(name)
+        if target.is_a?(Mongo::Database)
+          op_name = "db_#{op_name}"
+        elsif target.is_a?(Mongo::Client)
+          op_name= "client_#{op_name}"
+        end
+        send(op_name, target, Context.new)
+      end
+
+      def collection_options
+        Utils.convert_operation_options(@spec['collectionOptions'])
+      end
+
+      private
+
+      # read operations
+
+      def aggregate(collection, context)
+        collection.aggregate(arguments['pipeline'], context.transform_arguments(options)).to_a
+      end
+
+      def db_aggregate(database, context)
+        database.aggregate(arguments['pipeline'], context.transform_arguments(options)).to_a
+      end
+
+      def count(collection, context)
+        collection.count(arguments['filter'], context.transform_arguments(options))
+      end
+
+      def count_documents(collection, context)
+        collection.count_documents(arguments['filter'], context.transform_arguments(options))
+      end
+
+      def distinct(collection, context)
+        collection.distinct(arguments['fieldName'], arguments['filter'], context.transform_arguments(options))
+      end
+
+      def estimated_document_count(collection, context)
+        collection.estimated_document_count(context.transform_arguments(options))
+      end
+
+      def find(collection, context)
+        opts = context.transform_arguments(options)
+        if arguments['modifiers']
+          opts = opts.merge(modifiers: BSON::Document.new(arguments['modifiers']))
+        end
+        if read_preference
+          collection = collection.with(read: read_preference)
+        end
+        collection.find(arguments['filter'], opts).to_a
+      end
+
+      def find_one(collection, context)
+        find(collection, context).first
+      end
+
+      def client_list_databases(client, context)
+        client.list_databases
+      end
+
+      def client_list_database_names(client, context)
+        client.list_databases({}, true)
+      end
+
+      def client_list_database_objects(client, context)
+        client.list_mongo_databases
+      end
+
+      def db_list_collections(database, context)
+        database.list_collections
+      end
+
+      def db_list_collection_names(database, context)
+        database.collection_names
+      end
+
+      def db_list_collection_objects(database, context)
+        database.collections
+      end
+
+      def list_indexes(collection, context)
+        collection.indexes.to_a
+      end
+
+      def watch(collection, context)
+        collection.watch
+      end
+
+      def db_watch(database, context)
+        database.watch
+      end
+
+      def client_watch(client, context)
+        client.watch
+      end
+
+      def download(fs_bucket, context)
+        stream = fs_bucket.open_download_stream(BSON::ObjectId.from_string(arguments['id']['$oid']))
+        stream.read
+      end
+
+      def download_by_name(fs_bucket, context)
+        stream = fs_bucket.open_download_stream_by_name(arguments['filename'])
+        stream.read
+      end
+
+      def map_reduce(collection, context)
+        view = Mongo::Collection::View.new(collection)
+        mr = Mongo::Collection::View::MapReduce.new(view, arguments['map']['$code'], arguments['reduce']['$code'])
+        mr.to_a
+      end
+
+      # write operations
+
+      def bulk_write(collection, context)
+        result = collection.bulk_write(requests, context.transform_arguments(options))
+        return_doc = {}
+        return_doc['deletedCount'] = result.deleted_count || 0
+        return_doc['insertedIds'] = result.inserted_ids if result.inserted_ids
+        return_doc['insertedCount'] = result.inserted_count || 0
+        return_doc['upsertedId'] = result.upserted_id if arguments['upsert']
+        return_doc['upsertedIds'] = result.upserted_ids if result.upserted_ids
+        return_doc['upsertedCount'] = result.upserted_count || 0
+        return_doc['matchedCount'] = result.matched_count || 0
+        return_doc['modifiedCount'] = result.modified_count || 0
+        return_doc
+      end
+
+      def delete_many(collection, context)
+        result = collection.delete_many(arguments['filter'], context.transform_arguments(options))
+        { 'deletedCount' => result.deleted_count }
+      end
+
+      def delete_one(collection, context)
+        result = collection.delete_one(arguments['filter'], context.transform_arguments(options))
+        { 'deletedCount' => result.deleted_count }
+      end
+
+      def insert_many(collection, context)
+        result = collection.insert_many(arguments['documents'], context.transform_arguments(options))
+        { 'insertedIds' => result.inserted_ids }
+      end
+
+      def insert_one(collection, context)
+        result = collection.insert_one(arguments['document'], context.transform_arguments(options))
+        { 'insertedId' => result.inserted_id }
+      end
+
+      def replace_one(collection, context)
+        result = collection.replace_one(arguments['filter'], arguments['replacement'], context.transform_arguments(options))
+        update_return_doc(result)
+      end
+
+      def update_many(collection, context)
+        result = collection.update_many(arguments['filter'], arguments['update'], context.transform_arguments(options))
+        update_return_doc(result)
+      end
+
+      def update_one(collection, context)
+        result = collection.update_one(arguments['filter'], arguments['update'], context.transform_arguments(options))
+        update_return_doc(result)
+      end
+
+      def find_one_and_delete(collection, context)
+        collection.find_one_and_delete(arguments['filter'], context.transform_arguments(options))
+      end
+
+      def find_one_and_replace(collection, context)
+        collection.find_one_and_replace(arguments['filter'], arguments['replacement'], context.transform_arguments(options))
+      end
+
+      def find_one_and_update(collection, context)
+        collection.find_one_and_update(arguments['filter'], arguments['update'], context.transform_arguments(options))
+      end
+
+      # options & arguments
+
+      def options
+        out = {}
+        # Most tests have an "arguments" key which is a hash of options to
+        # be provided to the operation. The command monitoring unacknowledged
+        # bulk write test is an exception in that it has an "options" key
+        # with the options.
+        arguments.merge(arguments['options'] || {}).each do |spec_k, v|
+          ruby_k = Utils.underscore(spec_k).to_sym
+
+          if v.is_a?(Hash) && v['$numberLong']
+            v = v['$numberLong'].to_i
+          end
+
+          if respond_to?("transform_#{ruby_k}", true)
+            v = send("transform_#{ruby_k}", v)
+          end
+
+          out[ruby_k] = v
+        end
+        out
+      end
+
+      def requests
+        arguments['requests'].map do |request|
+          case request.keys.first
+          when 'insertOne' then
+            { insert_one: request['insertOne']['document'] }
+          when 'updateOne' then
+            update = request['updateOne']
+            { update_one: { filter: update['filter'], update: update['update'] } }
+          when 'name' then
+            bulk_request(request)
+          end
         end
       end
+
+      def bulk_request(request)
+        op_name = Utils.underscore(request['name'])
+        args = Utils.shallow_snakeize_hash(request['arguments'])
+        if args[:document]
+          unless args.keys == [:document]
+            raise "If :document is given, it must be the only key"
+          end
+          args = args[:document]
+        end
+        { op_name => args }
+      end
+
+      def upsert
+        arguments['upsert']
+      end
+
+      def transform_return_document(v)
+        Utils.underscore(v).to_sym
+      end
+
+      def update
+        arguments['update']
+      end
+
+      def transform_read_preference(v)
+        Utils.snakeize_hash(v)
+      end
+
+      def read_preference
+        transform_read_preference(@spec['read_preference'])
+      end
+
+      def update_return_doc(result)
+        return_doc = {}
+        return_doc['upsertedId'] = result.upserted_id if arguments['upsert']
+        return_doc['upsertedCount'] = result.upserted_count
+        return_doc['matchedCount'] = result.matched_count
+        return_doc['modifiedCount'] = result.modified_count if result.modified_count
+        return_doc
+      end
     end
   end
 end