mongo 2.8.0 → 2.9.0.rc0

data/spec/support/crud/verifier.rb

@@ -27,8 +27,7 @@ module Mongo
       #
       # Uses RSpec matchers and raises expectation failures if there is a
       # mismatch.
-      def verify_collection_data(actual_collection_data)
-        expected_collection_data = test_instance.outcome_collection_data
+      def verify_collection_data(expected_collection_data, actual_collection_data)
         if expected_collection_data.nil?
           expect(actual_collection_data).to be nil
         elsif expected_collection_data.empty?
@@ -48,14 +47,18 @@ module Mongo
       #
       # Uses RSpec matchers and raises expectation failures if there is a
       # mismatch.
-      def verify_operation_result(actual)
-        expected = test_instance.outcome['result']
+      def verify_operation_result(expected, actual)
         if expected.is_a?(Array)
           if expected.empty?
             expect(actual).to be_empty
           else
             expected.each_with_index do |expected_elt, i|
-              verify_result(expected_elt, actual[i])
+              # If the YAML spec test does not define a result,
+              # do not assert the operation's result - the operation may
+              # have produced a result, the test just does not care what it is
+              if expected_elt
+                verify_result(expected_elt, actual[i])
+              end
             end
           end
         else
@@ -63,6 +66,50 @@ module Mongo
         end
       end
 
+      def verify_command_started_event_count(expected_events, actual_events)
+        expect(actual_events.length).to eq(expected_events.length)
+      end
+
+      def verify_command_started_event(expected_events, actual_events, i)
+        expect(expected_events.length).to be > i
+        expect(actual_events.length).to be > i
+
+        expectation = expected_events[i]
+        actual_event = actual_events[i]['command_started_event'].dup
+
+        expect(expectation.keys).to eq(%w(command_started_event))
+        expected_event = expectation['command_started_event'].dup
+        # Retryable reads tests' YAML assertions omit some of the keys
+        # that are included in the actual command events.
+        # Transactions and transactions API tests specify all keys
+        # in YAML that are present in actual command events.
+        actual_event.keys.each do |key|
+          unless expected_event.key?(key)
+            actual_event.delete(key)
+          end
+        end
+        expect(actual_event).not_to be nil
+        expect(actual_event.keys).to eq(expected_event.keys)
+
+        expected_command = expected_event.delete('command')
+        actual_command = actual_event.delete('command')
+
+        # Hash#compact is ruby 2.4+
+        expected_presence = expected_command.select { |k, v| !v.nil? }
+        expected_absence = expected_command.select { |k, v| v.nil? }
+
+        expected_presence.each do |k, v|
+          expect(k => actual_command[k]).to eq(k => v)
+        end
+
+        expected_absence.each do |k, v|
+          expect(actual_command).not_to have_key(k)
+        end
+
+        # this compares remaining fields in events after command is removed
+        expect(actual_event).to eq(expected_event)
+      end
+
       private
 
       def verify_result(expected, actual)
@@ -70,28 +117,59 @@ module Mongo
         when nil
           expect(actual).to be nil
         when Hash
-          actual.each do |k, v|
-            ok = expected[k] == v || handle_upserted_id(k, expected[k], v) || handle_inserted_ids(k, expected[k], v)
-            expect(ok).to be true
+          expected.each do |k, v|
+            case k
+            when 'errorContains'
+              expect(actual['errorContains']).to include(v)
+            when 'errorLabelsContain'
+              v.each do |label|
+                expect(actual['errorLabels']).to include(label)
+              end
+            when 'errorLabelsOmit'
+              v.each do |label|
+                if actual['errorLabels']
+                  expect(actual['errorLabels']).not_to include(label)
+                end
+              end
+            else
+              verify_hash_items_equal(expected, actual, k)
+            end
           end
-        when Integer
+        else
           expect(actual).to eq(expected)
         end
       end
 
-      def handle_upserted_id(field, expected_id, actual_id)
-        return true if expected_id.nil?
-        if field == 'upsertedId'
-          if expected_id.is_a?(Integer)
-            actual_id.is_a?(BSON::ObjectId) || actual_id.nil?
+      def verify_hash_items_equal(expected, actual, k)
+        expect(actual).to be_a(Hash)
+
+        if expected[k] == actual[k]
+          return
+        end
+
+        if %w(deletedCount matchedCount modifiedCount upsertedCount).include?(k)
+          # Some tests assert that some of these counts are zero.
+          # The driver may omit the respective key, which is fine.
+          if expected[k] == 0
+            expect([0, nil]).to include(actual[k])
+            return
           end
         end
-      end
 
-      def handle_inserted_ids(field, expected, actual)
-        if field == 'insertedIds'
-          expected.values == actual
+        if %w(insertedIds upsertedIds).include?(k)
+          if expected[k] == {}
+            # Like with the counts, allow a response to not specify the
+            # ids in question if the expectation is for an empty id map.
+            expect([nil, []]).to include(actual[k])
+          else
+            expect(actual[k]).to eq(expected[k].values)
+          end
+          return
         end
+
+        # This should produce a meaningful error message,
+        # even though we do not actually require that expected[k] == actual[k]
+        expect({k => expected[k]}).to eq({k => actual[k]})
       end
     end
   end

data/spec/support/crud/write.rb

@@ -61,15 +61,28 @@ module Mongo
 
         # Instantiate the operation.
         #
-        # @return [ Hash ] spec The operation spec.
+        # @param [ Hash ] spec The operation specification.
+        # @param [ Hash ] outcome_spec The outcome specification.
+        #   If not provided, outcome is taken out of operation specification.
         #
         # @since 2.0.0
-        def initialize(spec)
+        def initialize(spec, outcome_spec = nil)
           @spec = spec
           @name = spec['name']
+          @outcome = Outcome.new(outcome_spec || spec)
         end
 
-        # Whether the operation is expected to have restuls.
+        attr_reader :outcome
+
+        def object
+          'collection'
+        end
+
+        def verify_collection_name
+          'crud_spec_test'
+        end
+
+        # Whether the operation is expected to have results.
         #
         # @example Whether the operation is expected to have results.
         #   operation.has_results?
@@ -103,8 +116,10 @@ module Mongo
           return_doc = {}
           return_doc['deletedCount'] = result.deleted_count if result.deleted_count
           return_doc['insertedIds'] = result.inserted_ids if result.inserted_ids
+          return_doc['upsertedIds'] = result.upserted_ids if result.upserted_ids
           return_doc['upsertedId'] = result.upserted_id if upsert
           return_doc['upsertedCount'] = result.upserted_count if result.upserted_count
+          return_doc['insertedCount'] = result.inserted_count if result.inserted_count
           return_doc['matchedCount'] = result.matched_count if result.matched_count
           return_doc['modifiedCount'] = result.modified_count if result.modified_count
           return_doc

data/spec/support/event_subscriber.rb

@@ -20,6 +20,8 @@ class EventSubscriber
     # @since 2.5.0
     attr_reader :failed_events
 
+    attr_reader :published_events
+
     # Cache the succeeded event.
     #
     # @param [ Event ] event The event.
@@ -53,6 +55,18 @@ class EventSubscriber
       end
     end
 
+    def select_published_events(cls)
+      @published_events.select do |event|
+        event.is_a?(cls)
+      end
+    end
+
+    def published(event)
+      @mutex.synchronize do
+        @published_events << event
+      end
+    end
+
     # Clear all cached events.
     #
     # @since 2.5.1
@@ -60,6 +74,7 @@ class EventSubscriber
       @started_events = []
       @succeeded_events = []
       @failed_events = []
+      @published_events = []
       self
     end
 

data/spec/support/primary_socket.rb

@@ -7,9 +7,9 @@ module PrimarySocket
       end
 
       let(:primary_connection) do
-        connection = primary_server.pool.checkout
+        connection = primary_server.pool.check_out
         connection.connect!
-        primary_server.pool.checkin(connection)
+        primary_server.pool.check_in(connection)
         connection
       end
 

data/spec/support/spec_config.rb

@@ -10,10 +10,14 @@ class SpecConfig
       if @uri_options[:replica_set]
         @addresses = @mongodb_uri.servers
         @connect_options = { connect: :replica_set, replica_set: @uri_options[:replica_set] }
-      elsif ENV['TOPOLOGY'] == 'sharded_cluster'
-        @addresses = [ @mongodb_uri.servers.first ] # See SERVER-16836 for why we can only use one host:port
+      elsif @uri_options[:connect] == :sharded || ENV['TOPOLOGY'] == 'sharded_cluster'
+        # See SERVER-16836 for why we can only use one host:port
+        if @mongodb_uri.servers.length > 1
+          warn "Using only the first mongos (#{@mongodb_uri.servers.first})"
+        end
+        @addresses = [ @mongodb_uri.servers.first ]
         @connect_options = { connect: :sharded }
-      else
+      elsif @uri_options[:connect] == :direct
         @addresses = @mongodb_uri.servers
         @connect_options = { connect: :direct }
       end
@@ -22,7 +26,7 @@ class SpecConfig
       else
         @ssl = @uri_options[:ssl]
       end
-    else
+    elsif ENV['MONGODB_ADDRESSES']
       @addresses = ENV['MONGODB_ADDRESSES'] ? ENV['MONGODB_ADDRESSES'].split(',').freeze : [ '127.0.0.1:27017' ].freeze
       if ENV['RS_ENABLED']
         @connect_options = { connect: :replica_set, replica_set: ENV['RS_NAME'] }
@@ -32,6 +36,36 @@ class SpecConfig
         @connect_options = { connect: :direct }
       end
     end
+
+    if @addresses.nil?
+      # Discover deployment topology
+      if @mongodb_uri
+        # TLS options need to be merged for evergreen due to
+        # https://github.com/10gen/mongo-orchestration/issues/268
+        client = Mongo::Client.new(@mongodb_uri.servers, Mongo::Options::Redacted.new(
+          server_selection_timeout: 5,
+        ).merge(@mongodb_uri.uri_options).merge(ssl_options))
+        @addresses = @mongodb_uri.servers
+      else
+        client = Mongo::Client.new(['localhost:27017'], server_selection_timeout: 5)
+        @addresses = client.cluster.servers_list.map do |server|
+          server.address.to_s
+        end
+      end
+      client.cluster.next_primary
+      case client.cluster.topology.class.name
+      when /Replica/
+        @connect_options = { connect: :replica_set, replica_set: client.cluster.topology.replica_set_name }
+      when /Sharded/
+        @connect_options = { connect: :sharded }
+      when /Single/
+        @connect_options = { connect: :direct }
+      when /Unknown/
+        raise "Could not detect topology because the test client failed to connect to MongoDB deployment"
+      else
+        raise "Weird topology #{client.cluster.topology}"
+      end
+    end
   end
 
   attr_reader :uri_options, :addresses, :connect_options
@@ -77,8 +111,39 @@ class SpecConfig
     end
   end
 
+  def retry_reads
+    uri_option_or_env_var(:retry_reads, 'RETRY_READS')
+  end
+
+  def retry_writes
+    uri_option_or_env_var(:retry_writes, 'RETRY_WRITES')
+  end
+
+  def uri_option_or_env_var(driver_option_symbol, env_var_key)
+    case uri_options[driver_option_symbol]
+    when true
+      true
+    when false
+      false
+    else
+      case (ENV[env_var_key] || '').downcase
+      when 'yes', 'true', 'on', '1'
+        true
+      when 'no', 'false', 'off', '0'
+        false
+      else
+        nil
+      end
+    end
+  end
+
   def retry_writes?
-    %w(yes true on 1).include?((ENV['RETRY_WRITES'] || '').downcase)
+    if retry_writes == false
+      false
+    else
+      # Current default is to retry writes
+      true
+    end
   end
 
   def ssl?
@@ -102,17 +167,18 @@ class SpecConfig
     connect_options[:connect] == :replica_set
   end
 
-  # Derived data
-
-  # The write concern to use in the tests.
-  def write_concern
-    if connect_replica_set?
-      {w: 2}
-    else
-      {w: 1}
-    end
+  def print_summary
+    puts "Connection options: #{test_options}"
+    client = ClientRegistry.instance.global_client('basic')
+    client.cluster.next_primary
+    puts <<-EOT
+Topology: #{client.cluster.topology.class}
+connect: #{connect_options[:connect]}
+EOT
   end
 
+  # Derived data
+
   def any_port
     addresses.first.split(':')[1] || '27017'
   end
@@ -141,6 +207,14 @@ class SpecConfig
     end
   end
 
+  def client_cert_key_pem
+    if drivers_tools?
+      ENV['DRIVER_TOOLS_CLIENT_CERT_KEY_PEM']
+    else
+      "#{ssl_certs_dir}/client.pem"
+    end
+  end
+
   # The default test database for all specs.
   def test_db
     'ruby-driver'.freeze
@@ -178,18 +252,13 @@ class SpecConfig
   end
 
   def retry_writes_options
-    if retry_writes?
-      {retry_writes: true}
-    else
-      {}
-    end
+    {retry_writes: retry_writes}
   end
 
   # Base test options.
   def base_test_options
     {
       max_pool_size: 1,
-      write: write_concern,
       heartbeat_frequency: 20,
       max_read_retries: 5,
       # The test suite seems to perform a number of operations

data/spec/support/transactions.rb

@@ -13,309 +13,5 @@
 # limitations under the License.
 
 require 'support/transactions/operation'
-require 'support/transactions/verifier'
-
-module Mongo
-  module Transactions
-
-    # Represents a Transactions specification test.
-    #
-    # @since 2.6.0
-    class Spec
-
-      # The name of the collection to run the tests against.
-      #
-      # @since 2.6.0
-      COLLECTION_NAME = 'transactions-tests'.freeze
-
-      # @return [ String ] description The spec description.
-      #
-      # @since 2.6.0
-      attr_reader :description
-
-      # Instantiate the new spec.
-      #
-      # @example Create the spec.
-      #   Spec.new(file)
-      #
-      # @param [ String ] file The name of the file.
-      #
-      # @since 2.6.0
-      def initialize(file)
-        file = File.new(file)
-        contents = ERB.new(file.read).result
-        @spec = YAML.load(contents)
-        file.close
-        @description = File.basename(file)
-        @data = @spec['data']
-        @transaction_tests = @spec['tests']
-      end
-
-      # Get a list of TransactionTests for each test definition.
-      #
-      # @example Get the list of TransactionTests.
-      #   spec.tests
-      #
-      # @return [ Array<TransactionsTest> ] The list of TransactionTests.
-      #
-      # @since 2.6.0
-      def tests
-        @transaction_tests.map do |test|
-          Proc.new { Mongo::Transactions::TransactionsTest.new(@data, test, self) }
-        end.compact
-      end
-
-      def database_name
-        @spec['database_name']
-      end
-
-      def collection_name
-        @spec['collection_name']
-      end
-
-      def min_server_version
-        @spec['minServerVersion']
-      end
-    end
-
-    # Represents a single transaction test.
-    #
-    # @since 2.6.0
-    class TransactionsTest
-
-      # The test description.
-      #
-      # @return [ String ] description The test description.
-      #
-      # @since 2.6.0
-      attr_reader :description
-
-      # The expected command monitoring events
-      #
-      # @since 2.6.0
-      attr_reader :expectations
-
-      attr_reader :expected_results
-      attr_reader :skip_reason
-
-      # Instantiate the new CRUDTest.
-      #
-      # @example Create the test.
-      #   TransactionTest.new(data, test)
-      #
-      # @param [ Array<Hash> ] data The documents the collection
-      # must have before the test runs.
-      # @param [ Hash ] test The test specification.
-      # @param [ Hash ] spec The top level YAML specification.
-      #
-      # @since 2.6.0
-      def initialize(data, test, spec)
-        test = IceNine.deep_freeze(test)
-        @spec = spec
-        @data = data
-        @description = test['description']
-        @client_options = convert_client_options(test['clientOptions'] || {})
-        @session_options = snakeize_hash(test['sessionOptions'] || {})
-        @fail_point = test['failPoint']
-        @operations = test['operations']
-        @expectations = test['expectations']
-        @skip_reason = test['skipReason']
-        @outcome = test['outcome']
-        @expected_results = @operations.map do |o|
-          result = o['result']
-          next result unless result.class == Hash
-
-          # Change maps of result ids to arrays of ids
-          result.dup.tap do |r|
-            r.each do |k, v|
-              next unless ['insertedIds', 'upsertedIds'].include?(k)
-              r[k] = v.to_a.sort_by(&:first).map(&:last)
-            end
-          end
-        end
-      end
-
-      def support_client
-        @support_client ||= ClientRegistry.instance.global_client('root_authorized').use(@spec.database_name)
-      end
-
-      def admin_support_client
-        @admin_support_client ||= support_client.use('admin')
-      end
-
-      def test_client
-        @test_client ||= ClientRegistry.instance.global_client('authorized_without_retry_writes').with(
-          @client_options.merge(
-            database: @spec.database_name,
-            app_name: 'this is used solely to force the new client to create its own cluster'))
-      end
-
-      def event_subscriber
-        @event_subscriber ||= EventSubscriber.new
-      end
-
-      # Run the test.
-      #
-      # @example Run the test.
-      #   test.run
-      #
-      # @return [ Result ] The result of running the test.
-      #
-      # @since 2.6.0
-      def run
-        test_client.subscribe(Mongo::Monitoring::COMMAND, event_subscriber)
-
-        results = @ops.map do |op|
-          op.execute(@collection)
-        end
-
-        session0_id = @session0.session_id
-        session1_id = @session1.session_id
-
-        @session0.end_session
-        @session1.end_session
-
-        events = event_subscriber.started_events.map do |e|
-
-          # Convert txnNumber field from a BSON integer to an extended JSON int64
-          if e.command['txnNumber']
-            e.command['txnNumber'] = {
-              '$numberLong' => e.command['txnNumber'].instance_variable_get(:@integer).to_s
-            }
-          end
-
-          # Replace the session id placeholders with the actual session ids.
-          e.command['lsid'] = 'session0' if e.command['lsid'] == session0_id
-          e.command['lsid'] = 'session1' if e.command['lsid'] == session1_id
-
-
-          # The spec files don't include these fields, so we delete them.
-          e.command.delete('$readPreference')
-          e.command.delete('bypassDocumentValidation')
-          e.command.delete('$clusterTime')
-
-
-          if e.command['readConcern']
-            # The spec test use an afterClusterTime value of 42 to indicate that we need to assert
-            # that the field exists in the actual read concern rather than comparing the value, so
-            # we replace any afterClusterTime value with 42.
-            if e.command['readConcern']['afterClusterTime']
-              e.command['readConcern']['afterClusterTime'] = 42
-            end
-
-            # Convert the readConcern level from a symbol to a string.
-            if e.command['readConcern']['level']
-              e.command['readConcern']['level'] = e.command['readConcern']['level'].to_s
-            end
-          end
-
-          # This write concern is sent for some server topologies/configurations, but not all, so it
-          # doesn't appear in the expected events.
-          e.command.delete('writeConcern') if e.command['writeConcern'] == { 'w' => 2 }
-
-          # The spec tests use 42 as a placeholder value for any getMore cursorId.
-          e.command['getMore'] = { '$numberLong' => '42' } if e.command['getMore']
-
-          # Remove fields if empty
-          e.command.delete('cursor') if e.command['cursor'] && e.command['cursor'].empty?
-          e.command.delete('filter') if e.command['filter'] && e.command['filter'].empty?
-          e.command.delete('query') if e.command['query'] && e.command['query'].empty?
-
-          {
-            'command_started_event' => {
-              'command' => order_hash(e.command),
-              'command_name' => e.command_name.to_s,
-              'database_name' => e.database_name
-            }
-          }
-        end
-
-        # Remove any events from authentication commands.
-        events.reject! { |c| c['command_started_event']['command_name'].start_with?('sasl') }
-
-        if @fail_point
-          admin_support_client.command(configureFailPoint: 'failCommand', mode: 'off')
-        end
-
-        events.map! do |event|
-          event['command_started_event'] = order_hash(event['command_started_event'])
-        end
-
-        {
-          results: results,
-          contents: @collection.find.to_a,
-          events: events,
-        }
-      end
-
-      def setup_test
-        begin
-          admin_support_client.command(killAllSessions: [])
-        rescue Mongo::Error
-        end
-
-        coll = support_client[@spec.collection_name]
-        coll.database.drop
-        coll.with(write: { w: :majority }).drop
-        support_client.command(create: @spec.collection_name, writeConcern: { w: :majority })
-
-        coll.with(write: { w: :majority }).insert_many(@data) unless @data.empty?
-        admin_support_client.command(@fail_point) if @fail_point
-
-        @collection = test_client[@spec.collection_name]
-
-        @session0 = test_client.start_session(@session_options[:session0] || {})
-        @session1 = test_client.start_session(@session_options[:session1] || {})
-
-        @ops = @operations.map do |op|
-          Operation.new(op, @session0, @session1)
-        end
-      end
-
-      def teardown_test
-        if @admin_support_client
-          @admin_support_client.close
-        end
-        if @test_client
-          @test_client.close
-        end
-      end
-
-      # The expected data in the collection as an outcome after running this test.
-      #
-      # @example Get the outcome collection data
-      #   test.outcome_collection_data
-      #
-      # @return [ Array<Hash> ] The list of documents expected to be in the collection
-      #   after running this test.
-      #
-      # @since 2.6.0
-      def outcome_collection_data
-        @outcome['collection']['data'] if @outcome && @outcome['collection']
-      end
-
-      def order_hash(hash)
-        Hash[hash.to_a.sort]
-      end
-
-      private
-
-      def convert_client_options(client_options)
-        client_options.reduce({}) do |opts, kv|
-          case kv.first
-          when 'readConcernLevel'
-            kv = [:read_concern, { 'level' => kv.last }]
-          when 'readPreference'
-            kv = [:read, { 'mode' => kv.last }]
-          when 'w'
-            kv = [:write, { w: kv.last }]
-          else
-            kv[0] = camel_to_snake(kv[0])
-          end
-
-          opts.tap { |o| o[kv.first] = kv.last }
-        end
-      end
-    end
-  end
-end
+require 'support/transactions/spec'
+require 'support/transactions/test'