mongo 1.8.6 → 1.12.5

data/test/test_helper.rb

@@ -1,210 +1,61 @@
-begin
-  require 'pry-rescue'
-  require 'pry-nav'
-rescue LoadError
-  # failed to load, skipping pry
-end
+# Copyright (C) 2009-2013 MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
-# SimpleCov must load before our code - A coverage report summary line will print after each test suite
-if RUBY_VERSION >= '1.9.0' && RUBY_ENGINE == 'ruby'
-  if ENV.key?('COVERAGE')
-    require 'simplecov'
-    SimpleCov.start do
-      add_group "Mongo", 'lib/mongo'
-      add_group "BSON", 'lib/bson'
-      add_filter "/test/"
-      merge_timeout 3600
-      command_name ENV['SIMPLECOV_COMMAND_NAME'] if ENV.has_key?('SIMPLECOV_COMMAND_NAME')
-    end
-  end
-end
-gem 'test-unit' # Do NOT remove this line - gem version is needed for Test::Unit::TestCase.shutdown
-require 'test/unit'
-require 'tools/mongo_config'
-
-class Test::Unit::TestCase
-
-  TEST_DATA = File.join(File.dirname(__FILE__), 'data')
-
-  def ensure_cluster(kind=nil, opts={})
-    @@cluster ||= nil
+# NOTE: on ruby <1.9 you need to run individual tests with 'bundle exec'
 
-    unless @@cluster
-      if kind == :rs
-        cluster_opts = Mongo::Config::DEFAULT_REPLICA_SET.dup
-      else
-        cluster_opts = Mongo::Config::DEFAULT_SHARDED_SIMPLE.dup
-      end
+unless RUBY_VERSION < '1.9' || ENV.key?('JENKINS_CI')
+  require 'simplecov'
+  require 'coveralls'
 
-      cluster_opts.merge!(opts)
+  SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
+    SimpleCov::Formatter::HTMLFormatter,
+    Coveralls::SimpleCov::Formatter
+  ]
 
-      dbpath = ENV['DBPATH'] || 'data'
-      cluster_opts.merge!(:dbpath => dbpath)
+  SimpleCov.start do
+    add_group 'Driver', 'lib/mongo'
+    add_group 'BSON', 'lib/bson'
 
-      #debug 1, opts
-      config = Mongo::Config.cluster(cluster_opts)
-      #debug 1, config
-      @@cluster = Mongo::Config::ClusterManager.new(config)
-
-      Test::Unit::TestCase.class_eval do
-        @@force_shutdown = false
-
-        def self.shutdown
-          if @@force_shutdown || /rake_test_loader/ !~ $0
-            @@cluster.stop
-            @@cluster.clobber
-          end
-        end
-      end
-    end
-
-    @@cluster.start
-    instance_variable_set("@#{kind}", @@cluster)
+    add_filter 'tasks'
+    add_filter 'test'
+    add_filter 'bin'
   end
-
-  # Generic code for rescuing connection failures and retrying operations.
-  # This could be combined with some timeout functionality.
-  def rescue_connection_failure(max_retries=30)
-    retries = 0
-    begin
-      yield
-    rescue Mongo::ConnectionFailure => ex
-      #puts "Rescue attempt #{retries}: from #{ex}"
-      retries += 1
-      raise ex if retries > max_retries
-      sleep(2)
-      retry
-    end
-  end
-end
-
-def silently
-  warn_level = $VERBOSE
-  $VERBOSE = nil
-  begin
-    result = yield
-  ensure
-    $VERBOSE = warn_level
-  end
-  result
-end
-
-begin
-  silently { require 'shoulda' }
-  silently { require 'mocha/setup' }
-rescue LoadError
-  puts <<MSG
-
-This test suite requires shoulda and mocha.
-You can install them as follows:
-  gem install shoulda
-  gem install mocha
-
-MSG
-  exit
-end
-
-unless defined? MONGO_TEST_DB
-  MONGO_TEST_DB = 'ruby-test-db'
-end
-
-unless defined? TEST_PORT
-  TEST_PORT = ENV['MONGO_RUBY_DRIVER_PORT'] ? ENV['MONGO_RUBY_DRIVER_PORT'].to_i : Mongo::MongoClient::DEFAULT_PORT
-end
-
-unless defined? TEST_HOST
-  TEST_HOST = ENV['MONGO_RUBY_DRIVER_HOST'] || 'localhost'
 end
 
-class Test::Unit::TestCase
-  include Mongo
-  include BSON
+# required for at_exit, at_start hooks
+require 'test-unit'
 
-  def self.standard_connection(options={}, legacy=false)
-    if legacy
-      Connection.new(TEST_HOST, TEST_PORT, options)
-    else
-      MongoClient.new(TEST_HOST, TEST_PORT, options)
-    end
-  end
-
-  def standard_connection(options={}, legacy=false)
-    self.class.standard_connection(options, legacy)
-  end
-
-  def self.host_port
-    "#{mongo_host}:#{mongo_port}"
-  end
-
-  def self.mongo_host
-    TEST_HOST
-  end
-
-  def self.mongo_port
-    TEST_PORT
-  end
-
-  def host_port
-    self.class.host_port
-  end
-
-  def mongo_host
-    self.class.mongo_host
-  end
-
-  def mongo_port
-    self.class.mongo_port
-  end
-
-  def method_name
-    caller[0]=~/`(.*?)'/
-    $1
-  end
-
-  def step_down_command
-    # Adding force=true to avoid 'no secondaries within 10 seconds of my optime' errors
-    step_down_command = BSON::OrderedHash.new
-    step_down_command[:replSetStepDown] = 5
-    step_down_command[:force]           = true
-    step_down_command
-  end
-
-  def new_mock_socket(host='localhost', port=27017)
-    socket = Object.new
-    socket.stubs(:setsockopt).with(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
-    socket.stubs(:close)
-    socket.stubs(:closed?)
-    socket.stubs(:checkin)
-    socket.stubs(:pool)
-    socket
-  end
+require 'test/unit'
+require 'shoulda'
+require 'mocha/setup'
 
-  def new_mock_unix_socket(sockfile='/tmp/mongod.sock')
-    socket = Object.new
-    socket.stubs(:setsockopt).with(Socket::IPPROTO_TCP)
-    socket.stubs(:close)
-    socket.stubs(:closed?)
-    socket
-  end
+# cluster manager
+require 'tools/mongo_config'
 
-  def new_mock_db
-    Object.new
-  end
+# For kerberos testing.
+begin
+  require 'mongo_kerberos'
+rescue LoadError; end
 
-  def assert_raise_error(klass, message=nil)
-    begin
-      yield
-    rescue => e
-      if klass.to_s != e.class.to_s
-        flunk "Expected exception class #{klass} but got #{e.class}.\n #{e.backtrace}"
-      end
+# test helpers
+require 'helpers/general'
+require 'helpers/test_unit'
 
-      if message && !e.message.include?(message)
-        p e.backtrace
-        flunk "#{e.message} does not include #{message}.\n#{e.backtrace}"
-      end
-    else
-      flunk "Expected assertion #{klass} but none was raised."
-    end
-  end
+# optional development and debug utilities
+begin
+  require 'pry-rescue'
+  require 'pry-nav'
+rescue LoadError
+  # failed to load, skipping pry
 end

data/test/threading/basic_test.rb

@@ -1,12 +1,26 @@
+# Copyright (C) 2009-2013 MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 require 'test_helper'
 
-class TestThreading < Test::Unit::TestCase
+class ThreadingTest < Test::Unit::TestCase
 
   include Mongo
 
   def setup
     @client = standard_connection(:pool_size => 10, :pool_timeout => 30)
-    @db = @client.db(MONGO_TEST_DB)
+    @db = @client.db(TEST_DB)
     @coll = @db.collection('thread-test-collection')
     @coll.drop
 
@@ -77,4 +91,30 @@ class TestThreading < Test::Unit::TestCase
     assert thread_values.all?{|v| v == 1000}
     assert_equal thread_values.size, n_threads
   end
+
+  def test_threading
+    @coll.drop
+    @coll = @db.collection('thread-test-collection')
+
+    docs = []
+    1000.times {|i| docs << {:x => i}}
+    @coll.insert(docs)
+
+    threads = []
+
+    10.times do |i|
+      threads[i] = Thread.new do
+        sum = 0
+        @coll.find().each do |document|
+          sum += document["x"]
+        end
+        assert_equal 499500, sum
+      end
+    end
+
+    10.times do |i|
+      threads[i].join
+    end
+  end
+
 end

data/test/tools/mongo_config.rb

@@ -1,4 +1,19 @@
 #!/usr/bin/env ruby
+
+# Copyright (C) 2009-2013 MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 require 'socket'
 require 'fileutils'
 require 'mongo'
@@ -26,7 +41,7 @@ module Mongo
   class Config
     DEFAULT_BASE_OPTS = { :host => 'localhost', :dbpath => 'data', :logpath => 'data/log' }
     DEFAULT_REPLICA_SET = DEFAULT_BASE_OPTS.merge( :replicas => 3, :arbiters => 0 )
-    DEFAULT_SHARDED_SIMPLE = DEFAULT_BASE_OPTS.merge( :shards => 2, :configs => 1, :routers => 4 )
+    DEFAULT_SHARDED_SIMPLE = DEFAULT_BASE_OPTS.merge( :shards => 2, :configs => 1, :routers => 2 )
     DEFAULT_SHARDED_REPLICA = DEFAULT_SHARDED_SIMPLE.merge( :replicas => 3, :arbiters => 0)
 
     IGNORE_KEYS = [:host, :command, :_id]
@@ -35,7 +50,7 @@ module Mongo
     MONGODS_OPT_KEYS = [:mongods]
     CLUSTER_OPT_KEYS = SHARDING_OPT_KEYS + REPLICA_OPT_KEYS + MONGODS_OPT_KEYS
 
-    FLAGS = [:noprealloc, :smallfiles, :logappend, :configsvr, :shardsvr, :quiet, :fastsync, :auth]
+    FLAGS = [:noprealloc, :smallfiles, :logappend, :configsvr, :shardsvr, :quiet, :fastsync, :auth, :ipv6]
 
     DEFAULT_VERIFIES = 60
     BASE_PORT = 3000
@@ -49,6 +64,7 @@ module Mongo
       raise "missing required option" if [:host, :dbpath].any?{|k| !opts[k]}
 
       config = opts.reject {|k,v| CLUSTER_OPT_KEYS.include?(k)}
+
       kinds = CLUSTER_OPT_KEYS.select{|key| opts.has_key?(key)} # order is significant
 
       replica_count = 0
@@ -67,9 +83,11 @@ module Mongo
                 make_config(opts)
               when :routers
                 make_router(config, opts)
+              when :shards
+                make_standalone_shard(kind, opts)
               else
                 make_mongod(kind, opts)
-            end
+              end
 
             replica_count += 1 if [:replicas, :arbiters].member?(kind)
             node
@@ -102,6 +120,8 @@ module Mongo
       quiet      = opts[:quiet]      || true
       fast_sync  = opts[:fastsync]   || false
       auth       = opts[:auth]       || true
+      ipv6       = opts[:ipv6].nil? ? true : opts[:ipv6]
+      setParameter = opts[:setParameter] || 'enableTestCommands=1'
 
       params.merge(:command    => mongod,
                    :dbpath     => path,
@@ -109,7 +129,23 @@ module Mongo
                    :noprealloc => noprealloc,
                    :quiet      => quiet,
                    :fastsync   => fast_sync,
-                   :auth       => auth)
+                   :auth       => auth,
+                   :ipv6       => ipv6,
+                   :setParameter => setParameter)
+    end
+
+    def self.key_file(opts)
+      keyFile = opts[:key_file] || '/test/fixtures/auth/keyfile'
+      keyFile = Dir.pwd << keyFile
+      system "chmod 600 #{keyFile}"
+      keyFile
+    end
+
+    # A regular mongod minus --auth and plus --keyFile.
+    def self.make_standalone_shard(kind, opts)
+      params = make_mongod(kind, opts)
+      params.delete(:auth)
+      params.merge(:keyFile => key_file(opts))
     end
 
     def self.make_replica(opts, id)
@@ -117,20 +153,18 @@ module Mongo
 
       replSet    = opts[:replSet]    || 'ruby-driver-test'
       oplogSize  = opts[:oplog_size] || 5
-      keyFile    = opts[:key_file]   || '/test/tools/keyfile.txt'
-
-      keyFile    = Dir.pwd << keyFile
-      system "chmod 600 #{keyFile}"
 
       params.merge(:_id       => id,
                    :replSet   => replSet,
                    :oplogSize => oplogSize,
-                   :keyFile   => keyFile)
+                   :keyFile   => key_file(opts))
     end
 
     def self.make_config(opts)
       params = make_mongod('configs', opts)
-      params.merge(:configsvr => nil)
+      params.delete(:auth)
+      params.merge(:configsvr => true,
+                   :keyFile => key_file(opts))
     end
 
     def self.make_router(config, opts)
@@ -138,8 +172,9 @@ module Mongo
       mongos = ENV['MONGOS'] || 'mongos'
 
       params.merge(
-        :command => mongos,
-        :configdb => self.configdb(config)
+        :command  => mongos,
+        :configdb => self.configdb(config),
+        :keyFile  => key_file(opts)
       )
     end
 
@@ -268,6 +303,11 @@ module Mongo
 
       def initialize(config)
         @config = config
+        cmd = init_config!
+        super(cmd, @config[:host], @config[:port])
+      end
+
+      def init_config!
         dbpath = @config[:dbpath]
         [dbpath, File.dirname(@config[:logpath])].compact.each{|dir| FileUtils.mkdir_p(dir) unless File.directory?(dir) }
         command = @config[:command] || 'mongod'
@@ -281,7 +321,6 @@ module Mongo
           end
         end
         cmd = [command, arguments].flatten.compact
-        super(cmd, @config[:host], @config[:port])
       end
 
       def start(verifies = DEFAULT_VERIFIES)
@@ -301,8 +340,13 @@ module Mongo
             sleep 1
           end
         end
-        system "ps -fp #{@pid}; cat #{@config[:logpath]}"
-        raise Mongo::ConnectionFailure, "DbServer.start verify via connection probe failed - port:#{@port.inspect} @pid:#{@pid.inspect} kill:#{Process.kill(0, @pid).inspect} running?:#{running?.inspect} cmd:#{cmd.inspect}"
+        if @config.delete(:setParameter)
+          @cmd = init_config!
+          start(verifies)
+        else
+          system "ps -fp #{@pid}; cat #{@config[:logpath]}"
+          raise Mongo::ConnectionFailure, "DbServer.start verify via connection probe failed - port:#{@port.inspect} @pid:#{@pid.inspect} kill:#{Process.kill(0, @pid).inspect} running?:#{running?.inspect} cmd:#{cmd.inspect}"
+        end
       end
 
     end
@@ -313,7 +357,7 @@ module Mongo
         @config = config
         @servers = {}
         Mongo::Config::CLUSTER_OPT_KEYS.each do |key|
-          @servers[key] = @config[key].collect{|conf| DbServer.new(conf)} if @config[key]
+          @servers[key] = @config[key].collect{|conf| p conf; DbServer.new(conf)} if @config[key]
         end
       end
 
@@ -321,6 +365,35 @@ module Mongo
         @servers.collect{|k,v| (!key || key == k) ? v : nil}.flatten.compact
       end
 
+      def ensure_authenticated(client)
+        begin
+          client[TEST_DB].authenticate(TEST_USER, TEST_USER_PWD)
+        rescue Mongo::MongoArgumentError => ex
+          # client is already authenticated
+          raise ex unless ex.message =~ /already authenticated/
+        rescue Mongo::AuthenticationError => ex
+          # 1) The creds are wrong
+          # 2) Or the user doesn't exist
+          roles = [ 'dbAdminAnyDatabase',
+                    'userAdminAnyDatabase',
+                    'readWriteAnyDatabase',
+                    'clusterAdmin' ]
+          begin
+            # Try to add the user for case (2)
+            client[TEST_DB].add_user(TEST_USER, TEST_USER_PWD, nil, :roles => roles)
+            client[TEST_DB].authenticate(TEST_USER, TEST_USER_PWD)
+          rescue Mongo::ConnectionFailure, Mongo::OperationFailure => ex
+            # Maybe not master, so try to authenticate
+            # 2.2 throws an OperationFailure if add_user fails
+            begin
+              client[TEST_DB].authenticate(TEST_USER, TEST_USER_PWD)
+            rescue => ex
+              # Maybe creds are wrong, nothing we can do
+            end
+          end
+        end
+      end
+
       def command( cmd_servers, db_name, cmd, opts = {} )
         ret = []
         cmd = cmd.class == Array ? cmd : [ cmd ]
@@ -330,6 +403,7 @@ module Mongo
           debug 3, cmd_server.inspect
           cmd_server = cmd_server.config if cmd_server.is_a?(DbServer)
           client = Mongo::MongoClient.new(cmd_server[:host], cmd_server[:port])
+          ensure_authenticated(client)
           cmd.each do |c|
             debug 3,  "ClusterManager.command c:#{c.inspect}"
             response = client[db_name].command( c, opts )
@@ -343,6 +417,10 @@ module Mongo
         ret.size == 1 ? ret.first : ret
       end
 
+      def replica_set?
+        !!config[:replicas]
+      end
+
       def repl_set_get_status
         command( @config[:replicas], 'admin', { :replSetGetStatus => 1 }, {:check_response => false } )
       end
@@ -350,6 +428,7 @@ module Mongo
       def repl_set_get_config
         host, port = primary_name.split(":")
         client = Mongo::MongoClient.new(host, port)
+        ensure_authenticated(client)
         client['local']['system.replset'].find_one
       end
 
@@ -368,27 +447,49 @@ module Mongo
       end
 
       def repl_set_startup
-        states = nil
-        60.times do
-          states = repl_set_get_status.zip(repl_set_is_master)
+        states     = nil
+        healthy    = false
+
+        80.times do
+          # enter the thunderdome...
+          states  = repl_set_get_status.zip(repl_set_is_master)
           healthy = states.all? do |status, is_master|
-            members = status['members']
-            if status['ok'] == 1.0 && members.collect{|m| m['state']}.all?{|state| [1,2,7].index(state)}
-              members.any?{|m| m['state'] == 1} &&
-                case status['myState']
-                when 1
-                  is_master['ismaster'] == true && is_master['secondary'] == false
-                when 2
-                  is_master['ismaster'] == false && is_master['secondary'] == true
-                when 7
-                  is_master['ismaster'] == false && is_master['secondary'] == false
-                end
+            # check replica set status for member list
+            next unless status['ok'] == 1.0 && (members = status['members'])
+
+            # ensure all replica set members are in a valid state
+            next unless members.all? { |m| [1,2,7].include?(m['state']) }
+
+            # check for primary replica set member
+            next unless (primary = members.find { |m| m['state'] == 1 })
+
+            # check replica set member optimes
+            primary_optime = (primary['optime']['ts'] ? primary['optime']['ts'] : primary['optime']).seconds
+            next unless primary_optime && members.all? do |m|
+              m_optime = (m['optime']['ts'] ? m['optime']['ts'] : m['optime']).seconds
+              m['state'] == 7 || primary_optime - m_optime < 5
+            end
+
+            # check replica set state
+            case status['myState']
+              when 1
+                is_master['ismaster']  == true &&
+                is_master['secondary'] == false
+              when 2
+                is_master['ismaster']  == false &&
+                is_master['secondary'] == true
+              when 7
+                is_master['ismaster']  == false &&
+                is_master['secondary'] == false
             end
           end
-          return true if healthy
+
+          return healthy if healthy
           sleep(1)
         end
-        raise Mongo::OperationFailure, "replSet startup failed - status: #{states.inspect}"
+
+        raise Mongo::OperationFailure,
+          "replSet startup failed - status: #{states.inspect}"
       end
 
       def repl_set_seeds
@@ -403,13 +504,19 @@ module Mongo
         repl_set_seeds.join(',')
       end
 
+      def members_uri
+        members = @config[:replicas] || @config[:routers]
+        members.collect{|node| "#{node[:host]}:#{node[:port]}"}.join(',')
+      end
+
       def repl_set_name
         @config[:replicas].first[:replSet]
       end
 
       def member_names_by_state(state)
         states = Array(state)
-        status = repl_set_get_status.first
+        # Any status with a REMOVED node won't have the full cluster state
+        status = repl_set_get_status.find {|status| status['members'].find {|m| m['state'] == 'REMOVED'}.nil?}
         status['members'].find_all{|member| states.index(member['state']) }.collect{|member| member['name']}
       end
 
@@ -513,7 +620,15 @@ module Mongo
       end
 
       def addshards(shards = @config[:shards])
-        command( @config[:routers].first, 'admin', Array(shards).collect{|s| { :addshard => "#{s[:host]}:#{s[:port]}" } } )
+        begin
+          command( @config[:routers].first, 'admin', Array(shards).collect{|s| { :addshard => "#{s[:host]}:#{s[:port]}" } } )
+        rescue Mongo::OperationFailure => ex
+          # Because we cannot run the listshards command under the localhost
+          # exception in > 2.7.1, we run the risk of attempting to add the same shard twice.
+          # Our tests may add a local db to a shard, if the cluster is still up,
+          # then we can ignore this.
+          raise ex unless ex.message =~ /host already used/
+        end
       end
 
       def listshards
@@ -543,7 +658,8 @@ module Mongo
         servers.each{|server| server.start}
         # TODO - sharded replica sets - pending
         if @config[:replicas]
-          repl_set_initiate if repl_set_get_status.first['startupStatus'] == 3
+          repl_set_initiate if repl_set_get_status.first['code'] == 94 ||
+            (repl_set_get_status.first['startupStatus'] && repl_set_get_status.first['startupStatus'] == 3)
           repl_set_startup
         end
         if @config[:routers]
@@ -553,7 +669,31 @@ module Mongo
       end
       alias :restart :start
 
+      def delete_users
+        cmd_servers = replica_set? ? [ primary ] : routers
+
+        cmd_servers.each do |cmd_server|
+          next unless cmd_server
+          begin
+            client = Mongo::MongoClient.new(cmd_server.config[:host],
+                                            cmd_server.config[:port])
+            ensure_authenticated(client)
+            db = client[TEST_DB]
+
+            if client.server_version < '2.5'
+              db['system.users'].remove
+            else
+              db.command(:dropAllUsersFromDatabase => 1)
+            end
+            break
+          rescue Mongo::ConnectionFailure
+          end
+        end
+      end
+
       def stop
+        start
+        delete_users
         servers.each{|server| server.stop}
         self
       end