mongo 1.7.1 → 1.8.0

data/test/support/hash_with_indifferent_access.rb

@@ -183,4 +183,4 @@ module ActiveSupport
   end
 end
 
-HashWithIndifferentAccess = ActiveSupport::HashWithIndifferentAccess
+#HashWithIndifferentAccess = ActiveSupport::HashWithIndifferentAccess

data/test/test_helper.rb

@@ -1,8 +1,52 @@
-$:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 require 'rubygems'
-require 'mongo'
-gem 'test-unit'
 require 'test/unit'
+require 'tools/mongo_config'
+
+class Test::Unit::TestCase
+
+  TEST_DATA = File.join(File.dirname(__FILE__), 'data')
+
+  def ensure_cluster(kind=nil)
+    if defined?(@@current_class) and @@current_class == self.class
+      @@cluster.start
+    else
+      @@current_class = self.class
+
+      if kind == :rs
+        opts = Mongo::Config::DEFAULT_REPLICA_SET
+        opts.merge!(:arbiters => 2)
+      else
+        opts = Mongo::Config::DEFAULT_SHARDED_SIMPLE
+        opts.merge!(:routers => 4)
+      end
+
+      dbpath = ENV['DBPATH'] || 'data'
+      opts.merge!(:dbpath => dbpath)
+
+      #debug 1, opts
+      config = Mongo::Config.cluster(opts)
+      #debug 1, config
+      @@cluster = Mongo::Config::ClusterManager.new(config)
+      @@cluster.start
+    end
+    instance_variable_set("@#{kind}", @@cluster)
+  end
+
+  # Generic code for rescuing connection failures and retrying operations.
+  # This could be combined with some timeout functionality.
+  def rescue_connection_failure(max_retries=30)
+    retries = 0
+    begin
+      yield
+    rescue Mongo::ConnectionFailure => ex
+      #puts "Rescue attempt #{retries}: from #{ex}"
+      retries += 1
+      raise ex if retries > max_retries
+      sleep(2)
+      retry
+    end
+  end
+end
 
 def silently
   warn_level = $VERBOSE
@@ -16,7 +60,6 @@ def silently
 end
 
 begin
-  require 'rubygems' if RUBY_VERSION < "1.9.0" && !ENV['C_EXT']
   silently { require 'shoulda' }
   silently { require 'mocha' }
 rescue LoadError
@@ -38,7 +81,7 @@ unless defined? MONGO_TEST_DB
 end
 
 unless defined? TEST_PORT
-  TEST_PORT = ENV['MONGO_RUBY_DRIVER_PORT'] ? ENV['MONGO_RUBY_DRIVER_PORT'].to_i : Mongo::Connection::DEFAULT_PORT
+  TEST_PORT = ENV['MONGO_RUBY_DRIVER_PORT'] ? ENV['MONGO_RUBY_DRIVER_PORT'].to_i : Mongo::MongoClient::DEFAULT_PORT
 end
 
 unless defined? TEST_HOST
@@ -49,12 +92,16 @@ class Test::Unit::TestCase
   include Mongo
   include BSON
 
-  def self.standard_connection(options={})
-    Connection.new(TEST_HOST, TEST_PORT, options)
+  def self.standard_connection(options={}, legacy=false)
+    if legacy
+      Connection.new(TEST_HOST, TEST_PORT, options)
+    else
+      MongoClient.new(TEST_HOST, TEST_PORT, options)
+    end
   end
 
-  def standard_connection(options={})
-    self.class.standard_connection(options)
+  def standard_connection(options={}, legacy=false)
+    self.class.standard_connection(options, legacy)
   end
 
   def self.host_port

data/test/threading/threading_with_large_pool_test.rb

@@ -1,4 +1,4 @@
-require File.expand_path("../../test_helper", __FILE__)
+require 'test_helper'
 
 # Essentialy the same as test_threading.rb but with an expanded pool for
 # testing multiple connections.
@@ -6,7 +6,8 @@ class TestThreadingLargePool < Test::Unit::TestCase
 
   include Mongo
 
-  @@db = standard_connection(:pool_size => 50, :pool_timeout => 60).db(MONGO_TEST_DB)
+  @@client = standard_connection(:pool_size => 50, :pool_timeout => 60)
+  @@db = @@client.db(MONGO_TEST_DB)
   @@coll = @@db.collection('thread-test-collection')
 
   def set_up_safe_data
@@ -29,10 +30,10 @@ class TestThreadingLargePool < Test::Unit::TestCase
       threads[i] = Thread.new do
         if i % 2 == 0
           assert_raise Mongo::OperationFailure do
-            @unique.update({"test" => "insert"}, {"$set" => {"test" => "update"}}, :safe => true)
+            @unique.update({"test" => "insert"}, {"$set" => {"test" => "update"}})
           end
         else
-          @duplicate.update({"test" => "insert"}, {"$set" => {"test" => "update"}}, :safe => true)
+          @duplicate.update({"test" => "insert"}, {"$set" => {"test" => "update"}})
         end
       end
     end
@@ -49,10 +50,10 @@ class TestThreadingLargePool < Test::Unit::TestCase
       threads[i] = Thread.new do
         if i % 2 == 0
           assert_raise Mongo::OperationFailure do
-            @unique.insert({"test" => "insert"}, :safe => true)
+            @unique.insert({"test" => "insert"})
           end
         else
-          @duplicate.insert({"test" => "insert"}, :safe => true)
+          @duplicate.insert({"test" => "insert"})
         end
       end
     end
@@ -67,7 +68,7 @@ class TestThreadingLargePool < Test::Unit::TestCase
     @@coll = @@db.collection('thread-test-collection')
 
     1000.times do |i|
-      @@coll.insert("x" => i, :safe => true)
+      @@coll.insert({ "x" => i })
     end
 
     threads = []
@@ -86,5 +87,4 @@ class TestThreadingLargePool < Test::Unit::TestCase
       threads[i].join
     end
   end
-
 end

data/test/tools/mongo_config.rb

@@ -1,15 +1,12 @@
 #!/usr/bin/env ruby
-
 require 'socket'
 require 'fileutils'
+require 'mongo'
+require 'sfl'
 
 $debug_level = 2
 STDOUT.sync = true
 
-unless defined? Mongo
-  require File.join(File.dirname(__FILE__), '..', '..', 'lib', 'mongo')
-end
-
 def debug(level, arg)
   if level <= $debug_level
     file_line = caller[0][/(.*:\d+):/, 1]
@@ -18,19 +15,28 @@ def debug(level, arg)
   end
 end
 
+#
+# Design Notes
+#     Configuration and Cluster Management are modularized with the concept that the Cluster Manager
+#     can be supplied with any configuration to run.
+#     A configuration can be edited, modified, copied into a test file, and supplied to a cluster manager
+#     as a parameter.
+#
 module Mongo
   class Config
-    DEFAULT_BASE_OPTS = { :host => 'localhost', :logpath => 'data/log', :dbpath => 'data' }
-    DEFAULT_REPLICA_SET = DEFAULT_BASE_OPTS.merge( :replicas => 3 )
+    DEFAULT_BASE_OPTS = { :host => 'localhost', :dbpath => 'data', :logpath => 'data/log'}
+    DEFAULT_REPLICA_SET = DEFAULT_BASE_OPTS.merge( :replicas => 3, :arbiters => 0 )
     DEFAULT_SHARDED_SIMPLE = DEFAULT_BASE_OPTS.merge( :shards => 2, :configs => 1, :routers => 2 )
-    DEFAULT_SHARDED_REPLICA = DEFAULT_SHARDED_SIMPLE.merge( :replicas => 3 )
+    DEFAULT_SHARDED_REPLICA = DEFAULT_SHARDED_SIMPLE.merge( :replicas => 3, :arbiters => 0)
 
-    SERVER_PRELUDE_KEYS = [:host, :command]
+    IGNORE_KEYS = [:host, :command, :_id]
     SHARDING_OPT_KEYS = [:shards, :configs, :routers]
-    REPLICA_OPT_KEYS = [:replicas]
+    REPLICA_OPT_KEYS = [:replicas, :arbiters]
     MONGODS_OPT_KEYS = [:mongods]
     CLUSTER_OPT_KEYS = SHARDING_OPT_KEYS + REPLICA_OPT_KEYS + MONGODS_OPT_KEYS
 
+    FLAGS = [:noprealloc, :smallfiles, :logappend, :configsvr, :shardsvr, :quiet]
+
     DEFAULT_VERIFIES = 60
     BASE_PORT = 3000
     @@port = BASE_PORT
@@ -41,31 +47,99 @@ module Mongo
 
     def self.cluster(opts = DEFAULT_SHARDED_SIMPLE)
       raise "missing required option" if [:host, :dbpath].any?{|k| !opts[k]}
-      config = opts.reject{|k,v| CLUSTER_OPT_KEYS.include?(k)}
-      keys = SHARDING_OPT_KEYS.any?{|k| opts[k]} ? SHARDING_OPT_KEYS : nil
-      keys ||= REPLICA_OPT_KEYS.any?{|k| opts[k]} ? REPLICA_OPT_KEYS : nil
-      keys ||= MONGODS_OPT_KEYS
-      keys.each do |key|
-        config[key] = opts.fetch(key,1).times.collect do |i| #default to 1 of whatever
-          server_base = key.to_s.chop
-          dbpath = "#{opts[:dbpath]}/#{server_base}#{i}"
-          logpath = "#{dbpath}/#{server_base}.log"
-          if key == :shards && opts[:replicas]
-            self.cluster(opts.reject{|k,v| SHARDING_OPT_KEYS.include?(k)}.merge(:dbpath => dbpath))
+
+      config = opts.reject {|k,v| CLUSTER_OPT_KEYS.include?(k)}
+      kinds = CLUSTER_OPT_KEYS.select{|key| opts.has_key?(key)} # order is significant
+
+      replica_count = 0
+
+      kinds.each do |kind|
+        config[kind] = opts.fetch(kind,1).times.collect do |i| #default to 1 of whatever
+          if kind == :shards && opts[:replicas]
+            self.cluster(opts.reject{|k,v| SHARDING_OPT_KEYS.include?(k)}.merge(:dbpath => path))
           else
-            server_params = { :host => opts[:host], :port => self.get_available_port, :logpath => logpath }
-            case key
-              when :replicas; server_params.merge!( :command => 'mongod', :dbpath => dbpath, :replSet => File.basename(opts[:dbpath]) )
-              when :configs;  server_params.merge!( :command => 'mongod', :dbpath => dbpath, :configsvr => nil )
-              when :routers;  server_params.merge!( :command => 'mongos', :configdb => self.configdb(config) ) # mongos, NO dbpath
-              else            server_params.merge!( :command => 'mongod', :dbpath => dbpath ) # :mongods, :shards
+            node = case kind
+              when :replicas
+                make_replica(opts, replica_count)
+              when :arbiters
+                make_replica(opts, replica_count)
+              when :configs
+                make_config(opts)
+              when :routers
+                make_router(config, opts)
+              else
+                make_mongod(kind, opts)
             end
+
+            replica_count += 1 if [:replicas, :arbiters].member?(kind)
+            node
           end
         end
       end
       config
     end
 
+    def self.make_mongo(kind, opts)
+      dbpath = opts[:dbpath]
+      port = self.get_available_port
+      path = "#{dbpath}/#{kind}-#{port}"
+      logpath = "#{path}/#{kind}.log"
+
+      {
+        :host => opts[:host],
+        :port => port,
+        :logpath => logpath,
+        :logappend => true
+      }
+    end
+
+    def self.make_mongod(kind, opts)
+      params = make_mongo('mongods', opts)
+
+      mongod = ENV['MONGOD'] || 'mongod'
+      path = File.dirname(params[:logpath])
+
+      noprealloc = opts[:noprealloc] || true
+      smallfiles = opts[:smallfiles] || true
+      quiet      = opts[:quiet]      || true
+
+      params.merge(
+        :command => mongod,
+        :dbpath => path,
+        :smallfiles => smallfiles,
+        :noprealloc => noprealloc,
+        :quiet => quiet
+      )
+    end
+
+    def self.make_replica(opts, count)
+      params = make_mongod('replicas', opts)
+
+      replSet    = opts[:replSet]    || 'ruby-driver-test'
+      oplog_size = opts[:oplog_size] || 10
+
+      params.merge(
+        :_id => count,
+        :replSet => replSet,
+        :oplogSize => oplog_size
+      )
+    end
+
+    def self.make_config(opts)
+      params = make_mongod('configs', opts)
+      params.merge(:configsvr => nil)
+    end
+
+    def self.make_router(config, opts)
+      params = make_mongo('routers', opts)
+      mongos = ENV['MONGOS'] || 'mongos'
+
+      params.merge(
+        :command => mongos,
+        :configdb => self.configdb(config)
+      )
+    end
+
     def self.port_available?(port)
       ret = false
       socket = Socket.new(Socket::Constants::AF_INET, Socket::Constants::SOCK_STREAM, 0)
@@ -96,15 +170,23 @@ module Mongo
         @cmd = cmd
       end
 
+      def clear_zombie
+        if @pid
+          begin
+            pid = Process.waitpid(@pid, Process::WNOHANG)
+          rescue Errno::ECHILD
+            # JVM might have already reaped the exit status
+          end
+          @pid = nil if pid && pid > 0
+        end
+      end
+
       def start(verifies = 0)
+        clear_zombie
         return @pid if running?
         begin
-          @pid = fork do
-            STDIN.reopen '/dev/null'
-            STDOUT.reopen '/dev/null', 'a'
-            STDERR.reopen STDOUT
-            exec cmd # spawn(@cmd, [:in, :out, :err] => :close) #
-          end
+          @pid = Process.spawn(*@cmd) # redirection not supported in jruby
+          sleep 1
           verify(verifies) if verifies > 0
           @pid
         end
@@ -115,16 +197,20 @@ module Mongo
         wait
       end
 
-      def kill
+      def kill(signal_no = 2)
         begin
-          @pid && Process.kill(3, @pid) && true
+          @pid && Process.kill(signal_no, @pid) && true
         rescue Errno::ESRCH
           false
         end
       end
 
       def wait
-        Process.wait(@pid) if @pid
+        begin
+          Process.waitpid(@pid) if @pid
+        rescue Errno::ECHILD
+          # JVM might have already reaped the exit status
+        end
         @pid = nil
       end
 
@@ -153,17 +239,34 @@ module Mongo
         @host = host
         @port = port
       end
+
+      def host_port
+        [@host, @port].join(':')
+      end
+
+      def host_port_a # for old format
+        [@host, @port]
+      end
     end
 
     class DbServer < Server
       attr_accessor :config
+
       def initialize(config)
         @config = config
         dbpath = @config[:dbpath]
         [dbpath, File.dirname(@config[:logpath])].compact.each{|dir| FileUtils.mkdir_p(dir) unless File.directory?(dir) }
         command = @config[:command] || 'mongod'
-        arguments = @config.reject{|k,v| SERVER_PRELUDE_KEYS.include?(k)}
-        cmd = [command, arguments.collect{|k,v| ['--' + k.to_s, v ]}].flatten.join(' ')
+        params = @config.reject{|k,v| IGNORE_KEYS.include?(k)}
+        arguments = params.sort{|a, b| a[0].to_s <=> b[0].to_s}.collect do |arg, value| # sort block is needed for 1.8.7 which lacks Symbol#<=>
+          argument = '--' + arg.to_s
+          if FLAGS.member?(arg)
+            [argument]
+          else
+            [argument, value.to_s]
+          end
+        end
+        cmd = [command, arguments].flatten.compact
         super(cmd, @config[:host], @config[:port])
       end
 
@@ -172,19 +275,20 @@ module Mongo
         verify(verifies)
       end
 
-      def verify(verifies = 10)
+      def verify(verifies = 600)
         verifies.times do |i|
-          #puts "DbServer.verify - port: #{@port} iteration: #{i}"
+          #puts "DbServer.verify via connection probe - port:#{@port.inspect} iteration:#{i} @pid:#{@pid.inspect} kill:#{Process.kill(0, @pid).inspect} running?:#{running?.inspect} cmd:#{cmd.inspect}"
           begin
             raise Mongo::ConnectionFailure unless running?
-            Mongo::Connection.new(@host, @port).close
+            Mongo::MongoClient.new(@host, @port).close
             #puts "DbServer.verified via connection - port: #{@port} iteration: #{i}"
             return @pid
           rescue Mongo::ConnectionFailure
             sleep 1
           end
         end
-        raise Mongo::ConnectionFailure, "DbServer.start verification via connection failed - port: #{@port}"
+        system "ps -fp #{@pid}; cat #{@config[:logpath]}"
+        raise Mongo::ConnectionFailure, "DbServer.start verify via connection probe failed - port:#{@port.inspect} @pid:#{@pid.inspect} kill:#{Process.kill(0, @pid).inspect} running?:#{running?.inspect} cmd:#{cmd.inspect}"
       end
 
     end
@@ -210,15 +314,16 @@ module Mongo
         cmd_servers = cmd_servers.class == Array ? cmd_servers : [cmd_servers]
         cmd_servers.each do |cmd_server|
           debug 3, cmd_server.inspect
-          conn = Mongo::Connection.new(cmd_server[:host], cmd_server[:port])
+          cmd_server = cmd_server.config if cmd_server.is_a?(DbServer)
+          client = Mongo::MongoClient.new(cmd_server[:host], cmd_server[:port])
           cmd.each do |c|
             debug 3,  "ClusterManager.command c:#{c.inspect}"
-            response = conn[db_name].command( c, opts )
+            response = client[db_name].command( c, opts )
             debug 3,  "ClusterManager.command response:#{response.inspect}"
             raise Mongo::OperationFailure, "c:#{c.inspect} opts:#{opts.inspect} failed" unless response["ok"] == 1.0 || opts.fetch(:check_response, true) == false
             ret << response
           end
-          conn.close
+          client.close
         end
         debug 3, "command ret:#{ret.inspect}"
         ret.size == 1 ? ret.first : ret
@@ -228,26 +333,130 @@ module Mongo
         command( @config[:replicas].first, 'admin', { :replSetGetStatus => 1 }, {:check_response => false } )
       end
 
-      def repl_set_initiate( cfg = nil )
-        cfg ||= {
-            :_id => @config[:replicas].first[:replSet],
-            :members => @config[:replicas].each_with_index.collect{|s, i| { :_id => i, :host => "#{s[:host]}:#{s[:port]}" } },
+      def repl_set_get_config
+        host, port = primary_name.split(":")
+        client = Mongo::MongoClient.new(host, port)
+        client['local']['system.replset'].find_one
+      end
+
+      def repl_set_config
+        members = []
+        @config[:replicas].each{|s| members << { :_id => s[:_id], :host => "#{s[:host]}:#{s[:port]}" } }
+        @config[:arbiters].each{|s| members << { :_id => s[:_id], :host => "#{s[:host]}:#{s[:port]}", :arbiterOnly => true } }
+        {
+          :_id => @config[:replicas].first[:replSet],
+          :members => members
         }
-        command( @config[:replicas].first, 'admin', { :replSetInitiate => cfg } )
+      end
+
+      def repl_set_initiate( cfg = nil )
+        command( @config[:replicas].first, 'admin', { :replSetInitiate => cfg || repl_set_config } )
       end
 
       def repl_set_startup
         response = nil
         60.times do |i|
-          break if (response = repl_set_get_status)['ok'] == 1.0
+          response = repl_set_get_status
+          members = response['members']
+          if response['ok'] == 1.0 && members.collect{|m| m['state']}.all?{|state| [1,2,7].index(state)}
+            return response if members.any?{|m| m['state'] == 1}
+          end
           sleep 1
         end
-        raise Mongo::OperationFailure, "replSet startup failed - status: #{repsonse.inspect}" unless response && response['ok'] == 1.0
-        response
+        raise Mongo::OperationFailure, "replSet startup failed - status: #{response.inspect}"
+      end
+
+      def repl_set_seeds
+        @config[:replicas].collect{|node| "#{node[:host]}:#{node[:port]}"}
+      end
+
+      def repl_set_seeds_old
+        @config[:replicas].collect{|node| [node[:host], node[:port]]}
+      end
+
+      def repl_set_name
+        @config[:replicas].first[:replSet]
+      end
+
+      def member_names_by_state(state)
+        states = Array(state)
+        status = repl_set_get_status
+        status['members'].find_all{|member| states.index(member['state']) }.collect{|member| member['name']}
+      end
+
+      def primary_name
+        member_names_by_state(1).first
+      end
+
+      def secondary_names
+        member_names_by_state(2)
+      end
+
+      def replica_names
+        member_names_by_state([1,2])
+      end
+
+      def arbiter_names
+        member_names_by_state(7)
+      end
+
+      def members_by_name(names)
+        names.collect do |name|
+          servers.find{|server| server.host_port == name}
+        end.compact
+      end
+
+      def primary
+        members_by_name([primary_name]).first
+      end
+
+      def secondaries
+        members_by_name(secondary_names)
+      end
+
+      def replicas
+        members_by_name(replica_names)
+      end
+
+      def arbiters
+        members_by_name(arbiter_names)
+      end
+
+      def config_names_by_kind(kind)
+        @config[kind].collect{|conf| "#{conf[:host]}:#{conf[:port]}"}
+      end
+
+      def shards
+        members_by_name(config_names_by_kind(:shards))
+      end
+
+      def repl_set_reconfig(new_config)
+        new_config['version'] = repl_set_get_config['version'] + 1
+        command( primary, 'admin', { :replSetReconfig => new_config } )
+        repl_set_startup
+      end
+
+      def repl_set_remove_node(state = [1,2])
+        names = member_names_by_state(state)
+        name = names[rand(names.length)]
+
+        @config[:replicas].delete_if{|node| "#{node[:host]}:#{node[:port]}" == name}
+        repl_set_reconfig(repl_set_config)
+      end
+
+      def repl_set_add_node
+      end
+
+      def configs
+        members_by_name(config_names_by_kind(:configs))
+      end
+
+      def routers
+        members_by_name(config_names_by_kind(:routers))
       end
 
       def mongos_seeds
-        @config[:routers].collect{|router| "#{router[:host]}:#{router[:port]}"}
+        config_names_by_kind(:routers)
       end
 
       def ismaster
@@ -272,14 +481,16 @@ module Mongo
 
       def mongos_discover # can also do @config[:routers] find but only want mongos for connections
         (@config[:configs]).collect do |cmd_server|
-          conn = Mongo::Connection.new(cmd_server[:host], cmd_server[:port])
-          result = conn['config']['mongos'].find.to_a
-          conn.close
+          client = Mongo::MongoClient.new(cmd_server[:host], cmd_server[:port])
+          result = client['config']['mongos'].find.to_a
+          client.close
           result
         end
       end
 
       def start
+        # Must start configs before mongos -- hash order not guaranteed on 1.8.X
+        servers(:configs).each{|server| server.start}
         servers.each{|server| server.start}
         # TODO - sharded replica sets - pending
         if @config[:replicas]
@@ -291,6 +502,7 @@ module Mongo
         end
         self
       end
+      alias :restart :start
 
       def stop
         servers.each{|server| server.stop}
@@ -298,7 +510,7 @@ module Mongo
       end
 
       def clobber
-        system "rm -fr #{@config[:dbpath]}"
+        system "rm -fr #{@config[:dbpath]}/*"
         self
       end
     end