mongo 1.6.4 → 1.7.0.rc0

data/test/replica_sets/rs_test_helper.rb

@@ -5,10 +5,10 @@ require './test/tools/repl_set_manager'
 class Test::Unit::TestCase
   # Ensure replica set is available as an instance variable and that
   # a new set is spun up for each TestCase class
-  def ensure_rs
+  def ensure_rs(opts={})
     unless defined?(@@current_class) and @@current_class == self.class
       @@current_class = self.class 
-      @@rs = ReplSetManager.new
+      @@rs = ReplSetManager.new(opts)
       @@rs.start_set
     end
     @rs = @@rs

data/test/sharded_cluster/basic_test.rb

@@ -0,0 +1,112 @@
+$:.unshift(File.expand_path('../..', File.dirname(__FILE__)))
+require 'test/sharded_cluster/sc_test_helper'
+require 'test/tools/mongo_config'
+
+class BasicTest < Test::Unit::TestCase
+
+  def self.suite
+    s = super
+    def s.setup
+
+    end
+    def s.teardown
+      @@sc.stop
+      @@sc.clobber
+    end
+    def s.run(*args)
+      setup
+      super
+      teardown
+    end
+    s
+  end
+
+  def setup
+    ensure_sc
+  end
+
+  def teardown
+
+  end
+
+  # TODO member.primary? ==> true
+  def test_connect
+    seeds = @sc.mongos_seeds
+    @con = Mongo::ShardedConnection.new(seeds)
+    assert @con.connected?
+    assert_equal(seeds.size, @con.seeds.size)
+    probe(seeds.size)
+    @con.close
+  end
+
+  def test_hard_refresh
+    seeds = @sc.mongos_seeds
+    @con = Mongo::ShardedConnection.new(seeds)
+    assert @con.connected?
+    @con.hard_refresh!
+    assert @con.connected?
+    @con.close
+  end
+
+  def test_reconnect
+    seeds = @sc.mongos_seeds
+    @con = Mongo::ShardedConnection.new(seeds)
+    assert @con.connected?
+    router = @sc.servers(:routers).first
+    router.stop
+    probe(seeds.size)
+    assert @con.connected?
+    @con.close
+  end
+
+  def test_all_down
+    seeds = @sc.mongos_seeds
+    @con = Mongo::ShardedConnection.new(seeds)
+    assert @con.connected?
+    @sc.servers(:routers).each{|router| router.stop}
+    assert_raises Mongo::ConnectionFailure do
+      probe(seeds.size)
+    end
+    assert_false @con.connected?
+    @con.close
+  end
+
+  def test_cycle
+    seeds = @sc.mongos_seeds
+    @con = Mongo::ShardedConnection.new(seeds)
+    assert @con.connected?
+    routers = @sc.servers(:routers)
+    while routers.size > 0 do
+      rescue_connection_failure do
+        probe(seeds.size)
+      end
+      probe(seeds.size)
+      #p @con.manager.primary
+      router = routers.detect{|r| r.port == @con.manager.primary.last}
+      routers.delete(router)
+      router.stop
+    end
+    assert_raises Mongo::ConnectionFailure do
+      probe(seeds.size)
+    end
+    assert_false @con.connected?
+    routers = @sc.servers(:routers).reverse
+    routers.each do |r|
+      r.start
+      @con.hard_refresh!
+      #p @con.manager.primary
+      rescue_connection_failure do
+        probe(seeds.size)
+      end
+      probe(seeds.size)
+    end
+    @con.close
+  end
+
+  private
+
+  def probe(size)
+    assert_equal(size, @con['config']['mongos'].find.to_a.size)
+  end
+
+end

data/test/sharded_cluster/mongo_config_test.rb

@@ -0,0 +1,126 @@
+$:.unshift(File.expand_path('../../lib', File.dirname(__FILE__))).unshift(File.expand_path('../..', File.dirname(__FILE__)))
+require 'test-unit'
+require 'test/tools/mongo_config'
+
+class MongoConfig < Test::Unit::TestCase
+
+  def self.suite
+    s = super
+    def s.setup
+
+    end
+    def s.teardown
+
+    end
+    def s.run(*args)
+      setup
+      super
+      teardown
+    end
+    s
+  end
+
+  test "config defaults" do
+    [ Mongo::Config::DEFAULT_BASE_OPTS,
+      Mongo::Config::DEFAULT_REPLICA_SET,
+      Mongo::Config::DEFAULT_SHARDED_SIMPLE,
+      Mongo::Config::DEFAULT_SHARDED_REPLICA
+    ].each do |params|
+      config = Mongo::Config.cluster(params)
+      assert(config.size > 0)
+    end
+  end
+
+  test "get available port" do
+    assert_not_nil(Mongo::Config.get_available_port)
+  end
+
+  test "SysProc start" do
+    cmd = "true"
+    sys_proc = Mongo::Config::SysProc.new(cmd)
+    assert_equal(cmd, sys_proc.cmd)
+    assert_nil(sys_proc.pid)
+    assert_not_nil(sys_proc.start(0))
+    assert_not_nil(sys_proc.pid)
+  end
+
+  test "SysProc wait" do
+    sys_proc = Mongo::Config::SysProc.new("true")
+    assert_not_nil(sys_proc.start(0))
+    assert(sys_proc.running?)
+    sys_proc.wait
+    assert(!sys_proc.running?)
+  end
+
+  test "SysProc kill" do
+    sys_proc = Mongo::Config::SysProc.new("true")
+    assert_not_nil(sys_proc.start(0))
+    sys_proc.kill
+    sys_proc.wait
+    assert(!sys_proc.running?)
+  end
+
+  test "SysProc stop" do
+    sys_proc = Mongo::Config::SysProc.new("true")
+    assert_not_nil(sys_proc.start(0))
+    sys_proc.stop
+    assert(!sys_proc.running?)
+  end
+
+  test "Server" do
+    server = Mongo::Config::Server.new('a cmd', 'host', 1234)
+    assert_equal('a cmd', server.cmd)
+    assert_equal('host', server.host)
+    assert_equal(1234, server.port)
+  end
+
+  test "DbServer" do
+    config = Mongo::Config::DEFAULT_BASE_OPTS
+    server = Mongo::Config::DbServer.new(config)
+    assert_equal(config, server.config)
+    assert_equal("mongod --logpath data/log --dbpath data", server.cmd)
+    assert_equal(config[:host], server.host)
+    assert_equal(config[:port], server.port)
+  end
+
+  def cluster_test(opts)
+    #debug 1, opts.inspect
+    config =  Mongo::Config.cluster(opts)
+    #debug 1, config.inspect
+    manager = Mongo::Config::ClusterManager.new(config)
+    assert_equal(config, manager.config)
+    manager.start
+    manager.servers.each{|s| p s}
+    manager.stop
+    manager.servers.each{|s| assert_equal(false, s.running?)}
+    manager.clobber
+  end
+
+  test "cluster manager base" do
+    #cluster_test(Mongo::Config::DEFAULT_BASE_OPTS)
+  end
+
+  test "cluster manager replica set" do
+    #cluster_test(Mongo::Config::DEFAULT_REPLICA_SET)
+  end
+
+  test "cluster manager sharded simple" do
+    #manager = Mongo::Config::ClusterManager.new(Mongo::Config.cluster(Mongo::Config::DEFAULT_SHARDED_SIMPLE)).start
+    opts = Mongo::Config::DEFAULT_SHARDED_SIMPLE
+    #debug 1, opts.inspect
+    config =  Mongo::Config.cluster(opts)
+    #debug 1, config.inspect
+    manager = Mongo::Config::ClusterManager.new(config)
+    assert_equal(config, manager.config)
+    manager.start
+    #debug 1, manager.ismaster
+    #debug 1, manager.mongos_discover
+    manager.stop.clobber
+  end
+
+  test "cluster manager sharded replica" do
+    #cluster_test(Mongo::Config::DEFAULT_SHARDED_REPLICA)
+  end
+
+end
+

data/test/sharded_cluster/sc_test_helper.rb

@@ -0,0 +1,39 @@
+$:.unshift(File.expand_path('../../lib', File.dirname(__FILE__)))
+require File.expand_path("../../test_helper", __FILE__)
+require 'test/tools/mongo_config'
+
+class Test::Unit::TestCase
+  # Ensure sharded cluster is available as an instance variable and that
+  # a new set is spun up for each TestCase class
+  def ensure_sc
+    if defined?(@@current_class) and @@current_class == self.class
+      @@sc.start
+    else
+      @@current_class = self.class
+      dbpath = 'sc'
+      opts = Mongo::Config::DEFAULT_SHARDED_SIMPLE.merge(:dbpath => dbpath).merge(:routers => 4)
+      #debug 1, opts
+      config = Mongo::Config.cluster(opts)
+      #debug 1, config
+      @@sc = Mongo::Config::ClusterManager.new(config)
+      @@sc.start
+    end
+    @sc = @@sc
+  end
+
+  # Generic code for rescuing connection failures and retrying operations.
+  # This could be combined with some timeout functionality.
+  def rescue_connection_failure(max_retries=30)
+    retries = 0
+    begin
+      yield
+    rescue Mongo::ConnectionFailure => ex
+      #puts "Rescue attempt #{retries}: from #{ex}"
+      retries += 1
+      raise ex if retries > max_retries
+      sleep(2)
+      retry
+    end
+  end
+
+end

data/test/test_helper.rb

@@ -1,5 +1,5 @@
 $:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-require 'rubygems' if RUBY_VERSION
+require 'rubygems'
 require 'mongo'
 gem 'test-unit'
 require 'test/unit'
@@ -93,7 +93,7 @@ class Test::Unit::TestCase
     Object.new
   end
 
-  def assert_raise_error(klass, message)
+  def assert_raise_error(klass, message=nil)
     begin
       yield
     rescue => e
@@ -101,7 +101,7 @@ class Test::Unit::TestCase
         flunk "Expected exception class #{klass} but got #{e.class}.\n #{e.backtrace}"
       end
 
-      if !e.message.include?(message)
+      if message && !e.message.include?(message)
         p e.backtrace
         flunk "#{e.message} does not include #{message}.\n#{e.backtrace}"
       end

data/test/threading/threading_with_large_pool_test.rb

@@ -67,7 +67,7 @@ class TestThreadingLargePool < Test::Unit::TestCase
     @@coll = @@db.collection('thread-test-collection')
 
     1000.times do |i|
-      @@coll.insert("x" => i)
+      @@coll.insert("x" => i, :safe => true)
     end
 
     threads = []

data/test/tools/mongo_config.rb

@@ -0,0 +1,307 @@
+#!/usr/bin/env ruby
+
+require 'socket'
+require 'fileutils'
+
+$debug_level = 2
+STDOUT.sync = true
+
+unless defined? Mongo
+  require File.join(File.dirname(__FILE__), '..', '..', 'lib', 'mongo')
+end
+
+def debug(level, arg)
+  if level <= $debug_level
+    file_line = caller[0][/(.*:\d+):/, 1]
+    calling_method = caller[0][/`([^']*)'/, 1]
+    puts "#{file_line}:#{calling_method}:#{arg.class == String ? arg : arg.inspect}"
+  end
+end
+
+module Mongo
+  class Config
+    DEFAULT_BASE_OPTS = { :host => 'localhost', :logpath => 'data/log', :dbpath => 'data' }
+    DEFAULT_REPLICA_SET = DEFAULT_BASE_OPTS.merge( :replicas => 3 )
+    DEFAULT_SHARDED_SIMPLE = DEFAULT_BASE_OPTS.merge( :shards => 2, :configs => 1, :routers => 2 )
+    DEFAULT_SHARDED_REPLICA = DEFAULT_SHARDED_SIMPLE.merge( :replicas => 3 )
+
+    SERVER_PRELUDE_KEYS = [:host, :command]
+    SHARDING_OPT_KEYS = [:shards, :configs, :routers]
+    REPLICA_OPT_KEYS = [:replicas]
+    MONGODS_OPT_KEYS = [:mongods]
+    CLUSTER_OPT_KEYS = SHARDING_OPT_KEYS + REPLICA_OPT_KEYS + MONGODS_OPT_KEYS
+
+    DEFAULT_VERIFIES = 60
+    BASE_PORT = 3000
+    @@port = BASE_PORT
+
+    def self.configdb(config)
+      config[:configs].collect{|c|"#{c[:host]}:#{c[:port]}"}.join(' ')
+    end
+
+    def self.cluster(opts = DEFAULT_SHARDED_SIMPLE)
+      raise "missing required option" if [:host, :dbpath].any?{|k| !opts[k]}
+      config = opts.reject{|k,v| CLUSTER_OPT_KEYS.include?(k)}
+      keys = SHARDING_OPT_KEYS.any?{|k| opts[k]} ? SHARDING_OPT_KEYS : nil
+      keys ||= REPLICA_OPT_KEYS.any?{|k| opts[k]} ? REPLICA_OPT_KEYS : nil
+      keys ||= MONGODS_OPT_KEYS
+      keys.each do |key|
+        config[key] = opts.fetch(key,1).times.collect do |i| #default to 1 of whatever
+          server_base = key.to_s.chop
+          dbpath = "#{opts[:dbpath]}/#{server_base}#{i}"
+          logpath = "#{dbpath}/#{server_base}.log"
+          if key == :shards && opts[:replicas]
+            self.cluster(opts.reject{|k,v| SHARDING_OPT_KEYS.include?(k)}.merge(:dbpath => dbpath))
+          else
+            server_params = { :host => opts[:host], :port => self.get_available_port, :logpath => logpath }
+            case key
+              when :replicas; server_params.merge!( :command => 'mongod', :dbpath => dbpath, :replSet => File.basename(opts[:dbpath]) )
+              when :configs;  server_params.merge!( :command => 'mongod', :dbpath => dbpath, :configsvr => nil )
+              when :routers;  server_params.merge!( :command => 'mongos', :configdb => self.configdb(config) ) # mongos, NO dbpath
+              else            server_params.merge!( :command => 'mongod', :dbpath => dbpath ) # :mongods, :shards
+            end
+          end
+        end
+      end
+      config
+    end
+
+    def self.port_available?(port)
+      ret = false
+      socket = Socket.new(Socket::Constants::AF_INET, Socket::Constants::SOCK_STREAM, 0)
+      sockaddr = Socket.sockaddr_in(port, '0.0.0.0')
+      begin
+        socket.bind(sockaddr)
+        ret = true
+      rescue Exception
+      end
+      socket.close
+      ret
+    end
+
+    def self.get_available_port
+      while true
+        port = @@port
+        @@port += 1
+        break if port_available?(port)
+      end
+      port
+    end
+
+    class SysProc
+      attr_reader :pid, :cmd
+
+      def initialize(cmd = nil)
+        @pid = nil
+        @cmd = cmd
+      end
+
+      def start(verifies = 0)
+        return @pid if running?
+        begin
+          @pid = fork do
+            STDIN.reopen '/dev/null'
+            STDOUT.reopen '/dev/null', 'a'
+            STDERR.reopen STDOUT
+            exec cmd # spawn(@cmd, [:in, :out, :err] => :close) #
+          end
+          verify(verifies) if verifies > 0
+          @pid
+        end
+      end
+
+      def stop
+        kill
+        wait
+      end
+
+      def kill
+        begin
+          @pid && Process.kill(3, @pid) && true
+        rescue Errno::ESRCH
+          false
+        end
+      end
+
+      def wait
+        Process.wait(@pid) if @pid
+        @pid = nil
+      end
+
+      def running?
+        begin
+          @pid && Process.kill(0, @pid) && true
+        rescue Errno::ESRCH
+          false
+        end
+      end
+
+      def verify(verifies = DEFAULT_VERIFIES)
+        verifies.times do |i|
+          return @pid if running?
+          sleep 1
+        end
+        nil
+      end
+    end
+
+    class Server < SysProc
+      attr_reader :host, :port
+
+      def initialize(cmd = nil, host = nil, port = nil)
+        super(cmd)
+        @host = host
+        @port = port
+      end
+    end
+
+    class DbServer < Server
+      attr_accessor :config
+      def initialize(config)
+        @config = config
+        dbpath = @config[:dbpath]
+        [dbpath, File.dirname(@config[:logpath])].compact.each{|dir| FileUtils.mkdir_p(dir) unless File.directory?(dir) }
+        command = @config[:command] || 'mongod'
+        arguments = @config.reject{|k,v| SERVER_PRELUDE_KEYS.include?(k)}
+        cmd = [command, arguments.collect{|k,v| ['--' + k.to_s, v ]}].flatten.join(' ')
+        super(cmd, @config[:host], @config[:port])
+      end
+
+      def start(verifies = DEFAULT_VERIFIES)
+        super(verifies)
+        verify(verifies)
+      end
+
+      def verify(verifies = 10)
+        verifies.times do |i|
+          #puts "DbServer.verify - port: #{@port} iteration: #{i}"
+          begin
+            raise Mongo::ConnectionFailure unless running?
+            Mongo::Connection.new(@host, @port).close
+            #puts "DbServer.verified via connection - port: #{@port} iteration: #{i}"
+            return @pid
+          rescue Mongo::ConnectionFailure
+            sleep 1
+          end
+        end
+        raise Mongo::ConnectionFailure, "DbServer.start verification via connection failed - port: #{@port}"
+      end
+
+    end
+
+    class ClusterManager
+      attr_reader :config
+      def initialize(config)
+        @config = config
+        @servers = {}
+        Mongo::Config::CLUSTER_OPT_KEYS.each do |key|
+          @servers[key] = @config[key].collect{|conf| DbServer.new(conf)} if @config[key]
+        end
+      end
+
+      def servers(key = nil)
+        @servers.collect{|k,v| (!key || key == k) ? v : nil}.flatten.compact
+      end
+
+      def command( cmd_servers, db_name, cmd, opts = {} )
+        ret = []
+        cmd = cmd.class == Array ? cmd : [ cmd ]
+        debug 3, "ClusterManager.command cmd:#{cmd.inspect}"
+        cmd_servers = cmd_servers.class == Array ? cmd_servers : [cmd_servers]
+        cmd_servers.each do |cmd_server|
+          debug 3, cmd_server.inspect
+          conn = Mongo::Connection.new(cmd_server[:host], cmd_server[:port])
+          cmd.each do |c|
+            debug 3,  "ClusterManager.command c:#{c.inspect}"
+            response = conn[db_name].command( c, opts )
+            debug 3,  "ClusterManager.command response:#{response.inspect}"
+            raise Mongo::OperationFailure, "c:#{c.inspect} opts:#{opts.inspect} failed" unless response["ok"] == 1.0 || opts.fetch(:check_response, true) == false
+            ret << response
+          end
+          conn.close
+        end
+        debug 3, "command ret:#{ret.inspect}"
+        ret.size == 1 ? ret.first : ret
+      end
+
+      def repl_set_get_status
+        command( @config[:replicas].first, 'admin', { :replSetGetStatus => 1 }, {:check_response => false } )
+      end
+
+      def repl_set_initiate( cfg = nil )
+        cfg ||= {
+            :_id => @config[:replicas].first[:replSet],
+            :members => @config[:replicas].each_with_index.collect{|s, i| { :_id => i, :host => "#{s[:host]}:#{s[:port]}" } },
+        }
+        command( @config[:replicas].first, 'admin', { :replSetInitiate => cfg } )
+      end
+
+      def repl_set_startup
+        response = nil
+        60.times do |i|
+          break if (response = repl_set_get_status)['ok'] == 1.0
+          sleep 1
+        end
+        raise Mongo::OperationFailure, "replSet startup failed - status: #{repsonse.inspect}" unless response && response['ok'] == 1.0
+        response
+      end
+
+      def mongos_seeds
+        @config[:routers].collect{|router| "#{router[:host]}:#{router[:port]}"}
+      end
+
+      def ismaster
+        command( @config[:routers], 'admin', { :ismaster => 1 } )
+      end
+
+      def addshards(shards = @config[:shards])
+        command( @config[:routers].first, 'admin', Array(shards).collect{|s| { :addshard => "#{s[:host]}:#{s[:port]}" } } )
+      end
+
+      def listshards
+        command( @config[:routers].first, 'admin', { :listshards => 1 } )
+      end
+
+      def enablesharding( dbname )
+        command( @config[:routers].first, 'admin', { :enablesharding => dbname } )
+      end
+
+      def shardcollection( namespace, key, unique = false )
+        command( @config[:routers].first, 'admin', { :shardcollection => namespace, :key => key, :unique => unique } )
+      end
+
+      def mongos_discover # can also do @config[:routers] find but only want mongos for connections
+        (@config[:configs]).collect do |cmd_server|
+          conn = Mongo::Connection.new(cmd_server[:host], cmd_server[:port])
+          result = conn['config']['mongos'].find.to_a
+          conn.close
+          result
+        end
+      end
+
+      def start
+        servers.each{|server| server.start}
+        # TODO - sharded replica sets - pending
+        if @config[:replicas]
+          repl_set_initiate if repl_set_get_status['startupStatus'] == 3
+          repl_set_startup
+        end
+        if @config[:routers]
+          addshards if listshards['shards'].size == 0
+        end
+        self
+      end
+
+      def stop
+        servers.each{|server| server.stop}
+        self
+      end
+
+      def clobber
+        system "rm -fr #{@config[:dbpath]}"
+        self
+      end
+    end
+
+  end
+end