mongo 1.6.0 → 1.6.1

data/lib/mongo/util/logging.rb

@@ -4,8 +4,11 @@ module Mongo
     DEBUG_LEVEL = defined?(Logger) ? Logger::DEBUG : 0
 
     def write_logging_startup_message
-      log(:debug, "Please note that logging negatively impacts client-side performance. " +
-      "You should set your logging level no lower than :info in production.")
+      if @logger && (@logger.level == DEBUG_LEVEL)
+        log(:debug, "Logging level is currently :debug which could negatively impact " +
+          "client-side performance. You should set your logging level no lower than " +
+          ":info in production.")
+      end
     end
 
     # Log a message with the given level.

data/lib/mongo/util/node.rb

@@ -101,12 +101,12 @@ module Mongo
       rescue ConnectionFailure, OperationFailure, OperationTimeout, SocketError, SystemCallError, IOError => ex
         @connection.log(:warn, "Attempted connection to node #{host_string} raised " +
                             "#{ex.class}: #{ex.message}")
-        
+
         # Socket may already be nil from issuing command
         if @socket && !@socket.closed?
           @socket.close
         end
-        
+
         return nil
       end
 

data/lib/mongo/util/pool.rb

@@ -19,6 +19,7 @@ module Mongo
   class Pool
     PING_ATTEMPTS  = 6
     MAX_PING_TIME  = 1_000_000
+    PRUNE_INTERVAL = 10_000
 
     attr_accessor :host, :port, :address,
       :size, :timeout, :safe, :checked_out, :connection
@@ -36,8 +37,8 @@ module Mongo
       @address = "#{@host}:#{@port}"
 
       # Pool size and timeout.
-      @size      = opts[:size] || 10000
-      @timeout   = opts[:timeout]   || 5.0
+      @size    = opts.fetch(:size, 20)
+      @timeout = opts.fetch(:timeout, 30)
 
       # Mutex for synchronizing pool access
       @connection_mutex = Mutex.new
@@ -51,11 +52,11 @@ module Mongo
       @sockets      = []
       @pids         = {}
       @checked_out  = []
-      @threads      = {}
       @ping_time    = nil
       @last_ping    = nil
       @closed       = false
-      @last_pruning = Time.now
+      @threads_to_sockets = {}
+      @checkout_counter   = 0
     end
 
     # Close this pool.
@@ -64,22 +65,13 @@ module Mongo
     #   close only those sockets that are not checked out.
     def close(opts={})
       @connection_mutex.synchronize do
-        if opts[:soft]
-          sockets_to_close = @sockets - @checked_out
+        if opts[:soft] && !@checked_out.empty?
+          @closing = true
+          close_sockets(@sockets - @checked_out)
         else
-          sockets_to_close = @sockets
+          close_sockets(@sockets)
+          @closed = true
         end
-        sockets_to_close.each do |sock|
-          begin
-            sock.close unless sock.closed?
-          rescue IOError => ex
-            warn "IOError when attempting to close socket connected to #{@host}:#{@port}: #{ex.inspect}"
-          end
-        end
-        @sockets.clear
-        @pids.clear
-        @checked_out.clear
-        @closed = true
       end
     end
 
@@ -166,6 +158,7 @@ module Mongo
       begin
         socket = self.connection.socket_class.new(@host, @port)
         socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
+        socket.pool = self
       rescue => ex
         socket.close if socket
         raise ConnectionFailure, "Failed to connect to host #{@host} and port #{@port}: #{ex}"
@@ -179,7 +172,7 @@ module Mongo
       @sockets << socket
       @pids[socket] = Process.pid
       @checked_out << socket
-      @threads[socket] = Thread.current.object_id
+      @threads_to_sockets[Thread.current] = socket
       socket
     end
 
@@ -216,30 +209,33 @@ module Mongo
     #
     # This method is called exclusively from #checkout;
     # therefore, it runs within a mutex.
-    def checkout_existing_socket
-      socket = (@sockets - @checked_out).first
+    def checkout_existing_socket(socket=nil)
+      if !socket
+        socket = (@sockets - @checked_out).first
+      end
+
       if @pids[socket] != Process.pid
-         @pids[socket] = nil
-         @sockets.delete(socket)
-         socket.close if socket
-         checkout_new_socket
+        @pids[socket] = nil
+        @sockets.delete(socket)
+        socket.close if socket
+        checkout_new_socket
       else
         @checked_out << socket
-        @threads[socket] = Thread.current.object_id
+        @threads_to_sockets[Thread.current] = socket
         socket
       end
     end
 
-    # If we have more sockets than the soft limit specified
-    # by the max pool size, then we should prune those
-    # extraneous sockets.
-    #
-    # Note: this must be called from within a mutex.
-    def prune
-      idle_sockets = @sockets - @checked_out
-      idle_sockets.each do |socket|
-        socket.close unless socket.closed?
-        @sockets.delete(socket)
+    def prune_thread_socket_hash
+      map = {}
+      Thread.list.each do |t|
+        map[t] = 1
+      end
+
+      @threads_to_sockets.keys.each do |key|
+        if !map[key]
+          @threads_to_sockets.delete(key)
+        end
       end
     end
 
@@ -251,21 +247,33 @@ module Mongo
       start_time = Time.now
       loop do
         if (Time.now - start_time) > @timeout
-            raise ConnectionTimeoutError, "could not obtain connection within " +
-              "#{@timeout} seconds. The max pool size is currently #{@size}; " +
-              "consider increasing the pool size or timeout."
+          raise ConnectionTimeoutError, "could not obtain connection within " +
+            "#{@timeout} seconds. The max pool size is currently #{@size}; " +
+            "consider increasing the pool size or timeout."
         end
 
         @connection_mutex.synchronize do
-          if @sockets.size > @size * 1.5
-            prune
+          if @checkout_counter > PRUNE_INTERVAL
+            @checkout_counter = 0
+            prune_thread_socket_hash
+          else
+            @checkout_counter += 1
           end
 
-          socket = if @checked_out.size < @sockets.size
-                     checkout_existing_socket
-                   else
-                     checkout_new_socket
-                   end
+          if socket_for_thread = @threads_to_sockets[Thread.current]
+            if !@checked_out.include?(socket_for_thread)
+              socket = checkout_existing_socket(socket_for_thread)
+            end
+          else # First checkout for this thread
+            thread_length = @threads_to_sockets.keys.length
+            if (thread_length <= @sockets.size) && (@sockets.size < @size)
+              socket = checkout_new_socket
+            elsif @checked_out.size < @sockets.size
+              socket = checkout_existing_socket
+            elsif @sockets.size < @size
+              socket = checkout_new_socket
+            end
+          end
 
           if socket
             # This calls all procs, in order, scoped to existing sockets.
@@ -275,6 +283,18 @@ module Mongo
               op.call
             end
 
+            if socket.closed?
+              @checked_out.delete(socket)
+              @sockets.delete(socket)
+              @threads_to_sockets.each do |k,v|
+                if v == socket
+                  @threads_to_sockets.delete(k)
+                end
+              end
+
+              socket = checkout_new_socket
+            end
+            
             return socket
           else
             # Otherwise, wait
@@ -283,5 +303,19 @@ module Mongo
         end
       end
     end
+
+    private
+
+    def close_sockets(sockets)
+      sockets.each do |socket|
+        @sockets.delete(socket)
+        begin
+          socket.close unless socket.closed?
+        rescue IOError => ex
+          warn "IOError when attempting to close socket connected to #{@host}:#{@port}: #{ex.inspect}"
+        end
+      end
+    end
+
   end
 end

data/lib/mongo/util/pool_manager.rb

@@ -84,6 +84,10 @@ module Mongo
       @refresh_required
     end
 
+    def closed?
+      pools.all? { |pool| pool.closed? }
+    end
+
     def close(opts={})
       begin
         if @primary_pool
@@ -114,6 +118,10 @@ module Mongo
 
     private
 
+    def pools
+      [@primary_pool, *@secondary_pools]
+    end
+
     def validate_existing_member(member)
       config = member.set_config
       if !config

data/lib/mongo/util/ssl_socket.rb

@@ -7,6 +7,8 @@ module Mongo
   # mirroring Ruby's TCPSocket, vis., TCPSocket#send and TCPSocket#read.
   class SSLSocket
 
+    attr_accessor :pool
+
     def initialize(host, port)
       @socket = ::TCPSocket.new(host, port)
       @ssl = OpenSSL::SSL::SSLSocket.new(@socket)
@@ -33,6 +35,5 @@ module Mongo
     def close
       @ssl.close
     end
-
   end
 end

data/lib/mongo/util/tcp_socket.rb

@@ -0,0 +1,6 @@
+module Mongo
+ class TCPSocket < ::TCPSocket
+   attr_accessor :pool
+
+ end
+end

data/lib/mongo/version.rb

@@ -1,3 +1,3 @@
 module Mongo
-  VERSION = "1.6.0"
+  VERSION = "1.6.1"
 end

data/mongo.gemspec

@@ -30,5 +30,5 @@ Gem::Specification.new do |s|
   s.email = 'mongodb-dev@googlegroups.com'
   s.homepage = 'http://www.mongodb.org'
 
-  s.add_dependency('bson', Mongo::VERSION)
+  s.add_dependency('bson', "~> #{Mongo::VERSION}")
 end

data/test/connection_test.rb

@@ -238,7 +238,7 @@ class TestConnection < Test::Unit::TestCase
     assert !conn.active?
 
     # Simulate a dropped connection.
-    dropped_socket = Mocha::Mock.new
+    dropped_socket = mock('dropped_socket')
     dropped_socket.stubs(:read).raises(Errno::ECONNRESET)
     dropped_socket.stubs(:send).raises(Errno::ECONNRESET)
     dropped_socket.stub_everything
@@ -357,7 +357,7 @@ class TestConnection < Test::Unit::TestCase
     end
 
     should "show a proper exception message if an IOError is raised while closing a socket" do
-      fake_socket = Mocha::Mock.new
+      fake_socket = mock('fake_socket')
       fake_socket.stubs(:close).raises(IOError.new)
       fake_socket.stub_everything
       TCPSocket.stubs(:new).returns(fake_socket)

data/test/pool_test.rb

@@ -0,0 +1,57 @@
+require './test/test_helper'
+require 'thread'
+
+class PoolTest < Test::Unit::TestCase
+  include Mongo
+
+  def setup
+    @connection = standard_connection
+  end
+
+  def test_pool_affinity
+    @pool = Pool.new(@connection, TEST_HOST, TEST_PORT, :size => 5)
+
+    @threads    = []
+    @sockets    = []
+
+    10.times do
+      @threads << Thread.new do
+        original_socket = @pool.checkout
+        @sockets << original_socket
+        @pool.checkin(original_socket)
+        5000.times do
+          socket = @pool.checkout
+          assert_equal original_socket, socket
+          @pool.checkin(socket)
+        end
+      end
+    end
+
+    @threads.each { |t| t.join }
+  end
+
+  def test_pool_thread_pruning
+    @pool = Pool.new(@connection, TEST_HOST, TEST_PORT, :size => 5)
+
+    @threads = []
+
+    10.times do
+      @threads << Thread.new do
+        50.times do
+          socket = @pool.checkout
+          @pool.checkin(socket)
+        end
+      end
+    end
+
+    @threads.each { |t| t.join }
+    assert_equal 10, @pool.instance_variable_get(:@threads_to_sockets).size
+
+    # Thread-socket pool
+    10000.times do
+      @pool.checkin(@pool.checkout)
+    end
+
+    assert_equal 1, @pool.instance_variable_get(:@threads_to_sockets).size
+  end
+end

data/test/replica_sets/pooled_insert_test.rb

@@ -7,7 +7,7 @@ class ReplicaSetPooledInsertTest < Test::Unit::TestCase
 
   def setup
     ensure_rs
-    @conn = ReplSetConnection.new(build_seeds(3), :pool_size => 5, :timeout => 5, :refresh_mode => false)
+    @conn = ReplSetConnection.new(build_seeds(3), :pool_size => 10, :timeout => 5, :refresh_mode => false)
     @db = @conn.db(MONGO_TEST_DB)
     @db.drop_collection("test-sets")
     @coll = @db.collection("test-sets")

data/test/replica_sets/read_preference_test.rb

@@ -50,7 +50,7 @@ class ReadPreferenceTest < Test::Unit::TestCase
     @coll.save({:a => 20}, :safe => {:w => 2})
 
     # Test that reads are going to secondary on ReplSetConnection
-    @secondary = Connection.new(@rs.host, @conn.read_pool.port, :slave_ok => true)
+    @secondary = Connection.new(@rs.host, @conn.secondary_pool.port, :slave_ok => true)
     queries_before = @secondary['admin'].command({:serverStatus => 1})['opcounters']['query']
     @coll.find_one
     queries_after = @secondary['admin'].command({:serverStatus => 1})['opcounters']['query']
@@ -60,7 +60,7 @@ class ReadPreferenceTest < Test::Unit::TestCase
     @conn.refresh
     
     # Test that reads are only allowed from secondaries
-    assert_raise ConnectionFailure.new("Could not connect to a secondary for reading.") do
+    assert_raise ConnectionFailure.new("Could not checkout a socket.") do
       @coll.find_one
     end
         

data/test/replica_sets/refresh_with_threads_test.rb

@@ -45,8 +45,16 @@ class ReplicaSetRefreshWithThreadsTest < Test::Unit::TestCase
       end
     end
 
-    @rs.add_node
-    threads.each {|t| t.join }
+    # MongoDB < 2.0 will disconnect clients on rs.reconfig()
+    if @rs.version.first < 2
+      assert_raise Mongo::ConnectionFailure do
+        @rs.add_node
+        threads.each {|t| t.join }
+      end
+    else
+      @rs.add_node
+      threads.each {|t| t.join }
+    end
 
     config = @conn['admin'].command({:ismaster => 1})
 

data/test/unit/connection_test.rb

@@ -26,6 +26,34 @@ class ConnectionTest < Test::Unit::TestCase
       should "default slave_ok to false" do
         assert !@conn.slave_ok?
       end
+
+      should "warn if invalid options are specified" do
+        conn = Connection.allocate
+        opts = {:connect => false}
+
+        ReplSetConnection::REPL_SET_OPTS.each do |opt|
+          conn.expects(:warn).with("#{opt} is not a valid option for #{conn.class}")
+          opts[opt] = true
+        end
+
+        args = ['localhost', 27017, opts]
+        conn.send(:initialize, *args)
+      end
+
+      context "given a replica set" do
+        should "warn if invalid options are specified" do
+          conn = ReplSetConnection.allocate
+          opts = {:connect => false}
+
+          Connection::CONNECTION_OPTS.each do |opt|
+            conn.expects(:warn).with("#{opt} is not a valid option for #{conn.class}")
+            opts[opt] = true
+          end
+
+          args = [['localhost:27017'], opts]
+          conn.send(:initialize, *args)
+        end
+      end
     end
 
     context "initializing with a mongodb uri" do
@@ -45,21 +73,21 @@ class ConnectionTest < Test::Unit::TestCase
         @conn = Connection.from_uri("mongodb://#{host_name}/foo", :connect => false)
         assert_equal [host_name, 27017], @conn.host_to_try
       end
-      
+
       should "set safe options on connection" do
         host_name = "localhost"
         opts = "safe=true&w=2&wtimeoutMS=1000&fsync=true&journal=true"
         @conn = Connection.from_uri("mongodb://#{host_name}/foo?#{opts}", :connect => false)
         assert_equal({:w => 2, :wtimeout => 1000, :fsync => true, :j => true}, @conn.safe)
       end
-      
+
       should "have wtimeoutMS take precidence over the depricated wtimeout" do
         host_name = "localhost"
         opts = "safe=true&wtimeout=100&wtimeoutMS=500"
         @conn = Connection.from_uri("mongodb://#{host_name}/foo?#{opts}", :connect => false)
         assert_equal({:wtimeout => 500}, @conn.safe)
       end
-      
+
       should "set timeout options on connection" do
         host_name = "localhost"
         opts = "connectTimeoutMS=1000&socketTimeoutMS=5000"