mongo 1.8.2 → 1.8.3.rc0

data/lib/mongo/networking.rb

@@ -39,7 +39,7 @@ module Mongo
         raise ex
       ensure
         if sock
-          sock.pool.checkin(sock)
+          sock.checkin
         end
       end
     end
@@ -116,6 +116,7 @@ module Mongo
         send_message_on_socket(packed_message, socket)
         result = receive(socket, request_id, exhaust)
       rescue ConnectionFailure => ex
+        socket.close
         checkin(socket)
         raise ex
       rescue SystemStackError, NoMemoryError, SystemCallError => ex
@@ -209,13 +210,13 @@ module Mongo
     end
 
     def build_command_message(db_name, query, projection=nil, skip=0, limit=-1)
-      message = BSON::ByteBuffer.new
+      message = BSON::ByteBuffer.new("", max_message_size)
       message.put_int(0)
       BSON::BSON_RUBY.serialize_cstr(message, "#{db_name}.$cmd")
       message.put_int(skip)
       message.put_int(limit)
-      message.put_binary(BSON::BSON_CODER.serialize(query, false).to_s)
-      message.put_binary(BSON::BSON_CODER.serialize(projection, false).to_s) if projection
+      message.put_binary(BSON::BSON_CODER.serialize(query, false, false, max_bson_size).to_s)
+      message.put_binary(BSON::BSON_CODER.serialize(projection, false, false, max_bson_size).to_s) if projection
       message
     end
 

data/lib/mongo/util/conversions.rb

@@ -1,20 +1,3 @@
-# encoding: UTF-8
-
-# --
-# Copyright (C) 2008-2012 10gen Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ++
 module Mongo #:nodoc:
 
   # Utility module to include when needing to convert certain types of

data/lib/mongo/util/core_ext.rb

@@ -1,21 +1,3 @@
-# encoding: UTF-8
-
-# --
-# Copyright (C) 2008-2012 10gen Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ++
-
 #:nodoc:
 class Object
 

data/lib/mongo/util/node.rb

@@ -13,17 +13,26 @@ module Mongo
     end
 
     def eql?(other)
-      other.is_a?(Node) && @address == other.address
+      (other.is_a?(Node) && @address == other.address)
     end
     alias :== :eql?
 
+    def =~(other)
+      if other.is_a?(String)
+        h, p = split_node(other)
+        h == @host && p == @port
+      else
+        false
+      end
+    end
+
     def host_string
       address
     end
 
     def config
       connect unless connected?
-      set_config unless @config
+      set_config unless @config || !connected?
       @config
     end
 
@@ -57,7 +66,7 @@ module Mongo
     end
 
     def connected?
-      @socket != nil
+      @socket != nil && !@socket.closed?
     end
 
     def active?
@@ -75,6 +84,10 @@ module Mongo
     def set_config
       @node_mutex.synchronize do
         begin
+          if @config
+            @last_state = @config['ismaster'] ? :primary : :other
+          end
+
           @config = @client['admin'].command({:ismaster => 1}, :socket => @socket)
 
           if @config['msg']

data/lib/mongo/util/pool.rb

@@ -1,23 +1,8 @@
-# --
-# Copyright (C) 2008-2012 10gen Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
 module Mongo
   class Pool
     PING_ATTEMPTS  = 6
     MAX_PING_TIME  = 1_000_000
-    include ThreadLocalVariableManager
+    PRUNE_INTERVAL = 10_000
 
     attr_accessor :host,
                   :port,
@@ -25,7 +10,8 @@ module Mongo
                   :size,
                   :timeout,
                   :checked_out,
-                  :client
+                  :client,
+                  :node
 
     # Create a new pool of connections.
     def initialize(client, host, port, opts={})
@@ -55,12 +41,13 @@ module Mongo
       # Operations to perform on a socket
       @socket_ops = Hash.new { |h, k| h[k] = [] }
 
-      @sockets            = []
-      @checked_out        = []
-      @ping_time          = nil
-      @last_ping          = nil
-      @closed             = false
-      @checkout_counter   = 0
+      @sockets               = []
+      @checked_out           = []
+      @ping_time             = nil
+      @last_ping             = nil
+      @closed                = false
+      @thread_ids_to_sockets = {}
+      @checkout_counter      = 0
     end
 
     # Close this pool.
@@ -85,6 +72,11 @@ module Mongo
       @node.tags
     end
 
+    def healthy?
+      close if @sockets.all?(&:closed?)
+      !closed? && node.healthy?
+    end
+
     def closed?
       @closed
     end
@@ -174,7 +166,7 @@ module Mongo
     def checkin(socket)
       @connection_mutex.synchronize do
         if @checked_out.delete(socket)
-          @queue.signal
+          @queue.broadcast
         else
           return false
         end
@@ -202,8 +194,7 @@ module Mongo
 
       @sockets << socket
       @checked_out << socket
-
-      thread_local[:sockets][self.object_id] = socket
+      @thread_ids_to_sockets[Thread.current.object_id] = socket
       socket
     end
 
@@ -242,7 +233,8 @@ module Mongo
     # therefore, it runs within a mutex.
     def checkout_existing_socket(socket=nil)
       if !socket
-        socket = (@sockets - @checked_out).first
+        available = @sockets - @checked_out
+        socket = available[rand(available.length)]
       end
 
       if socket.pid != Process.pid
@@ -253,11 +245,27 @@ module Mongo
         checkout_new_socket
       else
         @checked_out << socket
-        thread_local[:sockets][self.object_id] = socket
+        @thread_ids_to_sockets[Thread.current.object_id] = socket
         socket
       end
     end
 
+    def prune_threads
+      live_threads = Thread.list.map(&:object_id)
+      @thread_ids_to_sockets.reject! do |key, value|
+        !live_threads.include?(key)
+      end
+    end
+
+    def check_prune
+      if @checkout_counter > PRUNE_INTERVAL
+          @checkout_counter = 0
+          prune_threads
+      else
+        @checkout_counter += 1
+      end
+    end
+
     # Check out an existing socket or create a new socket if the maximum
     # pool size has not been exceeded. Otherwise, wait for the next
     # available socket.
@@ -265,8 +273,16 @@ module Mongo
       @client.connect if !@client.connected?
       start_time = Time.now
       loop do
+        if (Time.now - start_time) > @timeout
+          raise ConnectionTimeoutError, "could not obtain connection within " +
+            "#{@timeout} seconds. The max pool size is currently #{@size}; " +
+            "consider increasing the pool size or timeout."
+        end
+
         @connection_mutex.synchronize do
-          if socket_for_thread = thread_local[:sockets][self.object_id]
+          check_prune
+          socket = nil
+          if socket_for_thread = @thread_ids_to_sockets[Thread.current.object_id]
             if !@checked_out.include?(socket_for_thread)
               socket = checkout_existing_socket(socket_for_thread)
             end
@@ -289,7 +305,7 @@ module Mongo
             if socket.closed?
               @checked_out.delete(socket)
               @sockets.delete(socket)
-              thread_local[:sockets].delete self.object_id
+              @thread_ids_to_sockets.delete(Thread.current.object_id)
               socket = checkout_new_socket
             end
 
@@ -299,12 +315,6 @@ module Mongo
             @queue.wait(@connection_mutex)
           end
         end
-
-        if (Time.now - start_time) > @timeout
-          raise ConnectionTimeoutError, "could not obtain connection within " +
-            "#{@timeout} seconds. The max pool size is currently #{@size}; " +
-            "consider increasing the pool size or timeout."
-        end
       end
     end
 

data/lib/mongo/util/pool_manager.rb

@@ -3,9 +3,17 @@ module Mongo
     include Mongo::ReadPreference
     include ThreadLocalVariableManager
 
-    attr_reader :client, :arbiters, :primary, :secondaries, :primary_pool,
-      :secondary_pool, :secondary_pools, :hosts, :nodes, :members, :seeds,
-      :max_bson_size
+    attr_reader :client,
+                :arbiters,
+                :primary,
+                :secondaries,
+                :primary_pool,
+                :secondary_pools,
+                :hosts,
+                :nodes,
+                :members,
+                :seeds,
+                :pools
 
     # Create a new set of connection pools.
     #
@@ -17,7 +25,15 @@ module Mongo
     def initialize(client, seeds=[])
       @client               = client
       @seeds                = seeds
-      @previously_connected = false
+
+      @pools                = Set.new
+      @primary              = nil
+      @primary_pool         = nil
+      @secondaries          = Set.new
+      @secondary_pools      = []
+      @hosts                = Set.new
+      @members              = Set.new
+      @refresh_required     = false
     end
 
     def inspect
@@ -25,15 +41,16 @@ module Mongo
     end
 
     def connect
-      close if @previously_connected
-
-      initialize_data
-      members = connect_to_members
-      initialize_pools(members)
-      cache_discovered_seeds(members)
+      @refresh_required = false
+      disconnect_old_members
+      connect_to_members
+      initialize_pools(@members)
+      cache_discovered_seeds
+    end
 
-      @members = members
-      @previously_connected = true
+    def refresh!(additional_seeds)
+      @seeds |= additional_seeds
+      connect
     end
 
     # We're healthy if all members are pingable and if the view
@@ -48,25 +65,24 @@ module Mongo
         return
       end
 
-      config = seed.config
-      if config
+      unless current_config = seed.config
         @refresh_required = true
         seed.close
         return
       end
 
-      if config['hosts'].length != @members.length
+      if current_config['hosts'].length != @members.length
         @refresh_required = true
         seed.close
         return
       end
 
-      config['hosts'].each do |host|
+      current_config['hosts'].each do |host|
         member = @members.detect do |m|
           m.address == host
         end
 
-        if member && validate_existing_member(member)
+        if member && validate_existing_member(current_config, member)
           next
         else
           @refresh_required = true
@@ -118,71 +134,73 @@ module Mongo
       pool
     end
 
-    def pools
-      [@primary_pool, *@secondary_pools].compact
+    def max_bson_size
+      @max_bson_size ||= config_min('maxBsonObjectSize', DEFAULT_MAX_BSON_SIZE)
+    end
+
+    def max_message_size
+      @max_message_size ||= config_min('maxMessageSizeBytes', DEFAULT_MAX_MESSAGE_SIZE)
     end
 
     private
 
-    def validate_existing_member(member)
-      config = member.config
-      if config
+    def validate_existing_member(current_config, member)
+      if current_config['ismaster'] && member.last_state != :primary
+        return false
+      elsif member.last_state != :other
         return false
-      else
-        if member.primary?
-          if member.last_state == :primary
-            return true
-          else # This node is now primary, but didn't used to be.
-            return false
-          end
-        elsif member.last_state == :secondary &&
-          member.secondary?
-          return true
-        else # This node isn't what it used to be.
-          return false
-        end
       end
+      return true
     end
 
-    def initialize_data
-      @primary          = nil
-      @primary_pool     = nil
-      @read             = nil
-      @read_pool        = nil
-      @arbiters         = []
-      @secondaries      = Set.new
-      @secondary_pool   = nil
-      @secondary_pools  = []
-      @hosts            = Set.new
-      @members          = Set.new
-      @refresh_required = false
+    # For any existing members, close and remove any that are unhealthy or already closed.
+    def disconnect_old_members
+      @pools.reject!   {|pool| !pool.healthy? }
+      @members.reject! {|node| !node.healthy? }
     end
 
     # Connect to each member of the replica set
-    # as reported by the given seed node, and return
-    # as a list of Mongo::Node objects.
+    # as reported by the given seed node.
     def connect_to_members
-      members = []
-
       seed = get_valid_seed_node
-
       seed.node_list.each do |host|
+        if existing = @members.detect {|node| node =~ host }
+          if existing.healthy?
+            # Refresh this node's configuration
+            existing.set_config
+            # If we are unhealthy after refreshing our config, drop from the set.
+            if !existing.healthy?
+              @members.delete existing
+            else
+              next
+            end
+          else
+            existing.close
+            @members.delete existing
+          end
+        end
+
         node = Mongo::Node.new(self.client, host)
         node.connect
-        members << node if node.healthy?
+        @members << node if node.healthy?
       end
       seed.close
 
-      if members.empty?
+      if @members.empty?
         raise ConnectionFailure, "Failed to connect to any given member."
       end
-
-      members
     end
 
     # Initialize the connection pools for the primary and secondary nodes.
     def initialize_pools(members)
+      @primary_pool = nil
+      @primary = nil
+      @secondaries.clear
+      @secondary_pools.clear
+      @hosts.clear
+
       members.each do |member|
+        member.last_state = nil
         @hosts << member.host_string
         if member.primary?
           assign_primary(member)
@@ -192,30 +210,43 @@ module Mongo
         end
       end
 
-      @max_bson_size = members.first.config['maxBsonObjectSize'] ||
-        Mongo::DEFAULT_MAX_BSON_SIZE
       @arbiters = members.first.arbiters
     end
 
+    def config_min(attribute, default)
+      @members.reject {|m| !m.config[attribute]}
+      @members.map {|m| m.config[attribute]}.min || default
+    end
+
     def assign_primary(member)
       member.last_state = :primary
       @primary = member.host_port
-      @primary_pool = Pool.new(self.client, member.host, member.port,
-        :size => self.client.pool_size,
-        :timeout => self.client.pool_timeout,
-        :node => member
-      )
+      if existing = @pools.detect {|pool| pool.node == member }
+        @primary_pool = existing
+      else
+        @primary_pool = Pool.new(self.client, member.host, member.port,
+          :size => self.client.pool_size,
+          :timeout => self.client.pool_timeout,
+          :node => member
+        )
+        @pools << @primary_pool
+      end
     end
 
     def assign_secondary(member)
       member.last_state = :secondary
       @secondaries << member.host_port
-      pool = Pool.new(self.client, member.host, member.port,
-        :size => self.client.pool_size,
-        :timeout => self.client.pool_timeout,
-        :node => member
-      )
-      @secondary_pools << pool
+      if existing = @pools.detect {|pool| pool.node == member }
+        @secondary_pools << existing
+      else
+        pool = Pool.new(self.client, member.host, member.port,
+          :size => self.client.pool_size,
+          :timeout => self.client.pool_timeout,
+          :node => member
+        )
+        @secondary_pools << pool
+        @pools << pool
+      end
     end
 
     # Iterate through the list of provided seed
@@ -236,8 +267,8 @@ module Mongo
 
     private
 
-    def cache_discovered_seeds(members)
-      @seeds = members.map { |n| n.host_port }
+    def cache_discovered_seeds
+      @seeds = @members.map &:host_port
     end
 
   end