mongo 1.3.0 → 1.12.5

data/lib/mongo/connection/pool.rb

@@ -0,0 +1,340 @@
+# Copyright (C) 2009-2013 MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  class Pool
+    PING_ATTEMPTS  = 6
+    MAX_PING_TIME  = 1_000_000
+    PRUNE_INTERVAL = 10_000
+
+    attr_accessor :host,
+                  :port,
+                  :address,
+                  :size,
+                  :timeout,
+                  :checked_out,
+                  :client,
+                  :node
+
+    # Create a new pool of connections.
+    def initialize(client, host, port, opts={})
+      @client = client
+
+      @host, @port = host, port
+
+      # A Mongo::Node object.
+      @node = opts[:node]
+
+      # The string address
+      @address = "#{@host}:#{@port}"
+
+      # Pool size and timeout.
+      @size    = opts.fetch(:size, 20)
+      @timeout = opts.fetch(:timeout, 30)
+
+      # Mutex for synchronizing pool access
+      @connection_mutex = Mutex.new
+
+      # Mutex for synchronizing pings
+      @ping_mutex = Mutex.new
+
+      # Condition variable for signal and wait
+      @queue = ConditionVariable.new
+
+      @sockets               = []
+      @checked_out           = []
+      @ping_time             = nil
+      @last_ping             = nil
+      @closed                = false
+      @thread_ids_to_sockets = {}
+      @checkout_counter      = 0
+    end
+
+    # Close this pool.
+    #
+    # @option opts [Boolean]:soft (false) If true,
+    #   close only those sockets that are not checked out.
+    def close(opts={})
+      @connection_mutex.synchronize do
+        if opts[:soft] && !@checked_out.empty?
+          @closing = true
+          close_sockets(@sockets - @checked_out)
+        else
+          close_sockets(@sockets)
+          @closed = true
+        end
+        @node.close if @node
+      end
+      true
+    end
+
+    def tags
+      @node.tags
+    end
+
+    def healthy?
+      close if @sockets.all?(&:closed?)
+      !closed? && node.healthy?
+    end
+
+    def closed?
+      @closed
+    end
+
+    def up?
+      !@closed
+    end
+
+    def inspect
+      "#<Mongo::Pool:0x#{self.object_id.to_s(16)} @host=#{@host} @port=#{port} " +
+        "@ping_time=#{@ping_time} #{@checked_out.size}/#{@size} sockets available " +
+        "up=#{!closed?}>"
+    end
+
+    def host_string
+      "#{@host}:#{@port}"
+    end
+
+    def host_port
+      [@host, @port]
+    end
+
+    # Refresh ping time only if we haven't
+    # checked within the last five minutes.
+    def ping_time
+      @ping_mutex.synchronize do
+        if !@last_ping || (Time.now - @last_ping) > 300
+          @ping_time = refresh_ping_time
+          @last_ping = Time.now
+        end
+      end
+      @ping_time
+    end
+
+    # Return the time it takes on average
+    # to do a round-trip against this node.
+    def refresh_ping_time
+      trials = []
+      PING_ATTEMPTS.times do
+        t1 = Time.now
+        if !self.ping
+          return MAX_PING_TIME
+        end
+        trials << (Time.now - t1) * 1000
+      end
+
+      trials.sort!
+
+      # Delete shortest and longest times
+      trials.delete_at(trials.length-1)
+      trials.delete_at(0)
+
+      total = 0.0
+      trials.each { |t| total += t }
+
+      (total / trials.length).ceil
+    end
+
+    def ping
+      begin
+        return self.client['admin'].command({:ping => 1}, :socket => @node.socket,
+                                                          :timeout => client.op_timeout || MongoClient::DEFAULT_OP_TIMEOUT)
+      rescue ConnectionFailure, OperationFailure, SocketError, SystemCallError, IOError
+        return false
+      end
+    end
+
+    # Return a socket to the pool.
+    def checkin(socket)
+      @connection_mutex.synchronize do
+        if @checked_out.delete(socket)
+          @queue.broadcast
+        else
+          return false
+        end
+      end
+      true
+    end
+
+    # Adds a new socket to the pool and checks it out.
+    #
+    # This method is called exclusively from #checkout;
+    # therefore, it runs within a mutex.
+    def checkout_new_socket
+      begin
+        socket = @client.socket_class.new(@host, @port, @client.op_timeout,
+                                                        @client.connect_timeout,
+                                                        @client.socket_opts)
+        socket.pool = self
+      rescue => ex
+        socket.close if socket
+        @node.close if @node
+        raise ConnectionFailure, "Failed to connect to host #{@host} and port #{@port}: #{ex}"
+      end
+
+      @sockets << socket
+      @checked_out << socket
+      @thread_ids_to_sockets[Thread.current.object_id] = socket
+      socket
+    end
+
+    # If a user calls DB#authenticate, and several sockets exist,
+    # then we need a way to apply the authentication on each socket.
+    # So we store the apply_authentication method, and this will be
+    # applied right before the next use of each socket.
+    #
+    # @deprecated This method has been replaced by Pool#check_auths (private)
+    # and it isn't necessary to ever invoke this method directly.
+    # Authentication of sockets is handled upon checkout and checkin.
+    def authenticate_existing
+    end
+
+    # Store the logout op for each existing socket to be applied before
+    # the next use of each socket.
+    #
+    # @deprecated This method has been replaced by Pool#check_auths (private)
+    # and it isn't necessary to ever invoke this method directly.
+    # Authentication of sockets is handled upon checkout and checkin.
+    def logout_existing(database)
+    end
+
+    # Checks out the first available socket from the pool.
+    #
+    # If the pid has changed, remove the socket and check out
+    # new one.
+    #
+    # This method is called exclusively from #checkout;
+    # therefore, it runs within a mutex.
+    def checkout_existing_socket(socket=nil)
+      if !socket
+        available = @sockets - @checked_out
+        socket = available[rand(available.length)]
+      end
+
+      if socket.pid != Process.pid
+        @sockets.delete(socket)
+        if socket
+          socket.close unless socket.closed?
+        end
+        checkout_new_socket
+      else
+        @checked_out << socket
+        @thread_ids_to_sockets[Thread.current.object_id] = socket
+        socket
+      end
+    end
+
+    def prune_threads
+      live_threads = Thread.list.map(&:object_id)
+      @thread_ids_to_sockets.reject! do |key, value|
+        !live_threads.include?(key)
+      end
+    end
+
+    def check_prune
+      if @checkout_counter > PRUNE_INTERVAL
+          @checkout_counter = 0
+          prune_threads
+      else
+        @checkout_counter += 1
+      end
+    end
+
+    # Check out an existing socket or create a new socket if the maximum
+    # pool size has not been exceeded. Otherwise, wait for the next
+    # available socket.
+    def checkout
+      @client.connect if !@client.connected?
+      start_time = Time.now
+      loop do
+        if (Time.now - start_time) > @timeout
+          raise ConnectionTimeoutError, "could not obtain connection within " +
+            "#{@timeout} seconds. The max pool size is currently #{@size}; " +
+            "consider increasing the pool size or timeout."
+        end
+
+        @connection_mutex.synchronize do
+          check_prune
+          socket = nil
+          if socket_for_thread = @thread_ids_to_sockets[Thread.current.object_id]
+            if !@checked_out.include?(socket_for_thread)
+              socket = checkout_existing_socket(socket_for_thread)
+            end
+          else
+            if @sockets.size < @size
+              socket = checkout_new_socket
+            elsif @checked_out.size < @sockets.size
+              socket = checkout_existing_socket
+            end
+          end
+
+          if socket
+            if !socket.closed?
+              begin
+                check_auths(socket)
+                return socket
+              rescue ConnectionFailure
+                # Socket failed authentication and will be cleaned up below
+              end
+            end
+
+            # Socket was closed from earlier network error, or just now from
+            # a network error when authenticating.
+            @checked_out.delete(socket)
+            @sockets.delete(socket)
+            @thread_ids_to_sockets.delete(Thread.current.object_id)
+          else
+            # Otherwise, wait
+            @queue.wait(@connection_mutex)
+          end
+        end
+      end
+    end
+
+    private
+
+    # Helper method to handle keeping track of auths/logouts for sockets.
+    #
+    # @param socket [Socket] The socket instance to be checked.
+    #
+    # @return [Socket] The authenticated socket instance.
+    def check_auths(socket)
+      # find and handle logouts
+      (socket.auths - @client.auths).each do |auth|
+        @client.issue_logout(auth[:source], :socket => socket)
+        socket.auths.delete(auth)
+      end
+
+      # find and handle new auths
+      (@client.auths - socket.auths).each do |auth|
+        @client.issue_authentication(auth, :socket => socket)
+        socket.auths.add(auth)
+      end
+
+      socket
+    end
+
+    def close_sockets(sockets)
+      sockets.each do |socket|
+        @sockets.delete(socket)
+        begin
+          socket.close unless socket.closed?
+        rescue IOError => ex
+          warn "IOError when attempting to close socket connected to #{@host}:#{@port}: #{ex.inspect}"
+        end
+      end
+    end
+
+  end
+end

data/lib/mongo/connection/pool_manager.rb

@@ -0,0 +1,320 @@
+# Copyright (C) 2009-2013 MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  class PoolManager
+    include ThreadLocalVariableManager
+
+    attr_reader :client,
+                :hosts,
+                :pools,
+                :secondaries,
+                :secondary_pools,
+                :arbiters,
+                :primary,
+                :primary_pool,
+                :seeds,
+                :max_bson_size,
+                :max_message_size,
+                :max_wire_version,
+                :min_wire_version
+
+    # Create a new set of connection pools.
+    #
+    # The pool manager will by default use the original seed list passed
+    # to the connection objects, accessible via connection.seeds. In addition,
+    # the user may pass an additional list of seeds nodes discovered in real
+    # time. The union of these lists will be used when attempting to connect,
+    # with the newly-discovered nodes being used first.
+    def initialize(client, seeds=[])
+      @client                                   = client
+      @seeds                                    = seeds
+
+      initialize_immutable_state
+      initialize_mutable_state
+
+      @pools                                    = Set.new
+      @primary                                  = nil
+      @primary_pool                             = nil
+      @members                                  = Set.new
+      @refresh_required                         = false
+      @max_bson_size                            = DEFAULT_MAX_BSON_SIZE
+      @max_message_size                         = @max_bson_size * MESSAGE_SIZE_FACTOR
+      @max_wire_version                         = 0
+      @min_wire_version                         = 0
+      @connect_mutex                            = Mutex.new
+      thread_local[:locks][:connecting_manager] = false
+    end
+
+    def inspect
+      "<Mongo::PoolManager:0x#{self.object_id.to_s(16)} @seeds=#{@seeds}>"
+    end
+
+    def connect
+      @connect_mutex.synchronize do
+        begin
+          thread_local[:locks][:connecting_manager] = true
+          @refresh_required = false
+          disconnect_old_members
+          connect_to_members
+          initialize_pools(@members)
+          update_max_sizes
+          @seeds = discovered_seeds
+        ensure
+          thread_local[:locks][:connecting_manager] = false
+        end
+      end
+      clone_state
+    end
+
+    def refresh!(additional_seeds)
+      @seeds |= additional_seeds
+      connect
+    end
+
+    # We're healthy if all members are pingable and if the view
+    # of the replica set returned by isMaster is equivalent
+    # to our view. If any of these isn't the case,
+    # set @refresh_required to true, and return.
+    def check_connection_health
+      return if thread_local[:locks][:connecting_manager]
+      members = copy_members
+      begin
+        seed = get_valid_seed_node
+      rescue ConnectionFailure
+        @refresh_required = true
+        return
+      end
+
+      unless current_config = seed.config
+        @refresh_required = true
+        seed.close
+        return
+      end
+
+      if current_config['hosts'].length != members.length
+        @refresh_required = true
+        seed.close
+        return
+      end
+
+      current_config['hosts'].each do |host|
+        member = members.detect do |m|
+          m.address == host
+        end
+
+        if member && validate_existing_member(current_config, member)
+          next
+        else
+          @refresh_required = true
+          seed.close
+          return
+        end
+      end
+
+      seed.close
+    end
+
+    # The replica set connection should initiate a full refresh.
+    def refresh_required?
+      @refresh_required
+    end
+
+    def closed?
+      pools.all? { |pool| pool.closed? }
+    end
+
+    def close(opts={})
+      begin
+        pools.each { |pool| pool.close(opts) }
+      rescue ConnectionFailure
+      end
+    end
+
+    def read
+      read_pool.host_port
+    end
+
+    private
+
+    def update_max_sizes
+      unless @members.size == 0
+        @max_bson_size = @members.map(&:max_bson_size).min
+        @max_message_size = @members.map(&:max_message_size).min
+        @max_wire_version = @members.map(&:max_wire_version).min
+        @min_wire_version = @members.map(&:min_wire_version).max
+      end
+    end
+
+    def validate_existing_member(current_config, member)
+      if current_config['ismaster'] && member.last_state != :primary
+        return false
+      elsif member.last_state != :other
+        return false
+      end
+      return true
+    end
+
+    # For any existing members, close and remove any that are unhealthy or already closed.
+    def disconnect_old_members
+      @pools_mutable.reject!   {|pool| !pool.healthy? }
+      @members.reject! {|node| !node.healthy? }
+    end
+
+    # Connect to each member of the replica set
+    # as reported by the given seed node.
+    def connect_to_members
+      seed = get_valid_seed_node
+      seed.node_list.each do |host|
+        if existing = @members.detect {|node| node =~ host }
+          if existing.healthy?
+            # Refresh this node's configuration
+            existing.set_config
+            # If we are unhealthy after refreshing our config, drop from the set.
+            if !existing.healthy?
+              @members.delete(existing)
+            else
+              next
+            end
+          else
+            existing.close
+            @members.delete(existing)
+          end
+        end
+
+        node = Mongo::Node.new(self.client, host)
+        node.connect
+        @members << node if node.healthy?
+      end
+      seed.close
+
+      if @members.empty?
+        raise ConnectionFailure, "Failed to connect to any given member."
+      end
+    end
+
+    # Initialize the connection pools for the primary and secondary nodes.
+    def initialize_pools(members)
+      @primary_pool = nil
+      @primary = nil
+      @secondaries_mutable.clear
+      @secondary_pools_mutable.clear
+      @hosts_mutable.clear
+
+      members.each do |member|
+        member.last_state = nil
+        @hosts_mutable << member.host_string
+        if member.primary?
+          assign_primary(member)
+        elsif member.secondary?
+          # member could be not primary but secondary still is false
+          assign_secondary(member)
+        end
+      end
+
+      @arbiters_mutable = members.first.arbiters
+    end
+
+    def assign_primary(member)
+      member.last_state = :primary
+      @primary = member.host_port
+      if existing = @pools_mutable.detect {|pool| pool.node == member }
+        @primary_pool = existing
+      else
+        @primary_pool = Pool.new(self.client, member.host, member.port,
+          :size => self.client.pool_size,
+          :timeout => self.client.pool_timeout,
+          :node => member
+        )
+        @pools_mutable << @primary_pool
+      end
+    end
+
+    def assign_secondary(member)
+      member.last_state = :secondary
+      @secondaries_mutable << member.host_port
+      if existing = @pools_mutable.detect {|pool| pool.node == member }
+        @secondary_pools_mutable << existing
+      else
+        pool = Pool.new(self.client, member.host, member.port,
+          :size => self.client.pool_size,
+          :timeout => self.client.pool_timeout,
+          :node => member
+        )
+        @secondary_pools_mutable << pool
+        @pools_mutable << pool
+      end
+    end
+
+    # Iterate through the list of provided seed
+    # nodes until we've gotten a response from the
+    # replica set we're trying to connect to.
+    #
+    # If we don't get a response, raise an exception.
+    def get_valid_seed_node
+      @seeds.each do |seed|
+        node = Mongo::Node.new(self.client, seed)
+        node.connect
+        return node if node.healthy?
+      end
+
+      raise ConnectionFailure, "Cannot connect to a replica set using seeds " +
+        "#{@seeds.map {|s| "#{s[0]}:#{s[1]}" }.join(', ')}"
+    end
+
+    def discovered_seeds
+      @members.map(&:host_port)
+    end
+
+    def copy_members
+       members = Set.new
+       @connect_mutex.synchronize do
+         @members.map do |m|
+           members << m.dup
+         end
+       end
+       members
+    end
+
+    def initialize_immutable_state
+      @hosts           = Set.new.freeze
+      @pools           = Set.new.freeze
+      @secondaries     = Set.new.freeze
+      @secondary_pools = [].freeze
+      @arbiters        = [].freeze
+    end
+
+    def initialize_mutable_state
+      @hosts_mutable           = Set.new
+      @pools_mutable           = Set.new
+      @secondaries_mutable     = Set.new
+      @secondary_pools_mutable = []
+      @arbiters_mutable        = []
+    end
+
+    def clone_state
+      @hosts           = @hosts_mutable.clone
+      @pools           = @pools_mutable.clone
+      @secondaries     = @secondaries_mutable.clone
+      @secondary_pools = @secondary_pools_mutable.clone
+      @arbiters        = @arbiters_mutable.clone
+
+      @hosts.freeze
+      @pools.freeze
+      @secondaries.freeze
+      @secondary_pools.freeze
+      @arbiters.freeze
+    end
+  end
+end