pools 0.0.3

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) Michael Rykov
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,16 @@
+pools
+-----
+
+Provides connection pooling for multiple services that
+use persistent connections
+
+Installation
+============
+
+    $ gem install pools
+
+
+Author
+=====
+
+Michael Rykov :: mrykov@gmail.com

data/Rakefile

@@ -0,0 +1,10 @@
+require 'rspec/core'
+require 'rspec/core/rake_task'
+
+task :default => :spec
+task :test    => :spec
+
+task :noop do; end
+
+desc "Run all specs in spec directory (excluding plugin specs)"
+RSpec::Core::RakeTask.new(:spec => :noop)

data/lib/pools.rb

@@ -0,0 +1,4 @@
+require 'pools/connection_pool'
+require 'pools/handler'
+require 'pools/middleware'
+require 'pools/pooled'

data/lib/pools/connection_pool.rb

@@ -0,0 +1,236 @@
+##
+# This file adapted from activerecord gem
+#
+
+require 'thread'
+require 'monitor'
+require 'set'
+require 'active_support/core_ext/module/synchronization'
+
+module Pools
+  # Raised when a connection could not be obtained within the connection
+  # acquisition timeout period.
+  ConnectionNotEstablished = Class.new(StandardError)
+  ConnectionTimeoutError = Class.new(ConnectionNotEstablished)
+
+  # Connection pool base class for managing Active Record database
+  # connections.
+  #
+  # == Introduction
+  #
+  # A connection pool synchronizes thread access to a limited number of
+  # database connections. The basic idea is that each thread checks out a
+  # database connection from the pool, uses that connection, and checks the
+  # connection back in. ConnectionPool is completely thread-safe, and will
+  # ensure that a connection cannot be used by two threads at the same time,
+  # as long as ConnectionPool's contract is correctly followed. It will also
+  # handle cases in which there are more threads than connections: if all
+  # connections have been checked out, and a thread tries to checkout a
+  # connection anyway, then ConnectionPool will wait until some other thread
+  # has checked in a connection.
+  #
+  # == Obtaining (checking out) a connection
+  #
+  # Connections can be obtained and used from a connection pool in several
+  # ways:
+  #
+  # 1. Simply use ActiveRecord::Base.connection as with Active Record 2.1 and
+  #    earlier (pre-connection-pooling). Eventually, when you're done with
+  #    the connection(s) and wish it to be returned to the pool, you call
+  #    ActiveRecord::Base.clear_active_connections!. This will be the
+  #    default behavior for Active Record when used in conjunction with
+  #    Action Pack's request handling cycle.
+  # 2. Manually check out a connection from the pool with
+  #    ActiveRecord::Base.connection_pool.checkout. You are responsible for
+  #    returning this connection to the pool when finished by calling
+  #    ActiveRecord::Base.connection_pool.checkin(connection).
+  # 3. Use ActiveRecord::Base.connection_pool.with_connection(&block), which
+  #    obtains a connection, yields it as the sole argument to the block,
+  #    and returns it to the pool after the block completes.
+  #
+  # Connections in the pool are actually AbstractAdapter objects (or objects
+  # compatible with AbstractAdapter's interface).
+  #
+  # == Options
+  #
+  # There are two connection-pooling-related options that you can add to
+  # your database connection configuration:
+  #
+  # * +pool+: number indicating size of connection pool (default 5)
+  # * +wait_timeout+: number of seconds to block and wait for a connection
+  #   before giving up and raising a timeout error (default 5 seconds).
+  class ConnectionPool
+    attr_reader :options, :connections
+
+    # Creates a new ConnectionPool object. +spec+ is a ConnectionSpecification
+    # object which describes database connection information (e.g. adapter,
+    # host name, username, password, etc), as well as the maximum size for
+    # this ConnectionPool.
+    #
+    # The default ConnectionPool maximum size is 5.
+    def initialize(pooled, options)
+      @pooled = pooled
+      @options = options
+
+      # The cache of reserved connections mapped to threads
+      @reserved_connections = {}
+
+      # The mutex used to synchronize pool access
+      @connection_mutex = Monitor.new
+      @queue = @connection_mutex.new_cond
+
+      # default 5 second timeout unless on ruby 1.9
+      @timeout = options[:wait_timeout] || 5
+
+      # default max pool size to 5
+      @size = (options[:pool] && options[:pool].to_i) || 5
+
+      @connections = []
+      @checked_out = []
+    end
+
+    # Retrieve the connection associated with the current thread, or call
+    # #checkout to obtain one if necessary.
+    #
+    # #connection can be called any number of times; the connection is
+    # held in a hash keyed by the thread id.
+    def connection
+      @reserved_connections[current_connection_id] ||= checkout
+    end
+
+    # Signal that the thread is finished with the current connection.
+    # #release_connection releases the connection-thread association
+    # and returns the connection to the pool.
+    def release_connection(with_id = current_connection_id)
+      conn = @reserved_connections.delete(with_id)
+      checkin conn if conn
+    end
+
+    # If a connection already exists yield it to the block.  If no connection
+    # exists checkout a connection, yield it to the block, and checkin the
+    # connection when finished.
+    def with_connection
+      connection_id = current_connection_id
+      fresh_connection = true unless @reserved_connections[connection_id]
+      yield connection
+    ensure
+      release_connection(connection_id) if fresh_connection
+    end
+
+    # Returns true if a connection has already been opened.
+    def connected?
+      !@connections.empty?
+    end
+
+    # Disconnects all connections in the pool, and clears the pool.
+    def disconnect!
+      @reserved_connections.each do |name,conn|
+        checkin conn
+      end
+      @reserved_connections = {}
+      @connections.each do |conn|
+        @pooled.__disconnect(conn)
+      end
+      @connections = []
+    end
+
+    # Verify active connections and remove and disconnect connections
+    # associated with stale threads.
+    def verify_active_connections! #:nodoc:
+      clear_stale_cached_connections!
+      @connections.each do |connection|
+        @pooled.__disconnect(connection)
+      end
+    end
+
+    # Return any checked-out connections back to the pool by threads that
+    # are no longer alive.
+    def clear_stale_cached_connections!
+      keys = @reserved_connections.keys - Thread.list.find_all { |t|
+        t.alive?
+      }.map { |thread| thread.object_id }
+      keys.each do |key|
+        checkin @reserved_connections[key]
+        @reserved_connections.delete(key)
+      end
+    end
+
+    # Check-out a database connection from the pool, indicating that you want
+    # to use it. You should call #checkin when you no longer need this.
+    #
+    # This is done by either returning an existing connection, or by creating
+    # a new connection. If the maximum number of connections for this pool has
+    # already been reached, but the pool is empty (i.e. they're all being used),
+    # then this method will wait until a thread has checked in a connection.
+    # The wait time is bounded however: if no connection can be checked out
+    # within the timeout specified for this pool, then a ConnectionTimeoutError
+    # exception will be raised.
+    #
+    # Returns: an AbstractAdapter object.
+    #
+    # Raises:
+    # - ConnectionTimeoutError: no connection can be obtained from the pool
+    #   within the timeout period.
+    def checkout
+      # Checkout an available connection
+      @connection_mutex.synchronize do
+        loop do
+          conn = if @checked_out.size < @connections.size
+                   checkout_existing_connection
+                 elsif @connections.size < @size
+                   checkout_new_connection
+                 end
+          return conn if conn
+
+          @queue.wait(@timeout)
+
+          if(@checked_out.size < @connections.size)
+            next
+          else
+            clear_stale_cached_connections!
+            if @size == @checked_out.size
+              raise ConnectionTimeoutError, "could not obtain a pooled connection#{" within #{@timeout} seconds" if @timeout}.  The max pool size is currently #{@size}; consider increasing it."
+            end
+          end
+
+        end
+      end
+    end
+
+    # Check-in a database connection back into the pool, indicating that you
+    # no longer need this connection.
+    #
+    # +conn+: an AbstractAdapter object, which was obtained by earlier by
+    # calling +checkout+ on this pool.
+    def checkin(conn)
+      @connection_mutex.synchronize do
+        @checked_out.delete conn
+        @queue.signal
+      end
+    end
+
+    synchronize :verify_active_connections!, :connected?, :disconnect!,
+                :with => :@connection_mutex
+
+  private
+    def current_connection_id #:nodoc:
+      Thread.current.object_id
+    end
+
+    def checkout_new_connection
+      c = @pooled.__connection
+      @connections << c
+      checkout_connection(c)
+    end
+
+    def checkout_existing_connection
+      c = (@connections - @checked_out).first
+      checkout_connection(c)
+    end
+
+    def checkout_connection(c)
+      @checked_out << c
+      c
+    end
+  end
+end

data/lib/pools/handler.rb

@@ -0,0 +1,62 @@
+##
+# This file adapted from activerecord gem
+#
+
+module Pools
+  class Handler
+    attr_reader :pools
+
+    def initialize(pools = {})
+      @pools = pools
+    end
+
+    # Add a new connection pool to the mix
+    def add(pool, name = nil)
+      @pools[name || pool.object_id] = pool
+    end
+
+    # Returns any connections in use by the current thread back to the
+    # pool, and also returns connections to the pool cached by threads
+    # that are no longer alive.
+    def clear_active_connections!
+      @pools.each_value {|pool| pool.release_connection }
+    end
+
+    def clear_all_connections!
+      @pools.each_value {|pool| pool.disconnect! }
+    end
+
+    # Verify active connections.
+    def verify_active_connections! #:nodoc:
+      @pools.each_value {|pool| pool.verify_active_connections! }
+    end
+
+    # Returns true if a connection that's accessible to this class has
+    # already been opened.
+    def connected?(name)
+      conn = retrieve_connection_pool(name)
+      conn && conn.connected?
+    end
+
+    # Remove the connection for this class. This will close the active
+    # connection and the defined connection (if they exist). The result
+    # can be used as an argument for establish_connection, for easily
+    # re-establishing the connection.
+    def remove_connection(name)
+      pool = retrieve_connection_pool(name)
+      return nil unless pool
+
+      @pools.delete_if { |key, value| value == pool }
+      pool.disconnect!
+    end
+
+    def retrieve_connection_pool(name)
+      pool = @pools[name]
+      return pool if pool
+    end
+  end
+
+  def self.handler
+    @@pool_handler ||= Handler.new
+  end
+end

data/lib/pools/middleware.rb

@@ -0,0 +1,17 @@
+##
+# This file adapted from activerecord gem
+#
+
+module Pools
+  class Middleware
+    def initialize(app)
+      @app = app
+    end
+
+    def call(env)
+      @app.call(env)
+    ensure
+      Pools.handler.clear_active_connections!
+    end
+  end
+end

data/lib/pools/pooled.rb

@@ -0,0 +1,39 @@
+require 'active_support/core_ext/array/extract_options'
+require 'active_support/concern'
+
+module Pools
+  module Pooled
+    extend ActiveSupport::Concern
+    attr_reader :connection_pool
+
+    def initialize(*args)
+      options = args.extract_options!
+      @connection_pool = ConnectionPool.new(self, options)
+      Pools.handler.add(@connection_pool, options[:pool_name])
+    end
+
+    def with_connection(&block)
+      @connection_pool.with_connection(&block)
+    end
+
+    def __connection
+      # Override in parent
+    end
+
+    def __disconnect(connection)
+      # Override in parent
+    end
+
+    module ClassMethods
+      def connection_methods(*methods)
+        methods.each do |method|
+          define_method(method) do |*params|
+            with_connection do |client|
+              client.send(method, *params)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/redis/pooled.rb

@@ -0,0 +1,43 @@
+require 'redis'
+
+class Redis
+  class Pooled
+    include ::Pools::Pooled
+
+    def initialize(options = {})
+      @redis_options = options
+      super
+    end
+
+    def __connection
+      Redis.connect(@redis_options)
+    end
+
+    def __disconnect(client)
+      client.quit if client
+    end
+
+    # Method not supported:
+    # Subscribe/Unsubscribe methods and the following...
+    # :auth, :select, :discard, :quit, :watch, :unwatch
+    # :exec, :multi, :disconnect
+
+    connection_methods :info, :config, :flushdb, :flushall, :save,
+      :bgsave, :bgrewriteaof, :get, :getset, :mget, :append, :substr,
+      :strlen, :hgetall, :hget, :hdel, :hkeys, :keys, :randomkey,
+      :echo, :ping, :lastsave, :dbsize, :exists, :llen, :lrange,
+      :ltrim, :lindex, :linsert, :lset, :lrem, :rpush, :rpushx,
+      :lpush, :lpushx, :rpop, :blpop, :brpop, :rpoplpush, :lpop,
+      :smembers, :sismember, :sadd, :srem, :smove, :sdiff, :sdiffstore,
+      :sinter, :sinterstore, :sunion, :sunionstore, :spop, :scard,
+      :srandmember, :zadd, :zrank, :zrevrank, :zincrby, :zcard,
+      :zrange, :zrangebyscore, :zcount, :zrevrange, :zremrangebyscore,
+      :zremrangebyrank, :zscore, :zrem, :zinterstore, :zunionstore,
+      :move, :setnx, :del, :rename, :renamenx, :expire, :persist,
+      :ttl, :expireat, :hset, :hsetnx, :hmset, :mapped_hmset, :hmget,
+      :mapped_hmget, :hlen, :hvals, :hincrby, :hexists, :monitor,
+      :debug, :sync, :[], :[]=, :set, :setex, :mset, :mapped_mset,
+      :msetnx, :mapped_msetnx, :mapped_mget, :sort, :incr, :incrby,
+      :decr, :decrby, :type, :publish, :id
+  end
+end

data/lib/redis/pooled_store.rb

@@ -0,0 +1,40 @@
+require 'redis-store'
+require 'redis/pooled'
+
+class Redis
+  class PooledStore < Pooled
+    include Store::Ttl, Store::Interface
+
+    def initialize(options = { })
+      super
+      _extend_marshalling options
+    end
+
+    def self.rails3? #:nodoc:
+      defined?(::Rails) && ::Rails::VERSION::MAJOR == 3
+    end
+
+    def to_s
+      with_connection do |c|
+        "Redis::Pooled => #{c.host}:#{c.port} against DB #{c.db}"
+      end
+    end
+
+  private
+    def _extend_marshalling(options) # Copied from Store
+      @marshalling = !(options[:marshalling] === false)
+      extend Store::Marshalling if @marshalling
+    end
+  end
+
+  class << Store
+    def new(*args)
+      if args.size == 1 && args.first.is_a?(Hash) && args.first[:pool]
+        PooledStore.new(*args)
+      else
+        super(*args)
+      end
+    end
+  end
+end
+

metadata

@@ -0,0 +1,77 @@
+--- !ruby/object:Gem::Specification 
+name: pools
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.0.3
+platform: ruby
+authors: 
+  - Michael Rykov
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-03-13 00:00:00 -08:00
+default_executable: 
+dependencies: 
+  - !ruby/object:Gem::Dependency 
+    name: activesupport
+    prerelease: false
+    requirement: &id001 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 3.0.5
+    type: :runtime
+    version_requirements: *id001
+description: |
+  Generalized connection pooling
+
+email: mrykov@gmail
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+  - README.md
+  - Rakefile
+  - LICENSE
+  - lib/pools.rb
+  - lib/pools/connection_pool.rb
+  - lib/pools/handler.rb
+  - lib/pools/middleware.rb
+  - lib/pools/pooled.rb
+  - lib/redis/pooled.rb
+  - lib/redis/pooled_store.rb
+has_rdoc: true
+homepage: http://github.com/rykov/pools
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+  - lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.5.1
+signing_key: 
+specification_version: 3
+summary: Generalized connection pooling
+test_files: []
+