activerecord_threadsafe_fix 3.1.1

data/lib/active_record/queue.rb

@@ -0,0 +1,130 @@
+require 'thread'
+require 'monitor'
+require 'set'
+
+require 'active_record'
+require 'active_record/connection_adapters/abstract/connection_pool'
+
+module ActiveRecord
+  # Threadsafe, fair, FIFO queue.  Meant to be used by ConnectionPool
+  # with which it shares a Monitor.  But could be a generic Queue.
+  class Queue
+    def initialize(lock = Monitor.new)
+      @lock = lock
+      @cond = @lock.new_cond
+      @num_waiting = 0
+      @queue = []
+    end
+
+    # Test if any threads are currently waiting on the queue.
+    def any_waiting?
+      synchronize do
+        @num_waiting > 0
+      end
+    end
+
+    # Return the number of threads currently waiting on this
+    # queue.
+    def num_waiting
+      synchronize do
+        @num_waiting
+      end
+    end
+
+    # Add +element+ to the queue.  Never blocks.
+    def add(element)
+      synchronize do
+        @queue.push element
+        @cond.signal
+      end
+    end
+
+    # If +element+ is in the queue, remove and return it, or nil.
+    def delete(element)
+      synchronize do
+        @queue.delete(element)
+      end
+    end
+
+    # Remove all elements from the queue.
+    def clear
+      synchronize do
+        @queue.clear
+      end
+    end
+
+    # Remove the head of the queue.
+    #
+    # If +timeout+ is not given, remove and return the head the
+    # queue if the number of available elements is strictly
+    # greater than the number of threads currently waiting (that
+    # is, don't jump ahead in line).  Otherwise, return nil.
+    #
+    # If +timeout+ is given, block if it there is no element
+    # available, waiting up to +timeout+ seconds for an element to
+    # become available.
+    #
+    # Raises:
+    # - ConnectionTimeoutError if +timeout+ is given and no element
+    # becomes available after +timeout+ seconds,
+    def poll(timeout = nil)
+      synchronize do
+        if timeout
+          no_wait_poll || wait_poll(timeout)
+        else
+          no_wait_poll
+        end
+      end
+    end
+
+    private
+
+    def synchronize(&block)
+      @lock.synchronize(&block)
+    end
+
+    # Test if the queue currently contains any elements.
+    def any?
+      !@queue.empty?
+    end
+
+    # A thread can remove an element from the queue without
+    # waiting if an only if the number of currently available
+    # connections is strictly greater than the number of waiting
+    # threads.
+    def can_remove_no_wait?
+      @queue.size > @num_waiting
+    end
+
+    # Removes and returns the head of the queue if possible, or nil.
+    def remove
+      @queue.shift
+    end
+
+    # Remove and return the head the queue if the number of
+    # available elements is strictly greater than the number of
+    # threads currently waiting.  Otherwise, return nil.
+    def no_wait_poll
+      remove if can_remove_no_wait?
+    end
+
+    # Waits on the queue up to +timeout+ seconds, then removes and
+    # returns the head of the queue.
+    def wait_poll(timeout)
+      @num_waiting += 1
+
+      t0 = Time.now
+      elapsed = 0
+      loop do
+        @cond.wait(timeout - elapsed)
+
+        return remove if any?
+
+        elapsed = Time.now - t0
+        raise ConnectionTimeoutError if elapsed >= timeout
+      end
+    ensure
+      @num_waiting -= 1
+    end
+  end
+end

data/lib/activerecord_threadsafe_fix.rb

@@ -0,0 +1,12 @@
+case ActiveRecord::VERSION::MAJOR
+when 3
+  case ActiveRecord::VERSION::MINOR
+  when 0
+    require 'activerecord_threadsafe_fix_3_0'
+  when 1
+    require 'activerecord_threadsafe_fix_3_1'
+  else
+    raise 'ActiveRecord threadsafe fix required, but no fix for ActiveRecord' +
+      "#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}"
+  end
+end

data/lib/activerecord_threadsafe_fix_3_0.rb

@@ -0,0 +1,371 @@
+#######################################################################
+#
+# This is a fixed version of the connection pool making it threadsafe
+# and fair.  Mostly it's copied from 3.0.12's but with the Queue class
+# added -pmahoney
+#
+#######################################################################
+
+require 'active_record'
+require 'active_record/connection_adapters/abstract/connection_pool'
+require 'active_record/queue'
+
+require 'active_support/core_ext/module/synchronization'
+
+
+module ActiveRecord
+  # Raised when a connection could not be obtained within the connection
+  # acquisition timeout period.
+  class ConnectionTimeoutError < ConnectionNotEstablished
+  end
+
+  module ConnectionAdapters
+    # Connection pool base class for managing Active Record database
+    # connections.
+    #
+    # == Introduction
+    #
+    # A connection pool synchronizes thread access to a limited number of
+    # database connections. The basic idea is that each thread checks out a
+    # database connection from the pool, uses that connection, and checks the
+    # connection back in. ConnectionPool is completely thread-safe, and will
+    # ensure that a connection cannot be used by two threads at the same time,
+    # as long as ConnectionPool's contract is correctly followed. It will also
+    # handle cases in which there are more threads than connections: if all
+    # connections have been checked out, and a thread tries to checkout a
+    # connection anyway, then ConnectionPool will wait until some other thread
+    # has checked in a connection.
+    #
+    # == Obtaining (checking out) a connection
+    #
+    # Connections can be obtained and used from a connection pool in several
+    # ways:
+    #
+    # 1. Simply use ActiveRecord::Base.connection as with Active Record 2.1 and
+    #    earlier (pre-connection-pooling). Eventually, when you're done with
+    #    the connection(s) and wish it to be returned to the pool, you call
+    #    ActiveRecord::Base.clear_active_connections!. This will be the
+    #    default behavior for Active Record when used in conjunction with
+    #    Action Pack's request handling cycle.
+    # 2. Manually check out a connection from the pool with
+    #    ActiveRecord::Base.connection_pool.checkout. You are responsible for
+    #    returning this connection to the pool when finished by calling
+    #    ActiveRecord::Base.connection_pool.checkin(connection).
+    # 3. Use ActiveRecord::Base.connection_pool.with_connection(&block), which
+    #    obtains a connection, yields it as the sole argument to the block,
+    #    and returns it to the pool after the block completes.
+    #
+    # Connections in the pool are actually AbstractAdapter objects (or objects
+    # compatible with AbstractAdapter's interface).
+    #
+    # == Options
+    #
+    # There are two connection-pooling-related options that you can add to
+    # your database connection configuration:
+    #
+    # * +pool+: number indicating size of connection pool (default 5)
+    # * +wait_timeout+: number of seconds to block and wait for a connection
+    #   before giving up and raising a timeout error (default 5 seconds).
+    class ConnectionPool
+      attr_reader :spec, :connections
+
+      # Creates a new ConnectionPool object. +spec+ is a ConnectionSpecification
+      # object which describes database connection information (e.g. adapter,
+      # host name, username, password, etc), as well as the maximum size for
+      # this ConnectionPool.
+      #
+      # The default ConnectionPool maximum size is 5.
+      def initialize(spec)
+        @spec = spec
+
+        # The cache of reserved connections mapped to threads
+        @reserved_connections = {}
+
+        # The mutex used to synchronize pool access
+        @lock = Monitor.new
+        # @queue = @lock.new_cond
+        @queue = Queue.new(@lock)
+        
+
+        # default 5 second timeout unless on ruby 1.9
+        @timeout = spec.config[:wait_timeout] || 5
+
+        # default max pool size to 5
+        @size = (spec.config[:pool] && spec.config[:pool].to_i) || 5
+
+        @connections = []
+      end
+
+      # Retrieve the connection associated with the current thread, or call
+      # #checkout to obtain one if necessary.
+      #
+      # #connection can be called any number of times; the connection is
+      # held in a hash keyed by the thread id.
+      def connection
+        synchronize do
+          @reserved_connections[current_connection_id] ||= checkout
+        end
+      end
+
+      # Signal that the thread is finished with the current connection.
+      # #release_connection releases the connection-thread association
+      # and returns the connection to the pool.
+      def release_connection(with_id = current_connection_id)
+        synchronize do
+          conn = @reserved_connections.delete(with_id)
+          checkin conn if conn
+        end
+      end
+
+      # If a connection already exists yield it to the block.  If no connection
+      # exists checkout a connection, yield it to the block, and checkin the
+      # connection when finished.
+      def with_connection
+        connection_id = current_connection_id
+        fresh_connection = synchronize do
+          true unless @reserved_connections[connection_id]
+        end
+        yield connection
+      ensure
+        release_connection(connection_id) if fresh_connection
+      end
+
+      # Returns true if a connection has already been opened.
+      def connected?
+        synchronize do
+          !@connections.empty?
+        end
+      end
+
+      # Disconnects all connections in the pool, and clears the pool.
+      def disconnect!
+        synchronize do
+          @reserved_connections.each do |name,conn|
+            checkin conn
+          end
+          @reserved_connections = {}
+          @connections.each do |conn|
+            conn.disconnect!
+          end
+          @connections = []
+        end
+      end
+
+      # Clears the cache which maps classes
+      def clear_reloadable_connections!
+        synchronize do
+          @reserved_connections.each do |name, conn|
+            checkin conn
+          end
+          @reserved_connections = {}
+          @connections.each do |conn|
+            conn.disconnect! if conn.requires_reloading?
+          end
+          @connections.delete_if do |conn|
+            conn.requires_reloading?
+          end
+        end
+      end
+
+      # Verify active connections and remove and disconnect connections
+      # associated with stale threads.
+      def verify_active_connections! #:nodoc:
+        synchronize do
+          clear_stale_cached_connections!
+          @connections.each do |connection|
+            connection.verify!
+          end
+        end
+      end
+
+      # Return any checked-out connections back to the pool by threads that
+      # are no longer alive.
+      def clear_stale_cached_connections!
+        synchronize do
+          keys = @reserved_connections.keys - Thread.list.find_all { |t|
+            t.alive?
+          }.map { |thread| thread.object_id }
+          keys.each do |key|
+            checkin @reserved_connections[key]
+            @reserved_connections.delete(key)
+          end
+        end
+      end
+
+      # Check-out a database connection from the pool, indicating that you want
+      # to use it. You should call #checkin when you no longer need this.
+      #
+      # This is done by either returning an existing connection, or by creating
+      # a new connection. If the maximum number of connections for this pool has
+      # already been reached, but the pool is empty (i.e. they're all being used),
+      # then this method will wait until a thread has checked in a connection.
+      # The wait time is bounded however: if no connection can be checked out
+      # within the timeout specified for this pool, then a ConnectionTimeoutError
+      # exception will be raised.
+      #
+      # Returns: an AbstractAdapter object.
+      #
+      # Raises:
+      # - ConnectionTimeoutError: no connection can be obtained from the pool
+      #   within the timeout period.
+      def checkout
+        if conn = @queue.poll
+          checkout_and_verify(conn)
+        elsif @connections.size < @size
+          checkout_new_connection
+        else
+          # clear_stale_cached_connections!
+          checkout_and_verify(@queue.poll(@timeout))
+        end
+      end
+
+      # Check-in a database connection back into the pool, indicating that you
+      # no longer need this connection.
+      #
+      # +conn+: an AbstractAdapter object, which was obtained by earlier by
+      # calling +checkout+ on this pool.
+      def checkin(conn)
+        synchronize do
+          conn.send(:_run_checkin_callbacks) do
+            @queue.add conn
+          end
+        end
+      end
+
+      private
+
+      def synchronize(&block)
+        @lock.synchronize(&block)
+      end
+
+      def new_connection
+        ActiveRecord::Base.send(spec.adapter_method, spec.config)
+      end
+
+      def current_connection_id #:nodoc:
+        Thread.current.object_id
+      end
+
+      def checkout_new_connection
+        c = new_connection
+        @connections << c
+        checkout_and_verify(c)
+      end
+
+      def checkout_and_verify(c)
+        c.run_callbacks :checkout do
+          c.verify!
+        end
+        c
+      end
+    end
+
+    # ConnectionHandler is a collection of ConnectionPool objects. It is used
+    # for keeping separate connection pools for Active Record models that connect
+    # to different databases.
+    #
+    # For example, suppose that you have 5 models, with the following hierarchy:
+    #
+    #  |
+    #  +-- Book
+    #  |    |
+    #  |    +-- ScaryBook
+    #  |    +-- GoodBook
+    #  +-- Author
+    #  +-- BankAccount
+    #
+    # Suppose that Book is to connect to a separate database (i.e. one other
+    # than the default database). Then Book, ScaryBook and GoodBook will all use
+    # the same connection pool. Likewise, Author and BankAccount will use the
+    # same connection pool. However, the connection pool used by Author/BankAccount
+    # is not the same as the one used by Book/ScaryBook/GoodBook.
+    #
+    # Normally there is only a single ConnectionHandler instance, accessible via
+    # ActiveRecord::Base.connection_handler. Active Record models use this to
+    # determine that connection pool that they should use.
+    class ConnectionHandler
+      attr_reader :connection_pools
+
+      def initialize(pools = {})
+        @connection_pools = pools
+      end
+
+      def establish_connection(name, spec)
+        @connection_pools[name] = ConnectionAdapters::ConnectionPool.new(spec)
+      end
+
+      # Returns any connections in use by the current thread back to the pool,
+      # and also returns connections to the pool cached by threads that are no
+      # longer alive.
+      def clear_active_connections!
+        @connection_pools.each_value {|pool| pool.release_connection }
+      end
+
+      # Clears the cache which maps classes
+      def clear_reloadable_connections!
+        @connection_pools.each_value {|pool| pool.clear_reloadable_connections! }
+      end
+
+      def clear_all_connections!
+        @connection_pools.each_value {|pool| pool.disconnect! }
+      end
+
+      # Verify active connections.
+      def verify_active_connections! #:nodoc:
+        @connection_pools.each_value {|pool| pool.verify_active_connections! }
+      end
+
+      # Locate the connection of the nearest super class. This can be an
+      # active or defined connection: if it is the latter, it will be
+      # opened and set as the active connection for the class it was defined
+      # for (not necessarily the current class).
+      def retrieve_connection(klass) #:nodoc:
+        pool = retrieve_connection_pool(klass)
+        (pool && pool.connection) or raise ConnectionNotEstablished
+      end
+
+      # Returns true if a connection that's accessible to this class has
+      # already been opened.
+      def connected?(klass)
+        conn = retrieve_connection_pool(klass)
+        conn && conn.connected?
+      end
+
+      # Remove the connection for this class. This will close the active
+      # connection and the defined connection (if they exist). The result
+      # can be used as an argument for establish_connection, for easily
+      # re-establishing the connection.
+      def remove_connection(klass)
+        pool = @connection_pools[klass.name]
+        return nil unless pool
+
+        @connection_pools.delete_if { |key, value| value == pool }
+        pool.disconnect!
+        pool.spec.config
+      end
+
+      def retrieve_connection_pool(klass)
+        pool = @connection_pools[klass.name]
+        return pool if pool
+        return nil if ActiveRecord::Base == klass
+        retrieve_connection_pool klass.superclass
+      end
+    end
+
+    class ConnectionManagement
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        @app.call(env)
+      ensure
+        # Don't return connection (and perform implicit rollback) if
+        # this request is a part of integration test
+        unless env.key?("rack.test")
+          ActiveRecord::Base.clear_active_connections!
+        end
+      end
+    end
+  end
+end

data/lib/activerecord_threadsafe_fix_3_1.rb

@@ -0,0 +1,114 @@
+#######################################################################
+#
+# This is a fixed version of the connection pool making it threadsafe
+# and fair.  Mostly it's copied from 3.0.12's but with the Queue class
+# added -pmahoney
+#
+#######################################################################
+
+require 'active_record'
+require 'active_record/connection_adapters/abstract/connection_pool'
+require 'active_record/queue'
+
+require 'active_support/core_ext/module/synchronization'
+
+module ActiveRecord
+  module ConnectionAdapters
+    # Monkey Patch time!
+    class ConnectionPool
+
+      alias :orig_initialize :initialize
+
+      class << self
+        def make_synchronized!(*syms)
+          syms.each do |sym|
+            orig = "__orig_#{sym}__".to_sym
+            alias_method orig, sym
+            define_method(sym) do |*args, &block|
+              synchronize do
+                send(orig, *args, &block)
+              end
+            end
+          end
+        end
+      end
+
+      make_synchronized!(:connection, :active_connection?, :release_connection, :connected?,
+                         :disconnect!, :clear_reloadable_connections!, :verify_active_connections!,
+                         :clear_stale_cached_connections!)
+      
+
+      def initialize(spec)
+        orig_initialize(spec)
+
+        # The mutex used to synchronize pool access
+        @lock = Monitor.new
+
+        @queue = Queue.new(@lock)
+      end
+
+      def with_connection
+        connection_id = current_connection_id
+        fresh_connection = synchronize do
+          true unless @reserved_connections[connection_id]
+        end
+        yield connection
+      ensure
+        release_connection(connection_id) if fresh_connection
+      end
+
+      # Check-out a database connection from the pool, indicating that you want
+      # to use it. You should call #checkin when you no longer need this.
+      #
+      # This is done by either returning an existing connection, or by creating
+      # a new connection. If the maximum number of connections for this pool has
+      # already been reached, but the pool is empty (i.e. they're all being used),
+      # then this method will wait until a thread has checked in a connection.
+      # The wait time is bounded however: if no connection can be checked out
+      # within the timeout specified for this pool, then a ConnectionTimeoutError
+      # exception will be raised.
+      #
+      # Returns: an AbstractAdapter object.
+      #
+      # Raises:
+      # - ConnectionTimeoutError: no connection can be obtained from the pool
+      #   within the timeout period.
+      def checkout
+        if conn = @queue.poll
+          checkout_and_verify(conn)
+        elsif @connections.size < @size
+          checkout_new_connection
+        else
+          # clear_stale_cached_connections!
+          checkout_and_verify(@queue.poll(@timeout))
+        end
+      end
+
+      # Check-in a database connection back into the pool, indicating that you
+      # no longer need this connection.
+      #
+      # +conn+: an AbstractAdapter object, which was obtained by earlier by
+      # calling +checkout+ on this pool.
+      def checkin(conn)
+        synchronize do
+          conn.run_callbacks :checkin do
+            @queue.add conn
+          end
+        end
+      end
+
+      private
+
+      def synchronize(&block)
+        @lock.synchronize(&block)
+      end
+
+      def checkout_and_verify(c)
+        c.run_callbacks :checkout do
+          c.verify!
+        end
+        c
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,98 @@
+--- !ruby/object:Gem::Specification
+name: activerecord_threadsafe_fix
+version: !ruby/object:Gem::Version
+  version: 3.1.1
+  prerelease: 
+platform: ruby
+authors:
+- CG Labs
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-09-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: version
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1'
+description: Patch ActiveRecord to have a fair, threadsafe connection pool
+email:
+- eng@commongroundpublishing.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/active_record/queue.rb
+- lib/activerecord_threadsafe_fix.rb
+- lib/activerecord_threadsafe_fix_3_0.rb
+- lib/activerecord_threadsafe_fix_3_1.rb
+homepage: 
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Patch ActiveRecord to have a fair, threadsafe connection pool
+test_files: []
+has_rdoc: