activerecord-bogacs 0.1.0

data/lib/active_record/bogacs/false_pool.rb

@@ -0,0 +1,259 @@
+require 'thread_safe'
+require 'active_record/bogacs/pool_support'
+
+module ActiveRecord
+  module Bogacs
+    class FalsePool
+
+      include PoolSupport
+
+      include ThreadSafe::Util::CheapLockable
+      alias_method :synchronize, :cheap_synchronize
+
+      attr_accessor :automatic_reconnect
+
+      attr_reader :size, :spec
+
+      def initialize(spec)
+        @connected = nil
+
+        @spec = spec
+        @size = nil
+        #@automatic_reconnect = true
+
+        @reserved_connections = ThreadSafe::Cache.new #:initial_capacity => @size
+      end
+
+      # @private replacement for attr_reader :connections
+      def connections; @reserved_connections.values end
+
+      # @private attr_reader :reaper
+      def reaper; end
+
+      # @private
+      def checkout_timeout; end
+
+      # Retrieve the connection associated with the current thread, or call
+      # #checkout to obtain one if necessary.
+      #
+      # #connection can be called any number of times; the connection is
+      # held in a hash keyed by the thread id.
+      def connection
+        @reserved_connections[current_connection_id] ||= checkout
+      end
+
+      # Is there an open connection that is being used for the current thread?
+      def active_connection?
+        conn = @reserved_connections[current_connection_id]
+        conn ? conn.in_use? : false
+      end
+
+      # Signal that the thread is finished with the current connection.
+      # #release_connection releases the connection-thread association
+      # and returns the connection to the pool.
+      def release_connection(with_id = current_connection_id)
+        conn = @reserved_connections.delete(with_id)
+        checkin conn if conn
+      end
+
+      # If a connection already exists yield it to the block. If no connection
+      # exists checkout a connection, yield it to the block, and checkin the
+      # connection when finished.
+      def with_connection
+        connection_id = current_connection_id
+        fresh_connection = true unless active_connection?
+        yield connection
+      ensure
+        release_connection(connection_id) if fresh_connection
+      end
+
+      # Returns true if a connection has already been opened.
+      def connected?; @connected end
+
+      # Disconnects all connections in the pool, and clears the pool.
+      def disconnect!
+        synchronize do
+          @connected = false
+
+          connections = @reserved_connections.values
+          @reserved_connections.clear
+
+          connections.each do |conn|
+            checkin conn
+            conn.disconnect!
+          end
+        end
+      end
+
+      # Clears the cache which maps classes.
+      def clear_reloadable_connections!
+        synchronize do
+          @connected = false
+
+          connections = @reserved_connections.values
+          @reserved_connections.clear
+
+          connections.each do |conn|
+            checkin conn
+            conn.disconnect! if conn.requires_reloading?
+          end
+        end
+      end
+
+      # Verify active connections and remove and disconnect connections
+      # associated with stale threads.
+      # @private AR 3.2 compatibility
+      def verify_active_connections!
+        synchronize do
+          clear_stale_cached_connections!
+          connections = @reserved_connections.values
+          connections.each do |connection|
+            connection.verify!
+          end
+        end
+      end if ActiveRecord::VERSION::MAJOR < 4
+
+      # Return any checked-out connections back to the pool by threads that
+      # are no longer alive.
+      # @private AR 3.2 compatibility
+      def clear_stale_cached_connections!
+        keys = Thread.list.find_all { |t| t.alive? }.map(&:object_id)
+        keys = @reserved_connections.keys - keys
+        keys.each do |key|
+          if conn = @reserved_connections[key]
+            checkin conn, false # no release
+            @reserved_connections.delete(key)
+          end
+        end
+      end if ActiveRecord::VERSION::MAJOR < 4
+
+      # Check-out a database connection from the pool, indicating that you want
+      # to use it. You should call #checkin when you no longer need this.
+      #
+      # This is done by either returning and leasing existing connection, or by
+      # creating a new connection and leasing it.
+      #
+      # If all connections are leased and the pool is at capacity (meaning the
+      # number of currently leased connections is greater than or equal to the
+      # size limit set), an ActiveRecord::ConnectionTimeoutError exception will be raised.
+      #
+      # Returns: an AbstractAdapter object.
+      #
+      # Raises:
+      # - ConnectionTimeoutError: no connection can be obtained from the pool.
+      def checkout
+        #synchronize do
+          conn = checkout_new_connection # acquire_connection
+          conn.lease
+          conn.run_callbacks(:checkout) { conn.verify! } # checkout_and_verify(conn)
+          conn
+        #end
+      end
+
+      # Check-in a database connection back into the pool, indicating that you
+      # no longer need this connection.
+      #
+      # +conn+: an AbstractAdapter object, which was obtained by earlier by
+      # calling +checkout+ on this pool.
+      def checkin(conn, do_release = true)
+        release(conn) if do_release
+        #synchronize do
+          conn.run_callbacks(:checkin) { conn.expire }
+          #release conn
+          #@available.add conn
+        #end
+      end
+
+      # Remove a connection from the connection pool.  The connection will
+      # remain open and active but will no longer be managed by this pool.
+      def remove(conn)
+        release(conn)
+      end
+
+      # @private
+      def reap
+        # we do not really manage the connection pool - nothing to do ...
+      end
+
+      private
+
+      # Acquire a connection by one of 1) immediately removing one
+      # from the queue of available connections, 2) creating a new
+      # connection if the pool is not at capacity, 3) waiting on the
+      # queue for a connection to become available.
+      #
+      # Raises:
+      # - ConnectionTimeoutError if a connection could not be acquired
+      def acquire_connection
+        # underlying pool will poll and block if "empty" (all checked-out)
+        #if conn = @available.poll
+          #conn
+        #elsif @connections.size < @size
+          #checkout_new_connection
+        #else
+          #reap
+          #@available.poll(@checkout_timeout)
+        #end
+        checkout_new_connection
+      end
+
+      def release(conn, owner = nil)
+        thread_id = owner.object_id unless owner.nil?
+
+        thread_id ||=
+          if @reserved_connections[conn_id = current_connection_id] == conn
+            conn_id
+          else
+            connections = @reserved_connections
+            connections.keys.find { |k| connections[k] == conn }
+          end
+
+        @reserved_connections.delete thread_id if thread_id
+      end
+
+      def checkout_new_connection
+        # NOTE: automatic reconnect seems to make no sense for us!
+        #raise ConnectionNotEstablished unless @automatic_reconnect
+
+        begin
+          conn = new_connection
+        rescue ConnectionTimeoutError => e
+          raise e
+        rescue => e
+          raise ConnectionTimeoutError, e.message if _timeout_error?(e)
+          raise e
+        end
+        conn.pool = self
+        synchronize { @connected = true } if @connected != true
+        conn
+      end
+
+      #def checkout_and_verify(conn)
+      #  conn.run_callbacks(:checkout) { conn.verify! }
+      #  conn
+      #end
+
+      # sample on JRuby + Tomcat JDBC :
+      # ActiveRecord::JDBCError(<The driver encountered an unknown error:
+      #   org.apache.tomcat.jdbc.pool.PoolExhaustedException:
+      #   [main] Timeout: Pool empty. Unable to fetch a connection in 2 seconds,
+      #   none available[size:10; busy:10; idle:0; lastwait:2500].>
+      # )
+
+      def _timeout_error?(error)
+        error.inspect =~ /timeout/i
+      end
+
+#      def _timeout_error?(error); end # TODO: not sure what to do on MRI and friends
+#
+#      def _timeout_error?(error)
+#        if error.is_a?(JDBCError)
+#          if sql_exception = error.sql_exception
+#            return true if sql_exception.to_s =~ /timeout/i
+#          end
+#        end
+#      end if defined? ArJdbc
+
+    end
+  end
+end

data/lib/active_record/bogacs/pool_support.rb

@@ -0,0 +1,21 @@
+#require 'thread_safe'
+
+module ActiveRecord
+  module Bogacs
+    module PoolSupport
+
+      def self.included(base)
+        #base.send :include, ThreadSafe::Util::CheapLockable
+      end
+
+      def new_connection
+        Base.send(spec.adapter_method, spec.config)
+      end
+
+      def current_connection_id
+        Base.connection_id ||= Thread.current.object_id # TODO
+      end
+
+    end
+  end
+end

data/lib/active_record/bogacs/shareable_pool.rb

@@ -0,0 +1,255 @@
+require 'active_record/connection_adapters/abstract/connection_pool'
+require 'thread'
+require 'thread_safe'
+require 'atomic'
+
+# NOTE: needs explicit configuration - before connection gets established e.g.
+#
+#   pool_class = ActiveRecord::ConnectionAdapters::ShareableConnectionPool
+#   ActiveRecord::ConnectionAdapters::ConnectionHandler.connection_pool_class = pool_class
+#
+module ActiveRecord
+  module Bogacs
+    class ShareablePool < ConnectionAdapters::ConnectionPool # TODO do not override?!
+      include ThreadSafe::Util::CheapLockable
+
+      DEFAULT_SHARED_POOL = 0.25 # only allow 25% of the pool size to be shared
+      MAX_THREAD_SHARING = 5 # not really a strict limit but should hold
+
+      attr_reader :shared_size
+
+      # @override
+      def initialize(spec)
+        super(spec) # ConnectionPool#initialize
+        shared_size = spec.config[:shared_pool]
+        shared_size = shared_size ? shared_size.to_f : DEFAULT_SHARED_POOL
+        # size 0.0 - 1.0 assumes percentage of the pool size
+        shared_size = ( @size * shared_size ).round if shared_size <= 1.0
+        @shared_size = shared_size.to_i
+        @shared_connections = ThreadSafe::Cache.new # :initial_capacity => @shared_size, :concurrency_level => 20
+      end
+
+      # @override
+      def connection
+        # TODO we assume here a single pool - multiple pool support not implemented!
+        Thread.current[:shared_pool_connection] || begin # super - simplified :
+          super # @reserved_connections.compute_if_absent(current_connection_id) { checkout }
+        end
+      end
+
+      # @override
+      def active_connection?
+        if shared_conn = Thread.current[:shared_pool_connection]
+          return shared_conn.in_use?
+        end
+        super_active_connection? current_connection_id
+      end
+
+      # @override called from ConnectionManagement middle-ware (when finished)
+      def release_connection(with_id = current_connection_id)
+        if reserved_conn = @reserved_connections[with_id]
+          if shared_count = @shared_connections[reserved_conn]
+            cheap_synchronize do # lock due #get_shared_connection ... not needed ?!
+              # NOTE: the other option is to not care about shared here at all ...
+              if shared_count.get == 0 # releasing a shared connection
+                release_shared_connection(reserved_conn)
+              #else return false
+              end
+            end
+          else # check back-in non-shared connections
+            checkin reserved_conn # (what super does)
+          end
+        end
+      end
+
+      # @override
+      def disconnect!
+        cheap_synchronize { @shared_connections.clear; super }
+      end
+
+      # @override
+      def clear_reloadable_connections!
+        cheap_synchronize { @shared_connections.clear; super }
+      end
+
+      # @override
+      # @note called from #reap thus the pool should work with reaper
+      def remove(conn)
+        cheap_synchronize { @shared_connections.delete(conn); super }
+      end
+
+#      # Return any checked-out connections back to the pool by threads that
+#      # are no longer alive.
+#      # @private AR 3.2 compatibility
+#      def clear_stale_cached_connections!
+#        keys = Thread.list.find_all { |t| t.alive? }.map(&:object_id)
+#        keys = @reserved_connections.keys - keys
+#        keys.each do |key|
+#          release_connection(key)
+#          @reserved_connections.delete(key)
+#        end
+#      end if ActiveRecord::VERSION::MAJOR < 4
+
+      # TODO take care of explicit connection.close (`pool.checkin self`) ?
+
+      # Custom API :
+
+      def release_shared_connection(connection)
+        if connection == Thread.current[:shared_pool_connection]
+          Thread.current[:shared_pool_connection] = nil
+        end
+
+        @shared_connections.delete(connection)
+        checkin connection
+      end
+
+      def with_shared_connection
+        # with_shared_connection call nested in the same thread
+        if connection = Thread.current[:shared_pool_connection]
+          emulated_checkout(connection)
+          return yield connection
+        end
+
+        start = Time.now if DEBUG
+        begin
+          connection_id = current_connection_id
+          # if there's a 'regular' connection on the thread use it as super
+          if super_active_connection?(connection_id) # for current thread
+            connection = self.connection # do not mark as shared
+            DEBUG && debug("with_shared_conn 10 got active = #{connection.to_s}")
+          # otherwise if we have a shared connection - use that one :
+          elsif connection = get_shared_connection
+            emulated_checkout(connection); shared = true
+            DEBUG && debug("with_shared_conn 20 got shared = #{connection.to_s}")
+          else
+            cheap_synchronize do
+              # check shared again as/if threads end up sync-ing up here :
+              if connection = get_shared_connection
+                emulated_checkout(connection)
+                DEBUG && debug("with_shared_conn 21 got shared = #{connection.to_s}")
+              end # here we acquire but a connection from the pool
+              # TODO the bottle-neck for concurrency doing sync { checkout } :
+              unless connection # here we acquire a connection from the pool
+                connection = self.checkout # might block if pool fully used
+                add_shared_connection(connection)
+                DEBUG && debug("with_shared_conn 30 acq shared = #{connection.to_s}")
+              end
+            end
+            shared = true
+          end
+
+          Thread.current[:shared_pool_connection] = connection if shared
+
+          DEBUG && debug("with_shared_conn obtaining a connection took #{(Time.now - start) * 1000}ms")
+          yield connection
+        ensure
+          Thread.current[:shared_pool_connection] = nil # if shared
+          rem_shared_connection(connection) if shared
+        end
+      end
+
+      private
+
+      def super_active_connection?(connection_id = current_connection_id)
+        (@reserved_connections.get(connection_id) || ( return false )).in_use?
+      end
+
+      def super_active_connection?(connection_id = current_connection_id)
+        synchronize do
+          (@reserved_connections[connection_id] || ( return false )).in_use?
+        end
+      end if ActiveRecord::VERSION::MAJOR < 4
+
+      def acquire_connection_no_wait?
+        synchronize do
+          @available.send(:can_remove_no_wait?) || @connections.size < @size
+        end
+      end
+
+      # get a (shared) connection that is least shared among threads (or nil)
+      # nil gets returned if it's 'better' to checkout a new one to be shared
+      # ... to better utilize shared connection reuse among multiple threads
+      def get_shared_connection # (lock = nil)
+        least_count = MAX_THREAD_SHARING + 1; least_shared = nil
+        shared_connections_size = @shared_connections.size
+
+        @shared_connections.each_pair do |connection, shared_count|
+          next if shared_count.get >= MAX_THREAD_SHARING
+          if ( shared_count = shared_count.get ) < least_count
+            DEBUG && debug(" get_shared_conn loop : #{connection.to_s} shared #{shared_count}-time(s)")
+            # ! DO NOT return connection if shared_count == 0
+            least_count = shared_count; least_shared = connection
+          end
+        end
+
+        if least_count > 0
+          if shared_connections_size < @shared_connections.size
+            DEBUG && debug(" get_shared_conn retry (shared connection added)")
+            return get_shared_connection # someone else added something re-try
+          end
+          if ( @shared_connections.size < @shared_size ) && acquire_connection_no_wait?
+            DEBUG && debug(" get_shared_conn return none - acquire from pool")
+            return nil # we should rather 'get' a new shared one from the pool
+          end
+        end
+
+        # we did as much as could without a lock - now sync due possible release
+        cheap_synchronize do # TODO although this likely might be avoided ...
+          # should try again if possibly the same connection got released :
+          unless least_count = @shared_connections[least_shared]
+            DEBUG && debug(" get_shared_conn retry (connection got released)")
+            return get_shared_connection
+          end
+          least_count.update { |v| v + 1 }
+        end if least_shared
+
+        DEBUG && debug(" get_shared_conn least shared = #{least_shared.to_s}")
+        least_shared # might be nil in that case we'll likely wait (as super)
+      end
+
+      def add_shared_connection(connection)
+        @shared_connections[connection] = Atomic.new(1)
+      end
+
+      def rem_shared_connection(connection)
+        if shared_count = @shared_connections[connection]
+           # shared_count.update { |v| v - 1 } # NOTE: likely fine without lock!
+           cheap_synchronize do # give it back to the pool
+             shared_count.update { |v| v - 1 } # might give it back if :
+             release_shared_connection(connection) if shared_count.get == 0
+           end
+        end
+      end
+
+      def emulated_checkin(connection)
+        # NOTE: not sure we'd like to run `run_callbacks :checkin {}` here ...
+        connection.expire
+      end
+
+      def emulated_checkout(connection)
+        # NOTE: not sure we'd like to run `run_callbacks :checkout {}` here ...
+        connection.lease; # connection.verify! auto-reconnect should do this
+      end
+
+      DEBUG = begin
+        debug = ENV['DB_POOL_DEBUG'].to_s
+        if debug.to_s == 'false' then false
+        elsif ! debug.empty?
+          log_dev = case debug
+          when 'STDOUT', 'stdout' then STDOUT
+          when 'STDERR', 'stderr' then STDERR
+          when 'true' then ActiveRecord::Base.logger
+          else File.expand_path(debug)
+          end
+          require 'logger'; Logger.new log_dev
+        else nil
+        end
+      end
+
+      private
+
+      def debug(msg); DEBUG.debug msg end
+
+    end
+  end
+end