activerecord 5.0.7 → 5.1.7

data/lib/active_record/connection_adapters/abstract/connection_pool.rb

@@ -1,6 +1,6 @@
-require 'thread'
-require 'concurrent/map'
-require 'monitor'
+require "thread"
+require "concurrent/map"
+require "monitor"
 
 module ActiveRecord
   # Raised when a connection could not be obtained within the connection
@@ -69,12 +69,12 @@ module ActiveRecord
     #   threads, which can occur if a programmer forgets to close a
     #   connection at the end of a thread or a thread dies unexpectedly.
     #   Regardless of this setting, the Reaper will be invoked before every
-    #   blocking wait. (Default nil, which means don't schedule the Reaper).
+    #   blocking wait. (Default +nil+, which means don't schedule the Reaper).
     #
     #--
     # Synchronization policy:
     # * all public methods can be called outside +synchronize+
-    # * access to these i-vars needs to be in +synchronize+:
+    # * access to these instance variables needs to be in +synchronize+:
     #   * @connections
     #   * @now_connecting
     # * private methods that require being called in a +synchronize+ blocks
@@ -116,7 +116,7 @@ module ActiveRecord
           end
         end
 
-        # If +element+ is in the queue, remove and return it, or nil.
+        # If +element+ is in the queue, remove and return it, or +nil+.
         def delete(element)
           synchronize do
             @queue.delete(element)
@@ -135,7 +135,7 @@ module ActiveRecord
         # If +timeout+ is not given, remove and return the head the
         # queue if the number of available elements is strictly
         # greater than the number of threads currently waiting (that
-        # is, don't jump ahead in line).  Otherwise, return nil.
+        # is, don't jump ahead in line).  Otherwise, return +nil+.
         #
         # If +timeout+ is given, block if there is no element
         # available, waiting up to +timeout+ seconds for an element to
@@ -150,61 +150,63 @@ module ActiveRecord
 
         private
 
-        def internal_poll(timeout)
-          no_wait_poll || (timeout && wait_poll(timeout))
-        end
+          def internal_poll(timeout)
+            no_wait_poll || (timeout && wait_poll(timeout))
+          end
 
-        def synchronize(&block)
-          @lock.synchronize(&block)
-        end
+          def synchronize(&block)
+            @lock.synchronize(&block)
+          end
 
-        # Test if the queue currently contains any elements.
-        def any?
-          !@queue.empty?
-        end
+          # Test if the queue currently contains any elements.
+          def any?
+            !@queue.empty?
+          end
 
-        # A thread can remove an element from the queue without
-        # waiting if and only if the number of currently available
-        # connections is strictly greater than the number of waiting
-        # threads.
-        def can_remove_no_wait?
-          @queue.size > @num_waiting
-        end
+          # A thread can remove an element from the queue without
+          # waiting if and only if the number of currently available
+          # connections is strictly greater than the number of waiting
+          # threads.
+          def can_remove_no_wait?
+            @queue.size > @num_waiting
+          end
 
-        # Removes and returns the head of the queue if possible, or nil.
-        def remove
-          @queue.shift
-        end
+          # Removes and returns the head of the queue if possible, or +nil+.
+          def remove
+            @queue.shift
+          end
 
-        # Remove and return the head the queue if the number of
-        # available elements is strictly greater than the number of
-        # threads currently waiting.  Otherwise, return nil.
-        def no_wait_poll
-          remove if can_remove_no_wait?
-        end
+          # Remove and return the head the queue if the number of
+          # available elements is strictly greater than the number of
+          # threads currently waiting.  Otherwise, return +nil+.
+          def no_wait_poll
+            remove if can_remove_no_wait?
+          end
 
-        # Waits on the queue up to +timeout+ seconds, then removes and
-        # returns the head of the queue.
-        def wait_poll(timeout)
-          @num_waiting += 1
+          # Waits on the queue up to +timeout+ seconds, then removes and
+          # returns the head of the queue.
+          def wait_poll(timeout)
+            @num_waiting += 1
 
-          t0 = Time.now
-          elapsed = 0
-          loop do
-            @cond.wait(timeout - elapsed)
+            t0 = Time.now
+            elapsed = 0
+            loop do
+              ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
+                @cond.wait(timeout - elapsed)
+              end
 
-            return remove if any?
+              return remove if any?
 
-            elapsed = Time.now - t0
-            if elapsed >= timeout
-              msg = 'could not obtain a connection from the pool within %0.3f seconds (waited %0.3f seconds); all pooled connections were in use' %
-                [timeout, elapsed]
-              raise ConnectionTimeoutError, msg
+              elapsed = Time.now - t0
+              if elapsed >= timeout
+                msg = "could not obtain a connection from the pool within %0.3f seconds (waited %0.3f seconds); all pooled connections were in use" %
+                  [timeout, elapsed]
+                raise ConnectionTimeoutError, msg
+              end
             end
+          ensure
+            @num_waiting -= 1
           end
-        ensure
-          @num_waiting -= 1
-        end
       end
 
       # Adds the ability to turn a basic fair FIFO queue into one
@@ -274,15 +276,15 @@ module ActiveRecord
         include BiasableQueue
 
         private
-        def internal_poll(timeout)
-          conn = super
-          conn.lease if conn
-          conn
-        end
+          def internal_poll(timeout)
+            conn = super
+            conn.lease if conn
+            conn
+          end
       end
 
       # Every +frequency+ seconds, the reaper will call +reap+ on +pool+.
-      # A reaper instantiated with a nil frequency will never reap the
+      # A reaper instantiated with a +nil+ frequency will never reap the
       # connection pool.
       #
       # Configure the frequency by setting "reaping_frequency" in your
@@ -298,7 +300,7 @@ module ActiveRecord
         def run
           return unless frequency
           Thread.new(frequency, pool) { |t, p|
-            while true
+            loop do
               sleep t
               p.reap
             end
@@ -330,17 +332,17 @@ module ActiveRecord
         # default max pool size to 5
         @size = (spec.config[:pool] && spec.config[:pool].to_i) || 5
 
-        # The cache of threads mapped to reserved connections, the sole purpose
-        # of the cache is to speed-up +connection+ method, it is not the authoritative
-        # registry of which thread owns which connection, that is tracked by
-        # +connection.owner+ attr on each +connection+ instance.
+        # This variable tracks the cache of threads mapped to reserved connections, with the
+        # sole purpose of speeding up the +connection+ method. It is not the authoritative
+        # registry of which thread owns which connection. Connection ownership is tracked by
+        # the +connection.owner+ attr on each +connection+ instance.
         # The invariant works like this: if there is mapping of <tt>thread => conn</tt>,
-        # then that +thread+ does indeed own that +conn+, however an absence of a such
-        # mapping does not mean that the +thread+ doesn't own the said connection, in
+        # then that +thread+ does indeed own that +conn+. However, an absence of a such
+        # mapping does not mean that the +thread+ doesn't own the said connection. In
         # that case +conn.owner+ attr should be consulted.
         # Access and modification of +@thread_cached_conns+ does not require
         # synchronization.
-        @thread_cached_conns = Concurrent::Map.new(:initial_capacity => @size)
+        @thread_cached_conns = Concurrent::Map.new(initial_capacity: @size)
 
         @connections         = []
         @automatic_reconnect = true
@@ -353,6 +355,16 @@ module ActiveRecord
         @threads_blocking_new_connections = 0
 
         @available = ConnectionLeasingQueue.new self
+
+        @lock_thread = false
+      end
+
+      def lock_thread=(lock_thread)
+        if lock_thread
+          @lock_thread = Thread.current
+        else
+          @lock_thread = nil
+        end
       end
 
       # Retrieve the connection associated with the current thread, or call
@@ -361,13 +373,13 @@ module ActiveRecord
       # #connection can be called any number of times; the connection is
       # held in a cache keyed by a thread.
       def connection
-        @thread_cached_conns[connection_cache_key(Thread.current)] ||= checkout
+        @thread_cached_conns[connection_cache_key(@lock_thread || Thread.current)] ||= checkout
       end
 
-      # Is there an open connection that is being used for the current thread?
+      # Returns true if there is an open connection being used for the current thread.
       #
       # This method only works for connections that have been obtained through
-      # #connection or #with_connection methods, connections obtained through
+      # #connection or #with_connection methods. Connections obtained through
       # #checkout will not be detected by #active_connection?
       def active_connection?
         @thread_cached_conns[connection_cache_key(Thread.current)]
@@ -429,9 +441,9 @@ module ActiveRecord
 
       # Disconnects all connections in the pool, and clears the pool.
       #
-      # The pool first tries to gain ownership of all connections, if unable to
+      # The pool first tries to gain ownership of all connections. If unable to
       # do so within a timeout interval (default duration is
-      # <tt>spec.config[:checkout_timeout] * 2</tt> seconds), the pool is forcefully
+      # <tt>spec.config[:checkout_timeout] * 2</tt> seconds), then the pool is forcefully
       # disconnected without any regard for other connection owning threads.
       def disconnect!
         disconnect(false)
@@ -463,9 +475,9 @@ module ActiveRecord
       # Clears the cache which maps classes and re-connects connections that
       # require reloading.
       #
-      # The pool first tries to gain ownership of all connections, if unable to
+      # The pool first tries to gain ownership of all connections. If unable to
       # do so within a timeout interval (default duration is
-      # <tt>spec.config[:checkout_timeout] * 2</tt> seconds), the pool forcefully
+      # <tt>spec.config[:checkout_timeout] * 2</tt> seconds), then the pool forcefully
       # clears the cache and reloads connections without any regard for other
       # connection owning threads.
       def clear_reloadable_connections!
@@ -496,14 +508,16 @@ module ActiveRecord
       # +conn+: an AbstractAdapter object, which was obtained by earlier by
       # calling #checkout on this pool.
       def checkin(conn)
-        synchronize do
-          remove_connection_from_thread_cache conn
+        conn.lock.synchronize do
+          synchronize do
+            remove_connection_from_thread_cache conn
 
-          conn._run_checkin_callbacks do
-            conn.expire
-          end
+            conn._run_checkin_callbacks do
+              conn.expire
+            end
 
-          @available.add conn
+            @available.add conn
+          end
         end
       end
 
@@ -519,20 +533,20 @@ module ActiveRecord
           @available.delete conn
 
           # @available.any_waiting? => true means that prior to removing this
-          # conn, the pool was at its max size (@connections.size == @size)
-          # this would mean that any threads stuck waiting in the queue wouldn't
+          # conn, the pool was at its max size (@connections.size == @size).
+          # This would mean that any threads stuck waiting in the queue wouldn't
           # know they could checkout_new_connection, so let's do it for them.
           # Because condition-wait loop is encapsulated in the Queue class
           # (that in turn is oblivious to ConnectionPool implementation), threads
-          # that are "stuck" there are helpless, they have no way of creating
+          # that are "stuck" there are helpless. They have no way of creating
           # new connections and are completely reliant on us feeding available
           # connections into the Queue.
           needs_new_connection = @available.any_waiting?
         end
 
         # This is intentionally done outside of the synchronized section as we
-        # would like not to hold the main mutex while checking out new connections,
-        # thus there is some chance that needs_new_connection information is now
+        # would like not to hold the main mutex while checking out new connections.
+        # Thus there is some chance that needs_new_connection information is now
         # stale, we can live with that (bulk_make_new_connections will make
         # sure not to exceed the pool's @size limit).
         bulk_make_new_connections(1) if needs_new_connection
@@ -564,225 +578,243 @@ module ActiveRecord
         @available.num_waiting
       end
 
-      private
-      #--
-      # this is unfortunately not concurrent
-      def bulk_make_new_connections(num_new_conns_needed)
-        num_new_conns_needed.times do
-          # try_to_checkout_new_connection will not exceed pool's @size limit
-          if new_conn = try_to_checkout_new_connection
-            # make the new_conn available to the starving threads stuck @available Queue
-            checkin(new_conn)
-          end
-        end
-      end
-
-      #--
-      # From the discussion on GitHub:
-      #  https://github.com/rails/rails/pull/14938#commitcomment-6601951
-      # This hook-in method allows for easier monkey-patching fixes needed by
-      # JRuby users that use Fibers.
-      def connection_cache_key(thread)
-        thread
-      end
-
-      # Take control of all existing connections so a "group" action such as
-      # reload/disconnect can be performed safely. It is no longer enough to
-      # wrap it in +synchronize+ because some pool's actions are allowed
-      # to be performed outside of the main +synchronize+ block.
-      def with_exclusively_acquired_all_connections(raise_on_acquisition_timeout = true)
-        with_new_connections_blocked do
-          attempt_to_checkout_all_existing_connections(raise_on_acquisition_timeout)
-          yield
+      # Return connection pool's usage statistic
+      # Example:
+      #
+      #    ActiveRecord::Base.connection_pool.stat # => { size: 15, connections: 1, busy: 1, dead: 0, idle: 0, waiting: 0, checkout_timeout: 5 }
+      def stat
+        synchronize do
+          {
+            size: size,
+            connections: @connections.size,
+            busy: @connections.count { |c| c.in_use? && c.owner.alive? },
+            dead: @connections.count { |c| c.in_use? && !c.owner.alive? },
+            idle: @connections.count { |c| !c.in_use? },
+            waiting: num_waiting_in_queue,
+            checkout_timeout: checkout_timeout
+          }
         end
       end
 
-      def attempt_to_checkout_all_existing_connections(raise_on_acquisition_timeout = true)
-        collected_conns = synchronize do
-          # account for our own connections
-          @connections.select {|conn| conn.owner == Thread.current}
-        end
-
-        newly_checked_out = []
-        timeout_time      = Time.now + (@checkout_timeout * 2)
-
-        @available.with_a_bias_for(Thread.current) do
-          while true
-            synchronize do
-              return if collected_conns.size == @connections.size && @now_connecting == 0
-              remaining_timeout = timeout_time - Time.now
-              remaining_timeout = 0 if remaining_timeout < 0
-              conn = checkout_for_exclusive_access(remaining_timeout)
-              collected_conns   << conn
-              newly_checked_out << conn
+      private
+        #--
+        # this is unfortunately not concurrent
+        def bulk_make_new_connections(num_new_conns_needed)
+          num_new_conns_needed.times do
+            # try_to_checkout_new_connection will not exceed pool's @size limit
+            if new_conn = try_to_checkout_new_connection
+              # make the new_conn available to the starving threads stuck @available Queue
+              checkin(new_conn)
             end
           end
         end
-      rescue ExclusiveConnectionTimeoutError
-        # <tt>raise_on_acquisition_timeout == false</tt> means we are directed to ignore any
-        # timeouts and are expected to just give up: we've obtained as many connections
-        # as possible, note that in a case like that we don't return any of the
-        # +newly_checked_out+ connections.
 
-        if raise_on_acquisition_timeout
-          release_newly_checked_out = true
-          raise
+        #--
+        # From the discussion on GitHub:
+        #  https://github.com/rails/rails/pull/14938#commitcomment-6601951
+        # This hook-in method allows for easier monkey-patching fixes needed by
+        # JRuby users that use Fibers.
+        def connection_cache_key(thread)
+          thread
         end
-      rescue Exception # if something else went wrong
-        # this can't be a "naked" rescue, because we have should return conns
-        # even for non-StandardErrors
-        release_newly_checked_out = true
-        raise
-      ensure
-        if release_newly_checked_out && newly_checked_out
-          # releasing only those conns that were checked out in this method, conns
-          # checked outside this method (before it was called) are not for us to release
-          newly_checked_out.each {|conn| checkin(conn)}
+
+        # Take control of all existing connections so a "group" action such as
+        # reload/disconnect can be performed safely. It is no longer enough to
+        # wrap it in +synchronize+ because some pool's actions are allowed
+        # to be performed outside of the main +synchronize+ block.
+        def with_exclusively_acquired_all_connections(raise_on_acquisition_timeout = true)
+          with_new_connections_blocked do
+            attempt_to_checkout_all_existing_connections(raise_on_acquisition_timeout)
+            yield
+          end
         end
-      end
 
-      #--
-      # Must be called in a synchronize block.
-      def checkout_for_exclusive_access(checkout_timeout)
-        checkout(checkout_timeout)
-      rescue ConnectionTimeoutError
-        # this block can't be easily moved into attempt_to_checkout_all_existing_connections's
-        # rescue block, because doing so would put it outside of synchronize section, without
-        # being in a critical section thread_report might become inaccurate
-        msg = "could not obtain ownership of all database connections in #{checkout_timeout} seconds"
+        def attempt_to_checkout_all_existing_connections(raise_on_acquisition_timeout = true)
+          collected_conns = synchronize do
+            # account for our own connections
+            @connections.select { |conn| conn.owner == Thread.current }
+          end
 
-        thread_report = []
-        @connections.each do |conn|
-          unless conn.owner == Thread.current
-            thread_report << "#{conn} is owned by #{conn.owner}"
+          newly_checked_out = []
+          timeout_time      = Time.now + (@checkout_timeout * 2)
+
+          @available.with_a_bias_for(Thread.current) do
+            loop do
+              synchronize do
+                return if collected_conns.size == @connections.size && @now_connecting == 0
+                remaining_timeout = timeout_time - Time.now
+                remaining_timeout = 0 if remaining_timeout < 0
+                conn = checkout_for_exclusive_access(remaining_timeout)
+                collected_conns   << conn
+                newly_checked_out << conn
+              end
+            end
+          end
+        rescue ExclusiveConnectionTimeoutError
+          # <tt>raise_on_acquisition_timeout == false</tt> means we are directed to ignore any
+          # timeouts and are expected to just give up: we've obtained as many connections
+          # as possible, note that in a case like that we don't return any of the
+          # +newly_checked_out+ connections.
+
+          if raise_on_acquisition_timeout
+            release_newly_checked_out = true
+            raise
+          end
+        rescue Exception # if something else went wrong
+          # this can't be a "naked" rescue, because we have should return conns
+          # even for non-StandardErrors
+          release_newly_checked_out = true
+          raise
+        ensure
+          if release_newly_checked_out && newly_checked_out
+            # releasing only those conns that were checked out in this method, conns
+            # checked outside this method (before it was called) are not for us to release
+            newly_checked_out.each { |conn| checkin(conn) }
           end
         end
 
-        msg << " (#{thread_report.join(', ')})" if thread_report.any?
+        #--
+        # Must be called in a synchronize block.
+        def checkout_for_exclusive_access(checkout_timeout)
+          checkout(checkout_timeout)
+        rescue ConnectionTimeoutError
+          # this block can't be easily moved into attempt_to_checkout_all_existing_connections's
+          # rescue block, because doing so would put it outside of synchronize section, without
+          # being in a critical section thread_report might become inaccurate
+          msg = "could not obtain ownership of all database connections in #{checkout_timeout} seconds"
+
+          thread_report = []
+          @connections.each do |conn|
+            unless conn.owner == Thread.current
+              thread_report << "#{conn} is owned by #{conn.owner}"
+            end
+          end
 
-        raise ExclusiveConnectionTimeoutError, msg
-      end
+          msg << " (#{thread_report.join(', ')})" if thread_report.any?
 
-      def with_new_connections_blocked
-        synchronize do
-          @threads_blocking_new_connections += 1
+          raise ExclusiveConnectionTimeoutError, msg
         end
 
-        yield
-      ensure
-        num_new_conns_required = 0
+        def with_new_connections_blocked
+          synchronize do
+            @threads_blocking_new_connections += 1
+          end
 
-        synchronize do
-          @threads_blocking_new_connections -= 1
+          yield
+        ensure
+          num_new_conns_required = 0
 
-          if @threads_blocking_new_connections.zero?
-            @available.clear
+          synchronize do
+            @threads_blocking_new_connections -= 1
 
-            num_new_conns_required = num_waiting_in_queue
+            if @threads_blocking_new_connections.zero?
+              @available.clear
 
-            @connections.each do |conn|
-              next if conn.in_use?
+              num_new_conns_required = num_waiting_in_queue
+
+              @connections.each do |conn|
+                next if conn.in_use?
 
-              @available.add conn
-              num_new_conns_required -= 1
+                @available.add conn
+                num_new_conns_required -= 1
+              end
             end
           end
-        end
-
-        bulk_make_new_connections(num_new_conns_required) if num_new_conns_required > 0
-      end
 
-      # Acquire a connection by one of 1) immediately removing one
-      # from the queue of available connections, 2) creating a new
-      # connection if the pool is not at capacity, 3) waiting on the
-      # queue for a connection to become available.
-      #
-      # Raises:
-      # - ActiveRecord::ConnectionTimeoutError if a connection could not be acquired
-      #
-      #--
-      # Implementation detail: the connection returned by +acquire_connection+
-      # will already be "+connection.lease+ -ed" to the current thread.
-      def acquire_connection(checkout_timeout)
-        # NOTE: we rely on +@available.poll+ and +try_to_checkout_new_connection+ to
-        # +conn.lease+ the returned connection (and to do this in a +synchronized+
-        # section), this is not the cleanest implementation, as ideally we would
-        # <tt>synchronize { conn.lease }</tt> in this method, but by leaving it to +@available.poll+
-        # and +try_to_checkout_new_connection+ we can piggyback on +synchronize+ sections
-        # of the said methods and avoid an additional +synchronize+ overhead.
-        if conn = @available.poll || try_to_checkout_new_connection
-          conn
-        else
-          reap
-          @available.poll(checkout_timeout)
+          bulk_make_new_connections(num_new_conns_required) if num_new_conns_required > 0
         end
-      end
 
-      #--
-      # if owner_thread param is omitted, this must be called in synchronize block
-      def remove_connection_from_thread_cache(conn, owner_thread = conn.owner)
-        @thread_cached_conns.delete_pair(connection_cache_key(owner_thread), conn)
-      end
-      alias_method :release, :remove_connection_from_thread_cache
+        # Acquire a connection by one of 1) immediately removing one
+        # from the queue of available connections, 2) creating a new
+        # connection if the pool is not at capacity, 3) waiting on the
+        # queue for a connection to become available.
+        #
+        # Raises:
+        # - ActiveRecord::ConnectionTimeoutError if a connection could not be acquired
+        #
+        #--
+        # Implementation detail: the connection returned by +acquire_connection+
+        # will already be "+connection.lease+ -ed" to the current thread.
+        def acquire_connection(checkout_timeout)
+          # NOTE: we rely on +@available.poll+ and +try_to_checkout_new_connection+ to
+          # +conn.lease+ the returned connection (and to do this in a +synchronized+
+          # section). This is not the cleanest implementation, as ideally we would
+          # <tt>synchronize { conn.lease }</tt> in this method, but by leaving it to +@available.poll+
+          # and +try_to_checkout_new_connection+ we can piggyback on +synchronize+ sections
+          # of the said methods and avoid an additional +synchronize+ overhead.
+          if conn = @available.poll || try_to_checkout_new_connection
+            conn
+          else
+            reap
+            @available.poll(checkout_timeout)
+          end
+        end
 
-      def new_connection
-        Base.send(spec.adapter_method, spec.config).tap do |conn|
-          conn.schema_cache = schema_cache.dup if schema_cache
+        #--
+        # if owner_thread param is omitted, this must be called in synchronize block
+        def remove_connection_from_thread_cache(conn, owner_thread = conn.owner)
+          @thread_cached_conns.delete_pair(connection_cache_key(owner_thread), conn)
         end
-      end
+        alias_method :release, :remove_connection_from_thread_cache
 
-      # If the pool is not at a +@size+ limit, establish new connection. Connecting
-      # to the DB is done outside main synchronized section.
-      #--
-      # Implementation constraint: a newly established connection returned by this
-      # method must be in the +.leased+ state.
-      def try_to_checkout_new_connection
-        # first in synchronized section check if establishing new conns is allowed
-        # and increment @now_connecting, to prevent overstepping this pool's @size
-        # constraint
-        do_checkout = synchronize do
-          if @threads_blocking_new_connections.zero? && (@connections.size + @now_connecting) < @size
-            @now_connecting += 1
+        def new_connection
+          Base.send(spec.adapter_method, spec.config).tap do |conn|
+            conn.schema_cache = schema_cache.dup if schema_cache
           end
         end
-        if do_checkout
-          begin
-            # if successfully incremented @now_connecting establish new connection
-            # outside of synchronized section
-            conn = checkout_new_connection
-          ensure
-            synchronize do
-              if conn
-                adopt_connection(conn)
-                # returned conn needs to be already leased
-                conn.lease
+
+        # If the pool is not at a +@size+ limit, establish new connection. Connecting
+        # to the DB is done outside main synchronized section.
+        #--
+        # Implementation constraint: a newly established connection returned by this
+        # method must be in the +.leased+ state.
+        def try_to_checkout_new_connection
+          # first in synchronized section check if establishing new conns is allowed
+          # and increment @now_connecting, to prevent overstepping this pool's @size
+          # constraint
+          do_checkout = synchronize do
+            if @threads_blocking_new_connections.zero? && (@connections.size + @now_connecting) < @size
+              @now_connecting += 1
+            end
+          end
+          if do_checkout
+            begin
+              # if successfully incremented @now_connecting establish new connection
+              # outside of synchronized section
+              conn = checkout_new_connection
+            ensure
+              synchronize do
+                if conn
+                  adopt_connection(conn)
+                  # returned conn needs to be already leased
+                  conn.lease
+                end
+                @now_connecting -= 1
               end
-              @now_connecting -= 1
             end
           end
         end
-      end
 
-      def adopt_connection(conn)
-        conn.pool = self
-        @connections << conn
-      end
+        def adopt_connection(conn)
+          conn.pool = self
+          @connections << conn
+        end
 
-      def checkout_new_connection
-        raise ConnectionNotEstablished unless @automatic_reconnect
-        new_connection
-      end
+        def checkout_new_connection
+          raise ConnectionNotEstablished unless @automatic_reconnect
+          new_connection
+        end
 
-      def checkout_and_verify(c)
-        c._run_checkout_callbacks do
-          c.verify!
+        def checkout_and_verify(c)
+          c._run_checkout_callbacks do
+            c.verify!
+          end
+          c
+        rescue
+          remove c
+          c.disconnect!
+          raise
         end
-        c
-      rescue
-        remove c
-        c.disconnect!
-        raise
-      end
     end
 
     # ConnectionHandler is a collection of ConnectionPool objects. It is used
@@ -830,13 +862,13 @@ module ActiveRecord
     # should use.
     #
     # The ConnectionHandler class is not coupled with the Active models, as it has no knowlodge
-    # about the model. The model, needs to pass a specification name to the handler,
+    # about the model. The model needs to pass a specification name to the handler,
     # in order to lookup the correct connection pool.
     class ConnectionHandler
       def initialize
         # These caches are keyed by spec.name (ConnectionSpecification#name).
-        @owner_to_pool = Concurrent::Map.new(:initial_capacity => 2) do |h,k|
-          h[k] = Concurrent::Map.new(:initial_capacity => 2)
+        @owner_to_pool = Concurrent::Map.new(initial_capacity: 2) do |h, k|
+          h[k] = Concurrent::Map.new(initial_capacity: 2)
         end
       end
 
@@ -845,8 +877,26 @@ module ActiveRecord
       end
       alias :connection_pools :connection_pool_list
 
-      def establish_connection(spec)
-        owner_to_pool[spec.name] = ConnectionAdapters::ConnectionPool.new(spec)
+      def establish_connection(config)
+        resolver = ConnectionSpecification::Resolver.new(Base.configurations)
+        spec = resolver.spec(config)
+
+        remove_connection(spec.name)
+
+        message_bus = ActiveSupport::Notifications.instrumenter
+        payload = {
+          connection_id: object_id
+        }
+        if spec
+          payload[:spec_name] = spec.name
+          payload[:config] = spec.config
+        end
+
+        message_bus.instrument("!connection.active_record", payload) do
+          owner_to_pool[spec.name] = ConnectionAdapters::ConnectionPool.new(spec)
+        end
+
+        owner_to_pool[spec.name]
       end
 
       # Returns true if there are any active connections among the connection
@@ -879,9 +929,9 @@ module ActiveRecord
       # for (not necessarily the current class).
       def retrieve_connection(spec_name) #:nodoc:
         pool = retrieve_connection_pool(spec_name)
-        raise ConnectionNotEstablished, "No connection pool with id #{spec_name} found." unless pool
+        raise ConnectionNotEstablished, "No connection pool with '#{spec_name}' found." unless pool
         conn = pool.connection
-        raise ConnectionNotEstablished, "No connection for #{spec_name} in connection pool" unless conn
+        raise ConnectionNotEstablished, "No connection for '#{spec_name}' in connection pool" unless conn
         conn
       end
 
@@ -894,7 +944,7 @@ module ActiveRecord
 
       # Remove the connection for this class. This will close the active
       # connection and the defined connection (if they exist). The result
-      # can be used as an argument for establish_connection, for easily
+      # can be used as an argument for #establish_connection, for easily
       # re-establishing the connection.
       def remove_connection(spec_name)
         if pool = owner_to_pool.delete(spec_name)
@@ -904,22 +954,18 @@ module ActiveRecord
         end
       end
 
-      # Retrieving the connection pool happens a lot so we cache it in @class_to_pool.
+      # Retrieving the connection pool happens a lot, so we cache it in @owner_to_pool.
       # This makes retrieving the connection pool O(1) once the process is warm.
       # When a connection is established or removed, we invalidate the cache.
-      #
-      # Ideally we would use #fetch here, as class_to_pool[klass] may sometimes be nil.
-      # However, benchmarking (https://gist.github.com/jonleighton/3552829) showed that
-      # #fetch is significantly slower than #[]. So in the nil case, no caching will
-      # take place, but that's ok since the nil case is not the common one that we wish
-      # to optimise for.
       def retrieve_connection_pool(spec_name)
         owner_to_pool.fetch(spec_name) do
+          # Check if a connection was previously established in an ancestor process,
+          # which may have been forked.
           if ancestor_pool = pool_from_any_process_for(spec_name)
             # A connection was established in an ancestor process that must have
             # subsequently forked. We can't reuse the connection, but we can copy
             # the specification and establish a new connection with it.
-            establish_connection(ancestor_pool.spec).tap do |pool|
+            establish_connection(ancestor_pool.spec.to_hash).tap do |pool|
               pool.schema_cache = ancestor_pool.schema_cache if ancestor_pool.schema_cache
             end
           else
@@ -930,14 +976,14 @@ module ActiveRecord
 
       private
 
-      def owner_to_pool
-        @owner_to_pool[Process.pid]
-      end
+        def owner_to_pool
+          @owner_to_pool[Process.pid]
+        end
 
-      def pool_from_any_process_for(spec_name)
-        owner_to_pool = @owner_to_pool.values.reverse.find { |v| v[spec_name] }
-        owner_to_pool && owner_to_pool[spec_name]
-      end
+        def pool_from_any_process_for(spec_name)
+          owner_to_pool = @owner_to_pool.values.reverse.find { |v| v[spec_name] }
+          owner_to_pool && owner_to_pool[spec_name]
+        end
     end
   end
 end