activerecord 5.1.5 → 5.2.8.1

data/lib/active_record/collection_cache_key.rb

@@ -1,7 +1,9 @@
+# frozen_string_literal: true
+
 module ActiveRecord
   module CollectionCacheKey
     def collection_cache_key(collection = all, timestamp_column = :updated_at) # :nodoc:
-      query_signature = Digest::MD5.hexdigest(collection.to_sql)
+      query_signature = ActiveSupport::Digest.hexdigest(collection.to_sql)
       key = "#{collection.model_name.cache_key}/query-#{query_signature}"
 
       if collection.loaded? || collection.distinct_value
@@ -10,24 +12,26 @@ module ActiveRecord
           timestamp = collection.max_by(&timestamp_column)._read_attribute(timestamp_column)
         end
       else
-        column_type = type_for_attribute(timestamp_column.to_s)
-        column = "#{connection.quote_table_name(collection.table_name)}.#{connection.quote_column_name(timestamp_column)}"
+        if collection.eager_loading?
+          collection = collection.send(:apply_join_dependency)
+        end
+        column_type = type_for_attribute(timestamp_column)
+        column = connection.column_name_from_arel_node(collection.arel_attribute(timestamp_column))
         select_values = "COUNT(*) AS #{connection.quote_column_name("size")}, MAX(%s) AS timestamp"
 
         if collection.has_limit_or_offset?
-          query = collection.spawn
-          query.select_values = [column]
+          query = collection.select("#{column} AS collection_cache_key_timestamp")
           subquery_alias = "subquery_for_cache_key"
-          subquery_column = "#{subquery_alias}.#{timestamp_column}"
+          subquery_column = "#{subquery_alias}.collection_cache_key_timestamp"
           subquery = query.arel.as(subquery_alias)
-          arel = Arel::SelectManager.new(query.engine).project(select_values % subquery_column).from(subquery)
+          arel = Arel::SelectManager.new(subquery).project(select_values % subquery_column)
         else
           query = collection.unscope(:order)
           query.select_values = [select_values % column]
           arel = query.arel
         end
 
-        result = connection.select_one(arel, nil, query.bound_attributes)
+        result = connection.select_one(arel, nil)
 
         if result.blank?
           size = 0

data/lib/active_record/connection_adapters/abstract/connection_pool.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "thread"
 require "concurrent/map"
 require "monitor"
@@ -61,15 +63,13 @@ module ActiveRecord
     # There are several connection-pooling-related options that you can add to
     # your database connection configuration:
     #
-    # * +pool+: number indicating size of connection pool (default 5)
-    # * +checkout_timeout+: number of seconds to block and wait for a connection
-    #   before giving up and raising a timeout error (default 5 seconds).
-    # * +reaping_frequency+: frequency in seconds to periodically run the
-    #   Reaper, which attempts to find and recover connections from dead
-    #   threads, which can occur if a programmer forgets to close a
-    #   connection at the end of a thread or a thread dies unexpectedly.
-    #   Regardless of this setting, the Reaper will be invoked before every
-    #   blocking wait. (Default +nil+, which means don't schedule the Reaper).
+    # * +pool+: maximum number of connections the pool may manage (default 5).
+    # * +idle_timeout+: number of seconds that a connection will be kept
+    #   unused in the pool before it is automatically disconnected (default
+    #   300 seconds). Set this to zero to keep connections forever.
+    # * +checkout_timeout+: number of seconds to wait for a connection to
+    #   become available before giving up and raising a timeout error (default
+    #   5 seconds).
     #
     #--
     # Synchronization policy:
@@ -80,11 +80,8 @@ module ActiveRecord
     # * private methods that require being called in a +synchronize+ blocks
     #   are now explicitly documented
     class ConnectionPool
-      # Threadsafe, fair, FIFO queue.  Meant to be used by ConnectionPool
-      # with which it shares a Monitor.  But could be a generic Queue.
-      #
-      # The Queue in stdlib's 'thread' could replace this class except
-      # stdlib's doesn't support waiting with a timeout.
+      # Threadsafe, fair, LIFO queue.  Meant to be used by ConnectionPool
+      # with which it shares a Monitor.
       class Queue
         def initialize(lock = Monitor.new)
           @lock = lock
@@ -173,7 +170,7 @@ module ActiveRecord
 
           # Removes and returns the head of the queue if possible, or +nil+.
           def remove
-            @queue.shift
+            @queue.pop
           end
 
           # Remove and return the head the queue if the number of
@@ -191,7 +188,9 @@ module ActiveRecord
             t0 = Time.now
             elapsed = 0
             loop do
-              @cond.wait(timeout - elapsed)
+              ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
+                @cond.wait(timeout - elapsed)
+              end
 
               return remove if any?
 
@@ -268,7 +267,7 @@ module ActiveRecord
       # Connections must be leased while holding the main pool mutex. This is
       # an internal subclass that also +.leases+ returned connections while
       # still in queue's critical section (queue synchronizes with the same
-      # +@lock+ as the main pool) so that a returned connection is already
+      # <tt>@lock</tt> as the main pool) so that a returned connection is already
       # leased and there is no need to re-enter synchronized block.
       class ConnectionLeasingQueue < Queue # :nodoc:
         include BiasableQueue
@@ -281,12 +280,12 @@ module ActiveRecord
           end
       end
 
-      # Every +frequency+ seconds, the reaper will call +reap+ on +pool+.
-      # A reaper instantiated with a +nil+ frequency will never reap the
-      # connection pool.
+      # Every +frequency+ seconds, the reaper will call +reap+ and +flush+ on
+      # +pool+. A reaper instantiated with a zero frequency will never reap
+      # the connection pool.
       #
-      # Configure the frequency by setting "reaping_frequency" in your
-      # database yaml file.
+      # Configure the frequency by setting +reaping_frequency+ in your database
+      # yaml file (default 60 seconds).
       class Reaper
         attr_reader :pool, :frequency
 
@@ -296,11 +295,12 @@ module ActiveRecord
         end
 
         def run
-          return unless frequency
+          return unless frequency && frequency > 0
           Thread.new(frequency, pool) { |t, p|
             loop do
               sleep t
               p.reap
+              p.flush
             end
           }
         end
@@ -310,7 +310,7 @@ module ActiveRecord
       include QueryCache::ConnectionPoolConfiguration
 
       attr_accessor :automatic_reconnect, :checkout_timeout, :schema_cache
-      attr_reader :spec, :connections, :size, :reaper
+      attr_reader :spec, :size, :reaper
 
       # Creates a new ConnectionPool object. +spec+ is a ConnectionSpecification
       # object which describes database connection information (e.g. adapter,
@@ -324,8 +324,10 @@ module ActiveRecord
         @spec = spec
 
         @checkout_timeout = (spec.config[:checkout_timeout] && spec.config[:checkout_timeout].to_f) || 5
-        @reaper = Reaper.new(self, (spec.config[:reaping_frequency] && spec.config[:reaping_frequency].to_f))
-        @reaper.run
+        if @idle_timeout = spec.config.fetch(:idle_timeout, 300)
+          @idle_timeout = @idle_timeout.to_f
+          @idle_timeout = nil if @idle_timeout <= 0
+        end
 
         # default max pool size to 5
         @size = (spec.config[:pool] && spec.config[:pool].to_i) || 5
@@ -338,7 +340,7 @@ module ActiveRecord
         # then that +thread+ does indeed own that +conn+. However, an absence of a such
         # mapping does not mean that the +thread+ doesn't own the said connection. In
         # that case +conn.owner+ attr should be consulted.
-        # Access and modification of +@thread_cached_conns+ does not require
+        # Access and modification of <tt>@thread_cached_conns</tt> does not require
         # synchronization.
         @thread_cached_conns = Concurrent::Map.new(initial_capacity: @size)
 
@@ -355,6 +357,12 @@ module ActiveRecord
         @available = ConnectionLeasingQueue.new self
 
         @lock_thread = false
+
+        # +reaping_frequency+ is configurable mostly for historical reasons, but it could
+        # also be useful if someone wants a very low +idle_timeout+.
+        reaping_frequency = spec.config.fetch(:reaping_frequency, 60)
+        @reaper = Reaper.new(self, reaping_frequency && reaping_frequency.to_f)
+        @reaper.run
       end
 
       def lock_thread=(lock_thread)
@@ -371,7 +379,7 @@ module ActiveRecord
       # #connection can be called any number of times; the connection is
       # held in a cache keyed by a thread.
       def connection
-        @thread_cached_conns[connection_cache_key(@lock_thread || Thread.current)] ||= checkout
+        @thread_cached_conns[connection_cache_key(current_thread)] ||= checkout
       end
 
       # Returns true if there is an open connection being used for the current thread.
@@ -380,7 +388,7 @@ module ActiveRecord
       # #connection or #with_connection methods. Connections obtained through
       # #checkout will not be detected by #active_connection?
       def active_connection?
-        @thread_cached_conns[connection_cache_key(Thread.current)]
+        @thread_cached_conns[connection_cache_key(current_thread)]
       end
 
       # Signal that the thread is finished with the current connection.
@@ -415,6 +423,21 @@ module ActiveRecord
         synchronize { @connections.any? }
       end
 
+      # Returns an array containing the connections currently in the pool.
+      # Access to the array does not require synchronization on the pool because
+      # the array is newly created and not retained by the pool.
+      #
+      # However; this method bypasses the ConnectionPool's thread-safe connection
+      # access pattern. A returned connection may be owned by another thread,
+      # unowned, or by happen-stance owned by the calling thread.
+      #
+      # Calling methods on a connection without ownership is subject to the
+      # thread-safety guarantees of the underlying method. Many of the methods
+      # on connection adapter classes are inherently multi-thread unsafe.
+      def connections
+        synchronize { @connections.dup }
+      end
+
       # Disconnects all connections in the pool, and clears the pool.
       #
       # Raises:
@@ -447,6 +470,21 @@ module ActiveRecord
         disconnect(false)
       end
 
+      # Discards all connections in the pool (even if they're currently
+      # leased!), along with the pool itself. Any further interaction with the
+      # pool (except #spec and #schema_cache) is undefined.
+      #
+      # See AbstractAdapter#discard!
+      def discard! # :nodoc:
+        synchronize do
+          return if @connections.nil? # already discarded
+          @connections.each do |conn|
+            conn.discard!
+          end
+          @connections = @available = @thread_cached_conns = nil
+        end
+      end
+
       # Clears the cache which maps classes and re-connects connections that
       # require reloading.
       #
@@ -572,6 +610,35 @@ module ActiveRecord
         end
       end
 
+      # Disconnect all connections that have been idle for at least
+      # +minimum_idle+ seconds. Connections currently checked out, or that were
+      # checked in less than +minimum_idle+ seconds ago, are unaffected.
+      def flush(minimum_idle = @idle_timeout)
+        return if minimum_idle.nil?
+
+        idle_connections = synchronize do
+          @connections.select do |conn|
+            !conn.in_use? && conn.seconds_idle >= minimum_idle
+          end.each do |conn|
+            conn.lease
+
+            @available.delete conn
+            @connections.delete conn
+          end
+        end
+
+        idle_connections.each do |conn|
+          conn.disconnect!
+        end
+      end
+
+      # Disconnect all currently idle connections. Connections currently checked
+      # out are unaffected.
+      def flush!
+        reap
+        flush(-1)
+      end
+
       def num_waiting_in_queue # :nodoc:
         @available.num_waiting
       end
@@ -616,6 +683,10 @@ module ActiveRecord
           thread
         end
 
+        def current_thread
+          @lock_thread || Thread.current
+        end
+
         # Take control of all existing connections so a "group" action such as
         # reload/disconnect can be performed safely. It is no longer enough to
         # wrap it in +synchronize+ because some pool's actions are allowed
@@ -679,7 +750,7 @@ module ActiveRecord
           # this block can't be easily moved into attempt_to_checkout_all_existing_connections's
           # rescue block, because doing so would put it outside of synchronize section, without
           # being in a critical section thread_report might become inaccurate
-          msg = "could not obtain ownership of all database connections in #{checkout_timeout} seconds"
+          msg = "could not obtain ownership of all database connections in #{checkout_timeout} seconds".dup
 
           thread_report = []
           @connections.each do |conn|
@@ -734,10 +805,10 @@ module ActiveRecord
         # Implementation detail: the connection returned by +acquire_connection+
         # will already be "+connection.lease+ -ed" to the current thread.
         def acquire_connection(checkout_timeout)
-          # NOTE: we rely on +@available.poll+ and +try_to_checkout_new_connection+ to
+          # NOTE: we rely on <tt>@available.poll</tt> and +try_to_checkout_new_connection+ to
           # +conn.lease+ the returned connection (and to do this in a +synchronized+
           # section). This is not the cleanest implementation, as ideally we would
-          # <tt>synchronize { conn.lease }</tt> in this method, but by leaving it to +@available.poll+
+          # <tt>synchronize { conn.lease }</tt> in this method, but by leaving it to <tt>@available.poll</tt>
           # and +try_to_checkout_new_connection+ we can piggyback on +synchronize+ sections
           # of the said methods and avoid an additional +synchronize+ overhead.
           if conn = @available.poll || try_to_checkout_new_connection
@@ -761,7 +832,7 @@ module ActiveRecord
           end
         end
 
-        # If the pool is not at a +@size+ limit, establish new connection. Connecting
+        # If the pool is not at a <tt>@size</tt> limit, establish new connection. Connecting
         # to the DB is done outside main synchronized section.
         #--
         # Implementation constraint: a newly established connection returned by this
@@ -827,7 +898,7 @@ module ActiveRecord
     #   end
     #
     #   class Book < ActiveRecord::Base
-    #     establish_connection "library_db"
+    #     establish_connection :library_db
     #   end
     #
     #   class ScaryBook < Book
@@ -859,17 +930,41 @@ module ActiveRecord
     # All Active Record models use this handler to determine the connection pool that they
     # should use.
     #
-    # The ConnectionHandler class is not coupled with the Active models, as it has no knowlodge
+    # The ConnectionHandler class is not coupled with the Active models, as it has no knowledge
     # about the model. The model needs to pass a specification name to the handler,
-    # in order to lookup the correct connection pool.
+    # in order to look up the correct connection pool.
     class ConnectionHandler
-      def initialize
-        # These caches are keyed by spec.name (ConnectionSpecification#name).
-        @owner_to_pool = Concurrent::Map.new(initial_capacity: 2) do |h, k|
+      def self.create_owner_to_pool # :nodoc:
+        Concurrent::Map.new(initial_capacity: 2) do |h, k|
+          # Discard the parent's connection pools immediately; we have no need
+          # of them
+          discard_unowned_pools(h)
+
           h[k] = Concurrent::Map.new(initial_capacity: 2)
         end
       end
 
+      def self.unowned_pool_finalizer(pid_map) # :nodoc:
+        lambda do |_|
+          discard_unowned_pools(pid_map)
+        end
+      end
+
+      def self.discard_unowned_pools(pid_map) # :nodoc:
+        pid_map.each do |pid, pools|
+          pools.values.compact.each(&:discard!) unless pid == Process.pid
+        end
+      end
+
+      def initialize
+        # These caches are keyed by spec.name (ConnectionSpecification#name).
+        @owner_to_pool = ConnectionHandler.create_owner_to_pool
+
+        # Backup finalizer: if the forked child never needed a pool, the above
+        # early discard has not occurred
+        ObjectSpace.define_finalizer self, ConnectionHandler.unowned_pool_finalizer(@owner_to_pool)
+      end
+
       def connection_pool_list
         owner_to_pool.values.compact
       end
@@ -921,6 +1016,13 @@ module ActiveRecord
         connection_pool_list.each(&:disconnect!)
       end
 
+      # Disconnects all currently idle connections.
+      #
+      # See ConnectionPool#flush! for details.
+      def flush_idle_connections!
+        connection_pool_list.each(&:flush!)
+      end
+
       # Locate the connection of the nearest super class. This can be an
       # active or defined connection: if it is the latter, it will be
       # opened and set as the active connection for the class it was defined
@@ -928,9 +1030,7 @@ module ActiveRecord
       def retrieve_connection(spec_name) #:nodoc:
         pool = retrieve_connection_pool(spec_name)
         raise ConnectionNotEstablished, "No connection pool with '#{spec_name}' found." unless pool
-        conn = pool.connection
-        raise ConnectionNotEstablished, "No connection for '#{spec_name}' in connection pool" unless conn
-        conn
+        pool.connection
       end
 
       # Returns true if a connection that's accessible to this class has

data/lib/active_record/connection_adapters/abstract/database_limits.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ActiveRecord
   module ConnectionAdapters # :nodoc:
     module DatabaseLimits
@@ -60,6 +62,11 @@ module ActiveRecord
       def joins_per_query
         256
       end
+
+      private
+        def bind_params_length
+          65535
+        end
     end
   end
 end