activerecord 5.1.0 → 5.2.3

data/lib/active_record/connection_adapters/abstract/connection_pool.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "thread"
 require "concurrent/map"
 require "monitor"
@@ -61,15 +63,13 @@ module ActiveRecord
     # There are several connection-pooling-related options that you can add to
     # your database connection configuration:
     #
-    # * +pool+: number indicating size of connection pool (default 5)
-    # * +checkout_timeout+: number of seconds to block and wait for a connection
-    #   before giving up and raising a timeout error (default 5 seconds).
-    # * +reaping_frequency+: frequency in seconds to periodically run the
-    #   Reaper, which attempts to find and recover connections from dead
-    #   threads, which can occur if a programmer forgets to close a
-    #   connection at the end of a thread or a thread dies unexpectedly.
-    #   Regardless of this setting, the Reaper will be invoked before every
-    #   blocking wait. (Default +nil+, which means don't schedule the Reaper).
+    # * +pool+: maximum number of connections the pool may manage (default 5).
+    # * +idle_timeout+: number of seconds that a connection will be kept
+    #   unused in the pool before it is automatically disconnected (default
+    #   300 seconds). Set this to zero to keep connections forever.
+    # * +checkout_timeout+: number of seconds to wait for a connection to
+    #   become available before giving up and raising a timeout error (default
+    #   5 seconds).
     #
     #--
     # Synchronization policy:
@@ -80,11 +80,8 @@ module ActiveRecord
     # * private methods that require being called in a +synchronize+ blocks
     #   are now explicitly documented
     class ConnectionPool
-      # Threadsafe, fair, FIFO queue.  Meant to be used by ConnectionPool
-      # with which it shares a Monitor.  But could be a generic Queue.
-      #
-      # The Queue in stdlib's 'thread' could replace this class except
-      # stdlib's doesn't support waiting with a timeout.
+      # Threadsafe, fair, LIFO queue.  Meant to be used by ConnectionPool
+      # with which it shares a Monitor.
       class Queue
         def initialize(lock = Monitor.new)
           @lock = lock
@@ -173,7 +170,7 @@ module ActiveRecord
 
           # Removes and returns the head of the queue if possible, or +nil+.
           def remove
-            @queue.shift
+            @queue.pop
           end
 
           # Remove and return the head the queue if the number of
@@ -191,7 +188,9 @@ module ActiveRecord
             t0 = Time.now
             elapsed = 0
             loop do
-              @cond.wait(timeout - elapsed)
+              ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
+                @cond.wait(timeout - elapsed)
+              end
 
               return remove if any?
 
@@ -268,7 +267,7 @@ module ActiveRecord
       # Connections must be leased while holding the main pool mutex. This is
       # an internal subclass that also +.leases+ returned connections while
       # still in queue's critical section (queue synchronizes with the same
-      # +@lock+ as the main pool) so that a returned connection is already
+      # <tt>@lock</tt> as the main pool) so that a returned connection is already
       # leased and there is no need to re-enter synchronized block.
       class ConnectionLeasingQueue < Queue # :nodoc:
         include BiasableQueue
@@ -281,12 +280,12 @@ module ActiveRecord
           end
       end
 
-      # Every +frequency+ seconds, the reaper will call +reap+ on +pool+.
-      # A reaper instantiated with a +nil+ frequency will never reap the
-      # connection pool.
+      # Every +frequency+ seconds, the reaper will call +reap+ and +flush+ on
+      # +pool+. A reaper instantiated with a zero frequency will never reap
+      # the connection pool.
       #
-      # Configure the frequency by setting "reaping_frequency" in your
-      # database yaml file.
+      # Configure the frequency by setting +reaping_frequency+ in your database
+      # yaml file (default 60 seconds).
       class Reaper
         attr_reader :pool, :frequency
 
@@ -296,11 +295,12 @@ module ActiveRecord
         end
 
         def run
-          return unless frequency
+          return unless frequency && frequency > 0
           Thread.new(frequency, pool) { |t, p|
             loop do
               sleep t
               p.reap
+              p.flush
             end
           }
         end
@@ -324,8 +324,10 @@ module ActiveRecord
         @spec = spec
 
         @checkout_timeout = (spec.config[:checkout_timeout] && spec.config[:checkout_timeout].to_f) || 5
-        @reaper = Reaper.new(self, (spec.config[:reaping_frequency] && spec.config[:reaping_frequency].to_f))
-        @reaper.run
+        if @idle_timeout = spec.config.fetch(:idle_timeout, 300)
+          @idle_timeout = @idle_timeout.to_f
+          @idle_timeout = nil if @idle_timeout <= 0
+        end
 
         # default max pool size to 5
         @size = (spec.config[:pool] && spec.config[:pool].to_i) || 5
@@ -338,7 +340,7 @@ module ActiveRecord
         # then that +thread+ does indeed own that +conn+. However, an absence of a such
         # mapping does not mean that the +thread+ doesn't own the said connection. In
         # that case +conn.owner+ attr should be consulted.
-        # Access and modification of +@thread_cached_conns+ does not require
+        # Access and modification of <tt>@thread_cached_conns</tt> does not require
         # synchronization.
         @thread_cached_conns = Concurrent::Map.new(initial_capacity: @size)
 
@@ -355,6 +357,12 @@ module ActiveRecord
         @available = ConnectionLeasingQueue.new self
 
         @lock_thread = false
+
+        # +reaping_frequency+ is configurable mostly for historical reasons, but it could
+        # also be useful if someone wants a very low +idle_timeout+.
+        reaping_frequency = spec.config.fetch(:reaping_frequency, 60)
+        @reaper = Reaper.new(self, reaping_frequency && reaping_frequency.to_f)
+        @reaper.run
       end
 
       def lock_thread=(lock_thread)
@@ -447,6 +455,21 @@ module ActiveRecord
         disconnect(false)
       end
 
+      # Discards all connections in the pool (even if they're currently
+      # leased!), along with the pool itself. Any further interaction with the
+      # pool (except #spec and #schema_cache) is undefined.
+      #
+      # See AbstractAdapter#discard!
+      def discard! # :nodoc:
+        synchronize do
+          return if @connections.nil? # already discarded
+          @connections.each do |conn|
+            conn.discard!
+          end
+          @connections = @available = @thread_cached_conns = nil
+        end
+      end
+
       # Clears the cache which maps classes and re-connects connections that
       # require reloading.
       #
@@ -572,6 +595,35 @@ module ActiveRecord
         end
       end
 
+      # Disconnect all connections that have been idle for at least
+      # +minimum_idle+ seconds. Connections currently checked out, or that were
+      # checked in less than +minimum_idle+ seconds ago, are unaffected.
+      def flush(minimum_idle = @idle_timeout)
+        return if minimum_idle.nil?
+
+        idle_connections = synchronize do
+          @connections.select do |conn|
+            !conn.in_use? && conn.seconds_idle >= minimum_idle
+          end.each do |conn|
+            conn.lease
+
+            @available.delete conn
+            @connections.delete conn
+          end
+        end
+
+        idle_connections.each do |conn|
+          conn.disconnect!
+        end
+      end
+
+      # Disconnect all currently idle connections. Connections currently checked
+      # out are unaffected.
+      def flush!
+        reap
+        flush(-1)
+      end
+
       def num_waiting_in_queue # :nodoc:
         @available.num_waiting
       end
@@ -679,7 +731,7 @@ module ActiveRecord
           # this block can't be easily moved into attempt_to_checkout_all_existing_connections's
           # rescue block, because doing so would put it outside of synchronize section, without
           # being in a critical section thread_report might become inaccurate
-          msg = "could not obtain ownership of all database connections in #{checkout_timeout} seconds"
+          msg = "could not obtain ownership of all database connections in #{checkout_timeout} seconds".dup
 
           thread_report = []
           @connections.each do |conn|
@@ -734,10 +786,10 @@ module ActiveRecord
         # Implementation detail: the connection returned by +acquire_connection+
         # will already be "+connection.lease+ -ed" to the current thread.
         def acquire_connection(checkout_timeout)
-          # NOTE: we rely on +@available.poll+ and +try_to_checkout_new_connection+ to
+          # NOTE: we rely on <tt>@available.poll</tt> and +try_to_checkout_new_connection+ to
           # +conn.lease+ the returned connection (and to do this in a +synchronized+
           # section). This is not the cleanest implementation, as ideally we would
-          # <tt>synchronize { conn.lease }</tt> in this method, but by leaving it to +@available.poll+
+          # <tt>synchronize { conn.lease }</tt> in this method, but by leaving it to <tt>@available.poll</tt>
           # and +try_to_checkout_new_connection+ we can piggyback on +synchronize+ sections
           # of the said methods and avoid an additional +synchronize+ overhead.
           if conn = @available.poll || try_to_checkout_new_connection
@@ -761,7 +813,7 @@ module ActiveRecord
           end
         end
 
-        # If the pool is not at a +@size+ limit, establish new connection. Connecting
+        # If the pool is not at a <tt>@size</tt> limit, establish new connection. Connecting
         # to the DB is done outside main synchronized section.
         #--
         # Implementation constraint: a newly established connection returned by this
@@ -827,7 +879,7 @@ module ActiveRecord
     #   end
     #
     #   class Book < ActiveRecord::Base
-    #     establish_connection "library_db"
+    #     establish_connection :library_db
     #   end
     #
     #   class ScaryBook < Book
@@ -859,17 +911,41 @@ module ActiveRecord
     # All Active Record models use this handler to determine the connection pool that they
     # should use.
     #
-    # The ConnectionHandler class is not coupled with the Active models, as it has no knowlodge
+    # The ConnectionHandler class is not coupled with the Active models, as it has no knowledge
     # about the model. The model needs to pass a specification name to the handler,
-    # in order to lookup the correct connection pool.
+    # in order to look up the correct connection pool.
     class ConnectionHandler
-      def initialize
-        # These caches are keyed by spec.name (ConnectionSpecification#name).
-        @owner_to_pool = Concurrent::Map.new(initial_capacity: 2) do |h, k|
+      def self.create_owner_to_pool # :nodoc:
+        Concurrent::Map.new(initial_capacity: 2) do |h, k|
+          # Discard the parent's connection pools immediately; we have no need
+          # of them
+          discard_unowned_pools(h)
+
           h[k] = Concurrent::Map.new(initial_capacity: 2)
         end
       end
 
+      def self.unowned_pool_finalizer(pid_map) # :nodoc:
+        lambda do |_|
+          discard_unowned_pools(pid_map)
+        end
+      end
+
+      def self.discard_unowned_pools(pid_map) # :nodoc:
+        pid_map.each do |pid, pools|
+          pools.values.compact.each(&:discard!) unless pid == Process.pid
+        end
+      end
+
+      def initialize
+        # These caches are keyed by spec.name (ConnectionSpecification#name).
+        @owner_to_pool = ConnectionHandler.create_owner_to_pool
+
+        # Backup finalizer: if the forked child never needed a pool, the above
+        # early discard has not occurred
+        ObjectSpace.define_finalizer self, ConnectionHandler.unowned_pool_finalizer(@owner_to_pool)
+      end
+
       def connection_pool_list
         owner_to_pool.values.compact
       end
@@ -921,6 +997,13 @@ module ActiveRecord
         connection_pool_list.each(&:disconnect!)
       end
 
+      # Disconnects all currently idle connections.
+      #
+      # See ConnectionPool#flush! for details.
+      def flush_idle_connections!
+        connection_pool_list.each(&:flush!)
+      end
+
       # Locate the connection of the nearest super class. This can be an
       # active or defined connection: if it is the latter, it will be
       # opened and set as the active connection for the class it was defined
@@ -928,9 +1011,7 @@ module ActiveRecord
       def retrieve_connection(spec_name) #:nodoc:
         pool = retrieve_connection_pool(spec_name)
         raise ConnectionNotEstablished, "No connection pool with '#{spec_name}' found." unless pool
-        conn = pool.connection
-        raise ConnectionNotEstablished, "No connection for '#{spec_name}' in connection pool" unless conn
-        conn
+        pool.connection
       end
 
       # Returns true if a connection that's accessible to this class has

data/lib/active_record/connection_adapters/abstract/database_limits.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ActiveRecord
   module ConnectionAdapters # :nodoc:
     module DatabaseLimits
@@ -60,6 +62,11 @@ module ActiveRecord
       def joins_per_query
         256
       end
+
+      private
+        def bind_params_length
+          65535
+        end
     end
   end
 end

data/lib/active_record/connection_adapters/abstract/database_statements.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ActiveRecord
   module ConnectionAdapters # :nodoc:
     module DatabaseStatements
@@ -7,35 +9,61 @@ module ActiveRecord
       end
 
       # Converts an arel AST to SQL
-      def to_sql(arel, binds = [])
-        if arel.respond_to?(:ast)
-          collected = visitor.accept(arel.ast, collector)
-          collected.compile(binds, self).freeze
+      def to_sql(arel_or_sql_string, binds = [])
+        sql, _ = to_sql_and_binds(arel_or_sql_string, binds)
+        sql
+      end
+
+      def to_sql_and_binds(arel_or_sql_string, binds = []) # :nodoc:
+        if arel_or_sql_string.respond_to?(:ast)
+          unless binds.empty?
+            raise "Passing bind parameters with an arel AST is forbidden. " \
+              "The values must be stored on the AST directly"
+          end
+
+          if prepared_statements
+            sql, binds = visitor.accept(arel_or_sql_string.ast, collector).value
+
+            if binds.length > bind_params_length
+              unprepared_statement do
+                sql, binds = to_sql_and_binds(arel_or_sql_string)
+                visitor.preparable = false
+              end
+            end
+          else
+            sql = visitor.accept(arel_or_sql_string.ast, collector).value
+          end
+          [sql.freeze, binds]
         else
-          arel.dup.freeze
+          visitor.preparable = false if prepared_statements
+          [arel_or_sql_string.dup.freeze, binds]
         end
       end
+      private :to_sql_and_binds
 
       # This is used in the StatementCache object. It returns an object that
       # can be used to query the database repeatedly.
       def cacheable_query(klass, arel) # :nodoc:
-        collected = visitor.accept(arel.ast, collector)
         if prepared_statements
-          klass.query(collected.value)
+          sql, binds = visitor.accept(arel.ast, collector).value
+          query = klass.query(sql)
         else
-          klass.partial_query(collected.value)
+          collector = PartialQueryCollector.new
+          parts, binds = visitor.accept(arel.ast, collector).value
+          query = klass.partial_query(parts)
         end
+        [query, binds]
       end
 
       # Returns an ActiveRecord::Result instance.
       def select_all(arel, name = nil, binds = [], preparable: nil)
-        arel, binds = binds_from_relation arel, binds
-        sql = to_sql(arel, binds)
-        if !prepared_statements || (arel.is_a?(String) && preparable.nil?)
-          preparable = false
-        else
-          preparable = visitor.preparable
+        arel = arel_from_relation(arel)
+        sql, binds = to_sql_and_binds(arel, binds)
+
+        if preparable.nil?
+          preparable = prepared_statements ? visitor.preparable : false
         end
+
         if prepared_statements && preparable
           select_prepared(sql, name, binds)
         else
@@ -51,9 +79,7 @@ module ActiveRecord
 
       # Returns a single value from a record
       def select_value(arel, name = nil, binds = [])
-        if result = select_rows(arel, name, binds).first
-          result.first
-        end
+        single_value_from_rows(select_rows(arel, name, binds))
       end
 
       # Returns an array of the values of the first column in a select:
@@ -68,6 +94,18 @@ module ActiveRecord
         select_all(arel, name, binds).rows
       end
 
+      def query_value(sql, name = nil) # :nodoc:
+        single_value_from_rows(query(sql, name))
+      end
+
+      def query_values(sql, name = nil) # :nodoc:
+        query(sql, name).map(&:first)
+      end
+
+      def query(sql, name = nil) # :nodoc:
+        exec_query(sql, name).rows
+      end
+
       # Executes the SQL statement in the context of this connection and returns
       # the raw result from the connection adapter.
       # Note: depending on your database connector, the result returned by this
@@ -120,26 +158,30 @@ module ActiveRecord
       # If the next id was calculated in advance (as in Oracle), it should be
       # passed in as +id_value+.
       def insert(arel, name = nil, pk = nil, id_value = nil, sequence_name = nil, binds = [])
-        value = exec_insert(to_sql(arel, binds), name, binds, pk, sequence_name)
+        sql, binds = to_sql_and_binds(arel, binds)
+        value = exec_insert(sql, name, binds, pk, sequence_name)
         id_value || last_inserted_id(value)
       end
       alias create insert
 
       # Executes the update statement and returns the number of rows affected.
       def update(arel, name = nil, binds = [])
-        exec_update(to_sql(arel, binds), name, binds)
+        sql, binds = to_sql_and_binds(arel, binds)
+        exec_update(sql, name, binds)
       end
 
       # Executes the delete statement and returns the number of rows affected.
       def delete(arel, name = nil, binds = [])
-        exec_delete(to_sql(arel, binds), name, binds)
+        sql, binds = to_sql_and_binds(arel, binds)
+        exec_delete(sql, name, binds)
       end
 
       # Returns +true+ when the connection adapter supports prepared statement
       # caching, otherwise returns +false+
-      def supports_statement_cache?
-        false
+      def supports_statement_cache? # :nodoc:
+        true
       end
+      deprecate :supports_statement_cache?
 
       # Runs the given block in a database transaction, and returns the result
       # of the block.
@@ -152,7 +194,7 @@ module ActiveRecord
       #
       # In order to get around this problem, #transaction will emulate the effect
       # of nested transactions, by using savepoints:
-      # http://dev.mysql.com/doc/refman/5.7/en/savepoint.html
+      # https://dev.mysql.com/doc/refman/5.7/en/savepoint.html
       # Savepoints are supported by MySQL and PostgreSQL. SQLite3 version >= '3.6.8'
       # supports savepoints.
       #
@@ -204,7 +246,7 @@ module ActiveRecord
       # You should consult the documentation for your database to understand the
       # semantics of these different levels:
       #
-      # * http://www.postgresql.org/docs/current/static/transaction-iso.html
+      # * https://www.postgresql.org/docs/current/static/transaction-iso.html
       # * https://dev.mysql.com/doc/refman/5.7/en/set-transaction.html
       #
       # An ActiveRecord::TransactionIsolationError will be raised if:
@@ -295,6 +337,9 @@ module ActiveRecord
 
       # Inserts the given fixture into the table. Overridden in adapters that require
       # something beyond a simple insert (eg. Oracle).
+      # Most of adapters should implement `insert_fixtures` that leverages bulk SQL insert.
+      # We keep this method to provide fallback
+      # for databases like sqlite that do not support bulk inserts.
       def insert_fixture(fixture, table_name)
         fixture = fixture.stringify_keys
 
@@ -307,16 +352,50 @@ module ActiveRecord
             raise Fixture::FixtureError, %(table "#{table_name}" has no column named #{name.inspect}.)
           end
         end
-        key_list = fixture.keys.map { |name| quote_column_name(name) }
-        value_list = binds.map(&:value_for_database).map do |value|
-          begin
-            quote(value)
-          rescue TypeError
-            quote(YAML.dump(value))
-          end
+
+        table = Arel::Table.new(table_name)
+
+        values = binds.map do |bind|
+          value = with_yaml_fallback(bind.value_for_database)
+          [table[bind.name], value]
         end
 
-        execute "INSERT INTO #{quote_table_name(table_name)} (#{key_list.join(', ')}) VALUES (#{value_list.join(', ')})", "Fixture Insert"
+        manager = Arel::InsertManager.new
+        manager.into(table)
+        manager.insert(values)
+        execute manager.to_sql, "Fixture Insert"
+      end
+
+      # Inserts a set of fixtures into the table. Overridden in adapters that require
+      # something beyond a simple insert (eg. Oracle).
+      def insert_fixtures(fixtures, table_name)
+        ActiveSupport::Deprecation.warn(<<-MSG.squish)
+          `insert_fixtures` is deprecated and will be removed in the next version of Rails.
+          Consider using `insert_fixtures_set` for performance improvement.
+        MSG
+        return if fixtures.empty?
+
+        execute(build_fixture_sql(fixtures, table_name), "Fixtures Insert")
+      end
+
+      def insert_fixtures_set(fixture_set, tables_to_delete = [])
+        fixture_inserts = fixture_set.map do |table_name, fixtures|
+          next if fixtures.empty?
+
+          build_fixture_sql(fixtures, table_name)
+        end.compact
+
+        table_deletes = tables_to_delete.map { |table| "DELETE FROM #{quote_table_name table}".dup }
+        total_sql = Array.wrap(combine_multi_statements(table_deletes + fixture_inserts))
+
+        disable_referential_integrity do
+          transaction(requires_new: true) do
+            total_sql.each do |sql|
+              execute sql, "Fixtures Load"
+              yield if block_given?
+            end
+          end
+        end
       end
 
       def empty_insert_statement_value
@@ -348,6 +427,44 @@ module ActiveRecord
       alias join_to_delete join_to_update
 
       private
+        def default_insert_value(column)
+          Arel.sql("DEFAULT")
+        end
+
+        def build_fixture_sql(fixtures, table_name)
+          columns = schema_cache.columns_hash(table_name)
+
+          values = fixtures.map do |fixture|
+            fixture = fixture.stringify_keys
+
+            unknown_columns = fixture.keys - columns.keys
+            if unknown_columns.any?
+              raise Fixture::FixtureError, %(table "#{table_name}" has no columns named #{unknown_columns.map(&:inspect).join(', ')}.)
+            end
+
+            columns.map do |name, column|
+              if fixture.key?(name)
+                type = lookup_cast_type_from_column(column)
+                bind = Relation::QueryAttribute.new(name, fixture[name], type)
+                with_yaml_fallback(bind.value_for_database)
+              else
+                default_insert_value(column)
+              end
+            end
+          end
+
+          table = Arel::Table.new(table_name)
+          manager = Arel::InsertManager.new
+          manager.into(table)
+          columns.each_key { |column| manager.columns << table[column] }
+          manager.values = manager.create_values_list(values)
+
+          manager.to_sql
+        end
+
+        def combine_multi_statements(total_sql)
+          total_sql.join(";\n")
+        end
 
         # Returns a subquery for the given key using the join information.
         def subquery_for(key, select)
@@ -370,15 +487,53 @@ module ActiveRecord
         end
 
         def last_inserted_id(result)
-          row = result.rows.first
+          single_value_from_rows(result.rows)
+        end
+
+        def single_value_from_rows(rows)
+          row = rows.first
           row && row.first
         end
 
-        def binds_from_relation(relation, binds)
-          if relation.is_a?(Relation) && binds.empty?
-            relation, binds = relation.arel, relation.bound_attributes
+        def arel_from_relation(relation)
+          if relation.is_a?(Relation)
+            relation.arel
+          else
+            relation
+          end
+        end
+
+        # Fixture value is quoted by Arel, however scalar values
+        # are not quotable. In this case we want to convert
+        # the column value to YAML.
+        def with_yaml_fallback(value)
+          if value.is_a?(Hash) || value.is_a?(Array)
+            YAML.dump(value)
+          else
+            value
+          end
+        end
+
+        class PartialQueryCollector
+          def initialize
+            @parts = []
+            @binds = []
+          end
+
+          def <<(str)
+            @parts << str
+            self
+          end
+
+          def add_bind(obj)
+            @binds << obj
+            @parts << Arel::Nodes::BindParam.new(1)
+            self
+          end
+
+          def value
+            [@parts, @binds]
           end
-          [relation, binds]
         end
     end
   end