activerecord 7.0.8.7 → 7.1.0.beta1

data/lib/active_record/connection_adapters/schema_cache.rb

@@ -4,8 +4,234 @@ require "active_support/core_ext/file/atomic"
 
 module ActiveRecord
   module ConnectionAdapters
+    class SchemaReflection
+      class << self
+        attr_accessor :use_schema_cache_dump
+        attr_accessor :check_schema_cache_dump_version
+      end
+
+      self.use_schema_cache_dump = true
+      self.check_schema_cache_dump_version = true
+
+      def initialize(cache_path, cache = nil)
+        @cache = cache
+        @cache_path = cache_path
+      end
+
+      def set_schema_cache(cache)
+        @cache = cache
+      end
+
+      def clear!
+        @cache = empty_cache
+
+        nil
+      end
+
+      def load!(connection)
+        cache(connection)
+
+        self
+      end
+
+      def primary_keys(connection, table_name)
+        cache(connection).primary_keys(connection, table_name)
+      end
+
+      def data_source_exists?(connection, name)
+        cache(connection).data_source_exists?(connection, name)
+      end
+
+      def add(connection, name)
+        cache(connection).add(connection, name)
+      end
+
+      def data_sources(connection, name)
+        cache(connection).data_sources(connection, name)
+      end
+
+      def columns(connection, table_name)
+        cache(connection).columns(connection, table_name)
+      end
+
+      def columns_hash(connection, table_name)
+        cache(connection).columns_hash(connection, table_name)
+      end
+
+      def columns_hash?(connection, table_name)
+        cache(connection).columns_hash?(connection, table_name)
+      end
+
+      def indexes(connection, table_name)
+        cache(connection).indexes(connection, table_name)
+      end
+
+      def database_version(connection)
+        cache(connection).database_version(connection)
+      end
+
+      def version(connection)
+        cache(connection).version(connection)
+      end
+
+      def size(connection)
+        cache(connection).size
+      end
+
+      def clear_data_source_cache!(connection, name)
+        return if @cache.nil? && !possible_cache_available?
+
+        cache(connection).clear_data_source_cache!(connection, name)
+      end
+
+      def cached?(table_name)
+        if @cache.nil?
+          # If `check_schema_cache_dump_version` is enabled we can't load
+          # the schema cache dump without connecting to the database.
+          unless self.class.check_schema_cache_dump_version
+            @cache = load_cache(nil)
+          end
+        end
+
+        @cache&.cached?(table_name)
+      end
+
+      def dump_to(connection, filename)
+        fresh_cache = empty_cache
+        fresh_cache.add_all(connection)
+        fresh_cache.dump_to(filename)
+
+        @cache = fresh_cache
+      end
+
+      private
+        def empty_cache
+          new_cache = SchemaCache.allocate
+          new_cache.send(:initialize)
+          new_cache
+        end
+
+        def cache(connection)
+          @cache ||= load_cache(connection) || empty_cache
+        end
+
+        def possible_cache_available?
+          self.class.use_schema_cache_dump &&
+            @cache_path &&
+            File.file?(@cache_path)
+        end
+
+        def load_cache(connection)
+          # Can't load if schema dumps are disabled
+          return unless possible_cache_available?
+
+          # Check we can find one
+          return unless new_cache = SchemaCache._load_from(@cache_path)
+
+          if self.class.check_schema_cache_dump_version
+            begin
+              current_version = connection.schema_version
+
+              if new_cache.version(connection) != current_version
+                warn "Ignoring #{@cache_path} because it has expired. The current schema version is #{current_version}, but the one in the schema cache file is #{new_cache.schema_version}."
+                return
+              end
+            rescue ActiveRecordError => error
+              warn "Failed to validate the schema cache because of #{error.class}: #{error.message}"
+              return
+            end
+          end
+
+          new_cache
+        end
+    end
+
+    class BoundSchemaReflection
+      def initialize(abstract_schema_reflection, connection)
+        @schema_reflection = abstract_schema_reflection
+        @connection = connection
+      end
+
+      def clear!
+        @schema_reflection.clear!
+      end
+
+      def load!
+        @schema_reflection.load!(@connection)
+      end
+
+      def cached?(table_name)
+        @schema_reflection.cached?(table_name)
+      end
+
+      def primary_keys(table_name)
+        @schema_reflection.primary_keys(@connection, table_name)
+      end
+
+      def data_source_exists?(name)
+        @schema_reflection.data_source_exists?(@connection, name)
+      end
+
+      def add(name)
+        @schema_reflection.add(@connection, name)
+      end
+
+      def data_sources(name)
+        @schema_reflection.data_sources(@connection, name)
+      end
+
+      def columns(table_name)
+        @schema_reflection.columns(@connection, table_name)
+      end
+
+      def columns_hash(table_name)
+        @schema_reflection.columns_hash(@connection, table_name)
+      end
+
+      def columns_hash?(table_name)
+        @schema_reflection.columns_hash?(@connection, table_name)
+      end
+
+      def indexes(table_name)
+        @schema_reflection.indexes(@connection, table_name)
+      end
+
+      def database_version
+        @schema_reflection.database_version(@connection)
+      end
+
+      def version
+        @schema_reflection.version(@connection)
+      end
+
+      def size
+        @schema_reflection.size(@connection)
+      end
+
+      def clear_data_source_cache!(name)
+        @schema_reflection.clear_data_source_cache!(@connection, name)
+      end
+
+      def dump_to(filename)
+        @schema_reflection.dump_to(@connection, filename)
+      end
+    end
+
+    # = Active Record Connection Adapters Schema Cache
     class SchemaCache
-      def self.load_from(filename)
+      class << self
+        def new(connection)
+          BoundSchemaReflection.new(SchemaReflection.new(nil), connection)
+        end
+        deprecate new: "use ActiveRecord::ConnectionAdapters::SchemaReflection instead", deprecator: ActiveRecord.deprecator
+
+        def load_from(filename) # :nodoc:
+          BoundSchemaReflection.new(SchemaReflection.new(filename), nil)
+        end
+        deprecate load_from: "use ActiveRecord::ConnectionAdapters::SchemaReflection instead", deprecator: ActiveRecord.deprecator
+      end
+
+      def self._load_from(filename) # :nodoc:
         return unless File.file?(filename)
 
         read(filename) do |file|
@@ -32,20 +258,17 @@ module ActiveRecord
       end
       private_class_method :read
 
-      attr_reader :version
-      attr_accessor :connection
-
-      def initialize(conn)
-        @connection = conn
-
+      def initialize
         @columns      = {}
         @columns_hash = {}
         @primary_keys = {}
         @data_sources = {}
         @indexes      = {}
+        @database_version = nil
+        @version = nil
       end
 
-      def initialize_dup(other)
+      def initialize_dup(other) # :nodoc:
         super
         @columns      = @columns.dup
         @columns_hash = @columns_hash.dup
@@ -54,61 +277,67 @@ module ActiveRecord
         @indexes      = @indexes.dup
       end
 
-      def encode_with(coder)
-        reset_version!
-
-        coder["columns"]          = @columns
-        coder["primary_keys"]     = @primary_keys
-        coder["data_sources"]     = @data_sources
-        coder["indexes"]          = @indexes
+      def encode_with(coder) # :nodoc:
+        coder["columns"]          = @columns.sort.to_h
+        coder["primary_keys"]     = @primary_keys.sort.to_h
+        coder["data_sources"]     = @data_sources.sort.to_h
+        coder["indexes"]          = @indexes.sort.to_h
         coder["version"]          = @version
-        coder["database_version"] = database_version
+        coder["database_version"] = @database_version
       end
 
       def init_with(coder)
         @columns          = coder["columns"]
+        @columns_hash     = coder["columns_hash"]
         @primary_keys     = coder["primary_keys"]
         @data_sources     = coder["data_sources"]
         @indexes          = coder["indexes"] || {}
         @version          = coder["version"]
         @database_version = coder["database_version"]
 
-        derive_columns_hash_and_deduplicate_values
+        unless coder["deduplicated"]
+          derive_columns_hash_and_deduplicate_values
+        end
       end
 
-      def primary_keys(table_name)
+      def cached?(table_name)
+        @columns.key?(table_name)
+      end
+
+      def primary_keys(connection, table_name)
         @primary_keys.fetch(table_name) do
-          if data_source_exists?(table_name)
+          if data_source_exists?(connection, table_name)
             @primary_keys[deep_deduplicate(table_name)] = deep_deduplicate(connection.primary_key(table_name))
           end
         end
       end
 
       # A cached lookup for table existence.
-      def data_source_exists?(name)
+      def data_source_exists?(connection, name)
         return if ignored_table?(name)
-        prepare_data_sources if @data_sources.empty?
+        prepare_data_sources(connection) if @data_sources.empty?
         return @data_sources[name] if @data_sources.key? name
 
         @data_sources[deep_deduplicate(name)] = connection.data_source_exists?(name)
       end
 
       # Add internal cache for table with +table_name+.
-      def add(table_name)
-        if data_source_exists?(table_name)
-          primary_keys(table_name)
-          columns(table_name)
-          columns_hash(table_name)
-          indexes(table_name)
+      def add(connection, table_name)
+        if data_source_exists?(connection, table_name)
+          primary_keys(connection, table_name)
+          columns(connection, table_name)
+          columns_hash(connection, table_name)
+          indexes(connection, table_name)
         end
       end
 
-      def data_sources(name)
+      def data_sources(_connection, name) # :nodoc:
         @data_sources[name]
       end
+      deprecate data_sources: :data_source_exists?, deprecator: ActiveRecord.deprecator
 
       # Get the columns for a table
-      def columns(table_name)
+      def columns(connection, table_name)
         if ignored_table?(table_name)
           raise ActiveRecord::StatementInvalid, "Table '#{table_name}' doesn't exist"
         end
@@ -120,20 +349,20 @@ module ActiveRecord
 
       # Get the columns for a table as a hash, key is the column name
       # value is the column object.
-      def columns_hash(table_name)
+      def columns_hash(connection, table_name)
         @columns_hash.fetch(table_name) do
-          @columns_hash[deep_deduplicate(table_name)] = columns(table_name).index_by(&:name).freeze
+          @columns_hash[deep_deduplicate(table_name)] = columns(connection, table_name).index_by(&:name).freeze
         end
       end
 
       # Checks whether the columns hash is already cached for a table.
-      def columns_hash?(table_name)
+      def columns_hash?(connection, table_name)
         @columns_hash.key?(table_name)
       end
 
-      def indexes(table_name)
+      def indexes(connection, table_name)
         @indexes.fetch(table_name) do
-          if data_source_exists?(table_name)
+          if data_source_exists?(connection, table_name)
             @indexes[deep_deduplicate(table_name)] = deep_deduplicate(connection.indexes(table_name))
           else
             []
@@ -141,19 +370,16 @@ module ActiveRecord
         end
       end
 
-      def database_version # :nodoc:
+      def database_version(connection) # :nodoc:
         @database_version ||= connection.get_database_version
       end
 
-      # Clears out internal caches
-      def clear!
-        @columns.clear
-        @columns_hash.clear
-        @primary_keys.clear
-        @data_sources.clear
-        @indexes.clear
-        @version = nil
-        @database_version = nil
+      def version(connection)
+        @version ||= connection.schema_version
+      end
+
+      def schema_version
+        @version
       end
 
       def size
@@ -161,7 +387,7 @@ module ActiveRecord
       end
 
       # Clear out internal caches for the data source +name+.
-      def clear_data_source_cache!(name)
+      def clear_data_source_cache!(_connection, name)
         @columns.delete name
         @columns_hash.delete name
         @primary_keys.delete name
@@ -169,9 +395,16 @@ module ActiveRecord
         @indexes.delete name
       end
 
+      def add_all(connection) # :nodoc:
+        tables_to_cache(connection).each do |table|
+          add(connection, table)
+        end
+
+        version(connection)
+        database_version(connection)
+      end
+
       def dump_to(filename)
-        clear!
-        tables_to_cache.each { |table| add(table) }
         open(filename) { |f|
           if filename.include?(".dump")
             f.write(Marshal.dump(self))
@@ -181,13 +414,11 @@ module ActiveRecord
         }
       end
 
-      def marshal_dump
-        reset_version!
-
-        [@version, @columns, {}, @primary_keys, @data_sources, @indexes, database_version]
+      def marshal_dump # :nodoc:
+        [@version, @columns, {}, @primary_keys, @data_sources, @indexes, @database_version]
       end
 
-      def marshal_load(array)
+      def marshal_load(array) # :nodoc:
         @version, @columns, _columns_hash, @primary_keys, @data_sources, @indexes, @database_version = array
         @indexes ||= {}
 
@@ -195,7 +426,7 @@ module ActiveRecord
       end
 
       private
-        def tables_to_cache
+        def tables_to_cache(connection)
           connection.data_sources.reject do |table|
             ignored_table?(table)
           end
@@ -207,10 +438,6 @@ module ActiveRecord
           end
         end
 
-        def reset_version!
-          @version = connection.schema_version
-        end
-
         def derive_columns_hash_and_deduplicate_values
           @columns      = deep_deduplicate(@columns)
           @columns_hash = @columns.transform_values { |columns| columns.index_by(&:name) }
@@ -232,8 +459,8 @@ module ActiveRecord
           end
         end
 
-        def prepare_data_sources
-          tables_to_cache.each do |source|
+        def prepare_data_sources(connection)
+          tables_to_cache(connection).each do |source|
             @data_sources[source] = true
           end
         end
@@ -244,6 +471,7 @@ module ActiveRecord
           File.atomic_write(filename) do |file|
             if File.extname(filename) == ".gz"
               zipper = Zlib::GzipWriter.new file
+              zipper.mtime = 0
               yield zipper
               zipper.flush
               zipper.close

data/lib/active_record/connection_adapters/sqlite3/column.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module ConnectionAdapters
+    module SQLite3
+      class Column < ConnectionAdapters::Column # :nodoc:
+        attr_reader :rowid
+
+        def initialize(*, auto_increment: nil, rowid: false, **)
+          super
+          @auto_increment = auto_increment
+          @rowid = rowid
+        end
+
+        def auto_increment?
+          @auto_increment
+        end
+
+        def auto_incremented_by_db?
+          auto_increment? || rowid
+        end
+
+        def init_with(coder)
+          @auto_increment = coder["auto_increment"]
+          super
+        end
+
+        def encode_with(coder)
+          coder["auto_increment"] = @auto_increment
+          super
+        end
+
+        def ==(other)
+          other.is_a?(Column) &&
+            super &&
+            auto_increment? == other.auto_increment?
+        end
+        alias :eql? :==
+
+        def hash
+          Column.hash ^
+            super.hash ^
+            auto_increment?.hash ^
+            rowid.hash
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/sqlite3/database_statements.rb

@@ -15,39 +15,25 @@ module ActiveRecord
           !READ_QUERY.match?(sql.b)
         end
 
-        def explain(arel, binds = [])
-          sql = "EXPLAIN QUERY PLAN #{to_sql(arel, binds)}"
-          SQLite3::ExplainPrettyPrinter.new.pp(exec_query(sql, "EXPLAIN", []))
+        def explain(arel, binds = [], _options = [])
+          sql    = "EXPLAIN QUERY PLAN " + to_sql(arel, binds)
+          result = internal_exec_query(sql, "EXPLAIN", [])
+          SQLite3::ExplainPrettyPrinter.new.pp(result)
         end
 
-        def execute(sql, name = nil) # :nodoc:
+        def internal_exec_query(sql, name = nil, binds = [], prepare: false, async: false) # :nodoc:
           sql = transform_query(sql)
           check_if_write_query(sql)
 
-          materialize_transactions
-          mark_transaction_written_if_write(sql)
-
-          log(sql, name) do
-            ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
-              @connection.execute(sql)
-            end
-          end
-        end
-
-        def exec_query(sql, name = nil, binds = [], prepare: false, async: false) # :nodoc:
-          sql = transform_query(sql)
-          check_if_write_query(sql)
-
-          materialize_transactions
           mark_transaction_written_if_write(sql)
 
           type_casted_binds = type_casted_binds(binds)
 
           log(sql, name, binds, type_casted_binds, async: async) do
-            ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
+            with_raw_connection do |conn|
               # Don't cache statements if they are not prepared
               unless prepare
-                stmt = @connection.prepare(sql)
+                stmt = conn.prepare(sql)
                 begin
                   cols = stmt.columns
                   unless without_prepared_statement?(binds)
@@ -58,7 +44,7 @@ module ActiveRecord
                   stmt.close
                 end
               else
-                stmt = @statements[sql] ||= @connection.prepare(sql)
+                stmt = @statements[sql] ||= conn.prepare(sql)
                 cols = stmt.columns
                 stmt.reset!
                 stmt.bind_params(type_casted_binds)
@@ -71,8 +57,8 @@ module ActiveRecord
         end
 
         def exec_delete(sql, name = "SQL", binds = []) # :nodoc:
-          exec_query(sql, name, binds)
-          @connection.changes
+          internal_exec_query(sql, name, binds)
+          @raw_connection.changes
         end
         alias :exec_update :exec_delete
 
@@ -80,22 +66,36 @@ module ActiveRecord
           raise TransactionIsolationError, "SQLite3 only supports the `read_uncommitted` transaction isolation level" if isolation != :read_uncommitted
           raise StandardError, "You need to enable the shared-cache mode in SQLite mode before attempting to change the transaction isolation level" unless shared_cache?
 
-          ActiveSupport::IsolatedExecutionState[:active_record_read_uncommitted] = @connection.get_first_value("PRAGMA read_uncommitted")
-          @connection.read_uncommitted = true
-          begin_db_transaction
+          with_raw_connection(allow_retry: true, materialize_transactions: false) do |conn|
+            ActiveSupport::IsolatedExecutionState[:active_record_read_uncommitted] = conn.get_first_value("PRAGMA read_uncommitted")
+            conn.read_uncommitted = true
+            begin_db_transaction
+          end
         end
 
         def begin_db_transaction # :nodoc:
-          log("begin transaction", "TRANSACTION") { @connection.transaction }
+          log("begin transaction", "TRANSACTION") do
+            with_raw_connection(allow_retry: true, materialize_transactions: false) do |conn|
+              conn.transaction
+            end
+          end
         end
 
         def commit_db_transaction # :nodoc:
-          log("commit transaction", "TRANSACTION") { @connection.commit }
+          log("commit transaction", "TRANSACTION") do
+            with_raw_connection(allow_retry: true, materialize_transactions: false) do |conn|
+              conn.commit
+            end
+          end
           reset_read_uncommitted
         end
 
         def exec_rollback_db_transaction # :nodoc:
-          log("rollback transaction", "TRANSACTION") { @connection.rollback }
+          log("rollback transaction", "TRANSACTION") do
+            with_raw_connection(allow_retry: true, materialize_transactions: false) do |conn|
+              conn.rollback
+            end
+          end
           reset_read_uncommitted
         end
 
@@ -109,11 +109,19 @@ module ActiveRecord
         end
 
         private
+          def raw_execute(sql, name, async: false, allow_retry: false, materialize_transactions: false)
+            log(sql, name, async: async) do
+              with_raw_connection(allow_retry: allow_retry, materialize_transactions: materialize_transactions) do |conn|
+                conn.execute(sql)
+              end
+            end
+          end
+
           def reset_read_uncommitted
             read_uncommitted = ActiveSupport::IsolatedExecutionState[:active_record_read_uncommitted]
             return unless read_uncommitted
 
-            @connection.read_uncommitted = read_uncommitted
+            @raw_connection&.read_uncommitted = read_uncommitted
           end
 
           def execute_batch(statements, name = nil)
@@ -121,19 +129,17 @@ module ActiveRecord
             sql = combine_multi_statements(statements)
 
             check_if_write_query(sql)
-
-            materialize_transactions
             mark_transaction_written_if_write(sql)
 
             log(sql, name) do
-              ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
-                @connection.execute_batch2(sql)
+              with_raw_connection do |conn|
+                conn.execute_batch2(sql)
               end
             end
           end
 
           def last_inserted_id(result)
-            @connection.last_insert_row_id
+            @raw_connection.last_insert_row_id
           end
 
           def build_fixture_statements(fixture_set)