activerecord 7.1.5.1 → 7.2.2.1

data/lib/active_record/connection_adapters/sqlite3/database_statements.rb

@@ -21,7 +21,7 @@ module ActiveRecord
           SQLite3::ExplainPrettyPrinter.new.pp(result)
         end
 
-        def internal_exec_query(sql, name = nil, binds = [], prepare: false, async: false) # :nodoc:
+        def internal_exec_query(sql, name = nil, binds = [], prepare: false, async: false, allow_retry: false) # :nodoc:
           sql = transform_query(sql)
           check_if_write_query(sql)
 
@@ -29,7 +29,7 @@ module ActiveRecord
 
           type_casted_binds = type_casted_binds(binds)
 
-          log(sql, name, binds, type_casted_binds, async: async) do
+          log(sql, name, binds, type_casted_binds, async: async) do |notification_payload|
             with_raw_connection do |conn|
               # Don't cache statements if they are not prepared
               unless prepare
@@ -52,7 +52,9 @@ module ActiveRecord
               end
               verified!
 
-              build_result(columns: cols, rows: records)
+              result = build_result(columns: cols, rows: records)
+              notification_payload[:row_count] = result.length
+              result
             end
           end
         end
@@ -104,7 +106,7 @@ module ActiveRecord
 
         # https://stackoverflow.com/questions/17574784
         # https://www.sqlite.org/lang_datefunc.html
-        HIGH_PRECISION_CURRENT_TIMESTAMP = Arel.sql("STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')").freeze # :nodoc:
+        HIGH_PRECISION_CURRENT_TIMESTAMP = Arel.sql("STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')", retryable: true).freeze # :nodoc:
         private_constant :HIGH_PRECISION_CURRENT_TIMESTAMP
 
         def high_precision_current_timestamp
@@ -113,10 +115,11 @@ module ActiveRecord
 
         private
           def raw_execute(sql, name, async: false, allow_retry: false, materialize_transactions: false)
-            log(sql, name, async: async) do
+            log(sql, name, async: async) do |notification_payload|
               with_raw_connection(allow_retry: allow_retry, materialize_transactions: materialize_transactions) do |conn|
                 result = conn.execute(sql)
                 verified!
+                notification_payload[:row_count] = result.length
                 result
               end
             end
@@ -136,10 +139,11 @@ module ActiveRecord
             check_if_write_query(sql)
             mark_transaction_written_if_write(sql)
 
-            log(sql, name) do
+            log(sql, name) do |notification_payload|
               with_raw_connection do |conn|
                 result = conn.execute_batch2(sql)
                 verified!
+                notification_payload[:row_count] = result.length
                 result
               end
             end

data/lib/active_record/connection_adapters/sqlite3/quoting.rb

@@ -4,9 +4,52 @@ module ActiveRecord
   module ConnectionAdapters
     module SQLite3
       module Quoting # :nodoc:
+        extend ActiveSupport::Concern
+
         QUOTED_COLUMN_NAMES = Concurrent::Map.new # :nodoc:
         QUOTED_TABLE_NAMES = Concurrent::Map.new # :nodoc:
 
+        module ClassMethods # :nodoc:
+          def column_name_matcher
+            /
+              \A
+              (
+                (?:
+                  # "table_name"."column_name" | function(one or no argument)
+                  ((?:\w+\.|"\w+"\.)?(?:\w+|"\w+") | \w+\((?:|\g<2>)\))
+                )
+                (?:(?:\s+AS)?\s+(?:\w+|"\w+"))?
+              )
+              (?:\s*,\s*\g<1>)*
+              \z
+            /ix
+          end
+
+          def column_name_with_order_matcher
+            /
+              \A
+              (
+                (?:
+                  # "table_name"."column_name" | function(one or no argument)
+                  ((?:\w+\.|"\w+"\.)?(?:\w+|"\w+") | \w+\((?:|\g<2>)\))
+                )
+                (?:\s+COLLATE\s+(?:\w+|"\w+"))?
+                (?:\s+ASC|\s+DESC)?
+              )
+              (?:\s*,\s*\g<1>)*
+              \z
+            /ix
+          end
+
+          def quote_column_name(name)
+            QUOTED_COLUMN_NAMES[name] ||= %Q("#{name.to_s.gsub('"', '""')}").freeze
+          end
+
+          def quote_table_name(name)
+            QUOTED_TABLE_NAMES[name] ||= %Q("#{name.to_s.gsub('"', '""').gsub(".", "\".\"")}").freeze
+          end
+        end
+
         def quote_string(s)
           ::SQLite3::Database.quote(s)
         end
@@ -15,14 +58,6 @@ module ActiveRecord
           quote_column_name(attr)
         end
 
-        def quote_table_name(name)
-          QUOTED_TABLE_NAMES[name] ||= super.gsub(".", "\".\"").freeze
-        end
-
-        def quote_column_name(name)
-          QUOTED_COLUMN_NAMES[name] ||= %Q("#{super.gsub('"', '""')}")
-        end
-
         def quoted_time(value)
           value = value.change(year: 2000, month: 1, day: 1)
           quoted_date(value).sub(/\A\d\d\d\d-\d\d-\d\d /, "2000-01-01 ")
@@ -63,7 +98,7 @@ module ActiveRecord
 
         def type_cast(value) # :nodoc:
           case value
-          when BigDecimal
+          when BigDecimal, Rational
             value.to_f
           when String
             if value.encoding == Encoding::ASCII_8BIT
@@ -75,43 +110,6 @@ module ActiveRecord
             super
           end
         end
-
-        def column_name_matcher
-          COLUMN_NAME
-        end
-
-        def column_name_with_order_matcher
-          COLUMN_NAME_WITH_ORDER
-        end
-
-        COLUMN_NAME = /
-          \A
-          (
-            (?:
-              # "table_name"."column_name" | function(one or no argument)
-              ((?:\w+\.|"\w+"\.)?(?:\w+|"\w+") | \w+\((?:|\g<2>)\))
-            )
-            (?:(?:\s+AS)?\s+(?:\w+|"\w+"))?
-          )
-          (?:\s*,\s*\g<1>)*
-          \z
-        /ix
-
-        COLUMN_NAME_WITH_ORDER = /
-          \A
-          (
-            (?:
-              # "table_name"."column_name" | function(one or no argument)
-              ((?:\w+\.|"\w+"\.)?(?:\w+|"\w+") | \w+\((?:|\g<2>)\))
-            )
-            (?:\s+COLLATE\s+(?:\w+|"\w+"))?
-            (?:\s+ASC|\s+DESC)?
-          )
-          (?:\s*,\s*\g<1>)*
-          \z
-        /ix
-
-        private_constant :COLUMN_NAME, :COLUMN_NAME_WITH_ORDER
       end
     end
   end

data/lib/active_record/connection_adapters/sqlite3/schema_creation.rb

@@ -5,6 +5,18 @@ module ActiveRecord
     module SQLite3
       class SchemaCreation < SchemaCreation # :nodoc:
         private
+          def visit_AddForeignKey(o)
+            super.dup.tap do |sql|
+              sql << " DEFERRABLE INITIALLY #{o.options[:deferrable].to_s.upcase}" if o.deferrable
+            end
+          end
+
+          def visit_ForeignKeyDefinition(o)
+            super.dup.tap do |sql|
+              sql << " DEFERRABLE INITIALLY #{o.deferrable.to_s.upcase}" if o.deferrable
+            end
+          end
+
           def supports_index_using?
             false
           end
@@ -13,6 +25,16 @@ module ActiveRecord
             if options[:collation]
               sql << " COLLATE \"#{options[:collation]}\""
             end
+
+            if as = options[:as]
+              sql << " GENERATED ALWAYS AS (#{as})"
+
+              if options[:stored]
+                sql << " STORED"
+              else
+                sql << " VIRTUAL"
+              end
+            end
             super
           end
       end

data/lib/active_record/connection_adapters/sqlite3/schema_definitions.rb

@@ -16,10 +16,23 @@ module ActiveRecord
         end
         alias :belongs_to :references
 
+        def new_column_definition(name, type, **options) # :nodoc:
+          case type
+          when :virtual
+            type = options[:type]
+          end
+
+          super
+        end
+
         private
           def integer_like_primary_key_type(type, options)
             :primary_key
           end
+
+          def valid_column_definition_options
+            super + [:as, :type, :stored]
+          end
       end
     end
   end

data/lib/active_record/connection_adapters/sqlite3/schema_dumper.rb

@@ -12,6 +12,22 @@ module ActiveRecord
           def explicit_primary_key_default?(column)
             column.bigint?
           end
+
+          def prepare_column_options(column)
+            spec = super
+
+            if @connection.supports_virtual_columns? && column.virtual?
+              spec[:as] = extract_expression_for_virtual_column(column)
+              spec[:stored] = column.virtual_stored?
+              spec = { type: schema_type(column).inspect }.merge!(spec)
+            end
+
+            spec
+          end
+
+          def extract_expression_for_virtual_column(column)
+            column.default_function.inspect
+          end
       end
     end
   end

data/lib/active_record/connection_adapters/sqlite3/schema_statements.rb

@@ -53,6 +53,8 @@ module ActiveRecord
         end
 
         def add_foreign_key(from_table, to_table, **options)
+          assert_valid_deferrable(options[:deferrable])
+
           alter_table(from_table) do |definition|
             to_table = strip_table_name_prefix_and_suffix(to_table)
             definition.foreign_key(to_table, **options)
@@ -137,7 +139,14 @@ module ActiveRecord
 
             type_metadata = fetch_type_metadata(field["type"])
             default_value = extract_value_from_default(default)
-            default_function = extract_default_function(default_value, default)
+            generated_type = extract_generated_type(field)
+
+            if generated_type.present?
+              default_function = default
+            else
+              default_function = extract_default_function(default_value, default)
+            end
+
             rowid = is_column_the_rowid?(field, definitions)
 
             Column.new(
@@ -148,7 +157,8 @@ module ActiveRecord
               default_function,
               collation: field["collation"],
               auto_increment: field["auto_increment"],
-              rowid: rowid
+              rowid: rowid,
+              generated_type: generated_type
             )
           end
 
@@ -185,6 +195,19 @@ module ActiveRecord
             scope[:type] = type if type
             scope
           end
+
+          def assert_valid_deferrable(deferrable)
+            return if !deferrable || %i(immediate deferred).include?(deferrable)
+
+            raise ArgumentError, "deferrable must be `:immediate` or `:deferred`, got: `#{deferrable.inspect}`"
+          end
+
+          def extract_generated_type(field)
+            case field["hidden"]
+            when 2 then :virtual
+            when 3 then :stored
+            end
+          end
       end
     end
   end

data/lib/active_record/connection_adapters/sqlite3_adapter.rb

@@ -15,16 +15,6 @@ gem "sqlite3", ">= 1.4"
 require "sqlite3"
 
 module ActiveRecord
-  module ConnectionHandling # :nodoc:
-    def sqlite3_adapter_class
-      ConnectionAdapters::SQLite3Adapter
-    end
-
-    def sqlite3_connection(config)
-      sqlite3_adapter_class.new(config)
-    end
-  end
-
   module ConnectionAdapters # :nodoc:
     # = Active Record SQLite3 Adapter
     #
@@ -88,6 +78,15 @@ module ActiveRecord
         json:         { name: "json" },
       }
 
+      DEFAULT_PRAGMAS = {
+        "foreign_keys"        => true,
+        "journal_mode"        => :wal,
+        "synchronous"         => :normal,
+        "mmap_size"           => 134217728, # 128 megabytes
+        "journal_size_limit"  => 67108864, # 64 megabytes
+        "cache_size"          => 2000
+      }
+
       class StatementPool < ConnectionAdapters::StatementPool # :nodoc:
         alias reset clear
 
@@ -113,13 +112,9 @@ module ActiveRecord
           dirname = File.dirname(@config[:database])
           unless File.directory?(dirname)
             begin
-              Dir.mkdir(dirname)
-            rescue Errno::ENOENT => error
-              if error.message.include?("No such file or directory")
-                raise ActiveRecord::NoDatabaseError.new(connection_pool: @pool)
-              else
-                raise
-              end
+              FileUtils.mkdir_p(dirname)
+            rescue SystemCallError
+              raise ActiveRecord::NoDatabaseError.new(connection_pool: @pool)
             end
           end
         end
@@ -196,14 +191,16 @@ module ActiveRecord
         !@memory_database
       end
 
-      def active?
-        @raw_connection && !@raw_connection.closed?
+      def supports_virtual_columns?
+        database_version >= "3.31.0"
       end
 
-      def return_value_after_insert?(column) # :nodoc:
-        column.auto_populated?
+      def connected?
+        !(@raw_connection.nil? || @raw_connection.closed?)
       end
 
+      alias_method :active?, :connected?
+
       alias :reset! :reconnect!
 
       # Disconnects from the database if already connected. Otherwise, this
@@ -236,6 +233,10 @@ module ActiveRecord
         true
       end
 
+      def supports_deferrable_constraints?
+        true
+      end
+
       # REFERENTIAL INTEGRITY ====================================
 
       def disable_referential_integrity # :nodoc:
@@ -258,7 +259,7 @@ module ActiveRecord
 
         unless result.blank?
           tables = result.map { |row| row["table"] }
-          raise ActiveRecord::StatementInvalid.new("Foreign key violations found: #{tables.join(", ")}", sql: sql)
+          raise ActiveRecord::StatementInvalid.new("Foreign key violations found: #{tables.join(", ")}", sql: sql, connection_pool: @pool)
         end
       end
 
@@ -290,6 +291,7 @@ module ActiveRecord
       end
 
       def add_column(table_name, column_name, type, **options) # :nodoc:
+        type = type.to_sym
         if invalid_alter_table_type?(type, options)
           alter_table(table_name) do |definition|
             definition.column(column_name, type, **options)
@@ -365,15 +367,31 @@ module ActiveRecord
       end
       alias :add_belongs_to :add_reference
 
+      FK_REGEX = /.*FOREIGN KEY\s+\("(\w+)"\)\s+REFERENCES\s+"(\w+)"\s+\("(\w+)"\)/
+      DEFERRABLE_REGEX = /DEFERRABLE INITIALLY (\w+)/
       def foreign_keys(table_name)
         # SQLite returns 1 row for each column of composite foreign keys.
         fk_info = internal_exec_query("PRAGMA foreign_key_list(#{quote(table_name)})", "SCHEMA")
+        # Deferred or immediate foreign keys can only be seen in the CREATE TABLE sql
+        fk_defs = table_structure_sql(table_name)
+                    .select do |column_string|
+                      column_string.start_with?("CONSTRAINT") &&
+                      column_string.include?("FOREIGN KEY")
+                    end
+                    .to_h do |fk_string|
+                      _, from, table, to = fk_string.match(FK_REGEX).to_a
+                      _, mode = fk_string.match(DEFERRABLE_REGEX).to_a
+                      deferred = mode&.downcase&.to_sym || false
+                      [[table, from, to], deferred]
+                    end
+
         grouped_fk = fk_info.group_by { |row| row["id"] }.values.each { |group| group.sort_by! { |row| row["seq"] } }
         grouped_fk.map do |group|
           row = group.first
           options = {
             on_delete: extract_foreign_key_action(row["on_delete"]),
-            on_update: extract_foreign_key_action(row["on_update"])
+            on_update: extract_foreign_key_action(row["on_update"]),
+            deferrable: fk_defs[[row["table"], row["from"], row["to"]]]
           }
 
           if group.one?
@@ -454,8 +472,8 @@ module ActiveRecord
         end
 
         def table_structure(table_name)
-          structure = internal_exec_query("PRAGMA table_info(#{quote_table_name(table_name)})", "SCHEMA")
-          raise(ActiveRecord::StatementInvalid, "Could not find table '#{table_name}'") if structure.empty?
+          structure = table_info(table_name)
+          raise ActiveRecord::StatementInvalid.new("Could not find table '#{table_name}'", connection_pool: @pool) if structure.empty?
           table_structure_with_collation(table_name, structure)
         end
         alias column_definitions table_structure
@@ -494,8 +512,9 @@ module ActiveRecord
         # See: https://www.sqlite.org/lang_altertable.html
         # SQLite has an additional restriction on the ALTER TABLE statement
         def invalid_alter_table_type?(type, options)
-          type.to_sym == :primary_key || options[:primary_key] ||
-            options[:null] == false && options[:default].nil?
+          type == :primary_key || options[:primary_key] ||
+            options[:null] == false && options[:default].nil? ||
+            (type == :virtual && options[:stored])
         end
 
         def alter_table(
@@ -551,12 +570,6 @@ module ActiveRecord
                  options[:rename][column.name.to_sym] ||
                  column.name) : column.name
 
-              if column.has_default?
-                type = lookup_cast_type_from_column(column)
-                default = type.deserialize(column.default)
-                default = -> { column.default_function } if default.nil?
-              end
-
               column_options = {
                 limit: column.limit,
                 precision: column.precision,
@@ -566,19 +579,31 @@ module ActiveRecord
                 primary_key: column_name == from_primary_key
               }
 
-              unless column.auto_increment?
-                column_options[:default] = default
+              if column.virtual?
+                column_options[:as] = column.default_function
+                column_options[:stored] = column.virtual_stored?
+                column_options[:type] = column.type
+              elsif column.has_default?
+                type = lookup_cast_type_from_column(column)
+                default = type.deserialize(column.default)
+                default = -> { column.default_function } if default.nil?
+
+                unless column.auto_increment?
+                  column_options[:default] = default
+                end
               end
 
-              column_type = column.bigint? ? :bigint : column.type
+              column_type = column.virtual? ? :virtual : (column.bigint? ? :bigint : column.type)
               @definition.column(column_name, column_type, **column_options)
             end
 
             yield @definition if block_given?
           end
           copy_table_indexes(from, to, options[:rename] || {})
+
+          columns_to_copy = @definition.columns.reject { |col| col.options.key?(:as) }.map(&:name)
           copy_table_contents(from, to,
-            @definition.columns.map(&:name),
+            columns_to_copy,
             options[:rename] || {})
         end
 
@@ -643,32 +668,22 @@ module ActiveRecord
 
         COLLATE_REGEX = /.*"(\w+)".*collate\s+"(\w+)".*/i
         PRIMARY_KEY_AUTOINCREMENT_REGEX = /.*"(\w+)".+PRIMARY KEY AUTOINCREMENT/i
+        GENERATED_ALWAYS_AS_REGEX = /.*"(\w+)".+GENERATED ALWAYS AS \((.+)\) (?:STORED|VIRTUAL)/i
 
         def table_structure_with_collation(table_name, basic_structure)
           collation_hash = {}
           auto_increments = {}
-          sql = <<~SQL
-            SELECT sql FROM
-              (SELECT * FROM sqlite_master UNION ALL
-               SELECT * FROM sqlite_temp_master)
-            WHERE type = 'table' AND name = #{quote(table_name)}
-          SQL
+          generated_columns = {}
 
-          # Result will have following sample string
-          # CREATE TABLE "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
-          #                       "password_digest" varchar COLLATE "NOCASE");
-          result = query_value(sql, "SCHEMA")
+          column_strings = table_structure_sql(table_name, basic_structure.map { |column| column["name"] })
 
-          if result
-            # Splitting with left parentheses and discarding the first part will return all
-            # columns separated with comma(,).
-            columns_string = result.split("(", 2).last
-
-            columns_string.split(",").each do |column_string|
+          if column_strings.any?
+            column_strings.each do |column_string|
               # This regex will match the column name and collation type and will save
               # the value in $1 and $2 respectively.
               collation_hash[$1] = $2 if COLLATE_REGEX =~ column_string
               auto_increments[$1] = true if PRIMARY_KEY_AUTOINCREMENT_REGEX =~ column_string
+              generated_columns[$1] = $2 if GENERATED_ALWAYS_AS_REGEX =~ column_string
             end
 
             basic_structure.map do |column|
@@ -682,6 +697,10 @@ module ActiveRecord
                 column["auto_increment"] = true
               end
 
+              if generated_columns.has_key?(column_name)
+                column["dflt_value"] = generated_columns[column_name]
+              end
+
               column
             end
           else
@@ -689,6 +708,50 @@ module ActiveRecord
           end
         end
 
+        UNQUOTED_OPEN_PARENS_REGEX = /\((?![^'"]*['"][^'"]*$)/
+        FINAL_CLOSE_PARENS_REGEX = /\);*\z/
+
+        def table_structure_sql(table_name, column_names = nil)
+          unless column_names
+            column_info = table_info(table_name)
+            column_names = column_info.map { |column| column["name"] }
+          end
+
+          sql = <<~SQL
+            SELECT sql FROM
+              (SELECT * FROM sqlite_master UNION ALL
+               SELECT * FROM sqlite_temp_master)
+            WHERE type = 'table' AND name = #{quote(table_name)}
+          SQL
+
+          # Result will have following sample string
+          # CREATE TABLE "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+          #                       "password_digest" varchar COLLATE "NOCASE",
+          #                       "o_id" integer,
+          #                       CONSTRAINT "fk_rails_78146ddd2e" FOREIGN KEY ("o_id") REFERENCES "os" ("id"));
+          result = query_value(sql, "SCHEMA")
+
+          return [] unless result
+
+          # Splitting with left parentheses and discarding the first part will return all
+          # columns separated with comma(,).
+          result.partition(UNQUOTED_OPEN_PARENS_REGEX)
+                .last
+                .sub(FINAL_CLOSE_PARENS_REGEX, "")
+                # column definitions can have a comma in them, so split on commas followed
+                # by a space and a column name in quotes or followed by the keyword CONSTRAINT
+                .split(/,(?=\s(?:CONSTRAINT|"(?:#{Regexp.union(column_names).source})"))/i)
+                .map(&:strip)
+        end
+
+        def table_info(table_name)
+          if supports_virtual_columns?
+            internal_exec_query("PRAGMA table_xinfo(#{quote_table_name(table_name)})", "SCHEMA")
+          else
+            internal_exec_query("PRAGMA table_info(#{quote_table_name(table_name)})", "SCHEMA")
+          end
+        end
+
         def arel_visitor
           Arel::Visitors::SQLite.new(self)
         end
@@ -723,29 +786,16 @@ module ActiveRecord
             end
           end
 
-          # Enforce foreign key constraints
-          # https://www.sqlite.org/pragma.html#pragma_foreign_keys
-          # https://www.sqlite.org/foreignkeys.html
-          raw_execute("PRAGMA foreign_keys = ON", "SCHEMA")
-          unless @memory_database
-            # Journal mode WAL allows for greater concurrency (many readers + one writer)
-            # https://www.sqlite.org/pragma.html#pragma_journal_mode
-            raw_execute("PRAGMA journal_mode = WAL", "SCHEMA")
-            # Set more relaxed level of database durability
-            # 2 = "FULL" (sync on every write), 1 = "NORMAL" (sync every 1000 written pages) and 0 = "NONE"
-            # https://www.sqlite.org/pragma.html#pragma_synchronous
-            raw_execute("PRAGMA synchronous = NORMAL", "SCHEMA")
-            # Set the global memory map so all processes can share some data
-            # https://www.sqlite.org/pragma.html#pragma_mmap_size
-            # https://www.sqlite.org/mmap.html
-            raw_execute("PRAGMA mmap_size = #{128.megabytes}", "SCHEMA")
+          super
+
+          pragmas = @config.fetch(:pragmas, {}).stringify_keys
+          DEFAULT_PRAGMAS.merge(pragmas).each do |pragma, value|
+            if ::SQLite3::Pragmas.method_defined?("#{pragma}=")
+              @raw_connection.public_send("#{pragma}=", value)
+            else
+              warn "Unknown SQLite pragma: #{pragma}"
+            end
           end
-          # Impose a limit on the WAL file to prevent unlimited growth
-          # https://www.sqlite.org/pragma.html#pragma_journal_size_limit
-          raw_execute("PRAGMA journal_size_limit = #{64.megabytes}", "SCHEMA")
-          # Set the local connection cache to 2000 pages
-          # https://www.sqlite.org/pragma.html#pragma_cache_size
-          raw_execute("PRAGMA cache_size = 2000", "SCHEMA")
         end
     end
     ActiveSupport.run_load_hooks(:active_record_sqlite3adapter, SQLite3Adapter)