activerecord 7.1.5 → 7.2.0.beta1

data/lib/active_record/connection_adapters/sqlite3/schema_creation.rb

@@ -5,6 +5,18 @@ module ActiveRecord
     module SQLite3
       class SchemaCreation < SchemaCreation # :nodoc:
         private
+          def visit_AddForeignKey(o)
+            super.dup.tap do |sql|
+              sql << " DEFERRABLE INITIALLY #{o.options[:deferrable].to_s.upcase}" if o.deferrable
+            end
+          end
+
+          def visit_ForeignKeyDefinition(o)
+            super.dup.tap do |sql|
+              sql << " DEFERRABLE INITIALLY #{o.deferrable.to_s.upcase}" if o.deferrable
+            end
+          end
+
           def supports_index_using?
             false
           end
@@ -13,6 +25,16 @@ module ActiveRecord
             if options[:collation]
               sql << " COLLATE \"#{options[:collation]}\""
             end
+
+            if as = options[:as]
+              sql << " GENERATED ALWAYS AS (#{as})"
+
+              if options[:stored]
+                sql << " STORED"
+              else
+                sql << " VIRTUAL"
+              end
+            end
             super
           end
       end

data/lib/active_record/connection_adapters/sqlite3/schema_definitions.rb

@@ -16,10 +16,23 @@ module ActiveRecord
         end
         alias :belongs_to :references
 
+        def new_column_definition(name, type, **options) # :nodoc:
+          case type
+          when :virtual
+            type = options[:type]
+          end
+
+          super
+        end
+
         private
           def integer_like_primary_key_type(type, options)
             :primary_key
           end
+
+          def valid_column_definition_options
+            super + [:as, :type, :stored]
+          end
       end
     end
   end

data/lib/active_record/connection_adapters/sqlite3/schema_dumper.rb

@@ -12,6 +12,22 @@ module ActiveRecord
           def explicit_primary_key_default?(column)
             column.bigint?
           end
+
+          def prepare_column_options(column)
+            spec = super
+
+            if @connection.supports_virtual_columns? && column.virtual?
+              spec[:as] = extract_expression_for_virtual_column(column)
+              spec[:stored] = column.virtual_stored?
+              spec = { type: schema_type(column).inspect }.merge!(spec)
+            end
+
+            spec
+          end
+
+          def extract_expression_for_virtual_column(column)
+            column.default_function.inspect
+          end
       end
     end
   end

data/lib/active_record/connection_adapters/sqlite3/schema_statements.rb

@@ -53,6 +53,8 @@ module ActiveRecord
         end
 
         def add_foreign_key(from_table, to_table, **options)
+          assert_valid_deferrable(options[:deferrable])
+
           alter_table(from_table) do |definition|
             to_table = strip_table_name_prefix_and_suffix(to_table)
             definition.foreign_key(to_table, **options)
@@ -137,7 +139,14 @@ module ActiveRecord
 
             type_metadata = fetch_type_metadata(field["type"])
             default_value = extract_value_from_default(default)
-            default_function = extract_default_function(default_value, default)
+            generated_type = extract_generated_type(field)
+
+            if generated_type.present?
+              default_function = default
+            else
+              default_function = extract_default_function(default_value, default)
+            end
+
             rowid = is_column_the_rowid?(field, definitions)
 
             Column.new(
@@ -148,7 +157,8 @@ module ActiveRecord
               default_function,
               collation: field["collation"],
               auto_increment: field["auto_increment"],
-              rowid: rowid
+              rowid: rowid,
+              generated_type: generated_type
             )
           end
 
@@ -185,6 +195,19 @@ module ActiveRecord
             scope[:type] = type if type
             scope
           end
+
+          def assert_valid_deferrable(deferrable)
+            return if !deferrable || %i(immediate deferred).include?(deferrable)
+
+            raise ArgumentError, "deferrable must be `:immediate` or `:deferred`, got: `#{deferrable.inspect}`"
+          end
+
+          def extract_generated_type(field)
+            case field["hidden"]
+            when 2 then :virtual
+            when 3 then :stored
+            end
+          end
       end
     end
   end

data/lib/active_record/connection_adapters/sqlite3_adapter.rb

@@ -15,16 +15,6 @@ gem "sqlite3", ">= 1.4"
 require "sqlite3"
 
 module ActiveRecord
-  module ConnectionHandling # :nodoc:
-    def sqlite3_adapter_class
-      ConnectionAdapters::SQLite3Adapter
-    end
-
-    def sqlite3_connection(config)
-      sqlite3_adapter_class.new(config)
-    end
-  end
-
   module ConnectionAdapters # :nodoc:
     # = Active Record SQLite3 Adapter
     #
@@ -88,6 +78,15 @@ module ActiveRecord
         json:         { name: "json" },
       }
 
+      DEFAULT_PRAGMAS = {
+        "foreign_keys"        => true,
+        "journal_mode"        => :wal,
+        "synchronous"         => :normal,
+        "mmap_size"           => 134217728, # 128 megabytes
+        "journal_size_limit"  => 67108864, # 64 megabytes
+        "cache_size"          => 2000
+      }
+
       class StatementPool < ConnectionAdapters::StatementPool # :nodoc:
         alias reset clear
 
@@ -113,13 +112,9 @@ module ActiveRecord
           dirname = File.dirname(@config[:database])
           unless File.directory?(dirname)
             begin
-              Dir.mkdir(dirname)
-            rescue Errno::ENOENT => error
-              if error.message.include?("No such file or directory")
-                raise ActiveRecord::NoDatabaseError.new(connection_pool: @pool)
-              else
-                raise
-              end
+              FileUtils.mkdir_p(dirname)
+            rescue SystemCallError
+              raise ActiveRecord::NoDatabaseError.new(connection_pool: @pool)
             end
           end
         end
@@ -196,14 +191,16 @@ module ActiveRecord
         !@memory_database
       end
 
-      def active?
-        @raw_connection && !@raw_connection.closed?
+      def supports_virtual_columns?
+        database_version >= "3.31.0"
       end
 
-      def return_value_after_insert?(column) # :nodoc:
-        column.auto_populated?
+      def connected?
+        !(@raw_connection.nil? || @raw_connection.closed?)
       end
 
+      alias_method :active?, :connected?
+
       alias :reset! :reconnect!
 
       # Disconnects from the database if already connected. Otherwise, this
@@ -236,6 +233,10 @@ module ActiveRecord
         true
       end
 
+      def supports_deferrable_constraints?
+        true
+      end
+
       # REFERENTIAL INTEGRITY ====================================
 
       def disable_referential_integrity # :nodoc:
@@ -258,7 +259,7 @@ module ActiveRecord
 
         unless result.blank?
           tables = result.map { |row| row["table"] }
-          raise ActiveRecord::StatementInvalid.new("Foreign key violations found: #{tables.join(", ")}", sql: sql)
+          raise ActiveRecord::StatementInvalid.new("Foreign key violations found: #{tables.join(", ")}", sql: sql, connection_pool: @pool)
         end
       end
 
@@ -290,6 +291,7 @@ module ActiveRecord
       end
 
       def add_column(table_name, column_name, type, **options) # :nodoc:
+        type = type.to_sym
         if invalid_alter_table_type?(type, options)
           alter_table(table_name) do |definition|
             definition.column(column_name, type, **options)
@@ -365,15 +367,31 @@ module ActiveRecord
       end
       alias :add_belongs_to :add_reference
 
+      FK_REGEX = /.*FOREIGN KEY\s+\("(\w+)"\)\s+REFERENCES\s+"(\w+)"\s+\("(\w+)"\)/
+      DEFERRABLE_REGEX = /DEFERRABLE INITIALLY (\w+)/
       def foreign_keys(table_name)
         # SQLite returns 1 row for each column of composite foreign keys.
         fk_info = internal_exec_query("PRAGMA foreign_key_list(#{quote(table_name)})", "SCHEMA")
+        # Deferred or immediate foreign keys can only be seen in the CREATE TABLE sql
+        fk_defs = table_structure_sql(table_name)
+                    .select do |column_string|
+                      column_string.start_with?("CONSTRAINT") &&
+                      column_string.include?("FOREIGN KEY")
+                    end
+                    .to_h do |fk_string|
+                      _, from, table, to = fk_string.match(FK_REGEX).to_a
+                      _, mode = fk_string.match(DEFERRABLE_REGEX).to_a
+                      deferred = mode&.downcase&.to_sym || false
+                      [[table, from, to], deferred]
+                    end
+
         grouped_fk = fk_info.group_by { |row| row["id"] }.values.each { |group| group.sort_by! { |row| row["seq"] } }
         grouped_fk.map do |group|
           row = group.first
           options = {
             on_delete: extract_foreign_key_action(row["on_delete"]),
-            on_update: extract_foreign_key_action(row["on_update"])
+            on_update: extract_foreign_key_action(row["on_update"]),
+            deferrable: fk_defs[[row["table"], row["from"], row["to"]]]
           }
 
           if group.one?
@@ -454,8 +472,12 @@ module ActiveRecord
         end
 
         def table_structure(table_name)
-          structure = internal_exec_query("PRAGMA table_info(#{quote_table_name(table_name)})", "SCHEMA")
-          raise(ActiveRecord::StatementInvalid, "Could not find table '#{table_name}'") if structure.empty?
+          structure = if supports_virtual_columns?
+            internal_exec_query("PRAGMA table_xinfo(#{quote_table_name(table_name)})", "SCHEMA")
+          else
+            internal_exec_query("PRAGMA table_info(#{quote_table_name(table_name)})", "SCHEMA")
+          end
+          raise ActiveRecord::StatementInvalid.new("Could not find table '#{table_name}'", connection_pool: @pool) if structure.empty?
           table_structure_with_collation(table_name, structure)
         end
         alias column_definitions table_structure
@@ -494,8 +516,9 @@ module ActiveRecord
         # See: https://www.sqlite.org/lang_altertable.html
         # SQLite has an additional restriction on the ALTER TABLE statement
         def invalid_alter_table_type?(type, options)
-          type.to_sym == :primary_key || options[:primary_key] ||
-            options[:null] == false && options[:default].nil?
+          type == :primary_key || options[:primary_key] ||
+            options[:null] == false && options[:default].nil? ||
+            (type == :virtual && options[:stored])
         end
 
         def alter_table(
@@ -551,12 +574,6 @@ module ActiveRecord
                  options[:rename][column.name.to_sym] ||
                  column.name) : column.name
 
-              if column.has_default?
-                type = lookup_cast_type_from_column(column)
-                default = type.deserialize(column.default)
-                default = -> { column.default_function } if default.nil?
-              end
-
               column_options = {
                 limit: column.limit,
                 precision: column.precision,
@@ -566,19 +583,31 @@ module ActiveRecord
                 primary_key: column_name == from_primary_key
               }
 
-              unless column.auto_increment?
-                column_options[:default] = default
+              if column.virtual?
+                column_options[:as] = column.default_function
+                column_options[:stored] = column.virtual_stored?
+                column_options[:type] = column.type
+              elsif column.has_default?
+                type = lookup_cast_type_from_column(column)
+                default = type.deserialize(column.default)
+                default = -> { column.default_function } if default.nil?
+
+                unless column.auto_increment?
+                  column_options[:default] = default
+                end
               end
 
-              column_type = column.bigint? ? :bigint : column.type
+              column_type = column.virtual? ? :virtual : (column.bigint? ? :bigint : column.type)
               @definition.column(column_name, column_type, **column_options)
             end
 
             yield @definition if block_given?
           end
           copy_table_indexes(from, to, options[:rename] || {})
+
+          columns_to_copy = @definition.columns.reject { |col| col.options.key?(:as) }.map(&:name)
           copy_table_contents(from, to,
-            @definition.columns.map(&:name),
+            columns_to_copy,
             options[:rename] || {})
         end
 
@@ -643,32 +672,22 @@ module ActiveRecord
 
         COLLATE_REGEX = /.*"(\w+)".*collate\s+"(\w+)".*/i
         PRIMARY_KEY_AUTOINCREMENT_REGEX = /.*"(\w+)".+PRIMARY KEY AUTOINCREMENT/i
+        GENERATED_ALWAYS_AS_REGEX = /.*"(\w+)".+GENERATED ALWAYS AS \((.+)\) (?:STORED|VIRTUAL)/i
 
         def table_structure_with_collation(table_name, basic_structure)
           collation_hash = {}
           auto_increments = {}
-          sql = <<~SQL
-            SELECT sql FROM
-              (SELECT * FROM sqlite_master UNION ALL
-               SELECT * FROM sqlite_temp_master)
-            WHERE type = 'table' AND name = #{quote(table_name)}
-          SQL
-
-          # Result will have following sample string
-          # CREATE TABLE "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
-          #                       "password_digest" varchar COLLATE "NOCASE");
-          result = query_value(sql, "SCHEMA")
+          generated_columns = {}
 
-          if result
-            # Splitting with left parentheses and discarding the first part will return all
-            # columns separated with comma(,).
-            columns_string = result.split("(", 2).last
+          column_strings = table_structure_sql(table_name)
 
-            columns_string.split(",").each do |column_string|
+          if column_strings.any?
+            column_strings.each do |column_string|
               # This regex will match the column name and collation type and will save
               # the value in $1 and $2 respectively.
               collation_hash[$1] = $2 if COLLATE_REGEX =~ column_string
               auto_increments[$1] = true if PRIMARY_KEY_AUTOINCREMENT_REGEX =~ column_string
+              generated_columns[$1] = $2 if GENERATED_ALWAYS_AS_REGEX =~ column_string
             end
 
             basic_structure.map do |column|
@@ -682,6 +701,10 @@ module ActiveRecord
                 column["auto_increment"] = true
               end
 
+              if generated_columns.has_key?(column_name)
+                column["dflt_value"] = generated_columns[column_name]
+              end
+
               column
             end
           else
@@ -689,6 +712,28 @@ module ActiveRecord
           end
         end
 
+        def table_structure_sql(table_name)
+          sql = <<~SQL
+            SELECT sql FROM
+              (SELECT * FROM sqlite_master UNION ALL
+               SELECT * FROM sqlite_temp_master)
+            WHERE type = 'table' AND name = #{quote(table_name)}
+          SQL
+
+          # Result will have following sample string
+          # CREATE TABLE "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+          #                       "password_digest" varchar COLLATE "NOCASE");
+          result = query_value(sql, "SCHEMA")
+
+          return [] unless result
+
+          # Splitting with left parentheses and discarding the first part will return all
+          # columns separated with comma(,).
+          columns_string = result.split("(", 2).last
+
+          columns_string.split(",").map(&:strip)
+        end
+
         def arel_visitor
           Arel::Visitors::SQLite.new(self)
         end
@@ -723,29 +768,16 @@ module ActiveRecord
             end
           end
 
-          # Enforce foreign key constraints
-          # https://www.sqlite.org/pragma.html#pragma_foreign_keys
-          # https://www.sqlite.org/foreignkeys.html
-          raw_execute("PRAGMA foreign_keys = ON", "SCHEMA")
-          unless @memory_database
-            # Journal mode WAL allows for greater concurrency (many readers + one writer)
-            # https://www.sqlite.org/pragma.html#pragma_journal_mode
-            raw_execute("PRAGMA journal_mode = WAL", "SCHEMA")
-            # Set more relaxed level of database durability
-            # 2 = "FULL" (sync on every write), 1 = "NORMAL" (sync every 1000 written pages) and 0 = "NONE"
-            # https://www.sqlite.org/pragma.html#pragma_synchronous
-            raw_execute("PRAGMA synchronous = NORMAL", "SCHEMA")
-            # Set the global memory map so all processes can share some data
-            # https://www.sqlite.org/pragma.html#pragma_mmap_size
-            # https://www.sqlite.org/mmap.html
-            raw_execute("PRAGMA mmap_size = #{128.megabytes}", "SCHEMA")
+          super
+
+          pragmas = @config.fetch(:pragmas, {}).stringify_keys
+          DEFAULT_PRAGMAS.merge(pragmas).each do |pragma, value|
+            if ::SQLite3::Pragmas.method_defined?("#{pragma}=")
+              @raw_connection.public_send("#{pragma}=", value)
+            else
+              warn "Unknown SQLite pragma: #{pragma}"
+            end
           end
-          # Impose a limit on the WAL file to prevent unlimited growth
-          # https://www.sqlite.org/pragma.html#pragma_journal_size_limit
-          raw_execute("PRAGMA journal_size_limit = #{64.megabytes}", "SCHEMA")
-          # Set the local connection cache to 2000 pages
-          # https://www.sqlite.org/pragma.html#pragma_cache_size
-          raw_execute("PRAGMA cache_size = 2000", "SCHEMA")
         end
     end
     ActiveSupport.run_load_hooks(:active_record_sqlite3adapter, SQLite3Adapter)

data/lib/active_record/connection_adapters/trilogy/database_statements.rb

@@ -12,12 +12,12 @@ module ActiveRecord
           result
         end
 
-        def internal_exec_query(sql, name = "SQL", binds = [], prepare: false, async: false) # :nodoc:
+        def internal_exec_query(sql, name = "SQL", binds = [], prepare: false, async: false, allow_retry: false) # :nodoc:
           sql = transform_query(sql)
           check_if_write_query(sql)
           mark_transaction_written_if_write(sql)
 
-          result = raw_execute(sql, name, async: async)
+          result = raw_execute(sql, name, async: async, allow_retry: allow_retry)
           ActiveRecord::Result.new(result.fields, result.to_a)
         end
 
@@ -26,7 +26,8 @@ module ActiveRecord
           check_if_write_query(sql)
           mark_transaction_written_if_write(sql)
 
-          raw_execute(to_sql(sql, binds), name)
+          sql, _binds = sql_for_insert(sql, pk, binds, returning)
+          raw_execute(sql, name)
         end
 
         def exec_delete(sql, name = nil, binds = []) # :nodoc:
@@ -42,19 +43,24 @@ module ActiveRecord
 
         private
           def raw_execute(sql, name, async: false, allow_retry: false, materialize_transactions: true)
-            log(sql, name, async: async) do
+            log(sql, name, async: async) do |notification_payload|
               with_raw_connection(allow_retry: allow_retry, materialize_transactions: materialize_transactions) do |conn|
                 sync_timezone_changes(conn)
                 result = conn.query(sql)
                 verified!
                 handle_warnings(sql)
+                notification_payload[:row_count] = result.count
                 result
               end
             end
           end
 
           def last_inserted_id(result)
-            result.last_insert_id
+            if supports_insert_returning?
+              super
+            else
+              result.last_insert_id
+            end
           end
 
           def sync_timezone_changes(conn)
@@ -90,7 +96,7 @@ module ActiveRecord
 
               yield
             ensure
-              conn.set_server_option(::Trilogy::SET_SERVER_MULTI_STATEMENTS_OFF)
+              conn.set_server_option(::Trilogy::SET_SERVER_MULTI_STATEMENTS_OFF) if active?
             end
           end
       end

data/lib/active_record/connection_adapters/trilogy_adapter.rb

@@ -2,44 +2,17 @@
 
 require "active_record/connection_adapters/abstract_mysql_adapter"
 
-gem "trilogy", "~> 2.4"
+gem "trilogy", "~> 2.7"
 require "trilogy"
 
 require "active_record/connection_adapters/trilogy/database_statements"
 
 module ActiveRecord
-  module ConnectionHandling # :nodoc:
-    def trilogy_adapter_class
-      ConnectionAdapters::TrilogyAdapter
-    end
-
-    # Establishes a connection to the database that's used by all Active Record objects.
-    def trilogy_connection(config)
-      configuration = config.dup
-
-      # Set FOUND_ROWS capability on the connection so UPDATE queries returns number of rows
-      # matched rather than number of rows updated.
-      configuration[:found_rows] = true
-
-      options = [
-        configuration[:host],
-        configuration[:port],
-        configuration[:database],
-        configuration[:username],
-        configuration[:password],
-        configuration[:socket],
-        0
-      ]
-
-      trilogy_adapter_class.new nil, logger, options, configuration
-    end
-  end
   module ConnectionAdapters
     class TrilogyAdapter < AbstractMysqlAdapter
       ER_BAD_DB_ERROR = 1049
       ER_DBACCESS_DENIED_ERROR = 1044
       ER_ACCESS_DENIED_ERROR = 1045
-      ER_SERVER_SHUTDOWN = 1053
 
       ADAPTER_NAME = "Trilogy"
 
@@ -99,16 +72,22 @@ module ActiveRecord
           end
       end
 
-      def initialize(...)
-        super
+      def initialize(config, *)
+        config = config.dup
+
+        # Trilogy ignores `socket` if `host is set. We want the opposite to allow
+        # configuring UNIX domain sockets via `DATABASE_URL`.
+        config.delete(:host) if config[:socket]
 
-        if @config[:prepared_statements]
+        # Set FOUND_ROWS capability on the connection so UPDATE queries returns number of rows
+        # matched rather than number of rows updated.
+        config[:found_rows] = true
+
+        if config[:prepared_statements]
           raise ArgumentError, "Trilogy currently doesn't support prepared statements. Remove `prepared_statements: true` from your database configuration."
         end
 
-        # Trilogy ignore `socket` if `host is set. We want the opposite to allow
-        # configuring UNIX domain sockets via `DATABASE_URL`.
-        @config.delete(:host) if @config[:socket]
+        super
       end
 
       TYPE_MAP = Type::TypeMap.new.tap { |m| initialize_type_map(m) }
@@ -137,14 +116,12 @@ module ActiveRecord
         true
       end
 
-      def quote_string(string)
-        with_raw_connection(allow_retry: true, materialize_transactions: false) do |conn|
-          conn.escape(string)
-        end
+      def connected?
+        !(@raw_connection.nil? || @raw_connection.closed?)
       end
 
       def active?
-        !(@raw_connection.nil? || @raw_connection.closed?) && @lock.synchronize { @raw_connection&.ping } || false
+        connected? && @lock.synchronize { @raw_connection&.ping } || false
       rescue ::Trilogy::Error
         false
       end
@@ -206,7 +183,7 @@ module ActiveRecord
         end
 
         def full_version
-          schema_cache.database_version.full_version_string
+          database_version.full_version_string
         end
 
         def get_full_version
@@ -219,18 +196,12 @@ module ActiveRecord
           if exception.is_a?(::Trilogy::TimeoutError) && !exception.error_code
             return ActiveRecord::AdapterTimeout.new(message, sql: sql, binds: binds, connection_pool: @pool)
           end
-          error_code = exception.error_code if exception.respond_to?(:error_code)
-
-          case error_code
-          when ER_SERVER_SHUTDOWN
-            return ConnectionFailed.new(message, connection_pool: @pool)
-          end
 
           case exception
-          when Errno::EPIPE, SocketError, IOError
+          when ::Trilogy::ConnectionClosed, ::Trilogy::EOFError
             return ConnectionFailed.new(message, connection_pool: @pool)
           when ::Trilogy::Error
-            if /Connection reset by peer|TRILOGY_CLOSED_CONNECTION|TRILOGY_INVALID_SEQUENCE_ID|TRILOGY_UNEXPECTED_PACKET/.match?(exception.message)
+            if exception.is_a?(SystemCallError) || exception.message.include?("TRILOGY_INVALID_SEQUENCE_ID")
               return ConnectionFailed.new(message, connection_pool: @pool)
             end
           end