activerecord 7.1.5.1 → 8.0.2

data/lib/active_record/connection_adapters/sqlite3_adapter.rb

@@ -11,20 +11,13 @@ require "active_record/connection_adapters/sqlite3/schema_definitions"
 require "active_record/connection_adapters/sqlite3/schema_dumper"
 require "active_record/connection_adapters/sqlite3/schema_statements"
 
-gem "sqlite3", ">= 1.4"
+gem "sqlite3", ">= 2.1"
 require "sqlite3"
 
-module ActiveRecord
-  module ConnectionHandling # :nodoc:
-    def sqlite3_adapter_class
-      ConnectionAdapters::SQLite3Adapter
-    end
-
-    def sqlite3_connection(config)
-      sqlite3_adapter_class.new(config)
-    end
-  end
+# Suppress the warning that SQLite3 issues when open writable connections are carried across fork()
+SQLite3::ForkSafety.suppress_warnings!
 
+module ActiveRecord
   module ConnectionAdapters # :nodoc:
     # = Active Record SQLite3 Adapter
     #
@@ -55,7 +48,7 @@ module ActiveRecord
           args << "-header" if options[:header]
           args << File.expand_path(config.database, Rails.respond_to?(:root) ? Rails.root : nil)
 
-          find_cmd_and_exec("sqlite3", *args)
+          find_cmd_and_exec(ActiveRecord.database_cli[:sqlite], *args)
         end
       end
 
@@ -88,6 +81,15 @@ module ActiveRecord
         json:         { name: "json" },
       }
 
+      DEFAULT_PRAGMAS = {
+        "foreign_keys"        => true,
+        "journal_mode"        => :wal,
+        "synchronous"         => :normal,
+        "mmap_size"           => 134217728, # 128 megabytes
+        "journal_size_limit"  => 67108864, # 64 megabytes
+        "cache_size"          => 2000
+      }
+
       class StatementPool < ConnectionAdapters::StatementPool # :nodoc:
         alias reset clear
 
@@ -113,20 +115,21 @@ module ActiveRecord
           dirname = File.dirname(@config[:database])
           unless File.directory?(dirname)
             begin
-              Dir.mkdir(dirname)
-            rescue Errno::ENOENT => error
-              if error.message.include?("No such file or directory")
-                raise ActiveRecord::NoDatabaseError.new(connection_pool: @pool)
-              else
-                raise
-              end
+              FileUtils.mkdir_p(dirname)
+            rescue SystemCallError
+              raise ActiveRecord::NoDatabaseError.new(connection_pool: @pool)
             end
           end
         end
 
+        @last_affected_rows = nil
+        @previous_read_uncommitted = nil
         @config[:strict] = ConnectionAdapters::SQLite3Adapter.strict_strings_by_default unless @config.key?(:strict)
-        @connection_parameters = @config.merge(database: @config[:database].to_s, results_as_hash: true)
-        @use_insert_returning = @config.key?(:insert_returning) ? self.class.type_cast_config_to_boolean(@config[:insert_returning]) : true
+        @connection_parameters = @config.merge(
+          database: @config[:database].to_s,
+          results_as_hash: true,
+          default_transaction_mode: :immediate,
+        )
       end
 
       def database_exists?
@@ -196,12 +199,19 @@ module ActiveRecord
         !@memory_database
       end
 
-      def active?
-        @raw_connection && !@raw_connection.closed?
+      def supports_virtual_columns?
+        database_version >= "3.31.0"
+      end
+
+      def connected?
+        !(@raw_connection.nil? || @raw_connection.closed?)
       end
 
-      def return_value_after_insert?(column) # :nodoc:
-        column.auto_populated?
+      def active?
+        if connected?
+          verified!
+          true
+        end
       end
 
       alias :reset! :reconnect!
@@ -236,6 +246,10 @@ module ActiveRecord
         true
       end
 
+      def supports_deferrable_constraints?
+        true
+      end
+
       # REFERENTIAL INTEGRITY ====================================
 
       def disable_referential_integrity # :nodoc:
@@ -258,7 +272,7 @@ module ActiveRecord
 
         unless result.blank?
           tables = result.map { |row| row["table"] }
-          raise ActiveRecord::StatementInvalid.new("Foreign key violations found: #{tables.join(", ")}", sql: sql)
+          raise ActiveRecord::StatementInvalid.new("Foreign key violations found: #{tables.join(", ")}", sql: sql, connection_pool: @pool)
         end
       end
 
@@ -277,6 +291,38 @@ module ActiveRecord
         exec_query "DROP INDEX #{quote_column_name(index_name)}"
       end
 
+      VIRTUAL_TABLE_REGEX = /USING\s+(\w+)\s*\((.+)\)/i
+
+      # Returns a list of defined virtual tables
+      def virtual_tables
+        query = <<~SQL
+          SELECT name, sql FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL %';
+        SQL
+
+        exec_query(query, "SCHEMA").cast_values.each_with_object({}) do |row, memo|
+          table_name, sql = row[0], row[1]
+          _, module_name, arguments = sql.match(VIRTUAL_TABLE_REGEX).to_a
+          memo[table_name] = [module_name, arguments]
+        end.to_a
+      end
+
+      # Creates a virtual table
+      #
+      # Example:
+      #   create_virtual_table :emails, :fts5, ['sender', 'title',' body']
+      def create_virtual_table(table_name, module_name, values)
+        exec_query "CREATE VIRTUAL TABLE IF NOT EXISTS #{table_name} USING #{module_name} (#{values.join(", ")})"
+      end
+
+      # Drops a virtual table
+      #
+      # Although this command ignores +module_name+ and +values+,
+      # it can be helpful to provide these in a migration's +change+ method so it can be reverted.
+      # In that case, +module_name+, +values+ and +options+ will be used by #create_virtual_table.
+      def drop_virtual_table(table_name, module_name, values, **options)
+        drop_table(table_name)
+      end
+
       # Renames a table.
       #
       # Example:
@@ -290,6 +336,7 @@ module ActiveRecord
       end
 
       def add_column(table_name, column_name, type, **options) # :nodoc:
+        type = type.to_sym
         if invalid_alter_table_type?(type, options)
           alter_table(table_name) do |definition|
             definition.column(column_name, type, **options)
@@ -365,15 +412,31 @@ module ActiveRecord
       end
       alias :add_belongs_to :add_reference
 
+      FK_REGEX = /.*FOREIGN KEY\s+\("([^"]+)"\)\s+REFERENCES\s+"(\w+)"\s+\("(\w+)"\)/
+      DEFERRABLE_REGEX = /DEFERRABLE INITIALLY (\w+)/
       def foreign_keys(table_name)
         # SQLite returns 1 row for each column of composite foreign keys.
         fk_info = internal_exec_query("PRAGMA foreign_key_list(#{quote(table_name)})", "SCHEMA")
+        # Deferred or immediate foreign keys can only be seen in the CREATE TABLE sql
+        fk_defs = table_structure_sql(table_name)
+                    .select do |column_string|
+                      column_string.start_with?("CONSTRAINT") &&
+                      column_string.include?("FOREIGN KEY")
+                    end
+                    .to_h do |fk_string|
+                      _, from, table, to = fk_string.match(FK_REGEX).to_a
+                      _, mode = fk_string.match(DEFERRABLE_REGEX).to_a
+                      deferred = mode&.downcase&.to_sym || false
+                      [[table, from, to], deferred]
+                    end
+
         grouped_fk = fk_info.group_by { |row| row["id"] }.values.each { |group| group.sort_by! { |row| row["seq"] } }
         grouped_fk.map do |group|
           row = group.first
           options = {
             on_delete: extract_foreign_key_action(row["on_delete"]),
-            on_update: extract_foreign_key_action(row["on_update"])
+            on_update: extract_foreign_key_action(row["on_update"]),
+            deferrable: fk_defs[[row["table"], row["from"], row["to"]]]
           }
 
           if group.one?
@@ -410,10 +473,6 @@ module ActiveRecord
         @config.fetch(:flags, 0).anybits?(::SQLite3::Constants::Open::SHAREDCACHE)
       end
 
-      def use_insert_returning?
-        @use_insert_returning
-      end
-
       def get_database_version # :nodoc:
         SQLite3Adapter::Version.new(query_value("SELECT sqlite_version(*)", "SCHEMA"))
       end
@@ -454,8 +513,8 @@ module ActiveRecord
         end
 
         def table_structure(table_name)
-          structure = internal_exec_query("PRAGMA table_info(#{quote_table_name(table_name)})", "SCHEMA")
-          raise(ActiveRecord::StatementInvalid, "Could not find table '#{table_name}'") if structure.empty?
+          structure = table_info(table_name)
+          raise ActiveRecord::StatementInvalid.new("Could not find table '#{table_name}'", connection_pool: @pool) if structure.empty?
           table_structure_with_collation(table_name, structure)
         end
         alias column_definitions table_structure
@@ -494,8 +553,9 @@ module ActiveRecord
         # See: https://www.sqlite.org/lang_altertable.html
         # SQLite has an additional restriction on the ALTER TABLE statement
         def invalid_alter_table_type?(type, options)
-          type.to_sym == :primary_key || options[:primary_key] ||
-            options[:null] == false && options[:default].nil?
+          type == :primary_key || options[:primary_key] ||
+            options[:null] == false && options[:default].nil? ||
+            (type == :virtual && options[:stored])
         end
 
         def alter_table(
@@ -551,12 +611,6 @@ module ActiveRecord
                  options[:rename][column.name.to_sym] ||
                  column.name) : column.name
 
-              if column.has_default?
-                type = lookup_cast_type_from_column(column)
-                default = type.deserialize(column.default)
-                default = -> { column.default_function } if default.nil?
-              end
-
               column_options = {
                 limit: column.limit,
                 precision: column.precision,
@@ -566,19 +620,31 @@ module ActiveRecord
                 primary_key: column_name == from_primary_key
               }
 
-              unless column.auto_increment?
-                column_options[:default] = default
+              if column.virtual?
+                column_options[:as] = column.default_function
+                column_options[:stored] = column.virtual_stored?
+                column_options[:type] = column.type
+              elsif column.has_default?
+                type = lookup_cast_type_from_column(column)
+                default = type.deserialize(column.default)
+                default = -> { column.default_function } if default.nil?
+
+                unless column.auto_increment?
+                  column_options[:default] = default
+                end
               end
 
-              column_type = column.bigint? ? :bigint : column.type
+              column_type = column.virtual? ? :virtual : (column.bigint? ? :bigint : column.type)
               @definition.column(column_name, column_type, **column_options)
             end
 
             yield @definition if block_given?
           end
           copy_table_indexes(from, to, options[:rename] || {})
+
+          columns_to_copy = @definition.columns.reject { |col| col.options.key?(:as) }.map(&:name)
           copy_table_contents(from, to,
-            @definition.columns.map(&:name),
+            columns_to_copy,
             options[:rename] || {})
         end
 
@@ -636,6 +702,8 @@ module ActiveRecord
             InvalidForeignKey.new(message, sql: sql, binds: binds, connection_pool: @pool)
           elsif exception.message.match?(/called on a closed database/i)
             ConnectionNotEstablished.new(exception, connection_pool: @pool)
+          elsif exception.is_a?(::SQLite3::BusyException)
+            StatementTimeout.new(message, sql: sql, binds: binds, connection_pool: @pool)
           else
             super
           end
@@ -643,32 +711,22 @@ module ActiveRecord
 
         COLLATE_REGEX = /.*"(\w+)".*collate\s+"(\w+)".*/i
         PRIMARY_KEY_AUTOINCREMENT_REGEX = /.*"(\w+)".+PRIMARY KEY AUTOINCREMENT/i
+        GENERATED_ALWAYS_AS_REGEX = /.*"(\w+)".+GENERATED ALWAYS AS \((.+)\) (?:STORED|VIRTUAL)/i
 
         def table_structure_with_collation(table_name, basic_structure)
           collation_hash = {}
           auto_increments = {}
-          sql = <<~SQL
-            SELECT sql FROM
-              (SELECT * FROM sqlite_master UNION ALL
-               SELECT * FROM sqlite_temp_master)
-            WHERE type = 'table' AND name = #{quote(table_name)}
-          SQL
-
-          # Result will have following sample string
-          # CREATE TABLE "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
-          #                       "password_digest" varchar COLLATE "NOCASE");
-          result = query_value(sql, "SCHEMA")
+          generated_columns = {}
 
-          if result
-            # Splitting with left parentheses and discarding the first part will return all
-            # columns separated with comma(,).
-            columns_string = result.split("(", 2).last
+          column_strings = table_structure_sql(table_name, basic_structure.map { |column| column["name"] })
 
-            columns_string.split(",").each do |column_string|
+          if column_strings.any?
+            column_strings.each do |column_string|
               # This regex will match the column name and collation type and will save
               # the value in $1 and $2 respectively.
               collation_hash[$1] = $2 if COLLATE_REGEX =~ column_string
               auto_increments[$1] = true if PRIMARY_KEY_AUTOINCREMENT_REGEX =~ column_string
+              generated_columns[$1] = $2 if GENERATED_ALWAYS_AS_REGEX =~ column_string
             end
 
             basic_structure.map do |column|
@@ -682,6 +740,10 @@ module ActiveRecord
                 column["auto_increment"] = true
               end
 
+              if generated_columns.has_key?(column_name)
+                column["dflt_value"] = generated_columns[column_name]
+              end
+
               column
             end
           else
@@ -689,6 +751,50 @@ module ActiveRecord
           end
         end
 
+        UNQUOTED_OPEN_PARENS_REGEX = /\((?![^'"]*['"][^'"]*$)/
+        FINAL_CLOSE_PARENS_REGEX = /\);*\z/
+
+        def table_structure_sql(table_name, column_names = nil)
+          unless column_names
+            column_info = table_info(table_name)
+            column_names = column_info.map { |column| column["name"] }
+          end
+
+          sql = <<~SQL
+            SELECT sql FROM
+              (SELECT * FROM sqlite_master UNION ALL
+               SELECT * FROM sqlite_temp_master)
+            WHERE type = 'table' AND name = #{quote(table_name)}
+          SQL
+
+          # Result will have following sample string
+          # CREATE TABLE "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+          #                       "password_digest" varchar COLLATE "NOCASE",
+          #                       "o_id" integer,
+          #                       CONSTRAINT "fk_rails_78146ddd2e" FOREIGN KEY ("o_id") REFERENCES "os" ("id"));
+          result = query_value(sql, "SCHEMA")
+
+          return [] unless result
+
+          # Splitting with left parentheses and discarding the first part will return all
+          # columns separated with comma(,).
+          result.partition(UNQUOTED_OPEN_PARENS_REGEX)
+                .last
+                .sub(FINAL_CLOSE_PARENS_REGEX, "")
+                # column definitions can have a comma in them, so split on commas followed
+                # by a space and a column name in quotes or followed by the keyword CONSTRAINT
+                .split(/,(?=\s(?:CONSTRAINT|"(?:#{Regexp.union(column_names).source})"))/i)
+                .map(&:strip)
+        end
+
+        def table_info(table_name)
+          if supports_virtual_columns?
+            internal_exec_query("PRAGMA table_xinfo(#{quote_table_name(table_name)})", "SCHEMA")
+          else
+            internal_exec_query("PRAGMA table_info(#{quote_table_name(table_name)})", "SCHEMA")
+          end
+        end
+
         def arel_visitor
           Arel::Visitors::SQLite.new(self)
         end
@@ -715,37 +821,27 @@ module ActiveRecord
           if @config[:timeout] && @config[:retries]
             raise ArgumentError, "Cannot specify both timeout and retries arguments"
           elsif @config[:timeout]
-            @raw_connection.busy_timeout(self.class.type_cast_config_to_integer(@config[:timeout]))
+            timeout = self.class.type_cast_config_to_integer(@config[:timeout])
+            raise TypeError, "timeout must be integer, not #{timeout}" unless timeout.is_a?(Integer)
+            @raw_connection.busy_handler_timeout = timeout
           elsif @config[:retries]
+            ActiveRecord.deprecator.warn(<<~MSG)
+              The retries option is deprecated and will be removed in Rails 8.1. Use timeout instead.
+            MSG
             retries = self.class.type_cast_config_to_integer(@config[:retries])
-            raw_connection.busy_handler do |count|
-              count <= retries
-            end
+            raw_connection.busy_handler { |count| count <= retries }
           end
 
-          # Enforce foreign key constraints
-          # https://www.sqlite.org/pragma.html#pragma_foreign_keys
-          # https://www.sqlite.org/foreignkeys.html
-          raw_execute("PRAGMA foreign_keys = ON", "SCHEMA")
-          unless @memory_database
-            # Journal mode WAL allows for greater concurrency (many readers + one writer)
-            # https://www.sqlite.org/pragma.html#pragma_journal_mode
-            raw_execute("PRAGMA journal_mode = WAL", "SCHEMA")
-            # Set more relaxed level of database durability
-            # 2 = "FULL" (sync on every write), 1 = "NORMAL" (sync every 1000 written pages) and 0 = "NONE"
-            # https://www.sqlite.org/pragma.html#pragma_synchronous
-            raw_execute("PRAGMA synchronous = NORMAL", "SCHEMA")
-            # Set the global memory map so all processes can share some data
-            # https://www.sqlite.org/pragma.html#pragma_mmap_size
-            # https://www.sqlite.org/mmap.html
-            raw_execute("PRAGMA mmap_size = #{128.megabytes}", "SCHEMA")
+          super
+
+          pragmas = @config.fetch(:pragmas, {}).stringify_keys
+          DEFAULT_PRAGMAS.merge(pragmas).each do |pragma, value|
+            if ::SQLite3::Pragmas.method_defined?("#{pragma}=")
+              @raw_connection.public_send("#{pragma}=", value)
+            else
+              warn "Unknown SQLite pragma: #{pragma}"
+            end
           end
-          # Impose a limit on the WAL file to prevent unlimited growth
-          # https://www.sqlite.org/pragma.html#pragma_journal_size_limit
-          raw_execute("PRAGMA journal_size_limit = #{64.megabytes}", "SCHEMA")
-          # Set the local connection cache to 2000 pages
-          # https://www.sqlite.org/pragma.html#pragma_cache_size
-          raw_execute("PRAGMA cache_size = 2000", "SCHEMA")
         end
     end
     ActiveSupport.run_load_hooks(:active_record_sqlite3adapter, SQLite3Adapter)

data/lib/active_record/connection_adapters/statement_pool.rb

@@ -50,8 +50,10 @@ module ActiveRecord
       end
 
       def delete(key)
-        dealloc cache[key]
-        cache.delete(key)
+        if stmt = cache.delete(key)
+          dealloc(stmt)
+        end
+        stmt
       end
 
       private

data/lib/active_record/connection_adapters/trilogy/database_statements.rb

@@ -4,93 +4,63 @@ module ActiveRecord
   module ConnectionAdapters
     module Trilogy
       module DatabaseStatements
-        def select_all(*, **) # :nodoc:
-          result = super
-          with_raw_connection do |conn|
-            conn.next_result while conn.more_results_exist?
-          end
-          result
-        end
-
-        def internal_exec_query(sql, name = "SQL", binds = [], prepare: false, async: false) # :nodoc:
-          sql = transform_query(sql)
-          check_if_write_query(sql)
-          mark_transaction_written_if_write(sql)
-
-          result = raw_execute(sql, name, async: async)
-          ActiveRecord::Result.new(result.fields, result.to_a)
-        end
-
         def exec_insert(sql, name, binds, pk = nil, sequence_name = nil, returning: nil) # :nodoc:
-          sql = transform_query(sql)
-          check_if_write_query(sql)
-          mark_transaction_written_if_write(sql)
-
-          raw_execute(to_sql(sql, binds), name)
+          sql, _binds = sql_for_insert(sql, pk, binds, returning)
+          internal_execute(sql, name)
         end
 
-        def exec_delete(sql, name = nil, binds = []) # :nodoc:
-          sql = transform_query(sql)
-          check_if_write_query(sql)
-          mark_transaction_written_if_write(sql)
-
-          result = raw_execute(to_sql(sql, binds), name)
-          result.affected_rows
-        end
-
-        alias :exec_update :exec_delete # :nodoc:
-
         private
-          def raw_execute(sql, name, async: false, allow_retry: false, materialize_transactions: true)
-            log(sql, name, async: async) do
-              with_raw_connection(allow_retry: allow_retry, materialize_transactions: materialize_transactions) do |conn|
-                sync_timezone_changes(conn)
-                result = conn.query(sql)
-                verified!
-                handle_warnings(sql)
-                result
-              end
+          def perform_query(raw_connection, sql, binds, type_casted_binds, prepare:, notification_payload:, batch: false)
+            reset_multi_statement = if batch && !@config[:multi_statement]
+              raw_connection.set_server_option(::Trilogy::SET_SERVER_MULTI_STATEMENTS_ON)
+              true
             end
-          end
 
-          def last_inserted_id(result)
-            result.last_insert_id
-          end
-
-          def sync_timezone_changes(conn)
-            # Sync any changes since connection last established.
+            # Make sure we carry over any changes to ActiveRecord.default_timezone that have been
+            # made since we established the connection
             if default_timezone == :local
-              conn.query_flags |= ::Trilogy::QUERY_FLAGS_LOCAL_TIMEZONE
+              raw_connection.query_flags |= ::Trilogy::QUERY_FLAGS_LOCAL_TIMEZONE
             else
-              conn.query_flags &= ~::Trilogy::QUERY_FLAGS_LOCAL_TIMEZONE
+              raw_connection.query_flags &= ~::Trilogy::QUERY_FLAGS_LOCAL_TIMEZONE
+            end
+
+            result = raw_connection.query(sql)
+            while raw_connection.more_results_exist?
+              raw_connection.next_result
+            end
+            verified!
+            handle_warnings(sql)
+            notification_payload[:row_count] = result.count
+            result
+          ensure
+            if reset_multi_statement && active?
+              raw_connection.set_server_option(::Trilogy::SET_SERVER_MULTI_STATEMENTS_OFF)
             end
           end
 
-          def execute_batch(statements, name = nil)
-            statements = statements.map { |sql| transform_query(sql) }
-            combine_multi_statements(statements).each do |statement|
-              with_raw_connection do |conn|
-                raw_execute(statement, name)
-                conn.next_result while conn.more_results_exist?
-              end
+          def cast_result(result)
+            if result.fields.empty?
+              ActiveRecord::Result.empty
+            else
+              ActiveRecord::Result.new(result.fields, result.rows)
             end
           end
 
-          def multi_statements_enabled?
-            !!@config[:multi_statement]
+          def affected_rows(result)
+            result.affected_rows
           end
 
-          def with_multi_statements
-            if multi_statements_enabled?
-              return yield
+          def last_inserted_id(result)
+            if supports_insert_returning?
+              super
+            else
+              result.last_insert_id
             end
+          end
 
-            with_raw_connection do |conn|
-              conn.set_server_option(::Trilogy::SET_SERVER_MULTI_STATEMENTS_ON)
-
-              yield
-            ensure
-              conn.set_server_option(::Trilogy::SET_SERVER_MULTI_STATEMENTS_OFF)
+          def execute_batch(statements, name = nil, **kwargs)
+            combine_multi_statements(statements).each do |statement|
+              raw_execute(statement, name, batch: true, **kwargs)
             end
           end
       end