activerecord 7.2.0 → 8.0.0.beta1

data/lib/active_record/connection_adapters/sqlite3/database_statements.rb

@@ -21,87 +21,24 @@ module ActiveRecord
           SQLite3::ExplainPrettyPrinter.new.pp(result)
         end
 
-        def internal_exec_query(sql, name = nil, binds = [], prepare: false, async: false, allow_retry: false) # :nodoc:
-          sql = transform_query(sql)
-          check_if_write_query(sql)
-
-          mark_transaction_written_if_write(sql)
-
-          type_casted_binds = type_casted_binds(binds)
-
-          log(sql, name, binds, type_casted_binds, async: async) do |notification_payload|
-            with_raw_connection do |conn|
-              # Don't cache statements if they are not prepared
-              unless prepare
-                stmt = conn.prepare(sql)
-                begin
-                  cols = stmt.columns
-                  unless without_prepared_statement?(binds)
-                    stmt.bind_params(type_casted_binds)
-                  end
-                  records = stmt.to_a
-                ensure
-                  stmt.close
-                end
-              else
-                stmt = @statements[sql] ||= conn.prepare(sql)
-                cols = stmt.columns
-                stmt.reset!
-                stmt.bind_params(type_casted_binds)
-                records = stmt.to_a
-              end
-              verified!
-
-              result = build_result(columns: cols, rows: records)
-              notification_payload[:row_count] = result.length
-              result
-            end
-          end
-        end
-
-        def exec_delete(sql, name = "SQL", binds = []) # :nodoc:
-          internal_exec_query(sql, name, binds)
-          @raw_connection.changes
+        def begin_deferred_transaction(isolation = nil) # :nodoc:
+          internal_begin_transaction(:deferred, isolation)
         end
-        alias :exec_update :exec_delete
 
         def begin_isolated_db_transaction(isolation) # :nodoc:
-          raise TransactionIsolationError, "SQLite3 only supports the `read_uncommitted` transaction isolation level" if isolation != :read_uncommitted
-          raise StandardError, "You need to enable the shared-cache mode in SQLite mode before attempting to change the transaction isolation level" unless shared_cache?
-
-          with_raw_connection(allow_retry: true, materialize_transactions: false) do |conn|
-            ActiveSupport::IsolatedExecutionState[:active_record_read_uncommitted] = conn.get_first_value("PRAGMA read_uncommitted")
-            conn.read_uncommitted = true
-            begin_db_transaction
-          end
+          internal_begin_transaction(:deferred, isolation)
         end
 
         def begin_db_transaction # :nodoc:
-          log("begin transaction", "TRANSACTION") do
-            with_raw_connection(allow_retry: true, materialize_transactions: false) do |conn|
-              result = conn.transaction
-              verified!
-              result
-            end
-          end
+          internal_begin_transaction(:immediate, nil)
         end
 
         def commit_db_transaction # :nodoc:
-          log("commit transaction", "TRANSACTION") do
-            with_raw_connection(allow_retry: true, materialize_transactions: false) do |conn|
-              conn.commit
-            end
-          end
-          reset_read_uncommitted
+          internal_execute("COMMIT TRANSACTION", "TRANSACTION", allow_retry: true, materialize_transactions: false)
         end
 
         def exec_rollback_db_transaction # :nodoc:
-          log("rollback transaction", "TRANSACTION") do
-            with_raw_connection(allow_retry: true, materialize_transactions: false) do |conn|
-              conn.rollback
-            end
-          end
-          reset_read_uncommitted
+          internal_execute("ROLLBACK TRANSACTION", "TRANSACTION", allow_retry: true, materialize_transactions: false)
         end
 
         # https://stackoverflow.com/questions/17574784
@@ -113,47 +50,82 @@ module ActiveRecord
           HIGH_PRECISION_CURRENT_TIMESTAMP
         end
 
+        def execute(...) # :nodoc:
+          # SQLite3Adapter was refactored to use ActiveRecord::Result internally
+          # but for backward compatibility we have to keep returning arrays of hashes here
+          super&.to_a
+        end
+
+        def reset_isolation_level # :nodoc:
+          internal_execute("PRAGMA read_uncommitted=#{@previous_read_uncommitted}", "TRANSACTION", allow_retry: true, materialize_transactions: false)
+          @previous_read_uncommitted = nil
+        end
+
         private
-          def raw_execute(sql, name, async: false, allow_retry: false, materialize_transactions: false)
-            log(sql, name, async: async) do |notification_payload|
-              with_raw_connection(allow_retry: allow_retry, materialize_transactions: materialize_transactions) do |conn|
-                result = conn.execute(sql)
-                verified!
-                notification_payload[:row_count] = result.length
-                result
-              end
+          def internal_begin_transaction(mode, isolation)
+            if isolation
+              raise TransactionIsolationError, "SQLite3 only supports the `read_uncommitted` transaction isolation level" if isolation != :read_uncommitted
+              raise StandardError, "You need to enable the shared-cache mode in SQLite mode before attempting to change the transaction isolation level" unless shared_cache?
+            end
+
+            internal_execute("BEGIN #{mode} TRANSACTION", allow_retry: true, materialize_transactions: false)
+            if isolation
+              @previous_read_uncommitted = query_value("PRAGMA read_uncommitted")
+              internal_execute("PRAGMA read_uncommitted=ON", "TRANSACTION", allow_retry: true, materialize_transactions: false)
             end
           end
 
-          def reset_read_uncommitted
-            read_uncommitted = ActiveSupport::IsolatedExecutionState[:active_record_read_uncommitted]
-            return unless read_uncommitted
+          def perform_query(raw_connection, sql, binds, type_casted_binds, prepare:, notification_payload:, batch: false)
+            if batch
+              raw_connection.execute_batch2(sql)
+            elsif prepare
+              stmt = @statements[sql] ||= raw_connection.prepare(sql)
+              stmt.reset!
+              stmt.bind_params(type_casted_binds)
+
+              result = if stmt.column_count.zero? # No return
+                stmt.step
+                ActiveRecord::Result.empty
+              else
+                ActiveRecord::Result.new(stmt.columns, stmt.to_a)
+              end
+            else
+              # Don't cache statements if they are not prepared.
+              stmt = raw_connection.prepare(sql)
+              begin
+                unless binds.nil? || binds.empty?
+                  stmt.bind_params(type_casted_binds)
+                end
+                result = if stmt.column_count.zero? # No return
+                  stmt.step
+                  ActiveRecord::Result.empty
+                else
+                  ActiveRecord::Result.new(stmt.columns, stmt.to_a)
+                end
+              ensure
+                stmt.close
+              end
+            end
+            @last_affected_rows = raw_connection.changes
+            verified!
 
-            @raw_connection&.read_uncommitted = read_uncommitted
+            notification_payload[:row_count] = result&.length || 0
+            result
           end
 
-          def execute_batch(statements, name = nil)
-            statements = statements.map { |sql| transform_query(sql) }
-            sql = combine_multi_statements(statements)
-
-            check_if_write_query(sql)
-            mark_transaction_written_if_write(sql)
+          def cast_result(result)
+            # Given that SQLite3 doesn't really a Result type, raw_execute already return an ActiveRecord::Result
+            # and we have nothing to cast here.
+            result
+          end
 
-            log(sql, name) do |notification_payload|
-              with_raw_connection do |conn|
-                result = conn.execute_batch2(sql)
-                verified!
-                notification_payload[:row_count] = result.length
-                result
-              end
-            end
+          def affected_rows(result)
+            @last_affected_rows
           end
 
-          def build_fixture_statements(fixture_set)
-            fixture_set.flat_map do |table_name, fixtures|
-              next if fixtures.empty?
-              fixtures.map { |fixture| build_fixture_sql([fixture], table_name) }
-            end.compact
+          def execute_batch(statements, name = nil, **kwargs)
+            sql = combine_multi_statements(statements)
+            raw_execute(sql, name, batch: true, **kwargs)
           end
 
           def build_truncate_statement(table_name)
@@ -163,6 +135,10 @@ module ActiveRecord
           def returning_column_values(result)
             result.rows.first
           end
+
+          def default_insert_value(column)
+            column.default
+          end
       end
     end
   end

data/lib/active_record/connection_adapters/sqlite3/schema_creation.rb

@@ -5,12 +5,6 @@ module ActiveRecord
     module SQLite3
       class SchemaCreation < SchemaCreation # :nodoc:
         private
-          def visit_AddForeignKey(o)
-            super.dup.tap do |sql|
-              sql << " DEFERRABLE INITIALLY #{o.options[:deferrable].to_s.upcase}" if o.deferrable
-            end
-          end
-
           def visit_ForeignKeyDefinition(o)
             super.dup.tap do |sql|
               sql << " DEFERRABLE INITIALLY #{o.deferrable.to_s.upcase}" if o.deferrable

data/lib/active_record/connection_adapters/sqlite3/schema_dumper.rb

@@ -5,6 +5,19 @@ module ActiveRecord
     module SQLite3
       class SchemaDumper < ConnectionAdapters::SchemaDumper # :nodoc:
         private
+          def virtual_tables(stream)
+            virtual_tables = @connection.virtual_tables
+            if virtual_tables.any?
+              stream.puts
+              stream.puts "  # Virtual tables defined in this database."
+              stream.puts "  # Note that virtual tables may not work with other database engines. Be careful if changing database."
+              virtual_tables.sort.each do |table_name, options|
+                module_name, arguments = options
+                stream.puts "  create_virtual_table #{table_name.inspect}, #{module_name.inspect}, #{arguments.split(", ").inspect}"
+              end
+            end
+          end
+
           def default_primary_key?(column)
             schema_type(column) == :integer
           end

data/lib/active_record/connection_adapters/sqlite3/schema_statements.rb

@@ -82,6 +82,10 @@ module ActiveRecord
           alter_table(from_table, foreign_keys)
         end
 
+        def virtual_table_exists?(table_name)
+          query_values(data_source_sql(table_name, type: "VIRTUAL TABLE"), "SCHEMA").any?
+        end
+
         def check_constraints(table_name)
           table_sql = query_value(<<-SQL, "SCHEMA")
             SELECT sql
@@ -176,7 +180,8 @@ module ActiveRecord
             scope = quoted_scope(name, type: type)
             scope[:type] ||= "'table','view'"
 
-            sql = +"SELECT name FROM sqlite_master WHERE name <> 'sqlite_sequence'"
+            sql = +"SELECT name FROM pragma_table_list WHERE schema <> 'temp'"
+            sql << " AND name NOT IN ('sqlite_sequence', 'sqlite_schema')"
             sql << " AND name = #{scope[:name]}" if scope[:name]
             sql << " AND type IN (#{scope[:type]})"
             sql
@@ -189,6 +194,8 @@ module ActiveRecord
                 "'table'"
               when "VIEW"
                 "'view'"
+              when "VIRTUAL TABLE"
+                "'virtual'"
               end
             scope = {}
             scope[:name] = quote(name) if name

data/lib/active_record/connection_adapters/sqlite3_adapter.rb

@@ -11,9 +11,12 @@ require "active_record/connection_adapters/sqlite3/schema_definitions"
 require "active_record/connection_adapters/sqlite3/schema_dumper"
 require "active_record/connection_adapters/sqlite3/schema_statements"
 
-gem "sqlite3", ">= 1.4"
+gem "sqlite3", ">= 2.1"
 require "sqlite3"
 
+# Suppress the warning that SQLite3 issues when open writable connections are carried across fork()
+SQLite3::ForkSafety.suppress_warnings!
+
 module ActiveRecord
   module ConnectionAdapters # :nodoc:
     # = Active Record SQLite3 Adapter
@@ -45,7 +48,7 @@ module ActiveRecord
           args << "-header" if options[:header]
           args << File.expand_path(config.database, Rails.respond_to?(:root) ? Rails.root : nil)
 
-          find_cmd_and_exec("sqlite3", *args)
+          find_cmd_and_exec(ActiveRecord.database_cli[:sqlite], *args)
         end
       end
 
@@ -119,9 +122,14 @@ module ActiveRecord
           end
         end
 
+        @last_affected_rows = nil
+        @previous_read_uncommitted = nil
         @config[:strict] = ConnectionAdapters::SQLite3Adapter.strict_strings_by_default unless @config.key?(:strict)
-        @connection_parameters = @config.merge(database: @config[:database].to_s, results_as_hash: true)
-        @use_insert_returning = @config.key?(:insert_returning) ? self.class.type_cast_config_to_boolean(@config[:insert_returning]) : true
+        @connection_parameters = @config.merge(
+          database: @config[:database].to_s,
+          results_as_hash: true,
+          default_transaction_mode: :immediate,
+        )
       end
 
       def database_exists?
@@ -278,6 +286,38 @@ module ActiveRecord
         exec_query "DROP INDEX #{quote_column_name(index_name)}"
       end
 
+      VIRTUAL_TABLE_REGEX = /USING\s+(\w+)\s*\((.+)\)/i
+
+      # Returns a list of defined virtual tables
+      def virtual_tables
+        query = <<~SQL
+          SELECT name, sql FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL %';
+        SQL
+
+        exec_query(query, "SCHEMA").cast_values.each_with_object({}) do |row, memo|
+          table_name, sql = row[0], row[1]
+          _, module_name, arguments = sql.match(VIRTUAL_TABLE_REGEX).to_a
+          memo[table_name] = [module_name, arguments]
+        end.to_a
+      end
+
+      # Creates a virtual table
+      #
+      # Example:
+      #   create_virtual_table :emails, :fts5, ['sender', 'title',' body']
+      def create_virtual_table(table_name, module_name, values)
+        exec_query "CREATE VIRTUAL TABLE IF NOT EXISTS #{table_name} USING #{module_name} (#{values.join(", ")})"
+      end
+
+      # Drops a virtual table
+      #
+      # Although this command ignores +module_name+ and +values+,
+      # it can be helpful to provide these in a migration's +change+ method so it can be reverted.
+      # In that case, +module_name+, +values+ and +options+ will be used by #create_virtual_table.
+      def drop_virtual_table(table_name, module_name, values, **options)
+        drop_table(table_name)
+      end
+
       # Renames a table.
       #
       # Example:
@@ -428,10 +468,6 @@ module ActiveRecord
         @config.fetch(:flags, 0).anybits?(::SQLite3::Constants::Open::SHAREDCACHE)
       end
 
-      def use_insert_returning?
-        @use_insert_returning
-      end
-
       def get_database_version # :nodoc:
         SQLite3Adapter::Version.new(query_value("SELECT sqlite_version(*)", "SCHEMA"))
       end
@@ -661,6 +697,8 @@ module ActiveRecord
             InvalidForeignKey.new(message, sql: sql, binds: binds, connection_pool: @pool)
           elsif exception.message.match?(/called on a closed database/i)
             ConnectionNotEstablished.new(exception, connection_pool: @pool)
+          elsif exception.is_a?(::SQLite3::BusyException)
+            StatementTimeout.new(message, sql: sql, binds: binds, connection_pool: @pool)
           else
             super
           end
@@ -687,6 +725,8 @@ module ActiveRecord
             end
 
             basic_structure.map do |column|
+              column = column.to_h
+
               column_name = column["name"]
 
               if collation_hash.has_key? column_name
@@ -778,12 +818,15 @@ module ActiveRecord
           if @config[:timeout] && @config[:retries]
             raise ArgumentError, "Cannot specify both timeout and retries arguments"
           elsif @config[:timeout]
-            @raw_connection.busy_timeout(self.class.type_cast_config_to_integer(@config[:timeout]))
+            timeout = self.class.type_cast_config_to_integer(@config[:timeout])
+            raise TypeError, "timeout must be integer, not #{timeout}" unless timeout.is_a?(Integer)
+            @raw_connection.busy_handler_timeout = timeout
           elsif @config[:retries]
+            ActiveRecord.deprecator.warn(<<~MSG)
+              The retries option is deprecated and will be removed in Rails 8.1. Use timeout instead.
+            MSG
             retries = self.class.type_cast_config_to_integer(@config[:retries])
-            raw_connection.busy_handler do |count|
-              count <= retries
-            end
+            raw_connection.busy_handler { |count| count <= retries }
           end
 
           super

data/lib/active_record/connection_adapters/trilogy/database_statements.rb

@@ -4,93 +4,63 @@ module ActiveRecord
   module ConnectionAdapters
     module Trilogy
       module DatabaseStatements
-        def internal_exec_query(sql, name = "SQL", binds = [], prepare: false, async: false, allow_retry: false) # :nodoc:
-          sql = transform_query(sql)
-          check_if_write_query(sql)
-          mark_transaction_written_if_write(sql)
-
-          result = raw_execute(sql, name, async: async, allow_retry: allow_retry)
-          ActiveRecord::Result.new(result.fields, result.to_a)
-        end
-
         def exec_insert(sql, name, binds, pk = nil, sequence_name = nil, returning: nil) # :nodoc:
-          sql = transform_query(sql)
-          check_if_write_query(sql)
-          mark_transaction_written_if_write(sql)
-
           sql, _binds = sql_for_insert(sql, pk, binds, returning)
-          raw_execute(sql, name)
-        end
-
-        def exec_delete(sql, name = nil, binds = []) # :nodoc:
-          sql = transform_query(sql)
-          check_if_write_query(sql)
-          mark_transaction_written_if_write(sql)
-
-          result = raw_execute(to_sql(sql, binds), name)
-          result.affected_rows
+          internal_execute(sql, name)
         end
 
-        alias :exec_update :exec_delete # :nodoc:
-
         private
-          def raw_execute(sql, name, async: false, allow_retry: false, materialize_transactions: true)
-            log(sql, name, async: async) do |notification_payload|
-              with_raw_connection(allow_retry: allow_retry, materialize_transactions: materialize_transactions) do |conn|
-                sync_timezone_changes(conn)
-                result = conn.query(sql)
-                while conn.more_results_exist?
-                  conn.next_result
-                end
-                verified!
-                handle_warnings(sql)
-                notification_payload[:row_count] = result.count
-                result
-              end
+          def perform_query(raw_connection, sql, binds, type_casted_binds, prepare:, notification_payload:, batch: false)
+            reset_multi_statement = if batch && !@config[:multi_statement]
+              raw_connection.set_server_option(::Trilogy::SET_SERVER_MULTI_STATEMENTS_ON)
+              true
             end
-          end
 
-          def last_inserted_id(result)
-            if supports_insert_returning?
-              super
+            # Make sure we carry over any changes to ActiveRecord.default_timezone that have been
+            # made since we established the connection
+            if default_timezone == :local
+              raw_connection.query_flags |= ::Trilogy::QUERY_FLAGS_LOCAL_TIMEZONE
             else
-              result.last_insert_id
+              raw_connection.query_flags &= ~::Trilogy::QUERY_FLAGS_LOCAL_TIMEZONE
             end
-          end
 
-          def sync_timezone_changes(conn)
-            # Sync any changes since connection last established.
-            if default_timezone == :local
-              conn.query_flags |= ::Trilogy::QUERY_FLAGS_LOCAL_TIMEZONE
-            else
-              conn.query_flags &= ~::Trilogy::QUERY_FLAGS_LOCAL_TIMEZONE
+            result = raw_connection.query(sql)
+            while raw_connection.more_results_exist?
+              raw_connection.next_result
+            end
+            verified!
+            handle_warnings(sql)
+            notification_payload[:row_count] = result.count
+            result
+          ensure
+            if reset_multi_statement && active?
+              raw_connection.set_server_option(::Trilogy::SET_SERVER_MULTI_STATEMENTS_OFF)
             end
           end
 
-          def execute_batch(statements, name = nil)
-            statements = statements.map { |sql| transform_query(sql) }
-            combine_multi_statements(statements).each do |statement|
-              with_raw_connection do |conn|
-                raw_execute(statement, name)
-              end
+          def cast_result(result)
+            if result.fields.empty?
+              ActiveRecord::Result.empty
+            else
+              ActiveRecord::Result.new(result.fields, result.rows)
             end
           end
 
-          def multi_statements_enabled?
-            !!@config[:multi_statement]
+          def affected_rows(result)
+            result.affected_rows
           end
 
-          def with_multi_statements
-            if multi_statements_enabled?
-              return yield
+          def last_inserted_id(result)
+            if supports_insert_returning?
+              super
+            else
+              result.last_insert_id
             end
+          end
 
-            with_raw_connection do |conn|
-              conn.set_server_option(::Trilogy::SET_SERVER_MULTI_STATEMENTS_ON)
-
-              yield
-            ensure
-              conn.set_server_option(::Trilogy::SET_SERVER_MULTI_STATEMENTS_OFF) if active?
+          def execute_batch(statements, name = nil, **kwargs)
+            combine_multi_statements(statements).each do |statement|
+              raw_execute(statement, name, batch: true, **kwargs)
             end
           end
       end

data/lib/active_record/connection_adapters/trilogy_adapter.rb

@@ -149,23 +149,6 @@ module ActiveRecord
           TYPE_MAP.lookup(type).is_a?(Type::String) || TYPE_MAP.lookup(type).is_a?(Type::Text)
         end
 
-        def each_hash(result)
-          return to_enum(:each_hash, result) unless block_given?
-
-          keys = result.fields.map(&:to_sym)
-          result.rows.each do |row|
-            hash = {}
-            idx = 0
-            row.each do |value|
-              hash[keys[idx]] = value
-              idx += 1
-            end
-            yield hash
-          end
-
-          nil
-        end
-
         def error_number(exception)
           exception.error_code if exception.respond_to?(:error_code)
         end

data/lib/active_record/connection_handling.rb

@@ -87,6 +87,8 @@ module ActiveRecord
 
       connections = []
 
+      @shard_keys = shards.keys
+
       if shards.empty?
         shards[:default] = database
       end
@@ -175,6 +177,18 @@ module ActiveRecord
       connected_to_stack.pop
     end
 
+    # Passes the block to +connected_to+ for every +shard+ the
+    # model is configured to connect to (if any), and returns the
+    # results in an array.
+    #
+    # Optionally, +role+ and/or +prevent_writes+ can be passed which
+    # will be forwarded to each +connected_to+ call.
+    def connected_to_all_shards(role: nil, prevent_writes: false, &blk)
+      shard_keys.map do |shard|
+        connected_to(shard: shard, role: role, prevent_writes: prevent_writes, &blk)
+      end
+    end
+
     # Use a specified connection.
     #
     # This method is useful for ensuring that a specific connection is
@@ -359,6 +373,14 @@ module ActiveRecord
       connection_pool.schema_cache.clear!
     end
 
+    def shard_keys
+      connection_class_for_self.instance_variable_get(:@shard_keys) || []
+    end
+
+    def sharded?
+      shard_keys.any?
+    end
+
     private
       def resolve_config_for_connection(config_or_env)
         raise "Anonymous class is not allowed." unless name