activerecord 6.1.7 → 7.2.0

data/lib/active_record/connection_adapters/sqlite3/database_statements.rb

@@ -15,41 +15,25 @@ module ActiveRecord
           !READ_QUERY.match?(sql.b)
         end
 
-        def explain(arel, binds = [])
-          sql = "EXPLAIN QUERY PLAN #{to_sql(arel, binds)}"
-          SQLite3::ExplainPrettyPrinter.new.pp(exec_query(sql, "EXPLAIN", []))
+        def explain(arel, binds = [], _options = [])
+          sql    = "EXPLAIN QUERY PLAN " + to_sql(arel, binds)
+          result = internal_exec_query(sql, "EXPLAIN", [])
+          SQLite3::ExplainPrettyPrinter.new.pp(result)
         end
 
-        def execute(sql, name = nil) #:nodoc:
-          if preventing_writes? && write_query?(sql)
-            raise ActiveRecord::ReadOnlyError, "Write query attempted while in readonly mode: #{sql}"
-          end
+        def internal_exec_query(sql, name = nil, binds = [], prepare: false, async: false, allow_retry: false) # :nodoc:
+          sql = transform_query(sql)
+          check_if_write_query(sql)
 
-          materialize_transactions
-          mark_transaction_written_if_write(sql)
-
-          log(sql, name) do
-            ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
-              @connection.execute(sql)
-            end
-          end
-        end
-
-        def exec_query(sql, name = nil, binds = [], prepare: false)
-          if preventing_writes? && write_query?(sql)
-            raise ActiveRecord::ReadOnlyError, "Write query attempted while in readonly mode: #{sql}"
-          end
-
-          materialize_transactions
           mark_transaction_written_if_write(sql)
 
           type_casted_binds = type_casted_binds(binds)
 
-          log(sql, name, binds, type_casted_binds) do
-            ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
+          log(sql, name, binds, type_casted_binds, async: async) do |notification_payload|
+            with_raw_connection do |conn|
               # Don't cache statements if they are not prepared
               unless prepare
-                stmt = @connection.prepare(sql)
+                stmt = conn.prepare(sql)
                 begin
                   cols = stmt.columns
                   unless without_prepared_statement?(binds)
@@ -60,76 +44,111 @@ module ActiveRecord
                   stmt.close
                 end
               else
-                stmt = @statements[sql] ||= @connection.prepare(sql)
+                stmt = @statements[sql] ||= conn.prepare(sql)
                 cols = stmt.columns
                 stmt.reset!
                 stmt.bind_params(type_casted_binds)
                 records = stmt.to_a
               end
+              verified!
 
-              build_result(columns: cols, rows: records)
+              result = build_result(columns: cols, rows: records)
+              notification_payload[:row_count] = result.length
+              result
             end
           end
         end
 
-        def exec_delete(sql, name = "SQL", binds = [])
-          exec_query(sql, name, binds)
-          @connection.changes
+        def exec_delete(sql, name = "SQL", binds = []) # :nodoc:
+          internal_exec_query(sql, name, binds)
+          @raw_connection.changes
         end
         alias :exec_update :exec_delete
 
-        def begin_isolated_db_transaction(isolation) #:nodoc
+        def begin_isolated_db_transaction(isolation) # :nodoc:
           raise TransactionIsolationError, "SQLite3 only supports the `read_uncommitted` transaction isolation level" if isolation != :read_uncommitted
           raise StandardError, "You need to enable the shared-cache mode in SQLite mode before attempting to change the transaction isolation level" unless shared_cache?
 
-          Thread.current.thread_variable_set("read_uncommitted", @connection.get_first_value("PRAGMA read_uncommitted"))
-          @connection.read_uncommitted = true
-          begin_db_transaction
+          with_raw_connection(allow_retry: true, materialize_transactions: false) do |conn|
+            ActiveSupport::IsolatedExecutionState[:active_record_read_uncommitted] = conn.get_first_value("PRAGMA read_uncommitted")
+            conn.read_uncommitted = true
+            begin_db_transaction
+          end
         end
 
-        def begin_db_transaction #:nodoc:
-          log("begin transaction", "TRANSACTION") { @connection.transaction }
+        def begin_db_transaction # :nodoc:
+          log("begin transaction", "TRANSACTION") do
+            with_raw_connection(allow_retry: true, materialize_transactions: false) do |conn|
+              result = conn.transaction
+              verified!
+              result
+            end
+          end
         end
 
-        def commit_db_transaction #:nodoc:
-          log("commit transaction", "TRANSACTION") { @connection.commit }
+        def commit_db_transaction # :nodoc:
+          log("commit transaction", "TRANSACTION") do
+            with_raw_connection(allow_retry: true, materialize_transactions: false) do |conn|
+              conn.commit
+            end
+          end
           reset_read_uncommitted
         end
 
-        def exec_rollback_db_transaction #:nodoc:
-          log("rollback transaction", "TRANSACTION") { @connection.rollback }
+        def exec_rollback_db_transaction # :nodoc:
+          log("rollback transaction", "TRANSACTION") do
+            with_raw_connection(allow_retry: true, materialize_transactions: false) do |conn|
+              conn.rollback
+            end
+          end
           reset_read_uncommitted
         end
 
+        # https://stackoverflow.com/questions/17574784
+        # https://www.sqlite.org/lang_datefunc.html
+        HIGH_PRECISION_CURRENT_TIMESTAMP = Arel.sql("STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')", retryable: true).freeze # :nodoc:
+        private_constant :HIGH_PRECISION_CURRENT_TIMESTAMP
+
+        def high_precision_current_timestamp
+          HIGH_PRECISION_CURRENT_TIMESTAMP
+        end
+
         private
+          def raw_execute(sql, name, async: false, allow_retry: false, materialize_transactions: false)
+            log(sql, name, async: async) do |notification_payload|
+              with_raw_connection(allow_retry: allow_retry, materialize_transactions: materialize_transactions) do |conn|
+                result = conn.execute(sql)
+                verified!
+                notification_payload[:row_count] = result.length
+                result
+              end
+            end
+          end
+
           def reset_read_uncommitted
-            read_uncommitted = Thread.current.thread_variable_get("read_uncommitted")
+            read_uncommitted = ActiveSupport::IsolatedExecutionState[:active_record_read_uncommitted]
             return unless read_uncommitted
 
-            @connection.read_uncommitted = read_uncommitted
+            @raw_connection&.read_uncommitted = read_uncommitted
           end
 
           def execute_batch(statements, name = nil)
+            statements = statements.map { |sql| transform_query(sql) }
             sql = combine_multi_statements(statements)
 
-            if preventing_writes? && write_query?(sql)
-              raise ActiveRecord::ReadOnlyError, "Write query attempted while in readonly mode: #{sql}"
-            end
-
-            materialize_transactions
+            check_if_write_query(sql)
             mark_transaction_written_if_write(sql)
 
-            log(sql, name) do
-              ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
-                @connection.execute_batch2(sql)
+            log(sql, name) do |notification_payload|
+              with_raw_connection do |conn|
+                result = conn.execute_batch2(sql)
+                verified!
+                notification_payload[:row_count] = result.length
+                result
               end
             end
           end
 
-          def last_inserted_id(result)
-            @connection.last_insert_row_id
-          end
-
           def build_fixture_statements(fixture_set)
             fixture_set.flat_map do |table_name, fixtures|
               next if fixtures.empty?
@@ -140,6 +159,10 @@ module ActiveRecord
           def build_truncate_statement(table_name)
             "DELETE FROM #{quote_table_name(table_name)}"
           end
+
+          def returning_column_values(result)
+            result.rows.first
+          end
       end
     end
   end

data/lib/active_record/connection_adapters/sqlite3/quoting.rb

@@ -4,20 +4,58 @@ module ActiveRecord
   module ConnectionAdapters
     module SQLite3
       module Quoting # :nodoc:
-        def quote_string(s)
-          @connection.class.quote(s)
-        end
+        extend ActiveSupport::Concern
+
+        QUOTED_COLUMN_NAMES = Concurrent::Map.new # :nodoc:
+        QUOTED_TABLE_NAMES = Concurrent::Map.new # :nodoc:
+
+        module ClassMethods # :nodoc:
+          def column_name_matcher
+            /
+              \A
+              (
+                (?:
+                  # "table_name"."column_name" | function(one or no argument)
+                  ((?:\w+\.|"\w+"\.)?(?:\w+|"\w+") | \w+\((?:|\g<2>)\))
+                )
+                (?:(?:\s+AS)?\s+(?:\w+|"\w+"))?
+              )
+              (?:\s*,\s*\g<1>)*
+              \z
+            /ix
+          end
 
-        def quote_table_name_for_assignment(table, attr)
-          quote_column_name(attr)
+          def column_name_with_order_matcher
+            /
+              \A
+              (
+                (?:
+                  # "table_name"."column_name" | function(one or no argument)
+                  ((?:\w+\.|"\w+"\.)?(?:\w+|"\w+") | \w+\((?:|\g<2>)\))
+                )
+                (?:\s+COLLATE\s+(?:\w+|"\w+"))?
+                (?:\s+ASC|\s+DESC)?
+              )
+              (?:\s*,\s*\g<1>)*
+              \z
+            /ix
+          end
+
+          def quote_column_name(name)
+            QUOTED_COLUMN_NAMES[name] ||= %Q("#{name.to_s.gsub('"', '""')}").freeze
+          end
+
+          def quote_table_name(name)
+            QUOTED_TABLE_NAMES[name] ||= %Q("#{name.to_s.gsub('"', '""').gsub(".", "\".\"")}").freeze
+          end
         end
 
-        def quote_table_name(name)
-          self.class.quoted_table_names[name] ||= super.gsub(".", "\".\"").freeze
+        def quote_string(s)
+          ::SQLite3::Database.quote(s)
         end
 
-        def quote_column_name(name)
-          self.class.quoted_column_names[name] ||= %Q("#{super.gsub('"', '""')}")
+        def quote_table_name_for_assignment(table, attr)
+          quote_column_name(attr)
         end
 
         def quoted_time(value)
@@ -45,57 +83,33 @@ module ActiveRecord
           0
         end
 
-        def column_name_matcher
-          COLUMN_NAME
-        end
-
-        def column_name_with_order_matcher
-          COLUMN_NAME_WITH_ORDER
+        def quote_default_expression(value, column) # :nodoc:
+          if value.is_a?(Proc)
+            value = value.call
+            if value.match?(/\A\w+\(.*\)\z/)
+              "(#{value})"
+            else
+              value
+            end
+          else
+            super
+          end
         end
 
-        COLUMN_NAME = /
-          \A
-          (
-            (?:
-              # "table_name"."column_name" | function(one or no argument)
-              ((?:\w+\.|"\w+"\.)?(?:\w+|"\w+")) | \w+\((?:|\g<2>)\)
-            )
-            (?:(?:\s+AS)?\s+(?:\w+|"\w+"))?
-          )
-          (?:\s*,\s*\g<1>)*
-          \z
-        /ix
-
-        COLUMN_NAME_WITH_ORDER = /
-          \A
-          (
-            (?:
-              # "table_name"."column_name" | function(one or no argument)
-              ((?:\w+\.|"\w+"\.)?(?:\w+|"\w+")) | \w+\((?:|\g<2>)\)
-            )
-            (?:\s+ASC|\s+DESC)?
-          )
-          (?:\s*,\s*\g<1>)*
-          \z
-        /ix
-
-        private_constant :COLUMN_NAME, :COLUMN_NAME_WITH_ORDER
-
-        private
-          def _type_cast(value)
-            case value
-            when BigDecimal
-              value.to_f
-            when String
-              if value.encoding == Encoding::ASCII_8BIT
-                super(value.encode(Encoding::UTF_8))
-              else
-                super
-              end
+        def type_cast(value) # :nodoc:
+          case value
+          when BigDecimal, Rational
+            value.to_f
+          when String
+            if value.encoding == Encoding::ASCII_8BIT
+              super(value.encode(Encoding::UTF_8))
             else
               super
             end
+          else
+            super
           end
+        end
       end
     end
   end

data/lib/active_record/connection_adapters/sqlite3/schema_creation.rb

@@ -5,6 +5,18 @@ module ActiveRecord
     module SQLite3
       class SchemaCreation < SchemaCreation # :nodoc:
         private
+          def visit_AddForeignKey(o)
+            super.dup.tap do |sql|
+              sql << " DEFERRABLE INITIALLY #{o.options[:deferrable].to_s.upcase}" if o.deferrable
+            end
+          end
+
+          def visit_ForeignKeyDefinition(o)
+            super.dup.tap do |sql|
+              sql << " DEFERRABLE INITIALLY #{o.deferrable.to_s.upcase}" if o.deferrable
+            end
+          end
+
           def supports_index_using?
             false
           end
@@ -13,6 +25,16 @@ module ActiveRecord
             if options[:collation]
               sql << " COLLATE \"#{options[:collation]}\""
             end
+
+            if as = options[:as]
+              sql << " GENERATED ALWAYS AS (#{as})"
+
+              if options[:stored]
+                sql << " STORED"
+              else
+                sql << " VIRTUAL"
+              end
+            end
             super
           end
       end

data/lib/active_record/connection_adapters/sqlite3/schema_definitions.rb

@@ -3,16 +3,36 @@
 module ActiveRecord
   module ConnectionAdapters
     module SQLite3
+      # = Active Record SQLite3 Adapter \Table Definition
       class TableDefinition < ActiveRecord::ConnectionAdapters::TableDefinition
+        def change_column(column_name, type, **options)
+          name = column_name.to_s
+          @columns_hash[name] = nil
+          column(name, type, **options)
+        end
+
         def references(*args, **options)
           super(*args, type: :integer, **options)
         end
         alias :belongs_to :references
 
+        def new_column_definition(name, type, **options) # :nodoc:
+          case type
+          when :virtual
+            type = options[:type]
+          end
+
+          super
+        end
+
         private
           def integer_like_primary_key_type(type, options)
             :primary_key
           end
+
+          def valid_column_definition_options
+            super + [:as, :type, :stored]
+          end
       end
     end
   end

data/lib/active_record/connection_adapters/sqlite3/schema_dumper.rb

@@ -12,6 +12,22 @@ module ActiveRecord
           def explicit_primary_key_default?(column)
             column.bigint?
           end
+
+          def prepare_column_options(column)
+            spec = super
+
+            if @connection.supports_virtual_columns? && column.virtual?
+              spec[:as] = extract_expression_for_virtual_column(column)
+              spec[:stored] = column.virtual_stored?
+              spec = { type: schema_type(column).inspect }.merge!(spec)
+            end
+
+            spec
+          end
+
+          def extract_expression_for_virtual_column(column)
+            column.default_function.inspect
+          end
       end
     end
   end

data/lib/active_record/connection_adapters/sqlite3/schema_statements.rb

@@ -6,7 +6,7 @@ module ActiveRecord
       module SchemaStatements # :nodoc:
         # Returns an array of indexes for the given table.
         def indexes(table_name)
-          exec_query("PRAGMA index_list(#{quote_table_name(table_name)})", "SCHEMA").map do |row|
+          internal_exec_query("PRAGMA index_list(#{quote_table_name(table_name)})", "SCHEMA").filter_map do |row|
             # Indexes SQLite creates implicitly for internal use start with "sqlite_".
             # See https://www.sqlite.org/fileformat2.html#intschema
             next if row["name"].start_with?("sqlite_")
@@ -21,9 +21,9 @@ module ActiveRecord
               WHERE name = #{quote(row['name'])} AND type = 'index'
             SQL
 
-            /\bON\b\s*"?(\w+?)"?\s*\((?<expressions>.+?)\)(?:\s*WHERE\b\s*(?<where>.+))?\z/i =~ index_sql
+            /\bON\b\s*"?(\w+?)"?\s*\((?<expressions>.+?)\)(?:\s*WHERE\b\s*(?<where>.+))?(?:\s*\/\*.*\*\/)?\z/i =~ index_sql
 
-            columns = exec_query("PRAGMA index_info(#{quote(row['name'])})", "SCHEMA").map do |col|
+            columns = internal_exec_query("PRAGMA index_info(#{quote(row['name'])})", "SCHEMA").map do |col|
               col["name"]
             end
 
@@ -49,10 +49,12 @@ module ActiveRecord
               where: where,
               orders: orders
             )
-          end.compact
+          end
         end
 
         def add_foreign_key(from_table, to_table, **options)
+          assert_valid_deferrable(options[:deferrable])
+
           alter_table(from_table) do |definition|
             to_table = strip_table_name_prefix_and_suffix(to_table)
             definition.foreign_key(to_table, **options)
@@ -60,6 +62,8 @@ module ActiveRecord
         end
 
         def remove_foreign_key(from_table, to_table = nil, **options)
+          return if options.delete(:if_exists) == true && !foreign_key_exists?(from_table, to_table)
+
           to_table ||= options[:to_table]
           options = options.except(:name, :to_table, :validate)
           foreign_keys = foreign_keys(from_table)
@@ -82,11 +86,11 @@ module ActiveRecord
           table_sql = query_value(<<-SQL, "SCHEMA")
             SELECT sql
             FROM sqlite_master
-            WHERE name = #{quote_table_name(table_name)} AND type = 'table'
+            WHERE name = #{quote(table_name)} AND type = 'table'
             UNION ALL
             SELECT sql
             FROM sqlite_temp_master
-            WHERE name = #{quote_table_name(table_name)} AND type = 'table'
+            WHERE name = #{quote(table_name)} AND type = 'table'
           SQL
 
           table_sql.to_s.scan(/CONSTRAINT\s+(?<name>\w+)\s+CHECK\s+\((?<expression>(:?[^()]|\(\g<expression>\))+)\)/i).map do |name, expression|
@@ -100,7 +104,9 @@ module ActiveRecord
           end
         end
 
-        def remove_check_constraint(table_name, expression = nil, **options)
+        def remove_check_constraint(table_name, expression = nil, if_exists: false, **options)
+          return if if_exists && !check_constraint_exists?(table_name, **options)
+
           check_constraints = check_constraints(table_name)
           chk_name_to_delete = check_constraint_for!(table_name, expression: expression, **options).name
           check_constraints.delete_if { |chk| chk.name == chk_name_to_delete }
@@ -111,9 +117,13 @@ module ActiveRecord
           SQLite3::SchemaDumper.create(self, options)
         end
 
+        def schema_creation # :nodoc
+          SQLite3::SchemaCreation.new(self)
+        end
+
         private
-          def schema_creation
-            SQLite3::SchemaCreation.new(self)
+          def valid_table_definition_options
+            super + [:rename]
           end
 
           def create_table_definition(name, **options)
@@ -124,21 +134,42 @@ module ActiveRecord
             super unless internal
           end
 
-          def new_column_from_field(table_name, field)
-            default = \
-              case field["dflt_value"]
-              when /^null$/i
-                nil
-              when /^'(.*)'$/m
-                $1.gsub("''", "'")
-              when /^"(.*)"$/m
-                $1.gsub('""', '"')
-              else
-                field["dflt_value"]
-              end
+          def new_column_from_field(table_name, field, definitions)
+            default = field["dflt_value"]
 
             type_metadata = fetch_type_metadata(field["type"])
-            Column.new(field["name"], default, type_metadata, field["notnull"].to_i == 0, collation: field["collation"])
+            default_value = extract_value_from_default(default)
+            generated_type = extract_generated_type(field)
+
+            if generated_type.present?
+              default_function = default
+            else
+              default_function = extract_default_function(default_value, default)
+            end
+
+            rowid = is_column_the_rowid?(field, definitions)
+
+            Column.new(
+              field["name"],
+              default_value,
+              type_metadata,
+              field["notnull"].to_i == 0,
+              default_function,
+              collation: field["collation"],
+              auto_increment: field["auto_increment"],
+              rowid: rowid,
+              generated_type: generated_type
+            )
+          end
+
+          INTEGER_REGEX = /integer/i
+          # if a rowid table has a primary key that consists of a single column
+          # and the declared type of that column is "INTEGER" in any mixture of upper and lower case,
+          # then the column becomes an alias for the rowid.
+          def is_column_the_rowid?(field, column_definitions)
+            return false unless INTEGER_REGEX.match?(field["type"]) && field["pk"] == 1
+            # is the primary key a single column?
+            column_definitions.one? { |c| c["pk"] > 0 }
           end
 
           def data_source_sql(name = nil, type: nil)
@@ -164,6 +195,19 @@ module ActiveRecord
             scope[:type] = type if type
             scope
           end
+
+          def assert_valid_deferrable(deferrable)
+            return if !deferrable || %i(immediate deferred).include?(deferrable)
+
+            raise ArgumentError, "deferrable must be `:immediate` or `:deferred`, got: `#{deferrable.inspect}`"
+          end
+
+          def extract_generated_type(field)
+            case field["hidden"]
+            when 2 then :virtual
+            when 3 then :stored
+            end
+          end
       end
     end
   end