activerecord-materialized 0.1.0

data/lib/activerecord/materialized/aggregate_analysis.rb

@@ -0,0 +1,132 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # Decides whether a view can be maintained by applying signed deltas
+    # (summary-delta IVM) instead of re-aggregating a partition's base rows. True
+    # only for a single-table GROUP BY without HAVING whose aggregates are all
+    # distributive (SUM/COUNT/COUNT(*)) and which carries a trustworthy row count
+    # so emptied partitions can be detected; everything else falls back to scoped
+    # recompute, which is always correct.
+    class AggregateAnalysis
+      extend T::Sig
+
+      # One classified aggregate column from a view's projection.
+      #
+      # @api private
+      class Column < T::Struct
+        const :name, String
+        const :function, Symbol
+        const :attribute, T.nilable(String)
+        const :counts_rows, T::Boolean, default: false
+      end
+
+      sig { params(relation: ::ActiveRecord::Relation).void }
+      def initialize(relation)
+        @relation = relation
+        @aggregate_columns = T.let(nil, T.nilable(T::Array[Column]))
+      end
+
+      sig { returns(T::Array[Column]) }
+      def aggregate_columns
+        @aggregate_columns ||= T.unsafe(@relation).select_values.filter_map { |value| classify(value) }
+      end
+
+      sig { returns(T::Boolean) }
+      def delta_maintainable?
+        single_table? && grouped? && !having? && distributive_aggregates? && aggregate_columns.any?(&:counts_rows)
+      end
+
+      sig { returns(T::Boolean) }
+      def distributive_aggregates?
+        aggregate_columns.any? && aggregate_columns.all? { |column| distributive?(column) }
+      end
+
+      sig { returns(T.nilable(Column)) }
+      def row_count_column
+        aggregate_columns.find(&:counts_rows)
+      end
+
+      private
+
+      sig { params(value: T.untyped).returns(T.nilable(Column)) }
+      def classify(value)
+        return nil unless value.is_a?(::Arel::Nodes::As)
+
+        node = value.left
+        name = alias_name(value)
+        case node
+        when ::Arel::Nodes::Sum then Column.new(name: name, function: :sum, attribute: attribute_name(node))
+        when ::Arel::Nodes::Avg then Column.new(name: name, function: :avg, attribute: attribute_name(node))
+        when ::Arel::Nodes::Min then Column.new(name: name, function: :min, attribute: attribute_name(node))
+        when ::Arel::Nodes::Max then Column.new(name: name, function: :max, attribute: attribute_name(node))
+        when ::Arel::Nodes::Count then count_column(name, node)
+        else Column.new(name: name, function: :other)
+        end
+      end
+
+      sig { params(name: String, node: T.untyped).returns(Column) }
+      def count_column(name, node)
+        return Column.new(name: name, function: :count_distinct, attribute: attribute_name(node)) if node.distinct
+        return Column.new(name: name, function: :count_star, counts_rows: true) if star?(node)
+
+        attribute = attribute_name(node)
+        Column.new(name: name, function: :count, attribute: attribute, counts_rows: not_null_column?(attribute))
+      end
+
+      sig { params(column: Column).returns(T::Boolean) }
+      def distributive?(column)
+        case column.function
+        # A SUM over a nullable column can be NULL, which a zero delta can't
+        # distinguish from 0, so only NOT NULL sums are delta-maintainable.
+        when :sum then !column.attribute.nil? && not_null_column?(column.attribute)
+        when :count then !column.attribute.nil?
+        when :count_star then true
+        else false
+        end
+      end
+
+      sig { params(node: T.untyped).returns(T.nilable(String)) }
+      def attribute_name(node)
+        inner = node.expressions.first
+        inner.is_a?(::Arel::Attributes::Attribute) ? T.unsafe(inner).name.to_s : nil
+      end
+
+      sig { params(node: T.untyped).returns(T::Boolean) }
+      def star?(node)
+        inner = node.expressions.first
+        !!(inner.is_a?(::Arel::Nodes::SqlLiteral) && inner.to_s == "*")
+      end
+
+      sig { params(value: ::Arel::Nodes::As).returns(String) }
+      def alias_name(value)
+        right = T.unsafe(value).right
+        right.respond_to?(:name) ? right.name.to_s : right.to_s
+      end
+
+      sig { params(attribute: T.nilable(String)).returns(T::Boolean) }
+      def not_null_column?(attribute)
+        return false if attribute.nil?
+
+        column = T.unsafe(@relation).klass.columns_hash[attribute]
+        !column.nil? && !column.null
+      end
+
+      sig { returns(T::Boolean) }
+      def single_table?
+        T.unsafe(@relation).joins_values.empty? && T.unsafe(@relation).from_clause.value.nil?
+      end
+
+      sig { returns(T::Boolean) }
+      def grouped?
+        T.unsafe(@relation).group_values.any?
+      end
+
+      sig { returns(T::Boolean) }
+      def having?
+        !T.unsafe(@relation).having_clause.empty?
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/async_refresher.rb

@@ -0,0 +1,105 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # In-process, debounced background refresher — the default `:async` dispatcher.
+    #
+    # @api private
+    class AsyncRefresher
+      class << self
+        extend T::Sig
+
+        sig { params(view_class: ViewClass).void }
+        def enqueue(view_class)
+          interval = view_class.resolved_refresh_debounce
+
+          mutex.synchronize do
+            pending[view_class.view_key] = view_class
+            schedule_unlocked(interval)
+          end
+        end
+
+        sig { void }
+        def flush!
+          mutex.synchronize do
+            cancel_timer_unlocked
+            drain_pending_unlocked
+          end
+        end
+
+        sig { returns(Integer) }
+        def pending_count
+          mutex.synchronize { pending.size }
+        end
+
+        sig { void }
+        def reset!
+          mutex.synchronize do
+            cancel_timer_unlocked
+            pending.clear
+          end
+        end
+
+        # When paused, refreshes accumulate and run only on an explicit flush! —
+        # no background timer fires.
+        sig { params(value: T::Boolean).void }
+        def paused=(value)
+          @paused = T.let(value, T.nilable(T::Boolean))
+        end
+
+        sig { returns(T::Boolean) }
+        def paused?
+          @paused = T.let(@paused, T.nilable(T::Boolean))
+          @paused || false
+        end
+
+        private
+
+        sig { returns(T::Hash[String, ViewClass]) }
+        def pending
+          @pending ||= T.let({}, T.nilable(T::Hash[String, ViewClass]))
+        end
+
+        sig { returns(Mutex) }
+        def mutex
+          @mutex ||= T.let(Mutex.new, T.nilable(Mutex))
+        end
+
+        sig { params(interval: T.any(Integer, Float)).void }
+        def schedule_unlocked(interval)
+          cancel_timer_unlocked
+          return if paused?
+
+          @timer_thread = T.let(
+            Thread.new do
+              sleep(interval) unless interval.zero?
+              mutex.synchronize { drain_pending_unlocked }
+            end,
+            T.nilable(Thread)
+          )
+        end
+
+        sig { void }
+        def cancel_timer_unlocked
+          return unless @timer_thread&.alive?
+
+          @timer_thread.kill
+          @timer_thread = nil
+        end
+
+        sig { void }
+        def drain_pending_unlocked
+          views = pending.values
+          pending.clear
+
+          views.each do |view_class|
+            next unless view_class.dirty?
+
+            view_class.refresh!
+          end
+        end
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/cache_table_schema.rb

@@ -0,0 +1,67 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # Infers a view's cache-table columns from its source relation and provisions the table.
+    #
+    # @api private
+    module CacheTableSchema
+      extend T::Sig
+
+      # An inferred cache-table column: name from the relation projection, type
+      # one of :integer, :decimal, :boolean, :datetime, :string.
+      class ColumnDefinition < T::Struct
+        const :name, String
+        const :type, Symbol
+      end
+
+      module_function
+
+      sig { params(view_class: T.class_of(::ActiveRecord::Base), relation: ::ActiveRecord::Relation).void }
+      def ensure_table!(view_class, relation)
+        return if view_class.table_exists?
+
+        build_table!(view_class.connection, view_class.table_name, relation)
+        view_class.reset_column_information
+      end
+
+      sig { params(view_class: T.class_of(::ActiveRecord::Base), table_name: String, relation: ::ActiveRecord::Relation).void }
+      def create_table!(view_class, table_name, relation)
+        build_table!(view_class.connection, table_name, relation)
+      end
+
+      # The inferred MV columns (names from the projection, types from a one-row
+      # probe). Shared by table creation, migration generation, and drift checks.
+      sig { params(connection: Connection, relation: ::ActiveRecord::Relation).returns(T::Array[ColumnDefinition]) }
+      def column_definitions(connection, relation)
+        result = connection.exec_query(relation.limit(1).to_sql)
+        sample = result.rows.first
+        result.columns.each_with_index.map do |name, index|
+          ColumnDefinition.new(name: name, type: type_for_value(sample&.at(index)))
+        end
+      end
+
+      sig { params(connection: Connection, table_name: String, relation: ::ActiveRecord::Relation).void }
+      def build_table!(connection, table_name, relation)
+        definitions = column_definitions(connection, relation)
+        connection.create_table(table_name) do |table|
+          definitions.each { |definition| T.unsafe(table).public_send(definition.type, definition.name) }
+        end
+      end
+      private_class_method :build_table!
+
+      sig { params(value: T.untyped).returns(Symbol) }
+      def type_for_value(value)
+        case value
+        when Integer then :integer
+        when Float, BigDecimal then :decimal
+        when TrueClass, FalseClass then :boolean
+        when Time, Date, DateTime, ::ActiveSupport::TimeWithZone then :datetime
+        else :string
+        end
+      end
+      private_class_method :type_for_value
+    end
+  end
+end

data/lib/activerecord/materialized/cold_read.rb

@@ -0,0 +1,60 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # The relation a read is served from when a view is not yet materialized,
+    # per its cold_read strategy. Read-through wraps the live source as a derived
+    # table aliased to the cache table name, so where/order/limit/count keep
+    # working against the same column names.
+    class ColdRead
+      extend T::Sig
+
+      sig { params(view_class: ViewClass).void }
+      def initialize(view_class)
+        @view_class = view_class
+      end
+
+      sig { returns(T.untyped) }
+      def scope
+        case @view_class.resolved_cold_read_strategy
+        when :read_through
+          ensure_skeleton!
+          unscoped.from(source_derived_table)
+        when :serve_stale
+          ensure_skeleton!
+          unscoped
+        when :raise
+          Kernel.raise NotMaterializedError, not_materialized_message
+        else
+          Kernel.raise ArgumentError, "Unknown cold_read strategy: #{@view_class.resolved_cold_read_strategy}"
+        end
+      end
+
+      private
+
+      sig { returns(T.untyped) }
+      def unscoped
+        T.unsafe(@view_class).unscoped
+      end
+
+      sig { returns(Arel::Nodes::SqlLiteral) }
+      def source_derived_table
+        Arel.sql("(#{@view_class.resolved_source.to_sql}) #{@view_class.quoted_table_name}")
+      end
+
+      # Provisions an empty cache table for column metadata — cheap DDL, no data.
+      sig { void }
+      def ensure_skeleton!
+        return if @view_class.table_exists?
+
+        CacheTableSchema.ensure_table!(@view_class, @view_class.resolved_source)
+      end
+
+      sig { returns(String) }
+      def not_materialized_message
+        "#{@view_class.name} is not materialized; run #{@view_class.name}.rebuild!(confirm: true)"
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/configuration.rb

@@ -0,0 +1,80 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # Global, app-wide defaults, set via {Materialized.configure}. Individual
+    # views can override most of these with the corresponding DSL macro.
+    class Configuration
+      extend T::Sig
+
+      # @return [String] name of the metadata table (default "ar_materialized_view_metadata")
+      sig { returns(String) }
+      attr_accessor :metadata_table_name
+
+      # @return [String] name of the per-partition freshness table
+      sig { returns(String) }
+      attr_accessor :partition_table_name
+
+      # @return [Numeric, ActiveSupport::Duration, nil] default {ViewRefreshPolicyClassMethods::ClassMethods#max_staleness max_staleness}
+      sig { returns(T.nilable(DebounceInterval)) }
+      attr_accessor :default_max_staleness
+
+      # @return [Integer, nil] optional per-refresh timeout in seconds
+      sig { returns(T.nilable(Integer)) }
+      attr_accessor :refresh_timeout
+
+      # @return [Boolean] whether a full refresh swaps a freshly built table in atomically
+      sig { returns(T::Boolean) }
+      attr_accessor :atomic_swap_refresh
+
+      # @return [Symbol] default refresh strategy: +:async+, +:immediate+, or +:manual+
+      sig { returns(Symbol) }
+      attr_accessor :default_refresh_strategy
+
+      # @return [Numeric] default debounce window (seconds) for coalescing async refreshes
+      sig { returns(DebounceInterval) }
+      attr_accessor :default_refresh_debounce
+
+      # @return [Symbol] background dispatcher: +:async+ (in-process thread) or +:active_job+
+      sig { returns(Symbol) }
+      attr_accessor :refresh_dispatcher
+
+      # @return [Symbol] ActiveJob queue name used when +refresh_dispatcher+ is +:active_job+
+      sig { returns(Symbol) }
+      attr_accessor :refresh_queue_name
+
+      # Cold-read behavior: :read_through (serve from source), :serve_stale
+      # (serve the cache as-is), or :raise.
+      sig { returns(Symbol) }
+      attr_accessor :default_cold_read_strategy
+
+      # Cap on distinct partitions tracked in a view's pending maintenance before
+      # it collapses to a single full recompute. Bounds the per-write cost of a
+      # bulk write that spans many partitions. Defaults to 1000.
+      #
+      # @return [Integer]
+      sig { params(max_tracked_partitions: Integer).void }
+      attr_writer :max_tracked_partitions
+
+      sig { returns(Integer) }
+      def max_tracked_partitions
+        @max_tracked_partitions ||= T.let(1_000, T.nilable(Integer))
+      end
+
+      sig { void }
+      def initialize
+        @metadata_table_name = T.let("ar_materialized_view_metadata", String)
+        @partition_table_name = T.let("ar_materialized_view_partitions", String)
+        @default_max_staleness = T.let(nil, T.nilable(DebounceInterval))
+        @refresh_timeout = T.let(nil, T.nilable(Integer))
+        @atomic_swap_refresh = T.let(true, T::Boolean)
+        @default_refresh_strategy = T.let(:async, Symbol)
+        @default_refresh_debounce = T.let(2, DebounceInterval)
+        @refresh_dispatcher = T.let(:async, Symbol)
+        @refresh_queue_name = T.let(:materialized_views, Symbol)
+        @default_cold_read_strategy = T.let(:read_through, Symbol)
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/delta_maintainer.rb

@@ -0,0 +1,74 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # Applies a SummaryDelta to a delta-maintainable view's cache table without
+    # re-reading base rows: new partitions inserted, existing ones incremented in
+    # place (NULL-safe for SUM), and emptied partitions deleted.
+    class DeltaMaintainer
+      extend T::Sig
+
+      sig { params(view_class: ViewClass).void }
+      def initialize(view_class)
+        @view_class = view_class
+        @analysis = T.let(AggregateAnalysis.new(view_class.resolved_source), AggregateAnalysis)
+      end
+
+      sig { params(summary: SummaryDelta).returns(Integer) }
+      def apply!(summary)
+        summary.buckets.each { |key_tuple, column_deltas| apply_partition(key_tuple, column_deltas) }
+        T.unsafe(@view_class).unscoped.count
+      end
+
+      private
+
+      sig { returns(T::Array[String]) }
+      def group_columns
+        @view_class.maintenance_key_columns
+      end
+
+      sig { params(key_tuple: SummaryDelta::KeyTuple, column_deltas: T::Hash[String, Numeric]).void }
+      def apply_partition(key_tuple, column_deltas)
+        existing = partition_scope(key_tuple).first
+        if existing.nil?
+          insert_partition(key_tuple, column_deltas)
+        elsif emptied?(existing, column_deltas)
+          existing.destroy!
+        else
+          add_deltas!(existing, column_deltas)
+        end
+      end
+
+      sig { params(key_tuple: SummaryDelta::KeyTuple).returns(::ActiveRecord::Relation) }
+      def partition_scope(key_tuple)
+        T.unsafe(@view_class).unscoped.where(group_columns.zip(key_tuple).to_h)
+      end
+
+      sig { params(key_tuple: SummaryDelta::KeyTuple, column_deltas: T::Hash[String, Numeric]).void }
+      def insert_partition(key_tuple, column_deltas)
+        # A new partition's deltas are its absolute values; aggregates with no
+        # delta default to 0, not NULL, since the partition has rows.
+        defaults = @analysis.aggregate_columns.to_h { |column| [column.name, 0] }
+        row = group_columns.zip(key_tuple).to_h.merge(defaults).merge(column_deltas)
+        T.unsafe(@view_class).create!(row)
+      end
+
+      # Adds each delta to the current value, treating a NULL SUM as zero.
+      sig { params(existing: T.untyped, column_deltas: T::Hash[String, Numeric]).void }
+      def add_deltas!(existing, column_deltas)
+        new_values = column_deltas.to_h { |column, amount| [column, (existing[column] || 0) + amount] }
+        existing.update!(new_values)
+      end
+
+      sig { params(existing: T.untyped, column_deltas: T::Hash[String, Numeric]).returns(T::Boolean) }
+      def emptied?(existing, column_deltas)
+        column = @analysis.row_count_column
+        return false if column.nil?
+
+        delta = column_deltas[column.name] || 0
+        (existing[column.name].to_i + delta) <= 0
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/dependency_registry.rb

@@ -0,0 +1,107 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # Maps dependency tables to the views that depend on them and publishes committed writes to them.
+    #
+    # @api private
+    class DependencyRegistry
+      class << self
+        extend T::Sig
+
+        sig do
+          params(
+            view_class: ViewClass,
+            tables: T.any(
+              Symbol,
+              String,
+              T.class_of(::ActiveRecord::Base),
+              T::Array[T.any(Symbol, String, T.class_of(::ActiveRecord::Base))]
+            )
+          ).void
+        end
+        def register(view_class, tables)
+          normalized = Array(tables).flat_map { |entry| normalize_dependency(entry) }
+          T.unsafe(view_class).instance_variable_set(:@dependency_tables, normalized)
+
+          normalized.each do |table|
+            bucket = T.cast(dependency_index[table], T::Array[ViewClass])
+            bucket << view_class unless bucket.include?(view_class)
+            subscribe_source_table!(table)
+          end
+        end
+
+        sig { params(table: String).returns(T::Array[ViewClass]) }
+        def views_for_table(table)
+          T.must(dependency_index[normalize_table(table)])
+        end
+
+        sig { params(change: WriteChange).void }
+        def publish_write_change!(change)
+          affected_views([change.table_name]).each do |view|
+            view.record_write_change!(change)
+            RefreshScheduler.schedule(view)
+          end
+        end
+
+        sig { params(tables: T::Array[String]).void }
+        def mark_dirty_for_tables!(tables)
+          affected_views(tables).each(&:mark_dependencies_changed!)
+        end
+
+        sig { void }
+        def reset!
+          @dependency_index = Hash.new { |hash, key| hash[key] = [] }
+          ActiveRecord::Materialized::DependencyTrackable.reset!
+        end
+
+        private
+
+        sig { returns(T::Hash[String, T::Array[ViewClass]]) }
+        def dependency_index
+          @dependency_index ||= T.let(
+            Hash.new { |hash, key| hash[key] = [] },
+            T.nilable(T::Hash[String, T::Array[ViewClass]])
+          )
+        end
+
+        sig { params(tables: T::Array[String]).returns(T::Array[ViewClass]) }
+        def affected_views(tables)
+          Array(tables).flat_map do |table|
+            next [] if skip_table?(table)
+
+            views_for_table(table)
+          end.uniq
+        end
+
+        sig { params(entry: T.untyped).returns(T::Array[String]) }
+        def normalize_dependency(entry)
+          if entry.is_a?(Class) && entry < ::ActiveRecord::Base
+            TableModelRegistry.register(entry)
+            return [entry.table_name]
+          end
+
+          [normalize_table(entry)]
+        end
+
+        sig { params(table: String).void }
+        def subscribe_source_table!(table)
+          model = TableModelRegistry.resolve(table)
+          ActiveRecord::Materialized::DependencyTrackable.subscribe(model) if model
+        end
+
+        sig { params(table: T.any(Symbol, String)).returns(String) }
+        def normalize_table(table)
+          ::ActiveSupport::Inflector.underscore(table.to_s.delete_prefix(":"))
+        end
+
+        sig { params(table: T.any(Symbol, String)).returns(T::Boolean) }
+        def skip_table?(table)
+          name = normalize_table(table)
+          name.start_with?("mv_") || name == ::ActiveRecord::Materialized.metadata_table_name
+        end
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/dependency_trackable.rb

@@ -0,0 +1,48 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # Installs `after_*_commit` callbacks on `depends_on` models so their writes schedule maintenance.
+    #
+    # @api private
+    module DependencyTrackable
+      extend T::Sig
+
+      TRACKABLE_FLAG = :@ar_materialized_dependency_trackable
+
+      class << self
+        extend T::Sig
+
+        sig { params(model_class: T.class_of(::ActiveRecord::Base)).void }
+        def subscribe(model_class)
+          return if model_class.instance_variable_get(TRACKABLE_FLAG)
+
+          install_callbacks!(model_class)
+          model_class.instance_variable_set(TRACKABLE_FLAG, true)
+        end
+
+        # Invoked from the model commit callbacks; `record` is the committed instance.
+        sig { params(record: ::ActiveRecord::Base, operation: WriteChange::Operation).void }
+        def publish(record, operation)
+          DependencyRegistry.publish_write_change!(WriteChange.from_record(record, operation))
+        end
+
+        sig { void }
+        def reset!
+          nil
+        end
+
+        private
+
+        sig { params(model_class: T.class_of(::ActiveRecord::Base)).void }
+        def install_callbacks!(model_class)
+          model = T.unsafe(model_class)
+          model.after_create_commit { DependencyTrackable.publish(T.unsafe(self), :create) }
+          model.after_update_commit { DependencyTrackable.publish(T.unsafe(self), :update) }
+          model.after_destroy_commit { DependencyTrackable.publish(T.unsafe(self), :destroy) }
+        end
+      end
+    end
+  end
+end