activerecord-materialized 0.1.0

data/lib/activerecord/materialized/registry.rb

@@ -0,0 +1,74 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # Tracks every defined view class and provides bulk operations
+    # (refresh / rebuild / warm-up / verify) across all of them.
+    #
+    # @api private
+    class Registry
+      class << self
+        extend T::Sig
+
+        sig { params(view_class: ViewClass).void }
+        def register(view_class)
+          views[view_class.view_key] = view_class
+        end
+
+        sig { params(view_class: ViewClass).void }
+        def unregister(view_class)
+          views.delete(view_class.view_key)
+        end
+
+        sig { params(key: T.any(String, Symbol)).returns(T.nilable(ViewClass)) }
+        def find(key)
+          views[key.to_s]
+        end
+
+        sig { params(class_name: String).returns(T.nilable(ViewClass)) }
+        def for_class_name(class_name)
+          all.find { |view| view.name == class_name }
+        end
+
+        sig { returns(T::Array[ViewClass]) }
+        def all
+          views.values
+        end
+
+        # Incremental pass over every registered view; rebuild_all! for a full one.
+        sig { returns(T::Array[RefreshResult]) }
+        def refresh_all!
+          all.map(&:refresh!)
+        end
+
+        sig { returns(T::Array[RefreshResult]) }
+        def refresh_stale!
+          all.select(&:stale?).map(&:refresh!)
+        end
+
+        sig { returns(T::Array[RefreshResult]) }
+        def rebuild_all!
+          all.map { |view| view.rebuild!(confirm: true) }
+        end
+
+        sig { returns(T::Array[T.nilable(RefreshResult)]) }
+        def warm_up_all!
+          all.map(&:warm_up!)
+        end
+
+        sig { void }
+        def reset!
+          @views = {}
+        end
+
+        private
+
+        sig { returns(T::Hash[String, ViewClass]) }
+        def views
+          @views ||= T.let({}, T.nilable(T::Hash[String, ViewClass]))
+        end
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/relation_cache_writer.rb

@@ -0,0 +1,137 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # Materializes a relation into a cache table via `INSERT … SELECT`, with an atomic swap on full refresh.
+    #
+    # @api private
+    class RelationCacheWriter
+      extend T::Sig
+
+      sig { params(view_class: T.class_of(::ActiveRecord::Base)).void }
+      def initialize(view_class)
+        @view_class = view_class
+      end
+
+      sig { params(relation: ::ActiveRecord::Relation).returns(Integer) }
+      def bootstrap!(relation)
+        CacheTableSchema.ensure_table!(view_class, relation)
+        replace_all!(relation)
+      end
+
+      sig { params(relation: ::ActiveRecord::Relation).returns(Integer) }
+      def replace_all!(relation)
+        view_class.transaction do
+          view_class.delete_all
+          insert_rows!(relation)
+        end
+        cache_row_count
+      end
+
+      sig do
+        params(
+          relation: ::ActiveRecord::Relation,
+          key_tuples: T::Array[T::Array[String]],
+          full_partition: T::Boolean
+        ).returns(Integer)
+      end
+      def replace_partitions!(relation, key_tuples:, full_partition:)
+        view_class.transaction do
+          if full_partition
+            view_class.delete_all
+          else
+            delete_partitions!(key_tuples)
+          end
+          insert_rows!(relation)
+        end
+        cache_row_count
+      end
+
+      sig { params(relation: ::ActiveRecord::Relation).returns(Integer) }
+      def atomic_swap!(relation)
+        connection = view_class.connection
+        temp_table = refresh_temp_table_name
+        old_table = refresh_old_table_name
+
+        populate_temp_table!(temp_table, relation)
+        swap_tables!(connection, temp_table, old_table)
+
+        view_class.reset_column_information
+        cache_row_count
+      end
+
+      sig { params(temp_table: String, relation: ::ActiveRecord::Relation).void }
+      def populate_temp_table!(temp_table, relation)
+        CacheTableSchema.create_table!(T.cast(view_class, ViewClass), temp_table, relation)
+        temp_model = temporary_model(temp_table)
+        self.class.new(temp_model).replace_all!(relation)
+      end
+
+      sig { params(connection: Connection, temp_table: String, old_table: String).void }
+      def swap_tables!(connection, temp_table, old_table)
+        connection.transaction do
+          connection.rename_table(view_class.table_name, old_table) if view_class.table_exists?
+          connection.rename_table(temp_table, view_class.table_name)
+          connection.drop_table(old_table, if_exists: true)
+        end
+      end
+
+      sig { returns(String) }
+      def refresh_temp_table_name
+        "#{view_class.table_name}_refresh_#{SecureRandom.hex(4)}"
+      end
+
+      sig { returns(String) }
+      def refresh_old_table_name
+        "#{view_class.table_name}_old_#{SecureRandom.hex(4)}"
+      end
+
+      private
+
+      sig { returns(T.class_of(::ActiveRecord::Base)) }
+      attr_reader :view_class
+
+      # Count straight from the cache table, bypassing the View read routing
+      # (during a rebuild the view is not warm yet, so a routed count would read
+      # through to the source).
+      sig { returns(Integer) }
+      def cache_row_count
+        T.unsafe(view_class).unscoped.count
+      end
+
+      sig { params(key_tuples: T::Array[T::Array[String]]).void }
+      def delete_partitions!(key_tuples)
+        materialized_view = T.cast(view_class, ViewClass)
+        materialized_view.view_definition.partition_scope_on(view_class, key_tuples).delete_all
+      end
+
+      # INSERT ... SELECT entirely in the database; the result set never crosses
+      # into Ruby. Cache columns share the relation's projection order, so the
+      # SELECT list maps onto them positionally.
+      sig { params(relation: ::ActiveRecord::Relation).void }
+      def insert_rows!(relation)
+        columns = view_class.column_names - ["id"]
+        return if columns.empty?
+
+        T.unsafe(view_class.connection).execute(insert_select_sql(relation, columns))
+      end
+
+      sig { params(relation: ::ActiveRecord::Relation, columns: T::Array[String]).returns(String) }
+      def insert_select_sql(relation, columns)
+        manager = Arel::InsertManager.new
+        manager.into(view_class.arel_table)
+        T.unsafe(manager).columns.concat(columns.map { |name| view_class.arel_table[name] })
+        manager.select(Arel.sql(relation.to_sql))
+        manager.to_sql
+      end
+
+      sig { params(table_name: String).returns(T.class_of(::ActiveRecord::Base)) }
+      def temporary_model(table_name)
+        klass = Class.new(::ActiveRecord::Base)
+        T.unsafe(klass).table_name = table_name
+        klass
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/schema_verifier.rb

@@ -0,0 +1,64 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # Raises when a provisioned cache table no longer matches the columns its
+    # source relation projects (drift); never alters the table or rebuilds data.
+    class SchemaVerifier
+      extend T::Sig
+
+      # Raised when a cache table no longer matches the columns its source relation projects.
+      class SchemaDriftError < StandardError; end
+
+      sig { params(view_class: ViewClass).void }
+      def initialize(view_class)
+        @view_class = view_class
+      end
+
+      # An unprovisioned cache table is absent, not drifted, so this is a no-op.
+      sig { void }
+      def verify!
+        return unless @view_class.table_exists?
+
+        missing = expected_columns - actual_columns
+        extra = actual_columns - expected_columns
+        return if missing.empty? && extra.empty?
+
+        Kernel.raise SchemaDriftError, drift_message(missing, extra)
+      end
+
+      sig { returns(T::Boolean) }
+      def drifted?
+        verify!
+        false
+      rescue SchemaDriftError
+        true
+      end
+
+      private
+
+      sig { returns(T::Array[String]) }
+      def expected_columns
+        CacheTableSchema
+          .column_definitions(@view_class.connection, @view_class.resolved_source)
+          .map(&:name).sort
+      end
+
+      sig { returns(T::Array[String]) }
+      def actual_columns
+        @view_class.connection.columns(@view_class.table_name).map(&:name).reject { |name| name == "id" }.sort
+      end
+
+      sig { params(missing: T::Array[String], extra: T::Array[String]).returns(String) }
+      def drift_message(missing, extra)
+        details = []
+        details << "missing columns: #{missing.join(', ')}" if missing.any?
+        details << "unexpected columns: #{extra.join(', ')}" if extra.any?
+        "#{@view_class.name} cache table #{@view_class.table_name} is out of date " \
+          "(#{details.join('; ')}). Generate and run a migration: " \
+          "bin/rails generate activerecord_materialized:migration #{@view_class.name}"
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/summary_delta.rb

@@ -0,0 +1,76 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # Accumulates signed per-partition, per-column numeric changes for a
+    # delta-maintainable view (`partition key tuple => mv column => amount`).
+    #
+    # @api private
+    class SummaryDelta
+      extend T::Sig
+
+      KeyTuple = T.type_alias { T::Array[T.untyped] }
+      Columns = T.type_alias { T::Hash[String, Numeric] }
+      Buckets = T.type_alias { T::Hash[KeyTuple, Columns] }
+
+      sig { returns(Buckets) }
+      attr_reader :buckets
+
+      sig { params(buckets: Buckets).void }
+      def initialize(buckets = {})
+        @buckets = buckets
+      end
+
+      sig { params(key_tuple: KeyTuple, column: String, amount: Numeric).void }
+      def add(key_tuple, column, amount)
+        bucket = (@buckets[key_tuple] ||= {})
+        bucket[column] = (bucket[column] || 0) + amount
+      end
+
+      sig { returns(T::Boolean) }
+      def empty?
+        @buckets.empty?
+      end
+
+      # Number of distinct partitions (buckets) accumulated so far.
+      sig { returns(Integer) }
+      def tracked_partition_count
+        @buckets.size
+      end
+
+      # Drops net-zero columns (no-op changes) and the partitions left empty.
+      sig { returns(SummaryDelta) }
+      def prune!
+        @buckets.each_value { |columns| columns.reject! { |_column, amount| amount.zero? } }
+        @buckets.reject! { |_key_tuple, columns| columns.empty? }
+        self
+      end
+
+      sig { params(other: SummaryDelta).returns(SummaryDelta) }
+      def merge(other)
+        merged = SummaryDelta.new(@buckets.transform_values(&:dup))
+        other.buckets.each do |key_tuple, columns|
+          columns.each { |column, amount| merged.add(key_tuple, column, amount) }
+        end
+        merged
+      end
+
+      # A list of `"key" => tuple, "columns" => …` entries so array keys survive JSON.
+      sig { returns(T::Hash[String, T.untyped]) }
+      def serialize
+        { "summary" => @buckets.map { |key_tuple, columns| { "key" => key_tuple, "columns" => columns } } }
+      end
+
+      sig { params(payload: T.nilable(T::Hash[String, T.untyped])).returns(T.nilable(SummaryDelta)) }
+      def self.deserialize(payload)
+        rows = payload && payload["summary"]
+        return nil if rows.nil?
+
+        buckets = T.let({}, Buckets)
+        rows.each { |row| buckets[row.fetch("key")] = row.fetch("columns") }
+        new(buckets)
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/summary_delta_builder.rb

@@ -0,0 +1,58 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # The SummaryDelta a single write contributes. Subtracting the "before"
+    # snapshot and adding the "after" uniformly handles inserts, deletes, in-place
+    # updates, and group-key changes (a move between partitions).
+    class SummaryDeltaBuilder
+      extend T::Sig
+
+      sig { params(change: WriteChange, analysis: AggregateAnalysis, group_columns: T::Array[String]).void }
+      def initialize(change, analysis, group_columns)
+        @change = change
+        @analysis = analysis
+        @group_columns = group_columns
+      end
+
+      sig { returns(SummaryDelta) }
+      def build
+        delta = SummaryDelta.new
+        apply!(delta, @change.before, -1) unless @change.before.empty?
+        apply!(delta, @change.after, 1) unless @change.after.empty?
+        delta.prune!
+      end
+
+      private
+
+      sig { params(delta: SummaryDelta, snapshot: T::Hash[String, T.untyped], sign: Integer).void }
+      def apply!(delta, snapshot, sign)
+        key_tuple = key_tuple(snapshot)
+        return if key_tuple.nil?
+
+        @analysis.aggregate_columns.each do |column|
+          contribution = contribution_for(column, snapshot)
+          delta.add(key_tuple, column.name, sign * contribution) unless contribution.zero?
+        end
+      end
+
+      sig { params(column: AggregateAnalysis::Column, snapshot: T::Hash[String, T.untyped]).returns(Numeric) }
+      def contribution_for(column, snapshot)
+        case column.function
+        when :count_star then 1
+        when :count then snapshot[T.must(column.attribute)].nil? ? 0 : 1
+        when :sum then snapshot.fetch(T.must(column.attribute), 0) || 0
+        else 0
+        end
+      end
+
+      sig { params(snapshot: T::Hash[String, T.untyped]).returns(T.nilable(SummaryDelta::KeyTuple)) }
+      def key_tuple(snapshot)
+        return nil unless @group_columns.all? { |column| snapshot.key?(column) }
+
+        @group_columns.map { |column| snapshot.fetch(column) }
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/table_model_registry.rb

@@ -0,0 +1,43 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # Maps table names to their ActiveRecord models for dependency wiring.
+    #
+    # @api private
+    class TableModelRegistry
+      class << self
+        extend T::Sig
+
+        sig { params(model_class: T.class_of(::ActiveRecord::Base)).void }
+        def register(model_class)
+          return if model_class.abstract_class?
+
+          explicit[model_class.table_name] = model_class
+        end
+
+        sig { params(table_name: String).returns(T.nilable(T.class_of(::ActiveRecord::Base))) }
+        def resolve(table_name)
+          explicit[table_name] || find_descendant(table_name)
+        end
+
+        private
+
+        sig { returns(T::Hash[String, T.class_of(::ActiveRecord::Base)]) }
+        def explicit
+          @explicit ||= T.let({}, T.nilable(T::Hash[String, T.class_of(::ActiveRecord::Base)]))
+        end
+
+        sig { params(table_name: String).returns(T.nilable(T.class_of(::ActiveRecord::Base))) }
+        def find_descendant(table_name)
+          ::ActiveRecord::Base.descendants.find do |model_class|
+            !model_class.abstract_class? && model_class.table_name == table_name
+          end
+        rescue StandardError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/tasks.rb

@@ -0,0 +1,79 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # Defines the `materialized:*` rake tasks (refresh_all, refresh_stale, rebuild, verify, warm_up).
+    module Tasks
+      extend T::Sig
+
+      DEFINITIONS = T.let(
+        {
+          refresh_all: "Refresh all registered materialized views",
+          refresh_stale: "Refresh stale materialized views",
+          rebuild: "Rebuild (fully materialize) all registered materialized views",
+          verify: "Verify materialized view cache tables match their source relations",
+          warm_up: "Materialize each view's configured warm_up partitions"
+        }.freeze,
+        T::Hash[Symbol, String]
+      )
+
+      sig { void }
+      def self.define!
+        application = T.let(T.unsafe(::Rake.application), T.untyped)
+        application.instance_eval do
+          T.bind(self, T.untyped)
+
+          namespace :materialized do
+            DEFINITIONS.each do |task_name, description|
+              desc description
+              task(task_name => :environment) { Tasks.run!(task_name) }
+            end
+          end
+        end
+      end
+
+      sig { params(task_name: Symbol).void }
+      def self.run!(task_name)
+        case task_name
+        when :refresh_all then run_refresh_all!
+        when :refresh_stale then run_refresh_stale!
+        when :rebuild then run_rebuild_all!
+        when :verify then run_verify!
+        when :warm_up then run_warm_up_all!
+        end
+      end
+
+      sig { void }
+      def self.run_refresh_all!
+        Registry.refresh_all!
+        T.unsafe(Rails).logger.debug { "Refreshed #{Registry.all.size} materialized view(s)." }
+      end
+
+      sig { void }
+      def self.run_refresh_stale!
+        stale = Registry.all.select(&:stale?)
+        stale.each(&:refresh!)
+        T.unsafe(Rails).logger.debug { "Refreshed #{stale.size} stale materialized view(s)." }
+      end
+
+      sig { void }
+      def self.run_rebuild_all!
+        Registry.rebuild_all!
+        T.unsafe(Rails).logger.debug { "Rebuilt #{Registry.all.size} materialized view(s)." }
+      end
+
+      sig { void }
+      def self.run_verify!
+        ActiveRecord::Materialized.verify_schema!
+        T.unsafe(Rails).logger.debug { "Verified #{Registry.all.size} materialized view schema(s)." }
+      end
+
+      sig { void }
+      def self.run_warm_up_all!
+        Registry.warm_up_all!
+        T.unsafe(Rails).logger.debug { "Warmed up #{Registry.all.size} materialized view(s)." }
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/type_reexports.rb

@@ -0,0 +1,14 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    DebounceInterval = T.type_alias { ::ActiveRecordMaterializedTypes::DebounceInterval }
+    StalenessDuration = T.type_alias { ::ActiveRecordMaterializedTypes::StalenessDuration }
+    SourceDefinition = T.type_alias { ::ActiveRecordMaterializedTypes::SourceDefinition }
+    RefreshMode = T.type_alias { ::ActiveRecordMaterializedTypes::RefreshMode }
+    RefreshCallbackName = T.type_alias { ::ActiveRecordMaterializedTypes::RefreshCallbackName }
+    Connection = T.type_alias { ::ActiveRecordMaterializedTypes::Connection }
+    Timestamp = T.type_alias { ::ActiveRecordMaterializedTypes::Timestamp }
+  end
+end

data/lib/activerecord/materialized/version.rb

@@ -0,0 +1,9 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # The gem version.
+    VERSION = "0.1.0"
+  end
+end

data/lib/activerecord/materialized/view.rb

@@ -0,0 +1,79 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # The base class for an application-level materialized view. Subclass it,
+    # point {ViewConfigurationClassMethods::ClassMethods#materialized_from} at an
+    # `ActiveRecord::Relation`, declare the models it
+    # {ViewConfigurationClassMethods::ClassMethods#depends_on depends_on}, and then
+    # read it like any ActiveRecord model. Reads are served from a cache table the
+    # gem maintains incrementally as the underlying data changes; a full
+    # materialization happens only via an explicit
+    # {ViewQueryAccessClassMethods::ClassMethods#rebuild! rebuild!}, and until then
+    # reads transparently fall through to the source query.
+    #
+    # @example Define a view and use it
+    #   class RegionRevenue < ActiveRecord::Materialized::View
+    #     extend ActiveRecord::Materialized::QueryExpressions
+    #     self.table_name = "mv_region_revenue"
+    #
+    #     materialized_from do
+    #       sales = Sale.arel_table
+    #       Sale.group(:region).select(sales[:region], sum_as(sales[:amount], as: :revenue))
+    #     end
+    #
+    #     depends_on Sale            # writes to Sale schedule maintenance
+    #     refresh_on_change :async   # refresh in the background after commit
+    #     max_staleness 6.hours      # optional time-based safety net
+    #   end
+    #
+    #   RegionRevenue.rebuild!(confirm: true)              # materialize once (e.g. at deploy)
+    #   RegionRevenue.where(region: "west").pick(:revenue) # served from the cache table
+    #
+    # @see ViewConfigurationClassMethods::ClassMethods +materialized_from+ / +depends_on+ DSL
+    # @see ViewRefreshPolicyClassMethods::ClassMethods refresh strategy, staleness, and warm-up
+    # @see ViewIncrementalClassMethods::ClassMethods incremental-maintenance configuration
+    # @see ViewQueryAccessClassMethods::ClassMethods +rebuild!+, +refresh!+, +materialized?+, …
+    class View < ::ActiveRecord::Base
+      extend T::Sig
+      include RefreshCallbacks
+      include ViewConfigurationClassMethods
+      include ViewQueryAccessClassMethods
+
+      self.abstract_class = true
+
+      class << self
+        extend T::Sig
+
+        @source_definition = T.let(nil, T.nilable(SourceDefinition))
+        @max_staleness_setting = T.let(nil, T.nilable(T.any(StalenessDuration, Proc)))
+        @dependency_tables = T.let(nil, T.nilable(T::Array[String]))
+        @refresh_strategy = T.let(nil, T.nilable(Symbol))
+        @refresh_debounce = T.let(nil, T.nilable(DebounceInterval))
+        @refresh_mode = T.let(nil, T.nilable(RefreshMode))
+        @incremental_source_definition = T.let(nil, T.nilable(SourceDefinition))
+        @incremental_key_columns = T.let(nil, T.nilable(T::Array[String]))
+        @table_name = T.let(nil, T.nilable(String))
+
+        sig { returns(T.nilable(SourceDefinition)) }
+        attr_reader :source_definition
+
+        sig { returns(T.nilable(T.any(StalenessDuration, Proc))) }
+        attr_reader :max_staleness_setting
+
+        sig { returns(T::Array[String]) }
+        def dependency_tables
+          tables = T.let(T.unsafe(self).instance_variable_get(:@dependency_tables), T.nilable(T::Array[String]))
+          tables.nil? ? [] : tables
+        end
+      end
+
+      sig { returns(T::Boolean) }
+      def stale?
+        T.bind(self, View)
+        self.class.stale?
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/view_class.rb

@@ -0,0 +1,8 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    ViewClass = T.type_alias { T.class_of(View) }
+  end
+end