activerecord-materialized 0.1.0

data/lib/activerecord/materialized/module_api.rb

@@ -0,0 +1,82 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  # Application-level materialized views for Rails/ActiveRecord on databases
+  # without native materialized-view support (MySQL, MariaDB, SQLite).
+  #
+  # Define views by subclassing {Materialized::View}. This module is the
+  # top-level entry point for global {configure configuration} and operational
+  # helpers such as {verify_schema!}.
+  #
+  # @see Materialized::View defining a view
+  # @see Materialized::Configuration the configurable settings
+  module Materialized
+    class << self
+      extend T::Sig
+
+      @configuration = T.let(nil, T.nilable(Configuration))
+
+      # The global configuration object. Prefer {configure} for setting values.
+      #
+      # @return [Configuration] the current configuration (created on first use)
+      sig { returns(Configuration) }
+      def configuration
+        config = @configuration
+        if config.nil?
+          config = Configuration.new
+          @configuration = T.let(config, T.nilable(Configuration))
+        end
+        config
+      end
+
+      # Configure the gem, typically from an initializer.
+      #
+      # @example config/initializers/activerecord_materialized.rb
+      #   ActiveRecord::Materialized.configure do |config|
+      #     config.default_refresh_strategy = :async
+      #     config.refresh_dispatcher       = :active_job
+      #     config.default_max_staleness    = 12.hours
+      #   end
+      #
+      # @yieldparam config [Configuration]
+      # @return [void]
+      sig { params(block: T.proc.params(config: Configuration).void).void }
+      def configure(&block)
+        yield(configuration)
+      end
+
+      sig { returns(String) }
+      def metadata_table_name
+        configuration.metadata_table_name
+      end
+
+      sig { returns(String) }
+      def partition_table_name
+        configuration.partition_table_name
+      end
+
+      # Verifies every registered view's cache table still matches the columns its
+      # source relation projects — run it at boot or in CI to catch a view whose
+      # definition changed without a migration. Never alters tables.
+      #
+      # @raise [SchemaVerifier::SchemaDriftError] on the first drifted view
+      # @return [void]
+      sig { void }
+      def verify_schema!
+        registered = Registry.all
+        registered.each { |view_class| SchemaVerifier.new(view_class).verify! }
+      end
+
+      sig { returns(T::Boolean) }
+      def atomic_swap_refresh?
+        configuration.atomic_swap_refresh
+      end
+
+      sig { params(value: Configuration).void }
+      def configuration=(value)
+        @configuration = T.let(value, T.nilable(Configuration))
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/partition_record.rb

@@ -0,0 +1,27 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # One row per fresh partition of a cold view; presence means the partition is
+    # materialized and current, absence means it is not.
+    class PartitionRecord < ::ActiveRecord::Base
+      extend T::Sig
+
+      @table_name_override = T.let(nil, T.nilable(String))
+
+      self.table_name = ::ActiveRecord::Materialized.partition_table_name
+
+      sig { params(name: String).void }
+      def self.table_name=(name)
+        @table_name_override = name
+      end
+
+      sig { returns(String) }
+      def self.table_name
+        override = @table_name_override
+        override.nil? ? ::ActiveRecord::Materialized.partition_table_name : override
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/partition_state.rb

@@ -0,0 +1,127 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # Tracks which partitions of a cold view have been materialized ("fresh") so
+    # a read can decide whether a partition is served from the cache or read
+    # through to the source. Warm views are fully materialized and ignore this.
+    class PartitionState
+      extend T::Sig
+
+      KeyTuple = T.type_alias { T::Array[T.untyped] }
+
+      sig { params(view_class: ViewClass).void }
+      def initialize(view_class)
+        @view_class = view_class
+      end
+
+      sig { params(key_tuples: T::Array[KeyTuple]).returns(T::Boolean) }
+      def all_fresh?(key_tuples)
+        return false if key_tuples.empty?
+
+        ensure_table!
+        serialized = key_tuples.map { |tuple| serialize(tuple) }.uniq
+        scope.where(partition_key: serialized).count == serialized.size
+      end
+
+      sig { params(key_tuples: T::Array[KeyTuple]).void }
+      def mark_fresh!(key_tuples)
+        return if key_tuples.empty?
+
+        ensure_table!
+        key_tuples.uniq.each do |tuple|
+          PartitionRecord.create_or_find_by(view_name: view_key, partition_key: serialize(tuple))
+        end
+      end
+
+      sig { params(key_tuples: T::Array[KeyTuple]).void }
+      def mark_stale!(key_tuples)
+        return if key_tuples.empty?
+
+        ensure_table!
+        scope.where(partition_key: key_tuples.map { |tuple| serialize(tuple) }).delete_all
+      end
+
+      sig { void }
+      def reset!
+        ensure_table!
+        scope.delete_all
+      end
+
+      # The partition key tuples a query touches, or nil unless the conditions are
+      # an exact match on the GROUP BY columns (the only case the fast path serves).
+      sig { params(view_class: ViewClass, args: T::Array[T.untyped]).returns(T.nilable(T::Array[KeyTuple])) }
+      def self.keys_from(view_class, args)
+        conditions = single_hash(args)
+        return nil if conditions.nil?
+
+        group_keys = view_class.maintenance_key_columns
+        return nil if group_keys.empty?
+
+        value_lists = key_value_lists(conditions, group_keys)
+        value_lists.nil? ? nil : cartesian(value_lists)
+      end
+
+      sig { params(args: T::Array[T.untyped]).returns(T.nilable(T::Hash[T.untyped, T.untyped])) }
+      def self.single_hash(args)
+        return nil unless args.length == 1
+
+        conditions = args.fetch(0)
+        conditions.is_a?(Hash) ? conditions : nil
+      end
+
+      sig do
+        params(conditions: T::Hash[T.untyped, T.untyped], group_keys: T::Array[String])
+          .returns(T.nilable(T::Array[T::Array[String]]))
+      end
+      def self.key_value_lists(conditions, group_keys)
+        normalized = conditions.transform_keys(&:to_s)
+        return nil unless normalized.keys.sort == group_keys.sort
+
+        group_keys.map { |column| Array(normalized.fetch(column)).map(&:to_s) }
+      end
+
+      sig { params(value_lists: T::Array[T::Array[String]]).returns(T.nilable(T::Array[KeyTuple])) }
+      def self.cartesian(value_lists)
+        return nil if value_lists.any?(&:empty?)
+
+        value_lists.reduce([[]]) do |tuples, values|
+          tuples.flat_map { |tuple| values.map { |value| tuple + [value] } }
+        end
+      end
+
+      private
+
+      sig { returns(String) }
+      def view_key
+        @view_class.view_key
+      end
+
+      sig { params(key_tuple: KeyTuple).returns(String) }
+      def serialize(key_tuple)
+        key_tuple.map(&:to_s).to_json
+      end
+
+      sig { returns(::ActiveRecord::Relation) }
+      def scope
+        PartitionRecord.where(view_name: view_key)
+      end
+
+      sig { void }
+      def ensure_table!
+        connection = @view_class.connection
+        return if PartitionRecord.table_exists?
+
+        table = ::ActiveRecord::Materialized.partition_table_name
+        connection.create_table(table) do |t|
+          t.string :view_name, null: false
+          t.string :partition_key, null: false
+          t.datetime :created_at, null: false
+        end
+        connection.add_index(table, %i[view_name partition_key], unique: true)
+        PartitionRecord.reset_column_information
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/query_expressions.rb

@@ -0,0 +1,83 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # Portable, aliased Arel aggregate helpers for building a view's
+    # {ViewConfigurationClassMethods::ClassMethods#materialized_from source
+    # relation} without raw SQL. `extend` it into a view (or any object) so the
+    # helpers are available where you build the relation.
+    #
+    # @example
+    #   class SalesByCategory < ActiveRecord::Materialized::View
+    #     extend ActiveRecord::Materialized::QueryExpressions
+    #     materialized_from do
+    #       items = Item.arel_table
+    #       Item.group(:category).select(
+    #         items[:category],
+    #         sum_as(items[:amount], as: :revenue),
+    #         count_all_as(as: :order_count)
+    #       )
+    #     end
+    #   end
+    module QueryExpressions
+      extend T::Sig
+
+      module_function
+
+      # @return [Arel::Nodes::As] +SUM(attribute) AS <as>+
+      sig { params(attribute: Arel::Attributes::Attribute, as: T.any(Symbol, String)).returns(Arel::Nodes::As) }
+      def sum_as(attribute, as:)
+        attribute.sum.as(as.to_s)
+      end
+
+      # @return [Arel::Nodes::As] +AVG(attribute) AS <as>+
+      sig { params(attribute: Arel::Attributes::Attribute, as: T.any(Symbol, String)).returns(Arel::Nodes::As) }
+      def avg_as(attribute, as:)
+        attribute.average.as(as.to_s)
+      end
+
+      # @return [Arel::Nodes::As] +MIN(attribute) AS <as>+
+      sig { params(attribute: Arel::Attributes::Attribute, as: T.any(Symbol, String)).returns(Arel::Nodes::As) }
+      def min_as(attribute, as:)
+        attribute.minimum.as(as.to_s)
+      end
+
+      # @return [Arel::Nodes::As] +MAX(attribute) AS <as>+
+      sig { params(attribute: Arel::Attributes::Attribute, as: T.any(Symbol, String)).returns(Arel::Nodes::As) }
+      def max_as(attribute, as:)
+        attribute.maximum.as(as.to_s)
+      end
+
+      # @return [Arel::Nodes::As] +COUNT(*) AS <as>+ — a trustworthy per-partition row count
+      sig { params(as: T.any(Symbol, String)).returns(Arel::Nodes::As) }
+      def count_all_as(as:)
+        Arel.star.count.as(as.to_s)
+      end
+
+      # @return [Arel::Nodes::As] +COUNT(attribute) AS <as>+ (non-null values)
+      sig { params(attribute: Arel::Attributes::Attribute, as: T.any(Symbol, String)).returns(Arel::Nodes::As) }
+      def count_as(attribute, as:)
+        attribute.count.as(as.to_s)
+      end
+
+      # @return [Arel::Nodes::As] +COUNT(DISTINCT attribute) AS <as>+
+      sig { params(attribute: Arel::Attributes::Attribute, as: T.any(Symbol, String)).returns(Arel::Nodes::As) }
+      def count_distinct_as(attribute, as:)
+        attribute.count(true).as(as.to_s)
+      end
+
+      # @return [Arel::Nodes::NamedFunction] +LENGTH(attribute)+
+      sig { params(attribute: Arel::Attributes::Attribute).returns(Arel::Nodes::NamedFunction) }
+      def length(attribute)
+        Arel::Nodes::NamedFunction.new("LENGTH", [attribute])
+      end
+
+      # @return [Arel::Nodes::As] +SUM(LENGTH(attribute)) AS <as>+
+      sig { params(attribute: Arel::Attributes::Attribute, as: T.any(Symbol, String)).returns(Arel::Nodes::As) }
+      def sum_length_as(attribute, as:)
+        length(attribute).sum.as(as.to_s)
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/railtie.rb

@@ -0,0 +1,16 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # Rails integration: wires the gem's load hooks and rake tasks into a host application.
+    class Railtie < ::Rails::Railtie
+      extend T::Sig
+
+      rake_tasks do
+        require_relative "tasks"
+        Tasks.define!
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/refresh_callbacks.rb

@@ -0,0 +1,62 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # Adds `before_refresh` / `after_refresh` lifecycle callbacks to a {View}.
+    module RefreshCallbacks
+      extend T::Sig
+
+      sig { params(base: T.class_of(View)).void }
+      def self.included(base)
+        base.extend(ClassMethods)
+      end
+
+      # The callback-registration methods available on a {View} subclass.
+      module ClassMethods
+        extend T::Sig
+
+        sig { returns(T::Hash[Symbol, T::Array[RefreshCallbackName]]) }
+        def refresh_callback_store
+          @refresh_callback_store ||= T.let(
+            { before_refresh: [], after_refresh: [] },
+            T.nilable(T::Hash[Symbol, T::Array[RefreshCallbackName]])
+          )
+        end
+
+        sig { params(methods: Symbol, block: T.nilable(T.proc.void)).void }
+        def before_refresh(*methods, &block)
+          register_refresh_callback(:before_refresh, methods, block)
+        end
+
+        sig { params(methods: Symbol, block: T.nilable(T.proc.void)).void }
+        def after_refresh(*methods, &block)
+          register_refresh_callback(:after_refresh, methods, block)
+        end
+
+        sig { params(name: Symbol).void }
+        def run_refresh_callbacks(name)
+          callbacks = refresh_callback_store.fetch(name, [])
+          callbacks.each do |callback|
+            case callback
+            when Symbol
+              T.unsafe(self).public_send(callback)
+            when Proc
+              T.unsafe(self).instance_eval(&callback)
+            end
+          end
+        end
+
+        private
+
+        sig { params(name: Symbol, methods: T::Array[Symbol], block: T.nilable(T.proc.void)).void }
+        def register_refresh_callback(name, methods, block)
+          callbacks = T.must(refresh_callback_store[name]).dup
+          methods.each { |method| callbacks << method }
+          callbacks << block if block
+          refresh_callback_store[name] = callbacks
+        end
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/refresh_job.rb

@@ -0,0 +1,22 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # ActiveJob wrapper that runs a view's incremental refresh on a background worker.
+    class RefreshJob < ::ActiveJob::Base
+      extend T::Sig
+
+      queue_as { ::ActiveRecord::Materialized.configuration.refresh_queue_name }
+
+      sig { params(view_key: String).void }
+      def perform(view_key)
+        view_class = Registry.find(view_key)
+        return if view_class.nil?
+        return unless view_class.dirty?
+
+        view_class.refresh!
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/refresh_result.rb

@@ -0,0 +1,40 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # The outcome of a refresh or rebuild, returned by
+    # {ViewQueryAccessClassMethods::ClassMethods#refresh! refresh!},
+    # {ViewQueryAccessClassMethods::ClassMethods#rebuild! rebuild!}, and
+    # {ViewQueryAccessClassMethods::ClassMethods#refresh_if_stale! refresh_if_stale!}.
+    #
+    # @!attribute [r] view_class
+    #   @return [Class] the view that was refreshed
+    # @!attribute [r] row_count
+    #   @return [Integer] rows in the cache table after the operation
+    # @!attribute [r] duration_ms
+    #   @return [Integer] wall-clock duration in milliseconds
+    # @!attribute [r] refreshed_at
+    #   @return [Time, nil] when the refresh completed (nil when skipped)
+    # @!attribute [r] skipped
+    #   @return [Boolean] true when there was nothing to do (e.g. an unmaintainable view)
+    class RefreshResult < T::Struct
+      extend T::Sig
+
+      const :view_class, T.class_of(View)
+      const :row_count, Integer
+      const :duration_ms, Integer
+      const :refreshed_at, T.nilable(Timestamp)
+      const :skipped, T::Boolean, default: false
+
+      # A no-op result, returned when refresh! was requested on a view that is
+      # not maintainable.
+      #
+      # @return [RefreshResult]
+      sig { params(view_class: T.class_of(View)).returns(RefreshResult) }
+      def self.skipped(view_class)
+        new(view_class: view_class, row_count: 0, duration_ms: 0, refreshed_at: nil, skipped: true)
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/refresh_scheduler.rb

@@ -0,0 +1,54 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # Dispatches a view's configured refresh strategy (`:async` / `:immediate` / `:manual`) after a write.
+    #
+    # @api private
+    class RefreshScheduler
+      class << self
+        extend T::Sig
+
+        sig { params(view_class: ViewClass).void }
+        def schedule(view_class)
+          # Capture the transition before marking dirty so the async dispatcher
+          # can coalesce: a bulk write only needs one job for the whole burst.
+          newly_dirty = !view_class.dirty?
+          view_class.mark_dependencies_changed!
+
+          case view_class.resolved_refresh_strategy
+          when :manual
+            nil
+          when :immediate
+            view_class.refresh!
+          when :async
+            dispatch_async(view_class, newly_dirty)
+          else
+            raise ArgumentError, "Unknown refresh strategy: #{view_class.resolved_refresh_strategy}"
+          end
+        end
+
+        private
+
+        sig { params(view_class: ViewClass, newly_dirty: T::Boolean).void }
+        def dispatch_async(view_class, newly_dirty)
+          if use_active_job?
+            # ActiveJob has no enqueue-level coalescing, so only enqueue when the
+            # view first goes dirty; the job drains the accumulated payload. The
+            # in-process refresher already coalesces via its debounce timer.
+            T.unsafe(RefreshJob).perform_later(view_class.view_key) if newly_dirty
+          else
+            AsyncRefresher.enqueue(view_class)
+          end
+        end
+
+        sig { returns(T::Boolean) }
+        def use_active_job?
+          config = ActiveRecord::Materialized.configuration
+          !!(config.refresh_dispatcher == :active_job && defined?(ActiveJob::Base))
+        end
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/refresher.rb

@@ -0,0 +1,139 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # Orchestrates explicit rebuilds and incremental maintenance for a single view.
+    #
+    # @api private
+    class Refresher
+      extend T::Sig
+
+      # Raised when a refresh or rebuild fails.
+      class RefreshError < StandardError; end
+
+      sig { returns(ViewClass) }
+      attr_reader :view_class
+
+      sig { params(view_class: ViewClass).void }
+      def initialize(view_class)
+        @view_class = view_class
+        @metadata = T.let(nil, T.nilable(Metadata))
+      end
+
+      # Full materialization — the only path that scans all base data.
+      sig { returns(RefreshResult) }
+      def rebuild!
+        run_cycle(-> { perform_rebuild! })
+      rescue StandardError => e
+        fail_refresh!(e)
+      end
+
+      # Incremental maintenance only; a no-op when the view is not maintainable.
+      sig { returns(RefreshResult) }
+      def refresh!
+        return RefreshResult.skipped(view_class) unless maintainable?
+
+        run_cycle(-> { incremental_refresh! })
+      rescue StandardError => e
+        fail_refresh!(e)
+      end
+
+      private
+
+      sig { returns(T::Boolean) }
+      def maintainable?
+        return false unless view_class.incrementally_maintainable?
+
+        pending = MaintenanceStore.new(view_class).pending
+        return false if pending.nil?
+
+        # Never full-populate a cold view from maintenance — reads fall through
+        # to the source instead. Scoped deltas populate just their partitions.
+        return false if !view_class.materialized? && pending.is_a?(MaintenanceDelta) && pending.full_partition?
+
+        true
+      end
+
+      sig { params(operation: T.proc.returns(Integer)).returns(RefreshResult) }
+      def run_cycle(operation)
+        raise RefreshError, "#{view_class.name} is already refreshing" if metadata.refreshing?
+
+        started_at = monotonic_clock
+        metadata.mark_refreshing!
+        view_class.run_refresh_callbacks(:before_refresh)
+
+        row_count = operation.call
+        result = complete_refresh!(row_count: row_count, duration_ms: elapsed_milliseconds(started_at))
+        view_class.run_refresh_callbacks(:after_refresh)
+        result
+      end
+
+      sig { returns(Integer) }
+      def perform_rebuild!
+        row_count = RelationCacheWriter.new(view_class).atomic_swap!(view_class.resolved_source)
+        metadata.mark_warm!
+        # Fully materialized now, so the cold-view partition exceptions no longer apply.
+        PartitionState.new(view_class).reset!
+        row_count
+      end
+
+      sig { returns(Integer) }
+      def incremental_refresh!
+        ensure_cache_table!
+
+        store = MaintenanceStore.new(view_class)
+        pending = store.pending
+        return apply_summary_delta!(store, pending) if pending.is_a?(SummaryDelta)
+
+        IncrementalMaintainer.new(view_class).maintain!(view_class.connection, view_class.table_name)
+      end
+
+      # Cheap DDL so partition maintenance has somewhere to write — never a populate.
+      sig { void }
+      def ensure_cache_table!
+        return if view_class.table_exists?
+
+        CacheTableSchema.ensure_table!(view_class, view_class.resolved_source)
+      end
+
+      sig { params(store: MaintenanceStore, summary: SummaryDelta).returns(Integer) }
+      def apply_summary_delta!(store, summary)
+        store.clear!
+        DeltaMaintainer.new(view_class).apply!(summary)
+      end
+
+      sig { params(error: StandardError).returns(T.noreturn) }
+      def fail_refresh!(error)
+        metadata.mark_failed!(error)
+        raise RefreshError, "Failed to refresh #{view_class.name}: #{error.message}", error.backtrace
+      end
+
+      sig { returns(Metadata) }
+      def metadata
+        @metadata ||= view_class.metadata
+      end
+
+      sig { returns(Float) }
+      def monotonic_clock
+        T.cast(Process.clock_gettime(Process::CLOCK_MONOTONIC), Float)
+      end
+
+      sig { params(started_at: Float).returns(Integer) }
+      def elapsed_milliseconds(started_at)
+        ((monotonic_clock - started_at) * 1000).round
+      end
+
+      sig { params(row_count: Integer, duration_ms: Integer).returns(RefreshResult) }
+      def complete_refresh!(row_count:, duration_ms:)
+        metadata.mark_refreshed!(row_count: row_count, duration_ms: duration_ms)
+        RefreshResult.new(
+          view_class: view_class,
+          row_count: row_count,
+          duration_ms: duration_ms,
+          refreshed_at: metadata.last_refreshed_at
+        )
+      end
+    end
+  end
+end