activerecord-materialized 0.1.0

data/lib/activerecord/materialized/incremental_maintainer.rb

@@ -0,0 +1,58 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # Hot-path scoped recompute: deletes and re-aggregates only the affected partitions in place.
+    #
+    # @api private
+    class IncrementalMaintainer
+      extend T::Sig
+
+      sig { params(view_class: ViewClass).void }
+      def initialize(view_class)
+        @view_class = view_class
+      end
+
+      sig { params(_connection: Connection, _table_name: String).returns(Integer) }
+      def maintain!(_connection, _table_name)
+        delta = maintenance_store.consume_pending_delta!
+        relation = resolve_maintenance_relation(delta)
+
+        row_count = RelationCacheWriter.new(view_class).replace_partitions!(
+          relation,
+          key_tuples: delta.key_tuples,
+          full_partition: delta.full_partition?
+        )
+
+        # On a cold view the maintained partitions are now fresh.
+        unless delta.full_partition? || view_class.materialized?
+          PartitionState.new(view_class).mark_fresh!(delta.key_tuples)
+        end
+
+        row_count
+      end
+
+      private
+
+      sig { returns(ViewClass) }
+      attr_reader :view_class
+
+      sig { returns(MaintenanceStore) }
+      def maintenance_store
+        MaintenanceStore.new(view_class)
+      end
+
+      sig { params(delta: MaintenanceDelta).returns(::ActiveRecord::Relation) }
+      def resolve_maintenance_relation(delta)
+        if view_class.incremental_source_override?
+          view_class.resolved_incremental_source
+        elsif delta.full_partition?
+          view_class.resolved_source
+        else
+          view_class.view_definition.partition_scope(delta.key_tuples)
+        end
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/maintenance_delta.rb

@@ -0,0 +1,82 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # Pending scoped-recompute maintenance: the affected partition keys, or a full-partition marker.
+    #
+    # @api private
+    class MaintenanceDelta
+      extend T::Sig
+
+      SCOPED = T.let(:scoped, Symbol)
+      FULL = T.let(:full_partition, Symbol)
+
+      sig { returns(Symbol) }
+      attr_reader :scope
+
+      sig { returns(T::Array[T::Array[String]]) }
+      attr_reader :key_tuples
+
+      sig { params(scope: Symbol, key_tuples: T::Array[T::Array[String]]).void }
+      def initialize(scope:, key_tuples: [])
+        @scope = scope
+        @key_tuples = key_tuples
+      end
+
+      sig { params(key_tuples: T::Array[T::Array[String]]).returns(MaintenanceDelta) }
+      def self.scoped(key_tuples)
+        new(scope: SCOPED, key_tuples: key_tuples)
+      end
+
+      sig { returns(MaintenanceDelta) }
+      def self.full_partition
+        new(scope: FULL)
+      end
+
+      sig { returns(T::Boolean) }
+      def full_partition?
+        scope == FULL
+      end
+
+      # How many distinct partitions this pending maintenance tracks. A
+      # full-partition recompute tracks none — it is already the collapsed form.
+      sig { returns(Integer) }
+      def tracked_partition_count
+        full_partition? ? 0 : key_tuples.size
+      end
+
+      sig { params(other: MaintenanceDelta).returns(MaintenanceDelta) }
+      def merge(other)
+        return other if other.full_partition?
+        return self if full_partition?
+
+        combined = (key_tuples + other.key_tuples).uniq
+        self.class.scoped(combined)
+      end
+
+      sig { returns(T::Hash[String, T.untyped]) }
+      def serialize
+        {
+          "scope" => scope.to_s,
+          "key_tuples" => key_tuples
+        }
+      end
+
+      sig { params(payload: T.nilable(T::Hash[String, T.untyped])).returns(T.nilable(MaintenanceDelta)) }
+      def self.deserialize(payload)
+        return nil if payload.blank?
+
+        scope_name = payload["scope"]&.to_sym
+        return nil if scope_name.nil?
+
+        if scope_name == FULL
+          full_partition
+        else
+          tuples = T.cast(payload["key_tuples"], T.nilable(T::Array[T::Array[String]])) || []
+          scoped(tuples)
+        end
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/maintenance_delta_builder.rb

@@ -0,0 +1,62 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # Derives the affected partition keys for a write from its ActiveRecord change payload.
+    #
+    # @api private
+    class MaintenanceDeltaBuilder
+      extend T::Sig
+
+      sig { params(change: WriteChange, key_columns: T::Array[String]).void }
+      def initialize(change, key_columns)
+        @change = change
+        @key_columns = key_columns
+      end
+
+      sig { returns(MaintenanceDelta) }
+      def build
+        return MaintenanceDelta.full_partition if @key_columns.empty?
+
+        tuples = extract_tuples
+        tuples.empty? ? MaintenanceDelta.full_partition : MaintenanceDelta.scoped(tuples.uniq)
+      end
+
+      private
+
+      sig { returns(T::Array[T::Array[T.untyped]]) }
+      def extract_tuples
+        snapshots = case @change.operation.to_sym
+                    when :create then [@change.after]
+                    when :destroy then [@change.before]
+                    when :update then [@change.before, @change.after]
+                    else []
+                    end
+        snapshots.filter_map { |attributes| key_tuple(attributes) }.uniq
+      end
+
+      sig { params(attributes: T::Hash[String, T.untyped]).returns(T::Boolean) }
+      def keys_present?(attributes)
+        @key_columns.all? do |column|
+          attributes.key?(column) || T.unsafe(attributes).key?(column.to_sym)
+        end
+      end
+
+      sig { params(attributes: T::Hash[String, T.untyped]).returns(T.nilable(T::Array[T.untyped])) }
+      def key_tuple(attributes)
+        return nil unless keys_present?(attributes)
+
+        @key_columns.map { |column| attribute_value(attributes, column) }
+      end
+
+      sig { params(attributes: T::Hash[String, T.untyped], column: String).returns(T.untyped) }
+      def attribute_value(attributes, column)
+        # Look up by presence so falsey group-key values map to their partition.
+        return attributes[column] if attributes.key?(column)
+
+        T.unsafe(attributes)[column.to_sym]
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/maintenance_store.rb

@@ -0,0 +1,82 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # Persists a view's pending maintenance (a delta or a scope) in its metadata row.
+    #
+    # @api private
+    class MaintenanceStore
+      extend T::Sig
+
+      Pending = T.type_alias { T.any(SummaryDelta, MaintenanceDelta) }
+
+      sig { params(view_class: ViewClass).void }
+      def initialize(view_class)
+        @view_class = view_class
+      end
+
+      # Accumulates pending maintenance of either kind. A view's mode is fixed
+      # within a window, so existing pending is always the same kind. Once the
+      # tracked partitions exceed the configured cap, the payload collapses to a
+      # single full recompute, so a bulk write spanning many partitions stays
+      # O(1) per write instead of re-serializing an ever-growing blob.
+      sig { params(delta: Pending).void }
+      def merge!(delta)
+        metadata.record_maintenance_payload!(T.unsafe(combine(pending, delta)).serialize)
+      end
+
+      sig { returns(T.nilable(Pending)) }
+      def pending
+        payload = metadata.maintenance_payload
+        SummaryDelta.deserialize(payload) || MaintenanceDelta.deserialize(payload)
+      end
+
+      sig { returns(T.nilable(MaintenanceDelta)) }
+      def pending_delta
+        MaintenanceDelta.deserialize(metadata.maintenance_payload)
+      end
+
+      sig { returns(MaintenanceDelta) }
+      def consume_pending_delta!
+        delta = pending_delta || MaintenanceDelta.full_partition
+        clear!
+        delta
+      end
+
+      sig { void }
+      def clear!
+        metadata.clear_maintenance_payload!
+      end
+
+      private
+
+      sig { params(current: T.nilable(Pending), delta: Pending).returns(Pending) }
+      def combine(current, delta)
+        return delta if current.nil?
+        return current if recompute_all?(current) # terminal: absorb everything
+
+        merged = current.instance_of?(delta.class) ? T.unsafe(current).merge(delta) : delta
+        oversized?(merged) ? MaintenanceDelta.full_partition : merged
+      end
+
+      sig { params(pending: T.nilable(Pending)).returns(T::Boolean) }
+      def recompute_all?(pending)
+        pending.is_a?(MaintenanceDelta) && pending.full_partition?
+      end
+
+      sig { params(merged: Pending).returns(T::Boolean) }
+      def oversized?(merged)
+        merged.tracked_partition_count > ActiveRecord::Materialized.configuration.max_tracked_partitions
+      end
+
+      sig { returns(ViewClass) }
+      attr_reader :view_class
+
+      sig { returns(Metadata) }
+      def metadata
+        view_class.metadata
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/metadata/maintenance_payload.rb

@@ -0,0 +1,33 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    class Metadata
+      # Reads and writes the serialized pending-maintenance payload on the metadata row.
+      #
+      # @api private
+      module MaintenancePayload
+        extend T::Sig
+
+        sig { params(metadata: Metadata, payload: T::Hash[String, T.untyped]).void }
+        def self.record!(metadata, payload)
+          metadata.record.update!(maintenance_payload: payload.to_json)
+        end
+
+        sig { params(metadata: Metadata).returns(T.nilable(T::Hash[String, T.untyped])) }
+        def self.fetch(metadata)
+          raw = T.unsafe(metadata.record).maintenance_payload
+          return nil if raw.blank?
+
+          JSON.parse(raw)
+        end
+
+        sig { params(metadata: Metadata).void }
+        def self.clear!(metadata)
+          metadata.record.update!(maintenance_payload: nil)
+        end
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/metadata/schema.rb

@@ -0,0 +1,84 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    class Metadata
+      # Lazily provisions and migrates the materialized-view metadata table.
+      #
+      # @api private
+      module Schema
+        extend T::Sig
+
+        module_function
+
+        sig { params(view_class: ViewClass).void }
+        def ensure_table!(view_class)
+          connection = view_class.connection
+          create_metadata_table!(connection) unless MetadataRecord.table_exists?
+          ensure_dirty_column!(connection)
+          ensure_maintenance_payload_column!(connection)
+          ensure_warm_column!(connection)
+          MetadataRecord.reset_column_information
+        end
+
+        sig { params(connection: Connection).void }
+        def create_metadata_table!(connection)
+          connection.create_table(::ActiveRecord::Materialized.metadata_table_name, force: :cascade) do |t|
+            t.string :view_name, null: false
+            t.datetime :last_refreshed_at
+            t.boolean :refreshing, null: false, default: false
+            t.boolean :dirty, null: false, default: true
+            t.boolean :warm, null: false, default: false
+            t.integer :row_count
+            t.integer :refresh_duration_ms
+            t.text :last_error
+            t.text :maintenance_payload
+            t.timestamps
+          end
+          connection.add_index(::ActiveRecord::Materialized.metadata_table_name, :view_name, unique: true)
+        end
+
+        sig { params(connection: Connection).void }
+        def ensure_dirty_column!(connection)
+          return unless MetadataRecord.table_exists?
+          return if MetadataRecord.column_names.include?("dirty")
+
+          connection.add_column(
+            ::ActiveRecord::Materialized.metadata_table_name,
+            :dirty,
+            :boolean,
+            default: true,
+            null: false
+          )
+        end
+
+        sig { params(connection: Connection).void }
+        def ensure_maintenance_payload_column!(connection)
+          return unless MetadataRecord.table_exists?
+          return if MetadataRecord.column_names.include?("maintenance_payload")
+
+          connection.add_column(
+            ::ActiveRecord::Materialized.metadata_table_name,
+            :maintenance_payload,
+            :text
+          )
+        end
+
+        sig { params(connection: Connection).void }
+        def ensure_warm_column!(connection)
+          return unless MetadataRecord.table_exists?
+          return if MetadataRecord.column_names.include?("warm")
+
+          connection.add_column(
+            ::ActiveRecord::Materialized.metadata_table_name,
+            :warm,
+            :boolean,
+            default: false,
+            null: false
+          )
+        end
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/metadata/timestamps.rb

@@ -0,0 +1,31 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    class Metadata
+      # Current-time and staleness-threshold helpers for metadata timestamps.
+      #
+      # @api private
+      module Timestamps
+        extend T::Sig
+
+        module_function
+
+        sig { returns(Timestamp) }
+        def current
+          ::Time.zone&.now || ::Time.now.utc
+        end
+
+        sig { params(staleness: StalenessDuration).returns(Timestamp) }
+        def threshold(staleness)
+          if staleness.is_a?(Integer)
+            ::ActiveSupport::Duration.seconds(staleness).ago
+          else
+            staleness.ago
+          end
+        end
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/metadata.rb

@@ -0,0 +1,138 @@
+# typed: strict
+# frozen_string_literal: true
+
+require_relative "metadata/schema"
+require_relative "metadata/maintenance_payload"
+require_relative "metadata/timestamps"
+
+module ActiveRecord
+  module Materialized
+    # Reads and writes a view's freshness metadata row (dirty, warm, last_refreshed_at, …).
+    #
+    # @api private
+    class Metadata
+      extend T::Sig
+
+      sig { returns(ViewClass) }
+      attr_reader :view_class
+
+      sig { params(view_class: ViewClass).void }
+      def initialize(view_class)
+        @view_class = view_class
+        @schema_ensured = T.let(false, T::Boolean)
+      end
+
+      # Single entry point for the metadata row. Provision the schema once per
+      # instance — re-ensuring on every access thrashes the schema cache and
+      # makes high-write workloads quadratic.
+      sig { returns(MetadataRecord) }
+      def record
+        ensure_schema!
+        MetadataRecord.find_or_initialize_by(view_name: view_class.view_key)
+      end
+
+      sig { void }
+      def ensure_schema!
+        return if @schema_ensured
+
+        Schema.ensure_table!(view_class)
+        @schema_ensured = true
+      end
+
+      sig { returns(T.nilable(Timestamp)) }
+      def last_refreshed_at
+        record.last_refreshed_at
+      end
+
+      sig { returns(T::Boolean) }
+      def refreshing?
+        !!record.refreshing?
+      end
+
+      sig { returns(T.nilable(Integer)) }
+      def row_count
+        record.row_count
+      end
+
+      sig { returns(T.nilable(Integer)) }
+      def refresh_duration_ms
+        record.refresh_duration_ms
+      end
+
+      sig { returns(T::Boolean) }
+      def dirty?
+        !!record.dirty?
+      end
+
+      # Warm once fully materialized via rebuild!; cold views read through.
+      sig { returns(T::Boolean) }
+      def warm?
+        !!record.warm?
+      end
+
+      sig { void }
+      def mark_warm!
+        record.update!(warm: true)
+      end
+
+      sig { params(max_staleness: T.nilable(StalenessDuration)).returns(T::Boolean) }
+      def stale?(max_staleness: view_class.resolved_max_staleness)
+        return true if dirty?
+        return true if last_refreshed_at.nil?
+        return false if max_staleness.nil?
+
+        refreshed_at = T.must(last_refreshed_at)
+        refreshed_at.to_time < Timestamps.threshold(max_staleness).to_time
+      end
+
+      sig { void }
+      def mark_dirty!
+        record.update!(dirty: true)
+      end
+
+      sig { params(payload: T::Hash[String, T.untyped]).void }
+      def record_maintenance_payload!(payload)
+        MaintenancePayload.record!(self, payload)
+      end
+
+      sig { returns(T.nilable(T::Hash[String, T.untyped])) }
+      def maintenance_payload
+        MaintenancePayload.fetch(self)
+      end
+
+      sig { void }
+      def clear_maintenance_payload!
+        MaintenancePayload.clear!(self)
+      end
+
+      sig { void }
+      def mark_refreshing!
+        record.update!(
+          refreshing: true,
+          last_error: nil
+        )
+      end
+
+      sig { params(row_count: Integer, duration_ms: Integer).void }
+      def mark_refreshed!(row_count:, duration_ms:)
+        record.update!(
+          last_refreshed_at: Timestamps.current,
+          refreshing: false,
+          dirty: false,
+          row_count: row_count,
+          refresh_duration_ms: duration_ms,
+          last_error: nil,
+          maintenance_payload: nil
+        )
+      end
+
+      sig { params(error: StandardError).void }
+      def mark_failed!(error)
+        record.update!(
+          refreshing: false,
+          last_error: error.message
+        )
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/metadata_record.rb

@@ -0,0 +1,28 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # ActiveRecord model backing the materialized-view metadata table.
+    #
+    # @api private
+    class MetadataRecord < ::ActiveRecord::Base
+      extend T::Sig
+
+      @table_name_override = T.let(nil, T.nilable(String))
+
+      self.table_name = ::ActiveRecord::Materialized.metadata_table_name
+
+      sig { params(name: String).void }
+      def self.table_name=(name)
+        @table_name_override = name
+      end
+
+      sig { returns(String) }
+      def self.table_name
+        override = @table_name_override
+        override.nil? ? ::ActiveRecord::Materialized.metadata_table_name : override
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/migration_builder.rb

@@ -0,0 +1,38 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # The data a generated migration needs to provision a view's empty cache
+    # table: table name, migration class name, target version, and inferred
+    # columns/types. The file itself is produced by Rails' generator tooling.
+    class MigrationBuilder
+      extend T::Sig
+
+      sig { params(view_class: ViewClass).void }
+      def initialize(view_class)
+        @view_class = view_class
+      end
+
+      sig { returns(String) }
+      def table_name
+        @view_class.table_name
+      end
+
+      sig { returns(String) }
+      def migration_class_name
+        "Create#{table_name.camelize}"
+      end
+
+      sig { returns(T.any(String, Float)) }
+      def migration_version
+        ::ActiveRecord::Migration.current_version
+      end
+
+      sig { returns(T::Array[CacheTableSchema::ColumnDefinition]) }
+      def column_definitions
+        CacheTableSchema.column_definitions(@view_class.connection, @view_class.resolved_source)
+      end
+    end
+  end
+end