activerecord-materialized 0.1.1 → 0.2.0

data/lib/activerecord/materialized/maintenance_store.rb

@@ -1,4 +1,3 @@
-# typed: strict
 # frozen_string_literal: true
 
 module ActiveRecord
@@ -7,11 +6,6 @@ module ActiveRecord
     #
     # @api private
     class MaintenanceStore
-      extend T::Sig
-
-      Pending = T.type_alias { T.any(SummaryDelta, MaintenanceDelta) }
-
-      sig { params(view_class: ViewClass).void }
       def initialize(view_class)
         @view_class = view_class
       end
@@ -21,59 +15,102 @@ module ActiveRecord
       # tracked partitions exceed the configured cap, the payload collapses to a
       # single full recompute, so a bulk write spanning many partitions stays
       # O(1) per write instead of re-serializing an ever-growing blob.
-      sig { params(delta: Pending).void }
       def merge!(delta)
-        metadata.record_maintenance_payload!(T.unsafe(combine(pending, delta)).serialize)
+        combined = combine(pending, delta)
+        return drop_cold_full_recompute! if cold_full_recompute?(combined)
+
+        metadata.record_maintenance_payload!(combined.serialize)
       end
 
-      sig { returns(T.nilable(Pending)) }
       def pending
         payload = metadata.maintenance_payload
         SummaryDelta.deserialize(payload) || MaintenanceDelta.deserialize(payload)
       end
 
-      sig { returns(T.nilable(MaintenanceDelta)) }
       def pending_delta
         MaintenanceDelta.deserialize(metadata.maintenance_payload)
       end
 
-      sig { returns(MaintenanceDelta) }
+      # Atomically consume the pending scoped delta under a row lock on the metadata row, so two
+      # cross-process cycles can't both consume it and recompute the same partition twice — which
+      # duplicated rows on Postgres (no unique constraint, no gap locks under READ COMMITTED). Whoever
+      # clears the payload owns the delta; a loser reads an empty payload and gets nil (a benign
+      # no-op). Returns the delta, or nil. Unlike the additive summary path (which applies inside its
+      # lock for crash-atomicity), the scoped recompute is idempotent (delete + re-aggregate), so the
+      # caller applies OUTSIDE this lock, keeping it short; a crash between consume and apply loses the
+      # delta, which self-healing reconciliation (#64) repairs on the next tick.
       def consume_pending_delta!
-        delta = pending_delta || MaintenanceDelta.full_partition
-        clear!
-        delta
+        metadata.record.with_lock do # blocking FOR UPDATE on the metadata row + a transaction
+          delta = pending_delta
+          next nil if delta.nil?
+
+          clear!
+          delta
+        end
+      end
+
+      # Atomically consume the pending SummaryDelta and apply it under a row lock on the metadata
+      # row, so concurrent cycles across servers can't apply the same additive delta twice. Yields
+      # the delta to the block (which applies it) inside the locked transaction, so a failed apply
+      # rolls back the clear and the delta is retried. Returns the block's result, or nil when another
+      # cycle already consumed it — the loser blocks on the lock, then reads an empty payload (a
+      # benign no-op). DML-only (no DDL/callbacks), so the lock holds for the whole critical section.
+      def with_consumed_summary_delta
+        metadata.record.with_lock do # blocking FOR UPDATE on the metadata row + a transaction
+          delta = SummaryDelta.deserialize(metadata.maintenance_payload)
+          next nil if delta.nil?
+
+          clear!
+          yield delta
+        end
       end
 
-      sig { void }
       def clear!
         metadata.clear_maintenance_payload!
       end
 
       private
 
-      sig { params(current: T.nilable(Pending), delta: Pending).returns(Pending) }
       def combine(current, delta)
         return delta if current.nil?
         return current if recompute_all?(current) # terminal: absorb everything
+        # Different kinds can't merge (a summary delta meeting a scoped recompute — e.g.
+        # reconcile's repair racing a callback write); widen to a full recompute rather
+        # than dropping either side's pending maintenance.
+        return MaintenanceDelta.full_partition unless current.instance_of?(delta.class)
 
-        merged = current.instance_of?(delta.class) ? T.unsafe(current).merge(delta) : delta
+        merged = current.merge(delta)
         oversized?(merged) ? MaintenanceDelta.full_partition : merged
       end
 
-      sig { params(pending: T.nilable(Pending)).returns(T::Boolean) }
       def recompute_all?(pending)
         pending.is_a?(MaintenanceDelta) && pending.full_partition?
       end
 
-      sig { params(merged: Pending).returns(T::Boolean) }
+      # A cold view can't apply a full-partition recompute (Refresher#maintainable? skips it) and storing
+      # one gums up the payload: combine's terminal absorb (see #recompute_all?) then swallows every later
+      # scoped read-miss delta, so populate-on-read never repopulates. Every cold widen funnels through
+      # merge! — an ingested widening write, DependencyRegistry.mark_dirty_for_tables!, or a scoped payload
+      # that overflows max_tracked_partitions inside #combine — so this one guard covers them all. The
+      # cheap in-memory recompute_all? short-circuits before materialized?, so the scoped hot path is free.
+      def cold_full_recompute?(combined)
+        recompute_all?(combined) && !view_class.materialized?
+      end
+
+      # Drop the un-appliable recompute and invalidate the whole fresh set so every partition falls
+      # through to the source until a read-miss repopulates it. Reset first, then clear: a crash between
+      # the two leaves reads correct-by-fallback with only an un-appliable payload the next merge! re-drops.
+      def drop_cold_full_recompute!
+        PartitionState.new(view_class).reset!
+        clear!
+      end
+
       def oversized?(merged)
         merged.tracked_partition_count > ActiveRecord::Materialized.configuration.max_tracked_partitions
       end
 
-      sig { returns(ViewClass) }
       attr_reader :view_class
 
-      sig { returns(Metadata) }
       def metadata
         view_class.metadata
       end

data/lib/activerecord/materialized/metadata/maintenance_payload.rb

@@ -1,4 +1,3 @@
-# typed: strict
 # frozen_string_literal: true
 
 module ActiveRecord
@@ -8,22 +7,17 @@ module ActiveRecord
       #
       # @api private
       module MaintenancePayload
-        extend T::Sig
-
-        sig { params(metadata: Metadata, payload: T::Hash[String, T.untyped]).void }
         def self.record!(metadata, payload)
           metadata.record.update!(maintenance_payload: payload.to_json)
         end
 
-        sig { params(metadata: Metadata).returns(T.nilable(T::Hash[String, T.untyped])) }
         def self.fetch(metadata)
-          raw = T.unsafe(metadata.record).maintenance_payload
+          raw = metadata.record.maintenance_payload
           return nil if raw.blank?
 
           JSON.parse(raw)
         end
 
-        sig { params(metadata: Metadata).void }
         def self.clear!(metadata)
           metadata.record.update!(maintenance_payload: nil)
         end

data/lib/activerecord/materialized/metadata/reconciliation.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    class Metadata
+      # Writes a view's reconciliation bookkeeping — +last_reconciled_at+ and
+      # +reconciled_partition_count+ — on the metadata row. Kept out of {Metadata}
+      # itself the way {MaintenancePayload} is; reads go through the metadata record.
+      #
+      # @api private
+      module Reconciliation
+        module_function
+
+        # Stamp a completed reconciliation pass, resetting the staleness clock even
+        # when no partition needed repair.
+        def mark!(metadata, repaired_partition_count:)
+          metadata.record.update!(
+            last_reconciled_at: Timestamps.current,
+            reconciled_partition_count: repaired_partition_count
+          )
+        end
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/metadata/schema.rb

@@ -1,4 +1,3 @@
-# typed: strict
 # frozen_string_literal: true
 
 module ActiveRecord
@@ -8,23 +7,18 @@ module ActiveRecord
       #
       # @api private
       module Schema
-        extend T::Sig
-
         module_function
 
-        sig { params(view_class: ViewClass).void }
         def ensure_table!(view_class)
           connection = view_class.connection
           create_metadata_table!(connection) unless MetadataRecord.table_exists?
-          ensure_dirty_column!(connection)
-          ensure_maintenance_payload_column!(connection)
-          ensure_warm_column!(connection)
+          ensure_columns!(connection)
           MetadataRecord.reset_column_information
         end
 
-        sig { params(connection: Connection).void }
         def create_metadata_table!(connection)
-          connection.create_table(::ActiveRecord::Materialized.metadata_table_name, force: :cascade) do |t|
+          table = ::ActiveRecord::Materialized.metadata_table_name
+          connection.create_table(table, force: :cascade) do |t|
             t.string :view_name, null: false
             t.datetime :last_refreshed_at
             t.boolean :refreshing, null: false, default: false
@@ -34,48 +28,32 @@ module ActiveRecord
             t.integer :refresh_duration_ms
             t.text :last_error
             t.text :maintenance_payload
+            t.datetime :last_reconciled_at
+            t.integer :reconciled_partition_count
+            t.integer :fresh_set_generation, null: false, default: 0
             t.timestamps
           end
-          connection.add_index(::ActiveRecord::Materialized.metadata_table_name, :view_name, unique: true)
-        end
-
-        sig { params(connection: Connection).void }
-        def ensure_dirty_column!(connection)
-          return unless MetadataRecord.table_exists?
-          return if MetadataRecord.column_names.include?("dirty")
-
-          connection.add_column(
-            ::ActiveRecord::Materialized.metadata_table_name,
-            :dirty,
-            :boolean,
-            default: true,
-            null: false
-          )
+          connection.add_index(table, :view_name, unique: true)
         end
 
-        sig { params(connection: Connection).void }
-        def ensure_maintenance_payload_column!(connection)
+        # Lazily add columns introduced after a table was first provisioned, so an app
+        # migrated from an earlier version picks them up without a new migration.
+        def ensure_columns!(connection)
           return unless MetadataRecord.table_exists?
-          return if MetadataRecord.column_names.include?("maintenance_payload")
 
-          connection.add_column(
-            ::ActiveRecord::Materialized.metadata_table_name,
-            :maintenance_payload,
-            :text
-          )
+          ensure_column!(connection, :dirty, :boolean, default: true, null: false)
+          ensure_column!(connection, :warm, :boolean, default: false, null: false)
+          ensure_column!(connection, :maintenance_payload, :text)
+          ensure_column!(connection, :last_reconciled_at, :datetime)
+          ensure_column!(connection, :reconciled_partition_count, :integer)
+          ensure_column!(connection, :fresh_set_generation, :integer, default: 0, null: false)
         end
 
-        sig { params(connection: Connection).void }
-        def ensure_warm_column!(connection)
-          return unless MetadataRecord.table_exists?
-          return if MetadataRecord.column_names.include?("warm")
+        def ensure_column!(connection, name, type, default: nil, null: true)
+          return if MetadataRecord.column_names.include?(name.to_s)
 
           connection.add_column(
-            ::ActiveRecord::Materialized.metadata_table_name,
-            :warm,
-            :boolean,
-            default: false,
-            null: false
+            ::ActiveRecord::Materialized.metadata_table_name, name, type, default: default, null: null
           )
         end
       end

data/lib/activerecord/materialized/metadata/timestamps.rb

@@ -1,4 +1,3 @@
-# typed: strict
 # frozen_string_literal: true
 
 module ActiveRecord
@@ -8,16 +7,12 @@ module ActiveRecord
       #
       # @api private
       module Timestamps
-        extend T::Sig
-
         module_function
 
-        sig { returns(Timestamp) }
         def current
           ::Time.zone&.now || ::Time.now.utc
         end
 
-        sig { params(staleness: StalenessDuration).returns(Timestamp) }
         def threshold(staleness)
           if staleness.is_a?(Integer)
             ::ActiveSupport::Duration.seconds(staleness).ago

data/lib/activerecord/materialized/metadata.rb

@@ -1,8 +1,8 @@
-# typed: strict
 # frozen_string_literal: true
 
 require_relative "metadata/schema"
 require_relative "metadata/maintenance_payload"
+require_relative "metadata/reconciliation"
 require_relative "metadata/timestamps"
 
 module ActiveRecord
@@ -11,27 +11,21 @@ module ActiveRecord
     #
     # @api private
     class Metadata
-      extend T::Sig
-
-      sig { returns(ViewClass) }
       attr_reader :view_class
 
-      sig { params(view_class: ViewClass).void }
       def initialize(view_class)
         @view_class = view_class
-        @schema_ensured = T.let(false, T::Boolean)
+        @schema_ensured = false
       end
 
       # Single entry point for the metadata row. Provision the schema once per
       # instance — re-ensuring on every access thrashes the schema cache and
       # makes high-write workloads quadratic.
-      sig { returns(MetadataRecord) }
       def record
         ensure_schema!
         MetadataRecord.find_or_initialize_by(view_name: view_class.view_key)
       end
 
-      sig { void }
       def ensure_schema!
         return if @schema_ensured
 
@@ -39,73 +33,86 @@ module ActiveRecord
         @schema_ensured = true
       end
 
-      sig { returns(T.nilable(Timestamp)) }
       def last_refreshed_at
         record.last_refreshed_at
       end
 
-      sig { returns(T::Boolean) }
       def refreshing?
         !!record.refreshing?
       end
 
-      sig { returns(T.nilable(Integer)) }
       def row_count
         record.row_count
       end
 
-      sig { returns(T.nilable(Integer)) }
       def refresh_duration_ms
         record.refresh_duration_ms
       end
 
-      sig { returns(T::Boolean) }
       def dirty?
         !!record.dirty?
       end
 
       # Warm once fully materialized via rebuild!; cold views read through.
-      sig { returns(T::Boolean) }
       def warm?
         !!record.warm?
       end
 
-      sig { void }
       def mark_warm!
         record.update!(warm: true)
       end
 
-      sig { params(max_staleness: T.nilable(StalenessDuration)).returns(T::Boolean) }
+      # The current fresh-set epoch for this view (see PartitionState). A populate captures it before
+      # its source read; all_fresh? honours only marks stamped with the current value. (#120)
+      def fresh_set_generation
+        record.fresh_set_generation
+      end
+
+      # Advance the fresh-set epoch atomically so every mark stamped with an earlier value is treated
+      # as stale — used by PartitionState#reset! to invalidate a cold view's whole fresh set on a widen.
+      # Ensures the row exists first (create_or_find_by absorbs a concurrent first insert), then a single
+      # atomic SQL increment (never a read-then-write), so two concurrent resets can't lost-update the
+      # epoch — a populate that captured any earlier value still fails all_fresh?. Portable across adapters.
+      def bump_fresh_set_generation!
+        ensure_schema!
+        MetadataRecord.create_or_find_by(view_name: view_class.view_key)
+        connection = view_class.connection
+        table = connection.quote_table_name(::ActiveRecord::Materialized.metadata_table_name)
+        connection.execute(
+          "UPDATE #{table} SET fresh_set_generation = fresh_set_generation + 1 " \
+          "WHERE view_name = #{connection.quote(view_class.view_key)}"
+        )
+      end
+
       def stale?(max_staleness: view_class.resolved_max_staleness)
         return true if dirty?
         return true if last_refreshed_at.nil?
         return false if max_staleness.nil?
 
-        refreshed_at = T.must(last_refreshed_at)
-        refreshed_at.to_time < Timestamps.threshold(max_staleness).to_time
+        # A reconcile verifies contents against the source, so it resets the staleness clock like a
+        # refresh — measure age from whichever happened later. The replica-lag budget tightens the
+        # window: a replica read trails the primary by the lag, so the view goes stale that much
+        # sooner to keep replica reads within max_staleness.
+        freshest = [last_refreshed_at, record.last_reconciled_at].compact.max_by(&:to_time)
+        freshest.to_time < Timestamps.threshold(effective_staleness(max_staleness)).to_time
       end
 
-      sig { void }
       def mark_dirty!
         record.update!(dirty: true)
       end
 
-      sig { params(payload: T::Hash[String, T.untyped]).void }
       def record_maintenance_payload!(payload)
         MaintenancePayload.record!(self, payload)
       end
 
-      sig { returns(T.nilable(T::Hash[String, T.untyped])) }
       def maintenance_payload
         MaintenancePayload.fetch(self)
       end
 
-      sig { void }
       def clear_maintenance_payload!
         MaintenancePayload.clear!(self)
       end
 
-      sig { void }
       def mark_refreshing!
         record.update!(
           refreshing: true,
@@ -113,7 +120,6 @@ module ActiveRecord
         )
       end
 
-      sig { params(row_count: Integer, duration_ms: Integer).void }
       def mark_refreshed!(row_count:, duration_ms:)
         record.update!(
           last_refreshed_at: Timestamps.current,
@@ -126,13 +132,23 @@ module ActiveRecord
         )
       end
 
-      sig { params(error: StandardError).void }
       def mark_failed!(error)
         record.update!(
           refreshing: false,
           last_error: error.message
         )
       end
+
+      private
+
+      # max_staleness minus the configured replica-lag budget (clamped at 0), so replica reads stay
+      # within the window. Zero lag (the default) leaves max_staleness unchanged.
+      def effective_staleness(max_staleness)
+        lag = ActiveRecord::Materialized.configuration.replica_lag
+        return max_staleness if lag.nil? || lag.zero?
+
+        [max_staleness - lag, 0].max
+      end
     end
   end
 end

data/lib/activerecord/materialized/metadata_record.rb

@@ -1,4 +1,3 @@
-# typed: strict
 # frozen_string_literal: true
 
 module ActiveRecord
@@ -7,22 +6,9 @@ module ActiveRecord
     #
     # @api private
     class MetadataRecord < ::ActiveRecord::Base
-      extend T::Sig
+      include ConfigurableTableName
 
-      @table_name_override = T.let(nil, T.nilable(String))
-
-      self.table_name = ::ActiveRecord::Materialized.metadata_table_name
-
-      sig { params(name: String).void }
-      def self.table_name=(name)
-        @table_name_override = name
-      end
-
-      sig { returns(String) }
-      def self.table_name
-        override = @table_name_override
-        override.nil? ? ::ActiveRecord::Materialized.metadata_table_name : override
-      end
+      configurable_table_name { ::ActiveRecord::Materialized.metadata_table_name }
     end
   end
 end

data/lib/activerecord/materialized/migration_builder.rb

@@ -1,4 +1,3 @@
-# typed: strict
 # frozen_string_literal: true
 
 module ActiveRecord
@@ -7,32 +6,33 @@ module ActiveRecord
     # table: table name, migration class name, target version, and inferred
     # columns/types. The file itself is produced by Rails' generator tooling.
     class MigrationBuilder
-      extend T::Sig
-
-      sig { params(view_class: ViewClass).void }
       def initialize(view_class)
         @view_class = view_class
       end
 
-      sig { returns(String) }
       def table_name
         @view_class.table_name
       end
 
-      sig { returns(String) }
       def migration_class_name
         "Create#{table_name.camelize}"
       end
 
-      sig { returns(T.any(String, Float)) }
       def migration_version
         ::ActiveRecord::Migration.current_version
       end
 
-      sig { returns(T::Array[CacheTableSchema::ColumnDefinition]) }
       def column_definitions
         CacheTableSchema.column_definitions(@view_class.connection, @view_class.resolved_source)
       end
+
+      # The partition-key index the migration should add (nil for a non-grouped view). Incremental
+      # maintenance is keyed on these columns, so the generated migration indexes them by default.
+      #
+      # @return [CacheTableSchema::IndexDefinition, nil]
+      def index_definition
+        CacheTableSchema.index_definition(@view_class)
+      end
     end
   end
 end