activerecord-materialized 0.1.1 → 0.2.0

data/lib/activerecord/materialized/query_expressions.rb

@@ -1,4 +1,3 @@
-# typed: strict
 # frozen_string_literal: true
 
 module ActiveRecord
@@ -21,60 +20,49 @@ module ActiveRecord
     #     end
     #   end
     module QueryExpressions
-      extend T::Sig
-
       module_function
 
       # @return [Arel::Nodes::As] +SUM(attribute) AS <as>+
-      sig { params(attribute: Arel::Attributes::Attribute, as: T.any(Symbol, String)).returns(Arel::Nodes::As) }
       def sum_as(attribute, as:)
         attribute.sum.as(as.to_s)
       end
 
       # @return [Arel::Nodes::As] +AVG(attribute) AS <as>+
-      sig { params(attribute: Arel::Attributes::Attribute, as: T.any(Symbol, String)).returns(Arel::Nodes::As) }
       def avg_as(attribute, as:)
         attribute.average.as(as.to_s)
       end
 
       # @return [Arel::Nodes::As] +MIN(attribute) AS <as>+
-      sig { params(attribute: Arel::Attributes::Attribute, as: T.any(Symbol, String)).returns(Arel::Nodes::As) }
       def min_as(attribute, as:)
         attribute.minimum.as(as.to_s)
       end
 
       # @return [Arel::Nodes::As] +MAX(attribute) AS <as>+
-      sig { params(attribute: Arel::Attributes::Attribute, as: T.any(Symbol, String)).returns(Arel::Nodes::As) }
       def max_as(attribute, as:)
         attribute.maximum.as(as.to_s)
       end
 
       # @return [Arel::Nodes::As] +COUNT(*) AS <as>+ — a trustworthy per-partition row count
-      sig { params(as: T.any(Symbol, String)).returns(Arel::Nodes::As) }
       def count_all_as(as:)
         Arel.star.count.as(as.to_s)
       end
 
       # @return [Arel::Nodes::As] +COUNT(attribute) AS <as>+ (non-null values)
-      sig { params(attribute: Arel::Attributes::Attribute, as: T.any(Symbol, String)).returns(Arel::Nodes::As) }
       def count_as(attribute, as:)
         attribute.count.as(as.to_s)
       end
 
       # @return [Arel::Nodes::As] +COUNT(DISTINCT attribute) AS <as>+
-      sig { params(attribute: Arel::Attributes::Attribute, as: T.any(Symbol, String)).returns(Arel::Nodes::As) }
       def count_distinct_as(attribute, as:)
         attribute.count(true).as(as.to_s)
       end
 
       # @return [Arel::Nodes::NamedFunction] +LENGTH(attribute)+
-      sig { params(attribute: Arel::Attributes::Attribute).returns(Arel::Nodes::NamedFunction) }
       def length(attribute)
         Arel::Nodes::NamedFunction.new("LENGTH", [attribute])
       end
 
       # @return [Arel::Nodes::As] +SUM(LENGTH(attribute)) AS <as>+
-      sig { params(attribute: Arel::Attributes::Attribute, as: T.any(Symbol, String)).returns(Arel::Nodes::As) }
       def sum_length_as(attribute, as:)
         length(attribute).sum.as(as.to_s)
       end

data/lib/activerecord/materialized/railtie.rb

@@ -1,16 +1,28 @@
-# typed: strict
 # frozen_string_literal: true
 
 module ActiveRecord
   module Materialized
     # Rails integration: wires the gem's load hooks and rake tasks into a host application.
     class Railtie < ::Rails::Railtie
-      extend T::Sig
-
       rake_tasks do
         require_relative "tasks"
         Tasks.define!
       end
+
+      # Load view classes on boot and on every dev reload so their depends_on commit callbacks are
+      # installed even under Zeitwerk's lazy loading (config.eager_load = false). Without this, a view
+      # whose constant nothing has referenced yet is dormant and writes to its dependencies don't
+      # schedule maintenance. to_prepare runs after the autoloaders are set up (once in production,
+      # on each reload in development), and ViewLoader.load! is idempotent.
+      config.to_prepare do
+        ActiveRecord::Materialized::ViewLoader.load!
+      end
+
+      # After the host's initializers have run (so config.refresh_dispatcher is final),
+      # warn if the in-process refresher is active — it is single-process-only.
+      config.after_initialize do
+        ActiveRecord::Materialized.warn_if_in_process_dispatcher!
+      end
     end
   end
 end

data/lib/activerecord/materialized/read_router.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # Routes a read to the cache table or to the cold-read fallback
+    # (populate-on-read for keyed reads), emitting the read instrumentation event
+    # for whichever path it serves.
+    #
+    # @api private
+    class ReadRouter
+      def initialize(view_class)
+        @view_class = view_class
+      end
+
+      # Routing for reads with no partition predicate to exploit.
+      def scope
+        @view_class.materialized? ? cache : cold
+      end
+
+      # Per-partition fast path for keyed reads: serve fresh partitions from the
+      # cache, otherwise read through and enqueue maintenance (populate-on-read).
+      def partition_scope(args)
+        return cache if @view_class.materialized?
+
+        keys = PartitionState.keys_from(@view_class, args)
+        return cold if keys.nil?
+        return cache if PartitionState.new(@view_class).all_fresh?(keys)
+
+        enqueue_partition_maintenance(keys)
+        cold
+      end
+
+      private
+
+      def cache
+        Instrumentation.read(@view_class, source: :cache)
+        @view_class.unscoped
+      end
+
+      def cold
+        Instrumentation.read(@view_class, source: @view_class.resolved_cold_read_strategy)
+        ColdRead.new(@view_class).scope
+      end
+
+      def enqueue_partition_maintenance(keys)
+        MaintenanceStore.new(@view_class).merge!(MaintenanceDelta.scoped(keys))
+        RefreshScheduler.schedule(@view_class)
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/reconcile_job.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # ActiveJob wrapper that reconciles a single view on a background worker — the
+    # per-view fan-out unit for the periodic bounded-staleness backstop across a
+    # fleet, mirroring {RefreshJob} for the write path.
+    class ReconcileJob < ::ActiveJob::Base
+      queue_as { ::ActiveRecord::Materialized.configuration.reconcile_queue_name }
+
+      def perform(view_key, mode: :checksum, sample: nil)
+        view_class = Registry.find(view_key)
+        return if view_class.nil?
+        # Another server in the fleet may have refreshed or reconciled it since this
+        # job was enqueued; skip the redundant (expensive) verification if so.
+        return unless view_class.stale?
+
+        view_class.reconcile!(mode: mode, sample: sample)
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/reconcile_result.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # The outcome of a {Reconciler} run: which partitions were found divergent and
+    # repaired with scoped maintenance, and whether the run was deferred because a
+    # refresh was already in flight (in which case nothing was changed — the next
+    # scheduled tick reconciles).
+    ReconcileResult = Data.define(
+      :view_name,
+      :mode,
+      # The divergent partition keys reconciliation repaired (missing, extra, or mismatched).
+      :repaired_keys,
+      # True when a concurrent refresh deferred the run; the queued repair drains later.
+      :deferred,
+      # Set when a batch run (reconcile_all!/reconcile_stale!) caught an error for this
+      # view, so one failing view doesn't abort reconciliation for the rest of the fleet.
+      :error
+    ) do
+      def initialize(view_name:, mode:, repaired_keys:, deferred: false, error: nil) = super
+
+      def repaired_partition_count
+        repaired_keys.size
+      end
+
+      # Repaired real drift this run — divergence was found and applied, not deferred.
+      def repaired?
+        repaired_keys.any? && !deferred
+      end
+
+      def failed?
+        !error.nil?
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/reconciler.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # Bounds a view's staleness in time: verifies its materialized contents against
+    # the source (via {DataVerifier}) and repairs whatever the change source missed,
+    # using SCOPED maintenance — never a full rebuild. Meant to run on a schedule as a
+    # backstop for writes any change source can miss (a bulk/raw write, a dropped or
+    # lagging CDC event).
+    #
+    # Safe alongside normal refresh. It drains pending maintenance first, so remaining
+    # drift is genuinely missed writes rather than un-applied maintenance, then repairs
+    # through the same guarded, transactional maintenance path. When a refresh is
+    # already in flight it defers to the next tick rather than verifying a cache
+    # mid-mutation or racing that cycle's payload — so it can neither corrupt the cache
+    # nor double-maintain.
+    #
+    # @api private
+    class Reconciler
+      def initialize(view_class, mode: :checksum, sample: nil)
+        @view_class = view_class
+        @mode = mode
+        @sample = sample
+      end
+
+      def reconcile!
+        # Route the whole cycle to the writer so the repair writes (merge!/mark_reconciled!/refresh!)
+        # land on the primary; DataVerifier.verify nests its own reading role, so verification still
+        # reads from the replica while repair stays on the primary.
+        ConnectionRouting.maintenance do
+          divergent = []
+          next emit(build) unless @view_class.materialized? # cold view: no cache to verify or repair
+
+          @view_class.refresh! # drain pending maintenance so any remaining drift is genuine
+          divergent = detect_drift
+          repair!(divergent) unless divergent.empty?
+          mark_reconciled!(divergent)
+          emit(build(repaired_keys: divergent))
+        rescue Refresher::AlreadyRefreshingError
+          # A live refresh owns the cycle; the scoped repair we queued (if any) drains on
+          # that cycle or the next tick. Defer without corrupting or double-maintaining.
+          emit(build(repaired_keys: divergent || [], deferred: true))
+        end
+      end
+
+      private
+
+      def detect_drift
+        DataVerifier.new(@view_class, mode: @mode, sample: @sample).verify.divergent_keys
+      end
+
+      # Queue a scoped recompute of the divergent partitions and apply it in place —
+      # re-aggregating missing/mismatched partitions and dropping extra ones uniformly.
+      def repair!(divergent)
+        MaintenanceStore.new(@view_class).merge!(MaintenanceDelta.scoped(divergent))
+        @view_class.refresh!
+      end
+
+      def mark_reconciled!(divergent)
+        Metadata::Reconciliation.mark!(@view_class.metadata, repaired_partition_count: divergent.size)
+      end
+
+      def build(repaired_keys: [], deferred: false)
+        ReconcileResult.new(
+          view_name: @view_class.view_key, mode: @mode, repaired_keys: repaired_keys, deferred: deferred
+        )
+      end
+
+      def emit(outcome)
+        Instrumentation.reconcile(
+          @view_class,
+          mode: outcome.mode, repaired_partition_count: outcome.repaired_partition_count,
+          deferred: outcome.deferred
+        )
+        outcome
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/refresh_callbacks.rb

@@ -1,57 +1,43 @@
-# typed: strict
 # frozen_string_literal: true
 
 module ActiveRecord
   module Materialized
     # Adds `before_refresh` / `after_refresh` lifecycle callbacks to a {View}.
     module RefreshCallbacks
-      extend T::Sig
-
-      sig { params(base: T.class_of(View)).void }
       def self.included(base)
         base.extend(ClassMethods)
       end
 
       # The callback-registration methods available on a {View} subclass.
       module ClassMethods
-        extend T::Sig
-
-        sig { returns(T::Hash[Symbol, T::Array[RefreshCallbackName]]) }
         def refresh_callback_store
-          @refresh_callback_store ||= T.let(
-            { before_refresh: [], after_refresh: [] },
-            T.nilable(T::Hash[Symbol, T::Array[RefreshCallbackName]])
-          )
+          @refresh_callback_store ||= { before_refresh: [], after_refresh: [] }
         end
 
-        sig { params(methods: Symbol, block: T.nilable(T.proc.void)).void }
         def before_refresh(*methods, &block)
           register_refresh_callback(:before_refresh, methods, block)
         end
 
-        sig { params(methods: Symbol, block: T.nilable(T.proc.void)).void }
         def after_refresh(*methods, &block)
           register_refresh_callback(:after_refresh, methods, block)
         end
 
-        sig { params(name: Symbol).void }
         def run_refresh_callbacks(name)
           callbacks = refresh_callback_store.fetch(name, [])
           callbacks.each do |callback|
             case callback
             when Symbol
-              T.unsafe(self).public_send(callback)
+              public_send(callback)
             when Proc
-              T.unsafe(self).instance_eval(&callback)
+              instance_eval(&callback)
             end
           end
         end
 
         private
 
-        sig { params(name: Symbol, methods: T::Array[Symbol], block: T.nilable(T.proc.void)).void }
         def register_refresh_callback(name, methods, block)
-          callbacks = T.must(refresh_callback_store[name]).dup
+          callbacks = refresh_callback_store[name].dup
           methods.each { |method| callbacks << method }
           callbacks << block if block
           refresh_callback_store[name] = callbacks

data/lib/activerecord/materialized/refresh_job.rb

@@ -1,15 +1,11 @@
-# typed: strict
 # frozen_string_literal: true
 
 module ActiveRecord
   module Materialized
     # ActiveJob wrapper that runs a view's incremental refresh on a background worker.
     class RefreshJob < ::ActiveJob::Base
-      extend T::Sig
-
       queue_as { ::ActiveRecord::Materialized.configuration.refresh_queue_name }
 
-      sig { params(view_key: String).void }
       def perform(view_key)
         view_class = Registry.find(view_key)
         return if view_class.nil?

data/lib/activerecord/materialized/refresh_result.rb

@@ -1,4 +1,3 @@
-# typed: strict
 # frozen_string_literal: true
 
 module ActiveRecord
@@ -18,20 +17,13 @@ module ActiveRecord
     #   @return [Time, nil] when the refresh completed (nil when skipped)
     # @!attribute [r] skipped
     #   @return [Boolean] true when there was nothing to do (e.g. an unmaintainable view)
-    class RefreshResult < T::Struct
-      extend T::Sig
-
-      const :view_class, T.class_of(View)
-      const :row_count, Integer
-      const :duration_ms, Integer
-      const :refreshed_at, T.nilable(Timestamp)
-      const :skipped, T::Boolean, default: false
+    RefreshResult = Data.define(:view_class, :row_count, :duration_ms, :refreshed_at, :skipped) do
+      def initialize(view_class:, row_count:, duration_ms:, refreshed_at: nil, skipped: false) = super
 
       # A no-op result, returned when refresh! was requested on a view that is
       # not maintainable.
       #
       # @return [RefreshResult]
-      sig { params(view_class: T.class_of(View)).returns(RefreshResult) }
       def self.skipped(view_class)
         new(view_class: view_class, row_count: 0, duration_ms: 0, refreshed_at: nil, skipped: true)
       end

data/lib/activerecord/materialized/refresh_scheduler.rb

@@ -1,4 +1,3 @@
-# typed: strict
 # frozen_string_literal: true
 
 module ActiveRecord
@@ -7,10 +6,11 @@ module ActiveRecord
     #
     # @api private
     class RefreshScheduler
-      class << self
-        extend T::Sig
+      # The refresh strategies a view may declare; the single source of truth shared by
+      # the +refresh_on_change+ DSL (fail-fast validation) and this dispatcher.
+      STRATEGIES = %i[async immediate manual].freeze
 
-        sig { params(view_class: ViewClass).void }
+      class << self
         def schedule(view_class)
           # Capture the transition before marking dirty so the async dispatcher
           # can coalesce: a bulk write only needs one job for the whole burst.
@@ -31,19 +31,17 @@ module ActiveRecord
 
         private
 
-        sig { params(view_class: ViewClass, newly_dirty: T::Boolean).void }
         def dispatch_async(view_class, newly_dirty)
           if use_active_job?
             # ActiveJob has no enqueue-level coalescing, so only enqueue when the
             # view first goes dirty; the job drains the accumulated payload. The
             # in-process refresher already coalesces via its debounce timer.
-            T.unsafe(RefreshJob).perform_later(view_class.view_key) if newly_dirty
+            RefreshJob.perform_later(view_class.view_key) if newly_dirty
           else
             AsyncRefresher.enqueue(view_class)
           end
         end
 
-        sig { returns(T::Boolean) }
         def use_active_job?
           config = ActiveRecord::Materialized.configuration
           !!(config.refresh_dispatcher == :active_job && defined?(ActiveJob::Base))

data/lib/activerecord/materialized/refresher.rb

@@ -1,4 +1,3 @@
-# typed: strict
 # frozen_string_literal: true
 
 module ActiveRecord
@@ -7,41 +6,59 @@ module ActiveRecord
     #
     # @api private
     class Refresher
-      extend T::Sig
-
       # Raised when a refresh or rebuild fails.
       class RefreshError < StandardError; end
 
-      sig { returns(ViewClass) }
+      # Raised when a cycle can't start because another is already running for the
+      # view. A subclass of {RefreshError} so existing rescues still catch it, but
+      # distinct so a caller (reconciliation) can treat an overlap as a benign,
+      # retryable defer — and so it is re-raised without {#fail_refresh!}, which would
+      # otherwise mark the view failed and clear the *live* cycle's `refreshing` guard.
+      class AlreadyRefreshingError < RefreshError; end
+
       attr_reader :view_class
 
-      sig { params(view_class: ViewClass).void }
       def initialize(view_class)
         @view_class = view_class
-        @metadata = T.let(nil, T.nilable(Metadata))
+        @metadata = nil
       end
 
       # Full materialization — the only path that scans all base data.
-      sig { returns(RefreshResult) }
       def rebuild!
-        run_cycle(-> { perform_rebuild! })
-      rescue StandardError => e
-        fail_refresh!(e)
+        guarded_maintenance do
+          Instrumentation.refresh(view_class, operation: :rebuild) do |payload|
+            payload[:mode] = :full
+            run_cycle(-> { perform_rebuild! })
+          end
+        end
       end
 
       # Incremental maintenance only; a no-op when the view is not maintainable.
-      sig { returns(RefreshResult) }
       def refresh!
-        return RefreshResult.skipped(view_class) unless maintainable?
+        guarded_maintenance do
+          Instrumentation.refresh(view_class, operation: :incremental) do |payload|
+            next RefreshResult.skipped(view_class) unless maintainable?
 
-        run_cycle(-> { incremental_refresh! })
-      rescue StandardError => e
-        fail_refresh!(e)
+            run_cycle(-> { IncrementalRefresh.new(view_class, payload).call })
+          end
+        end
       end
 
       private
 
-      sig { returns(T::Boolean) }
+      # Runs a maintenance cycle under the configured writer role, translating an overlap into a
+      # benign re-raise (a live cycle owns the guard) and any other failure into fail_refresh!. The
+      # rescue lives inside the routed connection so fail_refresh!'s write also lands on the writer.
+      def guarded_maintenance
+        ConnectionRouting.maintenance do
+          yield
+        rescue AlreadyRefreshingError
+          raise # a live cycle owns the guard; don't fail_refresh! it (that would clear the guard)
+        rescue StandardError => e
+          fail_refresh!(e)
+        end
+      end
+
       def maintainable?
         return false unless view_class.incrementally_maintainable?
 
@@ -55,9 +72,8 @@ module ActiveRecord
         true
       end
 
-      sig { params(operation: T.proc.returns(Integer)).returns(RefreshResult) }
       def run_cycle(operation)
-        raise RefreshError, "#{view_class.name} is already refreshing" if metadata.refreshing?
+        raise AlreadyRefreshingError, "#{view_class.name} is already refreshing" if metadata.refreshing?
 
         started_at = monotonic_clock
         metadata.mark_refreshing!
@@ -69,7 +85,6 @@ module ActiveRecord
         result
       end
 
-      sig { returns(Integer) }
       def perform_rebuild!
         row_count = RelationCacheWriter.new(view_class).atomic_swap!(view_class.resolved_source)
         metadata.mark_warm!
@@ -78,53 +93,23 @@ module ActiveRecord
         row_count
       end
 
-      sig { returns(Integer) }
-      def incremental_refresh!
-        ensure_cache_table!
-
-        store = MaintenanceStore.new(view_class)
-        pending = store.pending
-        return apply_summary_delta!(store, pending) if pending.is_a?(SummaryDelta)
-
-        IncrementalMaintainer.new(view_class).maintain!(view_class.connection, view_class.table_name)
-      end
-
-      # Cheap DDL so partition maintenance has somewhere to write — never a populate.
-      sig { void }
-      def ensure_cache_table!
-        return if view_class.table_exists?
-
-        CacheTableSchema.ensure_table!(view_class, view_class.resolved_source)
-      end
-
-      sig { params(store: MaintenanceStore, summary: SummaryDelta).returns(Integer) }
-      def apply_summary_delta!(store, summary)
-        store.clear!
-        DeltaMaintainer.new(view_class).apply!(summary)
-      end
-
-      sig { params(error: StandardError).returns(T.noreturn) }
       def fail_refresh!(error)
         metadata.mark_failed!(error)
         raise RefreshError, "Failed to refresh #{view_class.name}: #{error.message}", error.backtrace
       end
 
-      sig { returns(Metadata) }
       def metadata
         @metadata ||= view_class.metadata
       end
 
-      sig { returns(Float) }
       def monotonic_clock
-        T.cast(Process.clock_gettime(Process::CLOCK_MONOTONIC), Float)
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
       end
 
-      sig { params(started_at: Float).returns(Integer) }
       def elapsed_milliseconds(started_at)
         ((monotonic_clock - started_at) * 1000).round
       end
 
-      sig { params(row_count: Integer, duration_ms: Integer).returns(RefreshResult) }
       def complete_refresh!(row_count:, duration_ms:)
         metadata.mark_refreshed!(row_count: row_count, duration_ms: duration_ms)
         RefreshResult.new(