activerecord-materialized 0.1.1 → 0.2.0

data/lib/activerecord/materialized/tasks.rb

@@ -1,29 +1,24 @@
-# typed: strict
 # frozen_string_literal: true
 
 module ActiveRecord
   module Materialized
     # Defines the `materialized:*` rake tasks (refresh_all, refresh_stale, rebuild, verify, warm_up).
     module Tasks
-      extend T::Sig
+      DEFINITIONS = {
+        refresh_all: "Refresh all registered materialized views",
+        refresh_stale: "Refresh stale materialized views",
+        enqueue_refreshes: "Enqueue a background RefreshJob per stale view (fleet fan-out; run from one owner)",
+        rebuild: "Rebuild (fully materialize) all registered materialized views",
+        verify: "Verify materialized view cache tables match their source relations",
+        audit: "Verify materialized view contents against their source relations (data drift)",
+        reconcile: "Reconcile stale materialized views: verify contents and repair drift (scoped)",
+        enqueue_reconciles: "Enqueue a background ReconcileJob per stale view (fleet fan-out; run from one owner)",
+        warm_up: "Materialize each view's configured warm_up partitions"
+      }.freeze
 
-      DEFINITIONS = T.let(
-        {
-          refresh_all: "Refresh all registered materialized views",
-          refresh_stale: "Refresh stale materialized views",
-          rebuild: "Rebuild (fully materialize) all registered materialized views",
-          verify: "Verify materialized view cache tables match their source relations",
-          warm_up: "Materialize each view's configured warm_up partitions"
-        }.freeze,
-        T::Hash[Symbol, String]
-      )
-
-      sig { void }
       def self.define!
-        application = T.let(T.unsafe(::Rake.application), T.untyped)
+        application = ::Rake.application
         application.instance_eval do
-          T.bind(self, T.untyped)
-
           namespace :materialized do
             DEFINITIONS.each do |task_name, description|
               desc description
@@ -33,46 +28,59 @@ module ActiveRecord
         end
       end
 
-      sig { params(task_name: Symbol).void }
+      # Each task name maps by convention to its `run_<task_name>!` module method.
       def self.run!(task_name)
-        case task_name
-        when :refresh_all then run_refresh_all!
-        when :refresh_stale then run_refresh_stale!
-        when :rebuild then run_rebuild_all!
-        when :verify then run_verify!
-        when :warm_up then run_warm_up_all!
-        end
+        public_send(:"run_#{task_name}!")
       end
 
-      sig { void }
       def self.run_refresh_all!
         Registry.refresh_all!
-        T.unsafe(Rails).logger.debug { "Refreshed #{Registry.all.size} materialized view(s)." }
+        Rails.logger.debug { "Refreshed #{Registry.all.size} materialized view(s)." }
       end
 
-      sig { void }
       def self.run_refresh_stale!
-        stale = Registry.all.select(&:stale?)
+        stale = Registry.stale_views
         stale.each(&:refresh!)
-        T.unsafe(Rails).logger.debug { "Refreshed #{stale.size} stale materialized view(s)." }
+        Rails.logger.debug { "Refreshed #{stale.size} stale materialized view(s)." }
       end
 
-      sig { void }
-      def self.run_rebuild_all!
+      def self.run_rebuild!
         Registry.rebuild_all!
-        T.unsafe(Rails).logger.debug { "Rebuilt #{Registry.all.size} materialized view(s)." }
+        Rails.logger.debug { "Rebuilt #{Registry.all.size} materialized view(s)." }
       end
 
-      sig { void }
       def self.run_verify!
         ActiveRecord::Materialized.verify_schema!
-        T.unsafe(Rails).logger.debug { "Verified #{Registry.all.size} materialized view schema(s)." }
+        Rails.logger.debug { "Verified #{Registry.all.size} materialized view schema(s)." }
+      end
+
+      def self.run_audit!
+        ActiveRecord::Materialized.verify_data!
+        Rails.logger.debug { "Audited data for #{Registry.all.size} materialized view(s)." }
+      end
+
+      def self.run_reconcile!
+        results = ActiveRecord::Materialized.reconcile_stale!
+        repaired = results.sum(&:repaired_partition_count)
+        failed = results.count(&:failed?)
+        Rails.logger.debug do
+          "Reconciled #{results.size} stale view(s); repaired #{repaired} partition(s); #{failed} failed."
+        end
+      end
+
+      def self.run_enqueue_refreshes!
+        keys = ActiveRecord::Materialized.enqueue_stale_refreshes!
+        Rails.logger.debug { "Enqueued #{keys.size} refresh job(s) for stale view(s)." }
+      end
+
+      def self.run_enqueue_reconciles!
+        keys = ActiveRecord::Materialized.enqueue_stale_reconciles!
+        Rails.logger.debug { "Enqueued #{keys.size} reconcile job(s) for stale view(s)." }
       end
 
-      sig { void }
-      def self.run_warm_up_all!
+      def self.run_warm_up!
         Registry.warm_up_all!
-        T.unsafe(Rails).logger.debug { "Warmed up #{Registry.all.size} materialized view(s)." }
+        Rails.logger.debug { "Warmed up #{Registry.all.size} materialized view(s)." }
       end
     end
   end

data/lib/activerecord/materialized/version.rb

@@ -1,9 +1,8 @@
-# typed: strict
 # frozen_string_literal: true
 
 module ActiveRecord
   module Materialized
     # The gem version.
-    VERSION = "0.1.1"
+    VERSION = "0.2.0"
   end
 end

data/lib/activerecord/materialized/view.rb

@@ -1,4 +1,3 @@
-# typed: strict
 # frozen_string_literal: true
 
 module ActiveRecord
@@ -36,7 +35,6 @@ module ActiveRecord
     # @see ViewIncrementalClassMethods::ClassMethods incremental-maintenance configuration
     # @see ViewQueryAccessClassMethods::ClassMethods +rebuild!+, +refresh!+, +materialized?+, …
     class View < ::ActiveRecord::Base
-      extend T::Sig
       include RefreshCallbacks
       include ViewConfigurationClassMethods
       include ViewQueryAccessClassMethods
@@ -44,34 +42,25 @@ module ActiveRecord
       self.abstract_class = true
 
       class << self
-        extend T::Sig
+        @source_definition = nil
+        @max_staleness_setting = nil
+        @dependency_tables = nil
+        @refresh_strategy = nil
+        @refresh_debounce = nil
+        @refresh_mode = nil
+        @incremental_source_definition = nil
+        @incremental_key_columns = nil
+        @table_name = nil
 
-        @source_definition = T.let(nil, T.nilable(SourceDefinition))
-        @max_staleness_setting = T.let(nil, T.nilable(T.any(StalenessDuration, Proc)))
-        @dependency_tables = T.let(nil, T.nilable(T::Array[String]))
-        @refresh_strategy = T.let(nil, T.nilable(Symbol))
-        @refresh_debounce = T.let(nil, T.nilable(DebounceInterval))
-        @refresh_mode = T.let(nil, T.nilable(RefreshMode))
-        @incremental_source_definition = T.let(nil, T.nilable(SourceDefinition))
-        @incremental_key_columns = T.let(nil, T.nilable(T::Array[String]))
-        @table_name = T.let(nil, T.nilable(String))
+        attr_reader :source_definition, :max_staleness_setting
 
-        sig { returns(T.nilable(SourceDefinition)) }
-        attr_reader :source_definition
-
-        sig { returns(T.nilable(T.any(StalenessDuration, Proc))) }
-        attr_reader :max_staleness_setting
-
-        sig { returns(T::Array[String]) }
         def dependency_tables
-          tables = T.let(T.unsafe(self).instance_variable_get(:@dependency_tables), T.nilable(T::Array[String]))
+          tables = instance_variable_get(:@dependency_tables)
           tables.nil? ? [] : tables
         end
       end
 
-      sig { returns(T::Boolean) }
       def stale?
-        T.bind(self, View)
         self.class.stale?
       end
     end

data/lib/activerecord/materialized/view_configuration_class_methods.rb

@@ -1,62 +1,61 @@
-# typed: strict
 # frozen_string_literal: true
 
 module ActiveRecord
   module Materialized
     # The `materialized_from` / `depends_on` DSL and source/metadata accessors mixed into every {View}.
     module ViewConfigurationClassMethods
-      extend T::Sig
-      extend T::Helpers
-
-      sig { params(base: T.class_of(View)).void }
       def self.included(base)
         base.extend(ClassMethods)
       end
 
       # The configuration DSL methods available on a {View} subclass.
       module ClassMethods
-        extend T::Sig
         include ViewIncrementalClassMethods::ClassMethods
         include ViewRefreshPolicyClassMethods::ClassMethods
 
-        sig { returns(T.class_of(View)) }
+        # Per-subclass DSL ivars reset to nil on inheritance so a subclass never
+        # inherits another view's configuration. (`@dependency_tables` resets to
+        # an empty list instead — see {inherited}.)
+        NIL_RESET_IVARS = %i[
+          @refresh_strategy @refresh_debounce @refresh_mode @incremental_source_definition
+          @incremental_key_columns @partition_key_resolvers @cold_read_strategy @change_source
+          @warm_up_definition
+        ].freeze
+
         def view_class
-          T.cast(self, T.class_of(View))
+          self
         end
 
-        sig { params(subclass: T.class_of(View)).void }
         def inherited(subclass)
           super
-          T.unsafe(subclass).instance_variable_set(:@dependency_tables, [])
-          T.unsafe(subclass).instance_variable_set(:@refresh_strategy, nil)
-          T.unsafe(subclass).instance_variable_set(:@refresh_debounce, nil)
-          T.unsafe(subclass).instance_variable_set(:@refresh_mode, nil)
-          T.unsafe(subclass).instance_variable_set(:@incremental_source_definition, nil)
-          T.unsafe(subclass).instance_variable_set(:@incremental_key_columns, nil)
-          T.unsafe(subclass).instance_variable_set(:@cold_read_strategy, nil)
-          T.unsafe(subclass).instance_variable_set(:@warm_up_definition, nil)
+          subclass.instance_variable_set(:@dependency_tables, [])
+          NIL_RESET_IVARS.each { |ivar| subclass.instance_variable_set(ivar, nil) }
         end
 
-        sig { returns(String) }
         def view_key
-          return T.must(view_class.name).underscore if view_class.name.present?
+          return view_class.name.underscore if view_class.name.present?
 
-          table = T.let(T.unsafe(view_class).instance_variable_get(:@table_name), T.nilable(String))
+          table = view_class.instance_variable_get(:@table_name)
           table.presence || "anonymous_view_#{view_class.object_id}"
         end
 
-        sig { params(block: T.proc.returns(::ActiveRecord::Relation)).void }
+        # Declares the source relation this view materializes and registers the view unless it is abstract.
+        #
+        # @yieldreturn [ActiveRecord::Relation] the query whose result set is cached in the view's table
+        # @return [void]
         def materialized_from(&block)
-          @source_definition = T.let(block, T.nilable(SourceDefinition))
+          @source_definition = block
           Registry.register(view_class) unless view_class.abstract_class?
         end
 
-        sig { params(tables: T.any(Symbol, String, T.class_of(::ActiveRecord::Base))).void }
+        # Declares the dependency tables whose committed writes trigger maintenance of this view.
+        #
+        # @param tables [Array<Symbol, String, Class<ActiveRecord::Base>>] dependency tables this view reads from
+        # @return [void]
         def depends_on(*tables)
           DependencyRegistry.register(view_class, tables)
         end
 
-        sig { returns(::ActiveRecord::Relation) }
         def resolved_source
           resolve_source_definition(
             @source_definition,
@@ -64,20 +63,12 @@ module ActiveRecord
           )
         end
 
-        sig { returns(Metadata) }
         def metadata
-          @metadata = T.let(@metadata, T.nilable(ActiveRecord::Materialized::Metadata))
           @metadata ||= ActiveRecord::Materialized::Metadata.new(view_class)
         end
 
         private
 
-        sig do
-          params(
-            definition: T.nilable(SourceDefinition),
-            empty_message: String
-          ).returns(::ActiveRecord::Relation)
-        end
         def resolve_source_definition(definition, empty_message)
           source = coerce_source(definition)
           Kernel.raise ArgumentError, empty_message if source.nil?
@@ -88,7 +79,6 @@ module ActiveRecord
           source
         end
 
-        sig { params(definition: T.nilable(SourceDefinition)).returns(T.untyped) }
         def coerce_source(definition)
           source = definition
           return source unless source.is_a?(Proc)
@@ -96,8 +86,6 @@ module ActiveRecord
           source.call
         end
       end
-
-      mixes_in_class_methods ClassMethods
     end
   end
 end

data/lib/activerecord/materialized/view_definition.rb

@@ -1,131 +1,142 @@
-# typed: strict
 # frozen_string_literal: true
 
 module ActiveRecord
   module Materialized
-    # Inspects a source relation for its GROUP BY maintenance keys and builds partition scopes.
+    # Inspects a source relation for its partition (maintenance) keys and builds partition scopes.
+    # The keys are the relation's GROUP BY columns, or — for a `SELECT DISTINCT a, b` with no GROUP BY
+    # and no aggregate — its projected columns, since a distinct lookup partitions exactly like a
+    # GROUP BY of those columns.
     #
     # @api private
     class ViewDefinition
-      extend T::Sig
-
-      sig do
-        params(
-          source: ::ActiveRecord::Relation,
-          explicit_group_keys: T.nilable(T::Array[String])
-        ).void
-      end
       def initialize(source, explicit_group_keys: nil)
         @source = source
         @explicit_group_keys = explicit_group_keys
       end
 
-      sig { returns(T::Boolean) }
       def incrementally_maintainable?
         group_key_columns.any?
       end
 
-      sig { returns(T::Array[String]) }
       def group_key_columns
-        @group_key_columns = T.let(@group_key_columns, T.nilable(T::Array[String]))
         @group_key_columns ||= resolve_group_key_columns
       end
 
       # Restrict a model/cache table to the given partitions. The partition
       # columns are real columns on `model`, so qualify them to its own table.
-      sig do
-        params(
-          model: T.class_of(::ActiveRecord::Base),
-          key_tuples: T::Array[T::Array[T.untyped]]
-        ).returns(::ActiveRecord::Relation)
-      end
       def partition_scope_on(model, key_tuples)
         validate_partition_keys!(key_tuples)
-        attributes = group_key_columns.map { |column| T.unsafe(model).arel_table[column] }
-        filter_partitions(model.unscoped, attributes, key_tuples)
+        # On the cache/model table the key is the projected column, so a qualified
+        # GROUP BY name (e.g. "authors.country") maps to the bare column ("country").
+        attributes = group_key_columns.map { |column| model.arel_table[unqualified(column)] }
+        PartitionFilter.new(attributes, key_tuples).apply(model.unscoped)
       end
 
       # Restrict the source relation to the given partitions. Qualify each key to
       # its GROUP BY attribute's own table, which may be a joined table (e.g.
       # `name.gender`) rather than the source's base table.
-      sig { params(key_tuples: T::Array[T::Array[T.untyped]]).returns(::ActiveRecord::Relation) }
       def partition_scope(key_tuples)
         validate_partition_keys!(key_tuples)
-        base = T.unsafe(source).klass.arel_table
-        attributes = group_key_columns.map { |column| group_attributes[column] || base[column] }
-        filter_partitions(source, attributes, key_tuples)
+        base = source.klass.arel_table
+        attributes = group_key_columns.map { |column| source_attribute(column, base) }
+        PartitionFilter.new(attributes, key_tuples).apply(source)
       end
 
       private
 
-      sig { returns(::ActiveRecord::Relation) }
+      # The Arel attribute for a GROUP BY key on the source side: the captured Arel
+      # attribute if the key was given as one; otherwise a qualified string like
+      # "authors.country" builds that table's attribute, and a bare name uses the
+      # source's base table.
+      def source_attribute(column, base)
+        return group_attributes[column] if group_attributes.key?(column)
+
+        table, separator, name = column.rpartition(".")
+        separator.empty? ? base[column] : ::Arel::Table.new(table)[name]
+      end
+
+      # The bare column name, dropping any "table." qualifier.
+      def unqualified(column)
+        column.rpartition(".").last
+      end
+
       attr_reader :source
 
-      sig { returns(T::Array[String]) }
+      # A bare or single-dot-qualified SQL identifier — "category" or
+      # "orders.region". Used to tell a plain key column in a DISTINCT projection
+      # from an expression (parens, star, spaces, operators), so a raw SQL string
+      # in the SELECT list is never mistaken for a partition key.
+      PLAIN_COLUMN_PATTERN = /\A\w+(?:\.\w+)?\z/
+      private_constant :PLAIN_COLUMN_PATTERN
+
       def resolve_group_key_columns
         return @explicit_group_keys if @explicit_group_keys&.any?
 
         relation_group_columns
       end
 
-      sig { returns(T::Array[String]) }
       def relation_group_columns
-        source.group_values.filter_map { |group_value| group_column_name(group_value) }
+        key_value_nodes.filter_map { |node| group_column_name(node) }
+      end
+
+      # The nodes that identify a partition: the GROUP BY values, or — for a
+      # DISTINCT projection of plain key columns with no GROUP BY — the projected
+      # columns themselves. `SELECT DISTINCT a, b` partitions identically to
+      # `GROUP BY a, b` (no aggregates), so a distinct lookup is maintained
+      # incrementally through the same scoped-recompute path. Anything else (an
+      # aggregate in the projection, `DISTINCT *`, no distinct) yields no keys and
+      # falls back to full refresh.
+      def key_value_nodes
+        return source.group_values if source.group_values.any?
+
+        distinct_key_projection? ? source.select_values : []
+      end
+
+      def distinct_key_projection?
+        return false unless source.distinct_value && source.group_values.empty?
+
+        nodes = source.select_values
+        nodes.any? && nodes.all? { |node| plain_key_column?(node) }
+      end
+
+      # Whether a projected node names a plain key column (safe as a DISTINCT
+      # partition key) rather than an aggregate or expression. A Symbol or Arel
+      # attribute is always a plain column; a String only when it is a bare or
+      # table-qualified identifier (never "COUNT(*)", "a + b", etc.).
+      def plain_key_column?(node)
+        case node
+        when Symbol, ::Arel::Attributes::Attribute
+          true
+        when String
+          node.match?(PLAIN_COLUMN_PATTERN)
+        else
+          false
+        end
       end
 
-      sig { params(group_value: T.untyped).returns(T.nilable(String)) }
       def group_column_name(group_value)
         case group_value
         when String, Symbol
           group_value.to_s
         when ::Arel::Attributes::Attribute
-          T.unsafe(group_value).name.to_s
+          group_value.name.to_s
         else
           group_value.to_s if group_value.respond_to?(:to_s)
         end
       end
 
-      sig { params(key_tuples: T::Array[T::Array[T.untyped]]).void }
       def validate_partition_keys!(key_tuples)
         raise ArgumentError, "scoped maintenance requires GROUP BY keys" unless incrementally_maintainable?
         raise ArgumentError, "scoped maintenance requires at least one partition key" if key_tuples.empty?
       end
 
-      # GROUP BY attributes keyed by name; an Arel attribute carries its real
+      # Partition-key attributes keyed by name; an Arel attribute carries its real
       # table (e.g. a joined `name.gender`) that the bare base table column lacks.
-      sig { returns(T::Hash[String, T.untyped]) }
+      # Reads the same nodes as the key columns, so a DISTINCT projection's joined
+      # key attribute (e.g. `orders.region`) qualifies to its own table too.
       def group_attributes
-        @group_attributes = T.let(@group_attributes, T.nilable(T::Hash[String, T.untyped]))
-        @group_attributes ||= T.unsafe(source).group_values.each_with_object({}) do |value, map|
-          map[T.unsafe(value).name.to_s] = value if value.is_a?(::Arel::Attributes::Attribute)
-        end
-      end
-
-      sig do
-        params(
-          scope: ::ActiveRecord::Relation,
-          attributes: T::Array[T.untyped],
-          key_tuples: T::Array[T::Array[T.untyped]]
-        ).returns(::ActiveRecord::Relation)
-      end
-      def filter_partitions(scope, attributes, key_tuples)
-        return multi_partition_filter(scope, attributes, key_tuples) if attributes.size > 1
-
-        scope.where(T.unsafe(attributes.fetch(0)).in(key_tuples.map(&:first)))
-      end
-
-      sig do
-        params(
-          scope: ::ActiveRecord::Relation, attributes: T::Array[T.untyped],
-          key_tuples: T::Array[T::Array[T.untyped]]
-        ).returns(::ActiveRecord::Relation)
-      end
-      def multi_partition_filter(scope, attributes, key_tuples)
-        key_tuples.reduce(T.unsafe(nil)) do |merged_scope, tuple|
-          branch = attributes.each_with_index.reduce(scope) do |relation, (attribute, index)|
-            relation.where(T.unsafe(attribute).eq(tuple[index]))
-          end
-          merged_scope.nil? ? branch : merged_scope.or(branch)
+        @group_attributes ||= key_value_nodes.each_with_object({}) do |value, map|
+          map[value.name.to_s] = value if value.is_a?(::Arel::Attributes::Attribute)
         end
       end
     end