activerecord-materialized 0.1.0

data/lib/activerecord/materialized/view_configuration_class_methods.rb

@@ -0,0 +1,103 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # The `materialized_from` / `depends_on` DSL and source/metadata accessors mixed into every {View}.
+    module ViewConfigurationClassMethods
+      extend T::Sig
+      extend T::Helpers
+
+      sig { params(base: T.class_of(View)).void }
+      def self.included(base)
+        base.extend(ClassMethods)
+      end
+
+      # The configuration DSL methods available on a {View} subclass.
+      module ClassMethods
+        extend T::Sig
+        include ViewIncrementalClassMethods::ClassMethods
+        include ViewRefreshPolicyClassMethods::ClassMethods
+
+        sig { returns(T.class_of(View)) }
+        def view_class
+          T.cast(self, T.class_of(View))
+        end
+
+        sig { params(subclass: T.class_of(View)).void }
+        def inherited(subclass)
+          super
+          T.unsafe(subclass).instance_variable_set(:@dependency_tables, [])
+          T.unsafe(subclass).instance_variable_set(:@refresh_strategy, nil)
+          T.unsafe(subclass).instance_variable_set(:@refresh_debounce, nil)
+          T.unsafe(subclass).instance_variable_set(:@refresh_mode, nil)
+          T.unsafe(subclass).instance_variable_set(:@incremental_source_definition, nil)
+          T.unsafe(subclass).instance_variable_set(:@incremental_key_columns, nil)
+          T.unsafe(subclass).instance_variable_set(:@cold_read_strategy, nil)
+          T.unsafe(subclass).instance_variable_set(:@warm_up_definition, nil)
+        end
+
+        sig { returns(String) }
+        def view_key
+          return T.must(view_class.name).underscore if view_class.name.present?
+
+          table = T.let(T.unsafe(view_class).instance_variable_get(:@table_name), T.nilable(String))
+          table.presence || "anonymous_view_#{view_class.object_id}"
+        end
+
+        sig { params(block: T.proc.returns(::ActiveRecord::Relation)).void }
+        def materialized_from(&block)
+          @source_definition = T.let(block, T.nilable(SourceDefinition))
+          Registry.register(view_class) unless view_class.abstract_class?
+        end
+
+        sig { params(tables: T.any(Symbol, String, T.class_of(::ActiveRecord::Base))).void }
+        def depends_on(*tables)
+          DependencyRegistry.register(view_class, tables)
+        end
+
+        sig { returns(::ActiveRecord::Relation) }
+        def resolved_source
+          resolve_source_definition(
+            @source_definition,
+            "materialized_from is required for #{view_class.name || view_class.view_key}"
+          )
+        end
+
+        sig { returns(Metadata) }
+        def metadata
+          @metadata = T.let(@metadata, T.nilable(ActiveRecord::Materialized::Metadata))
+          @metadata ||= ActiveRecord::Materialized::Metadata.new(view_class)
+        end
+
+        private
+
+        sig do
+          params(
+            definition: T.nilable(SourceDefinition),
+            empty_message: String
+          ).returns(::ActiveRecord::Relation)
+        end
+        def resolve_source_definition(definition, empty_message)
+          source = coerce_source(definition)
+          Kernel.raise ArgumentError, empty_message if source.nil?
+          unless source.is_a?(::ActiveRecord::Relation)
+            Kernel.raise ArgumentError, "#{empty_message}: expected ActiveRecord::Relation, got #{source.class}"
+          end
+
+          source
+        end
+
+        sig { params(definition: T.nilable(SourceDefinition)).returns(T.untyped) }
+        def coerce_source(definition)
+          source = definition
+          return source unless source.is_a?(Proc)
+
+          source.call
+        end
+      end
+
+      mixes_in_class_methods ClassMethods
+    end
+  end
+end

data/lib/activerecord/materialized/view_definition.rb

@@ -0,0 +1,133 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # Inspects a source relation for its GROUP BY maintenance keys and builds partition scopes.
+    #
+    # @api private
+    class ViewDefinition
+      extend T::Sig
+
+      sig do
+        params(
+          source: ::ActiveRecord::Relation,
+          explicit_group_keys: T.nilable(T::Array[String])
+        ).void
+      end
+      def initialize(source, explicit_group_keys: nil)
+        @source = source
+        @explicit_group_keys = explicit_group_keys
+      end
+
+      sig { returns(T::Boolean) }
+      def incrementally_maintainable?
+        group_key_columns.any?
+      end
+
+      sig { returns(T::Array[String]) }
+      def group_key_columns
+        @group_key_columns = T.let(@group_key_columns, T.nilable(T::Array[String]))
+        @group_key_columns ||= resolve_group_key_columns
+      end
+
+      # Restrict a model/cache table to the given partitions. The partition
+      # columns are real columns on `model`, so qualify them to its own table.
+      sig do
+        params(
+          model: T.class_of(::ActiveRecord::Base),
+          key_tuples: T::Array[T::Array[T.untyped]]
+        ).returns(::ActiveRecord::Relation)
+      end
+      def partition_scope_on(model, key_tuples)
+        validate_partition_keys!(key_tuples)
+        attributes = group_key_columns.map { |column| T.unsafe(model).arel_table[column] }
+        filter_partitions(model.unscoped, attributes, key_tuples)
+      end
+
+      # Restrict the source relation to the given partitions. Qualify each key to
+      # its GROUP BY attribute's own table, which may be a joined table (e.g.
+      # `name.gender`) rather than the source's base table.
+      sig { params(key_tuples: T::Array[T::Array[T.untyped]]).returns(::ActiveRecord::Relation) }
+      def partition_scope(key_tuples)
+        validate_partition_keys!(key_tuples)
+        base = T.unsafe(source).klass.arel_table
+        attributes = group_key_columns.map { |column| group_attributes[column] || base[column] }
+        filter_partitions(source, attributes, key_tuples)
+      end
+
+      private
+
+      sig { returns(::ActiveRecord::Relation) }
+      attr_reader :source
+
+      sig { returns(T::Array[String]) }
+      def resolve_group_key_columns
+        return @explicit_group_keys if @explicit_group_keys&.any?
+
+        relation_group_columns
+      end
+
+      sig { returns(T::Array[String]) }
+      def relation_group_columns
+        source.group_values.filter_map { |group_value| group_column_name(group_value) }
+      end
+
+      sig { params(group_value: T.untyped).returns(T.nilable(String)) }
+      def group_column_name(group_value)
+        case group_value
+        when String, Symbol
+          group_value.to_s
+        when ::Arel::Attributes::Attribute
+          T.unsafe(group_value).name.to_s
+        else
+          group_value.to_s if group_value.respond_to?(:to_s)
+        end
+      end
+
+      sig { params(key_tuples: T::Array[T::Array[T.untyped]]).void }
+      def validate_partition_keys!(key_tuples)
+        raise ArgumentError, "scoped maintenance requires GROUP BY keys" unless incrementally_maintainable?
+        raise ArgumentError, "scoped maintenance requires at least one partition key" if key_tuples.empty?
+      end
+
+      # GROUP BY attributes keyed by name; an Arel attribute carries its real
+      # table (e.g. a joined `name.gender`) that the bare base table column lacks.
+      sig { returns(T::Hash[String, T.untyped]) }
+      def group_attributes
+        @group_attributes = T.let(@group_attributes, T.nilable(T::Hash[String, T.untyped]))
+        @group_attributes ||= T.unsafe(source).group_values.each_with_object({}) do |value, map|
+          map[T.unsafe(value).name.to_s] = value if value.is_a?(::Arel::Attributes::Attribute)
+        end
+      end
+
+      sig do
+        params(
+          scope: ::ActiveRecord::Relation,
+          attributes: T::Array[T.untyped],
+          key_tuples: T::Array[T::Array[T.untyped]]
+        ).returns(::ActiveRecord::Relation)
+      end
+      def filter_partitions(scope, attributes, key_tuples)
+        return multi_partition_filter(scope, attributes, key_tuples) if attributes.size > 1
+
+        scope.where(T.unsafe(attributes.fetch(0)).in(key_tuples.map(&:first)))
+      end
+
+      sig do
+        params(
+          scope: ::ActiveRecord::Relation, attributes: T::Array[T.untyped],
+          key_tuples: T::Array[T::Array[T.untyped]]
+        ).returns(::ActiveRecord::Relation)
+      end
+      def multi_partition_filter(scope, attributes, key_tuples)
+        key_tuples.reduce(T.unsafe(nil)) do |merged_scope, tuple|
+          branch = attributes.each_with_index.reduce(scope) do |relation, (attribute, index)|
+            relation.where(T.unsafe(attribute).eq(tuple[index]))
+          end
+          merged_scope.nil? ? branch : merged_scope.or(branch)
+        end
+      end
+    end
+  end
+end

data/lib/activerecord/materialized/view_incremental_class_methods.rb

@@ -0,0 +1,142 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # Incremental-maintenance DSL mixed into a {View}: `refresh_mode`, `incremental_keys`,
+    # `incremental_from`, and the per-write maintenance entry points.
+    module ViewIncrementalClassMethods
+      extend T::Sig
+      extend T::Helpers
+
+      sig { params(base: T.class_of(View)).void }
+      def self.included(base)
+        base.extend(ClassMethods)
+      end
+
+      # The incremental-maintenance DSL methods available on a {View} subclass.
+      module ClassMethods
+        extend T::Sig
+
+        sig { returns(T.class_of(View)) }
+        def view_class
+          T.cast(self, T.class_of(View))
+        end
+
+        sig { params(mode: RefreshMode).void }
+        def refresh_mode(mode)
+          T.unsafe(self).instance_variable_set(:@refresh_mode, mode.to_sym)
+        end
+
+        sig { params(block: T.proc.returns(::ActiveRecord::Relation)).void }
+        def incremental_from(&block)
+          @incremental_source_definition = T.let(block, T.nilable(SourceDefinition))
+        end
+
+        sig { params(columns: T.any(Symbol, String)).void }
+        def incremental_keys(*columns)
+          @incremental_key_columns = T.let(columns.map(&:to_s), T.nilable(T::Array[String]))
+        end
+
+        sig { returns(RefreshMode) }
+        def resolved_refresh_mode
+          mode = T.let(
+            T.unsafe(self).instance_variable_get(:@refresh_mode),
+            T.nilable(RefreshMode)
+          )
+          mode || :incremental
+        end
+
+        sig { returns(ViewDefinition) }
+        def view_definition
+          ViewDefinition.new(
+            view_class.resolved_source,
+            explicit_group_keys: incremental_key_columns.presence
+          )
+        end
+
+        sig { returns(T::Array[String]) }
+        def maintenance_key_columns
+          return incremental_key_columns if incremental_key_columns.any?
+
+          view_definition.group_key_columns
+        end
+
+        sig { returns(T::Boolean) }
+        def incrementally_maintainable?
+          resolved_refresh_mode != :full && view_definition.incrementally_maintainable?
+        end
+
+        sig { returns(AggregateAnalysis) }
+        def aggregate_analysis
+          AggregateAnalysis.new(view_class.resolved_source)
+        end
+
+        # A warm view with all-distributive aggregates uses summary-delta IVM;
+        # otherwise writes drive scoped recompute.
+        sig { returns(T::Boolean) }
+        def delta_maintaining?
+          resolved_refresh_mode != :full && view_class.materialized? && aggregate_analysis.delta_maintainable?
+        end
+
+        sig { returns(T::Boolean) }
+        def incremental_source_override?
+          !@incremental_source_definition.nil?
+        end
+
+        sig { params(change: WriteChange).void }
+        def record_write_change!(change)
+          return record_summary_delta!(change) if delta_maintaining?
+          return unless incrementally_maintainable?
+
+          delta = MaintenanceDeltaBuilder.new(change, maintenance_key_columns).build
+          record_write_delta!(delta)
+
+          # On a cold view the written partitions are no longer current; drop them
+          # from the fresh set until re-maintained.
+          return if view_class.materialized? || delta.full_partition?
+
+          PartitionState.new(view_class).mark_stale!(delta.key_tuples)
+        end
+
+        sig { params(change: WriteChange).void }
+        def record_summary_delta!(change)
+          summary = SummaryDeltaBuilder.new(change, aggregate_analysis, maintenance_key_columns).build
+          MaintenanceStore.new(view_class).merge!(summary) unless summary.empty?
+        end
+
+        sig { params(delta: MaintenanceDelta).void }
+        def record_write_delta!(delta)
+          return unless incrementally_maintainable?
+
+          MaintenanceStore.new(view_class).merge!(delta)
+        end
+
+        sig { returns(T::Array[String]) }
+        def incremental_key_columns
+          columns = T.let(
+            T.unsafe(self).instance_variable_get(:@incremental_key_columns),
+            T.nilable(T::Array[String])
+          )
+          columns.nil? ? [] : columns
+        end
+
+        sig { returns(::ActiveRecord::Relation) }
+        def resolved_incremental_source
+          unless incremental_source_override?
+            Kernel.raise ArgumentError,
+                         "incremental_from override is not configured for #{view_class.name || view_class.view_key}"
+          end
+
+          view_class.send(
+            :resolve_source_definition,
+            @incremental_source_definition,
+            "incremental_from is required for #{view_class.name || view_class.view_key}"
+          )
+        end
+      end
+
+      mixes_in_class_methods ClassMethods
+    end
+  end
+end

data/lib/activerecord/materialized/view_query_access_class_methods.rb

@@ -0,0 +1,160 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # Raised when a read hits a cold view under the :raise cold-read strategy.
+    class NotMaterializedError < StandardError; end
+
+    # Read and refresh API mixed into a {View}: `rebuild!`, `refresh!`, `refresh_if_stale!`,
+    # `materialized?`, `stale?`, `dirty?`, and the routed query methods.
+    module ViewQueryAccessClassMethods
+      extend T::Sig
+      extend T::Helpers
+
+      sig { params(base: T.class_of(View)).void }
+      def self.included(base)
+        base.extend(ClassMethods)
+      end
+
+      # The read and refresh methods available on a {View} subclass.
+      module ClassMethods
+        extend T::Sig
+
+        sig { returns(T.class_of(View)) }
+        def view_class
+          T.cast(self, T.class_of(View))
+        end
+
+        sig { returns(T::Boolean) }
+        def stale?
+          view_class.metadata.stale?
+        end
+
+        sig { returns(T::Boolean) }
+        def dirty?
+          view_class.metadata.dirty?
+        end
+
+        sig { returns(T::Boolean) }
+        def warm?
+          view_class.metadata.warm?
+        end
+
+        # Reads are served from the cache only once warmed and the table exists;
+        # otherwise they fall through to the cold-read path.
+        sig { returns(T::Boolean) }
+        def materialized?
+          view_class.table_exists? && view_class.metadata.warm?
+        end
+
+        sig { returns(T.nilable(Timestamp)) }
+        def last_refreshed_at
+          view_class.metadata.last_refreshed_at
+        end
+
+        sig { returns(T::Boolean) }
+        def refreshing?
+          view_class.metadata.refreshing?
+        end
+
+        sig { void }
+        def mark_dependencies_changed!
+          view_class.metadata.mark_dirty!
+        end
+
+        sig { returns(T::Boolean) }
+        def table_exists?
+          view_class.connection.data_source_exists?(view_class.table_name)
+        end
+
+        # Incremental maintenance only — never scans all base data.
+        sig { returns(RefreshResult) }
+        def refresh!
+          Refresher.new(view_class).refresh!
+        end
+
+        sig { returns(T.nilable(RefreshResult)) }
+        def refresh_if_stale!
+          refresh! if materialized? && stale?
+        end
+
+        # The only path that scans all base data; `confirm:` guards against
+        # firing a full materialization by accident.
+        sig { params(confirm: T::Boolean).returns(RefreshResult) }
+        def rebuild!(confirm: false)
+          unless confirm
+            Kernel.raise ArgumentError,
+                         "#{view_class.name}.rebuild! performs a full materialization; call rebuild!(confirm: true)"
+          end
+
+          Refresher.new(view_class).rebuild!
+        end
+
+        sig { params(args: T.untyped).returns(T.untyped) }
+        def all(*args)
+          read_scope.all(*args)
+        end
+
+        sig { params(args: T.untyped).returns(T.untyped) }
+        def where(*args)
+          partition_scope(args).where(*args)
+        end
+
+        sig { params(args: T.untyped).returns(T.untyped) }
+        def find(*args)
+          read_scope.find(*args)
+        end
+
+        sig { params(args: T.untyped).returns(T.untyped) }
+        def find_by(*args)
+          partition_scope(args).find_by(*args)
+        end
+
+        sig { params(args: T.untyped).returns(T.untyped) }
+        def count(*args)
+          read_scope.count(*args)
+        end
+
+        private
+
+        sig { returns(T.untyped) }
+        def read_scope
+          materialized? ? cache_scope : cold_scope
+        end
+
+        # Per-partition fast path for keyed reads: serve fresh partitions from the
+        # cache, otherwise read through and enqueue maintenance (populate-on-read).
+        sig { params(args: T::Array[T.untyped]).returns(T.untyped) }
+        def partition_scope(args)
+          return cache_scope if materialized?
+
+          keys = PartitionState.keys_from(view_class, args)
+          return cold_scope if keys.nil?
+          return cache_scope if PartitionState.new(view_class).all_fresh?(keys)
+
+          enqueue_partition_maintenance(keys)
+          cold_scope
+        end
+
+        sig { returns(T.untyped) }
+        def cache_scope
+          T.unsafe(view_class).unscoped
+        end
+
+        sig { returns(T.untyped) }
+        def cold_scope
+          ColdRead.new(view_class).scope
+        end
+
+        sig { params(keys: T::Array[T.untyped]).void }
+        def enqueue_partition_maintenance(keys)
+          MaintenanceStore.new(view_class).merge!(MaintenanceDelta.scoped(keys))
+          RefreshScheduler.schedule(view_class)
+        end
+      end
+
+      mixes_in_class_methods ClassMethods
+    end
+  end
+end

data/lib/activerecord/materialized/view_refresh_policy_class_methods.rb

@@ -0,0 +1,109 @@
+# typed: strict
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Materialized
+    # Refresh-policy DSL mixed into a {View}: `refresh_on_change`, `refresh_debounce`, `cold_read`,
+    # `warm_up`, `max_staleness`, plus `warm_up!`.
+    module ViewRefreshPolicyClassMethods
+      extend T::Sig
+      extend T::Helpers
+
+      sig { params(base: T.class_of(View)).void }
+      def self.included(base)
+        base.extend(ClassMethods)
+      end
+
+      # The refresh-policy DSL methods available on a {View} subclass.
+      module ClassMethods
+        extend T::Sig
+
+        sig { returns(T.class_of(View)) }
+        def view_class
+          T.cast(self, T.class_of(View))
+        end
+
+        sig { params(strategy: Symbol).void }
+        def refresh_on_change(strategy = :async)
+          @refresh_strategy = T.let(strategy.to_sym, T.nilable(Symbol))
+        end
+
+        sig { params(seconds: DebounceInterval).void }
+        def refresh_debounce(seconds)
+          @refresh_debounce = T.let(seconds, T.nilable(DebounceInterval))
+        end
+
+        sig { params(strategy: Symbol).void }
+        def cold_read(strategy)
+          @cold_read_strategy = T.let(strategy.to_sym, T.nilable(Symbol))
+        end
+
+        sig { returns(Symbol) }
+        def resolved_cold_read_strategy
+          T.let(@cold_read_strategy, T.nilable(Symbol)) ||
+            ActiveRecord::Materialized.configuration.default_cold_read_strategy
+        end
+
+        # Queries warm_up! runs to materialize a cold view's hot partitions, e.g.:
+        #   warm_up { [where(region: "us"), order(revenue: :desc).limit(50)] }
+        sig { params(block: T.proc.returns(T.untyped)).void }
+        def warm_up(&block)
+          @warm_up_definition = T.let(block, T.nilable(Proc))
+        end
+
+        sig { returns(T::Array[::ActiveRecord::Relation]) }
+        def resolved_warm_up_queries
+          block = T.let(@warm_up_definition, T.nilable(Proc))
+          return [] if block.nil?
+
+          Kernel.Array(T.unsafe(view_class).instance_eval(&block))
+        end
+
+        # Running each warm_up query enqueues scoped maintenance for the
+        # partitions it touches; refresh! then applies it.
+        sig { returns(T.nilable(RefreshResult)) }
+        def warm_up!
+          resolved_warm_up_queries.each(&:to_a)
+          view_class.refresh!
+        end
+
+        sig { returns(Symbol) }
+        def resolved_refresh_strategy
+          @refresh_strategy || ActiveRecord::Materialized.configuration.default_refresh_strategy
+        end
+
+        sig { returns(T.any(Integer, Float)) }
+        def resolved_refresh_debounce
+          interval = if @refresh_debounce.nil?
+                       ActiveRecord::Materialized.configuration.default_refresh_debounce
+                     else
+                       @refresh_debounce
+                     end
+          interval.respond_to?(:to_f) ? interval.to_f : interval.to_i
+        end
+
+        sig do
+          params(
+            duration: T.nilable(StalenessDuration),
+            block: T.nilable(T.proc.returns(StalenessDuration))
+          ).void
+        end
+        def max_staleness(duration = nil, &block)
+          @max_staleness_setting = T.let(duration || block, T.nilable(T.any(StalenessDuration, Proc)))
+        end
+
+        sig { returns(T.nilable(StalenessDuration)) }
+        def resolved_max_staleness
+          setting = @max_staleness_setting
+          default = ActiveRecord::Materialized.configuration.default_max_staleness
+          return T.cast(default, T.nilable(StalenessDuration)) if setting.nil?
+          return T.unsafe(view_class).instance_eval(&setting) if setting.is_a?(Proc)
+
+          setting
+        end
+      end
+
+      mixes_in_class_methods ClassMethods
+    end
+  end
+end