activerecord-materialized 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +93 -0
  3. data/README.md +138 -323
  4. data/lib/activerecord/materialized/aggregate_analysis.rb +10 -31
  5. data/lib/activerecord/materialized/async_refresher.rb +18 -27
  6. data/lib/activerecord/materialized/cache_table_schema.rb +53 -34
  7. data/lib/activerecord/materialized/change_source.rb +25 -0
  8. data/lib/activerecord/materialized/cold_read.rb +1 -10
  9. data/lib/activerecord/materialized/column_type_inference.rb +167 -0
  10. data/lib/activerecord/materialized/configurable_table_name.rb +45 -0
  11. data/lib/activerecord/materialized/configuration.rb +104 -29
  12. data/lib/activerecord/materialized/connection_routing.rb +29 -0
  13. data/lib/activerecord/materialized/data_verification_result.rb +41 -0
  14. data/lib/activerecord/materialized/data_verifier.rb +162 -0
  15. data/lib/activerecord/materialized/debezium_envelope.rb +105 -0
  16. data/lib/activerecord/materialized/delta_maintainer.rb +4 -15
  17. data/lib/activerecord/materialized/dependency_registry.rb +43 -35
  18. data/lib/activerecord/materialized/dependency_trackable.rb +7 -14
  19. data/lib/activerecord/materialized/incremental_maintainer.rb +45 -28
  20. data/lib/activerecord/materialized/incremental_refresh.rb +63 -0
  21. data/lib/activerecord/materialized/instrumentation.rb +102 -0
  22. data/lib/activerecord/materialized/maintenance_delta.rb +4 -19
  23. data/lib/activerecord/materialized/maintenance_delta_builder.rb +46 -14
  24. data/lib/activerecord/materialized/maintenance_store.rb +58 -21
  25. data/lib/activerecord/materialized/metadata/maintenance_payload.rb +1 -7
  26. data/lib/activerecord/materialized/metadata/reconciliation.rb +25 -0
  27. data/lib/activerecord/materialized/metadata/schema.rb +19 -41
  28. data/lib/activerecord/materialized/metadata/timestamps.rb +0 -5
  29. data/lib/activerecord/materialized/metadata.rb +41 -25
  30. data/lib/activerecord/materialized/metadata_record.rb +2 -16
  31. data/lib/activerecord/materialized/migration_builder.rb +8 -8
  32. data/lib/activerecord/materialized/module_api.rb +219 -22
  33. data/lib/activerecord/materialized/partition_filter.rb +45 -0
  34. data/lib/activerecord/materialized/partition_keyed_store.rb +72 -0
  35. data/lib/activerecord/materialized/partition_record.rb +2 -16
  36. data/lib/activerecord/materialized/partition_snapshot.rb +79 -0
  37. data/lib/activerecord/materialized/partition_state.rb +47 -46
  38. data/lib/activerecord/materialized/query_expressions.rb +0 -12
  39. data/lib/activerecord/materialized/railtie.rb +15 -3
  40. data/lib/activerecord/materialized/read_router.rb +51 -0
  41. data/lib/activerecord/materialized/reconcile_job.rb +22 -0
  42. data/lib/activerecord/materialized/reconcile_result.rb +36 -0
  43. data/lib/activerecord/materialized/reconciler.rb +79 -0
  44. data/lib/activerecord/materialized/refresh_callbacks.rb +4 -18
  45. data/lib/activerecord/materialized/refresh_job.rb +0 -4
  46. data/lib/activerecord/materialized/refresh_result.rb +2 -10
  47. data/lib/activerecord/materialized/refresh_scheduler.rb +5 -7
  48. data/lib/activerecord/materialized/refresher.rb +35 -50
  49. data/lib/activerecord/materialized/registry.rb +26 -16
  50. data/lib/activerecord/materialized/relation_cache_writer.rb +12 -46
  51. data/lib/activerecord/materialized/schema_verifier.rb +0 -9
  52. data/lib/activerecord/materialized/source_watermark.rb +93 -0
  53. data/lib/activerecord/materialized/source_watermark_record.rb +16 -0
  54. data/lib/activerecord/materialized/summary_delta.rb +1 -17
  55. data/lib/activerecord/materialized/summary_delta_builder.rb +2 -10
  56. data/lib/activerecord/materialized/table_model_registry.rb +1 -8
  57. data/lib/activerecord/materialized/table_swap.rb +106 -0
  58. data/lib/activerecord/materialized/tasks.rb +46 -38
  59. data/lib/activerecord/materialized/version.rb +1 -2
  60. data/lib/activerecord/materialized/view.rb +11 -22
  61. data/lib/activerecord/materialized/view_configuration_class_methods.rb +23 -35
  62. data/lib/activerecord/materialized/view_definition.rb +76 -65
  63. data/lib/activerecord/materialized/view_incremental_class_methods.rb +57 -60
  64. data/lib/activerecord/materialized/view_loader.rb +49 -0
  65. data/lib/activerecord/materialized/view_query_access_class_methods.rb +100 -57
  66. data/lib/activerecord/materialized/view_refresh_policy_class_methods.rb +70 -37
  67. data/lib/activerecord/materialized/write_change.rb +75 -26
  68. data/lib/activerecord/materialized/write_maintenance.rb +73 -0
  69. data/lib/activerecord/materialized/write_outbox.rb +345 -0
  70. data/lib/activerecord/materialized/write_outbox_record.rb +17 -0
  71. data/lib/activerecord/materialized.rb +27 -6
  72. data/lib/generators/activerecord_materialized/install/templates/create_ar_materialized_view_metadata.rb.erb +14 -0
  73. data/lib/generators/activerecord_materialized/install_generator.rb +1 -6
  74. data/lib/generators/activerecord_materialized/migration_generator.rb +3 -9
  75. data/lib/generators/activerecord_materialized/outbox_generator.rb +36 -0
  76. data/lib/generators/activerecord_materialized/templates/materialized_view_migration.rb.erb +5 -1
  77. data/lib/generators/activerecord_materialized/templates/write_outbox_migration.rb.erb +21 -0
  78. data/lib/generators/activerecord_materialized/view_generator.rb +0 -4
  79. metadata +27 -18
  80. data/lib/activerecord/materialized/type_reexports.rb +0 -14
  81. data/lib/activerecord/materialized/view_class.rb +0 -8
  82. data/lib/activerecord_materialized_types.rb +0 -18
@@ -1,4 +1,3 @@
1
- # typed: strict
2
1
  # frozen_string_literal: true
3
2
 
4
3
  module ActiveRecord
@@ -8,49 +7,52 @@ module ActiveRecord
8
7
  # @api private
9
8
  class DependencyRegistry
10
9
  class << self
11
- extend T::Sig
12
-
13
- sig do
14
- params(
15
- view_class: ViewClass,
16
- tables: T.any(
17
- Symbol,
18
- String,
19
- T.class_of(::ActiveRecord::Base),
20
- T::Array[T.any(Symbol, String, T.class_of(::ActiveRecord::Base))]
21
- )
22
- ).void
23
- end
24
10
  def register(view_class, tables)
25
11
  normalized = Array(tables).flat_map { |entry| normalize_dependency(entry) }
26
- T.unsafe(view_class).instance_variable_set(:@dependency_tables, normalized)
12
+ view_class.instance_variable_set(:@dependency_tables, normalized)
27
13
 
28
14
  normalized.each do |table|
29
- bucket = T.cast(dependency_index[table], T::Array[ViewClass])
15
+ bucket = dependency_index[table]
30
16
  bucket << view_class unless bucket.include?(view_class)
31
- subscribe_source_table!(table)
17
+ subscribe_source_table!(table, view_class)
32
18
  end
33
19
  end
34
20
 
35
- sig { params(table: String).returns(T::Array[ViewClass]) }
36
21
  def views_for_table(table)
37
- T.must(dependency_index[normalize_table(table)])
22
+ dependency_index[normalize_table(table)]
38
23
  end
39
24
 
40
- sig { params(change: WriteChange).void }
41
- def publish_write_change!(change)
25
+ # Records a committed write and schedules refresh for the affected views.
26
+ # `source` identifies the publisher so no view is maintained by two sources:
27
+ # a callback publish (`:callbacks`) drives only callback-backed views, and an
28
+ # explicit ingestion-API publish (`nil`) drives only externally-fed views.
29
+ def publish_write_change!(change, source: nil)
42
30
  affected_views([change.table_name]).each do |view|
31
+ next unless delivers_to?(view, source)
32
+
43
33
  view.record_write_change!(change)
44
34
  RefreshScheduler.schedule(view)
45
35
  end
46
36
  end
47
37
 
48
- sig { params(tables: T::Array[String]).void }
38
+ # Coarse ingestion signal: something changed in these tables but the caller
39
+ # cannot describe the individual write. Enqueues a full recompute (the only
40
+ # correct scope without a partition key) and schedules it — idempotent, so
41
+ # safe to call repeatedly and after callback-skipping bulk loads.
49
42
  def mark_dirty_for_tables!(tables)
50
- affected_views(tables).each(&:mark_dependencies_changed!)
43
+ affected_views(tables).each do |view|
44
+ MaintenanceStore.new(view).merge!(MaintenanceDelta.full_partition)
45
+ RefreshScheduler.schedule(view)
46
+ end
47
+ end
48
+
49
+ # (Re)installs the built-in callback tracker for a view's already-declared
50
+ # dependency tables. Invoked by `change_source :callbacks` so opting in works
51
+ # whether it precedes or follows `depends_on`.
52
+ def install_callbacks_for(view_class)
53
+ view_class.dependency_tables.each { |table| subscribe_source_table!(table, view_class) }
51
54
  end
52
55
 
53
- sig { void }
54
56
  def reset!
55
57
  @dependency_index = Hash.new { |hash, key| hash[key] = [] }
56
58
  ActiveRecord::Materialized::DependencyTrackable.reset!
@@ -58,15 +60,19 @@ module ActiveRecord
58
60
 
59
61
  private
60
62
 
61
- sig { returns(T::Hash[String, T::Array[ViewClass]]) }
62
63
  def dependency_index
63
- @dependency_index ||= T.let(
64
- Hash.new { |hash, key| hash[key] = [] },
65
- T.nilable(T::Hash[String, T::Array[ViewClass]])
66
- )
64
+ @dependency_index ||= Hash.new { |hash, key| hash[key] = [] }
65
+ end
66
+
67
+ # A callback publish reaches callback-backed views; any other (external)
68
+ # publish reaches views that are NOT callback-backed. This keeps each view
69
+ # tied to a single source, so the additive summary-delta path is never
70
+ # applied twice for one write.
71
+ def delivers_to?(view, source)
72
+ callback_backed = view.resolved_change_source == ChangeSource::CALLBACKS
73
+ source == ChangeSource::CALLBACKS ? callback_backed : !callback_backed
67
74
  end
68
75
 
69
- sig { params(tables: T::Array[String]).returns(T::Array[ViewClass]) }
70
76
  def affected_views(tables)
71
77
  Array(tables).flat_map do |table|
72
78
  next [] if skip_table?(table)
@@ -75,7 +81,6 @@ module ActiveRecord
75
81
  end.uniq
76
82
  end
77
83
 
78
- sig { params(entry: T.untyped).returns(T::Array[String]) }
79
84
  def normalize_dependency(entry)
80
85
  if entry.is_a?(Class) && entry < ::ActiveRecord::Base
81
86
  TableModelRegistry.register(entry)
@@ -85,18 +90,21 @@ module ActiveRecord
85
90
  [normalize_table(entry)]
86
91
  end
87
92
 
88
- sig { params(table: String).void }
89
- def subscribe_source_table!(table)
93
+ # Installs the built-in commit-callback tracker for a table's model, unless
94
+ # the view opts out of callbacks (`change_source :none` / a `:none` global
95
+ # default) — in which case the dependency is still indexed for scoping and
96
+ # metadata, but its writes are expected to arrive via the ingestion API.
97
+ def subscribe_source_table!(table, view_class)
98
+ return unless view_class.resolved_change_source == ChangeSource::CALLBACKS
99
+
90
100
  model = TableModelRegistry.resolve(table)
91
101
  ActiveRecord::Materialized::DependencyTrackable.subscribe(model) if model
92
102
  end
93
103
 
94
- sig { params(table: T.any(Symbol, String)).returns(String) }
95
104
  def normalize_table(table)
96
105
  ::ActiveSupport::Inflector.underscore(table.to_s.delete_prefix(":"))
97
106
  end
98
107
 
99
- sig { params(table: T.any(Symbol, String)).returns(T::Boolean) }
100
108
  def skip_table?(table)
101
109
  name = normalize_table(table)
102
110
  name.start_with?("mv_") || name == ::ActiveRecord::Materialized.metadata_table_name
@@ -1,4 +1,3 @@
1
- # typed: strict
2
1
  # frozen_string_literal: true
3
2
 
4
3
  module ActiveRecord
@@ -7,14 +6,9 @@ module ActiveRecord
7
6
  #
8
7
  # @api private
9
8
  module DependencyTrackable
10
- extend T::Sig
11
-
12
9
  TRACKABLE_FLAG = :@ar_materialized_dependency_trackable
13
10
 
14
11
  class << self
15
- extend T::Sig
16
-
17
- sig { params(model_class: T.class_of(::ActiveRecord::Base)).void }
18
12
  def subscribe(model_class)
19
13
  return if model_class.instance_variable_get(TRACKABLE_FLAG)
20
14
 
@@ -23,24 +17,23 @@ module ActiveRecord
23
17
  end
24
18
 
25
19
  # Invoked from the model commit callbacks; `record` is the committed instance.
26
- sig { params(record: ::ActiveRecord::Base, operation: WriteChange::Operation).void }
20
+ # Publishes as the `:callbacks` source so views fed by another change source
21
+ # are not maintained twice.
27
22
  def publish(record, operation)
28
- DependencyRegistry.publish_write_change!(WriteChange.from_record(record, operation))
23
+ DependencyRegistry.publish_write_change!(WriteChange.from_record(record, operation), source: :callbacks)
29
24
  end
30
25
 
31
- sig { void }
32
26
  def reset!
33
27
  nil
34
28
  end
35
29
 
36
30
  private
37
31
 
38
- sig { params(model_class: T.class_of(::ActiveRecord::Base)).void }
39
32
  def install_callbacks!(model_class)
40
- model = T.unsafe(model_class)
41
- model.after_create_commit { DependencyTrackable.publish(T.unsafe(self), :create) }
42
- model.after_update_commit { DependencyTrackable.publish(T.unsafe(self), :update) }
43
- model.after_destroy_commit { DependencyTrackable.publish(T.unsafe(self), :destroy) }
33
+ model = model_class
34
+ model.after_create_commit { DependencyTrackable.publish(self, :create) }
35
+ model.after_update_commit { DependencyTrackable.publish(self, :update) }
36
+ model.after_destroy_commit { DependencyTrackable.publish(self, :destroy) }
44
37
  end
45
38
  end
46
39
  end
@@ -1,4 +1,3 @@
1
- # typed: strict
2
1
  # frozen_string_literal: true
3
2
 
4
3
  module ActiveRecord
@@ -7,51 +6,69 @@ module ActiveRecord
7
6
  #
8
7
  # @api private
9
8
  class IncrementalMaintainer
10
- extend T::Sig
11
-
12
- sig { params(view_class: ViewClass).void }
13
9
  def initialize(view_class)
14
10
  @view_class = view_class
15
11
  end
16
12
 
17
- sig { params(_connection: Connection, _table_name: String).returns(Integer) }
18
13
  def maintain!(_connection, _table_name)
19
14
  delta = maintenance_store.consume_pending_delta!
20
- relation = resolve_maintenance_relation(delta)
15
+ return view_class.unscoped.count if delta.nil? # another cross-process cycle consumed it — no-op
16
+
17
+ apply_delta!(delta)
18
+ end
19
+
20
+ private
21
+
22
+ attr_reader :view_class
23
+
24
+ def apply_delta!(delta)
25
+ # A widen (full_partition) on a warm view is a full recompute — the same scan as rebuild!, so
26
+ # route it through the atomic build-and-swap instead of an in-place delete_all + re-insert. That
27
+ # keeps readers on the old snapshot until an instant rename, rather than exposing an empty/locked
28
+ # table mid-recompute. (A cold view can't reach here with full_partition: Refresher#maintainable?
29
+ # declines it, #120. So this branch is always the warm full-recompute case.)
30
+ return recompute_all! if delta.full_partition?
31
+
32
+ apply_scoped_delta!(delta)
33
+ end
34
+
35
+ def recompute_all!
36
+ RelationCacheWriter.new(view_class).atomic_swap!(full_source_relation)
37
+ end
38
+
39
+ def apply_scoped_delta!(delta)
40
+ partition_state = PartitionState.new(view_class)
41
+ cold_scoped = !view_class.materialized?
42
+ # Capture the fresh-set epoch BEFORE the source read, so a widen committing during the read
43
+ # advances the epoch and leaves this populate's mark un-served (the populate-vs-widen race, #120).
44
+ generation = partition_state.current_generation if cold_scoped
21
45
 
22
46
  row_count = RelationCacheWriter.new(view_class).replace_partitions!(
23
- relation,
24
- key_tuples: delta.key_tuples,
25
- full_partition: delta.full_partition?
47
+ scoped_source_relation(delta.key_tuples),
48
+ key_tuples: delta.key_tuples
26
49
  )
27
50
 
28
- # On a cold view the maintained partitions are now fresh.
29
- unless delta.full_partition? || view_class.materialized?
30
- PartitionState.new(view_class).mark_fresh!(delta.key_tuples)
31
- end
51
+ # On a cold view the maintained partitions are now fresh, stamped with the captured epoch.
52
+ partition_state.mark_fresh!(delta.key_tuples, generation: generation) if cold_scoped
32
53
 
33
54
  row_count
34
55
  end
35
56
 
36
- private
37
-
38
- sig { returns(ViewClass) }
39
- attr_reader :view_class
40
-
41
- sig { returns(MaintenanceStore) }
42
57
  def maintenance_store
43
58
  MaintenanceStore.new(view_class)
44
59
  end
45
60
 
46
- sig { params(delta: MaintenanceDelta).returns(::ActiveRecord::Relation) }
47
- def resolve_maintenance_relation(delta)
48
- if view_class.incremental_source_override?
49
- view_class.resolved_incremental_source
50
- elsif delta.full_partition?
51
- view_class.resolved_source
52
- else
53
- view_class.view_definition.partition_scope(delta.key_tuples)
54
- end
61
+ # The whole-view source for a full recompute (the incremental override wins when configured).
62
+ def full_source_relation
63
+ view_class.incremental_source_override? ? view_class.resolved_incremental_source : view_class.resolved_source
64
+ end
65
+
66
+ # The source restricted to the affected partitions (the incremental override wins when configured;
67
+ # it is already the maintainable relation, so it is not re-scoped here).
68
+ def scoped_source_relation(key_tuples)
69
+ return view_class.resolved_incremental_source if view_class.incremental_source_override?
70
+
71
+ view_class.view_definition.partition_scope(key_tuples)
55
72
  end
56
73
  end
57
74
  end
@@ -0,0 +1,63 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveRecord
4
+ module Materialized
5
+ # Runs one incremental-maintenance pass for a view, choosing the summary-delta
6
+ # or scoped-recompute path from the pending maintenance and annotating the
7
+ # refresh instrumentation payload with the mode and partitions it touched.
8
+ #
9
+ # @api private
10
+ class IncrementalRefresh
11
+ def initialize(view_class, payload)
12
+ @view_class = view_class
13
+ @payload = payload
14
+ end
15
+
16
+ def call
17
+ ensure_cache_table!
18
+ store = MaintenanceStore.new(@view_class)
19
+ pending = store.pending
20
+ @payload[:mode] = mode(pending)
21
+ @payload[:partition_count] = partition_count(pending)
22
+ apply!(store, pending)
23
+ end
24
+
25
+ private
26
+
27
+ def apply!(store, pending)
28
+ if pending.is_a?(SummaryDelta)
29
+ # Consume+apply the additive delta under a row lock so concurrent cross-process cycles
30
+ # can't apply it twice; a loser finds an empty payload and no-ops. The row count is read
31
+ # after the lock releases (keeping the full-table COUNT out of the critical section) and
32
+ # reflects the cache's true total for winner and loser alike — never 0, which would
33
+ # clobber a populated view's row_count. The scoped path below serializes its own consume
34
+ # (MaintenanceStore#consume_pending_delta! takes the same row lock, #95), so a concurrent
35
+ # cross-process double-run recomputes a partition once rather than duplicating its rows.
36
+ store.with_consumed_summary_delta { |delta| DeltaMaintainer.new(@view_class).apply!(delta) }
37
+ return @view_class.unscoped.count
38
+ end
39
+
40
+ IncrementalMaintainer.new(@view_class).maintain!(@view_class.connection, @view_class.table_name)
41
+ end
42
+
43
+ def mode(pending)
44
+ pending.is_a?(SummaryDelta) ? :summary_delta : :scoped_recompute
45
+ end
46
+
47
+ # Partitions this pass recomputes; nil when it widened to a full recompute.
48
+ def partition_count(pending)
49
+ return pending.tracked_partition_count if pending.is_a?(SummaryDelta)
50
+ return nil unless pending.is_a?(MaintenanceDelta)
51
+
52
+ pending.full_partition? ? nil : pending.key_tuples.size
53
+ end
54
+
55
+ # Cheap DDL so partition maintenance has somewhere to write — never a populate.
56
+ def ensure_cache_table!
57
+ return if @view_class.table_exists?
58
+
59
+ CacheTableSchema.ensure_table!(@view_class, @view_class.resolved_source)
60
+ end
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,102 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveRecord
4
+ module Materialized
5
+ # Emits `ActiveSupport::Notifications` events at the read / refresh /
6
+ # maintenance lifecycle points so adopters can wire freshness and behavior
7
+ # SLIs to any APM / StatsD / OpenTelemetry backend without the gem taking a
8
+ # dependency on a specific telemetry vendor.
9
+ #
10
+ # The event names and payload keys documented here are a stable contract —
11
+ # subscribe with {https://api.rubyonrails.org/classes/ActiveSupport/Notifications.html
12
+ # ActiveSupport::Notifications}:
13
+ #
14
+ # ActiveSupport::Notifications.subscribe("read.active_record_materialized") do |event|
15
+ # StatsD.increment("mv.read.#{event.payload[:source]}", tags: ["view:#{event.payload[:view]}"])
16
+ # end
17
+ #
18
+ # Events:
19
+ #
20
+ # * +read.active_record_materialized+ — one per routed read. Payload:
21
+ # +:view+ (the view class), +:source+ (+:cache+, +:read_through+,
22
+ # +:serve_stale+, or +:raise+), +:staleness+ (seconds since the last
23
+ # successful refresh, or +nil+ when never refreshed).
24
+ # * +refresh.active_record_materialized+ — one per +refresh!+/+rebuild!+,
25
+ # timed. Payload: +:view+, +:operation+ (+:incremental+ or +:rebuild+),
26
+ # +:mode+ (+:summary_delta+, +:scoped_recompute+, or +:full+),
27
+ # +:partition_count+ (partitions recomputed, or +nil+ for a full pass),
28
+ # +:row_count+, +:skipped+, and — on failure — the standard
29
+ # +:exception+/+:exception_object+ keys set by +ActiveSupport::Notifications+.
30
+ # * +maintenance.active_record_materialized+ — one per dependency write that
31
+ # records pending maintenance. Payload: +:view+, +:table+, +:operation+
32
+ # (+:create+/+:update+/+:destroy+), +:path+ (+:summary_delta+ or
33
+ # +:scoped_recompute+), +:scope+ (+:scoped+, or +:full+ when the write
34
+ # widened to a full recompute because its partition key could not be
35
+ # derived), +:partition_count+ (distinct partitions scoped, 0 on widen).
36
+ # * +reconcile.active_record_materialized+ — one per {Reconciler} run. Payload:
37
+ # +:view+, +:mode+ (the drift-check depth), +:repaired_partition_count+
38
+ # (partitions found divergent and repaired with scoped maintenance), +:deferred+
39
+ # (+true+ when a concurrent refresh deferred the run to the next tick).
40
+ module Instrumentation
41
+ READ = "read.active_record_materialized"
42
+ REFRESH = "refresh.active_record_materialized"
43
+ MAINTENANCE = "maintenance.active_record_materialized"
44
+ RECONCILE = "reconcile.active_record_materialized"
45
+
46
+ class << self
47
+ # Fired once per routed read. The staleness lookup costs a metadata read,
48
+ # so it is skipped entirely unless a subscriber is attached.
49
+ def read(view_class, source:)
50
+ return unless ::ActiveSupport::Notifications.notifier.listening?(READ)
51
+
52
+ ::ActiveSupport::Notifications.instrument(
53
+ READ,
54
+ view: view_class, source: source, staleness: staleness_for(view_class)
55
+ )
56
+ end
57
+
58
+ # Wraps a refresh/rebuild so the event carries its wall-clock duration and
59
+ # any raised exception. The block runs the refresh and returns its
60
+ # {RefreshResult}; the row count and skipped flag are read back from it,
61
+ # while the block annotates the payload with the +:mode+ and
62
+ # +:partition_count+ it can only know mid-flight.
63
+ def refresh(view_class, operation:, &block)
64
+ ::ActiveSupport::Notifications.instrument(REFRESH, view: view_class, operation: operation) do |payload|
65
+ result = yield(payload)
66
+ payload[:row_count] = result.row_count
67
+ payload[:skipped] = result.skipped
68
+ result
69
+ end
70
+ end
71
+
72
+ # Fired once per dependency write that records pending maintenance.
73
+ def maintenance(view_class, change:, path:, scope:, partition_count:)
74
+ ::ActiveSupport::Notifications.instrument(
75
+ MAINTENANCE,
76
+ view: view_class, table: change.table_name, operation: change.operation.to_sym,
77
+ path: path, scope: scope, partition_count: partition_count
78
+ )
79
+ end
80
+
81
+ # Fired once per {Reconciler} run — after it verifies and (if needed) repairs
82
+ # via scoped maintenance — carrying the run's mode, repaired count, and defer flag.
83
+ def reconcile(view_class, mode:, repaired_partition_count:, deferred:)
84
+ ::ActiveSupport::Notifications.instrument(
85
+ RECONCILE,
86
+ view: view_class, mode: mode,
87
+ repaired_partition_count: repaired_partition_count, deferred: deferred
88
+ )
89
+ end
90
+
91
+ private
92
+
93
+ def staleness_for(view_class)
94
+ last_refreshed_at = view_class.metadata.last_refreshed_at
95
+ return nil if last_refreshed_at.nil?
96
+
97
+ (Metadata::Timestamps.current.to_time - last_refreshed_at.to_time).to_f
98
+ end
99
+ end
100
+ end
101
+ end
102
+ end
@@ -1,4 +1,3 @@
1
- # typed: strict
2
1
  # frozen_string_literal: true
3
2
 
4
3
  module ActiveRecord
@@ -7,46 +6,34 @@ module ActiveRecord
7
6
  #
8
7
  # @api private
9
8
  class MaintenanceDelta
10
- extend T::Sig
9
+ SCOPED = :scoped
10
+ FULL = :full_partition
11
11
 
12
- SCOPED = T.let(:scoped, Symbol)
13
- FULL = T.let(:full_partition, Symbol)
12
+ attr_reader :scope, :key_tuples
14
13
 
15
- sig { returns(Symbol) }
16
- attr_reader :scope
17
-
18
- sig { returns(T::Array[T::Array[String]]) }
19
- attr_reader :key_tuples
20
-
21
- sig { params(scope: Symbol, key_tuples: T::Array[T::Array[String]]).void }
22
14
  def initialize(scope:, key_tuples: [])
23
15
  @scope = scope
24
16
  @key_tuples = key_tuples
25
17
  end
26
18
 
27
- sig { params(key_tuples: T::Array[T::Array[String]]).returns(MaintenanceDelta) }
28
19
  def self.scoped(key_tuples)
29
20
  new(scope: SCOPED, key_tuples: key_tuples)
30
21
  end
31
22
 
32
- sig { returns(MaintenanceDelta) }
33
23
  def self.full_partition
34
24
  new(scope: FULL)
35
25
  end
36
26
 
37
- sig { returns(T::Boolean) }
38
27
  def full_partition?
39
28
  scope == FULL
40
29
  end
41
30
 
42
31
  # How many distinct partitions this pending maintenance tracks. A
43
32
  # full-partition recompute tracks none — it is already the collapsed form.
44
- sig { returns(Integer) }
45
33
  def tracked_partition_count
46
34
  full_partition? ? 0 : key_tuples.size
47
35
  end
48
36
 
49
- sig { params(other: MaintenanceDelta).returns(MaintenanceDelta) }
50
37
  def merge(other)
51
38
  return other if other.full_partition?
52
39
  return self if full_partition?
@@ -55,7 +42,6 @@ module ActiveRecord
55
42
  self.class.scoped(combined)
56
43
  end
57
44
 
58
- sig { returns(T::Hash[String, T.untyped]) }
59
45
  def serialize
60
46
  {
61
47
  "scope" => scope.to_s,
@@ -63,7 +49,6 @@ module ActiveRecord
63
49
  }
64
50
  end
65
51
 
66
- sig { params(payload: T.nilable(T::Hash[String, T.untyped])).returns(T.nilable(MaintenanceDelta)) }
67
52
  def self.deserialize(payload)
68
53
  return nil if payload.blank?
69
54
 
@@ -73,7 +58,7 @@ module ActiveRecord
73
58
  if scope_name == FULL
74
59
  full_partition
75
60
  else
76
- tuples = T.cast(payload["key_tuples"], T.nilable(T::Array[T::Array[String]])) || []
61
+ tuples = payload["key_tuples"] || []
77
62
  scoped(tuples)
78
63
  end
79
64
  end
@@ -1,4 +1,3 @@
1
- # typed: strict
2
1
  # frozen_string_literal: true
3
2
 
4
3
  module ActiveRecord
@@ -7,25 +6,25 @@ module ActiveRecord
7
6
  #
8
7
  # @api private
9
8
  class MaintenanceDeltaBuilder
10
- extend T::Sig
11
-
12
- sig { params(change: WriteChange, key_columns: T::Array[String]).void }
13
- def initialize(change, key_columns)
9
+ def initialize(change, key_columns, resolver: nil)
14
10
  @change = change
15
11
  @key_columns = key_columns
12
+ @resolver = resolver
16
13
  end
17
14
 
18
- sig { returns(MaintenanceDelta) }
19
15
  def build
20
16
  return MaintenanceDelta.full_partition if @key_columns.empty?
21
17
 
22
- tuples = extract_tuples
18
+ # A resolver is an explicit per-table mapping, so it is authoritative for
19
+ # that table's writes (it maps to the group key wherever it lives, even if
20
+ # the written row happens to carry a same-named column); otherwise the key
21
+ # comes from the written row's own payload.
22
+ tuples = @resolver ? resolved_tuples : extract_tuples
23
23
  tuples.empty? ? MaintenanceDelta.full_partition : MaintenanceDelta.scoped(tuples.uniq)
24
24
  end
25
25
 
26
26
  private
27
27
 
28
- sig { returns(T::Array[T::Array[T.untyped]]) }
29
28
  def extract_tuples
30
29
  snapshots = case @change.operation.to_sym
31
30
  when :create then [@change.after]
@@ -33,29 +32,62 @@ module ActiveRecord
33
32
  when :update then [@change.before, @change.after]
34
33
  else []
35
34
  end
36
- snapshots.filter_map { |attributes| key_tuple(attributes) }.uniq
35
+ tuples = snapshots.map { |attributes| key_tuple(attributes) }
36
+ # An update touches two partitions (old + new), a create/destroy one. If ANY affected snapshot
37
+ # can't yield its partition key — e.g. a non-FULL CDC before- or after-image with only the
38
+ # primary key — we can't identify every affected partition (a partial after-image can't even
39
+ # rule out a move), so return no tuples and let +build+ widen to a full recompute rather than
40
+ # under-scope to only the derivable side (leaving the other partition stale). +build+
41
+ # de-duplicates the tuples it scopes, so no +uniq+ here.
42
+ return [] if tuples.any?(&:nil?)
43
+
44
+ tuples
45
+ end
46
+
47
+ # Partition key tuple(s) from the configured resolver, or [] (=> full
48
+ # recompute) when none is set or it yields nothing.
49
+ def resolved_tuples
50
+ resolver = @resolver
51
+ return [] if resolver.nil?
52
+
53
+ normalize_keys(resolver.call(@change))
54
+ end
55
+
56
+ # Coerces a resolver's return into partition-key tuples, using the group key's
57
+ # arity to tell a single composite tuple from a list of tuples. A nil/empty
58
+ # return widens (=> full recompute); a returned nil value is a real key (the
59
+ # NULL partition), so it is kept rather than dropped.
60
+ def normalize_keys(result)
61
+ return [] if result.nil?
62
+
63
+ @key_columns.one? ? Array(result).zip : composite_tuples(result)
64
+ end
65
+
66
+ # A composite key expects a tuple or an array of tuples; anything else (a bare
67
+ # scalar, nil, empty) can't be scoped, so it widens rather than crashing.
68
+ def composite_tuples(result)
69
+ return [] unless result.is_a?(Array) && result.any?
70
+
71
+ result.first.is_a?(Array) ? result : [result]
37
72
  end
38
73
 
39
- sig { params(attributes: T::Hash[String, T.untyped]).returns(T::Boolean) }
40
74
  def keys_present?(attributes)
41
75
  @key_columns.all? do |column|
42
- attributes.key?(column) || T.unsafe(attributes).key?(column.to_sym)
76
+ attributes.key?(column) || attributes.key?(column.to_sym)
43
77
  end
44
78
  end
45
79
 
46
- sig { params(attributes: T::Hash[String, T.untyped]).returns(T.nilable(T::Array[T.untyped])) }
47
80
  def key_tuple(attributes)
48
81
  return nil unless keys_present?(attributes)
49
82
 
50
83
  @key_columns.map { |column| attribute_value(attributes, column) }
51
84
  end
52
85
 
53
- sig { params(attributes: T::Hash[String, T.untyped], column: String).returns(T.untyped) }
54
86
  def attribute_value(attributes, column)
55
87
  # Look up by presence so falsey group-key values map to their partition.
56
88
  return attributes[column] if attributes.key?(column)
57
89
 
58
- T.unsafe(attributes)[column.to_sym]
90
+ attributes[column.to_sym]
59
91
  end
60
92
  end
61
93
  end