activerecord-materialized 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +93 -0
- data/README.md +139 -324
- data/lib/activerecord/materialized/aggregate_analysis.rb +10 -31
- data/lib/activerecord/materialized/async_refresher.rb +18 -27
- data/lib/activerecord/materialized/cache_table_schema.rb +53 -34
- data/lib/activerecord/materialized/change_source.rb +25 -0
- data/lib/activerecord/materialized/cold_read.rb +1 -10
- data/lib/activerecord/materialized/column_type_inference.rb +167 -0
- data/lib/activerecord/materialized/configurable_table_name.rb +45 -0
- data/lib/activerecord/materialized/configuration.rb +104 -29
- data/lib/activerecord/materialized/connection_routing.rb +29 -0
- data/lib/activerecord/materialized/data_verification_result.rb +41 -0
- data/lib/activerecord/materialized/data_verifier.rb +162 -0
- data/lib/activerecord/materialized/debezium_envelope.rb +105 -0
- data/lib/activerecord/materialized/delta_maintainer.rb +4 -15
- data/lib/activerecord/materialized/dependency_registry.rb +43 -35
- data/lib/activerecord/materialized/dependency_trackable.rb +7 -14
- data/lib/activerecord/materialized/incremental_maintainer.rb +45 -28
- data/lib/activerecord/materialized/incremental_refresh.rb +63 -0
- data/lib/activerecord/materialized/instrumentation.rb +102 -0
- data/lib/activerecord/materialized/maintenance_delta.rb +4 -19
- data/lib/activerecord/materialized/maintenance_delta_builder.rb +46 -14
- data/lib/activerecord/materialized/maintenance_store.rb +58 -21
- data/lib/activerecord/materialized/metadata/maintenance_payload.rb +1 -7
- data/lib/activerecord/materialized/metadata/reconciliation.rb +25 -0
- data/lib/activerecord/materialized/metadata/schema.rb +19 -41
- data/lib/activerecord/materialized/metadata/timestamps.rb +0 -5
- data/lib/activerecord/materialized/metadata.rb +41 -25
- data/lib/activerecord/materialized/metadata_record.rb +2 -16
- data/lib/activerecord/materialized/migration_builder.rb +8 -8
- data/lib/activerecord/materialized/module_api.rb +219 -22
- data/lib/activerecord/materialized/partition_filter.rb +45 -0
- data/lib/activerecord/materialized/partition_keyed_store.rb +72 -0
- data/lib/activerecord/materialized/partition_record.rb +2 -16
- data/lib/activerecord/materialized/partition_snapshot.rb +79 -0
- data/lib/activerecord/materialized/partition_state.rb +47 -46
- data/lib/activerecord/materialized/query_expressions.rb +0 -12
- data/lib/activerecord/materialized/railtie.rb +15 -3
- data/lib/activerecord/materialized/read_router.rb +51 -0
- data/lib/activerecord/materialized/reconcile_job.rb +22 -0
- data/lib/activerecord/materialized/reconcile_result.rb +36 -0
- data/lib/activerecord/materialized/reconciler.rb +79 -0
- data/lib/activerecord/materialized/refresh_callbacks.rb +4 -18
- data/lib/activerecord/materialized/refresh_job.rb +0 -4
- data/lib/activerecord/materialized/refresh_result.rb +2 -10
- data/lib/activerecord/materialized/refresh_scheduler.rb +5 -7
- data/lib/activerecord/materialized/refresher.rb +35 -50
- data/lib/activerecord/materialized/registry.rb +26 -16
- data/lib/activerecord/materialized/relation_cache_writer.rb +12 -46
- data/lib/activerecord/materialized/schema_verifier.rb +0 -9
- data/lib/activerecord/materialized/source_watermark.rb +93 -0
- data/lib/activerecord/materialized/source_watermark_record.rb +16 -0
- data/lib/activerecord/materialized/summary_delta.rb +1 -17
- data/lib/activerecord/materialized/summary_delta_builder.rb +2 -10
- data/lib/activerecord/materialized/table_model_registry.rb +1 -8
- data/lib/activerecord/materialized/table_swap.rb +106 -0
- data/lib/activerecord/materialized/tasks.rb +46 -38
- data/lib/activerecord/materialized/version.rb +1 -2
- data/lib/activerecord/materialized/view.rb +11 -22
- data/lib/activerecord/materialized/view_configuration_class_methods.rb +23 -35
- data/lib/activerecord/materialized/view_definition.rb +76 -65
- data/lib/activerecord/materialized/view_incremental_class_methods.rb +57 -60
- data/lib/activerecord/materialized/view_loader.rb +49 -0
- data/lib/activerecord/materialized/view_query_access_class_methods.rb +100 -57
- data/lib/activerecord/materialized/view_refresh_policy_class_methods.rb +70 -37
- data/lib/activerecord/materialized/write_change.rb +75 -26
- data/lib/activerecord/materialized/write_maintenance.rb +73 -0
- data/lib/activerecord/materialized/write_outbox.rb +345 -0
- data/lib/activerecord/materialized/write_outbox_record.rb +17 -0
- data/lib/activerecord/materialized.rb +27 -6
- data/lib/generators/activerecord_materialized/install/templates/create_ar_materialized_view_metadata.rb.erb +14 -0
- data/lib/generators/activerecord_materialized/install_generator.rb +1 -6
- data/lib/generators/activerecord_materialized/migration_generator.rb +3 -9
- data/lib/generators/activerecord_materialized/outbox_generator.rb +36 -0
- data/lib/generators/activerecord_materialized/templates/materialized_view_migration.rb.erb +5 -1
- data/lib/generators/activerecord_materialized/templates/write_outbox_migration.rb.erb +21 -0
- data/lib/generators/activerecord_materialized/view_generator.rb +0 -4
- metadata +28 -19
- data/lib/activerecord/materialized/type_reexports.rb +0 -14
- data/lib/activerecord/materialized/view_class.rb +0 -8
- data/lib/activerecord_materialized_types.rb +0 -18
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# typed: strict
|
|
2
1
|
# frozen_string_literal: true
|
|
3
2
|
|
|
4
3
|
module ActiveRecord
|
|
@@ -8,49 +7,52 @@ module ActiveRecord
|
|
|
8
7
|
# @api private
|
|
9
8
|
class DependencyRegistry
|
|
10
9
|
class << self
|
|
11
|
-
extend T::Sig
|
|
12
|
-
|
|
13
|
-
sig do
|
|
14
|
-
params(
|
|
15
|
-
view_class: ViewClass,
|
|
16
|
-
tables: T.any(
|
|
17
|
-
Symbol,
|
|
18
|
-
String,
|
|
19
|
-
T.class_of(::ActiveRecord::Base),
|
|
20
|
-
T::Array[T.any(Symbol, String, T.class_of(::ActiveRecord::Base))]
|
|
21
|
-
)
|
|
22
|
-
).void
|
|
23
|
-
end
|
|
24
10
|
def register(view_class, tables)
|
|
25
11
|
normalized = Array(tables).flat_map { |entry| normalize_dependency(entry) }
|
|
26
|
-
|
|
12
|
+
view_class.instance_variable_set(:@dependency_tables, normalized)
|
|
27
13
|
|
|
28
14
|
normalized.each do |table|
|
|
29
|
-
bucket =
|
|
15
|
+
bucket = dependency_index[table]
|
|
30
16
|
bucket << view_class unless bucket.include?(view_class)
|
|
31
|
-
subscribe_source_table!(table)
|
|
17
|
+
subscribe_source_table!(table, view_class)
|
|
32
18
|
end
|
|
33
19
|
end
|
|
34
20
|
|
|
35
|
-
sig { params(table: String).returns(T::Array[ViewClass]) }
|
|
36
21
|
def views_for_table(table)
|
|
37
|
-
|
|
22
|
+
dependency_index[normalize_table(table)]
|
|
38
23
|
end
|
|
39
24
|
|
|
40
|
-
|
|
41
|
-
|
|
25
|
+
# Records a committed write and schedules refresh for the affected views.
|
|
26
|
+
# `source` identifies the publisher so no view is maintained by two sources:
|
|
27
|
+
# a callback publish (`:callbacks`) drives only callback-backed views, and an
|
|
28
|
+
# explicit ingestion-API publish (`nil`) drives only externally-fed views.
|
|
29
|
+
def publish_write_change!(change, source: nil)
|
|
42
30
|
affected_views([change.table_name]).each do |view|
|
|
31
|
+
next unless delivers_to?(view, source)
|
|
32
|
+
|
|
43
33
|
view.record_write_change!(change)
|
|
44
34
|
RefreshScheduler.schedule(view)
|
|
45
35
|
end
|
|
46
36
|
end
|
|
47
37
|
|
|
48
|
-
|
|
38
|
+
# Coarse ingestion signal: something changed in these tables but the caller
|
|
39
|
+
# cannot describe the individual write. Enqueues a full recompute (the only
|
|
40
|
+
# correct scope without a partition key) and schedules it — idempotent, so
|
|
41
|
+
# safe to call repeatedly and after callback-skipping bulk loads.
|
|
49
42
|
def mark_dirty_for_tables!(tables)
|
|
50
|
-
affected_views(tables).each
|
|
43
|
+
affected_views(tables).each do |view|
|
|
44
|
+
MaintenanceStore.new(view).merge!(MaintenanceDelta.full_partition)
|
|
45
|
+
RefreshScheduler.schedule(view)
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# (Re)installs the built-in callback tracker for a view's already-declared
|
|
50
|
+
# dependency tables. Invoked by `change_source :callbacks` so opting in works
|
|
51
|
+
# whether it precedes or follows `depends_on`.
|
|
52
|
+
def install_callbacks_for(view_class)
|
|
53
|
+
view_class.dependency_tables.each { |table| subscribe_source_table!(table, view_class) }
|
|
51
54
|
end
|
|
52
55
|
|
|
53
|
-
sig { void }
|
|
54
56
|
def reset!
|
|
55
57
|
@dependency_index = Hash.new { |hash, key| hash[key] = [] }
|
|
56
58
|
ActiveRecord::Materialized::DependencyTrackable.reset!
|
|
@@ -58,15 +60,19 @@ module ActiveRecord
|
|
|
58
60
|
|
|
59
61
|
private
|
|
60
62
|
|
|
61
|
-
sig { returns(T::Hash[String, T::Array[ViewClass]]) }
|
|
62
63
|
def dependency_index
|
|
63
|
-
@dependency_index ||=
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
64
|
+
@dependency_index ||= Hash.new { |hash, key| hash[key] = [] }
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# A callback publish reaches callback-backed views; any other (external)
|
|
68
|
+
# publish reaches views that are NOT callback-backed. This keeps each view
|
|
69
|
+
# tied to a single source, so the additive summary-delta path is never
|
|
70
|
+
# applied twice for one write.
|
|
71
|
+
def delivers_to?(view, source)
|
|
72
|
+
callback_backed = view.resolved_change_source == ChangeSource::CALLBACKS
|
|
73
|
+
source == ChangeSource::CALLBACKS ? callback_backed : !callback_backed
|
|
67
74
|
end
|
|
68
75
|
|
|
69
|
-
sig { params(tables: T::Array[String]).returns(T::Array[ViewClass]) }
|
|
70
76
|
def affected_views(tables)
|
|
71
77
|
Array(tables).flat_map do |table|
|
|
72
78
|
next [] if skip_table?(table)
|
|
@@ -75,7 +81,6 @@ module ActiveRecord
|
|
|
75
81
|
end.uniq
|
|
76
82
|
end
|
|
77
83
|
|
|
78
|
-
sig { params(entry: T.untyped).returns(T::Array[String]) }
|
|
79
84
|
def normalize_dependency(entry)
|
|
80
85
|
if entry.is_a?(Class) && entry < ::ActiveRecord::Base
|
|
81
86
|
TableModelRegistry.register(entry)
|
|
@@ -85,18 +90,21 @@ module ActiveRecord
|
|
|
85
90
|
[normalize_table(entry)]
|
|
86
91
|
end
|
|
87
92
|
|
|
88
|
-
|
|
89
|
-
|
|
93
|
+
# Installs the built-in commit-callback tracker for a table's model, unless
|
|
94
|
+
# the view opts out of callbacks (`change_source :none` / a `:none` global
|
|
95
|
+
# default) — in which case the dependency is still indexed for scoping and
|
|
96
|
+
# metadata, but its writes are expected to arrive via the ingestion API.
|
|
97
|
+
def subscribe_source_table!(table, view_class)
|
|
98
|
+
return unless view_class.resolved_change_source == ChangeSource::CALLBACKS
|
|
99
|
+
|
|
90
100
|
model = TableModelRegistry.resolve(table)
|
|
91
101
|
ActiveRecord::Materialized::DependencyTrackable.subscribe(model) if model
|
|
92
102
|
end
|
|
93
103
|
|
|
94
|
-
sig { params(table: T.any(Symbol, String)).returns(String) }
|
|
95
104
|
def normalize_table(table)
|
|
96
105
|
::ActiveSupport::Inflector.underscore(table.to_s.delete_prefix(":"))
|
|
97
106
|
end
|
|
98
107
|
|
|
99
|
-
sig { params(table: T.any(Symbol, String)).returns(T::Boolean) }
|
|
100
108
|
def skip_table?(table)
|
|
101
109
|
name = normalize_table(table)
|
|
102
110
|
name.start_with?("mv_") || name == ::ActiveRecord::Materialized.metadata_table_name
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# typed: strict
|
|
2
1
|
# frozen_string_literal: true
|
|
3
2
|
|
|
4
3
|
module ActiveRecord
|
|
@@ -7,14 +6,9 @@ module ActiveRecord
|
|
|
7
6
|
#
|
|
8
7
|
# @api private
|
|
9
8
|
module DependencyTrackable
|
|
10
|
-
extend T::Sig
|
|
11
|
-
|
|
12
9
|
TRACKABLE_FLAG = :@ar_materialized_dependency_trackable
|
|
13
10
|
|
|
14
11
|
class << self
|
|
15
|
-
extend T::Sig
|
|
16
|
-
|
|
17
|
-
sig { params(model_class: T.class_of(::ActiveRecord::Base)).void }
|
|
18
12
|
def subscribe(model_class)
|
|
19
13
|
return if model_class.instance_variable_get(TRACKABLE_FLAG)
|
|
20
14
|
|
|
@@ -23,24 +17,23 @@ module ActiveRecord
|
|
|
23
17
|
end
|
|
24
18
|
|
|
25
19
|
# Invoked from the model commit callbacks; `record` is the committed instance.
|
|
26
|
-
|
|
20
|
+
# Publishes as the `:callbacks` source so views fed by another change source
|
|
21
|
+
# are not maintained twice.
|
|
27
22
|
def publish(record, operation)
|
|
28
|
-
DependencyRegistry.publish_write_change!(WriteChange.from_record(record, operation))
|
|
23
|
+
DependencyRegistry.publish_write_change!(WriteChange.from_record(record, operation), source: :callbacks)
|
|
29
24
|
end
|
|
30
25
|
|
|
31
|
-
sig { void }
|
|
32
26
|
def reset!
|
|
33
27
|
nil
|
|
34
28
|
end
|
|
35
29
|
|
|
36
30
|
private
|
|
37
31
|
|
|
38
|
-
sig { params(model_class: T.class_of(::ActiveRecord::Base)).void }
|
|
39
32
|
def install_callbacks!(model_class)
|
|
40
|
-
model =
|
|
41
|
-
model.after_create_commit { DependencyTrackable.publish(
|
|
42
|
-
model.after_update_commit { DependencyTrackable.publish(
|
|
43
|
-
model.after_destroy_commit { DependencyTrackable.publish(
|
|
33
|
+
model = model_class
|
|
34
|
+
model.after_create_commit { DependencyTrackable.publish(self, :create) }
|
|
35
|
+
model.after_update_commit { DependencyTrackable.publish(self, :update) }
|
|
36
|
+
model.after_destroy_commit { DependencyTrackable.publish(self, :destroy) }
|
|
44
37
|
end
|
|
45
38
|
end
|
|
46
39
|
end
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# typed: strict
|
|
2
1
|
# frozen_string_literal: true
|
|
3
2
|
|
|
4
3
|
module ActiveRecord
|
|
@@ -7,51 +6,69 @@ module ActiveRecord
|
|
|
7
6
|
#
|
|
8
7
|
# @api private
|
|
9
8
|
class IncrementalMaintainer
|
|
10
|
-
extend T::Sig
|
|
11
|
-
|
|
12
|
-
sig { params(view_class: ViewClass).void }
|
|
13
9
|
def initialize(view_class)
|
|
14
10
|
@view_class = view_class
|
|
15
11
|
end
|
|
16
12
|
|
|
17
|
-
sig { params(_connection: Connection, _table_name: String).returns(Integer) }
|
|
18
13
|
def maintain!(_connection, _table_name)
|
|
19
14
|
delta = maintenance_store.consume_pending_delta!
|
|
20
|
-
|
|
15
|
+
return view_class.unscoped.count if delta.nil? # another cross-process cycle consumed it — no-op
|
|
16
|
+
|
|
17
|
+
apply_delta!(delta)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
private
|
|
21
|
+
|
|
22
|
+
attr_reader :view_class
|
|
23
|
+
|
|
24
|
+
def apply_delta!(delta)
|
|
25
|
+
# A widen (full_partition) on a warm view is a full recompute — the same scan as rebuild!, so
|
|
26
|
+
# route it through the atomic build-and-swap instead of an in-place delete_all + re-insert. That
|
|
27
|
+
# keeps readers on the old snapshot until an instant rename, rather than exposing an empty/locked
|
|
28
|
+
# table mid-recompute. (A cold view can't reach here with full_partition: Refresher#maintainable?
|
|
29
|
+
# declines it, #120. So this branch is always the warm full-recompute case.)
|
|
30
|
+
return recompute_all! if delta.full_partition?
|
|
31
|
+
|
|
32
|
+
apply_scoped_delta!(delta)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def recompute_all!
|
|
36
|
+
RelationCacheWriter.new(view_class).atomic_swap!(full_source_relation)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def apply_scoped_delta!(delta)
|
|
40
|
+
partition_state = PartitionState.new(view_class)
|
|
41
|
+
cold_scoped = !view_class.materialized?
|
|
42
|
+
# Capture the fresh-set epoch BEFORE the source read, so a widen committing during the read
|
|
43
|
+
# advances the epoch and leaves this populate's mark un-served (the populate-vs-widen race, #120).
|
|
44
|
+
generation = partition_state.current_generation if cold_scoped
|
|
21
45
|
|
|
22
46
|
row_count = RelationCacheWriter.new(view_class).replace_partitions!(
|
|
23
|
-
|
|
24
|
-
key_tuples: delta.key_tuples
|
|
25
|
-
full_partition: delta.full_partition?
|
|
47
|
+
scoped_source_relation(delta.key_tuples),
|
|
48
|
+
key_tuples: delta.key_tuples
|
|
26
49
|
)
|
|
27
50
|
|
|
28
|
-
# On a cold view the maintained partitions are now fresh.
|
|
29
|
-
|
|
30
|
-
PartitionState.new(view_class).mark_fresh!(delta.key_tuples)
|
|
31
|
-
end
|
|
51
|
+
# On a cold view the maintained partitions are now fresh, stamped with the captured epoch.
|
|
52
|
+
partition_state.mark_fresh!(delta.key_tuples, generation: generation) if cold_scoped
|
|
32
53
|
|
|
33
54
|
row_count
|
|
34
55
|
end
|
|
35
56
|
|
|
36
|
-
private
|
|
37
|
-
|
|
38
|
-
sig { returns(ViewClass) }
|
|
39
|
-
attr_reader :view_class
|
|
40
|
-
|
|
41
|
-
sig { returns(MaintenanceStore) }
|
|
42
57
|
def maintenance_store
|
|
43
58
|
MaintenanceStore.new(view_class)
|
|
44
59
|
end
|
|
45
60
|
|
|
46
|
-
|
|
47
|
-
def
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
61
|
+
# The whole-view source for a full recompute (the incremental override wins when configured).
|
|
62
|
+
def full_source_relation
|
|
63
|
+
view_class.incremental_source_override? ? view_class.resolved_incremental_source : view_class.resolved_source
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# The source restricted to the affected partitions (the incremental override wins when configured;
|
|
67
|
+
# it is already the maintainable relation, so it is not re-scoped here).
|
|
68
|
+
def scoped_source_relation(key_tuples)
|
|
69
|
+
return view_class.resolved_incremental_source if view_class.incremental_source_override?
|
|
70
|
+
|
|
71
|
+
view_class.view_definition.partition_scope(key_tuples)
|
|
55
72
|
end
|
|
56
73
|
end
|
|
57
74
|
end
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActiveRecord
|
|
4
|
+
module Materialized
|
|
5
|
+
# Runs one incremental-maintenance pass for a view, choosing the summary-delta
|
|
6
|
+
# or scoped-recompute path from the pending maintenance and annotating the
|
|
7
|
+
# refresh instrumentation payload with the mode and partitions it touched.
|
|
8
|
+
#
|
|
9
|
+
# @api private
|
|
10
|
+
class IncrementalRefresh
|
|
11
|
+
def initialize(view_class, payload)
|
|
12
|
+
@view_class = view_class
|
|
13
|
+
@payload = payload
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def call
|
|
17
|
+
ensure_cache_table!
|
|
18
|
+
store = MaintenanceStore.new(@view_class)
|
|
19
|
+
pending = store.pending
|
|
20
|
+
@payload[:mode] = mode(pending)
|
|
21
|
+
@payload[:partition_count] = partition_count(pending)
|
|
22
|
+
apply!(store, pending)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
private
|
|
26
|
+
|
|
27
|
+
def apply!(store, pending)
|
|
28
|
+
if pending.is_a?(SummaryDelta)
|
|
29
|
+
# Consume+apply the additive delta under a row lock so concurrent cross-process cycles
|
|
30
|
+
# can't apply it twice; a loser finds an empty payload and no-ops. The row count is read
|
|
31
|
+
# after the lock releases (keeping the full-table COUNT out of the critical section) and
|
|
32
|
+
# reflects the cache's true total for winner and loser alike — never 0, which would
|
|
33
|
+
# clobber a populated view's row_count. The scoped path below serializes its own consume
|
|
34
|
+
# (MaintenanceStore#consume_pending_delta! takes the same row lock, #95), so a concurrent
|
|
35
|
+
# cross-process double-run recomputes a partition once rather than duplicating its rows.
|
|
36
|
+
store.with_consumed_summary_delta { |delta| DeltaMaintainer.new(@view_class).apply!(delta) }
|
|
37
|
+
return @view_class.unscoped.count
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
IncrementalMaintainer.new(@view_class).maintain!(@view_class.connection, @view_class.table_name)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def mode(pending)
|
|
44
|
+
pending.is_a?(SummaryDelta) ? :summary_delta : :scoped_recompute
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Partitions this pass recomputes; nil when it widened to a full recompute.
|
|
48
|
+
def partition_count(pending)
|
|
49
|
+
return pending.tracked_partition_count if pending.is_a?(SummaryDelta)
|
|
50
|
+
return nil unless pending.is_a?(MaintenanceDelta)
|
|
51
|
+
|
|
52
|
+
pending.full_partition? ? nil : pending.key_tuples.size
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Cheap DDL so partition maintenance has somewhere to write — never a populate.
|
|
56
|
+
def ensure_cache_table!
|
|
57
|
+
return if @view_class.table_exists?
|
|
58
|
+
|
|
59
|
+
CacheTableSchema.ensure_table!(@view_class, @view_class.resolved_source)
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActiveRecord
|
|
4
|
+
module Materialized
|
|
5
|
+
# Emits `ActiveSupport::Notifications` events at the read / refresh /
|
|
6
|
+
# maintenance lifecycle points so adopters can wire freshness and behavior
|
|
7
|
+
# SLIs to any APM / StatsD / OpenTelemetry backend without the gem taking a
|
|
8
|
+
# dependency on a specific telemetry vendor.
|
|
9
|
+
#
|
|
10
|
+
# The event names and payload keys documented here are a stable contract —
|
|
11
|
+
# subscribe with {https://api.rubyonrails.org/classes/ActiveSupport/Notifications.html
|
|
12
|
+
# ActiveSupport::Notifications}:
|
|
13
|
+
#
|
|
14
|
+
# ActiveSupport::Notifications.subscribe("read.active_record_materialized") do |event|
|
|
15
|
+
# StatsD.increment("mv.read.#{event.payload[:source]}", tags: ["view:#{event.payload[:view]}"])
|
|
16
|
+
# end
|
|
17
|
+
#
|
|
18
|
+
# Events:
|
|
19
|
+
#
|
|
20
|
+
# * +read.active_record_materialized+ — one per routed read. Payload:
|
|
21
|
+
# +:view+ (the view class), +:source+ (+:cache+, +:read_through+,
|
|
22
|
+
# +:serve_stale+, or +:raise+), +:staleness+ (seconds since the last
|
|
23
|
+
# successful refresh, or +nil+ when never refreshed).
|
|
24
|
+
# * +refresh.active_record_materialized+ — one per +refresh!+/+rebuild!+,
|
|
25
|
+
# timed. Payload: +:view+, +:operation+ (+:incremental+ or +:rebuild+),
|
|
26
|
+
# +:mode+ (+:summary_delta+, +:scoped_recompute+, or +:full+),
|
|
27
|
+
# +:partition_count+ (partitions recomputed, or +nil+ for a full pass),
|
|
28
|
+
# +:row_count+, +:skipped+, and — on failure — the standard
|
|
29
|
+
# +:exception+/+:exception_object+ keys set by +ActiveSupport::Notifications+.
|
|
30
|
+
# * +maintenance.active_record_materialized+ — one per dependency write that
|
|
31
|
+
# records pending maintenance. Payload: +:view+, +:table+, +:operation+
|
|
32
|
+
# (+:create+/+:update+/+:destroy+), +:path+ (+:summary_delta+ or
|
|
33
|
+
# +:scoped_recompute+), +:scope+ (+:scoped+, or +:full+ when the write
|
|
34
|
+
# widened to a full recompute because its partition key could not be
|
|
35
|
+
# derived), +:partition_count+ (distinct partitions scoped, 0 on widen).
|
|
36
|
+
# * +reconcile.active_record_materialized+ — one per {Reconciler} run. Payload:
|
|
37
|
+
# +:view+, +:mode+ (the drift-check depth), +:repaired_partition_count+
|
|
38
|
+
# (partitions found divergent and repaired with scoped maintenance), +:deferred+
|
|
39
|
+
# (+true+ when a concurrent refresh deferred the run to the next tick).
|
|
40
|
+
module Instrumentation
|
|
41
|
+
READ = "read.active_record_materialized"
|
|
42
|
+
REFRESH = "refresh.active_record_materialized"
|
|
43
|
+
MAINTENANCE = "maintenance.active_record_materialized"
|
|
44
|
+
RECONCILE = "reconcile.active_record_materialized"
|
|
45
|
+
|
|
46
|
+
class << self
|
|
47
|
+
# Fired once per routed read. The staleness lookup costs a metadata read,
|
|
48
|
+
# so it is skipped entirely unless a subscriber is attached.
|
|
49
|
+
def read(view_class, source:)
|
|
50
|
+
return unless ::ActiveSupport::Notifications.notifier.listening?(READ)
|
|
51
|
+
|
|
52
|
+
::ActiveSupport::Notifications.instrument(
|
|
53
|
+
READ,
|
|
54
|
+
view: view_class, source: source, staleness: staleness_for(view_class)
|
|
55
|
+
)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Wraps a refresh/rebuild so the event carries its wall-clock duration and
|
|
59
|
+
# any raised exception. The block runs the refresh and returns its
|
|
60
|
+
# {RefreshResult}; the row count and skipped flag are read back from it,
|
|
61
|
+
# while the block annotates the payload with the +:mode+ and
|
|
62
|
+
# +:partition_count+ it can only know mid-flight.
|
|
63
|
+
def refresh(view_class, operation:, &block)
|
|
64
|
+
::ActiveSupport::Notifications.instrument(REFRESH, view: view_class, operation: operation) do |payload|
|
|
65
|
+
result = yield(payload)
|
|
66
|
+
payload[:row_count] = result.row_count
|
|
67
|
+
payload[:skipped] = result.skipped
|
|
68
|
+
result
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Fired once per dependency write that records pending maintenance.
|
|
73
|
+
def maintenance(view_class, change:, path:, scope:, partition_count:)
|
|
74
|
+
::ActiveSupport::Notifications.instrument(
|
|
75
|
+
MAINTENANCE,
|
|
76
|
+
view: view_class, table: change.table_name, operation: change.operation.to_sym,
|
|
77
|
+
path: path, scope: scope, partition_count: partition_count
|
|
78
|
+
)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# Fired once per {Reconciler} run — after it verifies and (if needed) repairs
|
|
82
|
+
# via scoped maintenance — carrying the run's mode, repaired count, and defer flag.
|
|
83
|
+
def reconcile(view_class, mode:, repaired_partition_count:, deferred:)
|
|
84
|
+
::ActiveSupport::Notifications.instrument(
|
|
85
|
+
RECONCILE,
|
|
86
|
+
view: view_class, mode: mode,
|
|
87
|
+
repaired_partition_count: repaired_partition_count, deferred: deferred
|
|
88
|
+
)
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
private
|
|
92
|
+
|
|
93
|
+
def staleness_for(view_class)
|
|
94
|
+
last_refreshed_at = view_class.metadata.last_refreshed_at
|
|
95
|
+
return nil if last_refreshed_at.nil?
|
|
96
|
+
|
|
97
|
+
(Metadata::Timestamps.current.to_time - last_refreshed_at.to_time).to_f
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
end
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# typed: strict
|
|
2
1
|
# frozen_string_literal: true
|
|
3
2
|
|
|
4
3
|
module ActiveRecord
|
|
@@ -7,46 +6,34 @@ module ActiveRecord
|
|
|
7
6
|
#
|
|
8
7
|
# @api private
|
|
9
8
|
class MaintenanceDelta
|
|
10
|
-
|
|
9
|
+
SCOPED = :scoped
|
|
10
|
+
FULL = :full_partition
|
|
11
11
|
|
|
12
|
-
|
|
13
|
-
FULL = T.let(:full_partition, Symbol)
|
|
12
|
+
attr_reader :scope, :key_tuples
|
|
14
13
|
|
|
15
|
-
sig { returns(Symbol) }
|
|
16
|
-
attr_reader :scope
|
|
17
|
-
|
|
18
|
-
sig { returns(T::Array[T::Array[String]]) }
|
|
19
|
-
attr_reader :key_tuples
|
|
20
|
-
|
|
21
|
-
sig { params(scope: Symbol, key_tuples: T::Array[T::Array[String]]).void }
|
|
22
14
|
def initialize(scope:, key_tuples: [])
|
|
23
15
|
@scope = scope
|
|
24
16
|
@key_tuples = key_tuples
|
|
25
17
|
end
|
|
26
18
|
|
|
27
|
-
sig { params(key_tuples: T::Array[T::Array[String]]).returns(MaintenanceDelta) }
|
|
28
19
|
def self.scoped(key_tuples)
|
|
29
20
|
new(scope: SCOPED, key_tuples: key_tuples)
|
|
30
21
|
end
|
|
31
22
|
|
|
32
|
-
sig { returns(MaintenanceDelta) }
|
|
33
23
|
def self.full_partition
|
|
34
24
|
new(scope: FULL)
|
|
35
25
|
end
|
|
36
26
|
|
|
37
|
-
sig { returns(T::Boolean) }
|
|
38
27
|
def full_partition?
|
|
39
28
|
scope == FULL
|
|
40
29
|
end
|
|
41
30
|
|
|
42
31
|
# How many distinct partitions this pending maintenance tracks. A
|
|
43
32
|
# full-partition recompute tracks none — it is already the collapsed form.
|
|
44
|
-
sig { returns(Integer) }
|
|
45
33
|
def tracked_partition_count
|
|
46
34
|
full_partition? ? 0 : key_tuples.size
|
|
47
35
|
end
|
|
48
36
|
|
|
49
|
-
sig { params(other: MaintenanceDelta).returns(MaintenanceDelta) }
|
|
50
37
|
def merge(other)
|
|
51
38
|
return other if other.full_partition?
|
|
52
39
|
return self if full_partition?
|
|
@@ -55,7 +42,6 @@ module ActiveRecord
|
|
|
55
42
|
self.class.scoped(combined)
|
|
56
43
|
end
|
|
57
44
|
|
|
58
|
-
sig { returns(T::Hash[String, T.untyped]) }
|
|
59
45
|
def serialize
|
|
60
46
|
{
|
|
61
47
|
"scope" => scope.to_s,
|
|
@@ -63,7 +49,6 @@ module ActiveRecord
|
|
|
63
49
|
}
|
|
64
50
|
end
|
|
65
51
|
|
|
66
|
-
sig { params(payload: T.nilable(T::Hash[String, T.untyped])).returns(T.nilable(MaintenanceDelta)) }
|
|
67
52
|
def self.deserialize(payload)
|
|
68
53
|
return nil if payload.blank?
|
|
69
54
|
|
|
@@ -73,7 +58,7 @@ module ActiveRecord
|
|
|
73
58
|
if scope_name == FULL
|
|
74
59
|
full_partition
|
|
75
60
|
else
|
|
76
|
-
tuples =
|
|
61
|
+
tuples = payload["key_tuples"] || []
|
|
77
62
|
scoped(tuples)
|
|
78
63
|
end
|
|
79
64
|
end
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# typed: strict
|
|
2
1
|
# frozen_string_literal: true
|
|
3
2
|
|
|
4
3
|
module ActiveRecord
|
|
@@ -7,25 +6,25 @@ module ActiveRecord
|
|
|
7
6
|
#
|
|
8
7
|
# @api private
|
|
9
8
|
class MaintenanceDeltaBuilder
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
sig { params(change: WriteChange, key_columns: T::Array[String]).void }
|
|
13
|
-
def initialize(change, key_columns)
|
|
9
|
+
def initialize(change, key_columns, resolver: nil)
|
|
14
10
|
@change = change
|
|
15
11
|
@key_columns = key_columns
|
|
12
|
+
@resolver = resolver
|
|
16
13
|
end
|
|
17
14
|
|
|
18
|
-
sig { returns(MaintenanceDelta) }
|
|
19
15
|
def build
|
|
20
16
|
return MaintenanceDelta.full_partition if @key_columns.empty?
|
|
21
17
|
|
|
22
|
-
|
|
18
|
+
# A resolver is an explicit per-table mapping, so it is authoritative for
|
|
19
|
+
# that table's writes (it maps to the group key wherever it lives, even if
|
|
20
|
+
# the written row happens to carry a same-named column); otherwise the key
|
|
21
|
+
# comes from the written row's own payload.
|
|
22
|
+
tuples = @resolver ? resolved_tuples : extract_tuples
|
|
23
23
|
tuples.empty? ? MaintenanceDelta.full_partition : MaintenanceDelta.scoped(tuples.uniq)
|
|
24
24
|
end
|
|
25
25
|
|
|
26
26
|
private
|
|
27
27
|
|
|
28
|
-
sig { returns(T::Array[T::Array[T.untyped]]) }
|
|
29
28
|
def extract_tuples
|
|
30
29
|
snapshots = case @change.operation.to_sym
|
|
31
30
|
when :create then [@change.after]
|
|
@@ -33,29 +32,62 @@ module ActiveRecord
|
|
|
33
32
|
when :update then [@change.before, @change.after]
|
|
34
33
|
else []
|
|
35
34
|
end
|
|
36
|
-
snapshots.
|
|
35
|
+
tuples = snapshots.map { |attributes| key_tuple(attributes) }
|
|
36
|
+
# An update touches two partitions (old + new), a create/destroy one. If ANY affected snapshot
|
|
37
|
+
# can't yield its partition key — e.g. a non-FULL CDC before- or after-image with only the
|
|
38
|
+
# primary key — we can't identify every affected partition (a partial after-image can't even
|
|
39
|
+
# rule out a move), so return no tuples and let +build+ widen to a full recompute rather than
|
|
40
|
+
# under-scope to only the derivable side (leaving the other partition stale). +build+
|
|
41
|
+
# de-duplicates the tuples it scopes, so no +uniq+ here.
|
|
42
|
+
return [] if tuples.any?(&:nil?)
|
|
43
|
+
|
|
44
|
+
tuples
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Partition key tuple(s) from the configured resolver, or [] (=> full
|
|
48
|
+
# recompute) when none is set or it yields nothing.
|
|
49
|
+
def resolved_tuples
|
|
50
|
+
resolver = @resolver
|
|
51
|
+
return [] if resolver.nil?
|
|
52
|
+
|
|
53
|
+
normalize_keys(resolver.call(@change))
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Coerces a resolver's return into partition-key tuples, using the group key's
|
|
57
|
+
# arity to tell a single composite tuple from a list of tuples. A nil/empty
|
|
58
|
+
# return widens (=> full recompute); a returned nil value is a real key (the
|
|
59
|
+
# NULL partition), so it is kept rather than dropped.
|
|
60
|
+
def normalize_keys(result)
|
|
61
|
+
return [] if result.nil?
|
|
62
|
+
|
|
63
|
+
@key_columns.one? ? Array(result).zip : composite_tuples(result)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# A composite key expects a tuple or an array of tuples; anything else (a bare
|
|
67
|
+
# scalar, nil, empty) can't be scoped, so it widens rather than crashing.
|
|
68
|
+
def composite_tuples(result)
|
|
69
|
+
return [] unless result.is_a?(Array) && result.any?
|
|
70
|
+
|
|
71
|
+
result.first.is_a?(Array) ? result : [result]
|
|
37
72
|
end
|
|
38
73
|
|
|
39
|
-
sig { params(attributes: T::Hash[String, T.untyped]).returns(T::Boolean) }
|
|
40
74
|
def keys_present?(attributes)
|
|
41
75
|
@key_columns.all? do |column|
|
|
42
|
-
attributes.key?(column) ||
|
|
76
|
+
attributes.key?(column) || attributes.key?(column.to_sym)
|
|
43
77
|
end
|
|
44
78
|
end
|
|
45
79
|
|
|
46
|
-
sig { params(attributes: T::Hash[String, T.untyped]).returns(T.nilable(T::Array[T.untyped])) }
|
|
47
80
|
def key_tuple(attributes)
|
|
48
81
|
return nil unless keys_present?(attributes)
|
|
49
82
|
|
|
50
83
|
@key_columns.map { |column| attribute_value(attributes, column) }
|
|
51
84
|
end
|
|
52
85
|
|
|
53
|
-
sig { params(attributes: T::Hash[String, T.untyped], column: String).returns(T.untyped) }
|
|
54
86
|
def attribute_value(attributes, column)
|
|
55
87
|
# Look up by presence so falsey group-key values map to their partition.
|
|
56
88
|
return attributes[column] if attributes.key?(column)
|
|
57
89
|
|
|
58
|
-
|
|
90
|
+
attributes[column.to_sym]
|
|
59
91
|
end
|
|
60
92
|
end
|
|
61
93
|
end
|