activerecord-materialized 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +93 -0
- data/README.md +139 -324
- data/lib/activerecord/materialized/aggregate_analysis.rb +10 -31
- data/lib/activerecord/materialized/async_refresher.rb +18 -27
- data/lib/activerecord/materialized/cache_table_schema.rb +53 -34
- data/lib/activerecord/materialized/change_source.rb +25 -0
- data/lib/activerecord/materialized/cold_read.rb +1 -10
- data/lib/activerecord/materialized/column_type_inference.rb +167 -0
- data/lib/activerecord/materialized/configurable_table_name.rb +45 -0
- data/lib/activerecord/materialized/configuration.rb +104 -29
- data/lib/activerecord/materialized/connection_routing.rb +29 -0
- data/lib/activerecord/materialized/data_verification_result.rb +41 -0
- data/lib/activerecord/materialized/data_verifier.rb +162 -0
- data/lib/activerecord/materialized/debezium_envelope.rb +105 -0
- data/lib/activerecord/materialized/delta_maintainer.rb +4 -15
- data/lib/activerecord/materialized/dependency_registry.rb +43 -35
- data/lib/activerecord/materialized/dependency_trackable.rb +7 -14
- data/lib/activerecord/materialized/incremental_maintainer.rb +45 -28
- data/lib/activerecord/materialized/incremental_refresh.rb +63 -0
- data/lib/activerecord/materialized/instrumentation.rb +102 -0
- data/lib/activerecord/materialized/maintenance_delta.rb +4 -19
- data/lib/activerecord/materialized/maintenance_delta_builder.rb +46 -14
- data/lib/activerecord/materialized/maintenance_store.rb +58 -21
- data/lib/activerecord/materialized/metadata/maintenance_payload.rb +1 -7
- data/lib/activerecord/materialized/metadata/reconciliation.rb +25 -0
- data/lib/activerecord/materialized/metadata/schema.rb +19 -41
- data/lib/activerecord/materialized/metadata/timestamps.rb +0 -5
- data/lib/activerecord/materialized/metadata.rb +41 -25
- data/lib/activerecord/materialized/metadata_record.rb +2 -16
- data/lib/activerecord/materialized/migration_builder.rb +8 -8
- data/lib/activerecord/materialized/module_api.rb +219 -22
- data/lib/activerecord/materialized/partition_filter.rb +45 -0
- data/lib/activerecord/materialized/partition_keyed_store.rb +72 -0
- data/lib/activerecord/materialized/partition_record.rb +2 -16
- data/lib/activerecord/materialized/partition_snapshot.rb +79 -0
- data/lib/activerecord/materialized/partition_state.rb +47 -46
- data/lib/activerecord/materialized/query_expressions.rb +0 -12
- data/lib/activerecord/materialized/railtie.rb +15 -3
- data/lib/activerecord/materialized/read_router.rb +51 -0
- data/lib/activerecord/materialized/reconcile_job.rb +22 -0
- data/lib/activerecord/materialized/reconcile_result.rb +36 -0
- data/lib/activerecord/materialized/reconciler.rb +79 -0
- data/lib/activerecord/materialized/refresh_callbacks.rb +4 -18
- data/lib/activerecord/materialized/refresh_job.rb +0 -4
- data/lib/activerecord/materialized/refresh_result.rb +2 -10
- data/lib/activerecord/materialized/refresh_scheduler.rb +5 -7
- data/lib/activerecord/materialized/refresher.rb +35 -50
- data/lib/activerecord/materialized/registry.rb +26 -16
- data/lib/activerecord/materialized/relation_cache_writer.rb +12 -46
- data/lib/activerecord/materialized/schema_verifier.rb +0 -9
- data/lib/activerecord/materialized/source_watermark.rb +93 -0
- data/lib/activerecord/materialized/source_watermark_record.rb +16 -0
- data/lib/activerecord/materialized/summary_delta.rb +1 -17
- data/lib/activerecord/materialized/summary_delta_builder.rb +2 -10
- data/lib/activerecord/materialized/table_model_registry.rb +1 -8
- data/lib/activerecord/materialized/table_swap.rb +106 -0
- data/lib/activerecord/materialized/tasks.rb +46 -38
- data/lib/activerecord/materialized/version.rb +1 -2
- data/lib/activerecord/materialized/view.rb +11 -22
- data/lib/activerecord/materialized/view_configuration_class_methods.rb +23 -35
- data/lib/activerecord/materialized/view_definition.rb +76 -65
- data/lib/activerecord/materialized/view_incremental_class_methods.rb +57 -60
- data/lib/activerecord/materialized/view_loader.rb +49 -0
- data/lib/activerecord/materialized/view_query_access_class_methods.rb +100 -57
- data/lib/activerecord/materialized/view_refresh_policy_class_methods.rb +70 -37
- data/lib/activerecord/materialized/write_change.rb +75 -26
- data/lib/activerecord/materialized/write_maintenance.rb +73 -0
- data/lib/activerecord/materialized/write_outbox.rb +345 -0
- data/lib/activerecord/materialized/write_outbox_record.rb +17 -0
- data/lib/activerecord/materialized.rb +27 -6
- data/lib/generators/activerecord_materialized/install/templates/create_ar_materialized_view_metadata.rb.erb +14 -0
- data/lib/generators/activerecord_materialized/install_generator.rb +1 -6
- data/lib/generators/activerecord_materialized/migration_generator.rb +3 -9
- data/lib/generators/activerecord_materialized/outbox_generator.rb +36 -0
- data/lib/generators/activerecord_materialized/templates/materialized_view_migration.rb.erb +5 -1
- data/lib/generators/activerecord_materialized/templates/write_outbox_migration.rb.erb +21 -0
- data/lib/generators/activerecord_materialized/view_generator.rb +0 -4
- metadata +28 -19
- data/lib/activerecord/materialized/type_reexports.rb +0 -14
- data/lib/activerecord/materialized/view_class.rb +0 -8
- data/lib/activerecord_materialized_types.rb +0 -18
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# typed: strict
|
|
2
1
|
# frozen_string_literal: true
|
|
3
2
|
|
|
4
3
|
module ActiveRecord
|
|
@@ -9,64 +8,75 @@ module ActiveRecord
|
|
|
9
8
|
# @api private
|
|
10
9
|
class Registry
|
|
11
10
|
class << self
|
|
12
|
-
extend T::Sig
|
|
13
|
-
|
|
14
|
-
sig { params(view_class: ViewClass).void }
|
|
15
11
|
def register(view_class)
|
|
16
12
|
views[view_class.view_key] = view_class
|
|
17
13
|
end
|
|
18
14
|
|
|
19
|
-
sig { params(view_class: ViewClass).void }
|
|
20
15
|
def unregister(view_class)
|
|
21
16
|
views.delete(view_class.view_key)
|
|
22
17
|
end
|
|
23
18
|
|
|
24
|
-
sig { params(key: T.any(String, Symbol)).returns(T.nilable(ViewClass)) }
|
|
25
19
|
def find(key)
|
|
26
20
|
views[key.to_s]
|
|
27
21
|
end
|
|
28
22
|
|
|
29
|
-
sig { params(class_name: String).returns(T.nilable(ViewClass)) }
|
|
30
23
|
def for_class_name(class_name)
|
|
31
24
|
all.find { |view| view.name == class_name }
|
|
32
25
|
end
|
|
33
26
|
|
|
34
|
-
sig { returns(T::Array[ViewClass]) }
|
|
35
27
|
def all
|
|
36
28
|
views.values
|
|
37
29
|
end
|
|
38
30
|
|
|
39
31
|
# Incremental pass over every registered view; rebuild_all! for a full one.
|
|
40
|
-
sig { returns(T::Array[RefreshResult]) }
|
|
41
32
|
def refresh_all!
|
|
42
33
|
all.map(&:refresh!)
|
|
43
34
|
end
|
|
44
35
|
|
|
45
|
-
|
|
36
|
+
# Registered views that need maintenance — dirty, never refreshed, or past max_staleness.
|
|
37
|
+
# The single stale-view selection shared by the periodic refresh/reconcile paths, their
|
|
38
|
+
# rake tasks, and the job fan-out (ActiveRecord::Materialized.enqueue_stale_*!).
|
|
39
|
+
def stale_views
|
|
40
|
+
all.select(&:stale?)
|
|
41
|
+
end
|
|
42
|
+
|
|
46
43
|
def refresh_stale!
|
|
47
|
-
|
|
44
|
+
stale_views.map(&:refresh!)
|
|
48
45
|
end
|
|
49
46
|
|
|
50
|
-
sig { returns(T::Array[RefreshResult]) }
|
|
51
47
|
def rebuild_all!
|
|
52
48
|
all.map { |view| view.rebuild!(confirm: true) }
|
|
53
49
|
end
|
|
54
50
|
|
|
55
|
-
|
|
51
|
+
def reconcile_all!(mode: :checksum, sample: nil)
|
|
52
|
+
all.map { |view| reconcile_view(view, mode: mode, sample: sample) }
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def reconcile_stale!(mode: :checksum, sample: nil)
|
|
56
|
+
stale_views.map { |view| reconcile_view(view, mode: mode, sample: sample) }
|
|
57
|
+
end
|
|
58
|
+
|
|
56
59
|
def warm_up_all!
|
|
57
60
|
all.map(&:warm_up!)
|
|
58
61
|
end
|
|
59
62
|
|
|
60
|
-
sig { void }
|
|
61
63
|
def reset!
|
|
62
64
|
@views = {}
|
|
63
65
|
end
|
|
64
66
|
|
|
65
67
|
private
|
|
66
68
|
|
|
67
|
-
|
|
69
|
+
# Reconcile one view, isolating a failure so a single unhealthy view (e.g. a
|
|
70
|
+
# transiently broken source relation) can't abort the scheduled backstop for
|
|
71
|
+
# the rest of the fleet; the error is reported on the result, not swallowed.
|
|
72
|
+
def reconcile_view(view, mode:, sample:)
|
|
73
|
+
view.reconcile!(mode: mode, sample: sample)
|
|
74
|
+
rescue StandardError => e
|
|
75
|
+
ReconcileResult.new(view_name: view.view_key, mode: mode, repaired_keys: [], error: e.message)
|
|
76
|
+
end
|
|
77
|
+
|
|
68
78
|
def views
|
|
69
|
-
@views ||=
|
|
79
|
+
@views ||= {}
|
|
70
80
|
end
|
|
71
81
|
end
|
|
72
82
|
end
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# typed: strict
|
|
2
1
|
# frozen_string_literal: true
|
|
3
2
|
|
|
4
3
|
module ActiveRecord
|
|
@@ -7,20 +6,15 @@ module ActiveRecord
|
|
|
7
6
|
#
|
|
8
7
|
# @api private
|
|
9
8
|
class RelationCacheWriter
|
|
10
|
-
extend T::Sig
|
|
11
|
-
|
|
12
|
-
sig { params(view_class: T.class_of(::ActiveRecord::Base)).void }
|
|
13
9
|
def initialize(view_class)
|
|
14
10
|
@view_class = view_class
|
|
15
11
|
end
|
|
16
12
|
|
|
17
|
-
sig { params(relation: ::ActiveRecord::Relation).returns(Integer) }
|
|
18
13
|
def bootstrap!(relation)
|
|
19
14
|
CacheTableSchema.ensure_table!(view_class, relation)
|
|
20
15
|
replace_all!(relation)
|
|
21
16
|
end
|
|
22
17
|
|
|
23
|
-
sig { params(relation: ::ActiveRecord::Relation).returns(Integer) }
|
|
24
18
|
def replace_all!(relation)
|
|
25
19
|
view_class.transaction do
|
|
26
20
|
view_class.delete_all
|
|
@@ -29,107 +23,79 @@ module ActiveRecord
|
|
|
29
23
|
cache_row_count
|
|
30
24
|
end
|
|
31
25
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
full_partition: T::Boolean
|
|
37
|
-
).returns(Integer)
|
|
38
|
-
end
|
|
39
|
-
def replace_partitions!(relation, key_tuples:, full_partition:)
|
|
26
|
+
# Scoped in-place maintenance: delete + re-insert only the affected partitions. A full recompute
|
|
27
|
+
# goes through +atomic_swap!+ instead (see IncrementalMaintainer#recompute_all!), so this only
|
|
28
|
+
# ever handles a bounded set of partition keys.
|
|
29
|
+
def replace_partitions!(relation, key_tuples:)
|
|
40
30
|
view_class.transaction do
|
|
41
|
-
|
|
42
|
-
view_class.delete_all
|
|
43
|
-
else
|
|
44
|
-
delete_partitions!(key_tuples)
|
|
45
|
-
end
|
|
31
|
+
delete_partitions!(key_tuples)
|
|
46
32
|
insert_rows!(relation)
|
|
47
33
|
end
|
|
48
34
|
cache_row_count
|
|
49
35
|
end
|
|
50
36
|
|
|
51
|
-
sig { params(relation: ::ActiveRecord::Relation).returns(Integer) }
|
|
52
37
|
def atomic_swap!(relation)
|
|
53
|
-
connection = view_class.connection
|
|
54
38
|
temp_table = refresh_temp_table_name
|
|
55
39
|
old_table = refresh_old_table_name
|
|
56
40
|
|
|
57
41
|
populate_temp_table!(temp_table, relation)
|
|
58
|
-
|
|
42
|
+
TableSwap.new(view_class).swap!(temp_table, old_table)
|
|
59
43
|
|
|
60
44
|
view_class.reset_column_information
|
|
61
45
|
cache_row_count
|
|
62
46
|
end
|
|
63
47
|
|
|
64
|
-
sig { params(temp_table: String, relation: ::ActiveRecord::Relation).void }
|
|
65
48
|
def populate_temp_table!(temp_table, relation)
|
|
66
|
-
CacheTableSchema.create_table!(
|
|
49
|
+
CacheTableSchema.create_table!(view_class, temp_table, relation)
|
|
67
50
|
temp_model = temporary_model(temp_table)
|
|
68
51
|
self.class.new(temp_model).replace_all!(relation)
|
|
69
52
|
end
|
|
70
53
|
|
|
71
|
-
sig { params(connection: Connection, temp_table: String, old_table: String).void }
|
|
72
|
-
def swap_tables!(connection, temp_table, old_table)
|
|
73
|
-
connection.transaction do
|
|
74
|
-
connection.rename_table(view_class.table_name, old_table) if view_class.table_exists?
|
|
75
|
-
connection.rename_table(temp_table, view_class.table_name)
|
|
76
|
-
connection.drop_table(old_table, if_exists: true)
|
|
77
|
-
end
|
|
78
|
-
end
|
|
79
|
-
|
|
80
|
-
sig { returns(String) }
|
|
81
54
|
def refresh_temp_table_name
|
|
82
55
|
"#{view_class.table_name}_refresh_#{SecureRandom.hex(4)}"
|
|
83
56
|
end
|
|
84
57
|
|
|
85
|
-
sig { returns(String) }
|
|
86
58
|
def refresh_old_table_name
|
|
87
59
|
"#{view_class.table_name}_old_#{SecureRandom.hex(4)}"
|
|
88
60
|
end
|
|
89
61
|
|
|
90
62
|
private
|
|
91
63
|
|
|
92
|
-
sig { returns(T.class_of(::ActiveRecord::Base)) }
|
|
93
64
|
attr_reader :view_class
|
|
94
65
|
|
|
95
66
|
# Count straight from the cache table, bypassing the View read routing
|
|
96
67
|
# (during a rebuild the view is not warm yet, so a routed count would read
|
|
97
68
|
# through to the source).
|
|
98
|
-
sig { returns(Integer) }
|
|
99
69
|
def cache_row_count
|
|
100
|
-
|
|
70
|
+
view_class.unscoped.count
|
|
101
71
|
end
|
|
102
72
|
|
|
103
|
-
sig { params(key_tuples: T::Array[T::Array[String]]).void }
|
|
104
73
|
def delete_partitions!(key_tuples)
|
|
105
|
-
materialized_view =
|
|
74
|
+
materialized_view = view_class
|
|
106
75
|
materialized_view.view_definition.partition_scope_on(view_class, key_tuples).delete_all
|
|
107
76
|
end
|
|
108
77
|
|
|
109
78
|
# INSERT ... SELECT entirely in the database; the result set never crosses
|
|
110
79
|
# into Ruby. Cache columns share the relation's projection order, so the
|
|
111
80
|
# SELECT list maps onto them positionally.
|
|
112
|
-
sig { params(relation: ::ActiveRecord::Relation).void }
|
|
113
81
|
def insert_rows!(relation)
|
|
114
82
|
columns = view_class.column_names - ["id"]
|
|
115
83
|
return if columns.empty?
|
|
116
84
|
|
|
117
|
-
|
|
85
|
+
view_class.connection.execute(insert_select_sql(relation, columns))
|
|
118
86
|
end
|
|
119
87
|
|
|
120
|
-
sig { params(relation: ::ActiveRecord::Relation, columns: T::Array[String]).returns(String) }
|
|
121
88
|
def insert_select_sql(relation, columns)
|
|
122
89
|
manager = Arel::InsertManager.new
|
|
123
90
|
manager.into(view_class.arel_table)
|
|
124
|
-
|
|
91
|
+
manager.columns.concat(columns.map { |name| view_class.arel_table[name] })
|
|
125
92
|
manager.select(Arel.sql(relation.to_sql))
|
|
126
93
|
manager.to_sql
|
|
127
94
|
end
|
|
128
95
|
|
|
129
|
-
sig { params(table_name: String).returns(T.class_of(::ActiveRecord::Base)) }
|
|
130
96
|
def temporary_model(table_name)
|
|
131
97
|
klass = Class.new(::ActiveRecord::Base)
|
|
132
|
-
|
|
98
|
+
klass.table_name = table_name
|
|
133
99
|
klass
|
|
134
100
|
end
|
|
135
101
|
end
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# typed: strict
|
|
2
1
|
# frozen_string_literal: true
|
|
3
2
|
|
|
4
3
|
module ActiveRecord
|
|
@@ -6,18 +5,14 @@ module ActiveRecord
|
|
|
6
5
|
# Raises when a provisioned cache table no longer matches the columns its
|
|
7
6
|
# source relation projects (drift); never alters the table or rebuilds data.
|
|
8
7
|
class SchemaVerifier
|
|
9
|
-
extend T::Sig
|
|
10
|
-
|
|
11
8
|
# Raised when a cache table no longer matches the columns its source relation projects.
|
|
12
9
|
class SchemaDriftError < StandardError; end
|
|
13
10
|
|
|
14
|
-
sig { params(view_class: ViewClass).void }
|
|
15
11
|
def initialize(view_class)
|
|
16
12
|
@view_class = view_class
|
|
17
13
|
end
|
|
18
14
|
|
|
19
15
|
# An unprovisioned cache table is absent, not drifted, so this is a no-op.
|
|
20
|
-
sig { void }
|
|
21
16
|
def verify!
|
|
22
17
|
return unless @view_class.table_exists?
|
|
23
18
|
|
|
@@ -28,7 +23,6 @@ module ActiveRecord
|
|
|
28
23
|
Kernel.raise SchemaDriftError, drift_message(missing, extra)
|
|
29
24
|
end
|
|
30
25
|
|
|
31
|
-
sig { returns(T::Boolean) }
|
|
32
26
|
def drifted?
|
|
33
27
|
verify!
|
|
34
28
|
false
|
|
@@ -38,19 +32,16 @@ module ActiveRecord
|
|
|
38
32
|
|
|
39
33
|
private
|
|
40
34
|
|
|
41
|
-
sig { returns(T::Array[String]) }
|
|
42
35
|
def expected_columns
|
|
43
36
|
CacheTableSchema
|
|
44
37
|
.column_definitions(@view_class.connection, @view_class.resolved_source)
|
|
45
38
|
.map(&:name).sort
|
|
46
39
|
end
|
|
47
40
|
|
|
48
|
-
sig { returns(T::Array[String]) }
|
|
49
41
|
def actual_columns
|
|
50
42
|
@view_class.connection.columns(@view_class.table_name).map(&:name).reject { |name| name == "id" }.sort
|
|
51
43
|
end
|
|
52
44
|
|
|
53
|
-
sig { params(missing: T::Array[String], extra: T::Array[String]).returns(String) }
|
|
54
45
|
def drift_message(missing, extra)
|
|
55
46
|
details = []
|
|
56
47
|
details << "missing columns: #{missing.join(', ')}" if missing.any?
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActiveRecord
|
|
4
|
+
module Materialized
|
|
5
|
+
# Per-partition CDC source watermarks: the max applied +source_ts+ for each (view, partition).
|
|
6
|
+
# Two jobs, both opt-in via a +source_ts+ on {Materialized.ingest_change}:
|
|
7
|
+
#
|
|
8
|
+
# - **Suppression** — a redelivered or out-of-order change whose watermark is `<` the partition's
|
|
9
|
+
# applied watermark is provably redundant (the cache already reflects a strictly-newer state), so
|
|
10
|
+
# its recompute is skipped. A change at the *same* watermark still applies: a coarse (e.g.
|
|
11
|
+
# second-granular) +source_ts+ such as Debezium's MySQL +source.ts_ms+ can tie two distinct
|
|
12
|
+
# commits, so an equal value is not treated as a redelivery. Best-effort and **reconcile-backed**:
|
|
13
|
+
# the watermark advances at ingest, so a crash before the recompute can over-suppress a partition
|
|
14
|
+
# until reconciliation (#64) heals it — an optimization over always-recompute, never a correctness
|
|
15
|
+
# substitute.
|
|
16
|
+
# - **Observability** — {#oldest} reports the view's most-behind partition watermark, so a caller can
|
|
17
|
+
# compute freshness/lag against its own source clock (in the same unit as +source_ts+).
|
|
18
|
+
#
|
|
19
|
+
# Only the scoped-recompute path (a +change_source :none+ CDC-fed view) carries a watermark; a full
|
|
20
|
+
# recompute (no partition key) is never suppressed. +source_ts+ must be monotonic (non-decreasing)
|
|
21
|
+
# per partition (e.g. a Debezium +source.ts_ms+ or a per-key Kafka offset); ties are safe (only a
|
|
22
|
+
# strictly-older value is suppressed). Mirrors {PartitionState}'s per-partition store.
|
|
23
|
+
#
|
|
24
|
+
# @api private
|
|
25
|
+
class SourceWatermark < PartitionKeyedStore
|
|
26
|
+
# Drop the delta's partitions that already applied a strictly-newer +source_ts+, and advance the
|
|
27
|
+
# watermark for the survivors — a change at the same +source_ts+ still applies, so a distinct write
|
|
28
|
+
# sharing a coarse (e.g. second-granular) timestamp is never suppressed. A full-partition delta
|
|
29
|
+
# (no key) is returned unchanged — it can't be scoped.
|
|
30
|
+
#
|
|
31
|
+
# @return [MaintenanceDelta, nil] the surviving partitions, or nil when every one was suppressed
|
|
32
|
+
def suppress(source_ts, delta)
|
|
33
|
+
return delta if delta.full_partition?
|
|
34
|
+
|
|
35
|
+
ensure_table!
|
|
36
|
+
current = current_watermarks(delta.key_tuples) # one read for the whole decision
|
|
37
|
+
fresh = delta.key_tuples.reject { |tuple| (ts = current[serialize(tuple)]) && ts > source_ts }
|
|
38
|
+
return nil if fresh.empty?
|
|
39
|
+
|
|
40
|
+
advance!(fresh, source_ts)
|
|
41
|
+
MaintenanceDelta.scoped(fresh)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# The oldest applied watermark across the view's partitions that have received a watermarked
|
|
45
|
+
# change (its most-behind such partition), or nil when none have. Subtract from the source clock
|
|
46
|
+
# (same unit as +source_ts+) for lag. A partition maintained only via widen-to-full changes records
|
|
47
|
+
# no watermark and so is not reflected here; a stalled *stream* (no events arriving) is a
|
|
48
|
+
# pipeline-level concern, monitored at the consumer. A read only — never provisions the table.
|
|
49
|
+
#
|
|
50
|
+
# @return [Integer, nil]
|
|
51
|
+
def oldest
|
|
52
|
+
return nil unless record_class.table_exists?
|
|
53
|
+
|
|
54
|
+
scope.minimum(:source_ts)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
private
|
|
58
|
+
|
|
59
|
+
# The stored watermark for each of +tuples+, in one query, as { serialized_key => source_ts }.
|
|
60
|
+
def current_watermarks(tuples)
|
|
61
|
+
scope.where(partition_key: tuples.map { |tuple| serialize(tuple) }).pluck(:partition_key, :source_ts).to_h
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# Advance each partition's watermark to source_ts. +create_or_find_by+ absorbs a concurrent
|
|
65
|
+
# first-insert (the unique index would otherwise raise RecordNotUnique); the guarded +update!+ only
|
|
66
|
+
# moves a watermark forward. Advancement is best-effort (like the feature): a concurrent advance
|
|
67
|
+
# read as stale could briefly regress a watermark, at worst causing a later redundant recompute
|
|
68
|
+
# (never wrong data) — reconciliation is the backstop.
|
|
69
|
+
def advance!(tuples, source_ts)
|
|
70
|
+
tuples.each do |tuple|
|
|
71
|
+
key = serialize(tuple)
|
|
72
|
+
row = record_class.create_or_find_by(view_name: view_key, partition_key: key) do |record|
|
|
73
|
+
record.source_ts = source_ts
|
|
74
|
+
end
|
|
75
|
+
row.update!(source_ts: source_ts) if row.source_ts < source_ts
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def record_class
|
|
80
|
+
SourceWatermarkRecord
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def table_name
|
|
84
|
+
::ActiveRecord::Materialized.configuration.source_watermark_table_name
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def define_columns(table)
|
|
88
|
+
table.bigint :source_ts, null: false
|
|
89
|
+
table.timestamps
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActiveRecord
|
|
4
|
+
module Materialized
|
|
5
|
+
# ActiveRecord model backing the CDC source-watermark table — the max applied +source_ts+ per
|
|
6
|
+
# (view, partition). Used to suppress stale/out-of-order CDC events and to report a view's
|
|
7
|
+
# freshness. Uses {ConfigurableTableName}'s dynamic resolution so a host app can rename the table.
|
|
8
|
+
#
|
|
9
|
+
# @api private
|
|
10
|
+
class SourceWatermarkRecord < ::ActiveRecord::Base
|
|
11
|
+
include ConfigurableTableName
|
|
12
|
+
|
|
13
|
+
configurable_table_name { ::ActiveRecord::Materialized.configuration.source_watermark_table_name }
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# typed: strict
|
|
2
1
|
# frozen_string_literal: true
|
|
3
2
|
|
|
4
3
|
module ActiveRecord
|
|
@@ -8,46 +7,33 @@ module ActiveRecord
|
|
|
8
7
|
#
|
|
9
8
|
# @api private
|
|
10
9
|
class SummaryDelta
|
|
11
|
-
extend T::Sig
|
|
12
|
-
|
|
13
|
-
KeyTuple = T.type_alias { T::Array[T.untyped] }
|
|
14
|
-
Columns = T.type_alias { T::Hash[String, Numeric] }
|
|
15
|
-
Buckets = T.type_alias { T::Hash[KeyTuple, Columns] }
|
|
16
|
-
|
|
17
|
-
sig { returns(Buckets) }
|
|
18
10
|
attr_reader :buckets
|
|
19
11
|
|
|
20
|
-
sig { params(buckets: Buckets).void }
|
|
21
12
|
def initialize(buckets = {})
|
|
22
13
|
@buckets = buckets
|
|
23
14
|
end
|
|
24
15
|
|
|
25
|
-
sig { params(key_tuple: KeyTuple, column: String, amount: Numeric).void }
|
|
26
16
|
def add(key_tuple, column, amount)
|
|
27
17
|
bucket = (@buckets[key_tuple] ||= {})
|
|
28
18
|
bucket[column] = (bucket[column] || 0) + amount
|
|
29
19
|
end
|
|
30
20
|
|
|
31
|
-
sig { returns(T::Boolean) }
|
|
32
21
|
def empty?
|
|
33
22
|
@buckets.empty?
|
|
34
23
|
end
|
|
35
24
|
|
|
36
25
|
# Number of distinct partitions (buckets) accumulated so far.
|
|
37
|
-
sig { returns(Integer) }
|
|
38
26
|
def tracked_partition_count
|
|
39
27
|
@buckets.size
|
|
40
28
|
end
|
|
41
29
|
|
|
42
30
|
# Drops net-zero columns (no-op changes) and the partitions left empty.
|
|
43
|
-
sig { returns(SummaryDelta) }
|
|
44
31
|
def prune!
|
|
45
32
|
@buckets.each_value { |columns| columns.reject! { |_column, amount| amount.zero? } }
|
|
46
33
|
@buckets.reject! { |_key_tuple, columns| columns.empty? }
|
|
47
34
|
self
|
|
48
35
|
end
|
|
49
36
|
|
|
50
|
-
sig { params(other: SummaryDelta).returns(SummaryDelta) }
|
|
51
37
|
def merge(other)
|
|
52
38
|
merged = SummaryDelta.new(@buckets.transform_values(&:dup))
|
|
53
39
|
other.buckets.each do |key_tuple, columns|
|
|
@@ -57,17 +43,15 @@ module ActiveRecord
|
|
|
57
43
|
end
|
|
58
44
|
|
|
59
45
|
# A list of `"key" => tuple, "columns" => …` entries so array keys survive JSON.
|
|
60
|
-
sig { returns(T::Hash[String, T.untyped]) }
|
|
61
46
|
def serialize
|
|
62
47
|
{ "summary" => @buckets.map { |key_tuple, columns| { "key" => key_tuple, "columns" => columns } } }
|
|
63
48
|
end
|
|
64
49
|
|
|
65
|
-
sig { params(payload: T.nilable(T::Hash[String, T.untyped])).returns(T.nilable(SummaryDelta)) }
|
|
66
50
|
def self.deserialize(payload)
|
|
67
51
|
rows = payload && payload["summary"]
|
|
68
52
|
return nil if rows.nil?
|
|
69
53
|
|
|
70
|
-
buckets =
|
|
54
|
+
buckets = {}
|
|
71
55
|
rows.each { |row| buckets[row.fetch("key")] = row.fetch("columns") }
|
|
72
56
|
new(buckets)
|
|
73
57
|
end
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# typed: strict
|
|
2
1
|
# frozen_string_literal: true
|
|
3
2
|
|
|
4
3
|
module ActiveRecord
|
|
@@ -7,16 +6,12 @@ module ActiveRecord
|
|
|
7
6
|
# snapshot and adding the "after" uniformly handles inserts, deletes, in-place
|
|
8
7
|
# updates, and group-key changes (a move between partitions).
|
|
9
8
|
class SummaryDeltaBuilder
|
|
10
|
-
extend T::Sig
|
|
11
|
-
|
|
12
|
-
sig { params(change: WriteChange, analysis: AggregateAnalysis, group_columns: T::Array[String]).void }
|
|
13
9
|
def initialize(change, analysis, group_columns)
|
|
14
10
|
@change = change
|
|
15
11
|
@analysis = analysis
|
|
16
12
|
@group_columns = group_columns
|
|
17
13
|
end
|
|
18
14
|
|
|
19
|
-
sig { returns(SummaryDelta) }
|
|
20
15
|
def build
|
|
21
16
|
delta = SummaryDelta.new
|
|
22
17
|
apply!(delta, @change.before, -1) unless @change.before.empty?
|
|
@@ -26,7 +21,6 @@ module ActiveRecord
|
|
|
26
21
|
|
|
27
22
|
private
|
|
28
23
|
|
|
29
|
-
sig { params(delta: SummaryDelta, snapshot: T::Hash[String, T.untyped], sign: Integer).void }
|
|
30
24
|
def apply!(delta, snapshot, sign)
|
|
31
25
|
key_tuple = key_tuple(snapshot)
|
|
32
26
|
return if key_tuple.nil?
|
|
@@ -37,17 +31,15 @@ module ActiveRecord
|
|
|
37
31
|
end
|
|
38
32
|
end
|
|
39
33
|
|
|
40
|
-
sig { params(column: AggregateAnalysis::Column, snapshot: T::Hash[String, T.untyped]).returns(Numeric) }
|
|
41
34
|
def contribution_for(column, snapshot)
|
|
42
35
|
case column.function
|
|
43
36
|
when :count_star then 1
|
|
44
|
-
when :count then snapshot[
|
|
45
|
-
when :sum then snapshot.fetch(
|
|
37
|
+
when :count then snapshot[column.attribute].nil? ? 0 : 1
|
|
38
|
+
when :sum then snapshot.fetch(column.attribute, 0) || 0
|
|
46
39
|
else 0
|
|
47
40
|
end
|
|
48
41
|
end
|
|
49
42
|
|
|
50
|
-
sig { params(snapshot: T::Hash[String, T.untyped]).returns(T.nilable(SummaryDelta::KeyTuple)) }
|
|
51
43
|
def key_tuple(snapshot)
|
|
52
44
|
return nil unless @group_columns.all? { |column| snapshot.key?(column) }
|
|
53
45
|
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# typed: strict
|
|
2
1
|
# frozen_string_literal: true
|
|
3
2
|
|
|
4
3
|
module ActiveRecord
|
|
@@ -8,28 +7,22 @@ module ActiveRecord
|
|
|
8
7
|
# @api private
|
|
9
8
|
class TableModelRegistry
|
|
10
9
|
class << self
|
|
11
|
-
extend T::Sig
|
|
12
|
-
|
|
13
|
-
sig { params(model_class: T.class_of(::ActiveRecord::Base)).void }
|
|
14
10
|
def register(model_class)
|
|
15
11
|
return if model_class.abstract_class?
|
|
16
12
|
|
|
17
13
|
explicit[model_class.table_name] = model_class
|
|
18
14
|
end
|
|
19
15
|
|
|
20
|
-
sig { params(table_name: String).returns(T.nilable(T.class_of(::ActiveRecord::Base))) }
|
|
21
16
|
def resolve(table_name)
|
|
22
17
|
explicit[table_name] || find_descendant(table_name)
|
|
23
18
|
end
|
|
24
19
|
|
|
25
20
|
private
|
|
26
21
|
|
|
27
|
-
sig { returns(T::Hash[String, T.class_of(::ActiveRecord::Base)]) }
|
|
28
22
|
def explicit
|
|
29
|
-
@explicit ||=
|
|
23
|
+
@explicit ||= {}
|
|
30
24
|
end
|
|
31
25
|
|
|
32
|
-
sig { params(table_name: String).returns(T.nilable(T.class_of(::ActiveRecord::Base))) }
|
|
33
26
|
def find_descendant(table_name)
|
|
34
27
|
::ActiveRecord::Base.descendants.find do |model_class|
|
|
35
28
|
!model_class.abstract_class? && model_class.table_name == table_name
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActiveRecord
|
|
4
|
+
module Materialized
|
|
5
|
+
# Swaps a view's cache table for a freshly-built temp table, atomically per the
|
|
6
|
+
# engine's DDL semantics: a transaction on SQLite/Postgres (transactional DDL),
|
|
7
|
+
# or a single multi-table RENAME TABLE on MySQL — whose DDL auto-commits, so a
|
|
8
|
+
# transaction could not make two separate renames atomic. Extracted from
|
|
9
|
+
# RelationCacheWriter, which builds the temp table and delegates the swap here.
|
|
10
|
+
#
|
|
11
|
+
# @api private
|
|
12
|
+
class TableSwap
|
|
13
|
+
def initialize(view_class)
|
|
14
|
+
@view_class = view_class
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# Replace the view table with temp_table; the displaced view table becomes
|
|
18
|
+
# old_table and is dropped. Indexes on the displaced table are preserved: the freshly built
|
|
19
|
+
# temp table only carries the schema-default index, so any index a user added to the cache table
|
|
20
|
+
# (or the default one, on a table built before it existed) would otherwise be lost with the
|
|
21
|
+
# dropped old table — unlike PostgreSQL's REFRESH MATERIALIZED VIEW, which keeps them.
|
|
22
|
+
def swap!(temp_table, old_table)
|
|
23
|
+
preserved = current_indexes
|
|
24
|
+
if connection.supports_ddl_transactions?
|
|
25
|
+
# SQLite/Postgres: rename + index restore commit together, so readers only ever see the old
|
|
26
|
+
# table or the fully-indexed new one.
|
|
27
|
+
connection.transaction do
|
|
28
|
+
rename_in_place!(temp_table, old_table)
|
|
29
|
+
restore_missing_indexes!(preserved)
|
|
30
|
+
end
|
|
31
|
+
else
|
|
32
|
+
# MySQL/MariaDB: RENAME TABLE auto-commits (a transaction can't make it atomic with the
|
|
33
|
+
# index rebuild), so the restore runs after. A crash in this window leaves the new table live
|
|
34
|
+
# but missing the preserved indexes until the next rebuild! — degraded read/maintenance
|
|
35
|
+
# performance, never data loss or an empty table (the rename itself is atomic).
|
|
36
|
+
atomic_rename!(temp_table, old_table)
|
|
37
|
+
restore_missing_indexes!(preserved)
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
private
|
|
42
|
+
|
|
43
|
+
attr_reader :view_class
|
|
44
|
+
|
|
45
|
+
def connection = view_class.connection
|
|
46
|
+
|
|
47
|
+
# The indexes on the current live cache table (empty on first build, when there is none), captured
|
|
48
|
+
# before the swap so they can be re-created on the table that replaces it.
|
|
49
|
+
def current_indexes
|
|
50
|
+
return [] unless view_class.table_exists?
|
|
51
|
+
|
|
52
|
+
connection.indexes(view_class.table_name)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Re-create every preserved index the swapped-in table does not already carry (matched on full
|
|
56
|
+
# signature, so the temp table's default index isn't duplicated). Index names are free again now
|
|
57
|
+
# that the old table is dropped, so each keeps its original name.
|
|
58
|
+
def restore_missing_indexes!(preserved)
|
|
59
|
+
existing = connection.indexes(view_class.table_name)
|
|
60
|
+
preserved.each do |index|
|
|
61
|
+
next if existing.any? { |candidate| same_index?(candidate, index) }
|
|
62
|
+
|
|
63
|
+
connection.add_index(view_class.table_name, index.columns, **index_options(index))
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def same_index?(one, other)
|
|
68
|
+
one.columns == other.columns && one.unique == other.unique && one.where == other.where &&
|
|
69
|
+
one.orders == other.orders && one.using == other.using
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def index_options(index)
|
|
73
|
+
options = { name: index.name, unique: index.unique }
|
|
74
|
+
options[:where] = index.where if index.where
|
|
75
|
+
options[:using] = index.using if index.using
|
|
76
|
+
options[:order] = index.orders if index.orders.present?
|
|
77
|
+
options
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def rename_in_place!(temp_table, old_table)
|
|
81
|
+
connection.rename_table(view_class.table_name, old_table) if view_class.table_exists?
|
|
82
|
+
connection.rename_table(temp_table, view_class.table_name)
|
|
83
|
+
connection.drop_table(old_table, if_exists: true)
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# MySQL: `RENAME TABLE current TO old, temp TO current` swaps in one atomic
|
|
87
|
+
# statement (a single metadata lock over all tables), then the old table is
|
|
88
|
+
# dropped. On first build there is no current table, so just rename temp in.
|
|
89
|
+
def atomic_rename!(temp_table, old_table)
|
|
90
|
+
view = view_class.table_name
|
|
91
|
+
unless view_class.table_exists?
|
|
92
|
+
connection.rename_table(temp_table, view)
|
|
93
|
+
return
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
connection.execute(rename_sql(view => old_table, temp_table => view))
|
|
97
|
+
connection.drop_table(old_table, if_exists: true)
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def rename_sql(pairs)
|
|
101
|
+
clauses = pairs.map { |from, to| "#{connection.quote_table_name(from)} TO #{connection.quote_table_name(to)}" }
|
|
102
|
+
"RENAME TABLE #{clauses.join(', ')}"
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
end
|