activerecord-materialized 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +93 -0
- data/README.md +138 -323
- data/lib/activerecord/materialized/aggregate_analysis.rb +10 -31
- data/lib/activerecord/materialized/async_refresher.rb +18 -27
- data/lib/activerecord/materialized/cache_table_schema.rb +53 -34
- data/lib/activerecord/materialized/change_source.rb +25 -0
- data/lib/activerecord/materialized/cold_read.rb +1 -10
- data/lib/activerecord/materialized/column_type_inference.rb +167 -0
- data/lib/activerecord/materialized/configurable_table_name.rb +45 -0
- data/lib/activerecord/materialized/configuration.rb +104 -29
- data/lib/activerecord/materialized/connection_routing.rb +29 -0
- data/lib/activerecord/materialized/data_verification_result.rb +41 -0
- data/lib/activerecord/materialized/data_verifier.rb +162 -0
- data/lib/activerecord/materialized/debezium_envelope.rb +105 -0
- data/lib/activerecord/materialized/delta_maintainer.rb +4 -15
- data/lib/activerecord/materialized/dependency_registry.rb +43 -35
- data/lib/activerecord/materialized/dependency_trackable.rb +7 -14
- data/lib/activerecord/materialized/incremental_maintainer.rb +45 -28
- data/lib/activerecord/materialized/incremental_refresh.rb +63 -0
- data/lib/activerecord/materialized/instrumentation.rb +102 -0
- data/lib/activerecord/materialized/maintenance_delta.rb +4 -19
- data/lib/activerecord/materialized/maintenance_delta_builder.rb +46 -14
- data/lib/activerecord/materialized/maintenance_store.rb +58 -21
- data/lib/activerecord/materialized/metadata/maintenance_payload.rb +1 -7
- data/lib/activerecord/materialized/metadata/reconciliation.rb +25 -0
- data/lib/activerecord/materialized/metadata/schema.rb +19 -41
- data/lib/activerecord/materialized/metadata/timestamps.rb +0 -5
- data/lib/activerecord/materialized/metadata.rb +41 -25
- data/lib/activerecord/materialized/metadata_record.rb +2 -16
- data/lib/activerecord/materialized/migration_builder.rb +8 -8
- data/lib/activerecord/materialized/module_api.rb +219 -22
- data/lib/activerecord/materialized/partition_filter.rb +45 -0
- data/lib/activerecord/materialized/partition_keyed_store.rb +72 -0
- data/lib/activerecord/materialized/partition_record.rb +2 -16
- data/lib/activerecord/materialized/partition_snapshot.rb +79 -0
- data/lib/activerecord/materialized/partition_state.rb +47 -46
- data/lib/activerecord/materialized/query_expressions.rb +0 -12
- data/lib/activerecord/materialized/railtie.rb +15 -3
- data/lib/activerecord/materialized/read_router.rb +51 -0
- data/lib/activerecord/materialized/reconcile_job.rb +22 -0
- data/lib/activerecord/materialized/reconcile_result.rb +36 -0
- data/lib/activerecord/materialized/reconciler.rb +79 -0
- data/lib/activerecord/materialized/refresh_callbacks.rb +4 -18
- data/lib/activerecord/materialized/refresh_job.rb +0 -4
- data/lib/activerecord/materialized/refresh_result.rb +2 -10
- data/lib/activerecord/materialized/refresh_scheduler.rb +5 -7
- data/lib/activerecord/materialized/refresher.rb +35 -50
- data/lib/activerecord/materialized/registry.rb +26 -16
- data/lib/activerecord/materialized/relation_cache_writer.rb +12 -46
- data/lib/activerecord/materialized/schema_verifier.rb +0 -9
- data/lib/activerecord/materialized/source_watermark.rb +93 -0
- data/lib/activerecord/materialized/source_watermark_record.rb +16 -0
- data/lib/activerecord/materialized/summary_delta.rb +1 -17
- data/lib/activerecord/materialized/summary_delta_builder.rb +2 -10
- data/lib/activerecord/materialized/table_model_registry.rb +1 -8
- data/lib/activerecord/materialized/table_swap.rb +106 -0
- data/lib/activerecord/materialized/tasks.rb +46 -38
- data/lib/activerecord/materialized/version.rb +1 -2
- data/lib/activerecord/materialized/view.rb +11 -22
- data/lib/activerecord/materialized/view_configuration_class_methods.rb +23 -35
- data/lib/activerecord/materialized/view_definition.rb +76 -65
- data/lib/activerecord/materialized/view_incremental_class_methods.rb +57 -60
- data/lib/activerecord/materialized/view_loader.rb +49 -0
- data/lib/activerecord/materialized/view_query_access_class_methods.rb +100 -57
- data/lib/activerecord/materialized/view_refresh_policy_class_methods.rb +70 -37
- data/lib/activerecord/materialized/write_change.rb +75 -26
- data/lib/activerecord/materialized/write_maintenance.rb +73 -0
- data/lib/activerecord/materialized/write_outbox.rb +345 -0
- data/lib/activerecord/materialized/write_outbox_record.rb +17 -0
- data/lib/activerecord/materialized.rb +27 -6
- data/lib/generators/activerecord_materialized/install/templates/create_ar_materialized_view_metadata.rb.erb +14 -0
- data/lib/generators/activerecord_materialized/install_generator.rb +1 -6
- data/lib/generators/activerecord_materialized/migration_generator.rb +3 -9
- data/lib/generators/activerecord_materialized/outbox_generator.rb +36 -0
- data/lib/generators/activerecord_materialized/templates/materialized_view_migration.rb.erb +5 -1
- data/lib/generators/activerecord_materialized/templates/write_outbox_migration.rb.erb +21 -0
- data/lib/generators/activerecord_materialized/view_generator.rb +0 -4
- metadata +27 -18
- data/lib/activerecord/materialized/type_reexports.rb +0 -14
- data/lib/activerecord/materialized/view_class.rb +0 -8
- data/lib/activerecord_materialized_types.rb +0 -18
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# typed: strict
|
|
2
1
|
# frozen_string_literal: true
|
|
3
2
|
|
|
4
3
|
module ActiveRecord
|
|
@@ -6,74 +5,150 @@ module ActiveRecord
|
|
|
6
5
|
# Global, app-wide defaults, set via {Materialized.configure}. Individual
|
|
7
6
|
# views can override most of these with the corresponding DSL macro.
|
|
8
7
|
class Configuration
|
|
9
|
-
extend T::Sig
|
|
10
|
-
|
|
11
8
|
# @return [String] name of the metadata table (default "ar_materialized_view_metadata")
|
|
12
|
-
sig { returns(String) }
|
|
13
9
|
attr_accessor :metadata_table_name
|
|
14
10
|
|
|
15
11
|
# @return [String] name of the per-partition freshness table
|
|
16
|
-
sig { returns(String) }
|
|
17
12
|
attr_accessor :partition_table_name
|
|
18
13
|
|
|
14
|
+
# @return [String] name of the write-outbox table (the optional trigger/outbox change source)
|
|
15
|
+
attr_accessor :write_outbox_table_name
|
|
16
|
+
|
|
17
|
+
# @return [String] name of the CDC source-watermark table (per-partition applied-watermark tracking)
|
|
18
|
+
attr_accessor :source_watermark_table_name
|
|
19
|
+
|
|
19
20
|
# @return [Numeric, ActiveSupport::Duration, nil] default {ViewRefreshPolicyClassMethods::ClassMethods#max_staleness max_staleness}
|
|
20
|
-
sig { returns(T.nilable(DebounceInterval)) }
|
|
21
21
|
attr_accessor :default_max_staleness
|
|
22
22
|
|
|
23
23
|
# @return [Integer, nil] optional per-refresh timeout in seconds
|
|
24
|
-
sig { returns(T.nilable(Integer)) }
|
|
25
24
|
attr_accessor :refresh_timeout
|
|
26
25
|
|
|
27
26
|
# @return [Boolean] whether a full refresh swaps a freshly built table in atomically
|
|
28
|
-
sig { returns(T::Boolean) }
|
|
29
27
|
attr_accessor :atomic_swap_refresh
|
|
30
28
|
|
|
31
29
|
# @return [Symbol] default refresh strategy: +:async+, +:immediate+, or +:manual+
|
|
32
|
-
sig { returns(Symbol) }
|
|
33
30
|
attr_accessor :default_refresh_strategy
|
|
34
31
|
|
|
35
32
|
# @return [Numeric] default debounce window (seconds) for coalescing async refreshes
|
|
36
|
-
sig { returns(DebounceInterval) }
|
|
37
33
|
attr_accessor :default_refresh_debounce
|
|
38
34
|
|
|
39
|
-
#
|
|
40
|
-
|
|
41
|
-
|
|
35
|
+
# Background dispatcher: +:active_job+ or +:async+ (in-process thread). Unset, it resolves
|
|
36
|
+
# to +:active_job+ when ActiveJob is loaded and +:async+ otherwise; an explicit assignment
|
|
37
|
+
# always wins. The in-process +:async+ dispatcher does not coordinate across processes and
|
|
38
|
+
# its queue is lost on restart, so it is single-process-only — multi-process deployments
|
|
39
|
+
# should run +:active_job+.
|
|
40
|
+
#
|
|
41
|
+
# @return [Symbol]
|
|
42
|
+
attr_writer :refresh_dispatcher
|
|
43
|
+
|
|
44
|
+
def refresh_dispatcher
|
|
45
|
+
@refresh_dispatcher || (active_job_available? ? :active_job : :async)
|
|
46
|
+
end
|
|
42
47
|
|
|
43
48
|
# @return [Symbol] ActiveJob queue name used when +refresh_dispatcher+ is +:active_job+
|
|
44
|
-
sig { returns(Symbol) }
|
|
45
49
|
attr_accessor :refresh_queue_name
|
|
46
50
|
|
|
51
|
+
# ActiveJob queue for {ReconcileJob} (the reconcile fan-out); falls back to
|
|
52
|
+
# +refresh_queue_name+ when unset so both maintenance jobs share a queue by default.
|
|
53
|
+
#
|
|
54
|
+
# @return [Symbol]
|
|
55
|
+
attr_writer :reconcile_queue_name
|
|
56
|
+
|
|
57
|
+
def reconcile_queue_name
|
|
58
|
+
@reconcile_queue_name || refresh_queue_name
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# Rails multi-database role that maintenance writes (refresh/reconcile/rebuild) run under, so
|
|
62
|
+
# they target the primary in a writer/replica topology. +nil+ (default) yields on the current
|
|
63
|
+
# connection — no routing. Requires the host app to have declared the role via +connects_to+.
|
|
64
|
+
#
|
|
65
|
+
# @return [Symbol, nil]
|
|
66
|
+
attr_accessor :maintenance_role
|
|
67
|
+
|
|
68
|
+
# Rails multi-database role that {DataVerifier} reads run under, so the (expensive) drift
|
|
69
|
+
# verification can be offloaded to a replica — the verify-on-replica / repair-on-primary split.
|
|
70
|
+
# +nil+ (default) yields on the current connection.
|
|
71
|
+
#
|
|
72
|
+
# @return [Symbol, nil]
|
|
73
|
+
attr_accessor :verification_role
|
|
74
|
+
|
|
75
|
+
# Replication lag budget folded into time-based staleness: a view read from a replica is
|
|
76
|
+
# effectively `view staleness + replication lag` stale, so this tightens the effective
|
|
77
|
+
# +max_staleness+ (a view goes stale this much sooner) to keep replica reads within budget.
|
|
78
|
+
# Keep it well below your smallest +max_staleness+ — a value at or above it drives the effective
|
|
79
|
+
# window to zero, making that view perpetually stale (reconciled every tick). Defaults to 0 (no
|
|
80
|
+
# adjustment); a static estimate of a dynamic quantity — see the distributed-deployment guide.
|
|
81
|
+
#
|
|
82
|
+
# @return [Numeric, ActiveSupport::Duration]
|
|
83
|
+
attr_accessor :replica_lag
|
|
84
|
+
|
|
47
85
|
# Cold-read behavior: :read_through (serve from source), :serve_stale
|
|
48
86
|
# (serve the cache as-is), or :raise.
|
|
49
|
-
sig { returns(Symbol) }
|
|
50
87
|
attr_accessor :default_cold_read_strategy
|
|
51
88
|
|
|
89
|
+
# Default change source for views: +:callbacks+ (install ActiveRecord
|
|
90
|
+
# commit callbacks on +depends_on+ models) or +:none+ (install no
|
|
91
|
+
# callbacks; drive maintenance through the public ingestion API from an
|
|
92
|
+
# external adapter). A view can override this with +change_source+.
|
|
93
|
+
# Lazily defaulted (like +max_tracked_partitions+) so it stays out of
|
|
94
|
+
# +initialize+.
|
|
95
|
+
#
|
|
96
|
+
# @return [Symbol]
|
|
97
|
+
def default_change_source=(value)
|
|
98
|
+
@default_change_source = ChangeSource.cast(value)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def default_change_source
|
|
102
|
+
@default_change_source ||= ChangeSource::CALLBACKS
|
|
103
|
+
end
|
|
104
|
+
|
|
52
105
|
# Cap on distinct partitions tracked in a view's pending maintenance before
|
|
53
106
|
# it collapses to a single full recompute. Bounds the per-write cost of a
|
|
54
107
|
# bulk write that spans many partitions. Defaults to 1000.
|
|
55
108
|
#
|
|
56
109
|
# @return [Integer]
|
|
57
|
-
sig { params(max_tracked_partitions: Integer).void }
|
|
58
110
|
attr_writer :max_tracked_partitions
|
|
59
111
|
|
|
60
|
-
sig { returns(Integer) }
|
|
61
112
|
def max_tracked_partitions
|
|
62
|
-
@max_tracked_partitions ||=
|
|
113
|
+
@max_tracked_partitions ||= 1_000
|
|
63
114
|
end
|
|
64
115
|
|
|
65
|
-
|
|
116
|
+
# App-relative directories the Railtie eager-loads on boot (and on each dev reload) so every
|
|
117
|
+
# view class is loaded — and its +depends_on+ commit callbacks installed — even under Zeitwerk's
|
|
118
|
+
# lazy loading (`config.eager_load = false`, i.e. development and test). Without this, a view
|
|
119
|
+
# whose constant nothing has referenced yet has no callbacks, so writes to its dependencies
|
|
120
|
+
# silently don't schedule maintenance until something first touches the class. Defaults to
|
|
121
|
+
# +["app/models"]+ (where view classes conventionally live); set to +[]+ to disable, or list your
|
|
122
|
+
# own directories. Only existing directories that are Zeitwerk autoload roots are loaded — a path
|
|
123
|
+
# outside the autoloader (or a production app that already eager-loads) is a safe no-op.
|
|
124
|
+
#
|
|
125
|
+
# @return [Array<String>]
|
|
126
|
+
attr_accessor :view_load_paths
|
|
127
|
+
|
|
66
128
|
def initialize
|
|
67
|
-
@metadata_table_name =
|
|
68
|
-
@partition_table_name =
|
|
69
|
-
@
|
|
70
|
-
@
|
|
71
|
-
@
|
|
72
|
-
@
|
|
73
|
-
@
|
|
74
|
-
@
|
|
75
|
-
@
|
|
76
|
-
@
|
|
129
|
+
@metadata_table_name = "ar_materialized_view_metadata"
|
|
130
|
+
@partition_table_name = "ar_materialized_view_partitions"
|
|
131
|
+
@write_outbox_table_name = "ar_materialized_view_write_outbox"
|
|
132
|
+
@source_watermark_table_name = "ar_materialized_view_source_watermarks"
|
|
133
|
+
@default_max_staleness = nil
|
|
134
|
+
@refresh_timeout = nil
|
|
135
|
+
@atomic_swap_refresh = true
|
|
136
|
+
@default_refresh_strategy = :async
|
|
137
|
+
@default_refresh_debounce = 2
|
|
138
|
+
# @refresh_dispatcher intentionally unset — lazily resolved from ActiveJob availability.
|
|
139
|
+
@refresh_queue_name = :materialized_views
|
|
140
|
+
@default_cold_read_strategy = :read_through
|
|
141
|
+
@replica_lag = 0
|
|
142
|
+
@view_load_paths = ["app/models"]
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
# Whether ActiveJob is loaded — the single source of truth for the dispatcher default and
|
|
146
|
+
# the module's dispatch/enqueue guards. Extracted (rather than an inline +defined?+) so it is
|
|
147
|
+
# testable regardless of whether the host app pulled in ActiveJob.
|
|
148
|
+
#
|
|
149
|
+
# @return [Boolean]
|
|
150
|
+
def active_job_available?
|
|
151
|
+
!defined?(ActiveJob::Base).nil?
|
|
77
152
|
end
|
|
78
153
|
end
|
|
79
154
|
end
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActiveRecord
|
|
4
|
+
module Materialized
|
|
5
|
+
# Routes work to a Rails multi-database role when the app has configured one: maintenance writes
|
|
6
|
+
# to the primary, verification reads to a replica (the verify-on-replica / repair-on-primary
|
|
7
|
+
# split). A nil role — the default — yields on the current connection, so single-database apps
|
|
8
|
+
# and those relying on Rails' automatic role switching are unaffected.
|
|
9
|
+
#
|
|
10
|
+
# @api private
|
|
11
|
+
module ConnectionRouting
|
|
12
|
+
module_function
|
|
13
|
+
|
|
14
|
+
def maintenance(&block)
|
|
15
|
+
with_role(ActiveRecord::Materialized.configuration.maintenance_role, &block)
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def verification(&block)
|
|
19
|
+
with_role(ActiveRecord::Materialized.configuration.verification_role, &block)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def with_role(role, &block)
|
|
23
|
+
return yield if role.nil?
|
|
24
|
+
|
|
25
|
+
ActiveRecord::Base.connected_to(role: role, &block)
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActiveRecord
|
|
4
|
+
module Materialized
|
|
5
|
+
# The outcome of a {DataVerifier} run: the partition keys whose materialized
|
|
6
|
+
# contents diverge from the source relation, plus how much was checked.
|
|
7
|
+
DataVerificationResult = Data.define(
|
|
8
|
+
:view_name,
|
|
9
|
+
:mode,
|
|
10
|
+
:total_partition_count,
|
|
11
|
+
:checked_partition_count,
|
|
12
|
+
# In the source relation but absent from the cache.
|
|
13
|
+
:missing_keys,
|
|
14
|
+
# In the cache but absent from the source relation.
|
|
15
|
+
:extra_keys,
|
|
16
|
+
# Present in both, but the partition's cache contents diverge. In +:full+ and
|
|
17
|
+
# +:checksum+ this means the value columns differ; in every mode (including
|
|
18
|
+
# +:row_count+) it also flags a partition the cache holds a different *number*
|
|
19
|
+
# of rows for than the source — a duplicated or lost row within the partition.
|
|
20
|
+
:mismatched_keys
|
|
21
|
+
) do
|
|
22
|
+
# A clean result covering nothing — for a view that isn't materialized yet.
|
|
23
|
+
def self.empty(view_name:, mode:)
|
|
24
|
+
new(
|
|
25
|
+
view_name: view_name, mode: mode, total_partition_count: 0, checked_partition_count: 0,
|
|
26
|
+
missing_keys: [], extra_keys: [], mismatched_keys: []
|
|
27
|
+
)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def drifted?
|
|
31
|
+
missing_keys.any? || extra_keys.any? || mismatched_keys.any?
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Every diverging partition key, however it diverged — the set a reconciliation
|
|
35
|
+
# re-maintains (see {Reconciler}).
|
|
36
|
+
def divergent_keys
|
|
37
|
+
(missing_keys + extra_keys + mismatched_keys).uniq
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActiveRecord
|
|
4
|
+
module Materialized
|
|
5
|
+
# Detects DATA drift — the materialized cache diverging from what the source
|
|
6
|
+
# relation would produce right now — as opposed to {SchemaVerifier}'s schema
|
|
7
|
+
# drift. Stateless: it only reads and compares.
|
|
8
|
+
#
|
|
9
|
+
# Modes trade cost for depth:
|
|
10
|
+
# * +:row_count+ — which partitions exist (missing/extra); cheapest.
|
|
11
|
+
# * +:checksum+ — a per-partition digest of the value columns.
|
|
12
|
+
# * +:full+ — the value columns exactly.
|
|
13
|
+
#
|
|
14
|
+
# +sample:+ (Integer count / Float fraction) value-checks a random subset of
|
|
15
|
+
# partitions for large views; a sample covering every partition runs exhaustive.
|
|
16
|
+
#
|
|
17
|
+
# Covers grouped/aggregate views whose GROUP BY keys map to projected columns
|
|
18
|
+
# (the incremental-maintenance target). A non-grouped view, or one whose group
|
|
19
|
+
# key can't be matched to a projected column, is skipped (an empty result).
|
|
20
|
+
class DataVerifier
|
|
21
|
+
# Raised (by {Materialized.verify_data!}) when a view's contents have drifted.
|
|
22
|
+
class DataDriftError < StandardError; end
|
|
23
|
+
|
|
24
|
+
MODES = %i[row_count checksum full].freeze
|
|
25
|
+
|
|
26
|
+
def initialize(view_class, mode: :checksum, sample: nil)
|
|
27
|
+
raise ArgumentError, "unknown data-verification mode #{mode.inspect}" unless MODES.include?(mode)
|
|
28
|
+
|
|
29
|
+
@view_class = view_class
|
|
30
|
+
@mode = mode
|
|
31
|
+
@sample = sample
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def verify
|
|
35
|
+
# verifiable? reads metadata/schema (and may provision the metadata table on first use), so it
|
|
36
|
+
# runs on the ambient/primary connection — never the read-only replica. Only the cache-vs-source
|
|
37
|
+
# comparison is routed to the verification role, inside one snapshot.
|
|
38
|
+
return empty_result unless verifiable?
|
|
39
|
+
|
|
40
|
+
ConnectionRouting.verification { in_consistent_snapshot { verify_within_snapshot } }
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
private
|
|
44
|
+
|
|
45
|
+
# Read the cache and the recomputed source inside one transaction, so both sides come from a
|
|
46
|
+
# single snapshot: a write landing mid-verify — or replication lag, when reads are pinned to a
|
|
47
|
+
# replica — can't make consistent data look like drift. SQLite has no explicit isolation levels
|
|
48
|
+
# (and its single-writer model already reads consistently), and an already-open transaction
|
|
49
|
+
# can't raise its level, so both fall back to a plain transaction.
|
|
50
|
+
def in_consistent_snapshot(&block)
|
|
51
|
+
@view_class.transaction(isolation: snapshot_isolation, &block)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def snapshot_isolation
|
|
55
|
+
connection = @view_class.connection
|
|
56
|
+
return nil if connection.open_transactions.positive? # can't raise isolation in a nested transaction
|
|
57
|
+
return nil if connection.adapter_name.match?(/sqlite/i) # /sqlite/i: SQLite has no explicit isolation levels
|
|
58
|
+
|
|
59
|
+
:repeatable_read
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def verify_within_snapshot
|
|
63
|
+
return verify_exhaustive if @sample.nil?
|
|
64
|
+
|
|
65
|
+
keys = cache_partition_keys
|
|
66
|
+
size = sample_size(keys.size)
|
|
67
|
+
return verify_exhaustive if size >= keys.size # a sample covering everything is exhaustive
|
|
68
|
+
return empty_result if size.zero? # sample: 0 => nothing checked
|
|
69
|
+
|
|
70
|
+
verify_sampled(Array(keys.sample(size)), keys.size)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# Compares every partition, so it detects missing, extra, and mismatched.
|
|
74
|
+
def verify_exhaustive
|
|
75
|
+
cache = snapshot(@view_class.unscoped)
|
|
76
|
+
source = snapshot(@view_class.resolved_source)
|
|
77
|
+
total = (cache.keys | source.keys).size # every partition on either side
|
|
78
|
+
build_result(cache, source, total: total, checked: total, exhaustive: true)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# A value-drift probe over a random subset of materialized partitions. It
|
|
82
|
+
# cannot detect missing (source-only) partitions — that needs a full source
|
|
83
|
+
# scan — so it never reports them.
|
|
84
|
+
def verify_sampled(checked, total)
|
|
85
|
+
cache = snapshot(view_definition.partition_scope_on(@view_class, checked))
|
|
86
|
+
source = snapshot(view_definition.partition_scope(checked))
|
|
87
|
+
build_result(cache, source, total: total, checked: checked.size, exhaustive: false)
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def build_result(cache, source, total:, checked:, exhaustive:)
|
|
91
|
+
shared = cache.keys & source.keys
|
|
92
|
+
DataVerificationResult.new(
|
|
93
|
+
view_name: @view_class.view_key, mode: @mode,
|
|
94
|
+
total_partition_count: total, checked_partition_count: checked,
|
|
95
|
+
missing_keys: exhaustive ? source.keys - cache.keys : [],
|
|
96
|
+
extra_keys: cache.keys - source.keys,
|
|
97
|
+
# Signature tallies differ on value drift and on a duplicated cache row
|
|
98
|
+
# (row_count's tally is of row presence, so it catches duplicates too).
|
|
99
|
+
mismatched_keys: shared.reject { |key| cache[key] == source[key] }
|
|
100
|
+
)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def snapshot(relation)
|
|
104
|
+
PartitionSnapshot.new(@view_class, key_columns, projected_columns - key_columns, @mode).of(relation)
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# Verifiable when the view is a materialized aggregate whose group keys map to
|
|
108
|
+
# projected columns and whose cache table actually has those columns (a
|
|
109
|
+
# schema-drifted cache is skipped — run `verify_schema!` for that).
|
|
110
|
+
def verifiable?
|
|
111
|
+
keys = view_definition.group_key_columns
|
|
112
|
+
@view_class.materialized? && keys.any? && key_columns.size == keys.size &&
|
|
113
|
+
(projected_columns - @view_class.column_names).empty?
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# Distinct group-key tuples in the cache — the population a sample is drawn from.
|
|
117
|
+
# `distinct` dedups in the database, so only one row per partition crosses into
|
|
118
|
+
# Ruby; the random draw stays in Ruby because a DB-side random order is neither an
|
|
119
|
+
# ActiveRecord primitive nor portable across adapters. `pluck` returns scalars for
|
|
120
|
+
# a single key (wrapped to 1-tuples so a NULL key becomes [nil], not the [] that
|
|
121
|
+
# Array(nil) would give) and tuples for a composite key.
|
|
122
|
+
def cache_partition_keys
|
|
123
|
+
values = @view_class.unscoped.distinct.pluck(*key_columns)
|
|
124
|
+
key_columns.one? ? values.zip : values
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# GROUP BY keys as their projected/cache column names (qualifier stripped), so a
|
|
128
|
+
# joined key like "authors.country" matches the projected "country".
|
|
129
|
+
def key_columns
|
|
130
|
+
@key_columns ||= view_definition.group_key_columns.filter_map { |column| projected_column_for(column) }
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
def projected_column_for(column)
|
|
134
|
+
return column if projected_columns.include?(column)
|
|
135
|
+
|
|
136
|
+
stripped = column.split(".").last
|
|
137
|
+
projected_columns.include?(stripped) ? stripped : nil
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def projected_columns
|
|
141
|
+
@projected_columns ||= CacheTableSchema.column_definitions(@view_class.connection,
|
|
142
|
+
@view_class.resolved_source).map(&:name)
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
def view_definition
|
|
146
|
+
@view_definition ||= @view_class.view_definition
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def sample_size(total)
|
|
150
|
+
sample = @sample
|
|
151
|
+
return total if sample.nil?
|
|
152
|
+
|
|
153
|
+
size = sample.is_a?(Float) ? (total * sample).ceil : sample.to_i
|
|
154
|
+
size.clamp(0, total)
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
def empty_result
|
|
158
|
+
DataVerificationResult.empty(view_name: @view_class.view_key, mode: @mode)
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
end
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActiveRecord
|
|
4
|
+
module Materialized
|
|
5
|
+
# Maps a Debezium change envelope to an {Materialized.ingest_change} descriptor. Pure and
|
|
6
|
+
# dependency-free: no Kafka / Debezium / Kafka-Connect runtime — just a mapping over the decoded
|
|
7
|
+
# envelope hash a consumer already holds. It removes the boilerplate (and the easy-to-miss cases:
|
|
8
|
+
# snapshot reads, tombstones, the nested/unwrapped shapes) every consumer otherwise hand-writes.
|
|
9
|
+
#
|
|
10
|
+
# The envelope shape (a Debezium MySQL/Postgres connector event, whether or not the
|
|
11
|
+
# ExtractNewRecordState "unwrap" SMT has been applied — a nested +payload+ is unwrapped here):
|
|
12
|
+
#
|
|
13
|
+
# { "op" => "u",
|
|
14
|
+
# "before" => { "id" => 1, "category" => "books", ... }, # full row image (see below)
|
|
15
|
+
# "after" => { "id" => 1, "category" => "games", ... },
|
|
16
|
+
# "ts_ms" => 1710000000000, # connector processing time (unused)
|
|
17
|
+
# "source" => { "table" => "line_items", "ts_ms" => 1710000000000, ... } }
|
|
18
|
+
#
|
|
19
|
+
# The +source.ts_ms+ (the DB commit time, monotonic within a source) is forwarded as the +source_ts+
|
|
20
|
+
# watermark, so a consumer relaying envelopes gets out-of-order suppression + freshness
|
|
21
|
+
# ({SourceWatermark}) for free. The top-level +ts_ms+ is a *different* clock (the connector's
|
|
22
|
+
# processing time), so it is not used as a fallback — mixing clocks would break monotonicity.
|
|
23
|
+
# Correct **partition-moving** updates require a full +before+ image (+binlog-row-image=FULL+ on
|
|
24
|
+
# MySQL, +REPLICA IDENTITY FULL+ on Postgres); with only the primary key in +before+, the old
|
|
25
|
+
# partition is under-maintained until reconciliation heals it. See +docs/out-of-band-writes.md+
|
|
26
|
+
# and the CDC section of the README. This adapter is Debezium-specific; a Maxwell/other envelope
|
|
27
|
+
# (no +op+) raises rather than being silently dropped.
|
|
28
|
+
module DebeziumEnvelope
|
|
29
|
+
# Debezium +op+ → the gem's operation. A snapshot read ("r") is a create; "d" is a destroy.
|
|
30
|
+
OPERATIONS = { "c" => :create, "r" => :create, "u" => :update, "d" => :destroy }.freeze
|
|
31
|
+
private_constant :OPERATIONS
|
|
32
|
+
|
|
33
|
+
# @param envelope [Hash, nil] a decoded Debezium change event (string or symbol keys), or +nil+
|
|
34
|
+
# for a Kafka tombstone (the null-value message emitted after a delete for log compaction)
|
|
35
|
+
# @param table [String, Symbol, nil] target-table override; defaults to the envelope's +source.table+
|
|
36
|
+
# @return [Hash, nil] +{ table:, operation:, before:, after: }+ (plus +source_ts:+ when the
|
|
37
|
+
# envelope carries a usable +ts_ms+) for {Materialized.ingest_change}, or +nil+ for a
|
|
38
|
+
# tombstone (nothing to relay)
|
|
39
|
+
# @raise [ArgumentError] for a non-tombstone envelope with no +op+ (not a Debezium change event,
|
|
40
|
+
# or not unwrapped), an unsupported +op+, or when the target table can't be determined
|
|
41
|
+
def self.to_change_descriptor(envelope, table = nil)
|
|
42
|
+
return nil if envelope.nil? # Kafka tombstone
|
|
43
|
+
|
|
44
|
+
change = unwrap(envelope)
|
|
45
|
+
op = fetch(change, "op")
|
|
46
|
+
raise ArgumentError, "not a Debezium change envelope (no op)" if op.nil?
|
|
47
|
+
|
|
48
|
+
descriptor = { table: resolve_table(table, change), operation: operation_for(op),
|
|
49
|
+
before: fetch(change, "before"), after: fetch(change, "after") }
|
|
50
|
+
ts = source_ts(change)
|
|
51
|
+
ts ? descriptor.merge(source_ts: ts) : descriptor
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Debezium's default envelope nests the change under +payload+; the ExtractNewRecordState SMT
|
|
55
|
+
# unwraps it to the top level. Accept either by unwrapping a +payload+ that carries the +op+.
|
|
56
|
+
def self.unwrap(envelope)
|
|
57
|
+
payload = fetch(envelope, "payload")
|
|
58
|
+
payload.is_a?(Hash) && fetch(payload, "op") ? payload : envelope
|
|
59
|
+
end
|
|
60
|
+
private_class_method :unwrap
|
|
61
|
+
|
|
62
|
+
# Read a key from an envelope that may use string or symbol keys, without deep-copying the hash
|
|
63
|
+
# (before/after images can be wide, and are copied again downstream by WriteChange).
|
|
64
|
+
def self.fetch(hash, key)
|
|
65
|
+
return nil unless hash.is_a?(Hash)
|
|
66
|
+
|
|
67
|
+
hash.key?(key) ? hash[key] : hash[key.to_sym]
|
|
68
|
+
end
|
|
69
|
+
private_class_method :fetch
|
|
70
|
+
|
|
71
|
+
# A field from the envelope's +source+ metadata block (string/symbol keys), or nil if absent.
|
|
72
|
+
def self.source_field(change, key)
|
|
73
|
+
fetch(fetch(change, "source"), key)
|
|
74
|
+
end
|
|
75
|
+
private_class_method :source_field
|
|
76
|
+
|
|
77
|
+
def self.operation_for(op_code)
|
|
78
|
+
OPERATIONS.fetch(op_code.to_s) do
|
|
79
|
+
raise ArgumentError, "unsupported Debezium op #{op_code.inspect} (expected one of c, r, u, d)"
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
private_class_method :operation_for
|
|
83
|
+
|
|
84
|
+
def self.resolve_table(table, change)
|
|
85
|
+
resolved = table.presence || source_field(change, "table")
|
|
86
|
+
return resolved.to_s if resolved.present?
|
|
87
|
+
|
|
88
|
+
raise ArgumentError, "could not determine the table (pass table or include source.table in the envelope)"
|
|
89
|
+
end
|
|
90
|
+
private_class_method :resolve_table
|
|
91
|
+
|
|
92
|
+
# The per-partition source watermark for this change: Debezium's +source.ts_ms+ (the DB commit
|
|
93
|
+
# time, monotonic within a source). Only an Integer is forwarded — a missing or non-integer value
|
|
94
|
+
# yields no watermark, so the change is maintained exactly as before (see {SourceWatermark}). The
|
|
95
|
+
# top-level +ts_ms+ (the connector's *processing* wall-clock) is intentionally NOT a fallback: it
|
|
96
|
+
# is a different clock, so mixing the two per partition would break the monotonicity suppression
|
|
97
|
+
# relies on. A real Debezium change event always carries +source.ts_ms+.
|
|
98
|
+
def self.source_ts(change)
|
|
99
|
+
ts = source_field(change, "ts_ms")
|
|
100
|
+
ts if ts.is_a?(Integer)
|
|
101
|
+
end
|
|
102
|
+
private_class_method :source_ts
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
end
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# typed: strict
|
|
2
1
|
# frozen_string_literal: true
|
|
3
2
|
|
|
4
3
|
module ActiveRecord
|
|
@@ -7,28 +6,22 @@ module ActiveRecord
|
|
|
7
6
|
# re-reading base rows: new partitions inserted, existing ones incremented in
|
|
8
7
|
# place (NULL-safe for SUM), and emptied partitions deleted.
|
|
9
8
|
class DeltaMaintainer
|
|
10
|
-
extend T::Sig
|
|
11
|
-
|
|
12
|
-
sig { params(view_class: ViewClass).void }
|
|
13
9
|
def initialize(view_class)
|
|
14
10
|
@view_class = view_class
|
|
15
|
-
@analysis =
|
|
11
|
+
@analysis = AggregateAnalysis.new(view_class.resolved_source)
|
|
16
12
|
end
|
|
17
13
|
|
|
18
|
-
sig { params(summary: SummaryDelta).returns(Integer) }
|
|
19
14
|
def apply!(summary)
|
|
20
15
|
summary.buckets.each { |key_tuple, column_deltas| apply_partition(key_tuple, column_deltas) }
|
|
21
|
-
|
|
16
|
+
summary.buckets.size # partitions applied; row-count reporting is the caller's job, outside the lock
|
|
22
17
|
end
|
|
23
18
|
|
|
24
19
|
private
|
|
25
20
|
|
|
26
|
-
sig { returns(T::Array[String]) }
|
|
27
21
|
def group_columns
|
|
28
22
|
@view_class.maintenance_key_columns
|
|
29
23
|
end
|
|
30
24
|
|
|
31
|
-
sig { params(key_tuple: SummaryDelta::KeyTuple, column_deltas: T::Hash[String, Numeric]).void }
|
|
32
25
|
def apply_partition(key_tuple, column_deltas)
|
|
33
26
|
existing = partition_scope(key_tuple).first
|
|
34
27
|
if existing.nil?
|
|
@@ -40,28 +33,24 @@ module ActiveRecord
|
|
|
40
33
|
end
|
|
41
34
|
end
|
|
42
35
|
|
|
43
|
-
sig { params(key_tuple: SummaryDelta::KeyTuple).returns(::ActiveRecord::Relation) }
|
|
44
36
|
def partition_scope(key_tuple)
|
|
45
|
-
|
|
37
|
+
@view_class.unscoped.where(group_columns.zip(key_tuple).to_h)
|
|
46
38
|
end
|
|
47
39
|
|
|
48
|
-
sig { params(key_tuple: SummaryDelta::KeyTuple, column_deltas: T::Hash[String, Numeric]).void }
|
|
49
40
|
def insert_partition(key_tuple, column_deltas)
|
|
50
41
|
# A new partition's deltas are its absolute values; aggregates with no
|
|
51
42
|
# delta default to 0, not NULL, since the partition has rows.
|
|
52
43
|
defaults = @analysis.aggregate_columns.to_h { |column| [column.name, 0] }
|
|
53
44
|
row = group_columns.zip(key_tuple).to_h.merge(defaults).merge(column_deltas)
|
|
54
|
-
|
|
45
|
+
@view_class.create!(row)
|
|
55
46
|
end
|
|
56
47
|
|
|
57
48
|
# Adds each delta to the current value, treating a NULL SUM as zero.
|
|
58
|
-
sig { params(existing: T.untyped, column_deltas: T::Hash[String, Numeric]).void }
|
|
59
49
|
def add_deltas!(existing, column_deltas)
|
|
60
50
|
new_values = column_deltas.to_h { |column, amount| [column, (existing[column] || 0) + amount] }
|
|
61
51
|
existing.update!(new_values)
|
|
62
52
|
end
|
|
63
53
|
|
|
64
|
-
sig { params(existing: T.untyped, column_deltas: T::Hash[String, Numeric]).returns(T::Boolean) }
|
|
65
54
|
def emptied?(existing, column_deltas)
|
|
66
55
|
column = @analysis.row_count_column
|
|
67
56
|
return false if column.nil?
|