activerecord-materialized 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +93 -0
  3. data/README.md +138 -323
  4. data/lib/activerecord/materialized/aggregate_analysis.rb +10 -31
  5. data/lib/activerecord/materialized/async_refresher.rb +18 -27
  6. data/lib/activerecord/materialized/cache_table_schema.rb +53 -34
  7. data/lib/activerecord/materialized/change_source.rb +25 -0
  8. data/lib/activerecord/materialized/cold_read.rb +1 -10
  9. data/lib/activerecord/materialized/column_type_inference.rb +167 -0
  10. data/lib/activerecord/materialized/configurable_table_name.rb +45 -0
  11. data/lib/activerecord/materialized/configuration.rb +104 -29
  12. data/lib/activerecord/materialized/connection_routing.rb +29 -0
  13. data/lib/activerecord/materialized/data_verification_result.rb +41 -0
  14. data/lib/activerecord/materialized/data_verifier.rb +162 -0
  15. data/lib/activerecord/materialized/debezium_envelope.rb +105 -0
  16. data/lib/activerecord/materialized/delta_maintainer.rb +4 -15
  17. data/lib/activerecord/materialized/dependency_registry.rb +43 -35
  18. data/lib/activerecord/materialized/dependency_trackable.rb +7 -14
  19. data/lib/activerecord/materialized/incremental_maintainer.rb +45 -28
  20. data/lib/activerecord/materialized/incremental_refresh.rb +63 -0
  21. data/lib/activerecord/materialized/instrumentation.rb +102 -0
  22. data/lib/activerecord/materialized/maintenance_delta.rb +4 -19
  23. data/lib/activerecord/materialized/maintenance_delta_builder.rb +46 -14
  24. data/lib/activerecord/materialized/maintenance_store.rb +58 -21
  25. data/lib/activerecord/materialized/metadata/maintenance_payload.rb +1 -7
  26. data/lib/activerecord/materialized/metadata/reconciliation.rb +25 -0
  27. data/lib/activerecord/materialized/metadata/schema.rb +19 -41
  28. data/lib/activerecord/materialized/metadata/timestamps.rb +0 -5
  29. data/lib/activerecord/materialized/metadata.rb +41 -25
  30. data/lib/activerecord/materialized/metadata_record.rb +2 -16
  31. data/lib/activerecord/materialized/migration_builder.rb +8 -8
  32. data/lib/activerecord/materialized/module_api.rb +219 -22
  33. data/lib/activerecord/materialized/partition_filter.rb +45 -0
  34. data/lib/activerecord/materialized/partition_keyed_store.rb +72 -0
  35. data/lib/activerecord/materialized/partition_record.rb +2 -16
  36. data/lib/activerecord/materialized/partition_snapshot.rb +79 -0
  37. data/lib/activerecord/materialized/partition_state.rb +47 -46
  38. data/lib/activerecord/materialized/query_expressions.rb +0 -12
  39. data/lib/activerecord/materialized/railtie.rb +15 -3
  40. data/lib/activerecord/materialized/read_router.rb +51 -0
  41. data/lib/activerecord/materialized/reconcile_job.rb +22 -0
  42. data/lib/activerecord/materialized/reconcile_result.rb +36 -0
  43. data/lib/activerecord/materialized/reconciler.rb +79 -0
  44. data/lib/activerecord/materialized/refresh_callbacks.rb +4 -18
  45. data/lib/activerecord/materialized/refresh_job.rb +0 -4
  46. data/lib/activerecord/materialized/refresh_result.rb +2 -10
  47. data/lib/activerecord/materialized/refresh_scheduler.rb +5 -7
  48. data/lib/activerecord/materialized/refresher.rb +35 -50
  49. data/lib/activerecord/materialized/registry.rb +26 -16
  50. data/lib/activerecord/materialized/relation_cache_writer.rb +12 -46
  51. data/lib/activerecord/materialized/schema_verifier.rb +0 -9
  52. data/lib/activerecord/materialized/source_watermark.rb +93 -0
  53. data/lib/activerecord/materialized/source_watermark_record.rb +16 -0
  54. data/lib/activerecord/materialized/summary_delta.rb +1 -17
  55. data/lib/activerecord/materialized/summary_delta_builder.rb +2 -10
  56. data/lib/activerecord/materialized/table_model_registry.rb +1 -8
  57. data/lib/activerecord/materialized/table_swap.rb +106 -0
  58. data/lib/activerecord/materialized/tasks.rb +46 -38
  59. data/lib/activerecord/materialized/version.rb +1 -2
  60. data/lib/activerecord/materialized/view.rb +11 -22
  61. data/lib/activerecord/materialized/view_configuration_class_methods.rb +23 -35
  62. data/lib/activerecord/materialized/view_definition.rb +76 -65
  63. data/lib/activerecord/materialized/view_incremental_class_methods.rb +57 -60
  64. data/lib/activerecord/materialized/view_loader.rb +49 -0
  65. data/lib/activerecord/materialized/view_query_access_class_methods.rb +100 -57
  66. data/lib/activerecord/materialized/view_refresh_policy_class_methods.rb +70 -37
  67. data/lib/activerecord/materialized/write_change.rb +75 -26
  68. data/lib/activerecord/materialized/write_maintenance.rb +73 -0
  69. data/lib/activerecord/materialized/write_outbox.rb +345 -0
  70. data/lib/activerecord/materialized/write_outbox_record.rb +17 -0
  71. data/lib/activerecord/materialized.rb +27 -6
  72. data/lib/generators/activerecord_materialized/install/templates/create_ar_materialized_view_metadata.rb.erb +14 -0
  73. data/lib/generators/activerecord_materialized/install_generator.rb +1 -6
  74. data/lib/generators/activerecord_materialized/migration_generator.rb +3 -9
  75. data/lib/generators/activerecord_materialized/outbox_generator.rb +36 -0
  76. data/lib/generators/activerecord_materialized/templates/materialized_view_migration.rb.erb +5 -1
  77. data/lib/generators/activerecord_materialized/templates/write_outbox_migration.rb.erb +21 -0
  78. data/lib/generators/activerecord_materialized/view_generator.rb +0 -4
  79. metadata +27 -18
  80. data/lib/activerecord/materialized/type_reexports.rb +0 -14
  81. data/lib/activerecord/materialized/view_class.rb +0 -8
  82. data/lib/activerecord_materialized_types.rb +0 -18
@@ -1,4 +1,3 @@
1
- # typed: strict
2
1
  # frozen_string_literal: true
3
2
 
4
3
  module ActiveRecord
@@ -6,74 +5,150 @@ module ActiveRecord
6
5
  # Global, app-wide defaults, set via {Materialized.configure}. Individual
7
6
  # views can override most of these with the corresponding DSL macro.
8
7
  class Configuration
9
- extend T::Sig
10
-
11
8
  # @return [String] name of the metadata table (default "ar_materialized_view_metadata")
12
- sig { returns(String) }
13
9
  attr_accessor :metadata_table_name
14
10
 
15
11
  # @return [String] name of the per-partition freshness table
16
- sig { returns(String) }
17
12
  attr_accessor :partition_table_name
18
13
 
14
+ # @return [String] name of the write-outbox table (the optional trigger/outbox change source)
15
+ attr_accessor :write_outbox_table_name
16
+
17
+ # @return [String] name of the CDC source-watermark table (per-partition applied-watermark tracking)
18
+ attr_accessor :source_watermark_table_name
19
+
19
20
  # @return [Numeric, ActiveSupport::Duration, nil] default {ViewRefreshPolicyClassMethods::ClassMethods#max_staleness max_staleness}
20
- sig { returns(T.nilable(DebounceInterval)) }
21
21
  attr_accessor :default_max_staleness
22
22
 
23
23
  # @return [Integer, nil] optional per-refresh timeout in seconds
24
- sig { returns(T.nilable(Integer)) }
25
24
  attr_accessor :refresh_timeout
26
25
 
27
26
  # @return [Boolean] whether a full refresh swaps a freshly built table in atomically
28
- sig { returns(T::Boolean) }
29
27
  attr_accessor :atomic_swap_refresh
30
28
 
31
29
  # @return [Symbol] default refresh strategy: +:async+, +:immediate+, or +:manual+
32
- sig { returns(Symbol) }
33
30
  attr_accessor :default_refresh_strategy
34
31
 
35
32
  # @return [Numeric] default debounce window (seconds) for coalescing async refreshes
36
- sig { returns(DebounceInterval) }
37
33
  attr_accessor :default_refresh_debounce
38
34
 
39
- # @return [Symbol] background dispatcher: +:async+ (in-process thread) or +:active_job+
40
- sig { returns(Symbol) }
41
- attr_accessor :refresh_dispatcher
35
+ # Background dispatcher: +:active_job+ or +:async+ (in-process thread). Unset, it resolves
36
+ # to +:active_job+ when ActiveJob is loaded and +:async+ otherwise; an explicit assignment
37
+ # always wins. The in-process +:async+ dispatcher does not coordinate across processes and
38
+ # its queue is lost on restart, so it is single-process-only — multi-process deployments
39
+ # should run +:active_job+.
40
+ #
41
+ # @return [Symbol]
42
+ attr_writer :refresh_dispatcher
43
+
44
+ def refresh_dispatcher
45
+ @refresh_dispatcher || (active_job_available? ? :active_job : :async)
46
+ end
42
47
 
43
48
  # @return [Symbol] ActiveJob queue name used when +refresh_dispatcher+ is +:active_job+
44
- sig { returns(Symbol) }
45
49
  attr_accessor :refresh_queue_name
46
50
 
51
+ # ActiveJob queue for {ReconcileJob} (the reconcile fan-out); falls back to
52
+ # +refresh_queue_name+ when unset so both maintenance jobs share a queue by default.
53
+ #
54
+ # @return [Symbol]
55
+ attr_writer :reconcile_queue_name
56
+
57
+ def reconcile_queue_name
58
+ @reconcile_queue_name || refresh_queue_name
59
+ end
60
+
61
+ # Rails multi-database role that maintenance writes (refresh/reconcile/rebuild) run under, so
62
+ # they target the primary in a writer/replica topology. +nil+ (default) yields on the current
63
+ # connection — no routing. Requires the host app to have declared the role via +connects_to+.
64
+ #
65
+ # @return [Symbol, nil]
66
+ attr_accessor :maintenance_role
67
+
68
+ # Rails multi-database role that {DataVerifier} reads run under, so the (expensive) drift
69
+ # verification can be offloaded to a replica — the verify-on-replica / repair-on-primary split.
70
+ # +nil+ (default) yields on the current connection.
71
+ #
72
+ # @return [Symbol, nil]
73
+ attr_accessor :verification_role
74
+
75
+ # Replication lag budget folded into time-based staleness: a view read from a replica is
76
+ # effectively `view staleness + replication lag` stale, so this tightens the effective
77
+ # +max_staleness+ (a view goes stale this much sooner) to keep replica reads within budget.
78
+ # Keep it well below your smallest +max_staleness+ — a value at or above it drives the effective
79
+ # window to zero, making that view perpetually stale (reconciled every tick). Defaults to 0 (no
80
+ # adjustment); a static estimate of a dynamic quantity — see the distributed-deployment guide.
81
+ #
82
+ # @return [Numeric, ActiveSupport::Duration]
83
+ attr_accessor :replica_lag
84
+
47
85
  # Cold-read behavior: :read_through (serve from source), :serve_stale
48
86
  # (serve the cache as-is), or :raise.
49
- sig { returns(Symbol) }
50
87
  attr_accessor :default_cold_read_strategy
51
88
 
89
+ # Default change source for views: +:callbacks+ (install ActiveRecord
90
+ # commit callbacks on +depends_on+ models) or +:none+ (install no
91
+ # callbacks; drive maintenance through the public ingestion API from an
92
+ # external adapter). A view can override this with +change_source+.
93
+ # Lazily defaulted (like +max_tracked_partitions+) so it stays out of
94
+ # +initialize+.
95
+ #
96
+ # @return [Symbol]
97
+ def default_change_source=(value)
98
+ @default_change_source = ChangeSource.cast(value)
99
+ end
100
+
101
+ def default_change_source
102
+ @default_change_source ||= ChangeSource::CALLBACKS
103
+ end
104
+
52
105
  # Cap on distinct partitions tracked in a view's pending maintenance before
53
106
  # it collapses to a single full recompute. Bounds the per-write cost of a
54
107
  # bulk write that spans many partitions. Defaults to 1000.
55
108
  #
56
109
  # @return [Integer]
57
- sig { params(max_tracked_partitions: Integer).void }
58
110
  attr_writer :max_tracked_partitions
59
111
 
60
- sig { returns(Integer) }
61
112
  def max_tracked_partitions
62
- @max_tracked_partitions ||= T.let(1_000, T.nilable(Integer))
113
+ @max_tracked_partitions ||= 1_000
63
114
  end
64
115
 
65
- sig { void }
116
+ # App-relative directories the Railtie eager-loads on boot (and on each dev reload) so every
117
+ # view class is loaded — and its +depends_on+ commit callbacks installed — even under Zeitwerk's
118
+ # lazy loading (`config.eager_load = false`, i.e. development and test). Without this, a view
119
+ # whose constant nothing has referenced yet has no callbacks, so writes to its dependencies
120
+ # silently don't schedule maintenance until something first touches the class. Defaults to
121
+ # +["app/models"]+ (where view classes conventionally live); set to +[]+ to disable, or list your
122
+ # own directories. Only existing directories that are Zeitwerk autoload roots are loaded — a path
123
+ # outside the autoloader (or a production app that already eager-loads) is a safe no-op.
124
+ #
125
+ # @return [Array<String>]
126
+ attr_accessor :view_load_paths
127
+
66
128
  def initialize
67
- @metadata_table_name = T.let("ar_materialized_view_metadata", String)
68
- @partition_table_name = T.let("ar_materialized_view_partitions", String)
69
- @default_max_staleness = T.let(nil, T.nilable(DebounceInterval))
70
- @refresh_timeout = T.let(nil, T.nilable(Integer))
71
- @atomic_swap_refresh = T.let(true, T::Boolean)
72
- @default_refresh_strategy = T.let(:async, Symbol)
73
- @default_refresh_debounce = T.let(2, DebounceInterval)
74
- @refresh_dispatcher = T.let(:async, Symbol)
75
- @refresh_queue_name = T.let(:materialized_views, Symbol)
76
- @default_cold_read_strategy = T.let(:read_through, Symbol)
129
+ @metadata_table_name = "ar_materialized_view_metadata"
130
+ @partition_table_name = "ar_materialized_view_partitions"
131
+ @write_outbox_table_name = "ar_materialized_view_write_outbox"
132
+ @source_watermark_table_name = "ar_materialized_view_source_watermarks"
133
+ @default_max_staleness = nil
134
+ @refresh_timeout = nil
135
+ @atomic_swap_refresh = true
136
+ @default_refresh_strategy = :async
137
+ @default_refresh_debounce = 2
138
+ # @refresh_dispatcher intentionally unset — lazily resolved from ActiveJob availability.
139
+ @refresh_queue_name = :materialized_views
140
+ @default_cold_read_strategy = :read_through
141
+ @replica_lag = 0
142
+ @view_load_paths = ["app/models"]
143
+ end
144
+
145
+ # Whether ActiveJob is loaded — the single source of truth for the dispatcher default and
146
+ # the module's dispatch/enqueue guards. Extracted (rather than an inline +defined?+) so it is
147
+ # testable regardless of whether the host app pulled in ActiveJob.
148
+ #
149
+ # @return [Boolean]
150
+ def active_job_available?
151
+ !defined?(ActiveJob::Base).nil?
77
152
  end
78
153
  end
79
154
  end
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveRecord
4
+ module Materialized
5
+ # Routes work to a Rails multi-database role when the app has configured one: maintenance writes
6
+ # to the primary, verification reads to a replica (the verify-on-replica / repair-on-primary
7
+ # split). A nil role — the default — yields on the current connection, so single-database apps
8
+ # and those relying on Rails' automatic role switching are unaffected.
9
+ #
10
+ # @api private
11
+ module ConnectionRouting
12
+ module_function
13
+
14
+ def maintenance(&block)
15
+ with_role(ActiveRecord::Materialized.configuration.maintenance_role, &block)
16
+ end
17
+
18
+ def verification(&block)
19
+ with_role(ActiveRecord::Materialized.configuration.verification_role, &block)
20
+ end
21
+
22
+ def with_role(role, &block)
23
+ return yield if role.nil?
24
+
25
+ ActiveRecord::Base.connected_to(role: role, &block)
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,41 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveRecord
4
+ module Materialized
5
+ # The outcome of a {DataVerifier} run: the partition keys whose materialized
6
+ # contents diverge from the source relation, plus how much was checked.
7
+ DataVerificationResult = Data.define(
8
+ :view_name,
9
+ :mode,
10
+ :total_partition_count,
11
+ :checked_partition_count,
12
+ # In the source relation but absent from the cache.
13
+ :missing_keys,
14
+ # In the cache but absent from the source relation.
15
+ :extra_keys,
16
+ # Present in both, but the partition's cache contents diverge. In +:full+ and
17
+ # +:checksum+ this means the value columns differ; in every mode (including
18
+ # +:row_count+) it also flags a partition the cache holds a different *number*
19
+ # of rows for than the source — a duplicated or lost row within the partition.
20
+ :mismatched_keys
21
+ ) do
22
+ # A clean result covering nothing — for a view that isn't materialized yet.
23
+ def self.empty(view_name:, mode:)
24
+ new(
25
+ view_name: view_name, mode: mode, total_partition_count: 0, checked_partition_count: 0,
26
+ missing_keys: [], extra_keys: [], mismatched_keys: []
27
+ )
28
+ end
29
+
30
+ def drifted?
31
+ missing_keys.any? || extra_keys.any? || mismatched_keys.any?
32
+ end
33
+
34
+ # Every diverging partition key, however it diverged — the set a reconciliation
35
+ # re-maintains (see {Reconciler}).
36
+ def divergent_keys
37
+ (missing_keys + extra_keys + mismatched_keys).uniq
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,162 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveRecord
4
+ module Materialized
5
+ # Detects DATA drift — the materialized cache diverging from what the source
6
+ # relation would produce right now — as opposed to {SchemaVerifier}'s schema
7
+ # drift. Stateless: it only reads and compares.
8
+ #
9
+ # Modes trade cost for depth:
10
+ # * +:row_count+ — which partitions exist (missing/extra); cheapest.
11
+ # * +:checksum+ — a per-partition digest of the value columns.
12
+ # * +:full+ — the value columns exactly.
13
+ #
14
+ # +sample:+ (Integer count / Float fraction) value-checks a random subset of
15
+ # partitions for large views; a sample covering every partition runs exhaustive.
16
+ #
17
+ # Covers grouped/aggregate views whose GROUP BY keys map to projected columns
18
+ # (the incremental-maintenance target). A non-grouped view, or one whose group
19
+ # key can't be matched to a projected column, is skipped (an empty result).
20
+ class DataVerifier
21
+ # Raised (by {Materialized.verify_data!}) when a view's contents have drifted.
22
+ class DataDriftError < StandardError; end
23
+
24
+ MODES = %i[row_count checksum full].freeze
25
+
26
+ def initialize(view_class, mode: :checksum, sample: nil)
27
+ raise ArgumentError, "unknown data-verification mode #{mode.inspect}" unless MODES.include?(mode)
28
+
29
+ @view_class = view_class
30
+ @mode = mode
31
+ @sample = sample
32
+ end
33
+
34
+ def verify
35
+ # verifiable? reads metadata/schema (and may provision the metadata table on first use), so it
36
+ # runs on the ambient/primary connection — never the read-only replica. Only the cache-vs-source
37
+ # comparison is routed to the verification role, inside one snapshot.
38
+ return empty_result unless verifiable?
39
+
40
+ ConnectionRouting.verification { in_consistent_snapshot { verify_within_snapshot } }
41
+ end
42
+
43
+ private
44
+
45
+ # Read the cache and the recomputed source inside one transaction, so both sides come from a
46
+ # single snapshot: a write landing mid-verify — or replication lag, when reads are pinned to a
47
+ # replica — can't make consistent data look like drift. SQLite has no explicit isolation levels
48
+ # (and its single-writer model already reads consistently), and an already-open transaction
49
+ # can't raise its level, so both fall back to a plain transaction.
50
+ def in_consistent_snapshot(&block)
51
+ @view_class.transaction(isolation: snapshot_isolation, &block)
52
+ end
53
+
54
+ def snapshot_isolation
55
+ connection = @view_class.connection
56
+ return nil if connection.open_transactions.positive? # can't raise isolation in a nested transaction
57
+ return nil if connection.adapter_name.match?(/sqlite/i) # /sqlite/i: SQLite has no explicit isolation levels
58
+
59
+ :repeatable_read
60
+ end
61
+
62
+ def verify_within_snapshot
63
+ return verify_exhaustive if @sample.nil?
64
+
65
+ keys = cache_partition_keys
66
+ size = sample_size(keys.size)
67
+ return verify_exhaustive if size >= keys.size # a sample covering everything is exhaustive
68
+ return empty_result if size.zero? # sample: 0 => nothing checked
69
+
70
+ verify_sampled(Array(keys.sample(size)), keys.size)
71
+ end
72
+
73
+ # Compares every partition, so it detects missing, extra, and mismatched.
74
+ def verify_exhaustive
75
+ cache = snapshot(@view_class.unscoped)
76
+ source = snapshot(@view_class.resolved_source)
77
+ total = (cache.keys | source.keys).size # every partition on either side
78
+ build_result(cache, source, total: total, checked: total, exhaustive: true)
79
+ end
80
+
81
+ # A value-drift probe over a random subset of materialized partitions. It
82
+ # cannot detect missing (source-only) partitions — that needs a full source
83
+ # scan — so it never reports them.
84
+ def verify_sampled(checked, total)
85
+ cache = snapshot(view_definition.partition_scope_on(@view_class, checked))
86
+ source = snapshot(view_definition.partition_scope(checked))
87
+ build_result(cache, source, total: total, checked: checked.size, exhaustive: false)
88
+ end
89
+
90
+ def build_result(cache, source, total:, checked:, exhaustive:)
91
+ shared = cache.keys & source.keys
92
+ DataVerificationResult.new(
93
+ view_name: @view_class.view_key, mode: @mode,
94
+ total_partition_count: total, checked_partition_count: checked,
95
+ missing_keys: exhaustive ? source.keys - cache.keys : [],
96
+ extra_keys: cache.keys - source.keys,
97
+ # Signature tallies differ on value drift and on a duplicated cache row
98
+ # (row_count's tally is of row presence, so it catches duplicates too).
99
+ mismatched_keys: shared.reject { |key| cache[key] == source[key] }
100
+ )
101
+ end
102
+
103
+ def snapshot(relation)
104
+ PartitionSnapshot.new(@view_class, key_columns, projected_columns - key_columns, @mode).of(relation)
105
+ end
106
+
107
+ # Verifiable when the view is a materialized aggregate whose group keys map to
108
+ # projected columns and whose cache table actually has those columns (a
109
+ # schema-drifted cache is skipped — run `verify_schema!` for that).
110
+ def verifiable?
111
+ keys = view_definition.group_key_columns
112
+ @view_class.materialized? && keys.any? && key_columns.size == keys.size &&
113
+ (projected_columns - @view_class.column_names).empty?
114
+ end
115
+
116
+ # Distinct group-key tuples in the cache — the population a sample is drawn from.
117
+ # `distinct` dedups in the database, so only one row per partition crosses into
118
+ # Ruby; the random draw stays in Ruby because a DB-side random order is neither an
119
+ # ActiveRecord primitive nor portable across adapters. `pluck` returns scalars for
120
+ # a single key (wrapped to 1-tuples so a NULL key becomes [nil], not the [] that
121
+ # Array(nil) would give) and tuples for a composite key.
122
+ def cache_partition_keys
123
+ values = @view_class.unscoped.distinct.pluck(*key_columns)
124
+ key_columns.one? ? values.zip : values
125
+ end
126
+
127
+ # GROUP BY keys as their projected/cache column names (qualifier stripped), so a
128
+ # joined key like "authors.country" matches the projected "country".
129
+ def key_columns
130
+ @key_columns ||= view_definition.group_key_columns.filter_map { |column| projected_column_for(column) }
131
+ end
132
+
133
+ def projected_column_for(column)
134
+ return column if projected_columns.include?(column)
135
+
136
+ stripped = column.split(".").last
137
+ projected_columns.include?(stripped) ? stripped : nil
138
+ end
139
+
140
+ def projected_columns
141
+ @projected_columns ||= CacheTableSchema.column_definitions(@view_class.connection,
142
+ @view_class.resolved_source).map(&:name)
143
+ end
144
+
145
+ def view_definition
146
+ @view_definition ||= @view_class.view_definition
147
+ end
148
+
149
+ def sample_size(total)
150
+ sample = @sample
151
+ return total if sample.nil?
152
+
153
+ size = sample.is_a?(Float) ? (total * sample).ceil : sample.to_i
154
+ size.clamp(0, total)
155
+ end
156
+
157
+ def empty_result
158
+ DataVerificationResult.empty(view_name: @view_class.view_key, mode: @mode)
159
+ end
160
+ end
161
+ end
162
+ end
@@ -0,0 +1,105 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveRecord
4
+ module Materialized
5
+ # Maps a Debezium change envelope to an {Materialized.ingest_change} descriptor. Pure and
6
+ # dependency-free: no Kafka / Debezium / Kafka-Connect runtime — just a mapping over the decoded
7
+ # envelope hash a consumer already holds. It removes the boilerplate (and the easy-to-miss cases:
8
+ # snapshot reads, tombstones, the nested/unwrapped shapes) every consumer otherwise hand-writes.
9
+ #
10
+ # The envelope shape (a Debezium MySQL/Postgres connector event, whether or not the
11
+ # ExtractNewRecordState "unwrap" SMT has been applied — a nested +payload+ is unwrapped here):
12
+ #
13
+ # { "op" => "u",
14
+ # "before" => { "id" => 1, "category" => "books", ... }, # full row image (see below)
15
+ # "after" => { "id" => 1, "category" => "games", ... },
16
+ # "ts_ms" => 1710000000000, # connector processing time (unused)
17
+ # "source" => { "table" => "line_items", "ts_ms" => 1710000000000, ... } }
18
+ #
19
+ # The +source.ts_ms+ (the DB commit time, monotonic within a source) is forwarded as the +source_ts+
20
+ # watermark, so a consumer relaying envelopes gets out-of-order suppression + freshness
21
+ # ({SourceWatermark}) for free. The top-level +ts_ms+ is a *different* clock (the connector's
22
+ # processing time), so it is not used as a fallback — mixing clocks would break monotonicity.
23
+ # Correct **partition-moving** updates require a full +before+ image (+binlog-row-image=FULL+ on
24
+ # MySQL, +REPLICA IDENTITY FULL+ on Postgres); with only the primary key in +before+, the old
25
+ # partition is under-maintained until reconciliation heals it. See +docs/out-of-band-writes.md+
26
+ # and the CDC section of the README. This adapter is Debezium-specific; a Maxwell/other envelope
27
+ # (no +op+) raises rather than being silently dropped.
28
+ module DebeziumEnvelope
29
+ # Debezium +op+ → the gem's operation. A snapshot read ("r") is a create; "d" is a destroy.
30
+ OPERATIONS = { "c" => :create, "r" => :create, "u" => :update, "d" => :destroy }.freeze
31
+ private_constant :OPERATIONS
32
+
33
+ # @param envelope [Hash, nil] a decoded Debezium change event (string or symbol keys), or +nil+
34
+ # for a Kafka tombstone (the null-value message emitted after a delete for log compaction)
35
+ # @param table [String, Symbol, nil] target-table override; defaults to the envelope's +source.table+
36
+ # @return [Hash, nil] +{ table:, operation:, before:, after: }+ (plus +source_ts:+ when the
37
+ # envelope carries a usable +ts_ms+) for {Materialized.ingest_change}, or +nil+ for a
38
+ # tombstone (nothing to relay)
39
+ # @raise [ArgumentError] for a non-tombstone envelope with no +op+ (not a Debezium change event,
40
+ # or not unwrapped), an unsupported +op+, or when the target table can't be determined
41
+ def self.to_change_descriptor(envelope, table = nil)
42
+ return nil if envelope.nil? # Kafka tombstone
43
+
44
+ change = unwrap(envelope)
45
+ op = fetch(change, "op")
46
+ raise ArgumentError, "not a Debezium change envelope (no op)" if op.nil?
47
+
48
+ descriptor = { table: resolve_table(table, change), operation: operation_for(op),
49
+ before: fetch(change, "before"), after: fetch(change, "after") }
50
+ ts = source_ts(change)
51
+ ts ? descriptor.merge(source_ts: ts) : descriptor
52
+ end
53
+
54
+ # Debezium's default envelope nests the change under +payload+; the ExtractNewRecordState SMT
55
+ # unwraps it to the top level. Accept either by unwrapping a +payload+ that carries the +op+.
56
+ def self.unwrap(envelope)
57
+ payload = fetch(envelope, "payload")
58
+ payload.is_a?(Hash) && fetch(payload, "op") ? payload : envelope
59
+ end
60
+ private_class_method :unwrap
61
+
62
+ # Read a key from an envelope that may use string or symbol keys, without deep-copying the hash
63
+ # (before/after images can be wide, and are copied again downstream by WriteChange).
64
+ def self.fetch(hash, key)
65
+ return nil unless hash.is_a?(Hash)
66
+
67
+ hash.key?(key) ? hash[key] : hash[key.to_sym]
68
+ end
69
+ private_class_method :fetch
70
+
71
+ # A field from the envelope's +source+ metadata block (string/symbol keys), or nil if absent.
72
+ def self.source_field(change, key)
73
+ fetch(fetch(change, "source"), key)
74
+ end
75
+ private_class_method :source_field
76
+
77
+ def self.operation_for(op_code)
78
+ OPERATIONS.fetch(op_code.to_s) do
79
+ raise ArgumentError, "unsupported Debezium op #{op_code.inspect} (expected one of c, r, u, d)"
80
+ end
81
+ end
82
+ private_class_method :operation_for
83
+
84
+ def self.resolve_table(table, change)
85
+ resolved = table.presence || source_field(change, "table")
86
+ return resolved.to_s if resolved.present?
87
+
88
+ raise ArgumentError, "could not determine the table (pass table or include source.table in the envelope)"
89
+ end
90
+ private_class_method :resolve_table
91
+
92
+ # The per-partition source watermark for this change: Debezium's +source.ts_ms+ (the DB commit
93
+ # time, monotonic within a source). Only an Integer is forwarded — a missing or non-integer value
94
+ # yields no watermark, so the change is maintained exactly as before (see {SourceWatermark}). The
95
+ # top-level +ts_ms+ (the connector's *processing* wall-clock) is intentionally NOT a fallback: it
96
+ # is a different clock, so mixing the two per partition would break the monotonicity suppression
97
+ # relies on. A real Debezium change event always carries +source.ts_ms+.
98
+ def self.source_ts(change)
99
+ ts = source_field(change, "ts_ms")
100
+ ts if ts.is_a?(Integer)
101
+ end
102
+ private_class_method :source_ts
103
+ end
104
+ end
105
+ end
@@ -1,4 +1,3 @@
1
- # typed: strict
2
1
  # frozen_string_literal: true
3
2
 
4
3
  module ActiveRecord
@@ -7,28 +6,22 @@ module ActiveRecord
7
6
  # re-reading base rows: new partitions inserted, existing ones incremented in
8
7
  # place (NULL-safe for SUM), and emptied partitions deleted.
9
8
  class DeltaMaintainer
10
- extend T::Sig
11
-
12
- sig { params(view_class: ViewClass).void }
13
9
  def initialize(view_class)
14
10
  @view_class = view_class
15
- @analysis = T.let(AggregateAnalysis.new(view_class.resolved_source), AggregateAnalysis)
11
+ @analysis = AggregateAnalysis.new(view_class.resolved_source)
16
12
  end
17
13
 
18
- sig { params(summary: SummaryDelta).returns(Integer) }
19
14
  def apply!(summary)
20
15
  summary.buckets.each { |key_tuple, column_deltas| apply_partition(key_tuple, column_deltas) }
21
- T.unsafe(@view_class).unscoped.count
16
+ summary.buckets.size # partitions applied; row-count reporting is the caller's job, outside the lock
22
17
  end
23
18
 
24
19
  private
25
20
 
26
- sig { returns(T::Array[String]) }
27
21
  def group_columns
28
22
  @view_class.maintenance_key_columns
29
23
  end
30
24
 
31
- sig { params(key_tuple: SummaryDelta::KeyTuple, column_deltas: T::Hash[String, Numeric]).void }
32
25
  def apply_partition(key_tuple, column_deltas)
33
26
  existing = partition_scope(key_tuple).first
34
27
  if existing.nil?
@@ -40,28 +33,24 @@ module ActiveRecord
40
33
  end
41
34
  end
42
35
 
43
- sig { params(key_tuple: SummaryDelta::KeyTuple).returns(::ActiveRecord::Relation) }
44
36
  def partition_scope(key_tuple)
45
- T.unsafe(@view_class).unscoped.where(group_columns.zip(key_tuple).to_h)
37
+ @view_class.unscoped.where(group_columns.zip(key_tuple).to_h)
46
38
  end
47
39
 
48
- sig { params(key_tuple: SummaryDelta::KeyTuple, column_deltas: T::Hash[String, Numeric]).void }
49
40
  def insert_partition(key_tuple, column_deltas)
50
41
  # A new partition's deltas are its absolute values; aggregates with no
51
42
  # delta default to 0, not NULL, since the partition has rows.
52
43
  defaults = @analysis.aggregate_columns.to_h { |column| [column.name, 0] }
53
44
  row = group_columns.zip(key_tuple).to_h.merge(defaults).merge(column_deltas)
54
- T.unsafe(@view_class).create!(row)
45
+ @view_class.create!(row)
55
46
  end
56
47
 
57
48
  # Adds each delta to the current value, treating a NULL SUM as zero.
58
- sig { params(existing: T.untyped, column_deltas: T::Hash[String, Numeric]).void }
59
49
  def add_deltas!(existing, column_deltas)
60
50
  new_values = column_deltas.to_h { |column, amount| [column, (existing[column] || 0) + amount] }
61
51
  existing.update!(new_values)
62
52
  end
63
53
 
64
- sig { params(existing: T.untyped, column_deltas: T::Hash[String, Numeric]).returns(T::Boolean) }
65
54
  def emptied?(existing, column_deltas)
66
55
  column = @analysis.row_count_column
67
56
  return false if column.nil?