activerecord-materialized 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +93 -0
- data/README.md +138 -323
- data/lib/activerecord/materialized/aggregate_analysis.rb +10 -31
- data/lib/activerecord/materialized/async_refresher.rb +18 -27
- data/lib/activerecord/materialized/cache_table_schema.rb +53 -34
- data/lib/activerecord/materialized/change_source.rb +25 -0
- data/lib/activerecord/materialized/cold_read.rb +1 -10
- data/lib/activerecord/materialized/column_type_inference.rb +167 -0
- data/lib/activerecord/materialized/configurable_table_name.rb +45 -0
- data/lib/activerecord/materialized/configuration.rb +104 -29
- data/lib/activerecord/materialized/connection_routing.rb +29 -0
- data/lib/activerecord/materialized/data_verification_result.rb +41 -0
- data/lib/activerecord/materialized/data_verifier.rb +162 -0
- data/lib/activerecord/materialized/debezium_envelope.rb +105 -0
- data/lib/activerecord/materialized/delta_maintainer.rb +4 -15
- data/lib/activerecord/materialized/dependency_registry.rb +43 -35
- data/lib/activerecord/materialized/dependency_trackable.rb +7 -14
- data/lib/activerecord/materialized/incremental_maintainer.rb +45 -28
- data/lib/activerecord/materialized/incremental_refresh.rb +63 -0
- data/lib/activerecord/materialized/instrumentation.rb +102 -0
- data/lib/activerecord/materialized/maintenance_delta.rb +4 -19
- data/lib/activerecord/materialized/maintenance_delta_builder.rb +46 -14
- data/lib/activerecord/materialized/maintenance_store.rb +58 -21
- data/lib/activerecord/materialized/metadata/maintenance_payload.rb +1 -7
- data/lib/activerecord/materialized/metadata/reconciliation.rb +25 -0
- data/lib/activerecord/materialized/metadata/schema.rb +19 -41
- data/lib/activerecord/materialized/metadata/timestamps.rb +0 -5
- data/lib/activerecord/materialized/metadata.rb +41 -25
- data/lib/activerecord/materialized/metadata_record.rb +2 -16
- data/lib/activerecord/materialized/migration_builder.rb +8 -8
- data/lib/activerecord/materialized/module_api.rb +219 -22
- data/lib/activerecord/materialized/partition_filter.rb +45 -0
- data/lib/activerecord/materialized/partition_keyed_store.rb +72 -0
- data/lib/activerecord/materialized/partition_record.rb +2 -16
- data/lib/activerecord/materialized/partition_snapshot.rb +79 -0
- data/lib/activerecord/materialized/partition_state.rb +47 -46
- data/lib/activerecord/materialized/query_expressions.rb +0 -12
- data/lib/activerecord/materialized/railtie.rb +15 -3
- data/lib/activerecord/materialized/read_router.rb +51 -0
- data/lib/activerecord/materialized/reconcile_job.rb +22 -0
- data/lib/activerecord/materialized/reconcile_result.rb +36 -0
- data/lib/activerecord/materialized/reconciler.rb +79 -0
- data/lib/activerecord/materialized/refresh_callbacks.rb +4 -18
- data/lib/activerecord/materialized/refresh_job.rb +0 -4
- data/lib/activerecord/materialized/refresh_result.rb +2 -10
- data/lib/activerecord/materialized/refresh_scheduler.rb +5 -7
- data/lib/activerecord/materialized/refresher.rb +35 -50
- data/lib/activerecord/materialized/registry.rb +26 -16
- data/lib/activerecord/materialized/relation_cache_writer.rb +12 -46
- data/lib/activerecord/materialized/schema_verifier.rb +0 -9
- data/lib/activerecord/materialized/source_watermark.rb +93 -0
- data/lib/activerecord/materialized/source_watermark_record.rb +16 -0
- data/lib/activerecord/materialized/summary_delta.rb +1 -17
- data/lib/activerecord/materialized/summary_delta_builder.rb +2 -10
- data/lib/activerecord/materialized/table_model_registry.rb +1 -8
- data/lib/activerecord/materialized/table_swap.rb +106 -0
- data/lib/activerecord/materialized/tasks.rb +46 -38
- data/lib/activerecord/materialized/version.rb +1 -2
- data/lib/activerecord/materialized/view.rb +11 -22
- data/lib/activerecord/materialized/view_configuration_class_methods.rb +23 -35
- data/lib/activerecord/materialized/view_definition.rb +76 -65
- data/lib/activerecord/materialized/view_incremental_class_methods.rb +57 -60
- data/lib/activerecord/materialized/view_loader.rb +49 -0
- data/lib/activerecord/materialized/view_query_access_class_methods.rb +100 -57
- data/lib/activerecord/materialized/view_refresh_policy_class_methods.rb +70 -37
- data/lib/activerecord/materialized/write_change.rb +75 -26
- data/lib/activerecord/materialized/write_maintenance.rb +73 -0
- data/lib/activerecord/materialized/write_outbox.rb +345 -0
- data/lib/activerecord/materialized/write_outbox_record.rb +17 -0
- data/lib/activerecord/materialized.rb +27 -6
- data/lib/generators/activerecord_materialized/install/templates/create_ar_materialized_view_metadata.rb.erb +14 -0
- data/lib/generators/activerecord_materialized/install_generator.rb +1 -6
- data/lib/generators/activerecord_materialized/migration_generator.rb +3 -9
- data/lib/generators/activerecord_materialized/outbox_generator.rb +36 -0
- data/lib/generators/activerecord_materialized/templates/materialized_view_migration.rb.erb +5 -1
- data/lib/generators/activerecord_materialized/templates/write_outbox_migration.rb.erb +21 -0
- data/lib/generators/activerecord_materialized/view_generator.rb +0 -4
- metadata +27 -18
- data/lib/activerecord/materialized/type_reexports.rb +0 -14
- data/lib/activerecord/materialized/view_class.rb +0 -8
- data/lib/activerecord_materialized_types.rb +0 -18
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# typed: strict
|
|
2
1
|
# frozen_string_literal: true
|
|
3
2
|
|
|
4
3
|
module ActiveRecord
|
|
@@ -6,48 +5,76 @@ module ActiveRecord
|
|
|
6
5
|
# Incremental-maintenance DSL mixed into a {View}: `refresh_mode`, `incremental_keys`,
|
|
7
6
|
# `incremental_from`, and the per-write maintenance entry points.
|
|
8
7
|
module ViewIncrementalClassMethods
|
|
9
|
-
|
|
10
|
-
|
|
8
|
+
# The maintenance modes a view may declare (validated fail-fast by refresh_mode).
|
|
9
|
+
REFRESH_MODES = %i[incremental full].freeze
|
|
11
10
|
|
|
12
|
-
sig { params(base: T.class_of(View)).void }
|
|
13
11
|
def self.included(base)
|
|
14
12
|
base.extend(ClassMethods)
|
|
15
13
|
end
|
|
16
14
|
|
|
17
15
|
# The incremental-maintenance DSL methods available on a {View} subclass.
|
|
18
16
|
module ClassMethods
|
|
19
|
-
extend T::Sig
|
|
20
|
-
|
|
21
|
-
sig { returns(T.class_of(View)) }
|
|
22
17
|
def view_class
|
|
23
|
-
|
|
18
|
+
self
|
|
24
19
|
end
|
|
25
20
|
|
|
26
|
-
|
|
21
|
+
# Sets whether maintenance is incremental (default) or a full recompute on every change.
|
|
22
|
+
#
|
|
23
|
+
# @param mode [Symbol] +:incremental+ (default) or +:full+
|
|
24
|
+
# @return [void]
|
|
27
25
|
def refresh_mode(mode)
|
|
28
|
-
|
|
26
|
+
mode = mode.to_sym
|
|
27
|
+
unless REFRESH_MODES.include?(mode)
|
|
28
|
+
raise ArgumentError, "unknown refresh mode #{mode.inspect}; expected one of #{REFRESH_MODES.inspect}"
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
instance_variable_set(:@refresh_mode, mode)
|
|
29
32
|
end
|
|
30
33
|
|
|
31
|
-
|
|
34
|
+
# Overrides the source relation used for incremental maintenance (defaults to +materialized_from+).
|
|
35
|
+
#
|
|
36
|
+
# @yieldreturn [ActiveRecord::Relation] the relation to maintain incrementally
|
|
37
|
+
# @return [void]
|
|
32
38
|
def incremental_from(&block)
|
|
33
|
-
@incremental_source_definition =
|
|
39
|
+
@incremental_source_definition = block
|
|
34
40
|
end
|
|
35
41
|
|
|
36
|
-
|
|
42
|
+
# Declares the explicit GROUP BY key columns used to scope incremental maintenance.
|
|
43
|
+
#
|
|
44
|
+
# @param columns [Array<Symbol>] the key columns identifying a partition
|
|
45
|
+
# @return [void]
|
|
37
46
|
def incremental_keys(*columns)
|
|
38
|
-
@incremental_key_columns =
|
|
47
|
+
@incremental_key_columns = columns.map(&:to_s)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Maps a write on `table` to the partition key(s) it affects — for a view
|
|
51
|
+
# whose GROUP BY key lives on a joined/parent table, so the written row's own
|
|
52
|
+
# payload can't supply it. The block receives the {WriteChange} and returns
|
|
53
|
+
# the key value(s): a scalar (or array of scalars) for a single-column key, a
|
|
54
|
+
# tuple (or array of tuples) for a composite key; returning nothing falls back
|
|
55
|
+
# to a full recompute. Trades a small lookup per write for avoiding one.
|
|
56
|
+
#
|
|
57
|
+
# @param table [Symbol, String] the dependency table the resolver applies to
|
|
58
|
+
# @yieldparam change [WriteChange] the committed write on +table+
|
|
59
|
+
# @yieldreturn [Object, Array, nil] the affected partition key(s), or +nil+ to force a full recompute
|
|
60
|
+
# @return [void]
|
|
61
|
+
def partition_key_for(table, &block)
|
|
62
|
+
partition_key_resolvers[table.to_s] = block
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def partition_key_resolver_for(table_name)
|
|
66
|
+
partition_key_resolvers[table_name]
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def partition_key_resolvers
|
|
70
|
+
@partition_key_resolvers ||= {}
|
|
39
71
|
end
|
|
40
72
|
|
|
41
|
-
sig { returns(RefreshMode) }
|
|
42
73
|
def resolved_refresh_mode
|
|
43
|
-
mode =
|
|
44
|
-
T.unsafe(self).instance_variable_get(:@refresh_mode),
|
|
45
|
-
T.nilable(RefreshMode)
|
|
46
|
-
)
|
|
74
|
+
mode = instance_variable_get(:@refresh_mode)
|
|
47
75
|
mode || :incremental
|
|
48
76
|
end
|
|
49
77
|
|
|
50
|
-
sig { returns(ViewDefinition) }
|
|
51
78
|
def view_definition
|
|
52
79
|
ViewDefinition.new(
|
|
53
80
|
view_class.resolved_source,
|
|
@@ -55,73 +82,45 @@ module ActiveRecord
|
|
|
55
82
|
)
|
|
56
83
|
end
|
|
57
84
|
|
|
58
|
-
sig { returns(T::Array[String]) }
|
|
59
85
|
def maintenance_key_columns
|
|
60
86
|
return incremental_key_columns if incremental_key_columns.any?
|
|
61
87
|
|
|
62
88
|
view_definition.group_key_columns
|
|
63
89
|
end
|
|
64
90
|
|
|
65
|
-
sig { returns(T::Boolean) }
|
|
66
91
|
def incrementally_maintainable?
|
|
67
92
|
resolved_refresh_mode != :full && view_definition.incrementally_maintainable?
|
|
68
93
|
end
|
|
69
94
|
|
|
70
|
-
sig { returns(AggregateAnalysis) }
|
|
71
95
|
def aggregate_analysis
|
|
72
96
|
AggregateAnalysis.new(view_class.resolved_source)
|
|
73
97
|
end
|
|
74
98
|
|
|
75
99
|
# A warm view with all-distributive aggregates uses summary-delta IVM;
|
|
76
|
-
# otherwise writes drive scoped recompute.
|
|
77
|
-
|
|
100
|
+
# otherwise writes drive scoped recompute. Restricted to callback-backed
|
|
101
|
+
# views: their in-app writes arrive exactly once, so applying signed deltas
|
|
102
|
+
# is safe. Externally fed views recompute their partitions instead, so
|
|
103
|
+
# at-least-once or duplicate delivery converges rather than double-counting.
|
|
78
104
|
def delta_maintaining?
|
|
79
|
-
resolved_refresh_mode != :full &&
|
|
105
|
+
resolved_refresh_mode != :full &&
|
|
106
|
+
view_class.resolved_change_source == ChangeSource::CALLBACKS &&
|
|
107
|
+
view_class.materialized? &&
|
|
108
|
+
aggregate_analysis.delta_maintainable?
|
|
80
109
|
end
|
|
81
110
|
|
|
82
|
-
sig { returns(T::Boolean) }
|
|
83
111
|
def incremental_source_override?
|
|
84
112
|
!@incremental_source_definition.nil?
|
|
85
113
|
end
|
|
86
114
|
|
|
87
|
-
sig { params(change: WriteChange).void }
|
|
88
115
|
def record_write_change!(change)
|
|
89
|
-
|
|
90
|
-
return unless incrementally_maintainable?
|
|
91
|
-
|
|
92
|
-
delta = MaintenanceDeltaBuilder.new(change, maintenance_key_columns).build
|
|
93
|
-
record_write_delta!(delta)
|
|
94
|
-
|
|
95
|
-
# On a cold view the written partitions are no longer current; drop them
|
|
96
|
-
# from the fresh set until re-maintained.
|
|
97
|
-
return if view_class.materialized? || delta.full_partition?
|
|
98
|
-
|
|
99
|
-
PartitionState.new(view_class).mark_stale!(delta.key_tuples)
|
|
100
|
-
end
|
|
101
|
-
|
|
102
|
-
sig { params(change: WriteChange).void }
|
|
103
|
-
def record_summary_delta!(change)
|
|
104
|
-
summary = SummaryDeltaBuilder.new(change, aggregate_analysis, maintenance_key_columns).build
|
|
105
|
-
MaintenanceStore.new(view_class).merge!(summary) unless summary.empty?
|
|
116
|
+
WriteMaintenance.new(view_class).record!(change)
|
|
106
117
|
end
|
|
107
118
|
|
|
108
|
-
sig { params(delta: MaintenanceDelta).void }
|
|
109
|
-
def record_write_delta!(delta)
|
|
110
|
-
return unless incrementally_maintainable?
|
|
111
|
-
|
|
112
|
-
MaintenanceStore.new(view_class).merge!(delta)
|
|
113
|
-
end
|
|
114
|
-
|
|
115
|
-
sig { returns(T::Array[String]) }
|
|
116
119
|
def incremental_key_columns
|
|
117
|
-
columns =
|
|
118
|
-
T.unsafe(self).instance_variable_get(:@incremental_key_columns),
|
|
119
|
-
T.nilable(T::Array[String])
|
|
120
|
-
)
|
|
120
|
+
columns = instance_variable_get(:@incremental_key_columns)
|
|
121
121
|
columns.nil? ? [] : columns
|
|
122
122
|
end
|
|
123
123
|
|
|
124
|
-
sig { returns(::ActiveRecord::Relation) }
|
|
125
124
|
def resolved_incremental_source
|
|
126
125
|
unless incremental_source_override?
|
|
127
126
|
Kernel.raise ArgumentError,
|
|
@@ -135,8 +134,6 @@ module ActiveRecord
|
|
|
135
134
|
)
|
|
136
135
|
end
|
|
137
136
|
end
|
|
138
|
-
|
|
139
|
-
mixes_in_class_methods ClassMethods
|
|
140
137
|
end
|
|
141
138
|
end
|
|
142
139
|
end
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActiveRecord
|
|
4
|
+
module Materialized
|
|
5
|
+
# Eager-loads the view classes under {Configuration#view_load_paths} so their +depends_on+ commit
|
|
6
|
+
# callbacks are installed even under Zeitwerk's lazy loading (development/test). Invoked by the
|
|
7
|
+
# Railtie on boot and on each code reload (via +config.to_prepare+); idempotent.
|
|
8
|
+
#
|
|
9
|
+
# Without this, a view whose constant nothing has referenced yet is dormant — its +after_*_commit+
|
|
10
|
+
# hooks are never installed — so writes to its dependencies silently don't schedule maintenance
|
|
11
|
+
# until something first touches the class. In production (+config.eager_load = true+) every view
|
|
12
|
+
# already loads at boot, so this is a no-op there.
|
|
13
|
+
#
|
|
14
|
+
# @api private
|
|
15
|
+
class ViewLoader
|
|
16
|
+
class << self
|
|
17
|
+
# Load the configured view directories through Zeitwerk. A no-op outside Rails or when
|
|
18
|
+
# {Configuration#view_load_paths} is empty.
|
|
19
|
+
#
|
|
20
|
+
# @return [void]
|
|
21
|
+
def load!
|
|
22
|
+
return unless rails_application?
|
|
23
|
+
|
|
24
|
+
autoloader = ::Rails.autoloaders.main
|
|
25
|
+
ActiveRecord::Materialized.configuration.view_load_paths.each do |path|
|
|
26
|
+
absolute = ::Rails.root.join(path).to_s
|
|
27
|
+
eager_load_dir(autoloader, absolute) if File.directory?(absolute)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
def rails_application?
|
|
34
|
+
defined?(::Rails) && ::Rails.respond_to?(:application) && ::Rails.application
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Eager-load one directory through Zeitwerk so the view constants under it load and install
|
|
38
|
+
# their callbacks. +eager_load_dir+ is Rails' own mechanism — idempotent and reload-safe — but
|
|
39
|
+
# it raises for a directory this autoloader does not manage (an engine path, a non-autoloaded
|
|
40
|
+
# location), which is not ours to load, so that is ignored.
|
|
41
|
+
def eager_load_dir(autoloader, absolute)
|
|
42
|
+
autoloader.eager_load_dir(absolute)
|
|
43
|
+
rescue ::Zeitwerk::Error
|
|
44
|
+
nil
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# typed: strict
|
|
2
1
|
# frozen_string_literal: true
|
|
3
2
|
|
|
4
3
|
module ActiveRecord
|
|
@@ -9,79 +8,107 @@ module ActiveRecord
|
|
|
9
8
|
# Read and refresh API mixed into a {View}: `rebuild!`, `refresh!`, `refresh_if_stale!`,
|
|
10
9
|
# `materialized?`, `stale?`, `dirty?`, and the routed query methods.
|
|
11
10
|
module ViewQueryAccessClassMethods
|
|
12
|
-
extend T::Sig
|
|
13
|
-
extend T::Helpers
|
|
14
|
-
|
|
15
|
-
sig { params(base: T.class_of(View)).void }
|
|
16
11
|
def self.included(base)
|
|
17
12
|
base.extend(ClassMethods)
|
|
18
13
|
end
|
|
19
14
|
|
|
20
15
|
# The read and refresh methods available on a {View} subclass.
|
|
21
16
|
module ClassMethods
|
|
22
|
-
extend T::Sig
|
|
23
|
-
|
|
24
|
-
sig { returns(T.class_of(View)) }
|
|
25
17
|
def view_class
|
|
26
|
-
|
|
18
|
+
self
|
|
27
19
|
end
|
|
28
20
|
|
|
29
|
-
|
|
21
|
+
# Whether the view needs refreshing — dirty, never refreshed, or past its +max_staleness+.
|
|
22
|
+
#
|
|
23
|
+
# @return [Boolean]
|
|
30
24
|
def stale?
|
|
31
25
|
view_class.metadata.stale?
|
|
32
26
|
end
|
|
33
27
|
|
|
34
|
-
|
|
28
|
+
# Whether dependency writes have marked the view dirty since its last refresh.
|
|
29
|
+
#
|
|
30
|
+
# @return [Boolean]
|
|
35
31
|
def dirty?
|
|
36
32
|
view_class.metadata.dirty?
|
|
37
33
|
end
|
|
38
34
|
|
|
39
|
-
|
|
35
|
+
# Whether the view has been fully materialized; cold views are served read-through.
|
|
36
|
+
#
|
|
37
|
+
# @return [Boolean]
|
|
40
38
|
def warm?
|
|
41
39
|
view_class.metadata.warm?
|
|
42
40
|
end
|
|
43
41
|
|
|
44
42
|
# Reads are served from the cache only once warmed and the table exists;
|
|
45
43
|
# otherwise they fall through to the cold-read path.
|
|
46
|
-
|
|
44
|
+
#
|
|
45
|
+
# @return [Boolean]
|
|
47
46
|
def materialized?
|
|
48
47
|
view_class.table_exists? && view_class.metadata.warm?
|
|
49
48
|
end
|
|
50
49
|
|
|
51
|
-
|
|
50
|
+
# When the view was last refreshed, or +nil+ if it never has been.
|
|
51
|
+
#
|
|
52
|
+
# @return [Time, nil]
|
|
52
53
|
def last_refreshed_at
|
|
53
54
|
view_class.metadata.last_refreshed_at
|
|
54
55
|
end
|
|
55
56
|
|
|
56
|
-
|
|
57
|
+
# The oldest applied CDC source watermark across the view's partitions — its most-behind
|
|
58
|
+
# partition — or +nil+ if no watermarked change has been ingested (see {SourceWatermark}).
|
|
59
|
+
# Subtract from your source clock (same unit as the +source_ts+ you pass {Materialized.ingest_change})
|
|
60
|
+
# to get the view's freshness/lag.
|
|
61
|
+
#
|
|
62
|
+
# @return [Integer, nil]
|
|
63
|
+
def source_watermark
|
|
64
|
+
SourceWatermark.new(view_class).oldest
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# Whether a refresh is currently in progress for the view.
|
|
68
|
+
#
|
|
69
|
+
# @return [Boolean]
|
|
57
70
|
def refreshing?
|
|
58
71
|
view_class.metadata.refreshing?
|
|
59
72
|
end
|
|
60
73
|
|
|
61
|
-
sig { void }
|
|
62
74
|
def mark_dependencies_changed!
|
|
63
75
|
view_class.metadata.mark_dirty!
|
|
64
76
|
end
|
|
65
77
|
|
|
66
|
-
sig { returns(T::Boolean) }
|
|
67
78
|
def table_exists?
|
|
68
79
|
view_class.connection.data_source_exists?(view_class.table_name)
|
|
69
80
|
end
|
|
70
81
|
|
|
71
82
|
# Incremental maintenance only — never scans all base data.
|
|
72
|
-
|
|
83
|
+
#
|
|
84
|
+
# @return [RefreshResult]
|
|
73
85
|
def refresh!
|
|
74
86
|
Refresher.new(view_class).refresh!
|
|
75
87
|
end
|
|
76
88
|
|
|
77
|
-
|
|
89
|
+
# Refreshes the view only when it is materialized and stale; otherwise a no-op.
|
|
90
|
+
#
|
|
91
|
+
# @return [RefreshResult, nil] the refresh result, or +nil+ when no refresh was needed
|
|
78
92
|
def refresh_if_stale!
|
|
79
93
|
refresh! if materialized? && stale?
|
|
80
94
|
end
|
|
81
95
|
|
|
96
|
+
# Verify this view's contents against its source and repair any drift with
|
|
97
|
+
# scoped maintenance (never a full rebuild). See {Reconciler}.
|
|
98
|
+
#
|
|
99
|
+
# @param mode [Symbol] drift-check depth: +:row_count+, +:checksum+, or +:full+
|
|
100
|
+
# @param sample [Numeric, nil] verify a random subset (Integer count / Float fraction)
|
|
101
|
+
# @return [ReconcileResult]
|
|
102
|
+
def reconcile!(mode: :checksum, sample: nil)
|
|
103
|
+
Reconciler.new(view_class, mode: mode, sample: sample).reconcile!
|
|
104
|
+
end
|
|
105
|
+
|
|
82
106
|
# The only path that scans all base data; `confirm:` guards against
|
|
83
107
|
# firing a full materialization by accident.
|
|
84
|
-
|
|
108
|
+
#
|
|
109
|
+
# @param confirm [Boolean] must be +true+ to run the full materialization
|
|
110
|
+
# @raise [ArgumentError] unless +confirm+ is true
|
|
111
|
+
# @return [RefreshResult]
|
|
85
112
|
def rebuild!(confirm: false)
|
|
86
113
|
unless confirm
|
|
87
114
|
Kernel.raise ArgumentError,
|
|
@@ -91,70 +118,86 @@ module ActiveRecord
|
|
|
91
118
|
Refresher.new(view_class).rebuild!
|
|
92
119
|
end
|
|
93
120
|
|
|
94
|
-
sig { params(args: T.untyped).returns(T.untyped) }
|
|
95
121
|
def all(*args)
|
|
96
|
-
|
|
122
|
+
read_router.scope.all(*args)
|
|
97
123
|
end
|
|
98
124
|
|
|
99
|
-
sig { params(args: T.untyped).returns(T.untyped) }
|
|
100
125
|
def where(*args)
|
|
101
|
-
partition_scope(args).where(*args)
|
|
126
|
+
read_router.partition_scope(args).where(*args)
|
|
102
127
|
end
|
|
103
128
|
|
|
104
|
-
sig { params(args: T.untyped).returns(T.untyped) }
|
|
105
129
|
def find(*args)
|
|
106
|
-
|
|
130
|
+
read_router.scope.find(*args)
|
|
107
131
|
end
|
|
108
132
|
|
|
109
|
-
sig { params(args: T.untyped).returns(T.untyped) }
|
|
110
133
|
def find_by(*args)
|
|
111
|
-
partition_scope(args).find_by(*args)
|
|
134
|
+
read_router.partition_scope(args).find_by(*args)
|
|
112
135
|
end
|
|
113
136
|
|
|
114
|
-
sig { params(args: T.untyped).returns(T.untyped) }
|
|
115
137
|
def count(*args)
|
|
116
|
-
|
|
138
|
+
read_router.scope.count(*args)
|
|
117
139
|
end
|
|
118
140
|
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
141
|
+
# Order-dependent finders (+first+, +last+, +second+, …) fall back to an implicit
|
|
142
|
+
# +ORDER BY <primary key>+ when the relation carries no order. A cold view reads through to
|
|
143
|
+
# the source as a derived table with no +id+ column, so that implicit order references a
|
|
144
|
+
# column that does not exist and the query blows up. Order by the GROUP BY key(s) instead —
|
|
145
|
+
# they exist on both the cold derived table and the warm cache — and the trailing +nil+ tells
|
|
146
|
+
# ActiveRecord's +_order_columns+ to use only these columns and NOT append the primary key.
|
|
147
|
+
# A non-grouped view has no such key, so it keeps the default primary-key behavior.
|
|
148
|
+
#
|
|
149
|
+
# The keys are unqualified (a dotted GROUP BY like +"items.category"+ maps to the projected
|
|
150
|
+
# +category+ on the cache/derived table), matching how {ViewDefinition} and {CacheTableSchema}
|
|
151
|
+
# resolve them — a qualified name would reference a column neither table has, on warm and cold.
|
|
152
|
+
#
|
|
153
|
+
# @return [Array, Object] the group-by key columns (plus a trailing nil), or the default
|
|
154
|
+
def implicit_order_column
|
|
155
|
+
keys = maintenance_key_columns.map { |key| key.rpartition(".").last }
|
|
156
|
+
keys.any? ? [*keys, nil] : super
|
|
124
157
|
end
|
|
125
158
|
|
|
126
|
-
#
|
|
127
|
-
#
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
159
|
+
# +ids+ and batch iteration (+find_each+/+find_in_batches+/+in_batches+) are defined in terms
|
|
160
|
+
# of the primary key: +ids+ plucks it, and batching cursors on it and requires the cursor to be
|
|
161
|
+
# a unique column. A cold view's read-through derived table has no +id+ (and no index to make a
|
|
162
|
+
# group-key cursor unique), so none of these can be served correctly — a warm, materialized
|
|
163
|
+
# cache table is required. Refuse with a clear, actionable error rather than emitting the
|
|
164
|
+
# cryptic "no such column: id" the underlying query would raise.
|
|
165
|
+
def ids(*args)
|
|
166
|
+
require_materialized!(:ids)
|
|
167
|
+
super
|
|
168
|
+
end
|
|
131
169
|
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
170
|
+
def find_each(*args, **opts, &block)
|
|
171
|
+
require_materialized!(:find_each)
|
|
172
|
+
super
|
|
173
|
+
end
|
|
135
174
|
|
|
136
|
-
|
|
137
|
-
|
|
175
|
+
def find_in_batches(*args, **opts, &block)
|
|
176
|
+
require_materialized!(:find_in_batches)
|
|
177
|
+
super
|
|
138
178
|
end
|
|
139
179
|
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
180
|
+
def in_batches(*args, **opts, &block)
|
|
181
|
+
require_materialized!(:in_batches)
|
|
182
|
+
super
|
|
143
183
|
end
|
|
144
184
|
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
185
|
+
private
|
|
186
|
+
|
|
187
|
+
# These methods need the materialized cache table (a stable primary key / unique cursor);
|
|
188
|
+
# a cold read-through has none, so refuse with guidance instead of a misleading SQL error.
|
|
189
|
+
def require_materialized!(method_name)
|
|
190
|
+
return if materialized?
|
|
191
|
+
|
|
192
|
+
Kernel.raise NotMaterializedError,
|
|
193
|
+
"#{view_class.name}.#{method_name} requires a materialized view; " \
|
|
194
|
+
"run #{view_class.name}.rebuild!(confirm: true) first"
|
|
148
195
|
end
|
|
149
196
|
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
MaintenanceStore.new(view_class).merge!(MaintenanceDelta.scoped(keys))
|
|
153
|
-
RefreshScheduler.schedule(view_class)
|
|
197
|
+
def read_router
|
|
198
|
+
ReadRouter.new(view_class)
|
|
154
199
|
end
|
|
155
200
|
end
|
|
156
|
-
|
|
157
|
-
mixes_in_class_methods ClassMethods
|
|
158
201
|
end
|
|
159
202
|
end
|
|
160
203
|
end
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# typed: strict
|
|
2
1
|
# frozen_string_literal: true
|
|
3
2
|
|
|
4
3
|
module ActiveRecord
|
|
@@ -6,73 +5,104 @@ module ActiveRecord
|
|
|
6
5
|
# Refresh-policy DSL mixed into a {View}: `refresh_on_change`, `refresh_debounce`, `cold_read`,
|
|
7
6
|
# `warm_up`, `max_staleness`, plus `warm_up!`.
|
|
8
7
|
module ViewRefreshPolicyClassMethods
|
|
9
|
-
extend T::Sig
|
|
10
|
-
extend T::Helpers
|
|
11
|
-
|
|
12
|
-
sig { params(base: T.class_of(View)).void }
|
|
13
8
|
def self.included(base)
|
|
14
9
|
base.extend(ClassMethods)
|
|
15
10
|
end
|
|
16
11
|
|
|
17
12
|
# The refresh-policy DSL methods available on a {View} subclass.
|
|
18
13
|
module ClassMethods
|
|
19
|
-
extend T::Sig
|
|
20
|
-
|
|
21
|
-
sig { returns(T.class_of(View)) }
|
|
22
14
|
def view_class
|
|
23
|
-
|
|
15
|
+
self
|
|
24
16
|
end
|
|
25
17
|
|
|
26
|
-
|
|
18
|
+
# Sets the strategy used to refresh the view when a dependency changes.
|
|
19
|
+
#
|
|
20
|
+
# @param strategy [Symbol] one of +:async+ (default), +:immediate+, or +:manual+
|
|
21
|
+
# @raise [ArgumentError] if +strategy+ is not a known refresh strategy
|
|
22
|
+
# @return [void]
|
|
27
23
|
def refresh_on_change(strategy = :async)
|
|
28
|
-
|
|
24
|
+
strategy = strategy.to_sym
|
|
25
|
+
unless RefreshScheduler::STRATEGIES.include?(strategy)
|
|
26
|
+
raise ArgumentError,
|
|
27
|
+
"unknown refresh strategy #{strategy.inspect}; expected one of #{RefreshScheduler::STRATEGIES.inspect}"
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
@refresh_strategy = strategy
|
|
29
31
|
end
|
|
30
32
|
|
|
31
|
-
|
|
33
|
+
# Sets how long successive async refreshes are coalesced before one runs.
|
|
34
|
+
#
|
|
35
|
+
# @param seconds [Numeric, ActiveSupport::Duration] the debounce interval
|
|
36
|
+
# @return [void]
|
|
32
37
|
def refresh_debounce(seconds)
|
|
33
|
-
|
|
38
|
+
unless seconds.is_a?(Numeric) || seconds.is_a?(::ActiveSupport::Duration)
|
|
39
|
+
raise ArgumentError, "refresh_debounce expects seconds (a number or Duration), got #{seconds.inspect}"
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
@refresh_debounce = seconds
|
|
34
43
|
end
|
|
35
44
|
|
|
36
|
-
|
|
45
|
+
# Sets how reads are served before the view has been materialized.
|
|
46
|
+
#
|
|
47
|
+
# @param strategy [Symbol] one of +:read_through+ (default), +:serve_stale+, or +:raise+
|
|
48
|
+
# @return [void]
|
|
37
49
|
def cold_read(strategy)
|
|
38
|
-
@cold_read_strategy =
|
|
50
|
+
@cold_read_strategy = strategy.to_sym
|
|
39
51
|
end
|
|
40
52
|
|
|
41
|
-
sig { returns(Symbol) }
|
|
42
53
|
def resolved_cold_read_strategy
|
|
43
|
-
|
|
54
|
+
@cold_read_strategy ||
|
|
44
55
|
ActiveRecord::Materialized.configuration.default_cold_read_strategy
|
|
45
56
|
end
|
|
46
57
|
|
|
58
|
+
# Selects where this view's changes come from: +:callbacks+ (the default
|
|
59
|
+
# built-in tracker installs commit callbacks on +depends_on+ models) or
|
|
60
|
+
# +:none+ (install no callbacks; feed changes through the public
|
|
61
|
+
# ingestion API from an external adapter — e.g. a CDC stream).
|
|
62
|
+
#
|
|
63
|
+
# Setting +:callbacks+ (re)installs callbacks for any dependencies already
|
|
64
|
+
# declared, so it works regardless of whether it precedes or follows
|
|
65
|
+
# +depends_on+ — important when the global default is +:none+.
|
|
66
|
+
#
|
|
67
|
+
# @param source [Symbol] +:callbacks+ or +:none+
|
|
68
|
+
# @return [void]
|
|
69
|
+
def change_source(source)
|
|
70
|
+
@change_source = ChangeSource.cast(source)
|
|
71
|
+
DependencyRegistry.install_callbacks_for(view_class) if @change_source == ChangeSource::CALLBACKS
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def resolved_change_source
|
|
75
|
+
@change_source ||
|
|
76
|
+
ActiveRecord::Materialized.configuration.default_change_source
|
|
77
|
+
end
|
|
78
|
+
|
|
47
79
|
# Queries warm_up! runs to materialize a cold view's hot partitions, e.g.:
|
|
48
80
|
# warm_up { [where(region: "us"), order(revenue: :desc).limit(50)] }
|
|
49
|
-
|
|
81
|
+
#
|
|
82
|
+
# @yieldreturn [Array<ActiveRecord::Relation>] the relations whose partitions to warm
|
|
83
|
+
# @return [void]
|
|
50
84
|
def warm_up(&block)
|
|
51
|
-
@warm_up_definition =
|
|
85
|
+
@warm_up_definition = block
|
|
52
86
|
end
|
|
53
87
|
|
|
54
|
-
sig { returns(T::Array[::ActiveRecord::Relation]) }
|
|
55
88
|
def resolved_warm_up_queries
|
|
56
|
-
block =
|
|
89
|
+
block = @warm_up_definition
|
|
57
90
|
return [] if block.nil?
|
|
58
91
|
|
|
59
|
-
Kernel.Array(
|
|
92
|
+
Kernel.Array(view_class.instance_eval(&block))
|
|
60
93
|
end
|
|
61
94
|
|
|
62
95
|
# Running each warm_up query enqueues scoped maintenance for the
|
|
63
96
|
# partitions it touches; refresh! then applies it.
|
|
64
|
-
sig { returns(T.nilable(RefreshResult)) }
|
|
65
97
|
def warm_up!
|
|
66
98
|
resolved_warm_up_queries.each(&:to_a)
|
|
67
99
|
view_class.refresh!
|
|
68
100
|
end
|
|
69
101
|
|
|
70
|
-
sig { returns(Symbol) }
|
|
71
102
|
def resolved_refresh_strategy
|
|
72
103
|
@refresh_strategy || ActiveRecord::Materialized.configuration.default_refresh_strategy
|
|
73
104
|
end
|
|
74
105
|
|
|
75
|
-
sig { returns(T.any(Integer, Float)) }
|
|
76
106
|
def resolved_refresh_debounce
|
|
77
107
|
interval = if @refresh_debounce.nil?
|
|
78
108
|
ActiveRecord::Materialized.configuration.default_refresh_debounce
|
|
@@ -82,28 +112,31 @@ module ActiveRecord
|
|
|
82
112
|
interval.respond_to?(:to_f) ? interval.to_f : interval.to_i
|
|
83
113
|
end
|
|
84
114
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
115
|
+
# Sets the maximum staleness window before the view is treated as stale — a static
|
|
116
|
+
# duration, or a block evaluated in the view's context for a dynamic window.
|
|
117
|
+
#
|
|
118
|
+
# @param duration [Integer, ActiveSupport::Duration, nil] a static staleness window (seconds)
|
|
119
|
+
# @yieldreturn [Integer, ActiveSupport::Duration] a dynamically computed staleness window
|
|
120
|
+
# @return [void]
|
|
91
121
|
def max_staleness(duration = nil, &block)
|
|
92
|
-
|
|
122
|
+
setting = duration || block
|
|
123
|
+
unless setting.nil? || [Integer, ::ActiveSupport::Duration, Proc].any? { |type| setting.is_a?(type) }
|
|
124
|
+
raise ArgumentError,
|
|
125
|
+
"max_staleness expects an Integer (seconds), a Duration, or a block, got #{duration.inspect}"
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
@max_staleness_setting = setting
|
|
93
129
|
end
|
|
94
130
|
|
|
95
|
-
sig { returns(T.nilable(StalenessDuration)) }
|
|
96
131
|
def resolved_max_staleness
|
|
97
132
|
setting = @max_staleness_setting
|
|
98
133
|
default = ActiveRecord::Materialized.configuration.default_max_staleness
|
|
99
|
-
return
|
|
100
|
-
return
|
|
134
|
+
return default if setting.nil?
|
|
135
|
+
return view_class.instance_eval(&setting) if setting.is_a?(Proc)
|
|
101
136
|
|
|
102
137
|
setting
|
|
103
138
|
end
|
|
104
139
|
end
|
|
105
|
-
|
|
106
|
-
mixes_in_class_methods ClassMethods
|
|
107
140
|
end
|
|
108
141
|
end
|
|
109
142
|
end
|