activerecord-materialized 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +93 -0
- data/README.md +139 -324
- data/lib/activerecord/materialized/aggregate_analysis.rb +10 -31
- data/lib/activerecord/materialized/async_refresher.rb +18 -27
- data/lib/activerecord/materialized/cache_table_schema.rb +53 -34
- data/lib/activerecord/materialized/change_source.rb +25 -0
- data/lib/activerecord/materialized/cold_read.rb +1 -10
- data/lib/activerecord/materialized/column_type_inference.rb +167 -0
- data/lib/activerecord/materialized/configurable_table_name.rb +45 -0
- data/lib/activerecord/materialized/configuration.rb +104 -29
- data/lib/activerecord/materialized/connection_routing.rb +29 -0
- data/lib/activerecord/materialized/data_verification_result.rb +41 -0
- data/lib/activerecord/materialized/data_verifier.rb +162 -0
- data/lib/activerecord/materialized/debezium_envelope.rb +105 -0
- data/lib/activerecord/materialized/delta_maintainer.rb +4 -15
- data/lib/activerecord/materialized/dependency_registry.rb +43 -35
- data/lib/activerecord/materialized/dependency_trackable.rb +7 -14
- data/lib/activerecord/materialized/incremental_maintainer.rb +45 -28
- data/lib/activerecord/materialized/incremental_refresh.rb +63 -0
- data/lib/activerecord/materialized/instrumentation.rb +102 -0
- data/lib/activerecord/materialized/maintenance_delta.rb +4 -19
- data/lib/activerecord/materialized/maintenance_delta_builder.rb +46 -14
- data/lib/activerecord/materialized/maintenance_store.rb +58 -21
- data/lib/activerecord/materialized/metadata/maintenance_payload.rb +1 -7
- data/lib/activerecord/materialized/metadata/reconciliation.rb +25 -0
- data/lib/activerecord/materialized/metadata/schema.rb +19 -41
- data/lib/activerecord/materialized/metadata/timestamps.rb +0 -5
- data/lib/activerecord/materialized/metadata.rb +41 -25
- data/lib/activerecord/materialized/metadata_record.rb +2 -16
- data/lib/activerecord/materialized/migration_builder.rb +8 -8
- data/lib/activerecord/materialized/module_api.rb +219 -22
- data/lib/activerecord/materialized/partition_filter.rb +45 -0
- data/lib/activerecord/materialized/partition_keyed_store.rb +72 -0
- data/lib/activerecord/materialized/partition_record.rb +2 -16
- data/lib/activerecord/materialized/partition_snapshot.rb +79 -0
- data/lib/activerecord/materialized/partition_state.rb +47 -46
- data/lib/activerecord/materialized/query_expressions.rb +0 -12
- data/lib/activerecord/materialized/railtie.rb +15 -3
- data/lib/activerecord/materialized/read_router.rb +51 -0
- data/lib/activerecord/materialized/reconcile_job.rb +22 -0
- data/lib/activerecord/materialized/reconcile_result.rb +36 -0
- data/lib/activerecord/materialized/reconciler.rb +79 -0
- data/lib/activerecord/materialized/refresh_callbacks.rb +4 -18
- data/lib/activerecord/materialized/refresh_job.rb +0 -4
- data/lib/activerecord/materialized/refresh_result.rb +2 -10
- data/lib/activerecord/materialized/refresh_scheduler.rb +5 -7
- data/lib/activerecord/materialized/refresher.rb +35 -50
- data/lib/activerecord/materialized/registry.rb +26 -16
- data/lib/activerecord/materialized/relation_cache_writer.rb +12 -46
- data/lib/activerecord/materialized/schema_verifier.rb +0 -9
- data/lib/activerecord/materialized/source_watermark.rb +93 -0
- data/lib/activerecord/materialized/source_watermark_record.rb +16 -0
- data/lib/activerecord/materialized/summary_delta.rb +1 -17
- data/lib/activerecord/materialized/summary_delta_builder.rb +2 -10
- data/lib/activerecord/materialized/table_model_registry.rb +1 -8
- data/lib/activerecord/materialized/table_swap.rb +106 -0
- data/lib/activerecord/materialized/tasks.rb +46 -38
- data/lib/activerecord/materialized/version.rb +1 -2
- data/lib/activerecord/materialized/view.rb +11 -22
- data/lib/activerecord/materialized/view_configuration_class_methods.rb +23 -35
- data/lib/activerecord/materialized/view_definition.rb +76 -65
- data/lib/activerecord/materialized/view_incremental_class_methods.rb +57 -60
- data/lib/activerecord/materialized/view_loader.rb +49 -0
- data/lib/activerecord/materialized/view_query_access_class_methods.rb +100 -57
- data/lib/activerecord/materialized/view_refresh_policy_class_methods.rb +70 -37
- data/lib/activerecord/materialized/write_change.rb +75 -26
- data/lib/activerecord/materialized/write_maintenance.rb +73 -0
- data/lib/activerecord/materialized/write_outbox.rb +345 -0
- data/lib/activerecord/materialized/write_outbox_record.rb +17 -0
- data/lib/activerecord/materialized.rb +27 -6
- data/lib/generators/activerecord_materialized/install/templates/create_ar_materialized_view_metadata.rb.erb +14 -0
- data/lib/generators/activerecord_materialized/install_generator.rb +1 -6
- data/lib/generators/activerecord_materialized/migration_generator.rb +3 -9
- data/lib/generators/activerecord_materialized/outbox_generator.rb +36 -0
- data/lib/generators/activerecord_materialized/templates/materialized_view_migration.rb.erb +5 -1
- data/lib/generators/activerecord_materialized/templates/write_outbox_migration.rb.erb +21 -0
- data/lib/generators/activerecord_materialized/view_generator.rb +0 -4
- metadata +28 -19
- data/lib/activerecord/materialized/type_reexports.rb +0 -14
- data/lib/activerecord/materialized/view_class.rb +0 -8
- data/lib/activerecord_materialized_types.rb +0 -18
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# typed: strict
|
|
2
1
|
# frozen_string_literal: true
|
|
3
2
|
|
|
4
3
|
module ActiveRecord
|
|
@@ -10,47 +9,36 @@ module ActiveRecord
|
|
|
10
9
|
# so emptied partitions can be detected; everything else falls back to scoped
|
|
11
10
|
# recompute, which is always correct.
|
|
12
11
|
class AggregateAnalysis
|
|
13
|
-
extend T::Sig
|
|
14
|
-
|
|
15
12
|
# One classified aggregate column from a view's projection.
|
|
16
13
|
#
|
|
17
14
|
# @api private
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
const :function, Symbol
|
|
21
|
-
const :attribute, T.nilable(String)
|
|
22
|
-
const :counts_rows, T::Boolean, default: false
|
|
15
|
+
Column = Data.define(:name, :function, :attribute, :counts_rows) do
|
|
16
|
+
def initialize(name:, function:, attribute: nil, counts_rows: false) = super
|
|
23
17
|
end
|
|
24
18
|
|
|
25
|
-
sig { params(relation: ::ActiveRecord::Relation).void }
|
|
26
19
|
def initialize(relation)
|
|
27
20
|
@relation = relation
|
|
28
|
-
@aggregate_columns =
|
|
21
|
+
@aggregate_columns = nil
|
|
29
22
|
end
|
|
30
23
|
|
|
31
|
-
sig { returns(T::Array[Column]) }
|
|
32
24
|
def aggregate_columns
|
|
33
|
-
@aggregate_columns ||=
|
|
25
|
+
@aggregate_columns ||= @relation.select_values.filter_map { |value| classify(value) }
|
|
34
26
|
end
|
|
35
27
|
|
|
36
|
-
sig { returns(T::Boolean) }
|
|
37
28
|
def delta_maintainable?
|
|
38
29
|
single_table? && grouped? && !having? && distributive_aggregates? && aggregate_columns.any?(&:counts_rows)
|
|
39
30
|
end
|
|
40
31
|
|
|
41
|
-
sig { returns(T::Boolean) }
|
|
42
32
|
def distributive_aggregates?
|
|
43
33
|
aggregate_columns.any? && aggregate_columns.all? { |column| distributive?(column) }
|
|
44
34
|
end
|
|
45
35
|
|
|
46
|
-
sig { returns(T.nilable(Column)) }
|
|
47
36
|
def row_count_column
|
|
48
37
|
aggregate_columns.find(&:counts_rows)
|
|
49
38
|
end
|
|
50
39
|
|
|
51
40
|
private
|
|
52
41
|
|
|
53
|
-
sig { params(value: T.untyped).returns(T.nilable(Column)) }
|
|
54
42
|
def classify(value)
|
|
55
43
|
return nil unless value.is_a?(::Arel::Nodes::As)
|
|
56
44
|
|
|
@@ -66,7 +54,6 @@ module ActiveRecord
|
|
|
66
54
|
end
|
|
67
55
|
end
|
|
68
56
|
|
|
69
|
-
sig { params(name: String, node: T.untyped).returns(Column) }
|
|
70
57
|
def count_column(name, node)
|
|
71
58
|
return Column.new(name: name, function: :count_distinct, attribute: attribute_name(node)) if node.distinct
|
|
72
59
|
return Column.new(name: name, function: :count_star, counts_rows: true) if star?(node)
|
|
@@ -75,7 +62,6 @@ module ActiveRecord
|
|
|
75
62
|
Column.new(name: name, function: :count, attribute: attribute, counts_rows: not_null_column?(attribute))
|
|
76
63
|
end
|
|
77
64
|
|
|
78
|
-
sig { params(column: Column).returns(T::Boolean) }
|
|
79
65
|
def distributive?(column)
|
|
80
66
|
case column.function
|
|
81
67
|
# A SUM over a nullable column can be NULL, which a zero delta can't
|
|
@@ -87,45 +73,38 @@ module ActiveRecord
|
|
|
87
73
|
end
|
|
88
74
|
end
|
|
89
75
|
|
|
90
|
-
sig { params(node: T.untyped).returns(T.nilable(String)) }
|
|
91
76
|
def attribute_name(node)
|
|
92
77
|
inner = node.expressions.first
|
|
93
|
-
inner.is_a?(::Arel::Attributes::Attribute) ?
|
|
78
|
+
inner.is_a?(::Arel::Attributes::Attribute) ? inner.name.to_s : nil
|
|
94
79
|
end
|
|
95
80
|
|
|
96
|
-
sig { params(node: T.untyped).returns(T::Boolean) }
|
|
97
81
|
def star?(node)
|
|
98
82
|
inner = node.expressions.first
|
|
99
83
|
!!(inner.is_a?(::Arel::Nodes::SqlLiteral) && inner.to_s == "*")
|
|
100
84
|
end
|
|
101
85
|
|
|
102
|
-
sig { params(value: ::Arel::Nodes::As).returns(String) }
|
|
103
86
|
def alias_name(value)
|
|
104
|
-
right =
|
|
87
|
+
right = value.right
|
|
105
88
|
right.respond_to?(:name) ? right.name.to_s : right.to_s
|
|
106
89
|
end
|
|
107
90
|
|
|
108
|
-
sig { params(attribute: T.nilable(String)).returns(T::Boolean) }
|
|
109
91
|
def not_null_column?(attribute)
|
|
110
92
|
return false if attribute.nil?
|
|
111
93
|
|
|
112
|
-
column =
|
|
94
|
+
column = @relation.klass.columns_hash[attribute]
|
|
113
95
|
!column.nil? && !column.null
|
|
114
96
|
end
|
|
115
97
|
|
|
116
|
-
sig { returns(T::Boolean) }
|
|
117
98
|
def single_table?
|
|
118
|
-
|
|
99
|
+
@relation.joins_values.empty? && @relation.from_clause.value.nil?
|
|
119
100
|
end
|
|
120
101
|
|
|
121
|
-
sig { returns(T::Boolean) }
|
|
122
102
|
def grouped?
|
|
123
|
-
|
|
103
|
+
@relation.group_values.any?
|
|
124
104
|
end
|
|
125
105
|
|
|
126
|
-
sig { returns(T::Boolean) }
|
|
127
106
|
def having?
|
|
128
|
-
|
|
107
|
+
!@relation.having_clause.empty?
|
|
129
108
|
end
|
|
130
109
|
end
|
|
131
110
|
end
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# typed: strict
|
|
2
1
|
# frozen_string_literal: true
|
|
3
2
|
|
|
4
3
|
module ActiveRecord
|
|
@@ -8,9 +7,6 @@ module ActiveRecord
|
|
|
8
7
|
# @api private
|
|
9
8
|
class AsyncRefresher
|
|
10
9
|
class << self
|
|
11
|
-
extend T::Sig
|
|
12
|
-
|
|
13
|
-
sig { params(view_class: ViewClass).void }
|
|
14
10
|
def enqueue(view_class)
|
|
15
11
|
interval = view_class.resolved_refresh_debounce
|
|
16
12
|
|
|
@@ -20,7 +16,6 @@ module ActiveRecord
|
|
|
20
16
|
end
|
|
21
17
|
end
|
|
22
18
|
|
|
23
|
-
sig { void }
|
|
24
19
|
def flush!
|
|
25
20
|
mutex.synchronize do
|
|
26
21
|
cancel_timer_unlocked
|
|
@@ -28,12 +23,10 @@ module ActiveRecord
|
|
|
28
23
|
end
|
|
29
24
|
end
|
|
30
25
|
|
|
31
|
-
sig { returns(Integer) }
|
|
32
26
|
def pending_count
|
|
33
27
|
mutex.synchronize { pending.size }
|
|
34
28
|
end
|
|
35
29
|
|
|
36
|
-
sig { void }
|
|
37
30
|
def reset!
|
|
38
31
|
mutex.synchronize do
|
|
39
32
|
cancel_timer_unlocked
|
|
@@ -43,44 +36,43 @@ module ActiveRecord
|
|
|
43
36
|
|
|
44
37
|
# When paused, refreshes accumulate and run only on an explicit flush! —
|
|
45
38
|
# no background timer fires.
|
|
46
|
-
|
|
47
|
-
def paused=(value)
|
|
48
|
-
@paused = T.let(value, T.nilable(T::Boolean))
|
|
49
|
-
end
|
|
39
|
+
attr_writer :paused
|
|
50
40
|
|
|
51
|
-
sig { returns(T::Boolean) }
|
|
52
41
|
def paused?
|
|
53
|
-
@paused = T.let(@paused, T.nilable(T::Boolean))
|
|
54
42
|
@paused || false
|
|
55
43
|
end
|
|
56
44
|
|
|
57
45
|
private
|
|
58
46
|
|
|
59
|
-
sig { returns(T::Hash[String, ViewClass]) }
|
|
60
47
|
def pending
|
|
61
|
-
@pending ||=
|
|
48
|
+
@pending ||= {}
|
|
62
49
|
end
|
|
63
50
|
|
|
64
|
-
sig { returns(Mutex) }
|
|
65
51
|
def mutex
|
|
66
|
-
@mutex ||=
|
|
52
|
+
@mutex ||= Mutex.new
|
|
67
53
|
end
|
|
68
54
|
|
|
69
|
-
sig { params(interval: T.any(Integer, Float)).void }
|
|
70
55
|
def schedule_unlocked(interval)
|
|
71
56
|
cancel_timer_unlocked
|
|
72
57
|
return if paused?
|
|
73
58
|
|
|
74
|
-
@timer_thread =
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
59
|
+
@timer_thread = Thread.new do
|
|
60
|
+
sleep(interval) unless interval.zero?
|
|
61
|
+
drain_on_pooled_connection
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# The background drain runs `refresh!` (real DML) on a thread of our own, so it must lease a
|
|
66
|
+
# pooled connection and return it. `with_connection` checks one out for the block and releases
|
|
67
|
+
# it in an ensure — even if a refresh raises — so a burst of writes can't leak the pool dry.
|
|
68
|
+
# (The synchronous `flush!` path drains on the caller's already-managed thread, so it is left
|
|
69
|
+
# to that thread's connection lifecycle.)
|
|
70
|
+
def drain_on_pooled_connection
|
|
71
|
+
::ActiveRecord::Base.connection_pool.with_connection do
|
|
72
|
+
mutex.synchronize { drain_pending_unlocked }
|
|
73
|
+
end
|
|
81
74
|
end
|
|
82
75
|
|
|
83
|
-
sig { void }
|
|
84
76
|
def cancel_timer_unlocked
|
|
85
77
|
return unless @timer_thread&.alive?
|
|
86
78
|
|
|
@@ -88,7 +80,6 @@ module ActiveRecord
|
|
|
88
80
|
@timer_thread = nil
|
|
89
81
|
end
|
|
90
82
|
|
|
91
|
-
sig { void }
|
|
92
83
|
def drain_pending_unlocked
|
|
93
84
|
views = pending.values
|
|
94
85
|
pending.clear
|
|
@@ -1,67 +1,86 @@
|
|
|
1
|
-
# typed: strict
|
|
2
1
|
# frozen_string_literal: true
|
|
3
2
|
|
|
4
3
|
module ActiveRecord
|
|
5
4
|
module Materialized
|
|
6
|
-
#
|
|
5
|
+
# Provisions a view's cache table from its source relation. The per-column type
|
|
6
|
+
# inference lives in ColumnTypeInference; this module maps the inferred definitions
|
|
7
|
+
# onto create_table (and, via MigrationBuilder, a generated migration).
|
|
7
8
|
#
|
|
8
9
|
# @api private
|
|
9
10
|
module CacheTableSchema
|
|
10
|
-
|
|
11
|
+
# An inferred cache-table column: name from the relation projection, a create_table
|
|
12
|
+
# column type, and — for a :decimal — the precision/scale that preserve value
|
|
13
|
+
# fidelity (a bare `decimal` is DECIMAL(10,0) on MySQL, which truncates).
|
|
14
|
+
ColumnDefinition = Data.define(:name, :type, :precision, :scale) do
|
|
15
|
+
def initialize(name:, type:, precision: nil, scale: nil) = super
|
|
11
16
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
+
# create_table options for this column (precision/scale for a decimal; none otherwise).
|
|
18
|
+
def options
|
|
19
|
+
{ precision: precision, scale: scale }.compact
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# The trailing arguments for a `t.<type> :<name>` migration line ("" when none).
|
|
23
|
+
def migration_arguments
|
|
24
|
+
options.map { |key, value| ", #{key}: #{value}" }.join
|
|
25
|
+
end
|
|
17
26
|
end
|
|
18
27
|
|
|
28
|
+
# The partition-key index a cache table is built with: the GROUP BY / maintenance-key columns
|
|
29
|
+
# (unqualified to their cache-column names), unique when the key is the whole partition identity
|
|
30
|
+
# (a plain GROUP BY, one row per group). Incremental maintenance filters/deletes/re-selects by
|
|
31
|
+
# exactly these columns, so without this index every partition operation is a full cache scan.
|
|
32
|
+
#
|
|
33
|
+
# @api private
|
|
34
|
+
IndexDefinition = Data.define(:columns, :unique)
|
|
35
|
+
|
|
19
36
|
module_function
|
|
20
37
|
|
|
21
|
-
sig { params(view_class: T.class_of(::ActiveRecord::Base), relation: ::ActiveRecord::Relation).void }
|
|
22
38
|
def ensure_table!(view_class, relation)
|
|
23
39
|
return if view_class.table_exists?
|
|
24
40
|
|
|
25
|
-
build_table!(view_class.connection, view_class.table_name, relation)
|
|
41
|
+
build_table!(view_class.connection, view_class.table_name, relation, index: index_definition(view_class))
|
|
26
42
|
view_class.reset_column_information
|
|
27
43
|
end
|
|
28
44
|
|
|
29
|
-
sig { params(view_class: T.class_of(::ActiveRecord::Base), table_name: String, relation: ::ActiveRecord::Relation).void }
|
|
30
45
|
def create_table!(view_class, table_name, relation)
|
|
31
|
-
build_table!(view_class.connection, table_name, relation)
|
|
46
|
+
build_table!(view_class.connection, table_name, relation, index: index_definition(view_class))
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# The partition-key index for a view, or nil for a non-grouped view (no maintenance key to index).
|
|
50
|
+
# A key derived purely from the GROUP BY is the partition identity, so the index is UNIQUE — which
|
|
51
|
+
# also structurally prevents duplicate partition rows under concurrent maintenance (see #95). An
|
|
52
|
+
# explicit +incremental_keys+ override may be coarser than the grouping, so that index is non-unique.
|
|
53
|
+
#
|
|
54
|
+
# @return [IndexDefinition, nil]
|
|
55
|
+
def index_definition(view_class)
|
|
56
|
+
keys = view_class.maintenance_key_columns
|
|
57
|
+
return nil if keys.empty?
|
|
58
|
+
|
|
59
|
+
columns = keys.map { |key| key.rpartition(".").last }
|
|
60
|
+
IndexDefinition.new(columns: columns, unique: view_class.incremental_key_columns.empty?)
|
|
32
61
|
end
|
|
33
62
|
|
|
34
|
-
# The inferred MV columns
|
|
35
|
-
# probe)
|
|
36
|
-
|
|
63
|
+
# The inferred MV columns. Names come from the query's projection (authoritative, via
|
|
64
|
+
# a zero-row probe); each column's type is inferred structurally from its projected
|
|
65
|
+
# node by ColumnTypeInference. Shared by table creation, migration generation, and
|
|
66
|
+
# drift checks.
|
|
37
67
|
def column_definitions(connection, relation)
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
ColumnDefinition.new(name: name, type: type_for_value(sample&.at(index)))
|
|
68
|
+
nodes = relation.select_values
|
|
69
|
+
connection.exec_query(relation.limit(0).to_sql).columns.each_with_index.map do |name, index|
|
|
70
|
+
ColumnTypeInference.definition_for(connection, relation, nodes[index], name)
|
|
42
71
|
end
|
|
43
72
|
end
|
|
44
73
|
|
|
45
|
-
|
|
46
|
-
def build_table!(connection, table_name, relation)
|
|
74
|
+
def build_table!(connection, table_name, relation, index: nil)
|
|
47
75
|
definitions = column_definitions(connection, relation)
|
|
48
76
|
connection.create_table(table_name) do |table|
|
|
49
|
-
definitions.each
|
|
77
|
+
definitions.each do |definition|
|
|
78
|
+
table.public_send(definition.type, definition.name, **definition.options)
|
|
79
|
+
end
|
|
80
|
+
table.index(index.columns, unique: index.unique) if index
|
|
50
81
|
end
|
|
51
82
|
end
|
|
52
83
|
private_class_method :build_table!
|
|
53
|
-
|
|
54
|
-
sig { params(value: T.untyped).returns(Symbol) }
|
|
55
|
-
def type_for_value(value)
|
|
56
|
-
case value
|
|
57
|
-
when Integer then :integer
|
|
58
|
-
when Float, BigDecimal then :decimal
|
|
59
|
-
when TrueClass, FalseClass then :boolean
|
|
60
|
-
when Time, Date, DateTime, ::ActiveSupport::TimeWithZone then :datetime
|
|
61
|
-
else :string
|
|
62
|
-
end
|
|
63
|
-
end
|
|
64
|
-
private_class_method :type_for_value
|
|
65
84
|
end
|
|
66
85
|
end
|
|
67
86
|
end
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActiveRecord
|
|
4
|
+
module Materialized
|
|
5
|
+
# The change sources a view can declare. `:callbacks` (the default) installs
|
|
6
|
+
# the built-in ActiveRecord commit-callback tracker on `depends_on` models;
|
|
7
|
+
# `:none` installs no callbacks and expects changes through the public
|
|
8
|
+
# ingestion API from an external adapter.
|
|
9
|
+
module ChangeSource
|
|
10
|
+
CALLBACKS = :callbacks
|
|
11
|
+
NONE = :none
|
|
12
|
+
NAMES = [CALLBACKS, NONE].freeze
|
|
13
|
+
|
|
14
|
+
# Validates and normalizes a change-source name, raising on an unknown one
|
|
15
|
+
# so a typo fails loudly at definition/configuration time instead of
|
|
16
|
+
# silently disabling all maintenance for the view.
|
|
17
|
+
def self.cast(source)
|
|
18
|
+
name = source.to_sym
|
|
19
|
+
return name if NAMES.include?(name)
|
|
20
|
+
|
|
21
|
+
raise ArgumentError, "Unknown change source #{source.inspect} (expected one of #{NAMES.inspect})"
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# typed: strict
|
|
2
1
|
# frozen_string_literal: true
|
|
3
2
|
|
|
4
3
|
module ActiveRecord
|
|
@@ -8,14 +7,10 @@ module ActiveRecord
|
|
|
8
7
|
# table aliased to the cache table name, so where/order/limit/count keep
|
|
9
8
|
# working against the same column names.
|
|
10
9
|
class ColdRead
|
|
11
|
-
extend T::Sig
|
|
12
|
-
|
|
13
|
-
sig { params(view_class: ViewClass).void }
|
|
14
10
|
def initialize(view_class)
|
|
15
11
|
@view_class = view_class
|
|
16
12
|
end
|
|
17
13
|
|
|
18
|
-
sig { returns(T.untyped) }
|
|
19
14
|
def scope
|
|
20
15
|
case @view_class.resolved_cold_read_strategy
|
|
21
16
|
when :read_through
|
|
@@ -33,25 +28,21 @@ module ActiveRecord
|
|
|
33
28
|
|
|
34
29
|
private
|
|
35
30
|
|
|
36
|
-
sig { returns(T.untyped) }
|
|
37
31
|
def unscoped
|
|
38
|
-
|
|
32
|
+
@view_class.unscoped
|
|
39
33
|
end
|
|
40
34
|
|
|
41
|
-
sig { returns(Arel::Nodes::SqlLiteral) }
|
|
42
35
|
def source_derived_table
|
|
43
36
|
Arel.sql("(#{@view_class.resolved_source.to_sql}) #{@view_class.quoted_table_name}")
|
|
44
37
|
end
|
|
45
38
|
|
|
46
39
|
# Provisions an empty cache table for column metadata — cheap DDL, no data.
|
|
47
|
-
sig { void }
|
|
48
40
|
def ensure_skeleton!
|
|
49
41
|
return if @view_class.table_exists?
|
|
50
42
|
|
|
51
43
|
CacheTableSchema.ensure_table!(@view_class, @view_class.resolved_source)
|
|
52
44
|
end
|
|
53
45
|
|
|
54
|
-
sig { returns(String) }
|
|
55
46
|
def not_materialized_message
|
|
56
47
|
"#{@view_class.name} is not materialized; run #{@view_class.name}.rebuild!(confirm: true)"
|
|
57
48
|
end
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActiveRecord
|
|
4
|
+
module Materialized
|
|
5
|
+
# Infers one cache-table column (a CacheTableSchema::ColumnDefinition) from a single
|
|
6
|
+
# projected node of a view's source relation, deriving from the aggregated/grouped
|
|
7
|
+
# SOURCE column so the cache mirrors the real data's constraints:
|
|
8
|
+
# - a group key or plain projection takes its source column's declared type, and for
|
|
9
|
+
# a decimal its exact precision/scale;
|
|
10
|
+
# - MIN/MAX return a source value unchanged, so they mirror the source column too;
|
|
11
|
+
# - SUM/AVG derive a wide DECIMAL from the aggregated column (a sum outgrows the
|
|
12
|
+
# source's integer digits; an average adds fractional scale), or :float when the
|
|
13
|
+
# source itself is a float;
|
|
14
|
+
# - COUNT is an integer count.
|
|
15
|
+
# Deterministic and correct even when the source is empty (a sampled-row probe types
|
|
16
|
+
# everything :string). Used by CacheTableSchema to build tables, generate migrations,
|
|
17
|
+
# and check drift.
|
|
18
|
+
#
|
|
19
|
+
# @api private
|
|
20
|
+
module ColumnTypeInference
|
|
21
|
+
# Shorthand for the value type this module produces.
|
|
22
|
+
Definition = CacheTableSchema::ColumnDefinition
|
|
23
|
+
|
|
24
|
+
# DECIMAL precision for a SUM/AVG column, whose result exceeds the aggregated source
|
|
25
|
+
# column's own budget — a sum grows past its integer-digit range, an average past its
|
|
26
|
+
# scale. Wide enough for any realistic aggregate and within MySQL's DECIMAL(65) cap.
|
|
27
|
+
AGGREGATE_PRECISION = 38
|
|
28
|
+
|
|
29
|
+
# Extra fractional digits an average introduces beyond its source column's scale
|
|
30
|
+
# (matching MySQL's own AVG widening). A materialized average is exact for a
|
|
31
|
+
# terminating result; a non-terminating one (e.g. PostgreSQL AVG of 1, 2, 2) is
|
|
32
|
+
# rounded at this scale — inherent to storing a division result in a fixed column.
|
|
33
|
+
AVG_SCALE_HEADROOM = 4
|
|
34
|
+
|
|
35
|
+
# Fractional scale for a SUM/AVG over an unscaled numeric source (one whose own
|
|
36
|
+
# scale the schema does not record), so its fractional part is not truncated to zero.
|
|
37
|
+
DEFAULT_DECIMAL_SCALE = 6
|
|
38
|
+
|
|
39
|
+
# Largest DECIMAL scale every target engine accepts (MySQL caps at 30); a derived
|
|
40
|
+
# scale is clamped to it so table creation never fails on an out-of-range scale.
|
|
41
|
+
MAX_DECIMAL_SCALE = 30
|
|
42
|
+
|
|
43
|
+
# Source column types that map directly to a create_table column builder. A type
|
|
44
|
+
# outside this set (e.g. a PostgreSQL enum, whose declared type is the enum's own
|
|
45
|
+
# name) has no `t.<type>` shorthand, so it degrades to :string rather than crashing
|
|
46
|
+
# table creation.
|
|
47
|
+
SAFE_TYPES = %i[integer bigint decimal float boolean string text binary datetime date time].freeze
|
|
48
|
+
|
|
49
|
+
module_function
|
|
50
|
+
|
|
51
|
+
# The cache column for one projected node — an Arel aggregate, a plain attribute, an
|
|
52
|
+
# aliased wrapper of either, or a bare Symbol resolved against the source schema.
|
|
53
|
+
# Falls back to a :string column when a node can't be classified.
|
|
54
|
+
def definition_for(connection, relation, node, name)
|
|
55
|
+
node = node.left if node.is_a?(::Arel::Nodes::As)
|
|
56
|
+
aggregate_column(connection, node, name) ||
|
|
57
|
+
attribute_column(connection, node, name) ||
|
|
58
|
+
named_column(relation, node, name) ||
|
|
59
|
+
Definition.new(name: name, type: :string)
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# A column for an Arel aggregate function, or nil when the node is not one (so the
|
|
63
|
+
# caller falls through to plain-attribute / named-column handling).
|
|
64
|
+
def aggregate_column(connection, node, name)
|
|
65
|
+
case node
|
|
66
|
+
when ::Arel::Nodes::Count then Definition.new(name: name, type: :integer)
|
|
67
|
+
when ::Arel::Nodes::Sum then sum_avg_column(connection, node, name, 0)
|
|
68
|
+
when ::Arel::Nodes::Avg then sum_avg_column(connection, node, name, AVG_SCALE_HEADROOM)
|
|
69
|
+
when ::Arel::Nodes::Min, ::Arel::Nodes::Max then min_max_column(connection, node, name)
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# SUM/AVG over a float source stays :float — a fixed DECIMAL scale would truncate it.
|
|
74
|
+
# Otherwise a wide DECIMAL keeping the source's fractional scale plus AVG's headroom
|
|
75
|
+
# (an integer or unresolved source contributes scale 0, so a sum stays exact).
|
|
76
|
+
def sum_avg_column(connection, node, name, extra_scale)
|
|
77
|
+
column = aggregate_source_column(connection, node)
|
|
78
|
+
return Definition.new(name: name, type: :float) if column&.type == :float
|
|
79
|
+
|
|
80
|
+
decimal_aggregate(name, source_scale(column) + extra_scale)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# A wide DECIMAL for a SUM/AVG result, with the scale clamped to what every engine
|
|
84
|
+
# accepts (MAX_DECIMAL_SCALE, always <= the precision) so table creation can't fail
|
|
85
|
+
# on an out-of-range scale.
|
|
86
|
+
def decimal_aggregate(name, scale)
|
|
87
|
+
Definition.new(
|
|
88
|
+
name: name, type: :decimal, precision: AGGREGATE_PRECISION, scale: scale.clamp(0, MAX_DECIMAL_SCALE)
|
|
89
|
+
)
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# MIN/MAX return one of the source values unchanged, so the column mirrors the
|
|
93
|
+
# aggregated attribute exactly; a wide integer-scale decimal when it can't be resolved.
|
|
94
|
+
def min_max_column(connection, node, name)
|
|
95
|
+
attribute_column(connection, node.expressions.first, name) || decimal_aggregate(name, 0)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# The live source Column an aggregate is computed over (e.g. amount for SUM(amount)),
|
|
99
|
+
# or nil for COUNT(*) or an aggregate not over a plain attribute.
|
|
100
|
+
def aggregate_source_column(connection, node)
|
|
101
|
+
inner = node.expressions.first
|
|
102
|
+
inner.is_a?(::Arel::Attributes::Attribute) ? source_column(connection, inner) : nil
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# The fractional scale to carry from an aggregate's source column: an integer or
|
|
106
|
+
# unresolved source contributes 0 (a sum stays whole); a decimal contributes its own
|
|
107
|
+
# scale, or DEFAULT_DECIMAL_SCALE for an unscaled numeric whose scale is unknown.
|
|
108
|
+
def source_scale(column)
|
|
109
|
+
return 0 unless column&.type == :decimal
|
|
110
|
+
|
|
111
|
+
column.scale || DEFAULT_DECIMAL_SCALE
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# A column mirroring the source column an Arel attribute names, read from the live
|
|
115
|
+
# schema so a JOINED group key (e.g. authors.country) resolves too. nil when the node
|
|
116
|
+
# is not an attribute or its table/column can't be found.
|
|
117
|
+
def attribute_column(connection, node, name)
|
|
118
|
+
return nil unless node.is_a?(::Arel::Attributes::Attribute)
|
|
119
|
+
|
|
120
|
+
column = source_column(connection, node)
|
|
121
|
+
column ? column_from_source(column, name) : nil
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# A column projected directly from the base table — a bare Symbol group key
|
|
125
|
+
# (`group(:region).select(:region, ...)`) or an implicit SELECT * (no projection
|
|
126
|
+
# node, so `node` is nil) — resolved against the source model's own schema. A
|
|
127
|
+
# raw-SQL/String or computed projection is NOT resolved this way: it may name a
|
|
128
|
+
# JOINED column that merely collides with a base column, so it falls through to
|
|
129
|
+
# :string rather than risk typing it from the wrong table.
|
|
130
|
+
def named_column(relation, node, name)
|
|
131
|
+
return nil unless node.nil? || node.is_a?(::Symbol)
|
|
132
|
+
|
|
133
|
+
column = relation.klass.columns_hash[name]
|
|
134
|
+
column ? column_from_source(column, name) : nil
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# Map a live source Column to a cache Definition: an allowlisted create_table type
|
|
138
|
+
# (see SAFE_TYPES), carrying a decimal's exact precision/scale so the cache preserves
|
|
139
|
+
# the source column's value fidelity.
|
|
140
|
+
def column_from_source(column, name)
|
|
141
|
+
type = SAFE_TYPES.include?(column.type) ? column.type : :string
|
|
142
|
+
return Definition.new(name: name, type: type) unless type == :decimal
|
|
143
|
+
|
|
144
|
+
Definition.new(name: name, type: :decimal, precision: column.precision, scale: column.scale)
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# The live source Column an Arel attribute names, or nil when its table/column can't be found.
|
|
148
|
+
def source_column(connection, node)
|
|
149
|
+
columns = connection.columns(source_relation_name(node.relation))
|
|
150
|
+
columns.find { |candidate| candidate.name == node.name.to_s }
|
|
151
|
+
rescue ::ActiveRecord::StatementInvalid
|
|
152
|
+
nil
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# The real table name an Arel relation refers to, unwrapping a TableAlias (e.g. a self-join
|
|
156
|
+
# `Model.arel_table.alias("t2")`) to its underlying table — so an aliased attribute types against
|
|
157
|
+
# the real column instead of failing to resolve against the alias (which is not a real relation).
|
|
158
|
+
def source_relation_name(relation)
|
|
159
|
+
relation.is_a?(::Arel::Nodes::TableAlias) ? relation.relation.name : relation.name
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
private_class_method :aggregate_column, :sum_avg_column, :decimal_aggregate, :min_max_column,
|
|
163
|
+
:aggregate_source_column, :source_scale, :attribute_column, :named_column,
|
|
164
|
+
:column_from_source, :source_column, :source_relation_name
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
end
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "active_support/concern"
|
|
4
|
+
|
|
5
|
+
module ActiveRecord
|
|
6
|
+
module Materialized
|
|
7
|
+
# Dynamic table-name resolution shared by the gem's internal AR models (metadata, partition,
|
|
8
|
+
# write-outbox). The default name is re-read from its configured source on every access, so a host
|
|
9
|
+
# app can rename the table via {Materialized.configure}; a test or app can still pin an explicit
|
|
10
|
+
# name with +self.table_name = "x"+, which then wins. Factored out of the three models so the
|
|
11
|
+
# order-sensitive idiom lives in one place.
|
|
12
|
+
#
|
|
13
|
+
# @api private
|
|
14
|
+
module ConfigurableTableName
|
|
15
|
+
extend ActiveSupport::Concern
|
|
16
|
+
|
|
17
|
+
included do
|
|
18
|
+
# A class_attribute (not a bare class-instance var) so a subclass inherits the resolver and
|
|
19
|
+
# resolves the default instead of raising NoMethodError on +table_name+.
|
|
20
|
+
class_attribute :table_name_resolver, instance_accessor: false
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
class_methods do
|
|
24
|
+
# Declare how the default table name is resolved. The +resolver+ block is called on each
|
|
25
|
+
# {table_name} read (dynamic), unless an explicit name was assigned via +self.table_name=+.
|
|
26
|
+
#
|
|
27
|
+
# @yieldreturn [String] the currently-configured table name
|
|
28
|
+
def configurable_table_name(&resolver)
|
|
29
|
+
self.table_name_resolver = resolver
|
|
30
|
+
# Seed ActiveRecord's internal @table_name once through its real setter, before the reader
|
|
31
|
+
# below shadows it, so any internal that reads the raw ivar sees a value. (arel_table /
|
|
32
|
+
# quoted_table_name are nilled by the setter and rebuild lazily through the overridden reader.)
|
|
33
|
+
self.table_name = table_name_resolver.call
|
|
34
|
+
@table_name_override = nil
|
|
35
|
+
|
|
36
|
+
define_singleton_method(:table_name) do
|
|
37
|
+
override = @table_name_override
|
|
38
|
+
override.nil? ? table_name_resolver.call : override
|
|
39
|
+
end
|
|
40
|
+
define_singleton_method(:table_name=) { |name| @table_name_override = name }
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|