orfeas_lyra 0.9.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +2021 -17
- data/README.md +22 -9
- data/Rakefile +40 -1
- data/app/controllers/lyra/application_controller.rb +127 -0
- data/app/controllers/lyra/dashboard_controller.rb +33 -31
- data/app/controllers/lyra/flow_controller.rb +32 -34
- data/app/controllers/lyra/privacy_controller.rb +57 -18
- data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
- data/app/views/lyra/dashboard/index.html.erb +1 -0
- data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
- data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
- data/app/views/lyra/privacy/policy.html.erb +7 -0
- data/app/views/lyra/privacy/rights.html.erb +78 -0
- data/config/privacy_policies.rb +0 -3
- data/config/routes.rb +1 -0
- data/lib/lyra/access_log.rb +187 -6
- data/lib/lyra/advisory_lock.rb +236 -0
- data/lib/lyra/aggregate.rb +1 -1
- data/lib/lyra/associations/habtm_links.rb +382 -19
- data/lib/lyra/associations/maintenance.rb +229 -0
- data/lib/lyra/attribution.rb +48 -0
- data/lib/lyra/bypass_events.rb +446 -46
- data/lib/lyra/command_handler.rb +19 -14
- data/lib/lyra/configuration.rb +237 -10
- data/lib/lyra/consistency/read_your_writes.rb +4 -2
- data/lib/lyra/correlation.rb +121 -0
- data/lib/lyra/domain_events.rb +4 -0
- data/lib/lyra/dual_view.rb +53 -16
- data/lib/lyra/encrypted_attributes.rb +5 -0
- data/lib/lyra/engine.rb +67 -0
- data/lib/lyra/erasure.rb +1011 -106
- data/lib/lyra/event.rb +35 -1
- data/lib/lyra/event_data.rb +38 -0
- data/lib/lyra/event_flow.rb +31 -15
- data/lib/lyra/event_store_adapter.rb +28 -0
- data/lib/lyra/event_window.rb +58 -0
- data/lib/lyra/genesis.rb +548 -44
- data/lib/lyra/interceptors/association_interceptor.rb +689 -34
- data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
- data/lib/lyra/internal.rb +26 -0
- data/lib/lyra/mode_sync.rb +51 -3
- data/lib/lyra/mode_transition.rb +257 -30
- data/lib/lyra/model_selection.rb +35 -0
- data/lib/lyra/own_connection.rb +17 -2
- data/lib/lyra/privacy/adapters/pam.rb +25 -2
- data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
- data/lib/lyra/privacy/interface.rb +27 -7
- data/lib/lyra/privacy/subject_links.rb +143 -0
- data/lib/lyra/privacy/subject_records.rb +369 -0
- data/lib/lyra/projection.rb +7 -1
- data/lib/lyra/projections/async_checkpoint.rb +32 -0
- data/lib/lyra/projections/async_projection_job.rb +166 -23
- data/lib/lyra/projections/cached_joins.rb +33 -18
- data/lib/lyra/projections/cached_projection.rb +74 -33
- data/lib/lyra/projections/cached_relation.rb +3175 -321
- data/lib/lyra/projections/default_scopes.rb +254 -0
- data/lib/lyra/projections/deferred_relation.rb +23 -3
- data/lib/lyra/projections/event_store_reader.rb +47 -8
- data/lib/lyra/projections/foreign_keys.rb +92 -0
- data/lib/lyra/projections/lazy_projection.rb +165 -27
- data/lib/lyra/projections/model_projection.rb +165 -14
- data/lib/lyra/projections/rebuild.rb +34 -6
- data/lib/lyra/projections/reference_links.rb +463 -72
- data/lib/lyra/projections/typed_values.rb +629 -0
- data/lib/lyra/projections/unique_indexes.rb +595 -0
- data/lib/lyra/projections/uniqueness.rb +784 -0
- data/lib/lyra/projections/value_digest.rb +93 -0
- data/lib/lyra/purpose_bound_reads.rb +912 -77
- data/lib/lyra/repair.rb +119 -28
- data/lib/lyra/retention.rb +114 -8
- data/lib/lyra/rights/consent_store.rb +415 -0
- data/lib/lyra/rights/erasure_guard.rb +135 -0
- data/lib/lyra/rights/pending.rb +74 -0
- data/lib/lyra/rights/queries.rb +269 -0
- data/lib/lyra/rights/store.rb +512 -0
- data/lib/lyra/rights/subject_keys.rb +224 -0
- data/lib/lyra/rights.rb +281 -0
- data/lib/lyra/saves_without_effect.rb +69 -0
- data/lib/lyra/stream_tail.rb +319 -0
- data/lib/lyra/strict_data_access.rb +916 -106
- data/lib/lyra/temporal.rb +4 -2
- data/lib/lyra/touch_registry.rb +404 -0
- data/lib/lyra/verification/bypass_workflow.rb +452 -42
- data/lib/lyra/verification/rights_workflows.rb +170 -0
- data/lib/lyra/version.rb +1 -1
- data/lib/lyra/visualization/event_graph.rb +1 -1
- data/lib/lyra/visualization/timeline.rb +2 -2
- data/lib/lyra.rb +43 -2
- data/lib/tasks/lyra_mode.rake +110 -8
- data/lib/tasks/lyra_projections.rake +43 -2
- metadata +34 -4
- data/lib/lyra/associations/event_aware.rb +0 -224
|
@@ -0,0 +1,595 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Lyra
|
|
4
|
+
module Projections
|
|
5
|
+
# A monitored model's unique indexes, enforced in the event-sourcing modes
|
|
6
|
+
# on the writes that record one record's change: create and update
|
|
7
|
+
# through callbacks (CrudInterceptor), update_columns and update_column
|
|
8
|
+
# (StrictDataAccess), and update_all (CachedRelation in ES-NoProj,
|
|
9
|
+
# StrictDataAccessRelation elsewhere; update_counters reaches it). In
|
|
10
|
+
# Monitor and Hijack the table's index refuses a duplicate: PostgreSQL
|
|
11
|
+
# raises ActiveRecord::RecordNotUnique from the INSERT or UPDATE, before
|
|
12
|
+
# the event. ES-NoProj writes no row, so a duplicate was saved, unless a
|
|
13
|
+
# uniqueness validation caught it; ES-Sync, ES-Async and ES-Lazy append
|
|
14
|
+
# the event first and project it after, so the duplicate's event was
|
|
15
|
+
# appended and its projection met the index (logged with
|
|
16
|
+
# strict_projections off: the log held a record the table could not, and
|
|
17
|
+
# ES-Lazy's next catch-up failed on it, for every model's reads).
|
|
18
|
+
# insert_all, insert_all! and upsert_all are checked against the same
|
|
19
|
+
# indexes (StrictDataAccessRelation#events_only_unique_rows in ES-NoProj,
|
|
20
|
+
# #projected_unique_rows elsewhere).
|
|
21
|
+
#
|
|
22
|
+
# ES-Sync looks the value up in its table (in_table?): its projection
|
|
23
|
+
# writes the row in the writer's transaction, so once the value's lock is
|
|
24
|
+
# held (see Races) the table holds every committed claim of it and the
|
|
25
|
+
# transaction's own, compared by the database as its index compares
|
|
26
|
+
# them; a write that skips callbacks takes the lock, and the table's
|
|
27
|
+
# statement checks. ES-Async and ES-Lazy, whose tables lag the event
|
|
28
|
+
# store (a job not run yet, a read not made since), look it up in the
|
|
29
|
+
# value streams, as ES-NoProj does (see Lookup), and ES-Async brings up to
|
|
30
|
+
# date, where it waits for no lock, the rows of records that gave the
|
|
31
|
+
# value up (bring_rows_up_to_date), so the table does not refuse a value
|
|
32
|
+
# no record holds. A transaction at REPEATABLE READ or SERIALIZABLE reads
|
|
33
|
+
# its snapshot, which misses a value committed after it began: not
|
|
34
|
+
# covered (ES-Sync's write then fails at its projection, which raises;
|
|
35
|
+
# ES-Async's and ES-Lazy's projection of the second record fails,
|
|
36
|
+
# ModelProjection.refuse_skipped_create!).
|
|
37
|
+
#
|
|
38
|
+
# Which indexes: unique, on columns (not an expression), not partial (no
|
|
39
|
+
# WHERE), other than the primary key, and on no column with a
|
|
40
|
+
# nondeterministic collation (whose equality only the database knows:
|
|
41
|
+
# Uniqueness refuses such columns too). They are read from the database
|
|
42
|
+
# once per model and process, again when its columns change
|
|
43
|
+
# (reset_column_information) and every config.unique_index_refresh
|
|
44
|
+
# seconds (60), so an index a migration adds is enforced in a process
|
|
45
|
+
# that started before it. The indexes left out are not enforced before
|
|
46
|
+
# the event (named in the log once per model). NULL conflicts with
|
|
47
|
+
# nothing, unless the index is NULLS NOT
|
|
48
|
+
# DISTINCT. A value is compared as the column holds it
|
|
49
|
+
# (Uniqueness.canonical: "250" and 250.00 on a decimal are one value, a
|
|
50
|
+
# time in UTC, a character(n) value without its trailing spaces); a
|
|
51
|
+
# citext column's by its LOWER as the database computes it, as citext
|
|
52
|
+
# compares.
|
|
53
|
+
#
|
|
54
|
+
# When: an index is checked when the write sets one of its columns (a
|
|
55
|
+
# create sets every column; an update, the columns it changes), at the
|
|
56
|
+
# point where the table modes write the row: after every before_*
|
|
57
|
+
# callback, before the event is built, so after_* callbacks do not run
|
|
58
|
+
# and nothing is appended (nor a projection job enqueued). The tuple an
|
|
59
|
+
# update claims takes the columns it does not write from the record's
|
|
60
|
+
# stream (ES-Sync: its row, which the update has locked), read under the
|
|
61
|
+
# record's aggregate lock, not from the copy being saved: a copy read before
|
|
62
|
+
# another writer moved the record claims the tuple the row would hold.
|
|
63
|
+
# A record that keeps its value conflicts with nothing; a destroyed
|
|
64
|
+
# record holds no value.
|
|
65
|
+
#
|
|
66
|
+
# Races. Each claimed value is locked (Uniqueness.lock_all!: a
|
|
67
|
+
# transaction-level advisory lock on the index's first column and a
|
|
68
|
+
# keyed digest of its value, the key a uniqueness validation of that
|
|
69
|
+
# column and insert_all take), so a second writer of the value waits
|
|
70
|
+
# until the first's transaction ends, and then finds its record. An
|
|
71
|
+
# update takes the value locks after the record's aggregate lock (and
|
|
72
|
+
# update_all after its records'), as Monitor's UPDATE takes the row
|
|
73
|
+
# lock and then the index entry: two writers of one record, or of two
|
|
74
|
+
# records and one value, wait for each other as they would there, and
|
|
75
|
+
# do not deadlock where Monitor does not. An update of a model with a
|
|
76
|
+
# uniqueness validation takes the record's aggregate lock before its
|
|
77
|
+
# validations (CrudInterceptor#lyra_lock_aggregate_before_validation),
|
|
78
|
+
# so the validation's value lock comes after it too. Several values are
|
|
79
|
+
# locked all or none (xact_lock_all). In ES-Sync update_columns locks its
|
|
80
|
+
# row (FOR NO KEY UPDATE) before the values, and update_all after its
|
|
81
|
+
# rows, as Monitor's UPDATE takes them.
|
|
82
|
+
#
|
|
83
|
+
# Lookup. Each event that sets an indexed column's value is linked into a
|
|
84
|
+
# value stream, "Model!column$<digest>" (ReferenceLinks, beside its
|
|
85
|
+
# foreign-key links in ES-NoProj; alone in ES-Async and ES-Lazy:
|
|
86
|
+
# ReferenceLinks.values_enabled?), named by a keyed digest of the value as compared
|
|
87
|
+
# (ValueDigest: an HMAC under a secret of the installation), so the
|
|
88
|
+
# stream table holds no value, nor a hash anyone can reverse by trying
|
|
89
|
+
# values. A check reads the stream of the value (one query; nothing more
|
|
90
|
+
# when no event ever set it), rebuilds the records whose events it holds
|
|
91
|
+
# (CachedProjection.find_many), and keeps those that hold the value now:
|
|
92
|
+
# its cost is that of the value's history, not the model's size. A
|
|
93
|
+
# multi-column index reads the stream of the column with the fewest
|
|
94
|
+
# events (one query to count them). Where links are off
|
|
95
|
+
# (config.reference_links false) or no column of the tuple has a value
|
|
96
|
+
# (all NULL, NULLS NOT DISTINCT), every record of the model is read
|
|
97
|
+
# (Uniqueness::Snapshot). Before the first check of a model in a process
|
|
98
|
+
# (and before its value locks), the events stored before its columns
|
|
99
|
+
# were linked are linked (ReferenceLinks.catch_up: per column).
|
|
100
|
+
module UniqueIndexes
|
|
101
|
+
Index = Struct.new(:name, :columns, :nulls_not_distinct)
|
|
102
|
+
|
|
103
|
+
# A value a write claims: the index, the tuple's comparison keys (nil
|
|
104
|
+
# for NULL), and the values as given (for the lock and the message).
|
|
105
|
+
Claim = Struct.new(:index, :keys, :values)
|
|
106
|
+
|
|
107
|
+
# One model's indexes, the attribute types they were read with, and
|
|
108
|
+
# when (monotonic seconds).
|
|
109
|
+
Facts = Struct.new(:types, :indexes, :columns, :read_at)
|
|
110
|
+
|
|
111
|
+
@facts = ObjectSpace::WeakKeyMap.new
|
|
112
|
+
@lock = Mutex.new
|
|
113
|
+
|
|
114
|
+
class << self
|
|
115
|
+
# Whether +klass+'s writes are checked here: a monitored model, in an
|
|
116
|
+
# event-sourcing mode (ES-NoProj, ES-Sync, ES-Async, ES-Lazy), where
|
|
117
|
+
# the event is appended before any row is written, or none is.
|
|
118
|
+
def enforced?(klass)
|
|
119
|
+
klass.respond_to?(:lyra_monitored) && klass.lyra_monitored && Lyra.event_sourcing_mode?
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# Whether the check reads the table rather than the value streams:
|
|
123
|
+
# ES-Sync, whose projection writes the row in the writer's
|
|
124
|
+
# transaction, so that under the value lock the table holds every
|
|
125
|
+
# committed claim and the transaction's own (see Lookup).
|
|
126
|
+
def in_table?(_klass = nil) = Lyra.config.projection_mode == :sync
|
|
127
|
+
|
|
128
|
+
# The mode's name, for the error's message.
|
|
129
|
+
def mode_name
|
|
130
|
+
{ sync: "ES-Sync", async: "ES-Async", lazy: "ES-Lazy", disabled: "ES-NoProj" }
|
|
131
|
+
.fetch(Lyra.config.projection_mode, "event sourcing")
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# The unique indexes checked (see the module's comment).
|
|
135
|
+
def of(klass) = facts(klass).indexes
|
|
136
|
+
|
|
137
|
+
# The columns of those indexes, whose values are linked.
|
|
138
|
+
def columns(klass) = facts(klass).columns
|
|
139
|
+
|
|
140
|
+
# The comparison key of +value+ (as the record holds it) in +column+;
|
|
141
|
+
# nil for NULL. A citext column's is its LOWER, as the database
|
|
142
|
+
# computes it (Uniqueness.lower_texts, remembered per process).
|
|
143
|
+
def key(klass, column, value)
|
|
144
|
+
if Uniqueness.citext?(klass, column)
|
|
145
|
+
text = Uniqueness.lowered_key_text(klass, column, value)
|
|
146
|
+
return text && "/#{Uniqueness.lower_texts(klass, [text], name: column).first}"
|
|
147
|
+
end
|
|
148
|
+
text = Uniqueness.canonical(klass, column, value)
|
|
149
|
+
text == "#null" ? nil : text
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
# The value stream an event that sets +column+ to the value of +key+
|
|
153
|
+
# is linked into.
|
|
154
|
+
def stream(klass, column, key)
|
|
155
|
+
prefix = "#{Lyra.stream_owner_name(klass)}!#{column}"
|
|
156
|
+
"#{prefix}$#{ValueDigest.hex("#{prefix}#{key}")}"
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# The value streams of +values+ ({ column => value as an event gives
|
|
160
|
+
# it }): one for each indexed column with a value. Each value is read
|
|
161
|
+
# as the projection cache reads an entry (in the table's form: an
|
|
162
|
+
# enum's value; then through the attribute's type: a ciphertext
|
|
163
|
+
# decrypted), so it meets the key the check computes for a holder.
|
|
164
|
+
def streams_for(klass, values)
|
|
165
|
+
wanted = columns(klass)
|
|
166
|
+
return [] if wanted.empty?
|
|
167
|
+
|
|
168
|
+
given = values.slice(*wanted).compact
|
|
169
|
+
return [] if given.empty?
|
|
170
|
+
|
|
171
|
+
table = TypedValues.database_attributes(klass, given)
|
|
172
|
+
reader = TypedValues::EventReader.new(klass)
|
|
173
|
+
given.each_key.filter_map do |column|
|
|
174
|
+
key = key(klass, column, reader.read(table, column))
|
|
175
|
+
key && stream(klass, column, key)
|
|
176
|
+
end
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
# The claims of one record's write: for each index whose columns
|
|
180
|
+
# +touched+ includes (nil: every index, a create), the tuple the
|
|
181
|
+
# record will hold, read by the block (column => value as the record
|
|
182
|
+
# holds it). A tuple with a NULL claims nothing, unless the index is
|
|
183
|
+
# NULLS NOT DISTINCT.
|
|
184
|
+
def claims(klass, touched = nil)
|
|
185
|
+
of(klass).filter_map do |index|
|
|
186
|
+
next unless touched.nil? || index.columns.any? { touched.include?(_1) }
|
|
187
|
+
|
|
188
|
+
values = index.columns.map { yield _1 }
|
|
189
|
+
keys = index.columns.each_with_index.map { |column, i| key(klass, column, values[i]) }
|
|
190
|
+
next if keys.any?(&:nil?) && !index.nulls_not_distinct
|
|
191
|
+
|
|
192
|
+
Claim.new(index, keys, values)
|
|
193
|
+
end
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
# The claims of an update of the record +id+ that writes +written+
|
|
197
|
+
# ({ column => value as the record will hold it }): the indexes with a
|
|
198
|
+
# column it changes, each tuple completed from the record's stream
|
|
199
|
+
# (current, read only when a tuple needs a column not written);
|
|
200
|
+
# +current+: that state, if the caller has read it. Called under the
|
|
201
|
+
# record's aggregate lock.
|
|
202
|
+
def update_claims(klass, id, written, current: nil)
|
|
203
|
+
indexes = of(klass).select { |index| index.columns.any? { written.key?(_1) } }
|
|
204
|
+
return [] if indexes.empty?
|
|
205
|
+
|
|
206
|
+
stored = lambda do |column|
|
|
207
|
+
current ||= current_state(klass, id)
|
|
208
|
+
current[column]
|
|
209
|
+
end
|
|
210
|
+
indexes.filter_map do |index|
|
|
211
|
+
values = index.columns.map { |column| written.key?(column) ? written[column] : stored.(column) }
|
|
212
|
+
keys = index.columns.each_with_index.map { |column, i| key(klass, column, values[i]) }
|
|
213
|
+
next if keys.any?(&:nil?) && !index.nulls_not_distinct
|
|
214
|
+
|
|
215
|
+
Claim.new(index, keys, values)
|
|
216
|
+
end
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
# { column => value as the record holds it } of the record +id+ as its
|
|
220
|
+
# stream has it now ({} for a record the store does not hold).
|
|
221
|
+
def current_state(klass, id)
|
|
222
|
+
table = owner(klass)
|
|
223
|
+
return row_state(table, id) if in_table?(table)
|
|
224
|
+
|
|
225
|
+
attributes = Lyra::PurposeBoundReads.internal(INTERNAL) { CachedProjection.find(table, id) }
|
|
226
|
+
return {} unless attributes
|
|
227
|
+
|
|
228
|
+
reader = TypedValues::EventReader.new(table)
|
|
229
|
+
columns(table).to_h { |column| [column, reader.read(attributes, column)] }
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
# The value locks of +claims+, all or none, after the model's value
|
|
233
|
+
# streams are caught up (outside the locks: a first use links the
|
|
234
|
+
# events stored before).
|
|
235
|
+
def lock!(klass, claims)
|
|
236
|
+
return if claims.blank?
|
|
237
|
+
|
|
238
|
+
table = owner(klass)
|
|
239
|
+
ReferenceLinks.catch_up(table) if !in_table?(table) && ReferenceLinks.values_enabled?
|
|
240
|
+
Uniqueness.lock_all!(table, claims.map { |claim| [claim.index.columns.first, claim.values.first] })
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
# Raise ActiveRecord::RecordNotUnique, as PostgreSQL's unique index
|
|
244
|
+
# would, for the first claim a live record other than those of
|
|
245
|
+
# +except+ (primary keys) holds.
|
|
246
|
+
def refuse_taken!(klass, claims, except: [])
|
|
247
|
+
claim = first_taken(klass, claims, except: except)
|
|
248
|
+
raise not_unique(claim) if claim
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
# The first of +claims+ a live record other than those of +except+
|
|
252
|
+
# holds, or nil. In ES-Async the rows of the records whose events
|
|
253
|
+
# set a claimed value and that no longer hold it (destroyed, or
|
|
254
|
+
# moved to another value) are brought up to date first, where that
|
|
255
|
+
# waits for no one (bring_rows_up_to_date), so neither this write's
|
|
256
|
+
# projection nor the table write of one that skips callbacks meets
|
|
257
|
+
# the value in a row whose job has still to delete or change it.
|
|
258
|
+
def first_taken(klass, claims, except: [])
|
|
259
|
+
return nil if claims.blank?
|
|
260
|
+
|
|
261
|
+
others = Array(except).map(&:to_s)
|
|
262
|
+
claims.find do |claim|
|
|
263
|
+
held, gave_up = scan(klass, claim)
|
|
264
|
+
bring_rows_up_to_date(klass, gave_up - others)
|
|
265
|
+
(held - others).any?
|
|
266
|
+
end
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
# [the primary keys (strings) of the live records that hold +claim+'s
|
|
270
|
+
# tuple, those of the records whose events set one of its values and
|
|
271
|
+
# that no longer hold it]. ES-Sync reads the table (in_table?), whose
|
|
272
|
+
# rows are its records: the second list is empty there.
|
|
273
|
+
def scan(klass, claim)
|
|
274
|
+
table = owner(klass)
|
|
275
|
+
return [table_holders(table, claim), []] if in_table?(table)
|
|
276
|
+
|
|
277
|
+
Lyra::PurposeBoundReads.internal(INTERNAL) do
|
|
278
|
+
Genesis.first_use(table)
|
|
279
|
+
records, named = candidates(table, claim.index, claim.keys)
|
|
280
|
+
reader = TypedValues::EventReader.new(table)
|
|
281
|
+
fold_ahead(table, claim.index, records, reader)
|
|
282
|
+
pk = table.primary_key
|
|
283
|
+
held = records.filter_map do |attributes|
|
|
284
|
+
holds = claim.index.columns.each_with_index.all? { |column, i| key(table, column, reader.read(attributes, column)) == claim.keys[i] }
|
|
285
|
+
attributes[pk].to_s if holds
|
|
286
|
+
end
|
|
287
|
+
[held, named - held]
|
|
288
|
+
end
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
# ES-Async: bring the rows of +ids+ (records of +klass+ whose events
|
|
292
|
+
# set a value a write claims, and that no longer hold it) up to date
|
|
293
|
+
# with their streams, as their projection jobs will, where a row
|
|
294
|
+
# still waits for its job (AsyncCheckpoint.behind: one query) and
|
|
295
|
+
# nothing has to be waited for: the record's aggregate lock, its row
|
|
296
|
+
# and its stream's projection lock are each only tried
|
|
297
|
+
# (pg_try_advisory_xact_lock, FOR UPDATE SKIP LOCKED), and a record
|
|
298
|
+
# one of them is taken for is left to its job. Never waiting, it
|
|
299
|
+
# adds no deadlock, whatever the caller holds. In the caller's
|
|
300
|
+
# transaction (AsyncProjectionJob.project_stream moves the
|
|
301
|
+
# checkpoint with the row: the jobs then find nothing new; rolled
|
|
302
|
+
# back, both go back). Returns the ids brought up to date.
|
|
303
|
+
def bring_rows_up_to_date(klass, ids)
|
|
304
|
+
return [] if ids.empty? || !Lyra::BypassEvents.projects_pending?
|
|
305
|
+
|
|
306
|
+
table = owner(klass)
|
|
307
|
+
table.with_connection do |connection|
|
|
308
|
+
next [] unless connection.transaction_open? && connection.adapter_name.match?(/postgres/i)
|
|
309
|
+
|
|
310
|
+
prefix = Lyra.stream_prefix(table)
|
|
311
|
+
behind = AsyncCheckpoint.behind(connection, ids.uniq.map { Lyra.stream_name(table, _1) })
|
|
312
|
+
behind.filter_map do |stream|
|
|
313
|
+
id = stream.delete_prefix(prefix)
|
|
314
|
+
next unless Lyra::AdvisoryLock.try_xact_lock(connection, "lyra_aggregate/#{stream}", purpose: "aggregate write")
|
|
315
|
+
next unless row_free?(connection, table, id)
|
|
316
|
+
next unless Lyra::AdvisoryLock.try_xact_lock(connection, stream, purpose: "ES-Async projection")
|
|
317
|
+
|
|
318
|
+
AsyncProjectionJob.project_stream(table, stream)
|
|
319
|
+
id
|
|
320
|
+
end
|
|
321
|
+
end
|
|
322
|
+
end
|
|
323
|
+
|
|
324
|
+
# The error PostgreSQL's unique index raises for +claim+, with its
|
|
325
|
+
# message, naming the index.
|
|
326
|
+
def not_unique(claim)
|
|
327
|
+
shown = claim.values.map { _1.nil? ? "null" : _1.to_s }
|
|
328
|
+
ActiveRecord::RecordNotUnique.new(
|
|
329
|
+
"Lyra (#{mode_name}): duplicate key value violates unique constraint \"#{claim.index.name}\"\n" \
|
|
330
|
+
"DETAIL: Key (#{claim.index.columns.join(', ')})=(#{shown.join(', ')}) already exists " \
|
|
331
|
+
"#{in_table? ? 'in the table' : 'in the event store'}."
|
|
332
|
+
)
|
|
333
|
+
end
|
|
334
|
+
|
|
335
|
+
# update_all(+updates+) on records of +klass+ (CachedRelation, in the
|
|
336
|
+
# events-only store): a BulkUpdate when it sets a column of an index;
|
|
337
|
+
# nil otherwise.
|
|
338
|
+
def bulk_update(klass, updates)
|
|
339
|
+
return nil unless updates.is_a?(Hash) && enforced?(klass)
|
|
340
|
+
|
|
341
|
+
set = updates.to_h { |name, value| [klass.attribute_aliases.fetch(name.to_s, name.to_s), value] }
|
|
342
|
+
indexes = of(klass).select { |index| index.columns.any? { set.key?(_1) } }
|
|
343
|
+
return nil if indexes.empty?
|
|
344
|
+
|
|
345
|
+
BulkUpdate.new(klass, set)
|
|
346
|
+
end
|
|
347
|
+
|
|
348
|
+
# Forget the indexes read, and the warnings given (tests).
|
|
349
|
+
def reset!
|
|
350
|
+
@lock.synchronize do
|
|
351
|
+
@facts = ObjectSpace::WeakKeyMap.new
|
|
352
|
+
@warned = Set.new
|
|
353
|
+
end
|
|
354
|
+
end
|
|
355
|
+
|
|
356
|
+
# The live records (attribute hashes, as the projection cache holds
|
|
357
|
+
# them) that may hold any of +wanted+ ([[index, keys], ...]), for a
|
|
358
|
+
# bulk insert (StrictDataAccessRelation#events_only_unique_rows): those
|
|
359
|
+
# whose events are in the streams of the tuples' values (each tuple's
|
|
360
|
+
# first column with a value), read together, and those of +ids+.
|
|
361
|
+
# Every record of the model where links are off or a tuple is all
|
|
362
|
+
# NULL (NULLS NOT DISTINCT).
|
|
363
|
+
def candidate_records(klass, wanted, ids: [])
|
|
364
|
+
table = owner(klass)
|
|
365
|
+
if !ReferenceLinks.values_enabled? || wanted.any? { |_, keys| keys.all?(&:nil?) }
|
|
366
|
+
return Uniqueness::Snapshot.records(table)
|
|
367
|
+
end
|
|
368
|
+
|
|
369
|
+
streams = wanted.filter_map do |index, keys|
|
|
370
|
+
i = keys.index { !_1.nil? }
|
|
371
|
+
stream(table, index.columns[i], keys[i])
|
|
372
|
+
end.uniq
|
|
373
|
+
# Links not caught up (another thread is linking): every record.
|
|
374
|
+
return Uniqueness::Snapshot.records(table) unless ReferenceLinks.catch_up(table)
|
|
375
|
+
|
|
376
|
+
found = streams.empty? ? [] : ReferenceLinks.record_ids_in_any(table, streams)
|
|
377
|
+
all = (found + ids.map(&:to_s)).uniq
|
|
378
|
+
all.empty? ? [] : CachedProjection.find_many(table, all)
|
|
379
|
+
end
|
|
380
|
+
|
|
381
|
+
private
|
|
382
|
+
|
|
383
|
+
def owner(klass) = klass.respond_to?(:base_class) ? klass.base_class : klass
|
|
384
|
+
|
|
385
|
+
# ES-Sync (in_table?): the primary keys (strings) of the rows that hold
|
|
386
|
+
# +claim+'s tuple, read now (never from the query cache: the value
|
|
387
|
+
# lock just taken may have waited for a writer that committed one),
|
|
388
|
+
# compared by the database as its index compares them (a citext
|
|
389
|
+
# column without case).
|
|
390
|
+
def table_holders(table, claim)
|
|
391
|
+
conditions = claim.index.columns.zip(claim.values).to_h
|
|
392
|
+
pk = table.primary_key
|
|
393
|
+
Lyra::PurposeBoundReads.internal(INTERNAL) do
|
|
394
|
+
table.uncached { table.unscoped.where(conditions).limit(2).pluck(pk) }.map(&:to_s)
|
|
395
|
+
end
|
|
396
|
+
end
|
|
397
|
+
|
|
398
|
+
# ES-Sync: { column => value as the record holds it } of the row of
|
|
399
|
+
# +id+, for the columns of the indexes ({} when there is no row). The
|
|
400
|
+
# row is the record's: written in the writers' transactions, and
|
|
401
|
+
# locked by this update (CrudInterceptor#lyra_lock_own_row).
|
|
402
|
+
def row_state(table, id)
|
|
403
|
+
wanted = columns(table)
|
|
404
|
+
return {} if wanted.empty?
|
|
405
|
+
|
|
406
|
+
values = Lyra::PurposeBoundReads.internal(INTERNAL) do
|
|
407
|
+
table.uncached { table.unscoped.where(table.primary_key => id).pluck(*wanted).first }
|
|
408
|
+
end
|
|
409
|
+
return {} if values.nil?
|
|
410
|
+
|
|
411
|
+
wanted.zip(wanted.one? ? [values] : values).to_h
|
|
412
|
+
end
|
|
413
|
+
|
|
414
|
+
# Whether the row of +id+ is absent, or now locked by this
|
|
415
|
+
# transaction (FOR UPDATE SKIP LOCKED): false when another holds it.
|
|
416
|
+
def row_free?(connection, table, id)
|
|
417
|
+
pk = table.primary_key
|
|
418
|
+
bound = table.type_for_attribute(pk).serialize(table.type_for_attribute(pk).cast(id))
|
|
419
|
+
from = "FROM #{connection.quote_table_name(table.table_name)} " \
|
|
420
|
+
"WHERE #{connection.quote_column_name(pk)} = #{connection.quote(bound)}"
|
|
421
|
+
present, held = connection.uncached do
|
|
422
|
+
connection.select_rows("SELECT (SELECT 1 #{from}), (SELECT 1 #{from} FOR UPDATE SKIP LOCKED)", "Lyra unique index row").first
|
|
423
|
+
end
|
|
424
|
+
present.nil? || !held.nil?
|
|
425
|
+
end
|
|
426
|
+
|
|
427
|
+
# The records that may hold the tuple: those whose events set one of
|
|
428
|
+
# its values (the value stream with the fewest events), or every
|
|
429
|
+
# record (see Lookup above); and the primary keys (strings) of the
|
|
430
|
+
# records named, destroyed ones included.
|
|
431
|
+
def candidates(table, index, keys)
|
|
432
|
+
streams = index.columns.each_with_index.filter_map { |column, i| keys[i] && stream(table, column, keys[i]) }
|
|
433
|
+
return every_record(table) if streams.empty? || !ReferenceLinks.values_enabled?
|
|
434
|
+
return every_record(table) unless ReferenceLinks.catch_up(table)
|
|
435
|
+
|
|
436
|
+
ids = ReferenceLinks.record_ids_in(table, streams)
|
|
437
|
+
[ids.empty? ? [] : CachedProjection.find_many(table, ids), ids.map(&:to_s)]
|
|
438
|
+
end
|
|
439
|
+
|
|
440
|
+
def every_record(table)
|
|
441
|
+
records = Uniqueness::Snapshot.records(table)
|
|
442
|
+
[records, records.map { _1[table.primary_key].to_s }]
|
|
443
|
+
end
|
|
444
|
+
|
|
445
|
+
# The LOWER of the candidates' citext values, in one query.
|
|
446
|
+
def fold_ahead(table, index, records, reader)
|
|
447
|
+
folded = index.columns.select { Uniqueness.citext?(table, _1) }
|
|
448
|
+
return if folded.empty? || records.empty?
|
|
449
|
+
|
|
450
|
+
folded.each do |column|
|
|
451
|
+
texts = records.filter_map { |attributes| Uniqueness.lowered_key_text(table, column, reader.read(attributes, column)) }
|
|
452
|
+
Uniqueness.lower_texts(table, texts.uniq, name: column) if texts.any?
|
|
453
|
+
end
|
|
454
|
+
end
|
|
455
|
+
|
|
456
|
+
# The model's facts, read again when its attribute types change or
|
|
457
|
+
# config.unique_index_refresh has passed. A column of an index that
|
|
458
|
+
# the facts did not have before is new to this process: its events
|
|
459
|
+
# may not all be linked (written here, or by a process that read the
|
|
460
|
+
# indexes earlier), so ReferenceLinks links its values again from the
|
|
461
|
+
# first event (ReferenceLinks.unlinked!).
|
|
462
|
+
def facts(klass)
|
|
463
|
+
table = owner(klass)
|
|
464
|
+
types = table.attribute_types
|
|
465
|
+
now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
466
|
+
found = @lock.synchronize { @facts[table] }
|
|
467
|
+
refresh = Lyra.config.unique_index_refresh
|
|
468
|
+
return found if found&.types.equal?(types) && (refresh.nil? || now - found.read_at < refresh.to_f)
|
|
469
|
+
|
|
470
|
+
indexes = read_indexes(table)
|
|
471
|
+
columns = indexes.flat_map(&:columns).uniq.freeze
|
|
472
|
+
gained = found ? columns - found.columns : []
|
|
473
|
+
fresh = Facts.new(types, indexes, columns, now).freeze
|
|
474
|
+
@lock.synchronize { @facts[table] = fresh }
|
|
475
|
+
ReferenceLinks.unlinked!(table, gained) if gained.any?
|
|
476
|
+
fresh
|
|
477
|
+
end
|
|
478
|
+
|
|
479
|
+
def read_indexes(table)
|
|
480
|
+
primary_key = Array(table.primary_key).map(&:to_s)
|
|
481
|
+
table.with_connection do |connection|
|
|
482
|
+
listed = connection.indexes(table.table_name)
|
|
483
|
+
unique = listed.select(&:unique)
|
|
484
|
+
candidates = unique.select { |index| index.columns.is_a?(Array) && index.where.nil? }
|
|
485
|
+
nondeterministic = nondeterministic_columns(connection, table, candidates.flat_map(&:columns))
|
|
486
|
+
skipped = (unique - candidates).map { |index| [index.name, index.where ? "partial" : "an expression"] }
|
|
487
|
+
indexes = candidates.filter_map do |index|
|
|
488
|
+
columns = index.columns.map(&:to_s)
|
|
489
|
+
next if columns == primary_key
|
|
490
|
+
|
|
491
|
+
if columns.intersect?(nondeterministic)
|
|
492
|
+
skipped << [index.name, "on a column with a nondeterministic collation"]
|
|
493
|
+
next
|
|
494
|
+
end
|
|
495
|
+
nulls = index.respond_to?(:nulls_not_distinct) && index.nulls_not_distinct ? true : false
|
|
496
|
+
Index.new(index.name, columns.freeze, nulls).freeze
|
|
497
|
+
end.freeze
|
|
498
|
+
warn_unenforced(table, skipped)
|
|
499
|
+
indexes
|
|
500
|
+
end
|
|
501
|
+
end
|
|
502
|
+
|
|
503
|
+
# Once per model and process, in an event-sourcing mode: the unique
|
|
504
|
+
# indexes it does not check before a write's event is appended, by
|
|
505
|
+
# name (the table modes' table does; ES-Sync's, ES-Async's and
|
|
506
|
+
# ES-Lazy's projection meets such a duplicate after its event).
|
|
507
|
+
def warn_unenforced(table, skipped)
|
|
508
|
+
return if skipped.empty? || !enforced?(table)
|
|
509
|
+
|
|
510
|
+
@warned ||= Set.new
|
|
511
|
+
fresh = @lock.synchronize { @warned.add?(table.name) }
|
|
512
|
+
return unless fresh
|
|
513
|
+
|
|
514
|
+
listed = skipped.map { |name, why| "#{name} (#{why})" }.join(", ")
|
|
515
|
+
message = "Lyra: #{mode_name} does not enforce these unique indexes of #{table.name}: #{listed}; " \
|
|
516
|
+
"a duplicate is refused only by a uniqueness validation there" \
|
|
517
|
+
"#{Lyra.config.projection_mode == :disabled ? '' : ' (its projection meets the index after the event)'}"
|
|
518
|
+
defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger ? Rails.logger.warn(message) : Kernel.warn(message)
|
|
519
|
+
end
|
|
520
|
+
|
|
521
|
+
# Of +names+, the columns with a collation of their own that is
|
|
522
|
+
# nondeterministic (PostgreSQL 12+): one query, only when a column
|
|
523
|
+
# has a collation.
|
|
524
|
+
def nondeterministic_columns(connection, table, names)
|
|
525
|
+
collations = names.uniq.filter_map do |name|
|
|
526
|
+
column = table.columns_hash[name.to_s]
|
|
527
|
+
[name.to_s, column.collation] if column.respond_to?(:collation) && column.collation
|
|
528
|
+
end.to_h
|
|
529
|
+
return [] if collations.empty? || !connection.adapter_name.match?(/postgres/i)
|
|
530
|
+
|
|
531
|
+
wanted = collations.values.uniq.map { connection.quote(_1.delete('"')) }.join(", ")
|
|
532
|
+
loose = connection.select_values(
|
|
533
|
+
"SELECT collname FROM pg_collation WHERE NOT collisdeterministic AND collname IN (#{wanted})", "Lyra unique indexes"
|
|
534
|
+
)
|
|
535
|
+
collations.select { |_, collation| loose.include?(collation.delete('"')) }.keys
|
|
536
|
+
rescue ActiveRecord::StatementInvalid
|
|
537
|
+
collations.keys
|
|
538
|
+
end
|
|
539
|
+
end
|
|
540
|
+
|
|
541
|
+
# The unique-index check of one update_all (UniqueIndexes.bulk_update).
|
|
542
|
+
class BulkUpdate
|
|
543
|
+
def initialize(klass, set)
|
|
544
|
+
@klass = klass
|
|
545
|
+
# The values set, as the records will hold them.
|
|
546
|
+
@set = set.to_h { |column, value| [column, klass.type_for_attribute(column).cast(value)] }
|
|
547
|
+
end
|
|
548
|
+
|
|
549
|
+
# Before the events of +records+ (the records written, locked and
|
|
550
|
+
# read now, CachedRelation#locked_records_now) are appended: the
|
|
551
|
+
# tuples they will hold locked (after the records' aggregate locks,
|
|
552
|
+
# as Monitor's UPDATE takes the rows, then the index entries) and
|
|
553
|
+
# checked, against each other and against the records that keep
|
|
554
|
+
# theirs, as the table's index checks the rows an UPDATE writes (the
|
|
555
|
+
# state after it; the order in which PostgreSQL visits the rows is
|
|
556
|
+
# not followed). A record whose tuple does not change claims nothing.
|
|
557
|
+
#
|
|
558
|
+
# +records+: records, or attribute hashes as a snapshot of the rows
|
|
559
|
+
# holds them (StrictDataAccessRelation#update_all, in ES-Sync,
|
|
560
|
+
# ES-Async and ES-Lazy, after the rows and their aggregate locks are
|
|
561
|
+
# taken). +check+ false (ES-Sync, whose table is current): the values
|
|
562
|
+
# locked only, so a write through callbacks checking the table waits
|
|
563
|
+
# for this one; the table's UPDATE checks them.
|
|
564
|
+
def check!(records, check: true)
|
|
565
|
+
return if records.empty?
|
|
566
|
+
|
|
567
|
+
pk = @klass.primary_key
|
|
568
|
+
touched = @set.keys
|
|
569
|
+
claims = records.flat_map do |record|
|
|
570
|
+
read = record.is_a?(Hash) ? ->(column) { record[column] } : ->(column) { record.read_attribute(column) }
|
|
571
|
+
id = read.(pk).to_s
|
|
572
|
+
held = UniqueIndexes.claims(@klass, touched) { read.(_1) }.to_h { [_1.index, _1.keys] }
|
|
573
|
+
UniqueIndexes.claims(@klass, touched) { |column| @set.key?(column) ? @set[column] : read.(column) }
|
|
574
|
+
.reject { |claim| held[claim.index] == claim.keys }
|
|
575
|
+
.map { |claim| [id, claim] }
|
|
576
|
+
end
|
|
577
|
+
return if claims.empty?
|
|
578
|
+
|
|
579
|
+
Lyra::BypassEvents.check_lock_limit!(@klass, claims.size, "update_all")
|
|
580
|
+
UniqueIndexes.lock!(@klass, claims.map(&:last))
|
|
581
|
+
return unless check
|
|
582
|
+
|
|
583
|
+
claims.group_by { |_, claim| claim.index }.each_value do |of_index|
|
|
584
|
+
moving = of_index.map(&:first)
|
|
585
|
+
of_index.group_by { |_, claim| claim.keys }.each_value do |same|
|
|
586
|
+
raise UniqueIndexes.not_unique(same.first.last) if same.size > 1
|
|
587
|
+
|
|
588
|
+
UniqueIndexes.refuse_taken!(@klass, [same.first.last], except: moving)
|
|
589
|
+
end
|
|
590
|
+
end
|
|
591
|
+
end
|
|
592
|
+
end
|
|
593
|
+
end
|
|
594
|
+
end
|
|
595
|
+
end
|