orfeas_lyra 0.9.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +2021 -17
  3. data/README.md +22 -9
  4. data/Rakefile +40 -1
  5. data/app/controllers/lyra/application_controller.rb +127 -0
  6. data/app/controllers/lyra/dashboard_controller.rb +33 -31
  7. data/app/controllers/lyra/flow_controller.rb +32 -34
  8. data/app/controllers/lyra/privacy_controller.rb +57 -18
  9. data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
  10. data/app/views/lyra/dashboard/index.html.erb +1 -0
  11. data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
  12. data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
  13. data/app/views/lyra/privacy/policy.html.erb +7 -0
  14. data/app/views/lyra/privacy/rights.html.erb +78 -0
  15. data/config/privacy_policies.rb +0 -3
  16. data/config/routes.rb +1 -0
  17. data/lib/lyra/access_log.rb +187 -6
  18. data/lib/lyra/advisory_lock.rb +236 -0
  19. data/lib/lyra/aggregate.rb +1 -1
  20. data/lib/lyra/associations/habtm_links.rb +382 -19
  21. data/lib/lyra/associations/maintenance.rb +229 -0
  22. data/lib/lyra/attribution.rb +48 -0
  23. data/lib/lyra/bypass_events.rb +446 -46
  24. data/lib/lyra/command_handler.rb +19 -14
  25. data/lib/lyra/configuration.rb +237 -10
  26. data/lib/lyra/consistency/read_your_writes.rb +4 -2
  27. data/lib/lyra/correlation.rb +121 -0
  28. data/lib/lyra/domain_events.rb +4 -0
  29. data/lib/lyra/dual_view.rb +53 -16
  30. data/lib/lyra/encrypted_attributes.rb +5 -0
  31. data/lib/lyra/engine.rb +67 -0
  32. data/lib/lyra/erasure.rb +1011 -106
  33. data/lib/lyra/event.rb +35 -1
  34. data/lib/lyra/event_data.rb +38 -0
  35. data/lib/lyra/event_flow.rb +31 -15
  36. data/lib/lyra/event_store_adapter.rb +28 -0
  37. data/lib/lyra/event_window.rb +58 -0
  38. data/lib/lyra/genesis.rb +548 -44
  39. data/lib/lyra/interceptors/association_interceptor.rb +689 -34
  40. data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
  41. data/lib/lyra/internal.rb +26 -0
  42. data/lib/lyra/mode_sync.rb +51 -3
  43. data/lib/lyra/mode_transition.rb +257 -30
  44. data/lib/lyra/model_selection.rb +35 -0
  45. data/lib/lyra/own_connection.rb +17 -2
  46. data/lib/lyra/privacy/adapters/pam.rb +25 -2
  47. data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
  48. data/lib/lyra/privacy/interface.rb +27 -7
  49. data/lib/lyra/privacy/subject_links.rb +143 -0
  50. data/lib/lyra/privacy/subject_records.rb +369 -0
  51. data/lib/lyra/projection.rb +7 -1
  52. data/lib/lyra/projections/async_checkpoint.rb +32 -0
  53. data/lib/lyra/projections/async_projection_job.rb +166 -23
  54. data/lib/lyra/projections/cached_joins.rb +33 -18
  55. data/lib/lyra/projections/cached_projection.rb +74 -33
  56. data/lib/lyra/projections/cached_relation.rb +3175 -321
  57. data/lib/lyra/projections/default_scopes.rb +254 -0
  58. data/lib/lyra/projections/deferred_relation.rb +23 -3
  59. data/lib/lyra/projections/event_store_reader.rb +47 -8
  60. data/lib/lyra/projections/foreign_keys.rb +92 -0
  61. data/lib/lyra/projections/lazy_projection.rb +165 -27
  62. data/lib/lyra/projections/model_projection.rb +165 -14
  63. data/lib/lyra/projections/rebuild.rb +34 -6
  64. data/lib/lyra/projections/reference_links.rb +463 -72
  65. data/lib/lyra/projections/typed_values.rb +629 -0
  66. data/lib/lyra/projections/unique_indexes.rb +595 -0
  67. data/lib/lyra/projections/uniqueness.rb +784 -0
  68. data/lib/lyra/projections/value_digest.rb +93 -0
  69. data/lib/lyra/purpose_bound_reads.rb +912 -77
  70. data/lib/lyra/repair.rb +119 -28
  71. data/lib/lyra/retention.rb +114 -8
  72. data/lib/lyra/rights/consent_store.rb +415 -0
  73. data/lib/lyra/rights/erasure_guard.rb +135 -0
  74. data/lib/lyra/rights/pending.rb +74 -0
  75. data/lib/lyra/rights/queries.rb +269 -0
  76. data/lib/lyra/rights/store.rb +512 -0
  77. data/lib/lyra/rights/subject_keys.rb +224 -0
  78. data/lib/lyra/rights.rb +281 -0
  79. data/lib/lyra/saves_without_effect.rb +69 -0
  80. data/lib/lyra/stream_tail.rb +319 -0
  81. data/lib/lyra/strict_data_access.rb +916 -106
  82. data/lib/lyra/temporal.rb +4 -2
  83. data/lib/lyra/touch_registry.rb +404 -0
  84. data/lib/lyra/verification/bypass_workflow.rb +452 -42
  85. data/lib/lyra/verification/rights_workflows.rb +170 -0
  86. data/lib/lyra/version.rb +1 -1
  87. data/lib/lyra/visualization/event_graph.rb +1 -1
  88. data/lib/lyra/visualization/timeline.rb +2 -2
  89. data/lib/lyra.rb +43 -2
  90. data/lib/tasks/lyra_mode.rake +110 -8
  91. data/lib/tasks/lyra_projections.rake +43 -2
  92. metadata +34 -4
  93. data/lib/lyra/associations/event_aware.rb +0 -224
@@ -0,0 +1,595 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Lyra
4
+ module Projections
5
+ # A monitored model's unique indexes, enforced in the event-sourcing modes
6
+ # on the writes that record one record's change: create and update
7
+ # through callbacks (CrudInterceptor), update_columns and update_column
8
+ # (StrictDataAccess), and update_all (CachedRelation in ES-NoProj,
9
+ # StrictDataAccessRelation elsewhere; update_counters reaches it). In
10
+ # Monitor and Hijack the table's index refuses a duplicate: PostgreSQL
11
+ # raises ActiveRecord::RecordNotUnique from the INSERT or UPDATE, before
12
+ # the event. ES-NoProj writes no row, so a duplicate was saved, unless a
13
+ # uniqueness validation caught it; ES-Sync, ES-Async and ES-Lazy append
14
+ # the event first and project it after, so the duplicate's event was
15
+ # appended and its projection met the index (logged with
16
+ # strict_projections off: the log held a record the table could not, and
17
+ # ES-Lazy's next catch-up failed on it, for every model's reads).
18
+ # insert_all, insert_all! and upsert_all are checked against the same
19
+ # indexes (StrictDataAccessRelation#events_only_unique_rows in ES-NoProj,
20
+ # #projected_unique_rows elsewhere).
21
+ #
22
+ # ES-Sync looks the value up in its table (in_table?): its projection
23
+ # writes the row in the writer's transaction, so once the value's lock is
24
+ # held (see Races) the table holds every committed claim of it and the
25
+ # transaction's own, compared by the database as its index compares
26
+ # them; a write that skips callbacks takes the lock, and the table's
27
+ # statement checks. ES-Async and ES-Lazy, whose tables lag the event
28
+ # store (a job not run yet, a read not made since), look it up in the
29
+ # value streams, as ES-NoProj does (see Lookup), and ES-Async brings up to
30
+ # date, where it waits for no lock, the rows of records that gave the
31
+ # value up (bring_rows_up_to_date), so the table does not refuse a value
32
+ # no record holds. A transaction at REPEATABLE READ or SERIALIZABLE reads
33
+ # its snapshot, which misses a value committed after it began: not
34
+ # covered (ES-Sync's write then fails at its projection, which raises;
35
+ # ES-Async's and ES-Lazy's projection of the second record fails,
36
+ # ModelProjection.refuse_skipped_create!).
37
+ #
38
+ # Which indexes: unique, on columns (not an expression), not partial (no
39
+ # WHERE), other than the primary key, and on no column with a
40
+ # nondeterministic collation (whose equality only the database knows:
41
+ # Uniqueness refuses such columns too). They are read from the database
42
+ # once per model and process, again when its columns change
43
+ # (reset_column_information) and every config.unique_index_refresh
44
+ # seconds (60), so an index a migration adds is enforced in a process
45
+ # that started before it. The indexes left out are not enforced before
46
+ # the event (named in the log once per model). NULL conflicts with
47
+ # nothing, unless the index is NULLS NOT
48
+ # DISTINCT. A value is compared as the column holds it
49
+ # (Uniqueness.canonical: "250" and 250.00 on a decimal are one value, a
50
+ # time in UTC, a character(n) value without its trailing spaces); a
51
+ # citext column's by its LOWER as the database computes it, as citext
52
+ # compares.
53
+ #
54
+ # When: an index is checked when the write sets one of its columns (a
55
+ # create sets every column; an update, the columns it changes), at the
56
+ # point where the table modes write the row: after every before_*
57
+ # callback, before the event is built, so after_* callbacks do not run
58
+ # and nothing is appended (nor a projection job enqueued). The tuple an
59
+ # update claims takes the columns it does not write from the record's
60
+ # stream (ES-Sync: its row, which the update has locked), read under the
61
+ # record's aggregate lock, not from the copy being saved: a copy read before
62
+ # another writer moved the record claims the tuple the row would hold.
63
+ # A record that keeps its value conflicts with nothing; a destroyed
64
+ # record holds no value.
65
+ #
66
+ # Races. Each claimed value is locked (Uniqueness.lock_all!: a
67
+ # transaction-level advisory lock on the index's first column and a
68
+ # keyed digest of its value, the key a uniqueness validation of that
69
+ # column and insert_all take), so a second writer of the value waits
70
+ # until the first's transaction ends, and then finds its record. An
71
+ # update takes the value locks after the record's aggregate lock (and
72
+ # update_all after its records'), as Monitor's UPDATE takes the row
73
+ # lock and then the index entry: two writers of one record, or of two
74
+ # records and one value, wait for each other as they would there, and
75
+ # do not deadlock where Monitor does not. An update of a model with a
76
+ # uniqueness validation takes the record's aggregate lock before its
77
+ # validations (CrudInterceptor#lyra_lock_aggregate_before_validation),
78
+ # so the validation's value lock comes after it too. Several values are
79
+ # locked all or none (xact_lock_all). In ES-Sync update_columns locks its
80
+ # row (FOR NO KEY UPDATE) before the values, and update_all after its
81
+ # rows, as Monitor's UPDATE takes them.
82
+ #
83
+ # Lookup. Each event that sets an indexed column's value is linked into a
84
+ # value stream, "Model!column$<digest>" (ReferenceLinks, beside its
85
+ # foreign-key links in ES-NoProj; alone in ES-Async and ES-Lazy:
86
+ # ReferenceLinks.values_enabled?), named by a keyed digest of the value as compared
87
+ # (ValueDigest: an HMAC under a secret of the installation), so the
88
+ # stream table holds no value, nor a hash anyone can reverse by trying
89
+ # values. A check reads the stream of the value (one query; nothing more
90
+ # when no event ever set it), rebuilds the records whose events it holds
91
+ # (CachedProjection.find_many), and keeps those that hold the value now:
92
+ # its cost is that of the value's history, not the model's size. A
93
+ # multi-column index reads the stream of the column with the fewest
94
+ # events (one query to count them). Where links are off
95
+ # (config.reference_links false) or no column of the tuple has a value
96
+ # (all NULL, NULLS NOT DISTINCT), every record of the model is read
97
+ # (Uniqueness::Snapshot). Before the first check of a model in a process
98
+ # (and before its value locks), the events stored before its columns
99
+ # were linked are linked (ReferenceLinks.catch_up: per column).
100
+ module UniqueIndexes
101
+ Index = Struct.new(:name, :columns, :nulls_not_distinct)
102
+
103
+ # A value a write claims: the index, the tuple's comparison keys (nil
104
+ # for NULL), and the values as given (for the lock and the message).
105
+ Claim = Struct.new(:index, :keys, :values)
106
+
107
+ # One model's indexes, the attribute types they were read with, and
108
+ # when (monotonic seconds).
109
+ Facts = Struct.new(:types, :indexes, :columns, :read_at)
110
+
111
+ @facts = ObjectSpace::WeakKeyMap.new
112
+ @lock = Mutex.new
113
+
114
+ class << self
115
+ # Whether +klass+'s writes are checked here: a monitored model, in an
116
+ # event-sourcing mode (ES-NoProj, ES-Sync, ES-Async, ES-Lazy), where
117
+ # the event is appended before any row is written, or none is.
118
+ def enforced?(klass)
119
+ klass.respond_to?(:lyra_monitored) && klass.lyra_monitored && Lyra.event_sourcing_mode?
120
+ end
121
+
122
+ # Whether the check reads the table rather than the value streams:
123
+ # ES-Sync, whose projection writes the row in the writer's
124
+ # transaction, so that under the value lock the table holds every
125
+ # committed claim and the transaction's own (see Lookup).
126
+ def in_table?(_klass = nil) = Lyra.config.projection_mode == :sync
127
+
128
+ # The mode's name, for the error's message.
129
+ def mode_name
130
+ { sync: "ES-Sync", async: "ES-Async", lazy: "ES-Lazy", disabled: "ES-NoProj" }
131
+ .fetch(Lyra.config.projection_mode, "event sourcing")
132
+ end
133
+
134
+ # The unique indexes checked (see the module's comment).
135
+ def of(klass) = facts(klass).indexes
136
+
137
+ # The columns of those indexes, whose values are linked.
138
+ def columns(klass) = facts(klass).columns
139
+
140
+ # The comparison key of +value+ (as the record holds it) in +column+;
141
+ # nil for NULL. A citext column's is its LOWER, as the database
142
+ # computes it (Uniqueness.lower_texts, remembered per process).
143
+ def key(klass, column, value)
144
+ if Uniqueness.citext?(klass, column)
145
+ text = Uniqueness.lowered_key_text(klass, column, value)
146
+ return text && "/#{Uniqueness.lower_texts(klass, [text], name: column).first}"
147
+ end
148
+ text = Uniqueness.canonical(klass, column, value)
149
+ text == "#null" ? nil : text
150
+ end
151
+
152
+ # The value stream an event that sets +column+ to the value of +key+
153
+ # is linked into.
154
+ def stream(klass, column, key)
155
+ prefix = "#{Lyra.stream_owner_name(klass)}!#{column}"
156
+ "#{prefix}$#{ValueDigest.hex("#{prefix}#{key}")}"
157
+ end
158
+
159
+ # The value streams of +values+ ({ column => value as an event gives
160
+ # it }): one for each indexed column with a value. Each value is read
161
+ # as the projection cache reads an entry (in the table's form: an
162
+ # enum's value; then through the attribute's type: a ciphertext
163
+ # decrypted), so it meets the key the check computes for a holder.
164
+ def streams_for(klass, values)
165
+ wanted = columns(klass)
166
+ return [] if wanted.empty?
167
+
168
+ given = values.slice(*wanted).compact
169
+ return [] if given.empty?
170
+
171
+ table = TypedValues.database_attributes(klass, given)
172
+ reader = TypedValues::EventReader.new(klass)
173
+ given.each_key.filter_map do |column|
174
+ key = key(klass, column, reader.read(table, column))
175
+ key && stream(klass, column, key)
176
+ end
177
+ end
178
+
179
+ # The claims of one record's write: for each index whose columns
180
+ # +touched+ includes (nil: every index, a create), the tuple the
181
+ # record will hold, read by the block (column => value as the record
182
+ # holds it). A tuple with a NULL claims nothing, unless the index is
183
+ # NULLS NOT DISTINCT.
184
+ def claims(klass, touched = nil)
185
+ of(klass).filter_map do |index|
186
+ next unless touched.nil? || index.columns.any? { touched.include?(_1) }
187
+
188
+ values = index.columns.map { yield _1 }
189
+ keys = index.columns.each_with_index.map { |column, i| key(klass, column, values[i]) }
190
+ next if keys.any?(&:nil?) && !index.nulls_not_distinct
191
+
192
+ Claim.new(index, keys, values)
193
+ end
194
+ end
195
+
196
+ # The claims of an update of the record +id+ that writes +written+
197
+ # ({ column => value as the record will hold it }): the indexes with a
198
+ # column it changes, each tuple completed from the record's stream
199
+ # (current, read only when a tuple needs a column not written);
200
+ # +current+: that state, if the caller has read it. Called under the
201
+ # record's aggregate lock.
202
+ def update_claims(klass, id, written, current: nil)
203
+ indexes = of(klass).select { |index| index.columns.any? { written.key?(_1) } }
204
+ return [] if indexes.empty?
205
+
206
+ stored = lambda do |column|
207
+ current ||= current_state(klass, id)
208
+ current[column]
209
+ end
210
+ indexes.filter_map do |index|
211
+ values = index.columns.map { |column| written.key?(column) ? written[column] : stored.(column) }
212
+ keys = index.columns.each_with_index.map { |column, i| key(klass, column, values[i]) }
213
+ next if keys.any?(&:nil?) && !index.nulls_not_distinct
214
+
215
+ Claim.new(index, keys, values)
216
+ end
217
+ end
218
+
219
+ # { column => value as the record holds it } of the record +id+ as its
220
+ # stream has it now ({} for a record the store does not hold).
221
+ def current_state(klass, id)
222
+ table = owner(klass)
223
+ return row_state(table, id) if in_table?(table)
224
+
225
+ attributes = Lyra::PurposeBoundReads.internal(INTERNAL) { CachedProjection.find(table, id) }
226
+ return {} unless attributes
227
+
228
+ reader = TypedValues::EventReader.new(table)
229
+ columns(table).to_h { |column| [column, reader.read(attributes, column)] }
230
+ end
231
+
232
+ # The value locks of +claims+, all or none, after the model's value
233
+ # streams are caught up (outside the locks: a first use links the
234
+ # events stored before).
235
+ def lock!(klass, claims)
236
+ return if claims.blank?
237
+
238
+ table = owner(klass)
239
+ ReferenceLinks.catch_up(table) if !in_table?(table) && ReferenceLinks.values_enabled?
240
+ Uniqueness.lock_all!(table, claims.map { |claim| [claim.index.columns.first, claim.values.first] })
241
+ end
242
+
243
+ # Raise ActiveRecord::RecordNotUnique, as PostgreSQL's unique index
244
+ # would, for the first claim a live record other than those of
245
+ # +except+ (primary keys) holds.
246
+ def refuse_taken!(klass, claims, except: [])
247
+ claim = first_taken(klass, claims, except: except)
248
+ raise not_unique(claim) if claim
249
+ end
250
+
251
+ # The first of +claims+ a live record other than those of +except+
252
+ # holds, or nil. In ES-Async the rows of the records whose events
253
+ # set a claimed value and that no longer hold it (destroyed, or
254
+ # moved to another value) are brought up to date first, where that
255
+ # waits for no one (bring_rows_up_to_date), so neither this write's
256
+ # projection nor the table write of one that skips callbacks meets
257
+ # the value in a row whose job has still to delete or change it.
258
+ def first_taken(klass, claims, except: [])
259
+ return nil if claims.blank?
260
+
261
+ others = Array(except).map(&:to_s)
262
+ claims.find do |claim|
263
+ held, gave_up = scan(klass, claim)
264
+ bring_rows_up_to_date(klass, gave_up - others)
265
+ (held - others).any?
266
+ end
267
+ end
268
+
269
+ # [the primary keys (strings) of the live records that hold +claim+'s
270
+ # tuple, those of the records whose events set one of its values and
271
+ # that no longer hold it]. ES-Sync reads the table (in_table?), whose
272
+ # rows are its records: the second list is empty there.
273
+ def scan(klass, claim)
274
+ table = owner(klass)
275
+ return [table_holders(table, claim), []] if in_table?(table)
276
+
277
+ Lyra::PurposeBoundReads.internal(INTERNAL) do
278
+ Genesis.first_use(table)
279
+ records, named = candidates(table, claim.index, claim.keys)
280
+ reader = TypedValues::EventReader.new(table)
281
+ fold_ahead(table, claim.index, records, reader)
282
+ pk = table.primary_key
283
+ held = records.filter_map do |attributes|
284
+ holds = claim.index.columns.each_with_index.all? { |column, i| key(table, column, reader.read(attributes, column)) == claim.keys[i] }
285
+ attributes[pk].to_s if holds
286
+ end
287
+ [held, named - held]
288
+ end
289
+ end
290
+
291
+ # ES-Async: bring the rows of +ids+ (records of +klass+ whose events
292
+ # set a value a write claims, and that no longer hold it) up to date
293
+ # with their streams, as their projection jobs will, where a row
294
+ # still waits for its job (AsyncCheckpoint.behind: one query) and
295
+ # nothing has to be waited for: the record's aggregate lock, its row
296
+ # and its stream's projection lock are each only tried
297
+ # (pg_try_advisory_xact_lock, FOR UPDATE SKIP LOCKED), and a record
298
+ # one of them is taken for is left to its job. Never waiting, it
299
+ # adds no deadlock, whatever the caller holds. In the caller's
300
+ # transaction (AsyncProjectionJob.project_stream moves the
301
+ # checkpoint with the row: the jobs then find nothing new; rolled
302
+ # back, both go back). Returns the ids brought up to date.
303
+ def bring_rows_up_to_date(klass, ids)
304
+ return [] if ids.empty? || !Lyra::BypassEvents.projects_pending?
305
+
306
+ table = owner(klass)
307
+ table.with_connection do |connection|
308
+ next [] unless connection.transaction_open? && connection.adapter_name.match?(/postgres/i)
309
+
310
+ prefix = Lyra.stream_prefix(table)
311
+ behind = AsyncCheckpoint.behind(connection, ids.uniq.map { Lyra.stream_name(table, _1) })
312
+ behind.filter_map do |stream|
313
+ id = stream.delete_prefix(prefix)
314
+ next unless Lyra::AdvisoryLock.try_xact_lock(connection, "lyra_aggregate/#{stream}", purpose: "aggregate write")
315
+ next unless row_free?(connection, table, id)
316
+ next unless Lyra::AdvisoryLock.try_xact_lock(connection, stream, purpose: "ES-Async projection")
317
+
318
+ AsyncProjectionJob.project_stream(table, stream)
319
+ id
320
+ end
321
+ end
322
+ end
323
+
324
+ # The error PostgreSQL's unique index raises for +claim+, with its
325
+ # message, naming the index.
326
+ def not_unique(claim)
327
+ shown = claim.values.map { _1.nil? ? "null" : _1.to_s }
328
+ ActiveRecord::RecordNotUnique.new(
329
+ "Lyra (#{mode_name}): duplicate key value violates unique constraint \"#{claim.index.name}\"\n" \
330
+ "DETAIL: Key (#{claim.index.columns.join(', ')})=(#{shown.join(', ')}) already exists " \
331
+ "#{in_table? ? 'in the table' : 'in the event store'}."
332
+ )
333
+ end
334
+
335
+ # update_all(+updates+) on records of +klass+ (CachedRelation, in the
336
+ # events-only store): a BulkUpdate when it sets a column of an index;
337
+ # nil otherwise.
338
+ def bulk_update(klass, updates)
339
+ return nil unless updates.is_a?(Hash) && enforced?(klass)
340
+
341
+ set = updates.to_h { |name, value| [klass.attribute_aliases.fetch(name.to_s, name.to_s), value] }
342
+ indexes = of(klass).select { |index| index.columns.any? { set.key?(_1) } }
343
+ return nil if indexes.empty?
344
+
345
+ BulkUpdate.new(klass, set)
346
+ end
347
+
348
+ # Forget the indexes read, and the warnings given (tests).
349
+ def reset!
350
+ @lock.synchronize do
351
+ @facts = ObjectSpace::WeakKeyMap.new
352
+ @warned = Set.new
353
+ end
354
+ end
355
+
356
+ # The live records (attribute hashes, as the projection cache holds
357
+ # them) that may hold any of +wanted+ ([[index, keys], ...]), for a
358
+ # bulk insert (StrictDataAccessRelation#events_only_unique_rows): those
359
+ # whose events are in the streams of the tuples' values (each tuple's
360
+ # first column with a value), read together, and those of +ids+.
361
+ # Every record of the model where links are off or a tuple is all
362
+ # NULL (NULLS NOT DISTINCT).
363
+ def candidate_records(klass, wanted, ids: [])
364
+ table = owner(klass)
365
+ if !ReferenceLinks.values_enabled? || wanted.any? { |_, keys| keys.all?(&:nil?) }
366
+ return Uniqueness::Snapshot.records(table)
367
+ end
368
+
369
+ streams = wanted.filter_map do |index, keys|
370
+ i = keys.index { !_1.nil? }
371
+ stream(table, index.columns[i], keys[i])
372
+ end.uniq
373
+ # Links not caught up (another thread is linking): every record.
374
+ return Uniqueness::Snapshot.records(table) unless ReferenceLinks.catch_up(table)
375
+
376
+ found = streams.empty? ? [] : ReferenceLinks.record_ids_in_any(table, streams)
377
+ all = (found + ids.map(&:to_s)).uniq
378
+ all.empty? ? [] : CachedProjection.find_many(table, all)
379
+ end
380
+
381
+ private
382
+
383
+ def owner(klass) = klass.respond_to?(:base_class) ? klass.base_class : klass
384
+
385
+ # ES-Sync (in_table?): the primary keys (strings) of the rows that hold
386
+ # +claim+'s tuple, read now (never from the query cache: the value
387
+ # lock just taken may have waited for a writer that committed one),
388
+ # compared by the database as its index compares them (a citext
389
+ # column without case).
390
+ def table_holders(table, claim)
391
+ conditions = claim.index.columns.zip(claim.values).to_h
392
+ pk = table.primary_key
393
+ Lyra::PurposeBoundReads.internal(INTERNAL) do
394
+ table.uncached { table.unscoped.where(conditions).limit(2).pluck(pk) }.map(&:to_s)
395
+ end
396
+ end
397
+
398
+ # ES-Sync: { column => value as the record holds it } of the row of
399
+ # +id+, for the columns of the indexes ({} when there is no row). The
400
+ # row is the record's: written in the writers' transactions, and
401
+ # locked by this update (CrudInterceptor#lyra_lock_own_row).
402
+ def row_state(table, id)
403
+ wanted = columns(table)
404
+ return {} if wanted.empty?
405
+
406
+ values = Lyra::PurposeBoundReads.internal(INTERNAL) do
407
+ table.uncached { table.unscoped.where(table.primary_key => id).pluck(*wanted).first }
408
+ end
409
+ return {} if values.nil?
410
+
411
+ wanted.zip(wanted.one? ? [values] : values).to_h
412
+ end
413
+
414
+ # Whether the row of +id+ is absent, or now locked by this
415
+ # transaction (FOR UPDATE SKIP LOCKED): false when another holds it.
416
+ def row_free?(connection, table, id)
417
+ pk = table.primary_key
418
+ bound = table.type_for_attribute(pk).serialize(table.type_for_attribute(pk).cast(id))
419
+ from = "FROM #{connection.quote_table_name(table.table_name)} " \
420
+ "WHERE #{connection.quote_column_name(pk)} = #{connection.quote(bound)}"
421
+ present, held = connection.uncached do
422
+ connection.select_rows("SELECT (SELECT 1 #{from}), (SELECT 1 #{from} FOR UPDATE SKIP LOCKED)", "Lyra unique index row").first
423
+ end
424
+ present.nil? || !held.nil?
425
+ end
426
+
427
+ # The records that may hold the tuple: those whose events set one of
428
+ # its values (the value stream with the fewest events), or every
429
+ # record (see Lookup above); and the primary keys (strings) of the
430
+ # records named, destroyed ones included.
431
+ def candidates(table, index, keys)
432
+ streams = index.columns.each_with_index.filter_map { |column, i| keys[i] && stream(table, column, keys[i]) }
433
+ return every_record(table) if streams.empty? || !ReferenceLinks.values_enabled?
434
+ return every_record(table) unless ReferenceLinks.catch_up(table)
435
+
436
+ ids = ReferenceLinks.record_ids_in(table, streams)
437
+ [ids.empty? ? [] : CachedProjection.find_many(table, ids), ids.map(&:to_s)]
438
+ end
439
+
440
+ def every_record(table)
441
+ records = Uniqueness::Snapshot.records(table)
442
+ [records, records.map { _1[table.primary_key].to_s }]
443
+ end
444
+
445
+ # The LOWER of the candidates' citext values, in one query.
446
+ def fold_ahead(table, index, records, reader)
447
+ folded = index.columns.select { Uniqueness.citext?(table, _1) }
448
+ return if folded.empty? || records.empty?
449
+
450
+ folded.each do |column|
451
+ texts = records.filter_map { |attributes| Uniqueness.lowered_key_text(table, column, reader.read(attributes, column)) }
452
+ Uniqueness.lower_texts(table, texts.uniq, name: column) if texts.any?
453
+ end
454
+ end
455
+
456
+ # The model's facts, read again when its attribute types change or
457
+ # config.unique_index_refresh has passed. A column of an index that
458
+ # the facts did not have before is new to this process: its events
459
+ # may not all be linked (written here, or by a process that read the
460
+ # indexes earlier), so ReferenceLinks links its values again from the
461
+ # first event (ReferenceLinks.unlinked!).
462
+ def facts(klass)
463
+ table = owner(klass)
464
+ types = table.attribute_types
465
+ now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
466
+ found = @lock.synchronize { @facts[table] }
467
+ refresh = Lyra.config.unique_index_refresh
468
+ return found if found&.types.equal?(types) && (refresh.nil? || now - found.read_at < refresh.to_f)
469
+
470
+ indexes = read_indexes(table)
471
+ columns = indexes.flat_map(&:columns).uniq.freeze
472
+ gained = found ? columns - found.columns : []
473
+ fresh = Facts.new(types, indexes, columns, now).freeze
474
+ @lock.synchronize { @facts[table] = fresh }
475
+ ReferenceLinks.unlinked!(table, gained) if gained.any?
476
+ fresh
477
+ end
478
+
479
+ def read_indexes(table)
480
+ primary_key = Array(table.primary_key).map(&:to_s)
481
+ table.with_connection do |connection|
482
+ listed = connection.indexes(table.table_name)
483
+ unique = listed.select(&:unique)
484
+ candidates = unique.select { |index| index.columns.is_a?(Array) && index.where.nil? }
485
+ nondeterministic = nondeterministic_columns(connection, table, candidates.flat_map(&:columns))
486
+ skipped = (unique - candidates).map { |index| [index.name, index.where ? "partial" : "an expression"] }
487
+ indexes = candidates.filter_map do |index|
488
+ columns = index.columns.map(&:to_s)
489
+ next if columns == primary_key
490
+
491
+ if columns.intersect?(nondeterministic)
492
+ skipped << [index.name, "on a column with a nondeterministic collation"]
493
+ next
494
+ end
495
+ nulls = index.respond_to?(:nulls_not_distinct) && index.nulls_not_distinct ? true : false
496
+ Index.new(index.name, columns.freeze, nulls).freeze
497
+ end.freeze
498
+ warn_unenforced(table, skipped)
499
+ indexes
500
+ end
501
+ end
502
+
503
+ # Once per model and process, in an event-sourcing mode: the unique
504
+ # indexes it does not check before a write's event is appended, by
505
+ # name (the table modes' table does; ES-Sync's, ES-Async's and
506
+ # ES-Lazy's projection meets such a duplicate after its event).
507
+ def warn_unenforced(table, skipped)
508
+ return if skipped.empty? || !enforced?(table)
509
+
510
+ @warned ||= Set.new
511
+ fresh = @lock.synchronize { @warned.add?(table.name) }
512
+ return unless fresh
513
+
514
+ listed = skipped.map { |name, why| "#{name} (#{why})" }.join(", ")
515
+ message = "Lyra: #{mode_name} does not enforce these unique indexes of #{table.name}: #{listed}; " \
516
+ "a duplicate is refused only by a uniqueness validation there" \
517
+ "#{Lyra.config.projection_mode == :disabled ? '' : ' (its projection meets the index after the event)'}"
518
+ defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger ? Rails.logger.warn(message) : Kernel.warn(message)
519
+ end
520
+
521
+ # Of +names+, the columns with a collation of their own that is
522
+ # nondeterministic (PostgreSQL 12+): one query, only when a column
523
+ # has a collation.
524
+ def nondeterministic_columns(connection, table, names)
525
+ collations = names.uniq.filter_map do |name|
526
+ column = table.columns_hash[name.to_s]
527
+ [name.to_s, column.collation] if column.respond_to?(:collation) && column.collation
528
+ end.to_h
529
+ return [] if collations.empty? || !connection.adapter_name.match?(/postgres/i)
530
+
531
+ wanted = collations.values.uniq.map { connection.quote(_1.delete('"')) }.join(", ")
532
+ loose = connection.select_values(
533
+ "SELECT collname FROM pg_collation WHERE NOT collisdeterministic AND collname IN (#{wanted})", "Lyra unique indexes"
534
+ )
535
+ collations.select { |_, collation| loose.include?(collation.delete('"')) }.keys
536
+ rescue ActiveRecord::StatementInvalid
537
+ collations.keys
538
+ end
539
+ end
540
+
541
+ # The unique-index check of one update_all (UniqueIndexes.bulk_update).
542
+ class BulkUpdate
543
+ def initialize(klass, set)
544
+ @klass = klass
545
+ # The values set, as the records will hold them.
546
+ @set = set.to_h { |column, value| [column, klass.type_for_attribute(column).cast(value)] }
547
+ end
548
+
549
+ # Before the events of +records+ (the records written, locked and
550
+ # read now, CachedRelation#locked_records_now) are appended: the
551
+ # tuples they will hold locked (after the records' aggregate locks,
552
+ # as Monitor's UPDATE takes the rows, then the index entries) and
553
+ # checked, against each other and against the records that keep
554
+ # theirs, as the table's index checks the rows an UPDATE writes (the
555
+ # state after it; the order in which PostgreSQL visits the rows is
556
+ # not followed). A record whose tuple does not change claims nothing.
557
+ #
558
+ # +records+: records, or attribute hashes as a snapshot of the rows
559
+ # holds them (StrictDataAccessRelation#update_all, in ES-Sync,
560
+ # ES-Async and ES-Lazy, after the rows and their aggregate locks are
561
+ # taken). +check+ false (ES-Sync, whose table is current): the values
562
+ # locked only, so a write through callbacks checking the table waits
563
+ # for this one; the table's UPDATE checks them.
564
+ def check!(records, check: true)
565
+ return if records.empty?
566
+
567
+ pk = @klass.primary_key
568
+ touched = @set.keys
569
+ claims = records.flat_map do |record|
570
+ read = record.is_a?(Hash) ? ->(column) { record[column] } : ->(column) { record.read_attribute(column) }
571
+ id = read.(pk).to_s
572
+ held = UniqueIndexes.claims(@klass, touched) { read.(_1) }.to_h { [_1.index, _1.keys] }
573
+ UniqueIndexes.claims(@klass, touched) { |column| @set.key?(column) ? @set[column] : read.(column) }
574
+ .reject { |claim| held[claim.index] == claim.keys }
575
+ .map { |claim| [id, claim] }
576
+ end
577
+ return if claims.empty?
578
+
579
+ Lyra::BypassEvents.check_lock_limit!(@klass, claims.size, "update_all")
580
+ UniqueIndexes.lock!(@klass, claims.map(&:last))
581
+ return unless check
582
+
583
+ claims.group_by { |_, claim| claim.index }.each_value do |of_index|
584
+ moving = of_index.map(&:first)
585
+ of_index.group_by { |_, claim| claim.keys }.each_value do |same|
586
+ raise UniqueIndexes.not_unique(same.first.last) if same.size > 1
587
+
588
+ UniqueIndexes.refuse_taken!(@klass, [same.first.last], except: moving)
589
+ end
590
+ end
591
+ end
592
+ end
593
+ end
594
+ end
595
+ end