orfeas_lyra 0.9.3 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +1923 -17
  3. data/README.md +15 -8
  4. data/Rakefile +40 -1
  5. data/app/controllers/lyra/application_controller.rb +108 -0
  6. data/app/controllers/lyra/dashboard_controller.rb +17 -15
  7. data/app/controllers/lyra/flow_controller.rb +30 -33
  8. data/app/controllers/lyra/privacy_controller.rb +57 -18
  9. data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
  10. data/app/views/lyra/dashboard/index.html.erb +1 -0
  11. data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
  12. data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
  13. data/app/views/lyra/privacy/policy.html.erb +7 -0
  14. data/app/views/lyra/privacy/rights.html.erb +78 -0
  15. data/config/privacy_policies.rb +0 -3
  16. data/config/routes.rb +1 -0
  17. data/lib/lyra/access_log.rb +187 -6
  18. data/lib/lyra/advisory_lock.rb +236 -0
  19. data/lib/lyra/aggregate.rb +1 -1
  20. data/lib/lyra/associations/habtm_links.rb +382 -19
  21. data/lib/lyra/associations/maintenance.rb +229 -0
  22. data/lib/lyra/attribution.rb +48 -0
  23. data/lib/lyra/bypass_events.rb +426 -45
  24. data/lib/lyra/command_handler.rb +19 -14
  25. data/lib/lyra/configuration.rb +237 -10
  26. data/lib/lyra/consistency/read_your_writes.rb +4 -2
  27. data/lib/lyra/correlation.rb +121 -0
  28. data/lib/lyra/domain_events.rb +4 -0
  29. data/lib/lyra/dual_view.rb +53 -16
  30. data/lib/lyra/encrypted_attributes.rb +5 -0
  31. data/lib/lyra/engine.rb +67 -0
  32. data/lib/lyra/erasure.rb +1002 -116
  33. data/lib/lyra/event.rb +35 -1
  34. data/lib/lyra/event_data.rb +38 -0
  35. data/lib/lyra/event_flow.rb +31 -15
  36. data/lib/lyra/event_store_adapter.rb +28 -0
  37. data/lib/lyra/event_window.rb +58 -0
  38. data/lib/lyra/genesis.rb +548 -44
  39. data/lib/lyra/interceptors/association_interceptor.rb +689 -34
  40. data/lib/lyra/interceptors/crud_interceptor.rb +547 -95
  41. data/lib/lyra/internal.rb +26 -0
  42. data/lib/lyra/mode_sync.rb +51 -3
  43. data/lib/lyra/mode_transition.rb +245 -34
  44. data/lib/lyra/model_selection.rb +35 -0
  45. data/lib/lyra/own_connection.rb +17 -2
  46. data/lib/lyra/privacy/adapters/pam.rb +25 -2
  47. data/lib/lyra/privacy/gdpr_compliance.rb +264 -36
  48. data/lib/lyra/privacy/interface.rb +27 -7
  49. data/lib/lyra/privacy/subject_links.rb +143 -0
  50. data/lib/lyra/privacy/subject_records.rb +369 -0
  51. data/lib/lyra/projection.rb +7 -1
  52. data/lib/lyra/projections/async_checkpoint.rb +32 -0
  53. data/lib/lyra/projections/async_projection_job.rb +166 -23
  54. data/lib/lyra/projections/cached_joins.rb +33 -18
  55. data/lib/lyra/projections/cached_projection.rb +74 -33
  56. data/lib/lyra/projections/cached_relation.rb +3175 -321
  57. data/lib/lyra/projections/default_scopes.rb +254 -0
  58. data/lib/lyra/projections/deferred_relation.rb +23 -3
  59. data/lib/lyra/projections/event_store_reader.rb +47 -8
  60. data/lib/lyra/projections/foreign_keys.rb +92 -0
  61. data/lib/lyra/projections/lazy_projection.rb +165 -27
  62. data/lib/lyra/projections/model_projection.rb +165 -14
  63. data/lib/lyra/projections/rebuild.rb +34 -6
  64. data/lib/lyra/projections/reference_links.rb +463 -72
  65. data/lib/lyra/projections/typed_values.rb +629 -0
  66. data/lib/lyra/projections/unique_indexes.rb +595 -0
  67. data/lib/lyra/projections/uniqueness.rb +784 -0
  68. data/lib/lyra/projections/value_digest.rb +93 -0
  69. data/lib/lyra/purpose_bound_reads.rb +912 -77
  70. data/lib/lyra/repair.rb +119 -28
  71. data/lib/lyra/retention.rb +114 -8
  72. data/lib/lyra/rights/consent_store.rb +415 -0
  73. data/lib/lyra/rights/erasure_guard.rb +135 -0
  74. data/lib/lyra/rights/pending.rb +74 -0
  75. data/lib/lyra/rights/queries.rb +269 -0
  76. data/lib/lyra/rights/store.rb +512 -0
  77. data/lib/lyra/rights/subject_keys.rb +224 -0
  78. data/lib/lyra/rights.rb +281 -0
  79. data/lib/lyra/saves_without_effect.rb +69 -0
  80. data/lib/lyra/stream_tail.rb +319 -0
  81. data/lib/lyra/strict_data_access.rb +906 -107
  82. data/lib/lyra/temporal.rb +4 -2
  83. data/lib/lyra/touch_registry.rb +404 -0
  84. data/lib/lyra/verification/bypass_workflow.rb +452 -42
  85. data/lib/lyra/verification/rights_workflows.rb +170 -0
  86. data/lib/lyra/version.rb +1 -1
  87. data/lib/lyra/visualization/event_graph.rb +1 -1
  88. data/lib/lyra/visualization/timeline.rb +2 -2
  89. data/lib/lyra.rb +43 -2
  90. data/lib/tasks/lyra_mode.rake +107 -8
  91. data/lib/tasks/lyra_projections.rake +43 -2
  92. metadata +34 -4
  93. data/lib/lyra/associations/event_aware.rb +0 -224
@@ -55,9 +55,15 @@ module Lyra
55
55
  # was seen, and no event was published (Solidus's
56
56
  # Shipment#persist_amounts). Building after the write also covers the
57
57
  # timestamps that touch: true sets.
58
+ #
59
+ # A has_one's dependent: :nullify clears its record's key with
60
+ # update_columns: Rails' association code (Lyra::Associations::Maintenance),
61
+ # allowed in strict mode as a has_many's nullify is (a polymorphic key's
62
+ # two columns included). It was refused, and the owner could not be
63
+ # destroyed. Its event is update_columns', as before.
58
64
  def update_columns(attributes)
59
- raise_strict_violation!(:update_columns)
60
- with_bypass_event(bypass_column_names(attributes), "update_columns") { super }
65
+ raise_strict_violation!(:update_columns) unless Lyra::Associations::Maintenance.claim(:nullify, self)
66
+ with_bypass_event(bypass_column_names(attributes), "update_columns", unique: attributes) { super }
61
67
  end
62
68
 
63
69
  # Rails implements update_column(name, value, touch:) as update_columns,
@@ -77,15 +83,49 @@ module Lyra
77
83
  self.class.attribute_aliases[name] || name
78
84
  end
79
85
  columns |= [self.class.locking_column] if Lyra::OptimisticLocking.enabled?(self.class)
80
- with_bypass_event(columns, "touch") { super }
86
+ deferred, @lyra_deferred_touch_before = @lyra_deferred_touch_before, nil
87
+ with_bypass_event(columns, "touch", before: deferred) { super }
81
88
  end
82
89
 
90
+ # A copy of a record destroyed since it was read (its stream ends in a
91
+ # destroy, read under the aggregate lock: Lyra::BypassEvents
92
+ # .checks_streams?) records nothing and deletes nothing, as Active
93
+ # Record's delete of a row already gone: it appended a second Destroyed,
94
+ # a stream no replay can apply. The record is frozen, as Active Record
95
+ # leaves it; a row still in the table (ES-Async, before the destroy's
96
+ # job runs) is left to the job (Lyra::BypassEvents.unwritten).
83
97
  def delete
84
98
  raise_strict_violation!(:delete)
99
+ if persisted? && Lyra::BypassEvents.enabled_for?(self.class) && Lyra::BypassEvents.checks_streams?
100
+ return self.class.transaction do
101
+ if Lyra::BypassEvents.destroyed_ids(self.class, [id], write: "delete").any?
102
+ Lyra::BypassEvents.unwritten(self.class) { super }
103
+ self
104
+ else
105
+ safely_publish("delete") do
106
+ Lyra::BypassEvents.publish(self.class, id, :destroyed, attributes: attributes, source: "delete")
107
+ end
108
+ super
109
+ end
110
+ end
111
+ end
112
+ if persisted? && Lyra::BypassEvents.enabled_for?(self.class) && Lyra::BypassEvents.locks_rows?
113
+ # Where the rows are the store: the row locked first; a copy whose
114
+ # row another write deleted records nothing (it appended a second
115
+ # Destroyed), as Active Record's delete reaches no row.
116
+ return Lyra::BypassEvents.atomically(self.class) do
117
+ if lyra_lock_row_for_delete
118
+ safely_publish("delete") do
119
+ Lyra::BypassEvents.publish(self.class, id, :destroyed, attributes: attributes, source: "delete")
120
+ end
121
+ end
122
+ super
123
+ end
124
+ end
85
125
  if Lyra::BypassEvents.enabled_for?(self.class)
86
126
  safely_publish("delete") do
87
127
  Lyra::BypassEvents.publish(self.class, id, :destroyed,
88
- attributes: attributes.except(*Lyra::BypassEvents::TIMESTAMP_COLUMNS),
128
+ attributes: attributes,
89
129
  source: "delete")
90
130
  end
91
131
  end
@@ -94,6 +134,34 @@ module Lyra
94
134
 
95
135
  private
96
136
 
137
+ # belongs_to ... touch: true runs through Rails' touch_later: it writes
138
+ # the timestamps into the record at once and clears their change
139
+ # (surreptitiously_touch), and calls touch when the transaction commits.
140
+ # By then attribute_in_database holds the new value, so touch saw no change
141
+ # and the cascade published nothing: the parent's row moved and its stream
142
+ # did not. The value the row held before the first deferred touch is kept
143
+ # here, and the next touch takes it as the old value.
144
+ def surreptitiously_touch(attr_names)
145
+ @lyra_deferred_touch_before ||= {}
146
+ attr_names.each do |name|
147
+ name = name.to_s
148
+ @lyra_deferred_touch_before[name] = attribute_in_database(name) unless @lyra_deferred_touch_before.key?(name)
149
+ end
150
+ super
151
+ end
152
+
153
+ # Whether the record's row is there, locked (FOR UPDATE) until the
154
+ # transaction ends.
155
+ def lyra_lock_row_for_delete
156
+ klass = self.class
157
+ key = respond_to?(:id_in_database) ? id_in_database : id
158
+ klass.with_connection do |connection|
159
+ sql = "SELECT 1 FROM #{connection.quote_table_name(klass.table_name)} " \
160
+ "WHERE #{connection.quote_column_name(klass.primary_key)} = #{connection.quote(key)} FOR UPDATE"
161
+ connection.uncached { connection.select_value(sql, "Lyra delete row lock") }.present?
162
+ end
163
+ end
164
+
97
165
  def raise_strict_violation!(method_name)
98
166
  return unless Lyra.config.strict_data_access
99
167
  return if Thread.current[:lyra_bypass_strict_access]
@@ -115,10 +183,50 @@ module Lyra
115
183
  # In an events-only store (ES-NoProj) there is no row: the UPDATE matches
116
184
  # nothing and returns false, and the event is the write. Publishing only
117
185
  # on a matched row lost every update_columns and touch there.
118
- def with_bypass_event(columns, source)
186
+ # +before+: old values known from elsewhere (a deferred touch's), which
187
+ # attribute_in_database no longer holds.
188
+ #
189
+ # A copy of a record destroyed since it was read (its stream ends in a
190
+ # destroy, read under the aggregate lock where the store can hold a
191
+ # destroy whose row is not gone: Lyra::BypassEvents.checks_streams?)
192
+ # records nothing, writes no row, and returns false, as Active Record's
193
+ # update_columns and touch of a row already gone write nothing and
194
+ # return false (the attributes are set in memory, as there). It
195
+ # appended an update after the destroy and brought the record back.
196
+ #
197
+ # +unique+: the attributes update_columns writes. In ES-NoProj, after the
198
+ # stream check (under the record's aggregate lock, where streams are
199
+ # checked) and before anything is written or recorded: where a value
200
+ # given is the one the copy holds, the record is read from its stream (a
201
+ # copy read before another writer changed it held another value, which
202
+ # its UPDATE in the table modes overwrites: the write recorded nothing);
203
+ # and the values it claims in the table's unique indexes are locked (after
204
+ # the aggregate lock, as Monitor's UPDATE takes the row, then the index
205
+ # entries) and checked against the event store (Lyra::Projections
206
+ # ::UniqueIndexes): a value a live record holds raises
207
+ # ActiveRecord::RecordNotUnique, as the table's UPDATE raises it in the
208
+ # other modes. Before, the duplicate was recorded.
209
+ def with_bypass_event(columns, source, before: nil, unique: nil)
119
210
  Lyra::BypassEvents.atomically(self.class) do
211
+ enabled = persisted? && Lyra::BypassEvents.enabled_for?(self.class)
212
+ Lyra::BypassEvents.admit(self.class, [id]) if enabled
213
+ if enabled && Lyra::BypassEvents.checks_streams? &&
214
+ Lyra::BypassEvents.destroyed_ids(self.class, [id], write: source).any?
215
+ Lyra::BypassEvents.unwritten(self.class) { yield }
216
+ next false
217
+ end
120
218
  events_only = Lyra.event_sourcing_mode? && Lyra.config.projection_mode == :disabled
121
- before = columns.to_h { |c| [c, attribute_in_database(c)] }
219
+ stored = lyra_events_only_columns(unique) if enabled && events_only && unique
220
+ # ES-Lazy: the row as the stream has it, under the lock just taken.
221
+ Lyra::Projections::LazyProjection.refresh_records(self.class, [id]) if enabled && Lyra::BypassEvents.checks_streams?
222
+ lyra_projected_unique_columns(unique) if enabled && unique && !events_only && Lyra.event_sourcing_mode?
223
+ known = before || {}
224
+ before = columns.to_h do |c|
225
+ [c, if known.key?(c) then known[c]
226
+ elsif stored&.key?(c) then stored[c]
227
+ else attribute_in_database(c)
228
+ end]
229
+ end
122
230
  result = yield
123
231
  written = result || events_only
124
232
  changes = columns.each_with_object({}) do |c, h|
@@ -130,10 +238,96 @@ module Lyra
130
238
  end
131
239
  end
132
240
 
241
+ # ES-NoProj's update_columns(+attributes+), before its write (see
242
+ # with_bypass_event): the record's values as its stream has them, when a
243
+ # value given is the copy's own ({ column => value }, else nil), and the
244
+ # unique values it claims locked and checked.
245
+ def lyra_events_only_columns(attributes)
246
+ klass = self.class
247
+ given = lyra_given_columns(attributes)
248
+ stored = nil
249
+ if given.any? { |column, value| value == attribute_in_database(column) }
250
+ record = Lyra::PurposeBoundReads.internal(INTERNAL) { Lyra::Projections::EventStoreReader.find(klass, id) }
251
+ stored = record && klass.column_names.to_h { |column| [column, record.read_attribute(column)] }
252
+ end
253
+ claims = lyra_unique_column_claims(given, stored)
254
+ if claims.present?
255
+ Lyra::Projections::UniqueIndexes.lock!(klass, claims)
256
+ Lyra::Projections::UniqueIndexes.refuse_taken!(klass, claims, except: id)
257
+ end
258
+ stored
259
+ end
260
+
261
+ # { column => value as the record will hold it } of update_columns'
262
+ # +attributes+ (aliases resolved; touch: and unknown names left out).
263
+ def lyra_given_columns(attributes)
264
+ klass = self.class
265
+ attributes.to_h.each_with_object({}) do |(name, value), acc|
266
+ column = klass.attribute_aliases.fetch(name.to_s, name.to_s)
267
+ next if column == "touch" || !klass.column_names.include?(column)
268
+
269
+ acc[column] = klass.type_for_attribute(column).cast(value)
270
+ end
271
+ end
272
+
273
+ # update_columns(+attributes+) in ES-Sync, ES-Async and ES-Lazy, before
274
+ # its table write (with_bypass_event): the values it sets in the table's
275
+ # unique indexes (Lyra::Projections::UniqueIndexes) are locked, so a
276
+ # write through callbacks checking the value waits for this one, as this
277
+ # one waits for it. Where the table lags the event store (ES-Async: a
278
+ # record created or moved there whose job has not run; ES-Lazy: one
279
+ # written after the write's catch-up) they are then checked against the
280
+ # event store, under the record's aggregate lock: the table's UPDATE
281
+ # missed such a holder and recorded the duplicate, which the holder's
282
+ # projection then met (ES-Async also brings up to date, where it waits
283
+ # for no one, the rows of records that gave the value up, which the
284
+ # UPDATE would meet). ES-Sync's table is current: its UPDATE checks, and
285
+ # its row is locked first (FOR NO KEY UPDATE, as the UPDATE and a save
286
+ # through callbacks lock it), then the values, as Monitor's UPDATE takes
287
+ # the row, then the index entries.
288
+ def lyra_projected_unique_columns(attributes)
289
+ klass = self.class
290
+ indexes = Lyra::Projections::UniqueIndexes
291
+ return unless indexes.enforced?(klass) && indexes.of(klass).any?
292
+
293
+ written = lyra_given_columns(attributes).slice(*indexes.columns(klass))
294
+ return if written.empty?
295
+
296
+ in_table = indexes.in_table?(klass)
297
+ lyra_lock_own_row(:update) if in_table && respond_to?(:lyra_lock_own_row, true)
298
+ claims = indexes.update_claims(klass, id, written)
299
+ return if claims.empty?
300
+
301
+ indexes.lock!(klass, claims)
302
+ indexes.refuse_taken!(klass, claims, except: id) unless in_table
303
+ end
304
+
305
+ # The values update_columns claims in the table's unique indexes, in
306
+ # ES-NoProj (Lyra::Projections::UniqueIndexes): for each index with a
307
+ # column whose value it changes (from the record's stream, +stored+, if
308
+ # read), the tuple the record will hold (the values given, the others as
309
+ # the stream has them). nil for a model without such an index.
310
+ def lyra_unique_column_claims(given, stored)
311
+ klass = self.class
312
+ indexes = Lyra::Projections::UniqueIndexes
313
+ return nil unless indexes.enforced?(klass) && indexes.of(klass).any?
314
+
315
+ written = given.slice(*indexes.columns(klass)).reject do |column, value|
316
+ held = stored ? stored[column] : attribute_in_database(column)
317
+ indexes.key(klass, column, value) == indexes.key(klass, column, held)
318
+ end
319
+ return nil if written.empty?
320
+
321
+ indexes.update_claims(klass, id, written, current: stored)
322
+ end
323
+
133
324
  # Publish an "updated" event for a write that bypassed callbacks.
134
325
  # +changes+ maps column => [old, new].
135
326
  def publish_bypass_update_event(changes, source)
136
327
  return unless Lyra::BypassEvents.enabled_for?(self.class)
328
+ # A touch (instance, or belongs_to ... touch: true) that changed only the
329
+ # update timestamps, with config.record_touches false in Monitor.
330
+ return if source == "touch" && Lyra::BypassEvents.skip_touch?(self.class, changes)
137
331
 
138
332
  safely_publish(source) do
139
333
  Lyra::BypassEvents.publish(self.class, id, :updated,
@@ -158,65 +352,142 @@ module Lyra
158
352
  def update_all(updates)
159
353
  return super unless lyra_guarded?
160
354
 
161
- # dependent: :nullify is Rails' own bookkeeping when a parent is
162
- # destroyed through callbacks. It is always allowed, but the children
163
- # it rewrites still get events.
164
- if association_nullify_update?(updates)
165
- publish_nullify_events(updates) if Lyra::BypassEvents.enabled_for?(klass)
166
- return super
355
+ # Rails' own nullify (a has_many's dependent: :nullify, and children
356
+ # removed from the collection; a has_many :through's
357
+ # dependent: :nullify of its through records), the relation it writes
358
+ # marked by its association code (Lyra::Associations::Maintenance). It
359
+ # is always allowed, but the records it rewrites still get events. The
360
+ # events and the write are one transaction where events are required:
361
+ # a write that fails after its events were published takes them back.
362
+ # The same write made by the application is an update_all like any
363
+ # other, below.
364
+ if Lyra::Associations::Maintenance.claim(:nullify, self) && updates.is_a?(Hash)
365
+ return super unless Lyra::BypassEvents.enabled_for?(klass)
366
+
367
+ return Lyra::BypassEvents.atomically(klass) do
368
+ skipped = publish_nullify_events(updates)
369
+ # A child whose stream ends in a destroy (ES-Async, before its
370
+ # job deletes the row) is left as it is, as its event was.
371
+ next super if skipped.blank?
372
+
373
+ Lyra.projection_write(INTERNAL) { where.not(klass.primary_key => skipped).update_all(updates) }
374
+ end
167
375
  end
168
376
 
169
377
  check_strict_mode!(:update_all)
170
378
  return super unless Lyra::BypassEvents.enabled_for?(klass)
379
+ # ES-NoProj: an Active Record relation of the model (one built past
380
+ # Lyra's reads) clearing a foreign key writes the records the event
381
+ # store holds that it selects, as it did when it was taken for a
382
+ # nullify, now as an update_all (CachedRelation#update_all: strict
383
+ # mode, the stream checks, the lock limit, an event per record).
384
+ return events_only_relation("update_all(#{updates.keys.first}: nil)").update_all(updates) if events_only_nullify?(updates)
171
385
 
172
386
  Lyra::BypassEvents.atomically(klass) do
173
387
  lock = Lyra::BypassEvents.locks_rows?
174
- before = Lyra::BypassEvents.snapshot(klass, self, lock: lock)
388
+ before = lock ? Lyra::BypassEvents.locked_snapshot(klass, self, write: "update_all") : Lyra::BypassEvents.snapshot(klass, self)
389
+ projected_unique_update(before, updates) if lock
175
390
  result = lock ? write_snapshotted(before) { |rows| rows.update_all(updates) } : super
176
391
  safely_publish_bulk("update_all") { Lyra::BypassEvents.publish_updates(klass, before, source: "update_all") }
177
392
  result
178
393
  end
179
394
  end
180
395
 
396
+ # Rails's touch_all is update_all of the timestamp columns, recorded as
397
+ # update_all. With config.record_touches false (Monitor), the rows whose
398
+ # only change is the update timestamp are left out of the log
399
+ # (BypassEvents.skip_touch?).
400
+ def touch_all(*names, time: nil)
401
+ return super unless Lyra.config.skip_touches? && lyra_guarded?
402
+
403
+ Lyra::BypassEvents.touching_all { super }
404
+ end
405
+
181
406
  def delete_all
182
- # Deleting has_and_belongs_to_many join rows unlinks their owners.
183
- return Lyra::Associations::HabtmLinks.delete_all(self) { super } if Lyra::Associations::HabtmLinks.join_model?(klass)
407
+ # Deleting has_and_belongs_to_many join rows unlinks their owners:
408
+ # always allowed when Rails' association code deletes them (delete,
409
+ # clear, assigning, destroying the owner: Lyra::Associations::Maintenance),
410
+ # refused in strict mode when the application does, through the join
411
+ # model or a model of its own on the join table.
412
+ join_model = Lyra::Associations::HabtmLinks.join_model_of(klass)
413
+ if join_model
414
+ association = Lyra::Associations::Maintenance.claim(:unlink, self)
415
+ check_strict_mode!(:delete_all) unless association || !Lyra::Associations::HabtmLinks.guarded?(join_model)
416
+ # A join model's rows unlink its owners; an application model's
417
+ # rows of a join table, every monitored owner they name (several
418
+ # has_and_belongs_to_many can share the table).
419
+ return Lyra::Associations::HabtmLinks.delete_all(self) { super } if join_model.equal?(klass)
420
+
421
+ return Lyra::Associations::HabtmLinks.delete_all_on_table(self) { super }
422
+ end
184
423
  return super unless lyra_guarded?
185
424
 
186
- check_strict_mode!(:delete_all)
425
+ # A has_many :through's through records, deleted when records are
426
+ # removed from it (Lyra::Associations::Maintenance): allowed, and
427
+ # recorded as any delete_all.
428
+ check_strict_mode!(:delete_all) unless Lyra::Associations::Maintenance.claim(:unlink, self)
187
429
  return super unless Lyra::BypassEvents.enabled_for?(klass)
188
430
 
189
431
  Lyra::BypassEvents.atomically(klass) do
190
432
  lock = Lyra::BypassEvents.locks_rows?
191
- before = Lyra::BypassEvents.snapshot(klass, self, lock: lock)
433
+ before = lock ? Lyra::BypassEvents.locked_snapshot(klass, self, write: "delete_all") : Lyra::BypassEvents.snapshot(klass, self)
192
434
  result = lock ? write_snapshotted(before) { |rows| rows.delete_all } : super
193
435
  safely_publish_bulk("delete_all") { Lyra::BypassEvents.publish_destroys(klass, before, source: "delete_all") }
194
436
  result
195
437
  end
196
438
  end
197
439
 
440
+ # In ES-NoProj a row's primary key conflicts as the event store has it
441
+ # (events_only_inserts): a record it holds is skipped, as the table's
442
+ # ON CONFLICT DO NOTHING skips a row already there in Monitor; a key
443
+ # whose record was destroyed is inserted again, its stream going on with
444
+ # a Created after the Destroyed, as Monitor's log does. In ES-Async the
445
+ # rows of the keys given are brought up to date with their streams
446
+ # first (project_pending_inserts), so the insert conflicts as Monitor's
447
+ # does there too.
448
+ #
449
+ # On a join table (Lyra::Associations::HabtmLinks.join_model_of): guarded
450
+ # as the application's delete_all of join rows is, and recorded as the
451
+ # links the rows written make (join_insert).
198
452
  def insert_all(attributes, returning: nil, **options)
453
+ if (join_model = Lyra::Associations::HabtmLinks.join_model_of(klass))
454
+ return join_insert(:insert_all, join_model, attributes, returning) { |ret| super(attributes, returning: ret, **options) }
455
+ end
199
456
  return super unless lyra_guarded?
200
457
 
201
458
  check_strict_mode!(:insert_all)
202
459
  return super unless Lyra::BypassEvents.enabled_for?(klass)
203
460
 
461
+ refuse_composite_key!(:insert_all)
204
462
  Lyra::BypassEvents.atomically(klass) do
463
+ skip = options[:unique_by].nil? ? :any : conflict_columns(options[:unique_by])
464
+ attributes = events_only_inserts(attributes, "insert_all", skip: skip)
465
+ locked = project_pending_inserts(attributes, "insert_all")
466
+ attributes, = projected_unique_rows(attributes, "insert_all", skip: skip, keys: locked)
205
467
  result, ids = with_primary_keys(returning) { |ret| super(attributes, returning: ret, **options) }
206
- safely_publish_bulk("insert_all") { Lyra::BypassEvents.publish_upserts(klass, {}, ids, source: "insert_all") }
468
+ published = safely_publish_bulk("insert_all") { Lyra::BypassEvents.publish_upserts(klass, {}, ids, source: "insert_all") }
469
+ advance_checkpoints(published, locked)
207
470
  result
208
471
  end
209
472
  end
210
473
 
211
474
  def insert_all!(attributes, returning: nil, **options)
475
+ if (join_model = Lyra::Associations::HabtmLinks.join_model_of(klass))
476
+ return join_insert(:insert_all!, join_model, attributes, returning) { |ret| super(attributes, returning: ret, **options) }
477
+ end
212
478
  return super unless lyra_guarded?
213
479
 
214
480
  check_strict_mode!(:insert_all!)
215
481
  return super unless Lyra::BypassEvents.enabled_for?(klass)
216
482
 
483
+ refuse_composite_key!(:insert_all!)
217
484
  Lyra::BypassEvents.atomically(klass) do
485
+ attributes = events_only_inserts(attributes, "insert_all!", skip: nil)
486
+ locked = project_pending_inserts(attributes, "insert_all!")
487
+ attributes, = projected_unique_rows(attributes, "insert_all!", skip: nil, keys: locked)
218
488
  result, ids = with_primary_keys(returning) { |ret| super(attributes, returning: ret, **options) }
219
- safely_publish_bulk("insert_all!") { Lyra::BypassEvents.publish_upserts(klass, {}, ids, source: "insert_all!") }
489
+ published = safely_publish_bulk("insert_all!") { Lyra::BypassEvents.publish_upserts(klass, {}, ids, source: "insert_all!") }
490
+ advance_checkpoints(published, locked)
220
491
  result
221
492
  end
222
493
  end
@@ -225,11 +496,17 @@ module Lyra
225
496
  # already existed are found by the conflict key before the write; after
226
497
  # it, those are "updated" (with their changes) and the rest "created".
227
498
  def upsert_all(attributes, returning: nil, unique_by: nil, **options)
499
+ if (join_model = Lyra::Associations::HabtmLinks.join_model_of(klass))
500
+ return join_insert(:upsert_all, join_model, attributes, returning, unique_by: unique_by) do |ret|
501
+ super(attributes, returning: ret, unique_by: unique_by, **options)
502
+ end
503
+ end
228
504
  return super unless lyra_guarded?
229
505
 
230
506
  check_strict_mode!(:upsert_all)
231
507
  return super unless Lyra::BypassEvents.enabled_for?(klass)
232
508
 
509
+ refuse_composite_key!(:upsert_all)
233
510
  if Lyra.event_sourcing_mode? && Lyra.config.projection_mode == :disabled
234
511
  raise Lyra::Projections::UnsupportedQuery.new(
235
512
  klass,
@@ -242,54 +519,556 @@ module Lyra
242
519
  keys = conflict_columns(unique_by)
243
520
  Lyra::BypassEvents.atomically(klass) do
244
521
  before = Lyra::BypassEvents.snapshot_by_keys(klass, rows, keys)
522
+ before, locked = lock_upserted_rows(before, rows, keys) if Lyra::BypassEvents.checks_streams?
523
+ _, brought = projected_unique_rows(attributes, "upsert_all", upsert: keys, keys: Array(locked))
524
+ # A record whose row the check brought up to date (ES-Async: its
525
+ # create's job had not run) is updated by the upsert, not created.
526
+ before = Lyra::BypassEvents.snapshot_by_keys(klass, rows, keys) if brought.any?
245
527
  result, ids = with_primary_keys(returning) do |ret|
246
528
  super(attributes, returning: ret, unique_by: unique_by, **options)
247
529
  end
248
530
  ids ||= Lyra::BypassEvents.snapshot_by_keys(klass, rows, keys).keys
249
- safely_publish_bulk("upsert_all") { Lyra::BypassEvents.publish_upserts(klass, before, ids, source: "upsert_all") }
531
+ published = safely_publish_bulk("upsert_all") { Lyra::BypassEvents.publish_upserts(klass, before, ids, source: "upsert_all") }
532
+ advance_checkpoints(published, locked)
250
533
  result
251
534
  end
252
535
  end
253
536
 
254
537
  private
255
538
 
539
+ # Where the table can hold the row of a record whose stream ends in a
540
+ # destroy (Lyra::BypassEvents.checks_streams?: ES-Async before the
541
+ # destroy's projection job runs, ES-Lazy before its next catch-up; in
542
+ # ES-NoProj upsert_all is refused above), the records whose rows the
543
+ # upsert conflicts with have their aggregate locks taken (all or none,
544
+ # within config.bulk_write_lock_limit) and their streams read. ON CONFLICT
545
+ # updated such a row and appended an Updated after the Destroyed,
546
+ # bringing the record back (and the destroy's job then deleted the row,
547
+ # leaving the log and the table apart). Active Record, whose row is
548
+ # gone, inserts a new row there: with the column defaults for what the
549
+ # upsert does not name, under unique_by another key perhaps.
550
+ #
551
+ # In ES-Async (Lyra::BypassEvents.projects_pending?) the records of the
552
+ # keys the rows give are locked too, and every locked record's row is
553
+ # brought up to date with its stream
554
+ # (Lyra::Projections::AsyncProjectionJob.project_pending), then the rows
555
+ # are read again: a destroyed record's row is gone, and the upsert
556
+ # inserts it again, as Active Record does (a Created after the
557
+ # Destroyed); a record whose create was pending has its row, and is
558
+ # updated; one with an update pending is updated from the row its stream
559
+ # has, which the update's job no longer overwrites. Before, a destroy
560
+ # still to be projected refused the upsert (Lyra::RecordDestroyedError).
561
+ #
562
+ # In ES-Lazy the upsert is refused there (Lyra::RecordDestroyedError)
563
+ # before anything is written, as an update of a destroyed record is
564
+ # (BypassEvents.projects_pending? says why it is not projected there).
565
+ # Once the destroy is in the table (the next catch-up), the upsert
566
+ # inserts. The locked rows are then brought up to date with their
567
+ # streams and read again, so the events carry the values the stream has.
568
+ #
569
+ # The rows are read again once the locks are held, whatever was brought
570
+ # up to date: the first read takes no lock, and a projection job (or an
571
+ # ES-Lazy catch-up) that committed between it and the locks changed the
572
+ # rows it read (a destroyed record's row gone, a created one's there).
573
+ # From the stale read the upsert's row was recorded as an update of the
574
+ # destroyed record, or as a second create. Returns [rows read, the keys
575
+ # locked].
576
+ def lock_upserted_rows(before, rows, keys)
577
+ if Lyra::BypassEvents.projects_pending?
578
+ ids = before.keys | explicit_keys(rows)
579
+ return [before, []] if ids.empty?
580
+
581
+ Lyra::BypassEvents.lock_streams!(klass, ids, write: "upsert_all")
582
+ Lyra::Projections::AsyncProjectionJob.project_pending(klass, ids)
583
+ return [Lyra::BypassEvents.snapshot_by_keys(klass, rows, keys), ids]
584
+ end
585
+ return [before, []] if before.empty?
586
+
587
+ dead = Lyra::BypassEvents.destroyed_ids(klass, before.keys, write: "upsert_all")
588
+ if dead.any?
589
+ raise Lyra::RecordDestroyedError,
590
+ "#{klass.name} #{dead.join(', ')} #{dead.one? ? 'was' : 'were'} destroyed and #{dead.one? ? 'its row is' : 'their rows are'} " \
591
+ "not yet gone from the table (a projection still to run); upsert_all would update #{dead.one? ? 'it' : 'them'} " \
592
+ "and bring #{dead.one? ? 'it' : 'them'} back, so it is refused. Retry once the destroy is projected, " \
593
+ "or create the records again."
594
+ end
595
+ Lyra::Projections::LazyProjection.refresh_records(klass, before.keys)
596
+ [Lyra::BypassEvents.snapshot_by_keys(klass, rows, keys), before.keys]
597
+ end
598
+
256
599
  def lyra_guarded?
257
600
  klass.respond_to?(:lyra_monitored) && klass.lyra_monitored
258
601
  end
259
602
 
603
+ # update_all(+updates+) in ES-Sync, ES-Async and ES-Lazy, once the rows
604
+ # it writes (+before+, the snapshot) and their aggregate locks are held
605
+ # (BypassEvents.locked_snapshot): the tuples the records will hold in the
606
+ # table's unique indexes (Lyra::Projections::UniqueIndexes::BulkUpdate)
607
+ # are locked, after the rows, as Monitor's UPDATE takes the rows and then
608
+ # the index entries, so a write through callbacks checking a value waits
609
+ # for this one. Where the table lags the event store (ES-Async, ES-Lazy)
610
+ # they are checked against each other and the event store before the
611
+ # write, as for update_columns (StrictDataAccess
612
+ # #lyra_projected_unique_columns); ES-Sync's UPDATE checks them.
613
+ def projected_unique_update(before, updates)
614
+ return if before.empty? || !Lyra.event_sourcing_mode?
615
+
616
+ unique = Lyra::Projections::UniqueIndexes.bulk_update(klass, updates)
617
+ unique&.check!(before.values, check: !Lyra::Projections::UniqueIndexes.in_table?(klass))
618
+ end
619
+
620
+ # insert_all, insert_all! (+skip+: the conflicts insert_all's ON
621
+ # CONFLICT DO NOTHING skips: :any, or its unique_by's columns; nil for
622
+ # insert_all!) and upsert_all (+upsert+: its conflict columns) in
623
+ # ES-Sync, ES-Async and ES-Lazy, after the keys the rows give are locked
624
+ # (+keys+): the values each row claims in the table's unique indexes
625
+ # (Lyra::Projections::UniqueIndexes; the primary key is the keys'), as
626
+ # the record would hold them (the column defaults and the relation's
627
+ # scope for what the row does not give), are locked, together counted
628
+ # against config.bulk_write_lock_limit with the keys, so a write through
629
+ # callbacks checking a value waits for the insert, as it waits for that
630
+ # write. ES-Sync's table is current, and its INSERT checks them.
631
+ #
632
+ # Where the table lags the event store (ES-Async: a record whose create
633
+ # or update waits for its job; ES-Lazy: one written after the catch-up
634
+ # the write began with), each value is then looked up in the event store
635
+ # (its value stream, under its lock): the table missed such a holder and
636
+ # inserted the duplicate, which the holder's projection then met. A row
637
+ # whose value a live record other than the row's own key holds is left
638
+ # out of insert_all (as ON CONFLICT DO NOTHING leaves it, on any index,
639
+ # or on its unique_by), and raises ActiveRecord::RecordNotUnique,
640
+ # before anything is written, for insert_all!, a conflict on an index
641
+ # other than insert_all's unique_by, and an upsert's conflict on an
642
+ # index other than its conflict columns (the record the row updates
643
+ # excepted). An upsert's conflict on its conflict columns updates the
644
+ # holder's row: in ES-Async that row is brought up to date first where
645
+ # it waits for its job (UniqueIndexes.bring_rows_up_to_date, which waits
646
+ # for no one), and where it is still not in the table (taken, or in
647
+ # ES-Lazy written after the catch-up) the upsert raises
648
+ # ActiveRecord::RecordNotUnique rather than insert a second record with
649
+ # the value. In ES-Async the rows of records that gave a value up and
650
+ # whose jobs have not run are brought up to date too, so the table does
651
+ # not skip or refuse a row for a value no record holds.
652
+ #
653
+ # Returns [the rows to write, the ids of the records whose rows were
654
+ # brought up to date].
655
+ def projected_unique_rows(attributes, write, skip: nil, upsert: nil, keys: [])
656
+ unique = Lyra::Projections::UniqueIndexes
657
+ return [attributes, []] if Lyra.config.projection_mode == :disabled || !unique.enforced?(klass)
658
+
659
+ indexes = unique.of(klass)
660
+ rows = attributes.is_a?(Hash) ? [attributes] : Array(attributes)
661
+ return [attributes, []] if indexes.empty? || rows.empty?
662
+
663
+ scope = scope_for_create.transform_keys(&:to_s)
664
+ defaults = klass.column_defaults
665
+ columns = indexes.flat_map(&:columns).uniq
666
+ claims = rows.map do |row|
667
+ values = defaults.merge(scope.merge(row.to_h.to_h { |k, v| [klass.attribute_aliases.fetch(k.to_s, k.to_s), v] }))
668
+ given = columns.to_h { |column| [column, klass.type_for_attribute(column).cast(values[column])] }
669
+ unique.claims(klass) { given[_1] }
670
+ end
671
+ all = claims.flatten
672
+ return [attributes, []] if all.empty?
673
+
674
+ Lyra::BypassEvents.check_lock_limit!(klass, all.map { [_1.index.columns.first, _1.values.first] }.uniq.size + keys.size, write)
675
+ unique.lock!(klass, all)
676
+ return [attributes, []] if unique.in_table?(klass)
677
+
678
+ pk = klass.primary_key
679
+ type = pk.is_a?(String) ? klass.type_for_attribute(pk) : nil
680
+ brought = []
681
+ seen = {}
682
+ holders = lambda do |claim|
683
+ seen[[claim.index.name, claim.keys]] ||= begin
684
+ held, gave_up = unique.scan(klass, claim)
685
+ brought.concat(unique.bring_rows_up_to_date(klass, gave_up))
686
+ held
687
+ end
688
+ end
689
+ target = upsert&.sort
690
+ kept = rows.each_index.select do |r|
691
+ own = type && explicit_key(rows[r], pk, type)&.to_s
692
+ if upsert
693
+ updated = target == [pk] ? own : nil
694
+ aim = claims[r].find { _1.index.columns.sort == target }
695
+ if aim && (held = holders.(aim) - [own].compact).any?
696
+ updated = held.first
697
+ unless upserted_row_present?(updated, brought)
698
+ raise unique.not_unique(aim)
699
+ end
700
+ end
701
+ refused = claims[r].find { |claim| claim != aim && (holders.(claim) - [own, updated].compact).any? }
702
+ raise unique.not_unique(refused) if refused
703
+
704
+ next true
705
+ end
706
+ taken = claims[r].select { |claim| (holders.(claim) - [own].compact).any? }
707
+ next true if taken.empty?
708
+
709
+ refused = taken.find { |claim| !(skip == :any || (skip && skip.sort == claim.index.columns.sort)) }
710
+ raise unique.not_unique(refused) if refused
711
+
712
+ false
713
+ end
714
+ return [attributes, brought] if kept.size == rows.size
715
+
716
+ [kept.map { rows[_1] }, brought]
717
+ end
718
+
719
+ # Whether the row of the live record +id+, which an upsert updates by its
720
+ # conflict columns, is in the table: in ES-Async brought up to date
721
+ # first, where its job has not run and nothing has to be waited for.
722
+ def upserted_row_present?(id, brought)
723
+ brought.concat(Lyra::Projections::UniqueIndexes.bring_rows_up_to_date(klass, [id]))
724
+ Lyra::PurposeBoundReads.internal(INTERNAL) do
725
+ klass.uncached { klass.unscoped.where(klass.primary_key => id).exists? }
726
+ end
727
+ end
728
+
729
+ # The rows of an insert_all or insert_all! in the events-only store
730
+ # (ES-NoProj), where the event store, not the table, says which records
731
+ # exist. +skip+: the conflicts ON CONFLICT DO NOTHING skips, as Monitor's
732
+ # does: :any (insert_all), the columns of its unique_by, or nil
733
+ # (insert_all!: every conflict raises ActiveRecord::RecordNotUnique).
734
+ #
735
+ # The model's other unique indexes first (events_only_unique_rows),
736
+ # then its primary key. ON CONFLICT DO NOTHING covers a primary key
737
+ # conflict with no unique_by or unique_by the primary key; otherwise
738
+ # (+unique+ below: insert_all!, or an insert_all whose unique_by is
739
+ # another index) a key taken raises ActiveRecord::RecordNotUnique, as
740
+ # Monitor's insert raises it. The records whose keys the rows give are
741
+ # locked (aggregate locks, under config.bulk_write_lock_limit) and read
742
+ # from the store. A row whose record it holds is left out of the insert,
743
+ # as Monitor's ON CONFLICT DO NOTHING leaves it (when +unique+,
744
+ # ActiveRecord::RecordNotUnique is raised, as Monitor's insert raises it,
745
+ # before anything is written). The table's row of any other key whose
746
+ # stream ends in a destroy is deleted (Lyra's own write), so the row is
747
+ # inserted and recorded as Monitor inserts it; one with no stream (a row
748
+ # Genesis skipped) is a record the table holds, and conflicts
749
+ # (events_only_dead_keys).
750
+ # Before, the table answered: a record the store held but the table
751
+ # did not (every record created through callbacks there) was inserted
752
+ # again, a second Created on its live stream, and its attributes
753
+ # overwritten; a key whose record had been inserted with insert_all and
754
+ # then destroyed met that record's row and was skipped, with no event,
755
+ # where Monitor creates the record again. Rows that give no key take
756
+ # the table's sequence, as before. Elsewhere, the rows as given.
757
+ def events_only_inserts(attributes, write, skip:)
758
+ return attributes unless Lyra.event_sourcing_mode? && Lyra.config.projection_mode == :disabled
759
+
760
+ rows = attributes.is_a?(Hash) ? [attributes] : Array(attributes)
761
+ # The keys' aggregate locks before the values' (events_only_unique_rows),
762
+ # as an update through callbacks takes them: the record, then the
763
+ # index entry, as Monitor's writes take the row, then the entry.
764
+ given = explicit_keys(rows)
765
+ lock_keys = -> { Lyra::BypassEvents.lock_streams!(klass, given, write: write) if given.any? }
766
+ rows = events_only_unique_rows(rows, write, skip, keys: given.size, before_locks: lock_keys)
767
+ pk = klass.primary_key
768
+ return rows unless pk.is_a?(String)
769
+
770
+ unique = !(skip == :any || skip == [pk])
771
+ type = klass.type_for_attribute(pk)
772
+ keyed = rows.map { |row| [row, explicit_key(row, pk, type)] }
773
+ ids = keyed.filter_map(&:last).uniq
774
+ return rows if ids.empty?
775
+
776
+ Lyra::BypassEvents.lock_streams!(klass, ids, write: write)
777
+ held = Lyra::PurposeBoundReads.internal(INTERNAL) { Lyra::Projections::EventStoreReader.find_many(klass, ids) }
778
+ taken = held.to_set { _1.read_attribute(pk) }
779
+ dead = events_only_dead_keys(ids.reject { taken.include?(_1) }, taken)
780
+ if unique && taken.any?
781
+ raise ActiveRecord::RecordNotUnique,
782
+ "Lyra: #{klass.name} #{pk} #{taken.first.inspect} is already taken (#{write})"
783
+ end
784
+
785
+ Lyra.projection_write(INTERNAL) { klass.unscoped.where(pk => dead).delete_all } if dead.any?
786
+ keyed.reject { |_, id| taken.include?(id) }.map(&:first)
787
+ end
788
+
789
+ # Of +ids+ (keys the event store does not hold), those whose table row
790
+ # (left by a destroyed record) is to be deleted for the insert: the
791
+ # streams that end in a destroy. A row with no stream (from before Lyra,
792
+ # which Genesis left without one: config.genesis_unrepresentable :skip)
793
+ # is a record Monitor's table holds, and its key is added to +taken+;
794
+ # it was deleted as a destroyed record's, and its data lost.
795
+ def events_only_dead_keys(ids, taken)
796
+ return [] if ids.empty?
797
+
798
+ pk = klass.primary_key
799
+ present = Lyra::PurposeBoundReads.internal(INTERNAL) { klass.unscoped.where(pk => ids).pluck(pk) }
800
+ return [] if present.empty?
801
+
802
+ streams = present.to_h { [Lyra.stream_name(klass, _1), _1] }
803
+ dead = Lyra::StreamTail.destroyed_streams(streams.keys).map { streams[_1] }
804
+ taken.merge(present - dead)
805
+ dead
806
+ end
807
+
808
+ # The model's unique indexes other than its primary key, checked from
809
+ # the event store as Monitor's table checks them (events_only_inserts).
810
+ # Before, ES-NoProj checked them against its table, which holds no row
811
+ # of a record created through callbacks: a duplicate (an email taken,
812
+ # with no key given) was inserted, a second record with the value, where
813
+ # Monitor's insert_all skips it and insert_all! raises; and the row a
814
+ # destroyed record left there was taken for a live one, skipping (or
815
+ # refusing) a value that is free.
816
+ #
817
+ # Each row's values for each index (the relation's scope and the column
818
+ # defaults filling what it does not give; a row with a NULL among them
819
+ # conflicts with nothing, unless the index is NULLS NOT DISTINCT) are
820
+ # locked as a uniqueness validation of the value locks them
821
+ # (Lyra::Projections::Uniqueness.lock_all!: a multi-column index by its
822
+ # first column's value, coarser, never finer), so a concurrent insert or
823
+ # validated create of the value waits. Those locks and the aggregate
824
+ # locks of the keys the rows give (+keys+) count together against
825
+ # config.bulk_write_lock_limit, PostgreSQL's lock table being what they
826
+ # share: a model with several unique indexes reaches the limit at fewer
827
+ # rows. The keys' locks (+before_locks+) are taken once the limit is
828
+ # checked and before the values', as an update takes its record's lock
829
+ # and then its values'. Then the records that may hold the values are read: those whose
830
+ # events are in the values' streams (Lyra::Projections::UniqueIndexes
831
+ # .candidate_records: one query for the streams, one for their events,
832
+ # one for the records, whatever the model's size), and the table's rows
833
+ # that hold the values (one query per index), each index's values
834
+ # compared as UniqueIndexes.key compares them (as the column holds
835
+ # them; a citext column's by the database's LOWER). Every record was
836
+ # read before (Uniqueness::Snapshot), at a cost that grew with the
837
+ # model: 52.8 s at 10,000 records, cold.
838
+ #
839
+ # A row conflicts with a value a record the store holds has, and with a
840
+ # value a table row holds whose record the store neither holds nor has
841
+ # destroyed (a row from before Lyra that Genesis left without a stream,
842
+ # config.genesis_unrepresentable :skip): skipped where +skip+ covers that
843
+ # index, else ActiveRecord::RecordNotUnique before anything is written.
844
+ # The table's rows whose streams end in a destroy are deleted (Lyra's own
845
+ # write), and the rows of live records that hold a value their records
846
+ # no longer have (written by an insert, updated since through callbacks,
847
+ # which write no row) are brought up to the values the store has, so
848
+ # the insert meets the table as Monitor's does. Rows of the same insert
849
+ # that conflict with each other meet in the table, as in Monitor. Partial
850
+ # and expression indexes are left to the table.
851
+ def events_only_unique_rows(rows, write, skip, keys: 0, before_locks: nil)
852
+ pk = klass.primary_key
853
+ unique = Lyra::Projections::UniqueIndexes
854
+ indexes = unique.of(klass)
855
+ return rows if indexes.empty? || rows.empty?
856
+
857
+ scope = scope_for_create.transform_keys(&:to_s)
858
+ defaults = klass.column_defaults
859
+ columns = indexes.flat_map(&:columns).uniq
860
+ # Each row's values, as the record would hold them.
861
+ given = rows.map do |row|
862
+ values = defaults.merge(scope.merge(row.to_h.to_h { |k, v| [klass.attribute_aliases.fetch(k.to_s, k.to_s), v] }))
863
+ columns.to_h { |column| [column, klass.type_for_attribute(column).cast(values[column])] }
864
+ end
865
+ key_tuple = lambda do |index, values|
866
+ compared = index.columns.map { |column| unique.key(klass, column, values[column]) }
867
+ compared.any?(&:nil?) && !index.nulls_not_distinct ? nil : compared
868
+ end
869
+ tuples = given.map { |values| indexes.map { |index| key_tuple.(index, values) } }
870
+
871
+ claims = given.each_index.flat_map do |r|
872
+ indexes.each_index.filter_map { |i| tuples[r][i] && [indexes[i].columns.first, given[r][indexes[i].columns.first]] }
873
+ end.uniq
874
+ return rows if claims.empty?
875
+
876
+ Lyra::BypassEvents.check_lock_limit!(klass, claims.size + keys, write)
877
+ Lyra::Projections::ReferenceLinks.catch_up(klass) if Lyra::Projections::ReferenceLinks.enabled?
878
+ before_locks&.call
879
+ Lyra::Projections::Uniqueness.lock_all!(klass, claims)
880
+ table = table_rows_holding(indexes, tuples, given, key_tuple)
881
+ wanted = tuples.flat_map { |row| row.each_with_index.filter_map { |tuple, i| tuple && [indexes[i], tuple] } }.uniq
882
+ records = Lyra::PurposeBoundReads.internal(INTERNAL) do
883
+ Lyra::Genesis.first_use(klass)
884
+ unique.candidate_records(klass, wanted, ids: table.map { _2 })
885
+ end
886
+ reader = Lyra::Projections::TypedValues::EventReader.new(klass.base_class)
887
+ entry_tuple = ->(index, attrs) { key_tuple.(index, index.columns.to_h { [_1, reader.read(attrs, _1)] }) }
888
+ live = records.to_h { [_1[pk], _1] }
889
+ held = indexes.map { |index| records.filter_map { |attrs| entry_tuple.(index, attrs) }.to_set }
890
+ dead, outdated = sort_table_rows(indexes, table, live, held, entry_tuple)
891
+
892
+ kept = rows.each_index.reject do |r|
893
+ taken = indexes.each_index.select { |i| tuples[r][i] && held[i].include?(tuples[r][i]) }
894
+ next false if taken.empty?
895
+
896
+ refused = taken.find { |i| !(skip == :any || (skip && skip.sort == indexes[i].columns.sort)) }
897
+ if refused
898
+ index = indexes[refused]
899
+ raise unique.not_unique(unique::Claim.new(index, tuples[r][refused], index.columns.map { given[r][_1] }))
900
+ end
901
+ true
902
+ end
903
+ Lyra.projection_write(INTERNAL) do
904
+ klass.unscoped.where(pk => dead).delete_all if dead.any?
905
+ outdated.each { |id| klass.unscoped.where(pk => id).update_all(columns.to_h { [_1, live[id][_1]] }) }
906
+ end
907
+ kept.map { rows[_1] }
908
+ end
909
+
910
+ # The table's rows that hold any of +tuples+' values (comparison keys;
911
+ # +given+: the rows' values), per index, as [index position, primary
912
+ # key, tuple], one query per index.
913
+ def table_rows_holding(indexes, tuples, given, key_tuple)
914
+ pk = klass.primary_key
915
+ indexes.each_with_index.flat_map do |index, i|
916
+ wanted = tuples.filter_map { _1[i] }.to_set
917
+ next [] if wanted.empty?
918
+
919
+ first = index.columns.first
920
+ values = given.each_index.filter_map { |r| given[r][first] if tuples[r][i] }.uniq
921
+ found = Lyra::PurposeBoundReads.internal(INTERNAL) do
922
+ klass.unscoped.where(first => values).pluck(pk, *index.columns)
923
+ end
924
+ found.filter_map do |id, *held|
925
+ tuple = key_tuple.(index, index.columns.zip(held).to_h)
926
+ [i, id, tuple] if wanted.include?(tuple)
927
+ end
928
+ end
929
+ end
930
+
931
+ # Sort the table's rows that hold a value being inserted (+table+): a
932
+ # live record's row holding a value its record no longer has is
933
+ # +outdated+; a row whose record the store does not hold is +dead+ when
934
+ # its stream ends in a destroy, and otherwise (no stream: a row Genesis
935
+ # skipped) its value is taken, added to +held+. Returns [dead, outdated].
936
+ def sort_table_rows(indexes, table, live, held, entry_tuple)
937
+ outdated = table.filter_map { |i, id, tuple| id if live[id] && entry_tuple.(indexes[i], live[id]) != tuple }.uniq
938
+ unknown = table.reject { |_, id, _| live[id] }
939
+ return [[], outdated] if unknown.empty?
940
+
941
+ streams = unknown.map { |_, id, _| id }.uniq.to_h { [Lyra.stream_name(klass, _1), _1] }
942
+ dead = Lyra::StreamTail.destroyed_streams(streams.keys).map { streams[_1] }.to_set
943
+ unknown.each { |i, id, tuple| held[i] << tuple unless dead.include?(id) }
944
+ [dead.to_a, outdated]
945
+ end
946
+
947
+ # ES-Async (Lyra::BypassEvents.projects_pending?): the records whose
948
+ # keys the rows of an insert_all or insert_all! give are locked
949
+ # (aggregate locks, within config.bulk_write_lock_limit) and their rows
950
+ # brought up to date with their streams
951
+ # (Lyra::Projections::AsyncProjectionJob.project_pending), before the
952
+ # insert. Its ON CONFLICT then meets the rows Monitor's would: a record
953
+ # whose destroy was waiting for its job has no row, and is inserted
954
+ # again, a Created after the Destroyed, with the values given and the
955
+ # column defaults; a record whose create was waiting has its row, which
956
+ # insert_all skips and insert_all! raises ActiveRecord::RecordNotUnique
957
+ # on. Before, the first was skipped (no event; the destroy's job then
958
+ # deleted its row), the second inserted with a second Created on its
959
+ # stream. Rows that give no key take the table's sequence. A conflict on
960
+ # another unique index is looked up in the event store
961
+ # (projected_unique_rows), as a record whose create waits for its job
962
+ # holds its value with no row.
963
+ #
964
+ # Returns the keys locked.
965
+ def project_pending_inserts(attributes, write)
966
+ return [] unless Lyra::BypassEvents.projects_pending?
967
+
968
+ ids = explicit_keys(attributes)
969
+ return [] if ids.empty?
970
+
971
+ Lyra::BypassEvents.lock_streams!(klass, ids, write: write)
972
+ Lyra::Projections::AsyncProjectionJob.project_pending(klass, ids)
973
+ ids
974
+ end
975
+
976
+ # ES-Async (Lyra::BypassEvents.projects_pending?): the write wrote the
977
+ # rows and their events (+published+: publish_upserts) in one
978
+ # transaction, so a row whose stream was up to date before it is up to
979
+ # date with its new event, and its projection checkpoint is moved there,
980
+ # in one statement (AsyncProjectionJob.advance): the records it locked
981
+ # and brought up to date (+locked+), and every record it created, whose
982
+ # row it inserted with the very attributes of its Created. Others (a
983
+ # record an upsert matched by another unique index, not locked) keep
984
+ # their checkpoints. Before, a bypass write never moved a checkpoint:
985
+ # the next insert or upsert of the same keys found every stream behind
986
+ # and replayed the previous write's events onto rows that already had
987
+ # them, as each record's next projection job did.
988
+ def advance_checkpoints(published, locked)
989
+ return unless Lyra::BypassEvents.projects_pending? && published.present?
990
+
991
+ locked = locked.to_set
992
+ moves = published.filter_map { |id, (operation, event_id)| [id, event_id] if operation == :created || locked.include?(id) }
993
+ Lyra::Projections::AsyncProjectionJob.advance(klass, moves)
994
+ end
995
+
996
+ # insert_all, insert_all! and upsert_all of a model whose primary key
997
+ # has several columns (one its table declares, enrolled before Active
998
+ # Record read it; Lyra::ModelSelection refuses one it knows) are refused
999
+ # in the event-sourcing modes, before anything is written: their
1000
+ # Created events name the record by an array no projection, catch-up or
1001
+ # rebuild can apply (ES-Lazy's catch-up then failed on every read and
1002
+ # write of every monitored model, and a rebuild failed), and an update
1003
+ # of such a record failed in Genesis. Monitor and Hijack, which replay
1004
+ # nothing, write them and record their events.
1005
+ def refuse_composite_key!(write)
1006
+ key = klass.primary_key
1007
+ return unless key.is_a?(Array) && Lyra.event_sourcing_mode?
1008
+
1009
+ raise ArgumentError,
1010
+ "#{klass.name} has a composite primary key (#{key.join(', ')}), which Lyra does not support: in the " \
1011
+ "event-sourcing modes #{write} would record events that no projection or rebuild can apply, so it is " \
1012
+ "refused before anything is written. A monitored model needs a single-column primary key."
1013
+ end
1014
+
1015
+ # The primary keys the rows of an insert give, cast, each once.
1016
+ def explicit_keys(attributes)
1017
+ pk = klass.primary_key
1018
+ return [] unless pk.is_a?(String)
1019
+
1020
+ type = klass.type_for_attribute(pk)
1021
+ rows = attributes.is_a?(Hash) ? [attributes] : Array(attributes)
1022
+ rows.filter_map { explicit_key(_1, pk, type) }.uniq
1023
+ end
1024
+
1025
+ # The primary key a row of an insert gives, cast, or nil.
1026
+ def explicit_key(row, pk, type)
1027
+ row = row.to_h.transform_keys(&:to_s)
1028
+ name = row.key?(pk) ? pk : klass.attribute_aliases.key(pk)
1029
+ name && row.key?(name) ? type.cast(row[name]) : nil
1030
+ end
1031
+
260
1032
  # Run the write with the primary key in RETURNING, so the inserted and
261
1033
  # updated rows can be identified, and hand the caller the result they
262
- # asked for. Returns [caller_result, ids]; ids is nil only when the
263
- # caller passed raw SQL for returning without the primary key.
1034
+ # asked for (Lyra::BypassEvents.with_returning: raw SQL for returning
1035
+ # gets the key under an alias of Lyra's own; the caller's result keeps
1036
+ # its column types). Returns [caller_result, ids]; ids nil where no key
1037
+ # can be returned (an adapter without RETURNING), as before. A primary
1038
+ # key of several columns (a model whose table declares one, enrolled
1039
+ # before Active Record read it: Lyra::ModelSelection refuses one it
1040
+ # knows) has each column returned, and each id is the array of their
1041
+ # values, as record.id gives it. Its key was asked for as one quoted
1042
+ # name, and an explicit returning: raised PG::UndefinedColumn; without
1043
+ # one, no key was found in the result and nothing was recorded.
264
1044
  def with_primary_keys(returning)
265
1045
  pk = klass.primary_key
266
- request, strip = case returning
267
- when nil then [nil, nil]
268
- when false then [[pk], :all]
269
- when Arel::Nodes::SqlLiteral then [returning, nil]
270
- else
271
- columns = Array(returning).map(&:to_s)
272
- columns.include?(pk) ? [returning, nil] : [columns + [pk], :pk]
273
- end
274
-
275
- result = yield(request)
276
- index = result.columns.index(pk)
277
- ids = index && result.rows.map { |row| row[index] }
278
-
279
- caller_result = case strip
280
- when :all then ActiveRecord::Result.empty
281
- when :pk
282
- ActiveRecord::Result.new(result.columns.reject.with_index { |_, i| i == index },
283
- result.rows.map { |row| row.reject.with_index { |_, i| i == index } })
284
- else result
285
- end
286
- [caller_result, ids]
1046
+ return [yield(returning), nil] if pk.blank?
1047
+
1048
+ keys = Array(pk)
1049
+ result, rows = Lyra::BypassEvents.with_returning(klass, returning, keys) { |ret| yield ret }
1050
+ [result, rows&.map { |row| pk.is_a?(Array) ? row.values_at(*keys) : row[pk] }]
1051
+ end
1052
+
1053
+ # The application's insert_all, insert_all! or upsert_all (+write+) on a
1054
+ # join table: the join model of a has_and_belongs_to_many, or a model it
1055
+ # declared on the table (+join_model+: HabtmLinks.join_model_of). Refused
1056
+ # in strict mode, unless inside Lyra.without_strict_access, where either
1057
+ # side's model is monitored (HabtmLinks.guarded?), as its delete_all of
1058
+ # join rows is; Rails' own links are inserted one by one, through the
1059
+ # join model's callbacks (HabtmLinks.create), and never come here.
1060
+ # Where it runs, the rows it wrote are recorded as the links they make
1061
+ # (HabtmLinks.insert_all). Before, it was neither refused nor recorded.
1062
+ def join_insert(write, join_model, attributes, returning, unique_by: nil, &block)
1063
+ check_strict_mode!(write) if Lyra::Associations::HabtmLinks.guarded?(join_model)
1064
+ conflict = write == :upsert_all ? conflict_columns(unique_by) : nil
1065
+ Lyra::Associations::HabtmLinks.insert_all(self, attributes, returning: returning, conflict: conflict, &block)
287
1066
  end
288
1067
 
289
1068
  # The columns upsert_all's ON CONFLICT uses: unique_by as columns, or as
290
1069
  # an index name, or the primary key by default.
291
1070
  def conflict_columns(unique_by)
292
- return [klass.primary_key] if unique_by.nil?
1071
+ return Array(klass.primary_key) if unique_by.nil?
293
1072
 
294
1073
  columns = Array(unique_by).map(&:to_s)
295
1074
  return columns unless columns.size == 1 && !klass.column_names.include?(columns.first)
@@ -307,11 +1086,13 @@ module Lyra
307
1086
 
308
1087
  # The bulk write, on exactly the rows the locked snapshot holds: no row
309
1088
  # another transaction inserted since is written without its event. Runs as
310
- # Lyra's own write, so it is not intercepted a second time.
1089
+ # Lyra's own write, so it is not intercepted a second time. The snapshot
1090
+ # already took the relation's limit and offset: the write by key keeps
1091
+ # neither (an offset applied again to the snapshot's rows wrote none).
311
1092
  def write_snapshotted(before)
312
1093
  return 0 if before.empty?
313
-
314
- Lyra.projection_write { yield where(klass.primary_key => before.keys) }
1094
+
1095
+ Lyra.projection_write(INTERNAL) { yield unscope(:limit, :offset).where(klass.primary_key => before.keys) }
315
1096
  end
316
1097
 
317
1098
  def safely_publish_bulk(source)
@@ -320,9 +1101,33 @@ module Lyra
320
1101
  Lyra::BypassEvents.publish_failed!(e, source)
321
1102
  end
322
1103
 
323
- # Detect if this is an association nullify operation (dependent: :nullify)
324
- # Rails sets a single foreign key column to NULL when destroying parent records
1104
+ # The shape of the write a has_many's nullify makes: one foreign key
1105
+ # column set to NULL. Rails' association code marks its own
1106
+ # (Lyra::Associations::Maintenance); the shape alone does not make a
1107
+ # write a nullify.
325
1108
  def association_nullify_update?(updates)
1109
+ StrictDataAccessRelation.nullify_update?(updates)
1110
+ end
1111
+
1112
+ # In the events-only store, with events recorded: whether this is an
1113
+ # Active Record relation's update_all clearing a foreign key, which the
1114
+ # table cannot answer (it is empty or stale there).
1115
+ def events_only_nullify?(updates)
1116
+ Lyra.event_sourcing_mode? && Lyra.config.projection_mode == :disabled && association_nullify_update?(updates)
1117
+ end
1118
+
1119
+ # The records the event store holds that this relation selects, all of
1120
+ # it (CachedRelation#narrowed_by: its conditions on the model's columns,
1121
+ # order and limit; anything else raises UnsupportedQuery, naming +what+).
1122
+ def events_only_relation(what)
1123
+ Lyra::PurposeBoundReads.internal(INTERNAL) do
1124
+ Lyra::Projections::EventStoreReader.relation(klass).narrowed_by(self, what)
1125
+ end
1126
+ end
1127
+
1128
+ # The same test for the events-only store, where the association writes
1129
+ # through the event-backed relation (AssociationInterceptor).
1130
+ def self.nullify_update?(updates)
326
1131
  return false unless updates.is_a?(Hash)
327
1132
  return false unless updates.size == 1
328
1133
 
@@ -334,80 +1139,74 @@ module Lyra
334
1139
  # Publish "updated" events for each record affected by dependent: :nullify
335
1140
  # This captures the foreign key nullification in the event stream
336
1141
  def publish_nullify_events(updates)
337
- column = updates.keys.first.to_s
1142
+ updates = updates.to_h { |key, value| [klass.attribute_aliases[key.to_s] || key.to_s, value] }
1143
+ columns = updates.keys
338
1144
 
339
1145
  # Get the records being updated with their current values
340
1146
  # In ES Disabled mode, records don't exist in DB, so we need to query
341
1147
  # the event store via the model's read path (which uses CachedRelation)
342
- records_data = fetch_affected_records_for_nullify(column)
1148
+ # Lyra's own read, for the events: the write itself is checked
1149
+ # (Lyra::PurposeBoundReads::RelationCheck).
1150
+ records_data = Lyra::PurposeBoundReads.internal(INTERNAL) { fetch_affected_records_for_nullify(columns) }
1151
+
1152
+ # Where events are required, a row Genesis skipped refuses the whole
1153
+ # destroy before any event (Lyra::Genesis.admit_all!). In Monitor each
1154
+ # row's failure is logged on its own: one skipped child does not cost
1155
+ # the others their events.
1156
+ Lyra::Genesis.admit_all!(klass, records_data.map(&:first)) if Lyra::BypassEvents.required?
1157
+ # Where the store can hold a destroy whose row is not gone
1158
+ # (Lyra::BypassEvents.checks_streams?), a destroyed child gets no event:
1159
+ # it would bring the child back. Its key is returned, for the write to
1160
+ # leave its row as well.
1161
+ skipped = []
1162
+ if Lyra::BypassEvents.checks_streams? && records_data.any?
1163
+ skipped = Lyra::BypassEvents.destroyed_ids(klass, records_data.map(&:first), write: "dependent: :nullify")
1164
+ records_data = records_data.reject { |id, _| skipped.include?(id) } if skipped.any?
1165
+ end
1166
+ # The columns it changes: the foreign key, and for a polymorphic
1167
+ # association its type, whatever they are called.
1168
+ records_data.each do |id, old_values|
1169
+ changes = columns.zip(old_values).each_with_object({}) do |(column, old_value), acc|
1170
+ acc[column] = [old_value, updates[column]] unless old_value == updates[column]
1171
+ end
1172
+ next if changes.empty?
343
1173
 
344
- records_data.each do |id, old_value|
345
1174
  Lyra::BypassEvents.publish(klass, id, :updated,
346
- attributes: { column => nil },
347
- changes: { column => [old_value, nil] },
1175
+ attributes: changes.transform_values(&:last),
1176
+ changes: changes,
348
1177
  source: "dependent_association")
1178
+ rescue => e
1179
+ Lyra::BypassEvents.publish_failed!(e, "dependent_association")
349
1180
  end
1181
+ skipped
350
1182
  rescue => e
351
1183
  # Fails the parent's destroy in Hijack and the event-sourcing modes
352
1184
  # (it runs inside its transaction); logged in Monitor.
353
1185
  Lyra::BypassEvents.publish_failed!(e, "dependent_association")
1186
+ []
354
1187
  end
355
1188
 
356
- # Fetch affected records for nullify operation
357
- # Handles both DB-backed mode and ES Disabled mode (event store only)
358
- def fetch_affected_records_for_nullify(column)
359
- # First, try to get records from DB (works in non-ES-Disabled modes)
360
- records_data = self.pluck(:id, column)
361
- return records_data unless records_data.empty?
362
-
363
- # In ES Disabled mode, records don't exist in DB
364
- # Extract the foreign key value from the where clause and query event store
365
- foreign_key_value = extract_foreign_key_from_where_clause(column)
366
- return [] unless foreign_key_value
367
-
368
- # Query the model using its normal read path (which uses EventStoreReader in ES mode)
369
- # This will return records from the event store cache
370
- if Lyra.event_sourcing_mode? && Lyra.config.projection_mode == :disabled
371
- cached_records = klass.where(column.to_sym => foreign_key_value)
372
- cached_records.map { |r| [r.id, r.send(column)] }
373
- else
374
- []
375
- end
376
- end
377
-
378
- # Extract the foreign key value from the relation's where clause
379
- # The relation is something like: Registration.where(group_id: 123)
380
- def extract_foreign_key_from_where_clause(column)
381
- return nil unless respond_to?(:where_clause)
382
-
383
- where_clause.send(:predicates).each do |predicate|
384
- next unless predicate.is_a?(Arel::Nodes::Equality)
385
- next unless predicate.left.respond_to?(:name)
386
- next unless predicate.left.name.to_s == column
387
-
388
- # Extract the value - Rails 7/8 uses QueryAttribute
389
- right = predicate.right
390
- case right
391
- when Arel::Nodes::Casted
392
- return right.value
393
- when Arel::Nodes::BindParam
394
- return right.value.value_before_type_cast
395
- when Integer, String
396
- return right
397
- else
398
- # Rails 7/8: ActiveRecord::Relation::QueryAttribute
399
- if right.respond_to?(:value)
400
- return right.value
401
- elsif right.respond_to?(:value_before_type_cast)
402
- return right.value_before_type_cast
403
- end
404
- end
1189
+ # The records the nullify rewrites, as [id, [old values of +columns+]]: the table's rows
1190
+ # the relation selects, or, in the events-only store (ES-NoProj), where
1191
+ # the table is empty or stale, the records the event store holds that
1192
+ # the relation's where clause selects, all of it (CachedRelation's
1193
+ # narrowed_by: its conditions on the model's columns, order and limit;
1194
+ # anything else it holds is refused with UnsupportedQuery, which fails
1195
+ # the write). A parent's dependent: :nullify is written by
1196
+ # AssociationInterceptor there; this is reached when Rails' association
1197
+ # code writes a table relation (Lyra's own projection writes aside). It
1198
+ # used to read the foreign key's equality alone, skipping any other
1199
+ # condition (where(author_id: 1, status: "published") nullified the
1200
+ # author's drafts too in the log, and matched no row in the table), and
1201
+ # to read the table first.
1202
+ def fetch_affected_records_for_nullify(columns)
1203
+ pk = klass.primary_key
1204
+ unless Lyra.event_sourcing_mode? && Lyra.config.projection_mode == :disabled
1205
+ return pluck(pk, *columns).map { |id, *values| [id, values] }
405
1206
  end
406
1207
 
407
- nil
408
- rescue => e
409
- Rails.logger.debug("Lyra: Could not extract foreign key from where clause - #{e.message}")
410
- nil
1208
+ selected = Lyra::Projections::EventStoreReader.relation(klass).narrowed_by(self, "update_all(#{columns.join(', ')}: nil)")
1209
+ selected.map { |record| [record.read_attribute(pk), columns.map { record.read_attribute(_1) }] }
411
1210
  end
412
1211
  end
413
1212