orfeas_lyra 0.9.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +2021 -17
- data/README.md +22 -9
- data/Rakefile +40 -1
- data/app/controllers/lyra/application_controller.rb +127 -0
- data/app/controllers/lyra/dashboard_controller.rb +33 -31
- data/app/controllers/lyra/flow_controller.rb +32 -34
- data/app/controllers/lyra/privacy_controller.rb +57 -18
- data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
- data/app/views/lyra/dashboard/index.html.erb +1 -0
- data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
- data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
- data/app/views/lyra/privacy/policy.html.erb +7 -0
- data/app/views/lyra/privacy/rights.html.erb +78 -0
- data/config/privacy_policies.rb +0 -3
- data/config/routes.rb +1 -0
- data/lib/lyra/access_log.rb +187 -6
- data/lib/lyra/advisory_lock.rb +236 -0
- data/lib/lyra/aggregate.rb +1 -1
- data/lib/lyra/associations/habtm_links.rb +382 -19
- data/lib/lyra/associations/maintenance.rb +229 -0
- data/lib/lyra/attribution.rb +48 -0
- data/lib/lyra/bypass_events.rb +446 -46
- data/lib/lyra/command_handler.rb +19 -14
- data/lib/lyra/configuration.rb +237 -10
- data/lib/lyra/consistency/read_your_writes.rb +4 -2
- data/lib/lyra/correlation.rb +121 -0
- data/lib/lyra/domain_events.rb +4 -0
- data/lib/lyra/dual_view.rb +53 -16
- data/lib/lyra/encrypted_attributes.rb +5 -0
- data/lib/lyra/engine.rb +67 -0
- data/lib/lyra/erasure.rb +1011 -106
- data/lib/lyra/event.rb +35 -1
- data/lib/lyra/event_data.rb +38 -0
- data/lib/lyra/event_flow.rb +31 -15
- data/lib/lyra/event_store_adapter.rb +28 -0
- data/lib/lyra/event_window.rb +58 -0
- data/lib/lyra/genesis.rb +548 -44
- data/lib/lyra/interceptors/association_interceptor.rb +689 -34
- data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
- data/lib/lyra/internal.rb +26 -0
- data/lib/lyra/mode_sync.rb +51 -3
- data/lib/lyra/mode_transition.rb +257 -30
- data/lib/lyra/model_selection.rb +35 -0
- data/lib/lyra/own_connection.rb +17 -2
- data/lib/lyra/privacy/adapters/pam.rb +25 -2
- data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
- data/lib/lyra/privacy/interface.rb +27 -7
- data/lib/lyra/privacy/subject_links.rb +143 -0
- data/lib/lyra/privacy/subject_records.rb +369 -0
- data/lib/lyra/projection.rb +7 -1
- data/lib/lyra/projections/async_checkpoint.rb +32 -0
- data/lib/lyra/projections/async_projection_job.rb +166 -23
- data/lib/lyra/projections/cached_joins.rb +33 -18
- data/lib/lyra/projections/cached_projection.rb +74 -33
- data/lib/lyra/projections/cached_relation.rb +3175 -321
- data/lib/lyra/projections/default_scopes.rb +254 -0
- data/lib/lyra/projections/deferred_relation.rb +23 -3
- data/lib/lyra/projections/event_store_reader.rb +47 -8
- data/lib/lyra/projections/foreign_keys.rb +92 -0
- data/lib/lyra/projections/lazy_projection.rb +165 -27
- data/lib/lyra/projections/model_projection.rb +165 -14
- data/lib/lyra/projections/rebuild.rb +34 -6
- data/lib/lyra/projections/reference_links.rb +463 -72
- data/lib/lyra/projections/typed_values.rb +629 -0
- data/lib/lyra/projections/unique_indexes.rb +595 -0
- data/lib/lyra/projections/uniqueness.rb +784 -0
- data/lib/lyra/projections/value_digest.rb +93 -0
- data/lib/lyra/purpose_bound_reads.rb +912 -77
- data/lib/lyra/repair.rb +119 -28
- data/lib/lyra/retention.rb +114 -8
- data/lib/lyra/rights/consent_store.rb +415 -0
- data/lib/lyra/rights/erasure_guard.rb +135 -0
- data/lib/lyra/rights/pending.rb +74 -0
- data/lib/lyra/rights/queries.rb +269 -0
- data/lib/lyra/rights/store.rb +512 -0
- data/lib/lyra/rights/subject_keys.rb +224 -0
- data/lib/lyra/rights.rb +281 -0
- data/lib/lyra/saves_without_effect.rb +69 -0
- data/lib/lyra/stream_tail.rb +319 -0
- data/lib/lyra/strict_data_access.rb +916 -106
- data/lib/lyra/temporal.rb +4 -2
- data/lib/lyra/touch_registry.rb +404 -0
- data/lib/lyra/verification/bypass_workflow.rb +452 -42
- data/lib/lyra/verification/rights_workflows.rb +170 -0
- data/lib/lyra/version.rb +1 -1
- data/lib/lyra/visualization/event_graph.rb +1 -1
- data/lib/lyra/visualization/timeline.rb +2 -2
- data/lib/lyra.rb +43 -2
- data/lib/tasks/lyra_mode.rake +110 -8
- data/lib/tasks/lyra_projections.rake +43 -2
- metadata +34 -4
- data/lib/lyra/associations/event_aware.rb +0 -224
|
@@ -55,9 +55,15 @@ module Lyra
|
|
|
55
55
|
# was seen, and no event was published (Solidus's
|
|
56
56
|
# Shipment#persist_amounts). Building after the write also covers the
|
|
57
57
|
# timestamps that touch: true sets.
|
|
58
|
+
#
|
|
59
|
+
# A has_one's dependent: :nullify clears its record's key with
|
|
60
|
+
# update_columns: Rails' association code (Lyra::Associations::Maintenance),
|
|
61
|
+
# allowed in strict mode as a has_many's nullify is (a polymorphic key's
|
|
62
|
+
# two columns included). It was refused, and the owner could not be
|
|
63
|
+
# destroyed. Its event is update_columns', as before.
|
|
58
64
|
def update_columns(attributes)
|
|
59
|
-
raise_strict_violation!(:update_columns)
|
|
60
|
-
with_bypass_event(bypass_column_names(attributes), "update_columns") { super }
|
|
65
|
+
raise_strict_violation!(:update_columns) unless Lyra::Associations::Maintenance.claim(:nullify, self)
|
|
66
|
+
with_bypass_event(bypass_column_names(attributes), "update_columns", unique: attributes) { super }
|
|
61
67
|
end
|
|
62
68
|
|
|
63
69
|
# Rails implements update_column(name, value, touch:) as update_columns,
|
|
@@ -77,15 +83,49 @@ module Lyra
|
|
|
77
83
|
self.class.attribute_aliases[name] || name
|
|
78
84
|
end
|
|
79
85
|
columns |= [self.class.locking_column] if Lyra::OptimisticLocking.enabled?(self.class)
|
|
80
|
-
|
|
86
|
+
deferred, @lyra_deferred_touch_before = @lyra_deferred_touch_before, nil
|
|
87
|
+
with_bypass_event(columns, "touch", before: deferred) { super }
|
|
81
88
|
end
|
|
82
89
|
|
|
90
|
+
# A copy of a record destroyed since it was read (its stream ends in a
|
|
91
|
+
# destroy, read under the aggregate lock: Lyra::BypassEvents
|
|
92
|
+
# .checks_streams?) records nothing and deletes nothing, as Active
|
|
93
|
+
# Record's delete of a row already gone: it appended a second Destroyed,
|
|
94
|
+
# a stream no replay can apply. The record is frozen, as Active Record
|
|
95
|
+
# leaves it; a row still in the table (ES-Async, before the destroy's
|
|
96
|
+
# job runs) is left to the job (Lyra::BypassEvents.unwritten).
|
|
83
97
|
def delete
|
|
84
98
|
raise_strict_violation!(:delete)
|
|
99
|
+
if persisted? && Lyra::BypassEvents.enabled_for?(self.class) && Lyra::BypassEvents.checks_streams?
|
|
100
|
+
return self.class.transaction do
|
|
101
|
+
if Lyra::BypassEvents.destroyed_ids(self.class, [id], write: "delete").any?
|
|
102
|
+
Lyra::BypassEvents.unwritten(self.class) { super }
|
|
103
|
+
self
|
|
104
|
+
else
|
|
105
|
+
safely_publish("delete") do
|
|
106
|
+
Lyra::BypassEvents.publish(self.class, id, :destroyed, attributes: attributes, source: "delete")
|
|
107
|
+
end
|
|
108
|
+
super
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
if persisted? && Lyra::BypassEvents.enabled_for?(self.class) && Lyra::BypassEvents.locks_rows?
|
|
113
|
+
# Where the rows are the store: the row locked first; a copy whose
|
|
114
|
+
# row another write deleted records nothing (it appended a second
|
|
115
|
+
# Destroyed), as Active Record's delete reaches no row.
|
|
116
|
+
return Lyra::BypassEvents.atomically(self.class) do
|
|
117
|
+
if lyra_lock_row_for_delete
|
|
118
|
+
safely_publish("delete") do
|
|
119
|
+
Lyra::BypassEvents.publish(self.class, id, :destroyed, attributes: attributes, source: "delete")
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
super
|
|
123
|
+
end
|
|
124
|
+
end
|
|
85
125
|
if Lyra::BypassEvents.enabled_for?(self.class)
|
|
86
126
|
safely_publish("delete") do
|
|
87
127
|
Lyra::BypassEvents.publish(self.class, id, :destroyed,
|
|
88
|
-
attributes: attributes
|
|
128
|
+
attributes: attributes,
|
|
89
129
|
source: "delete")
|
|
90
130
|
end
|
|
91
131
|
end
|
|
@@ -94,6 +134,34 @@ module Lyra
|
|
|
94
134
|
|
|
95
135
|
private
|
|
96
136
|
|
|
137
|
+
# belongs_to ... touch: true runs through Rails' touch_later: it writes
|
|
138
|
+
# the timestamps into the record at once and clears their change
|
|
139
|
+
# (surreptitiously_touch), and calls touch when the transaction commits.
|
|
140
|
+
# By then attribute_in_database holds the new value, so touch saw no change
|
|
141
|
+
# and the cascade published nothing: the parent's row moved and its stream
|
|
142
|
+
# did not. The value the row held before the first deferred touch is kept
|
|
143
|
+
# here, and the next touch takes it as the old value.
|
|
144
|
+
def surreptitiously_touch(attr_names)
|
|
145
|
+
@lyra_deferred_touch_before ||= {}
|
|
146
|
+
attr_names.each do |name|
|
|
147
|
+
name = name.to_s
|
|
148
|
+
@lyra_deferred_touch_before[name] = attribute_in_database(name) unless @lyra_deferred_touch_before.key?(name)
|
|
149
|
+
end
|
|
150
|
+
super
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# Whether the record's row is there, locked (FOR UPDATE) until the
|
|
154
|
+
# transaction ends.
|
|
155
|
+
def lyra_lock_row_for_delete
|
|
156
|
+
klass = self.class
|
|
157
|
+
key = respond_to?(:id_in_database) ? id_in_database : id
|
|
158
|
+
klass.with_connection do |connection|
|
|
159
|
+
sql = "SELECT 1 FROM #{connection.quote_table_name(klass.table_name)} " \
|
|
160
|
+
"WHERE #{connection.quote_column_name(klass.primary_key)} = #{connection.quote(key)} FOR UPDATE"
|
|
161
|
+
connection.uncached { connection.select_value(sql, "Lyra delete row lock") }.present?
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
|
|
97
165
|
def raise_strict_violation!(method_name)
|
|
98
166
|
return unless Lyra.config.strict_data_access
|
|
99
167
|
return if Thread.current[:lyra_bypass_strict_access]
|
|
@@ -115,10 +183,50 @@ module Lyra
|
|
|
115
183
|
# In an events-only store (ES-NoProj) there is no row: the UPDATE matches
|
|
116
184
|
# nothing and returns false, and the event is the write. Publishing only
|
|
117
185
|
# on a matched row lost every update_columns and touch there.
|
|
118
|
-
|
|
186
|
+
# +before+: old values known from elsewhere (a deferred touch's), which
|
|
187
|
+
# attribute_in_database no longer holds.
|
|
188
|
+
#
|
|
189
|
+
# A copy of a record destroyed since it was read (its stream ends in a
|
|
190
|
+
# destroy, read under the aggregate lock where the store can hold a
|
|
191
|
+
# destroy whose row is not gone: Lyra::BypassEvents.checks_streams?)
|
|
192
|
+
# records nothing, writes no row, and returns false, as Active Record's
|
|
193
|
+
# update_columns and touch of a row already gone write nothing and
|
|
194
|
+
# return false (the attributes are set in memory, as there). It
|
|
195
|
+
# appended an update after the destroy and brought the record back.
|
|
196
|
+
#
|
|
197
|
+
# +unique+: the attributes update_columns writes. In ES-NoProj, after the
|
|
198
|
+
# stream check (under the record's aggregate lock, where streams are
|
|
199
|
+
# checked) and before anything is written or recorded: where a value
|
|
200
|
+
# given is the one the copy holds, the record is read from its stream (a
|
|
201
|
+
# copy read before another writer changed it held another value, which
|
|
202
|
+
# its UPDATE in the table modes overwrites: the write recorded nothing);
|
|
203
|
+
# and the values it claims in the table's unique indexes are locked (after
|
|
204
|
+
# the aggregate lock, as Monitor's UPDATE takes the row, then the index
|
|
205
|
+
# entries) and checked against the event store (Lyra::Projections
|
|
206
|
+
# ::UniqueIndexes): a value a live record holds raises
|
|
207
|
+
# ActiveRecord::RecordNotUnique, as the table's UPDATE raises it in the
|
|
208
|
+
# other modes. Before, the duplicate was recorded.
|
|
209
|
+
def with_bypass_event(columns, source, before: nil, unique: nil)
|
|
119
210
|
Lyra::BypassEvents.atomically(self.class) do
|
|
211
|
+
enabled = persisted? && Lyra::BypassEvents.enabled_for?(self.class)
|
|
212
|
+
Lyra::BypassEvents.admit(self.class, [id]) if enabled
|
|
213
|
+
if enabled && Lyra::BypassEvents.checks_streams? &&
|
|
214
|
+
Lyra::BypassEvents.destroyed_ids(self.class, [id], write: source).any?
|
|
215
|
+
Lyra::BypassEvents.unwritten(self.class) { yield }
|
|
216
|
+
next false
|
|
217
|
+
end
|
|
120
218
|
events_only = Lyra.event_sourcing_mode? && Lyra.config.projection_mode == :disabled
|
|
121
|
-
|
|
219
|
+
stored = lyra_events_only_columns(unique) if enabled && events_only && unique
|
|
220
|
+
# ES-Lazy: the row as the stream has it, under the lock just taken.
|
|
221
|
+
Lyra::Projections::LazyProjection.refresh_records(self.class, [id]) if enabled && Lyra::BypassEvents.checks_streams?
|
|
222
|
+
lyra_projected_unique_columns(unique) if enabled && unique && !events_only && Lyra.event_sourcing_mode?
|
|
223
|
+
known = before || {}
|
|
224
|
+
before = columns.to_h do |c|
|
|
225
|
+
[c, if known.key?(c) then known[c]
|
|
226
|
+
elsif stored&.key?(c) then stored[c]
|
|
227
|
+
else attribute_in_database(c)
|
|
228
|
+
end]
|
|
229
|
+
end
|
|
122
230
|
result = yield
|
|
123
231
|
written = result || events_only
|
|
124
232
|
changes = columns.each_with_object({}) do |c, h|
|
|
@@ -130,10 +238,96 @@ module Lyra
|
|
|
130
238
|
end
|
|
131
239
|
end
|
|
132
240
|
|
|
241
|
+
# ES-NoProj's update_columns(+attributes+), before its write (see
|
|
242
|
+
# with_bypass_event): the record's values as its stream has them, when a
|
|
243
|
+
# value given is the copy's own ({ column => value }, else nil), and the
|
|
244
|
+
# unique values it claims locked and checked.
|
|
245
|
+
def lyra_events_only_columns(attributes)
|
|
246
|
+
klass = self.class
|
|
247
|
+
given = lyra_given_columns(attributes)
|
|
248
|
+
stored = nil
|
|
249
|
+
if given.any? { |column, value| value == attribute_in_database(column) }
|
|
250
|
+
record = Lyra::PurposeBoundReads.internal(INTERNAL) { Lyra::Projections::EventStoreReader.find(klass, id) }
|
|
251
|
+
stored = record && klass.column_names.to_h { |column| [column, record.read_attribute(column)] }
|
|
252
|
+
end
|
|
253
|
+
claims = lyra_unique_column_claims(given, stored)
|
|
254
|
+
if claims.present?
|
|
255
|
+
Lyra::Projections::UniqueIndexes.lock!(klass, claims)
|
|
256
|
+
Lyra::Projections::UniqueIndexes.refuse_taken!(klass, claims, except: id)
|
|
257
|
+
end
|
|
258
|
+
stored
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
# { column => value as the record will hold it } of update_columns'
|
|
262
|
+
# +attributes+ (aliases resolved; touch: and unknown names left out).
|
|
263
|
+
def lyra_given_columns(attributes)
|
|
264
|
+
klass = self.class
|
|
265
|
+
attributes.to_h.each_with_object({}) do |(name, value), acc|
|
|
266
|
+
column = klass.attribute_aliases.fetch(name.to_s, name.to_s)
|
|
267
|
+
next if column == "touch" || !klass.column_names.include?(column)
|
|
268
|
+
|
|
269
|
+
acc[column] = klass.type_for_attribute(column).cast(value)
|
|
270
|
+
end
|
|
271
|
+
end
|
|
272
|
+
|
|
273
|
+
# update_columns(+attributes+) in ES-Sync, ES-Async and ES-Lazy, before
|
|
274
|
+
# its table write (with_bypass_event): the values it sets in the table's
|
|
275
|
+
# unique indexes (Lyra::Projections::UniqueIndexes) are locked, so a
|
|
276
|
+
# write through callbacks checking the value waits for this one, as this
|
|
277
|
+
# one waits for it. Where the table lags the event store (ES-Async: a
|
|
278
|
+
# record created or moved there whose job has not run; ES-Lazy: one
|
|
279
|
+
# written after the write's catch-up) they are then checked against the
|
|
280
|
+
# event store, under the record's aggregate lock: the table's UPDATE
|
|
281
|
+
# missed such a holder and recorded the duplicate, which the holder's
|
|
282
|
+
# projection then met (ES-Async also brings up to date, where it waits
|
|
283
|
+
# for no one, the rows of records that gave the value up, which the
|
|
284
|
+
# UPDATE would meet). ES-Sync's table is current: its UPDATE checks, and
|
|
285
|
+
# its row is locked first (FOR NO KEY UPDATE, as the UPDATE and a save
|
|
286
|
+
# through callbacks lock it), then the values, as Monitor's UPDATE takes
|
|
287
|
+
# the row, then the index entries.
|
|
288
|
+
def lyra_projected_unique_columns(attributes)
|
|
289
|
+
klass = self.class
|
|
290
|
+
indexes = Lyra::Projections::UniqueIndexes
|
|
291
|
+
return unless indexes.enforced?(klass) && indexes.of(klass).any?
|
|
292
|
+
|
|
293
|
+
written = lyra_given_columns(attributes).slice(*indexes.columns(klass))
|
|
294
|
+
return if written.empty?
|
|
295
|
+
|
|
296
|
+
in_table = indexes.in_table?(klass)
|
|
297
|
+
lyra_lock_own_row(:update) if in_table && respond_to?(:lyra_lock_own_row, true)
|
|
298
|
+
claims = indexes.update_claims(klass, id, written)
|
|
299
|
+
return if claims.empty?
|
|
300
|
+
|
|
301
|
+
indexes.lock!(klass, claims)
|
|
302
|
+
indexes.refuse_taken!(klass, claims, except: id) unless in_table
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
# The values update_columns claims in the table's unique indexes, in
|
|
306
|
+
# ES-NoProj (Lyra::Projections::UniqueIndexes): for each index with a
|
|
307
|
+
# column whose value it changes (from the record's stream, +stored+, if
|
|
308
|
+
# read), the tuple the record will hold (the values given, the others as
|
|
309
|
+
# the stream has them). nil for a model without such an index.
|
|
310
|
+
def lyra_unique_column_claims(given, stored)
|
|
311
|
+
klass = self.class
|
|
312
|
+
indexes = Lyra::Projections::UniqueIndexes
|
|
313
|
+
return nil unless indexes.enforced?(klass) && indexes.of(klass).any?
|
|
314
|
+
|
|
315
|
+
written = given.slice(*indexes.columns(klass)).reject do |column, value|
|
|
316
|
+
held = stored ? stored[column] : attribute_in_database(column)
|
|
317
|
+
indexes.key(klass, column, value) == indexes.key(klass, column, held)
|
|
318
|
+
end
|
|
319
|
+
return nil if written.empty?
|
|
320
|
+
|
|
321
|
+
indexes.update_claims(klass, id, written, current: stored)
|
|
322
|
+
end
|
|
323
|
+
|
|
133
324
|
# Publish an "updated" event for a write that bypassed callbacks.
|
|
134
325
|
# +changes+ maps column => [old, new].
|
|
135
326
|
def publish_bypass_update_event(changes, source)
|
|
136
327
|
return unless Lyra::BypassEvents.enabled_for?(self.class)
|
|
328
|
+
# A touch (instance, or belongs_to ... touch: true) that changed only the
|
|
329
|
+
# update timestamps, with config.record_touches false in Monitor.
|
|
330
|
+
return if source == "touch" && Lyra::BypassEvents.skip_touch?(self.class, changes)
|
|
137
331
|
|
|
138
332
|
safely_publish(source) do
|
|
139
333
|
Lyra::BypassEvents.publish(self.class, id, :updated,
|
|
@@ -158,63 +352,142 @@ module Lyra
|
|
|
158
352
|
def update_all(updates)
|
|
159
353
|
return super unless lyra_guarded?
|
|
160
354
|
|
|
161
|
-
#
|
|
162
|
-
#
|
|
163
|
-
#
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
355
|
+
# Rails' own nullify (a has_many's dependent: :nullify, and children
|
|
356
|
+
# removed from the collection; a has_many :through's
|
|
357
|
+
# dependent: :nullify of its through records), the relation it writes
|
|
358
|
+
# marked by its association code (Lyra::Associations::Maintenance). It
|
|
359
|
+
# is always allowed, but the records it rewrites still get events. The
|
|
360
|
+
# events and the write are one transaction where events are required:
|
|
361
|
+
# a write that fails after its events were published takes them back.
|
|
362
|
+
# The same write made by the application is an update_all like any
|
|
363
|
+
# other, below.
|
|
364
|
+
if Lyra::Associations::Maintenance.claim(:nullify, self) && updates.is_a?(Hash)
|
|
365
|
+
return super unless Lyra::BypassEvents.enabled_for?(klass)
|
|
366
|
+
|
|
367
|
+
return Lyra::BypassEvents.atomically(klass) do
|
|
368
|
+
skipped = publish_nullify_events(updates)
|
|
369
|
+
# A child whose stream ends in a destroy (ES-Async, before its
|
|
370
|
+
# job deletes the row) is left as it is, as its event was.
|
|
371
|
+
next super if skipped.blank?
|
|
372
|
+
|
|
373
|
+
Lyra.projection_write(INTERNAL) { where.not(klass.primary_key => skipped).update_all(updates) }
|
|
374
|
+
end
|
|
167
375
|
end
|
|
168
376
|
|
|
169
377
|
check_strict_mode!(:update_all)
|
|
170
378
|
return super unless Lyra::BypassEvents.enabled_for?(klass)
|
|
379
|
+
# ES-NoProj: an Active Record relation of the model (one built past
|
|
380
|
+
# Lyra's reads) clearing a foreign key writes the records the event
|
|
381
|
+
# store holds that it selects, as it did when it was taken for a
|
|
382
|
+
# nullify, now as an update_all (CachedRelation#update_all: strict
|
|
383
|
+
# mode, the stream checks, the lock limit, an event per record).
|
|
384
|
+
return events_only_relation("update_all(#{updates.keys.first}: nil)").update_all(updates) if events_only_nullify?(updates)
|
|
171
385
|
|
|
172
386
|
Lyra::BypassEvents.atomically(klass) do
|
|
173
|
-
|
|
174
|
-
|
|
387
|
+
lock = Lyra::BypassEvents.locks_rows?
|
|
388
|
+
before = lock ? Lyra::BypassEvents.locked_snapshot(klass, self, write: "update_all") : Lyra::BypassEvents.snapshot(klass, self)
|
|
389
|
+
projected_unique_update(before, updates) if lock
|
|
390
|
+
result = lock ? write_snapshotted(before) { |rows| rows.update_all(updates) } : super
|
|
175
391
|
safely_publish_bulk("update_all") { Lyra::BypassEvents.publish_updates(klass, before, source: "update_all") }
|
|
176
392
|
result
|
|
177
393
|
end
|
|
178
394
|
end
|
|
179
395
|
|
|
396
|
+
# Rails's touch_all is update_all of the timestamp columns, recorded as
|
|
397
|
+
# update_all. With config.record_touches false (Monitor), the rows whose
|
|
398
|
+
# only change is the update timestamp are left out of the log
|
|
399
|
+
# (BypassEvents.skip_touch?).
|
|
400
|
+
def touch_all(*names, time: nil)
|
|
401
|
+
return super unless Lyra.config.skip_touches? && lyra_guarded?
|
|
402
|
+
|
|
403
|
+
Lyra::BypassEvents.touching_all { super }
|
|
404
|
+
end
|
|
405
|
+
|
|
180
406
|
def delete_all
|
|
181
|
-
# Deleting has_and_belongs_to_many join rows unlinks their owners
|
|
182
|
-
|
|
407
|
+
# Deleting has_and_belongs_to_many join rows unlinks their owners:
|
|
408
|
+
# always allowed when Rails' association code deletes them (delete,
|
|
409
|
+
# clear, assigning, destroying the owner: Lyra::Associations::Maintenance),
|
|
410
|
+
# refused in strict mode when the application does, through the join
|
|
411
|
+
# model or a model of its own on the join table.
|
|
412
|
+
join_model = Lyra::Associations::HabtmLinks.join_model_of(klass)
|
|
413
|
+
if join_model
|
|
414
|
+
association = Lyra::Associations::Maintenance.claim(:unlink, self)
|
|
415
|
+
check_strict_mode!(:delete_all) unless association || !Lyra::Associations::HabtmLinks.guarded?(join_model)
|
|
416
|
+
# A join model's rows unlink its owners; an application model's
|
|
417
|
+
# rows of a join table, every monitored owner they name (several
|
|
418
|
+
# has_and_belongs_to_many can share the table).
|
|
419
|
+
return Lyra::Associations::HabtmLinks.delete_all(self) { super } if join_model.equal?(klass)
|
|
420
|
+
|
|
421
|
+
return Lyra::Associations::HabtmLinks.delete_all_on_table(self) { super }
|
|
422
|
+
end
|
|
183
423
|
return super unless lyra_guarded?
|
|
184
424
|
|
|
185
|
-
|
|
425
|
+
# A has_many :through's through records, deleted when records are
|
|
426
|
+
# removed from it (Lyra::Associations::Maintenance): allowed, and
|
|
427
|
+
# recorded as any delete_all.
|
|
428
|
+
check_strict_mode!(:delete_all) unless Lyra::Associations::Maintenance.claim(:unlink, self)
|
|
186
429
|
return super unless Lyra::BypassEvents.enabled_for?(klass)
|
|
187
430
|
|
|
188
431
|
Lyra::BypassEvents.atomically(klass) do
|
|
189
|
-
|
|
190
|
-
|
|
432
|
+
lock = Lyra::BypassEvents.locks_rows?
|
|
433
|
+
before = lock ? Lyra::BypassEvents.locked_snapshot(klass, self, write: "delete_all") : Lyra::BypassEvents.snapshot(klass, self)
|
|
434
|
+
result = lock ? write_snapshotted(before) { |rows| rows.delete_all } : super
|
|
191
435
|
safely_publish_bulk("delete_all") { Lyra::BypassEvents.publish_destroys(klass, before, source: "delete_all") }
|
|
192
436
|
result
|
|
193
437
|
end
|
|
194
438
|
end
|
|
195
439
|
|
|
440
|
+
# In ES-NoProj a row's primary key conflicts as the event store has it
|
|
441
|
+
# (events_only_inserts): a record it holds is skipped, as the table's
|
|
442
|
+
# ON CONFLICT DO NOTHING skips a row already there in Monitor; a key
|
|
443
|
+
# whose record was destroyed is inserted again, its stream going on with
|
|
444
|
+
# a Created after the Destroyed, as Monitor's log does. In ES-Async the
|
|
445
|
+
# rows of the keys given are brought up to date with their streams
|
|
446
|
+
# first (project_pending_inserts), so the insert conflicts as Monitor's
|
|
447
|
+
# does there too.
|
|
448
|
+
#
|
|
449
|
+
# On a join table (Lyra::Associations::HabtmLinks.join_model_of): guarded
|
|
450
|
+
# as the application's delete_all of join rows is, and recorded as the
|
|
451
|
+
# links the rows written make (join_insert).
|
|
196
452
|
def insert_all(attributes, returning: nil, **options)
|
|
453
|
+
if (join_model = Lyra::Associations::HabtmLinks.join_model_of(klass))
|
|
454
|
+
return join_insert(:insert_all, join_model, attributes, returning) { |ret| super(attributes, returning: ret, **options) }
|
|
455
|
+
end
|
|
197
456
|
return super unless lyra_guarded?
|
|
198
457
|
|
|
199
458
|
check_strict_mode!(:insert_all)
|
|
200
459
|
return super unless Lyra::BypassEvents.enabled_for?(klass)
|
|
201
460
|
|
|
461
|
+
refuse_composite_key!(:insert_all)
|
|
202
462
|
Lyra::BypassEvents.atomically(klass) do
|
|
463
|
+
skip = options[:unique_by].nil? ? :any : conflict_columns(options[:unique_by])
|
|
464
|
+
attributes = events_only_inserts(attributes, "insert_all", skip: skip)
|
|
465
|
+
locked = project_pending_inserts(attributes, "insert_all")
|
|
466
|
+
attributes, = projected_unique_rows(attributes, "insert_all", skip: skip, keys: locked)
|
|
203
467
|
result, ids = with_primary_keys(returning) { |ret| super(attributes, returning: ret, **options) }
|
|
204
|
-
safely_publish_bulk("insert_all") { Lyra::BypassEvents.publish_upserts(klass, {}, ids, source: "insert_all") }
|
|
468
|
+
published = safely_publish_bulk("insert_all") { Lyra::BypassEvents.publish_upserts(klass, {}, ids, source: "insert_all") }
|
|
469
|
+
advance_checkpoints(published, locked)
|
|
205
470
|
result
|
|
206
471
|
end
|
|
207
472
|
end
|
|
208
473
|
|
|
209
474
|
def insert_all!(attributes, returning: nil, **options)
|
|
475
|
+
if (join_model = Lyra::Associations::HabtmLinks.join_model_of(klass))
|
|
476
|
+
return join_insert(:insert_all!, join_model, attributes, returning) { |ret| super(attributes, returning: ret, **options) }
|
|
477
|
+
end
|
|
210
478
|
return super unless lyra_guarded?
|
|
211
479
|
|
|
212
480
|
check_strict_mode!(:insert_all!)
|
|
213
481
|
return super unless Lyra::BypassEvents.enabled_for?(klass)
|
|
214
482
|
|
|
483
|
+
refuse_composite_key!(:insert_all!)
|
|
215
484
|
Lyra::BypassEvents.atomically(klass) do
|
|
485
|
+
attributes = events_only_inserts(attributes, "insert_all!", skip: nil)
|
|
486
|
+
locked = project_pending_inserts(attributes, "insert_all!")
|
|
487
|
+
attributes, = projected_unique_rows(attributes, "insert_all!", skip: nil, keys: locked)
|
|
216
488
|
result, ids = with_primary_keys(returning) { |ret| super(attributes, returning: ret, **options) }
|
|
217
|
-
safely_publish_bulk("insert_all!") { Lyra::BypassEvents.publish_upserts(klass, {}, ids, source: "insert_all!") }
|
|
489
|
+
published = safely_publish_bulk("insert_all!") { Lyra::BypassEvents.publish_upserts(klass, {}, ids, source: "insert_all!") }
|
|
490
|
+
advance_checkpoints(published, locked)
|
|
218
491
|
result
|
|
219
492
|
end
|
|
220
493
|
end
|
|
@@ -223,11 +496,17 @@ module Lyra
|
|
|
223
496
|
# already existed are found by the conflict key before the write; after
|
|
224
497
|
# it, those are "updated" (with their changes) and the rest "created".
|
|
225
498
|
def upsert_all(attributes, returning: nil, unique_by: nil, **options)
|
|
499
|
+
if (join_model = Lyra::Associations::HabtmLinks.join_model_of(klass))
|
|
500
|
+
return join_insert(:upsert_all, join_model, attributes, returning, unique_by: unique_by) do |ret|
|
|
501
|
+
super(attributes, returning: ret, unique_by: unique_by, **options)
|
|
502
|
+
end
|
|
503
|
+
end
|
|
226
504
|
return super unless lyra_guarded?
|
|
227
505
|
|
|
228
506
|
check_strict_mode!(:upsert_all)
|
|
229
507
|
return super unless Lyra::BypassEvents.enabled_for?(klass)
|
|
230
508
|
|
|
509
|
+
refuse_composite_key!(:upsert_all)
|
|
231
510
|
if Lyra.event_sourcing_mode? && Lyra.config.projection_mode == :disabled
|
|
232
511
|
raise Lyra::Projections::UnsupportedQuery.new(
|
|
233
512
|
klass,
|
|
@@ -240,54 +519,556 @@ module Lyra
|
|
|
240
519
|
keys = conflict_columns(unique_by)
|
|
241
520
|
Lyra::BypassEvents.atomically(klass) do
|
|
242
521
|
before = Lyra::BypassEvents.snapshot_by_keys(klass, rows, keys)
|
|
522
|
+
before, locked = lock_upserted_rows(before, rows, keys) if Lyra::BypassEvents.checks_streams?
|
|
523
|
+
_, brought = projected_unique_rows(attributes, "upsert_all", upsert: keys, keys: Array(locked))
|
|
524
|
+
# A record whose row the check brought up to date (ES-Async: its
|
|
525
|
+
# create's job had not run) is updated by the upsert, not created.
|
|
526
|
+
before = Lyra::BypassEvents.snapshot_by_keys(klass, rows, keys) if brought.any?
|
|
243
527
|
result, ids = with_primary_keys(returning) do |ret|
|
|
244
528
|
super(attributes, returning: ret, unique_by: unique_by, **options)
|
|
245
529
|
end
|
|
246
530
|
ids ||= Lyra::BypassEvents.snapshot_by_keys(klass, rows, keys).keys
|
|
247
|
-
safely_publish_bulk("upsert_all") { Lyra::BypassEvents.publish_upserts(klass, before, ids, source: "upsert_all") }
|
|
531
|
+
published = safely_publish_bulk("upsert_all") { Lyra::BypassEvents.publish_upserts(klass, before, ids, source: "upsert_all") }
|
|
532
|
+
advance_checkpoints(published, locked)
|
|
248
533
|
result
|
|
249
534
|
end
|
|
250
535
|
end
|
|
251
536
|
|
|
252
537
|
private
|
|
253
538
|
|
|
539
|
+
# Where the table can hold the row of a record whose stream ends in a
|
|
540
|
+
# destroy (Lyra::BypassEvents.checks_streams?: ES-Async before the
|
|
541
|
+
# destroy's projection job runs, ES-Lazy before its next catch-up; in
|
|
542
|
+
# ES-NoProj upsert_all is refused above), the records whose rows the
|
|
543
|
+
# upsert conflicts with have their aggregate locks taken (all or none,
|
|
544
|
+
# within config.bulk_write_lock_limit) and their streams read. ON CONFLICT
|
|
545
|
+
# updated such a row and appended an Updated after the Destroyed,
|
|
546
|
+
# bringing the record back (and the destroy's job then deleted the row,
|
|
547
|
+
# leaving the log and the table apart). Active Record, whose row is
|
|
548
|
+
# gone, inserts a new row there: with the column defaults for what the
|
|
549
|
+
# upsert does not name, under unique_by another key perhaps.
|
|
550
|
+
#
|
|
551
|
+
# In ES-Async (Lyra::BypassEvents.projects_pending?) the records of the
|
|
552
|
+
# keys the rows give are locked too, and every locked record's row is
|
|
553
|
+
# brought up to date with its stream
|
|
554
|
+
# (Lyra::Projections::AsyncProjectionJob.project_pending), then the rows
|
|
555
|
+
# are read again: a destroyed record's row is gone, and the upsert
|
|
556
|
+
# inserts it again, as Active Record does (a Created after the
|
|
557
|
+
# Destroyed); a record whose create was pending has its row, and is
|
|
558
|
+
# updated; one with an update pending is updated from the row its stream
|
|
559
|
+
# has, which the update's job no longer overwrites. Before, a destroy
|
|
560
|
+
# still to be projected refused the upsert (Lyra::RecordDestroyedError).
|
|
561
|
+
#
|
|
562
|
+
# In ES-Lazy the upsert is refused there (Lyra::RecordDestroyedError)
|
|
563
|
+
# before anything is written, as an update of a destroyed record is
|
|
564
|
+
# (BypassEvents.projects_pending? says why it is not projected there).
|
|
565
|
+
# Once the destroy is in the table (the next catch-up), the upsert
|
|
566
|
+
# inserts. The locked rows are then brought up to date with their
|
|
567
|
+
# streams and read again, so the events carry the values the stream has.
|
|
568
|
+
#
|
|
569
|
+
# The rows are read again once the locks are held, whatever was brought
|
|
570
|
+
# up to date: the first read takes no lock, and a projection job (or an
|
|
571
|
+
# ES-Lazy catch-up) that committed between it and the locks changed the
|
|
572
|
+
# rows it read (a destroyed record's row gone, a created one's there).
|
|
573
|
+
# From the stale read the upsert's row was recorded as an update of the
|
|
574
|
+
# destroyed record, or as a second create. Returns [rows read, the keys
|
|
575
|
+
# locked].
|
|
576
|
+
def lock_upserted_rows(before, rows, keys)
|
|
577
|
+
if Lyra::BypassEvents.projects_pending?
|
|
578
|
+
ids = before.keys | explicit_keys(rows)
|
|
579
|
+
return [before, []] if ids.empty?
|
|
580
|
+
|
|
581
|
+
Lyra::BypassEvents.lock_streams!(klass, ids, write: "upsert_all")
|
|
582
|
+
Lyra::Projections::AsyncProjectionJob.project_pending(klass, ids)
|
|
583
|
+
return [Lyra::BypassEvents.snapshot_by_keys(klass, rows, keys), ids]
|
|
584
|
+
end
|
|
585
|
+
return [before, []] if before.empty?
|
|
586
|
+
|
|
587
|
+
dead = Lyra::BypassEvents.destroyed_ids(klass, before.keys, write: "upsert_all")
|
|
588
|
+
if dead.any?
|
|
589
|
+
raise Lyra::RecordDestroyedError,
|
|
590
|
+
"#{klass.name} #{dead.join(', ')} #{dead.one? ? 'was' : 'were'} destroyed and #{dead.one? ? 'its row is' : 'their rows are'} " \
|
|
591
|
+
"not yet gone from the table (a projection still to run); upsert_all would update #{dead.one? ? 'it' : 'them'} " \
|
|
592
|
+
"and bring #{dead.one? ? 'it' : 'them'} back, so it is refused. Retry once the destroy is projected, " \
|
|
593
|
+
"or create the records again."
|
|
594
|
+
end
|
|
595
|
+
Lyra::Projections::LazyProjection.refresh_records(klass, before.keys)
|
|
596
|
+
[Lyra::BypassEvents.snapshot_by_keys(klass, rows, keys), before.keys]
|
|
597
|
+
end
|
|
598
|
+
|
|
254
599
|
def lyra_guarded?
|
|
255
600
|
klass.respond_to?(:lyra_monitored) && klass.lyra_monitored
|
|
256
601
|
end
|
|
257
602
|
|
|
603
|
+
# update_all(+updates+) in ES-Sync, ES-Async and ES-Lazy, once the rows
|
|
604
|
+
# it writes (+before+, the snapshot) and their aggregate locks are held
|
|
605
|
+
# (BypassEvents.locked_snapshot): the tuples the records will hold in the
|
|
606
|
+
# table's unique indexes (Lyra::Projections::UniqueIndexes::BulkUpdate)
|
|
607
|
+
# are locked, after the rows, as Monitor's UPDATE takes the rows and then
|
|
608
|
+
# the index entries, so a write through callbacks checking a value waits
|
|
609
|
+
# for this one. Where the table lags the event store (ES-Async, ES-Lazy)
|
|
610
|
+
# they are checked against each other and the event store before the
|
|
611
|
+
# write, as for update_columns (StrictDataAccess
|
|
612
|
+
# #lyra_projected_unique_columns); ES-Sync's UPDATE checks them.
|
|
613
|
+
def projected_unique_update(before, updates)
|
|
614
|
+
return if before.empty? || !Lyra.event_sourcing_mode?
|
|
615
|
+
|
|
616
|
+
unique = Lyra::Projections::UniqueIndexes.bulk_update(klass, updates)
|
|
617
|
+
unique&.check!(before.values, check: !Lyra::Projections::UniqueIndexes.in_table?(klass))
|
|
618
|
+
end
|
|
619
|
+
|
|
620
|
+
# insert_all, insert_all! (+skip+: the conflicts insert_all's ON
|
|
621
|
+
# CONFLICT DO NOTHING skips: :any, or its unique_by's columns; nil for
|
|
622
|
+
# insert_all!) and upsert_all (+upsert+: its conflict columns) in
|
|
623
|
+
# ES-Sync, ES-Async and ES-Lazy, after the keys the rows give are locked
|
|
624
|
+
# (+keys+): the values each row claims in the table's unique indexes
|
|
625
|
+
# (Lyra::Projections::UniqueIndexes; the primary key is the keys'), as
|
|
626
|
+
# the record would hold them (the column defaults and the relation's
|
|
627
|
+
# scope for what the row does not give), are locked, together counted
|
|
628
|
+
# against config.bulk_write_lock_limit with the keys, so a write through
|
|
629
|
+
# callbacks checking a value waits for the insert, as it waits for that
|
|
630
|
+
# write. ES-Sync's table is current, and its INSERT checks them.
|
|
631
|
+
#
|
|
632
|
+
# Where the table lags the event store (ES-Async: a record whose create
|
|
633
|
+
# or update waits for its job; ES-Lazy: one written after the catch-up
|
|
634
|
+
# the write began with), each value is then looked up in the event store
|
|
635
|
+
# (its value stream, under its lock): the table missed such a holder and
|
|
636
|
+
# inserted the duplicate, which the holder's projection then met. A row
|
|
637
|
+
# whose value a live record other than the row's own key holds is left
|
|
638
|
+
# out of insert_all (as ON CONFLICT DO NOTHING leaves it, on any index,
|
|
639
|
+
# or on its unique_by), and raises ActiveRecord::RecordNotUnique,
|
|
640
|
+
# before anything is written, for insert_all!, a conflict on an index
|
|
641
|
+
# other than insert_all's unique_by, and an upsert's conflict on an
|
|
642
|
+
# index other than its conflict columns (the record the row updates
|
|
643
|
+
# excepted). An upsert's conflict on its conflict columns updates the
|
|
644
|
+
# holder's row: in ES-Async that row is brought up to date first where
|
|
645
|
+
# it waits for its job (UniqueIndexes.bring_rows_up_to_date, which waits
|
|
646
|
+
# for no one), and where it is still not in the table (taken, or in
|
|
647
|
+
# ES-Lazy written after the catch-up) the upsert raises
|
|
648
|
+
# ActiveRecord::RecordNotUnique rather than insert a second record with
|
|
649
|
+
# the value. In ES-Async the rows of records that gave a value up and
|
|
650
|
+
# whose jobs have not run are brought up to date too, so the table does
|
|
651
|
+
# not skip or refuse a row for a value no record holds.
|
|
652
|
+
#
|
|
653
|
+
# Returns [the rows to write, the ids of the records whose rows were
|
|
654
|
+
# brought up to date].
|
|
655
|
+
def projected_unique_rows(attributes, write, skip: nil, upsert: nil, keys: [])
|
|
656
|
+
unique = Lyra::Projections::UniqueIndexes
|
|
657
|
+
return [attributes, []] if Lyra.config.projection_mode == :disabled || !unique.enforced?(klass)
|
|
658
|
+
|
|
659
|
+
indexes = unique.of(klass)
|
|
660
|
+
rows = attributes.is_a?(Hash) ? [attributes] : Array(attributes)
|
|
661
|
+
return [attributes, []] if indexes.empty? || rows.empty?
|
|
662
|
+
|
|
663
|
+
scope = scope_for_create.transform_keys(&:to_s)
|
|
664
|
+
defaults = klass.column_defaults
|
|
665
|
+
columns = indexes.flat_map(&:columns).uniq
|
|
666
|
+
claims = rows.map do |row|
|
|
667
|
+
values = defaults.merge(scope.merge(row.to_h.to_h { |k, v| [klass.attribute_aliases.fetch(k.to_s, k.to_s), v] }))
|
|
668
|
+
given = columns.to_h { |column| [column, klass.type_for_attribute(column).cast(values[column])] }
|
|
669
|
+
unique.claims(klass) { given[_1] }
|
|
670
|
+
end
|
|
671
|
+
all = claims.flatten
|
|
672
|
+
return [attributes, []] if all.empty?
|
|
673
|
+
|
|
674
|
+
Lyra::BypassEvents.check_lock_limit!(klass, all.map { [_1.index.columns.first, _1.values.first] }.uniq.size + keys.size, write)
|
|
675
|
+
unique.lock!(klass, all)
|
|
676
|
+
return [attributes, []] if unique.in_table?(klass)
|
|
677
|
+
|
|
678
|
+
pk = klass.primary_key
|
|
679
|
+
type = pk.is_a?(String) ? klass.type_for_attribute(pk) : nil
|
|
680
|
+
brought = []
|
|
681
|
+
seen = {}
|
|
682
|
+
holders = lambda do |claim|
|
|
683
|
+
seen[[claim.index.name, claim.keys]] ||= begin
|
|
684
|
+
held, gave_up = unique.scan(klass, claim)
|
|
685
|
+
brought.concat(unique.bring_rows_up_to_date(klass, gave_up))
|
|
686
|
+
held
|
|
687
|
+
end
|
|
688
|
+
end
|
|
689
|
+
target = upsert&.sort
|
|
690
|
+
kept = rows.each_index.select do |r|
|
|
691
|
+
own = type && explicit_key(rows[r], pk, type)&.to_s
|
|
692
|
+
if upsert
|
|
693
|
+
updated = target == [pk] ? own : nil
|
|
694
|
+
aim = claims[r].find { _1.index.columns.sort == target }
|
|
695
|
+
if aim && (held = holders.(aim) - [own].compact).any?
|
|
696
|
+
updated = held.first
|
|
697
|
+
unless upserted_row_present?(updated, brought)
|
|
698
|
+
raise unique.not_unique(aim)
|
|
699
|
+
end
|
|
700
|
+
end
|
|
701
|
+
refused = claims[r].find { |claim| claim != aim && (holders.(claim) - [own, updated].compact).any? }
|
|
702
|
+
raise unique.not_unique(refused) if refused
|
|
703
|
+
|
|
704
|
+
next true
|
|
705
|
+
end
|
|
706
|
+
taken = claims[r].select { |claim| (holders.(claim) - [own].compact).any? }
|
|
707
|
+
next true if taken.empty?
|
|
708
|
+
|
|
709
|
+
refused = taken.find { |claim| !(skip == :any || (skip && skip.sort == claim.index.columns.sort)) }
|
|
710
|
+
raise unique.not_unique(refused) if refused
|
|
711
|
+
|
|
712
|
+
false
|
|
713
|
+
end
|
|
714
|
+
return [attributes, brought] if kept.size == rows.size
|
|
715
|
+
|
|
716
|
+
[kept.map { rows[_1] }, brought]
|
|
717
|
+
end
|
|
718
|
+
|
|
719
|
+
# Whether the row of the live record +id+, which an upsert updates by its
|
|
720
|
+
# conflict columns, is in the table: in ES-Async brought up to date
|
|
721
|
+
# first, where its job has not run and nothing has to be waited for.
|
|
722
|
+
def upserted_row_present?(id, brought)
|
|
723
|
+
brought.concat(Lyra::Projections::UniqueIndexes.bring_rows_up_to_date(klass, [id]))
|
|
724
|
+
Lyra::PurposeBoundReads.internal(INTERNAL) do
|
|
725
|
+
klass.uncached { klass.unscoped.where(klass.primary_key => id).exists? }
|
|
726
|
+
end
|
|
727
|
+
end
|
|
728
|
+
|
|
729
|
+
# The rows of an insert_all or insert_all! in the events-only store
|
|
730
|
+
# (ES-NoProj), where the event store, not the table, says which records
|
|
731
|
+
# exist. +skip+: the conflicts ON CONFLICT DO NOTHING skips, as Monitor's
|
|
732
|
+
# does: :any (insert_all), the columns of its unique_by, or nil
|
|
733
|
+
# (insert_all!: every conflict raises ActiveRecord::RecordNotUnique).
|
|
734
|
+
#
|
|
735
|
+
# The model's other unique indexes first (events_only_unique_rows),
|
|
736
|
+
# then its primary key. ON CONFLICT DO NOTHING covers a primary key
|
|
737
|
+
# conflict with no unique_by or unique_by the primary key; otherwise
|
|
738
|
+
# (+unique+ below: insert_all!, or an insert_all whose unique_by is
|
|
739
|
+
# another index) a key taken raises ActiveRecord::RecordNotUnique, as
|
|
740
|
+
# Monitor's insert raises it. The records whose keys the rows give are
|
|
741
|
+
# locked (aggregate locks, under config.bulk_write_lock_limit) and read
|
|
742
|
+
# from the store. A row whose record it holds is left out of the insert,
|
|
743
|
+
# as Monitor's ON CONFLICT DO NOTHING leaves it (when +unique+,
|
|
744
|
+
# ActiveRecord::RecordNotUnique is raised, as Monitor's insert raises it,
|
|
745
|
+
# before anything is written). The table's row of any other key whose
|
|
746
|
+
# stream ends in a destroy is deleted (Lyra's own write), so the row is
|
|
747
|
+
# inserted and recorded as Monitor inserts it; one with no stream (a row
|
|
748
|
+
# Genesis skipped) is a record the table holds, and conflicts
|
|
749
|
+
# (events_only_dead_keys).
|
|
750
|
+
# Before, the table answered: a record the store held but the table
|
|
751
|
+
# did not (every record created through callbacks there) was inserted
|
|
752
|
+
# again, a second Created on its live stream, and its attributes
|
|
753
|
+
# overwritten; a key whose record had been inserted with insert_all and
|
|
754
|
+
# then destroyed met that record's row and was skipped, with no event,
|
|
755
|
+
# where Monitor creates the record again. Rows that give no key take
|
|
756
|
+
# the table's sequence, as before. Elsewhere, the rows as given.
|
|
757
|
+
def events_only_inserts(attributes, write, skip:)
|
|
758
|
+
return attributes unless Lyra.event_sourcing_mode? && Lyra.config.projection_mode == :disabled
|
|
759
|
+
|
|
760
|
+
rows = attributes.is_a?(Hash) ? [attributes] : Array(attributes)
|
|
761
|
+
# The keys' aggregate locks before the values' (events_only_unique_rows),
|
|
762
|
+
# as an update through callbacks takes them: the record, then the
|
|
763
|
+
# index entry, as Monitor's writes take the row, then the entry.
|
|
764
|
+
given = explicit_keys(rows)
|
|
765
|
+
lock_keys = -> { Lyra::BypassEvents.lock_streams!(klass, given, write: write) if given.any? }
|
|
766
|
+
rows = events_only_unique_rows(rows, write, skip, keys: given.size, before_locks: lock_keys)
|
|
767
|
+
pk = klass.primary_key
|
|
768
|
+
return rows unless pk.is_a?(String)
|
|
769
|
+
|
|
770
|
+
unique = !(skip == :any || skip == [pk])
|
|
771
|
+
type = klass.type_for_attribute(pk)
|
|
772
|
+
keyed = rows.map { |row| [row, explicit_key(row, pk, type)] }
|
|
773
|
+
ids = keyed.filter_map(&:last).uniq
|
|
774
|
+
return rows if ids.empty?
|
|
775
|
+
|
|
776
|
+
Lyra::BypassEvents.lock_streams!(klass, ids, write: write)
|
|
777
|
+
held = Lyra::PurposeBoundReads.internal(INTERNAL) { Lyra::Projections::EventStoreReader.find_many(klass, ids) }
|
|
778
|
+
taken = held.to_set { _1.read_attribute(pk) }
|
|
779
|
+
dead = events_only_dead_keys(ids.reject { taken.include?(_1) }, taken)
|
|
780
|
+
if unique && taken.any?
|
|
781
|
+
raise ActiveRecord::RecordNotUnique,
|
|
782
|
+
"Lyra: #{klass.name} #{pk} #{taken.first.inspect} is already taken (#{write})"
|
|
783
|
+
end
|
|
784
|
+
|
|
785
|
+
Lyra.projection_write(INTERNAL) { klass.unscoped.where(pk => dead).delete_all } if dead.any?
|
|
786
|
+
keyed.reject { |_, id| taken.include?(id) }.map(&:first)
|
|
787
|
+
end
|
|
788
|
+
|
|
789
|
+
# Of +ids+ (keys the event store does not hold), those whose table row
|
|
790
|
+
# (left by a destroyed record) is to be deleted for the insert: the
|
|
791
|
+
# streams that end in a destroy. A row with no stream (from before Lyra,
|
|
792
|
+
# which Genesis left without one: config.genesis_unrepresentable :skip)
|
|
793
|
+
# is a record Monitor's table holds, and its key is added to +taken+;
|
|
794
|
+
# it was deleted as a destroyed record's, and its data lost.
|
|
795
|
+
def events_only_dead_keys(ids, taken)
|
|
796
|
+
return [] if ids.empty?
|
|
797
|
+
|
|
798
|
+
pk = klass.primary_key
|
|
799
|
+
present = Lyra::PurposeBoundReads.internal(INTERNAL) { klass.unscoped.where(pk => ids).pluck(pk) }
|
|
800
|
+
return [] if present.empty?
|
|
801
|
+
|
|
802
|
+
streams = present.to_h { [Lyra.stream_name(klass, _1), _1] }
|
|
803
|
+
dead = Lyra::StreamTail.destroyed_streams(streams.keys).map { streams[_1] }
|
|
804
|
+
taken.merge(present - dead)
|
|
805
|
+
dead
|
|
806
|
+
end
|
|
807
|
+
|
|
808
|
+
# The model's unique indexes other than its primary key, checked from
|
|
809
|
+
# the event store as Monitor's table checks them (events_only_inserts).
|
|
810
|
+
# Before, ES-NoProj checked them against its table, which holds no row
|
|
811
|
+
# of a record created through callbacks: a duplicate (an email taken,
|
|
812
|
+
# with no key given) was inserted, a second record with the value, where
|
|
813
|
+
# Monitor's insert_all skips it and insert_all! raises; and the row a
|
|
814
|
+
# destroyed record left there was taken for a live one, skipping (or
|
|
815
|
+
# refusing) a value that is free.
|
|
816
|
+
#
|
|
817
|
+
# Each row's values for each index (the relation's scope and the column
|
|
818
|
+
# defaults filling what it does not give; a row with a NULL among them
|
|
819
|
+
# conflicts with nothing, unless the index is NULLS NOT DISTINCT) are
|
|
820
|
+
# locked as a uniqueness validation of the value locks them
|
|
821
|
+
# (Lyra::Projections::Uniqueness.lock_all!: a multi-column index by its
|
|
822
|
+
# first column's value, coarser, never finer), so a concurrent insert or
|
|
823
|
+
# validated create of the value waits. Those locks and the aggregate
|
|
824
|
+
# locks of the keys the rows give (+keys+) count together against
|
|
825
|
+
# config.bulk_write_lock_limit, PostgreSQL's lock table being what they
|
|
826
|
+
# share: a model with several unique indexes reaches the limit at fewer
|
|
827
|
+
# rows. The keys' locks (+before_locks+) are taken once the limit is
|
|
828
|
+
# checked and before the values', as an update takes its record's lock
|
|
829
|
+
# and then its values'. Then the records that may hold the values are read: those whose
|
|
830
|
+
# events are in the values' streams (Lyra::Projections::UniqueIndexes
|
|
831
|
+
# .candidate_records: one query for the streams, one for their events,
|
|
832
|
+
# one for the records, whatever the model's size), and the table's rows
|
|
833
|
+
# that hold the values (one query per index), each index's values
|
|
834
|
+
# compared as UniqueIndexes.key compares them (as the column holds
|
|
835
|
+
# them; a citext column's by the database's LOWER). Every record was
|
|
836
|
+
# read before (Uniqueness::Snapshot), at a cost that grew with the
|
|
837
|
+
# model: 52.8 s at 10,000 records, cold.
|
|
838
|
+
#
|
|
839
|
+
# A row conflicts with a value a record the store holds has, and with a
|
|
840
|
+
# value a table row holds whose record the store neither holds nor has
|
|
841
|
+
# destroyed (a row from before Lyra that Genesis left without a stream,
|
|
842
|
+
# config.genesis_unrepresentable :skip): skipped where +skip+ covers that
|
|
843
|
+
# index, else ActiveRecord::RecordNotUnique before anything is written.
|
|
844
|
+
# The table's rows whose streams end in a destroy are deleted (Lyra's own
|
|
845
|
+
# write), and the rows of live records that hold a value their records
|
|
846
|
+
# no longer have (written by an insert, updated since through callbacks,
|
|
847
|
+
# which write no row) are brought up to the values the store has, so
|
|
848
|
+
# the insert meets the table as Monitor's does. Rows of the same insert
|
|
849
|
+
# that conflict with each other meet in the table, as in Monitor. Partial
|
|
850
|
+
# and expression indexes are left to the table.
|
|
851
|
+
def events_only_unique_rows(rows, write, skip, keys: 0, before_locks: nil)
|
|
852
|
+
pk = klass.primary_key
|
|
853
|
+
unique = Lyra::Projections::UniqueIndexes
|
|
854
|
+
indexes = unique.of(klass)
|
|
855
|
+
return rows if indexes.empty? || rows.empty?
|
|
856
|
+
|
|
857
|
+
scope = scope_for_create.transform_keys(&:to_s)
|
|
858
|
+
defaults = klass.column_defaults
|
|
859
|
+
columns = indexes.flat_map(&:columns).uniq
|
|
860
|
+
# Each row's values, as the record would hold them.
|
|
861
|
+
given = rows.map do |row|
|
|
862
|
+
values = defaults.merge(scope.merge(row.to_h.to_h { |k, v| [klass.attribute_aliases.fetch(k.to_s, k.to_s), v] }))
|
|
863
|
+
columns.to_h { |column| [column, klass.type_for_attribute(column).cast(values[column])] }
|
|
864
|
+
end
|
|
865
|
+
key_tuple = lambda do |index, values|
|
|
866
|
+
compared = index.columns.map { |column| unique.key(klass, column, values[column]) }
|
|
867
|
+
compared.any?(&:nil?) && !index.nulls_not_distinct ? nil : compared
|
|
868
|
+
end
|
|
869
|
+
tuples = given.map { |values| indexes.map { |index| key_tuple.(index, values) } }
|
|
870
|
+
|
|
871
|
+
claims = given.each_index.flat_map do |r|
|
|
872
|
+
indexes.each_index.filter_map { |i| tuples[r][i] && [indexes[i].columns.first, given[r][indexes[i].columns.first]] }
|
|
873
|
+
end.uniq
|
|
874
|
+
return rows if claims.empty?
|
|
875
|
+
|
|
876
|
+
Lyra::BypassEvents.check_lock_limit!(klass, claims.size + keys, write)
|
|
877
|
+
Lyra::Projections::ReferenceLinks.catch_up(klass) if Lyra::Projections::ReferenceLinks.enabled?
|
|
878
|
+
before_locks&.call
|
|
879
|
+
Lyra::Projections::Uniqueness.lock_all!(klass, claims)
|
|
880
|
+
table = table_rows_holding(indexes, tuples, given, key_tuple)
|
|
881
|
+
wanted = tuples.flat_map { |row| row.each_with_index.filter_map { |tuple, i| tuple && [indexes[i], tuple] } }.uniq
|
|
882
|
+
records = Lyra::PurposeBoundReads.internal(INTERNAL) do
|
|
883
|
+
Lyra::Genesis.first_use(klass)
|
|
884
|
+
unique.candidate_records(klass, wanted, ids: table.map { _2 })
|
|
885
|
+
end
|
|
886
|
+
reader = Lyra::Projections::TypedValues::EventReader.new(klass.base_class)
|
|
887
|
+
entry_tuple = ->(index, attrs) { key_tuple.(index, index.columns.to_h { [_1, reader.read(attrs, _1)] }) }
|
|
888
|
+
live = records.to_h { [_1[pk], _1] }
|
|
889
|
+
held = indexes.map { |index| records.filter_map { |attrs| entry_tuple.(index, attrs) }.to_set }
|
|
890
|
+
dead, outdated = sort_table_rows(indexes, table, live, held, entry_tuple)
|
|
891
|
+
|
|
892
|
+
kept = rows.each_index.reject do |r|
|
|
893
|
+
taken = indexes.each_index.select { |i| tuples[r][i] && held[i].include?(tuples[r][i]) }
|
|
894
|
+
next false if taken.empty?
|
|
895
|
+
|
|
896
|
+
refused = taken.find { |i| !(skip == :any || (skip && skip.sort == indexes[i].columns.sort)) }
|
|
897
|
+
if refused
|
|
898
|
+
index = indexes[refused]
|
|
899
|
+
raise unique.not_unique(unique::Claim.new(index, tuples[r][refused], index.columns.map { given[r][_1] }))
|
|
900
|
+
end
|
|
901
|
+
true
|
|
902
|
+
end
|
|
903
|
+
Lyra.projection_write(INTERNAL) do
|
|
904
|
+
klass.unscoped.where(pk => dead).delete_all if dead.any?
|
|
905
|
+
outdated.each { |id| klass.unscoped.where(pk => id).update_all(columns.to_h { [_1, live[id][_1]] }) }
|
|
906
|
+
end
|
|
907
|
+
kept.map { rows[_1] }
|
|
908
|
+
end
|
|
909
|
+
|
|
910
|
+
# The table's rows that hold any of +tuples+' values (comparison keys;
|
|
911
|
+
# +given+: the rows' values), per index, as [index position, primary
|
|
912
|
+
# key, tuple], one query per index.
|
|
913
|
+
def table_rows_holding(indexes, tuples, given, key_tuple)
|
|
914
|
+
pk = klass.primary_key
|
|
915
|
+
indexes.each_with_index.flat_map do |index, i|
|
|
916
|
+
wanted = tuples.filter_map { _1[i] }.to_set
|
|
917
|
+
next [] if wanted.empty?
|
|
918
|
+
|
|
919
|
+
first = index.columns.first
|
|
920
|
+
values = given.each_index.filter_map { |r| given[r][first] if tuples[r][i] }.uniq
|
|
921
|
+
found = Lyra::PurposeBoundReads.internal(INTERNAL) do
|
|
922
|
+
klass.unscoped.where(first => values).pluck(pk, *index.columns)
|
|
923
|
+
end
|
|
924
|
+
found.filter_map do |id, *held|
|
|
925
|
+
tuple = key_tuple.(index, index.columns.zip(held).to_h)
|
|
926
|
+
[i, id, tuple] if wanted.include?(tuple)
|
|
927
|
+
end
|
|
928
|
+
end
|
|
929
|
+
end
|
|
930
|
+
|
|
931
|
+
# Sort the table's rows that hold a value being inserted (+table+): a
|
|
932
|
+
# live record's row holding a value its record no longer has is
|
|
933
|
+
# +outdated+; a row whose record the store does not hold is +dead+ when
|
|
934
|
+
# its stream ends in a destroy, and otherwise (no stream: a row Genesis
|
|
935
|
+
# skipped) its value is taken, added to +held+. Returns [dead, outdated].
|
|
936
|
+
def sort_table_rows(indexes, table, live, held, entry_tuple)
|
|
937
|
+
outdated = table.filter_map { |i, id, tuple| id if live[id] && entry_tuple.(indexes[i], live[id]) != tuple }.uniq
|
|
938
|
+
unknown = table.reject { |_, id, _| live[id] }
|
|
939
|
+
return [[], outdated] if unknown.empty?
|
|
940
|
+
|
|
941
|
+
streams = unknown.map { |_, id, _| id }.uniq.to_h { [Lyra.stream_name(klass, _1), _1] }
|
|
942
|
+
dead = Lyra::StreamTail.destroyed_streams(streams.keys).map { streams[_1] }.to_set
|
|
943
|
+
unknown.each { |i, id, tuple| held[i] << tuple unless dead.include?(id) }
|
|
944
|
+
[dead.to_a, outdated]
|
|
945
|
+
end
|
|
946
|
+
|
|
947
|
+
# ES-Async (Lyra::BypassEvents.projects_pending?): the records whose
|
|
948
|
+
# keys the rows of an insert_all or insert_all! give are locked
|
|
949
|
+
# (aggregate locks, within config.bulk_write_lock_limit) and their rows
|
|
950
|
+
# brought up to date with their streams
|
|
951
|
+
# (Lyra::Projections::AsyncProjectionJob.project_pending), before the
|
|
952
|
+
# insert. Its ON CONFLICT then meets the rows Monitor's would: a record
|
|
953
|
+
# whose destroy was waiting for its job has no row, and is inserted
|
|
954
|
+
# again, a Created after the Destroyed, with the values given and the
|
|
955
|
+
# column defaults; a record whose create was waiting has its row, which
|
|
956
|
+
# insert_all skips and insert_all! raises ActiveRecord::RecordNotUnique
|
|
957
|
+
# on. Before, the first was skipped (no event; the destroy's job then
|
|
958
|
+
# deleted its row), the second inserted with a second Created on its
|
|
959
|
+
# stream. Rows that give no key take the table's sequence. A conflict on
|
|
960
|
+
# another unique index is looked up in the event store
|
|
961
|
+
# (projected_unique_rows), as a record whose create waits for its job
|
|
962
|
+
# holds its value with no row.
|
|
963
|
+
#
|
|
964
|
+
# Returns the keys locked.
|
|
965
|
+
def project_pending_inserts(attributes, write)
|
|
966
|
+
return [] unless Lyra::BypassEvents.projects_pending?
|
|
967
|
+
|
|
968
|
+
ids = explicit_keys(attributes)
|
|
969
|
+
return [] if ids.empty?
|
|
970
|
+
|
|
971
|
+
Lyra::BypassEvents.lock_streams!(klass, ids, write: write)
|
|
972
|
+
Lyra::Projections::AsyncProjectionJob.project_pending(klass, ids)
|
|
973
|
+
ids
|
|
974
|
+
end
|
|
975
|
+
|
|
976
|
+
# ES-Async (Lyra::BypassEvents.projects_pending?): the write wrote the
|
|
977
|
+
# rows and their events (+published+: publish_upserts) in one
|
|
978
|
+
# transaction, so a row whose stream was up to date before it is up to
|
|
979
|
+
# date with its new event, and its projection checkpoint is moved there,
|
|
980
|
+
# in one statement (AsyncProjectionJob.advance): the records it locked
|
|
981
|
+
# and brought up to date (+locked+), and every record it created, whose
|
|
982
|
+
# row it inserted with the very attributes of its Created. Others (a
|
|
983
|
+
# record an upsert matched by another unique index, not locked) keep
|
|
984
|
+
# their checkpoints. Before, a bypass write never moved a checkpoint:
|
|
985
|
+
# the next insert or upsert of the same keys found every stream behind
|
|
986
|
+
# and replayed the previous write's events onto rows that already had
|
|
987
|
+
# them, as each record's next projection job did.
|
|
988
|
+
def advance_checkpoints(published, locked)
|
|
989
|
+
return unless Lyra::BypassEvents.projects_pending? && published.present?
|
|
990
|
+
|
|
991
|
+
locked = locked.to_set
|
|
992
|
+
moves = published.filter_map { |id, (operation, event_id)| [id, event_id] if operation == :created || locked.include?(id) }
|
|
993
|
+
Lyra::Projections::AsyncProjectionJob.advance(klass, moves)
|
|
994
|
+
end
|
|
995
|
+
|
|
996
|
+
# insert_all, insert_all! and upsert_all of a model whose primary key
|
|
997
|
+
# has several columns (one its table declares, enrolled before Active
|
|
998
|
+
# Record read it; Lyra::ModelSelection refuses one it knows) are refused
|
|
999
|
+
# in the event-sourcing modes, before anything is written: their
|
|
1000
|
+
# Created events name the record by an array no projection, catch-up or
|
|
1001
|
+
# rebuild can apply (ES-Lazy's catch-up then failed on every read and
|
|
1002
|
+
# write of every monitored model, and a rebuild failed), and an update
|
|
1003
|
+
# of such a record failed in Genesis. Monitor and Hijack, which replay
|
|
1004
|
+
# nothing, write them and record their events.
|
|
1005
|
+
def refuse_composite_key!(write)
|
|
1006
|
+
key = klass.primary_key
|
|
1007
|
+
return unless key.is_a?(Array) && Lyra.event_sourcing_mode?
|
|
1008
|
+
|
|
1009
|
+
raise ArgumentError,
|
|
1010
|
+
"#{klass.name} has a composite primary key (#{key.join(', ')}), which Lyra does not support: in the " \
|
|
1011
|
+
"event-sourcing modes #{write} would record events that no projection or rebuild can apply, so it is " \
|
|
1012
|
+
"refused before anything is written. A monitored model needs a single-column primary key."
|
|
1013
|
+
end
|
|
1014
|
+
|
|
1015
|
+
# The primary keys the rows of an insert give, cast, each once.
|
|
1016
|
+
def explicit_keys(attributes)
|
|
1017
|
+
pk = klass.primary_key
|
|
1018
|
+
return [] unless pk.is_a?(String)
|
|
1019
|
+
|
|
1020
|
+
type = klass.type_for_attribute(pk)
|
|
1021
|
+
rows = attributes.is_a?(Hash) ? [attributes] : Array(attributes)
|
|
1022
|
+
rows.filter_map { explicit_key(_1, pk, type) }.uniq
|
|
1023
|
+
end
|
|
1024
|
+
|
|
1025
|
+
# The primary key a row of an insert gives, cast, or nil.
|
|
1026
|
+
def explicit_key(row, pk, type)
|
|
1027
|
+
row = row.to_h.transform_keys(&:to_s)
|
|
1028
|
+
name = row.key?(pk) ? pk : klass.attribute_aliases.key(pk)
|
|
1029
|
+
name && row.key?(name) ? type.cast(row[name]) : nil
|
|
1030
|
+
end
|
|
1031
|
+
|
|
258
1032
|
# Run the write with the primary key in RETURNING, so the inserted and
|
|
259
1033
|
# updated rows can be identified, and hand the caller the result they
|
|
260
|
-
# asked for.
|
|
261
|
-
#
|
|
1034
|
+
# asked for (Lyra::BypassEvents.with_returning: raw SQL for returning
|
|
1035
|
+
# gets the key under an alias of Lyra's own; the caller's result keeps
|
|
1036
|
+
# its column types). Returns [caller_result, ids]; ids nil where no key
|
|
1037
|
+
# can be returned (an adapter without RETURNING), as before. A primary
|
|
1038
|
+
# key of several columns (a model whose table declares one, enrolled
|
|
1039
|
+
# before Active Record read it: Lyra::ModelSelection refuses one it
|
|
1040
|
+
# knows) has each column returned, and each id is the array of their
|
|
1041
|
+
# values, as record.id gives it. Its key was asked for as one quoted
|
|
1042
|
+
# name, and an explicit returning: raised PG::UndefinedColumn; without
|
|
1043
|
+
# one, no key was found in the result and nothing was recorded.
|
|
262
1044
|
def with_primary_keys(returning)
|
|
263
1045
|
pk = klass.primary_key
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
[caller_result, ids]
|
|
1046
|
+
return [yield(returning), nil] if pk.blank?
|
|
1047
|
+
|
|
1048
|
+
keys = Array(pk)
|
|
1049
|
+
result, rows = Lyra::BypassEvents.with_returning(klass, returning, keys) { |ret| yield ret }
|
|
1050
|
+
[result, rows&.map { |row| pk.is_a?(Array) ? row.values_at(*keys) : row[pk] }]
|
|
1051
|
+
end
|
|
1052
|
+
|
|
1053
|
+
# The application's insert_all, insert_all! or upsert_all (+write+) on a
|
|
1054
|
+
# join table: the join model of a has_and_belongs_to_many, or a model it
|
|
1055
|
+
# declared on the table (+join_model+: HabtmLinks.join_model_of). Refused
|
|
1056
|
+
# in strict mode, unless inside Lyra.without_strict_access, where either
|
|
1057
|
+
# side's model is monitored (HabtmLinks.guarded?), as its delete_all of
|
|
1058
|
+
# join rows is; Rails' own links are inserted one by one, through the
|
|
1059
|
+
# join model's callbacks (HabtmLinks.create), and never come here.
|
|
1060
|
+
# Where it runs, the rows it wrote are recorded as the links they make
|
|
1061
|
+
# (HabtmLinks.insert_all). Before, it was neither refused nor recorded.
|
|
1062
|
+
def join_insert(write, join_model, attributes, returning, unique_by: nil, &block)
|
|
1063
|
+
check_strict_mode!(write) if Lyra::Associations::HabtmLinks.guarded?(join_model)
|
|
1064
|
+
conflict = write == :upsert_all ? conflict_columns(unique_by) : nil
|
|
1065
|
+
Lyra::Associations::HabtmLinks.insert_all(self, attributes, returning: returning, conflict: conflict, &block)
|
|
285
1066
|
end
|
|
286
1067
|
|
|
287
1068
|
# The columns upsert_all's ON CONFLICT uses: unique_by as columns, or as
|
|
288
1069
|
# an index name, or the primary key by default.
|
|
289
1070
|
def conflict_columns(unique_by)
|
|
290
|
-
return
|
|
1071
|
+
return Array(klass.primary_key) if unique_by.nil?
|
|
291
1072
|
|
|
292
1073
|
columns = Array(unique_by).map(&:to_s)
|
|
293
1074
|
return columns unless columns.size == 1 && !klass.column_names.include?(columns.first)
|
|
@@ -303,15 +1084,50 @@ module Lyra
|
|
|
303
1084
|
raise StrictDataAccessViolation.new(method_name, klass)
|
|
304
1085
|
end
|
|
305
1086
|
|
|
1087
|
+
# The bulk write, on exactly the rows the locked snapshot holds: no row
|
|
1088
|
+
# another transaction inserted since is written without its event. Runs as
|
|
1089
|
+
# Lyra's own write, so it is not intercepted a second time. The snapshot
|
|
1090
|
+
# already took the relation's limit and offset: the write by key keeps
|
|
1091
|
+
# neither (an offset applied again to the snapshot's rows wrote none).
|
|
1092
|
+
def write_snapshotted(before)
|
|
1093
|
+
return 0 if before.empty?
|
|
1094
|
+
|
|
1095
|
+
Lyra.projection_write(INTERNAL) { yield unscope(:limit, :offset).where(klass.primary_key => before.keys) }
|
|
1096
|
+
end
|
|
1097
|
+
|
|
306
1098
|
def safely_publish_bulk(source)
|
|
307
1099
|
yield
|
|
308
1100
|
rescue => e
|
|
309
1101
|
Lyra::BypassEvents.publish_failed!(e, source)
|
|
310
1102
|
end
|
|
311
1103
|
|
|
312
|
-
#
|
|
313
|
-
#
|
|
1104
|
+
# The shape of the write a has_many's nullify makes: one foreign key
|
|
1105
|
+
# column set to NULL. Rails' association code marks its own
|
|
1106
|
+
# (Lyra::Associations::Maintenance); the shape alone does not make a
|
|
1107
|
+
# write a nullify.
|
|
314
1108
|
def association_nullify_update?(updates)
|
|
1109
|
+
StrictDataAccessRelation.nullify_update?(updates)
|
|
1110
|
+
end
|
|
1111
|
+
|
|
1112
|
+
# In the events-only store, with events recorded: whether this is an
|
|
1113
|
+
# Active Record relation's update_all clearing a foreign key, which the
|
|
1114
|
+
# table cannot answer (it is empty or stale there).
|
|
1115
|
+
def events_only_nullify?(updates)
|
|
1116
|
+
Lyra.event_sourcing_mode? && Lyra.config.projection_mode == :disabled && association_nullify_update?(updates)
|
|
1117
|
+
end
|
|
1118
|
+
|
|
1119
|
+
# The records the event store holds that this relation selects, all of
|
|
1120
|
+
# it (CachedRelation#narrowed_by: its conditions on the model's columns,
|
|
1121
|
+
# order and limit; anything else raises UnsupportedQuery, naming +what+).
|
|
1122
|
+
def events_only_relation(what)
|
|
1123
|
+
Lyra::PurposeBoundReads.internal(INTERNAL) do
|
|
1124
|
+
Lyra::Projections::EventStoreReader.relation(klass).narrowed_by(self, what)
|
|
1125
|
+
end
|
|
1126
|
+
end
|
|
1127
|
+
|
|
1128
|
+
# The same test for the events-only store, where the association writes
|
|
1129
|
+
# through the event-backed relation (AssociationInterceptor).
|
|
1130
|
+
def self.nullify_update?(updates)
|
|
315
1131
|
return false unless updates.is_a?(Hash)
|
|
316
1132
|
return false unless updates.size == 1
|
|
317
1133
|
|
|
@@ -323,80 +1139,74 @@ module Lyra
|
|
|
323
1139
|
# Publish "updated" events for each record affected by dependent: :nullify
|
|
324
1140
|
# This captures the foreign key nullification in the event stream
|
|
325
1141
|
def publish_nullify_events(updates)
|
|
326
|
-
|
|
1142
|
+
updates = updates.to_h { |key, value| [klass.attribute_aliases[key.to_s] || key.to_s, value] }
|
|
1143
|
+
columns = updates.keys
|
|
327
1144
|
|
|
328
1145
|
# Get the records being updated with their current values
|
|
329
1146
|
# In ES Disabled mode, records don't exist in DB, so we need to query
|
|
330
1147
|
# the event store via the model's read path (which uses CachedRelation)
|
|
331
|
-
|
|
1148
|
+
# Lyra's own read, for the events: the write itself is checked
|
|
1149
|
+
# (Lyra::PurposeBoundReads::RelationCheck).
|
|
1150
|
+
records_data = Lyra::PurposeBoundReads.internal(INTERNAL) { fetch_affected_records_for_nullify(columns) }
|
|
1151
|
+
|
|
1152
|
+
# Where events are required, a row Genesis skipped refuses the whole
|
|
1153
|
+
# destroy before any event (Lyra::Genesis.admit_all!). In Monitor each
|
|
1154
|
+
# row's failure is logged on its own: one skipped child does not cost
|
|
1155
|
+
# the others their events.
|
|
1156
|
+
Lyra::Genesis.admit_all!(klass, records_data.map(&:first)) if Lyra::BypassEvents.required?
|
|
1157
|
+
# Where the store can hold a destroy whose row is not gone
|
|
1158
|
+
# (Lyra::BypassEvents.checks_streams?), a destroyed child gets no event:
|
|
1159
|
+
# it would bring the child back. Its key is returned, for the write to
|
|
1160
|
+
# leave its row as well.
|
|
1161
|
+
skipped = []
|
|
1162
|
+
if Lyra::BypassEvents.checks_streams? && records_data.any?
|
|
1163
|
+
skipped = Lyra::BypassEvents.destroyed_ids(klass, records_data.map(&:first), write: "dependent: :nullify")
|
|
1164
|
+
records_data = records_data.reject { |id, _| skipped.include?(id) } if skipped.any?
|
|
1165
|
+
end
|
|
1166
|
+
# The columns it changes: the foreign key, and for a polymorphic
|
|
1167
|
+
# association its type, whatever they are called.
|
|
1168
|
+
records_data.each do |id, old_values|
|
|
1169
|
+
changes = columns.zip(old_values).each_with_object({}) do |(column, old_value), acc|
|
|
1170
|
+
acc[column] = [old_value, updates[column]] unless old_value == updates[column]
|
|
1171
|
+
end
|
|
1172
|
+
next if changes.empty?
|
|
332
1173
|
|
|
333
|
-
records_data.each do |id, old_value|
|
|
334
1174
|
Lyra::BypassEvents.publish(klass, id, :updated,
|
|
335
|
-
attributes:
|
|
336
|
-
changes:
|
|
1175
|
+
attributes: changes.transform_values(&:last),
|
|
1176
|
+
changes: changes,
|
|
337
1177
|
source: "dependent_association")
|
|
1178
|
+
rescue => e
|
|
1179
|
+
Lyra::BypassEvents.publish_failed!(e, "dependent_association")
|
|
338
1180
|
end
|
|
1181
|
+
skipped
|
|
339
1182
|
rescue => e
|
|
340
1183
|
# Fails the parent's destroy in Hijack and the event-sourcing modes
|
|
341
1184
|
# (it runs inside its transaction); logged in Monitor.
|
|
342
1185
|
Lyra::BypassEvents.publish_failed!(e, "dependent_association")
|
|
1186
|
+
[]
|
|
343
1187
|
end
|
|
344
1188
|
|
|
345
|
-
#
|
|
346
|
-
#
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
else
|
|
363
|
-
[]
|
|
364
|
-
end
|
|
365
|
-
end
|
|
366
|
-
|
|
367
|
-
# Extract the foreign key value from the relation's where clause
|
|
368
|
-
# The relation is something like: Registration.where(group_id: 123)
|
|
369
|
-
def extract_foreign_key_from_where_clause(column)
|
|
370
|
-
return nil unless respond_to?(:where_clause)
|
|
371
|
-
|
|
372
|
-
where_clause.send(:predicates).each do |predicate|
|
|
373
|
-
next unless predicate.is_a?(Arel::Nodes::Equality)
|
|
374
|
-
next unless predicate.left.respond_to?(:name)
|
|
375
|
-
next unless predicate.left.name.to_s == column
|
|
376
|
-
|
|
377
|
-
# Extract the value - Rails 7/8 uses QueryAttribute
|
|
378
|
-
right = predicate.right
|
|
379
|
-
case right
|
|
380
|
-
when Arel::Nodes::Casted
|
|
381
|
-
return right.value
|
|
382
|
-
when Arel::Nodes::BindParam
|
|
383
|
-
return right.value.value_before_type_cast
|
|
384
|
-
when Integer, String
|
|
385
|
-
return right
|
|
386
|
-
else
|
|
387
|
-
# Rails 7/8: ActiveRecord::Relation::QueryAttribute
|
|
388
|
-
if right.respond_to?(:value)
|
|
389
|
-
return right.value
|
|
390
|
-
elsif right.respond_to?(:value_before_type_cast)
|
|
391
|
-
return right.value_before_type_cast
|
|
392
|
-
end
|
|
393
|
-
end
|
|
1189
|
+
# The records the nullify rewrites, as [id, [old values of +columns+]]: the table's rows
|
|
1190
|
+
# the relation selects, or, in the events-only store (ES-NoProj), where
|
|
1191
|
+
# the table is empty or stale, the records the event store holds that
|
|
1192
|
+
# the relation's where clause selects, all of it (CachedRelation's
|
|
1193
|
+
# narrowed_by: its conditions on the model's columns, order and limit;
|
|
1194
|
+
# anything else it holds is refused with UnsupportedQuery, which fails
|
|
1195
|
+
# the write). A parent's dependent: :nullify is written by
|
|
1196
|
+
# AssociationInterceptor there; this is reached when Rails' association
|
|
1197
|
+
# code writes a table relation (Lyra's own projection writes aside). It
|
|
1198
|
+
# used to read the foreign key's equality alone, skipping any other
|
|
1199
|
+
# condition (where(author_id: 1, status: "published") nullified the
|
|
1200
|
+
# author's drafts too in the log, and matched no row in the table), and
|
|
1201
|
+
# to read the table first.
|
|
1202
|
+
def fetch_affected_records_for_nullify(columns)
|
|
1203
|
+
pk = klass.primary_key
|
|
1204
|
+
unless Lyra.event_sourcing_mode? && Lyra.config.projection_mode == :disabled
|
|
1205
|
+
return pluck(pk, *columns).map { |id, *values| [id, values] }
|
|
394
1206
|
end
|
|
395
1207
|
|
|
396
|
-
nil
|
|
397
|
-
|
|
398
|
-
Rails.logger.debug("Lyra: Could not extract foreign key from where clause - #{e.message}")
|
|
399
|
-
nil
|
|
1208
|
+
selected = Lyra::Projections::EventStoreReader.relation(klass).narrowed_by(self, "update_all(#{columns.join(', ')}: nil)")
|
|
1209
|
+
selected.map { |record| [record.read_attribute(pk), columns.map { record.read_attribute(_1) }] }
|
|
400
1210
|
end
|
|
401
1211
|
end
|
|
402
1212
|
|