orfeas_lyra 0.9.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +2021 -17
  3. data/README.md +22 -9
  4. data/Rakefile +40 -1
  5. data/app/controllers/lyra/application_controller.rb +127 -0
  6. data/app/controllers/lyra/dashboard_controller.rb +33 -31
  7. data/app/controllers/lyra/flow_controller.rb +32 -34
  8. data/app/controllers/lyra/privacy_controller.rb +57 -18
  9. data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
  10. data/app/views/lyra/dashboard/index.html.erb +1 -0
  11. data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
  12. data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
  13. data/app/views/lyra/privacy/policy.html.erb +7 -0
  14. data/app/views/lyra/privacy/rights.html.erb +78 -0
  15. data/config/privacy_policies.rb +0 -3
  16. data/config/routes.rb +1 -0
  17. data/lib/lyra/access_log.rb +187 -6
  18. data/lib/lyra/advisory_lock.rb +236 -0
  19. data/lib/lyra/aggregate.rb +1 -1
  20. data/lib/lyra/associations/habtm_links.rb +382 -19
  21. data/lib/lyra/associations/maintenance.rb +229 -0
  22. data/lib/lyra/attribution.rb +48 -0
  23. data/lib/lyra/bypass_events.rb +446 -46
  24. data/lib/lyra/command_handler.rb +19 -14
  25. data/lib/lyra/configuration.rb +237 -10
  26. data/lib/lyra/consistency/read_your_writes.rb +4 -2
  27. data/lib/lyra/correlation.rb +121 -0
  28. data/lib/lyra/domain_events.rb +4 -0
  29. data/lib/lyra/dual_view.rb +53 -16
  30. data/lib/lyra/encrypted_attributes.rb +5 -0
  31. data/lib/lyra/engine.rb +67 -0
  32. data/lib/lyra/erasure.rb +1011 -106
  33. data/lib/lyra/event.rb +35 -1
  34. data/lib/lyra/event_data.rb +38 -0
  35. data/lib/lyra/event_flow.rb +31 -15
  36. data/lib/lyra/event_store_adapter.rb +28 -0
  37. data/lib/lyra/event_window.rb +58 -0
  38. data/lib/lyra/genesis.rb +548 -44
  39. data/lib/lyra/interceptors/association_interceptor.rb +689 -34
  40. data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
  41. data/lib/lyra/internal.rb +26 -0
  42. data/lib/lyra/mode_sync.rb +51 -3
  43. data/lib/lyra/mode_transition.rb +257 -30
  44. data/lib/lyra/model_selection.rb +35 -0
  45. data/lib/lyra/own_connection.rb +17 -2
  46. data/lib/lyra/privacy/adapters/pam.rb +25 -2
  47. data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
  48. data/lib/lyra/privacy/interface.rb +27 -7
  49. data/lib/lyra/privacy/subject_links.rb +143 -0
  50. data/lib/lyra/privacy/subject_records.rb +369 -0
  51. data/lib/lyra/projection.rb +7 -1
  52. data/lib/lyra/projections/async_checkpoint.rb +32 -0
  53. data/lib/lyra/projections/async_projection_job.rb +166 -23
  54. data/lib/lyra/projections/cached_joins.rb +33 -18
  55. data/lib/lyra/projections/cached_projection.rb +74 -33
  56. data/lib/lyra/projections/cached_relation.rb +3175 -321
  57. data/lib/lyra/projections/default_scopes.rb +254 -0
  58. data/lib/lyra/projections/deferred_relation.rb +23 -3
  59. data/lib/lyra/projections/event_store_reader.rb +47 -8
  60. data/lib/lyra/projections/foreign_keys.rb +92 -0
  61. data/lib/lyra/projections/lazy_projection.rb +165 -27
  62. data/lib/lyra/projections/model_projection.rb +165 -14
  63. data/lib/lyra/projections/rebuild.rb +34 -6
  64. data/lib/lyra/projections/reference_links.rb +463 -72
  65. data/lib/lyra/projections/typed_values.rb +629 -0
  66. data/lib/lyra/projections/unique_indexes.rb +595 -0
  67. data/lib/lyra/projections/uniqueness.rb +784 -0
  68. data/lib/lyra/projections/value_digest.rb +93 -0
  69. data/lib/lyra/purpose_bound_reads.rb +912 -77
  70. data/lib/lyra/repair.rb +119 -28
  71. data/lib/lyra/retention.rb +114 -8
  72. data/lib/lyra/rights/consent_store.rb +415 -0
  73. data/lib/lyra/rights/erasure_guard.rb +135 -0
  74. data/lib/lyra/rights/pending.rb +74 -0
  75. data/lib/lyra/rights/queries.rb +269 -0
  76. data/lib/lyra/rights/store.rb +512 -0
  77. data/lib/lyra/rights/subject_keys.rb +224 -0
  78. data/lib/lyra/rights.rb +281 -0
  79. data/lib/lyra/saves_without_effect.rb +69 -0
  80. data/lib/lyra/stream_tail.rb +319 -0
  81. data/lib/lyra/strict_data_access.rb +916 -106
  82. data/lib/lyra/temporal.rb +4 -2
  83. data/lib/lyra/touch_registry.rb +404 -0
  84. data/lib/lyra/verification/bypass_workflow.rb +452 -42
  85. data/lib/lyra/verification/rights_workflows.rb +170 -0
  86. data/lib/lyra/version.rb +1 -1
  87. data/lib/lyra/visualization/event_graph.rb +1 -1
  88. data/lib/lyra/visualization/timeline.rb +2 -2
  89. data/lib/lyra.rb +43 -2
  90. data/lib/tasks/lyra_mode.rake +110 -8
  91. data/lib/tasks/lyra_projections.rake +43 -2
  92. metadata +34 -4
  93. data/lib/lyra/associations/event_aware.rb +0 -224
@@ -17,6 +17,16 @@ module Lyra
17
17
  # store's own stream table; the event is not copied, no table is added, and
18
18
  # the index is append-only, rebuildable from the events.
19
19
  #
20
+ # An event that sets a column of one of the model's unique indexes
21
+ # (UniqueIndexes) is linked the same way into a value stream,
22
+ # "Model!column$<digest>", named by a digest of the value as the index
23
+ # compares it, never by the value itself (an email, an IBAN): the
24
+ # unique-index check of ES-NoProj's, ES-Async's and ES-Lazy's writes
25
+ # reads it (record_ids_in). The checkpoint names the columns whose value
26
+ # streams it covers; one stored
27
+ # before a column was indexed (or before 1.0.0) is caught up from the
28
+ # first event.
29
+ #
20
30
  # A lookup by that key reads the reference stream for candidates, rebuilds
21
31
  # each with CachedProjection.find and keeps the records that still match:
22
32
  # a child moved to another parent stays linked under the old one (streams
@@ -25,7 +35,11 @@ module Lyra
25
35
  # the candidates include every match and the answer is exact.
26
36
  #
27
37
  # Completeness. Links are written while config.reference_links says so
28
- # (:auto, the default: in ES-NoProj only, so no other mode pays for them).
38
+ # (:auto, the default: in ES-NoProj only, so no other mode pays for them;
39
+ # the value streams also in ES-Async and ES-Lazy, whose unique-index
40
+ # check reads them, values_enabled?: there a model without a unique
41
+ # index links nothing, and the foreign keys' checkpoint is left where
42
+ # it was).
29
43
  # The switch into ES-NoProj (Lyra::ModeTransition.to!) links every event
30
44
  # of every monitored model first, so the index is complete when the mode
31
45
  # starts. As a safety net, events written while linking was off are also
@@ -51,6 +65,21 @@ module Lyra
51
65
  end
52
66
  end
53
67
 
68
+ # Whether events are linked into their value streams as they are
69
+ # stored (UniqueIndexes: the columns of a model's unique indexes):
70
+ # wherever all links are (enabled?), and in ES-Async and ES-Lazy, whose
71
+ # unique-index check reads the value streams too (their tables lag
72
+ # the event store), while config.reference_links is not false. Only
73
+ # the value streams there: the foreign-key links serve ES-NoProj's
74
+ # reads, which those modes do not make, so a model without a unique
75
+ # index links nothing. ES-Sync checks its table.
76
+ def values_enabled?
77
+ return true if enabled?
78
+
79
+ !(Lyra.config.reference_links == false || Lyra.config.reference_links.nil?) &&
80
+ Lyra.event_sourcing_mode? && %i[async lazy].include?(Lyra.config.projection_mode)
81
+ end
82
+
54
83
  # Named after the class that owns the table (the STI base class), as
55
84
  # the record streams are (Lyra.stream_name).
56
85
  def stream_name(model_class, foreign_key, value)
@@ -59,22 +88,42 @@ module Lyra
59
88
 
60
89
  # Called by Lyra.append_events after +events+ are stored in
61
90
  # +stream_name+: link those that set a foreign key, or, with linking
62
- # off, note that the model's links fell behind in this process.
91
+ # off, note that the model's links fell behind in this process. In
92
+ # ES-Async and ES-Lazy (values_enabled? only) the events that set a
93
+ # column of a unique index are linked into its value streams, and the
94
+ # foreign keys are noted as behind.
63
95
  def after_append(events, stream_name, store)
64
96
  linking = enabled?
97
+ values = !linking && values_enabled?
65
98
  # With linking off, only a model this process took for linked needs
66
99
  # noting; otherwise nothing is done, so other modes pay nothing.
67
- return if !linking && MUTEX.synchronize { ready.empty? }
100
+ return if !linking && !values && MUTEX.synchronize { ready.empty? }
68
101
 
69
102
  model_class = model_for_stream(stream_name)
70
103
  return unless model_class
71
104
 
105
+ events = Array(events)
72
106
  if linking
73
- link_events(Array(events), model_class, store)
74
- else
75
- if Array(events).any? { |event| reference_values(model_class, event).any? }
76
- MUTEX.synchronize { ready.delete(model_class.name) }
77
- end
107
+ # Columns new to this process (UniqueIndexes.facts): linked from the
108
+ # first event, before this write's own links.
109
+ catch_up(model_class) if unlinked?(model_class)
110
+ link_events(events, model_class, store)
111
+ return
112
+ end
113
+ if values && !UniqueIndexes.columns(model_class).empty?
114
+ catch_up(model_class) if unlinked?(model_class)
115
+ link_events(events, model_class, store, keys: false)
116
+ end
117
+ if values
118
+ # Only a model this process took for linked by its foreign keys
119
+ # (in ES-NoProj, earlier) needs noting.
120
+ return unless MUTEX.synchronize { ready.any? { _1.end_with?("|k") } }
121
+
122
+ forget_ready(model_class, keys_only: true) if events.any? { |event| reference_values(model_class, event).any? }
123
+ elsif MUTEX.synchronize { ready.empty? }
124
+ nil
125
+ elsif events.any? { |event| link_streams(model_class, event).any? }
126
+ forget_ready(model_class)
78
127
  end
79
128
  end
80
129
 
@@ -93,7 +142,10 @@ module Lyra
93
142
  indexed = indexed_conditions(table, conditions)
94
143
  return nil if indexed.empty?
95
144
 
96
- catch_up(table)
145
+ # Without the links caught up (another thread is linking), every
146
+ # record is considered.
147
+ return nil unless catch_up(table)
148
+
97
149
  # Every condition on an indexed key narrows the candidates: a record
98
150
  # that matches them all is in each key's stream, so the streams are
99
151
  # intersected (stock_location_id and variant_id together name one
@@ -115,30 +167,174 @@ module Lyra
115
167
  # none free) it runs in the caller's transaction, and the model counts
116
168
  # as linked only once that transaction commits (Lyra::OwnConnection).
117
169
  #
118
- # One thread per model and process catches up; the others wait for it
119
- # without taking a connection, so a burst of first lookups needs one
120
- # connection beyond the callers' own, not one each.
121
- def catch_up(model_class, force: false)
170
+ # One thread per model and process catches up. A caller never waits
171
+ # for it, nor for the catch-up's advisory lock, nor for a connection
172
+ # (unless +wait+: an explicit catch-up, ModeTransition's, the rake
173
+ # task's): when another thread or process is catching up, or the pool
174
+ # has no connection free and the lock is not to be had in the
175
+ # caller's transaction, it returns nil and the caller answers without
176
+ # the links (a lookup from every record). A caller that waited held
177
+ # its transaction, and its locks, while it did: with every connection
178
+ # of the pool taken by writers of a model's first use, the writer
179
+ # that caught up inline kept the advisory lock until it committed,
180
+ # and called catch_up again in the same write, waiting on the Ruby
181
+ # mutex of a thread whose connection waited on that lock; PostgreSQL
182
+ # could not see the cycle, and every writer hung. A thread that
183
+ # caught up inline takes the model for linked until its transaction
184
+ # ends; other threads, once it commits.
185
+ #
186
+ # Returns the number of events linked, 0 when there was nothing to
187
+ # do, nil when it did not catch up.
188
+ def catch_up(model_class, force: false, wait: force)
122
189
  model_class = owner(model_class)
123
- return 0 if !force && ready?(model_class)
190
+ return 0 if !force && (ready?(model_class) || caught_up_here?(model_class))
124
191
 
125
- lock_for(model_class).synchronize do
126
- return 0 if !force && ready?(model_class)
192
+ mutex = lock_for(model_class)
193
+ if wait
194
+ mutex.lock
195
+ elsif !mutex.try_lock
196
+ return nil
197
+ end
198
+ begin
199
+ return 0 if !force && (ready?(model_class) || caught_up_here?(model_class))
127
200
 
201
+ key = ready_key(model_class, keys: force || enabled?)
202
+ unlinked = take_unlinked(model_class)
128
203
  inline = false
129
- linked = Lyra::OwnConnection.run do |in_caller_transaction|
130
- inline = in_caller_transaction
131
- link_and_checkpoint(model_class, force)
204
+ begin
205
+ linked = Lyra::OwnConnection.run(wait_for_connection: wait) do |in_caller_transaction|
206
+ inline = in_caller_transaction
207
+ link_and_checkpoint(model_class, force, unlinked, wait: wait)
208
+ end
209
+ rescue StandardError
210
+ unlinked!(model_class, unlinked) if unlinked.any?
211
+ raise
212
+ end
213
+ if linked.nil?
214
+ unlinked!(model_class, unlinked) if unlinked.any?
215
+ return nil
216
+ end
217
+ mark_ready = -> { MUTEX.synchronize { ready << key } }
218
+ if inline
219
+ caught_up_here!(model_class, key)
220
+ Lyra::OwnConnection.after_commit(&mark_ready)
221
+ else
222
+ mark_ready.call
132
223
  end
133
- mark_ready = -> { MUTEX.synchronize { ready << model_class.name } }
134
- inline ? Lyra::OwnConnection.after_commit(&mark_ready) : mark_ready.call
135
224
  linked
225
+ ensure
226
+ mutex.unlock
227
+ end
228
+ end
229
+
230
+ # The columns of +model_class+'s unique indexes whose value streams may
231
+ # miss events (new to this process: UniqueIndexes.facts): its next
232
+ # catch-up links them from the first event.
233
+ def unlinked!(model_class, columns)
234
+ name = owner(model_class).name
235
+ MUTEX.synchronize do
236
+ @unlinked ||= {}
237
+ @unlinked[name] = (@unlinked[name] || []) | columns.map(&:to_s)
238
+ end
239
+ end
240
+
241
+ # Link every value stream of +model_class+ again, from the first event:
242
+ # after a rolling deploy in which processes running a Lyra without value
243
+ # streams, or reading the indexes before a migration added one, wrote
244
+ # (bin/rails lyra:link_values).
245
+ def relink_values!(model_class)
246
+ table = owner(model_class)
247
+ unlinked!(table, UniqueIndexes.columns(table))
248
+ catch_up(table, wait: true)
249
+ end
250
+
251
+ # The ids of the records whose events are linked in every one of
252
+ # +streams+ could be (a superset): those of the stream with the
253
+ # fewest links, whose events are read (each names its record:
254
+ # model_id); none when one of them has no link. One query when no
255
+ # event set the value (two for several streams), one more for the
256
+ # events otherwise. For the unique-index check (UniqueIndexes): its
257
+ # value streams.
258
+ def record_ids_in(model_class, streams)
259
+ stream = streams.one? ? streams.first : fewest_links(streams)
260
+ return [] unless stream
261
+
262
+ # Never from the query cache: read under the value's lock, which may
263
+ # have waited for a writer that linked the value since.
264
+ event_ids = ActiveRecord::Base.with_connection do |connection|
265
+ connection.uncached do
266
+ connection.select_values("SELECT event_id FROM event_store_events_in_streams WHERE stream = #{connection.quote(stream)}")
267
+ end
268
+ end
269
+ return [] if event_ids.empty?
270
+
271
+ events = Lyra.config.event_store.read.events(event_ids.uniq).to_a
272
+ ids = events.map { |event| event.data[:model_id] || event.data["model_id"] }
273
+ return candidate_streams(owner(model_class), stream) if ids.any?(&:nil?) || ids.size < event_ids.uniq.size
274
+
275
+ ids.map(&:to_s).uniq
276
+ end
277
+
278
+ # The ids of the records whose events are linked in any of +streams+:
279
+ # one query per 1,000 streams, one per 500 events read (each names its
280
+ # record). For a bulk insert's unique-index check (UniqueIndexes
281
+ # .candidate_records).
282
+ def record_ids_in_any(model_class, streams)
283
+ rows = ActiveRecord::Base.with_connection do |connection|
284
+ streams.each_slice(1_000).flat_map do |slice|
285
+ connection.select_rows(
286
+ "SELECT stream, event_id FROM event_store_events_in_streams WHERE stream IN (#{slice.map { connection.quote(_1) }.join(', ')})"
287
+ )
288
+ end
289
+ end
290
+ return [] if rows.empty?
291
+
292
+ ids = []
293
+ unnamed = []
294
+ rows.map(&:last).uniq.each_slice(500) do |event_ids|
295
+ events = Lyra.config.event_store.read.events(event_ids).to_a
296
+ events.each do |event|
297
+ id = event.data[:model_id] || event.data["model_id"]
298
+ id.nil? ? unnamed << event.event_id : ids << id.to_s
299
+ end
300
+ unnamed.concat(event_ids - events.map(&:event_id))
301
+ end
302
+ if unnamed.any?
303
+ rows.each { |stream, event_id| ids.concat(candidate_streams(owner(model_class), stream)) if unnamed.include?(event_id) }
304
+ end
305
+ ids.uniq
306
+ end
307
+
308
+ # After +events+ were rewritten in place (Lyra::Erasure), their value
309
+ # stream links follow their new data: the links of the values they
310
+ # held are deleted (no digest of an erased value is kept) and those
311
+ # of the values they hold now inserted, where links are written or
312
+ # the events had some (the rewritten events are behind the
313
+ # checkpoint, and no catch-up would link them again).
314
+ def relink_values(events)
315
+ events.group_by { |event| value_owner(event) }.each do |table, rewritten|
316
+ next unless table && !UniqueIndexes.columns(table).empty?
317
+
318
+ ActiveRecord::Base.with_connection do |connection|
319
+ pattern = connection.quote("#{ActiveRecord::Base.sanitize_sql_like(Lyra.stream_owner_name(table))}!%")
320
+ ids = rewritten.map { connection.quote(_1.event_id) }.join(", ")
321
+ deleted = connection.exec_delete(
322
+ "DELETE FROM event_store_events_in_streams WHERE event_id IN (#{ids}) AND stream LIKE #{pattern}", "Lyra value links"
323
+ )
324
+ next unless deleted.positive? || values_enabled?
325
+
326
+ pairs = rewritten.flat_map { |event| UniqueIndexes.streams_for(table, set_values(event)).map { [_1, event.event_id] } }
327
+ insert_links(connection, pairs) if pairs.any?
328
+ end
136
329
  end
137
330
  end
138
331
 
139
332
  # Forget which models are linked in this process (tests, truncation).
140
333
  def reset!
141
- MUTEX.synchronize { @ready = Set.new }
334
+ MUTEX.synchronize do
335
+ @ready = Set.new
336
+ @unlinked = {}
337
+ end
142
338
  end
143
339
 
144
340
  # Foreign keys of +model_class+ the index covers.
@@ -156,8 +352,53 @@ module Lyra
156
352
  model_class.respond_to?(:base_class) ? model_class.base_class : model_class
157
353
  end
158
354
 
355
+ # Linked, in this process, as the model's foreign keys and unique-index
356
+ # columns are now, under the digest key in use: a column added to an
357
+ # index (or a key rotated) makes it not ready again.
358
+ # Where only the value streams are written (values_enabled? without
359
+ # enabled?: ES-Async, ES-Lazy), a model linked with its foreign keys
360
+ # is ready too; the reverse does not hold.
159
361
  def ready?(model_class)
160
- MUTEX.synchronize { ready.include?(model_class.name) }
362
+ key = ready_key(model_class)
363
+ also = key.end_with?("|v") ? key.sub(/\|v\z/, "|k") : nil
364
+ MUTEX.synchronize { (ready.include?(key) || (also && ready.include?(also))) && !(@unlinked || {}).key?(model_class.name) }
365
+ end
366
+
367
+ # This thread caught the model up inline, in the transaction it is in
368
+ # (caught_up_here!): until that transaction ends.
369
+ def caught_up_here?(model_class)
370
+ (Thread.current[:lyra_links_caught_up] || {}).key?(ready_key(model_class))
371
+ end
372
+
373
+ def caught_up_here!(model_class, key)
374
+ here = (Thread.current[:lyra_links_caught_up] ||= {})
375
+ here[key] = true
376
+ transaction = model_class.current_transaction
377
+ forget = -> { here.delete(key) }
378
+ transaction.after_commit(&forget)
379
+ transaction.after_rollback(&forget)
380
+ end
381
+
382
+ # Ends in "|k" where the foreign keys are linked too (enabled?), "|v"
383
+ # where only the value streams are.
384
+ def ready_key(model_class, keys: enabled?)
385
+ columns = UniqueIndexes.columns(model_class)
386
+ kind = keys ? "k" : "v"
387
+ columns.empty? ? "#{model_class.name}||#{kind}" : "#{model_class.name}|#{columns.join(',')}|#{ValueDigest.key_id}|#{kind}"
388
+ end
389
+
390
+ # +keys_only+: the foreign keys fell behind, the value streams did not.
391
+ def forget_ready(model_class, keys_only: false)
392
+ prefix = "#{owner(model_class).name}|"
393
+ MUTEX.synchronize { ready.delete_if { _1.start_with?(prefix) && (!keys_only || _1.end_with?("|k")) } }
394
+ end
395
+
396
+ def unlinked?(model_class)
397
+ MUTEX.synchronize { (@unlinked || {}).key?(owner(model_class).name) }
398
+ end
399
+
400
+ def take_unlinked(model_class)
401
+ MUTEX.synchronize { (@unlinked || {}).delete(model_class.name) || [] }
161
402
  end
162
403
 
163
404
  # The per-model lock of #catch_up.
@@ -165,21 +406,110 @@ module Lyra
165
406
  MUTEX.synchronize { (@locks ||= {})[model_class.name] ||= Mutex.new }
166
407
  end
167
408
 
168
- def link_and_checkpoint(model_class, force)
409
+ # The foreign keys from the checkpoint's position; each unique-index
410
+ # column from its own (columns linked under another digest key, or new to
411
+ # this process, from the first event, their old links deleted), so a
412
+ # column added to an index does not make the foreign keys be read again.
413
+ def link_and_checkpoint(model_class, force, unlinked = [], wait: true)
169
414
  ActiveRecord::Base.with_connection do |connection|
170
415
  ActiveRecord::Base.transaction do
171
- Lyra::AdvisoryLock.xact_lock(connection, "#{LOCK}/#{model_class.name}", purpose: "reference links")
172
- position = force ? 0 : checkpoint(model_class)
416
+ lock = "#{LOCK}/#{model_class.name}"
417
+ if wait
418
+ Lyra::AdvisoryLock.xact_lock(connection, lock, purpose: "reference links")
419
+ elsif !Lyra::AdvisoryLock.try_xact_lock(connection, lock, purpose: "reference links")
420
+ next nil
421
+ end
422
+ stored = checkpoint_state(model_class)
423
+ position = force ? 0 : stored[:position]
173
424
  upto = connection.select_value("SELECT MAX(id) FROM event_store_events").to_i
174
- next 0 unless upto > position
175
-
176
- linked = link_since(model_class, position, upto)
177
- store_checkpoint(model_class, upto)
425
+ next 0 if upto.zero?
426
+
427
+ kid = ValueDigest.key_id
428
+ columns = UniqueIndexes.columns(model_class)
429
+ relinked = columns.select do |column|
430
+ entry = stored[:values][column]
431
+ # No entry: linked, if at all, under a key no checkpoint names.
432
+ force || unlinked.include?(column) || entry.nil? || entry["key"] != kid
433
+ end
434
+ from = columns.to_h do |column|
435
+ entry = stored[:values][column]
436
+ [column, relinked.include?(column) || entry.nil? ? 0 : entry["position"].to_i]
437
+ end
438
+ drop_value_links(connection, model_class, relinked) if relinked.any?
439
+ behind = from.select { |_, at| upto > at }
440
+ current = stored[:values].keys.sort == columns.sort
441
+ # The foreign keys where they are linked (enabled?) or in a
442
+ # forced pass (the switch into ES-NoProj); in ES-Async and
443
+ # ES-Lazy the value streams alone, and the keys' position is
444
+ # kept.
445
+ keys = force || enabled?
446
+ next 0 unless (keys && upto > position) || behind.any? || relinked.any? || !current
447
+
448
+ # One read of the events behind, for both kinds of link; none
449
+ # for the foreign keys of a model that has none.
450
+ keys_behind = keys && upto > position && foreign_keys(model_class).any? ? position : nil
451
+ linked = keys_behind || behind.any? ? link_since(connection, model_class, keys_behind, behind, upto) : 0
452
+ store_checkpoint(model_class, keys ? [upto, position].max : stored[:position],
453
+ columns.to_h { [_1, { "position" => upto, "key" => kid }] })
178
454
  linked
179
455
  end
180
456
  end
181
457
  end
182
458
 
459
+ # Delete +columns+' value links (a key rotated, a relink).
460
+ def drop_value_links(connection, model_class, columns)
461
+ columns.each do |column|
462
+ pattern = connection.quote("#{ActiveRecord::Base.sanitize_sql_like("#{Lyra.stream_owner_name(model_class)}!#{column}$")}%")
463
+ connection.exec_delete("DELETE FROM event_store_events_in_streams WHERE stream LIKE #{pattern}", "Lyra value links")
464
+ end
465
+ end
466
+
467
+ # Link the model's events up to +upto+: those after +keys_from+ (nil:
468
+ # none) into their foreign-key streams (each stream read, the links
469
+ # already there left out), those after each column's position
470
+ # (+behind+: { column => position }) into the column's value streams
471
+ # (500 links per statement, those already there left as they are).
472
+ # The events are read once, 500 at a time.
473
+ def link_since(connection, model_class, keys_from, behind, upto)
474
+ from = [keys_from, *behind.values].compact.min
475
+ prefix = connection.quote("#{ActiveRecord::Base.sanitize_sql_like(Lyra.stream_prefix(model_class))}%")
476
+ rows = connection.select_rows(<<~SQL.squish)
477
+ SELECT e.id, e.event_id FROM event_store_events e
478
+ JOIN event_store_events_in_streams s ON s.event_id = e.event_id
479
+ WHERE s.stream LIKE #{prefix} AND e.id > #{Integer(from)} AND e.id <= #{Integer(upto)}
480
+ ORDER BY e.id
481
+ SQL
482
+ return 0 if rows.empty?
483
+
484
+ store = Lyra.config.event_store
485
+ position = rows.to_h { |id, event_id| [event_id, id.to_i] }
486
+ wanted = Hash.new { |h, k| h[k] = [] }
487
+ pairs = []
488
+ position.keys.each_slice(500) do |ids|
489
+ store.read.events(ids).each do |event|
490
+ at = position[event.event_id]
491
+ if keys_from && at > keys_from
492
+ reference_values(model_class, event).each { |key, value| wanted[stream_name(model_class, key, value)] << event.event_id }
493
+ end
494
+ columns = behind.select { |_, column_at| at > column_at }.keys
495
+ next if columns.empty?
496
+
497
+ UniqueIndexes.streams_for(model_class, set_values(event).slice(*columns)).each { pairs << [_1, event.event_id] }
498
+ end
499
+ end
500
+
501
+ linked = pairs.each_slice(500).sum { |slice| insert_links(connection, slice) }
502
+ wanted.each do |stream, ids|
503
+ present = store.read.stream(stream).to_a.map(&:event_id).to_set
504
+ missing = ids.uniq.reject { present.include?(_1) }
505
+ next if missing.empty?
506
+
507
+ store.link(missing, stream_name: stream)
508
+ linked += missing.size
509
+ end
510
+ linked
511
+ end
512
+
183
513
  # [[key, value], ...] for the conditions that name one value of an
184
514
  # indexed foreign key.
185
515
  def indexed_conditions(model_class, conditions)
@@ -188,7 +518,13 @@ module Lyra
188
518
  next unless keys.include?(key.to_s)
189
519
 
190
520
  value = value.id if value.is_a?(ActiveRecord::Base)
191
- [key.to_s, value] if value.is_a?(Integer) || value.is_a?(String)
521
+ next unless value.is_a?(Integer) || value.is_a?(String)
522
+
523
+ # As the key's type casts it ("5" and 5 name one stream); a value
524
+ # the type binds as NULL ("abc" on an integer key) names none, and
525
+ # is left to the conditions.
526
+ bound = TypedValues.query_value(model_class.type_for_attribute(key.to_s), value)
527
+ [key.to_s, bound] if bound.is_a?(Integer) || bound.is_a?(String)
192
528
  end
193
529
  end
194
530
 
@@ -196,19 +532,68 @@ module Lyra
196
532
  # records whose own streams hold the events linked in the reference
197
533
  # stream. One query on the stream table; the events are not read.
198
534
  def candidate_ids(model_class, key, value)
535
+ candidate_streams(model_class, stream_name(model_class, key, value))
536
+ end
537
+
538
+ # Ids of the records whose own streams hold the events linked in
539
+ # +stream+.
540
+ def candidate_streams(model_class, stream)
199
541
  ActiveRecord::Base.with_connection do |connection|
200
542
  own = Lyra.stream_prefix(model_class)
201
- connection.select_values(<<~SQL.squish).map { |stream| stream.delete_prefix(own) }
543
+ connection.select_values(<<~SQL.squish).map { |found| found.delete_prefix(own) }
202
544
  SELECT DISTINCT s.stream FROM event_store_events_in_streams l
203
545
  JOIN event_store_events_in_streams s
204
546
  ON s.event_id = l.event_id AND s.stream LIKE #{connection.quote("#{ActiveRecord::Base.sanitize_sql_like(own)}%")}
205
- WHERE l.stream = #{connection.quote(stream_name(model_class, key, value))}
547
+ WHERE l.stream = #{connection.quote(stream)}
206
548
  SQL
207
549
  end
208
550
  end
209
551
 
552
+ # Of +streams+, the one with the fewest links; nil when one has
553
+ # none. One query (the stream table's index on the stream).
554
+ def fewest_links(streams)
555
+ counts = ActiveRecord::Base.with_connection do |connection|
556
+ connection.uncached do
557
+ connection.select_rows(<<~SQL.squish).to_h
558
+ SELECT stream, COUNT(*) FROM event_store_events_in_streams
559
+ WHERE stream IN (#{streams.map { connection.quote(_1) }.join(', ')}) GROUP BY stream
560
+ SQL
561
+ end
562
+ end
563
+ return nil if counts.size < streams.uniq.size
564
+
565
+ counts.min_by { |_, count| count.to_i }.first
566
+ end
567
+
568
+ # The streams +event+ is linked into: one per foreign key it sets
569
+ # (stream_name), and one per value of a unique index's column it sets
570
+ # (UniqueIndexes.streams_for).
571
+ def link_streams(model_class, event)
572
+ references = reference_values(model_class, event).map { |key, value| stream_name(model_class, key, value) }
573
+ return references if UniqueIndexes.columns(model_class).empty?
574
+
575
+ references + UniqueIndexes.streams_for(model_class, set_values(event))
576
+ end
577
+
210
578
  # { foreign key => value } that +event+ sets, for the model's keys.
211
579
  def reference_values(model_class, event)
580
+ set_values(event).slice(*foreign_keys(model_class)).compact
581
+ end
582
+
583
+ # The monitored table (its stream owner) whose record an event of a
584
+ # write names (its data's model_class), or nil.
585
+ def value_owner(event)
586
+ data = event.data
587
+ name = data.is_a?(Hash) ? (data[:model_class] || data["model_class"]) : nil
588
+ model = name.to_s.safe_constantize
589
+ return nil unless model.is_a?(Class) && model < ActiveRecord::Base && model.respond_to?(:lyra_monitored)
590
+
591
+ owner(model)
592
+ end
593
+
594
+ # { attribute => value } that +event+ sets: a create's or an
595
+ # import's attributes, an update's new values.
596
+ def set_values(event)
212
597
  data = event.data
213
598
  values =
214
599
  case Lyra::Event.operation_of(event)
@@ -221,60 +606,66 @@ module Lyra
221
606
  else
222
607
  {}
223
608
  end
224
- values.slice(*foreign_keys(model_class)).compact
609
+ values
225
610
  end
226
611
 
227
- def link_events(events, model_class, store)
612
+ def link_events(events, model_class, store, keys: true)
613
+ values = !UniqueIndexes.columns(model_class).empty?
228
614
  events.each do |event|
229
- reference_values(model_class, event).each do |key, value|
230
- store.link([event.event_id], stream_name: stream_name(model_class, key, value))
615
+ if keys
616
+ reference_values(model_class, event).each do |key, value|
617
+ store.link([event.event_id], stream_name: stream_name(model_class, key, value))
618
+ end
231
619
  end
620
+ link_values(event, UniqueIndexes.streams_for(model_class, set_values(event))) if values
232
621
  end
233
622
  end
234
623
 
235
- def link_since(model_class, position, upto)
236
- ActiveRecord::Base.with_connection do |connection|
237
- prefix = connection.quote("#{ActiveRecord::Base.sanitize_sql_like(Lyra.stream_prefix(model_class))}%")
238
- event_ids = connection.select_values(<<~SQL.squish)
239
- SELECT e.event_id FROM event_store_events e
240
- JOIN event_store_events_in_streams s ON s.event_id = e.event_id
241
- WHERE s.stream LIKE #{prefix} AND e.id > #{Integer(position)} AND e.id <= #{Integer(upto)}
242
- ORDER BY e.id
243
- SQL
244
- return 0 if event_ids.empty?
245
-
246
- store = Lyra.config.event_store
247
- wanted = Hash.new { |h, k| h[k] = [] }
248
- event_ids.each_slice(500) do |ids|
249
- store.read.events(ids).each do |event|
250
- reference_values(model_class, event).each do |key, value|
251
- wanted[stream_name(model_class, key, value)] << event.event_id
252
- end
253
- end
254
- end
624
+ # An event just stored, linked into its value streams in one
625
+ # statement: the rows RES's link inserts, without the existence
626
+ # check and the savepoint it takes for each stream (three round
627
+ # trips per stream on every create of a model with a unique index).
628
+ # The stream table is the one every ES-NoProj read queries.
629
+ def link_values(event, streams)
630
+ return if streams.empty?
255
631
 
256
- linked = 0
257
- wanted.each do |stream, ids|
258
- present = store.read.stream(stream).to_a.map(&:event_id).to_set
259
- missing = ids.uniq.reject { present.include?(_1) }
260
- next if missing.empty?
632
+ ActiveRecord::Base.with_connection { |connection| insert_links(connection, streams.map { [_1, event.event_id] }) }
633
+ end
261
634
 
262
- store.link(missing, stream_name: stream)
263
- linked += missing.size
264
- end
265
- linked
266
- end
635
+ # Insert [stream, event id] links, those already there left as they
636
+ # are; returns how many were inserted.
637
+ def insert_links(connection, pairs)
638
+ now = connection.quote(Time.now.utc)
639
+ rows = pairs.map { |stream, event_id| "(#{connection.quote(stream)}, NULL, #{connection.quote(event_id)}, #{now})" }
640
+ connection.exec_update(
641
+ "INSERT INTO event_store_events_in_streams (stream, position, event_id, created_at) " \
642
+ "VALUES #{rows.join(', ')} ON CONFLICT DO NOTHING", "Lyra value links"
643
+ )
267
644
  end
268
645
 
269
- def checkpoint(model_class)
646
+
647
+ # The position up to which the model's foreign keys are linked.
648
+ def checkpoint(model_class) = checkpoint_state(model_class)[:position]
649
+
650
+ # { position: the foreign keys' position, values: { column =>
651
+ # { "position" =>, "key" => the digest key's id } } }, from the model's
652
+ # last checkpoint (values empty for one stored before 1.0.0's value
653
+ # streams).
654
+ def checkpoint_state(model_class)
270
655
  last = Lyra.config.event_store.read.stream("#{CHECKPOINT_PREFIX}#{model_class.name}").last
271
- last ? (last.data[:position] || last.data["position"]).to_i : 0
656
+ return { position: 0, values: {} } unless last
657
+
658
+ data = last.data
659
+ values = (data[:values] || data["values"] || {}).to_h do |column, entry|
660
+ [column.to_s, entry.to_h.transform_keys(&:to_s)]
661
+ end
662
+ { position: (data[:position] || data["position"]).to_i, values: values }
272
663
  rescue RubyEventStore::EventNotFoundInStream
273
- 0
664
+ { position: 0, values: {} }
274
665
  end
275
666
 
276
- def store_checkpoint(model_class, position)
277
- event = Lyra::Events::ReferenceLinksIndexed.new(data: { model_class: model_class.name, position: position })
667
+ def store_checkpoint(model_class, position, values)
668
+ event = Lyra::Events::ReferenceLinksIndexed.new(data: { model_class: model_class.name, position: position, values: values })
278
669
  Lyra.config.event_store.publish(event, stream_name: "#{CHECKPOINT_PREFIX}#{model_class.name}")
279
670
  end
280
671