orfeas_lyra 0.9.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +2021 -17
  3. data/README.md +22 -9
  4. data/Rakefile +40 -1
  5. data/app/controllers/lyra/application_controller.rb +127 -0
  6. data/app/controllers/lyra/dashboard_controller.rb +33 -31
  7. data/app/controllers/lyra/flow_controller.rb +32 -34
  8. data/app/controllers/lyra/privacy_controller.rb +57 -18
  9. data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
  10. data/app/views/lyra/dashboard/index.html.erb +1 -0
  11. data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
  12. data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
  13. data/app/views/lyra/privacy/policy.html.erb +7 -0
  14. data/app/views/lyra/privacy/rights.html.erb +78 -0
  15. data/config/privacy_policies.rb +0 -3
  16. data/config/routes.rb +1 -0
  17. data/lib/lyra/access_log.rb +187 -6
  18. data/lib/lyra/advisory_lock.rb +236 -0
  19. data/lib/lyra/aggregate.rb +1 -1
  20. data/lib/lyra/associations/habtm_links.rb +382 -19
  21. data/lib/lyra/associations/maintenance.rb +229 -0
  22. data/lib/lyra/attribution.rb +48 -0
  23. data/lib/lyra/bypass_events.rb +446 -46
  24. data/lib/lyra/command_handler.rb +19 -14
  25. data/lib/lyra/configuration.rb +237 -10
  26. data/lib/lyra/consistency/read_your_writes.rb +4 -2
  27. data/lib/lyra/correlation.rb +121 -0
  28. data/lib/lyra/domain_events.rb +4 -0
  29. data/lib/lyra/dual_view.rb +53 -16
  30. data/lib/lyra/encrypted_attributes.rb +5 -0
  31. data/lib/lyra/engine.rb +67 -0
  32. data/lib/lyra/erasure.rb +1011 -106
  33. data/lib/lyra/event.rb +35 -1
  34. data/lib/lyra/event_data.rb +38 -0
  35. data/lib/lyra/event_flow.rb +31 -15
  36. data/lib/lyra/event_store_adapter.rb +28 -0
  37. data/lib/lyra/event_window.rb +58 -0
  38. data/lib/lyra/genesis.rb +548 -44
  39. data/lib/lyra/interceptors/association_interceptor.rb +689 -34
  40. data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
  41. data/lib/lyra/internal.rb +26 -0
  42. data/lib/lyra/mode_sync.rb +51 -3
  43. data/lib/lyra/mode_transition.rb +257 -30
  44. data/lib/lyra/model_selection.rb +35 -0
  45. data/lib/lyra/own_connection.rb +17 -2
  46. data/lib/lyra/privacy/adapters/pam.rb +25 -2
  47. data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
  48. data/lib/lyra/privacy/interface.rb +27 -7
  49. data/lib/lyra/privacy/subject_links.rb +143 -0
  50. data/lib/lyra/privacy/subject_records.rb +369 -0
  51. data/lib/lyra/projection.rb +7 -1
  52. data/lib/lyra/projections/async_checkpoint.rb +32 -0
  53. data/lib/lyra/projections/async_projection_job.rb +166 -23
  54. data/lib/lyra/projections/cached_joins.rb +33 -18
  55. data/lib/lyra/projections/cached_projection.rb +74 -33
  56. data/lib/lyra/projections/cached_relation.rb +3175 -321
  57. data/lib/lyra/projections/default_scopes.rb +254 -0
  58. data/lib/lyra/projections/deferred_relation.rb +23 -3
  59. data/lib/lyra/projections/event_store_reader.rb +47 -8
  60. data/lib/lyra/projections/foreign_keys.rb +92 -0
  61. data/lib/lyra/projections/lazy_projection.rb +165 -27
  62. data/lib/lyra/projections/model_projection.rb +165 -14
  63. data/lib/lyra/projections/rebuild.rb +34 -6
  64. data/lib/lyra/projections/reference_links.rb +463 -72
  65. data/lib/lyra/projections/typed_values.rb +629 -0
  66. data/lib/lyra/projections/unique_indexes.rb +595 -0
  67. data/lib/lyra/projections/uniqueness.rb +784 -0
  68. data/lib/lyra/projections/value_digest.rb +93 -0
  69. data/lib/lyra/purpose_bound_reads.rb +912 -77
  70. data/lib/lyra/repair.rb +119 -28
  71. data/lib/lyra/retention.rb +114 -8
  72. data/lib/lyra/rights/consent_store.rb +415 -0
  73. data/lib/lyra/rights/erasure_guard.rb +135 -0
  74. data/lib/lyra/rights/pending.rb +74 -0
  75. data/lib/lyra/rights/queries.rb +269 -0
  76. data/lib/lyra/rights/store.rb +512 -0
  77. data/lib/lyra/rights/subject_keys.rb +224 -0
  78. data/lib/lyra/rights.rb +281 -0
  79. data/lib/lyra/saves_without_effect.rb +69 -0
  80. data/lib/lyra/stream_tail.rb +319 -0
  81. data/lib/lyra/strict_data_access.rb +916 -106
  82. data/lib/lyra/temporal.rb +4 -2
  83. data/lib/lyra/touch_registry.rb +404 -0
  84. data/lib/lyra/verification/bypass_workflow.rb +452 -42
  85. data/lib/lyra/verification/rights_workflows.rb +170 -0
  86. data/lib/lyra/version.rb +1 -1
  87. data/lib/lyra/visualization/event_graph.rb +1 -1
  88. data/lib/lyra/visualization/timeline.rb +2 -2
  89. data/lib/lyra.rb +43 -2
  90. data/lib/tasks/lyra_mode.rake +110 -8
  91. data/lib/tasks/lyra_projections.rake +43 -2
  92. metadata +34 -4
  93. data/lib/lyra/associations/event_aware.rb +0 -224
@@ -33,9 +33,13 @@ module Lyra
33
33
  # becomes visible within about GAP_RECHECK; a process's own writes, above
34
34
  # the checkpoint, are visible immediately.
35
35
  #
36
- # Cost: every read first checks whether the log has moved on (two small
37
- # queries). Catch-up happens only when it has. The checkpoint table is
38
- # created on first use, so enabling the mode needs no migration.
36
+ # Cost: every read first checks whether the log has moved on, in one
37
+ # query that reads the checkpoint and the newest event id together.
38
+ # Catch-up happens only when it has. A reader that finds another reader
39
+ # catching up does not queue to repeat its work: it waits for that
40
+ # catch-up to end and reads the checkpoint again (wait_for_catch_up). A
41
+ # catch-up that applies nothing writes no checkpoint. The checkpoint
42
+ # table is created on first use, so enabling the mode needs no migration.
39
43
  class LazyProjection
40
44
  TABLE = "lyra_projection_checkpoints"
41
45
  NAME = "lazy"
@@ -44,12 +48,33 @@ module Lyra
44
48
  GAP_RECHECK = 1.0
45
49
  FRESH_WINDOW = 100
46
50
  BATCH = 1_000
51
+ # How often a reader waits for another reader's catch-up before it
52
+ # queues for the lock itself, as every reader did before. Two waits
53
+ # suffice when every catch-up commits: the second holder took the lock
54
+ # after this reader's check, so it saw every event this reader must
55
+ # see. The third allows for one that rolled back, or for a try that
56
+ # failed only because another waiter held the lock shared for a moment.
57
+ WAITS = 3
47
58
 
48
59
  class << self
49
60
  def active?
50
61
  Lyra.event_sourcing_mode? && Lyra.config.projection_mode == :lazy
51
62
  end
52
63
 
64
+ # Run the block with no catch-up before its reads (a write that
65
+ # caught up when it began: BypassEvents.atomically).
66
+ def without_catch_up
67
+ return yield unless active?
68
+
69
+ previous = Thread.current[:lyra_lazy_catching_up]
70
+ Thread.current[:lyra_lazy_catching_up] = true
71
+ begin
72
+ yield
73
+ ensure
74
+ Thread.current[:lyra_lazy_catching_up] = previous
75
+ end
76
+ end
77
+
53
78
  # Called before a read on +model_class+ (see Interceptors::LazyReads).
54
79
  def before_read(model_class)
55
80
  return unless active?
@@ -67,20 +92,18 @@ module Lyra
67
92
  return 0 unless force || active?
68
93
  return 0 if Thread.current[:lyra_lazy_catching_up]
69
94
 
70
- Thread.current[:lyra_lazy_catching_up] = true
95
+ began = Thread.current[:lyra_lazy_catching_up] = true
71
96
  ActiveRecord::Base.with_connection do |connection|
72
97
  ensure_table!(connection)
73
- next 0 if up_to_date?(connection)
98
+ target = catch_up_target(connection)
99
+ next 0 unless target
74
100
 
75
- applied = 0
76
- ActiveRecord::Base.transaction do
77
- lock!(connection)
78
- applied = apply_pending(connection)
79
- end
80
- applied
101
+ ActiveRecord::Base.transaction { catch_up_to(connection, target) }
81
102
  end
82
103
  ensure
83
- Thread.current[:lyra_lazy_catching_up] = nil
104
+ # Only the call that set it (a call refused above left it to its
105
+ # owner: a catch-up under way, or LazyProjection.without_catch_up).
106
+ Thread.current[:lyra_lazy_catching_up] = nil if began
84
107
  end
85
108
 
86
109
  # The current checkpoint: [position, { gap_id => first_seen_epoch }].
@@ -91,7 +114,38 @@ module Lyra
91
114
  row = connection.select_one("SELECT position, gaps FROM #{TABLE} WHERE name = #{connection.quote(NAME)}")
92
115
  return [0, {}] unless row
93
116
 
94
- [row["position"].to_i, JSON.parse(row["gaps"] || "{}").transform_keys(&:to_i)]
117
+ parse_checkpoint(row["position"], row["gaps"])
118
+ end
119
+
120
+ # Bring the rows of +ids+ (records of +model_class+) up to date with
121
+ # their streams, where a stream has an event the tables have not had
122
+ # yet (past the checkpoint, or among its gaps): that record is
123
+ # replayed onto its row (Rebuild.replay_record), as a catch-up would.
124
+ # For a write that skips callbacks, once it holds the records'
125
+ # aggregate locks (Lyra::BypassEvents): an update committed after the
126
+ # catch-up the write began with is then in the row it writes, not
127
+ # lost under it. One query when every row is up to date. Returns how
128
+ # many records were replayed.
129
+ def refresh_records(model_class, ids)
130
+ return 0 unless active? && ids.any?
131
+
132
+ ActiveRecord::Base.with_connection do |connection|
133
+ position, gaps = checkpoint(connection)
134
+ prefix = Lyra.stream_prefix(model_class)
135
+ streams = ids.map { |id| connection.quote("#{prefix}#{id}") }.join(", ")
136
+ pending = ["e.id > #{Integer(position)}"]
137
+ pending << "e.id IN (#{gaps.keys.map { Integer(_1) }.join(',')})" if gaps.any?
138
+ behind = connection.uncached do
139
+ connection.select_values(
140
+ "SELECT DISTINCT s.stream FROM event_store_events_in_streams s " \
141
+ "JOIN event_store_events e ON e.event_id = s.event_id " \
142
+ "WHERE s.stream IN (#{streams}) AND (#{pending.join(' OR ')})",
143
+ "Lyra ES-Lazy rows behind"
144
+ )
145
+ end
146
+ behind.each { |stream| Rebuild.replay_record(model_class, stream.delete_prefix(prefix)) }
147
+ behind.size
148
+ end
95
149
  end
96
150
 
97
151
  # Forget the checkpoint, e.g. after the tables were truncated or rebuilt.
@@ -109,11 +163,24 @@ module Lyra
109
163
  table.start_with?("event_store_") || table == TABLE
110
164
  end
111
165
 
112
- # Fast path, no lock: nothing new, and open gaps (if any) were
113
- # re-checked less than GAP_RECHECK seconds ago.
114
- def up_to_date?(connection)
115
- position, gaps = checkpoint(connection)
116
- newest = max_event_id(connection)
166
+ def parse_checkpoint(position, gaps)
167
+ [position.to_i, JSON.parse(gaps || "{}").transform_keys(&:to_i)]
168
+ end
169
+
170
+ # Fast path, no lock: nil when nothing is new and open gaps (if any)
171
+ # were re-checked less than GAP_RECHECK seconds ago. Otherwise the
172
+ # newest event id this read must see applied: the catch-up target.
173
+ #
174
+ # One statement reads the checkpoint and the newest event id, so both
175
+ # come from one snapshot. It decides as the two queries it replaces
176
+ # did, so the fast path skips exactly what it skipped before.
177
+ def catch_up_target(connection)
178
+ row = connection.select_rows(<<~SQL.squish).first
179
+ SELECT c.position, c.gaps, (SELECT MAX(id) FROM event_store_events)
180
+ FROM (SELECT 1) AS one LEFT JOIN #{TABLE} c ON c.name = #{connection.quote(NAME)}
181
+ SQL
182
+ position, gaps = parse_checkpoint(row[0], row[1])
183
+ newest = row[2].to_i
117
184
  # A checkpoint past the newest event cannot be right: the event
118
185
  # store was emptied or restored without this table. Taken at its
119
186
  # word, every new event would look applied and nothing would be
@@ -121,15 +188,68 @@ module Lyra
121
188
  if position > newest
122
189
  warn_stale_checkpoint(position, newest)
123
190
  reset!(connection)
124
- return newest.zero?
191
+ return newest.zero? ? nil : newest
125
192
  end
126
- return false if newest > position
193
+ return newest if newest > position
127
194
 
195
+ gaps_rechecked?(gaps) ? nil : newest
196
+ end
197
+
198
+ def gaps_rechecked?(gaps)
128
199
  gaps.empty? || (Process.clock_gettime(Process::CLOCK_MONOTONIC) - (@gaps_checked_at || 0)) < GAP_RECHECK
129
200
  end
130
201
 
202
+ # Bring the tables up to +target+, in the reader's transaction.
203
+ #
204
+ # The reader that gets the lock catches up, as before. A reader that
205
+ # finds the lock taken does not queue behind it to repeat the same
206
+ # work: it waits for that catch-up to end (AdvisoryLock.wait_for),
207
+ # then reads the checkpoint again and is done if nothing it must see
208
+ # is left (caught_up_to?). Otherwise it tries again, and after WAITS
209
+ # waits it queues for the lock and catches up itself, as before.
210
+ #
211
+ # The rules hold: only the lock's holder applies events or writes the
212
+ # checkpoint, in its own transaction, in the log's order, from the
213
+ # checkpoint it reads under the lock (rules 1 and 2); a reader that
214
+ # waited applies and writes nothing, so its transaction has nothing
215
+ # to roll back. Waiting skips work only when the work is done, by a
216
+ # catch-up that committed (the waiter reads the committed checkpoint
217
+ # after the holder's transaction ended), and it is judged with the
218
+ # gap list (rule 3), more strictly than the fast path judges it.
219
+ def catch_up_to(connection, target)
220
+ WAITS.times do
221
+ return apply_pending(connection) if try_lock!(connection)
222
+
223
+ wait_for_catch_up(connection)
224
+ return 0 if caught_up_to?(connection, target)
225
+ end
226
+ lock!(connection)
227
+ apply_pending(connection)
228
+ end
229
+
230
+ # After waiting: has a committed catch-up applied every event this
231
+ # reader must see? Those are the events visible to it, up to the
232
+ # newest it saw before waiting (+target+; events committed later are
233
+ # not owed to this read). The checkpoint must have reached +target+,
234
+ # and no event the checkpoint lists as missing may be visible to this
235
+ # reader: such an event committed (or was written by this reader's
236
+ # own open transaction) after the catch-up passed its id, and only a
237
+ # catch-up under the lock replays it. So this reader sees what it
238
+ # would have seen had it queued for the lock and caught up itself;
239
+ # unlike the fast path, it does not wait GAP_RECHECK for a late event.
240
+ def caught_up_to?(connection, target)
241
+ position, gaps = checkpoint(connection)
242
+ return false if position < target
243
+ return true if gaps.empty?
244
+
245
+ connection.select_value(
246
+ "SELECT 1 FROM event_store_events WHERE id IN (#{gaps.keys.map { Integer(_1) }.join(',')}) LIMIT 1"
247
+ ).nil?
248
+ end
249
+
131
250
  def apply_pending(connection)
132
251
  position, gaps = checkpoint(connection)
252
+ started_at = [position, gaps.dup]
133
253
  applied = 0
134
254
 
135
255
  loop do
@@ -154,7 +274,11 @@ module Lyra
154
274
  end
155
275
 
156
276
  applied += fill_gaps(connection, gaps)
157
- save_checkpoint(connection, position, gaps)
277
+ # A catch-up that found nothing new (another reader got there
278
+ # first) leaves the checkpoint row alone: rewriting the same values
279
+ # cost a row write and a commit that must reach the disk, all under
280
+ # the lock every other catching-up reader waits on.
281
+ save_checkpoint(connection, position, gaps) unless [position, gaps] == started_at
158
282
  applied
159
283
  end
160
284
 
@@ -186,8 +310,13 @@ module Lyra
186
310
  return false unless model_class && operation
187
311
 
188
312
  ModelProjection.project(model_class, operation, Struct.new(:events, :attributes, :success?, keyword_init: true).new(
189
- events: [event], attributes: event.data[:attributes] || event.data["attributes"] || {}, success?: true
190
- ))
313
+ events: [event], success?: true,
314
+ attributes: if %i[created imported].include?(operation)
315
+ Lyra::Event.created_attributes(event.data)
316
+ else
317
+ event.data[:attributes] || event.data["attributes"] || {}
318
+ end
319
+ ), INTERNAL)
191
320
  true
192
321
  end
193
322
 
@@ -212,10 +341,6 @@ module Lyra
212
341
  Lyra.config.event_store.read.events(event_ids).to_a.to_h { |e| [e.event_id, e] }
213
342
  end
214
343
 
215
- def max_event_id(connection)
216
- connection.select_value("SELECT MAX(id) FROM event_store_events").to_i
217
- end
218
-
219
344
  def warn_stale_checkpoint(position, newest)
220
345
  message = "Lyra: ES-Lazy checkpoint #{position} is past the newest event (#{newest}); " \
221
346
  "the event store was emptied or restored without #{TABLE}. Catching up from the start."
@@ -226,6 +351,19 @@ module Lyra
226
351
  Lyra::AdvisoryLock.xact_lock(connection, LOCK, purpose: "ES-Lazy catch-up")
227
352
  end
228
353
 
354
+ # True when this transaction now holds the catch-up lock (or the
355
+ # adapter has none, as lock! then takes none); false when another
356
+ # reader holds it.
357
+ def try_lock!(connection)
358
+ Lyra::AdvisoryLock.try_xact_lock(connection, LOCK, purpose: "ES-Lazy catch-up")
359
+ end
360
+
361
+ # Wait until the reader that holds the catch-up lock has committed or
362
+ # rolled back. Several waiters go on together.
363
+ def wait_for_catch_up(connection)
364
+ Lyra::AdvisoryLock.wait_for(connection, LOCK)
365
+ end
366
+
229
367
  def save_checkpoint(connection, position, gaps)
230
368
  name = connection.quote(NAME)
231
369
  json = connection.quote(JSON.generate(gaps))
@@ -2,6 +2,20 @@
2
2
 
3
3
  module Lyra
4
4
  module Projections
5
+ # A projected create met, in a unique index, the row of a record whose
6
+ # stream no longer holds the value (destroyed, or moved to another): the
7
+ # row waits for that record's projection (ES-Async: its job). Projecting
8
+ # again later converges; AsyncProjectionJob retries it with no limit on
9
+ # the attempts (ModelProjection.refuse_skipped_create!).
10
+ class ProjectionBehind < ActiveRecord::RecordNotUnique; end
11
+
12
+ # A projected create met, in a unique index, the row of a live record
13
+ # whose stream holds the value too: two records of the log hold one
14
+ # value (written under REPEATABLE READ or SERIALIZABLE, or before 1.0.0),
15
+ # which no projection can hold. Logged once per record and process; not
16
+ # retried (ModelProjection.refuse_skipped_create!).
17
+ class DuplicateInLog < ActiveRecord::RecordNotUnique; end
18
+
5
19
  # Synchronous projection handler for updating model tables from events.
6
20
  #
7
21
  # In event_sourcing mode, this class is responsible for keeping the model
@@ -21,7 +35,12 @@ module Lyra
21
35
  # @param model_class [Class] The ActiveRecord model class
22
36
  # @param operation [Symbol] :create, :update, or :destroy
23
37
  # @param result [CommandResult] The result from CommandHandler
24
- def project(model_class, operation, result)
38
+ # Writes the table with no event, so only Lyra calls it (Lyra::INTERNAL),
39
+ # as for Lyra.projection_write; project_create, project_update and
40
+ # project_destroy are private.
41
+ def project(model_class, operation, result, token = nil)
42
+ raise InternalApiError, "ModelProjection.project is for Lyra's own projections" unless token.equal?(INTERNAL)
43
+
25
44
  # Projection writes the table, so it must address the table. With
26
45
  # projection_mode :disabled (ES-NoProj), the model's where/all answer
27
46
  # from the event store instead; update and destroy below go through
@@ -31,7 +50,7 @@ module Lyra
31
50
  # both project through here, whatever the projection mode.
32
51
  previous = Thread.current[:lyra_bypass_read_override]
33
52
  Thread.current[:lyra_bypass_read_override] = true
34
- Lyra::PurposeBoundReads.internal do
53
+ Lyra::PurposeBoundReads.internal(INTERNAL) do
35
54
  case operation
36
55
  when :create
37
56
  project_create(model_class, result)
@@ -54,22 +73,150 @@ module Lyra
54
73
  def project_create(model_class, result)
55
74
  # Encrypted attributes arrive from the log as ciphertext; Active
56
75
  # Record encrypts what it inserts, so they are decrypted first.
57
- attributes = Lyra::EncryptedAttributes.open(model_class, result.attributes.dup)
76
+ # String keys throughout: an event's attributes come from the store, or
77
+ # from a record, with string keys, and a symbol-keyed default added
78
+ # beside a string-keyed value overwrote it when the keys were
79
+ # stringified (every created_at and updated_at became the projection
80
+ # time).
81
+ attributes = Lyra::EncryptedAttributes.open(model_class, result.attributes.dup).stringify_keys
58
82
 
59
- # Ensure we have an ID
60
- raise "Cannot project create without ID" unless attributes[:id] || attributes["id"]
83
+ # Ensure we have the record's key (the primary key column, whatever
84
+ # its name)
85
+ raise "Cannot project create without ID" unless attributes[Lyra.key_column(model_class)]
61
86
 
62
- # Add timestamps if not present
63
- now = Time.current
64
- attributes[:created_at] ||= now
65
- attributes[:updated_at] ||= now
87
+ # No default timestamps: a create's event carries created_at and
88
+ # updated_at like any other column (from 1.0.0), and a value made up
89
+ # here would differ from the row's.
66
90
 
67
91
  # Convert to database-safe format
68
92
  insert_attrs = sanitize_attributes(model_class, attributes)
69
93
 
70
94
  # Use insert to bypass callbacks
71
95
  # Lyra.projection_write: skips strict access and publishes no bypass events
72
- Lyra.projection_write { model_class.insert(insert_attrs) }
96
+ #
97
+ # ON CONFLICT DO NOTHING, so a create projected again (a replay, a
98
+ # job after a write that projected its own row) inserts nothing.
99
+ inserted = Lyra.projection_write(INTERNAL) { model_class.insert(insert_attrs) }
100
+ refuse_skipped_create!(model_class, insert_attrs, inserted)
101
+ inserted
102
+ end
103
+
104
+ # A create that inserted nothing (no key returned) although no row has
105
+ # its key (on the table, whatever the record's class now: an STI
106
+ # record whose type changed is still its row) conflicted on another
107
+ # unique index: a row another record holds. ON CONFLICT DO NOTHING
108
+ # skipped it silently: a projection job that ran before the job of
109
+ # the holder its record replaced (the holder's row busy when the
110
+ # write checked it) completed and moved its checkpoint past the
111
+ # create, and the log held a record the table never got. One query,
112
+ # only when nothing was inserted; then the rows holding its values
113
+ # are read (one query per unique index whose tuple the create gives
114
+ # and that a NULL in it does not exempt) and their records' streams:
115
+ #
116
+ # - Every such row is of a record whose stream no longer holds the
117
+ # value (destroyed, or moved): its projection lags. ES-Async brings
118
+ # those rows up to date with their streams here, where nothing has
119
+ # to be waited for (UniqueIndexes.bring_rows_up_to_date: a holder
120
+ # whose own job was lost no longer blocks), and inserts again. A row
121
+ # still held by another transaction raises ProjectionBehind, which
122
+ # AsyncProjectionJob retries (BEHIND_ATTEMPTS: about a day, then
123
+ # reported and given up); ES-Sync's write fails with it.
124
+ # - A live record holds the value in the log too, or a row with no
125
+ # stream (Monitor's table holds it), or no holder is found once the
126
+ # insert has been tried again (an expression index, which Lyra does
127
+ # not read): DuplicateInLog. No projection can hold both. Reported
128
+ # once per record and process (report_duplicate); raised for a job,
129
+ # which discards it, and for ES-Sync's write.
130
+ #
131
+ # In ES-Lazy's catch-up neither is raised: raising there would stop
132
+ # every read. It is reported (once) and the catch-up goes on.
133
+ def refuse_skipped_create!(model_class, attributes, inserted)
134
+ key = model_class.primary_key
135
+ return unless key.is_a?(String) && inserted.respond_to?(:rows) && inserted.rows.empty?
136
+ return unless model_class.with_connection(&:supports_insert_returning?)
137
+
138
+ table = model_class.base_class
139
+ id = attributes[key]
140
+ present = table.uncached { table.unscoped.where(key => id).exists? }
141
+ return if present
142
+
143
+ holders = holder_rows(table, attributes)
144
+ lagging = holders.any? && holders.all? { |holder, held| lagging?(table, holder, held) }
145
+ ids = holders.map(&:first).uniq
146
+ # ES-Async: the lagging rows brought up to date, then the insert
147
+ # again; with no holder found, the insert again (a row gone since).
148
+ if holders.empty? || (lagging && UniqueIndexes.bring_rows_up_to_date(table, ids.map(&:to_s)).any?)
149
+ again = Lyra.projection_write(INTERNAL) { model_class.insert(attributes) }
150
+ return if again.rows.any?
151
+ end
152
+
153
+ error = if lagging
154
+ ProjectionBehind.new("Lyra: the projection of #{model_class.name} #{id}'s create met the row of " \
155
+ "#{table.name} #{ids.join(', ')}, which gave the value up but whose projection lags " \
156
+ "(its row held by another transaction); nothing was inserted")
157
+ else
158
+ DuplicateInLog.new("Lyra: the projection of #{model_class.name} #{id}'s create met a value of a unique index " \
159
+ "of #{model_class.table_name} that #{holders.empty? ? 'a row Lyra cannot match (an expression index)' : 'another live record holds in the log too'} " \
160
+ "holds; nothing was inserted")
161
+ end
162
+ lazy = Lyra.event_sourcing_mode? && Lyra.config.projection_mode == :lazy
163
+ report_duplicate(table, id, error) if !lagging || lazy
164
+ raise error unless lazy
165
+ end
166
+
167
+ # [[primary key, { column => value }]] of the rows that hold one of
168
+ # +attributes+' unique values: one query per unique index (other than
169
+ # the primary key, on columns) whose columns the create gives, a
170
+ # tuple with a NULL left out unless the index is NULLS NOT DISTINCT
171
+ # (it conflicts with nothing).
172
+ def holder_rows(table, attributes)
173
+ key = table.primary_key
174
+ indexes = table.with_connection { |c| c.indexes(table.table_name) }.select do |index|
175
+ index.unique && index.columns.is_a?(Array) && index.columns != [key]
176
+ end
177
+ indexes.flat_map do |index|
178
+ next [] unless index.columns.all? { attributes.key?(_1) }
179
+
180
+ distinct = !(index.respond_to?(:nulls_not_distinct) && index.nulls_not_distinct)
181
+ next [] if distinct && index.columns.any? { attributes[_1].nil? }
182
+
183
+ scope = table.unscoped.where(index.columns.to_h { [_1, attributes[_1]] })
184
+ scope = scope.where(Arel.sql(index.where)) if index.where
185
+ table.uncached { scope.pluck(key, *index.columns) }.map { |id, *values| [id, index.columns.zip(values).to_h] }
186
+ end
187
+ end
188
+
189
+ # Whether the row of +id+, holding +held+, is of a record whose stream
190
+ # no longer holds those values: destroyed, or moved to another.
191
+ def lagging?(table, id, held)
192
+ return true if Lyra::StreamTail.destroyed?(Lyra.stream_name(table, id))
193
+
194
+ now = Lyra::PurposeBoundReads.internal(INTERNAL) { CachedProjection.find(table, id) }
195
+ return false unless now # no stream: a row the log does not know, which holds the value
196
+
197
+ reader = TypedValues::EventReader.new(table)
198
+ held.any? { |column, value| UniqueIndexes.key(table, column, reader.read(now, column)) != UniqueIndexes.key(table, column, value) }
199
+ end
200
+
201
+ # How many creates report_duplicate remembers, per process.
202
+ REPORTED_LIMIT = 10_000
203
+
204
+ # A create the projection could not insert: logged at error level once
205
+ # per record and process (debug after), and handed to
206
+ # config.projection_error_handler once. The records remembered are
207
+ # bounded (REPORTED_LIMIT): past it, they are forgotten and each is
208
+ # reported once more.
209
+ def report_duplicate(table, id, error)
210
+ reported = (@reported_duplicates ||= Concurrent::Set.new)
211
+ reported.clear if reported.size >= REPORTED_LIMIT
212
+ first = reported.add?([table.name, id.to_s])
213
+ logger = defined?(Rails) && Rails.logger
214
+ if first
215
+ logger&.error(error.message)
216
+ Lyra.config.projection_error_handler&.call(error, nil, :create)
217
+ else
218
+ logger&.debug(error.message)
219
+ end
73
220
  end
74
221
 
75
222
  # Update an existing record in the model table
@@ -92,13 +239,15 @@ module Lyra
92
239
  updates[field.to_sym] = new_value
93
240
  end
94
241
 
95
- # Ensure updated_at timestamp (may already be in changes)
96
- updates[:updated_at] ||= Time.current
242
+ # updated_at is set only when the event changed it: an update that
243
+ # left it alone (update_columns, an update_all that did not set it)
244
+ # left it alone in the row too, so stamping the projection time here
245
+ # made a rebuilt row differ from its events.
97
246
  updates = Lyra::EncryptedAttributes.open(model_class, updates)
98
247
 
99
248
  # Use update_all to bypass callbacks
100
249
  # Lyra.projection_write: skips strict access and publishes no bypass events
101
- Lyra.projection_write { model_class.where(id: model_id).update_all(updates) }
250
+ Lyra.projection_write(INTERNAL) { model_class.where(Lyra.key_column(model_class) => model_id).update_all(updates) }
102
251
  end
103
252
 
104
253
  # Delete a record from the model table
@@ -112,9 +261,11 @@ module Lyra
112
261
 
113
262
  # Use delete_all to bypass callbacks
114
263
  # Lyra.projection_write: skips strict access and publishes no bypass events
115
- Lyra.projection_write { model_class.where(id: model_id).delete_all }
264
+ Lyra.projection_write(INTERNAL) { model_class.where(Lyra.key_column(model_class) => model_id).delete_all }
116
265
  end
117
266
 
267
+ private :project_create, :project_update, :project_destroy, :refuse_skipped_create!, :holder_rows, :lagging?, :report_duplicate
268
+
118
269
  private
119
270
 
120
271
  # Sanitize attributes for database insertion
@@ -22,6 +22,10 @@ module Lyra
22
22
  # rake lyra:projections:rebuild # all monitored models
23
23
  #
24
24
  class Rebuild
25
+ # A rebuild that would undo what the log does not record
26
+ # (config.record_touches false).
27
+ class Refused < StandardError; end
28
+
25
29
  class << self
26
30
  # Rebuild a single model's read-model table from its event streams.
27
31
  #
@@ -30,13 +34,29 @@ module Lyra
30
34
  # set false to reconstruct only the streamed rows in place
31
35
  # @return [Hash] statistics for the rebuild
32
36
  def rebuild(model_class, truncate: true)
37
+ # With touches left out of the log, the rows' updated_at is newer
38
+ # than the events say, and a rebuild would set every touched row back.
39
+ if Lyra.config.record_touches == false
40
+ raise Lyra::Projections::Rebuild::Refused,
41
+ "Lyra refuses to rebuild #{model_class.name} from its events: config.record_touches is false, so " \
42
+ "the log misses touches and the rebuild would set each touched row's updated_at back. Set " \
43
+ "config.record_touches = true and run lyra:repair first."
44
+ end
45
+ live = Lyra::TouchRegistry.live_processes
46
+ if live.any?
47
+ raise Lyra::Projections::Rebuild::Refused,
48
+ "Lyra refuses to rebuild #{model_class.name} from its events: processes leave touches out of the " \
49
+ "log (config.record_touches = false): #{Lyra::TouchRegistry.describe(live)}. The rebuild would set " \
50
+ "each touched row's updated_at back. #{Lyra::TouchRegistry::STEPS}"
51
+ end
52
+
33
53
  stats = {
34
54
  model: model_class.name,
35
55
  streams: 0, events: 0, records: 0, destroyed: 0
36
56
  }
37
57
  rebuilt_ids = []
38
58
 
39
- Lyra::PurposeBoundReads.internal { rebuild_in_transaction(model_class, truncate, stats, rebuilt_ids) }
59
+ Lyra::PurposeBoundReads.internal(INTERNAL) { rebuild_in_transaction(model_class, truncate, stats, rebuilt_ids) }
40
60
  end
41
61
 
42
62
  def rebuild_in_transaction(model_class, truncate, stats, rebuilt_ids)
@@ -92,12 +112,16 @@ module Lyra
92
112
  # event the row already reflects is harmless: a create of an existing
93
113
  # row inserts nothing and an update sets the same values, so callers
94
114
  # may start from an earlier event than strictly needed.
95
- def project_events(model_class, events)
115
+ # Projects the events it is given, whether or not they are in the log,
116
+ # so only Lyra calls it (Lyra::INTERNAL).
117
+ def project_events(model_class, events, token = nil)
118
+ raise InternalApiError, "Rebuild.project_events is for Lyra's own projections" unless token.equal?(INTERNAL)
119
+
96
120
  events.count do |event|
97
121
  operation = operation_for(event)
98
122
  next false unless operation
99
123
 
100
- ModelProjection.project(model_class, operation, build_result_from_event(event))
124
+ ModelProjection.project(model_class, operation, build_result_from_event(event), INTERNAL)
101
125
  true
102
126
  end
103
127
  end
@@ -109,7 +133,7 @@ module Lyra
109
133
  def replay_stream(model_class, id)
110
134
  stream_name = Lyra.stream_name(model_class, id)
111
135
  events = Lyra.config.event_store.read.stream(stream_name).to_a
112
- replayed = project_events(model_class, events)
136
+ replayed = project_events(model_class, events, INTERNAL)
113
137
 
114
138
  last = events.reverse_each.find { Lyra::Event.operation_of(_1) }
115
139
  present = !(last && Lyra::Event.operation_of(last) == :destroyed)
@@ -132,7 +156,11 @@ module Lyra
132
156
  def build_result_from_event(event)
133
157
  Struct.new(:events, :attributes, :success?, keyword_init: true).new(
134
158
  events: [event],
135
- attributes: event.data[:attributes] || event.data["attributes"] || {},
159
+ attributes: if %i[created imported].include?(Lyra::Event.operation_of(event))
160
+ Lyra::Event.created_attributes(event.data)
161
+ else
162
+ event.data[:attributes] || event.data["attributes"] || {}
163
+ end,
136
164
  success?: true
137
165
  )
138
166
  end
@@ -149,7 +177,7 @@ module Lyra
149
177
  # Clear the read-model table, bypassing callbacks/strict-access guards
150
178
  # (this is a legitimate bulk projection operation, like ModelProjection).
151
179
  def clear_read_model(model_class)
152
- Lyra.projection_write { model_class.unscoped.delete_all }
180
+ Lyra.projection_write(INTERNAL) { model_class.unscoped.delete_all }
153
181
  end
154
182
 
155
183
  # Drop any cached (ES-NoProj) reconstructions so reads reflect the rebuild.