orfeas_lyra 0.9.3 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +1923 -17
  3. data/README.md +15 -8
  4. data/Rakefile +40 -1
  5. data/app/controllers/lyra/application_controller.rb +108 -0
  6. data/app/controllers/lyra/dashboard_controller.rb +17 -15
  7. data/app/controllers/lyra/flow_controller.rb +30 -33
  8. data/app/controllers/lyra/privacy_controller.rb +57 -18
  9. data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
  10. data/app/views/lyra/dashboard/index.html.erb +1 -0
  11. data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
  12. data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
  13. data/app/views/lyra/privacy/policy.html.erb +7 -0
  14. data/app/views/lyra/privacy/rights.html.erb +78 -0
  15. data/config/privacy_policies.rb +0 -3
  16. data/config/routes.rb +1 -0
  17. data/lib/lyra/access_log.rb +187 -6
  18. data/lib/lyra/advisory_lock.rb +236 -0
  19. data/lib/lyra/aggregate.rb +1 -1
  20. data/lib/lyra/associations/habtm_links.rb +382 -19
  21. data/lib/lyra/associations/maintenance.rb +229 -0
  22. data/lib/lyra/attribution.rb +48 -0
  23. data/lib/lyra/bypass_events.rb +426 -45
  24. data/lib/lyra/command_handler.rb +19 -14
  25. data/lib/lyra/configuration.rb +237 -10
  26. data/lib/lyra/consistency/read_your_writes.rb +4 -2
  27. data/lib/lyra/correlation.rb +121 -0
  28. data/lib/lyra/domain_events.rb +4 -0
  29. data/lib/lyra/dual_view.rb +53 -16
  30. data/lib/lyra/encrypted_attributes.rb +5 -0
  31. data/lib/lyra/engine.rb +67 -0
  32. data/lib/lyra/erasure.rb +1002 -116
  33. data/lib/lyra/event.rb +35 -1
  34. data/lib/lyra/event_data.rb +38 -0
  35. data/lib/lyra/event_flow.rb +31 -15
  36. data/lib/lyra/event_store_adapter.rb +28 -0
  37. data/lib/lyra/event_window.rb +58 -0
  38. data/lib/lyra/genesis.rb +548 -44
  39. data/lib/lyra/interceptors/association_interceptor.rb +689 -34
  40. data/lib/lyra/interceptors/crud_interceptor.rb +547 -95
  41. data/lib/lyra/internal.rb +26 -0
  42. data/lib/lyra/mode_sync.rb +51 -3
  43. data/lib/lyra/mode_transition.rb +245 -34
  44. data/lib/lyra/model_selection.rb +35 -0
  45. data/lib/lyra/own_connection.rb +17 -2
  46. data/lib/lyra/privacy/adapters/pam.rb +25 -2
  47. data/lib/lyra/privacy/gdpr_compliance.rb +264 -36
  48. data/lib/lyra/privacy/interface.rb +27 -7
  49. data/lib/lyra/privacy/subject_links.rb +143 -0
  50. data/lib/lyra/privacy/subject_records.rb +369 -0
  51. data/lib/lyra/projection.rb +7 -1
  52. data/lib/lyra/projections/async_checkpoint.rb +32 -0
  53. data/lib/lyra/projections/async_projection_job.rb +166 -23
  54. data/lib/lyra/projections/cached_joins.rb +33 -18
  55. data/lib/lyra/projections/cached_projection.rb +74 -33
  56. data/lib/lyra/projections/cached_relation.rb +3175 -321
  57. data/lib/lyra/projections/default_scopes.rb +254 -0
  58. data/lib/lyra/projections/deferred_relation.rb +23 -3
  59. data/lib/lyra/projections/event_store_reader.rb +47 -8
  60. data/lib/lyra/projections/foreign_keys.rb +92 -0
  61. data/lib/lyra/projections/lazy_projection.rb +165 -27
  62. data/lib/lyra/projections/model_projection.rb +165 -14
  63. data/lib/lyra/projections/rebuild.rb +34 -6
  64. data/lib/lyra/projections/reference_links.rb +463 -72
  65. data/lib/lyra/projections/typed_values.rb +629 -0
  66. data/lib/lyra/projections/unique_indexes.rb +595 -0
  67. data/lib/lyra/projections/uniqueness.rb +784 -0
  68. data/lib/lyra/projections/value_digest.rb +93 -0
  69. data/lib/lyra/purpose_bound_reads.rb +912 -77
  70. data/lib/lyra/repair.rb +119 -28
  71. data/lib/lyra/retention.rb +114 -8
  72. data/lib/lyra/rights/consent_store.rb +415 -0
  73. data/lib/lyra/rights/erasure_guard.rb +135 -0
  74. data/lib/lyra/rights/pending.rb +74 -0
  75. data/lib/lyra/rights/queries.rb +269 -0
  76. data/lib/lyra/rights/store.rb +512 -0
  77. data/lib/lyra/rights/subject_keys.rb +224 -0
  78. data/lib/lyra/rights.rb +281 -0
  79. data/lib/lyra/saves_without_effect.rb +69 -0
  80. data/lib/lyra/stream_tail.rb +319 -0
  81. data/lib/lyra/strict_data_access.rb +906 -107
  82. data/lib/lyra/temporal.rb +4 -2
  83. data/lib/lyra/touch_registry.rb +404 -0
  84. data/lib/lyra/verification/bypass_workflow.rb +452 -42
  85. data/lib/lyra/verification/rights_workflows.rb +170 -0
  86. data/lib/lyra/version.rb +1 -1
  87. data/lib/lyra/visualization/event_graph.rb +1 -1
  88. data/lib/lyra/visualization/timeline.rb +2 -2
  89. data/lib/lyra.rb +43 -2
  90. data/lib/tasks/lyra_mode.rake +107 -8
  91. data/lib/tasks/lyra_projections.rake +43 -2
  92. metadata +34 -4
  93. data/lib/lyra/associations/event_aware.rb +0 -224
@@ -1,6 +1,32 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Lyra
4
+ # Raised before a bulk write that would lock more records than
5
+ # config.bulk_write_lock_limit: the write holds one aggregate lock per
6
+ # record until its transaction ends, and PostgreSQL's lock table, which
7
+ # every session shares, has room for about max_locks_per_transaction ×
8
+ # (max_connections + background processes) locks. Past that, every
9
+ # session's writes fail ("out of shared memory").
10
+ class BulkWriteLockLimitError < StandardError
11
+ attr_reader :model_class, :count, :limit
12
+
13
+ def initialize(model_class, count, limit, write = "bulk write")
14
+ @model_class = model_class
15
+ @count = count
16
+ @limit = limit
17
+ name = model_class.respond_to?(:name) ? model_class.name : model_class.to_s
18
+ records = count == 1 ? "record" : "records"
19
+ locks = count == 1 ? "lock" : "locks"
20
+ super(
21
+ "Lyra: #{write} of #{count} #{name} #{records} would hold #{count} aggregate #{locks} until " \
22
+ "its transaction ends, more than config.bulk_write_lock_limit (#{limit}). Write in batches " \
23
+ "(in_batches(of: #{limit}) { |batch| batch.update_all(...) }): each batch commits, and gives its locks " \
24
+ "back, on its own, but not inside a transaction already open, which holds them all until it ends. " \
25
+ "Or raise the limit (nil: none), within PostgreSQL's lock table."
26
+ )
27
+ end
28
+ end
29
+
4
30
  # Publishes events for writes that skip ActiveRecord callbacks
5
31
  # (update_column(s), delete, update_all, delete_all, insert_all,
6
32
  # upsert_all, dependent: :nullify), so the event stream records every state change
@@ -11,8 +37,6 @@ module Lyra
11
37
  # read model. Those run inside Lyra.projection_write and must never
12
38
  # publish: the event they project already exists.
13
39
  module BypassEvents
14
- TIMESTAMP_COLUMNS = %w[created_at updated_at].freeze
15
-
16
40
  class << self
17
41
  # Whether a bypass write's events must be stored for the write to
18
42
  # stand. In Hijack and the event-sourcing modes the log is the record of
@@ -32,10 +56,19 @@ module Lyra
32
56
  # Under ES-Lazy the tables are brought up to date first, as before a
33
57
  # read: a write appends events only, so a record written since the last
34
58
  # read had no row yet, and update_columns, update_all or delete_all on it
35
- # matched nothing, recorded no event, and the change was lost.
59
+ # matched nothing, recorded no event, and the change was lost. Only
60
+ # then: the write's own reads (its snapshot, the rows it reads back for
61
+ # the events) do not catch up again. A catch-up there waited for the
62
+ # catch-up lock while the write held rows that a catch-up running
63
+ # elsewhere, holding that lock, waited for: a deadlock, which bulk
64
+ # writes and writes that skip callbacks met against catch-ups under
65
+ # load (before 1.0.0 too). A destroy committed after the catch-up is
66
+ # found by the stream check (checks_streams?).
36
67
  def atomically(model_class, &block)
37
68
  Lyra::Projections::LazyProjection.before_read(model_class)
38
- required? || locks_rows? ? model_class.transaction(&block) : yield
69
+ Lyra::Projections::LazyProjection.without_catch_up do
70
+ required? || locks_rows? ? model_class.transaction(&block) : yield
71
+ end
39
72
  end
40
73
 
41
74
  # Whether a bulk write locks the rows it matches and writes exactly
@@ -50,6 +83,220 @@ module Lyra
50
83
  !(Lyra.config.event_sourcing_mode? && Lyra.config.projection_mode == :disabled)
51
84
  end
52
85
 
86
+ # Whether a write that skips callbacks must ask the record's stream
87
+ # whether it was destroyed, under the aggregate lock, before it records
88
+ # anything: wherever a destroy can be in the store without its row
89
+ # being gone. In ES-NoProj there is no row (a stale copy, or a relation
90
+ # built before the destroy, still holds the record); in ES-Async the
91
+ # row stays until the destroy's projection job runs; in ES-Lazy until
92
+ # the next catch-up, and one committed after the catch-up a write
93
+ # began with is not in the table it writes. A write that reached the
94
+ # record there appended an update after its destroy, and brought it
95
+ # back. ES-Sync, Hijack and Monitor delete the row with the destroy.
96
+ # With lock_aggregate_before_append off, as for a write through
97
+ # callbacks, nothing is checked.
98
+ def checks_streams?
99
+ Lyra.config.lock_aggregate_before_append && Lyra.config.event_sourcing_mode? &&
100
+ %i[disabled async lazy].include?(Lyra.config.projection_mode)
101
+ end
102
+
103
+ # Whether an insert_all, insert_all! or upsert_all brings the rows of the
104
+ # keys it writes up to date with their streams first, under their
105
+ # aggregate locks (Lyra::Projections::AsyncProjectionJob
106
+ # .project_pending): in ES-Async, where a row waits for its projection
107
+ # job. A record whose destroy was pending kept its row, and ON CONFLICT
108
+ # skipped (insert_all) or updated (upsert_all) the dead record, where
109
+ # Monitor creates it again; a record whose create was pending had no
110
+ # row yet, and the insert logged a second Created, where Monitor's
111
+ # conflicts. Deleting the row alone was no answer: the destroy's job,
112
+ # run after, would delete the new row. ES-Lazy's rows are brought up to
113
+ # date when the write begins (atomically), and its upsert_all refuses a
114
+ # destroy committed since (StrictDataAccessRelation#lock_upserted_rows):
115
+ # replaying a destroy onto a row there would be applied again by the
116
+ # next catch-up, which replays from a checkpoint of its own, global, not
117
+ # per stream, and would delete the row created again.
118
+ def projects_pending?
119
+ checks_streams? && Lyra.config.projection_mode == :async
120
+ end
121
+
122
+ # Of +ids+ (records of +model_class+), those whose streams end in a
123
+ # destroy (Lyra::StreamTail), read under each stream's aggregate lock,
124
+ # the one a write through callbacks takes
125
+ # (CrudInterceptor#lyra_lock_aggregate_unless_destroyed!). The locks are
126
+ # taken in one statement and held until the caller's transaction ends,
127
+ # so no destroy can come between the read and the write. +write+ names
128
+ # the write in a BulkWriteLockLimitError.
129
+ def destroyed_ids(model_class, ids, write: "bulk write")
130
+ ids = ids.uniq
131
+ return [] if ids.empty?
132
+
133
+ lock_streams!(model_class, ids, write: write)
134
+ destroyed_among(model_class, ids)
135
+ end
136
+
137
+ # Of +ids+, those whose streams end in a destroy, read in one query
138
+ # where it can be (Lyra::StreamTail.destroyed_streams); the caller holds
139
+ # their aggregate locks.
140
+ def destroyed_among(model_class, ids)
141
+ return [] if ids.empty?
142
+
143
+ streams = ids.to_h { |id| [Lyra.stream_name(model_class, id), id] }
144
+ Lyra::StreamTail.destroyed_streams(streams.keys).map { streams[_1] }
145
+ end
146
+
147
+ # Run the write in the block (Active Record's own, on a record whose
148
+ # stream says it was destroyed) and take back what it wrote to the
149
+ # table, in a savepoint rolled back: a row still there (ES-Async, before
150
+ # the destroy's projection job runs; ES-NoProj, one left from an
151
+ # earlier mode) is not changed without an event. What the write set in
152
+ # memory stays, as on a row already gone.
153
+ def unwritten(model_class)
154
+ model_class.transaction(requires_new: true) do
155
+ yield
156
+ raise ActiveRecord::Rollback
157
+ end
158
+ nil
159
+ end
160
+
161
+ # The snapshot a bulk write writes (snapshot(lock: true)): the rows it
162
+ # matches, locked. Where the table can still hold the row of a record
163
+ # whose stream ends in a destroy (checks_streams?: ES-Async, before the
164
+ # destroy's projection job runs), less those rows, as Active Record's
165
+ # write reaches no row of a destroyed record; the write then writes,
166
+ # records and counts the others only.
167
+ #
168
+ # The records' aggregate locks are taken before their rows' locks, in
169
+ # Lyra's order (Lyra::AdvisoryLock): the keys the relation matches are
170
+ # read, their aggregate locks taken (all or none), then the rows
171
+ # locked. A row that came to match in between, whose aggregate lock is
172
+ # not held, sends the row locks back (a savepoint rolled back) and the
173
+ # keys are read again, up to ROW_LOCK_TRIES times; after that such a
174
+ # row is left out, as if it had matched after the write. In ES-Lazy,
175
+ # once the aggregate locks are held, each record whose stream has an
176
+ # event the tables have not had yet (one committed after the catch-up
177
+ # the write began with: atomically) is replayed onto its row
178
+ # (LazyProjection.refresh_records), so the write starts from the row
179
+ # as the stream has it: an update committed in between is not lost
180
+ # under it. The whole catch-up is not run again there: its lock is
181
+ # taken before the aggregate locks by every writer that reads first,
182
+ # and taking it after them deadlocked. Then the streams of the rows
183
+ # locked are read, in one query where it can be
184
+ # (Lyra::StreamTail.destroyed_streams). Genesis admits the rows
185
+ # (snapshot) before the streams are read.
186
+ def locked_snapshot(model_class, relation, write: "bulk write")
187
+ return snapshot(model_class, relation, lock: true) unless checks_streams?
188
+
189
+ pk = model_class.primary_key
190
+ locked = Set.new
191
+ ROW_LOCK_TRIES.times do |round|
192
+ keys = Lyra::PurposeBoundReads.internal(INTERNAL) do
193
+ model_class.unscoped.where(pk => relation.reselect(model_class.arel_table[pk])).pluck(pk)
194
+ end
195
+ added = keys.reject { locked.include?(_1) }
196
+ if added.any?
197
+ check_lock_limit!(model_class, locked.size + added.size, write)
198
+ lock_streams!(model_class, added, write: write)
199
+ locked.merge(added)
200
+ # ES-Lazy: their rows as their streams have them now, under the
201
+ # locks, before the rows are read and written.
202
+ Lyra::Projections::LazyProjection.refresh_records(model_class, added)
203
+ end
204
+
205
+ before = nil
206
+ model_class.transaction(requires_new: true) do
207
+ rows = snapshot(model_class, relation, lock: true)
208
+ raise ActiveRecord::Rollback if round < ROW_LOCK_TRIES - 1 && rows.each_key.any? { !locked.include?(_1) }
209
+
210
+ before = rows
211
+ end
212
+ next unless before
213
+
214
+ before = before.select { |id, _| locked.include?(id) }
215
+ dead = destroyed_among(model_class, before.keys)
216
+ return dead.empty? ? before : before.except(*dead)
217
+ end
218
+ end
219
+
220
+ # How many times locked_snapshot reads the keys again for a row that
221
+ # came to match while it took their aggregate locks.
222
+ ROW_LOCK_TRIES = 3
223
+
224
+ # Take the aggregate locks of +ids+' streams, in one statement, after
225
+ # checking their number against config.bulk_write_lock_limit. Returns
226
+ # the stream names, in the order of +ids+.
227
+ def lock_streams!(model_class, ids, write: "bulk write")
228
+ check_lock_limit!(model_class, ids.size, write)
229
+ streams = ids.map { |id| Lyra.stream_name(model_class, id) }
230
+ model_class.with_connection do |connection|
231
+ Lyra::AdvisoryLock.xact_lock_all(connection, streams.map { "lyra_aggregate/#{_1}" }, purpose: "aggregate write")
232
+ end
233
+ streams
234
+ end
235
+
236
+ # Refuse a write that would hold more than config.bulk_write_lock_limit
237
+ # aggregate locks (BulkWriteLockLimitError), before it locks or writes
238
+ # anything.
239
+ def check_lock_limit!(model_class, count, write = "bulk write")
240
+ limit = Lyra.config.bulk_write_lock_limit
241
+ return if limit.nil? || count <= limit
242
+
243
+ raise Lyra::BulkWriteLockLimitError.new(model_class, count, limit, write)
244
+ end
245
+
246
+ # Run an insert_all, insert_all! or upsert_all (the block, given the
247
+ # RETURNING to ask for) with +columns+ in its RETURNING, so the rows it
248
+ # wrote can be told, and hand the caller the result it asked for.
249
+ # Returns [caller_result, rows], rows as hashes of +columns+, each
250
+ # value cast as the result types it (as Result#cast_values does).
251
+ #
252
+ # +returning+ as Active Record takes it: nil (the primary key, where
253
+ # the adapter returns rows), false (nothing), column names, or raw SQL
254
+ # (Arel.sql("code"), Arel.sql("*")). A column the caller did not ask
255
+ # for is added (raw SQL: under an alias of Lyra's own, "lyra_returning_
256
+ # <column>") and taken out of the caller's result again, which keeps
257
+ # its column types (by name and by index, as Active Record's own),
258
+ # its affected rows, and, for returning: false, is Active Record's
259
+ # empty result. Before, it was rebuilt without the column types:
260
+ # result.column_types came back empty and cast_values cast nothing.
261
+ # Rows nil on an adapter with no RETURNING.
262
+ def with_returning(model_class, returning, columns)
263
+ # An adapter with no RETURNING (MySQL): the write as asked, and no
264
+ # rows to tell.
265
+ return [yield(returning), nil] unless model_class.with_connection(&:supports_insert_returning?)
266
+
267
+ returning = default_returning(model_class) if returning.nil?
268
+ quote = ->(name) { model_class.with_connection { |c| c.quote_column_name(name) } }
269
+ names = columns
270
+ case returning
271
+ when false
272
+ request = columns
273
+ when Arel::Nodes::SqlLiteral
274
+ names = columns.map { "#{RETURNING_ALIAS}#{_1}" }
275
+ request = Arel.sql([returning, *columns.zip(names).map { |c, n| "#{quote.(c)} AS #{quote.(n)}" }].join(", "))
276
+ added = columns.size
277
+ else
278
+ asked = Array(returning).map(&:to_s)
279
+ added = (columns - asked).size
280
+ request = added.zero? ? returning : asked + (columns - asked)
281
+ end
282
+
283
+ result = yield(request)
284
+ indexes = names.map { result.columns.index(_1) }
285
+ types = result.column_types
286
+ rows = result.rows.map do |row|
287
+ columns.each_with_index.to_h do |column, i|
288
+ index = indexes[i]
289
+ value = index && row[index]
290
+ [column, index && types[index] ? types[index].deserialize(value) : value]
291
+ end
292
+ end
293
+ [caller_result(result, returning == false ? nil : added.to_i), rows]
294
+ end
295
+
296
+ # The alias a column the caller's raw-SQL RETURNING did not name is
297
+ # returned under (with_returning).
298
+ RETURNING_ALIAS = "lyra_returning_"
299
+
53
300
  # Report a failed publish: raise when the events are required (see
54
301
  # required?), log otherwise.
55
302
  def publish_failed!(error, source)
@@ -58,6 +305,34 @@ module Lyra
58
305
  Rails.logger.error("Lyra: Failed to publish bypass event (#{source}) - #{error.message}")
59
306
  end
60
307
 
308
+ # Whether a touch's event is left out of the log (config.record_touches
309
+ # false, in Monitor: Configuration#skip_touches?): when every column it
310
+ # changed is one of the model's update timestamps (updated_at,
311
+ # updated_on). A touch that changed another column too (a named one,
312
+ # touch(:completed_at), or lock_version) is recorded in full. Only once
313
+ # this process's entry in the registry of processes that leave touches
314
+ # out is live (Lyra::TouchRegistry.live!), so that every process's mode
315
+ # gate sees it; when the entry cannot be written, the touch is recorded.
316
+ def skip_touch?(model_class, changes)
317
+ return false unless Lyra.config.skip_touches?
318
+ return false if changes.empty?
319
+
320
+ stamps = model_class.timestamp_attributes_for_update_in_model
321
+ return false unless changes.each_key.all? { |column| stamps.include?(column.to_s) }
322
+
323
+ Lyra::TouchRegistry.live!
324
+ end
325
+
326
+ # Run a touch_all, so that publish_updates can tell its rows from an
327
+ # update_all's (Rails's touch_all is update_all of the timestamps).
328
+ def touching_all
329
+ previous = Thread.current[:lyra_touch_all]
330
+ Thread.current[:lyra_touch_all] = true
331
+ yield
332
+ ensure
333
+ Thread.current[:lyra_touch_all] = previous
334
+ end
335
+
61
336
  def enabled_for?(model_class)
62
337
  return false if Thread.current[:lyra_projection_write]
63
338
  return false if Lyra.config.mode == :disabled
@@ -66,17 +341,14 @@ module Lyra
66
341
  end
67
342
 
68
343
  def publish(model_class, id, operation, attributes:, changes: {}, source:)
344
+ # Every path admits its rows before writing them; this catches one
345
+ # that does not (the write is in the same transaction, and rolls back
346
+ # with a refusal where events are required).
347
+ admit(model_class, [id]) unless operation.to_sym == :created
69
348
  event_class = event_class_for(model_class, operation)
70
349
  metadata = current_metadata.merge(source_key(source) => source)
71
350
 
72
- data = {
73
- model_class: model_class.name,
74
- model_id: id,
75
- operation: operation,
76
- attributes: attributes,
77
- changes: changes,
78
- timestamp: Time.current
79
- }
351
+ data = Lyra::EventData.build(model_class, id, operation, attributes: attributes, changes: changes)
80
352
  event = event_class.new(data: data, metadata: Lyra::Privacy.stamp(model_class, data, metadata))
81
353
  Lyra.append_events(event, stream_name: Lyra.stream_name(model_class, id))
82
354
 
@@ -85,6 +357,7 @@ module Lyra
85
357
  if Lyra.config.projection_mode == :disabled
86
358
  Lyra::Projections::EventStoreReader.invalidate(model_class, id)
87
359
  end
360
+ event
88
361
  end
89
362
 
90
363
  # Who made the change and what it belongs with, for a write no record's
@@ -94,6 +367,7 @@ module Lyra
94
367
  def current_metadata
95
368
  {
96
369
  user_id: current(:user)&.id,
370
+ user_type: Lyra::Attribution.actor_type(current(:user)),
97
371
  request_id: current(:request_id),
98
372
  correlation_id: Lyra::Correlation.current_id,
99
373
  causation_id: Lyra::Causation.current_id
@@ -111,8 +385,16 @@ module Lyra
111
385
  keys = relation.reselect(model_class.arel_table[pk])
112
386
  scope = model_class.unscoped.where(pk => keys).lock
113
387
  end
114
- rows = Lyra::PurposeBoundReads.internal { scope.to_a }
115
- rows.to_h { |record| [record.public_send(pk), record.attributes] }
388
+ rows = Lyra::PurposeBoundReads.internal(INTERNAL) { scope.to_a }
389
+ rows.to_h { |record| [record_key(model_class, record), record.attributes] }.tap { |by_id| admit(model_class, by_id.keys) }
390
+ end
391
+
392
+ # Before a bypass write records per-row events: rows that predate Lyra
393
+ # are imported first (Lyra::Genesis), as before a write through
394
+ # callbacks, and a row Genesis skipped is imported now or the write is
395
+ # refused, so a bypass event never begins a stream (Genesis.admit_all!).
396
+ def admit(model_class, ids)
397
+ Lyra::Genesis.admit_all!(model_class, ids) unless ids.empty?
116
398
  end
117
399
 
118
400
  # After update_all: compare each snapshotted row with its new state and
@@ -121,13 +403,14 @@ module Lyra
121
403
  return if before.empty?
122
404
 
123
405
  pk = model_class.primary_key
124
- after = Lyra::PurposeBoundReads.internal { model_class.unscoped.where(pk => before.keys).to_a }
125
- .to_h { |record| [record.public_send(pk), record.attributes] }
406
+ after = Lyra::PurposeBoundReads.internal(INTERNAL) { model_class.unscoped.where(pk => before.keys).to_a }
407
+ .to_h { |record| [record_key(model_class, record), record.attributes] }
126
408
 
127
409
  before.each do |id, old_attrs|
128
410
  new_attrs = after[id] or next
129
411
  changes = diff(old_attrs, new_attrs)
130
412
  next if changes.empty?
413
+ next if Thread.current[:lyra_touch_all] && skip_touch?(model_class, changes)
131
414
 
132
415
  publish(model_class, id, :updated,
133
416
  attributes: changes.transform_values(&:last), changes: changes, source: source)
@@ -138,7 +421,7 @@ module Lyra
138
421
  def publish_destroys(model_class, before, source:)
139
422
  before.each do |id, attrs|
140
423
  publish(model_class, id, :destroyed,
141
- attributes: attrs.except(*TIMESTAMP_COLUMNS), source: source)
424
+ attributes: attrs, source: source)
142
425
  end
143
426
  end
144
427
 
@@ -157,48 +440,92 @@ module Lyra
157
440
  return {} if tuples.empty?
158
441
 
159
442
  wanted = tuples.to_set
160
- pk = model_class.primary_key
161
- found = Lyra::PurposeBoundReads.internal { model_class.unscoped.where(keys.first => tuples.map(&:first).uniq).to_a }
443
+ found = Lyra::PurposeBoundReads.internal(INTERNAL) { model_class.unscoped.where(keys.first => tuples.map(&:first).uniq).to_a }
162
444
  found.each_with_object({}) do |record, acc|
163
445
  next unless wanted.include?(keys.map { |key| record.read_attribute(key) })
164
446
 
165
- acc[record.public_send(pk)] = record.attributes
166
- end
447
+ acc[record_key(model_class, record)] = record.attributes
448
+ end.tap { |by_id| admit(model_class, by_id.keys) }
167
449
  end
168
450
 
169
451
  # After insert_all / upsert_all: a "created" event for each written row
170
452
  # that wasn't in the snapshot, an "updated" event (with its changes)
171
- # for each that was and changed.
453
+ # for each that was and changed. The rows are reloaded as Lyra's own
454
+ # read, like the snapshots: before 1.0.0 the reload ran as the
455
+ # application's, so under a purpose that does not allow a declared
456
+ # field it was refused, and Monitor wrote the rows with no events.
457
+ #
458
+ # Returns what it published: { id => [:created or :updated, event id] }.
172
459
  def publish_upserts(model_class, before, ids, source:)
173
- return if ids.blank?
460
+ published = {}
461
+ return published if ids.blank?
174
462
 
175
463
  pk = model_class.primary_key
176
- model_class.unscoped.where(pk => ids).find_each do |record|
177
- id = record.public_send(pk)
178
- if (old_attrs = before[id])
179
- changes = diff(old_attrs, record.attributes)
180
- next if changes.empty?
181
-
182
- publish(model_class, id, :updated,
183
- attributes: changes.transform_values(&:last), changes: changes, source: source)
184
- else
185
- publish(model_class, id, :created,
186
- attributes: record.attributes.except(*TIMESTAMP_COLUMNS), source: source)
464
+ Lyra::PurposeBoundReads.internal(INTERNAL) do
465
+ model_class.unscoped.where(pk => ids).find_each do |record|
466
+ id = record_key(model_class, record)
467
+ if (old_attrs = before[id])
468
+ changes = diff(old_attrs, record.attributes)
469
+ next if changes.empty?
470
+
471
+ event = publish(model_class, id, :updated,
472
+ attributes: changes.transform_values(&:last), changes: changes, source: source)
473
+ published[id] = [:updated, event.event_id]
474
+ else
475
+ event = publish(model_class, id, :created,
476
+ attributes: record.attributes, source: source)
477
+ published[id] = [:created, event.event_id]
478
+ end
187
479
  end
188
480
  end
481
+ published
189
482
  end
190
483
 
191
484
  private
192
485
 
486
+ # The key a record's events name it by: its primary key's value, or,
487
+ # for a primary key of several columns, the array of their values
488
+ # (record.id), which no column's reader gives.
489
+ def record_key(model_class, record)
490
+ pk = model_class.primary_key
491
+ pk.is_a?(Array) ? record.id : record.public_send(pk)
492
+ end
493
+
494
+ # What Active Record returns when returning: is nil: the primary key,
495
+ # where the adapter returns rows; nothing otherwise (InsertAll).
496
+ def default_returning(model_class)
497
+ keys = Array(model_class.primary_key)
498
+ supported = model_class.with_connection(&:supports_insert_returning?)
499
+ supported && keys.any? ? keys : false
500
+ end
501
+
502
+ # The write's result as the caller asked for it: the last +added+
503
+ # columns (those with_returning added) taken out, the column types,
504
+ # by index, and the affected rows kept; for returning: false (+added+
505
+ # nil), Active Record's empty result.
506
+ def caller_result(result, added)
507
+ return ActiveRecord::Result.empty(affected_rows: result.affected_rows) if added.nil?
508
+ return result if added.zero?
509
+
510
+ kept = result.columns.size - added
511
+ types = result.column_types
512
+ ActiveRecord::Result.new(
513
+ result.columns.first(kept),
514
+ result.rows.map { _1.first(kept) },
515
+ types.empty? ? nil : Array.new(kept) { types[_1] },
516
+ affected_rows: result.affected_rows
517
+ )
518
+ end
519
+
193
520
  def current(attribute)
194
521
  defined?(::Current) && ::Current.respond_to?(attribute) ? ::Current.public_send(attribute) : nil
195
522
  end
196
523
 
197
- # Column => [old, new] for every non-timestamp column that changed.
524
+ # Column => [old, new] for every column that changed, created_at and
525
+ # updated_at included: a bare touch_all changes updated_at, and that is a
526
+ # change of the record's state.
198
527
  def diff(old_attrs, new_attrs)
199
528
  new_attrs.each_with_object({}) do |(column, new_value), acc|
200
- next if TIMESTAMP_COLUMNS.include?(column)
201
-
202
529
  old_value = old_attrs[column]
203
530
  acc[column] = [old_value, new_value] unless old_value == new_value
204
531
  end
@@ -226,15 +553,69 @@ module Lyra
226
553
  end
227
554
 
228
555
  # Run a block of Lyra's own read-model writes. Bypasses strict data
229
- # access and publishes no bypass events.
230
- def self.projection_write
556
+ # access and publishes no bypass events. Only Lyra's own call sites may call
557
+ # it (they pass Lyra::INTERNAL): application code that needs the same uses
558
+ # Lyra.unrecorded_write.
559
+ def self.projection_write(token = nil)
560
+ unless token.equal?(INTERNAL)
561
+ raise InternalApiError,
562
+ "Lyra.projection_write is for Lyra's own projections. To write without events " \
563
+ "(seeding, fixtures, a wipe whose history you do not want) use " \
564
+ "Lyra.unrecorded_write(reason: \"...\") { ... }, which records that it happened."
565
+ end
566
+
567
+ # Refused above, before the flags are touched: a refused call leaves the
568
+ # block it was called in as it was.
231
569
  previous_write = Thread.current[:lyra_projection_write]
232
570
  previous_bypass = Thread.current[:lyra_bypass_strict_access]
233
- Thread.current[:lyra_projection_write] = true
234
- Thread.current[:lyra_bypass_strict_access] = true
235
- yield
236
- ensure
237
- Thread.current[:lyra_projection_write] = previous_write
238
- Thread.current[:lyra_bypass_strict_access] = previous_bypass
571
+ begin
572
+ Thread.current[:lyra_projection_write] = true
573
+ Thread.current[:lyra_bypass_strict_access] = true
574
+ yield
575
+ ensure
576
+ Thread.current[:lyra_projection_write] = previous_write
577
+ Thread.current[:lyra_bypass_strict_access] = previous_bypass
578
+ end
579
+ end
580
+
581
+ UNRECORDED_WRITES_STREAM = "Lyra::UnrecordedWrites"
582
+ LIB_ROOT = File.expand_path("..", __dir__)
583
+
584
+ # The way application code writes without events: seeding, fixtures, a table
585
+ # wipe whose history is not wanted. The block's changes are not in the log, and
586
+ # in Monitor lyra:repair reports them; but the write itself is recorded, with
587
+ # its reason, who ran it and where it was called from, in the stream
588
+ # Lyra::UnrecordedWrites, so an unrecorded change leaves a record that it was
589
+ # made. config.unrecorded_writes = :deny refuses it (production hardening).
590
+ # Nothing is recorded in disabled mode, where nothing is.
591
+ def self.unrecorded_write(reason:, &block)
592
+ raise ArgumentError, "unrecorded_write needs a reason: why this write should not be in the log" if reason.to_s.strip.empty?
593
+ raise ArgumentError, "unrecorded_write needs a block" unless block
594
+
595
+ if config.unrecorded_writes == :deny
596
+ raise UnrecordedWriteDenied, "unrecorded writes are refused (config.unrecorded_writes = :deny): #{reason}"
597
+ end
598
+
599
+ record_unrecorded_write(reason) unless config.disabled_mode?
600
+ projection_write(INTERNAL, &block)
601
+ end
602
+
603
+ def self.record_unrecorded_write(reason)
604
+ caller_frame = caller_locations.find { |frame| !frame.absolute_path.to_s.start_with?(LIB_ROOT) }
605
+ event = Lyra::Events::UnrecordedWrite.new(
606
+ data: { reason: reason.to_s, called_from: caller_frame && "#{caller_frame.path}:#{caller_frame.lineno}", at: Time.current },
607
+ metadata: Lyra::BypassEvents.current_metadata.merge(source: "lyra_unrecorded_write")
608
+ )
609
+ append_events(event, stream_name: UNRECORDED_WRITES_STREAM)
610
+ rescue => e
611
+ # As for any event: Hijack and event sourcing fail the write, Monitor logs and goes on.
612
+ raise if config.events_required?
613
+
614
+ Rails.logger.error("Lyra: could not record an unrecorded write (#{reason}) - #{e.message}")
615
+ end
616
+ private_class_method :record_unrecorded_write
617
+
618
+ module Events
619
+ class UnrecordedWrite < Lyra::Event; end
239
620
  end
240
621
  end
@@ -57,7 +57,13 @@ module Lyra
57
57
  # table's PostgreSQL sequence). Elsewhere hijack falls back to the
58
58
  # "pending-<hex>" placeholder, with a one-time warning, rather than risk
59
59
  # a duplicate key.
60
- explicit_id = (command.attributes["id"] || command.attributes[:id]).presence
60
+ #
61
+ # The key is the model's primary key column, whatever its name ("code",
62
+ # a uuid column): the value the application gave it is kept, and the
63
+ # event and the result carry the key under that column, never under an
64
+ # "id" the table may not have.
65
+ key = Lyra.key_column(model_class)
66
+ explicit_id = (command.attributes[key] || command.attributes[key.to_sym]).presence
61
67
  id = explicit_id ||
62
68
  (IdGenerator.next_id(model_class) if Lyra.event_sourcing_mode? || IdGenerator.reserves_safely?(model_class))
63
69
  unless id
@@ -71,7 +77,7 @@ module Lyra
71
77
 
72
78
  # Create event with pre-generated ID
73
79
  # Use symbolize_keys for consistent key types in event data
74
- event_attrs = command.attributes.symbolize_keys.merge(id: id)
80
+ event_attrs = command.attributes.symbolize_keys.merge(key.to_sym => id)
75
81
  events = create_events(:created, id, event_attrs)
76
82
  event = events.first
77
83
 
@@ -89,10 +95,10 @@ module Lyra
89
95
  # Return result with the ID, so the caller inserts the row under the same
90
96
  # ID the event already carries.
91
97
  attributes = command.attributes.dup
92
- # Remove string "id" key to prevent conflict with symbol :id
98
+ # Remove the string key to prevent conflict with the symbol one
93
99
  # (AR attributes have string keys, but we add symbol keys)
94
- attributes.delete("id")
95
- attributes[:id] = id unless id.to_s.start_with?("pending-")
100
+ attributes.delete(key)
101
+ attributes[key.to_sym] = id unless id.to_s.start_with?("pending-")
96
102
  CommandResult.success(attributes: attributes, events: events)
97
103
  end
98
104
 
@@ -164,16 +170,15 @@ module Lyra
164
170
  end
165
171
 
166
172
  def create_event(operation, id, data)
167
- event_data = {
168
- model_class: command.model_class.name,
169
- model_id: id,
170
- operation: operation,
171
- attributes: data[:attributes] || data,
172
- changes: data[:changes] || {},
173
- timestamp: Time.current
174
- }
173
+ event_data = Lyra::EventData.build(
174
+ command.model_class, id, operation,
175
+ # A create's data is the attributes themselves; an update's is { changes: }.
176
+ attributes: operation.to_sym == :updated ? {} : (data[:attributes] || data),
177
+ changes: data[:changes] || {}
178
+ )
175
179
 
176
- event_metadata = attribution_metadata(operation).merge(source: 'lyra_command_handler')
180
+ # source is Lyra's default: a metadata_proc that sets its own wins.
181
+ event_metadata = { source: 'lyra_command_handler' }.merge(attribution_metadata(operation))
177
182
 
178
183
  config = Lyra.config.model_config(command.model_class)
179
184
  event_name = config.event_name_for(operation)