orfeas_lyra 0.9.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +2021 -17
  3. data/README.md +22 -9
  4. data/Rakefile +40 -1
  5. data/app/controllers/lyra/application_controller.rb +127 -0
  6. data/app/controllers/lyra/dashboard_controller.rb +33 -31
  7. data/app/controllers/lyra/flow_controller.rb +32 -34
  8. data/app/controllers/lyra/privacy_controller.rb +57 -18
  9. data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
  10. data/app/views/lyra/dashboard/index.html.erb +1 -0
  11. data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
  12. data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
  13. data/app/views/lyra/privacy/policy.html.erb +7 -0
  14. data/app/views/lyra/privacy/rights.html.erb +78 -0
  15. data/config/privacy_policies.rb +0 -3
  16. data/config/routes.rb +1 -0
  17. data/lib/lyra/access_log.rb +187 -6
  18. data/lib/lyra/advisory_lock.rb +236 -0
  19. data/lib/lyra/aggregate.rb +1 -1
  20. data/lib/lyra/associations/habtm_links.rb +382 -19
  21. data/lib/lyra/associations/maintenance.rb +229 -0
  22. data/lib/lyra/attribution.rb +48 -0
  23. data/lib/lyra/bypass_events.rb +446 -46
  24. data/lib/lyra/command_handler.rb +19 -14
  25. data/lib/lyra/configuration.rb +237 -10
  26. data/lib/lyra/consistency/read_your_writes.rb +4 -2
  27. data/lib/lyra/correlation.rb +121 -0
  28. data/lib/lyra/domain_events.rb +4 -0
  29. data/lib/lyra/dual_view.rb +53 -16
  30. data/lib/lyra/encrypted_attributes.rb +5 -0
  31. data/lib/lyra/engine.rb +67 -0
  32. data/lib/lyra/erasure.rb +1011 -106
  33. data/lib/lyra/event.rb +35 -1
  34. data/lib/lyra/event_data.rb +38 -0
  35. data/lib/lyra/event_flow.rb +31 -15
  36. data/lib/lyra/event_store_adapter.rb +28 -0
  37. data/lib/lyra/event_window.rb +58 -0
  38. data/lib/lyra/genesis.rb +548 -44
  39. data/lib/lyra/interceptors/association_interceptor.rb +689 -34
  40. data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
  41. data/lib/lyra/internal.rb +26 -0
  42. data/lib/lyra/mode_sync.rb +51 -3
  43. data/lib/lyra/mode_transition.rb +257 -30
  44. data/lib/lyra/model_selection.rb +35 -0
  45. data/lib/lyra/own_connection.rb +17 -2
  46. data/lib/lyra/privacy/adapters/pam.rb +25 -2
  47. data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
  48. data/lib/lyra/privacy/interface.rb +27 -7
  49. data/lib/lyra/privacy/subject_links.rb +143 -0
  50. data/lib/lyra/privacy/subject_records.rb +369 -0
  51. data/lib/lyra/projection.rb +7 -1
  52. data/lib/lyra/projections/async_checkpoint.rb +32 -0
  53. data/lib/lyra/projections/async_projection_job.rb +166 -23
  54. data/lib/lyra/projections/cached_joins.rb +33 -18
  55. data/lib/lyra/projections/cached_projection.rb +74 -33
  56. data/lib/lyra/projections/cached_relation.rb +3175 -321
  57. data/lib/lyra/projections/default_scopes.rb +254 -0
  58. data/lib/lyra/projections/deferred_relation.rb +23 -3
  59. data/lib/lyra/projections/event_store_reader.rb +47 -8
  60. data/lib/lyra/projections/foreign_keys.rb +92 -0
  61. data/lib/lyra/projections/lazy_projection.rb +165 -27
  62. data/lib/lyra/projections/model_projection.rb +165 -14
  63. data/lib/lyra/projections/rebuild.rb +34 -6
  64. data/lib/lyra/projections/reference_links.rb +463 -72
  65. data/lib/lyra/projections/typed_values.rb +629 -0
  66. data/lib/lyra/projections/unique_indexes.rb +595 -0
  67. data/lib/lyra/projections/uniqueness.rb +784 -0
  68. data/lib/lyra/projections/value_digest.rb +93 -0
  69. data/lib/lyra/purpose_bound_reads.rb +912 -77
  70. data/lib/lyra/repair.rb +119 -28
  71. data/lib/lyra/retention.rb +114 -8
  72. data/lib/lyra/rights/consent_store.rb +415 -0
  73. data/lib/lyra/rights/erasure_guard.rb +135 -0
  74. data/lib/lyra/rights/pending.rb +74 -0
  75. data/lib/lyra/rights/queries.rb +269 -0
  76. data/lib/lyra/rights/store.rb +512 -0
  77. data/lib/lyra/rights/subject_keys.rb +224 -0
  78. data/lib/lyra/rights.rb +281 -0
  79. data/lib/lyra/saves_without_effect.rb +69 -0
  80. data/lib/lyra/stream_tail.rb +319 -0
  81. data/lib/lyra/strict_data_access.rb +916 -106
  82. data/lib/lyra/temporal.rb +4 -2
  83. data/lib/lyra/touch_registry.rb +404 -0
  84. data/lib/lyra/verification/bypass_workflow.rb +452 -42
  85. data/lib/lyra/verification/rights_workflows.rb +170 -0
  86. data/lib/lyra/version.rb +1 -1
  87. data/lib/lyra/visualization/event_graph.rb +1 -1
  88. data/lib/lyra/visualization/timeline.rb +2 -2
  89. data/lib/lyra.rb +43 -2
  90. data/lib/tasks/lyra_mode.rake +110 -8
  91. data/lib/tasks/lyra_projections.rake +43 -2
  92. metadata +34 -4
  93. data/lib/lyra/associations/event_aware.rb +0 -224
data/lib/lyra/genesis.rb CHANGED
@@ -38,11 +38,130 @@ module Lyra
38
38
  # Enabled with config.genesis: :auto (the default) runs it in event-sourcing
39
39
  # mode, where the event log is the only source of truth; true also runs it in
40
40
  # Monitor and Hijack modes (for DualView and Rebuild); false never runs it.
41
+ #
42
+ # Values the model cannot represent (from 1.0.0). The Imported event holds
43
+ # the row as the model reads it. A stored value the model cannot read --
44
+ # refund_requests.status = '2' under a string enum without that value, which
45
+ # the model reads as nil -- would give an event that disagrees with its row,
46
+ # and the disagreement would surface later and elsewhere (an ES-Lazy
47
+ # catch-up writing NULL into a NOT NULL column). So every attribute of every
48
+ # row is checked first (loss): the value the model reads, written back by
49
+ # the model's type, must be the value stored, compared as the column's own
50
+ # type reads both or as the database would be sent them. '1' and 1, a
51
+ # timestamp and its time zone, a time of day, a decimal's trailing zeros or
52
+ # a jsonb object's key order are no difference; a lossy reading is (an enum
53
+ # value outside the mapping, `attribute :n, :integer` over 10.50, an inet
54
+ # value's host bits, which Rails drops, also in an inet[] array, a value out
55
+ # of the model's integer range). Whatever the type, a stored value the model
56
+ # writes back as NULL is lost, in a nullable column too: a JSON null in a
57
+ # json/jsonb column, an empty range, a blank or 'null' JSON text or an empty
58
+ # YAML hash in a serialized column (read as nil or {}). A serialized,
59
+ # encrypted or JSON attribute's stored text is otherwise one of many
60
+ # encodings of its value (JSON whitespace, a fresh ciphertext), so its text
61
+ # is not compared; a value of another class is. Columns the model ignores
62
+ # are imported as stored and not checked.
63
+ #
64
+ # What happens to such rows is config.genesis_unrepresentable (or the
65
+ # +unrepresentable:+ argument of import and import_all):
66
+ # - :refuse (default): every row is checked before anything is written, and
67
+ # the model's import raises Refused, listing the rows (model, id,
68
+ # attribute, stored value, value read; at most LISTED of them, with the
69
+ # count): nothing of the model is imported. The import runs in a savepoint
70
+ # inside a caller's transaction, so a caller that rescues the refusal
71
+ # commits nothing of it either. Fix the rows or the model, then import
72
+ # again. A refused first use is remembered for REFUSAL_RECHECK seconds
73
+ # from the scan that found it: the uses meanwhile raise it again without
74
+ # scanning; an explicit import (lyra:genesis) always scans. At a gated
75
+ # mode switch the refusal refuses the switch (ModeTransition);
76
+ # - :skip: those rows are left without a stream and the rest is imported;
77
+ # import's Result and `lyra:genesis` list them, and the log warns. A write
78
+ # to a skipped row -- through callbacks (update!, destroy!) or bypassing
79
+ # them (update_columns, touch, update_all, touch_all, delete_all,
80
+ # upsert_all, ...; admit_all!) -- or an ES-NoProj find of it, imports it
81
+ # if the model can now represent it, and is refused (Refused, naming the
82
+ # rows) while it cannot: its stream would otherwise begin with the
83
+ # write's event. A query over many records (where, count) does not see
84
+ # it. So a skipped row is fixed in the database (SQL), not through the
85
+ # model; the next use of it, or lyra:genesis, then imports it.
86
+ # A process imports on its own first use of a model and keeps what it
87
+ # skipped: a row written outside Lyra (SQL) after that first use, or
88
+ # skipped by another process, is unknown to it until lyra:genesis imports
89
+ # it. Values of attributes the model's privacy policy declares, or
90
+ # encrypted ones, are not shown in the listing.
41
91
  module Genesis
42
92
  BATCH = 1_000
43
93
  OPERATION = :imported
94
+ LISTED = 20
95
+
96
+ # One stored value the model cannot represent: +stored+ as the row holds
97
+ # it, +read_as+ as the model reads it (and the Imported event would hold
98
+ # it). +reason+: nil (the value read writes back to another value),
99
+ # :null (it would be written back as NULL),
100
+ # :out_of_range (the model's type cannot hold it), or a String. +personal+:
101
+ # the values are not shown.
102
+ Unrepresentable = Struct.new(:model, :id, :attribute, :stored, :read_as, :personal, :reason, keyword_init: true) do
103
+ def details
104
+ shown = personal ? "#{attribute} (a personal value, not shown)" : "#{attribute} stored #{stored.inspect}"
105
+ read = personal ? "" : ", read as #{read_as.inspect}"
106
+ case reason
107
+ when nil then "#{shown}#{read}"
108
+ when :null then "#{shown}#{read}, which would be written back as NULL"
109
+ when :out_of_range then "#{shown}: out of range for the model's type"
110
+ else "#{shown}: #{reason}"
111
+ end
112
+ end
113
+
114
+ def to_s = "#{model} #{id}: #{details}"
115
+ end
116
+
117
+ # The model holds rows it cannot represent; nothing of it was imported.
118
+ # +rows+: how many rows; +values+: the first LISTED Unrepresentable values
119
+ # of the +found+ in all. +id+: a row an import skipped, refused on a write
120
+ # or read of it (see admit!), or the rows (an Array) a bulk write would
121
+ # have written (admit_all!).
122
+ class Refused < StandardError
123
+ attr_reader :model, :rows, :values, :found, :id
124
+
125
+ def initialize(model, rows:, values:, found:, id: nil)
126
+ @model = model
127
+ @rows = rows
128
+ @values = values
129
+ @found = found
130
+ @id = id
131
+ listing = "#{values.map(&:to_s).join('; ')}#{" (and #{found - values.size} more)" if found > values.size}"
132
+ super(if id
133
+ "Lyra::Genesis left #{model} #{Array(id).join(', ')} without events (genesis_unrepresentable :skip): " \
134
+ "#{id.is_a?(Array) ? 'they hold' : 'it holds'} values the model cannot represent, so a write or read " \
135
+ "would start a stream without its state. Fix the row in the database (SQL) or the model; the next " \
136
+ "use imports it. #{listing}"
137
+ else
138
+ "Lyra::Genesis will not import #{model}: #{rows} row(s) hold values the model cannot represent, so " \
139
+ "their Imported events would not match them. Fix the rows or the model; or skip those rows " \
140
+ "(config.genesis_unrepresentable = :skip, or UNREPRESENTABLE=skip for lyra:genesis). #{listing}"
141
+ end)
142
+ end
143
+ end
144
+
145
+ # What import did: +imported+ rows; +skipped+ rows the model cannot
146
+ # represent (with :skip), and the first LISTED of their values.
147
+ Result = Struct.new(:model, :imported, :skipped, :unrepresentable, keyword_init: true) do
148
+ def summary
149
+ "#{model}: #{imported} rows imported#{", #{skipped} skipped (values the model cannot represent)" if skipped.positive?}"
150
+ end
151
+ end
152
+
153
+ # How an attribute's value goes back to its column (round_trip_type).
154
+ RoundTripType = Struct.new(:column, :writer, :opaque, :enum, :inet)
155
+ private_constant :RoundTripType
156
+
157
+ # A refused first use is not repeated for this long (seconds): the next
158
+ # uses raise the same refusal without scanning the table again. An
159
+ # explicit import (lyra:genesis) always scans.
160
+ REFUSAL_RECHECK = 30
44
161
 
45
162
  @imported = {}
163
+ @skipped = {}
164
+ @refused = {}
46
165
  @mutex = Mutex.new
47
166
 
48
167
  class << self
@@ -55,16 +174,220 @@ module Lyra
55
174
  end
56
175
 
57
176
  # Import +model_class+'s rows without a stream, once per process.
58
- # Returns the number of rows imported by this call.
59
- def first_use(model_class)
177
+ # Returns the number of rows imported by this call. +id+: the row about
178
+ # to be written or read, refused if an import skipped it and it still
179
+ # holds values the model cannot represent (admit!).
180
+ def first_use(model_class, id = nil)
60
181
  return 0 unless enabled?
61
- return 0 if imported?(model_class)
62
182
  return 0 if Thread.current[:lyra_genesis_running]
63
183
 
64
- # Usually every row already has a stream: then nothing is written and
65
- # no lock is needed. The model counts as done once the caller's
66
- # transaction, if any, commits: the streams it sees may be an inline
67
- # import's, uncommitted in that transaction (see below).
184
+ count = 0
185
+ unless imported?(model_class)
186
+ # Raised here, not in import_on_first_use, whose rescue would store
187
+ # it again as new: a busy process would then never scan again.
188
+ if (refusal = recent_refusal(model_class))
189
+ raise Refused.new(refusal.model, rows: refusal.rows, values: refusal.values, found: refusal.found)
190
+ end
191
+
192
+ count = import_on_first_use(model_class)
193
+ end
194
+ admit!(model_class, id) unless id.nil?
195
+ count
196
+ end
197
+
198
+ # Before a write that bypasses callbacks and records an event per row
199
+ # (update_all, update_columns, touch, delete_all, upsert_all, ...): the
200
+ # model's first use, as for a write through callbacks, and then admit!
201
+ # for each of +ids+ the import skipped. Rows the model can now represent
202
+ # are imported first; the write is refused (Refused, naming the rows)
203
+ # if any still cannot be, so a bypass event never begins a stream. Costs
204
+ # nothing more than first_use while no row was skipped.
205
+ def admit_all!(model_class, ids)
206
+ return unless enabled?
207
+ return if Thread.current[:lyra_genesis_running]
208
+
209
+ first_use(model_class)
210
+ return if @skipped.empty?
211
+
212
+ refusals = ids.filter_map do |id|
213
+ next unless skipped?(model_class, id)
214
+
215
+ admit!(model_class, id)
216
+ nil
217
+ rescue Refused => e
218
+ e
219
+ end
220
+ return if refusals.empty?
221
+
222
+ values = refusals.flat_map(&:values)
223
+ raise Refused.new(model_class.name, rows: refusals.size, values: values.first(LISTED),
224
+ found: refusals.sum(&:found),
225
+ id: refusals.one? ? refusals.first.id : refusals.map(&:id))
226
+ end
227
+
228
+ # Import every row of +model_class+ that has no stream, now, whatever the
229
+ # mode. Returns the number of rows imported (see import for the rest).
230
+ def import_all(model_class, unrepresentable: Lyra.config.genesis_unrepresentable)
231
+ import(model_class, unrepresentable: unrepresentable).imported
232
+ end
233
+
234
+ # Import every row of +model_class+ that has no stream, now, whatever the
235
+ # mode, in one transaction (a savepoint inside the caller's). With
236
+ # +unrepresentable+ :refuse every row is checked first, and one holding
237
+ # a value the model cannot represent (see above) raises Refused before
238
+ # anything is written; with :skip such rows are left out, and refused
239
+ # on a later write or read (admit!). Returns a Result.
240
+ def import(model_class, unrepresentable: Lyra.config.genesis_unrepresentable)
241
+ skip = unrepresentable_choice(unrepresentable) == :skip
242
+ running = Thread.current[:lyra_genesis_running]
243
+ Thread.current[:lyra_genesis_running] = true
244
+ imported = []
245
+ skipped = []
246
+ listed = []
247
+ found = 0
248
+
249
+ model_class.with_connection do |connection|
250
+ model_class.transaction(requires_new: true) do
251
+ lock!(connection, model_class)
252
+ types = {}
253
+ unless skip
254
+ bad = 0
255
+ each_row_without_stream(connection, model_class) do |record, row, attributes|
256
+ problems = unrepresentable_values(connection, types, record, row, attributes)
257
+ next if problems.empty?
258
+
259
+ bad += 1
260
+ found += problems.size
261
+ listed.concat(problems.first(LISTED - listed.size))
262
+ end
263
+ raise Refused.new(model_class.name, rows: bad, values: listed, found: found) if bad.positive?
264
+ end
265
+
266
+ each_row_without_stream(connection, model_class) do |record, row, attributes|
267
+ problems = unrepresentable_values(connection, types, record, row, attributes)
268
+ if problems.empty?
269
+ imported << import_record(model_class, record, attributes)
270
+ elsif skip
271
+ skipped << record.id.to_s
272
+ found += problems.size
273
+ listed.concat(problems.first(LISTED - listed.size))
274
+ else
275
+ # Changed since the check above; rolls back what was imported.
276
+ raise Refused.new(model_class.name, rows: 1, values: problems.first(LISTED), found: problems.size)
277
+ end
278
+ end
279
+ end
280
+ end
281
+
282
+ @mutex.synchronize do
283
+ skipped.empty? ? @skipped.delete(owner(model_class)) : @skipped[owner(model_class)] = skipped.to_set
284
+ @refused.delete(owner(model_class))
285
+ end
286
+ forget_cached(model_class, imported)
287
+ warn_unrepresentable(model_class, skipped.size, listed, found) if skipped.any?
288
+ Result.new(model: model_class.name, imported: imported.size, skipped: skipped.size, unrepresentable: listed)
289
+ ensure
290
+ Thread.current[:lyra_genesis_running] = running
291
+ end
292
+
293
+ # Import the one row +id+ of +model_class+ if it still has no stream, now,
294
+ # whatever the mode, under the model's Genesis lock (see hold_lock!).
295
+ # Lyra::Erasure imports a row it erased that had no stream, so that the
296
+ # Imported event holds the erased values, never the personal ones (from
297
+ # 1.0.0). Returns whether it imported the row. A value the model cannot
298
+ # represent is logged, not refused: an erasure must not fail on it.
299
+ def import_one(model_class, id)
300
+ running = Thread.current[:lyra_genesis_running]
301
+ Thread.current[:lyra_genesis_running] = true
302
+ model_class.with_connection do |connection|
303
+ lock!(connection, model_class)
304
+ row = rows_without_stream(connection, model_class, nil, limit: 1, id: id).first
305
+ record, attributes = read_row(model_class, row) if row
306
+ next false if record.nil? || record.id.nil?
307
+
308
+ problems = unrepresentable_values(connection, {}, record, row, attributes)
309
+ warn_unrepresentable(model_class, 0, problems, problems.size) if problems.any?
310
+ forget_cached(model_class, [import_record(model_class, record, attributes)])
311
+ forget_skipped(model_class, id)
312
+ true
313
+ end
314
+ ensure
315
+ Thread.current[:lyra_genesis_running] = running
316
+ end
317
+
318
+ # Before a write to, or a read of, the row +id+: if an import skipped it
319
+ # (:skip), import it now when the model can represent it, or else raise
320
+ # Refused for it. Its stream would otherwise begin with the write's
321
+ # Updated event, no Imported event giving the rest of its state, and an
322
+ # ES-NoProj read would build a partial record from it.
323
+ def admit!(model_class, id)
324
+ return unless skipped?(model_class, id)
325
+
326
+ inline = Lyra::OwnConnection.run(model_class) do |in_caller_transaction|
327
+ import_skipped(model_class, id)
328
+ in_caller_transaction
329
+ end
330
+ forget = -> { forget_skipped(model_class, id) }
331
+ inline ? Lyra::OwnConnection.after_commit(model_class, &forget) : forget.call
332
+ end
333
+
334
+ # Whether an import in this process skipped the row +id+ (:skip).
335
+ def skipped?(model_class, id)
336
+ return false if @skipped.empty? # every write asks: no lock while nothing was skipped
337
+
338
+ @mutex.synchronize { @skipped[owner(model_class)]&.include?(id.to_s) || false }
339
+ end
340
+
341
+ # Take +model_class+'s Genesis lock until the caller's transaction ends: no
342
+ # Genesis run imports one of its rows meanwhile. Lyra::Erasure takes it
343
+ # before erasing a row with no stream, so no import elsewhere reads the row
344
+ # as it was.
345
+ def hold_lock!(model_class)
346
+ model_class.with_connection { |connection| lock!(connection, model_class) }
347
+ end
348
+
349
+ def imported?(model_class)
350
+ @mutex.synchronize { @imported.key?(model_class.name) }
351
+ end
352
+
353
+ # Forget which models were imported, which rows were skipped and which
354
+ # imports were refused (tests, or after the event store or the tables
355
+ # were reset).
356
+ def reset!
357
+ @mutex.synchronize do
358
+ @imported.clear
359
+ @skipped.clear
360
+ @refused.clear
361
+ end
362
+ end
363
+
364
+ # The stored values of +record+, as loaded from its row and not changed
365
+ # since, that its model does not represent (the check above), as
366
+ # Unrepresentable; only those of +columns+ if given. Lyra::Repair checks
367
+ # what it writes into an event this way (from 1.0.0).
368
+ def unrepresentable(record, columns: nil)
369
+ row = record.attributes_before_type_cast
370
+ row = row.slice(*columns.map(&:to_s)) if columns
371
+ attributes = record.attributes
372
+ record.class.with_connection { |connection| unrepresentable_values(connection, {}, record, row, attributes) }
373
+ end
374
+
375
+ # :refuse or :skip; a string (from ENV, say) is taken as its symbol.
376
+ def unrepresentable_choice(value)
377
+ choice = value.is_a?(String) ? value.strip.to_sym : value
378
+ return choice if Lyra::Configuration::GENESIS_UNREPRESENTABLE.include?(choice)
379
+
380
+ raise ArgumentError, "unrepresentable must be :refuse or :skip (got #{value.inspect})"
381
+ end
382
+
383
+ private
384
+
385
+ # The first use's import. Usually every row already has a stream: then
386
+ # nothing is written and no lock is needed. The model counts as done
387
+ # once the caller's transaction, if any, commits: the streams it sees
388
+ # may be an inline import's, uncommitted in that transaction (see
389
+ # below). A refusal is remembered for REFUSAL_RECHECK seconds.
390
+ def import_on_first_use(model_class)
68
391
  if model_class.with_connection { |c| rows_without_stream(c, model_class, nil, limit: 1) }.empty?
69
392
  Lyra::OwnConnection.after_commit(model_class) { mark_imported(model_class) }
70
393
  return 0
@@ -81,55 +404,82 @@ module Lyra
81
404
  mark = -> { mark_imported(model_class) }
82
405
  inline ? Lyra::OwnConnection.after_commit(model_class, &mark) : mark.call
83
406
  count
407
+ rescue Refused => e
408
+ @mutex.synchronize { @refused[owner(model_class)] = [Process.clock_gettime(Process::CLOCK_MONOTONIC), e] } if e.id.nil?
409
+ raise
84
410
  end
85
411
 
86
- # Import every row of +model_class+ that has no stream, now, whatever the
87
- # mode. Returns the number of rows imported.
88
- def import_all(model_class)
89
- Thread.current[:lyra_genesis_running] = true
90
- imported = []
412
+ def recent_refusal(model_class)
413
+ at, refusal = @mutex.synchronize { @refused[owner(model_class)] }
414
+ refusal if at && Process.clock_gettime(Process::CLOCK_MONOTONIC) - at < REFUSAL_RECHECK
415
+ end
91
416
 
417
+ # admit!'s import of one skipped row, under the Genesis lock.
418
+ def import_skipped(model_class, id)
419
+ running = Thread.current[:lyra_genesis_running]
420
+ Thread.current[:lyra_genesis_running] = true
92
421
  model_class.with_connection do |connection|
93
- model_class.transaction do
422
+ model_class.transaction(requires_new: true) do
94
423
  lock!(connection, model_class)
95
- after = nil
96
- loop do
97
- rows = rows_without_stream(connection, model_class, after)
98
- break if rows.empty?
424
+ row = rows_without_stream(connection, model_class, nil, limit: 1, id: id).first
425
+ record, attributes = read_row(model_class, row) if row
426
+ next if record.nil? || record.id.nil? # imported meanwhile, or gone
99
427
 
100
- rows.each { |row| (id = import_row(model_class, row)) && imported << id }
101
- after = rows.last[model_class.primary_key]
428
+ problems = unrepresentable_values(connection, {}, record, row, attributes)
429
+ if problems.any?
430
+ raise Refused.new(model_class.name, rows: 1, values: problems.first(LISTED), found: problems.size, id: id)
102
431
  end
432
+
433
+ forget_cached(model_class, [import_record(model_class, record, attributes)])
103
434
  end
104
435
  end
105
-
106
- forget_cached(model_class, imported)
107
- imported.size
108
436
  ensure
109
- Thread.current[:lyra_genesis_running] = nil
437
+ Thread.current[:lyra_genesis_running] = running
110
438
  end
111
439
 
112
- def imported?(model_class)
113
- @mutex.synchronize { @imported.key?(model_class.name) }
440
+ def forget_skipped(model_class, id)
441
+ @mutex.synchronize do
442
+ rows = @skipped[owner(model_class)]
443
+ rows&.delete(id.to_s)
444
+ @skipped.delete(owner(model_class)) if rows&.empty?
445
+ end
114
446
  end
115
447
 
116
- # Forget which models were imported (tests, or after the event store or
117
- # the tables were reset).
118
- def reset!
119
- @mutex.synchronize { @imported.clear }
120
- end
448
+ # Skipped rows and refusals are kept per table (its stream owner): an STI
449
+ # subclass shares its base class's rows.
450
+ def owner(model_class) = Lyra.stream_owner_name(model_class)
121
451
 
122
- private
452
+ # Each row of +model_class+ without a stream, in batches, as the model
453
+ # reads it: yields the record, the row and the record's attributes.
454
+ def each_row_without_stream(connection, model_class)
455
+ after = nil
456
+ loop do
457
+ rows = rows_without_stream(connection, model_class, after)
458
+ break if rows.empty?
459
+
460
+ rows.each do |row|
461
+ record, attributes = read_row(model_class, row)
462
+ yield record, row, attributes unless record.id.nil?
463
+ end
464
+ after = rows.last[model_class.primary_key]
465
+ end
466
+ end
123
467
 
124
468
  def mark_imported(model_class)
125
469
  @mutex.synchronize { @imported[model_class.name] = true }
126
470
  end
127
471
 
128
- def rows_without_stream(connection, model_class, after, limit: BATCH)
472
+ def rows_without_stream(connection, model_class, after, limit: BATCH, id: nil)
473
+ # A primary key of several columns (a model enrolled before Active
474
+ # Record read its key): Lyra's refusal (ArgumentError), before the
475
+ # query fails on it (PG::UndefinedColumn) and takes the transaction
476
+ # it is in with it.
477
+ Lyra::ModelSelection.refuse_composite_key!(model_class) if model_class.primary_key.is_a?(Array)
129
478
  table = connection.quote_table_name(model_class.table_name)
130
479
  pk = connection.quote_column_name(model_class.primary_key)
131
480
  prefix = connection.quote(Lyra.stream_prefix(model_class))
132
481
  keyset = after.nil? ? "" : "AND t.#{pk} > #{connection.quote(after)}"
482
+ keyset += " AND t.#{pk} = #{connection.quote(id)}" unless id.nil?
133
483
 
134
484
  connection.select_all(<<~SQL.squish).to_a
135
485
  SELECT t.* FROM #{table} t
@@ -141,18 +491,14 @@ module Lyra
141
491
  SQL
142
492
  end
143
493
 
144
- def import_row(model_class, row)
145
- record = Lyra::PurposeBoundReads.internal { model_class.instantiate(row) }
146
- return nil if record.id.nil?
494
+ # The row as the model reads it: the record and its attributes.
495
+ def read_row(model_class, row)
496
+ record = Lyra::PurposeBoundReads.internal(INTERNAL) { model_class.instantiate(row) }
497
+ [record, record.id.nil? ? nil : record.attributes]
498
+ end
147
499
 
148
- data = {
149
- model_class: model_class.name,
150
- model_id: record.id,
151
- operation: OPERATION,
152
- attributes: record.attributes,
153
- changes: {},
154
- timestamp: Time.current
155
- }
500
+ def import_record(model_class, record, attributes)
501
+ data = Lyra::EventData.build(model_class, record.id, OPERATION, attributes: attributes)
156
502
  event = event_class(model_class).new(data: data, metadata: Lyra::Privacy.stamp(model_class, data, { genesis: true }))
157
503
  # The lock and the NOT EXISTS above make the stream empty here.
158
504
  stream_name = Lyra.stream_name(model_class, record.id)
@@ -162,6 +508,160 @@ module Lyra
162
508
  record.id
163
509
  end
164
510
 
511
+ # The values of +row+ that +record+, the model's reading of it, does not
512
+ # represent (see the module comment), as Unrepresentable. +types+ caches
513
+ # each attribute's types for one import.
514
+ def unrepresentable_values(connection, types, record, row, attributes)
515
+ klass = record.class
516
+ row.filter_map do |name, stored|
517
+ next if stored.nil?
518
+
519
+ type = types.fetch([klass, name]) { types[[klass, name]] = round_trip_type(connection, klass, name) }
520
+ next if type.nil?
521
+
522
+ reason = loss(connection, type, stored, attributes[name])
523
+ next if reason.nil?
524
+
525
+ personal = (types[[klass, :personal]] ||= personal_attributes(klass)).include?(name)
526
+ Unrepresentable.new(model: klass.name, id: record.id, attribute: name, stored: stored,
527
+ read_as: attributes[name], personal: personal, reason: (reason unless reason == :value))
528
+ end
529
+ end
530
+
531
+ # How +name+'s value goes back to its column: +column+, the column's own
532
+ # type (no enum, serializer, encryption or time zone); +writer+, the
533
+ # model's type without normalizes (which applies to assignment, not to a
534
+ # value read); +opaque+, a serialized, encrypted or JSON attribute (a
535
+ # stored text of many encodings); +enum+, an enum; +inet+, a PostgreSQL
536
+ # inet column or array of inet. nil for a column the model ignores.
537
+ def round_trip_type(connection, klass, name)
538
+ column = klass.columns_hash[name]
539
+ return nil if column.nil?
540
+
541
+ type = klass.attribute_types[name]
542
+ column_type = if column.respond_to?(:fetch_cast_type)
543
+ column.fetch_cast_type(connection)
544
+ else
545
+ connection.lookup_cast_type_from_column(column)
546
+ end
547
+ writer = type
548
+ writer = writer.cast_type while writer.class.name.to_s.end_with?("NormalizedValueType") && writer.respond_to?(:cast_type)
549
+ chain = type_chain(type)
550
+ opaque = chain.any? do |t|
551
+ t.is_a?(ActiveRecord::Type::Serialized) || t.is_a?(ActiveRecord::Type::Json) ||
552
+ t.class.name == "ActiveRecord::Encryption::EncryptedAttributeType"
553
+ end
554
+ inet = type_chain(column_type).any? { |t| t.class.name.to_s.end_with?("::OID::Inet") }
555
+ RoundTripType.new(column_type, writer, opaque, chain.any? { |t| t.is_a?(ActiveRecord::Enum::EnumType) }, inet)
556
+ end
557
+
558
+ # +type+ and the types it decorates (time zone, enum, serializer,
559
+ # encryption, normalizes, array element).
560
+ def type_chain(type)
561
+ chain = []
562
+ while type && chain.size < 8
563
+ chain << type
564
+ type = if type.respond_to?(:cast_type) then type.cast_type
565
+ elsif type.respond_to?(:subtype) then type.subtype
566
+ elsif type.is_a?(Delegator) then type.__getobj__
567
+ end
568
+ end
569
+ chain
570
+ end
571
+
572
+ # nil if +read_as+ writes back to +stored+ (not NULL: the caller skips
573
+ # NULLs): equal as the column's own type reads them, or as the database
574
+ # would be sent them (a time of day read in another time zone carries
575
+ # another date, which the column does not keep). Otherwise why not (see
576
+ # Unrepresentable#reason; :value: it writes back to another value).
577
+ #
578
+ # Whatever the type, a value the model writes back as NULL is lost: an
579
+ # enum value outside the mapping, a JSON null in a json/jsonb column, an
580
+ # empty range, a blank or 'null' JSON text or an empty YAML hash in a
581
+ # serialized column (read as nil or {}, written back as NULL), into a
582
+ # nullable column too (where(col: nil) would then find the row).
583
+ #
584
+ # A serialized, encrypted or JSON attribute's stored text is one of many
585
+ # encodings of its value (JSON whitespace, a fresh ciphertext), so a
586
+ # re-encoding need not equal it: there it is checked that the value
587
+ # comes back (not NULL, above, and not a value of another class).
588
+ def loss(connection, type, stored, read_as)
589
+ serialized = begin
590
+ type.writer.serialize(read_as)
591
+ rescue ActiveModel::RangeError
592
+ return :out_of_range
593
+ rescue StandardError => e
594
+ return "cannot be written back (#{e.class})"
595
+ end
596
+ return :value if type.enum && read_as.nil?
597
+ return :null if serialized.nil?
598
+
599
+ stored_value = type.column.deserialize(stored)
600
+ if same_written?(connection, type.column, stored_value, serialized)
601
+ return type.inet && host_bits_lost?(stored, stored_value) ? "host bits lost (read as #{stored_value.inspect})" : nil
602
+ end
603
+
604
+ written = written_value(type.column, serialized)
605
+ return :value unless type.opaque
606
+ return :value if !written.nil? && !stored_value.nil? && written.class != stored_value.class
607
+
608
+ nil
609
+ rescue StandardError
610
+ :value
611
+ end
612
+
613
+ # Whether +serialized+, the model's writing of the value it read, is the
614
+ # stored value: equal as the column's own type reads both, or as the
615
+ # database would be sent them.
616
+ def same_written?(connection, column_type, stored_value, serialized)
617
+ written = written_value(column_type, serialized)
618
+ return true if same_value?(stored_value, written)
619
+
620
+ sent = connection.type_cast(column_type.serialize(stored_value))
621
+ sent == connection.type_cast(column_type.serialize(written)) || sent == connection.type_cast(serialized)
622
+ rescue StandardError
623
+ false
624
+ end
625
+
626
+ def written_value(column_type, serialized)
627
+ column_type.deserialize(serialized)
628
+ rescue StandardError
629
+ serialized
630
+ end
631
+
632
+ def same_value?(stored, written)
633
+ stored == written || (stored.respond_to?(:nan?) && written.respond_to?(:nan?) && stored.nan? && written.nan?)
634
+ end
635
+
636
+ # An inet value keeps its host bits (192.168.1.5/24); Rails reads it as
637
+ # an IPAddr of the network (192.168.1.0/24), and writes that back.
638
+ # In an inet[] array, element by element.
639
+ def host_bits_lost?(stored, read)
640
+ return false unless stored.is_a?(String)
641
+ if read.is_a?(Array)
642
+ texts = PG::TextDecoder::Array.new.decode(stored).flatten
643
+ return texts.zip(read.flatten).any? { |text, element| text && host_bits_lost?(text, element) }
644
+ end
645
+ return false unless read.is_a?(IPAddr)
646
+
647
+ IPAddr.new(stored.split("/").first).to_i != read.to_i
648
+ rescue IPAddr::Error, PG::Error
649
+ false
650
+ end
651
+
652
+ # Attributes whose values a listing does not show: those the privacy
653
+ # policy declares, and encrypted ones.
654
+ def personal_attributes(klass)
655
+ Lyra::Privacy.policy_for(klass).declared_fields.map(&:to_s) | Lyra::EncryptedAttributes.names(klass)
656
+ end
657
+
658
+ def warn_unrepresentable(model_class, rows, listed, found)
659
+ what = rows.positive? ? "skipped #{rows} row(s) of #{model_class.name}" : "imported a row of #{model_class.name}"
660
+ more = found > listed.size ? " (and #{found - listed.size} more)" : ""
661
+ Rails.logger&.warn("Lyra::Genesis #{what} holding values the model cannot represent: " \
662
+ "#{listed.map(&:to_s).join('; ')}#{more}")
663
+ end
664
+
165
665
  # An ES-NoProj lookup made before the import (in an earlier process, or
166
666
  # before the event store was reset) may have cached "not found" for these
167
667
  # records. invalidate_all cannot drop per-record entries on every store
@@ -181,8 +681,12 @@ module Lyra
181
681
  Lyra::Events.const_set(name, Class.new(Lyra::Event))
182
682
  end
183
683
 
684
+ # One lock per table, named after its stream owner (the STI base class):
685
+ # an import through a subclass and one through the base class, or an
686
+ # erasure of one of its rows (hold_lock!), wait for each other. Before
687
+ # 1.0.0 it was named after the class given.
184
688
  def lock!(connection, model_class)
185
- Lyra::AdvisoryLock.xact_lock(connection, "lyra_genesis/#{model_class.name}", purpose: "Genesis")
689
+ Lyra::AdvisoryLock.xact_lock(connection, "lyra_genesis/#{Lyra.stream_owner_name(model_class)}", purpose: "Genesis")
186
690
  end
187
691
  end
188
692
  end