orfeas_lyra 0.9.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +2021 -17
  3. data/README.md +22 -9
  4. data/Rakefile +40 -1
  5. data/app/controllers/lyra/application_controller.rb +127 -0
  6. data/app/controllers/lyra/dashboard_controller.rb +33 -31
  7. data/app/controllers/lyra/flow_controller.rb +32 -34
  8. data/app/controllers/lyra/privacy_controller.rb +57 -18
  9. data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
  10. data/app/views/lyra/dashboard/index.html.erb +1 -0
  11. data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
  12. data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
  13. data/app/views/lyra/privacy/policy.html.erb +7 -0
  14. data/app/views/lyra/privacy/rights.html.erb +78 -0
  15. data/config/privacy_policies.rb +0 -3
  16. data/config/routes.rb +1 -0
  17. data/lib/lyra/access_log.rb +187 -6
  18. data/lib/lyra/advisory_lock.rb +236 -0
  19. data/lib/lyra/aggregate.rb +1 -1
  20. data/lib/lyra/associations/habtm_links.rb +382 -19
  21. data/lib/lyra/associations/maintenance.rb +229 -0
  22. data/lib/lyra/attribution.rb +48 -0
  23. data/lib/lyra/bypass_events.rb +446 -46
  24. data/lib/lyra/command_handler.rb +19 -14
  25. data/lib/lyra/configuration.rb +237 -10
  26. data/lib/lyra/consistency/read_your_writes.rb +4 -2
  27. data/lib/lyra/correlation.rb +121 -0
  28. data/lib/lyra/domain_events.rb +4 -0
  29. data/lib/lyra/dual_view.rb +53 -16
  30. data/lib/lyra/encrypted_attributes.rb +5 -0
  31. data/lib/lyra/engine.rb +67 -0
  32. data/lib/lyra/erasure.rb +1011 -106
  33. data/lib/lyra/event.rb +35 -1
  34. data/lib/lyra/event_data.rb +38 -0
  35. data/lib/lyra/event_flow.rb +31 -15
  36. data/lib/lyra/event_store_adapter.rb +28 -0
  37. data/lib/lyra/event_window.rb +58 -0
  38. data/lib/lyra/genesis.rb +548 -44
  39. data/lib/lyra/interceptors/association_interceptor.rb +689 -34
  40. data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
  41. data/lib/lyra/internal.rb +26 -0
  42. data/lib/lyra/mode_sync.rb +51 -3
  43. data/lib/lyra/mode_transition.rb +257 -30
  44. data/lib/lyra/model_selection.rb +35 -0
  45. data/lib/lyra/own_connection.rb +17 -2
  46. data/lib/lyra/privacy/adapters/pam.rb +25 -2
  47. data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
  48. data/lib/lyra/privacy/interface.rb +27 -7
  49. data/lib/lyra/privacy/subject_links.rb +143 -0
  50. data/lib/lyra/privacy/subject_records.rb +369 -0
  51. data/lib/lyra/projection.rb +7 -1
  52. data/lib/lyra/projections/async_checkpoint.rb +32 -0
  53. data/lib/lyra/projections/async_projection_job.rb +166 -23
  54. data/lib/lyra/projections/cached_joins.rb +33 -18
  55. data/lib/lyra/projections/cached_projection.rb +74 -33
  56. data/lib/lyra/projections/cached_relation.rb +3175 -321
  57. data/lib/lyra/projections/default_scopes.rb +254 -0
  58. data/lib/lyra/projections/deferred_relation.rb +23 -3
  59. data/lib/lyra/projections/event_store_reader.rb +47 -8
  60. data/lib/lyra/projections/foreign_keys.rb +92 -0
  61. data/lib/lyra/projections/lazy_projection.rb +165 -27
  62. data/lib/lyra/projections/model_projection.rb +165 -14
  63. data/lib/lyra/projections/rebuild.rb +34 -6
  64. data/lib/lyra/projections/reference_links.rb +463 -72
  65. data/lib/lyra/projections/typed_values.rb +629 -0
  66. data/lib/lyra/projections/unique_indexes.rb +595 -0
  67. data/lib/lyra/projections/uniqueness.rb +784 -0
  68. data/lib/lyra/projections/value_digest.rb +93 -0
  69. data/lib/lyra/purpose_bound_reads.rb +912 -77
  70. data/lib/lyra/repair.rb +119 -28
  71. data/lib/lyra/retention.rb +114 -8
  72. data/lib/lyra/rights/consent_store.rb +415 -0
  73. data/lib/lyra/rights/erasure_guard.rb +135 -0
  74. data/lib/lyra/rights/pending.rb +74 -0
  75. data/lib/lyra/rights/queries.rb +269 -0
  76. data/lib/lyra/rights/store.rb +512 -0
  77. data/lib/lyra/rights/subject_keys.rb +224 -0
  78. data/lib/lyra/rights.rb +281 -0
  79. data/lib/lyra/saves_without_effect.rb +69 -0
  80. data/lib/lyra/stream_tail.rb +319 -0
  81. data/lib/lyra/strict_data_access.rb +916 -106
  82. data/lib/lyra/temporal.rb +4 -2
  83. data/lib/lyra/touch_registry.rb +404 -0
  84. data/lib/lyra/verification/bypass_workflow.rb +452 -42
  85. data/lib/lyra/verification/rights_workflows.rb +170 -0
  86. data/lib/lyra/version.rb +1 -1
  87. data/lib/lyra/visualization/event_graph.rb +1 -1
  88. data/lib/lyra/visualization/timeline.rb +2 -2
  89. data/lib/lyra.rb +43 -2
  90. data/lib/tasks/lyra_mode.rake +110 -8
  91. data/lib/tasks/lyra_projections.rake +43 -2
  92. metadata +34 -4
  93. data/lib/lyra/associations/event_aware.rb +0 -224
@@ -5,15 +5,34 @@ module Lyra
5
5
  # Valid projection modes (event sourcing): :disabled is ES-NoProj, :lazy
6
6
  # is ES-Lazy (project on read).
7
7
  PROJECTION_MODES = [:sync, :async, :disabled, :lazy].freeze
8
+ RIGHTS_REFRESH_MODES = %i[scope interval].freeze
8
9
 
9
10
  attr_reader :mode, :projection_mode
10
- attr_accessor :event_store, :hijack_enabled, :retention_policy
11
+ attr_accessor :event_store, :retention_policy
12
+ attr_reader :hijack_enabled
11
13
  attr_accessor :strict_projections, :projection_error_handler, :async_projections_inline
12
14
  attr_accessor :strict_schema, :schema_path
13
15
  attr_accessor :strict_data_access # Raise on callback-bypassing operations
14
16
  attr_accessor :metadata_proc # Custom metadata proc for events
15
17
  attr_accessor :genesis # Imported events for pre-existing rows (see Lyra::Genesis)
18
+ # What Genesis does with a row holding a value its model cannot represent
19
+ # (an enum value outside the mapping, ...): :refuse (default) refuses the
20
+ # model's import, listing the rows; :skip leaves them without a stream
21
+ # and imports the rest. Lyra::Repair follows it for what it would write.
22
+ # See Lyra::Genesis.
23
+ GENESIS_UNREPRESENTABLE = %i[refuse skip].freeze
24
+ attr_reader :genesis_unrepresentable
16
25
  attr_accessor :reference_links # Foreign-key links for ES-NoProj lookups (see Lyra::Projections::ReferenceLinks)
26
+ # The secret the value streams of ES-NoProj's unique-index check and the
27
+ # value locks are keyed with (Lyra::Projections::ValueDigest): nil
28
+ # (default) derives one from the application's secret_key_base. Changing
29
+ # it relinks the value streams on their next use.
30
+ attr_accessor :value_digest_key
31
+ # Seconds after which a process reads a monitored model's unique indexes
32
+ # again (Lyra::Projections::UniqueIndexes), so an index added by a
33
+ # migration while it runs is enforced and its earlier writes linked:
34
+ # 60 (default), nil never (only reset_column_information does).
35
+ attr_accessor :unique_index_refresh
17
36
  attr_reader :monitored_models
18
37
  # Default privacy policy for monitored models that name none of their
19
38
  # own (monitor_with_lyra privacy_policy: ...). See Lyra::Privacy.policy_for.
@@ -50,9 +69,38 @@ module Lyra
50
69
  # Extra metadata for each recorded access, from a proc given the
51
70
  # PamDsl::Enforcement::Access: who accessed, where Current does not say.
52
71
  attr_accessor :access_metadata_proc
72
+ # Whether Lyra::Erasure.erase! also deletes the subject's access log
73
+ # (Lyra::DataAccess$<Model>$<id>), unless the call says (access_log:).
74
+ # Off by default: the log may be evidence the application must keep.
75
+ attr_accessor :erase_access_log
76
+ # How long, in seconds, a process answers the checks of the data subject's
77
+ # restrictions, objections (Condition 6) and consent from what it last read
78
+ # (Lyra::Rights). A right or consent recorded in this process is seen at
79
+ # once; one recorded in another process within this interval. Default 1.
80
+ attr_accessor :rights_refresh_interval
81
+ # When a process reads the rights and consent afresh (from 1.0.0):
82
+ # :scope (the default) at the first check of each checking scope (a
83
+ # Lyra.with_purpose block, a web request, a job, a batch of a batch loop:
84
+ # Lyra::Rights.scope), and otherwise after rights_refresh_interval;
85
+ # :interval only after rights_refresh_interval, so a right recorded in
86
+ # another process may apply up to that long later, in the middle of a
87
+ # request, and no scope costs a query of its own. Scopes open either way.
88
+ attr_reader :rights_refresh
89
+ # With Lyra::Erasure.erase!(…, everywhere: true): a value held by more
90
+ # other records than this (default 10) is taken to be shared by many people
91
+ # (a placeholder, a city) and left, and reported in Result#shared. The
92
+ # person's linked records (belongs_to the erased record) do not count. The
93
+ # call's max_copies: and the task's MAX_COPIES= override it. From 1.0.0.
94
+ attr_reader :erasure_max_copies
95
+ # Record each explicit Art. 15 copy (GDPRCompliance#data_export, not
96
+ # record: false) as an event in Lyra::SubjectAccessExports$<subject>, which
97
+ # access-log erasure leaves: who exported, when, which sections, never
98
+ # values (Lyra::AccessLog.record_export). On by default (accountability,
99
+ # Art. 5(2), and evidence the request was answered); false turns it off.
100
+ attr_accessor :record_subject_access_exports
53
101
  # What a read of a model with a privacy policy does when no purpose is
54
102
  # declared (Lyra::PurposeBoundReads): :allow (default), :audit or :deny.
55
- attr_reader :reads_without_purpose
103
+ attr_reader :reads_without_purpose, :unrecorded_writes
56
104
  # What Monitor does when a write's event cannot be stored: :log (default)
57
105
  # lets the write stand and logs the failure, the stream falling behind its
58
106
  # row until lyra:repair; :fail_write fails the write and rolls it back, as
@@ -60,14 +108,37 @@ module Lyra
60
108
  # together either way; this decides only what a failed append does.
61
109
  MONITOR_APPEND_FAILURES = %i[log fail_write].freeze
62
110
  attr_reader :monitor_append_failure
63
- # Take a lock on the aggregate before appending its event, in Hijack and
64
- # ES-Sync (assumption L of Theorem 5; Lyra::Interceptors::CrudInterceptor
65
- # #lyra_lock_aggregate!). On (default): two writers of one record append and
66
- # write its row in the same order, at the price of one lock statement per
67
- # update or destroy, which also makes the append nest in a savepoint. Off:
68
- # the order Hijack and ES-Sync had before, with a divergence between row and
69
- # log possible under same-record races, which DualView detects.
111
+ # Take a lock on the aggregate before appending its event, in Hijack,
112
+ # ES-Sync and ES-Async (assumption L of Theorem 5;
113
+ # Lyra::Interceptors::CrudInterceptor#lyra_lock_aggregate!). On (default):
114
+ # in Hijack and ES-Sync the record's row lock, so two writers of one record
115
+ # append and write its row in the same order; in ES-Async, whose writers do
116
+ # not write the row, an advisory lock on the stream, so a record's events
117
+ # commit in the order they were appended and the projection job applies
118
+ # only those after its checkpoint (AsyncProjectionJob). An update or a
119
+ # destroy also checks, under the lock, that the stream does not end in a
120
+ # Destroyed event, taking the advisory lock for that in ES-NoProj and
121
+ # ES-Lazy too (#lyra_lock_aggregate_unless_destroyed!). The price is one
122
+ # lock statement per update or destroy, which also makes the append nest
123
+ # in a savepoint. Off: the order these modes had before, with a divergence
124
+ # between row and log possible under same-record races in Hijack and
125
+ # ES-Sync (DualView detects it), no destroyed check, and ES-Async's job
126
+ # replaying the whole stream every time.
70
127
  attr_accessor :lock_aggregate_before_append
128
+ # The most records a bulk write may lock: in ES-NoProj update_all,
129
+ # delete_all, touch_all, update_counters, a dependent: :delete_all or
130
+ # :nullify, and a destroy_all inside a transaction already open; in
131
+ # ES-Async and ES-Lazy update_all, delete_all, touch_all, upsert_all and
132
+ # a dependent: :nullify (BypassEvents.checks_streams?). It holds one
133
+ # aggregate lock per record until its
134
+ # transaction ends, and PostgreSQL's lock table, shared by every session,
135
+ # holds about max_locks_per_transaction × (max_connections + background
136
+ # processes) locks (about 12,800 with the defaults). Past it every
137
+ # session's writes fail ("out of shared memory"). A larger write raises
138
+ # Lyra::BulkWriteLockLimitError before it locks or writes anything. nil:
139
+ # no limit. Only with lock_aggregate_before_append on. At least 1, so a
140
+ # write of one record is never refused (bulk_write_lock_limit=).
141
+ attr_reader :bulk_write_lock_limit
71
142
  # Apply the privacy policy's retention rules (Lyra::Retention). Off by
72
143
  # default: deleting data on a timer is the deployment's decision.
73
144
  attr_accessor :retention_executor
@@ -82,6 +153,32 @@ module Lyra
82
153
  # config.dashboard_authorization = ->(controller) { controller.current_user&.admin? }
83
154
  attr_accessor :dashboard_authorization
84
155
 
156
+ # The purpose the dashboard reads under when a request names none
157
+ # (?purpose=): one the policy declares for oversight. nil (default): such a
158
+ # request reads without a purpose, as reads_without_purpose decides.
159
+ # config.dashboard_purpose = :oversight
160
+ attr_accessor :dashboard_purpose
161
+ # The most events a dashboard page or JSON route reads from the log in one
162
+ # request (default 10,000); a request's ?limit= may only lower it. The routes
163
+ # that look across the log (the CRUD summary, a correlation, a user's
164
+ # actions, PII detection) read the most recent events up to it and say so
165
+ # (window: { limit:, returned:, truncated: }); before 1.0.0 they read the whole log.
166
+ attr_reader :dashboard_max_events
167
+ # Whether a touch whose only change is the model's update timestamp
168
+ # (updated_at, updated_on) is recorded as an event: an instance touch, a
169
+ # touch_all and a belongs_to ... touch: true cascade. True (the default)
170
+ # records every one, as before. False leaves those out of the log, in
171
+ # Monitor only, where the table is authoritative and the event a copy; a
172
+ # touch that also changes another column (touch(:completed_at), a
173
+ # lock_version) is recorded in full. DualView then ignores a difference
174
+ # confined to those columns. Refused with Hijack and event sourcing, whose
175
+ # log is the record of every change and is replayed to rebuild the rows: a
176
+ # missing touch would rebuild a stale updated_at. A process with it off
177
+ # registers (Lyra::TouchRegistry), and no process escalates while one is
178
+ # registered. Before escalating, restart every process with it true and
179
+ # run lyra:repair, which records the timestamps the log missed. From 1.0.0.
180
+ attr_reader :record_touches
181
+
85
182
  def initialize
86
183
  @mode = :monitor
87
184
  @hijack_enabled = false
@@ -107,9 +204,12 @@ module Lyra
107
204
  # Genesis: :auto (event-sourcing mode only), true (every event-producing
108
205
  # mode) or false. See Lyra::Genesis.
109
206
  @genesis = :auto
207
+ @genesis_unrepresentable = :refuse
110
208
  # Reference links: :auto (ES-NoProj only), true (every event-producing
111
209
  # mode) or false. See Lyra::Projections::ReferenceLinks.
112
210
  @reference_links = :auto
211
+ @value_digest_key = nil
212
+ @unique_index_refresh = 60
113
213
  @mode_transition_gate = nil
114
214
  @mode_transition_certificate_ttl = 3600
115
215
  @dual_view_sample_rate = 0.0
@@ -120,17 +220,29 @@ module Lyra
120
220
  @annotate_privacy = false
121
221
  @record_access_events = false
122
222
  @access_metadata_proc = nil
223
+ @erase_access_log = false
224
+ @rights_refresh_interval = 1
225
+ @rights_refresh = :scope
226
+ @erasure_max_copies = 10 # Lyra::Erasure::MAX_COPIES
227
+ @record_subject_access_exports = true
123
228
  @reads_without_purpose = :allow
229
+ @unrecorded_writes = :allow
124
230
  @monitor_append_failure = :log
125
231
  @lock_aggregate_before_append = true
232
+ @bulk_write_lock_limit = 5_000
126
233
  @retention_executor = false
127
234
  @retention_anchors = {}
128
235
  @dashboard_authorization = nil
236
+ @dashboard_purpose = nil
237
+ @dashboard_max_events = 10_000
238
+ @record_touches = true
129
239
  end
130
240
 
131
241
  # One of MODES; a string (from ENV, say) is taken as its symbol.
132
242
  def mode=(mode)
133
- @mode = validate_choice!(:mode, mode, MODES)
243
+ mode = validate_choice!(:mode, mode, MODES)
244
+ refuse_unrecorded_touches!(mode)
245
+ @mode = mode
134
246
  end
135
247
 
136
248
  # One of PROJECTION_MODES; a string is taken as its symbol.
@@ -138,10 +250,111 @@ module Lyra
138
250
  @projection_mode = validate_choice!(:projection_mode, mode, PROJECTION_MODES)
139
251
  end
140
252
 
253
+ # One of GENESIS_UNREPRESENTABLE; a string is taken as its symbol.
254
+ def genesis_unrepresentable=(choice)
255
+ @genesis_unrepresentable = validate_choice!(:genesis_unrepresentable, choice, GENESIS_UNREPRESENTABLE)
256
+ end
257
+
258
+ def rights_refresh=(mode)
259
+ @rights_refresh = validate_choice!(:rights_refresh, mode, RIGHTS_REFRESH_MODES)
260
+ end
261
+
262
+ def erasure_max_copies=(value)
263
+ count = value.is_a?(Integer) ? value : Integer(value.to_s, 10, exception: false)
264
+ raise ArgumentError, "erasure_max_copies must be a positive integer (got #{value.inspect})" unless count&.positive?
265
+
266
+ @erasure_max_copies = count
267
+ end
268
+
269
+ def dashboard_max_events=(value)
270
+ value = Integer(value)
271
+ raise ArgumentError, "dashboard_max_events must be positive" unless value.positive?
272
+
273
+ @dashboard_max_events = value
274
+ end
275
+
141
276
  def reads_without_purpose=(mode)
142
277
  @reads_without_purpose = Lyra::PurposeBoundReads.validate_mode!(mode)
143
278
  end
144
279
 
280
+ # Whether application code may write without events through
281
+ # Lyra.unrecorded_write: :allow (default; each use is recorded with its
282
+ # reason) or :deny (it raises: production hardening).
283
+ def unrecorded_writes=(mode)
284
+ mode = mode.to_sym
285
+ raise ArgumentError, "unrecorded_writes must be :allow or :deny, not #{mode.inspect}" unless %i[allow deny].include?(mode)
286
+
287
+ @unrecorded_writes = mode
288
+ end
289
+
290
+ # true or false (a string "true" or "false", from ENV, is taken as its
291
+ # value). False is refused outside Monitor and Disabled (touch_refusal).
292
+ def record_touches=(value)
293
+ value = { "true" => true, "false" => false }.fetch(value.strip.downcase, value) if value.is_a?(String)
294
+ raise ArgumentError, "record_touches must be true or false (got #{value.inspect})" unless [true, false].include?(value)
295
+
296
+ previous = @record_touches
297
+ @record_touches = value
298
+ refusal = touch_refusal(hijack_mode? ? :hijack : @mode)
299
+ if refusal
300
+ @record_touches = previous
301
+ raise ArgumentError, refusal
302
+ end
303
+ # This process registers as one that leaves touches out (after boot;
304
+ # at boot, Lyra::TouchRegistry.boot!), which is refused in an application
305
+ # whose latest applied mode is Hijack or event sourcing, and ends its
306
+ # entry when set back to true.
307
+ return unless defined?(Lyra::TouchRegistry) && value != previous
308
+ return Lyra::TouchRegistry.option_on! if value
309
+
310
+ begin
311
+ Lyra::TouchRegistry.option_off!
312
+ rescue StandardError
313
+ @record_touches = previous
314
+ raise
315
+ end
316
+ end
317
+
318
+ # Set by the mode helpers. Refused, like mode=, while record_touches is
319
+ # false: hijack_mode? is true whatever mode says.
320
+ def hijack_enabled=(value)
321
+ refuse_unrecorded_touches!(:hijack) if value
322
+ @hijack_enabled = value
323
+ end
324
+
325
+ # Why +mode+ (a mode, or a ModeTransition label such as
326
+ # "event_sourcing/lazy") cannot run with record_touches false, or nil when
327
+ # it can. Hijack and the event-sourcing modes take the log as the record of
328
+ # every change: the rows are rebuilt from it (a switch into event
329
+ # sourcing, which Hijack passes without a check, a Rebuild, ES-NoProj's
330
+ # reads). An unrecorded touch there would rebuild a stale updated_at.
331
+ def touch_refusal(mode)
332
+ return nil unless @record_touches == false
333
+ return nil if %w[disabled monitor].include?(mode.to_s.split("/").first)
334
+
335
+ "config.record_touches = false is refused in #{mode}: there the event log is the record of every change, " \
336
+ "and a touch left out of it would rebuild a stale updated_at. To escalate: restart every process with " \
337
+ "record_touches on, run `rake lyra:repair` in Monitor, then switch."
338
+ end
339
+
340
+ # Whether a touch whose only change is the update timestamp is left out
341
+ # of the log now: record_touches false, in Monitor.
342
+ def skip_touches?
343
+ @record_touches == false && monitor_mode? && !hijack_mode?
344
+ end
345
+
346
+ # nil (no limit) or a whole number of at least 1; a string (from ENV,
347
+ # say) is taken as its number. 0 refused every write that locks a
348
+ # record, update_columns, touch and delete included.
349
+ def bulk_write_lock_limit=(limit)
350
+ limit = Integer(limit, exception: false) || limit if limit.is_a?(String)
351
+ unless limit.nil? || (limit.is_a?(Integer) && limit >= 1)
352
+ raise ArgumentError, "bulk_write_lock_limit must be nil (no limit) or a whole number of at least 1, got #{limit.inspect}"
353
+ end
354
+
355
+ @bulk_write_lock_limit = limit
356
+ end
357
+
145
358
  # One of MONITOR_APPEND_FAILURES; a string (from ENV, say) is taken as its symbol.
146
359
  def monitor_append_failure=(policy)
147
360
  policy = policy.to_sym if policy.is_a?(String)
@@ -217,6 +430,9 @@ module Lyra
217
430
  @monitored_models.map! { |m| (m.name rescue m.object_id.to_s) == model_name ? model_class : m }
218
431
  end
219
432
  @model_configs[model_class] = ModelConfiguration.new(model_class, options)
433
+ # The join tables of the monitored models' has_and_belongs_to_many may
434
+ # have changed (a new model, or a class reloaded).
435
+ Lyra::Associations::HabtmLinks.reset_join_models! if defined?(Lyra::Associations::HabtmLinks)
220
436
  end
221
437
 
222
438
  # The monitored model an event's model_class names: a monitored model, or
@@ -272,6 +488,7 @@ module Lyra
272
488
  def enable_hijack!
273
489
  return Lyra::ModeTransition.to!(:hijack) if gated_switch?
274
490
 
491
+ refuse_unrecorded_touches!(:hijack)
275
492
  @hijack_enabled = true
276
493
  @mode = :hijack
277
494
  end
@@ -288,6 +505,7 @@ module Lyra
288
505
  def enable_event_sourcing!
289
506
  return Lyra::ModeTransition.to!(:event_sourcing) if gated_switch?
290
507
 
508
+ refuse_unrecorded_touches!(:event_sourcing)
291
509
  @mode = :event_sourcing
292
510
  @hijack_enabled = false
293
511
  end
@@ -300,6 +518,11 @@ module Lyra
300
518
 
301
519
  private
302
520
 
521
+ def refuse_unrecorded_touches!(mode)
522
+ refusal = touch_refusal(mode)
523
+ raise ArgumentError, refusal if refusal
524
+ end
525
+
303
526
  def validate_choice!(name, value, valid)
304
527
  choice = value.is_a?(String) ? value.strip.to_sym : value
305
528
  return choice if valid.include?(choice)
@@ -320,6 +543,10 @@ module Lyra
320
543
  @event_prefix = options[:event_prefix] || model_class.name
321
544
  @aggregate_class = options[:aggregate_class]
322
545
  @custom_event_mapping = options[:event_mapping] || {}
546
+ # Names Lyra keeps for its events that are never replayed (Lyra::StreamTail).
547
+ @custom_event_mapping.each_value do |name|
548
+ raise ArgumentError, "event_mapping: #{name} is reserved for Lyra's own events" if Lyra::StreamTail.reserved_name?(name)
549
+ end
323
550
  @privacy_policy = options[:privacy_policy]
324
551
  @domain_events = Lyra::DomainEvents.rules(options[:domain_events])
325
552
  end
@@ -9,7 +9,9 @@ module Lyra
9
9
  # projects the writes made inside a block itself: by default when the
10
10
  # block ends, so reads after the block see them (reads inside it do not);
11
11
  # with each_write: true as each write is made, so reads later in the same
12
- # block see them too.
12
+ # block see them too. Under ES-Sync a write's row is written with it,
13
+ # inside a block as outside one (CrudInterceptor#lyra_run_sync_projection),
14
+ # so the block has nothing to project there.
13
15
  #
14
16
  # Usage:
15
17
  # Lyra::Consistency::ReadYourWrites.with_guaranteed_read do
@@ -100,7 +102,7 @@ module Lyra
100
102
  model_class = write[:model_class]
101
103
  model_class.transaction do
102
104
  lock_stream(model_class, write[:result])
103
- Lyra::Projections::ModelProjection.project(model_class, write[:operation], write[:result])
105
+ Lyra::Projections::ModelProjection.project(model_class, write[:operation], write[:result], INTERNAL)
104
106
  end
105
107
  rescue => e
106
108
  Rails.logger.error("Lyra: Failed to ensure projection - #{e.message}")
@@ -124,12 +124,115 @@ module Lyra
124
124
  # └── InventoryReserved (causation_id: PaymentProcessed.id)
125
125
  #
126
126
  class Causation
127
+ # The cause a write scope (CrudInterceptor#lyra_write_scope) puts in scope
128
+ # for the writes made during the write (its callbacks, after_save and
129
+ # after_destroy included), decided when one of them needs it:
130
+ #
131
+ # - the write's own event id, once that event is in the log (appended);
132
+ # - the cause in scope when the write began, once the write is known to
133
+ # record no event (decided without an event: a save that changes
134
+ # nothing, a destroy or update that reached no row), or when its event
135
+ # did not reach the log after all (lost: Monitor's logged append
136
+ # failure, a Hijack row write that failed and took the event with it);
137
+ # - before either is known (the write's before_* callbacks: a belongs_to
138
+ # autosave, a dependent: :destroy; or a refused command), the write's
139
+ # own id as well, which it then owes: if the write ends without its
140
+ # event in the log, it appends a SaveWithoutEffect marker with that id
141
+ # (Lyra::SavesWithoutEffect).
142
+ #
143
+ # So the causation_id of every event a write scope gives names an event
144
+ # in the log once the transaction commits. The own id used to be the
145
+ # cause throughout, and from 1.0.0, when a save that changes nothing
146
+ # stopped recording an event, the writes its callbacks made (Solidus's
147
+ # Order#save updating its line items) named an event that never existed.
148
+ class WriteScope
149
+ attr_reader :event_id, :outer
150
+
151
+ def initialize(outer)
152
+ @outer = outer
153
+ @event_id = SecureRandom.uuid
154
+ @taken = false
155
+ @appended = false
156
+ @lost = false
157
+ @decided = false
158
+ @owed = false
159
+ end
160
+
161
+ # The write's own event id, for its own event: given once, so no two
162
+ # events can share it; nil once taken (the store then assigns one).
163
+ def take_event_id
164
+ return nil if @taken
165
+
166
+ @taken = true
167
+ @event_id
168
+ end
169
+
170
+ # The write's own event is in the log (in its transaction).
171
+ def appended!
172
+ @appended = true if @taken
173
+ end
174
+
175
+ # The write's own event is not in the log after all, and the write
176
+ # may stand: +reason+ is the marker's, if one is owed.
177
+ def lost!(reason = "append failed")
178
+ @lost = reason
179
+ end
180
+
181
+ # The write has reached its after_* callbacks: it has built its event,
182
+ # or it records none.
183
+ def decided!
184
+ @decided = true
185
+ end
186
+
187
+ def recorded? = @appended && !@lost
188
+
189
+ def lost? = !!@lost
190
+
191
+ def lost_reason = @lost || nil
192
+
193
+ def decided? = @decided
194
+
195
+ # Whether a cause handed out before the write's event was known still
196
+ # names no event: the marker is owed.
197
+ def owes_marker? = @owed && !recorded?
198
+
199
+ def cause_id
200
+ return @event_id if recorded?
201
+ return Causation.resolve(@outer) if @lost || (@decided && !@taken)
202
+
203
+ @owed = true
204
+ @event_id
205
+ end
206
+ end
207
+
127
208
  class << self
128
209
  # Current causation ID (the event that caused the current operation)
129
210
  def current_id
211
+ resolve(Thread.current[:lyra_causation_id])
212
+ end
213
+
214
+ # The cause in scope as set, a write scope unresolved: a scope opened
215
+ # inside another keeps it as its outer cause, and resolves it when its
216
+ # own event, or a write inside it, needs it.
217
+ def current
130
218
  Thread.current[:lyra_causation_id]
131
219
  end
132
220
 
221
+ def resolve(cause)
222
+ cause.is_a?(WriteScope) ? cause.cause_id : cause
223
+ end
224
+
225
+ # The cause in scope outside every write scope of this thread: one set
226
+ # by the application or carried by a job (Correlation::Job), or nil.
227
+ # For an entry that commits on a connection of its own
228
+ # (Lyra::AccessLog): a write in scope may still roll back, and an entry
229
+ # naming it would then name no event.
230
+ def outside_write_scopes
231
+ cause = current
232
+ cause = cause.outer while cause.is_a?(WriteScope)
233
+ cause
234
+ end
235
+
133
236
  # Set causation ID for current context (when processing/handling an event)
134
237
  def with_id(causation_id)
135
238
  previous_id = Thread.current[:lyra_causation_id]
@@ -169,6 +272,24 @@ module Lyra
169
272
  causation_store[event_id]
170
273
  end
171
274
 
275
+ # Whether every causation_id in +event_store+ names an event there: the
276
+ # number of events read and of those with a cause, and the events whose
277
+ # cause is not in the log (dangling: event_id, event_type,
278
+ # causation_id), none when the causal links are whole. A check for
279
+ # tests and replays (the Olist sweep); it reads the whole store.
280
+ def integrity(event_store = Lyra.config.event_store)
281
+ ids = Set.new
282
+ caused = []
283
+ event_store.read.in_batches(1000).each_batch do |batch|
284
+ batch.each do |event|
285
+ ids << event.event_id
286
+ cause = event.metadata[:causation_id]
287
+ caused << { event_id: event.event_id, event_type: event.event_type, causation_id: cause } if cause
288
+ end
289
+ end
290
+ { events: ids.size, caused: caused.size, dangling: caused.reject { ids.include?(_1[:causation_id]) } }
291
+ end
292
+
172
293
  private
173
294
 
174
295
  def causation_store
@@ -75,6 +75,10 @@ module Lyra
75
75
  if klass && !(klass.is_a?(Class) && klass <= RubyEventStore::Event)
76
76
  raise ArgumentError, "class: for #{name} must be a RubyEventStore::Event subclass"
77
77
  end
78
+ # Names Lyra keeps for its events that are never replayed (Lyra::StreamTail).
79
+ if klass ? Lyra::StreamTail::NEVER_REPLAYED.include?(klass.name) : Lyra::StreamTail.reserved_name?(name)
80
+ raise ArgumentError, "#{name} is reserved for Lyra's own events"
81
+ end
78
82
 
79
83
  operations = Array(definition.fetch(:on) { %i[create update destroy] }).map do |op|
80
84
  OPERATIONS.fetch(op.to_sym) { raise ArgumentError, "on: #{op.inspect} for #{name} is not create, update or destroy" }
@@ -25,19 +25,20 @@ module Lyra
25
25
  }
26
26
  end
27
27
 
28
- # CRUD view - current database state
28
+ # CRUD view - current database state. Read as the caller reads: under
29
+ # reads_without_purpose :deny it needs a purpose that allows the declared
30
+ # fields, as any read does. Lyra's own comparisons (the mode gate, the
31
+ # sampler, repair) read as Lyra. The row is looked up by the model's
32
+ # primary key, whatever its column is called.
29
33
  def crud_state
30
- record = Lyra::PurposeBoundReads.internal { model_class.find_by(id: model_id) }
34
+ record = model_class.find_by(Lyra.key_column(model_class) => model_id)
31
35
 
32
36
  return { exists: false } unless record
33
37
 
34
38
  {
35
39
  exists: true,
36
40
  attributes: record.attributes,
37
- timestamps: {
38
- created_at: record.created_at,
39
- updated_at: record.updated_at
40
- }
41
+ timestamps: timestamps_of(record)
41
42
  }
42
43
  end
43
44
 
@@ -94,12 +95,21 @@ module Lyra
94
95
  end
95
96
 
96
97
  # Calculate differences between CRUD and Event-sourced states
98
+ # Each attribute whose value differs, keyed by its column name as a
99
+ # string (as Active Record keys attributes), => { crud:, event_sourced: };
100
+ # {} when the row and its replayed state agree (both absent counts as
101
+ # agreeing). When only one of them has the record, one entry in the same
102
+ # shape under the symbol :exists, exists: { crud:, event_sourced: } (true or
103
+ # false each), which no column name (a string) can collide with: never
104
+ # empty, so a caller that checks for {} cannot take it for agreement.
105
+ # Before 1.0.0 agreement was { no_differences: true } and a missing record
106
+ # { exists_mismatch: true }.
97
107
  def calculate_differences
98
108
  crud = crud_state
99
109
  es = event_sourced_state
100
110
 
101
- return { exists_mismatch: true } if crud[:exists] != es[:exists]
102
- return { no_differences: true } if !crud[:exists] && !es[:exists]
111
+ return { exists: { crud: crud[:exists], event_sourced: es[:exists] } } if crud[:exists] != es[:exists]
112
+ return {} if !crud[:exists] && !es[:exists]
103
113
 
104
114
  crud_attrs = crud[:attributes] || {}
105
115
  es_attrs = es[:state] || {}
@@ -110,27 +120,33 @@ module Lyra
110
120
  crud_lookup = build_key_lookup(crud_attrs)
111
121
  es_lookup = build_key_lookup(es_attrs)
112
122
 
113
- # Compare union of all keys from both sources, excluding DB-managed timestamps
114
- ignored_keys = %i[created_at updated_at]
115
- all_keys = (crud_lookup.keys + es_lookup.keys).uniq - ignored_keys
123
+ # Compare the union of both sources' keys. created_at and updated_at are
124
+ # compared like any other attribute: every event carries them (1.0.0).
125
+ all_keys = (crud_lookup.keys + es_lookup.keys).uniq
116
126
 
117
127
  all_keys.each do |key|
118
128
  crud_val = crud_lookup[key]
119
129
  es_val = es_lookup[key]
120
130
 
121
131
  unless values_equal?(crud_val, es_val)
122
- differences[key] = {
132
+ differences[key.to_s] = {
123
133
  crud: crud_val,
124
134
  event_sourced: es_val
125
135
  }
126
136
  end
127
137
  end
128
138
 
139
+ # With config.record_touches false (Monitor only), a touch that changed
140
+ # only the update timestamps left no event, so a row whose only
141
+ # difference is in those columns agrees with its events. A difference in
142
+ # any other column is reported with the timestamps beside it.
143
+ differences = {} if !Lyra.config.record_touches && touch_only?(differences)
144
+
129
145
  # has_and_belongs_to_many links: the join table against the link
130
146
  # events (Lyra::Associations::HabtmLinks), by association name.
131
147
  differences.merge!(Associations::HabtmLinks.differences(model_class, model_id))
132
148
 
133
- differences.empty? ? { no_differences: true } : differences
149
+ differences
134
150
  end
135
151
 
136
152
  # Get audit trail from events
@@ -140,6 +156,27 @@ module Lyra
140
156
 
141
157
  private
142
158
 
159
+ # The record's timestamp columns, by name: those the model has
160
+ # (created_at, updated_at, updated_on, ...), none for a model without.
161
+ # Before 1.0.0 created_at and updated_at were read whatever the model had,
162
+ # and a model without one raised NoMethodError.
163
+ def timestamps_of(record)
164
+ klass = record.class
165
+ return {} unless klass.respond_to?(:timestamp_attributes_for_update_in_model)
166
+
167
+ names = klass.timestamp_attributes_for_create_in_model | klass.timestamp_attributes_for_update_in_model
168
+ names.to_h { |name| [name.to_sym, record.read_attribute(name)] }
169
+ end
170
+
171
+ # Whether every difference is in the model's update timestamps
172
+ # (updated_at, updated_on).
173
+ def touch_only?(differences)
174
+ return false if differences.empty? || !model_class.respond_to?(:timestamp_attributes_for_update_in_model)
175
+
176
+ stamps = model_class.timestamp_attributes_for_update_in_model
177
+ differences.each_key.all? { |key| stamps.include?(key.to_s) }
178
+ end
179
+
143
180
  # Build a normalized lookup table from attributes hash
144
181
  # Converts all keys to symbols for consistent access
145
182
  def build_key_lookup(attrs)
@@ -211,7 +248,7 @@ module Lyra
211
248
 
212
249
  def find_discrepancies(model_class)
213
250
  compare_all(model_class).select do |comparison|
214
- comparison[:differences] != { no_differences: true }
251
+ comparison[:differences].any?
215
252
  end
216
253
  end
217
254
  end
@@ -233,11 +270,11 @@ module Lyra
233
270
  comparison = view.compare
234
271
  recommendations = []
235
272
 
236
- if comparison[:differences][:exists_mismatch]
273
+ if comparison[:differences].key?(:exists)
237
274
  recommendations << "State existence mismatch - investigate data consistency"
238
275
  end
239
276
 
240
- if comparison[:differences] != { no_differences: true } && !comparison[:differences][:exists_mismatch]
277
+ if comparison[:differences].any? && !comparison[:differences].key?(:exists)
241
278
  recommendations << "Attribute differences detected - consider which source is authoritative"
242
279
  recommendations << "Differences: #{comparison[:differences].keys.join(', ')}"
243
280
  end