orfeas_lyra 0.9.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +2021 -17
  3. data/README.md +22 -9
  4. data/Rakefile +40 -1
  5. data/app/controllers/lyra/application_controller.rb +127 -0
  6. data/app/controllers/lyra/dashboard_controller.rb +33 -31
  7. data/app/controllers/lyra/flow_controller.rb +32 -34
  8. data/app/controllers/lyra/privacy_controller.rb +57 -18
  9. data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
  10. data/app/views/lyra/dashboard/index.html.erb +1 -0
  11. data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
  12. data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
  13. data/app/views/lyra/privacy/policy.html.erb +7 -0
  14. data/app/views/lyra/privacy/rights.html.erb +78 -0
  15. data/config/privacy_policies.rb +0 -3
  16. data/config/routes.rb +1 -0
  17. data/lib/lyra/access_log.rb +187 -6
  18. data/lib/lyra/advisory_lock.rb +236 -0
  19. data/lib/lyra/aggregate.rb +1 -1
  20. data/lib/lyra/associations/habtm_links.rb +382 -19
  21. data/lib/lyra/associations/maintenance.rb +229 -0
  22. data/lib/lyra/attribution.rb +48 -0
  23. data/lib/lyra/bypass_events.rb +446 -46
  24. data/lib/lyra/command_handler.rb +19 -14
  25. data/lib/lyra/configuration.rb +237 -10
  26. data/lib/lyra/consistency/read_your_writes.rb +4 -2
  27. data/lib/lyra/correlation.rb +121 -0
  28. data/lib/lyra/domain_events.rb +4 -0
  29. data/lib/lyra/dual_view.rb +53 -16
  30. data/lib/lyra/encrypted_attributes.rb +5 -0
  31. data/lib/lyra/engine.rb +67 -0
  32. data/lib/lyra/erasure.rb +1011 -106
  33. data/lib/lyra/event.rb +35 -1
  34. data/lib/lyra/event_data.rb +38 -0
  35. data/lib/lyra/event_flow.rb +31 -15
  36. data/lib/lyra/event_store_adapter.rb +28 -0
  37. data/lib/lyra/event_window.rb +58 -0
  38. data/lib/lyra/genesis.rb +548 -44
  39. data/lib/lyra/interceptors/association_interceptor.rb +689 -34
  40. data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
  41. data/lib/lyra/internal.rb +26 -0
  42. data/lib/lyra/mode_sync.rb +51 -3
  43. data/lib/lyra/mode_transition.rb +257 -30
  44. data/lib/lyra/model_selection.rb +35 -0
  45. data/lib/lyra/own_connection.rb +17 -2
  46. data/lib/lyra/privacy/adapters/pam.rb +25 -2
  47. data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
  48. data/lib/lyra/privacy/interface.rb +27 -7
  49. data/lib/lyra/privacy/subject_links.rb +143 -0
  50. data/lib/lyra/privacy/subject_records.rb +369 -0
  51. data/lib/lyra/projection.rb +7 -1
  52. data/lib/lyra/projections/async_checkpoint.rb +32 -0
  53. data/lib/lyra/projections/async_projection_job.rb +166 -23
  54. data/lib/lyra/projections/cached_joins.rb +33 -18
  55. data/lib/lyra/projections/cached_projection.rb +74 -33
  56. data/lib/lyra/projections/cached_relation.rb +3175 -321
  57. data/lib/lyra/projections/default_scopes.rb +254 -0
  58. data/lib/lyra/projections/deferred_relation.rb +23 -3
  59. data/lib/lyra/projections/event_store_reader.rb +47 -8
  60. data/lib/lyra/projections/foreign_keys.rb +92 -0
  61. data/lib/lyra/projections/lazy_projection.rb +165 -27
  62. data/lib/lyra/projections/model_projection.rb +165 -14
  63. data/lib/lyra/projections/rebuild.rb +34 -6
  64. data/lib/lyra/projections/reference_links.rb +463 -72
  65. data/lib/lyra/projections/typed_values.rb +629 -0
  66. data/lib/lyra/projections/unique_indexes.rb +595 -0
  67. data/lib/lyra/projections/uniqueness.rb +784 -0
  68. data/lib/lyra/projections/value_digest.rb +93 -0
  69. data/lib/lyra/purpose_bound_reads.rb +912 -77
  70. data/lib/lyra/repair.rb +119 -28
  71. data/lib/lyra/retention.rb +114 -8
  72. data/lib/lyra/rights/consent_store.rb +415 -0
  73. data/lib/lyra/rights/erasure_guard.rb +135 -0
  74. data/lib/lyra/rights/pending.rb +74 -0
  75. data/lib/lyra/rights/queries.rb +269 -0
  76. data/lib/lyra/rights/store.rb +512 -0
  77. data/lib/lyra/rights/subject_keys.rb +224 -0
  78. data/lib/lyra/rights.rb +281 -0
  79. data/lib/lyra/saves_without_effect.rb +69 -0
  80. data/lib/lyra/stream_tail.rb +319 -0
  81. data/lib/lyra/strict_data_access.rb +916 -106
  82. data/lib/lyra/temporal.rb +4 -2
  83. data/lib/lyra/touch_registry.rb +404 -0
  84. data/lib/lyra/verification/bypass_workflow.rb +452 -42
  85. data/lib/lyra/verification/rights_workflows.rb +170 -0
  86. data/lib/lyra/version.rb +1 -1
  87. data/lib/lyra/visualization/event_graph.rb +1 -1
  88. data/lib/lyra/visualization/timeline.rb +2 -2
  89. data/lib/lyra.rb +43 -2
  90. data/lib/tasks/lyra_mode.rake +110 -8
  91. data/lib/tasks/lyra_projections.rake +43 -2
  92. metadata +34 -4
  93. data/lib/lyra/associations/event_aware.rb +0 -224
data/lib/lyra/temporal.rb CHANGED
@@ -46,7 +46,7 @@ module Lyra
46
46
  return nil if replayed.empty? || Lyra::Event.operation_of(replayed.last) == :destroyed
47
47
 
48
48
  state = Lyra::StateProjection.new.rebuild_from_events(replayed).transform_keys(&:to_s)
49
- state.merge("id" => id.to_s.match?(/\A\d+\z/) ? id.to_i : id)
49
+ state.merge(Lyra.key_column(model_class) => Lyra.key_value(model_class, id))
50
50
  end
51
51
 
52
52
  def event_time(event)
@@ -104,7 +104,9 @@ module Lyra
104
104
  private
105
105
 
106
106
  def build(attributes)
107
- record = model_class.instantiate(attributes.slice(*model_class.column_names))
107
+ record = model_class.instantiate(
108
+ Lyra::Projections::TypedValues.database_attributes(model_class, attributes.slice(*model_class.column_names))
109
+ )
108
110
  record.readonly!
109
111
  record
110
112
  end
@@ -0,0 +1,404 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Lyra
4
+ # The processes that leave touches out of the log (config.record_touches =
5
+ # false), recorded where every process's mode gate reads them: rows of kind
6
+ # 'touch_skipping' in Lyra::ModeTransition::TABLE.
7
+ #
8
+ # Such a process changes rows in a way its events do not show (updated_at
9
+ # moves, the stream does not). That is acceptable in Monitor, where the table
10
+ # is authoritative, and only there: no process may switch the application
11
+ # into Hijack or event sourcing while one runs this way, or use a
12
+ # certificate that a check issued before or during such a period. The
13
+ # registry is how the gate knows:
14
+ #
15
+ # - a process registers when its option is off: at boot, or when the option
16
+ # is set false after boot, and in any case before its first touch is left
17
+ # out (live!). A touch is left out only once the entry is committed; when
18
+ # it cannot be (no connection of its own to be had, the database refuses),
19
+ # the touch is recorded;
20
+ # - every registry write commits on its own, never in the caller's
21
+ # transaction: on this thread's connection when no transaction is open on
22
+ # it, else on a second connection checked out for at most
23
+ # CHECKOUT_TIMEOUT seconds. Each runs under the registry's advisory lock
24
+ # (PostgreSQL), with a short lock_timeout, and re-reads the application's
25
+ # mode under it: a switch into Hijack or event sourcing takes the same lock
26
+ # for its last look at the registry and its record (serialize), so either
27
+ # the switch sees the entry and is refused, or the entry sees the switch,
28
+ # is ended at once, and the touch is recorded (and the process fenced);
29
+ # - it refreshes the entry at most every HEARTBEAT seconds: from the touch
30
+ # path, and from ModeSync's check before requests, jobs and writes. No
31
+ # thread. Times are the database's clock (clock_timestamp() on
32
+ # PostgreSQL), so hosts' clocks do not matter;
33
+ # - it ends the entry when the option is set back to true and at exit, best
34
+ # effort. An entry not refreshed for EXPIRY seconds is no longer live;
35
+ # - each refresh notes the table's latest id (position), and a certificate
36
+ # is taken only if it was issued after every entry's latest refresh or end
37
+ # (ModeTransition.certified_check).
38
+ #
39
+ # With the option true (the default) no write path reads or writes the
40
+ # registry. Each to! or check into Hijack or event sourcing, each ModeSync
41
+ # adoption of such a mode and each Rebuild.rebuild runs one registry SELECT;
42
+ # to! and check also take the advisory lock in a short transaction.
43
+ #
44
+ # The advisory lock is PostgreSQL's: on other adapters AdvisoryLock takes no
45
+ # lock (and warns once, except on SQLite), so the race between a registration and a switch is
46
+ # closed only on PostgreSQL.
47
+ module TouchRegistry
48
+ KIND = "touch_skipping"
49
+ TABLE = ModeTransition::TABLE
50
+ HEARTBEAT = 60
51
+ EXPIRY = 3 * HEARTBEAT
52
+ # How long a registry write waits for a connection of its own while the
53
+ # caller's transaction is open, and for a row lock.
54
+ CHECKOUT_TIMEOUT = 1
55
+ LOCK_TIMEOUT = "2s"
56
+ LOCK_KEY = "lyra:touch_registry"
57
+
58
+ STEPS = "To escalate: restart every process with record_touches on, run `rake lyra:repair` in Monitor, then switch."
59
+ WARNING = "Lyra: record_touches is off — touches that change only update timestamps are not recorded; escalation " \
60
+ "to Hijack or event sourcing is refused while any process runs this way. #{STEPS}"
61
+
62
+ @mutex = Mutex.new
63
+ @beat_mutex = Mutex.new
64
+ @row_id = nil
65
+ @pid = nil
66
+ @beat_at = nil
67
+ @started_at = nil
68
+ @warned_pid = nil
69
+ @exit_hook = false
70
+ @fence = nil
71
+
72
+ class << self
73
+ # At boot (Lyra::Engine), every process: with the option off, refuse to
74
+ # run in an application whose latest applied mode rebuilds rows from the
75
+ # log (it must not leave touches out there), otherwise register.
76
+ def boot!
77
+ register_or_refuse! if Lyra.config.record_touches == false
78
+ end
79
+
80
+ # config.record_touches set to false after boot: as at boot.
81
+ def option_off!
82
+ register_or_refuse! if Lyra.booted?
83
+ end
84
+
85
+ # config.record_touches set back to true: end this process's entry.
86
+ def option_on!
87
+ @fence = nil
88
+ deregister!
89
+ end
90
+
91
+ # Whether +label+ ("hijack", "event_sourcing/lazy") is a mode whose log
92
+ # rebuilds rows.
93
+ def authoritative?(label)
94
+ !label.nil? && !%w[disabled monitor].include?(label.to_s.split("/").first)
95
+ end
96
+
97
+ # Before a touch is left out: true once this process's entry is live.
98
+ # False when it cannot be written, or the application turns out to run in
99
+ # a mode whose log rebuilds rows: the touch is then recorded. Raises once
100
+ # this process has been fenced (fence!).
101
+ def live!
102
+ raise ModeTransition::Refused, @fence if @fence
103
+ return true if fresh?
104
+
105
+ beat!
106
+ end
107
+
108
+ # From ModeSync's check, outside any transaction: refresh the entry when
109
+ # due. Costs nothing with the option on.
110
+ def heartbeat_if_due
111
+ return unless Lyra.config.record_touches == false
112
+ return if @fence || fresh?
113
+
114
+ beat!
115
+ end
116
+
117
+ # The application runs in +label+, which this process cannot run with the
118
+ # option off: it refuses to go on leaving touches out. Every touch it
119
+ # would leave out raises from now on, and with it the write that cascades
120
+ # it (a child's save under belongs_to ... touch: true), until the option
121
+ # is set true or the process restarts with it.
122
+ def fence!(label)
123
+ @fence = process_refusal(label)
124
+ end
125
+
126
+ def fenced? = !@fence.nil?
127
+
128
+ # A switch into +to+'s last look at the registry and its record, under
129
+ # the registry's lock (PostgreSQL), in a transaction: a registration
130
+ # waits for it, and then sees the switch. Other targets run unlocked.
131
+ def serialize(to, &block)
132
+ return yield unless authoritative?(to)
133
+
134
+ ActiveRecord::Base.transaction do
135
+ ActiveRecord::Base.with_connection do |c|
136
+ Lyra::AdvisoryLock.xact_lock(c, LOCK_KEY, purpose: "touch registry")
137
+ end
138
+ block.call
139
+ end
140
+ end
141
+
142
+ # The live entries, each { id:, host:, pid:, since:, last_seen:, version:,
143
+ # actor: }; [] when there are none (or no table).
144
+ def live_processes
145
+ return [] unless ModeTransition.table_available?
146
+
147
+ rows = ActiveRecord::Base.with_connection do |c|
148
+ c.select_all(
149
+ "SELECT id, host, pid, os_user, actor, from_config, started_at, created_at, seen_at FROM #{TABLE} " \
150
+ "WHERE kind = #{c.quote(KIND)} AND ended_at IS NULL AND seen_at > #{expiry_boundary(c)} ORDER BY id"
151
+ ).to_a
152
+ end
153
+ rows.map do |row|
154
+ { id: row["id"].to_i, host: row["host"], pid: row["pid"]&.to_i, version: row["from_config"],
155
+ actor: row["actor"], since: time(row["started_at"] || row["created_at"]), last_seen: time(row["seen_at"]) }
156
+ end
157
+ end
158
+
159
+ # The table's id up to which certificates are not to be taken: the
160
+ # latest refresh or end of any entry. 0 when there is none.
161
+ def certificate_floor_sql
162
+ "COALESCE((SELECT MAX(position) FROM #{TABLE} WHERE kind = '#{KIND}'), 0)"
163
+ end
164
+
165
+ # A gate refusal naming the live entries, or nil when there are none.
166
+ def gate_refusal(from, to)
167
+ live = live_processes
168
+ return nil if live.empty?
169
+
170
+ "Lyra refuses #{from} -> #{to}: #{live.size} process#{'es' if live.size > 1} leave#{'s' if live.size == 1} " \
171
+ "touches out of the log (config.record_touches = false): #{describe(live)}. #{STEPS}"
172
+ end
173
+
174
+ def describe(live)
175
+ live.map do |p|
176
+ "#{p[:host]} pid #{p[:pid]} (since #{p[:since]&.utc&.iso8601}, last seen #{p[:last_seen]&.utc&.iso8601})"
177
+ end.join("; ")
178
+ end
179
+
180
+ # The refusal for a process with the option off in an application whose
181
+ # mode is +label+.
182
+ def process_refusal(label)
183
+ "Lyra refuses to run with config.record_touches = false: the application runs in #{label}, whose log " \
184
+ "rebuilds rows, and a touch left out of it would rebuild a stale updated_at. #{STEPS}"
185
+ end
186
+
187
+ # Testing and maintenance: forget this process's entry (the row stays).
188
+ def reset!
189
+ @mutex.synchronize do
190
+ @row_id = nil
191
+ @pid = nil
192
+ @beat_at = nil
193
+ @fence = nil
194
+ @warned_pid = nil
195
+ end
196
+ end
197
+
198
+ private
199
+
200
+ def fresh?
201
+ @row_id && @pid == Process.pid && (monotonic - @beat_at) < HEARTBEAT
202
+ end
203
+
204
+ # Register, or raise (leaving this process unfenced and unregistered)
205
+ # when the application runs in a mode whose log rebuilds rows.
206
+ def register_or_refuse!
207
+ label = ModeTransition.last_applied
208
+ raise ModeTransition::Refused, process_refusal(label) if authoritative?(label)
209
+
210
+ beat!
211
+ return unless @fence
212
+
213
+ message = @fence
214
+ @fence = nil
215
+ raise ModeTransition::Refused, message
216
+ end
217
+
218
+ # Write or refresh the entry (see the module's notes). Returns whether
219
+ # it is live.
220
+ def beat!
221
+ return false unless ModeTransition.table_available?
222
+
223
+ @beat_mutex.synchronize do
224
+ return true if fresh?
225
+
226
+ outcome = in_registry_transaction do |c|
227
+ id = refresh(c, @row_id) if @row_id && @pid == Process.pid
228
+ id ||= insert(c)
229
+ applied = last_applied(c)
230
+ if authoritative?(applied)
231
+ end_entry(c, id)
232
+ [:authoritative, applied]
233
+ else
234
+ [:live, id]
235
+ end
236
+ end
237
+
238
+ unless outcome
239
+ Rails.logger.warn("Lyra: no connection of its own for the registry of processes that leave touches out; " \
240
+ "this touch is recorded")
241
+ return false
242
+ end
243
+
244
+ status, value = outcome
245
+ if status == :authoritative
246
+ forget
247
+ fence!(value)
248
+ Rails.logger.error(@fence)
249
+ return false
250
+ end
251
+
252
+ remember(value)
253
+ warn_once
254
+ true
255
+ end
256
+ rescue StandardError => e
257
+ Rails.logger.error("Lyra: could not record that this process leaves touches out; it records them instead - " \
258
+ "#{e.class}: #{e.message}")
259
+ false
260
+ end
261
+
262
+ # Run the block with a connection in a transaction of its own, never the
263
+ # caller's: this thread's connection when no transaction is open on it,
264
+ # else a second one, checked out for at most CHECKOUT_TIMEOUT seconds
265
+ # (none when the connection is pinned, as in transactional tests).
266
+ # Under the registry's advisory lock, with LOCK_TIMEOUT. Returns the
267
+ # block's value, or nil when no such connection could be had.
268
+ def in_registry_transaction(&block)
269
+ where = ActiveRecord::Base.with_connection do |c|
270
+ next :here unless c.transaction_open?
271
+
272
+ c.pinned ? :none : :second
273
+ end
274
+ return nil if where == :none
275
+ return ActiveRecord::Base.with_connection { |c| locked_transaction(c, &block) } if where == :here
276
+
277
+ pool = ActiveRecord::Base.connection_pool
278
+ Thread.new do
279
+ Thread.current.report_on_exception = false # its error is raised to the caller by #value
280
+ connection = pool.checkout(CHECKOUT_TIMEOUT)
281
+ begin
282
+ locked_transaction(connection, &block)
283
+ ensure
284
+ pool.checkin(connection)
285
+ end
286
+ end.value
287
+ rescue ActiveRecord::ConnectionTimeoutError
288
+ nil
289
+ end
290
+
291
+ def locked_transaction(connection)
292
+ connection.transaction do
293
+ if postgres?(connection)
294
+ connection.execute("SET LOCAL lock_timeout = '#{LOCK_TIMEOUT}'")
295
+ Lyra::AdvisoryLock.xact_lock(connection, LOCK_KEY, purpose: "touch registry")
296
+ end
297
+ yield connection
298
+ end
299
+ end
300
+
301
+ def remember(id)
302
+ @mutex.synchronize do
303
+ @row_id = id
304
+ @pid = Process.pid
305
+ @beat_at = monotonic
306
+ end
307
+ return if @exit_hook
308
+
309
+ @exit_hook = true
310
+ at_exit { deregister! }
311
+ end
312
+
313
+ def forget
314
+ @mutex.synchronize do
315
+ @row_id = nil
316
+ @beat_at = nil
317
+ end
318
+ end
319
+
320
+ def insert(c)
321
+ @started_at = Time.current if @pid != Process.pid || @started_at.nil?
322
+ who = Lyra::Attribution.of
323
+ now = now_sql(c)
324
+ c.select_value(
325
+ "INSERT INTO #{TABLE} (kind, from_config, to_config, position, checked, created_at, seen_at, started_at, " \
326
+ "actor, os_user, host, pid) SELECT #{c.quote(KIND)}, #{c.quote(Lyra::VERSION)}, 'touches', " \
327
+ "COALESCE(MAX(id), 0), 0, #{now}, #{now}, #{c.quote(@started_at.utc)}, " \
328
+ "#{c.quote(who[:actor])}, #{c.quote(who[:os_user])}, #{c.quote(who[:host])}, #{Integer(who[:pid])} " \
329
+ "FROM #{TABLE} RETURNING id"
330
+ ).to_i
331
+ end
332
+
333
+ # The refreshed entry's id, or nil when it has ended or gone.
334
+ def refresh(c, id)
335
+ c.select_value(
336
+ "UPDATE #{TABLE} SET seen_at = #{now_sql(c)}, position = (SELECT COALESCE(MAX(id), 0) FROM #{TABLE}) " \
337
+ "WHERE id = #{Integer(id)} AND kind = #{c.quote(KIND)} AND ended_at IS NULL RETURNING id"
338
+ )&.to_i
339
+ end
340
+
341
+ def end_entry(c, id)
342
+ now = now_sql(c)
343
+ c.execute("UPDATE #{TABLE} SET ended_at = #{now}, seen_at = #{now}, " \
344
+ "position = (SELECT COALESCE(MAX(id), 0) FROM #{TABLE}) WHERE id = #{Integer(id)}")
345
+ end
346
+
347
+ def last_applied(c)
348
+ c.select_value("SELECT to_config FROM #{TABLE} WHERE kind = 'applied' ORDER BY id DESC LIMIT 1")
349
+ end
350
+
351
+ def deregister!
352
+ id = @mutex.synchronize do
353
+ next nil unless @row_id && @pid == Process.pid
354
+
355
+ found = @row_id
356
+ @row_id = nil
357
+ @beat_at = nil
358
+ found
359
+ end
360
+ return unless id
361
+
362
+ done = in_registry_transaction { |c| end_entry(c, id) || true }
363
+ return if done
364
+
365
+ Rails.logger&.warn("Lyra: no connection to end this process's entry among those that leave touches out; it " \
366
+ "expires in #{EXPIRY}s")
367
+ rescue StandardError => e
368
+ Rails.logger&.warn("Lyra: could not end this process's entry among those that leave touches out; it expires " \
369
+ "in #{EXPIRY}s - #{e.class}: #{e.message}")
370
+ end
371
+
372
+ def warn_once
373
+ return if @warned_pid == Process.pid
374
+
375
+ @warned_pid = Process.pid
376
+ Rails.logger.warn(WARNING)
377
+ end
378
+
379
+ def postgres?(connection) = connection.adapter_name.match?(/postgres/i)
380
+
381
+ # The database's current time, at the statement (not the transaction's
382
+ # start).
383
+ def now_sql(connection)
384
+ postgres?(connection) ? "clock_timestamp()" : "CURRENT_TIMESTAMP"
385
+ end
386
+
387
+ def expiry_boundary(connection)
388
+ if connection.adapter_name.match?(/sqlite/i)
389
+ "datetime('now', '-#{EXPIRY} seconds')"
390
+ else
391
+ "#{now_sql(connection)} - INTERVAL '#{EXPIRY}' SECOND"
392
+ end
393
+ end
394
+
395
+ def time(value)
396
+ return nil if value.nil?
397
+
398
+ value.is_a?(String) ? Time.zone.parse(value) : value
399
+ end
400
+
401
+ def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
402
+ end
403
+ end
404
+ end