orfeas_lyra 0.9.3 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +1923 -17
  3. data/README.md +15 -8
  4. data/Rakefile +40 -1
  5. data/app/controllers/lyra/application_controller.rb +108 -0
  6. data/app/controllers/lyra/dashboard_controller.rb +17 -15
  7. data/app/controllers/lyra/flow_controller.rb +30 -33
  8. data/app/controllers/lyra/privacy_controller.rb +57 -18
  9. data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
  10. data/app/views/lyra/dashboard/index.html.erb +1 -0
  11. data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
  12. data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
  13. data/app/views/lyra/privacy/policy.html.erb +7 -0
  14. data/app/views/lyra/privacy/rights.html.erb +78 -0
  15. data/config/privacy_policies.rb +0 -3
  16. data/config/routes.rb +1 -0
  17. data/lib/lyra/access_log.rb +187 -6
  18. data/lib/lyra/advisory_lock.rb +236 -0
  19. data/lib/lyra/aggregate.rb +1 -1
  20. data/lib/lyra/associations/habtm_links.rb +382 -19
  21. data/lib/lyra/associations/maintenance.rb +229 -0
  22. data/lib/lyra/attribution.rb +48 -0
  23. data/lib/lyra/bypass_events.rb +426 -45
  24. data/lib/lyra/command_handler.rb +19 -14
  25. data/lib/lyra/configuration.rb +237 -10
  26. data/lib/lyra/consistency/read_your_writes.rb +4 -2
  27. data/lib/lyra/correlation.rb +121 -0
  28. data/lib/lyra/domain_events.rb +4 -0
  29. data/lib/lyra/dual_view.rb +53 -16
  30. data/lib/lyra/encrypted_attributes.rb +5 -0
  31. data/lib/lyra/engine.rb +67 -0
  32. data/lib/lyra/erasure.rb +1002 -116
  33. data/lib/lyra/event.rb +35 -1
  34. data/lib/lyra/event_data.rb +38 -0
  35. data/lib/lyra/event_flow.rb +31 -15
  36. data/lib/lyra/event_store_adapter.rb +28 -0
  37. data/lib/lyra/event_window.rb +58 -0
  38. data/lib/lyra/genesis.rb +548 -44
  39. data/lib/lyra/interceptors/association_interceptor.rb +689 -34
  40. data/lib/lyra/interceptors/crud_interceptor.rb +547 -95
  41. data/lib/lyra/internal.rb +26 -0
  42. data/lib/lyra/mode_sync.rb +51 -3
  43. data/lib/lyra/mode_transition.rb +245 -34
  44. data/lib/lyra/model_selection.rb +35 -0
  45. data/lib/lyra/own_connection.rb +17 -2
  46. data/lib/lyra/privacy/adapters/pam.rb +25 -2
  47. data/lib/lyra/privacy/gdpr_compliance.rb +264 -36
  48. data/lib/lyra/privacy/interface.rb +27 -7
  49. data/lib/lyra/privacy/subject_links.rb +143 -0
  50. data/lib/lyra/privacy/subject_records.rb +369 -0
  51. data/lib/lyra/projection.rb +7 -1
  52. data/lib/lyra/projections/async_checkpoint.rb +32 -0
  53. data/lib/lyra/projections/async_projection_job.rb +166 -23
  54. data/lib/lyra/projections/cached_joins.rb +33 -18
  55. data/lib/lyra/projections/cached_projection.rb +74 -33
  56. data/lib/lyra/projections/cached_relation.rb +3175 -321
  57. data/lib/lyra/projections/default_scopes.rb +254 -0
  58. data/lib/lyra/projections/deferred_relation.rb +23 -3
  59. data/lib/lyra/projections/event_store_reader.rb +47 -8
  60. data/lib/lyra/projections/foreign_keys.rb +92 -0
  61. data/lib/lyra/projections/lazy_projection.rb +165 -27
  62. data/lib/lyra/projections/model_projection.rb +165 -14
  63. data/lib/lyra/projections/rebuild.rb +34 -6
  64. data/lib/lyra/projections/reference_links.rb +463 -72
  65. data/lib/lyra/projections/typed_values.rb +629 -0
  66. data/lib/lyra/projections/unique_indexes.rb +595 -0
  67. data/lib/lyra/projections/uniqueness.rb +784 -0
  68. data/lib/lyra/projections/value_digest.rb +93 -0
  69. data/lib/lyra/purpose_bound_reads.rb +912 -77
  70. data/lib/lyra/repair.rb +119 -28
  71. data/lib/lyra/retention.rb +114 -8
  72. data/lib/lyra/rights/consent_store.rb +415 -0
  73. data/lib/lyra/rights/erasure_guard.rb +135 -0
  74. data/lib/lyra/rights/pending.rb +74 -0
  75. data/lib/lyra/rights/queries.rb +269 -0
  76. data/lib/lyra/rights/store.rb +512 -0
  77. data/lib/lyra/rights/subject_keys.rb +224 -0
  78. data/lib/lyra/rights.rb +281 -0
  79. data/lib/lyra/saves_without_effect.rb +69 -0
  80. data/lib/lyra/stream_tail.rb +319 -0
  81. data/lib/lyra/strict_data_access.rb +906 -107
  82. data/lib/lyra/temporal.rb +4 -2
  83. data/lib/lyra/touch_registry.rb +404 -0
  84. data/lib/lyra/verification/bypass_workflow.rb +452 -42
  85. data/lib/lyra/verification/rights_workflows.rb +170 -0
  86. data/lib/lyra/version.rb +1 -1
  87. data/lib/lyra/visualization/event_graph.rb +1 -1
  88. data/lib/lyra/visualization/timeline.rb +2 -2
  89. data/lib/lyra.rb +43 -2
  90. data/lib/tasks/lyra_mode.rake +107 -8
  91. data/lib/tasks/lyra_projections.rake +43 -2
  92. metadata +34 -4
  93. data/lib/lyra/associations/event_aware.rb +0 -224
@@ -8,6 +8,28 @@ module Lyra
8
8
  # SQLite needs none: it serialises writers. Other adapters have no such lock
9
9
  # here, so the call does nothing there and warns once per purpose, since
10
10
  # those paths are then safe in a single process only.
11
+ #
12
+ # The order of a record's locks. A write that takes both a record's
13
+ # aggregate lock (this advisory lock on "lyra_aggregate/<stream>") and its
14
+ # row's lock takes the aggregate lock first: ES-Async's and ES-Lazy's
15
+ # writes through callbacks take the aggregate lock only (their rows are the
16
+ # projection's), Hijack and ES-Sync take the row's lock only (the row lock
17
+ # is their aggregate lock: CrudInterceptor#lyra_lock_aggregate!), and a
18
+ # write that skips callbacks (update_columns, touch, delete, and in ES-Async
19
+ # update_all and delete_all: Lyra::BypassEvents) and an erasure
20
+ # (Lyra::Erasure#lock_record) take the aggregate lock, then write or lock
21
+ # the row. A write that took them the other way round deadlocked against
22
+ # one of these. Several records' aggregate locks are taken together by
23
+ # xact_lock_all.
24
+ #
25
+ # Lyra orders the locks of one write, not those of an application's
26
+ # transaction: one that writes several records one after another takes
27
+ # their locks in the order it writes them, and can deadlock against
28
+ # another transaction, or a bulk write, that holds one of them and waits
29
+ # for another, as plain Active Record's row locks do. PostgreSQL detects
30
+ # it and aborts one of them (ActiveRecord::Deadlocked). Write records in
31
+ # one order (by primary key, say) and retry the transaction on
32
+ # ActiveRecord::Deadlocked; see docs/TROUBLESHOOTING.md.
11
33
  module AdvisoryLock
12
34
  @warned = {}
13
35
 
@@ -24,6 +46,162 @@ module Lyra
24
46
  false
25
47
  end
26
48
 
49
+ # The locks of every one of +keys+, as xact_lock takes each. Each is
50
+ # tried without waiting, in one statement, in a savepoint; when every
51
+ # one is free, that is all (one statement besides the savepoint's).
52
+ #
53
+ # When some are held by other sessions, what the call does next depends
54
+ # on whether this transaction holds advisory locks of its own already
55
+ # (wait_bound, one query, at the first contention only):
56
+ #
57
+ # - It holds none (a bulk write beginning its transaction's locking):
58
+ # it waits for the held ones, in the order of their hashes, keeping
59
+ # the ones it took, for at most MAX_WAIT_MS in all; a holder that
60
+ # ends meanwhile lets it through, in its place in that lock's queue.
61
+ # When the time is up, the savepoint is rolled back, which gives back
62
+ # every lock taken in it, and the next try first waits, holding none,
63
+ # for the first lock that was held, for at most half of
64
+ # deadlock_timeout, then tries them all again. Before, it gave every
65
+ # lock back whenever one was held and waited for that one without a
66
+ # limit: sessions that come and go with a few of its records kept it
67
+ # out (a bulk write of 8 records against three transactions that each
68
+ # held one and then bulk-wrote two others went through once in 15
69
+ # seconds in ES-NoProj), and, waiting in their locks' queues, it was
70
+ # the one PostgreSQL aborted for their deadlocks (5 or 6 times in
71
+ # those 15 seconds). No wait of the call's lasts as long as
72
+ # deadlock_timeout, so PostgreSQL never reports it deadlocked: its own
73
+ # check runs only after deadlock_timeout, and a session that waits for
74
+ # a lock the call took began waiting after the call took it, so its
75
+ # check finds the call gone. Two bulk writes, or a bulk write and a
76
+ # transaction that holds one of its records and asks for another,
77
+ # therefore never deadlock: the bulk write gives way.
78
+ # - It holds some (a transaction that wrote a record, or a bulk write's
79
+ # later round): whenever one is held, it gives back the ones it took
80
+ # in that try and waits for that one holding none of them, without a
81
+ # limit, as before 1.0.0. It holds its earlier locks anyway, and
82
+ # keeping the new ones as well while it waited made such transactions
83
+ # deadlock against each other.
84
+ #
85
+ # Nothing changes for the caller: a wait that ran out of time is not
86
+ # reported, and the call returns only once it holds every lock (the
87
+ # lock limit check comes before it, and nothing is appended before it
88
+ # returns). A lock_timeout of the caller's own that is shorter than a
89
+ # bounded wait's share stays in force, and its timeout raises
90
+ # ActiveRecord::LockWaitTimeout, as before. Under contention the bulk
91
+ # write's turns cost the transactions it shares records with some
92
+ # throughput: they wait while it holds its records, up to MAX_WAIT_MS
93
+ # at a time. After TRIES tries, or outside a transaction (no savepoint
94
+ # to give locks back with), the locks are taken waiting, in the order
95
+ # of their hashes. Returns true when the locks were taken, false
96
+ # otherwise.
97
+ def xact_lock_all(connection, keys, purpose:)
98
+ keys = keys.uniq
99
+ return false if keys.empty?
100
+ return xact_lock(connection, keys.first, purpose: purpose) if keys.size == 1
101
+
102
+ adapter = connection.adapter_name
103
+ unless adapter.match?(/postgres/i)
104
+ warn_once(adapter, purpose) unless adapter.match?(/sqlite/i)
105
+ return false
106
+ end
107
+
108
+ hashes = "SELECT DISTINCT hashtext(k) AS h FROM unnest(ARRAY[#{keys.map { connection.quote(_1) }.join(', ')}]::text[]) " \
109
+ "AS k ORDER BY h"
110
+ if connection.transaction_open?
111
+ contended = nil
112
+ # The bounds of its waits (wait_bound) when the caller held no
113
+ # advisory lock of its own at the first contention; false when it
114
+ # did; nil before the first.
115
+ bounded = nil
116
+ TRIES.times do
117
+ held = nil
118
+ connection.transaction(requires_new: true) do
119
+ # The lock another session held last time, waited for while
120
+ # this call holds none of the others (it gets its place in
121
+ # that lock's queue; at most half of deadlock_timeout where
122
+ # the waits are bounded), then the others tried.
123
+ if contended
124
+ if bounded
125
+ held = contended unless wait_briefly(connection, [contended], bounded, bounded.first)
126
+ else
127
+ connection.execute("SELECT pg_advisory_xact_lock(#{contended})")
128
+ end
129
+ end
130
+ raise ActiveRecord::Rollback if held
131
+
132
+ # Volatile functions are called on the subquery's rows, in
133
+ # their order, every one of them.
134
+ rows = connection.select_rows("SELECT h, pg_try_advisory_xact_lock(h) FROM (#{hashes}) AS keys")
135
+ missing = rows.reject { |_, taken| taken }.map { |hash, _| Integer(hash) }
136
+ next if missing.empty?
137
+
138
+ bounded = wait_bound(connection, rows.size - missing.size) if bounded.nil?
139
+ next if bounded && wait_briefly(connection, missing, bounded, MAX_WAIT_MS)
140
+
141
+ held = missing.first
142
+ raise ActiveRecord::Rollback
143
+ end
144
+ return true unless held
145
+
146
+ contended = held
147
+ end
148
+ end
149
+ connection.execute("SELECT count(pg_advisory_xact_lock(h)) FROM (#{hashes}) AS keys")
150
+ true
151
+ end
152
+
153
+ # How many times xact_lock_all tries all the locks before it waits for
154
+ # them in order.
155
+ TRIES = 200
156
+
157
+ # The longest a bulk write waits, in milliseconds, holding the locks it
158
+ # took, for the ones it did not (xact_lock_all), in all; never more
159
+ # than half of deadlock_timeout either.
160
+ MAX_WAIT_MS = 100
161
+
162
+ # Take the lock only if no other session holds it. Returns true when
163
+ # the caller may go on as the lock's holder: it took the lock, or there
164
+ # is no such lock on this adapter (as xact_lock, which then takes none).
165
+ # Returns false when another session holds it.
166
+ def try_xact_lock(connection, key, purpose:)
167
+ adapter = connection.adapter_name
168
+ if adapter.match?(/postgres/i)
169
+ return connection.select_value("SELECT pg_try_advisory_xact_lock(hashtext(#{connection.quote(key)}))") == true
170
+ end
171
+
172
+ warn_once(adapter, purpose) unless adapter.match?(/sqlite/i)
173
+ true
174
+ end
175
+
176
+ # Wait until no other session holds the lock, without keeping it: take
177
+ # it shared in a savepoint and roll the savepoint back, which gives the
178
+ # lock back. Shared requests do not exclude each other, so every
179
+ # session waiting on the same holder goes on when that holder's
180
+ # transaction ends. The lock belongs to the savepoint, not to the
181
+ # session, so it cannot outlive the caller's transaction whatever
182
+ # interrupts this call; a session-level lock left behind by an error or
183
+ # a Thread#raise between taking and giving it back would block every
184
+ # later holder for good. Keeping it until the caller's transaction ends
185
+ # would block every holder for as long as that transaction, and two
186
+ # waiters that each kept it shared and then asked for it exclusively
187
+ # would deadlock. The savepoint, the wait and the rollback go to the
188
+ # server as one string, one round trip; if the wait fails, the rest is
189
+ # not run, and the error rolls back the caller's transaction or
190
+ # savepoint, which also releases the lock. Returns false where there is
191
+ # no such lock.
192
+ def wait_for(connection, key)
193
+ return false unless connection.adapter_name.match?(/postgres/i)
194
+
195
+ connection.transaction do
196
+ connection.execute(
197
+ "SAVEPOINT lyra_wait_for; " \
198
+ "SELECT pg_advisory_xact_lock_shared(hashtext(#{connection.quote(key)})); " \
199
+ "ROLLBACK TO SAVEPOINT lyra_wait_for; RELEASE SAVEPOINT lyra_wait_for"
200
+ )
201
+ end
202
+ true
203
+ end
204
+
27
205
  # Forget which purposes have warned (tests).
28
206
  def reset_warnings!
29
207
  @warned = {}
@@ -31,6 +209,64 @@ module Lyra
31
209
 
32
210
  private
33
211
 
212
+ # xact_lock_all, at its first contention, having just taken +taken+
213
+ # of its locks: whether its waits are bounded. Only for a caller that
214
+ # holds no other advisory lock in this session (a bulk write that
215
+ # begins its transaction's locking): [half of deadlock_timeout in ms,
216
+ # the session's lock_timeout in ms (0: none), the lock_timeout setting
217
+ # to restore]. False for one that already holds some (a transaction
218
+ # that wrote a record and now writes others): it waits holding none of
219
+ # the new ones, as before 1.0.0. One query.
220
+ def wait_bound(connection, taken)
221
+ held, half_deadlock, lock, setting = connection.select_rows(
222
+ "SELECT (SELECT count(*) FROM pg_locks WHERE locktype = 'advisory' AND granted AND pid = pg_backend_pid()), " \
223
+ "(EXTRACT(epoch FROM current_setting('deadlock_timeout')::interval) * 500)::bigint, " \
224
+ "(EXTRACT(epoch FROM current_setting('lock_timeout')::interval) * 1000)::bigint, " \
225
+ "current_setting('lock_timeout')"
226
+ ).first
227
+ return false unless Integer(held) == taken
228
+
229
+ [Integer(half_deadlock), Integer(lock), setting]
230
+ end
231
+
232
+ # xact_lock_all, for a caller whose waits are bounded (wait_bound):
233
+ # wait for the locks of +hashes+, in their order, in the caller's
234
+ # savepoint, for at most +total+ milliseconds in all, never more than
235
+ # half of deadlock_timeout. The limit is lock_timeout, set for that
236
+ # statement only (SET LOCAL, then set back; the three go to the server
237
+ # as one string), to the total divided among the locks. lock_timeout,
238
+ # not statement_timeout: its timer runs only while a lock is waited
239
+ # for, and PostgreSQL reports one that fires as the lock is granted as
240
+ # a lock timeout, in this statement or the next of the string, never in
241
+ # a later statement of the caller's (a statement_timeout that fired as
242
+ # the statement ended cancelled the next one "due to user request").
243
+ # True when every one was taken; false when the time ran out, and the
244
+ # caller rolls its savepoint back, which also undoes the SET LOCAL.
245
+ #
246
+ # A lock_timeout of the caller's own (the session's or the
247
+ # transaction's) that is shorter than that share is left as it is, and
248
+ # its timeout is the caller's error (ActiveRecord::LockWaitTimeout), as
249
+ # for a wait without a bound: it is not taken for this call's own limit
250
+ # and tried again.
251
+ def wait_briefly(connection, hashes, (half_deadlock, lock, setting), total)
252
+ each = [[total, half_deadlock].min / hashes.size, 1].max
253
+ wait = "SELECT count(pg_advisory_xact_lock(h)) FROM (SELECT unnest(ARRAY[#{hashes.sort.join(', ')}]::int[]) AS h " \
254
+ "ORDER BY h) AS keys"
255
+ if lock.positive? && lock < each
256
+ connection.execute(wait)
257
+ return true
258
+ end
259
+
260
+ begin
261
+ connection.execute(
262
+ "SET LOCAL lock_timeout = #{Integer(each)}; #{wait}; SET LOCAL lock_timeout = #{connection.quote(setting)}"
263
+ )
264
+ rescue ActiveRecord::LockWaitTimeout
265
+ return false
266
+ end
267
+ true
268
+ end
269
+
34
270
  def warn_once(adapter, purpose)
35
271
  return if @warned[purpose]
36
272
 
@@ -121,7 +121,7 @@ module Lyra
121
121
 
122
122
  def apply_created(event)
123
123
  @id = event.data[:model_id] rescue event.model_id
124
- attrs = event.data[:attributes] rescue event.attributes
124
+ attrs = Lyra::Event.created_attributes(event.data) rescue event.attributes
125
125
  attrs&.each { |k, v| set_state(k, v) }
126
126
  end
127
127