orfeas_lyra 0.9.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +2021 -17
- data/README.md +22 -9
- data/Rakefile +40 -1
- data/app/controllers/lyra/application_controller.rb +127 -0
- data/app/controllers/lyra/dashboard_controller.rb +33 -31
- data/app/controllers/lyra/flow_controller.rb +32 -34
- data/app/controllers/lyra/privacy_controller.rb +57 -18
- data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
- data/app/views/lyra/dashboard/index.html.erb +1 -0
- data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
- data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
- data/app/views/lyra/privacy/policy.html.erb +7 -0
- data/app/views/lyra/privacy/rights.html.erb +78 -0
- data/config/privacy_policies.rb +0 -3
- data/config/routes.rb +1 -0
- data/lib/lyra/access_log.rb +187 -6
- data/lib/lyra/advisory_lock.rb +236 -0
- data/lib/lyra/aggregate.rb +1 -1
- data/lib/lyra/associations/habtm_links.rb +382 -19
- data/lib/lyra/associations/maintenance.rb +229 -0
- data/lib/lyra/attribution.rb +48 -0
- data/lib/lyra/bypass_events.rb +446 -46
- data/lib/lyra/command_handler.rb +19 -14
- data/lib/lyra/configuration.rb +237 -10
- data/lib/lyra/consistency/read_your_writes.rb +4 -2
- data/lib/lyra/correlation.rb +121 -0
- data/lib/lyra/domain_events.rb +4 -0
- data/lib/lyra/dual_view.rb +53 -16
- data/lib/lyra/encrypted_attributes.rb +5 -0
- data/lib/lyra/engine.rb +67 -0
- data/lib/lyra/erasure.rb +1011 -106
- data/lib/lyra/event.rb +35 -1
- data/lib/lyra/event_data.rb +38 -0
- data/lib/lyra/event_flow.rb +31 -15
- data/lib/lyra/event_store_adapter.rb +28 -0
- data/lib/lyra/event_window.rb +58 -0
- data/lib/lyra/genesis.rb +548 -44
- data/lib/lyra/interceptors/association_interceptor.rb +689 -34
- data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
- data/lib/lyra/internal.rb +26 -0
- data/lib/lyra/mode_sync.rb +51 -3
- data/lib/lyra/mode_transition.rb +257 -30
- data/lib/lyra/model_selection.rb +35 -0
- data/lib/lyra/own_connection.rb +17 -2
- data/lib/lyra/privacy/adapters/pam.rb +25 -2
- data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
- data/lib/lyra/privacy/interface.rb +27 -7
- data/lib/lyra/privacy/subject_links.rb +143 -0
- data/lib/lyra/privacy/subject_records.rb +369 -0
- data/lib/lyra/projection.rb +7 -1
- data/lib/lyra/projections/async_checkpoint.rb +32 -0
- data/lib/lyra/projections/async_projection_job.rb +166 -23
- data/lib/lyra/projections/cached_joins.rb +33 -18
- data/lib/lyra/projections/cached_projection.rb +74 -33
- data/lib/lyra/projections/cached_relation.rb +3175 -321
- data/lib/lyra/projections/default_scopes.rb +254 -0
- data/lib/lyra/projections/deferred_relation.rb +23 -3
- data/lib/lyra/projections/event_store_reader.rb +47 -8
- data/lib/lyra/projections/foreign_keys.rb +92 -0
- data/lib/lyra/projections/lazy_projection.rb +165 -27
- data/lib/lyra/projections/model_projection.rb +165 -14
- data/lib/lyra/projections/rebuild.rb +34 -6
- data/lib/lyra/projections/reference_links.rb +463 -72
- data/lib/lyra/projections/typed_values.rb +629 -0
- data/lib/lyra/projections/unique_indexes.rb +595 -0
- data/lib/lyra/projections/uniqueness.rb +784 -0
- data/lib/lyra/projections/value_digest.rb +93 -0
- data/lib/lyra/purpose_bound_reads.rb +912 -77
- data/lib/lyra/repair.rb +119 -28
- data/lib/lyra/retention.rb +114 -8
- data/lib/lyra/rights/consent_store.rb +415 -0
- data/lib/lyra/rights/erasure_guard.rb +135 -0
- data/lib/lyra/rights/pending.rb +74 -0
- data/lib/lyra/rights/queries.rb +269 -0
- data/lib/lyra/rights/store.rb +512 -0
- data/lib/lyra/rights/subject_keys.rb +224 -0
- data/lib/lyra/rights.rb +281 -0
- data/lib/lyra/saves_without_effect.rb +69 -0
- data/lib/lyra/stream_tail.rb +319 -0
- data/lib/lyra/strict_data_access.rb +916 -106
- data/lib/lyra/temporal.rb +4 -2
- data/lib/lyra/touch_registry.rb +404 -0
- data/lib/lyra/verification/bypass_workflow.rb +452 -42
- data/lib/lyra/verification/rights_workflows.rb +170 -0
- data/lib/lyra/version.rb +1 -1
- data/lib/lyra/visualization/event_graph.rb +1 -1
- data/lib/lyra/visualization/timeline.rb +2 -2
- data/lib/lyra.rb +43 -2
- data/lib/tasks/lyra_mode.rake +110 -8
- data/lib/tasks/lyra_projections.rake +43 -2
- metadata +34 -4
- data/lib/lyra/associations/event_aware.rb +0 -224
data/lib/lyra/advisory_lock.rb
CHANGED
|
@@ -8,6 +8,28 @@ module Lyra
|
|
|
8
8
|
# SQLite needs none: it serialises writers. Other adapters have no such lock
|
|
9
9
|
# here, so the call does nothing there and warns once per purpose, since
|
|
10
10
|
# those paths are then safe in a single process only.
|
|
11
|
+
#
|
|
12
|
+
# The order of a record's locks. A write that takes both a record's
|
|
13
|
+
# aggregate lock (this advisory lock on "lyra_aggregate/<stream>") and its
|
|
14
|
+
# row's lock takes the aggregate lock first: ES-Async's and ES-Lazy's
|
|
15
|
+
# writes through callbacks take the aggregate lock only (their rows are the
|
|
16
|
+
# projection's), Hijack and ES-Sync take the row's lock only (the row lock
|
|
17
|
+
# is their aggregate lock: CrudInterceptor#lyra_lock_aggregate!), and a
|
|
18
|
+
# write that skips callbacks (update_columns, touch, delete, and in ES-Async
|
|
19
|
+
# update_all and delete_all: Lyra::BypassEvents) and an erasure
|
|
20
|
+
# (Lyra::Erasure#lock_record) take the aggregate lock, then write or lock
|
|
21
|
+
# the row. A write that took them the other way round deadlocked against
|
|
22
|
+
# one of these. Several records' aggregate locks are taken together by
|
|
23
|
+
# xact_lock_all.
|
|
24
|
+
#
|
|
25
|
+
# Lyra orders the locks of one write, not those of an application's
|
|
26
|
+
# transaction: one that writes several records one after another takes
|
|
27
|
+
# their locks in the order it writes them, and can deadlock against
|
|
28
|
+
# another transaction, or a bulk write, that holds one of them and waits
|
|
29
|
+
# for another, as plain Active Record's row locks do. PostgreSQL detects
|
|
30
|
+
# it and aborts one of them (ActiveRecord::Deadlocked). Write records in
|
|
31
|
+
# one order (by primary key, say) and retry the transaction on
|
|
32
|
+
# ActiveRecord::Deadlocked; see docs/TROUBLESHOOTING.md.
|
|
11
33
|
module AdvisoryLock
|
|
12
34
|
@warned = {}
|
|
13
35
|
|
|
@@ -24,6 +46,162 @@ module Lyra
|
|
|
24
46
|
false
|
|
25
47
|
end
|
|
26
48
|
|
|
49
|
+
# The locks of every one of +keys+, as xact_lock takes each. Each is
|
|
50
|
+
# tried without waiting, in one statement, in a savepoint; when every
|
|
51
|
+
# one is free, that is all (one statement besides the savepoint's).
|
|
52
|
+
#
|
|
53
|
+
# When some are held by other sessions, what the call does next depends
|
|
54
|
+
# on whether this transaction holds advisory locks of its own already
|
|
55
|
+
# (wait_bound, one query, at the first contention only):
|
|
56
|
+
#
|
|
57
|
+
# - It holds none (a bulk write beginning its transaction's locking):
|
|
58
|
+
# it waits for the held ones, in the order of their hashes, keeping
|
|
59
|
+
# the ones it took, for at most MAX_WAIT_MS in all; a holder that
|
|
60
|
+
# ends meanwhile lets it through, in its place in that lock's queue.
|
|
61
|
+
# When the time is up, the savepoint is rolled back, which gives back
|
|
62
|
+
# every lock taken in it, and the next try first waits, holding none,
|
|
63
|
+
# for the first lock that was held, for at most half of
|
|
64
|
+
# deadlock_timeout, then tries them all again. Before, it gave every
|
|
65
|
+
# lock back whenever one was held and waited for that one without a
|
|
66
|
+
# limit: sessions that come and go with a few of its records kept it
|
|
67
|
+
# out (a bulk write of 8 records against three transactions that each
|
|
68
|
+
# held one and then bulk-wrote two others went through once in 15
|
|
69
|
+
# seconds in ES-NoProj), and, waiting in their locks' queues, it was
|
|
70
|
+
# the one PostgreSQL aborted for their deadlocks (5 or 6 times in
|
|
71
|
+
# those 15 seconds). No wait of the call's lasts as long as
|
|
72
|
+
# deadlock_timeout, so PostgreSQL never reports it deadlocked: its own
|
|
73
|
+
# check runs only after deadlock_timeout, and a session that waits for
|
|
74
|
+
# a lock the call took began waiting after the call took it, so its
|
|
75
|
+
# check finds the call gone. Two bulk writes, or a bulk write and a
|
|
76
|
+
# transaction that holds one of its records and asks for another,
|
|
77
|
+
# therefore never deadlock: the bulk write gives way.
|
|
78
|
+
# - It holds some (a transaction that wrote a record, or a bulk write's
|
|
79
|
+
# later round): whenever one is held, it gives back the ones it took
|
|
80
|
+
# in that try and waits for that one holding none of them, without a
|
|
81
|
+
# limit, as before 1.0.0. It holds its earlier locks anyway, and
|
|
82
|
+
# keeping the new ones as well while it waited made such transactions
|
|
83
|
+
# deadlock against each other.
|
|
84
|
+
#
|
|
85
|
+
# Nothing changes for the caller: a wait that ran out of time is not
|
|
86
|
+
# reported, and the call returns only once it holds every lock (the
|
|
87
|
+
# lock limit check comes before it, and nothing is appended before it
|
|
88
|
+
# returns). A lock_timeout of the caller's own that is shorter than a
|
|
89
|
+
# bounded wait's share stays in force, and its timeout raises
|
|
90
|
+
# ActiveRecord::LockWaitTimeout, as before. Under contention the bulk
|
|
91
|
+
# write's turns cost the transactions it shares records with some
|
|
92
|
+
# throughput: they wait while it holds its records, up to MAX_WAIT_MS
|
|
93
|
+
# at a time. After TRIES tries, or outside a transaction (no savepoint
|
|
94
|
+
# to give locks back with), the locks are taken waiting, in the order
|
|
95
|
+
# of their hashes. Returns true when the locks were taken, false
|
|
96
|
+
# otherwise.
|
|
97
|
+
def xact_lock_all(connection, keys, purpose:)
|
|
98
|
+
keys = keys.uniq
|
|
99
|
+
return false if keys.empty?
|
|
100
|
+
return xact_lock(connection, keys.first, purpose: purpose) if keys.size == 1
|
|
101
|
+
|
|
102
|
+
adapter = connection.adapter_name
|
|
103
|
+
unless adapter.match?(/postgres/i)
|
|
104
|
+
warn_once(adapter, purpose) unless adapter.match?(/sqlite/i)
|
|
105
|
+
return false
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
hashes = "SELECT DISTINCT hashtext(k) AS h FROM unnest(ARRAY[#{keys.map { connection.quote(_1) }.join(', ')}]::text[]) " \
|
|
109
|
+
"AS k ORDER BY h"
|
|
110
|
+
if connection.transaction_open?
|
|
111
|
+
contended = nil
|
|
112
|
+
# The bounds of its waits (wait_bound) when the caller held no
|
|
113
|
+
# advisory lock of its own at the first contention; false when it
|
|
114
|
+
# did; nil before the first.
|
|
115
|
+
bounded = nil
|
|
116
|
+
TRIES.times do
|
|
117
|
+
held = nil
|
|
118
|
+
connection.transaction(requires_new: true) do
|
|
119
|
+
# The lock another session held last time, waited for while
|
|
120
|
+
# this call holds none of the others (it gets its place in
|
|
121
|
+
# that lock's queue; at most half of deadlock_timeout where
|
|
122
|
+
# the waits are bounded), then the others tried.
|
|
123
|
+
if contended
|
|
124
|
+
if bounded
|
|
125
|
+
held = contended unless wait_briefly(connection, [contended], bounded, bounded.first)
|
|
126
|
+
else
|
|
127
|
+
connection.execute("SELECT pg_advisory_xact_lock(#{contended})")
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
raise ActiveRecord::Rollback if held
|
|
131
|
+
|
|
132
|
+
# Volatile functions are called on the subquery's rows, in
|
|
133
|
+
# their order, every one of them.
|
|
134
|
+
rows = connection.select_rows("SELECT h, pg_try_advisory_xact_lock(h) FROM (#{hashes}) AS keys")
|
|
135
|
+
missing = rows.reject { |_, taken| taken }.map { |hash, _| Integer(hash) }
|
|
136
|
+
next if missing.empty?
|
|
137
|
+
|
|
138
|
+
bounded = wait_bound(connection, rows.size - missing.size) if bounded.nil?
|
|
139
|
+
next if bounded && wait_briefly(connection, missing, bounded, MAX_WAIT_MS)
|
|
140
|
+
|
|
141
|
+
held = missing.first
|
|
142
|
+
raise ActiveRecord::Rollback
|
|
143
|
+
end
|
|
144
|
+
return true unless held
|
|
145
|
+
|
|
146
|
+
contended = held
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
connection.execute("SELECT count(pg_advisory_xact_lock(h)) FROM (#{hashes}) AS keys")
|
|
150
|
+
true
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# How many times xact_lock_all tries all the locks before it waits for
|
|
154
|
+
# them in order.
|
|
155
|
+
TRIES = 200
|
|
156
|
+
|
|
157
|
+
# The longest a bulk write waits, in milliseconds, holding the locks it
|
|
158
|
+
# took, for the ones it did not (xact_lock_all), in all; never more
|
|
159
|
+
# than half of deadlock_timeout either.
|
|
160
|
+
MAX_WAIT_MS = 100
|
|
161
|
+
|
|
162
|
+
# Take the lock only if no other session holds it. Returns true when
|
|
163
|
+
# the caller may go on as the lock's holder: it took the lock, or there
|
|
164
|
+
# is no such lock on this adapter (as xact_lock, which then takes none).
|
|
165
|
+
# Returns false when another session holds it.
|
|
166
|
+
def try_xact_lock(connection, key, purpose:)
|
|
167
|
+
adapter = connection.adapter_name
|
|
168
|
+
if adapter.match?(/postgres/i)
|
|
169
|
+
return connection.select_value("SELECT pg_try_advisory_xact_lock(hashtext(#{connection.quote(key)}))") == true
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
warn_once(adapter, purpose) unless adapter.match?(/sqlite/i)
|
|
173
|
+
true
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
# Wait until no other session holds the lock, without keeping it: take
|
|
177
|
+
# it shared in a savepoint and roll the savepoint back, which gives the
|
|
178
|
+
# lock back. Shared requests do not exclude each other, so every
|
|
179
|
+
# session waiting on the same holder goes on when that holder's
|
|
180
|
+
# transaction ends. The lock belongs to the savepoint, not to the
|
|
181
|
+
# session, so it cannot outlive the caller's transaction whatever
|
|
182
|
+
# interrupts this call; a session-level lock left behind by an error or
|
|
183
|
+
# a Thread#raise between taking and giving it back would block every
|
|
184
|
+
# later holder for good. Keeping it until the caller's transaction ends
|
|
185
|
+
# would block every holder for as long as that transaction, and two
|
|
186
|
+
# waiters that each kept it shared and then asked for it exclusively
|
|
187
|
+
# would deadlock. The savepoint, the wait and the rollback go to the
|
|
188
|
+
# server as one string, one round trip; if the wait fails, the rest is
|
|
189
|
+
# not run, and the error rolls back the caller's transaction or
|
|
190
|
+
# savepoint, which also releases the lock. Returns false where there is
|
|
191
|
+
# no such lock.
|
|
192
|
+
def wait_for(connection, key)
|
|
193
|
+
return false unless connection.adapter_name.match?(/postgres/i)
|
|
194
|
+
|
|
195
|
+
connection.transaction do
|
|
196
|
+
connection.execute(
|
|
197
|
+
"SAVEPOINT lyra_wait_for; " \
|
|
198
|
+
"SELECT pg_advisory_xact_lock_shared(hashtext(#{connection.quote(key)})); " \
|
|
199
|
+
"ROLLBACK TO SAVEPOINT lyra_wait_for; RELEASE SAVEPOINT lyra_wait_for"
|
|
200
|
+
)
|
|
201
|
+
end
|
|
202
|
+
true
|
|
203
|
+
end
|
|
204
|
+
|
|
27
205
|
# Forget which purposes have warned (tests).
|
|
28
206
|
def reset_warnings!
|
|
29
207
|
@warned = {}
|
|
@@ -31,6 +209,64 @@ module Lyra
|
|
|
31
209
|
|
|
32
210
|
private
|
|
33
211
|
|
|
212
|
+
# xact_lock_all, at its first contention, having just taken +taken+
|
|
213
|
+
# of its locks: whether its waits are bounded. Only for a caller that
|
|
214
|
+
# holds no other advisory lock in this session (a bulk write that
|
|
215
|
+
# begins its transaction's locking): [half of deadlock_timeout in ms,
|
|
216
|
+
# the session's lock_timeout in ms (0: none), the lock_timeout setting
|
|
217
|
+
# to restore]. False for one that already holds some (a transaction
|
|
218
|
+
# that wrote a record and now writes others): it waits holding none of
|
|
219
|
+
# the new ones, as before 1.0.0. One query.
|
|
220
|
+
def wait_bound(connection, taken)
|
|
221
|
+
held, half_deadlock, lock, setting = connection.select_rows(
|
|
222
|
+
"SELECT (SELECT count(*) FROM pg_locks WHERE locktype = 'advisory' AND granted AND pid = pg_backend_pid()), " \
|
|
223
|
+
"(EXTRACT(epoch FROM current_setting('deadlock_timeout')::interval) * 500)::bigint, " \
|
|
224
|
+
"(EXTRACT(epoch FROM current_setting('lock_timeout')::interval) * 1000)::bigint, " \
|
|
225
|
+
"current_setting('lock_timeout')"
|
|
226
|
+
).first
|
|
227
|
+
return false unless Integer(held) == taken
|
|
228
|
+
|
|
229
|
+
[Integer(half_deadlock), Integer(lock), setting]
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
# xact_lock_all, for a caller whose waits are bounded (wait_bound):
|
|
233
|
+
# wait for the locks of +hashes+, in their order, in the caller's
|
|
234
|
+
# savepoint, for at most +total+ milliseconds in all, never more than
|
|
235
|
+
# half of deadlock_timeout. The limit is lock_timeout, set for that
|
|
236
|
+
# statement only (SET LOCAL, then set back; the three go to the server
|
|
237
|
+
# as one string), to the total divided among the locks. lock_timeout,
|
|
238
|
+
# not statement_timeout: its timer runs only while a lock is waited
|
|
239
|
+
# for, and PostgreSQL reports one that fires as the lock is granted as
|
|
240
|
+
# a lock timeout, in this statement or the next of the string, never in
|
|
241
|
+
# a later statement of the caller's (a statement_timeout that fired as
|
|
242
|
+
# the statement ended cancelled the next one "due to user request").
|
|
243
|
+
# True when every one was taken; false when the time ran out, and the
|
|
244
|
+
# caller rolls its savepoint back, which also undoes the SET LOCAL.
|
|
245
|
+
#
|
|
246
|
+
# A lock_timeout of the caller's own (the session's or the
|
|
247
|
+
# transaction's) that is shorter than that share is left as it is, and
|
|
248
|
+
# its timeout is the caller's error (ActiveRecord::LockWaitTimeout), as
|
|
249
|
+
# for a wait without a bound: it is not taken for this call's own limit
|
|
250
|
+
# and tried again.
|
|
251
|
+
def wait_briefly(connection, hashes, (half_deadlock, lock, setting), total)
|
|
252
|
+
each = [[total, half_deadlock].min / hashes.size, 1].max
|
|
253
|
+
wait = "SELECT count(pg_advisory_xact_lock(h)) FROM (SELECT unnest(ARRAY[#{hashes.sort.join(', ')}]::int[]) AS h " \
|
|
254
|
+
"ORDER BY h) AS keys"
|
|
255
|
+
if lock.positive? && lock < each
|
|
256
|
+
connection.execute(wait)
|
|
257
|
+
return true
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
begin
|
|
261
|
+
connection.execute(
|
|
262
|
+
"SET LOCAL lock_timeout = #{Integer(each)}; #{wait}; SET LOCAL lock_timeout = #{connection.quote(setting)}"
|
|
263
|
+
)
|
|
264
|
+
rescue ActiveRecord::LockWaitTimeout
|
|
265
|
+
return false
|
|
266
|
+
end
|
|
267
|
+
true
|
|
268
|
+
end
|
|
269
|
+
|
|
34
270
|
def warn_once(adapter, purpose)
|
|
35
271
|
return if @warned[purpose]
|
|
36
272
|
|
data/lib/lyra/aggregate.rb
CHANGED
|
@@ -121,7 +121,7 @@ module Lyra
|
|
|
121
121
|
|
|
122
122
|
def apply_created(event)
|
|
123
123
|
@id = event.data[:model_id] rescue event.model_id
|
|
124
|
-
attrs = event.data
|
|
124
|
+
attrs = Lyra::Event.created_attributes(event.data) rescue event.attributes
|
|
125
125
|
attrs&.each { |k, v| set_state(k, v) }
|
|
126
126
|
end
|
|
127
127
|
|