orfeas_lyra 0.9.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +2021 -17
- data/README.md +22 -9
- data/Rakefile +40 -1
- data/app/controllers/lyra/application_controller.rb +127 -0
- data/app/controllers/lyra/dashboard_controller.rb +33 -31
- data/app/controllers/lyra/flow_controller.rb +32 -34
- data/app/controllers/lyra/privacy_controller.rb +57 -18
- data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
- data/app/views/lyra/dashboard/index.html.erb +1 -0
- data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
- data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
- data/app/views/lyra/privacy/policy.html.erb +7 -0
- data/app/views/lyra/privacy/rights.html.erb +78 -0
- data/config/privacy_policies.rb +0 -3
- data/config/routes.rb +1 -0
- data/lib/lyra/access_log.rb +187 -6
- data/lib/lyra/advisory_lock.rb +236 -0
- data/lib/lyra/aggregate.rb +1 -1
- data/lib/lyra/associations/habtm_links.rb +382 -19
- data/lib/lyra/associations/maintenance.rb +229 -0
- data/lib/lyra/attribution.rb +48 -0
- data/lib/lyra/bypass_events.rb +446 -46
- data/lib/lyra/command_handler.rb +19 -14
- data/lib/lyra/configuration.rb +237 -10
- data/lib/lyra/consistency/read_your_writes.rb +4 -2
- data/lib/lyra/correlation.rb +121 -0
- data/lib/lyra/domain_events.rb +4 -0
- data/lib/lyra/dual_view.rb +53 -16
- data/lib/lyra/encrypted_attributes.rb +5 -0
- data/lib/lyra/engine.rb +67 -0
- data/lib/lyra/erasure.rb +1011 -106
- data/lib/lyra/event.rb +35 -1
- data/lib/lyra/event_data.rb +38 -0
- data/lib/lyra/event_flow.rb +31 -15
- data/lib/lyra/event_store_adapter.rb +28 -0
- data/lib/lyra/event_window.rb +58 -0
- data/lib/lyra/genesis.rb +548 -44
- data/lib/lyra/interceptors/association_interceptor.rb +689 -34
- data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
- data/lib/lyra/internal.rb +26 -0
- data/lib/lyra/mode_sync.rb +51 -3
- data/lib/lyra/mode_transition.rb +257 -30
- data/lib/lyra/model_selection.rb +35 -0
- data/lib/lyra/own_connection.rb +17 -2
- data/lib/lyra/privacy/adapters/pam.rb +25 -2
- data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
- data/lib/lyra/privacy/interface.rb +27 -7
- data/lib/lyra/privacy/subject_links.rb +143 -0
- data/lib/lyra/privacy/subject_records.rb +369 -0
- data/lib/lyra/projection.rb +7 -1
- data/lib/lyra/projections/async_checkpoint.rb +32 -0
- data/lib/lyra/projections/async_projection_job.rb +166 -23
- data/lib/lyra/projections/cached_joins.rb +33 -18
- data/lib/lyra/projections/cached_projection.rb +74 -33
- data/lib/lyra/projections/cached_relation.rb +3175 -321
- data/lib/lyra/projections/default_scopes.rb +254 -0
- data/lib/lyra/projections/deferred_relation.rb +23 -3
- data/lib/lyra/projections/event_store_reader.rb +47 -8
- data/lib/lyra/projections/foreign_keys.rb +92 -0
- data/lib/lyra/projections/lazy_projection.rb +165 -27
- data/lib/lyra/projections/model_projection.rb +165 -14
- data/lib/lyra/projections/rebuild.rb +34 -6
- data/lib/lyra/projections/reference_links.rb +463 -72
- data/lib/lyra/projections/typed_values.rb +629 -0
- data/lib/lyra/projections/unique_indexes.rb +595 -0
- data/lib/lyra/projections/uniqueness.rb +784 -0
- data/lib/lyra/projections/value_digest.rb +93 -0
- data/lib/lyra/purpose_bound_reads.rb +912 -77
- data/lib/lyra/repair.rb +119 -28
- data/lib/lyra/retention.rb +114 -8
- data/lib/lyra/rights/consent_store.rb +415 -0
- data/lib/lyra/rights/erasure_guard.rb +135 -0
- data/lib/lyra/rights/pending.rb +74 -0
- data/lib/lyra/rights/queries.rb +269 -0
- data/lib/lyra/rights/store.rb +512 -0
- data/lib/lyra/rights/subject_keys.rb +224 -0
- data/lib/lyra/rights.rb +281 -0
- data/lib/lyra/saves_without_effect.rb +69 -0
- data/lib/lyra/stream_tail.rb +319 -0
- data/lib/lyra/strict_data_access.rb +916 -106
- data/lib/lyra/temporal.rb +4 -2
- data/lib/lyra/touch_registry.rb +404 -0
- data/lib/lyra/verification/bypass_workflow.rb +452 -42
- data/lib/lyra/verification/rights_workflows.rb +170 -0
- data/lib/lyra/version.rb +1 -1
- data/lib/lyra/visualization/event_graph.rb +1 -1
- data/lib/lyra/visualization/timeline.rb +2 -2
- data/lib/lyra.rb +43 -2
- data/lib/tasks/lyra_mode.rake +110 -8
- data/lib/tasks/lyra_projections.rake +43 -2
- metadata +34 -4
- data/lib/lyra/associations/event_aware.rb +0 -224
|
@@ -33,9 +33,13 @@ module Lyra
|
|
|
33
33
|
# becomes visible within about GAP_RECHECK; a process's own writes, above
|
|
34
34
|
# the checkpoint, are visible immediately.
|
|
35
35
|
#
|
|
36
|
-
# Cost: every read first checks whether the log has moved on
|
|
37
|
-
#
|
|
38
|
-
#
|
|
36
|
+
# Cost: every read first checks whether the log has moved on, in one
|
|
37
|
+
# query that reads the checkpoint and the newest event id together.
|
|
38
|
+
# Catch-up happens only when it has. A reader that finds another reader
|
|
39
|
+
# catching up does not queue to repeat its work: it waits for that
|
|
40
|
+
# catch-up to end and reads the checkpoint again (wait_for_catch_up). A
|
|
41
|
+
# catch-up that applies nothing writes no checkpoint. The checkpoint
|
|
42
|
+
# table is created on first use, so enabling the mode needs no migration.
|
|
39
43
|
class LazyProjection
|
|
40
44
|
TABLE = "lyra_projection_checkpoints"
|
|
41
45
|
NAME = "lazy"
|
|
@@ -44,12 +48,33 @@ module Lyra
|
|
|
44
48
|
GAP_RECHECK = 1.0
|
|
45
49
|
FRESH_WINDOW = 100
|
|
46
50
|
BATCH = 1_000
|
|
51
|
+
# How often a reader waits for another reader's catch-up before it
|
|
52
|
+
# queues for the lock itself, as every reader did before. Two waits
|
|
53
|
+
# suffice when every catch-up commits: the second holder took the lock
|
|
54
|
+
# after this reader's check, so it saw every event this reader must
|
|
55
|
+
# see. The third allows for one that rolled back, or for a try that
|
|
56
|
+
# failed only because another waiter held the lock shared for a moment.
|
|
57
|
+
WAITS = 3
|
|
47
58
|
|
|
48
59
|
class << self
|
|
49
60
|
def active?
|
|
50
61
|
Lyra.event_sourcing_mode? && Lyra.config.projection_mode == :lazy
|
|
51
62
|
end
|
|
52
63
|
|
|
64
|
+
# Run the block with no catch-up before its reads (a write that
|
|
65
|
+
# caught up when it began: BypassEvents.atomically).
|
|
66
|
+
def without_catch_up
|
|
67
|
+
return yield unless active?
|
|
68
|
+
|
|
69
|
+
previous = Thread.current[:lyra_lazy_catching_up]
|
|
70
|
+
Thread.current[:lyra_lazy_catching_up] = true
|
|
71
|
+
begin
|
|
72
|
+
yield
|
|
73
|
+
ensure
|
|
74
|
+
Thread.current[:lyra_lazy_catching_up] = previous
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
53
78
|
# Called before a read on +model_class+ (see Interceptors::LazyReads).
|
|
54
79
|
def before_read(model_class)
|
|
55
80
|
return unless active?
|
|
@@ -67,20 +92,18 @@ module Lyra
|
|
|
67
92
|
return 0 unless force || active?
|
|
68
93
|
return 0 if Thread.current[:lyra_lazy_catching_up]
|
|
69
94
|
|
|
70
|
-
Thread.current[:lyra_lazy_catching_up] = true
|
|
95
|
+
began = Thread.current[:lyra_lazy_catching_up] = true
|
|
71
96
|
ActiveRecord::Base.with_connection do |connection|
|
|
72
97
|
ensure_table!(connection)
|
|
73
|
-
|
|
98
|
+
target = catch_up_target(connection)
|
|
99
|
+
next 0 unless target
|
|
74
100
|
|
|
75
|
-
|
|
76
|
-
ActiveRecord::Base.transaction do
|
|
77
|
-
lock!(connection)
|
|
78
|
-
applied = apply_pending(connection)
|
|
79
|
-
end
|
|
80
|
-
applied
|
|
101
|
+
ActiveRecord::Base.transaction { catch_up_to(connection, target) }
|
|
81
102
|
end
|
|
82
103
|
ensure
|
|
83
|
-
|
|
104
|
+
# Only the call that set it (a call refused above left it to its
|
|
105
|
+
# owner: a catch-up under way, or LazyProjection.without_catch_up).
|
|
106
|
+
Thread.current[:lyra_lazy_catching_up] = nil if began
|
|
84
107
|
end
|
|
85
108
|
|
|
86
109
|
# The current checkpoint: [position, { gap_id => first_seen_epoch }].
|
|
@@ -91,7 +114,38 @@ module Lyra
|
|
|
91
114
|
row = connection.select_one("SELECT position, gaps FROM #{TABLE} WHERE name = #{connection.quote(NAME)}")
|
|
92
115
|
return [0, {}] unless row
|
|
93
116
|
|
|
94
|
-
|
|
117
|
+
parse_checkpoint(row["position"], row["gaps"])
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# Bring the rows of +ids+ (records of +model_class+) up to date with
|
|
121
|
+
# their streams, where a stream has an event the tables have not had
|
|
122
|
+
# yet (past the checkpoint, or among its gaps): that record is
|
|
123
|
+
# replayed onto its row (Rebuild.replay_record), as a catch-up would.
|
|
124
|
+
# For a write that skips callbacks, once it holds the records'
|
|
125
|
+
# aggregate locks (Lyra::BypassEvents): an update committed after the
|
|
126
|
+
# catch-up the write began with is then in the row it writes, not
|
|
127
|
+
# lost under it. One query when every row is up to date. Returns how
|
|
128
|
+
# many records were replayed.
|
|
129
|
+
def refresh_records(model_class, ids)
|
|
130
|
+
return 0 unless active? && ids.any?
|
|
131
|
+
|
|
132
|
+
ActiveRecord::Base.with_connection do |connection|
|
|
133
|
+
position, gaps = checkpoint(connection)
|
|
134
|
+
prefix = Lyra.stream_prefix(model_class)
|
|
135
|
+
streams = ids.map { |id| connection.quote("#{prefix}#{id}") }.join(", ")
|
|
136
|
+
pending = ["e.id > #{Integer(position)}"]
|
|
137
|
+
pending << "e.id IN (#{gaps.keys.map { Integer(_1) }.join(',')})" if gaps.any?
|
|
138
|
+
behind = connection.uncached do
|
|
139
|
+
connection.select_values(
|
|
140
|
+
"SELECT DISTINCT s.stream FROM event_store_events_in_streams s " \
|
|
141
|
+
"JOIN event_store_events e ON e.event_id = s.event_id " \
|
|
142
|
+
"WHERE s.stream IN (#{streams}) AND (#{pending.join(' OR ')})",
|
|
143
|
+
"Lyra ES-Lazy rows behind"
|
|
144
|
+
)
|
|
145
|
+
end
|
|
146
|
+
behind.each { |stream| Rebuild.replay_record(model_class, stream.delete_prefix(prefix)) }
|
|
147
|
+
behind.size
|
|
148
|
+
end
|
|
95
149
|
end
|
|
96
150
|
|
|
97
151
|
# Forget the checkpoint, e.g. after the tables were truncated or rebuilt.
|
|
@@ -109,11 +163,24 @@ module Lyra
|
|
|
109
163
|
table.start_with?("event_store_") || table == TABLE
|
|
110
164
|
end
|
|
111
165
|
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
166
|
+
def parse_checkpoint(position, gaps)
|
|
167
|
+
[position.to_i, JSON.parse(gaps || "{}").transform_keys(&:to_i)]
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
# Fast path, no lock: nil when nothing is new and open gaps (if any)
|
|
171
|
+
# were re-checked less than GAP_RECHECK seconds ago. Otherwise the
|
|
172
|
+
# newest event id this read must see applied: the catch-up target.
|
|
173
|
+
#
|
|
174
|
+
# One statement reads the checkpoint and the newest event id, so both
|
|
175
|
+
# come from one snapshot. It decides as the two queries it replaces
|
|
176
|
+
# did, so the fast path skips exactly what it skipped before.
|
|
177
|
+
def catch_up_target(connection)
|
|
178
|
+
row = connection.select_rows(<<~SQL.squish).first
|
|
179
|
+
SELECT c.position, c.gaps, (SELECT MAX(id) FROM event_store_events)
|
|
180
|
+
FROM (SELECT 1) AS one LEFT JOIN #{TABLE} c ON c.name = #{connection.quote(NAME)}
|
|
181
|
+
SQL
|
|
182
|
+
position, gaps = parse_checkpoint(row[0], row[1])
|
|
183
|
+
newest = row[2].to_i
|
|
117
184
|
# A checkpoint past the newest event cannot be right: the event
|
|
118
185
|
# store was emptied or restored without this table. Taken at its
|
|
119
186
|
# word, every new event would look applied and nothing would be
|
|
@@ -121,15 +188,68 @@ module Lyra
|
|
|
121
188
|
if position > newest
|
|
122
189
|
warn_stale_checkpoint(position, newest)
|
|
123
190
|
reset!(connection)
|
|
124
|
-
return newest.zero?
|
|
191
|
+
return newest.zero? ? nil : newest
|
|
125
192
|
end
|
|
126
|
-
return
|
|
193
|
+
return newest if newest > position
|
|
127
194
|
|
|
195
|
+
gaps_rechecked?(gaps) ? nil : newest
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
def gaps_rechecked?(gaps)
|
|
128
199
|
gaps.empty? || (Process.clock_gettime(Process::CLOCK_MONOTONIC) - (@gaps_checked_at || 0)) < GAP_RECHECK
|
|
129
200
|
end
|
|
130
201
|
|
|
202
|
+
# Bring the tables up to +target+, in the reader's transaction.
|
|
203
|
+
#
|
|
204
|
+
# The reader that gets the lock catches up, as before. A reader that
|
|
205
|
+
# finds the lock taken does not queue behind it to repeat the same
|
|
206
|
+
# work: it waits for that catch-up to end (AdvisoryLock.wait_for),
|
|
207
|
+
# then reads the checkpoint again and is done if nothing it must see
|
|
208
|
+
# is left (caught_up_to?). Otherwise it tries again, and after WAITS
|
|
209
|
+
# waits it queues for the lock and catches up itself, as before.
|
|
210
|
+
#
|
|
211
|
+
# The rules hold: only the lock's holder applies events or writes the
|
|
212
|
+
# checkpoint, in its own transaction, in the log's order, from the
|
|
213
|
+
# checkpoint it reads under the lock (rules 1 and 2); a reader that
|
|
214
|
+
# waited applies and writes nothing, so its transaction has nothing
|
|
215
|
+
# to roll back. Waiting skips work only when the work is done, by a
|
|
216
|
+
# catch-up that committed (the waiter reads the committed checkpoint
|
|
217
|
+
# after the holder's transaction ended), and it is judged with the
|
|
218
|
+
# gap list (rule 3), more strictly than the fast path judges it.
|
|
219
|
+
def catch_up_to(connection, target)
|
|
220
|
+
WAITS.times do
|
|
221
|
+
return apply_pending(connection) if try_lock!(connection)
|
|
222
|
+
|
|
223
|
+
wait_for_catch_up(connection)
|
|
224
|
+
return 0 if caught_up_to?(connection, target)
|
|
225
|
+
end
|
|
226
|
+
lock!(connection)
|
|
227
|
+
apply_pending(connection)
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
# After waiting: has a committed catch-up applied every event this
|
|
231
|
+
# reader must see? Those are the events visible to it, up to the
|
|
232
|
+
# newest it saw before waiting (+target+; events committed later are
|
|
233
|
+
# not owed to this read). The checkpoint must have reached +target+,
|
|
234
|
+
# and no event the checkpoint lists as missing may be visible to this
|
|
235
|
+
# reader: such an event committed (or was written by this reader's
|
|
236
|
+
# own open transaction) after the catch-up passed its id, and only a
|
|
237
|
+
# catch-up under the lock replays it. So this reader sees what it
|
|
238
|
+
# would have seen had it queued for the lock and caught up itself;
|
|
239
|
+
# unlike the fast path, it does not wait GAP_RECHECK for a late event.
|
|
240
|
+
def caught_up_to?(connection, target)
|
|
241
|
+
position, gaps = checkpoint(connection)
|
|
242
|
+
return false if position < target
|
|
243
|
+
return true if gaps.empty?
|
|
244
|
+
|
|
245
|
+
connection.select_value(
|
|
246
|
+
"SELECT 1 FROM event_store_events WHERE id IN (#{gaps.keys.map { Integer(_1) }.join(',')}) LIMIT 1"
|
|
247
|
+
).nil?
|
|
248
|
+
end
|
|
249
|
+
|
|
131
250
|
def apply_pending(connection)
|
|
132
251
|
position, gaps = checkpoint(connection)
|
|
252
|
+
started_at = [position, gaps.dup]
|
|
133
253
|
applied = 0
|
|
134
254
|
|
|
135
255
|
loop do
|
|
@@ -154,7 +274,11 @@ module Lyra
|
|
|
154
274
|
end
|
|
155
275
|
|
|
156
276
|
applied += fill_gaps(connection, gaps)
|
|
157
|
-
|
|
277
|
+
# A catch-up that found nothing new (another reader got there
|
|
278
|
+
# first) leaves the checkpoint row alone: rewriting the same values
|
|
279
|
+
# cost a row write and a commit that must reach the disk, all under
|
|
280
|
+
# the lock every other catching-up reader waits on.
|
|
281
|
+
save_checkpoint(connection, position, gaps) unless [position, gaps] == started_at
|
|
158
282
|
applied
|
|
159
283
|
end
|
|
160
284
|
|
|
@@ -186,8 +310,13 @@ module Lyra
|
|
|
186
310
|
return false unless model_class && operation
|
|
187
311
|
|
|
188
312
|
ModelProjection.project(model_class, operation, Struct.new(:events, :attributes, :success?, keyword_init: true).new(
|
|
189
|
-
events: [event],
|
|
190
|
-
|
|
313
|
+
events: [event], success?: true,
|
|
314
|
+
attributes: if %i[created imported].include?(operation)
|
|
315
|
+
Lyra::Event.created_attributes(event.data)
|
|
316
|
+
else
|
|
317
|
+
event.data[:attributes] || event.data["attributes"] || {}
|
|
318
|
+
end
|
|
319
|
+
), INTERNAL)
|
|
191
320
|
true
|
|
192
321
|
end
|
|
193
322
|
|
|
@@ -212,10 +341,6 @@ module Lyra
|
|
|
212
341
|
Lyra.config.event_store.read.events(event_ids).to_a.to_h { |e| [e.event_id, e] }
|
|
213
342
|
end
|
|
214
343
|
|
|
215
|
-
def max_event_id(connection)
|
|
216
|
-
connection.select_value("SELECT MAX(id) FROM event_store_events").to_i
|
|
217
|
-
end
|
|
218
|
-
|
|
219
344
|
def warn_stale_checkpoint(position, newest)
|
|
220
345
|
message = "Lyra: ES-Lazy checkpoint #{position} is past the newest event (#{newest}); " \
|
|
221
346
|
"the event store was emptied or restored without #{TABLE}. Catching up from the start."
|
|
@@ -226,6 +351,19 @@ module Lyra
|
|
|
226
351
|
Lyra::AdvisoryLock.xact_lock(connection, LOCK, purpose: "ES-Lazy catch-up")
|
|
227
352
|
end
|
|
228
353
|
|
|
354
|
+
# True when this transaction now holds the catch-up lock (or the
|
|
355
|
+
# adapter has none, as lock! then takes none); false when another
|
|
356
|
+
# reader holds it.
|
|
357
|
+
def try_lock!(connection)
|
|
358
|
+
Lyra::AdvisoryLock.try_xact_lock(connection, LOCK, purpose: "ES-Lazy catch-up")
|
|
359
|
+
end
|
|
360
|
+
|
|
361
|
+
# Wait until the reader that holds the catch-up lock has committed or
|
|
362
|
+
# rolled back. Several waiters go on together.
|
|
363
|
+
def wait_for_catch_up(connection)
|
|
364
|
+
Lyra::AdvisoryLock.wait_for(connection, LOCK)
|
|
365
|
+
end
|
|
366
|
+
|
|
229
367
|
def save_checkpoint(connection, position, gaps)
|
|
230
368
|
name = connection.quote(NAME)
|
|
231
369
|
json = connection.quote(JSON.generate(gaps))
|
|
@@ -2,6 +2,20 @@
|
|
|
2
2
|
|
|
3
3
|
module Lyra
|
|
4
4
|
module Projections
|
|
5
|
+
# A projected create met, in a unique index, the row of a record whose
|
|
6
|
+
# stream no longer holds the value (destroyed, or moved to another): the
|
|
7
|
+
# row waits for that record's projection (ES-Async: its job). Projecting
|
|
8
|
+
# again later converges; AsyncProjectionJob retries it with no limit on
|
|
9
|
+
# the attempts (ModelProjection.refuse_skipped_create!).
|
|
10
|
+
class ProjectionBehind < ActiveRecord::RecordNotUnique; end
|
|
11
|
+
|
|
12
|
+
# A projected create met, in a unique index, the row of a live record
|
|
13
|
+
# whose stream holds the value too: two records of the log hold one
|
|
14
|
+
# value (written under REPEATABLE READ or SERIALIZABLE, or before 1.0.0),
|
|
15
|
+
# which no projection can hold. Logged once per record and process; not
|
|
16
|
+
# retried (ModelProjection.refuse_skipped_create!).
|
|
17
|
+
class DuplicateInLog < ActiveRecord::RecordNotUnique; end
|
|
18
|
+
|
|
5
19
|
# Synchronous projection handler for updating model tables from events.
|
|
6
20
|
#
|
|
7
21
|
# In event_sourcing mode, this class is responsible for keeping the model
|
|
@@ -21,7 +35,12 @@ module Lyra
|
|
|
21
35
|
# @param model_class [Class] The ActiveRecord model class
|
|
22
36
|
# @param operation [Symbol] :create, :update, or :destroy
|
|
23
37
|
# @param result [CommandResult] The result from CommandHandler
|
|
24
|
-
|
|
38
|
+
# Writes the table with no event, so only Lyra calls it (Lyra::INTERNAL),
|
|
39
|
+
# as for Lyra.projection_write; project_create, project_update and
|
|
40
|
+
# project_destroy are private.
|
|
41
|
+
def project(model_class, operation, result, token = nil)
|
|
42
|
+
raise InternalApiError, "ModelProjection.project is for Lyra's own projections" unless token.equal?(INTERNAL)
|
|
43
|
+
|
|
25
44
|
# Projection writes the table, so it must address the table. With
|
|
26
45
|
# projection_mode :disabled (ES-NoProj), the model's where/all answer
|
|
27
46
|
# from the event store instead; update and destroy below go through
|
|
@@ -31,7 +50,7 @@ module Lyra
|
|
|
31
50
|
# both project through here, whatever the projection mode.
|
|
32
51
|
previous = Thread.current[:lyra_bypass_read_override]
|
|
33
52
|
Thread.current[:lyra_bypass_read_override] = true
|
|
34
|
-
Lyra::PurposeBoundReads.internal do
|
|
53
|
+
Lyra::PurposeBoundReads.internal(INTERNAL) do
|
|
35
54
|
case operation
|
|
36
55
|
when :create
|
|
37
56
|
project_create(model_class, result)
|
|
@@ -54,22 +73,150 @@ module Lyra
|
|
|
54
73
|
def project_create(model_class, result)
|
|
55
74
|
# Encrypted attributes arrive from the log as ciphertext; Active
|
|
56
75
|
# Record encrypts what it inserts, so they are decrypted first.
|
|
57
|
-
attributes
|
|
76
|
+
# String keys throughout: an event's attributes come from the store, or
|
|
77
|
+
# from a record, with string keys, and a symbol-keyed default added
|
|
78
|
+
# beside a string-keyed value overwrote it when the keys were
|
|
79
|
+
# stringified (every created_at and updated_at became the projection
|
|
80
|
+
# time).
|
|
81
|
+
attributes = Lyra::EncryptedAttributes.open(model_class, result.attributes.dup).stringify_keys
|
|
58
82
|
|
|
59
|
-
# Ensure we have
|
|
60
|
-
|
|
83
|
+
# Ensure we have the record's key (the primary key column, whatever
|
|
84
|
+
# its name)
|
|
85
|
+
raise "Cannot project create without ID" unless attributes[Lyra.key_column(model_class)]
|
|
61
86
|
|
|
62
|
-
#
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
attributes[:updated_at] ||= now
|
|
87
|
+
# No default timestamps: a create's event carries created_at and
|
|
88
|
+
# updated_at like any other column (from 1.0.0), and a value made up
|
|
89
|
+
# here would differ from the row's.
|
|
66
90
|
|
|
67
91
|
# Convert to database-safe format
|
|
68
92
|
insert_attrs = sanitize_attributes(model_class, attributes)
|
|
69
93
|
|
|
70
94
|
# Use insert to bypass callbacks
|
|
71
95
|
# Lyra.projection_write: skips strict access and publishes no bypass events
|
|
72
|
-
|
|
96
|
+
#
|
|
97
|
+
# ON CONFLICT DO NOTHING, so a create projected again (a replay, a
|
|
98
|
+
# job after a write that projected its own row) inserts nothing.
|
|
99
|
+
inserted = Lyra.projection_write(INTERNAL) { model_class.insert(insert_attrs) }
|
|
100
|
+
refuse_skipped_create!(model_class, insert_attrs, inserted)
|
|
101
|
+
inserted
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# A create that inserted nothing (no key returned) although no row has
|
|
105
|
+
# its key (on the table, whatever the record's class now: an STI
|
|
106
|
+
# record whose type changed is still its row) conflicted on another
|
|
107
|
+
# unique index: a row another record holds. ON CONFLICT DO NOTHING
|
|
108
|
+
# skipped it silently: a projection job that ran before the job of
|
|
109
|
+
# the holder its record replaced (the holder's row busy when the
|
|
110
|
+
# write checked it) completed and moved its checkpoint past the
|
|
111
|
+
# create, and the log held a record the table never got. One query,
|
|
112
|
+
# only when nothing was inserted; then the rows holding its values
|
|
113
|
+
# are read (one query per unique index whose tuple the create gives
|
|
114
|
+
# and that a NULL in it does not exempt) and their records' streams:
|
|
115
|
+
#
|
|
116
|
+
# - Every such row is of a record whose stream no longer holds the
|
|
117
|
+
# value (destroyed, or moved): its projection lags. ES-Async brings
|
|
118
|
+
# those rows up to date with their streams here, where nothing has
|
|
119
|
+
# to be waited for (UniqueIndexes.bring_rows_up_to_date: a holder
|
|
120
|
+
# whose own job was lost no longer blocks), and inserts again. A row
|
|
121
|
+
# still held by another transaction raises ProjectionBehind, which
|
|
122
|
+
# AsyncProjectionJob retries (BEHIND_ATTEMPTS: about a day, then
|
|
123
|
+
# reported and given up); ES-Sync's write fails with it.
|
|
124
|
+
# - A live record holds the value in the log too, or a row with no
|
|
125
|
+
# stream (Monitor's table holds it), or no holder is found once the
|
|
126
|
+
# insert has been tried again (an expression index, which Lyra does
|
|
127
|
+
# not read): DuplicateInLog. No projection can hold both. Reported
|
|
128
|
+
# once per record and process (report_duplicate); raised for a job,
|
|
129
|
+
# which discards it, and for ES-Sync's write.
|
|
130
|
+
#
|
|
131
|
+
# In ES-Lazy's catch-up neither is raised: raising there would stop
|
|
132
|
+
# every read. It is reported (once) and the catch-up goes on.
|
|
133
|
+
def refuse_skipped_create!(model_class, attributes, inserted)
|
|
134
|
+
key = model_class.primary_key
|
|
135
|
+
return unless key.is_a?(String) && inserted.respond_to?(:rows) && inserted.rows.empty?
|
|
136
|
+
return unless model_class.with_connection(&:supports_insert_returning?)
|
|
137
|
+
|
|
138
|
+
table = model_class.base_class
|
|
139
|
+
id = attributes[key]
|
|
140
|
+
present = table.uncached { table.unscoped.where(key => id).exists? }
|
|
141
|
+
return if present
|
|
142
|
+
|
|
143
|
+
holders = holder_rows(table, attributes)
|
|
144
|
+
lagging = holders.any? && holders.all? { |holder, held| lagging?(table, holder, held) }
|
|
145
|
+
ids = holders.map(&:first).uniq
|
|
146
|
+
# ES-Async: the lagging rows brought up to date, then the insert
|
|
147
|
+
# again; with no holder found, the insert again (a row gone since).
|
|
148
|
+
if holders.empty? || (lagging && UniqueIndexes.bring_rows_up_to_date(table, ids.map(&:to_s)).any?)
|
|
149
|
+
again = Lyra.projection_write(INTERNAL) { model_class.insert(attributes) }
|
|
150
|
+
return if again.rows.any?
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
error = if lagging
|
|
154
|
+
ProjectionBehind.new("Lyra: the projection of #{model_class.name} #{id}'s create met the row of " \
|
|
155
|
+
"#{table.name} #{ids.join(', ')}, which gave the value up but whose projection lags " \
|
|
156
|
+
"(its row held by another transaction); nothing was inserted")
|
|
157
|
+
else
|
|
158
|
+
DuplicateInLog.new("Lyra: the projection of #{model_class.name} #{id}'s create met a value of a unique index " \
|
|
159
|
+
"of #{model_class.table_name} that #{holders.empty? ? 'a row Lyra cannot match (an expression index)' : 'another live record holds in the log too'} " \
|
|
160
|
+
"holds; nothing was inserted")
|
|
161
|
+
end
|
|
162
|
+
lazy = Lyra.event_sourcing_mode? && Lyra.config.projection_mode == :lazy
|
|
163
|
+
report_duplicate(table, id, error) if !lagging || lazy
|
|
164
|
+
raise error unless lazy
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
# [[primary key, { column => value }]] of the rows that hold one of
|
|
168
|
+
# +attributes+' unique values: one query per unique index (other than
|
|
169
|
+
# the primary key, on columns) whose columns the create gives, a
|
|
170
|
+
# tuple with a NULL left out unless the index is NULLS NOT DISTINCT
|
|
171
|
+
# (it conflicts with nothing).
|
|
172
|
+
def holder_rows(table, attributes)
|
|
173
|
+
key = table.primary_key
|
|
174
|
+
indexes = table.with_connection { |c| c.indexes(table.table_name) }.select do |index|
|
|
175
|
+
index.unique && index.columns.is_a?(Array) && index.columns != [key]
|
|
176
|
+
end
|
|
177
|
+
indexes.flat_map do |index|
|
|
178
|
+
next [] unless index.columns.all? { attributes.key?(_1) }
|
|
179
|
+
|
|
180
|
+
distinct = !(index.respond_to?(:nulls_not_distinct) && index.nulls_not_distinct)
|
|
181
|
+
next [] if distinct && index.columns.any? { attributes[_1].nil? }
|
|
182
|
+
|
|
183
|
+
scope = table.unscoped.where(index.columns.to_h { [_1, attributes[_1]] })
|
|
184
|
+
scope = scope.where(Arel.sql(index.where)) if index.where
|
|
185
|
+
table.uncached { scope.pluck(key, *index.columns) }.map { |id, *values| [id, index.columns.zip(values).to_h] }
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
# Whether the row of +id+, holding +held+, is of a record whose stream
|
|
190
|
+
# no longer holds those values: destroyed, or moved to another.
|
|
191
|
+
def lagging?(table, id, held)
|
|
192
|
+
return true if Lyra::StreamTail.destroyed?(Lyra.stream_name(table, id))
|
|
193
|
+
|
|
194
|
+
now = Lyra::PurposeBoundReads.internal(INTERNAL) { CachedProjection.find(table, id) }
|
|
195
|
+
return false unless now # no stream: a row the log does not know, which holds the value
|
|
196
|
+
|
|
197
|
+
reader = TypedValues::EventReader.new(table)
|
|
198
|
+
held.any? { |column, value| UniqueIndexes.key(table, column, reader.read(now, column)) != UniqueIndexes.key(table, column, value) }
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
# How many creates report_duplicate remembers, per process.
|
|
202
|
+
REPORTED_LIMIT = 10_000
|
|
203
|
+
|
|
204
|
+
# A create the projection could not insert: logged at error level once
|
|
205
|
+
# per record and process (debug after), and handed to
|
|
206
|
+
# config.projection_error_handler once. The records remembered are
|
|
207
|
+
# bounded (REPORTED_LIMIT): past it, they are forgotten and each is
|
|
208
|
+
# reported once more.
|
|
209
|
+
def report_duplicate(table, id, error)
|
|
210
|
+
reported = (@reported_duplicates ||= Concurrent::Set.new)
|
|
211
|
+
reported.clear if reported.size >= REPORTED_LIMIT
|
|
212
|
+
first = reported.add?([table.name, id.to_s])
|
|
213
|
+
logger = defined?(Rails) && Rails.logger
|
|
214
|
+
if first
|
|
215
|
+
logger&.error(error.message)
|
|
216
|
+
Lyra.config.projection_error_handler&.call(error, nil, :create)
|
|
217
|
+
else
|
|
218
|
+
logger&.debug(error.message)
|
|
219
|
+
end
|
|
73
220
|
end
|
|
74
221
|
|
|
75
222
|
# Update an existing record in the model table
|
|
@@ -92,13 +239,15 @@ module Lyra
|
|
|
92
239
|
updates[field.to_sym] = new_value
|
|
93
240
|
end
|
|
94
241
|
|
|
95
|
-
#
|
|
96
|
-
|
|
242
|
+
# updated_at is set only when the event changed it: an update that
|
|
243
|
+
# left it alone (update_columns, an update_all that did not set it)
|
|
244
|
+
# left it alone in the row too, so stamping the projection time here
|
|
245
|
+
# made a rebuilt row differ from its events.
|
|
97
246
|
updates = Lyra::EncryptedAttributes.open(model_class, updates)
|
|
98
247
|
|
|
99
248
|
# Use update_all to bypass callbacks
|
|
100
249
|
# Lyra.projection_write: skips strict access and publishes no bypass events
|
|
101
|
-
Lyra.projection_write { model_class.where(
|
|
250
|
+
Lyra.projection_write(INTERNAL) { model_class.where(Lyra.key_column(model_class) => model_id).update_all(updates) }
|
|
102
251
|
end
|
|
103
252
|
|
|
104
253
|
# Delete a record from the model table
|
|
@@ -112,9 +261,11 @@ module Lyra
|
|
|
112
261
|
|
|
113
262
|
# Use delete_all to bypass callbacks
|
|
114
263
|
# Lyra.projection_write: skips strict access and publishes no bypass events
|
|
115
|
-
Lyra.projection_write { model_class.where(
|
|
264
|
+
Lyra.projection_write(INTERNAL) { model_class.where(Lyra.key_column(model_class) => model_id).delete_all }
|
|
116
265
|
end
|
|
117
266
|
|
|
267
|
+
private :project_create, :project_update, :project_destroy, :refuse_skipped_create!, :holder_rows, :lagging?, :report_duplicate
|
|
268
|
+
|
|
118
269
|
private
|
|
119
270
|
|
|
120
271
|
# Sanitize attributes for database insertion
|
|
@@ -22,6 +22,10 @@ module Lyra
|
|
|
22
22
|
# rake lyra:projections:rebuild # all monitored models
|
|
23
23
|
#
|
|
24
24
|
class Rebuild
|
|
25
|
+
# A rebuild that would undo what the log does not record
|
|
26
|
+
# (config.record_touches false).
|
|
27
|
+
class Refused < StandardError; end
|
|
28
|
+
|
|
25
29
|
class << self
|
|
26
30
|
# Rebuild a single model's read-model table from its event streams.
|
|
27
31
|
#
|
|
@@ -30,13 +34,29 @@ module Lyra
|
|
|
30
34
|
# set false to reconstruct only the streamed rows in place
|
|
31
35
|
# @return [Hash] statistics for the rebuild
|
|
32
36
|
def rebuild(model_class, truncate: true)
|
|
37
|
+
# With touches left out of the log, the rows' updated_at is newer
|
|
38
|
+
# than the events say, and a rebuild would set every touched row back.
|
|
39
|
+
if Lyra.config.record_touches == false
|
|
40
|
+
raise Lyra::Projections::Rebuild::Refused,
|
|
41
|
+
"Lyra refuses to rebuild #{model_class.name} from its events: config.record_touches is false, so " \
|
|
42
|
+
"the log misses touches and the rebuild would set each touched row's updated_at back. Set " \
|
|
43
|
+
"config.record_touches = true and run lyra:repair first."
|
|
44
|
+
end
|
|
45
|
+
live = Lyra::TouchRegistry.live_processes
|
|
46
|
+
if live.any?
|
|
47
|
+
raise Lyra::Projections::Rebuild::Refused,
|
|
48
|
+
"Lyra refuses to rebuild #{model_class.name} from its events: processes leave touches out of the " \
|
|
49
|
+
"log (config.record_touches = false): #{Lyra::TouchRegistry.describe(live)}. The rebuild would set " \
|
|
50
|
+
"each touched row's updated_at back. #{Lyra::TouchRegistry::STEPS}"
|
|
51
|
+
end
|
|
52
|
+
|
|
33
53
|
stats = {
|
|
34
54
|
model: model_class.name,
|
|
35
55
|
streams: 0, events: 0, records: 0, destroyed: 0
|
|
36
56
|
}
|
|
37
57
|
rebuilt_ids = []
|
|
38
58
|
|
|
39
|
-
Lyra::PurposeBoundReads.internal { rebuild_in_transaction(model_class, truncate, stats, rebuilt_ids) }
|
|
59
|
+
Lyra::PurposeBoundReads.internal(INTERNAL) { rebuild_in_transaction(model_class, truncate, stats, rebuilt_ids) }
|
|
40
60
|
end
|
|
41
61
|
|
|
42
62
|
def rebuild_in_transaction(model_class, truncate, stats, rebuilt_ids)
|
|
@@ -92,12 +112,16 @@ module Lyra
|
|
|
92
112
|
# event the row already reflects is harmless: a create of an existing
|
|
93
113
|
# row inserts nothing and an update sets the same values, so callers
|
|
94
114
|
# may start from an earlier event than strictly needed.
|
|
95
|
-
|
|
115
|
+
# Projects the events it is given, whether or not they are in the log,
|
|
116
|
+
# so only Lyra calls it (Lyra::INTERNAL).
|
|
117
|
+
def project_events(model_class, events, token = nil)
|
|
118
|
+
raise InternalApiError, "Rebuild.project_events is for Lyra's own projections" unless token.equal?(INTERNAL)
|
|
119
|
+
|
|
96
120
|
events.count do |event|
|
|
97
121
|
operation = operation_for(event)
|
|
98
122
|
next false unless operation
|
|
99
123
|
|
|
100
|
-
ModelProjection.project(model_class, operation, build_result_from_event(event))
|
|
124
|
+
ModelProjection.project(model_class, operation, build_result_from_event(event), INTERNAL)
|
|
101
125
|
true
|
|
102
126
|
end
|
|
103
127
|
end
|
|
@@ -109,7 +133,7 @@ module Lyra
|
|
|
109
133
|
def replay_stream(model_class, id)
|
|
110
134
|
stream_name = Lyra.stream_name(model_class, id)
|
|
111
135
|
events = Lyra.config.event_store.read.stream(stream_name).to_a
|
|
112
|
-
replayed = project_events(model_class, events)
|
|
136
|
+
replayed = project_events(model_class, events, INTERNAL)
|
|
113
137
|
|
|
114
138
|
last = events.reverse_each.find { Lyra::Event.operation_of(_1) }
|
|
115
139
|
present = !(last && Lyra::Event.operation_of(last) == :destroyed)
|
|
@@ -132,7 +156,11 @@ module Lyra
|
|
|
132
156
|
def build_result_from_event(event)
|
|
133
157
|
Struct.new(:events, :attributes, :success?, keyword_init: true).new(
|
|
134
158
|
events: [event],
|
|
135
|
-
attributes:
|
|
159
|
+
attributes: if %i[created imported].include?(Lyra::Event.operation_of(event))
|
|
160
|
+
Lyra::Event.created_attributes(event.data)
|
|
161
|
+
else
|
|
162
|
+
event.data[:attributes] || event.data["attributes"] || {}
|
|
163
|
+
end,
|
|
136
164
|
success?: true
|
|
137
165
|
)
|
|
138
166
|
end
|
|
@@ -149,7 +177,7 @@ module Lyra
|
|
|
149
177
|
# Clear the read-model table, bypassing callbacks/strict-access guards
|
|
150
178
|
# (this is a legitimate bulk projection operation, like ModelProjection).
|
|
151
179
|
def clear_read_model(model_class)
|
|
152
|
-
Lyra.projection_write { model_class.unscoped.delete_all }
|
|
180
|
+
Lyra.projection_write(INTERNAL) { model_class.unscoped.delete_all }
|
|
153
181
|
end
|
|
154
182
|
|
|
155
183
|
# Drop any cached (ES-NoProj) reconstructions so reads reflect the rebuild.
|