orfeas_lyra 0.9.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +2021 -17
- data/README.md +22 -9
- data/Rakefile +40 -1
- data/app/controllers/lyra/application_controller.rb +127 -0
- data/app/controllers/lyra/dashboard_controller.rb +33 -31
- data/app/controllers/lyra/flow_controller.rb +32 -34
- data/app/controllers/lyra/privacy_controller.rb +57 -18
- data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
- data/app/views/lyra/dashboard/index.html.erb +1 -0
- data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
- data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
- data/app/views/lyra/privacy/policy.html.erb +7 -0
- data/app/views/lyra/privacy/rights.html.erb +78 -0
- data/config/privacy_policies.rb +0 -3
- data/config/routes.rb +1 -0
- data/lib/lyra/access_log.rb +187 -6
- data/lib/lyra/advisory_lock.rb +236 -0
- data/lib/lyra/aggregate.rb +1 -1
- data/lib/lyra/associations/habtm_links.rb +382 -19
- data/lib/lyra/associations/maintenance.rb +229 -0
- data/lib/lyra/attribution.rb +48 -0
- data/lib/lyra/bypass_events.rb +446 -46
- data/lib/lyra/command_handler.rb +19 -14
- data/lib/lyra/configuration.rb +237 -10
- data/lib/lyra/consistency/read_your_writes.rb +4 -2
- data/lib/lyra/correlation.rb +121 -0
- data/lib/lyra/domain_events.rb +4 -0
- data/lib/lyra/dual_view.rb +53 -16
- data/lib/lyra/encrypted_attributes.rb +5 -0
- data/lib/lyra/engine.rb +67 -0
- data/lib/lyra/erasure.rb +1011 -106
- data/lib/lyra/event.rb +35 -1
- data/lib/lyra/event_data.rb +38 -0
- data/lib/lyra/event_flow.rb +31 -15
- data/lib/lyra/event_store_adapter.rb +28 -0
- data/lib/lyra/event_window.rb +58 -0
- data/lib/lyra/genesis.rb +548 -44
- data/lib/lyra/interceptors/association_interceptor.rb +689 -34
- data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
- data/lib/lyra/internal.rb +26 -0
- data/lib/lyra/mode_sync.rb +51 -3
- data/lib/lyra/mode_transition.rb +257 -30
- data/lib/lyra/model_selection.rb +35 -0
- data/lib/lyra/own_connection.rb +17 -2
- data/lib/lyra/privacy/adapters/pam.rb +25 -2
- data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
- data/lib/lyra/privacy/interface.rb +27 -7
- data/lib/lyra/privacy/subject_links.rb +143 -0
- data/lib/lyra/privacy/subject_records.rb +369 -0
- data/lib/lyra/projection.rb +7 -1
- data/lib/lyra/projections/async_checkpoint.rb +32 -0
- data/lib/lyra/projections/async_projection_job.rb +166 -23
- data/lib/lyra/projections/cached_joins.rb +33 -18
- data/lib/lyra/projections/cached_projection.rb +74 -33
- data/lib/lyra/projections/cached_relation.rb +3175 -321
- data/lib/lyra/projections/default_scopes.rb +254 -0
- data/lib/lyra/projections/deferred_relation.rb +23 -3
- data/lib/lyra/projections/event_store_reader.rb +47 -8
- data/lib/lyra/projections/foreign_keys.rb +92 -0
- data/lib/lyra/projections/lazy_projection.rb +165 -27
- data/lib/lyra/projections/model_projection.rb +165 -14
- data/lib/lyra/projections/rebuild.rb +34 -6
- data/lib/lyra/projections/reference_links.rb +463 -72
- data/lib/lyra/projections/typed_values.rb +629 -0
- data/lib/lyra/projections/unique_indexes.rb +595 -0
- data/lib/lyra/projections/uniqueness.rb +784 -0
- data/lib/lyra/projections/value_digest.rb +93 -0
- data/lib/lyra/purpose_bound_reads.rb +912 -77
- data/lib/lyra/repair.rb +119 -28
- data/lib/lyra/retention.rb +114 -8
- data/lib/lyra/rights/consent_store.rb +415 -0
- data/lib/lyra/rights/erasure_guard.rb +135 -0
- data/lib/lyra/rights/pending.rb +74 -0
- data/lib/lyra/rights/queries.rb +269 -0
- data/lib/lyra/rights/store.rb +512 -0
- data/lib/lyra/rights/subject_keys.rb +224 -0
- data/lib/lyra/rights.rb +281 -0
- data/lib/lyra/saves_without_effect.rb +69 -0
- data/lib/lyra/stream_tail.rb +319 -0
- data/lib/lyra/strict_data_access.rb +916 -106
- data/lib/lyra/temporal.rb +4 -2
- data/lib/lyra/touch_registry.rb +404 -0
- data/lib/lyra/verification/bypass_workflow.rb +452 -42
- data/lib/lyra/verification/rights_workflows.rb +170 -0
- data/lib/lyra/version.rb +1 -1
- data/lib/lyra/visualization/event_graph.rb +1 -1
- data/lib/lyra/visualization/timeline.rb +2 -2
- data/lib/lyra.rb +43 -2
- data/lib/tasks/lyra_mode.rake +110 -8
- data/lib/tasks/lyra_projections.rake +43 -2
- metadata +34 -4
- data/lib/lyra/associations/event_aware.rb +0 -224
|
@@ -17,6 +17,16 @@ module Lyra
|
|
|
17
17
|
# store's own stream table; the event is not copied, no table is added, and
|
|
18
18
|
# the index is append-only, rebuildable from the events.
|
|
19
19
|
#
|
|
20
|
+
# An event that sets a column of one of the model's unique indexes
|
|
21
|
+
# (UniqueIndexes) is linked the same way into a value stream,
|
|
22
|
+
# "Model!column$<digest>", named by a digest of the value as the index
|
|
23
|
+
# compares it, never by the value itself (an email, an IBAN): the
|
|
24
|
+
# unique-index check of ES-NoProj's, ES-Async's and ES-Lazy's writes
|
|
25
|
+
# reads it (record_ids_in). The checkpoint names the columns whose value
|
|
26
|
+
# streams it covers; one stored
|
|
27
|
+
# before a column was indexed (or before 1.0.0) is caught up from the
|
|
28
|
+
# first event.
|
|
29
|
+
#
|
|
20
30
|
# A lookup by that key reads the reference stream for candidates, rebuilds
|
|
21
31
|
# each with CachedProjection.find and keeps the records that still match:
|
|
22
32
|
# a child moved to another parent stays linked under the old one (streams
|
|
@@ -25,7 +35,11 @@ module Lyra
|
|
|
25
35
|
# the candidates include every match and the answer is exact.
|
|
26
36
|
#
|
|
27
37
|
# Completeness. Links are written while config.reference_links says so
|
|
28
|
-
# (:auto, the default: in ES-NoProj only, so no other mode pays for them
|
|
38
|
+
# (:auto, the default: in ES-NoProj only, so no other mode pays for them;
|
|
39
|
+
# the value streams also in ES-Async and ES-Lazy, whose unique-index
|
|
40
|
+
# check reads them, values_enabled?: there a model without a unique
|
|
41
|
+
# index links nothing, and the foreign keys' checkpoint is left where
|
|
42
|
+
# it was).
|
|
29
43
|
# The switch into ES-NoProj (Lyra::ModeTransition.to!) links every event
|
|
30
44
|
# of every monitored model first, so the index is complete when the mode
|
|
31
45
|
# starts. As a safety net, events written while linking was off are also
|
|
@@ -51,6 +65,21 @@ module Lyra
|
|
|
51
65
|
end
|
|
52
66
|
end
|
|
53
67
|
|
|
68
|
+
# Whether events are linked into their value streams as they are
|
|
69
|
+
# stored (UniqueIndexes: the columns of a model's unique indexes):
|
|
70
|
+
# wherever all links are (enabled?), and in ES-Async and ES-Lazy, whose
|
|
71
|
+
# unique-index check reads the value streams too (their tables lag
|
|
72
|
+
# the event store), while config.reference_links is not false. Only
|
|
73
|
+
# the value streams there: the foreign-key links serve ES-NoProj's
|
|
74
|
+
# reads, which those modes do not make, so a model without a unique
|
|
75
|
+
# index links nothing. ES-Sync checks its table.
|
|
76
|
+
def values_enabled?
|
|
77
|
+
return true if enabled?
|
|
78
|
+
|
|
79
|
+
!(Lyra.config.reference_links == false || Lyra.config.reference_links.nil?) &&
|
|
80
|
+
Lyra.event_sourcing_mode? && %i[async lazy].include?(Lyra.config.projection_mode)
|
|
81
|
+
end
|
|
82
|
+
|
|
54
83
|
# Named after the class that owns the table (the STI base class), as
|
|
55
84
|
# the record streams are (Lyra.stream_name).
|
|
56
85
|
def stream_name(model_class, foreign_key, value)
|
|
@@ -59,22 +88,42 @@ module Lyra
|
|
|
59
88
|
|
|
60
89
|
# Called by Lyra.append_events after +events+ are stored in
|
|
61
90
|
# +stream_name+: link those that set a foreign key, or, with linking
|
|
62
|
-
# off, note that the model's links fell behind in this process.
|
|
91
|
+
# off, note that the model's links fell behind in this process. In
|
|
92
|
+
# ES-Async and ES-Lazy (values_enabled? only) the events that set a
|
|
93
|
+
# column of a unique index are linked into its value streams, and the
|
|
94
|
+
# foreign keys are noted as behind.
|
|
63
95
|
def after_append(events, stream_name, store)
|
|
64
96
|
linking = enabled?
|
|
97
|
+
values = !linking && values_enabled?
|
|
65
98
|
# With linking off, only a model this process took for linked needs
|
|
66
99
|
# noting; otherwise nothing is done, so other modes pay nothing.
|
|
67
|
-
return if !linking && MUTEX.synchronize { ready.empty? }
|
|
100
|
+
return if !linking && !values && MUTEX.synchronize { ready.empty? }
|
|
68
101
|
|
|
69
102
|
model_class = model_for_stream(stream_name)
|
|
70
103
|
return unless model_class
|
|
71
104
|
|
|
105
|
+
events = Array(events)
|
|
72
106
|
if linking
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
107
|
+
# Columns new to this process (UniqueIndexes.facts): linked from the
|
|
108
|
+
# first event, before this write's own links.
|
|
109
|
+
catch_up(model_class) if unlinked?(model_class)
|
|
110
|
+
link_events(events, model_class, store)
|
|
111
|
+
return
|
|
112
|
+
end
|
|
113
|
+
if values && !UniqueIndexes.columns(model_class).empty?
|
|
114
|
+
catch_up(model_class) if unlinked?(model_class)
|
|
115
|
+
link_events(events, model_class, store, keys: false)
|
|
116
|
+
end
|
|
117
|
+
if values
|
|
118
|
+
# Only a model this process took for linked by its foreign keys
|
|
119
|
+
# (in ES-NoProj, earlier) needs noting.
|
|
120
|
+
return unless MUTEX.synchronize { ready.any? { _1.end_with?("|k") } }
|
|
121
|
+
|
|
122
|
+
forget_ready(model_class, keys_only: true) if events.any? { |event| reference_values(model_class, event).any? }
|
|
123
|
+
elsif MUTEX.synchronize { ready.empty? }
|
|
124
|
+
nil
|
|
125
|
+
elsif events.any? { |event| link_streams(model_class, event).any? }
|
|
126
|
+
forget_ready(model_class)
|
|
78
127
|
end
|
|
79
128
|
end
|
|
80
129
|
|
|
@@ -93,7 +142,10 @@ module Lyra
|
|
|
93
142
|
indexed = indexed_conditions(table, conditions)
|
|
94
143
|
return nil if indexed.empty?
|
|
95
144
|
|
|
96
|
-
|
|
145
|
+
# Without the links caught up (another thread is linking), every
|
|
146
|
+
# record is considered.
|
|
147
|
+
return nil unless catch_up(table)
|
|
148
|
+
|
|
97
149
|
# Every condition on an indexed key narrows the candidates: a record
|
|
98
150
|
# that matches them all is in each key's stream, so the streams are
|
|
99
151
|
# intersected (stock_location_id and variant_id together name one
|
|
@@ -115,30 +167,174 @@ module Lyra
|
|
|
115
167
|
# none free) it runs in the caller's transaction, and the model counts
|
|
116
168
|
# as linked only once that transaction commits (Lyra::OwnConnection).
|
|
117
169
|
#
|
|
118
|
-
# One thread per model and process catches up
|
|
119
|
-
#
|
|
120
|
-
#
|
|
121
|
-
|
|
170
|
+
# One thread per model and process catches up. A caller never waits
|
|
171
|
+
# for it, nor for the catch-up's advisory lock, nor for a connection
|
|
172
|
+
# (unless +wait+: an explicit catch-up, ModeTransition's, the rake
|
|
173
|
+
# task's): when another thread or process is catching up, or the pool
|
|
174
|
+
# has no connection free and the lock is not to be had in the
|
|
175
|
+
# caller's transaction, it returns nil and the caller answers without
|
|
176
|
+
# the links (a lookup from every record). A caller that waited held
|
|
177
|
+
# its transaction, and its locks, while it did: with every connection
|
|
178
|
+
# of the pool taken by writers of a model's first use, the writer
|
|
179
|
+
# that caught up inline kept the advisory lock until it committed,
|
|
180
|
+
# and called catch_up again in the same write, waiting on the Ruby
|
|
181
|
+
# mutex of a thread whose connection waited on that lock; PostgreSQL
|
|
182
|
+
# could not see the cycle, and every writer hung. A thread that
|
|
183
|
+
# caught up inline takes the model for linked until its transaction
|
|
184
|
+
# ends; other threads, once it commits.
|
|
185
|
+
#
|
|
186
|
+
# Returns the number of events linked, 0 when there was nothing to
|
|
187
|
+
# do, nil when it did not catch up.
|
|
188
|
+
def catch_up(model_class, force: false, wait: force)
|
|
122
189
|
model_class = owner(model_class)
|
|
123
|
-
return 0 if !force && ready?(model_class)
|
|
190
|
+
return 0 if !force && (ready?(model_class) || caught_up_here?(model_class))
|
|
124
191
|
|
|
125
|
-
lock_for(model_class)
|
|
126
|
-
|
|
192
|
+
mutex = lock_for(model_class)
|
|
193
|
+
if wait
|
|
194
|
+
mutex.lock
|
|
195
|
+
elsif !mutex.try_lock
|
|
196
|
+
return nil
|
|
197
|
+
end
|
|
198
|
+
begin
|
|
199
|
+
return 0 if !force && (ready?(model_class) || caught_up_here?(model_class))
|
|
127
200
|
|
|
201
|
+
key = ready_key(model_class, keys: force || enabled?)
|
|
202
|
+
unlinked = take_unlinked(model_class)
|
|
128
203
|
inline = false
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
204
|
+
begin
|
|
205
|
+
linked = Lyra::OwnConnection.run(wait_for_connection: wait) do |in_caller_transaction|
|
|
206
|
+
inline = in_caller_transaction
|
|
207
|
+
link_and_checkpoint(model_class, force, unlinked, wait: wait)
|
|
208
|
+
end
|
|
209
|
+
rescue StandardError
|
|
210
|
+
unlinked!(model_class, unlinked) if unlinked.any?
|
|
211
|
+
raise
|
|
212
|
+
end
|
|
213
|
+
if linked.nil?
|
|
214
|
+
unlinked!(model_class, unlinked) if unlinked.any?
|
|
215
|
+
return nil
|
|
216
|
+
end
|
|
217
|
+
mark_ready = -> { MUTEX.synchronize { ready << key } }
|
|
218
|
+
if inline
|
|
219
|
+
caught_up_here!(model_class, key)
|
|
220
|
+
Lyra::OwnConnection.after_commit(&mark_ready)
|
|
221
|
+
else
|
|
222
|
+
mark_ready.call
|
|
132
223
|
end
|
|
133
|
-
mark_ready = -> { MUTEX.synchronize { ready << model_class.name } }
|
|
134
|
-
inline ? Lyra::OwnConnection.after_commit(&mark_ready) : mark_ready.call
|
|
135
224
|
linked
|
|
225
|
+
ensure
|
|
226
|
+
mutex.unlock
|
|
227
|
+
end
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
# The columns of +model_class+'s unique indexes whose value streams may
|
|
231
|
+
# miss events (new to this process: UniqueIndexes.facts): its next
|
|
232
|
+
# catch-up links them from the first event.
|
|
233
|
+
def unlinked!(model_class, columns)
|
|
234
|
+
name = owner(model_class).name
|
|
235
|
+
MUTEX.synchronize do
|
|
236
|
+
@unlinked ||= {}
|
|
237
|
+
@unlinked[name] = (@unlinked[name] || []) | columns.map(&:to_s)
|
|
238
|
+
end
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
# Link every value stream of +model_class+ again, from the first event:
|
|
242
|
+
# after a rolling deploy in which processes running a Lyra without value
|
|
243
|
+
# streams, or reading the indexes before a migration added one, wrote
|
|
244
|
+
# (bin/rails lyra:link_values).
|
|
245
|
+
def relink_values!(model_class)
|
|
246
|
+
table = owner(model_class)
|
|
247
|
+
unlinked!(table, UniqueIndexes.columns(table))
|
|
248
|
+
catch_up(table, wait: true)
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
# The ids of the records whose events are linked in every one of
|
|
252
|
+
# +streams+ could be (a superset): those of the stream with the
|
|
253
|
+
# fewest links, whose events are read (each names its record:
|
|
254
|
+
# model_id); none when one of them has no link. One query when no
|
|
255
|
+
# event set the value (two for several streams), one more for the
|
|
256
|
+
# events otherwise. For the unique-index check (UniqueIndexes): its
|
|
257
|
+
# value streams.
|
|
258
|
+
def record_ids_in(model_class, streams)
|
|
259
|
+
stream = streams.one? ? streams.first : fewest_links(streams)
|
|
260
|
+
return [] unless stream
|
|
261
|
+
|
|
262
|
+
# Never from the query cache: read under the value's lock, which may
|
|
263
|
+
# have waited for a writer that linked the value since.
|
|
264
|
+
event_ids = ActiveRecord::Base.with_connection do |connection|
|
|
265
|
+
connection.uncached do
|
|
266
|
+
connection.select_values("SELECT event_id FROM event_store_events_in_streams WHERE stream = #{connection.quote(stream)}")
|
|
267
|
+
end
|
|
268
|
+
end
|
|
269
|
+
return [] if event_ids.empty?
|
|
270
|
+
|
|
271
|
+
events = Lyra.config.event_store.read.events(event_ids.uniq).to_a
|
|
272
|
+
ids = events.map { |event| event.data[:model_id] || event.data["model_id"] }
|
|
273
|
+
return candidate_streams(owner(model_class), stream) if ids.any?(&:nil?) || ids.size < event_ids.uniq.size
|
|
274
|
+
|
|
275
|
+
ids.map(&:to_s).uniq
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
# The ids of the records whose events are linked in any of +streams+:
|
|
279
|
+
# one query per 1,000 streams, one per 500 events read (each names its
|
|
280
|
+
# record). For a bulk insert's unique-index check (UniqueIndexes
|
|
281
|
+
# .candidate_records).
|
|
282
|
+
def record_ids_in_any(model_class, streams)
|
|
283
|
+
rows = ActiveRecord::Base.with_connection do |connection|
|
|
284
|
+
streams.each_slice(1_000).flat_map do |slice|
|
|
285
|
+
connection.select_rows(
|
|
286
|
+
"SELECT stream, event_id FROM event_store_events_in_streams WHERE stream IN (#{slice.map { connection.quote(_1) }.join(', ')})"
|
|
287
|
+
)
|
|
288
|
+
end
|
|
289
|
+
end
|
|
290
|
+
return [] if rows.empty?
|
|
291
|
+
|
|
292
|
+
ids = []
|
|
293
|
+
unnamed = []
|
|
294
|
+
rows.map(&:last).uniq.each_slice(500) do |event_ids|
|
|
295
|
+
events = Lyra.config.event_store.read.events(event_ids).to_a
|
|
296
|
+
events.each do |event|
|
|
297
|
+
id = event.data[:model_id] || event.data["model_id"]
|
|
298
|
+
id.nil? ? unnamed << event.event_id : ids << id.to_s
|
|
299
|
+
end
|
|
300
|
+
unnamed.concat(event_ids - events.map(&:event_id))
|
|
301
|
+
end
|
|
302
|
+
if unnamed.any?
|
|
303
|
+
rows.each { |stream, event_id| ids.concat(candidate_streams(owner(model_class), stream)) if unnamed.include?(event_id) }
|
|
304
|
+
end
|
|
305
|
+
ids.uniq
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
# After +events+ were rewritten in place (Lyra::Erasure), their value
|
|
309
|
+
# stream links follow their new data: the links of the values they
|
|
310
|
+
# held are deleted (no digest of an erased value is kept) and those
|
|
311
|
+
# of the values they hold now inserted, where links are written or
|
|
312
|
+
# the events had some (the rewritten events are behind the
|
|
313
|
+
# checkpoint, and no catch-up would link them again).
|
|
314
|
+
def relink_values(events)
|
|
315
|
+
events.group_by { |event| value_owner(event) }.each do |table, rewritten|
|
|
316
|
+
next unless table && !UniqueIndexes.columns(table).empty?
|
|
317
|
+
|
|
318
|
+
ActiveRecord::Base.with_connection do |connection|
|
|
319
|
+
pattern = connection.quote("#{ActiveRecord::Base.sanitize_sql_like(Lyra.stream_owner_name(table))}!%")
|
|
320
|
+
ids = rewritten.map { connection.quote(_1.event_id) }.join(", ")
|
|
321
|
+
deleted = connection.exec_delete(
|
|
322
|
+
"DELETE FROM event_store_events_in_streams WHERE event_id IN (#{ids}) AND stream LIKE #{pattern}", "Lyra value links"
|
|
323
|
+
)
|
|
324
|
+
next unless deleted.positive? || values_enabled?
|
|
325
|
+
|
|
326
|
+
pairs = rewritten.flat_map { |event| UniqueIndexes.streams_for(table, set_values(event)).map { [_1, event.event_id] } }
|
|
327
|
+
insert_links(connection, pairs) if pairs.any?
|
|
328
|
+
end
|
|
136
329
|
end
|
|
137
330
|
end
|
|
138
331
|
|
|
139
332
|
# Forget which models are linked in this process (tests, truncation).
|
|
140
333
|
def reset!
|
|
141
|
-
MUTEX.synchronize
|
|
334
|
+
MUTEX.synchronize do
|
|
335
|
+
@ready = Set.new
|
|
336
|
+
@unlinked = {}
|
|
337
|
+
end
|
|
142
338
|
end
|
|
143
339
|
|
|
144
340
|
# Foreign keys of +model_class+ the index covers.
|
|
@@ -156,8 +352,53 @@ module Lyra
|
|
|
156
352
|
model_class.respond_to?(:base_class) ? model_class.base_class : model_class
|
|
157
353
|
end
|
|
158
354
|
|
|
355
|
+
# Linked, in this process, as the model's foreign keys and unique-index
|
|
356
|
+
# columns are now, under the digest key in use: a column added to an
|
|
357
|
+
# index (or a key rotated) makes it not ready again.
|
|
358
|
+
# Where only the value streams are written (values_enabled? without
|
|
359
|
+
# enabled?: ES-Async, ES-Lazy), a model linked with its foreign keys
|
|
360
|
+
# is ready too; the reverse does not hold.
|
|
159
361
|
def ready?(model_class)
|
|
160
|
-
|
|
362
|
+
key = ready_key(model_class)
|
|
363
|
+
also = key.end_with?("|v") ? key.sub(/\|v\z/, "|k") : nil
|
|
364
|
+
MUTEX.synchronize { (ready.include?(key) || (also && ready.include?(also))) && !(@unlinked || {}).key?(model_class.name) }
|
|
365
|
+
end
|
|
366
|
+
|
|
367
|
+
# This thread caught the model up inline, in the transaction it is in
|
|
368
|
+
# (caught_up_here!): until that transaction ends.
|
|
369
|
+
def caught_up_here?(model_class)
|
|
370
|
+
(Thread.current[:lyra_links_caught_up] || {}).key?(ready_key(model_class))
|
|
371
|
+
end
|
|
372
|
+
|
|
373
|
+
def caught_up_here!(model_class, key)
|
|
374
|
+
here = (Thread.current[:lyra_links_caught_up] ||= {})
|
|
375
|
+
here[key] = true
|
|
376
|
+
transaction = model_class.current_transaction
|
|
377
|
+
forget = -> { here.delete(key) }
|
|
378
|
+
transaction.after_commit(&forget)
|
|
379
|
+
transaction.after_rollback(&forget)
|
|
380
|
+
end
|
|
381
|
+
|
|
382
|
+
# Ends in "|k" where the foreign keys are linked too (enabled?), "|v"
|
|
383
|
+
# where only the value streams are.
|
|
384
|
+
def ready_key(model_class, keys: enabled?)
|
|
385
|
+
columns = UniqueIndexes.columns(model_class)
|
|
386
|
+
kind = keys ? "k" : "v"
|
|
387
|
+
columns.empty? ? "#{model_class.name}||#{kind}" : "#{model_class.name}|#{columns.join(',')}|#{ValueDigest.key_id}|#{kind}"
|
|
388
|
+
end
|
|
389
|
+
|
|
390
|
+
# +keys_only+: the foreign keys fell behind, the value streams did not.
|
|
391
|
+
def forget_ready(model_class, keys_only: false)
|
|
392
|
+
prefix = "#{owner(model_class).name}|"
|
|
393
|
+
MUTEX.synchronize { ready.delete_if { _1.start_with?(prefix) && (!keys_only || _1.end_with?("|k")) } }
|
|
394
|
+
end
|
|
395
|
+
|
|
396
|
+
def unlinked?(model_class)
|
|
397
|
+
MUTEX.synchronize { (@unlinked || {}).key?(owner(model_class).name) }
|
|
398
|
+
end
|
|
399
|
+
|
|
400
|
+
def take_unlinked(model_class)
|
|
401
|
+
MUTEX.synchronize { (@unlinked || {}).delete(model_class.name) || [] }
|
|
161
402
|
end
|
|
162
403
|
|
|
163
404
|
# The per-model lock of #catch_up.
|
|
@@ -165,21 +406,110 @@ module Lyra
|
|
|
165
406
|
MUTEX.synchronize { (@locks ||= {})[model_class.name] ||= Mutex.new }
|
|
166
407
|
end
|
|
167
408
|
|
|
168
|
-
|
|
409
|
+
# The foreign keys from the checkpoint's position; each unique-index
|
|
410
|
+
# column from its own (columns linked under another digest key, or new to
|
|
411
|
+
# this process, from the first event, their old links deleted), so a
|
|
412
|
+
# column added to an index does not make the foreign keys be read again.
|
|
413
|
+
def link_and_checkpoint(model_class, force, unlinked = [], wait: true)
|
|
169
414
|
ActiveRecord::Base.with_connection do |connection|
|
|
170
415
|
ActiveRecord::Base.transaction do
|
|
171
|
-
|
|
172
|
-
|
|
416
|
+
lock = "#{LOCK}/#{model_class.name}"
|
|
417
|
+
if wait
|
|
418
|
+
Lyra::AdvisoryLock.xact_lock(connection, lock, purpose: "reference links")
|
|
419
|
+
elsif !Lyra::AdvisoryLock.try_xact_lock(connection, lock, purpose: "reference links")
|
|
420
|
+
next nil
|
|
421
|
+
end
|
|
422
|
+
stored = checkpoint_state(model_class)
|
|
423
|
+
position = force ? 0 : stored[:position]
|
|
173
424
|
upto = connection.select_value("SELECT MAX(id) FROM event_store_events").to_i
|
|
174
|
-
next 0
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
425
|
+
next 0 if upto.zero?
|
|
426
|
+
|
|
427
|
+
kid = ValueDigest.key_id
|
|
428
|
+
columns = UniqueIndexes.columns(model_class)
|
|
429
|
+
relinked = columns.select do |column|
|
|
430
|
+
entry = stored[:values][column]
|
|
431
|
+
# No entry: linked, if at all, under a key no checkpoint names.
|
|
432
|
+
force || unlinked.include?(column) || entry.nil? || entry["key"] != kid
|
|
433
|
+
end
|
|
434
|
+
from = columns.to_h do |column|
|
|
435
|
+
entry = stored[:values][column]
|
|
436
|
+
[column, relinked.include?(column) || entry.nil? ? 0 : entry["position"].to_i]
|
|
437
|
+
end
|
|
438
|
+
drop_value_links(connection, model_class, relinked) if relinked.any?
|
|
439
|
+
behind = from.select { |_, at| upto > at }
|
|
440
|
+
current = stored[:values].keys.sort == columns.sort
|
|
441
|
+
# The foreign keys where they are linked (enabled?) or in a
|
|
442
|
+
# forced pass (the switch into ES-NoProj); in ES-Async and
|
|
443
|
+
# ES-Lazy the value streams alone, and the keys' position is
|
|
444
|
+
# kept.
|
|
445
|
+
keys = force || enabled?
|
|
446
|
+
next 0 unless (keys && upto > position) || behind.any? || relinked.any? || !current
|
|
447
|
+
|
|
448
|
+
# One read of the events behind, for both kinds of link; none
|
|
449
|
+
# for the foreign keys of a model that has none.
|
|
450
|
+
keys_behind = keys && upto > position && foreign_keys(model_class).any? ? position : nil
|
|
451
|
+
linked = keys_behind || behind.any? ? link_since(connection, model_class, keys_behind, behind, upto) : 0
|
|
452
|
+
store_checkpoint(model_class, keys ? [upto, position].max : stored[:position],
|
|
453
|
+
columns.to_h { [_1, { "position" => upto, "key" => kid }] })
|
|
178
454
|
linked
|
|
179
455
|
end
|
|
180
456
|
end
|
|
181
457
|
end
|
|
182
458
|
|
|
459
|
+
# Delete +columns+' value links (a key rotated, a relink).
|
|
460
|
+
def drop_value_links(connection, model_class, columns)
|
|
461
|
+
columns.each do |column|
|
|
462
|
+
pattern = connection.quote("#{ActiveRecord::Base.sanitize_sql_like("#{Lyra.stream_owner_name(model_class)}!#{column}$")}%")
|
|
463
|
+
connection.exec_delete("DELETE FROM event_store_events_in_streams WHERE stream LIKE #{pattern}", "Lyra value links")
|
|
464
|
+
end
|
|
465
|
+
end
|
|
466
|
+
|
|
467
|
+
# Link the model's events up to +upto+: those after +keys_from+ (nil:
|
|
468
|
+
# none) into their foreign-key streams (each stream read, the links
|
|
469
|
+
# already there left out), those after each column's position
|
|
470
|
+
# (+behind+: { column => position }) into the column's value streams
|
|
471
|
+
# (500 links per statement, those already there left as they are).
|
|
472
|
+
# The events are read once, 500 at a time.
|
|
473
|
+
def link_since(connection, model_class, keys_from, behind, upto)
|
|
474
|
+
from = [keys_from, *behind.values].compact.min
|
|
475
|
+
prefix = connection.quote("#{ActiveRecord::Base.sanitize_sql_like(Lyra.stream_prefix(model_class))}%")
|
|
476
|
+
rows = connection.select_rows(<<~SQL.squish)
|
|
477
|
+
SELECT e.id, e.event_id FROM event_store_events e
|
|
478
|
+
JOIN event_store_events_in_streams s ON s.event_id = e.event_id
|
|
479
|
+
WHERE s.stream LIKE #{prefix} AND e.id > #{Integer(from)} AND e.id <= #{Integer(upto)}
|
|
480
|
+
ORDER BY e.id
|
|
481
|
+
SQL
|
|
482
|
+
return 0 if rows.empty?
|
|
483
|
+
|
|
484
|
+
store = Lyra.config.event_store
|
|
485
|
+
position = rows.to_h { |id, event_id| [event_id, id.to_i] }
|
|
486
|
+
wanted = Hash.new { |h, k| h[k] = [] }
|
|
487
|
+
pairs = []
|
|
488
|
+
position.keys.each_slice(500) do |ids|
|
|
489
|
+
store.read.events(ids).each do |event|
|
|
490
|
+
at = position[event.event_id]
|
|
491
|
+
if keys_from && at > keys_from
|
|
492
|
+
reference_values(model_class, event).each { |key, value| wanted[stream_name(model_class, key, value)] << event.event_id }
|
|
493
|
+
end
|
|
494
|
+
columns = behind.select { |_, column_at| at > column_at }.keys
|
|
495
|
+
next if columns.empty?
|
|
496
|
+
|
|
497
|
+
UniqueIndexes.streams_for(model_class, set_values(event).slice(*columns)).each { pairs << [_1, event.event_id] }
|
|
498
|
+
end
|
|
499
|
+
end
|
|
500
|
+
|
|
501
|
+
linked = pairs.each_slice(500).sum { |slice| insert_links(connection, slice) }
|
|
502
|
+
wanted.each do |stream, ids|
|
|
503
|
+
present = store.read.stream(stream).to_a.map(&:event_id).to_set
|
|
504
|
+
missing = ids.uniq.reject { present.include?(_1) }
|
|
505
|
+
next if missing.empty?
|
|
506
|
+
|
|
507
|
+
store.link(missing, stream_name: stream)
|
|
508
|
+
linked += missing.size
|
|
509
|
+
end
|
|
510
|
+
linked
|
|
511
|
+
end
|
|
512
|
+
|
|
183
513
|
# [[key, value], ...] for the conditions that name one value of an
|
|
184
514
|
# indexed foreign key.
|
|
185
515
|
def indexed_conditions(model_class, conditions)
|
|
@@ -188,7 +518,13 @@ module Lyra
|
|
|
188
518
|
next unless keys.include?(key.to_s)
|
|
189
519
|
|
|
190
520
|
value = value.id if value.is_a?(ActiveRecord::Base)
|
|
191
|
-
|
|
521
|
+
next unless value.is_a?(Integer) || value.is_a?(String)
|
|
522
|
+
|
|
523
|
+
# As the key's type casts it ("5" and 5 name one stream); a value
|
|
524
|
+
# the type binds as NULL ("abc" on an integer key) names none, and
|
|
525
|
+
# is left to the conditions.
|
|
526
|
+
bound = TypedValues.query_value(model_class.type_for_attribute(key.to_s), value)
|
|
527
|
+
[key.to_s, bound] if bound.is_a?(Integer) || bound.is_a?(String)
|
|
192
528
|
end
|
|
193
529
|
end
|
|
194
530
|
|
|
@@ -196,19 +532,68 @@ module Lyra
|
|
|
196
532
|
# records whose own streams hold the events linked in the reference
|
|
197
533
|
# stream. One query on the stream table; the events are not read.
|
|
198
534
|
def candidate_ids(model_class, key, value)
|
|
535
|
+
candidate_streams(model_class, stream_name(model_class, key, value))
|
|
536
|
+
end
|
|
537
|
+
|
|
538
|
+
# Ids of the records whose own streams hold the events linked in
|
|
539
|
+
# +stream+.
|
|
540
|
+
def candidate_streams(model_class, stream)
|
|
199
541
|
ActiveRecord::Base.with_connection do |connection|
|
|
200
542
|
own = Lyra.stream_prefix(model_class)
|
|
201
|
-
connection.select_values(<<~SQL.squish).map { |
|
|
543
|
+
connection.select_values(<<~SQL.squish).map { |found| found.delete_prefix(own) }
|
|
202
544
|
SELECT DISTINCT s.stream FROM event_store_events_in_streams l
|
|
203
545
|
JOIN event_store_events_in_streams s
|
|
204
546
|
ON s.event_id = l.event_id AND s.stream LIKE #{connection.quote("#{ActiveRecord::Base.sanitize_sql_like(own)}%")}
|
|
205
|
-
WHERE l.stream = #{connection.quote(
|
|
547
|
+
WHERE l.stream = #{connection.quote(stream)}
|
|
206
548
|
SQL
|
|
207
549
|
end
|
|
208
550
|
end
|
|
209
551
|
|
|
552
|
+
# Of +streams+, the one with the fewest links; nil when one has
|
|
553
|
+
# none. One query (the stream table's index on the stream).
|
|
554
|
+
def fewest_links(streams)
|
|
555
|
+
counts = ActiveRecord::Base.with_connection do |connection|
|
|
556
|
+
connection.uncached do
|
|
557
|
+
connection.select_rows(<<~SQL.squish).to_h
|
|
558
|
+
SELECT stream, COUNT(*) FROM event_store_events_in_streams
|
|
559
|
+
WHERE stream IN (#{streams.map { connection.quote(_1) }.join(', ')}) GROUP BY stream
|
|
560
|
+
SQL
|
|
561
|
+
end
|
|
562
|
+
end
|
|
563
|
+
return nil if counts.size < streams.uniq.size
|
|
564
|
+
|
|
565
|
+
counts.min_by { |_, count| count.to_i }.first
|
|
566
|
+
end
|
|
567
|
+
|
|
568
|
+
# The streams +event+ is linked into: one per foreign key it sets
|
|
569
|
+
# (stream_name), and one per value of a unique index's column it sets
|
|
570
|
+
# (UniqueIndexes.streams_for).
|
|
571
|
+
def link_streams(model_class, event)
|
|
572
|
+
references = reference_values(model_class, event).map { |key, value| stream_name(model_class, key, value) }
|
|
573
|
+
return references if UniqueIndexes.columns(model_class).empty?
|
|
574
|
+
|
|
575
|
+
references + UniqueIndexes.streams_for(model_class, set_values(event))
|
|
576
|
+
end
|
|
577
|
+
|
|
210
578
|
# { foreign key => value } that +event+ sets, for the model's keys.
|
|
211
579
|
def reference_values(model_class, event)
|
|
580
|
+
set_values(event).slice(*foreign_keys(model_class)).compact
|
|
581
|
+
end
|
|
582
|
+
|
|
583
|
+
# The monitored table (its stream owner) whose record an event of a
|
|
584
|
+
# write names (its data's model_class), or nil.
|
|
585
|
+
def value_owner(event)
|
|
586
|
+
data = event.data
|
|
587
|
+
name = data.is_a?(Hash) ? (data[:model_class] || data["model_class"]) : nil
|
|
588
|
+
model = name.to_s.safe_constantize
|
|
589
|
+
return nil unless model.is_a?(Class) && model < ActiveRecord::Base && model.respond_to?(:lyra_monitored)
|
|
590
|
+
|
|
591
|
+
owner(model)
|
|
592
|
+
end
|
|
593
|
+
|
|
594
|
+
# { attribute => value } that +event+ sets: a create's or an
|
|
595
|
+
# import's attributes, an update's new values.
|
|
596
|
+
def set_values(event)
|
|
212
597
|
data = event.data
|
|
213
598
|
values =
|
|
214
599
|
case Lyra::Event.operation_of(event)
|
|
@@ -221,60 +606,66 @@ module Lyra
|
|
|
221
606
|
else
|
|
222
607
|
{}
|
|
223
608
|
end
|
|
224
|
-
values
|
|
609
|
+
values
|
|
225
610
|
end
|
|
226
611
|
|
|
227
|
-
def link_events(events, model_class, store)
|
|
612
|
+
def link_events(events, model_class, store, keys: true)
|
|
613
|
+
values = !UniqueIndexes.columns(model_class).empty?
|
|
228
614
|
events.each do |event|
|
|
229
|
-
|
|
230
|
-
|
|
615
|
+
if keys
|
|
616
|
+
reference_values(model_class, event).each do |key, value|
|
|
617
|
+
store.link([event.event_id], stream_name: stream_name(model_class, key, value))
|
|
618
|
+
end
|
|
231
619
|
end
|
|
620
|
+
link_values(event, UniqueIndexes.streams_for(model_class, set_values(event))) if values
|
|
232
621
|
end
|
|
233
622
|
end
|
|
234
623
|
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
ORDER BY e.id
|
|
243
|
-
SQL
|
|
244
|
-
return 0 if event_ids.empty?
|
|
245
|
-
|
|
246
|
-
store = Lyra.config.event_store
|
|
247
|
-
wanted = Hash.new { |h, k| h[k] = [] }
|
|
248
|
-
event_ids.each_slice(500) do |ids|
|
|
249
|
-
store.read.events(ids).each do |event|
|
|
250
|
-
reference_values(model_class, event).each do |key, value|
|
|
251
|
-
wanted[stream_name(model_class, key, value)] << event.event_id
|
|
252
|
-
end
|
|
253
|
-
end
|
|
254
|
-
end
|
|
624
|
+
# An event just stored, linked into its value streams in one
|
|
625
|
+
# statement: the rows RES's link inserts, without the existence
|
|
626
|
+
# check and the savepoint it takes for each stream (three round
|
|
627
|
+
# trips per stream on every create of a model with a unique index).
|
|
628
|
+
# The stream table is the one every ES-NoProj read queries.
|
|
629
|
+
def link_values(event, streams)
|
|
630
|
+
return if streams.empty?
|
|
255
631
|
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
present = store.read.stream(stream).to_a.map(&:event_id).to_set
|
|
259
|
-
missing = ids.uniq.reject { present.include?(_1) }
|
|
260
|
-
next if missing.empty?
|
|
632
|
+
ActiveRecord::Base.with_connection { |connection| insert_links(connection, streams.map { [_1, event.event_id] }) }
|
|
633
|
+
end
|
|
261
634
|
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
635
|
+
# Insert [stream, event id] links, those already there left as they
|
|
636
|
+
# are; returns how many were inserted.
|
|
637
|
+
def insert_links(connection, pairs)
|
|
638
|
+
now = connection.quote(Time.now.utc)
|
|
639
|
+
rows = pairs.map { |stream, event_id| "(#{connection.quote(stream)}, NULL, #{connection.quote(event_id)}, #{now})" }
|
|
640
|
+
connection.exec_update(
|
|
641
|
+
"INSERT INTO event_store_events_in_streams (stream, position, event_id, created_at) " \
|
|
642
|
+
"VALUES #{rows.join(', ')} ON CONFLICT DO NOTHING", "Lyra value links"
|
|
643
|
+
)
|
|
267
644
|
end
|
|
268
645
|
|
|
269
|
-
|
|
646
|
+
|
|
647
|
+
# The position up to which the model's foreign keys are linked.
|
|
648
|
+
def checkpoint(model_class) = checkpoint_state(model_class)[:position]
|
|
649
|
+
|
|
650
|
+
# { position: the foreign keys' position, values: { column =>
|
|
651
|
+
# { "position" =>, "key" => the digest key's id } } }, from the model's
|
|
652
|
+
# last checkpoint (values empty for one stored before 1.0.0's value
|
|
653
|
+
# streams).
|
|
654
|
+
def checkpoint_state(model_class)
|
|
270
655
|
last = Lyra.config.event_store.read.stream("#{CHECKPOINT_PREFIX}#{model_class.name}").last
|
|
271
|
-
|
|
656
|
+
return { position: 0, values: {} } unless last
|
|
657
|
+
|
|
658
|
+
data = last.data
|
|
659
|
+
values = (data[:values] || data["values"] || {}).to_h do |column, entry|
|
|
660
|
+
[column.to_s, entry.to_h.transform_keys(&:to_s)]
|
|
661
|
+
end
|
|
662
|
+
{ position: (data[:position] || data["position"]).to_i, values: values }
|
|
272
663
|
rescue RubyEventStore::EventNotFoundInStream
|
|
273
|
-
0
|
|
664
|
+
{ position: 0, values: {} }
|
|
274
665
|
end
|
|
275
666
|
|
|
276
|
-
def store_checkpoint(model_class, position)
|
|
277
|
-
event = Lyra::Events::ReferenceLinksIndexed.new(data: { model_class: model_class.name, position: position })
|
|
667
|
+
def store_checkpoint(model_class, position, values)
|
|
668
|
+
event = Lyra::Events::ReferenceLinksIndexed.new(data: { model_class: model_class.name, position: position, values: values })
|
|
278
669
|
Lyra.config.event_store.publish(event, stream_name: "#{CHECKPOINT_PREFIX}#{model_class.name}")
|
|
279
670
|
end
|
|
280
671
|
|