orfeas_lyra 0.9.3 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +1923 -17
- data/README.md +15 -8
- data/Rakefile +40 -1
- data/app/controllers/lyra/application_controller.rb +108 -0
- data/app/controllers/lyra/dashboard_controller.rb +17 -15
- data/app/controllers/lyra/flow_controller.rb +30 -33
- data/app/controllers/lyra/privacy_controller.rb +57 -18
- data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
- data/app/views/lyra/dashboard/index.html.erb +1 -0
- data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
- data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
- data/app/views/lyra/privacy/policy.html.erb +7 -0
- data/app/views/lyra/privacy/rights.html.erb +78 -0
- data/config/privacy_policies.rb +0 -3
- data/config/routes.rb +1 -0
- data/lib/lyra/access_log.rb +187 -6
- data/lib/lyra/advisory_lock.rb +236 -0
- data/lib/lyra/aggregate.rb +1 -1
- data/lib/lyra/associations/habtm_links.rb +382 -19
- data/lib/lyra/associations/maintenance.rb +229 -0
- data/lib/lyra/attribution.rb +48 -0
- data/lib/lyra/bypass_events.rb +426 -45
- data/lib/lyra/command_handler.rb +19 -14
- data/lib/lyra/configuration.rb +237 -10
- data/lib/lyra/consistency/read_your_writes.rb +4 -2
- data/lib/lyra/correlation.rb +121 -0
- data/lib/lyra/domain_events.rb +4 -0
- data/lib/lyra/dual_view.rb +53 -16
- data/lib/lyra/encrypted_attributes.rb +5 -0
- data/lib/lyra/engine.rb +67 -0
- data/lib/lyra/erasure.rb +1002 -116
- data/lib/lyra/event.rb +35 -1
- data/lib/lyra/event_data.rb +38 -0
- data/lib/lyra/event_flow.rb +31 -15
- data/lib/lyra/event_store_adapter.rb +28 -0
- data/lib/lyra/event_window.rb +58 -0
- data/lib/lyra/genesis.rb +548 -44
- data/lib/lyra/interceptors/association_interceptor.rb +689 -34
- data/lib/lyra/interceptors/crud_interceptor.rb +547 -95
- data/lib/lyra/internal.rb +26 -0
- data/lib/lyra/mode_sync.rb +51 -3
- data/lib/lyra/mode_transition.rb +245 -34
- data/lib/lyra/model_selection.rb +35 -0
- data/lib/lyra/own_connection.rb +17 -2
- data/lib/lyra/privacy/adapters/pam.rb +25 -2
- data/lib/lyra/privacy/gdpr_compliance.rb +264 -36
- data/lib/lyra/privacy/interface.rb +27 -7
- data/lib/lyra/privacy/subject_links.rb +143 -0
- data/lib/lyra/privacy/subject_records.rb +369 -0
- data/lib/lyra/projection.rb +7 -1
- data/lib/lyra/projections/async_checkpoint.rb +32 -0
- data/lib/lyra/projections/async_projection_job.rb +166 -23
- data/lib/lyra/projections/cached_joins.rb +33 -18
- data/lib/lyra/projections/cached_projection.rb +74 -33
- data/lib/lyra/projections/cached_relation.rb +3175 -321
- data/lib/lyra/projections/default_scopes.rb +254 -0
- data/lib/lyra/projections/deferred_relation.rb +23 -3
- data/lib/lyra/projections/event_store_reader.rb +47 -8
- data/lib/lyra/projections/foreign_keys.rb +92 -0
- data/lib/lyra/projections/lazy_projection.rb +165 -27
- data/lib/lyra/projections/model_projection.rb +165 -14
- data/lib/lyra/projections/rebuild.rb +34 -6
- data/lib/lyra/projections/reference_links.rb +463 -72
- data/lib/lyra/projections/typed_values.rb +629 -0
- data/lib/lyra/projections/unique_indexes.rb +595 -0
- data/lib/lyra/projections/uniqueness.rb +784 -0
- data/lib/lyra/projections/value_digest.rb +93 -0
- data/lib/lyra/purpose_bound_reads.rb +912 -77
- data/lib/lyra/repair.rb +119 -28
- data/lib/lyra/retention.rb +114 -8
- data/lib/lyra/rights/consent_store.rb +415 -0
- data/lib/lyra/rights/erasure_guard.rb +135 -0
- data/lib/lyra/rights/pending.rb +74 -0
- data/lib/lyra/rights/queries.rb +269 -0
- data/lib/lyra/rights/store.rb +512 -0
- data/lib/lyra/rights/subject_keys.rb +224 -0
- data/lib/lyra/rights.rb +281 -0
- data/lib/lyra/saves_without_effect.rb +69 -0
- data/lib/lyra/stream_tail.rb +319 -0
- data/lib/lyra/strict_data_access.rb +906 -107
- data/lib/lyra/temporal.rb +4 -2
- data/lib/lyra/touch_registry.rb +404 -0
- data/lib/lyra/verification/bypass_workflow.rb +452 -42
- data/lib/lyra/verification/rights_workflows.rb +170 -0
- data/lib/lyra/version.rb +1 -1
- data/lib/lyra/visualization/event_graph.rb +1 -1
- data/lib/lyra/visualization/timeline.rb +2 -2
- data/lib/lyra.rb +43 -2
- data/lib/tasks/lyra_mode.rake +107 -8
- data/lib/tasks/lyra_projections.rake +43 -2
- metadata +34 -4
- data/lib/lyra/associations/event_aware.rb +0 -224
data/lib/lyra/bypass_events.rb
CHANGED
|
@@ -1,6 +1,32 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Lyra
|
|
4
|
+
# Raised before a bulk write that would lock more records than
|
|
5
|
+
# config.bulk_write_lock_limit: the write holds one aggregate lock per
|
|
6
|
+
# record until its transaction ends, and PostgreSQL's lock table, which
|
|
7
|
+
# every session shares, has room for about max_locks_per_transaction ×
|
|
8
|
+
# (max_connections + background processes) locks. Past that, every
|
|
9
|
+
# session's writes fail ("out of shared memory").
|
|
10
|
+
class BulkWriteLockLimitError < StandardError
|
|
11
|
+
attr_reader :model_class, :count, :limit
|
|
12
|
+
|
|
13
|
+
def initialize(model_class, count, limit, write = "bulk write")
|
|
14
|
+
@model_class = model_class
|
|
15
|
+
@count = count
|
|
16
|
+
@limit = limit
|
|
17
|
+
name = model_class.respond_to?(:name) ? model_class.name : model_class.to_s
|
|
18
|
+
records = count == 1 ? "record" : "records"
|
|
19
|
+
locks = count == 1 ? "lock" : "locks"
|
|
20
|
+
super(
|
|
21
|
+
"Lyra: #{write} of #{count} #{name} #{records} would hold #{count} aggregate #{locks} until " \
|
|
22
|
+
"its transaction ends, more than config.bulk_write_lock_limit (#{limit}). Write in batches " \
|
|
23
|
+
"(in_batches(of: #{limit}) { |batch| batch.update_all(...) }): each batch commits, and gives its locks " \
|
|
24
|
+
"back, on its own, but not inside a transaction already open, which holds them all until it ends. " \
|
|
25
|
+
"Or raise the limit (nil: none), within PostgreSQL's lock table."
|
|
26
|
+
)
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
4
30
|
# Publishes events for writes that skip ActiveRecord callbacks
|
|
5
31
|
# (update_column(s), delete, update_all, delete_all, insert_all,
|
|
6
32
|
# upsert_all, dependent: :nullify), so the event stream records every state change
|
|
@@ -11,8 +37,6 @@ module Lyra
|
|
|
11
37
|
# read model. Those run inside Lyra.projection_write and must never
|
|
12
38
|
# publish: the event they project already exists.
|
|
13
39
|
module BypassEvents
|
|
14
|
-
TIMESTAMP_COLUMNS = %w[created_at updated_at].freeze
|
|
15
|
-
|
|
16
40
|
class << self
|
|
17
41
|
# Whether a bypass write's events must be stored for the write to
|
|
18
42
|
# stand. In Hijack and the event-sourcing modes the log is the record of
|
|
@@ -32,10 +56,19 @@ module Lyra
|
|
|
32
56
|
# Under ES-Lazy the tables are brought up to date first, as before a
|
|
33
57
|
# read: a write appends events only, so a record written since the last
|
|
34
58
|
# read had no row yet, and update_columns, update_all or delete_all on it
|
|
35
|
-
# matched nothing, recorded no event, and the change was lost.
|
|
59
|
+
# matched nothing, recorded no event, and the change was lost. Only
|
|
60
|
+
# then: the write's own reads (its snapshot, the rows it reads back for
|
|
61
|
+
# the events) do not catch up again. A catch-up there waited for the
|
|
62
|
+
# catch-up lock while the write held rows that a catch-up running
|
|
63
|
+
# elsewhere, holding that lock, waited for: a deadlock, which bulk
|
|
64
|
+
# writes and writes that skip callbacks met against catch-ups under
|
|
65
|
+
# load (before 1.0.0 too). A destroy committed after the catch-up is
|
|
66
|
+
# found by the stream check (checks_streams?).
|
|
36
67
|
def atomically(model_class, &block)
|
|
37
68
|
Lyra::Projections::LazyProjection.before_read(model_class)
|
|
38
|
-
|
|
69
|
+
Lyra::Projections::LazyProjection.without_catch_up do
|
|
70
|
+
required? || locks_rows? ? model_class.transaction(&block) : yield
|
|
71
|
+
end
|
|
39
72
|
end
|
|
40
73
|
|
|
41
74
|
# Whether a bulk write locks the rows it matches and writes exactly
|
|
@@ -50,6 +83,220 @@ module Lyra
|
|
|
50
83
|
!(Lyra.config.event_sourcing_mode? && Lyra.config.projection_mode == :disabled)
|
|
51
84
|
end
|
|
52
85
|
|
|
86
|
+
# Whether a write that skips callbacks must ask the record's stream
|
|
87
|
+
# whether it was destroyed, under the aggregate lock, before it records
|
|
88
|
+
# anything: wherever a destroy can be in the store without its row
|
|
89
|
+
# being gone. In ES-NoProj there is no row (a stale copy, or a relation
|
|
90
|
+
# built before the destroy, still holds the record); in ES-Async the
|
|
91
|
+
# row stays until the destroy's projection job runs; in ES-Lazy until
|
|
92
|
+
# the next catch-up, and one committed after the catch-up a write
|
|
93
|
+
# began with is not in the table it writes. A write that reached the
|
|
94
|
+
# record there appended an update after its destroy, and brought it
|
|
95
|
+
# back. ES-Sync, Hijack and Monitor delete the row with the destroy.
|
|
96
|
+
# With lock_aggregate_before_append off, as for a write through
|
|
97
|
+
# callbacks, nothing is checked.
|
|
98
|
+
def checks_streams?
|
|
99
|
+
Lyra.config.lock_aggregate_before_append && Lyra.config.event_sourcing_mode? &&
|
|
100
|
+
%i[disabled async lazy].include?(Lyra.config.projection_mode)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Whether an insert_all, insert_all! or upsert_all brings the rows of the
|
|
104
|
+
# keys it writes up to date with their streams first, under their
|
|
105
|
+
# aggregate locks (Lyra::Projections::AsyncProjectionJob
|
|
106
|
+
# .project_pending): in ES-Async, where a row waits for its projection
|
|
107
|
+
# job. A record whose destroy was pending kept its row, and ON CONFLICT
|
|
108
|
+
# skipped (insert_all) or updated (upsert_all) the dead record, where
|
|
109
|
+
# Monitor creates it again; a record whose create was pending had no
|
|
110
|
+
# row yet, and the insert logged a second Created, where Monitor's
|
|
111
|
+
# conflicts. Deleting the row alone was no answer: the destroy's job,
|
|
112
|
+
# run after, would delete the new row. ES-Lazy's rows are brought up to
|
|
113
|
+
# date when the write begins (atomically), and its upsert_all refuses a
|
|
114
|
+
# destroy committed since (StrictDataAccessRelation#lock_upserted_rows):
|
|
115
|
+
# replaying a destroy onto a row there would be applied again by the
|
|
116
|
+
# next catch-up, which replays from a checkpoint of its own, global, not
|
|
117
|
+
# per stream, and would delete the row created again.
|
|
118
|
+
def projects_pending?
|
|
119
|
+
checks_streams? && Lyra.config.projection_mode == :async
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# Of +ids+ (records of +model_class+), those whose streams end in a
|
|
123
|
+
# destroy (Lyra::StreamTail), read under each stream's aggregate lock,
|
|
124
|
+
# the one a write through callbacks takes
|
|
125
|
+
# (CrudInterceptor#lyra_lock_aggregate_unless_destroyed!). The locks are
|
|
126
|
+
# taken in one statement and held until the caller's transaction ends,
|
|
127
|
+
# so no destroy can come between the read and the write. +write+ names
|
|
128
|
+
# the write in a BulkWriteLockLimitError.
|
|
129
|
+
def destroyed_ids(model_class, ids, write: "bulk write")
|
|
130
|
+
ids = ids.uniq
|
|
131
|
+
return [] if ids.empty?
|
|
132
|
+
|
|
133
|
+
lock_streams!(model_class, ids, write: write)
|
|
134
|
+
destroyed_among(model_class, ids)
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# Of +ids+, those whose streams end in a destroy, read in one query
|
|
138
|
+
# where it can be (Lyra::StreamTail.destroyed_streams); the caller holds
|
|
139
|
+
# their aggregate locks.
|
|
140
|
+
def destroyed_among(model_class, ids)
|
|
141
|
+
return [] if ids.empty?
|
|
142
|
+
|
|
143
|
+
streams = ids.to_h { |id| [Lyra.stream_name(model_class, id), id] }
|
|
144
|
+
Lyra::StreamTail.destroyed_streams(streams.keys).map { streams[_1] }
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# Run the write in the block (Active Record's own, on a record whose
|
|
148
|
+
# stream says it was destroyed) and take back what it wrote to the
|
|
149
|
+
# table, in a savepoint rolled back: a row still there (ES-Async, before
|
|
150
|
+
# the destroy's projection job runs; ES-NoProj, one left from an
|
|
151
|
+
# earlier mode) is not changed without an event. What the write set in
|
|
152
|
+
# memory stays, as on a row already gone.
|
|
153
|
+
def unwritten(model_class)
|
|
154
|
+
model_class.transaction(requires_new: true) do
|
|
155
|
+
yield
|
|
156
|
+
raise ActiveRecord::Rollback
|
|
157
|
+
end
|
|
158
|
+
nil
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# The snapshot a bulk write writes (snapshot(lock: true)): the rows it
|
|
162
|
+
# matches, locked. Where the table can still hold the row of a record
|
|
163
|
+
# whose stream ends in a destroy (checks_streams?: ES-Async, before the
|
|
164
|
+
# destroy's projection job runs), less those rows, as Active Record's
|
|
165
|
+
# write reaches no row of a destroyed record; the write then writes,
|
|
166
|
+
# records and counts the others only.
|
|
167
|
+
#
|
|
168
|
+
# The records' aggregate locks are taken before their rows' locks, in
|
|
169
|
+
# Lyra's order (Lyra::AdvisoryLock): the keys the relation matches are
|
|
170
|
+
# read, their aggregate locks taken (all or none), then the rows
|
|
171
|
+
# locked. A row that came to match in between, whose aggregate lock is
|
|
172
|
+
# not held, sends the row locks back (a savepoint rolled back) and the
|
|
173
|
+
# keys are read again, up to ROW_LOCK_TRIES times; after that such a
|
|
174
|
+
# row is left out, as if it had matched after the write. In ES-Lazy,
|
|
175
|
+
# once the aggregate locks are held, each record whose stream has an
|
|
176
|
+
# event the tables have not had yet (one committed after the catch-up
|
|
177
|
+
# the write began with: atomically) is replayed onto its row
|
|
178
|
+
# (LazyProjection.refresh_records), so the write starts from the row
|
|
179
|
+
# as the stream has it: an update committed in between is not lost
|
|
180
|
+
# under it. The whole catch-up is not run again there: its lock is
|
|
181
|
+
# taken before the aggregate locks by every writer that reads first,
|
|
182
|
+
# and taking it after them deadlocked. Then the streams of the rows
|
|
183
|
+
# locked are read, in one query where it can be
|
|
184
|
+
# (Lyra::StreamTail.destroyed_streams). Genesis admits the rows
|
|
185
|
+
# (snapshot) before the streams are read.
|
|
186
|
+
def locked_snapshot(model_class, relation, write: "bulk write")
|
|
187
|
+
return snapshot(model_class, relation, lock: true) unless checks_streams?
|
|
188
|
+
|
|
189
|
+
pk = model_class.primary_key
|
|
190
|
+
locked = Set.new
|
|
191
|
+
ROW_LOCK_TRIES.times do |round|
|
|
192
|
+
keys = Lyra::PurposeBoundReads.internal(INTERNAL) do
|
|
193
|
+
model_class.unscoped.where(pk => relation.reselect(model_class.arel_table[pk])).pluck(pk)
|
|
194
|
+
end
|
|
195
|
+
added = keys.reject { locked.include?(_1) }
|
|
196
|
+
if added.any?
|
|
197
|
+
check_lock_limit!(model_class, locked.size + added.size, write)
|
|
198
|
+
lock_streams!(model_class, added, write: write)
|
|
199
|
+
locked.merge(added)
|
|
200
|
+
# ES-Lazy: their rows as their streams have them now, under the
|
|
201
|
+
# locks, before the rows are read and written.
|
|
202
|
+
Lyra::Projections::LazyProjection.refresh_records(model_class, added)
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
before = nil
|
|
206
|
+
model_class.transaction(requires_new: true) do
|
|
207
|
+
rows = snapshot(model_class, relation, lock: true)
|
|
208
|
+
raise ActiveRecord::Rollback if round < ROW_LOCK_TRIES - 1 && rows.each_key.any? { !locked.include?(_1) }
|
|
209
|
+
|
|
210
|
+
before = rows
|
|
211
|
+
end
|
|
212
|
+
next unless before
|
|
213
|
+
|
|
214
|
+
before = before.select { |id, _| locked.include?(id) }
|
|
215
|
+
dead = destroyed_among(model_class, before.keys)
|
|
216
|
+
return dead.empty? ? before : before.except(*dead)
|
|
217
|
+
end
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
# How many times locked_snapshot reads the keys again for a row that
|
|
221
|
+
# came to match while it took their aggregate locks.
|
|
222
|
+
ROW_LOCK_TRIES = 3
|
|
223
|
+
|
|
224
|
+
# Take the aggregate locks of +ids+' streams, in one statement, after
|
|
225
|
+
# checking their number against config.bulk_write_lock_limit. Returns
|
|
226
|
+
# the stream names, in the order of +ids+.
|
|
227
|
+
def lock_streams!(model_class, ids, write: "bulk write")
|
|
228
|
+
check_lock_limit!(model_class, ids.size, write)
|
|
229
|
+
streams = ids.map { |id| Lyra.stream_name(model_class, id) }
|
|
230
|
+
model_class.with_connection do |connection|
|
|
231
|
+
Lyra::AdvisoryLock.xact_lock_all(connection, streams.map { "lyra_aggregate/#{_1}" }, purpose: "aggregate write")
|
|
232
|
+
end
|
|
233
|
+
streams
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
# Refuse a write that would hold more than config.bulk_write_lock_limit
|
|
237
|
+
# aggregate locks (BulkWriteLockLimitError), before it locks or writes
|
|
238
|
+
# anything.
|
|
239
|
+
def check_lock_limit!(model_class, count, write = "bulk write")
|
|
240
|
+
limit = Lyra.config.bulk_write_lock_limit
|
|
241
|
+
return if limit.nil? || count <= limit
|
|
242
|
+
|
|
243
|
+
raise Lyra::BulkWriteLockLimitError.new(model_class, count, limit, write)
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
# Run an insert_all, insert_all! or upsert_all (the block, given the
|
|
247
|
+
# RETURNING to ask for) with +columns+ in its RETURNING, so the rows it
|
|
248
|
+
# wrote can be told, and hand the caller the result it asked for.
|
|
249
|
+
# Returns [caller_result, rows], rows as hashes of +columns+, each
|
|
250
|
+
# value cast as the result types it (as Result#cast_values does).
|
|
251
|
+
#
|
|
252
|
+
# +returning+ as Active Record takes it: nil (the primary key, where
|
|
253
|
+
# the adapter returns rows), false (nothing), column names, or raw SQL
|
|
254
|
+
# (Arel.sql("code"), Arel.sql("*")). A column the caller did not ask
|
|
255
|
+
# for is added (raw SQL: under an alias of Lyra's own, "lyra_returning_
|
|
256
|
+
# <column>") and taken out of the caller's result again, which keeps
|
|
257
|
+
# its column types (by name and by index, as Active Record's own),
|
|
258
|
+
# its affected rows, and, for returning: false, is Active Record's
|
|
259
|
+
# empty result. Before, it was rebuilt without the column types:
|
|
260
|
+
# result.column_types came back empty and cast_values cast nothing.
|
|
261
|
+
# Rows nil on an adapter with no RETURNING.
|
|
262
|
+
def with_returning(model_class, returning, columns)
|
|
263
|
+
# An adapter with no RETURNING (MySQL): the write as asked, and no
|
|
264
|
+
# rows to tell.
|
|
265
|
+
return [yield(returning), nil] unless model_class.with_connection(&:supports_insert_returning?)
|
|
266
|
+
|
|
267
|
+
returning = default_returning(model_class) if returning.nil?
|
|
268
|
+
quote = ->(name) { model_class.with_connection { |c| c.quote_column_name(name) } }
|
|
269
|
+
names = columns
|
|
270
|
+
case returning
|
|
271
|
+
when false
|
|
272
|
+
request = columns
|
|
273
|
+
when Arel::Nodes::SqlLiteral
|
|
274
|
+
names = columns.map { "#{RETURNING_ALIAS}#{_1}" }
|
|
275
|
+
request = Arel.sql([returning, *columns.zip(names).map { |c, n| "#{quote.(c)} AS #{quote.(n)}" }].join(", "))
|
|
276
|
+
added = columns.size
|
|
277
|
+
else
|
|
278
|
+
asked = Array(returning).map(&:to_s)
|
|
279
|
+
added = (columns - asked).size
|
|
280
|
+
request = added.zero? ? returning : asked + (columns - asked)
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
result = yield(request)
|
|
284
|
+
indexes = names.map { result.columns.index(_1) }
|
|
285
|
+
types = result.column_types
|
|
286
|
+
rows = result.rows.map do |row|
|
|
287
|
+
columns.each_with_index.to_h do |column, i|
|
|
288
|
+
index = indexes[i]
|
|
289
|
+
value = index && row[index]
|
|
290
|
+
[column, index && types[index] ? types[index].deserialize(value) : value]
|
|
291
|
+
end
|
|
292
|
+
end
|
|
293
|
+
[caller_result(result, returning == false ? nil : added.to_i), rows]
|
|
294
|
+
end
|
|
295
|
+
|
|
296
|
+
# The alias a column the caller's raw-SQL RETURNING did not name is
|
|
297
|
+
# returned under (with_returning).
|
|
298
|
+
RETURNING_ALIAS = "lyra_returning_"
|
|
299
|
+
|
|
53
300
|
# Report a failed publish: raise when the events are required (see
|
|
54
301
|
# required?), log otherwise.
|
|
55
302
|
def publish_failed!(error, source)
|
|
@@ -58,6 +305,34 @@ module Lyra
|
|
|
58
305
|
Rails.logger.error("Lyra: Failed to publish bypass event (#{source}) - #{error.message}")
|
|
59
306
|
end
|
|
60
307
|
|
|
308
|
+
# Whether a touch's event is left out of the log (config.record_touches
|
|
309
|
+
# false, in Monitor: Configuration#skip_touches?): when every column it
|
|
310
|
+
# changed is one of the model's update timestamps (updated_at,
|
|
311
|
+
# updated_on). A touch that changed another column too (a named one,
|
|
312
|
+
# touch(:completed_at), or lock_version) is recorded in full. Only once
|
|
313
|
+
# this process's entry in the registry of processes that leave touches
|
|
314
|
+
# out is live (Lyra::TouchRegistry.live!), so that every process's mode
|
|
315
|
+
# gate sees it; when the entry cannot be written, the touch is recorded.
|
|
316
|
+
def skip_touch?(model_class, changes)
|
|
317
|
+
return false unless Lyra.config.skip_touches?
|
|
318
|
+
return false if changes.empty?
|
|
319
|
+
|
|
320
|
+
stamps = model_class.timestamp_attributes_for_update_in_model
|
|
321
|
+
return false unless changes.each_key.all? { |column| stamps.include?(column.to_s) }
|
|
322
|
+
|
|
323
|
+
Lyra::TouchRegistry.live!
|
|
324
|
+
end
|
|
325
|
+
|
|
326
|
+
# Run a touch_all, so that publish_updates can tell its rows from an
|
|
327
|
+
# update_all's (Rails's touch_all is update_all of the timestamps).
|
|
328
|
+
def touching_all
|
|
329
|
+
previous = Thread.current[:lyra_touch_all]
|
|
330
|
+
Thread.current[:lyra_touch_all] = true
|
|
331
|
+
yield
|
|
332
|
+
ensure
|
|
333
|
+
Thread.current[:lyra_touch_all] = previous
|
|
334
|
+
end
|
|
335
|
+
|
|
61
336
|
def enabled_for?(model_class)
|
|
62
337
|
return false if Thread.current[:lyra_projection_write]
|
|
63
338
|
return false if Lyra.config.mode == :disabled
|
|
@@ -66,17 +341,14 @@ module Lyra
|
|
|
66
341
|
end
|
|
67
342
|
|
|
68
343
|
def publish(model_class, id, operation, attributes:, changes: {}, source:)
|
|
344
|
+
# Every path admits its rows before writing them; this catches one
|
|
345
|
+
# that does not (the write is in the same transaction, and rolls back
|
|
346
|
+
# with a refusal where events are required).
|
|
347
|
+
admit(model_class, [id]) unless operation.to_sym == :created
|
|
69
348
|
event_class = event_class_for(model_class, operation)
|
|
70
349
|
metadata = current_metadata.merge(source_key(source) => source)
|
|
71
350
|
|
|
72
|
-
data =
|
|
73
|
-
model_class: model_class.name,
|
|
74
|
-
model_id: id,
|
|
75
|
-
operation: operation,
|
|
76
|
-
attributes: attributes,
|
|
77
|
-
changes: changes,
|
|
78
|
-
timestamp: Time.current
|
|
79
|
-
}
|
|
351
|
+
data = Lyra::EventData.build(model_class, id, operation, attributes: attributes, changes: changes)
|
|
80
352
|
event = event_class.new(data: data, metadata: Lyra::Privacy.stamp(model_class, data, metadata))
|
|
81
353
|
Lyra.append_events(event, stream_name: Lyra.stream_name(model_class, id))
|
|
82
354
|
|
|
@@ -85,6 +357,7 @@ module Lyra
|
|
|
85
357
|
if Lyra.config.projection_mode == :disabled
|
|
86
358
|
Lyra::Projections::EventStoreReader.invalidate(model_class, id)
|
|
87
359
|
end
|
|
360
|
+
event
|
|
88
361
|
end
|
|
89
362
|
|
|
90
363
|
# Who made the change and what it belongs with, for a write no record's
|
|
@@ -94,6 +367,7 @@ module Lyra
|
|
|
94
367
|
def current_metadata
|
|
95
368
|
{
|
|
96
369
|
user_id: current(:user)&.id,
|
|
370
|
+
user_type: Lyra::Attribution.actor_type(current(:user)),
|
|
97
371
|
request_id: current(:request_id),
|
|
98
372
|
correlation_id: Lyra::Correlation.current_id,
|
|
99
373
|
causation_id: Lyra::Causation.current_id
|
|
@@ -111,8 +385,16 @@ module Lyra
|
|
|
111
385
|
keys = relation.reselect(model_class.arel_table[pk])
|
|
112
386
|
scope = model_class.unscoped.where(pk => keys).lock
|
|
113
387
|
end
|
|
114
|
-
rows = Lyra::PurposeBoundReads.internal { scope.to_a }
|
|
115
|
-
rows.to_h { |record| [record
|
|
388
|
+
rows = Lyra::PurposeBoundReads.internal(INTERNAL) { scope.to_a }
|
|
389
|
+
rows.to_h { |record| [record_key(model_class, record), record.attributes] }.tap { |by_id| admit(model_class, by_id.keys) }
|
|
390
|
+
end
|
|
391
|
+
|
|
392
|
+
# Before a bypass write records per-row events: rows that predate Lyra
|
|
393
|
+
# are imported first (Lyra::Genesis), as before a write through
|
|
394
|
+
# callbacks, and a row Genesis skipped is imported now or the write is
|
|
395
|
+
# refused, so a bypass event never begins a stream (Genesis.admit_all!).
|
|
396
|
+
def admit(model_class, ids)
|
|
397
|
+
Lyra::Genesis.admit_all!(model_class, ids) unless ids.empty?
|
|
116
398
|
end
|
|
117
399
|
|
|
118
400
|
# After update_all: compare each snapshotted row with its new state and
|
|
@@ -121,13 +403,14 @@ module Lyra
|
|
|
121
403
|
return if before.empty?
|
|
122
404
|
|
|
123
405
|
pk = model_class.primary_key
|
|
124
|
-
after = Lyra::PurposeBoundReads.internal { model_class.unscoped.where(pk => before.keys).to_a }
|
|
125
|
-
.to_h { |record| [record
|
|
406
|
+
after = Lyra::PurposeBoundReads.internal(INTERNAL) { model_class.unscoped.where(pk => before.keys).to_a }
|
|
407
|
+
.to_h { |record| [record_key(model_class, record), record.attributes] }
|
|
126
408
|
|
|
127
409
|
before.each do |id, old_attrs|
|
|
128
410
|
new_attrs = after[id] or next
|
|
129
411
|
changes = diff(old_attrs, new_attrs)
|
|
130
412
|
next if changes.empty?
|
|
413
|
+
next if Thread.current[:lyra_touch_all] && skip_touch?(model_class, changes)
|
|
131
414
|
|
|
132
415
|
publish(model_class, id, :updated,
|
|
133
416
|
attributes: changes.transform_values(&:last), changes: changes, source: source)
|
|
@@ -138,7 +421,7 @@ module Lyra
|
|
|
138
421
|
def publish_destroys(model_class, before, source:)
|
|
139
422
|
before.each do |id, attrs|
|
|
140
423
|
publish(model_class, id, :destroyed,
|
|
141
|
-
attributes: attrs
|
|
424
|
+
attributes: attrs, source: source)
|
|
142
425
|
end
|
|
143
426
|
end
|
|
144
427
|
|
|
@@ -157,48 +440,92 @@ module Lyra
|
|
|
157
440
|
return {} if tuples.empty?
|
|
158
441
|
|
|
159
442
|
wanted = tuples.to_set
|
|
160
|
-
|
|
161
|
-
found = Lyra::PurposeBoundReads.internal { model_class.unscoped.where(keys.first => tuples.map(&:first).uniq).to_a }
|
|
443
|
+
found = Lyra::PurposeBoundReads.internal(INTERNAL) { model_class.unscoped.where(keys.first => tuples.map(&:first).uniq).to_a }
|
|
162
444
|
found.each_with_object({}) do |record, acc|
|
|
163
445
|
next unless wanted.include?(keys.map { |key| record.read_attribute(key) })
|
|
164
446
|
|
|
165
|
-
acc[record
|
|
166
|
-
end
|
|
447
|
+
acc[record_key(model_class, record)] = record.attributes
|
|
448
|
+
end.tap { |by_id| admit(model_class, by_id.keys) }
|
|
167
449
|
end
|
|
168
450
|
|
|
169
451
|
# After insert_all / upsert_all: a "created" event for each written row
|
|
170
452
|
# that wasn't in the snapshot, an "updated" event (with its changes)
|
|
171
|
-
# for each that was and changed.
|
|
453
|
+
# for each that was and changed. The rows are reloaded as Lyra's own
|
|
454
|
+
# read, like the snapshots: before 1.0.0 the reload ran as the
|
|
455
|
+
# application's, so under a purpose that does not allow a declared
|
|
456
|
+
# field it was refused, and Monitor wrote the rows with no events.
|
|
457
|
+
#
|
|
458
|
+
# Returns what it published: { id => [:created or :updated, event id] }.
|
|
172
459
|
def publish_upserts(model_class, before, ids, source:)
|
|
173
|
-
|
|
460
|
+
published = {}
|
|
461
|
+
return published if ids.blank?
|
|
174
462
|
|
|
175
463
|
pk = model_class.primary_key
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
464
|
+
Lyra::PurposeBoundReads.internal(INTERNAL) do
|
|
465
|
+
model_class.unscoped.where(pk => ids).find_each do |record|
|
|
466
|
+
id = record_key(model_class, record)
|
|
467
|
+
if (old_attrs = before[id])
|
|
468
|
+
changes = diff(old_attrs, record.attributes)
|
|
469
|
+
next if changes.empty?
|
|
470
|
+
|
|
471
|
+
event = publish(model_class, id, :updated,
|
|
472
|
+
attributes: changes.transform_values(&:last), changes: changes, source: source)
|
|
473
|
+
published[id] = [:updated, event.event_id]
|
|
474
|
+
else
|
|
475
|
+
event = publish(model_class, id, :created,
|
|
476
|
+
attributes: record.attributes, source: source)
|
|
477
|
+
published[id] = [:created, event.event_id]
|
|
478
|
+
end
|
|
187
479
|
end
|
|
188
480
|
end
|
|
481
|
+
published
|
|
189
482
|
end
|
|
190
483
|
|
|
191
484
|
private
|
|
192
485
|
|
|
486
|
+
# The key a record's events name it by: its primary key's value, or,
|
|
487
|
+
# for a primary key of several columns, the array of their values
|
|
488
|
+
# (record.id), which no column's reader gives.
|
|
489
|
+
def record_key(model_class, record)
|
|
490
|
+
pk = model_class.primary_key
|
|
491
|
+
pk.is_a?(Array) ? record.id : record.public_send(pk)
|
|
492
|
+
end
|
|
493
|
+
|
|
494
|
+
# What Active Record returns when returning: is nil: the primary key,
|
|
495
|
+
# where the adapter returns rows; nothing otherwise (InsertAll).
|
|
496
|
+
def default_returning(model_class)
|
|
497
|
+
keys = Array(model_class.primary_key)
|
|
498
|
+
supported = model_class.with_connection(&:supports_insert_returning?)
|
|
499
|
+
supported && keys.any? ? keys : false
|
|
500
|
+
end
|
|
501
|
+
|
|
502
|
+
# The write's result as the caller asked for it: the last +added+
|
|
503
|
+
# columns (those with_returning added) taken out, the column types,
|
|
504
|
+
# by index, and the affected rows kept; for returning: false (+added+
|
|
505
|
+
# nil), Active Record's empty result.
|
|
506
|
+
def caller_result(result, added)
|
|
507
|
+
return ActiveRecord::Result.empty(affected_rows: result.affected_rows) if added.nil?
|
|
508
|
+
return result if added.zero?
|
|
509
|
+
|
|
510
|
+
kept = result.columns.size - added
|
|
511
|
+
types = result.column_types
|
|
512
|
+
ActiveRecord::Result.new(
|
|
513
|
+
result.columns.first(kept),
|
|
514
|
+
result.rows.map { _1.first(kept) },
|
|
515
|
+
types.empty? ? nil : Array.new(kept) { types[_1] },
|
|
516
|
+
affected_rows: result.affected_rows
|
|
517
|
+
)
|
|
518
|
+
end
|
|
519
|
+
|
|
193
520
|
def current(attribute)
|
|
194
521
|
defined?(::Current) && ::Current.respond_to?(attribute) ? ::Current.public_send(attribute) : nil
|
|
195
522
|
end
|
|
196
523
|
|
|
197
|
-
# Column => [old, new] for every
|
|
524
|
+
# Column => [old, new] for every column that changed, created_at and
|
|
525
|
+
# updated_at included: a bare touch_all changes updated_at, and that is a
|
|
526
|
+
# change of the record's state.
|
|
198
527
|
def diff(old_attrs, new_attrs)
|
|
199
528
|
new_attrs.each_with_object({}) do |(column, new_value), acc|
|
|
200
|
-
next if TIMESTAMP_COLUMNS.include?(column)
|
|
201
|
-
|
|
202
529
|
old_value = old_attrs[column]
|
|
203
530
|
acc[column] = [old_value, new_value] unless old_value == new_value
|
|
204
531
|
end
|
|
@@ -226,15 +553,69 @@ module Lyra
|
|
|
226
553
|
end
|
|
227
554
|
|
|
228
555
|
# Run a block of Lyra's own read-model writes. Bypasses strict data
|
|
229
|
-
# access and publishes no bypass events.
|
|
230
|
-
|
|
556
|
+
# access and publishes no bypass events. Only Lyra's own call sites may call
|
|
557
|
+
# it (they pass Lyra::INTERNAL): application code that needs the same uses
|
|
558
|
+
# Lyra.unrecorded_write.
|
|
559
|
+
def self.projection_write(token = nil)
|
|
560
|
+
unless token.equal?(INTERNAL)
|
|
561
|
+
raise InternalApiError,
|
|
562
|
+
"Lyra.projection_write is for Lyra's own projections. To write without events " \
|
|
563
|
+
"(seeding, fixtures, a wipe whose history you do not want) use " \
|
|
564
|
+
"Lyra.unrecorded_write(reason: \"...\") { ... }, which records that it happened."
|
|
565
|
+
end
|
|
566
|
+
|
|
567
|
+
# Refused above, before the flags are touched: a refused call leaves the
|
|
568
|
+
# block it was called in as it was.
|
|
231
569
|
previous_write = Thread.current[:lyra_projection_write]
|
|
232
570
|
previous_bypass = Thread.current[:lyra_bypass_strict_access]
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
571
|
+
begin
|
|
572
|
+
Thread.current[:lyra_projection_write] = true
|
|
573
|
+
Thread.current[:lyra_bypass_strict_access] = true
|
|
574
|
+
yield
|
|
575
|
+
ensure
|
|
576
|
+
Thread.current[:lyra_projection_write] = previous_write
|
|
577
|
+
Thread.current[:lyra_bypass_strict_access] = previous_bypass
|
|
578
|
+
end
|
|
579
|
+
end
|
|
580
|
+
|
|
581
|
+
UNRECORDED_WRITES_STREAM = "Lyra::UnrecordedWrites"
|
|
582
|
+
LIB_ROOT = File.expand_path("..", __dir__)
|
|
583
|
+
|
|
584
|
+
# The way application code writes without events: seeding, fixtures, a table
|
|
585
|
+
# wipe whose history is not wanted. The block's changes are not in the log, and
|
|
586
|
+
# in Monitor lyra:repair reports them; but the write itself is recorded, with
|
|
587
|
+
# its reason, who ran it and where it was called from, in the stream
|
|
588
|
+
# Lyra::UnrecordedWrites, so an unrecorded change leaves a record that it was
|
|
589
|
+
# made. config.unrecorded_writes = :deny refuses it (production hardening).
|
|
590
|
+
# Nothing is recorded in disabled mode, where nothing is.
|
|
591
|
+
def self.unrecorded_write(reason:, &block)
|
|
592
|
+
raise ArgumentError, "unrecorded_write needs a reason: why this write should not be in the log" if reason.to_s.strip.empty?
|
|
593
|
+
raise ArgumentError, "unrecorded_write needs a block" unless block
|
|
594
|
+
|
|
595
|
+
if config.unrecorded_writes == :deny
|
|
596
|
+
raise UnrecordedWriteDenied, "unrecorded writes are refused (config.unrecorded_writes = :deny): #{reason}"
|
|
597
|
+
end
|
|
598
|
+
|
|
599
|
+
record_unrecorded_write(reason) unless config.disabled_mode?
|
|
600
|
+
projection_write(INTERNAL, &block)
|
|
601
|
+
end
|
|
602
|
+
|
|
603
|
+
def self.record_unrecorded_write(reason)
|
|
604
|
+
caller_frame = caller_locations.find { |frame| !frame.absolute_path.to_s.start_with?(LIB_ROOT) }
|
|
605
|
+
event = Lyra::Events::UnrecordedWrite.new(
|
|
606
|
+
data: { reason: reason.to_s, called_from: caller_frame && "#{caller_frame.path}:#{caller_frame.lineno}", at: Time.current },
|
|
607
|
+
metadata: Lyra::BypassEvents.current_metadata.merge(source: "lyra_unrecorded_write")
|
|
608
|
+
)
|
|
609
|
+
append_events(event, stream_name: UNRECORDED_WRITES_STREAM)
|
|
610
|
+
rescue => e
|
|
611
|
+
# As for any event: Hijack and event sourcing fail the write, Monitor logs and goes on.
|
|
612
|
+
raise if config.events_required?
|
|
613
|
+
|
|
614
|
+
Rails.logger.error("Lyra: could not record an unrecorded write (#{reason}) - #{e.message}")
|
|
615
|
+
end
|
|
616
|
+
private_class_method :record_unrecorded_write
|
|
617
|
+
|
|
618
|
+
module Events
|
|
619
|
+
class UnrecordedWrite < Lyra::Event; end
|
|
239
620
|
end
|
|
240
621
|
end
|
data/lib/lyra/command_handler.rb
CHANGED
|
@@ -57,7 +57,13 @@ module Lyra
|
|
|
57
57
|
# table's PostgreSQL sequence). Elsewhere hijack falls back to the
|
|
58
58
|
# "pending-<hex>" placeholder, with a one-time warning, rather than risk
|
|
59
59
|
# a duplicate key.
|
|
60
|
-
|
|
60
|
+
#
|
|
61
|
+
# The key is the model's primary key column, whatever its name ("code",
|
|
62
|
+
# a uuid column): the value the application gave it is kept, and the
|
|
63
|
+
# event and the result carry the key under that column, never under an
|
|
64
|
+
# "id" the table may not have.
|
|
65
|
+
key = Lyra.key_column(model_class)
|
|
66
|
+
explicit_id = (command.attributes[key] || command.attributes[key.to_sym]).presence
|
|
61
67
|
id = explicit_id ||
|
|
62
68
|
(IdGenerator.next_id(model_class) if Lyra.event_sourcing_mode? || IdGenerator.reserves_safely?(model_class))
|
|
63
69
|
unless id
|
|
@@ -71,7 +77,7 @@ module Lyra
|
|
|
71
77
|
|
|
72
78
|
# Create event with pre-generated ID
|
|
73
79
|
# Use symbolize_keys for consistent key types in event data
|
|
74
|
-
event_attrs = command.attributes.symbolize_keys.merge(
|
|
80
|
+
event_attrs = command.attributes.symbolize_keys.merge(key.to_sym => id)
|
|
75
81
|
events = create_events(:created, id, event_attrs)
|
|
76
82
|
event = events.first
|
|
77
83
|
|
|
@@ -89,10 +95,10 @@ module Lyra
|
|
|
89
95
|
# Return result with the ID, so the caller inserts the row under the same
|
|
90
96
|
# ID the event already carries.
|
|
91
97
|
attributes = command.attributes.dup
|
|
92
|
-
# Remove string
|
|
98
|
+
# Remove the string key to prevent conflict with the symbol one
|
|
93
99
|
# (AR attributes have string keys, but we add symbol keys)
|
|
94
|
-
attributes.delete(
|
|
95
|
-
attributes[
|
|
100
|
+
attributes.delete(key)
|
|
101
|
+
attributes[key.to_sym] = id unless id.to_s.start_with?("pending-")
|
|
96
102
|
CommandResult.success(attributes: attributes, events: events)
|
|
97
103
|
end
|
|
98
104
|
|
|
@@ -164,16 +170,15 @@ module Lyra
|
|
|
164
170
|
end
|
|
165
171
|
|
|
166
172
|
def create_event(operation, id, data)
|
|
167
|
-
event_data =
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
operation:
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
timestamp: Time.current
|
|
174
|
-
}
|
|
173
|
+
event_data = Lyra::EventData.build(
|
|
174
|
+
command.model_class, id, operation,
|
|
175
|
+
# A create's data is the attributes themselves; an update's is { changes: }.
|
|
176
|
+
attributes: operation.to_sym == :updated ? {} : (data[:attributes] || data),
|
|
177
|
+
changes: data[:changes] || {}
|
|
178
|
+
)
|
|
175
179
|
|
|
176
|
-
|
|
180
|
+
# source is Lyra's default: a metadata_proc that sets its own wins.
|
|
181
|
+
event_metadata = { source: 'lyra_command_handler' }.merge(attribution_metadata(operation))
|
|
177
182
|
|
|
178
183
|
config = Lyra.config.model_config(command.model_class)
|
|
179
184
|
event_name = config.event_name_for(operation)
|