orfeas_lyra 0.9.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +2021 -17
- data/README.md +22 -9
- data/Rakefile +40 -1
- data/app/controllers/lyra/application_controller.rb +127 -0
- data/app/controllers/lyra/dashboard_controller.rb +33 -31
- data/app/controllers/lyra/flow_controller.rb +32 -34
- data/app/controllers/lyra/privacy_controller.rb +57 -18
- data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
- data/app/views/lyra/dashboard/index.html.erb +1 -0
- data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
- data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
- data/app/views/lyra/privacy/policy.html.erb +7 -0
- data/app/views/lyra/privacy/rights.html.erb +78 -0
- data/config/privacy_policies.rb +0 -3
- data/config/routes.rb +1 -0
- data/lib/lyra/access_log.rb +187 -6
- data/lib/lyra/advisory_lock.rb +236 -0
- data/lib/lyra/aggregate.rb +1 -1
- data/lib/lyra/associations/habtm_links.rb +382 -19
- data/lib/lyra/associations/maintenance.rb +229 -0
- data/lib/lyra/attribution.rb +48 -0
- data/lib/lyra/bypass_events.rb +446 -46
- data/lib/lyra/command_handler.rb +19 -14
- data/lib/lyra/configuration.rb +237 -10
- data/lib/lyra/consistency/read_your_writes.rb +4 -2
- data/lib/lyra/correlation.rb +121 -0
- data/lib/lyra/domain_events.rb +4 -0
- data/lib/lyra/dual_view.rb +53 -16
- data/lib/lyra/encrypted_attributes.rb +5 -0
- data/lib/lyra/engine.rb +67 -0
- data/lib/lyra/erasure.rb +1011 -106
- data/lib/lyra/event.rb +35 -1
- data/lib/lyra/event_data.rb +38 -0
- data/lib/lyra/event_flow.rb +31 -15
- data/lib/lyra/event_store_adapter.rb +28 -0
- data/lib/lyra/event_window.rb +58 -0
- data/lib/lyra/genesis.rb +548 -44
- data/lib/lyra/interceptors/association_interceptor.rb +689 -34
- data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
- data/lib/lyra/internal.rb +26 -0
- data/lib/lyra/mode_sync.rb +51 -3
- data/lib/lyra/mode_transition.rb +257 -30
- data/lib/lyra/model_selection.rb +35 -0
- data/lib/lyra/own_connection.rb +17 -2
- data/lib/lyra/privacy/adapters/pam.rb +25 -2
- data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
- data/lib/lyra/privacy/interface.rb +27 -7
- data/lib/lyra/privacy/subject_links.rb +143 -0
- data/lib/lyra/privacy/subject_records.rb +369 -0
- data/lib/lyra/projection.rb +7 -1
- data/lib/lyra/projections/async_checkpoint.rb +32 -0
- data/lib/lyra/projections/async_projection_job.rb +166 -23
- data/lib/lyra/projections/cached_joins.rb +33 -18
- data/lib/lyra/projections/cached_projection.rb +74 -33
- data/lib/lyra/projections/cached_relation.rb +3175 -321
- data/lib/lyra/projections/default_scopes.rb +254 -0
- data/lib/lyra/projections/deferred_relation.rb +23 -3
- data/lib/lyra/projections/event_store_reader.rb +47 -8
- data/lib/lyra/projections/foreign_keys.rb +92 -0
- data/lib/lyra/projections/lazy_projection.rb +165 -27
- data/lib/lyra/projections/model_projection.rb +165 -14
- data/lib/lyra/projections/rebuild.rb +34 -6
- data/lib/lyra/projections/reference_links.rb +463 -72
- data/lib/lyra/projections/typed_values.rb +629 -0
- data/lib/lyra/projections/unique_indexes.rb +595 -0
- data/lib/lyra/projections/uniqueness.rb +784 -0
- data/lib/lyra/projections/value_digest.rb +93 -0
- data/lib/lyra/purpose_bound_reads.rb +912 -77
- data/lib/lyra/repair.rb +119 -28
- data/lib/lyra/retention.rb +114 -8
- data/lib/lyra/rights/consent_store.rb +415 -0
- data/lib/lyra/rights/erasure_guard.rb +135 -0
- data/lib/lyra/rights/pending.rb +74 -0
- data/lib/lyra/rights/queries.rb +269 -0
- data/lib/lyra/rights/store.rb +512 -0
- data/lib/lyra/rights/subject_keys.rb +224 -0
- data/lib/lyra/rights.rb +281 -0
- data/lib/lyra/saves_without_effect.rb +69 -0
- data/lib/lyra/stream_tail.rb +319 -0
- data/lib/lyra/strict_data_access.rb +916 -106
- data/lib/lyra/temporal.rb +4 -2
- data/lib/lyra/touch_registry.rb +404 -0
- data/lib/lyra/verification/bypass_workflow.rb +452 -42
- data/lib/lyra/verification/rights_workflows.rb +170 -0
- data/lib/lyra/version.rb +1 -1
- data/lib/lyra/visualization/event_graph.rb +1 -1
- data/lib/lyra/visualization/timeline.rb +2 -2
- data/lib/lyra.rb +43 -2
- data/lib/tasks/lyra_mode.rake +110 -8
- data/lib/tasks/lyra_projections.rake +43 -2
- metadata +34 -4
- data/lib/lyra/associations/event_aware.rb +0 -224
data/lib/lyra/genesis.rb
CHANGED
|
@@ -38,11 +38,130 @@ module Lyra
|
|
|
38
38
|
# Enabled with config.genesis: :auto (the default) runs it in event-sourcing
|
|
39
39
|
# mode, where the event log is the only source of truth; true also runs it in
|
|
40
40
|
# Monitor and Hijack modes (for DualView and Rebuild); false never runs it.
|
|
41
|
+
#
|
|
42
|
+
# Values the model cannot represent (from 1.0.0). The Imported event holds
|
|
43
|
+
# the row as the model reads it. A stored value the model cannot read --
|
|
44
|
+
# refund_requests.status = '2' under a string enum without that value, which
|
|
45
|
+
# the model reads as nil -- would give an event that disagrees with its row,
|
|
46
|
+
# and the disagreement would surface later and elsewhere (an ES-Lazy
|
|
47
|
+
# catch-up writing NULL into a NOT NULL column). So every attribute of every
|
|
48
|
+
# row is checked first (loss): the value the model reads, written back by
|
|
49
|
+
# the model's type, must be the value stored, compared as the column's own
|
|
50
|
+
# type reads both or as the database would be sent them. '1' and 1, a
|
|
51
|
+
# timestamp and its time zone, a time of day, a decimal's trailing zeros or
|
|
52
|
+
# a jsonb object's key order are no difference; a lossy reading is (an enum
|
|
53
|
+
# value outside the mapping, `attribute :n, :integer` over 10.50, an inet
|
|
54
|
+
# value's host bits, which Rails drops, also in an inet[] array, a value out
|
|
55
|
+
# of the model's integer range). Whatever the type, a stored value the model
|
|
56
|
+
# writes back as NULL is lost, in a nullable column too: a JSON null in a
|
|
57
|
+
# json/jsonb column, an empty range, a blank or 'null' JSON text or an empty
|
|
58
|
+
# YAML hash in a serialized column (read as nil or {}). A serialized,
|
|
59
|
+
# encrypted or JSON attribute's stored text is otherwise one of many
|
|
60
|
+
# encodings of its value (JSON whitespace, a fresh ciphertext), so its text
|
|
61
|
+
# is not compared; a value of another class is. Columns the model ignores
|
|
62
|
+
# are imported as stored and not checked.
|
|
63
|
+
#
|
|
64
|
+
# What happens to such rows is config.genesis_unrepresentable (or the
|
|
65
|
+
# +unrepresentable:+ argument of import and import_all):
|
|
66
|
+
# - :refuse (default): every row is checked before anything is written, and
|
|
67
|
+
# the model's import raises Refused, listing the rows (model, id,
|
|
68
|
+
# attribute, stored value, value read; at most LISTED of them, with the
|
|
69
|
+
# count): nothing of the model is imported. The import runs in a savepoint
|
|
70
|
+
# inside a caller's transaction, so a caller that rescues the refusal
|
|
71
|
+
# commits nothing of it either. Fix the rows or the model, then import
|
|
72
|
+
# again. A refused first use is remembered for REFUSAL_RECHECK seconds
|
|
73
|
+
# from the scan that found it: the uses meanwhile raise it again without
|
|
74
|
+
# scanning; an explicit import (lyra:genesis) always scans. At a gated
|
|
75
|
+
# mode switch the refusal refuses the switch (ModeTransition);
|
|
76
|
+
# - :skip: those rows are left without a stream and the rest is imported;
|
|
77
|
+
# import's Result and `lyra:genesis` list them, and the log warns. A write
|
|
78
|
+
# to a skipped row -- through callbacks (update!, destroy!) or bypassing
|
|
79
|
+
# them (update_columns, touch, update_all, touch_all, delete_all,
|
|
80
|
+
# upsert_all, ...; admit_all!) -- or an ES-NoProj find of it, imports it
|
|
81
|
+
# if the model can now represent it, and is refused (Refused, naming the
|
|
82
|
+
# rows) while it cannot: its stream would otherwise begin with the
|
|
83
|
+
# write's event. A query over many records (where, count) does not see
|
|
84
|
+
# it. So a skipped row is fixed in the database (SQL), not through the
|
|
85
|
+
# model; the next use of it, or lyra:genesis, then imports it.
|
|
86
|
+
# A process imports on its own first use of a model and keeps what it
|
|
87
|
+
# skipped: a row written outside Lyra (SQL) after that first use, or
|
|
88
|
+
# skipped by another process, is unknown to it until lyra:genesis imports
|
|
89
|
+
# it. Values of attributes the model's privacy policy declares, or
|
|
90
|
+
# encrypted ones, are not shown in the listing.
|
|
41
91
|
module Genesis
|
|
42
92
|
BATCH = 1_000
|
|
43
93
|
OPERATION = :imported
|
|
94
|
+
LISTED = 20
|
|
95
|
+
|
|
96
|
+
# One stored value the model cannot represent: +stored+ as the row holds
|
|
97
|
+
# it, +read_as+ as the model reads it (and the Imported event would hold
|
|
98
|
+
# it). +reason+: nil (the value read writes back to another value),
|
|
99
|
+
# :null (it would be written back as NULL),
|
|
100
|
+
# :out_of_range (the model's type cannot hold it), or a String. +personal+:
|
|
101
|
+
# the values are not shown.
|
|
102
|
+
Unrepresentable = Struct.new(:model, :id, :attribute, :stored, :read_as, :personal, :reason, keyword_init: true) do
|
|
103
|
+
def details
|
|
104
|
+
shown = personal ? "#{attribute} (a personal value, not shown)" : "#{attribute} stored #{stored.inspect}"
|
|
105
|
+
read = personal ? "" : ", read as #{read_as.inspect}"
|
|
106
|
+
case reason
|
|
107
|
+
when nil then "#{shown}#{read}"
|
|
108
|
+
when :null then "#{shown}#{read}, which would be written back as NULL"
|
|
109
|
+
when :out_of_range then "#{shown}: out of range for the model's type"
|
|
110
|
+
else "#{shown}: #{reason}"
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def to_s = "#{model} #{id}: #{details}"
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# The model holds rows it cannot represent; nothing of it was imported.
|
|
118
|
+
# +rows+: how many rows; +values+: the first LISTED Unrepresentable values
|
|
119
|
+
# of the +found+ in all. +id+: a row an import skipped, refused on a write
|
|
120
|
+
# or read of it (see admit!), or the rows (an Array) a bulk write would
|
|
121
|
+
# have written (admit_all!).
|
|
122
|
+
class Refused < StandardError
|
|
123
|
+
attr_reader :model, :rows, :values, :found, :id
|
|
124
|
+
|
|
125
|
+
def initialize(model, rows:, values:, found:, id: nil)
|
|
126
|
+
@model = model
|
|
127
|
+
@rows = rows
|
|
128
|
+
@values = values
|
|
129
|
+
@found = found
|
|
130
|
+
@id = id
|
|
131
|
+
listing = "#{values.map(&:to_s).join('; ')}#{" (and #{found - values.size} more)" if found > values.size}"
|
|
132
|
+
super(if id
|
|
133
|
+
"Lyra::Genesis left #{model} #{Array(id).join(', ')} without events (genesis_unrepresentable :skip): " \
|
|
134
|
+
"#{id.is_a?(Array) ? 'they hold' : 'it holds'} values the model cannot represent, so a write or read " \
|
|
135
|
+
"would start a stream without its state. Fix the row in the database (SQL) or the model; the next " \
|
|
136
|
+
"use imports it. #{listing}"
|
|
137
|
+
else
|
|
138
|
+
"Lyra::Genesis will not import #{model}: #{rows} row(s) hold values the model cannot represent, so " \
|
|
139
|
+
"their Imported events would not match them. Fix the rows or the model; or skip those rows " \
|
|
140
|
+
"(config.genesis_unrepresentable = :skip, or UNREPRESENTABLE=skip for lyra:genesis). #{listing}"
|
|
141
|
+
end)
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
# What import did: +imported+ rows; +skipped+ rows the model cannot
|
|
146
|
+
# represent (with :skip), and the first LISTED of their values.
|
|
147
|
+
Result = Struct.new(:model, :imported, :skipped, :unrepresentable, keyword_init: true) do
|
|
148
|
+
def summary
|
|
149
|
+
"#{model}: #{imported} rows imported#{", #{skipped} skipped (values the model cannot represent)" if skipped.positive?}"
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# How an attribute's value goes back to its column (round_trip_type).
|
|
154
|
+
RoundTripType = Struct.new(:column, :writer, :opaque, :enum, :inet)
|
|
155
|
+
private_constant :RoundTripType
|
|
156
|
+
|
|
157
|
+
# A refused first use is not repeated for this long (seconds): the next
|
|
158
|
+
# uses raise the same refusal without scanning the table again. An
|
|
159
|
+
# explicit import (lyra:genesis) always scans.
|
|
160
|
+
REFUSAL_RECHECK = 30
|
|
44
161
|
|
|
45
162
|
@imported = {}
|
|
163
|
+
@skipped = {}
|
|
164
|
+
@refused = {}
|
|
46
165
|
@mutex = Mutex.new
|
|
47
166
|
|
|
48
167
|
class << self
|
|
@@ -55,16 +174,220 @@ module Lyra
|
|
|
55
174
|
end
|
|
56
175
|
|
|
57
176
|
# Import +model_class+'s rows without a stream, once per process.
|
|
58
|
-
# Returns the number of rows imported by this call.
|
|
59
|
-
|
|
177
|
+
# Returns the number of rows imported by this call. +id+: the row about
|
|
178
|
+
# to be written or read, refused if an import skipped it and it still
|
|
179
|
+
# holds values the model cannot represent (admit!).
|
|
180
|
+
def first_use(model_class, id = nil)
|
|
60
181
|
return 0 unless enabled?
|
|
61
|
-
return 0 if imported?(model_class)
|
|
62
182
|
return 0 if Thread.current[:lyra_genesis_running]
|
|
63
183
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
184
|
+
count = 0
|
|
185
|
+
unless imported?(model_class)
|
|
186
|
+
# Raised here, not in import_on_first_use, whose rescue would store
|
|
187
|
+
# it again as new: a busy process would then never scan again.
|
|
188
|
+
if (refusal = recent_refusal(model_class))
|
|
189
|
+
raise Refused.new(refusal.model, rows: refusal.rows, values: refusal.values, found: refusal.found)
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
count = import_on_first_use(model_class)
|
|
193
|
+
end
|
|
194
|
+
admit!(model_class, id) unless id.nil?
|
|
195
|
+
count
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
# Before a write that bypasses callbacks and records an event per row
|
|
199
|
+
# (update_all, update_columns, touch, delete_all, upsert_all, ...): the
|
|
200
|
+
# model's first use, as for a write through callbacks, and then admit!
|
|
201
|
+
# for each of +ids+ the import skipped. Rows the model can now represent
|
|
202
|
+
# are imported first; the write is refused (Refused, naming the rows)
|
|
203
|
+
# if any still cannot be, so a bypass event never begins a stream. Costs
|
|
204
|
+
# nothing more than first_use while no row was skipped.
|
|
205
|
+
def admit_all!(model_class, ids)
|
|
206
|
+
return unless enabled?
|
|
207
|
+
return if Thread.current[:lyra_genesis_running]
|
|
208
|
+
|
|
209
|
+
first_use(model_class)
|
|
210
|
+
return if @skipped.empty?
|
|
211
|
+
|
|
212
|
+
refusals = ids.filter_map do |id|
|
|
213
|
+
next unless skipped?(model_class, id)
|
|
214
|
+
|
|
215
|
+
admit!(model_class, id)
|
|
216
|
+
nil
|
|
217
|
+
rescue Refused => e
|
|
218
|
+
e
|
|
219
|
+
end
|
|
220
|
+
return if refusals.empty?
|
|
221
|
+
|
|
222
|
+
values = refusals.flat_map(&:values)
|
|
223
|
+
raise Refused.new(model_class.name, rows: refusals.size, values: values.first(LISTED),
|
|
224
|
+
found: refusals.sum(&:found),
|
|
225
|
+
id: refusals.one? ? refusals.first.id : refusals.map(&:id))
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
# Import every row of +model_class+ that has no stream, now, whatever the
|
|
229
|
+
# mode. Returns the number of rows imported (see import for the rest).
|
|
230
|
+
def import_all(model_class, unrepresentable: Lyra.config.genesis_unrepresentable)
|
|
231
|
+
import(model_class, unrepresentable: unrepresentable).imported
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
# Import every row of +model_class+ that has no stream, now, whatever the
|
|
235
|
+
# mode, in one transaction (a savepoint inside the caller's). With
|
|
236
|
+
# +unrepresentable+ :refuse every row is checked first, and one holding
|
|
237
|
+
# a value the model cannot represent (see above) raises Refused before
|
|
238
|
+
# anything is written; with :skip such rows are left out, and refused
|
|
239
|
+
# on a later write or read (admit!). Returns a Result.
|
|
240
|
+
def import(model_class, unrepresentable: Lyra.config.genesis_unrepresentable)
|
|
241
|
+
skip = unrepresentable_choice(unrepresentable) == :skip
|
|
242
|
+
running = Thread.current[:lyra_genesis_running]
|
|
243
|
+
Thread.current[:lyra_genesis_running] = true
|
|
244
|
+
imported = []
|
|
245
|
+
skipped = []
|
|
246
|
+
listed = []
|
|
247
|
+
found = 0
|
|
248
|
+
|
|
249
|
+
model_class.with_connection do |connection|
|
|
250
|
+
model_class.transaction(requires_new: true) do
|
|
251
|
+
lock!(connection, model_class)
|
|
252
|
+
types = {}
|
|
253
|
+
unless skip
|
|
254
|
+
bad = 0
|
|
255
|
+
each_row_without_stream(connection, model_class) do |record, row, attributes|
|
|
256
|
+
problems = unrepresentable_values(connection, types, record, row, attributes)
|
|
257
|
+
next if problems.empty?
|
|
258
|
+
|
|
259
|
+
bad += 1
|
|
260
|
+
found += problems.size
|
|
261
|
+
listed.concat(problems.first(LISTED - listed.size))
|
|
262
|
+
end
|
|
263
|
+
raise Refused.new(model_class.name, rows: bad, values: listed, found: found) if bad.positive?
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
each_row_without_stream(connection, model_class) do |record, row, attributes|
|
|
267
|
+
problems = unrepresentable_values(connection, types, record, row, attributes)
|
|
268
|
+
if problems.empty?
|
|
269
|
+
imported << import_record(model_class, record, attributes)
|
|
270
|
+
elsif skip
|
|
271
|
+
skipped << record.id.to_s
|
|
272
|
+
found += problems.size
|
|
273
|
+
listed.concat(problems.first(LISTED - listed.size))
|
|
274
|
+
else
|
|
275
|
+
# Changed since the check above; rolls back what was imported.
|
|
276
|
+
raise Refused.new(model_class.name, rows: 1, values: problems.first(LISTED), found: problems.size)
|
|
277
|
+
end
|
|
278
|
+
end
|
|
279
|
+
end
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
@mutex.synchronize do
|
|
283
|
+
skipped.empty? ? @skipped.delete(owner(model_class)) : @skipped[owner(model_class)] = skipped.to_set
|
|
284
|
+
@refused.delete(owner(model_class))
|
|
285
|
+
end
|
|
286
|
+
forget_cached(model_class, imported)
|
|
287
|
+
warn_unrepresentable(model_class, skipped.size, listed, found) if skipped.any?
|
|
288
|
+
Result.new(model: model_class.name, imported: imported.size, skipped: skipped.size, unrepresentable: listed)
|
|
289
|
+
ensure
|
|
290
|
+
Thread.current[:lyra_genesis_running] = running
|
|
291
|
+
end
|
|
292
|
+
|
|
293
|
+
# Import the one row +id+ of +model_class+ if it still has no stream, now,
|
|
294
|
+
# whatever the mode, under the model's Genesis lock (see hold_lock!).
|
|
295
|
+
# Lyra::Erasure imports a row it erased that had no stream, so that the
|
|
296
|
+
# Imported event holds the erased values, never the personal ones (from
|
|
297
|
+
# 1.0.0). Returns whether it imported the row. A value the model cannot
|
|
298
|
+
# represent is logged, not refused: an erasure must not fail on it.
|
|
299
|
+
def import_one(model_class, id)
|
|
300
|
+
running = Thread.current[:lyra_genesis_running]
|
|
301
|
+
Thread.current[:lyra_genesis_running] = true
|
|
302
|
+
model_class.with_connection do |connection|
|
|
303
|
+
lock!(connection, model_class)
|
|
304
|
+
row = rows_without_stream(connection, model_class, nil, limit: 1, id: id).first
|
|
305
|
+
record, attributes = read_row(model_class, row) if row
|
|
306
|
+
next false if record.nil? || record.id.nil?
|
|
307
|
+
|
|
308
|
+
problems = unrepresentable_values(connection, {}, record, row, attributes)
|
|
309
|
+
warn_unrepresentable(model_class, 0, problems, problems.size) if problems.any?
|
|
310
|
+
forget_cached(model_class, [import_record(model_class, record, attributes)])
|
|
311
|
+
forget_skipped(model_class, id)
|
|
312
|
+
true
|
|
313
|
+
end
|
|
314
|
+
ensure
|
|
315
|
+
Thread.current[:lyra_genesis_running] = running
|
|
316
|
+
end
|
|
317
|
+
|
|
318
|
+
# Before a write to, or a read of, the row +id+: if an import skipped it
|
|
319
|
+
# (:skip), import it now when the model can represent it, or else raise
|
|
320
|
+
# Refused for it. Its stream would otherwise begin with the write's
|
|
321
|
+
# Updated event, no Imported event giving the rest of its state, and an
|
|
322
|
+
# ES-NoProj read would build a partial record from it.
|
|
323
|
+
def admit!(model_class, id)
|
|
324
|
+
return unless skipped?(model_class, id)
|
|
325
|
+
|
|
326
|
+
inline = Lyra::OwnConnection.run(model_class) do |in_caller_transaction|
|
|
327
|
+
import_skipped(model_class, id)
|
|
328
|
+
in_caller_transaction
|
|
329
|
+
end
|
|
330
|
+
forget = -> { forget_skipped(model_class, id) }
|
|
331
|
+
inline ? Lyra::OwnConnection.after_commit(model_class, &forget) : forget.call
|
|
332
|
+
end
|
|
333
|
+
|
|
334
|
+
# Whether an import in this process skipped the row +id+ (:skip).
|
|
335
|
+
def skipped?(model_class, id)
|
|
336
|
+
return false if @skipped.empty? # every write asks: no lock while nothing was skipped
|
|
337
|
+
|
|
338
|
+
@mutex.synchronize { @skipped[owner(model_class)]&.include?(id.to_s) || false }
|
|
339
|
+
end
|
|
340
|
+
|
|
341
|
+
# Take +model_class+'s Genesis lock until the caller's transaction ends: no
|
|
342
|
+
# Genesis run imports one of its rows meanwhile. Lyra::Erasure takes it
|
|
343
|
+
# before erasing a row with no stream, so no import elsewhere reads the row
|
|
344
|
+
# as it was.
|
|
345
|
+
def hold_lock!(model_class)
|
|
346
|
+
model_class.with_connection { |connection| lock!(connection, model_class) }
|
|
347
|
+
end
|
|
348
|
+
|
|
349
|
+
def imported?(model_class)
|
|
350
|
+
@mutex.synchronize { @imported.key?(model_class.name) }
|
|
351
|
+
end
|
|
352
|
+
|
|
353
|
+
# Forget which models were imported, which rows were skipped and which
|
|
354
|
+
# imports were refused (tests, or after the event store or the tables
|
|
355
|
+
# were reset).
|
|
356
|
+
def reset!
|
|
357
|
+
@mutex.synchronize do
|
|
358
|
+
@imported.clear
|
|
359
|
+
@skipped.clear
|
|
360
|
+
@refused.clear
|
|
361
|
+
end
|
|
362
|
+
end
|
|
363
|
+
|
|
364
|
+
# The stored values of +record+, as loaded from its row and not changed
|
|
365
|
+
# since, that its model does not represent (the check above), as
|
|
366
|
+
# Unrepresentable; only those of +columns+ if given. Lyra::Repair checks
|
|
367
|
+
# what it writes into an event this way (from 1.0.0).
|
|
368
|
+
def unrepresentable(record, columns: nil)
|
|
369
|
+
row = record.attributes_before_type_cast
|
|
370
|
+
row = row.slice(*columns.map(&:to_s)) if columns
|
|
371
|
+
attributes = record.attributes
|
|
372
|
+
record.class.with_connection { |connection| unrepresentable_values(connection, {}, record, row, attributes) }
|
|
373
|
+
end
|
|
374
|
+
|
|
375
|
+
# :refuse or :skip; a string (from ENV, say) is taken as its symbol.
|
|
376
|
+
def unrepresentable_choice(value)
|
|
377
|
+
choice = value.is_a?(String) ? value.strip.to_sym : value
|
|
378
|
+
return choice if Lyra::Configuration::GENESIS_UNREPRESENTABLE.include?(choice)
|
|
379
|
+
|
|
380
|
+
raise ArgumentError, "unrepresentable must be :refuse or :skip (got #{value.inspect})"
|
|
381
|
+
end
|
|
382
|
+
|
|
383
|
+
private
|
|
384
|
+
|
|
385
|
+
# The first use's import. Usually every row already has a stream: then
|
|
386
|
+
# nothing is written and no lock is needed. The model counts as done
|
|
387
|
+
# once the caller's transaction, if any, commits: the streams it sees
|
|
388
|
+
# may be an inline import's, uncommitted in that transaction (see
|
|
389
|
+
# below). A refusal is remembered for REFUSAL_RECHECK seconds.
|
|
390
|
+
def import_on_first_use(model_class)
|
|
68
391
|
if model_class.with_connection { |c| rows_without_stream(c, model_class, nil, limit: 1) }.empty?
|
|
69
392
|
Lyra::OwnConnection.after_commit(model_class) { mark_imported(model_class) }
|
|
70
393
|
return 0
|
|
@@ -81,55 +404,82 @@ module Lyra
|
|
|
81
404
|
mark = -> { mark_imported(model_class) }
|
|
82
405
|
inline ? Lyra::OwnConnection.after_commit(model_class, &mark) : mark.call
|
|
83
406
|
count
|
|
407
|
+
rescue Refused => e
|
|
408
|
+
@mutex.synchronize { @refused[owner(model_class)] = [Process.clock_gettime(Process::CLOCK_MONOTONIC), e] } if e.id.nil?
|
|
409
|
+
raise
|
|
84
410
|
end
|
|
85
411
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
imported = []
|
|
412
|
+
def recent_refusal(model_class)
|
|
413
|
+
at, refusal = @mutex.synchronize { @refused[owner(model_class)] }
|
|
414
|
+
refusal if at && Process.clock_gettime(Process::CLOCK_MONOTONIC) - at < REFUSAL_RECHECK
|
|
415
|
+
end
|
|
91
416
|
|
|
417
|
+
# admit!'s import of one skipped row, under the Genesis lock.
|
|
418
|
+
def import_skipped(model_class, id)
|
|
419
|
+
running = Thread.current[:lyra_genesis_running]
|
|
420
|
+
Thread.current[:lyra_genesis_running] = true
|
|
92
421
|
model_class.with_connection do |connection|
|
|
93
|
-
model_class.transaction do
|
|
422
|
+
model_class.transaction(requires_new: true) do
|
|
94
423
|
lock!(connection, model_class)
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
break if rows.empty?
|
|
424
|
+
row = rows_without_stream(connection, model_class, nil, limit: 1, id: id).first
|
|
425
|
+
record, attributes = read_row(model_class, row) if row
|
|
426
|
+
next if record.nil? || record.id.nil? # imported meanwhile, or gone
|
|
99
427
|
|
|
100
|
-
|
|
101
|
-
|
|
428
|
+
problems = unrepresentable_values(connection, {}, record, row, attributes)
|
|
429
|
+
if problems.any?
|
|
430
|
+
raise Refused.new(model_class.name, rows: 1, values: problems.first(LISTED), found: problems.size, id: id)
|
|
102
431
|
end
|
|
432
|
+
|
|
433
|
+
forget_cached(model_class, [import_record(model_class, record, attributes)])
|
|
103
434
|
end
|
|
104
435
|
end
|
|
105
|
-
|
|
106
|
-
forget_cached(model_class, imported)
|
|
107
|
-
imported.size
|
|
108
436
|
ensure
|
|
109
|
-
Thread.current[:lyra_genesis_running] =
|
|
437
|
+
Thread.current[:lyra_genesis_running] = running
|
|
110
438
|
end
|
|
111
439
|
|
|
112
|
-
def
|
|
113
|
-
@mutex.synchronize
|
|
440
|
+
def forget_skipped(model_class, id)
|
|
441
|
+
@mutex.synchronize do
|
|
442
|
+
rows = @skipped[owner(model_class)]
|
|
443
|
+
rows&.delete(id.to_s)
|
|
444
|
+
@skipped.delete(owner(model_class)) if rows&.empty?
|
|
445
|
+
end
|
|
114
446
|
end
|
|
115
447
|
|
|
116
|
-
#
|
|
117
|
-
#
|
|
118
|
-
def
|
|
119
|
-
@mutex.synchronize { @imported.clear }
|
|
120
|
-
end
|
|
448
|
+
# Skipped rows and refusals are kept per table (its stream owner): an STI
|
|
449
|
+
# subclass shares its base class's rows.
|
|
450
|
+
def owner(model_class) = Lyra.stream_owner_name(model_class)
|
|
121
451
|
|
|
122
|
-
|
|
452
|
+
# Each row of +model_class+ without a stream, in batches, as the model
|
|
453
|
+
# reads it: yields the record, the row and the record's attributes.
|
|
454
|
+
def each_row_without_stream(connection, model_class)
|
|
455
|
+
after = nil
|
|
456
|
+
loop do
|
|
457
|
+
rows = rows_without_stream(connection, model_class, after)
|
|
458
|
+
break if rows.empty?
|
|
459
|
+
|
|
460
|
+
rows.each do |row|
|
|
461
|
+
record, attributes = read_row(model_class, row)
|
|
462
|
+
yield record, row, attributes unless record.id.nil?
|
|
463
|
+
end
|
|
464
|
+
after = rows.last[model_class.primary_key]
|
|
465
|
+
end
|
|
466
|
+
end
|
|
123
467
|
|
|
124
468
|
def mark_imported(model_class)
|
|
125
469
|
@mutex.synchronize { @imported[model_class.name] = true }
|
|
126
470
|
end
|
|
127
471
|
|
|
128
|
-
def rows_without_stream(connection, model_class, after, limit: BATCH)
|
|
472
|
+
def rows_without_stream(connection, model_class, after, limit: BATCH, id: nil)
|
|
473
|
+
# A primary key of several columns (a model enrolled before Active
|
|
474
|
+
# Record read its key): Lyra's refusal (ArgumentError), before the
|
|
475
|
+
# query fails on it (PG::UndefinedColumn) and takes the transaction
|
|
476
|
+
# it is in with it.
|
|
477
|
+
Lyra::ModelSelection.refuse_composite_key!(model_class) if model_class.primary_key.is_a?(Array)
|
|
129
478
|
table = connection.quote_table_name(model_class.table_name)
|
|
130
479
|
pk = connection.quote_column_name(model_class.primary_key)
|
|
131
480
|
prefix = connection.quote(Lyra.stream_prefix(model_class))
|
|
132
481
|
keyset = after.nil? ? "" : "AND t.#{pk} > #{connection.quote(after)}"
|
|
482
|
+
keyset += " AND t.#{pk} = #{connection.quote(id)}" unless id.nil?
|
|
133
483
|
|
|
134
484
|
connection.select_all(<<~SQL.squish).to_a
|
|
135
485
|
SELECT t.* FROM #{table} t
|
|
@@ -141,18 +491,14 @@ module Lyra
|
|
|
141
491
|
SQL
|
|
142
492
|
end
|
|
143
493
|
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
494
|
+
# The row as the model reads it: the record and its attributes.
|
|
495
|
+
def read_row(model_class, row)
|
|
496
|
+
record = Lyra::PurposeBoundReads.internal(INTERNAL) { model_class.instantiate(row) }
|
|
497
|
+
[record, record.id.nil? ? nil : record.attributes]
|
|
498
|
+
end
|
|
147
499
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
model_id: record.id,
|
|
151
|
-
operation: OPERATION,
|
|
152
|
-
attributes: record.attributes,
|
|
153
|
-
changes: {},
|
|
154
|
-
timestamp: Time.current
|
|
155
|
-
}
|
|
500
|
+
def import_record(model_class, record, attributes)
|
|
501
|
+
data = Lyra::EventData.build(model_class, record.id, OPERATION, attributes: attributes)
|
|
156
502
|
event = event_class(model_class).new(data: data, metadata: Lyra::Privacy.stamp(model_class, data, { genesis: true }))
|
|
157
503
|
# The lock and the NOT EXISTS above make the stream empty here.
|
|
158
504
|
stream_name = Lyra.stream_name(model_class, record.id)
|
|
@@ -162,6 +508,160 @@ module Lyra
|
|
|
162
508
|
record.id
|
|
163
509
|
end
|
|
164
510
|
|
|
511
|
+
# The values of +row+ that +record+, the model's reading of it, does not
|
|
512
|
+
# represent (see the module comment), as Unrepresentable. +types+ caches
|
|
513
|
+
# each attribute's types for one import.
|
|
514
|
+
def unrepresentable_values(connection, types, record, row, attributes)
|
|
515
|
+
klass = record.class
|
|
516
|
+
row.filter_map do |name, stored|
|
|
517
|
+
next if stored.nil?
|
|
518
|
+
|
|
519
|
+
type = types.fetch([klass, name]) { types[[klass, name]] = round_trip_type(connection, klass, name) }
|
|
520
|
+
next if type.nil?
|
|
521
|
+
|
|
522
|
+
reason = loss(connection, type, stored, attributes[name])
|
|
523
|
+
next if reason.nil?
|
|
524
|
+
|
|
525
|
+
personal = (types[[klass, :personal]] ||= personal_attributes(klass)).include?(name)
|
|
526
|
+
Unrepresentable.new(model: klass.name, id: record.id, attribute: name, stored: stored,
|
|
527
|
+
read_as: attributes[name], personal: personal, reason: (reason unless reason == :value))
|
|
528
|
+
end
|
|
529
|
+
end
|
|
530
|
+
|
|
531
|
+
# How +name+'s value goes back to its column: +column+, the column's own
|
|
532
|
+
# type (no enum, serializer, encryption or time zone); +writer+, the
|
|
533
|
+
# model's type without normalizes (which applies to assignment, not to a
|
|
534
|
+
# value read); +opaque+, a serialized, encrypted or JSON attribute (a
|
|
535
|
+
# stored text of many encodings); +enum+, an enum; +inet+, a PostgreSQL
|
|
536
|
+
# inet column or array of inet. nil for a column the model ignores.
|
|
537
|
+
def round_trip_type(connection, klass, name)
|
|
538
|
+
column = klass.columns_hash[name]
|
|
539
|
+
return nil if column.nil?
|
|
540
|
+
|
|
541
|
+
type = klass.attribute_types[name]
|
|
542
|
+
column_type = if column.respond_to?(:fetch_cast_type)
|
|
543
|
+
column.fetch_cast_type(connection)
|
|
544
|
+
else
|
|
545
|
+
connection.lookup_cast_type_from_column(column)
|
|
546
|
+
end
|
|
547
|
+
writer = type
|
|
548
|
+
writer = writer.cast_type while writer.class.name.to_s.end_with?("NormalizedValueType") && writer.respond_to?(:cast_type)
|
|
549
|
+
chain = type_chain(type)
|
|
550
|
+
opaque = chain.any? do |t|
|
|
551
|
+
t.is_a?(ActiveRecord::Type::Serialized) || t.is_a?(ActiveRecord::Type::Json) ||
|
|
552
|
+
t.class.name == "ActiveRecord::Encryption::EncryptedAttributeType"
|
|
553
|
+
end
|
|
554
|
+
inet = type_chain(column_type).any? { |t| t.class.name.to_s.end_with?("::OID::Inet") }
|
|
555
|
+
RoundTripType.new(column_type, writer, opaque, chain.any? { |t| t.is_a?(ActiveRecord::Enum::EnumType) }, inet)
|
|
556
|
+
end
|
|
557
|
+
|
|
558
|
+
# +type+ and the types it decorates (time zone, enum, serializer,
|
|
559
|
+
# encryption, normalizes, array element).
|
|
560
|
+
def type_chain(type)
|
|
561
|
+
chain = []
|
|
562
|
+
while type && chain.size < 8
|
|
563
|
+
chain << type
|
|
564
|
+
type = if type.respond_to?(:cast_type) then type.cast_type
|
|
565
|
+
elsif type.respond_to?(:subtype) then type.subtype
|
|
566
|
+
elsif type.is_a?(Delegator) then type.__getobj__
|
|
567
|
+
end
|
|
568
|
+
end
|
|
569
|
+
chain
|
|
570
|
+
end
|
|
571
|
+
|
|
572
|
+
# nil if +read_as+ writes back to +stored+ (not NULL: the caller skips
|
|
573
|
+
# NULLs): equal as the column's own type reads them, or as the database
|
|
574
|
+
# would be sent them (a time of day read in another time zone carries
|
|
575
|
+
# another date, which the column does not keep). Otherwise why not (see
|
|
576
|
+
# Unrepresentable#reason; :value: it writes back to another value).
|
|
577
|
+
#
|
|
578
|
+
# Whatever the type, a value the model writes back as NULL is lost: an
|
|
579
|
+
# enum value outside the mapping, a JSON null in a json/jsonb column, an
|
|
580
|
+
# empty range, a blank or 'null' JSON text or an empty YAML hash in a
|
|
581
|
+
# serialized column (read as nil or {}, written back as NULL), into a
|
|
582
|
+
# nullable column too (where(col: nil) would then find the row).
|
|
583
|
+
#
|
|
584
|
+
# A serialized, encrypted or JSON attribute's stored text is one of many
|
|
585
|
+
# encodings of its value (JSON whitespace, a fresh ciphertext), so a
|
|
586
|
+
# re-encoding need not equal it: there it is checked that the value
|
|
587
|
+
# comes back (not NULL, above, and not a value of another class).
|
|
588
|
+
def loss(connection, type, stored, read_as)
|
|
589
|
+
serialized = begin
|
|
590
|
+
type.writer.serialize(read_as)
|
|
591
|
+
rescue ActiveModel::RangeError
|
|
592
|
+
return :out_of_range
|
|
593
|
+
rescue StandardError => e
|
|
594
|
+
return "cannot be written back (#{e.class})"
|
|
595
|
+
end
|
|
596
|
+
return :value if type.enum && read_as.nil?
|
|
597
|
+
return :null if serialized.nil?
|
|
598
|
+
|
|
599
|
+
stored_value = type.column.deserialize(stored)
|
|
600
|
+
if same_written?(connection, type.column, stored_value, serialized)
|
|
601
|
+
return type.inet && host_bits_lost?(stored, stored_value) ? "host bits lost (read as #{stored_value.inspect})" : nil
|
|
602
|
+
end
|
|
603
|
+
|
|
604
|
+
written = written_value(type.column, serialized)
|
|
605
|
+
return :value unless type.opaque
|
|
606
|
+
return :value if !written.nil? && !stored_value.nil? && written.class != stored_value.class
|
|
607
|
+
|
|
608
|
+
nil
|
|
609
|
+
rescue StandardError
|
|
610
|
+
:value
|
|
611
|
+
end
|
|
612
|
+
|
|
613
|
+
# Whether +serialized+, the model's writing of the value it read, is the
|
|
614
|
+
# stored value: equal as the column's own type reads both, or as the
|
|
615
|
+
# database would be sent them.
|
|
616
|
+
def same_written?(connection, column_type, stored_value, serialized)
|
|
617
|
+
written = written_value(column_type, serialized)
|
|
618
|
+
return true if same_value?(stored_value, written)
|
|
619
|
+
|
|
620
|
+
sent = connection.type_cast(column_type.serialize(stored_value))
|
|
621
|
+
sent == connection.type_cast(column_type.serialize(written)) || sent == connection.type_cast(serialized)
|
|
622
|
+
rescue StandardError
|
|
623
|
+
false
|
|
624
|
+
end
|
|
625
|
+
|
|
626
|
+
def written_value(column_type, serialized)
|
|
627
|
+
column_type.deserialize(serialized)
|
|
628
|
+
rescue StandardError
|
|
629
|
+
serialized
|
|
630
|
+
end
|
|
631
|
+
|
|
632
|
+
def same_value?(stored, written)
|
|
633
|
+
stored == written || (stored.respond_to?(:nan?) && written.respond_to?(:nan?) && stored.nan? && written.nan?)
|
|
634
|
+
end
|
|
635
|
+
|
|
636
|
+
# An inet value keeps its host bits (192.168.1.5/24); Rails reads it as
|
|
637
|
+
# an IPAddr of the network (192.168.1.0/24), and writes that back.
|
|
638
|
+
# In an inet[] array, element by element.
|
|
639
|
+
def host_bits_lost?(stored, read)
|
|
640
|
+
return false unless stored.is_a?(String)
|
|
641
|
+
if read.is_a?(Array)
|
|
642
|
+
texts = PG::TextDecoder::Array.new.decode(stored).flatten
|
|
643
|
+
return texts.zip(read.flatten).any? { |text, element| text && host_bits_lost?(text, element) }
|
|
644
|
+
end
|
|
645
|
+
return false unless read.is_a?(IPAddr)
|
|
646
|
+
|
|
647
|
+
IPAddr.new(stored.split("/").first).to_i != read.to_i
|
|
648
|
+
rescue IPAddr::Error, PG::Error
|
|
649
|
+
false
|
|
650
|
+
end
|
|
651
|
+
|
|
652
|
+
# Attributes whose values a listing does not show: those the privacy
|
|
653
|
+
# policy declares, and encrypted ones.
|
|
654
|
+
def personal_attributes(klass)
|
|
655
|
+
Lyra::Privacy.policy_for(klass).declared_fields.map(&:to_s) | Lyra::EncryptedAttributes.names(klass)
|
|
656
|
+
end
|
|
657
|
+
|
|
658
|
+
def warn_unrepresentable(model_class, rows, listed, found)
|
|
659
|
+
what = rows.positive? ? "skipped #{rows} row(s) of #{model_class.name}" : "imported a row of #{model_class.name}"
|
|
660
|
+
more = found > listed.size ? " (and #{found - listed.size} more)" : ""
|
|
661
|
+
Rails.logger&.warn("Lyra::Genesis #{what} holding values the model cannot represent: " \
|
|
662
|
+
"#{listed.map(&:to_s).join('; ')}#{more}")
|
|
663
|
+
end
|
|
664
|
+
|
|
165
665
|
# An ES-NoProj lookup made before the import (in an earlier process, or
|
|
166
666
|
# before the event store was reset) may have cached "not found" for these
|
|
167
667
|
# records. invalidate_all cannot drop per-record entries on every store
|
|
@@ -181,8 +681,12 @@ module Lyra
|
|
|
181
681
|
Lyra::Events.const_set(name, Class.new(Lyra::Event))
|
|
182
682
|
end
|
|
183
683
|
|
|
684
|
+
# One lock per table, named after its stream owner (the STI base class):
|
|
685
|
+
# an import through a subclass and one through the base class, or an
|
|
686
|
+
# erasure of one of its rows (hold_lock!), wait for each other. Before
|
|
687
|
+
# 1.0.0 it was named after the class given.
|
|
184
688
|
def lock!(connection, model_class)
|
|
185
|
-
Lyra::AdvisoryLock.xact_lock(connection, "lyra_genesis/#{model_class
|
|
689
|
+
Lyra::AdvisoryLock.xact_lock(connection, "lyra_genesis/#{Lyra.stream_owner_name(model_class)}", purpose: "Genesis")
|
|
186
690
|
end
|
|
187
691
|
end
|
|
188
692
|
end
|