orfeas_lyra 0.9.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +2021 -17
- data/README.md +22 -9
- data/Rakefile +40 -1
- data/app/controllers/lyra/application_controller.rb +127 -0
- data/app/controllers/lyra/dashboard_controller.rb +33 -31
- data/app/controllers/lyra/flow_controller.rb +32 -34
- data/app/controllers/lyra/privacy_controller.rb +57 -18
- data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
- data/app/views/lyra/dashboard/index.html.erb +1 -0
- data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
- data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
- data/app/views/lyra/privacy/policy.html.erb +7 -0
- data/app/views/lyra/privacy/rights.html.erb +78 -0
- data/config/privacy_policies.rb +0 -3
- data/config/routes.rb +1 -0
- data/lib/lyra/access_log.rb +187 -6
- data/lib/lyra/advisory_lock.rb +236 -0
- data/lib/lyra/aggregate.rb +1 -1
- data/lib/lyra/associations/habtm_links.rb +382 -19
- data/lib/lyra/associations/maintenance.rb +229 -0
- data/lib/lyra/attribution.rb +48 -0
- data/lib/lyra/bypass_events.rb +446 -46
- data/lib/lyra/command_handler.rb +19 -14
- data/lib/lyra/configuration.rb +237 -10
- data/lib/lyra/consistency/read_your_writes.rb +4 -2
- data/lib/lyra/correlation.rb +121 -0
- data/lib/lyra/domain_events.rb +4 -0
- data/lib/lyra/dual_view.rb +53 -16
- data/lib/lyra/encrypted_attributes.rb +5 -0
- data/lib/lyra/engine.rb +67 -0
- data/lib/lyra/erasure.rb +1011 -106
- data/lib/lyra/event.rb +35 -1
- data/lib/lyra/event_data.rb +38 -0
- data/lib/lyra/event_flow.rb +31 -15
- data/lib/lyra/event_store_adapter.rb +28 -0
- data/lib/lyra/event_window.rb +58 -0
- data/lib/lyra/genesis.rb +548 -44
- data/lib/lyra/interceptors/association_interceptor.rb +689 -34
- data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
- data/lib/lyra/internal.rb +26 -0
- data/lib/lyra/mode_sync.rb +51 -3
- data/lib/lyra/mode_transition.rb +257 -30
- data/lib/lyra/model_selection.rb +35 -0
- data/lib/lyra/own_connection.rb +17 -2
- data/lib/lyra/privacy/adapters/pam.rb +25 -2
- data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
- data/lib/lyra/privacy/interface.rb +27 -7
- data/lib/lyra/privacy/subject_links.rb +143 -0
- data/lib/lyra/privacy/subject_records.rb +369 -0
- data/lib/lyra/projection.rb +7 -1
- data/lib/lyra/projections/async_checkpoint.rb +32 -0
- data/lib/lyra/projections/async_projection_job.rb +166 -23
- data/lib/lyra/projections/cached_joins.rb +33 -18
- data/lib/lyra/projections/cached_projection.rb +74 -33
- data/lib/lyra/projections/cached_relation.rb +3175 -321
- data/lib/lyra/projections/default_scopes.rb +254 -0
- data/lib/lyra/projections/deferred_relation.rb +23 -3
- data/lib/lyra/projections/event_store_reader.rb +47 -8
- data/lib/lyra/projections/foreign_keys.rb +92 -0
- data/lib/lyra/projections/lazy_projection.rb +165 -27
- data/lib/lyra/projections/model_projection.rb +165 -14
- data/lib/lyra/projections/rebuild.rb +34 -6
- data/lib/lyra/projections/reference_links.rb +463 -72
- data/lib/lyra/projections/typed_values.rb +629 -0
- data/lib/lyra/projections/unique_indexes.rb +595 -0
- data/lib/lyra/projections/uniqueness.rb +784 -0
- data/lib/lyra/projections/value_digest.rb +93 -0
- data/lib/lyra/purpose_bound_reads.rb +912 -77
- data/lib/lyra/repair.rb +119 -28
- data/lib/lyra/retention.rb +114 -8
- data/lib/lyra/rights/consent_store.rb +415 -0
- data/lib/lyra/rights/erasure_guard.rb +135 -0
- data/lib/lyra/rights/pending.rb +74 -0
- data/lib/lyra/rights/queries.rb +269 -0
- data/lib/lyra/rights/store.rb +512 -0
- data/lib/lyra/rights/subject_keys.rb +224 -0
- data/lib/lyra/rights.rb +281 -0
- data/lib/lyra/saves_without_effect.rb +69 -0
- data/lib/lyra/stream_tail.rb +319 -0
- data/lib/lyra/strict_data_access.rb +916 -106
- data/lib/lyra/temporal.rb +4 -2
- data/lib/lyra/touch_registry.rb +404 -0
- data/lib/lyra/verification/bypass_workflow.rb +452 -42
- data/lib/lyra/verification/rights_workflows.rb +170 -0
- data/lib/lyra/version.rb +1 -1
- data/lib/lyra/visualization/event_graph.rb +1 -1
- data/lib/lyra/visualization/timeline.rb +2 -2
- data/lib/lyra.rb +43 -2
- data/lib/tasks/lyra_mode.rake +110 -8
- data/lib/tasks/lyra_projections.rake +43 -2
- metadata +34 -4
- data/lib/lyra/associations/event_aware.rb +0 -224
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Lyra
|
|
4
|
+
# What only Lyra's own call sites hold. Lyra.projection_write and
|
|
5
|
+
# PurposeBoundReads.internal switch off the checks that keep every write in
|
|
6
|
+
# the log and every read under its purpose; they are for Lyra's projections,
|
|
7
|
+
# replays and tasks, and application code that called them could change a row
|
|
8
|
+
# with no event or read a declared field under :deny.
|
|
9
|
+
#
|
|
10
|
+
# The constant is private, so only code written inside `module Lyra` can name it.
|
|
11
|
+
# That keeps application code from calling them by accident or by following the
|
|
12
|
+
# documentation; it is not a barrier against code that is out to defeat it
|
|
13
|
+
# (Lyra.const_get reaches it), which no Ruby code could provide.
|
|
14
|
+
#
|
|
15
|
+
# Application code that needs to write without events (seeding, fixtures, a
|
|
16
|
+
# table wipe whose history is not wanted) uses Lyra.unrecorded_write, which asks
|
|
17
|
+
# for a reason and records that it happened.
|
|
18
|
+
INTERNAL = Object.new.freeze
|
|
19
|
+
private_constant :INTERNAL
|
|
20
|
+
|
|
21
|
+
# Lyra's internal switches called from outside Lyra.
|
|
22
|
+
class InternalApiError < StandardError; end
|
|
23
|
+
|
|
24
|
+
# Lyra.unrecorded_write while config.unrecorded_writes is :deny.
|
|
25
|
+
class UnrecordedWriteDenied < StandardError; end
|
|
26
|
+
end
|
data/lib/lyra/mode_sync.rb
CHANGED
|
@@ -13,7 +13,13 @@ module Lyra
|
|
|
13
13
|
# Each process remembers the latest applied row it has seen. At most every
|
|
14
14
|
# config.mode_sync_interval seconds (default 5) it looks for a newer one,
|
|
15
15
|
# and adopts its mode: the process that recorded it already passed the
|
|
16
|
-
# gate, so the gate is not run again
|
|
16
|
+
# gate, so the gate is not run again; but a switch the gate checks (an
|
|
17
|
+
# escalation, or leaving a lagging mode) is adopted only if the
|
|
18
|
+
# row names the certificate of the clean check that cleared it (or the operator
|
|
19
|
+
# overrode the gate, which the row says), so one INSERT cannot move every
|
|
20
|
+
# process to a mode whose stores were never compared (0.9.3 adopted any row;
|
|
21
|
+
# ModeTransition.adoption_refusal).
|
|
22
|
+
# It looks:
|
|
17
23
|
# - at the start of each web request (Middleware);
|
|
18
24
|
# - before each background job (ActiveJob before_perform);
|
|
19
25
|
# - before each write to a monitored model, which covers consoles and
|
|
@@ -44,6 +50,11 @@ module Lyra
|
|
|
44
50
|
# Cheap when nothing is due: a clock read. Returns the label adopted, or
|
|
45
51
|
# nil.
|
|
46
52
|
def maybe_sync!
|
|
53
|
+
# A process that leaves touches out keeps its registry entry fresh from
|
|
54
|
+
# here, outside any transaction (nothing with config.record_touches true).
|
|
55
|
+
if Lyra.config.record_touches == false && !ActiveRecord::Base.with_connection(&:transaction_open?)
|
|
56
|
+
TouchRegistry.heartbeat_if_due
|
|
57
|
+
end
|
|
47
58
|
return nil unless enabled?
|
|
48
59
|
return nil if due_in.positive?
|
|
49
60
|
return nil if ActiveRecord::Base.with_connection(&:transaction_open?)
|
|
@@ -60,11 +71,48 @@ module Lyra
|
|
|
60
71
|
return nil unless row
|
|
61
72
|
return nil if @last_seen_id && row[:id] <= @last_seen_id
|
|
62
73
|
|
|
63
|
-
|
|
64
|
-
|
|
74
|
+
if row[:config] == ModeTransition.current
|
|
75
|
+
@last_seen_id = row[:id]
|
|
76
|
+
return nil
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# The row counts as seen once it is decided: adopted, or refused (logged
|
|
80
|
+
# once). An error while deciding leaves it unseen, so the next check tries
|
|
81
|
+
# again rather than leave this process in its mode for good.
|
|
82
|
+
# The application runs in a mode whose log rebuilds rows, and this
|
|
83
|
+
# process leaves touches out: it must not go on that way. It refuses
|
|
84
|
+
# every touch it would leave out from now on (TouchRegistry.fence!),
|
|
85
|
+
# and each check raises, until it runs with record_touches on.
|
|
86
|
+
if Lyra.config.record_touches == false && TouchRegistry.authoritative?(row[:config])
|
|
87
|
+
TouchRegistry.fence!(row[:config])
|
|
88
|
+
message = TouchRegistry.process_refusal(row[:config])
|
|
89
|
+
Rails.logger.error(message)
|
|
90
|
+
raise ModeTransition::Refused, message
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Not while another process leaves touches out of the log: the row
|
|
94
|
+
# stays unseen, and this process adopts it once that process's entry
|
|
95
|
+
# has ended or expired (as a fenced process retries).
|
|
96
|
+
if TouchRegistry.authoritative?(row[:config]) &&
|
|
97
|
+
(touches = TouchRegistry.gate_refusal(ModeTransition.current, row[:config]))
|
|
98
|
+
if @touch_refused_id != row[:id]
|
|
99
|
+
@touch_refused_id = row[:id]
|
|
100
|
+
Rails.logger.error("Lyra: did not adopt the application's mode #{row[:config]} yet; this process stays " \
|
|
101
|
+
"in #{ModeTransition.current} and tries again: #{touches}")
|
|
102
|
+
end
|
|
103
|
+
return nil
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
if (refusal = ModeTransition.adoption_refusal(row))
|
|
107
|
+
@last_seen_id = row[:id]
|
|
108
|
+
Rails.logger.error("Lyra: did not adopt the application's mode #{row[:config]}; this process stays in " \
|
|
109
|
+
"#{ModeTransition.current}: #{refusal}")
|
|
110
|
+
return nil
|
|
111
|
+
end
|
|
65
112
|
|
|
66
113
|
previous = ModeTransition.current
|
|
67
114
|
ModeTransition.apply_label(row[:config])
|
|
115
|
+
@last_seen_id = row[:id]
|
|
68
116
|
Rails.logger.warn("Lyra: adopted the application's mode #{row[:config]} (was #{previous}), switched elsewhere")
|
|
69
117
|
row[:config]
|
|
70
118
|
end
|
data/lib/lyra/mode_transition.rb
CHANGED
|
@@ -9,13 +9,17 @@ module Lyra
|
|
|
9
9
|
# - Escalation, from Disabled or Monitor to Hijack or event sourcing: the
|
|
10
10
|
# events become authoritative, so they must reproduce every row. Rows
|
|
11
11
|
# that predate Lyra are imported first (Lyra::Genesis), or each would
|
|
12
|
-
# count as a discrepancy.
|
|
12
|
+
# count as a discrepancy. A row holding a value its model cannot
|
|
13
|
+
# represent is not imported, and refuses the switch (from 1.0.0).
|
|
13
14
|
# - Leaving a mode whose tables lag the log (ES-NoProj, ES-Lazy, ES-Async)
|
|
14
15
|
# for one that reads them: the tables must catch up first. ES-Lazy
|
|
15
16
|
# catches up on read; ES-NoProj needs a Rebuild (rebuild: true).
|
|
16
17
|
# Every other switch (Monitor to Disabled, ES-Sync to ES-NoProj, ...) keeps
|
|
17
18
|
# the authoritative store as it is and passes freely. Entering ES-NoProj
|
|
18
|
-
# also links the events' foreign keys (Lyra::Projections::ReferenceLinks)
|
|
19
|
+
# also links the events' foreign keys (Lyra::Projections::ReferenceLinks),
|
|
20
|
+
# and is refused for a default scope ES-NoProj cannot evaluate and for a
|
|
21
|
+
# database foreign key from an unmonitored table into a monitored one
|
|
22
|
+
# (Lyra::Projections::ForeignKeys); force: true overrides both.
|
|
19
23
|
#
|
|
20
24
|
# A check compares every row with the state replayed from its stream, and
|
|
21
25
|
# every stream with its row, in batches, reading the real table (never the
|
|
@@ -55,13 +59,18 @@ module Lyra
|
|
|
55
59
|
def to_s = "#{model} #{id}: #{problem}#{" (#{details})" if details}"
|
|
56
60
|
end
|
|
57
61
|
|
|
62
|
+
# +unrepresentable+: rows Genesis would not import, holding values their
|
|
63
|
+
# model cannot represent (each listed value is also a discrepancy, up to
|
|
64
|
+
# Lyra::Genesis::LISTED per model).
|
|
58
65
|
Report = Struct.new(:from, :to, :checked, :rechecked, :imported, :rebuilt, :discrepancies,
|
|
59
|
-
:position, :started_at, :finished_at,
|
|
66
|
+
:position, :started_at, :finished_at, :certificate_id, :unrepresentable,
|
|
67
|
+
keyword_init: true) do
|
|
60
68
|
def clean? = discrepancies.empty?
|
|
61
69
|
|
|
62
70
|
def summary
|
|
63
71
|
"#{from} -> #{to}: #{checked} records checked, #{rechecked} re-checked, " \
|
|
64
|
-
"#{imported} imported, #{discrepancies.size} discrepancies"
|
|
72
|
+
"#{imported} imported, #{discrepancies.size} discrepancies" \
|
|
73
|
+
"#{", #{unrepresentable} rows with values their model cannot represent" if unrepresentable.to_i.positive?}"
|
|
65
74
|
end
|
|
66
75
|
end
|
|
67
76
|
|
|
@@ -99,6 +108,11 @@ module Lyra
|
|
|
99
108
|
from = current
|
|
100
109
|
to = label(mode, projection_mode)
|
|
101
110
|
report = nil
|
|
111
|
+
# Not overridden by force: no mode whose log rebuilds rows may be entered
|
|
112
|
+
# while touches are left out of it, here or in another process.
|
|
113
|
+
refuse_unrecorded_touches!(from, to)
|
|
114
|
+
refuse_unevaluable_default_scopes!(from, to) unless force
|
|
115
|
+
refuse_foreign_keys_into_event_only_tables!(from, to) unless force
|
|
102
116
|
|
|
103
117
|
if gate_enabled? && gate_required?(from, to) && !force
|
|
104
118
|
report = certified_check(from, to) || check(to: to, from: from, rebuild: rebuild)
|
|
@@ -109,8 +123,17 @@ module Lyra
|
|
|
109
123
|
end
|
|
110
124
|
|
|
111
125
|
link_references if to == "event_sourcing/disabled" && from != to
|
|
112
|
-
|
|
113
|
-
|
|
126
|
+
# The registry's last word and the switch's record, under the registry's
|
|
127
|
+
# lock: a process registering to leave touches out either sees this
|
|
128
|
+
# switch (and records its touch) or is seen here (and the switch is
|
|
129
|
+
# refused). TouchRegistry.serialize.
|
|
130
|
+
id = TouchRegistry.serialize(to) do
|
|
131
|
+
refuse_unrecorded_touches!(from, to)
|
|
132
|
+
apply(mode, projection_mode)
|
|
133
|
+
# forced: the gate was required and the operator overrode it.
|
|
134
|
+
record_applied(to, from: from, forced: force && gate_enabled? && gate_required?(from, to),
|
|
135
|
+
certificate_id: report&.certificate_id)
|
|
136
|
+
end
|
|
114
137
|
ModeSync.seen!(id)
|
|
115
138
|
announce(from, to)
|
|
116
139
|
report
|
|
@@ -124,49 +147,133 @@ module Lyra
|
|
|
124
147
|
apply(mode, projection_mode)
|
|
125
148
|
end
|
|
126
149
|
|
|
127
|
-
# The application-wide mode:
|
|
150
|
+
# The application-wide mode: the latest applied row, as { id:, config:, from:,
|
|
151
|
+
# forced:, certificate_id:, at: }. from and certificate_id are nil and forced
|
|
152
|
+
# false in a row an earlier version wrote.
|
|
128
153
|
def latest_applied
|
|
129
154
|
return nil unless table_ready?
|
|
130
155
|
|
|
131
|
-
row = sql(:select_one,
|
|
132
|
-
|
|
156
|
+
row = sql(:select_one,
|
|
157
|
+
"SELECT id, to_config, from_config, forced, certificate_id, created_at FROM #{TABLE} " \
|
|
158
|
+
"WHERE kind = 'applied' ORDER BY id DESC LIMIT 1")
|
|
159
|
+
return nil unless row
|
|
160
|
+
|
|
161
|
+
at = row["created_at"].is_a?(String) ? Time.zone.parse(row["created_at"]) : row["created_at"]
|
|
162
|
+
{ id: row["id"].to_i, config: row["to_config"], from: row["from_config"].presence,
|
|
163
|
+
forced: row["forced"] == true || row["forced"] == "t", certificate_id: row["certificate_id"]&.to_i, at: at }
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
# Why ModeSync must not adopt +row+ (latest_applied's hash), or nil when it may.
|
|
167
|
+
#
|
|
168
|
+
# With the gate off nothing is certified, and a switch is adopted as recorded.
|
|
169
|
+
# With it on, a switch the gate checks (gate_required?: an escalation from
|
|
170
|
+
# Disabled or Monitor, or leaving a lagging mode for one that reads the
|
|
171
|
+
# tables) must have passed the gate: the row names the certificate of
|
|
172
|
+
# the clean check that cleared it, one for the row's own from -> to, recorded
|
|
173
|
+
# before the row and named by no earlier switch. The switcher checked the
|
|
174
|
+
# certificate's age when it switched, so no clock is compared here. A row
|
|
175
|
+
# recorded with the gate overridden (forced) is adopted: the operator meant
|
|
176
|
+
# it, and it is on record.
|
|
177
|
+
#
|
|
178
|
+
# Whether the switch is gated is judged from the row's from and from where the
|
|
179
|
+
# switch before it left the application, so a row cannot name a from that
|
|
180
|
+
# makes it ungated. A row an earlier version wrote has no from and names no
|
|
181
|
+
# certificate; it is judged from this process's mode.
|
|
182
|
+
#
|
|
183
|
+
# This is a check against mistakes and naive writes, not against anyone who
|
|
184
|
+
# can write the table: such a writer can also write the certificate.
|
|
185
|
+
def adoption_refusal(row, from: current)
|
|
186
|
+
return nil unless gate_enabled?
|
|
187
|
+
|
|
188
|
+
to = row[:config]
|
|
189
|
+
from = row[:from] || from
|
|
190
|
+
previous = previous_applied(row[:id])
|
|
191
|
+
return nil unless gate_required?(from, to) || (previous && gate_required?(previous, to))
|
|
192
|
+
return nil if row[:forced]
|
|
193
|
+
return nil if names_its_certificate?(row, from)
|
|
194
|
+
|
|
195
|
+
named = row[:certificate_id] ? "certificate #{row[:certificate_id]}, which does not clear it" : "no certificate"
|
|
196
|
+
"no clean check certifies #{from} -> #{to} for this switch (row #{row[:id]} names #{named}); " \
|
|
197
|
+
"only a gated switch records both"
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
# Where the applied row before +id+ left the application, or nil.
|
|
201
|
+
def previous_applied(id)
|
|
202
|
+
sql(:select_value,
|
|
203
|
+
"SELECT to_config FROM #{TABLE} WHERE kind = 'applied' AND id < #{Integer(id)} ORDER BY id DESC LIMIT 1")
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
def names_its_certificate?(row, from)
|
|
207
|
+
return false unless row[:certificate_id]
|
|
208
|
+
|
|
209
|
+
cert = Integer(row[:certificate_id])
|
|
210
|
+
id = Integer(row[:id])
|
|
211
|
+
sql(:select_value,
|
|
212
|
+
"SELECT 1 FROM #{TABLE} c WHERE c.id = #{cert} AND c.kind = 'certificate' AND c.from_config = #{q(from)} " \
|
|
213
|
+
"AND c.to_config = #{q(row[:config])} AND c.id < #{id} AND NOT EXISTS (SELECT 1 FROM #{TABLE} a " \
|
|
214
|
+
"WHERE a.kind = 'applied' AND a.certificate_id = #{cert} AND a.id < #{id})").present?
|
|
133
215
|
end
|
|
134
216
|
|
|
135
217
|
# Compare every monitored record's row with its events. Returns a
|
|
136
218
|
# Report; stores a certificate when it is clean.
|
|
137
219
|
def check(to:, from: current, models: Lyra.config.monitored_models, rebuild: false, batch_size: 1_000)
|
|
220
|
+
# With record_touches false DualView takes timestamp-only differences
|
|
221
|
+
# as agreement, and a process that leaves touches out moves rows its
|
|
222
|
+
# events do not show: no certificate may come of either.
|
|
223
|
+
refuse_unrecorded_touches!(from, to)
|
|
138
224
|
started_at = Time.current
|
|
139
225
|
position = event_position
|
|
140
226
|
imported = 0
|
|
141
227
|
rebuilt = 0
|
|
142
|
-
|
|
228
|
+
discrepancies = []
|
|
229
|
+
unrepresentable = 0
|
|
230
|
+
|
|
231
|
+
# A row its model cannot represent refuses the model's import (nothing
|
|
232
|
+
# of the model is imported), whatever config.genesis_unrepresentable
|
|
233
|
+
# says: a skipped row would have no events. Each listed value is a
|
|
234
|
+
# discrepancy; the model's rows are not compared (each would be "row
|
|
235
|
+
# but no events").
|
|
236
|
+
refused = []
|
|
143
237
|
if !%w[disabled monitor].include?(to) && %w[disabled monitor].include?(from)
|
|
144
|
-
models.each
|
|
238
|
+
models.each do |model|
|
|
239
|
+
imported += Lyra::Genesis.import_all(model, unrepresentable: :refuse)
|
|
240
|
+
rescue Lyra::Genesis::Refused => e
|
|
241
|
+
refused << model
|
|
242
|
+
unrepresentable += e.rows
|
|
243
|
+
e.values.each do |value|
|
|
244
|
+
discrepancies << Discrepancy.new(model: value.model, id: value.id, problem: "a value the model cannot represent",
|
|
245
|
+
details: value.details)
|
|
246
|
+
end
|
|
247
|
+
end
|
|
145
248
|
end
|
|
146
249
|
if LAGGING.include?(from)
|
|
147
250
|
Lyra::Projections::LazyProjection.catch_up!(force: true) if from == "event_sourcing/lazy"
|
|
148
251
|
rebuilt = models.sum { |model| Lyra::Projections::Rebuild.rebuild(model)[:records] } if rebuild
|
|
149
252
|
end
|
|
150
253
|
|
|
151
|
-
discrepancies = []
|
|
152
254
|
checked = 0
|
|
153
|
-
models.each do |model|
|
|
255
|
+
(models - refused).each do |model|
|
|
154
256
|
ids_to_check(model).each_slice(batch_size) do |ids|
|
|
155
257
|
ids.each { |id| (problem = compare(model, id)) && discrepancies << problem }
|
|
156
258
|
checked += ids.size
|
|
157
259
|
end
|
|
158
260
|
end
|
|
159
261
|
|
|
160
|
-
changed = changed_since(models, position, started_at)
|
|
262
|
+
changed = changed_since(models - refused, position, started_at)
|
|
161
263
|
changed.each do |model, ids|
|
|
162
264
|
discrepancies.reject! { |d| d.model == model.name && ids.include?(d.id.to_s) }
|
|
163
265
|
ids.each { |id| (problem = compare(model, id)) && discrepancies << problem }
|
|
164
266
|
end
|
|
165
267
|
|
|
166
268
|
report = Report.new(from: from, to: to, checked: checked, rechecked: changed.values.sum(&:size),
|
|
167
|
-
imported: imported, rebuilt: rebuilt, discrepancies: discrepancies,
|
|
269
|
+
imported: imported, rebuilt: rebuilt, discrepancies: discrepancies, unrepresentable: unrepresentable,
|
|
168
270
|
position: event_position, started_at: started_at, finished_at: Time.current)
|
|
169
|
-
|
|
271
|
+
# A process may have begun leaving touches out while the check ran: the
|
|
272
|
+
# last look and the certificate under the registry's lock.
|
|
273
|
+
TouchRegistry.serialize(to) do
|
|
274
|
+
refuse_unrecorded_touches!(from, to)
|
|
275
|
+
certify(report) if report.clean?
|
|
276
|
+
end
|
|
170
277
|
report
|
|
171
278
|
end
|
|
172
279
|
|
|
@@ -178,14 +285,18 @@ module Lyra
|
|
|
178
285
|
|
|
179
286
|
to = current
|
|
180
287
|
last = last_applied
|
|
288
|
+
if TouchRegistry.authoritative?(to) && (touches = TouchRegistry.gate_refusal(last || to, to))
|
|
289
|
+
raise Refused, touches
|
|
290
|
+
end
|
|
181
291
|
if last.nil? || last == to || !gate_required?(last, to)
|
|
182
|
-
ModeSync.seen!(last == to ? latest_applied&.dig(:id) : record_applied(to))
|
|
292
|
+
ModeSync.seen!(last == to ? latest_applied&.dig(:id) : record_applied(to, from: last))
|
|
183
293
|
return
|
|
184
294
|
end
|
|
185
295
|
|
|
186
296
|
report = certified_check(last, to)
|
|
187
297
|
if report&.clean? || ENV["LYRA_FORCE_MODE_TRANSITION"].present?
|
|
188
|
-
ModeSync.seen!(record_applied(to
|
|
298
|
+
ModeSync.seen!(record_applied(to, from: last, forced: !report&.clean?,
|
|
299
|
+
certificate_id: (report.certificate_id if report&.clean?)))
|
|
189
300
|
return
|
|
190
301
|
end
|
|
191
302
|
|
|
@@ -194,6 +305,33 @@ module Lyra
|
|
|
194
305
|
"or set LYRA_FORCE_MODE_TRANSITION=1.", report)
|
|
195
306
|
end
|
|
196
307
|
|
|
308
|
+
# At boot, in ES-NoProj set by configuration (config.projection_mode =
|
|
309
|
+
# :disabled in an initializer) rather than reached by a switch: what
|
|
310
|
+
# the switch into ES-NoProj would refuse, as one warning each, for the
|
|
311
|
+
# models enrolled by then (with eager loading, all of them): a database
|
|
312
|
+
# foreign key into an event-only table, and a default scope ES-NoProj
|
|
313
|
+
# cannot evaluate. Never stops the boot: config.mode = ... is the raw,
|
|
314
|
+
# ungated setter, and a model enrolled later is not seen here.
|
|
315
|
+
# Returns the warnings.
|
|
316
|
+
def check_es_noproj!(models: Lyra.config.monitored_models)
|
|
317
|
+
return [] unless Lyra.config.event_sourcing_mode? && Lyra.config.projection_mode == :disabled
|
|
318
|
+
|
|
319
|
+
foreign_keys = Lyra::Projections::ForeignKeys
|
|
320
|
+
messages = foreign_keys.blocking(models).map do |fk|
|
|
321
|
+
"Lyra: in ES-NoProj every write to #{fk.from_table} that sets #{fk.column} will fail. " \
|
|
322
|
+
"#{foreign_keys.description([fk])} #{foreign_keys.remedy([fk])} " \
|
|
323
|
+
"The switch into ES-NoProj (Lyra::ModeTransition.to!) refuses this."
|
|
324
|
+
end
|
|
325
|
+
messages += Lyra::Projections::DefaultScopes.unevaluable(models).map do |model, reason|
|
|
326
|
+
"Lyra: in ES-NoProj every read of #{model.name} will be refused: ES-NoProj cannot evaluate its default " \
|
|
327
|
+
"scope from the event store (#{reason}). Write the default scope as hash conditions and column " \
|
|
328
|
+
"orderings, or use another projection mode."
|
|
329
|
+
end
|
|
330
|
+
logger = defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
|
|
331
|
+
messages.each { |message| logger ? logger.warn(message) : Kernel.warn(message) }
|
|
332
|
+
messages
|
|
333
|
+
end
|
|
334
|
+
|
|
197
335
|
def last_applied
|
|
198
336
|
return nil unless table_ready?
|
|
199
337
|
|
|
@@ -203,6 +341,9 @@ module Lyra
|
|
|
203
341
|
# Every id of +model+ with a row or a stream (Lyra::Repair).
|
|
204
342
|
def record_ids(model) = ids_to_check(model)
|
|
205
343
|
|
|
344
|
+
# Whether the table can be used (created on first use).
|
|
345
|
+
def table_available? = table_ready?
|
|
346
|
+
|
|
206
347
|
# nil if the record's row and events agree, else a Discrepancy
|
|
207
348
|
# (Lyra::Repair).
|
|
208
349
|
def discrepancy(model, id) = compare(model, id)
|
|
@@ -221,6 +362,53 @@ module Lyra
|
|
|
221
362
|
Lyra.config.monitored_models.each { |model| Lyra::Projections::ReferenceLinks.catch_up(model, force: true) }
|
|
222
363
|
end
|
|
223
364
|
|
|
365
|
+
# A mode whose log rebuilds rows is refused while touches are left out
|
|
366
|
+
# of the log: by this process (config.record_touches false,
|
|
367
|
+
# Configuration#touch_refusal) or by any live process
|
|
368
|
+
# (TouchRegistry.gate_refusal).
|
|
369
|
+
def refuse_unrecorded_touches!(from, to)
|
|
370
|
+
return unless TouchRegistry.authoritative?(to)
|
|
371
|
+
|
|
372
|
+
if (own = Lyra.config.touch_refusal(to))
|
|
373
|
+
raise Refused, "Lyra refuses #{from} -> #{to}: #{own}"
|
|
374
|
+
end
|
|
375
|
+
others = TouchRegistry.gate_refusal(from, to)
|
|
376
|
+
raise Refused, others if others
|
|
377
|
+
end
|
|
378
|
+
|
|
379
|
+
# ES-NoProj reads through each model's default scopes from the event
|
|
380
|
+
# store, and refuses a read whose default scope it cannot evaluate (an
|
|
381
|
+
# SQL string, a join, an ordering by an expression): every read of
|
|
382
|
+
# such a model would fail. The switch into ES-NoProj names them first.
|
|
383
|
+
# force: true switches anyway.
|
|
384
|
+
def refuse_unevaluable_default_scopes!(from, to)
|
|
385
|
+
return unless to == "event_sourcing/disabled" && from != to
|
|
386
|
+
|
|
387
|
+
refused = Lyra::Projections::DefaultScopes.unevaluable(Lyra.config.monitored_models)
|
|
388
|
+
return if refused.empty?
|
|
389
|
+
|
|
390
|
+
raise Refused.new("Lyra refuses #{from} -> #{to}: ES-NoProj cannot evaluate the default scope of " \
|
|
391
|
+
"#{refused.map { |model, _| model.name }.join(', ')} from the event store, and would refuse " \
|
|
392
|
+
"every read of #{refused.size == 1 ? 'that model' : 'those models'}. Write the default scope " \
|
|
393
|
+
"as hash conditions and column orderings, or use another projection mode. " \
|
|
394
|
+
"(#{refused.first.last})")
|
|
395
|
+
end
|
|
396
|
+
|
|
397
|
+
# ES-NoProj leaves the tables of monitored models empty, so a database
|
|
398
|
+
# foreign key into one from a table no monitored model writes refuses
|
|
399
|
+
# every write there that references a record
|
|
400
|
+
# (Lyra::Projections::ForeignKeys). The switch into ES-NoProj names
|
|
401
|
+
# each such constraint first. force: true switches anyway.
|
|
402
|
+
def refuse_foreign_keys_into_event_only_tables!(from, to)
|
|
403
|
+
return unless to == "event_sourcing/disabled" && from != to
|
|
404
|
+
|
|
405
|
+
blocking = Lyra::Projections::ForeignKeys.blocking(Lyra.config.monitored_models)
|
|
406
|
+
return if blocking.empty?
|
|
407
|
+
|
|
408
|
+
raise Refused.new("Lyra refuses #{from} -> #{to}: #{Lyra::Projections::ForeignKeys.description(blocking)} " +
|
|
409
|
+
Lyra::Projections::ForeignKeys.remedy(blocking))
|
|
410
|
+
end
|
|
411
|
+
|
|
224
412
|
def apply(mode, projection_mode)
|
|
225
413
|
Lyra.config.mode = mode.to_sym
|
|
226
414
|
Lyra.config.hijack_enabled = mode.to_sym == :hijack
|
|
@@ -228,12 +416,14 @@ module Lyra
|
|
|
228
416
|
end
|
|
229
417
|
|
|
230
418
|
# A fresh certificate for the switch, re-checked for what changed since.
|
|
419
|
+
# Only one issued after every period in which a process left touches out
|
|
420
|
+
# (its latest heartbeat or its end, TouchRegistry.certificate_floor_sql).
|
|
231
421
|
def certified_check(from, to)
|
|
232
422
|
return nil unless table_ready?
|
|
233
423
|
|
|
234
424
|
row = sql(:select_one,
|
|
235
|
-
"SELECT position, created_at FROM #{TABLE} WHERE kind = 'certificate' AND from_config = #{q(from)} " \
|
|
236
|
-
"AND to_config = #{q(to)} ORDER BY id DESC LIMIT 1"
|
|
425
|
+
"SELECT id, position, created_at FROM #{TABLE} WHERE kind = 'certificate' AND from_config = #{q(from)} " \
|
|
426
|
+
"AND to_config = #{q(to)} AND id > #{TouchRegistry.certificate_floor_sql} ORDER BY id DESC LIMIT 1"
|
|
237
427
|
)
|
|
238
428
|
return nil unless row
|
|
239
429
|
|
|
@@ -245,7 +435,7 @@ module Lyra
|
|
|
245
435
|
discrepancies = changed.flat_map { |model, ids| ids.filter_map { |id| compare(model, id) } }
|
|
246
436
|
Report.new(from: from, to: to, checked: 0, rechecked: changed.values.sum(&:size), imported: 0, rebuilt: 0,
|
|
247
437
|
discrepancies: discrepancies, position: event_position, started_at: certified_at,
|
|
248
|
-
finished_at: Time.current)
|
|
438
|
+
finished_at: Time.current, certificate_id: row["id"].to_i)
|
|
249
439
|
end
|
|
250
440
|
|
|
251
441
|
# Every id that has a row or a stream.
|
|
@@ -280,7 +470,7 @@ module Lyra
|
|
|
280
470
|
next Discrepancy.new(model: model.name, id: id, problem: "row of a destroyed record") if gone
|
|
281
471
|
|
|
282
472
|
differences = Lyra::DualView.new(model, id).compare[:differences]
|
|
283
|
-
next nil if differences
|
|
473
|
+
next nil if differences.empty?
|
|
284
474
|
|
|
285
475
|
Discrepancy.new(model: model.name, id: id, problem: "row differs from its events",
|
|
286
476
|
details: differences.keys.first(5).join(", "))
|
|
@@ -306,24 +496,47 @@ module Lyra
|
|
|
306
496
|
end
|
|
307
497
|
end
|
|
308
498
|
|
|
499
|
+
# Store a clean check's certificate; returns its id, which the report keeps
|
|
500
|
+
# and the switch it clears names.
|
|
309
501
|
def certify(report)
|
|
310
502
|
return unless table_ready?
|
|
311
503
|
|
|
312
|
-
sql(:
|
|
504
|
+
report.certificate_id = sql(:select_value,
|
|
313
505
|
"INSERT INTO #{TABLE} (kind, from_config, to_config, position, checked, created_at) VALUES " \
|
|
314
506
|
"('certificate', #{q(report.from)}, #{q(report.to)}, #{Integer(report.position)}, #{Integer(report.checked)}, " \
|
|
315
|
-
"#{q(report.finished_at.utc)})"
|
|
316
|
-
)
|
|
507
|
+
"#{q(report.finished_at.utc)}) RETURNING id"
|
|
508
|
+
).to_i
|
|
317
509
|
end
|
|
318
510
|
|
|
319
511
|
# Record +to+ as the application-wide mode; returns the row's id.
|
|
320
|
-
def record_applied(to)
|
|
512
|
+
def record_applied(to, from: nil, forced: false, certificate_id: nil)
|
|
321
513
|
return nil unless table_ready?
|
|
322
514
|
|
|
515
|
+
who = Lyra::Attribution.of
|
|
323
516
|
sql(:select_value,
|
|
324
|
-
"INSERT INTO #{TABLE} (kind, from_config, to_config, position, checked, created_at
|
|
325
|
-
"
|
|
326
|
-
|
|
517
|
+
"INSERT INTO #{TABLE} (kind, from_config, to_config, position, checked, created_at, actor, os_user, host, pid, forced, " \
|
|
518
|
+
"certificate_id) VALUES " \
|
|
519
|
+
"('applied', #{q(from)}, #{q(to)}, #{Integer(event_position)}, 0, #{q(Time.current.utc)}, #{q(who[:actor])}, " \
|
|
520
|
+
"#{q(who[:os_user])}, #{q(who[:host])}, #{Integer(who[:pid])}, #{forced ? "TRUE" : "FALSE"}, " \
|
|
521
|
+
"#{certificate_id ? Integer(certificate_id) : "NULL"}) RETURNING id"
|
|
522
|
+
).to_i.tap do |id|
|
|
523
|
+
Rails.logger&.warn("[Lyra] Mode switched to #{to} by #{who[:actor]} (os user #{who[:os_user]} on #{who[:host]}, pid #{who[:pid]}; " \
|
|
524
|
+
"lyra_mode_transitions #{id})")
|
|
525
|
+
end
|
|
526
|
+
end
|
|
527
|
+
|
|
528
|
+
# Who is making a change: the given operator, else the signed-in user
|
|
529
|
+
# (Current.user), else LYRA_OPERATOR, else the operating-system user and
|
|
530
|
+
# process of a task or console. A label nobody checks: Lyra::Attribution
|
|
531
|
+
# records the operating system's user and the host beside it. Recorded with
|
|
532
|
+
# every mode switch, every erasure (0.9.3) and every repair (1.0.0).
|
|
533
|
+
public def actor(operator = nil)
|
|
534
|
+
return operator.to_s if operator.present?
|
|
535
|
+
|
|
536
|
+
user = defined?(::Current) && ::Current.respond_to?(:user) ? ::Current.user : nil
|
|
537
|
+
return "user:#{user.respond_to?(:id) ? user.id : user}" if user
|
|
538
|
+
|
|
539
|
+
ENV["LYRA_OPERATOR"].presence || ["process", ENV["USER"], Process.pid].compact.join(":")
|
|
327
540
|
end
|
|
328
541
|
|
|
329
542
|
# Say what a switch reaches: with ModeSync, every process within its
|
|
@@ -358,8 +571,22 @@ module Lyra
|
|
|
358
571
|
t.bigint :position, null: false, default: 0
|
|
359
572
|
t.integer :checked, null: false, default: 0
|
|
360
573
|
t.datetime :created_at, null: false
|
|
574
|
+
t.string :actor
|
|
575
|
+
t.string :os_user
|
|
576
|
+
t.string :host
|
|
577
|
+
t.integer :pid
|
|
578
|
+
t.boolean :forced, null: false, default: false
|
|
579
|
+
t.bigint :certificate_id
|
|
361
580
|
end
|
|
362
581
|
end
|
|
582
|
+
# Columns later versions added: actor (0.9.3); what the operating system
|
|
583
|
+
# says, whether the gate was overridden, and the certificate that cleared
|
|
584
|
+
# a switch (1.0.0). Added outside the caller's transaction, so a rollback
|
|
585
|
+
# cannot take one back, and safely when two processes add one at once.
|
|
586
|
+
Lyra.add_own_columns(connection, TABLE,
|
|
587
|
+
actor: [:string], os_user: [:string], host: [:string], pid: [:integer],
|
|
588
|
+
forced: [:boolean, { null: false, default: false }], certificate_id: [:bigint],
|
|
589
|
+
started_at: [:datetime], seen_at: [:datetime], ended_at: [:datetime])
|
|
363
590
|
end
|
|
364
591
|
@table_ready = true
|
|
365
592
|
rescue ActiveRecord::NoDatabaseError, ActiveRecord::ConnectionNotEstablished
|
|
@@ -369,7 +596,7 @@ module Lyra
|
|
|
369
596
|
def bypass
|
|
370
597
|
previous = Thread.current[:lyra_bypass_read_override]
|
|
371
598
|
Thread.current[:lyra_bypass_read_override] = true
|
|
372
|
-
Lyra::PurposeBoundReads.internal { yield }
|
|
599
|
+
Lyra::PurposeBoundReads.internal(INTERNAL) { yield }
|
|
373
600
|
ensure
|
|
374
601
|
Thread.current[:lyra_bypass_read_override] = previous
|
|
375
602
|
end
|
data/lib/lyra/model_selection.rb
CHANGED
|
@@ -69,6 +69,7 @@ module Lyra
|
|
|
69
69
|
warn_about_mode(chosen.size) if spec.all
|
|
70
70
|
log(chosen.keys, spec)
|
|
71
71
|
chosen.map do |model, options|
|
|
72
|
+
refuse_composite_key!(model, resolve: true)
|
|
72
73
|
model.monitor_with_lyra(options) unless model.respond_to?(:lyra_monitored) && model.lyra_monitored
|
|
73
74
|
model
|
|
74
75
|
end
|
|
@@ -102,8 +103,42 @@ module Lyra
|
|
|
102
103
|
false # no database yet (assets:precompile), or a model that cannot reflect
|
|
103
104
|
end
|
|
104
105
|
|
|
106
|
+
# Refuse (ArgumentError) a model whose primary key is composite
|
|
107
|
+
# (self.primary_key = [:a, :b], or one its table declares): a record is
|
|
108
|
+
# one stream named by its key, and every lookup by key is by one column.
|
|
109
|
+
# A single primary key of any name and type is supported. Checks the key
|
|
110
|
+
# the model declared or has already read; with +resolve+, reads it from
|
|
111
|
+
# the schema now when the table exists (config.models does, once the
|
|
112
|
+
# application has loaded; monitor_with_lyra in a class body does not
|
|
113
|
+
# touch the database).
|
|
114
|
+
def refuse_composite_key!(model, resolve: false)
|
|
115
|
+
key = known_primary_key(model)
|
|
116
|
+
key = model.primary_key if key.nil? && resolve && table_present?(model)
|
|
117
|
+
return unless key.is_a?(Array)
|
|
118
|
+
|
|
119
|
+
raise ArgumentError, "#{model.name || model} has a composite primary key (#{key.join(', ')}), which Lyra does " \
|
|
120
|
+
"not support: a monitored model needs a single-column primary key"
|
|
121
|
+
end
|
|
122
|
+
|
|
105
123
|
private
|
|
106
124
|
|
|
125
|
+
# The primary key +model+ declared or has already read from the schema,
|
|
126
|
+
# without reading it now; nil when it is not known yet.
|
|
127
|
+
def known_primary_key(model)
|
|
128
|
+
return nil unless model.instance_variable_defined?(:@primary_key)
|
|
129
|
+
|
|
130
|
+
# Unread, Active Record holds a BasicObject placeholder here, which
|
|
131
|
+
# has no is_a?.
|
|
132
|
+
key = model.instance_variable_get(:@primary_key)
|
|
133
|
+
Array === key || String === key ? key : nil
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def table_present?(model)
|
|
137
|
+
model.table_exists?
|
|
138
|
+
rescue StandardError
|
|
139
|
+
false
|
|
140
|
+
end
|
|
141
|
+
|
|
107
142
|
def resolve(name)
|
|
108
143
|
model = begin
|
|
109
144
|
name.constantize
|
data/lib/lyra/own_connection.rb
CHANGED
|
@@ -25,7 +25,13 @@ module Lyra
|
|
|
25
25
|
# Runs the block, on a connection of its own when +model_class+'s
|
|
26
26
|
# connection has a transaction open. Yields +inline+; returns the block's
|
|
27
27
|
# value.
|
|
28
|
-
|
|
28
|
+
#
|
|
29
|
+
# +wait_for_connection+ false: with no connection free in the pool
|
|
30
|
+
# (none idle, and the pool at its size), run inline at once rather
|
|
31
|
+
# than wait up to the checkout timeout, holding the caller's
|
|
32
|
+
# transaction and its locks (ReferenceLinks.catch_up, which does not
|
|
33
|
+
# wait inline either).
|
|
34
|
+
def run(model_class = ActiveRecord::Base, wait_for_connection: true, &block)
|
|
29
35
|
where = model_class.with_connection do |connection|
|
|
30
36
|
next :here unless connection.transaction_open?
|
|
31
37
|
|
|
@@ -35,10 +41,19 @@ module Lyra
|
|
|
35
41
|
return yield(true) if where == :inline
|
|
36
42
|
|
|
37
43
|
pool = model_class.connection_pool
|
|
44
|
+
unless wait_for_connection
|
|
45
|
+
stat = pool.stat
|
|
46
|
+
return yield(true) unless stat[:idle].to_i.positive? || stat[:connections].to_i < stat[:size].to_i
|
|
47
|
+
end
|
|
38
48
|
begin
|
|
39
49
|
# prevent_permanent_checkout: whatever the block leases is returned
|
|
40
50
|
# with the connection, not left to the thread when it ends.
|
|
41
|
-
|
|
51
|
+
# The block's error is raised here, by value; the thread does not
|
|
52
|
+
# also report it (a refused Genesis import, say).
|
|
53
|
+
Thread.new do
|
|
54
|
+
Thread.current.report_on_exception = false
|
|
55
|
+
pool.with_connection(prevent_permanent_checkout: true) { block.call(false) }
|
|
56
|
+
end.value
|
|
42
57
|
rescue ActiveRecord::ConnectionTimeoutError
|
|
43
58
|
yield(true)
|
|
44
59
|
end
|