orfeas_lyra 0.9.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +2021 -17
- data/README.md +22 -9
- data/Rakefile +40 -1
- data/app/controllers/lyra/application_controller.rb +127 -0
- data/app/controllers/lyra/dashboard_controller.rb +33 -31
- data/app/controllers/lyra/flow_controller.rb +32 -34
- data/app/controllers/lyra/privacy_controller.rb +57 -18
- data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
- data/app/views/lyra/dashboard/index.html.erb +1 -0
- data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
- data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
- data/app/views/lyra/privacy/policy.html.erb +7 -0
- data/app/views/lyra/privacy/rights.html.erb +78 -0
- data/config/privacy_policies.rb +0 -3
- data/config/routes.rb +1 -0
- data/lib/lyra/access_log.rb +187 -6
- data/lib/lyra/advisory_lock.rb +236 -0
- data/lib/lyra/aggregate.rb +1 -1
- data/lib/lyra/associations/habtm_links.rb +382 -19
- data/lib/lyra/associations/maintenance.rb +229 -0
- data/lib/lyra/attribution.rb +48 -0
- data/lib/lyra/bypass_events.rb +446 -46
- data/lib/lyra/command_handler.rb +19 -14
- data/lib/lyra/configuration.rb +237 -10
- data/lib/lyra/consistency/read_your_writes.rb +4 -2
- data/lib/lyra/correlation.rb +121 -0
- data/lib/lyra/domain_events.rb +4 -0
- data/lib/lyra/dual_view.rb +53 -16
- data/lib/lyra/encrypted_attributes.rb +5 -0
- data/lib/lyra/engine.rb +67 -0
- data/lib/lyra/erasure.rb +1011 -106
- data/lib/lyra/event.rb +35 -1
- data/lib/lyra/event_data.rb +38 -0
- data/lib/lyra/event_flow.rb +31 -15
- data/lib/lyra/event_store_adapter.rb +28 -0
- data/lib/lyra/event_window.rb +58 -0
- data/lib/lyra/genesis.rb +548 -44
- data/lib/lyra/interceptors/association_interceptor.rb +689 -34
- data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
- data/lib/lyra/internal.rb +26 -0
- data/lib/lyra/mode_sync.rb +51 -3
- data/lib/lyra/mode_transition.rb +257 -30
- data/lib/lyra/model_selection.rb +35 -0
- data/lib/lyra/own_connection.rb +17 -2
- data/lib/lyra/privacy/adapters/pam.rb +25 -2
- data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
- data/lib/lyra/privacy/interface.rb +27 -7
- data/lib/lyra/privacy/subject_links.rb +143 -0
- data/lib/lyra/privacy/subject_records.rb +369 -0
- data/lib/lyra/projection.rb +7 -1
- data/lib/lyra/projections/async_checkpoint.rb +32 -0
- data/lib/lyra/projections/async_projection_job.rb +166 -23
- data/lib/lyra/projections/cached_joins.rb +33 -18
- data/lib/lyra/projections/cached_projection.rb +74 -33
- data/lib/lyra/projections/cached_relation.rb +3175 -321
- data/lib/lyra/projections/default_scopes.rb +254 -0
- data/lib/lyra/projections/deferred_relation.rb +23 -3
- data/lib/lyra/projections/event_store_reader.rb +47 -8
- data/lib/lyra/projections/foreign_keys.rb +92 -0
- data/lib/lyra/projections/lazy_projection.rb +165 -27
- data/lib/lyra/projections/model_projection.rb +165 -14
- data/lib/lyra/projections/rebuild.rb +34 -6
- data/lib/lyra/projections/reference_links.rb +463 -72
- data/lib/lyra/projections/typed_values.rb +629 -0
- data/lib/lyra/projections/unique_indexes.rb +595 -0
- data/lib/lyra/projections/uniqueness.rb +784 -0
- data/lib/lyra/projections/value_digest.rb +93 -0
- data/lib/lyra/purpose_bound_reads.rb +912 -77
- data/lib/lyra/repair.rb +119 -28
- data/lib/lyra/retention.rb +114 -8
- data/lib/lyra/rights/consent_store.rb +415 -0
- data/lib/lyra/rights/erasure_guard.rb +135 -0
- data/lib/lyra/rights/pending.rb +74 -0
- data/lib/lyra/rights/queries.rb +269 -0
- data/lib/lyra/rights/store.rb +512 -0
- data/lib/lyra/rights/subject_keys.rb +224 -0
- data/lib/lyra/rights.rb +281 -0
- data/lib/lyra/saves_without_effect.rb +69 -0
- data/lib/lyra/stream_tail.rb +319 -0
- data/lib/lyra/strict_data_access.rb +916 -106
- data/lib/lyra/temporal.rb +4 -2
- data/lib/lyra/touch_registry.rb +404 -0
- data/lib/lyra/verification/bypass_workflow.rb +452 -42
- data/lib/lyra/verification/rights_workflows.rb +170 -0
- data/lib/lyra/version.rb +1 -1
- data/lib/lyra/visualization/event_graph.rb +1 -1
- data/lib/lyra/visualization/timeline.rb +2 -2
- data/lib/lyra.rb +43 -2
- data/lib/tasks/lyra_mode.rake +110 -8
- data/lib/tasks/lyra_projections.rake +43 -2
- metadata +34 -4
- data/lib/lyra/associations/event_aware.rb +0 -224
data/lib/lyra/configuration.rb
CHANGED
|
@@ -5,15 +5,34 @@ module Lyra
|
|
|
5
5
|
# Valid projection modes (event sourcing): :disabled is ES-NoProj, :lazy
|
|
6
6
|
# is ES-Lazy (project on read).
|
|
7
7
|
PROJECTION_MODES = [:sync, :async, :disabled, :lazy].freeze
|
|
8
|
+
RIGHTS_REFRESH_MODES = %i[scope interval].freeze
|
|
8
9
|
|
|
9
10
|
attr_reader :mode, :projection_mode
|
|
10
|
-
attr_accessor :event_store, :
|
|
11
|
+
attr_accessor :event_store, :retention_policy
|
|
12
|
+
attr_reader :hijack_enabled
|
|
11
13
|
attr_accessor :strict_projections, :projection_error_handler, :async_projections_inline
|
|
12
14
|
attr_accessor :strict_schema, :schema_path
|
|
13
15
|
attr_accessor :strict_data_access # Raise on callback-bypassing operations
|
|
14
16
|
attr_accessor :metadata_proc # Custom metadata proc for events
|
|
15
17
|
attr_accessor :genesis # Imported events for pre-existing rows (see Lyra::Genesis)
|
|
18
|
+
# What Genesis does with a row holding a value its model cannot represent
|
|
19
|
+
# (an enum value outside the mapping, ...): :refuse (default) refuses the
|
|
20
|
+
# model's import, listing the rows; :skip leaves them without a stream
|
|
21
|
+
# and imports the rest. Lyra::Repair follows it for what it would write.
|
|
22
|
+
# See Lyra::Genesis.
|
|
23
|
+
GENESIS_UNREPRESENTABLE = %i[refuse skip].freeze
|
|
24
|
+
attr_reader :genesis_unrepresentable
|
|
16
25
|
attr_accessor :reference_links # Foreign-key links for ES-NoProj lookups (see Lyra::Projections::ReferenceLinks)
|
|
26
|
+
# The secret the value streams of ES-NoProj's unique-index check and the
|
|
27
|
+
# value locks are keyed with (Lyra::Projections::ValueDigest): nil
|
|
28
|
+
# (default) derives one from the application's secret_key_base. Changing
|
|
29
|
+
# it relinks the value streams on their next use.
|
|
30
|
+
attr_accessor :value_digest_key
|
|
31
|
+
# Seconds after which a process reads a monitored model's unique indexes
|
|
32
|
+
# again (Lyra::Projections::UniqueIndexes), so an index added by a
|
|
33
|
+
# migration while it runs is enforced and its earlier writes linked:
|
|
34
|
+
# 60 (default), nil never (only reset_column_information does).
|
|
35
|
+
attr_accessor :unique_index_refresh
|
|
17
36
|
attr_reader :monitored_models
|
|
18
37
|
# Default privacy policy for monitored models that name none of their
|
|
19
38
|
# own (monitor_with_lyra privacy_policy: ...). See Lyra::Privacy.policy_for.
|
|
@@ -50,9 +69,38 @@ module Lyra
|
|
|
50
69
|
# Extra metadata for each recorded access, from a proc given the
|
|
51
70
|
# PamDsl::Enforcement::Access: who accessed, where Current does not say.
|
|
52
71
|
attr_accessor :access_metadata_proc
|
|
72
|
+
# Whether Lyra::Erasure.erase! also deletes the subject's access log
|
|
73
|
+
# (Lyra::DataAccess$<Model>$<id>), unless the call says (access_log:).
|
|
74
|
+
# Off by default: the log may be evidence the application must keep.
|
|
75
|
+
attr_accessor :erase_access_log
|
|
76
|
+
# How long, in seconds, a process answers the checks of the data subject's
|
|
77
|
+
# restrictions, objections (Condition 6) and consent from what it last read
|
|
78
|
+
# (Lyra::Rights). A right or consent recorded in this process is seen at
|
|
79
|
+
# once; one recorded in another process within this interval. Default 1.
|
|
80
|
+
attr_accessor :rights_refresh_interval
|
|
81
|
+
# When a process reads the rights and consent afresh (from 1.0.0):
|
|
82
|
+
# :scope (the default) at the first check of each checking scope (a
|
|
83
|
+
# Lyra.with_purpose block, a web request, a job, a batch of a batch loop:
|
|
84
|
+
# Lyra::Rights.scope), and otherwise after rights_refresh_interval;
|
|
85
|
+
# :interval only after rights_refresh_interval, so a right recorded in
|
|
86
|
+
# another process may apply up to that long later, in the middle of a
|
|
87
|
+
# request, and no scope costs a query of its own. Scopes open either way.
|
|
88
|
+
attr_reader :rights_refresh
|
|
89
|
+
# With Lyra::Erasure.erase!(…, everywhere: true): a value held by more
|
|
90
|
+
# other records than this (default 10) is taken to be shared by many people
|
|
91
|
+
# (a placeholder, a city) and left, and reported in Result#shared. The
|
|
92
|
+
# person's linked records (belongs_to the erased record) do not count. The
|
|
93
|
+
# call's max_copies: and the task's MAX_COPIES= override it. From 1.0.0.
|
|
94
|
+
attr_reader :erasure_max_copies
|
|
95
|
+
# Record each explicit Art. 15 copy (GDPRCompliance#data_export, not
|
|
96
|
+
# record: false) as an event in Lyra::SubjectAccessExports$<subject>, which
|
|
97
|
+
# access-log erasure leaves: who exported, when, which sections, never
|
|
98
|
+
# values (Lyra::AccessLog.record_export). On by default (accountability,
|
|
99
|
+
# Art. 5(2), and evidence the request was answered); false turns it off.
|
|
100
|
+
attr_accessor :record_subject_access_exports
|
|
53
101
|
# What a read of a model with a privacy policy does when no purpose is
|
|
54
102
|
# declared (Lyra::PurposeBoundReads): :allow (default), :audit or :deny.
|
|
55
|
-
attr_reader :reads_without_purpose
|
|
103
|
+
attr_reader :reads_without_purpose, :unrecorded_writes
|
|
56
104
|
# What Monitor does when a write's event cannot be stored: :log (default)
|
|
57
105
|
# lets the write stand and logs the failure, the stream falling behind its
|
|
58
106
|
# row until lyra:repair; :fail_write fails the write and rolls it back, as
|
|
@@ -60,14 +108,37 @@ module Lyra
|
|
|
60
108
|
# together either way; this decides only what a failed append does.
|
|
61
109
|
MONITOR_APPEND_FAILURES = %i[log fail_write].freeze
|
|
62
110
|
attr_reader :monitor_append_failure
|
|
63
|
-
# Take a lock on the aggregate before appending its event, in Hijack
|
|
64
|
-
# ES-Sync (assumption L of Theorem 5;
|
|
65
|
-
# #lyra_lock_aggregate!). On (default):
|
|
66
|
-
#
|
|
67
|
-
#
|
|
68
|
-
# the
|
|
69
|
-
#
|
|
111
|
+
# Take a lock on the aggregate before appending its event, in Hijack,
|
|
112
|
+
# ES-Sync and ES-Async (assumption L of Theorem 5;
|
|
113
|
+
# Lyra::Interceptors::CrudInterceptor#lyra_lock_aggregate!). On (default):
|
|
114
|
+
# in Hijack and ES-Sync the record's row lock, so two writers of one record
|
|
115
|
+
# append and write its row in the same order; in ES-Async, whose writers do
|
|
116
|
+
# not write the row, an advisory lock on the stream, so a record's events
|
|
117
|
+
# commit in the order they were appended and the projection job applies
|
|
118
|
+
# only those after its checkpoint (AsyncProjectionJob). An update or a
|
|
119
|
+
# destroy also checks, under the lock, that the stream does not end in a
|
|
120
|
+
# Destroyed event, taking the advisory lock for that in ES-NoProj and
|
|
121
|
+
# ES-Lazy too (#lyra_lock_aggregate_unless_destroyed!). The price is one
|
|
122
|
+
# lock statement per update or destroy, which also makes the append nest
|
|
123
|
+
# in a savepoint. Off: the order these modes had before, with a divergence
|
|
124
|
+
# between row and log possible under same-record races in Hijack and
|
|
125
|
+
# ES-Sync (DualView detects it), no destroyed check, and ES-Async's job
|
|
126
|
+
# replaying the whole stream every time.
|
|
70
127
|
attr_accessor :lock_aggregate_before_append
|
|
128
|
+
# The most records a bulk write may lock: in ES-NoProj update_all,
|
|
129
|
+
# delete_all, touch_all, update_counters, a dependent: :delete_all or
|
|
130
|
+
# :nullify, and a destroy_all inside a transaction already open; in
|
|
131
|
+
# ES-Async and ES-Lazy update_all, delete_all, touch_all, upsert_all and
|
|
132
|
+
# a dependent: :nullify (BypassEvents.checks_streams?). It holds one
|
|
133
|
+
# aggregate lock per record until its
|
|
134
|
+
# transaction ends, and PostgreSQL's lock table, shared by every session,
|
|
135
|
+
# holds about max_locks_per_transaction × (max_connections + background
|
|
136
|
+
# processes) locks (about 12,800 with the defaults). Past it every
|
|
137
|
+
# session's writes fail ("out of shared memory"). A larger write raises
|
|
138
|
+
# Lyra::BulkWriteLockLimitError before it locks or writes anything. nil:
|
|
139
|
+
# no limit. Only with lock_aggregate_before_append on. At least 1, so a
|
|
140
|
+
# write of one record is never refused (bulk_write_lock_limit=).
|
|
141
|
+
attr_reader :bulk_write_lock_limit
|
|
71
142
|
# Apply the privacy policy's retention rules (Lyra::Retention). Off by
|
|
72
143
|
# default: deleting data on a timer is the deployment's decision.
|
|
73
144
|
attr_accessor :retention_executor
|
|
@@ -82,6 +153,32 @@ module Lyra
|
|
|
82
153
|
# config.dashboard_authorization = ->(controller) { controller.current_user&.admin? }
|
|
83
154
|
attr_accessor :dashboard_authorization
|
|
84
155
|
|
|
156
|
+
# The purpose the dashboard reads under when a request names none
|
|
157
|
+
# (?purpose=): one the policy declares for oversight. nil (default): such a
|
|
158
|
+
# request reads without a purpose, as reads_without_purpose decides.
|
|
159
|
+
# config.dashboard_purpose = :oversight
|
|
160
|
+
attr_accessor :dashboard_purpose
|
|
161
|
+
# The most events a dashboard page or JSON route reads from the log in one
|
|
162
|
+
# request (default 10,000); a request's ?limit= may only lower it. The routes
|
|
163
|
+
# that look across the log (the CRUD summary, a correlation, a user's
|
|
164
|
+
# actions, PII detection) read the most recent events up to it and say so
|
|
165
|
+
# (window: { limit:, returned:, truncated: }); before 1.0.0 they read the whole log.
|
|
166
|
+
attr_reader :dashboard_max_events
|
|
167
|
+
# Whether a touch whose only change is the model's update timestamp
|
|
168
|
+
# (updated_at, updated_on) is recorded as an event: an instance touch, a
|
|
169
|
+
# touch_all and a belongs_to ... touch: true cascade. True (the default)
|
|
170
|
+
# records every one, as before. False leaves those out of the log, in
|
|
171
|
+
# Monitor only, where the table is authoritative and the event a copy; a
|
|
172
|
+
# touch that also changes another column (touch(:completed_at), a
|
|
173
|
+
# lock_version) is recorded in full. DualView then ignores a difference
|
|
174
|
+
# confined to those columns. Refused with Hijack and event sourcing, whose
|
|
175
|
+
# log is the record of every change and is replayed to rebuild the rows: a
|
|
176
|
+
# missing touch would rebuild a stale updated_at. A process with it off
|
|
177
|
+
# registers (Lyra::TouchRegistry), and no process escalates while one is
|
|
178
|
+
# registered. Before escalating, restart every process with it true and
|
|
179
|
+
# run lyra:repair, which records the timestamps the log missed. From 1.0.0.
|
|
180
|
+
attr_reader :record_touches
|
|
181
|
+
|
|
85
182
|
def initialize
|
|
86
183
|
@mode = :monitor
|
|
87
184
|
@hijack_enabled = false
|
|
@@ -107,9 +204,12 @@ module Lyra
|
|
|
107
204
|
# Genesis: :auto (event-sourcing mode only), true (every event-producing
|
|
108
205
|
# mode) or false. See Lyra::Genesis.
|
|
109
206
|
@genesis = :auto
|
|
207
|
+
@genesis_unrepresentable = :refuse
|
|
110
208
|
# Reference links: :auto (ES-NoProj only), true (every event-producing
|
|
111
209
|
# mode) or false. See Lyra::Projections::ReferenceLinks.
|
|
112
210
|
@reference_links = :auto
|
|
211
|
+
@value_digest_key = nil
|
|
212
|
+
@unique_index_refresh = 60
|
|
113
213
|
@mode_transition_gate = nil
|
|
114
214
|
@mode_transition_certificate_ttl = 3600
|
|
115
215
|
@dual_view_sample_rate = 0.0
|
|
@@ -120,17 +220,29 @@ module Lyra
|
|
|
120
220
|
@annotate_privacy = false
|
|
121
221
|
@record_access_events = false
|
|
122
222
|
@access_metadata_proc = nil
|
|
223
|
+
@erase_access_log = false
|
|
224
|
+
@rights_refresh_interval = 1
|
|
225
|
+
@rights_refresh = :scope
|
|
226
|
+
@erasure_max_copies = 10 # Lyra::Erasure::MAX_COPIES
|
|
227
|
+
@record_subject_access_exports = true
|
|
123
228
|
@reads_without_purpose = :allow
|
|
229
|
+
@unrecorded_writes = :allow
|
|
124
230
|
@monitor_append_failure = :log
|
|
125
231
|
@lock_aggregate_before_append = true
|
|
232
|
+
@bulk_write_lock_limit = 5_000
|
|
126
233
|
@retention_executor = false
|
|
127
234
|
@retention_anchors = {}
|
|
128
235
|
@dashboard_authorization = nil
|
|
236
|
+
@dashboard_purpose = nil
|
|
237
|
+
@dashboard_max_events = 10_000
|
|
238
|
+
@record_touches = true
|
|
129
239
|
end
|
|
130
240
|
|
|
131
241
|
# One of MODES; a string (from ENV, say) is taken as its symbol.
|
|
132
242
|
def mode=(mode)
|
|
133
|
-
|
|
243
|
+
mode = validate_choice!(:mode, mode, MODES)
|
|
244
|
+
refuse_unrecorded_touches!(mode)
|
|
245
|
+
@mode = mode
|
|
134
246
|
end
|
|
135
247
|
|
|
136
248
|
# One of PROJECTION_MODES; a string is taken as its symbol.
|
|
@@ -138,10 +250,111 @@ module Lyra
|
|
|
138
250
|
@projection_mode = validate_choice!(:projection_mode, mode, PROJECTION_MODES)
|
|
139
251
|
end
|
|
140
252
|
|
|
253
|
+
# One of GENESIS_UNREPRESENTABLE; a string is taken as its symbol.
|
|
254
|
+
def genesis_unrepresentable=(choice)
|
|
255
|
+
@genesis_unrepresentable = validate_choice!(:genesis_unrepresentable, choice, GENESIS_UNREPRESENTABLE)
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
def rights_refresh=(mode)
|
|
259
|
+
@rights_refresh = validate_choice!(:rights_refresh, mode, RIGHTS_REFRESH_MODES)
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
def erasure_max_copies=(value)
|
|
263
|
+
count = value.is_a?(Integer) ? value : Integer(value.to_s, 10, exception: false)
|
|
264
|
+
raise ArgumentError, "erasure_max_copies must be a positive integer (got #{value.inspect})" unless count&.positive?
|
|
265
|
+
|
|
266
|
+
@erasure_max_copies = count
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
def dashboard_max_events=(value)
|
|
270
|
+
value = Integer(value)
|
|
271
|
+
raise ArgumentError, "dashboard_max_events must be positive" unless value.positive?
|
|
272
|
+
|
|
273
|
+
@dashboard_max_events = value
|
|
274
|
+
end
|
|
275
|
+
|
|
141
276
|
def reads_without_purpose=(mode)
|
|
142
277
|
@reads_without_purpose = Lyra::PurposeBoundReads.validate_mode!(mode)
|
|
143
278
|
end
|
|
144
279
|
|
|
280
|
+
# Whether application code may write without events through
|
|
281
|
+
# Lyra.unrecorded_write: :allow (default; each use is recorded with its
|
|
282
|
+
# reason) or :deny (it raises: production hardening).
|
|
283
|
+
def unrecorded_writes=(mode)
|
|
284
|
+
mode = mode.to_sym
|
|
285
|
+
raise ArgumentError, "unrecorded_writes must be :allow or :deny, not #{mode.inspect}" unless %i[allow deny].include?(mode)
|
|
286
|
+
|
|
287
|
+
@unrecorded_writes = mode
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
# true or false (a string "true" or "false", from ENV, is taken as its
|
|
291
|
+
# value). False is refused outside Monitor and Disabled (touch_refusal).
|
|
292
|
+
def record_touches=(value)
|
|
293
|
+
value = { "true" => true, "false" => false }.fetch(value.strip.downcase, value) if value.is_a?(String)
|
|
294
|
+
raise ArgumentError, "record_touches must be true or false (got #{value.inspect})" unless [true, false].include?(value)
|
|
295
|
+
|
|
296
|
+
previous = @record_touches
|
|
297
|
+
@record_touches = value
|
|
298
|
+
refusal = touch_refusal(hijack_mode? ? :hijack : @mode)
|
|
299
|
+
if refusal
|
|
300
|
+
@record_touches = previous
|
|
301
|
+
raise ArgumentError, refusal
|
|
302
|
+
end
|
|
303
|
+
# This process registers as one that leaves touches out (after boot;
|
|
304
|
+
# at boot, Lyra::TouchRegistry.boot!), which is refused in an application
|
|
305
|
+
# whose latest applied mode is Hijack or event sourcing, and ends its
|
|
306
|
+
# entry when set back to true.
|
|
307
|
+
return unless defined?(Lyra::TouchRegistry) && value != previous
|
|
308
|
+
return Lyra::TouchRegistry.option_on! if value
|
|
309
|
+
|
|
310
|
+
begin
|
|
311
|
+
Lyra::TouchRegistry.option_off!
|
|
312
|
+
rescue StandardError
|
|
313
|
+
@record_touches = previous
|
|
314
|
+
raise
|
|
315
|
+
end
|
|
316
|
+
end
|
|
317
|
+
|
|
318
|
+
# Set by the mode helpers. Refused, like mode=, while record_touches is
|
|
319
|
+
# false: hijack_mode? is true whatever mode says.
|
|
320
|
+
def hijack_enabled=(value)
|
|
321
|
+
refuse_unrecorded_touches!(:hijack) if value
|
|
322
|
+
@hijack_enabled = value
|
|
323
|
+
end
|
|
324
|
+
|
|
325
|
+
# Why +mode+ (a mode, or a ModeTransition label such as
|
|
326
|
+
# "event_sourcing/lazy") cannot run with record_touches false, or nil when
|
|
327
|
+
# it can. Hijack and the event-sourcing modes take the log as the record of
|
|
328
|
+
# every change: the rows are rebuilt from it (a switch into event
|
|
329
|
+
# sourcing, which Hijack passes without a check, a Rebuild, ES-NoProj's
|
|
330
|
+
# reads). An unrecorded touch there would rebuild a stale updated_at.
|
|
331
|
+
def touch_refusal(mode)
|
|
332
|
+
return nil unless @record_touches == false
|
|
333
|
+
return nil if %w[disabled monitor].include?(mode.to_s.split("/").first)
|
|
334
|
+
|
|
335
|
+
"config.record_touches = false is refused in #{mode}: there the event log is the record of every change, " \
|
|
336
|
+
"and a touch left out of it would rebuild a stale updated_at. To escalate: restart every process with " \
|
|
337
|
+
"record_touches on, run `rake lyra:repair` in Monitor, then switch."
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
# Whether a touch whose only change is the update timestamp is left out
|
|
341
|
+
# of the log now: record_touches false, in Monitor.
|
|
342
|
+
def skip_touches?
|
|
343
|
+
@record_touches == false && monitor_mode? && !hijack_mode?
|
|
344
|
+
end
|
|
345
|
+
|
|
346
|
+
# nil (no limit) or a whole number of at least 1; a string (from ENV,
|
|
347
|
+
# say) is taken as its number. 0 refused every write that locks a
|
|
348
|
+
# record, update_columns, touch and delete included.
|
|
349
|
+
def bulk_write_lock_limit=(limit)
|
|
350
|
+
limit = Integer(limit, exception: false) || limit if limit.is_a?(String)
|
|
351
|
+
unless limit.nil? || (limit.is_a?(Integer) && limit >= 1)
|
|
352
|
+
raise ArgumentError, "bulk_write_lock_limit must be nil (no limit) or a whole number of at least 1, got #{limit.inspect}"
|
|
353
|
+
end
|
|
354
|
+
|
|
355
|
+
@bulk_write_lock_limit = limit
|
|
356
|
+
end
|
|
357
|
+
|
|
145
358
|
# One of MONITOR_APPEND_FAILURES; a string (from ENV, say) is taken as its symbol.
|
|
146
359
|
def monitor_append_failure=(policy)
|
|
147
360
|
policy = policy.to_sym if policy.is_a?(String)
|
|
@@ -217,6 +430,9 @@ module Lyra
|
|
|
217
430
|
@monitored_models.map! { |m| (m.name rescue m.object_id.to_s) == model_name ? model_class : m }
|
|
218
431
|
end
|
|
219
432
|
@model_configs[model_class] = ModelConfiguration.new(model_class, options)
|
|
433
|
+
# The join tables of the monitored models' has_and_belongs_to_many may
|
|
434
|
+
# have changed (a new model, or a class reloaded).
|
|
435
|
+
Lyra::Associations::HabtmLinks.reset_join_models! if defined?(Lyra::Associations::HabtmLinks)
|
|
220
436
|
end
|
|
221
437
|
|
|
222
438
|
# The monitored model an event's model_class names: a monitored model, or
|
|
@@ -272,6 +488,7 @@ module Lyra
|
|
|
272
488
|
def enable_hijack!
|
|
273
489
|
return Lyra::ModeTransition.to!(:hijack) if gated_switch?
|
|
274
490
|
|
|
491
|
+
refuse_unrecorded_touches!(:hijack)
|
|
275
492
|
@hijack_enabled = true
|
|
276
493
|
@mode = :hijack
|
|
277
494
|
end
|
|
@@ -288,6 +505,7 @@ module Lyra
|
|
|
288
505
|
def enable_event_sourcing!
|
|
289
506
|
return Lyra::ModeTransition.to!(:event_sourcing) if gated_switch?
|
|
290
507
|
|
|
508
|
+
refuse_unrecorded_touches!(:event_sourcing)
|
|
291
509
|
@mode = :event_sourcing
|
|
292
510
|
@hijack_enabled = false
|
|
293
511
|
end
|
|
@@ -300,6 +518,11 @@ module Lyra
|
|
|
300
518
|
|
|
301
519
|
private
|
|
302
520
|
|
|
521
|
+
def refuse_unrecorded_touches!(mode)
|
|
522
|
+
refusal = touch_refusal(mode)
|
|
523
|
+
raise ArgumentError, refusal if refusal
|
|
524
|
+
end
|
|
525
|
+
|
|
303
526
|
def validate_choice!(name, value, valid)
|
|
304
527
|
choice = value.is_a?(String) ? value.strip.to_sym : value
|
|
305
528
|
return choice if valid.include?(choice)
|
|
@@ -320,6 +543,10 @@ module Lyra
|
|
|
320
543
|
@event_prefix = options[:event_prefix] || model_class.name
|
|
321
544
|
@aggregate_class = options[:aggregate_class]
|
|
322
545
|
@custom_event_mapping = options[:event_mapping] || {}
|
|
546
|
+
# Names Lyra keeps for its events that are never replayed (Lyra::StreamTail).
|
|
547
|
+
@custom_event_mapping.each_value do |name|
|
|
548
|
+
raise ArgumentError, "event_mapping: #{name} is reserved for Lyra's own events" if Lyra::StreamTail.reserved_name?(name)
|
|
549
|
+
end
|
|
323
550
|
@privacy_policy = options[:privacy_policy]
|
|
324
551
|
@domain_events = Lyra::DomainEvents.rules(options[:domain_events])
|
|
325
552
|
end
|
|
@@ -9,7 +9,9 @@ module Lyra
|
|
|
9
9
|
# projects the writes made inside a block itself: by default when the
|
|
10
10
|
# block ends, so reads after the block see them (reads inside it do not);
|
|
11
11
|
# with each_write: true as each write is made, so reads later in the same
|
|
12
|
-
# block see them too.
|
|
12
|
+
# block see them too. Under ES-Sync a write's row is written with it,
|
|
13
|
+
# inside a block as outside one (CrudInterceptor#lyra_run_sync_projection),
|
|
14
|
+
# so the block has nothing to project there.
|
|
13
15
|
#
|
|
14
16
|
# Usage:
|
|
15
17
|
# Lyra::Consistency::ReadYourWrites.with_guaranteed_read do
|
|
@@ -100,7 +102,7 @@ module Lyra
|
|
|
100
102
|
model_class = write[:model_class]
|
|
101
103
|
model_class.transaction do
|
|
102
104
|
lock_stream(model_class, write[:result])
|
|
103
|
-
Lyra::Projections::ModelProjection.project(model_class, write[:operation], write[:result])
|
|
105
|
+
Lyra::Projections::ModelProjection.project(model_class, write[:operation], write[:result], INTERNAL)
|
|
104
106
|
end
|
|
105
107
|
rescue => e
|
|
106
108
|
Rails.logger.error("Lyra: Failed to ensure projection - #{e.message}")
|
data/lib/lyra/correlation.rb
CHANGED
|
@@ -124,12 +124,115 @@ module Lyra
|
|
|
124
124
|
# └── InventoryReserved (causation_id: PaymentProcessed.id)
|
|
125
125
|
#
|
|
126
126
|
class Causation
|
|
127
|
+
# The cause a write scope (CrudInterceptor#lyra_write_scope) puts in scope
|
|
128
|
+
# for the writes made during the write (its callbacks, after_save and
|
|
129
|
+
# after_destroy included), decided when one of them needs it:
|
|
130
|
+
#
|
|
131
|
+
# - the write's own event id, once that event is in the log (appended);
|
|
132
|
+
# - the cause in scope when the write began, once the write is known to
|
|
133
|
+
# record no event (decided without an event: a save that changes
|
|
134
|
+
# nothing, a destroy or update that reached no row), or when its event
|
|
135
|
+
# did not reach the log after all (lost: Monitor's logged append
|
|
136
|
+
# failure, a Hijack row write that failed and took the event with it);
|
|
137
|
+
# - before either is known (the write's before_* callbacks: a belongs_to
|
|
138
|
+
# autosave, a dependent: :destroy; or a refused command), the write's
|
|
139
|
+
# own id as well, which it then owes: if the write ends without its
|
|
140
|
+
# event in the log, it appends a SaveWithoutEffect marker with that id
|
|
141
|
+
# (Lyra::SavesWithoutEffect).
|
|
142
|
+
#
|
|
143
|
+
# So the causation_id of every event a write scope gives names an event
|
|
144
|
+
# in the log once the transaction commits. The own id used to be the
|
|
145
|
+
# cause throughout, and from 1.0.0, when a save that changes nothing
|
|
146
|
+
# stopped recording an event, the writes its callbacks made (Solidus's
|
|
147
|
+
# Order#save updating its line items) named an event that never existed.
|
|
148
|
+
class WriteScope
|
|
149
|
+
attr_reader :event_id, :outer
|
|
150
|
+
|
|
151
|
+
def initialize(outer)
|
|
152
|
+
@outer = outer
|
|
153
|
+
@event_id = SecureRandom.uuid
|
|
154
|
+
@taken = false
|
|
155
|
+
@appended = false
|
|
156
|
+
@lost = false
|
|
157
|
+
@decided = false
|
|
158
|
+
@owed = false
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# The write's own event id, for its own event: given once, so no two
|
|
162
|
+
# events can share it; nil once taken (the store then assigns one).
|
|
163
|
+
def take_event_id
|
|
164
|
+
return nil if @taken
|
|
165
|
+
|
|
166
|
+
@taken = true
|
|
167
|
+
@event_id
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
# The write's own event is in the log (in its transaction).
|
|
171
|
+
def appended!
|
|
172
|
+
@appended = true if @taken
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# The write's own event is not in the log after all, and the write
|
|
176
|
+
# may stand: +reason+ is the marker's, if one is owed.
|
|
177
|
+
def lost!(reason = "append failed")
|
|
178
|
+
@lost = reason
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
# The write has reached its after_* callbacks: it has built its event,
|
|
182
|
+
# or it records none.
|
|
183
|
+
def decided!
|
|
184
|
+
@decided = true
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
def recorded? = @appended && !@lost
|
|
188
|
+
|
|
189
|
+
def lost? = !!@lost
|
|
190
|
+
|
|
191
|
+
def lost_reason = @lost || nil
|
|
192
|
+
|
|
193
|
+
def decided? = @decided
|
|
194
|
+
|
|
195
|
+
# Whether a cause handed out before the write's event was known still
|
|
196
|
+
# names no event: the marker is owed.
|
|
197
|
+
def owes_marker? = @owed && !recorded?
|
|
198
|
+
|
|
199
|
+
def cause_id
|
|
200
|
+
return @event_id if recorded?
|
|
201
|
+
return Causation.resolve(@outer) if @lost || (@decided && !@taken)
|
|
202
|
+
|
|
203
|
+
@owed = true
|
|
204
|
+
@event_id
|
|
205
|
+
end
|
|
206
|
+
end
|
|
207
|
+
|
|
127
208
|
class << self
|
|
128
209
|
# Current causation ID (the event that caused the current operation)
|
|
129
210
|
def current_id
|
|
211
|
+
resolve(Thread.current[:lyra_causation_id])
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
# The cause in scope as set, a write scope unresolved: a scope opened
|
|
215
|
+
# inside another keeps it as its outer cause, and resolves it when its
|
|
216
|
+
# own event, or a write inside it, needs it.
|
|
217
|
+
def current
|
|
130
218
|
Thread.current[:lyra_causation_id]
|
|
131
219
|
end
|
|
132
220
|
|
|
221
|
+
def resolve(cause)
|
|
222
|
+
cause.is_a?(WriteScope) ? cause.cause_id : cause
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
# The cause in scope outside every write scope of this thread: one set
|
|
226
|
+
# by the application or carried by a job (Correlation::Job), or nil.
|
|
227
|
+
# For an entry that commits on a connection of its own
|
|
228
|
+
# (Lyra::AccessLog): a write in scope may still roll back, and an entry
|
|
229
|
+
# naming it would then name no event.
|
|
230
|
+
def outside_write_scopes
|
|
231
|
+
cause = current
|
|
232
|
+
cause = cause.outer while cause.is_a?(WriteScope)
|
|
233
|
+
cause
|
|
234
|
+
end
|
|
235
|
+
|
|
133
236
|
# Set causation ID for current context (when processing/handling an event)
|
|
134
237
|
def with_id(causation_id)
|
|
135
238
|
previous_id = Thread.current[:lyra_causation_id]
|
|
@@ -169,6 +272,24 @@ module Lyra
|
|
|
169
272
|
causation_store[event_id]
|
|
170
273
|
end
|
|
171
274
|
|
|
275
|
+
# Whether every causation_id in +event_store+ names an event there: the
|
|
276
|
+
# number of events read and of those with a cause, and the events whose
|
|
277
|
+
# cause is not in the log (dangling: event_id, event_type,
|
|
278
|
+
# causation_id), none when the causal links are whole. A check for
|
|
279
|
+
# tests and replays (the Olist sweep); it reads the whole store.
|
|
280
|
+
def integrity(event_store = Lyra.config.event_store)
|
|
281
|
+
ids = Set.new
|
|
282
|
+
caused = []
|
|
283
|
+
event_store.read.in_batches(1000).each_batch do |batch|
|
|
284
|
+
batch.each do |event|
|
|
285
|
+
ids << event.event_id
|
|
286
|
+
cause = event.metadata[:causation_id]
|
|
287
|
+
caused << { event_id: event.event_id, event_type: event.event_type, causation_id: cause } if cause
|
|
288
|
+
end
|
|
289
|
+
end
|
|
290
|
+
{ events: ids.size, caused: caused.size, dangling: caused.reject { ids.include?(_1[:causation_id]) } }
|
|
291
|
+
end
|
|
292
|
+
|
|
172
293
|
private
|
|
173
294
|
|
|
174
295
|
def causation_store
|
data/lib/lyra/domain_events.rb
CHANGED
|
@@ -75,6 +75,10 @@ module Lyra
|
|
|
75
75
|
if klass && !(klass.is_a?(Class) && klass <= RubyEventStore::Event)
|
|
76
76
|
raise ArgumentError, "class: for #{name} must be a RubyEventStore::Event subclass"
|
|
77
77
|
end
|
|
78
|
+
# Names Lyra keeps for its events that are never replayed (Lyra::StreamTail).
|
|
79
|
+
if klass ? Lyra::StreamTail::NEVER_REPLAYED.include?(klass.name) : Lyra::StreamTail.reserved_name?(name)
|
|
80
|
+
raise ArgumentError, "#{name} is reserved for Lyra's own events"
|
|
81
|
+
end
|
|
78
82
|
|
|
79
83
|
operations = Array(definition.fetch(:on) { %i[create update destroy] }).map do |op|
|
|
80
84
|
OPERATIONS.fetch(op.to_sym) { raise ArgumentError, "on: #{op.inspect} for #{name} is not create, update or destroy" }
|
data/lib/lyra/dual_view.rb
CHANGED
|
@@ -25,19 +25,20 @@ module Lyra
|
|
|
25
25
|
}
|
|
26
26
|
end
|
|
27
27
|
|
|
28
|
-
# CRUD view - current database state
|
|
28
|
+
# CRUD view - current database state. Read as the caller reads: under
|
|
29
|
+
# reads_without_purpose :deny it needs a purpose that allows the declared
|
|
30
|
+
# fields, as any read does. Lyra's own comparisons (the mode gate, the
|
|
31
|
+
# sampler, repair) read as Lyra. The row is looked up by the model's
|
|
32
|
+
# primary key, whatever its column is called.
|
|
29
33
|
def crud_state
|
|
30
|
-
record =
|
|
34
|
+
record = model_class.find_by(Lyra.key_column(model_class) => model_id)
|
|
31
35
|
|
|
32
36
|
return { exists: false } unless record
|
|
33
37
|
|
|
34
38
|
{
|
|
35
39
|
exists: true,
|
|
36
40
|
attributes: record.attributes,
|
|
37
|
-
timestamps:
|
|
38
|
-
created_at: record.created_at,
|
|
39
|
-
updated_at: record.updated_at
|
|
40
|
-
}
|
|
41
|
+
timestamps: timestamps_of(record)
|
|
41
42
|
}
|
|
42
43
|
end
|
|
43
44
|
|
|
@@ -94,12 +95,21 @@ module Lyra
|
|
|
94
95
|
end
|
|
95
96
|
|
|
96
97
|
# Calculate differences between CRUD and Event-sourced states
|
|
98
|
+
# Each attribute whose value differs, keyed by its column name as a
|
|
99
|
+
# string (as Active Record keys attributes), => { crud:, event_sourced: };
|
|
100
|
+
# {} when the row and its replayed state agree (both absent counts as
|
|
101
|
+
# agreeing). When only one of them has the record, one entry in the same
|
|
102
|
+
# shape under the symbol :exists, exists: { crud:, event_sourced: } (true or
|
|
103
|
+
# false each), which no column name (a string) can collide with: never
|
|
104
|
+
# empty, so a caller that checks for {} cannot take it for agreement.
|
|
105
|
+
# Before 1.0.0 agreement was { no_differences: true } and a missing record
|
|
106
|
+
# { exists_mismatch: true }.
|
|
97
107
|
def calculate_differences
|
|
98
108
|
crud = crud_state
|
|
99
109
|
es = event_sourced_state
|
|
100
110
|
|
|
101
|
-
return {
|
|
102
|
-
return {
|
|
111
|
+
return { exists: { crud: crud[:exists], event_sourced: es[:exists] } } if crud[:exists] != es[:exists]
|
|
112
|
+
return {} if !crud[:exists] && !es[:exists]
|
|
103
113
|
|
|
104
114
|
crud_attrs = crud[:attributes] || {}
|
|
105
115
|
es_attrs = es[:state] || {}
|
|
@@ -110,27 +120,33 @@ module Lyra
|
|
|
110
120
|
crud_lookup = build_key_lookup(crud_attrs)
|
|
111
121
|
es_lookup = build_key_lookup(es_attrs)
|
|
112
122
|
|
|
113
|
-
# Compare union of
|
|
114
|
-
|
|
115
|
-
all_keys = (crud_lookup.keys + es_lookup.keys).uniq
|
|
123
|
+
# Compare the union of both sources' keys. created_at and updated_at are
|
|
124
|
+
# compared like any other attribute: every event carries them (1.0.0).
|
|
125
|
+
all_keys = (crud_lookup.keys + es_lookup.keys).uniq
|
|
116
126
|
|
|
117
127
|
all_keys.each do |key|
|
|
118
128
|
crud_val = crud_lookup[key]
|
|
119
129
|
es_val = es_lookup[key]
|
|
120
130
|
|
|
121
131
|
unless values_equal?(crud_val, es_val)
|
|
122
|
-
differences[key] = {
|
|
132
|
+
differences[key.to_s] = {
|
|
123
133
|
crud: crud_val,
|
|
124
134
|
event_sourced: es_val
|
|
125
135
|
}
|
|
126
136
|
end
|
|
127
137
|
end
|
|
128
138
|
|
|
139
|
+
# With config.record_touches false (Monitor only), a touch that changed
|
|
140
|
+
# only the update timestamps left no event, so a row whose only
|
|
141
|
+
# difference is in those columns agrees with its events. A difference in
|
|
142
|
+
# any other column is reported with the timestamps beside it.
|
|
143
|
+
differences = {} if !Lyra.config.record_touches && touch_only?(differences)
|
|
144
|
+
|
|
129
145
|
# has_and_belongs_to_many links: the join table against the link
|
|
130
146
|
# events (Lyra::Associations::HabtmLinks), by association name.
|
|
131
147
|
differences.merge!(Associations::HabtmLinks.differences(model_class, model_id))
|
|
132
148
|
|
|
133
|
-
differences
|
|
149
|
+
differences
|
|
134
150
|
end
|
|
135
151
|
|
|
136
152
|
# Get audit trail from events
|
|
@@ -140,6 +156,27 @@ module Lyra
|
|
|
140
156
|
|
|
141
157
|
private
|
|
142
158
|
|
|
159
|
+
# The record's timestamp columns, by name: those the model has
|
|
160
|
+
# (created_at, updated_at, updated_on, ...), none for a model without.
|
|
161
|
+
# Before 1.0.0 created_at and updated_at were read whatever the model had,
|
|
162
|
+
# and a model without one raised NoMethodError.
|
|
163
|
+
def timestamps_of(record)
|
|
164
|
+
klass = record.class
|
|
165
|
+
return {} unless klass.respond_to?(:timestamp_attributes_for_update_in_model)
|
|
166
|
+
|
|
167
|
+
names = klass.timestamp_attributes_for_create_in_model | klass.timestamp_attributes_for_update_in_model
|
|
168
|
+
names.to_h { |name| [name.to_sym, record.read_attribute(name)] }
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
# Whether every difference is in the model's update timestamps
|
|
172
|
+
# (updated_at, updated_on).
|
|
173
|
+
def touch_only?(differences)
|
|
174
|
+
return false if differences.empty? || !model_class.respond_to?(:timestamp_attributes_for_update_in_model)
|
|
175
|
+
|
|
176
|
+
stamps = model_class.timestamp_attributes_for_update_in_model
|
|
177
|
+
differences.each_key.all? { |key| stamps.include?(key.to_s) }
|
|
178
|
+
end
|
|
179
|
+
|
|
143
180
|
# Build a normalized lookup table from attributes hash
|
|
144
181
|
# Converts all keys to symbols for consistent access
|
|
145
182
|
def build_key_lookup(attrs)
|
|
@@ -211,7 +248,7 @@ module Lyra
|
|
|
211
248
|
|
|
212
249
|
def find_discrepancies(model_class)
|
|
213
250
|
compare_all(model_class).select do |comparison|
|
|
214
|
-
comparison[:differences]
|
|
251
|
+
comparison[:differences].any?
|
|
215
252
|
end
|
|
216
253
|
end
|
|
217
254
|
end
|
|
@@ -233,11 +270,11 @@ module Lyra
|
|
|
233
270
|
comparison = view.compare
|
|
234
271
|
recommendations = []
|
|
235
272
|
|
|
236
|
-
if comparison[:differences]
|
|
273
|
+
if comparison[:differences].key?(:exists)
|
|
237
274
|
recommendations << "State existence mismatch - investigate data consistency"
|
|
238
275
|
end
|
|
239
276
|
|
|
240
|
-
if comparison[:differences]
|
|
277
|
+
if comparison[:differences].any? && !comparison[:differences].key?(:exists)
|
|
241
278
|
recommendations << "Attribute differences detected - consider which source is authoritative"
|
|
242
279
|
recommendations << "Differences: #{comparison[:differences].keys.join(', ')}"
|
|
243
280
|
end
|