orfeas_lyra 0.9.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +2021 -17
- data/README.md +22 -9
- data/Rakefile +40 -1
- data/app/controllers/lyra/application_controller.rb +127 -0
- data/app/controllers/lyra/dashboard_controller.rb +33 -31
- data/app/controllers/lyra/flow_controller.rb +32 -34
- data/app/controllers/lyra/privacy_controller.rb +57 -18
- data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
- data/app/views/lyra/dashboard/index.html.erb +1 -0
- data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
- data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
- data/app/views/lyra/privacy/policy.html.erb +7 -0
- data/app/views/lyra/privacy/rights.html.erb +78 -0
- data/config/privacy_policies.rb +0 -3
- data/config/routes.rb +1 -0
- data/lib/lyra/access_log.rb +187 -6
- data/lib/lyra/advisory_lock.rb +236 -0
- data/lib/lyra/aggregate.rb +1 -1
- data/lib/lyra/associations/habtm_links.rb +382 -19
- data/lib/lyra/associations/maintenance.rb +229 -0
- data/lib/lyra/attribution.rb +48 -0
- data/lib/lyra/bypass_events.rb +446 -46
- data/lib/lyra/command_handler.rb +19 -14
- data/lib/lyra/configuration.rb +237 -10
- data/lib/lyra/consistency/read_your_writes.rb +4 -2
- data/lib/lyra/correlation.rb +121 -0
- data/lib/lyra/domain_events.rb +4 -0
- data/lib/lyra/dual_view.rb +53 -16
- data/lib/lyra/encrypted_attributes.rb +5 -0
- data/lib/lyra/engine.rb +67 -0
- data/lib/lyra/erasure.rb +1011 -106
- data/lib/lyra/event.rb +35 -1
- data/lib/lyra/event_data.rb +38 -0
- data/lib/lyra/event_flow.rb +31 -15
- data/lib/lyra/event_store_adapter.rb +28 -0
- data/lib/lyra/event_window.rb +58 -0
- data/lib/lyra/genesis.rb +548 -44
- data/lib/lyra/interceptors/association_interceptor.rb +689 -34
- data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
- data/lib/lyra/internal.rb +26 -0
- data/lib/lyra/mode_sync.rb +51 -3
- data/lib/lyra/mode_transition.rb +257 -30
- data/lib/lyra/model_selection.rb +35 -0
- data/lib/lyra/own_connection.rb +17 -2
- data/lib/lyra/privacy/adapters/pam.rb +25 -2
- data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
- data/lib/lyra/privacy/interface.rb +27 -7
- data/lib/lyra/privacy/subject_links.rb +143 -0
- data/lib/lyra/privacy/subject_records.rb +369 -0
- data/lib/lyra/projection.rb +7 -1
- data/lib/lyra/projections/async_checkpoint.rb +32 -0
- data/lib/lyra/projections/async_projection_job.rb +166 -23
- data/lib/lyra/projections/cached_joins.rb +33 -18
- data/lib/lyra/projections/cached_projection.rb +74 -33
- data/lib/lyra/projections/cached_relation.rb +3175 -321
- data/lib/lyra/projections/default_scopes.rb +254 -0
- data/lib/lyra/projections/deferred_relation.rb +23 -3
- data/lib/lyra/projections/event_store_reader.rb +47 -8
- data/lib/lyra/projections/foreign_keys.rb +92 -0
- data/lib/lyra/projections/lazy_projection.rb +165 -27
- data/lib/lyra/projections/model_projection.rb +165 -14
- data/lib/lyra/projections/rebuild.rb +34 -6
- data/lib/lyra/projections/reference_links.rb +463 -72
- data/lib/lyra/projections/typed_values.rb +629 -0
- data/lib/lyra/projections/unique_indexes.rb +595 -0
- data/lib/lyra/projections/uniqueness.rb +784 -0
- data/lib/lyra/projections/value_digest.rb +93 -0
- data/lib/lyra/purpose_bound_reads.rb +912 -77
- data/lib/lyra/repair.rb +119 -28
- data/lib/lyra/retention.rb +114 -8
- data/lib/lyra/rights/consent_store.rb +415 -0
- data/lib/lyra/rights/erasure_guard.rb +135 -0
- data/lib/lyra/rights/pending.rb +74 -0
- data/lib/lyra/rights/queries.rb +269 -0
- data/lib/lyra/rights/store.rb +512 -0
- data/lib/lyra/rights/subject_keys.rb +224 -0
- data/lib/lyra/rights.rb +281 -0
- data/lib/lyra/saves_without_effect.rb +69 -0
- data/lib/lyra/stream_tail.rb +319 -0
- data/lib/lyra/strict_data_access.rb +916 -106
- data/lib/lyra/temporal.rb +4 -2
- data/lib/lyra/touch_registry.rb +404 -0
- data/lib/lyra/verification/bypass_workflow.rb +452 -42
- data/lib/lyra/verification/rights_workflows.rb +170 -0
- data/lib/lyra/version.rb +1 -1
- data/lib/lyra/visualization/event_graph.rb +1 -1
- data/lib/lyra/visualization/timeline.rb +2 -2
- data/lib/lyra.rb +43 -2
- data/lib/tasks/lyra_mode.rake +110 -8
- data/lib/tasks/lyra_projections.rake +43 -2
- metadata +34 -4
- data/lib/lyra/associations/event_aware.rb +0 -224
|
@@ -35,6 +35,30 @@ module Lyra
|
|
|
35
35
|
# Don't retry on permanent failures
|
|
36
36
|
discard_on ActiveRecord::RecordNotFound
|
|
37
37
|
|
|
38
|
+
# A create that met, in a unique index, the row of a record whose
|
|
39
|
+
# stream gave the value up, and whose row is held by another
|
|
40
|
+
# transaction, so it could not be brought up to date (ModelProjection
|
|
41
|
+
# .refuse_skipped_create!, which brings it up to date when it is
|
|
42
|
+
# free, so a holder whose own job was lost does not block): tried
|
|
43
|
+
# again, at most every five minutes, for about a day (BEHIND_ATTEMPTS;
|
|
44
|
+
# five attempts gave up after about seven minutes, and the record never
|
|
45
|
+
# reached the table). Then it is reported (logged, and passed to
|
|
46
|
+
# config.projection_error_handler) and given up: the row was held for
|
|
47
|
+
# a day; a rebuild of the model projects the record. (Declared after
|
|
48
|
+
# the general retry: Active Job takes the last handler that matches.)
|
|
49
|
+
BEHIND_ATTEMPTS = 300
|
|
50
|
+
retry_on Lyra::Projections::ProjectionBehind, attempts: BEHIND_ATTEMPTS,
|
|
51
|
+
wait: ->(executions) { [(executions**4) + 2, 300].min } do |_job, error|
|
|
52
|
+
message = "#{error.message}; given up after #{BEHIND_ATTEMPTS} attempts: rebuild the model's projection " \
|
|
53
|
+
"(bin/rails lyra:projections:rebuild) once the row is free"
|
|
54
|
+
Rails.logger.error(message) if defined?(Rails) && Rails.logger
|
|
55
|
+
Lyra.config.projection_error_handler&.call(error, nil, :create)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Two live records of the log hold one value (DuplicateInLog): no
|
|
59
|
+
# attempt can project both. Logged once (ModelProjection), not retried.
|
|
60
|
+
discard_on Lyra::Projections::DuplicateInLog
|
|
61
|
+
|
|
38
62
|
# Bring the event's record up to date with its stream, rather than
|
|
39
63
|
# applying this one event.
|
|
40
64
|
#
|
|
@@ -57,6 +81,19 @@ module Lyra
|
|
|
57
81
|
# below a checkpoint already passed. With that lock off, or without a
|
|
58
82
|
# usable checkpoint, the job replays the whole stream as before.
|
|
59
83
|
# +operation+ is kept for job compatibility.
|
|
84
|
+
#
|
|
85
|
+
# The job does not wait for its row while it holds the stream's lock
|
|
86
|
+
# (PostgreSQL): once it has the lock it takes the row's, skipping it if
|
|
87
|
+
# another transaction holds it (row_taken?, the one statement this
|
|
88
|
+
# costs), and then lets the lock go (its transaction, which wrote
|
|
89
|
+
# nothing, commits), waits for the row with no lock of its own, and
|
|
90
|
+
# starts again, up to ROW_WAITS times; after that it waits holding
|
|
91
|
+
# the lock, as it always did. A transaction that held the row (an
|
|
92
|
+
# update_columns, touch, delete, update_all, lock! or with_lock) and
|
|
93
|
+
# then brought it up to date for an insert_all or upsert_all
|
|
94
|
+
# (project_pending, which takes the stream's lock) deadlocked against
|
|
95
|
+
# a job that had taken the lock and waited for the row. A job that
|
|
96
|
+
# holds the row takes no other lock, so the writer then waits for it.
|
|
60
97
|
def perform(event_id, model_class_name, operation = nil)
|
|
61
98
|
event = load_event(event_id)
|
|
62
99
|
return unless event
|
|
@@ -65,46 +102,152 @@ module Lyra
|
|
|
65
102
|
model_id = event.data[:model_id] || event.data["model_id"]
|
|
66
103
|
stream = Lyra.stream_name(model_class, model_id)
|
|
67
104
|
|
|
105
|
+
ROW_WAITS.times do
|
|
106
|
+
projected = model_class.transaction do
|
|
107
|
+
lock_stream!(model_class, stream)
|
|
108
|
+
next false if row_taken?(model_class, model_id)
|
|
109
|
+
|
|
110
|
+
self.class.project_stream(model_class, stream)
|
|
111
|
+
true
|
|
112
|
+
end
|
|
113
|
+
return if projected
|
|
114
|
+
|
|
115
|
+
wait_for_row(model_class, model_id)
|
|
116
|
+
end
|
|
68
117
|
model_class.transaction do
|
|
69
118
|
lock_stream!(model_class, stream)
|
|
70
|
-
project_stream(model_class,
|
|
119
|
+
self.class.project_stream(model_class, stream)
|
|
71
120
|
end
|
|
72
121
|
end
|
|
73
122
|
|
|
74
|
-
|
|
123
|
+
# How many times a job lets its stream's lock go for a row another
|
|
124
|
+
# transaction holds (perform).
|
|
125
|
+
ROW_WAITS = 3
|
|
75
126
|
|
|
76
|
-
|
|
77
|
-
model_class
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
127
|
+
class << self
|
|
128
|
+
# Bring the rows of +ids+ (records of +model_class+) up to date with
|
|
129
|
+
# their streams now, as their projection jobs would, for a write that
|
|
130
|
+
# skips callbacks and would otherwise meet a row its job has not
|
|
131
|
+
# written yet (Lyra::BypassEvents.projects_pending?). The caller holds
|
|
132
|
+
# the records' aggregate locks, in its transaction, so no event can
|
|
133
|
+
# be appended to those streams until it ends. The streams with events
|
|
134
|
+
# past their checkpoint (AsyncCheckpoint.behind, one query) are locked
|
|
135
|
+
# as a job locks them, in one statement, after the aggregate locks and
|
|
136
|
+
# before any row (Lyra::AdvisoryLock's order): a job in flight
|
|
137
|
+
# finishes first. Then, as perform does, the events after each
|
|
138
|
+
# checkpoint are applied and the checkpoint is moved to the last of
|
|
139
|
+
# them, in the caller's transaction.
|
|
140
|
+
#
|
|
141
|
+
# The pending jobs cannot apply those events again, nor undo the
|
|
142
|
+
# caller's write: a job applies only what is past the checkpoint,
|
|
143
|
+
# under the same lock, and that is the caller's own event at most,
|
|
144
|
+
# already in the row (a create of an existing row inserts nothing, an
|
|
145
|
+
# update sets the same values). Rolled back with the caller, the rows
|
|
146
|
+
# and the checkpoint go back together, and the jobs apply the events
|
|
147
|
+
# as before. Where checkpoints are not kept (incremental? false) every
|
|
148
|
+
# stream given is replayed, and so is it by its jobs; the row ends as
|
|
149
|
+
# its stream does either way. Returns how many records were brought
|
|
150
|
+
# up to date.
|
|
151
|
+
def project_pending(model_class, ids)
|
|
152
|
+
return 0 if ids.empty?
|
|
153
|
+
|
|
154
|
+
model_class.with_connection do |connection|
|
|
155
|
+
streams = ids.map { Lyra.stream_name(model_class, _1) }.uniq
|
|
156
|
+
behind = incremental?(connection) ? AsyncCheckpoint.behind(connection, streams) : streams
|
|
157
|
+
next 0 if behind.empty?
|
|
158
|
+
|
|
159
|
+
Lyra::AdvisoryLock.xact_lock_all(connection, behind, purpose: "ES-Async projection")
|
|
160
|
+
behind.count { |stream| project_stream(model_class, stream).positive? }
|
|
161
|
+
end
|
|
81
162
|
end
|
|
82
|
-
end
|
|
83
163
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
164
|
+
# Move the checkpoints of the records in +moves+ ([id, event id]) to
|
|
165
|
+
# those events, in one statement: a write that skips callbacks wrote
|
|
166
|
+
# their rows and those events, the last of their streams, in its
|
|
167
|
+
# transaction, from rows that were up to date
|
|
168
|
+
# (StrictDataAccessRelation#advance_checkpoints). Where checkpoints
|
|
169
|
+
# are kept (incremental?).
|
|
170
|
+
def advance(model_class, moves)
|
|
171
|
+
return if moves.empty?
|
|
90
172
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
173
|
+
model_class.with_connection do |connection|
|
|
174
|
+
next unless incremental?(connection)
|
|
175
|
+
|
|
176
|
+
AsyncCheckpoint.write_all(connection, moves.map { |id, event_id| [Lyra.stream_name(model_class, id), event_id] })
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
# Apply +stream+'s events after its checkpoint to its row and move the
|
|
181
|
+
# checkpoint to the last of them. The caller holds the stream's lock.
|
|
182
|
+
# Returns how many events there were.
|
|
183
|
+
def project_stream(model_class, stream)
|
|
184
|
+
model_class.with_connection do |connection|
|
|
185
|
+
events = events_to_apply(connection, stream)
|
|
186
|
+
Rebuild.project_events(model_class, events, INTERNAL)
|
|
187
|
+
AsyncCheckpoint.write(connection, stream, events.last.event_id) if events.any? && incremental?(connection)
|
|
188
|
+
events.size
|
|
189
|
+
end
|
|
190
|
+
end
|
|
95
191
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
192
|
+
private
|
|
193
|
+
|
|
194
|
+
# The events after the checkpoint when it can be trusted; the whole
|
|
195
|
+
# stream otherwise (the checkpoint's event is gone, or none was written).
|
|
196
|
+
def events_to_apply(connection, stream)
|
|
197
|
+
reader = Lyra.config.event_store.read.stream(stream)
|
|
198
|
+
position = incremental?(connection) && AsyncCheckpoint.read(connection, stream)
|
|
199
|
+
return reader.to_a unless position
|
|
200
|
+
|
|
201
|
+
reader.from(position).to_a
|
|
202
|
+
rescue RubyEventStore::EventNotFound
|
|
203
|
+
Lyra.config.event_store.read.stream(stream).to_a
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
# Checkpoints are exact only while writers take the lock on the
|
|
207
|
+
# aggregate, and need a database where it is taken (PostgreSQL) or
|
|
208
|
+
# writers are serialised anyway (SQLite).
|
|
209
|
+
def incremental?(connection)
|
|
210
|
+
Lyra.config.lock_aggregate_before_append && connection.adapter_name.match?(/postgres|sqlite/i)
|
|
211
|
+
end
|
|
101
212
|
end
|
|
102
213
|
|
|
214
|
+
private
|
|
215
|
+
|
|
103
216
|
# Serialise projection of one stream across workers (Lyra::AdvisoryLock).
|
|
104
217
|
def lock_stream!(model_class, stream)
|
|
105
218
|
model_class.with_connection { |c| Lyra::AdvisoryLock.xact_lock(c, stream, purpose: "ES-Async projection") }
|
|
106
219
|
end
|
|
107
220
|
|
|
221
|
+
# Take the record's row (FOR UPDATE), unless another transaction holds
|
|
222
|
+
# it: true then. One statement; a row not there (a create to project)
|
|
223
|
+
# is not taken. PostgreSQL only (SKIP LOCKED); elsewhere false.
|
|
224
|
+
def row_taken?(model_class, model_id)
|
|
225
|
+
model_class.with_connection do |c|
|
|
226
|
+
next false unless c.adapter_name.match?(/postgres/i)
|
|
227
|
+
|
|
228
|
+
from = "FROM #{c.quote_table_name(model_class.table_name)} " \
|
|
229
|
+
"WHERE #{c.quote_column_name(Lyra.key_column(model_class))} = #{c.quote(model_id)}"
|
|
230
|
+
present, held = c.uncached do
|
|
231
|
+
c.select_rows("SELECT (SELECT 1 #{from}), (SELECT 1 #{from} FOR UPDATE SKIP LOCKED)", "Lyra ES-Async row").first
|
|
232
|
+
end
|
|
233
|
+
!present.nil? && held.nil?
|
|
234
|
+
end
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
# Wait, holding no lock of the job's, until the transaction that holds
|
|
238
|
+
# the record's row ends.
|
|
239
|
+
def wait_for_row(model_class, model_id)
|
|
240
|
+
model_class.transaction do
|
|
241
|
+
model_class.with_connection do |c|
|
|
242
|
+
c.uncached do
|
|
243
|
+
c.select_value("SELECT 1 FROM #{c.quote_table_name(model_class.table_name)} " \
|
|
244
|
+
"WHERE #{c.quote_column_name(Lyra.key_column(model_class))} = #{c.quote(model_id)} FOR UPDATE",
|
|
245
|
+
"Lyra ES-Async row wait")
|
|
246
|
+
end
|
|
247
|
+
end
|
|
248
|
+
end
|
|
249
|
+
end
|
|
250
|
+
|
|
108
251
|
# read.event returns nil for a missing event (read.event! raises): the
|
|
109
252
|
# warning sat in a rescue of EventNotFound and was never logged.
|
|
110
253
|
def load_event(event_id)
|
|
@@ -114,16 +114,22 @@ module Lyra
|
|
|
114
114
|
read_partners(reflection.klass, key_column => keys).group_by { _1.read_attribute(key_column) }
|
|
115
115
|
end
|
|
116
116
|
|
|
117
|
+
# Lyra reads the partners to evaluate the join, as the database reads
|
|
118
|
+
# the joined rows; none is returned to the application. What the query
|
|
119
|
+
# names on the joined table is checked against that model's policy
|
|
120
|
+
# (Lyra::PurposeBoundReads), not the partners themselves.
|
|
117
121
|
def read_partners(klass, conditions)
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
122
|
+
Lyra::PurposeBoundReads.internal(INTERNAL) do
|
|
123
|
+
if events_only_model?(klass)
|
|
124
|
+
EventStoreReader.where(klass, conditions).to_a
|
|
125
|
+
else
|
|
126
|
+
begin
|
|
127
|
+
previous = Thread.current[:lyra_bypass_read_override]
|
|
128
|
+
Thread.current[:lyra_bypass_read_override] = true
|
|
129
|
+
klass.unscoped.where(conditions).to_a
|
|
130
|
+
ensure
|
|
131
|
+
Thread.current[:lyra_bypass_read_override] = previous
|
|
132
|
+
end
|
|
127
133
|
end
|
|
128
134
|
end
|
|
129
135
|
end
|
|
@@ -140,7 +146,9 @@ module Lyra
|
|
|
140
146
|
own = {}
|
|
141
147
|
joined = {}
|
|
142
148
|
conditions.each do |key, value|
|
|
143
|
-
|
|
149
|
+
# A Hash on a column of the model is a value (a serialized
|
|
150
|
+
# attribute's), as Active Record reads it (table.has_column?).
|
|
151
|
+
if value.is_a?(Hash) && !model_class.column_names.include?(TypedValues.attribute_name(model_class, key))
|
|
144
152
|
joined[joined_name_for(key)] = value
|
|
145
153
|
else
|
|
146
154
|
own[key] = value
|
|
@@ -162,22 +170,29 @@ module Lyra
|
|
|
162
170
|
# Whether a row's partners match the joined conditions, as SQL would
|
|
163
171
|
# decide it: true, false, or nil (unknown, a missing partner under a
|
|
164
172
|
# condition that is not IS NULL).
|
|
173
|
+
# +joined_conditions+ is what compile_joined made of the conditions on
|
|
174
|
+
# joined tables.
|
|
165
175
|
def joined_match(joins, joined_conditions)
|
|
166
|
-
joined_conditions.reduce(true) do |result, (name,
|
|
176
|
+
joined_conditions.reduce(true) do |result, (name, tests, null_partner)|
|
|
167
177
|
partner = joins[name]
|
|
178
|
+
outcome = partner.nil? ? null_partner : matches_all?(partner, tests)
|
|
179
|
+
combine_and(result, outcome)
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# The conditions on joined tables, compiled once per query: for each
|
|
184
|
+
# association, its tests on the partner, and the outcome for a missing
|
|
185
|
+
# partner (true when every condition is IS NULL, else unknown).
|
|
186
|
+
def compile_joined(joined_conditions)
|
|
187
|
+
joined_conditions.map do |name, conditions|
|
|
168
188
|
klass = joined_associations[name].reflection.klass
|
|
169
189
|
rewritten = AssociationConditions.rewrite(klass, conditions)
|
|
170
190
|
if rewritten.values.any? { _1.is_a?(Hash) }
|
|
171
191
|
raise UnsupportedQuery.new(model_class, "conditions nested more than one level on #{name}")
|
|
172
192
|
end
|
|
173
193
|
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
rewritten.values.all? { |v| v.nil? || (v.is_a?(Array) && v.include?(nil)) } ? true : nil
|
|
177
|
-
else
|
|
178
|
-
rewritten.all? { |column, v| matches_value?(partner, column, v) }
|
|
179
|
-
end
|
|
180
|
-
combine_and(result, outcome)
|
|
194
|
+
null_partner = rewritten.values.all? { |v| v.nil? || (v.is_a?(Array) && v.include?(nil)) } ? true : nil
|
|
195
|
+
[name, condition_tests(klass, rewritten), null_partner]
|
|
181
196
|
end
|
|
182
197
|
end
|
|
183
198
|
|
|
@@ -45,7 +45,9 @@ module Lyra
|
|
|
45
45
|
#
|
|
46
46
|
class CachedProjection
|
|
47
47
|
# 2: entries are { "v" => last event id, "a" => attributes or nil }.
|
|
48
|
-
|
|
48
|
+
# 3: the attributes as the table holds them (an enum's value, not its
|
|
49
|
+
# label; a serialized attribute's text), not as the events give them.
|
|
50
|
+
CACHE_VERSION = 3
|
|
49
51
|
DEFAULT_EXPIRES_IN = 1.hour
|
|
50
52
|
MEMORY_CACHE_BYTES = 32.megabytes
|
|
51
53
|
|
|
@@ -91,14 +93,16 @@ module Lyra
|
|
|
91
93
|
rest = attributes.reject { |key, _| key.to_s == primary_key }
|
|
92
94
|
return record if rest.empty?
|
|
93
95
|
|
|
94
|
-
return matches_attributes?(record, rest) ? record : nil
|
|
96
|
+
return matches_attributes?(TypedValues::EventReader.new(model_class), record, conditions_for(model_class, rest)) ? record : nil
|
|
95
97
|
end
|
|
96
98
|
|
|
97
99
|
# Without a primary key there is no way to know which stream holds
|
|
98
100
|
# the answer, so every record of the model is considered. That cost
|
|
99
101
|
# is inherent to querying by a non-key attribute with projections
|
|
100
102
|
# disabled; the entries keep it to one bulk read once warm.
|
|
101
|
-
|
|
103
|
+
tests = conditions_for(model_class, attributes)
|
|
104
|
+
reader = TypedValues::EventReader.new(model_class)
|
|
105
|
+
all(model_class).find { |record| matches_attributes?(reader, record, tests) }
|
|
102
106
|
end
|
|
103
107
|
|
|
104
108
|
# Every live record of the model, in primary-key order
|
|
@@ -132,7 +136,9 @@ module Lyra
|
|
|
132
136
|
# @return [Array<Hash>] Matching record attributes
|
|
133
137
|
def where(model_class, conditions)
|
|
134
138
|
candidates = ReferenceLinks.lookup(model_class, conditions) || all(model_class)
|
|
135
|
-
|
|
139
|
+
tests = conditions_for(model_class, conditions)
|
|
140
|
+
reader = TypedValues::EventReader.new(model_class)
|
|
141
|
+
candidates.select { |record| matches_attributes?(reader, record, tests) }
|
|
136
142
|
end
|
|
137
143
|
|
|
138
144
|
# Count records
|
|
@@ -184,6 +190,14 @@ module Lyra
|
|
|
184
190
|
find(model_class, id)
|
|
185
191
|
end
|
|
186
192
|
|
|
193
|
+
# For a reader that keeps its own copy of a model's records, kept
|
|
194
|
+
# current by the same stamps (Uniqueness::Snapshot): every stream's
|
|
195
|
+
# last event, the entries at given versions, and whether a record's
|
|
196
|
+
# attributes belong to +model_class+ (STI).
|
|
197
|
+
def stream_versions_for(model_class) = stream_versions(model_class)
|
|
198
|
+
def entries_for(model_class, versions) = entries_at(model_class, versions)
|
|
199
|
+
def of_class?(model_class, attributes) = !of_class(model_class, attributes).nil?
|
|
200
|
+
|
|
187
201
|
private
|
|
188
202
|
|
|
189
203
|
def fresh?(entry, version)
|
|
@@ -218,12 +232,20 @@ module Lyra
|
|
|
218
232
|
# The state reached by replaying the events after +entry+'s stamp onto
|
|
219
233
|
# its attributes, as [attributes or nil, version]; nil when that cannot
|
|
220
234
|
# be done exactly (no usable entry, a stamp that is no longer in the
|
|
221
|
-
# stream, nothing after it), and the caller
|
|
235
|
+
# stream, nothing after it, an erasure after it), and the caller
|
|
236
|
+
# replays the whole stream.
|
|
237
|
+
#
|
|
238
|
+
# An erasure rewrites the stream's events in place (Lyra::Erasure)
|
|
239
|
+
# and appends an ErasureApplied: the events before it no longer hold
|
|
240
|
+
# what the entry was built from. Replaying only the events after the
|
|
241
|
+
# stamp kept the erased values, in a process other than the eraser's
|
|
242
|
+
# (whose own entries the erasure drops).
|
|
222
243
|
def build_from_entry(model_class, id, stream, entry)
|
|
223
244
|
return nil unless entry.is_a?(Hash) && entry["v"] && entry["a"].is_a?(Hash)
|
|
224
245
|
|
|
225
246
|
events = load_events_after(stream, entry["v"])
|
|
226
247
|
return nil if events.nil? || events.empty?
|
|
248
|
+
return nil if events.any? { rewrites_stream?(_1) }
|
|
227
249
|
|
|
228
250
|
version = events.last.event_id
|
|
229
251
|
replayed = events.select { Lyra::Event.operation_of(_1) }
|
|
@@ -233,6 +255,10 @@ module Lyra
|
|
|
233
255
|
[reconstruct_state(model_class, id, replayed, entry["a"]), version]
|
|
234
256
|
end
|
|
235
257
|
|
|
258
|
+
def rewrites_stream?(event)
|
|
259
|
+
defined?(Lyra::Events::ErasureApplied) && event.is_a?(Lyra::Events::ErasureApplied)
|
|
260
|
+
end
|
|
261
|
+
|
|
236
262
|
# The id of the last event in a record's stream, or nil if it has none.
|
|
237
263
|
def stream_version(model_class, id)
|
|
238
264
|
ActiveRecord::Base.with_connection do |connection|
|
|
@@ -247,23 +273,33 @@ module Lyra
|
|
|
247
273
|
# id }): one bulk cache read, and a replay of only the streams whose
|
|
248
274
|
# entry is missing or out of date.
|
|
249
275
|
def records_at(model_class, versions)
|
|
250
|
-
|
|
276
|
+
entries_at(model_class, versions).filter_map { |_, (_, attributes)| attributes }
|
|
277
|
+
.select { |attributes| of_class(model_class, attributes) }
|
|
278
|
+
end
|
|
279
|
+
|
|
280
|
+
# { id => [the id of the last event the attributes were built from,
|
|
281
|
+
# attributes or nil (destroyed)] } for the streams whose last events
|
|
282
|
+
# are +versions+, in their order, whatever their class: one bulk
|
|
283
|
+
# cache read, and a replay of the streams whose entry is missing or
|
|
284
|
+
# out of date.
|
|
285
|
+
def entries_at(model_class, versions)
|
|
286
|
+
return {} if versions.empty?
|
|
251
287
|
|
|
252
288
|
keys = versions.keys.to_h { |id| [record_cache_key(model_class, id), id] }
|
|
253
289
|
cached = cache_read_multi(keys.keys)
|
|
254
290
|
rebuilt = {}
|
|
255
291
|
|
|
256
|
-
|
|
292
|
+
entries = keys.to_h do |key, id|
|
|
257
293
|
entry = cached[key]
|
|
258
|
-
next entry["a"] if fresh?(entry, versions[id])
|
|
294
|
+
next [id, [versions[id], entry["a"]]] if fresh?(entry, versions[id])
|
|
259
295
|
|
|
260
296
|
attributes, built_version = build_record(model_class, id, entry)
|
|
261
297
|
rebuilt[key] = entry_for(attributes, built_version) if built_version
|
|
262
|
-
attributes
|
|
298
|
+
[id, [built_version, attributes]]
|
|
263
299
|
end
|
|
264
300
|
|
|
265
301
|
cache_write_multi(rebuilt) if rebuilt.any?
|
|
266
|
-
|
|
302
|
+
entries
|
|
267
303
|
end
|
|
268
304
|
|
|
269
305
|
# +attributes+ if the record belongs to +model_class+, nil otherwise.
|
|
@@ -311,50 +347,55 @@ module Lyra
|
|
|
311
347
|
end
|
|
312
348
|
end
|
|
313
349
|
|
|
314
|
-
# Shared by find_by and where so both decide a match the same way
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
350
|
+
# Shared by find_by and where so both decide a match the same way, and
|
|
351
|
+
# as a CachedRelation of the built records decides it: each value
|
|
352
|
+
# read as the record built from +record+ (the event attributes) holds
|
|
353
|
+
# it (+reader+, a TypedValues::EventReader), and compared with the
|
|
354
|
+
# condition's value bound as Active Record binds it (TypedValues).
|
|
355
|
+
# The event's "250.0" against find_by(amount: 250.0) was a String
|
|
356
|
+
# against a Float, and missed.
|
|
357
|
+
def matches_attributes?(reader, record, tests)
|
|
358
|
+
true.equal?(TypedValues.all_hold(tests, reader, record))
|
|
319
359
|
end
|
|
320
360
|
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
case value
|
|
326
|
-
when Array
|
|
327
|
-
value.include?(record_value)
|
|
328
|
-
when Range
|
|
329
|
-
value.cover?(record_value)
|
|
330
|
-
else
|
|
331
|
-
record_value == value
|
|
332
|
-
end
|
|
333
|
-
end
|
|
361
|
+
# The conditions of a hash, compiled once for the records they are
|
|
362
|
+
# tested on (TypedValues.condition).
|
|
363
|
+
def conditions_for(model_class, conditions)
|
|
364
|
+
conditions.map { |key, value| TypedValues.condition(model_class, key, value) }
|
|
334
365
|
end
|
|
335
366
|
|
|
336
367
|
# Reconstruct record state from events
|
|
337
368
|
# +base+: the attributes the +events+ are applied onto (the state at
|
|
338
369
|
# an earlier event), instead of an empty record.
|
|
339
370
|
def reconstruct_state(model_class, id, events, base = nil)
|
|
340
|
-
|
|
371
|
+
key = Lyra.key_column(model_class)
|
|
372
|
+
attributes = base ? base.dup : { key => id }
|
|
373
|
+
# The attributes the events set (all of them, from a create).
|
|
374
|
+
set = base ? Set.new : nil
|
|
341
375
|
|
|
342
376
|
events.each do |event|
|
|
343
377
|
case Lyra::Event.operation_of(event)
|
|
344
378
|
when :created, :imported
|
|
345
|
-
event_attrs =
|
|
346
|
-
attributes.merge!(
|
|
379
|
+
event_attrs = stringify_keys(Lyra::Event.created_attributes(event.data))
|
|
380
|
+
attributes.merge!(event_attrs)
|
|
381
|
+
set&.merge(event_attrs.keys)
|
|
347
382
|
when :updated
|
|
348
383
|
changes = event.data[:changes] || event.data["changes"] || {}
|
|
349
384
|
changes.each do |field, change|
|
|
350
385
|
new_value = change.is_a?(Array) ? change.last : change
|
|
351
386
|
attributes[field.to_s] = new_value
|
|
387
|
+
set&.add(field.to_s)
|
|
352
388
|
end
|
|
353
389
|
end
|
|
354
390
|
end
|
|
355
391
|
|
|
356
|
-
#
|
|
357
|
-
|
|
392
|
+
# The key as the row holds it (an integer key from the stream name's
|
|
393
|
+
# text), under the primary key column
|
|
394
|
+
attributes[key] = Lyra.key_value(model_class, id)
|
|
395
|
+
# Entries hold the attributes as the table does (an enum's value,
|
|
396
|
+
# a serialized attribute's text), converted once here rather than
|
|
397
|
+
# on every read (TypedValues.convert_event_values!).
|
|
398
|
+
TypedValues.convert_event_values!(model_class, attributes, set)
|
|
358
399
|
attributes
|
|
359
400
|
end
|
|
360
401
|
|