orfeas_lyra 0.9.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +2021 -17
  3. data/README.md +22 -9
  4. data/Rakefile +40 -1
  5. data/app/controllers/lyra/application_controller.rb +127 -0
  6. data/app/controllers/lyra/dashboard_controller.rb +33 -31
  7. data/app/controllers/lyra/flow_controller.rb +32 -34
  8. data/app/controllers/lyra/privacy_controller.rb +57 -18
  9. data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
  10. data/app/views/lyra/dashboard/index.html.erb +1 -0
  11. data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
  12. data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
  13. data/app/views/lyra/privacy/policy.html.erb +7 -0
  14. data/app/views/lyra/privacy/rights.html.erb +78 -0
  15. data/config/privacy_policies.rb +0 -3
  16. data/config/routes.rb +1 -0
  17. data/lib/lyra/access_log.rb +187 -6
  18. data/lib/lyra/advisory_lock.rb +236 -0
  19. data/lib/lyra/aggregate.rb +1 -1
  20. data/lib/lyra/associations/habtm_links.rb +382 -19
  21. data/lib/lyra/associations/maintenance.rb +229 -0
  22. data/lib/lyra/attribution.rb +48 -0
  23. data/lib/lyra/bypass_events.rb +446 -46
  24. data/lib/lyra/command_handler.rb +19 -14
  25. data/lib/lyra/configuration.rb +237 -10
  26. data/lib/lyra/consistency/read_your_writes.rb +4 -2
  27. data/lib/lyra/correlation.rb +121 -0
  28. data/lib/lyra/domain_events.rb +4 -0
  29. data/lib/lyra/dual_view.rb +53 -16
  30. data/lib/lyra/encrypted_attributes.rb +5 -0
  31. data/lib/lyra/engine.rb +67 -0
  32. data/lib/lyra/erasure.rb +1011 -106
  33. data/lib/lyra/event.rb +35 -1
  34. data/lib/lyra/event_data.rb +38 -0
  35. data/lib/lyra/event_flow.rb +31 -15
  36. data/lib/lyra/event_store_adapter.rb +28 -0
  37. data/lib/lyra/event_window.rb +58 -0
  38. data/lib/lyra/genesis.rb +548 -44
  39. data/lib/lyra/interceptors/association_interceptor.rb +689 -34
  40. data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
  41. data/lib/lyra/internal.rb +26 -0
  42. data/lib/lyra/mode_sync.rb +51 -3
  43. data/lib/lyra/mode_transition.rb +257 -30
  44. data/lib/lyra/model_selection.rb +35 -0
  45. data/lib/lyra/own_connection.rb +17 -2
  46. data/lib/lyra/privacy/adapters/pam.rb +25 -2
  47. data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
  48. data/lib/lyra/privacy/interface.rb +27 -7
  49. data/lib/lyra/privacy/subject_links.rb +143 -0
  50. data/lib/lyra/privacy/subject_records.rb +369 -0
  51. data/lib/lyra/projection.rb +7 -1
  52. data/lib/lyra/projections/async_checkpoint.rb +32 -0
  53. data/lib/lyra/projections/async_projection_job.rb +166 -23
  54. data/lib/lyra/projections/cached_joins.rb +33 -18
  55. data/lib/lyra/projections/cached_projection.rb +74 -33
  56. data/lib/lyra/projections/cached_relation.rb +3175 -321
  57. data/lib/lyra/projections/default_scopes.rb +254 -0
  58. data/lib/lyra/projections/deferred_relation.rb +23 -3
  59. data/lib/lyra/projections/event_store_reader.rb +47 -8
  60. data/lib/lyra/projections/foreign_keys.rb +92 -0
  61. data/lib/lyra/projections/lazy_projection.rb +165 -27
  62. data/lib/lyra/projections/model_projection.rb +165 -14
  63. data/lib/lyra/projections/rebuild.rb +34 -6
  64. data/lib/lyra/projections/reference_links.rb +463 -72
  65. data/lib/lyra/projections/typed_values.rb +629 -0
  66. data/lib/lyra/projections/unique_indexes.rb +595 -0
  67. data/lib/lyra/projections/uniqueness.rb +784 -0
  68. data/lib/lyra/projections/value_digest.rb +93 -0
  69. data/lib/lyra/purpose_bound_reads.rb +912 -77
  70. data/lib/lyra/repair.rb +119 -28
  71. data/lib/lyra/retention.rb +114 -8
  72. data/lib/lyra/rights/consent_store.rb +415 -0
  73. data/lib/lyra/rights/erasure_guard.rb +135 -0
  74. data/lib/lyra/rights/pending.rb +74 -0
  75. data/lib/lyra/rights/queries.rb +269 -0
  76. data/lib/lyra/rights/store.rb +512 -0
  77. data/lib/lyra/rights/subject_keys.rb +224 -0
  78. data/lib/lyra/rights.rb +281 -0
  79. data/lib/lyra/saves_without_effect.rb +69 -0
  80. data/lib/lyra/stream_tail.rb +319 -0
  81. data/lib/lyra/strict_data_access.rb +916 -106
  82. data/lib/lyra/temporal.rb +4 -2
  83. data/lib/lyra/touch_registry.rb +404 -0
  84. data/lib/lyra/verification/bypass_workflow.rb +452 -42
  85. data/lib/lyra/verification/rights_workflows.rb +170 -0
  86. data/lib/lyra/version.rb +1 -1
  87. data/lib/lyra/visualization/event_graph.rb +1 -1
  88. data/lib/lyra/visualization/timeline.rb +2 -2
  89. data/lib/lyra.rb +43 -2
  90. data/lib/tasks/lyra_mode.rake +110 -8
  91. data/lib/tasks/lyra_projections.rake +43 -2
  92. metadata +34 -4
  93. data/lib/lyra/associations/event_aware.rb +0 -224
@@ -35,6 +35,30 @@ module Lyra
35
35
  # Don't retry on permanent failures
36
36
  discard_on ActiveRecord::RecordNotFound
37
37
 
38
+ # A create that met, in a unique index, the row of a record whose
39
+ # stream gave the value up, and whose row is held by another
40
+ # transaction, so it could not be brought up to date (ModelProjection
41
+ # .refuse_skipped_create!, which brings it up to date when it is
42
+ # free, so a holder whose own job was lost does not block): tried
43
+ # again, at most every five minutes, for about a day (BEHIND_ATTEMPTS;
44
+ # five attempts gave up after about seven minutes, and the record never
45
+ # reached the table). Then it is reported (logged, and passed to
46
+ # config.projection_error_handler) and given up: the row was held for
47
+ # a day; a rebuild of the model projects the record. (Declared after
48
+ # the general retry: Active Job takes the last handler that matches.)
49
+ BEHIND_ATTEMPTS = 300
50
+ retry_on Lyra::Projections::ProjectionBehind, attempts: BEHIND_ATTEMPTS,
51
+ wait: ->(executions) { [(executions**4) + 2, 300].min } do |_job, error|
52
+ message = "#{error.message}; given up after #{BEHIND_ATTEMPTS} attempts: rebuild the model's projection " \
53
+ "(bin/rails lyra:projections:rebuild) once the row is free"
54
+ Rails.logger.error(message) if defined?(Rails) && Rails.logger
55
+ Lyra.config.projection_error_handler&.call(error, nil, :create)
56
+ end
57
+
58
+ # Two live records of the log hold one value (DuplicateInLog): no
59
+ # attempt can project both. Logged once (ModelProjection), not retried.
60
+ discard_on Lyra::Projections::DuplicateInLog
61
+
38
62
  # Bring the event's record up to date with its stream, rather than
39
63
  # applying this one event.
40
64
  #
@@ -57,6 +81,19 @@ module Lyra
57
81
  # below a checkpoint already passed. With that lock off, or without a
58
82
  # usable checkpoint, the job replays the whole stream as before.
59
83
  # +operation+ is kept for job compatibility.
84
+ #
85
+ # The job does not wait for its row while it holds the stream's lock
86
+ # (PostgreSQL): once it has the lock it takes the row's, skipping it if
87
+ # another transaction holds it (row_taken?, the one statement this
88
+ # costs), and then lets the lock go (its transaction, which wrote
89
+ # nothing, commits), waits for the row with no lock of its own, and
90
+ # starts again, up to ROW_WAITS times; after that it waits holding
91
+ # the lock, as it always did. A transaction that held the row (an
92
+ # update_columns, touch, delete, update_all, lock! or with_lock) and
93
+ # then brought it up to date for an insert_all or upsert_all
94
+ # (project_pending, which takes the stream's lock) deadlocked against
95
+ # a job that had taken the lock and waited for the row. A job that
96
+ # holds the row takes no other lock, so the writer then waits for it.
60
97
  def perform(event_id, model_class_name, operation = nil)
61
98
  event = load_event(event_id)
62
99
  return unless event
@@ -65,46 +102,152 @@ module Lyra
65
102
  model_id = event.data[:model_id] || event.data["model_id"]
66
103
  stream = Lyra.stream_name(model_class, model_id)
67
104
 
105
+ ROW_WAITS.times do
106
+ projected = model_class.transaction do
107
+ lock_stream!(model_class, stream)
108
+ next false if row_taken?(model_class, model_id)
109
+
110
+ self.class.project_stream(model_class, stream)
111
+ true
112
+ end
113
+ return if projected
114
+
115
+ wait_for_row(model_class, model_id)
116
+ end
68
117
  model_class.transaction do
69
118
  lock_stream!(model_class, stream)
70
- project_stream(model_class, model_id, stream)
119
+ self.class.project_stream(model_class, stream)
71
120
  end
72
121
  end
73
122
 
74
- private
123
+ # How many times a job lets its stream's lock go for a row another
124
+ # transaction holds (perform).
125
+ ROW_WAITS = 3
75
126
 
76
- def project_stream(model_class, model_id, stream)
77
- model_class.with_connection do |connection|
78
- events = events_to_apply(connection, stream)
79
- Rebuild.project_events(model_class, events)
80
- AsyncCheckpoint.write(connection, stream, events.last.event_id) if events.any? && incremental?(connection)
127
+ class << self
128
+ # Bring the rows of +ids+ (records of +model_class+) up to date with
129
+ # their streams now, as their projection jobs would, for a write that
130
+ # skips callbacks and would otherwise meet a row its job has not
131
+ # written yet (Lyra::BypassEvents.projects_pending?). The caller holds
132
+ # the records' aggregate locks, in its transaction, so no event can
133
+ # be appended to those streams until it ends. The streams with events
134
+ # past their checkpoint (AsyncCheckpoint.behind, one query) are locked
135
+ # as a job locks them, in one statement, after the aggregate locks and
136
+ # before any row (Lyra::AdvisoryLock's order): a job in flight
137
+ # finishes first. Then, as perform does, the events after each
138
+ # checkpoint are applied and the checkpoint is moved to the last of
139
+ # them, in the caller's transaction.
140
+ #
141
+ # The pending jobs cannot apply those events again, nor undo the
142
+ # caller's write: a job applies only what is past the checkpoint,
143
+ # under the same lock, and that is the caller's own event at most,
144
+ # already in the row (a create of an existing row inserts nothing, an
145
+ # update sets the same values). Rolled back with the caller, the rows
146
+ # and the checkpoint go back together, and the jobs apply the events
147
+ # as before. Where checkpoints are not kept (incremental? false) every
148
+ # stream given is replayed, and so is it by its jobs; the row ends as
149
+ # its stream does either way. Returns how many records were brought
150
+ # up to date.
151
+ def project_pending(model_class, ids)
152
+ return 0 if ids.empty?
153
+
154
+ model_class.with_connection do |connection|
155
+ streams = ids.map { Lyra.stream_name(model_class, _1) }.uniq
156
+ behind = incremental?(connection) ? AsyncCheckpoint.behind(connection, streams) : streams
157
+ next 0 if behind.empty?
158
+
159
+ Lyra::AdvisoryLock.xact_lock_all(connection, behind, purpose: "ES-Async projection")
160
+ behind.count { |stream| project_stream(model_class, stream).positive? }
161
+ end
81
162
  end
82
- end
83
163
 
84
- # The events after the checkpoint when it can be trusted; the whole
85
- # stream otherwise (the checkpoint's event is gone, or none was written).
86
- def events_to_apply(connection, stream)
87
- reader = Lyra.config.event_store.read.stream(stream)
88
- position = incremental?(connection) && AsyncCheckpoint.read(connection, stream)
89
- return reader.to_a unless position
164
+ # Move the checkpoints of the records in +moves+ ([id, event id]) to
165
+ # those events, in one statement: a write that skips callbacks wrote
166
+ # their rows and those events, the last of their streams, in its
167
+ # transaction, from rows that were up to date
168
+ # (StrictDataAccessRelation#advance_checkpoints). Where checkpoints
169
+ # are kept (incremental?).
170
+ def advance(model_class, moves)
171
+ return if moves.empty?
90
172
 
91
- reader.from(position).to_a
92
- rescue RubyEventStore::EventNotFound
93
- Lyra.config.event_store.read.stream(stream).to_a
94
- end
173
+ model_class.with_connection do |connection|
174
+ next unless incremental?(connection)
175
+
176
+ AsyncCheckpoint.write_all(connection, moves.map { |id, event_id| [Lyra.stream_name(model_class, id), event_id] })
177
+ end
178
+ end
179
+
180
+ # Apply +stream+'s events after its checkpoint to its row and move the
181
+ # checkpoint to the last of them. The caller holds the stream's lock.
182
+ # Returns how many events there were.
183
+ def project_stream(model_class, stream)
184
+ model_class.with_connection do |connection|
185
+ events = events_to_apply(connection, stream)
186
+ Rebuild.project_events(model_class, events, INTERNAL)
187
+ AsyncCheckpoint.write(connection, stream, events.last.event_id) if events.any? && incremental?(connection)
188
+ events.size
189
+ end
190
+ end
95
191
 
96
- # Checkpoints are exact only while writers take the lock on the
97
- # aggregate, and need a database where it is taken (PostgreSQL) or
98
- # writers are serialised anyway (SQLite).
99
- def incremental?(connection)
100
- Lyra.config.lock_aggregate_before_append && connection.adapter_name.match?(/postgres|sqlite/i)
192
+ private
193
+
194
+ # The events after the checkpoint when it can be trusted; the whole
195
+ # stream otherwise (the checkpoint's event is gone, or none was written).
196
+ def events_to_apply(connection, stream)
197
+ reader = Lyra.config.event_store.read.stream(stream)
198
+ position = incremental?(connection) && AsyncCheckpoint.read(connection, stream)
199
+ return reader.to_a unless position
200
+
201
+ reader.from(position).to_a
202
+ rescue RubyEventStore::EventNotFound
203
+ Lyra.config.event_store.read.stream(stream).to_a
204
+ end
205
+
206
+ # Checkpoints are exact only while writers take the lock on the
207
+ # aggregate, and need a database where it is taken (PostgreSQL) or
208
+ # writers are serialised anyway (SQLite).
209
+ def incremental?(connection)
210
+ Lyra.config.lock_aggregate_before_append && connection.adapter_name.match?(/postgres|sqlite/i)
211
+ end
101
212
  end
102
213
 
214
+ private
215
+
103
216
  # Serialise projection of one stream across workers (Lyra::AdvisoryLock).
104
217
  def lock_stream!(model_class, stream)
105
218
  model_class.with_connection { |c| Lyra::AdvisoryLock.xact_lock(c, stream, purpose: "ES-Async projection") }
106
219
  end
107
220
 
221
+ # Take the record's row (FOR UPDATE), unless another transaction holds
222
+ # it: true then. One statement; a row not there (a create to project)
223
+ # is not taken. PostgreSQL only (SKIP LOCKED); elsewhere false.
224
+ def row_taken?(model_class, model_id)
225
+ model_class.with_connection do |c|
226
+ next false unless c.adapter_name.match?(/postgres/i)
227
+
228
+ from = "FROM #{c.quote_table_name(model_class.table_name)} " \
229
+ "WHERE #{c.quote_column_name(Lyra.key_column(model_class))} = #{c.quote(model_id)}"
230
+ present, held = c.uncached do
231
+ c.select_rows("SELECT (SELECT 1 #{from}), (SELECT 1 #{from} FOR UPDATE SKIP LOCKED)", "Lyra ES-Async row").first
232
+ end
233
+ !present.nil? && held.nil?
234
+ end
235
+ end
236
+
237
+ # Wait, holding no lock of the job's, until the transaction that holds
238
+ # the record's row ends.
239
+ def wait_for_row(model_class, model_id)
240
+ model_class.transaction do
241
+ model_class.with_connection do |c|
242
+ c.uncached do
243
+ c.select_value("SELECT 1 FROM #{c.quote_table_name(model_class.table_name)} " \
244
+ "WHERE #{c.quote_column_name(Lyra.key_column(model_class))} = #{c.quote(model_id)} FOR UPDATE",
245
+ "Lyra ES-Async row wait")
246
+ end
247
+ end
248
+ end
249
+ end
250
+
108
251
  # read.event returns nil for a missing event (read.event! raises): the
109
252
  # warning sat in a rescue of EventNotFound and was never logged.
110
253
  def load_event(event_id)
@@ -114,16 +114,22 @@ module Lyra
114
114
  read_partners(reflection.klass, key_column => keys).group_by { _1.read_attribute(key_column) }
115
115
  end
116
116
 
117
+ # Lyra reads the partners to evaluate the join, as the database reads
118
+ # the joined rows; none is returned to the application. What the query
119
+ # names on the joined table is checked against that model's policy
120
+ # (Lyra::PurposeBoundReads), not the partners themselves.
117
121
  def read_partners(klass, conditions)
118
- if events_only_model?(klass)
119
- EventStoreReader.where(klass, conditions).to_a
120
- else
121
- begin
122
- previous = Thread.current[:lyra_bypass_read_override]
123
- Thread.current[:lyra_bypass_read_override] = true
124
- klass.unscoped.where(conditions).to_a
125
- ensure
126
- Thread.current[:lyra_bypass_read_override] = previous
122
+ Lyra::PurposeBoundReads.internal(INTERNAL) do
123
+ if events_only_model?(klass)
124
+ EventStoreReader.where(klass, conditions).to_a
125
+ else
126
+ begin
127
+ previous = Thread.current[:lyra_bypass_read_override]
128
+ Thread.current[:lyra_bypass_read_override] = true
129
+ klass.unscoped.where(conditions).to_a
130
+ ensure
131
+ Thread.current[:lyra_bypass_read_override] = previous
132
+ end
127
133
  end
128
134
  end
129
135
  end
@@ -140,7 +146,9 @@ module Lyra
140
146
  own = {}
141
147
  joined = {}
142
148
  conditions.each do |key, value|
143
- if value.is_a?(Hash)
149
+ # A Hash on a column of the model is a value (a serialized
150
+ # attribute's), as Active Record reads it (table.has_column?).
151
+ if value.is_a?(Hash) && !model_class.column_names.include?(TypedValues.attribute_name(model_class, key))
144
152
  joined[joined_name_for(key)] = value
145
153
  else
146
154
  own[key] = value
@@ -162,22 +170,29 @@ module Lyra
162
170
  # Whether a row's partners match the joined conditions, as SQL would
163
171
  # decide it: true, false, or nil (unknown, a missing partner under a
164
172
  # condition that is not IS NULL).
173
+ # +joined_conditions+ is what compile_joined made of the conditions on
174
+ # joined tables.
165
175
  def joined_match(joins, joined_conditions)
166
- joined_conditions.reduce(true) do |result, (name, conditions)|
176
+ joined_conditions.reduce(true) do |result, (name, tests, null_partner)|
167
177
  partner = joins[name]
178
+ outcome = partner.nil? ? null_partner : matches_all?(partner, tests)
179
+ combine_and(result, outcome)
180
+ end
181
+ end
182
+
183
+ # The conditions on joined tables, compiled once per query: for each
184
+ # association, its tests on the partner, and the outcome for a missing
185
+ # partner (true when every condition is IS NULL, else unknown).
186
+ def compile_joined(joined_conditions)
187
+ joined_conditions.map do |name, conditions|
168
188
  klass = joined_associations[name].reflection.klass
169
189
  rewritten = AssociationConditions.rewrite(klass, conditions)
170
190
  if rewritten.values.any? { _1.is_a?(Hash) }
171
191
  raise UnsupportedQuery.new(model_class, "conditions nested more than one level on #{name}")
172
192
  end
173
193
 
174
- outcome =
175
- if partner.nil?
176
- rewritten.values.all? { |v| v.nil? || (v.is_a?(Array) && v.include?(nil)) } ? true : nil
177
- else
178
- rewritten.all? { |column, v| matches_value?(partner, column, v) }
179
- end
180
- combine_and(result, outcome)
194
+ null_partner = rewritten.values.all? { |v| v.nil? || (v.is_a?(Array) && v.include?(nil)) } ? true : nil
195
+ [name, condition_tests(klass, rewritten), null_partner]
181
196
  end
182
197
  end
183
198
 
@@ -45,7 +45,9 @@ module Lyra
45
45
  #
46
46
  class CachedProjection
47
47
  # 2: entries are { "v" => last event id, "a" => attributes or nil }.
48
- CACHE_VERSION = 2
48
+ # 3: the attributes as the table holds them (an enum's value, not its
49
+ # label; a serialized attribute's text), not as the events give them.
50
+ CACHE_VERSION = 3
49
51
  DEFAULT_EXPIRES_IN = 1.hour
50
52
  MEMORY_CACHE_BYTES = 32.megabytes
51
53
 
@@ -91,14 +93,16 @@ module Lyra
91
93
  rest = attributes.reject { |key, _| key.to_s == primary_key }
92
94
  return record if rest.empty?
93
95
 
94
- return matches_attributes?(record, rest) ? record : nil
96
+ return matches_attributes?(TypedValues::EventReader.new(model_class), record, conditions_for(model_class, rest)) ? record : nil
95
97
  end
96
98
 
97
99
  # Without a primary key there is no way to know which stream holds
98
100
  # the answer, so every record of the model is considered. That cost
99
101
  # is inherent to querying by a non-key attribute with projections
100
102
  # disabled; the entries keep it to one bulk read once warm.
101
- all(model_class).find { |record| matches_attributes?(record, attributes) }
103
+ tests = conditions_for(model_class, attributes)
104
+ reader = TypedValues::EventReader.new(model_class)
105
+ all(model_class).find { |record| matches_attributes?(reader, record, tests) }
102
106
  end
103
107
 
104
108
  # Every live record of the model, in primary-key order
@@ -132,7 +136,9 @@ module Lyra
132
136
  # @return [Array<Hash>] Matching record attributes
133
137
  def where(model_class, conditions)
134
138
  candidates = ReferenceLinks.lookup(model_class, conditions) || all(model_class)
135
- candidates.select { |record| matches_conditions?(record, conditions) }
139
+ tests = conditions_for(model_class, conditions)
140
+ reader = TypedValues::EventReader.new(model_class)
141
+ candidates.select { |record| matches_attributes?(reader, record, tests) }
136
142
  end
137
143
 
138
144
  # Count records
@@ -184,6 +190,14 @@ module Lyra
184
190
  find(model_class, id)
185
191
  end
186
192
 
193
+ # For a reader that keeps its own copy of a model's records, kept
194
+ # current by the same stamps (Uniqueness::Snapshot): every stream's
195
+ # last event, the entries at given versions, and whether a record's
196
+ # attributes belong to +model_class+ (STI).
197
+ def stream_versions_for(model_class) = stream_versions(model_class)
198
+ def entries_for(model_class, versions) = entries_at(model_class, versions)
199
+ def of_class?(model_class, attributes) = !of_class(model_class, attributes).nil?
200
+
187
201
  private
188
202
 
189
203
  def fresh?(entry, version)
@@ -218,12 +232,20 @@ module Lyra
218
232
  # The state reached by replaying the events after +entry+'s stamp onto
219
233
  # its attributes, as [attributes or nil, version]; nil when that cannot
220
234
  # be done exactly (no usable entry, a stamp that is no longer in the
221
- # stream, nothing after it), and the caller replays the whole stream.
235
+ # stream, nothing after it, an erasure after it), and the caller
236
+ # replays the whole stream.
237
+ #
238
+ # An erasure rewrites the stream's events in place (Lyra::Erasure)
239
+ # and appends an ErasureApplied: the events before it no longer hold
240
+ # what the entry was built from. Replaying only the events after the
241
+ # stamp kept the erased values, in a process other than the eraser's
242
+ # (whose own entries the erasure drops).
222
243
  def build_from_entry(model_class, id, stream, entry)
223
244
  return nil unless entry.is_a?(Hash) && entry["v"] && entry["a"].is_a?(Hash)
224
245
 
225
246
  events = load_events_after(stream, entry["v"])
226
247
  return nil if events.nil? || events.empty?
248
+ return nil if events.any? { rewrites_stream?(_1) }
227
249
 
228
250
  version = events.last.event_id
229
251
  replayed = events.select { Lyra::Event.operation_of(_1) }
@@ -233,6 +255,10 @@ module Lyra
233
255
  [reconstruct_state(model_class, id, replayed, entry["a"]), version]
234
256
  end
235
257
 
258
+ def rewrites_stream?(event)
259
+ defined?(Lyra::Events::ErasureApplied) && event.is_a?(Lyra::Events::ErasureApplied)
260
+ end
261
+
236
262
  # The id of the last event in a record's stream, or nil if it has none.
237
263
  def stream_version(model_class, id)
238
264
  ActiveRecord::Base.with_connection do |connection|
@@ -247,23 +273,33 @@ module Lyra
247
273
  # id }): one bulk cache read, and a replay of only the streams whose
248
274
  # entry is missing or out of date.
249
275
  def records_at(model_class, versions)
250
- return [] if versions.empty?
276
+ entries_at(model_class, versions).filter_map { |_, (_, attributes)| attributes }
277
+ .select { |attributes| of_class(model_class, attributes) }
278
+ end
279
+
280
+ # { id => [the id of the last event the attributes were built from,
281
+ # attributes or nil (destroyed)] } for the streams whose last events
282
+ # are +versions+, in their order, whatever their class: one bulk
283
+ # cache read, and a replay of the streams whose entry is missing or
284
+ # out of date.
285
+ def entries_at(model_class, versions)
286
+ return {} if versions.empty?
251
287
 
252
288
  keys = versions.keys.to_h { |id| [record_cache_key(model_class, id), id] }
253
289
  cached = cache_read_multi(keys.keys)
254
290
  rebuilt = {}
255
291
 
256
- records = keys.filter_map do |key, id|
292
+ entries = keys.to_h do |key, id|
257
293
  entry = cached[key]
258
- next entry["a"] if fresh?(entry, versions[id])
294
+ next [id, [versions[id], entry["a"]]] if fresh?(entry, versions[id])
259
295
 
260
296
  attributes, built_version = build_record(model_class, id, entry)
261
297
  rebuilt[key] = entry_for(attributes, built_version) if built_version
262
- attributes
298
+ [id, [built_version, attributes]]
263
299
  end
264
300
 
265
301
  cache_write_multi(rebuilt) if rebuilt.any?
266
- records.select { |attributes| of_class(model_class, attributes) }
302
+ entries
267
303
  end
268
304
 
269
305
  # +attributes+ if the record belongs to +model_class+, nil otherwise.
@@ -311,50 +347,55 @@ module Lyra
311
347
  end
312
348
  end
313
349
 
314
- # Shared by find_by and where so both decide a match the same way.
315
- def matches_attributes?(record, attributes)
316
- attributes.all? do |key, value|
317
- record[key.to_s] == value || record[key.to_sym] == value
318
- end
350
+ # Shared by find_by and where so both decide a match the same way, and
351
+ # as a CachedRelation of the built records decides it: each value
352
+ # read as the record built from +record+ (the event attributes) holds
353
+ # it (+reader+, a TypedValues::EventReader), and compared with the
354
+ # condition's value bound as Active Record binds it (TypedValues).
355
+ # The event's "250.0" against find_by(amount: 250.0) was a String
356
+ # against a Float, and missed.
357
+ def matches_attributes?(reader, record, tests)
358
+ true.equal?(TypedValues.all_hold(tests, reader, record))
319
359
  end
320
360
 
321
- def matches_conditions?(record, conditions)
322
- conditions.all? do |key, value|
323
- record_value = record[key.to_s] || record[key.to_sym]
324
-
325
- case value
326
- when Array
327
- value.include?(record_value)
328
- when Range
329
- value.cover?(record_value)
330
- else
331
- record_value == value
332
- end
333
- end
361
+ # The conditions of a hash, compiled once for the records they are
362
+ # tested on (TypedValues.condition).
363
+ def conditions_for(model_class, conditions)
364
+ conditions.map { |key, value| TypedValues.condition(model_class, key, value) }
334
365
  end
335
366
 
336
367
  # Reconstruct record state from events
337
368
  # +base+: the attributes the +events+ are applied onto (the state at
338
369
  # an earlier event), instead of an empty record.
339
370
  def reconstruct_state(model_class, id, events, base = nil)
340
- attributes = base ? base.dup : { "id" => id }
371
+ key = Lyra.key_column(model_class)
372
+ attributes = base ? base.dup : { key => id }
373
+ # The attributes the events set (all of them, from a create).
374
+ set = base ? Set.new : nil
341
375
 
342
376
  events.each do |event|
343
377
  case Lyra::Event.operation_of(event)
344
378
  when :created, :imported
345
- event_attrs = event.data[:attributes] || event.data["attributes"] || {}
346
- attributes.merge!(stringify_keys(event_attrs))
379
+ event_attrs = stringify_keys(Lyra::Event.created_attributes(event.data))
380
+ attributes.merge!(event_attrs)
381
+ set&.merge(event_attrs.keys)
347
382
  when :updated
348
383
  changes = event.data[:changes] || event.data["changes"] || {}
349
384
  changes.each do |field, change|
350
385
  new_value = change.is_a?(Array) ? change.last : change
351
386
  attributes[field.to_s] = new_value
387
+ set&.add(field.to_s)
352
388
  end
353
389
  end
354
390
  end
355
391
 
356
- # Convert id to integer if needed
357
- attributes["id"] = id.to_i if id.to_s =~ /^\d+$/
392
+ # The key as the row holds it (an integer key from the stream name's
393
+ # text), under the primary key column
394
+ attributes[key] = Lyra.key_value(model_class, id)
395
+ # Entries hold the attributes as the table does (an enum's value,
396
+ # a serialized attribute's text), converted once here rather than
397
+ # on every read (TypedValues.convert_event_values!).
398
+ TypedValues.convert_event_values!(model_class, attributes, set)
358
399
  attributes
359
400
  end
360
401