orfeas_lyra 0.9.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +2021 -17
  3. data/README.md +22 -9
  4. data/Rakefile +40 -1
  5. data/app/controllers/lyra/application_controller.rb +127 -0
  6. data/app/controllers/lyra/dashboard_controller.rb +33 -31
  7. data/app/controllers/lyra/flow_controller.rb +32 -34
  8. data/app/controllers/lyra/privacy_controller.rb +57 -18
  9. data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
  10. data/app/views/lyra/dashboard/index.html.erb +1 -0
  11. data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
  12. data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
  13. data/app/views/lyra/privacy/policy.html.erb +7 -0
  14. data/app/views/lyra/privacy/rights.html.erb +78 -0
  15. data/config/privacy_policies.rb +0 -3
  16. data/config/routes.rb +1 -0
  17. data/lib/lyra/access_log.rb +187 -6
  18. data/lib/lyra/advisory_lock.rb +236 -0
  19. data/lib/lyra/aggregate.rb +1 -1
  20. data/lib/lyra/associations/habtm_links.rb +382 -19
  21. data/lib/lyra/associations/maintenance.rb +229 -0
  22. data/lib/lyra/attribution.rb +48 -0
  23. data/lib/lyra/bypass_events.rb +446 -46
  24. data/lib/lyra/command_handler.rb +19 -14
  25. data/lib/lyra/configuration.rb +237 -10
  26. data/lib/lyra/consistency/read_your_writes.rb +4 -2
  27. data/lib/lyra/correlation.rb +121 -0
  28. data/lib/lyra/domain_events.rb +4 -0
  29. data/lib/lyra/dual_view.rb +53 -16
  30. data/lib/lyra/encrypted_attributes.rb +5 -0
  31. data/lib/lyra/engine.rb +67 -0
  32. data/lib/lyra/erasure.rb +1011 -106
  33. data/lib/lyra/event.rb +35 -1
  34. data/lib/lyra/event_data.rb +38 -0
  35. data/lib/lyra/event_flow.rb +31 -15
  36. data/lib/lyra/event_store_adapter.rb +28 -0
  37. data/lib/lyra/event_window.rb +58 -0
  38. data/lib/lyra/genesis.rb +548 -44
  39. data/lib/lyra/interceptors/association_interceptor.rb +689 -34
  40. data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
  41. data/lib/lyra/internal.rb +26 -0
  42. data/lib/lyra/mode_sync.rb +51 -3
  43. data/lib/lyra/mode_transition.rb +257 -30
  44. data/lib/lyra/model_selection.rb +35 -0
  45. data/lib/lyra/own_connection.rb +17 -2
  46. data/lib/lyra/privacy/adapters/pam.rb +25 -2
  47. data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
  48. data/lib/lyra/privacy/interface.rb +27 -7
  49. data/lib/lyra/privacy/subject_links.rb +143 -0
  50. data/lib/lyra/privacy/subject_records.rb +369 -0
  51. data/lib/lyra/projection.rb +7 -1
  52. data/lib/lyra/projections/async_checkpoint.rb +32 -0
  53. data/lib/lyra/projections/async_projection_job.rb +166 -23
  54. data/lib/lyra/projections/cached_joins.rb +33 -18
  55. data/lib/lyra/projections/cached_projection.rb +74 -33
  56. data/lib/lyra/projections/cached_relation.rb +3175 -321
  57. data/lib/lyra/projections/default_scopes.rb +254 -0
  58. data/lib/lyra/projections/deferred_relation.rb +23 -3
  59. data/lib/lyra/projections/event_store_reader.rb +47 -8
  60. data/lib/lyra/projections/foreign_keys.rb +92 -0
  61. data/lib/lyra/projections/lazy_projection.rb +165 -27
  62. data/lib/lyra/projections/model_projection.rb +165 -14
  63. data/lib/lyra/projections/rebuild.rb +34 -6
  64. data/lib/lyra/projections/reference_links.rb +463 -72
  65. data/lib/lyra/projections/typed_values.rb +629 -0
  66. data/lib/lyra/projections/unique_indexes.rb +595 -0
  67. data/lib/lyra/projections/uniqueness.rb +784 -0
  68. data/lib/lyra/projections/value_digest.rb +93 -0
  69. data/lib/lyra/purpose_bound_reads.rb +912 -77
  70. data/lib/lyra/repair.rb +119 -28
  71. data/lib/lyra/retention.rb +114 -8
  72. data/lib/lyra/rights/consent_store.rb +415 -0
  73. data/lib/lyra/rights/erasure_guard.rb +135 -0
  74. data/lib/lyra/rights/pending.rb +74 -0
  75. data/lib/lyra/rights/queries.rb +269 -0
  76. data/lib/lyra/rights/store.rb +512 -0
  77. data/lib/lyra/rights/subject_keys.rb +224 -0
  78. data/lib/lyra/rights.rb +281 -0
  79. data/lib/lyra/saves_without_effect.rb +69 -0
  80. data/lib/lyra/stream_tail.rb +319 -0
  81. data/lib/lyra/strict_data_access.rb +916 -106
  82. data/lib/lyra/temporal.rb +4 -2
  83. data/lib/lyra/touch_registry.rb +404 -0
  84. data/lib/lyra/verification/bypass_workflow.rb +452 -42
  85. data/lib/lyra/verification/rights_workflows.rb +170 -0
  86. data/lib/lyra/version.rb +1 -1
  87. data/lib/lyra/visualization/event_graph.rb +1 -1
  88. data/lib/lyra/visualization/timeline.rb +2 -2
  89. data/lib/lyra.rb +43 -2
  90. data/lib/tasks/lyra_mode.rake +110 -8
  91. data/lib/tasks/lyra_projections.rake +43 -2
  92. metadata +34 -4
  93. data/lib/lyra/associations/event_aware.rb +0 -224
@@ -0,0 +1,784 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "objspace"
4
+
5
+ module Lyra
6
+ module Projections
7
+ # Rails' uniqueness validation (validates :x, uniqueness: true) in
8
+ # ES-NoProj, where a monitored model's table stays empty and its records
9
+ # are rebuilt from events.
10
+ #
11
+ # The validator builds an Active Record relation (Model.unscoped, the
12
+ # value compared as the connection compares it, the scope: columns, the
13
+ # conditions: lambda, and on update where.not(primary key: the record's))
14
+ # and asks it exists?. That query read the empty table, so every value
15
+ # passed and duplicates were saved; the table's unique index could not
16
+ # catch them, since no row is written.
17
+ #
18
+ # The relation is still built by Rails, unchanged, so that scope,
19
+ # conditions and case sensitivity follow Rails' own construction. Only
20
+ # its exists? is answered here, from the event-sourced records: its where
21
+ # clause is evaluated on every record of the model, each predicate as the
22
+ # database evaluates it (TypedValues: the value bound through the
23
+ # attribute's type, SQL's three-valued logic). What cannot be evaluated
24
+ # exactly raises UnsupportedQuery instead of passing or refusing a value
25
+ # on a guess: an SQL fragment or a subquery in conditions:, a join, a
26
+ # comparison with another column, LIKE, a string column with a collation
27
+ # of its own, an adapter other than PostgreSQL or SQLite for a string
28
+ # comparison. LOWER(column) = LOWER(value) (case_sensitive: false) and a
29
+ # citext column's equality are evaluated by the database's own LOWER, on
30
+ # the values the records hold (Predicates#lower_all).
31
+ #
32
+ # The check reads the records as Rails reads the table: through
33
+ # Model.unscoped, so a default scope (a soft delete) is not applied, and
34
+ # with the type condition of the class Rails chooses (under STI, the one
35
+ # that declared the validation). Other modes are untouched: the relation
36
+ # is extended only when the model reads from events.
37
+ #
38
+ # A check is a read and then a write, and two writers of one value can
39
+ # both check before either commits. In the table modes a unique index
40
+ # refuses the second INSERT. ES-NoProj writes no row, so the check takes
41
+ # a transaction-level advisory lock on (model, attribute, value) before it
42
+ # reads (lock!): the second writer's check waits until the first's
43
+ # transaction ends, and then reads its events. See lock! for its limits.
44
+ #
45
+ # The purpose check exemption (PurposeBoundReads::UniquenessExemption)
46
+ # still applies: the validation is part of writing the record.
47
+ module Uniqueness
48
+ LOCK = "lyra_unique"
49
+
50
+ class << self
51
+ # Whether +klass+'s reads are answered from events (ES-NoProj), as
52
+ # the model's read overrides decide it, Lyra's own table reads aside:
53
+ # whoever builds the relation, the table has no rows to answer from.
54
+ def reads_events?(klass)
55
+ klass.respond_to?(:lyra_monitored) && klass.lyra_monitored &&
56
+ Lyra.event_sourcing_mode? && Lyra.config.projection_mode == :disabled
57
+ end
58
+
59
+ # LOWER of texts already sent to a database, per database, in this
60
+ # process (Predicates#lower_all): LOWER depends only on the text and
61
+ # the database's character type.
62
+ LOWER_MEMO_LIMIT = 200_000
63
+
64
+ def lower_memo(connection)
65
+ config = connection.pool.db_config
66
+ key = [config.env_name, config.name, config.database]
67
+ (@lower_memos ||= Concurrent::Map.new).compute_if_absent(key) { Concurrent::Map.new }
68
+ end
69
+
70
+ def remember_lower(memo, text, lowered)
71
+ memo.clear if memo.size >= LOWER_MEMO_LIMIT
72
+ memo[text] = lowered
73
+ end
74
+
75
+ # The current scope of +klass+ when it is a uniqueness check's
76
+ # relation: a named scope called in conditions: (-> { live }) runs
77
+ # under it (Relation#scoping), and Model.all must return it, as
78
+ # Active Record's all does, for the scope to build on it.
79
+ def scoped?(klass)
80
+ scope = klass.current_scope
81
+ scope.is_a?(Relation) && scope.model == klass
82
+ end
83
+
84
+ # Whether a record matches +relation+'s where clause: its exists?.
85
+ def exists?(relation)
86
+ klass = relation.model
87
+ refuse_unsupported!(relation)
88
+ return false if relation.limit_value&.zero?
89
+
90
+ predicates = Predicates.new(klass)
91
+ begin
92
+ test = predicates.compile_all(relation.where_clause.send(:predicates))
93
+ rescue Predicates::LiteralOutOfRange
94
+ # Active Record cannot quote the literal; select_all rescues the
95
+ # RangeError and answers no row: the value is not taken.
96
+ return false
97
+ end
98
+ Genesis.first_use(klass)
99
+ records = Snapshot.records(klass)
100
+ predicates.prepare(records)
101
+ records.any? { |attributes| true.equal?(test.call(attributes)) }
102
+ end
103
+
104
+ # A transaction-level advisory lock on (model, attribute, value),
105
+ # taken before the check reads, held until the writer's transaction
106
+ # ends: a second writer of the value waits in its check until the
107
+ # first commits or rolls back, and then reads the first's events.
108
+ #
109
+ # The value is the one the check binds, as the column holds it:
110
+ # the attribute's type serializes it ("250" and 250.00 on a decimal
111
+ # are one value, a time is taken in UTC, -0.0 is 0.0, a character(n)
112
+ # value without its trailing spaces), under the column an
113
+ # alias_attribute names. Under case_sensitive: false on a text
114
+ # column, and on a citext column, the key is made from LOWER(value)
115
+ # as the database computes it, as the check compares: values that
116
+ # differ only in case take one lock. The key holds a keyed digest of
117
+ # the value (value_lock_key), never the value: lock statements reach
118
+ # the database's log. The scope: and conditions: are
119
+ # left out of the key, which makes it coarser (writers of one value
120
+ # in two scopes wait for each other), never finer; keys are hashed
121
+ # to 32 bits (hashtext), so two values may rarely share one.
122
+ #
123
+ # Not taken for a persisted record whose attribute and scope columns
124
+ # did not change, with no conditions: (ValidatorPatch#lyra_claims?):
125
+ # it holds a value committed before (or written earlier in its own
126
+ # transaction, under the lock that write took), and claims nothing a
127
+ # second writer could also claim. Rails still runs its check whenever it would
128
+ # (validation_needed?, which runs it for every save under
129
+ # case_sensitive:), and it is still answered.
130
+ #
131
+ # Limits. PostgreSQL only (Lyra::AdvisoryLock; SQLite serialises
132
+ # writers). Taken only inside a transaction (save, create, update
133
+ # run their validations in one); valid? alone, then save(validate:
134
+ # false), is a race in every mode. Under REPEATABLE READ or
135
+ # SERIALIZABLE a transaction whose snapshot was taken before the
136
+ # first writer committed does not see its events after the wait
137
+ # (PostgreSQL's default, READ COMMITTED, takes a new snapshot per
138
+ # statement). Two transactions that each claim the same two values
139
+ # in opposite orders deadlock, as two inserts of those values into a
140
+ # unique index do, and PostgreSQL fails one of them
141
+ # (ActiveRecord::Deadlocked); a lock key coarser than the index (a
142
+ # scope or conditions left out, a hash shared) can make two writers
143
+ # wait, or deadlock, where an index on the scoped columns would not.
144
+ def lock!(klass, attribute, value, case_insensitive:)
145
+ klass.with_connection do |connection|
146
+ next unless connection.transaction_open?
147
+
148
+ key = value_lock_key(klass, attribute, value, fold: case_insensitive)
149
+ Lyra::AdvisoryLock.xact_lock(connection, key, purpose: "uniqueness check")
150
+ end
151
+ end
152
+
153
+ # The locks lock! takes for each [attribute, value] of +pairs+, as a
154
+ # case-sensitive check takes them (a citext column's folded, as the
155
+ # column compares), all or none in one statement
156
+ # (Lyra::AdvisoryLock.xact_lock_all). For the writes checked against
157
+ # the unique indexes from the event store (UniqueIndexes, and
158
+ # StrictDataAccessRelation#events_only_unique_rows): they wait for, and
159
+ # are waited for by, a uniqueness validation of the same value.
160
+ def lock_all!(klass, pairs)
161
+ klass.with_connection do |connection|
162
+ next unless connection.transaction_open?
163
+
164
+ keys = pairs.uniq.map { |attribute, value| value_lock_key(klass, attribute, value, fold: false) }
165
+ Lyra::AdvisoryLock.xact_lock_all(connection, keys, purpose: "unique index check")
166
+ end
167
+ end
168
+
169
+ # The advisory lock key of +value+ in +attribute+ (lock!, lock_all!):
170
+ # "lyra_unique/<model>/<column>/" and a keyed digest (ValueDigest) of
171
+ # the value as the column holds it (canonical), or, folded (+fold+, or
172
+ # a citext column), of its LOWER as the database computes it; never the
173
+ # value itself, which would reach the statement log. NULL is "#null".
174
+ def value_lock_key(klass, attribute, value, fold:)
175
+ name = klass.attribute_aliases.fetch(attribute.to_s, attribute.to_s)
176
+ prefix = "#{LOCK}/#{Lyra.stream_owner_name(klass)}/#{name}"
177
+ text = lowered_key_text(klass, name, value) if fold || citext?(klass, name)
178
+ return "#{prefix}/#{ValueDigest.hex("#{prefix}/#{lower_texts(klass, [text], name: name).first}")}" if text
179
+
180
+ bound = canonical(klass, name, value)
181
+ bound == "#null" ? "#{prefix}#null" : "#{prefix}/#{ValueDigest.hex("#{prefix}#{bound}")}"
182
+ end
183
+
184
+ # LOWER of each of +texts+ as +klass+'s database computes it, those
185
+ # lowered before (per database, in this process) not sent again; the
186
+ # rest in one query, as bind parameters named +name+ (a column's: the
187
+ # statement log shows them as it shows that column's, filtered where
188
+ # the application filters it).
189
+ def lower_texts(klass, texts, name: "value")
190
+ klass.with_connection do |connection|
191
+ memo = lower_memo(connection)
192
+ known = {}
193
+ texts.each { |text| (lowered = memo[text]) ? known[text] = lowered : nil }
194
+ missing = texts.reject { known.key?(_1) }.uniq
195
+ if missing.any?
196
+ lower_query(connection, missing, name.to_s).each_with_index do |lowered, i|
197
+ known[missing[i]] = lowered
198
+ remember_lower(memo, missing[i], lowered)
199
+ end
200
+ end
201
+ texts.map { known.fetch(_1) }
202
+ end
203
+ end
204
+
205
+ # "/" and the value as the column holds it; "#null" for NULL: the
206
+ # lock key's value, and the value a unique index compares in ES-NoProj
207
+ # (Lyra::Projections::UniqueIndexes).
208
+ def canonical(klass, name, value)
209
+ type = klass.type_for_attribute(name)
210
+ value = TypedValues.normalize(value)
211
+ bound = value.nil? || !type.serializable?(value) ? nil : type.serialize(value)
212
+ return "#null" if bound.nil?
213
+
214
+ bound = TypedValues.unpadded(bound) if TypedValues.schema(klass).blank_padded.include?(name)
215
+ bound = bound.utc if bound.is_a?(Time)
216
+ bound = 0.0 if bound.is_a?(Float) && bound.zero?
217
+ bound = bound.strftime("%Y-%m-%d %H:%M:%S.%N") if bound.is_a?(Time)
218
+ "/#{bound}"
219
+ rescue StandardError
220
+ "/#{value.inspect}"
221
+ end
222
+
223
+ # A citext column, wherever its type lives (public.citext, a schema
224
+ # outside the search path).
225
+ def citext?(klass, name)
226
+ column = klass.columns_hash[name]
227
+ return false unless column
228
+
229
+ column.type == :citext || column.sql_type.to_s.match?(/(\A|\.)"?citext"?\z/)
230
+ end
231
+
232
+ # The text LOWER is applied to in a lock's key or a citext comparison, for a value of a
233
+ # text column (nil otherwise, or for NULL: the canonical key then).
234
+ def lowered_key_text(klass, name, value)
235
+ column = klass.columns_hash[name]
236
+ return nil unless column && %i[string text citext].include?(column.type)
237
+
238
+ value = TypedValues.normalize(value)
239
+ type = klass.type_for_attribute(name)
240
+ return nil if value.nil? || !type.serializable?(value)
241
+
242
+ text = type.serialize(value)&.to_s
243
+ TypedValues.schema(klass).blank_padded.include?(name) ? TypedValues.unpadded(text) : text
244
+ end
245
+
246
+ private
247
+
248
+ # LOWER of +texts+ in one query (lower_texts), the texts bound.
249
+ def lower_query(connection, texts, name)
250
+ type = ActiveModel::Type::String.new
251
+ if connection.adapter_name.match?(/postgres/i)
252
+ array = PG::TextEncoder::Array.new.encode(texts)
253
+ connection.select_values(
254
+ "SELECT LOWER(t.x) FROM unnest($1::text[]) WITH ORDINALITY AS t(x, i) ORDER BY t.i",
255
+ "Lyra uniqueness LOWER", [ActiveRecord::Relation::QueryAttribute.new(name, array, type)]
256
+ )
257
+ else
258
+ texts.each_slice(500).flat_map do |slice|
259
+ binds = slice.map { ActiveRecord::Relation::QueryAttribute.new(name, _1, type) }
260
+ connection.select_rows("SELECT #{slice.map { 'LOWER(?)' }.join(', ')}", "Lyra uniqueness LOWER", binds).first
261
+ end
262
+ end
263
+ end
264
+
265
+ def refuse_unsupported!(relation)
266
+ used = {
267
+ "joins" => relation.joins_values.any? || relation.left_outer_joins_values.any?,
268
+ "eager loading" => relation.eager_loading?,
269
+ "group" => relation.group_values.any?,
270
+ "having" => !relation.having_clause.empty?,
271
+ "offset" => !relation.offset_value.nil?,
272
+ "from" => !relation.from_clause.empty?
273
+ }.select { |_, present| present }.keys
274
+ return if used.empty?
275
+
276
+ raise UnsupportedQuery.new(relation.model, "a uniqueness check (it uses #{used.join(', ')})")
277
+ end
278
+ end
279
+
280
+ # Every record of a model, as CachedProjection.all reads them, for the
281
+ # checks: each stream's last event is read (one query), and a copy of
282
+ # the records, stamped as the projection cache's entries are, is kept
283
+ # in this process. Only the streams whose last event is not the stamp
284
+ # are read from the cache or rebuilt (CachedProjection.entries_for);
285
+ # the others are used as they are. CachedProjection.all reads every
286
+ # entry from the cache, which copies each (Marshal) on every read: at
287
+ # 3,000 records that was 55 of a check's 94 ms.
288
+ #
289
+ # Each check reads every entry of the copy once, into a hash of its
290
+ # own, decides from it which are stale, and answers from it alone:
291
+ # another thread replacing an entry meanwhile cannot change what this
292
+ # check sees. An entry is used only while its stamp is its stream's
293
+ # last event as this transaction sees it. The copy can hold an entry
294
+ # built inside a transaction from that transaction's own uncommitted
295
+ # events (the projection cache's shared entries can too): it is
296
+ # stamped with an event no other transaction sees until it commits,
297
+ # and one that rolls back is never any stream's last event, so no
298
+ # other check uses it; a check after the commit uses it, and then it
299
+ # is committed state.
300
+ #
301
+ # Bounded: a model with more than LIMIT streams is read through
302
+ # CachedProjection.all, and the copies of all models together are kept
303
+ # under BYTES (as ObjectSpace measures their records), beside the
304
+ # projection cache's own MEMORY_CACHE_BYTES: past it the copies used
305
+ # least recently are dropped. A copy is dropped and rebuilt after
306
+ # EXPIRES_IN, as the projection cache's memory entries expire, and
307
+ # when its model's attribute types change (reset_column_information,
308
+ # a migration) or the projection cache's CACHE_VERSION does.
309
+ module Snapshot
310
+ LIMIT = 20_000
311
+ BYTES = 32.megabytes
312
+ EXPIRES_IN = CachedProjection::DEFAULT_EXPIRES_IN
313
+
314
+ # One model's copy: { id => [stamp, frozen attributes or nil] }.
315
+ Copy = Struct.new(:entries, :types, :version, :born, :used, :bytes)
316
+
317
+ @copies = {} # stream owner (a class) => Copy
318
+ @lock = Mutex.new
319
+
320
+ class << self
321
+ def records(klass)
322
+ versions = CachedProjection.stream_versions_for(klass)
323
+ return CachedProjection.all(klass) if versions.size > LIMIT
324
+
325
+ copy = copy_for(klass)
326
+ local = versions.each_key.to_h { |id| [id, copy.entries[id]] }
327
+ stale = versions.reject { |id, version| local[id]&.first == version }
328
+ if stale.any?
329
+ CachedProjection.entries_for(klass, stale).each do |id, (version, attributes)|
330
+ local[id] = [version, frozen(attributes)].freeze
331
+ end
332
+ keep(copy, local, stale.keys)
333
+ end
334
+ prune(copy, versions)
335
+ local.each_value.filter_map do |entry|
336
+ attributes = entry&.last
337
+ attributes if attributes && CachedProjection.of_class?(klass, attributes)
338
+ end
339
+ end
340
+
341
+ # Forget every copy (tests).
342
+ def reset!
343
+ @lock.synchronize { @copies = {} }
344
+ end
345
+
346
+ # The bytes the copies hold, as counted (tests, monitoring).
347
+ def bytes = @lock.synchronize { @copies.each_value.sum(&:bytes) }
348
+
349
+ private
350
+
351
+ # One copy per table: the stream owner (an STI base class). A new
352
+ # one when it has expired or its model's types have changed.
353
+ def copy_for(klass)
354
+ owner = klass.respond_to?(:base_class) ? klass.base_class : klass
355
+ now = monotonic
356
+ @lock.synchronize do
357
+ copy = @copies[owner]
358
+ if copy.nil? || !copy.types.equal?(owner.attribute_types) || copy.version != CachedProjection::CACHE_VERSION ||
359
+ now - copy.born > EXPIRES_IN.to_f
360
+ copy = @copies[owner] = Copy.new(Concurrent::Map.new, owner.attribute_types, CachedProjection::CACHE_VERSION, now, now, 0)
361
+ end
362
+ copy.used = now
363
+ copy
364
+ end
365
+ end
366
+
367
+ # The entries this check read again, into the shared copy, within
368
+ # the byte budget.
369
+ def keep(copy, local, ids)
370
+ @lock.synchronize do
371
+ ids.each do |id|
372
+ entry = local[id]
373
+ previous = copy.entries[id]
374
+ copy.bytes += size_of(entry) - (previous ? size_of(previous) : 0)
375
+ copy.entries[id] = entry
376
+ end
377
+ enforce_budget(copy)
378
+ end
379
+ end
380
+
381
+ # Past BYTES, drop the copies used least recently, this one last.
382
+ def enforce_budget(current)
383
+ total = @copies.each_value.sum(&:bytes)
384
+ return if total <= BYTES
385
+
386
+ (@copies.reject { |_, copy| copy.equal?(current) }.sort_by { |_, copy| copy.used } +
387
+ @copies.select { |_, copy| copy.equal?(current) }.to_a).each do |owner, copy|
388
+ break if total <= BYTES
389
+
390
+ total -= copy.bytes
391
+ @copies.delete(owner)
392
+ end
393
+ end
394
+
395
+ # Streams that are gone (an erasure that rewrote them) are dropped
396
+ # once the copy has grown well past the model's streams.
397
+ def prune(copy, versions)
398
+ return unless copy.entries.size > (versions.size * 2) + 100
399
+
400
+ @lock.synchronize do
401
+ copy.entries.each_key do |id|
402
+ next if versions.key?(id)
403
+
404
+ removed = copy.entries.delete(id)
405
+ copy.bytes -= size_of(removed) if removed
406
+ end
407
+ end
408
+ end
409
+
410
+ def frozen(attributes)
411
+ return nil unless attributes
412
+
413
+ attributes.each_value { |value| value.freeze if value.is_a?(String) }
414
+ attributes.freeze
415
+ end
416
+
417
+ # The memory an entry holds: its hash and its strings.
418
+ def size_of(entry)
419
+ attributes = entry&.last
420
+ return 40 unless attributes
421
+
422
+ attributes.sum(ObjectSpace.memsize_of(attributes)) do |key, value|
423
+ ObjectSpace.memsize_of(key) + (value.is_a?(String) ? ObjectSpace.memsize_of(value) : 0)
424
+ end
425
+ end
426
+
427
+ def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
428
+ end
429
+ end
430
+
431
+ # Extends the relation a uniqueness check builds (ValidatorPatch): its
432
+ # exists? is answered from the event-sourced records. The relations
433
+ # the validator derives from it (where, where.not, the conditions)
434
+ # are clones and keep the extension.
435
+ module Relation
436
+ def exists?(conditions = :none)
437
+ return super unless conditions == :none && !null_relation? && Uniqueness.reads_events?(model)
438
+
439
+ Uniqueness.exists?(self)
440
+ end
441
+ end
442
+
443
+ # Prepended to ActiveRecord::Validations::UniquenessValidator: the
444
+ # relation Rails builds for the check is extended (Relation), and the
445
+ # advisory lock taken, when the model reads from events. In other modes
446
+ # the relation is Rails' own.
447
+ module ValidatorPatch
448
+ RECORD = :lyra_uniqueness_record
449
+
450
+ # The record being validated, for build_relation (which Rails does
451
+ # not pass it): whether its save claims a value (lyra_claims?).
452
+ def validate_each(record, attribute, value)
453
+ previous = Thread.current[RECORD]
454
+ Thread.current[RECORD] = record
455
+ super
456
+ ensure
457
+ Thread.current[RECORD] = previous
458
+ end
459
+
460
+ private
461
+
462
+ def build_relation(klass, attribute, value)
463
+ relation = super
464
+ return relation unless Uniqueness.reads_events?(klass)
465
+ # An unboundable value (out of the column's range) matches no row.
466
+ return relation if relation.null_relation?
467
+
468
+ if lyra_claims?(Thread.current[RECORD], attribute)
469
+ case_insensitive = options.key?(:case_sensitive) && !options[:case_sensitive]
470
+ Uniqueness.lock!(klass, attribute, value, case_insensitive: case_insensitive)
471
+ end
472
+ relation.extending!(Relation)
473
+ end
474
+
475
+ # Whether saving +record+ claims its value: a new record, one whose
476
+ # attribute or scope columns change, or any record checked under
477
+ # conditions: (which may read any of its attributes). A persisted
478
+ # record that keeps them holds a value already committed.
479
+ def lyra_claims?(record, attribute)
480
+ return true if record.nil? || !record.persisted? || options[:conditions]
481
+
482
+ aliases = record.class.attribute_aliases
483
+ resolve_attributes(record, Array(options[:scope]) + [attribute]).any? do |name|
484
+ record.attribute_changed?(aliases.fetch(name.to_s, name.to_s))
485
+ end
486
+ end
487
+ end
488
+
489
+ # A relation's where clause compiled into a test of a record's event
490
+ # attributes (CachedProjection's), answering as SQL does: true, false
491
+ # or nil (unknown, which a WHERE does not select).
492
+ class Predicates
493
+ # An Arel literal out of its column's range (#literal).
494
+ class LiteralOutOfRange < StandardError; end
495
+
496
+ STRING_ADAPTERS = /postgres|sqlite/i
497
+
498
+ def initialize(klass)
499
+ @klass = klass
500
+ @reader = TypedValues::EventReader.new(klass)
501
+ @lowered = [] # [name, bound text, cell] for each LOWER comparison
502
+ end
503
+
504
+ # The AND of +predicates+.
505
+ def compile_all(predicates)
506
+ tests = predicates.map { compile(_1) }
507
+ ->(attributes) { TypedValues.all_of(tests.map { _1.call(attributes) }) }
508
+ end
509
+
510
+ # The values the LOWER comparisons need, worked out by the database
511
+ # once for the records to be tested.
512
+ def prepare(records)
513
+ @lowered.each do |name, bound, cell|
514
+ values = records.filter_map { |attributes| text(name, @reader.read(attributes, name)) }.uniq
515
+ lowered = lower_all([bound, *values], name)
516
+ cell[:target] = lowered.first
517
+ cell[:values] = values.zip(lowered.drop(1)).to_h
518
+ end
519
+ end
520
+
521
+ private
522
+
523
+ def compile(node)
524
+ case node
525
+ when Arel::Nodes::And then compile_all(node.children)
526
+ when Arel::Nodes::Or
527
+ tests = node.children.map { compile(_1) }
528
+ ->(attributes) { TypedValues.any_of(tests.map { _1.call(attributes) }) }
529
+ when Arel::Nodes::Grouping then compile(node.expr)
530
+ when Arel::Nodes::Not then negated(compile(node.expr))
531
+ when Arel::Nodes::Equality then equality(node)
532
+ when Arel::Nodes::NotEqual then not_equal(node)
533
+ when Arel::Nodes::HomogeneousIn then homogeneous_in(node)
534
+ when Arel::Nodes::In then in_list(node, negate: false)
535
+ when Arel::Nodes::NotIn then in_list(node, negate: true)
536
+ when Arel::Nodes::Between then between(node)
537
+ when Arel::Nodes::GreaterThanOrEqual then bounded(node) { |value| value.. }
538
+ when Arel::Nodes::LessThanOrEqual then bounded(node) { |value| ..value }
539
+ when Arel::Nodes::LessThan then bounded(node) { |value| ...value }
540
+ when Arel::Nodes::GreaterThan then greater_than(node)
541
+ else refuse(node)
542
+ end
543
+ end
544
+
545
+ # column = value, IS NULL when the bound value is nil (Arel's rule),
546
+ # no row when it is unboundable; LOWER(column) = LOWER(value).
547
+ def equality(node)
548
+ return lower_equality(node) if lower?(node.left) && lower?(node.right)
549
+
550
+ name = column(node.left, node)
551
+ return ->(_) { false } if unboundable?(node.right)
552
+ return is_null(name) if node.right.nil?
553
+ return citext_equality(name, node.right, node) if Uniqueness.citext?(@klass, name)
554
+
555
+ condition(name, value(node.right, node))
556
+ end
557
+
558
+ def not_equal(node)
559
+ name = column(node.left, node)
560
+ return ->(_) { true } if unboundable?(node.right)
561
+ return negated(is_null(name)) if node.right.nil?
562
+ return negated(citext_equality(name, node.right, node)) if Uniqueness.citext?(@klass, name)
563
+
564
+ negated(condition(name, value(node.right, node)))
565
+ end
566
+
567
+ # IN or NOT IN over the values the attribute's type binds
568
+ # (HomogeneousIn#casted_values: a value it cannot bind, or binds as
569
+ # NULL, is left out); with none, IN (NULL): unknown for every row.
570
+ def homogeneous_in(node)
571
+ name = column(node.attribute, node)
572
+ refuse(node) if Uniqueness.citext?(@klass, name)
573
+ type = @klass.type_for_attribute(name)
574
+ values = node.values.select { type.serializable?(_1) && !type.serialize(_1).nil? }
575
+ return ->(_) { nil } if values.empty?
576
+
577
+ test = condition(name, values)
578
+ node.type == :in ? test : negated(test)
579
+ end
580
+
581
+ # IN (...) or NOT IN (...) built with Arel: unboundable values are
582
+ # left out (an empty list is 1=0, or 1=1 for NOT IN), and each other
583
+ # is bound as it is, NULL included (x IN (1, NULL) is unknown where
584
+ # x is not 1), unlike a hash condition's list.
585
+ def in_list(node, negate:)
586
+ name = column(node.left, node)
587
+ refuse(node) unless node.right.is_a?(Array)
588
+ refuse(node) if Uniqueness.citext?(@klass, name)
589
+ tests = node.right.reject { unboundable?(_1) }.map { |bound| in_member(name, bound, node) }
590
+ return ->(_) { negate } if tests.empty?
591
+
592
+ test = ->(attributes) { TypedValues.any_of(tests.map { _1.call(attributes) }) }
593
+ negate ? negated(test) : test
594
+ end
595
+
596
+ # One member of an Arel IN list, compared with =: a value bound as
597
+ # NULL (nil, or one its type serializes to nil) is unknown.
598
+ def in_member(name, bound, node)
599
+ raw = value(bound, node)
600
+ type = @klass.type_for_attribute(name)
601
+ return ->(_) { nil } if raw.nil? || (type.serializable?(raw) && type.serialize(raw).nil?)
602
+
603
+ condition(name, raw)
604
+ end
605
+
606
+ def between(node)
607
+ name = column(node.left, node)
608
+ refuse(node) unless node.right.is_a?(Arel::Nodes::And) && node.right.children.size == 2
609
+ low, high = node.right.children.map { |bound| ranged_value(bound, node) }
610
+ condition(name, low..high)
611
+ end
612
+
613
+ def bounded(node)
614
+ name = column(node.left, node)
615
+ condition(name, yield(ranged_value(node.right, node)))
616
+ end
617
+
618
+ # > value: at least value, and not equal to it.
619
+ def greater_than(node)
620
+ name = column(node.left, node)
621
+ bound = ranged_value(node.right, node)
622
+ at_least = condition(name, bound..)
623
+ equal = condition(name, bound)
624
+ ->(attributes) { TypedValues.all_of([at_least.call(attributes), negated(equal).call(attributes)]) }
625
+ end
626
+
627
+ def condition(name, value)
628
+ string_comparison!(name)
629
+ compiled = TypedValues.condition(@klass, name, value)
630
+ ->(attributes) { compiled.evaluate(@reader, attributes) }
631
+ end
632
+
633
+ def is_null(name)
634
+ ->(attributes) { @reader.read(attributes, name).nil? }
635
+ end
636
+
637
+ def negated(test)
638
+ lambda do |attributes|
639
+ result = test.call(attributes)
640
+ result.nil? ? nil : !result
641
+ end
642
+ end
643
+
644
+ # LOWER(column) = LOWER(value): the database lowers the value and the
645
+ # values the records hold (prepare), and they are compared here.
646
+ def lower_equality(node)
647
+ name = column(single_argument(node.left, node), node)
648
+ bound = single_argument(node.right, node)
649
+ return ->(_) { false } if unboundable?(bound)
650
+ return is_null(name) if bound.nil?
651
+
652
+ lowered(name, database_text(bound, node))
653
+ end
654
+
655
+ # A citext column's = is case-insensitive: LOWER on both sides.
656
+ def citext_equality(name, bound, node)
657
+ lowered(name, database_text(bound, node))
658
+ end
659
+
660
+ def lowered(name, bound_text)
661
+ string_comparison!(name)
662
+ return ->(_) { nil } if bound_text.nil?
663
+
664
+ cell = {}
665
+ @lowered << [name, bound_text, cell]
666
+ lambda do |attributes|
667
+ stored = text(name, @reader.read(attributes, name))
668
+ next nil if stored.nil?
669
+
670
+ cell[:values].fetch(stored) == cell[:target]
671
+ end
672
+ end
673
+
674
+ # A string comparison is the database's: refused where it may not be
675
+ # Ruby's (a collation of the column's own; MySQL's case-insensitive
676
+ # default collations).
677
+ def string_comparison!(name)
678
+ column = @klass.columns_hash[name]
679
+ return unless column && %i[string text citext].include?(column.type)
680
+
681
+ if column.respond_to?(:collation) && column.collation
682
+ raise UnsupportedQuery.new(@klass, "a uniqueness check on #{name}, which has a collation of its own (#{column.collation})")
683
+ end
684
+ adapter = @klass.with_connection(&:adapter_name)
685
+ return if adapter.match?(STRING_ADAPTERS)
686
+
687
+ raise UnsupportedQuery.new(@klass, "a uniqueness check comparing strings on #{adapter}")
688
+ end
689
+
690
+ # The text LOWER(column) reads for an attribute's value: the text the
691
+ # column holds, a character(n) value without its trailing spaces
692
+ # (its cast to text drops them).
693
+ def text(name, value)
694
+ return nil if value.nil?
695
+
696
+ serialized = @klass.type_for_attribute(name).serialize(value)&.to_s
697
+ serialized && TypedValues.schema(@klass).blank_padded.include?(name) ? TypedValues.unpadded(serialized) : serialized
698
+ end
699
+
700
+ # LOWER of each of +texts+, as the database computes it. Texts
701
+ # lowered before (per database, in this process) are not sent
702
+ # again; the rest go in one query (PostgreSQL: one array
703
+ # parameter; SQLite: one SELECT of LOWER(...) per 500).
704
+ def lower_all(texts, name = "value") = Uniqueness.lower_texts(@klass, texts, name: name)
705
+
706
+ # The column an attribute node names: one of this model's own table.
707
+ def column(node, predicate)
708
+ unless node.is_a?(Arel::Attributes::Attribute) && node.relation.respond_to?(:name) &&
709
+ node.relation.name.to_s == @klass.table_name.to_s && !node.relation.is_a?(Arel::Nodes::TableAlias)
710
+ refuse(predicate)
711
+ end
712
+ TypedValues.attribute_name(@klass, node.name)
713
+ end
714
+
715
+ def lower?(node)
716
+ node.is_a?(Arel::Nodes::NamedFunction) && node.name.to_s.casecmp?("LOWER") && node.expressions.size == 1
717
+ end
718
+
719
+ def single_argument(node, predicate)
720
+ refuse(predicate) unless lower?(node)
721
+ node.expressions.first
722
+ end
723
+
724
+ def unboundable?(node)
725
+ node.respond_to?(:unboundable?) && node.unboundable?
726
+ end
727
+
728
+ # The value a bound node carries, before its type casts it (the
729
+ # condition casts it, as Active Record does). A literal quoted
730
+ # without a type, another column or an expression is refused.
731
+ def value(node, predicate)
732
+ return literal(node) if node.is_a?(Arel::Nodes::Casted)
733
+
734
+ bound_attribute(node, predicate).value_before_type_cast
735
+ end
736
+
737
+ # A literal Arel quotes through its attribute's type (attr.in([..])):
738
+ # one out of the type's range cannot be quoted, and the query is not
739
+ # run (LiteralOutOfRange).
740
+ def literal(node)
741
+ node.value_for_database
742
+ node.value
743
+ rescue ::RangeError
744
+ raise LiteralOutOfRange
745
+ end
746
+
747
+ def ranged_value(node, predicate)
748
+ refuse(predicate) if unboundable?(node)
749
+ value(node, predicate)
750
+ end
751
+
752
+ # The text the database is given for a bound value.
753
+ def database_text(node, predicate)
754
+ return literal(node).then { node.value_for_database&.to_s } if node.is_a?(Arel::Nodes::Casted)
755
+
756
+ bound_attribute(node, predicate).value_for_database&.to_s
757
+ end
758
+
759
+ # The attribute a bound value is (Active Record passes it to Arel as
760
+ # it is, or in a BindParam).
761
+ def bound_attribute(node, predicate)
762
+ bound = node.is_a?(Arel::Nodes::BindParam) ? node.value : node
763
+ refuse(predicate) unless bound.is_a?(ActiveModel::Attribute)
764
+ bound
765
+ end
766
+
767
+ def refuse(node)
768
+ described = node.respond_to?(:to_sql) ? (node.to_sql rescue node.class.name) : node.class.name
769
+ raise UnsupportedQuery.new(@klass, "a uniqueness check's condition (#{described.to_s[0, 80]})")
770
+ end
771
+ end
772
+
773
+ def self.install!
774
+ validator = ActiveRecord::Validations::UniquenessValidator
775
+ validator.prepend(ValidatorPatch) unless validator.ancestors.include?(ValidatorPatch)
776
+ end
777
+ end
778
+ end
779
+ end
780
+
781
+ # Under Rails the engine installs the validator patch (Lyra::Engine).
782
+ if defined?(ActiveSupport)
783
+ ActiveSupport.on_load(:active_record) { Lyra::Projections::Uniqueness.install! } unless defined?(Lyra::Engine)
784
+ end