orfeas_lyra 0.9.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +2021 -17
  3. data/README.md +22 -9
  4. data/Rakefile +40 -1
  5. data/app/controllers/lyra/application_controller.rb +127 -0
  6. data/app/controllers/lyra/dashboard_controller.rb +33 -31
  7. data/app/controllers/lyra/flow_controller.rb +32 -34
  8. data/app/controllers/lyra/privacy_controller.rb +57 -18
  9. data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
  10. data/app/views/lyra/dashboard/index.html.erb +1 -0
  11. data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
  12. data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
  13. data/app/views/lyra/privacy/policy.html.erb +7 -0
  14. data/app/views/lyra/privacy/rights.html.erb +78 -0
  15. data/config/privacy_policies.rb +0 -3
  16. data/config/routes.rb +1 -0
  17. data/lib/lyra/access_log.rb +187 -6
  18. data/lib/lyra/advisory_lock.rb +236 -0
  19. data/lib/lyra/aggregate.rb +1 -1
  20. data/lib/lyra/associations/habtm_links.rb +382 -19
  21. data/lib/lyra/associations/maintenance.rb +229 -0
  22. data/lib/lyra/attribution.rb +48 -0
  23. data/lib/lyra/bypass_events.rb +446 -46
  24. data/lib/lyra/command_handler.rb +19 -14
  25. data/lib/lyra/configuration.rb +237 -10
  26. data/lib/lyra/consistency/read_your_writes.rb +4 -2
  27. data/lib/lyra/correlation.rb +121 -0
  28. data/lib/lyra/domain_events.rb +4 -0
  29. data/lib/lyra/dual_view.rb +53 -16
  30. data/lib/lyra/encrypted_attributes.rb +5 -0
  31. data/lib/lyra/engine.rb +67 -0
  32. data/lib/lyra/erasure.rb +1011 -106
  33. data/lib/lyra/event.rb +35 -1
  34. data/lib/lyra/event_data.rb +38 -0
  35. data/lib/lyra/event_flow.rb +31 -15
  36. data/lib/lyra/event_store_adapter.rb +28 -0
  37. data/lib/lyra/event_window.rb +58 -0
  38. data/lib/lyra/genesis.rb +548 -44
  39. data/lib/lyra/interceptors/association_interceptor.rb +689 -34
  40. data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
  41. data/lib/lyra/internal.rb +26 -0
  42. data/lib/lyra/mode_sync.rb +51 -3
  43. data/lib/lyra/mode_transition.rb +257 -30
  44. data/lib/lyra/model_selection.rb +35 -0
  45. data/lib/lyra/own_connection.rb +17 -2
  46. data/lib/lyra/privacy/adapters/pam.rb +25 -2
  47. data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
  48. data/lib/lyra/privacy/interface.rb +27 -7
  49. data/lib/lyra/privacy/subject_links.rb +143 -0
  50. data/lib/lyra/privacy/subject_records.rb +369 -0
  51. data/lib/lyra/projection.rb +7 -1
  52. data/lib/lyra/projections/async_checkpoint.rb +32 -0
  53. data/lib/lyra/projections/async_projection_job.rb +166 -23
  54. data/lib/lyra/projections/cached_joins.rb +33 -18
  55. data/lib/lyra/projections/cached_projection.rb +74 -33
  56. data/lib/lyra/projections/cached_relation.rb +3175 -321
  57. data/lib/lyra/projections/default_scopes.rb +254 -0
  58. data/lib/lyra/projections/deferred_relation.rb +23 -3
  59. data/lib/lyra/projections/event_store_reader.rb +47 -8
  60. data/lib/lyra/projections/foreign_keys.rb +92 -0
  61. data/lib/lyra/projections/lazy_projection.rb +165 -27
  62. data/lib/lyra/projections/model_projection.rb +165 -14
  63. data/lib/lyra/projections/rebuild.rb +34 -6
  64. data/lib/lyra/projections/reference_links.rb +463 -72
  65. data/lib/lyra/projections/typed_values.rb +629 -0
  66. data/lib/lyra/projections/unique_indexes.rb +595 -0
  67. data/lib/lyra/projections/uniqueness.rb +784 -0
  68. data/lib/lyra/projections/value_digest.rb +93 -0
  69. data/lib/lyra/purpose_bound_reads.rb +912 -77
  70. data/lib/lyra/repair.rb +119 -28
  71. data/lib/lyra/retention.rb +114 -8
  72. data/lib/lyra/rights/consent_store.rb +415 -0
  73. data/lib/lyra/rights/erasure_guard.rb +135 -0
  74. data/lib/lyra/rights/pending.rb +74 -0
  75. data/lib/lyra/rights/queries.rb +269 -0
  76. data/lib/lyra/rights/store.rb +512 -0
  77. data/lib/lyra/rights/subject_keys.rb +224 -0
  78. data/lib/lyra/rights.rb +281 -0
  79. data/lib/lyra/saves_without_effect.rb +69 -0
  80. data/lib/lyra/stream_tail.rb +319 -0
  81. data/lib/lyra/strict_data_access.rb +916 -106
  82. data/lib/lyra/temporal.rb +4 -2
  83. data/lib/lyra/touch_registry.rb +404 -0
  84. data/lib/lyra/verification/bypass_workflow.rb +452 -42
  85. data/lib/lyra/verification/rights_workflows.rb +170 -0
  86. data/lib/lyra/version.rb +1 -1
  87. data/lib/lyra/visualization/event_graph.rb +1 -1
  88. data/lib/lyra/visualization/timeline.rb +2 -2
  89. data/lib/lyra.rb +43 -2
  90. data/lib/tasks/lyra_mode.rake +110 -8
  91. data/lib/tasks/lyra_projections.rake +43 -2
  92. metadata +34 -4
  93. data/lib/lyra/associations/event_aware.rb +0 -224
@@ -54,7 +54,16 @@ module Lyra
54
54
  target_klass = lyra_resolve_target_class
55
55
  return nil unless target_klass
56
56
 
57
- Lyra::Projections::EventStoreReader.find(target_klass, foreign_key_value)
57
+ # The record is returned to the application: checked as a loaded
58
+ # row is by after_find (Lyra::PurposeBoundReads), consent included.
59
+ # As ActiveRecord's association scope, it is read through the
60
+ # target's default scopes, less what the association's scope
61
+ # unscopes (belongs_to :variant, -> { with_discarded }).
62
+ record = Lyra::Projections::EventStoreReader.find(target_klass, foreign_key_value)
63
+ default_scope = record && Lyra::Projections::DefaultScopes.for_association(target_klass, reflection, owner)
64
+ record = nil if default_scope && !Lyra::Projections::DefaultScopes.admits?(target_klass, record, default_scope)
65
+ Lyra::PurposeBoundReads.check(record) if record
66
+ record
58
67
  end
59
68
  end
60
69
 
@@ -79,13 +88,77 @@ module Lyra
79
88
  end
80
89
 
81
90
  def lyra_find_target_from_cache
91
+ return lyra_find_target_through if reflection.through_reflection?
92
+
82
93
  owner_key_value = owner.read_attribute(reflection.active_record_primary_key)
83
94
  return nil if owner_key_value.nil?
84
95
 
85
- Lyra::Projections::EventStoreReader.find_by(
86
- reflection.klass,
87
- { reflection.foreign_key => owner_key_value }
88
- )
96
+ # Through the target's default scopes, as the belongs_to above: a
97
+ # soft-deleted record is not the one the owner has.
98
+ klass = reflection.klass
99
+ conditions = lyra_has_one_conditions(reflection, owner_key_value)
100
+ default = Lyra::Projections::DefaultScopes.for_association(klass, reflection, owner)
101
+ return lyra_scoped_has_one(klass, conditions, default) if reflection.scope
102
+
103
+ record = Lyra::Projections::DefaultScopes.find_by(klass, conditions, default)
104
+ # Checked as a loaded row is, with the key it was found by.
105
+ Lyra::PurposeBoundReads.check(record, Array(reflection.foreign_key).map(&:to_s)) if record
106
+ record
107
+ end
108
+
109
+ # A has_one with a scope of its own (-> { where(selected: true) },
110
+ # -> { order(created_at: :desc) }): the owner's records, through the
111
+ # target's default scopes, narrowed by the scope, and the first of
112
+ # them in its order, as Active Record's query with LIMIT 1. What the
113
+ # scope holds that CachedRelation cannot evaluate raises
114
+ # UnsupportedQuery. The scope used to be left out: any record of the
115
+ # owner's was the answer.
116
+ def lyra_scoped_has_one(klass, conditions, default)
117
+ relation = Lyra::Projections::DefaultScopes.narrow(Lyra::Projections::EventStoreReader.where(klass, conditions), default)
118
+ record = AssociationInterceptor.scoped(relation, reflection.scope, owner).first
119
+ Lyra::PurposeBoundReads.check(record, Array(reflection.foreign_key).map(&:to_s)) if record
120
+ record
121
+ end
122
+
123
+ # The foreign key of a has_one :through is on the intermediate record,
124
+ # not on the target. Solidus's InventoryUnit has_one :order, through:
125
+ # :shipment was looked up as Spree::Order.find_by(order_id: <the
126
+ # unit's id>), a column only the shipment has: 0.9.x answered nil, so
127
+ # carton.orders was empty and no shipped email went out.
128
+ #
129
+ # The targets are those Active Record's join reaches, in its order
130
+ # (AssociationInterceptor.through_scoped): through every intermediate
131
+ # record (every one a has_one selects, not only the one it reads),
132
+ # read afresh, and narrowed as the join narrows them. When a scope in
133
+ # the chain orders, the first target in that order is the answer;
134
+ # otherwise none or one target is, and more than one is refused
135
+ # (UnsupportedQuery): the join would choose among them. What a scope
136
+ # holds that CachedRelation cannot evaluate raises UnsupportedQuery.
137
+ # Only the answer is checked as a loaded row: the join loads nothing
138
+ # else.
139
+ def lyra_find_target_through
140
+ relation, ordered = AssociationInterceptor.through_scoped(owner, reflection)
141
+ joined = relation.is_a?(ThroughRelation)
142
+ targets = (joined ? relation.join_records : Lyra::PurposeBoundReads.internal(INTERNAL) { relation.to_a }).uniq
143
+ if targets.size > 1 && !ordered
144
+ raise Lyra::Projections::UnsupportedQuery.new(
145
+ owner.class, "#{reflection.name} (a has_one :through #{reflection.through_reflection.name}, reaching " \
146
+ "#{targets.size} #{reflection.klass.name} records for this owner in no order: " \
147
+ "Active Record's join would choose among them)"
148
+ )
149
+ end
150
+ if targets.size > 1 && joined && relation.unordered_reason
151
+ raise Lyra::Projections::UnsupportedQuery.new(owner.class, relation.unordered_reason)
152
+ end
153
+ # Delivered alone, as the join's LIMIT 1 row: checked as a
154
+ # loaded row, and as a query through the intermediate models.
155
+ targets.empty? ? nil : relation.spawn_records(targets.first(1)).first
156
+ end
157
+
158
+ # The key (and, for a polymorphic as: association, the owner's type)
159
+ # a direct has_one reads its record by.
160
+ def lyra_has_one_conditions(has_one, owner_key_value)
161
+ AssociationInterceptor.has_one_conditions(owner, has_one, owner_key_value)
89
162
  end
90
163
  end
91
164
 
@@ -107,8 +180,12 @@ module Lyra
107
180
  # scope (scope.update_all), where Lyra records them as the
108
181
  # dependent_association bypass they are. Given the event-backed scope
109
182
  # they became plain update_all bypasses, refused in strict mode.
183
+ # Recorded writes go to the event store instead
184
+ # (lyra_delete_or_nullify_from_events); only Lyra's own projection
185
+ # writes still use the table scope.
110
186
  def delete_or_nullify_all_records(method)
111
187
  return super unless lyra_should_use_cache?
188
+ return lyra_delete_or_nullify_from_events(method, lyra_dependent_children) unless Thread.current[:lyra_projection_write]
112
189
 
113
190
  previous = Thread.current[:lyra_association_own_scope]
114
191
  Thread.current[:lyra_association_own_scope] = true
@@ -172,37 +249,52 @@ module Lyra
172
249
  # The foreign key of a :through association lives on the intermediate
173
250
  # records, not on the target (shipment.orders was filtered on an
174
251
  # order_id Spree::Order does not have): they are read through that
175
- # association and its source. A polymorphic (as:) association matches
176
- # the owner's type too, and the association's own scope applies.
252
+ # association and its source, afresh, as Active Record's join reads
253
+ # them, narrowed by the target's default scopes as the join applies
254
+ # them, in its order and narrowed by the association's own scope
255
+ # (AssociationInterceptor.through_scoped). A direct polymorphic (as:)
256
+ # association matches the owner's type too. The
257
+ # target's default scopes apply, less what the association's scope
258
+ # unscopes, and then the association's own scope, as in
259
+ # ActiveRecord's association scope; reading, counting, plucking and
260
+ # the collection's queries all start here.
177
261
  def lyra_get_cached_relation
178
- relation =
179
- if reflection.through_reflection?
180
- lyra_records_through
181
- else
182
- owner_key_value = owner.read_attribute(reflection.active_record_primary_key)
183
- return Lyra::Projections::CachedRelation.new(reflection.klass, []) if owner_key_value.nil?
184
-
185
- conditions = { reflection.foreign_key => owner_key_value }
186
- conditions[reflection.type] = owner.class.polymorphic_name if reflection.type
187
- # Read when first needed, so that a narrowing where made on it
188
- # (stock_items.where(variant_id: ...)) reaches the reference
189
- # links with all its keys. As ActiveRecord does for what an
190
- # association loads, the records point back at this owner, not
191
- # at a copy of it read from the events: a child asks its
192
- # parent in memory (shipment.order.paid?), and the copy was the
193
- # parent before the change under way was recorded.
194
- Lyra::Projections::DeferredRelation.new(
195
- reflection.klass, conditions: conditions,
196
- on_load: ->(records) { records.each { |record| set_inverse_instance(record) } }
197
- )
262
+ if reflection.through_reflection?
263
+ # A write reads the join again (Lyra::Projections::Rebuildable)
264
+ # with the default scopes built now (DefaultScopes.remembering),
265
+ # where that runs none of the application's code again: no
266
+ # association on the way has a scope. Otherwise it writes the
267
+ # records the relation holds.
268
+ scopes = {}
269
+ relation = Lyra::Projections::DefaultScopes.remembering(scopes) do
270
+ AssociationInterceptor.through_scoped(owner, reflection).first
198
271
  end
199
- lyra_apply_reflection_scope(relation)
200
- end
272
+ if relation.is_a?(Lyra::Projections::CachedRelation) && AssociationInterceptor.plain_through?(reflection)
273
+ relation.rebuild_recipe = [lambda {
274
+ Lyra::Projections::DefaultScopes.remembering(scopes) { AssociationInterceptor.through_scoped(owner, reflection).first }
275
+ }, []]
276
+ end
277
+ return relation
278
+ end
201
279
 
202
- def lyra_records_through
203
- through = Array(owner.association(reflection.through_reflection.name).reader)
204
- records = through.flat_map { |record| Array(record.association(reflection.source_reflection.name).reader) }
205
- Lyra::Projections::CachedRelation.new(reflection.klass, records.compact.uniq)
280
+ owner_key_value = owner.read_attribute(reflection.active_record_primary_key)
281
+ return Lyra::Projections::CachedRelation.new(reflection.klass, []) if owner_key_value.nil?
282
+
283
+ conditions = { reflection.foreign_key => owner_key_value }
284
+ conditions[reflection.type] = owner.class.polymorphic_name if reflection.type
285
+ # Read when first needed, so that a narrowing where made on it
286
+ # (stock_items.where(variant_id: ...)) reaches the reference
287
+ # links with all its keys. As ActiveRecord does for what an
288
+ # association loads, the records point back at this owner, not
289
+ # at a copy of it read from the events: a child asks its
290
+ # parent in memory (shipment.order.paid?), and the copy was the
291
+ # parent before the change under way was recorded.
292
+ relation = Lyra::Projections::DeferredRelation.new(
293
+ reflection.klass, conditions: conditions,
294
+ on_load: ->(records) { records.each { |record| set_inverse_instance(record) } }
295
+ )
296
+ relation = Lyra::Projections::DefaultScopes.narrow(relation, lyra_target_default_scope)
297
+ lyra_apply_reflection_scope(relation)
206
298
  end
207
299
 
208
300
  def lyra_apply_reflection_scope(relation)
@@ -212,17 +304,106 @@ module Lyra
212
304
  scope.arity.zero? ? relation.instance_exec(&scope) : relation.instance_exec(owner, &scope)
213
305
  end
214
306
 
307
+ # The records are checked as loaded rows as the relation delivers
308
+ # them (CachedRelationDelivery), those returned only.
215
309
  def lyra_find_target_from_cache
216
310
  lyra_get_cached_relation.to_a
217
311
  end
312
+
313
+ # Removing records from the association (delete(record), or assigning
314
+ # a new set) deletes or nullifies them as dependent: says, through the
315
+ # association's scope narrowed to them. In the events-only store that
316
+ # is the same write as above, on those of the records the association
317
+ # selects.
318
+ def delete_records(records, method)
319
+ return super if method == :destroy || !lyra_should_use_cache? || Thread.current[:lyra_projection_write]
320
+
321
+ pk = reflection.klass.primary_key
322
+ return super unless pk.is_a?(String)
323
+
324
+ ids = records.map { |record| record.read_attribute(pk) }
325
+ lyra_delete_or_nullify_from_events(method, lyra_dependent_children.select { |child| ids.include?(child.read_attribute(pk)) })
326
+ end
327
+
328
+ # dependent: :delete_all and :nullify in the events-only store. The
329
+ # table scope ActiveRecord writes through matches no row there (or
330
+ # stale ones), so the children were left alive in the log, and a
331
+ # nullify found them by foreign key alone, past the association's
332
+ # scope. Each child gets the event delete_all or update_all writes for
333
+ # it there. A nullify is Rails' own write, the dependent_association
334
+ # bypass, outside strict mode, whatever its columns (a polymorphic
335
+ # key's type, a key not named *_id), as StrictDataAccessRelation
336
+ # takes the marked nullify (Lyra::Associations::Maintenance); a
337
+ # polymorphic one was an update_all, refused in strict mode.
338
+ #
339
+ # The owner's counter cache is left as it is: ES-NoProj does not
340
+ # maintain counter caches, so the log still holds the old count, and
341
+ # lowering the one in memory made it disagree (and go negative).
342
+ def lyra_delete_or_nullify_from_events(method, children)
343
+ relation = Lyra::Projections::CachedRelation.new(reflection.klass, children)
344
+ if method == :delete_all
345
+ relation.delete_all
346
+ else
347
+ Lyra.without_strict_access do
348
+ relation.send(:bulk_mutate_as, :update_all, "dependent_association", nullified_owner_attributes)
349
+ end
350
+ end
351
+ end
352
+
353
+ # The children a dependent write reaches: those ActiveRecord's scope
354
+ # for the association selects, read from the event store as Lyra's
355
+ # own read (as a bulk write's snapshot is). That is the owner's
356
+ # records, narrowed by the target's default scopes (less what the
357
+ # association's scope unscopes) and then by the association's scope.
358
+ # Leaving the default scopes out deleted or nullified children the
359
+ # application had put out of scope (a soft-deleted one). A scope the event store cannot evaluate
360
+ # (an SQL string) raises UnsupportedQuery: deleting or nullifying
361
+ # other children than the association selects is worse than refusing
362
+ # the parent's destroy.
363
+ def lyra_dependent_children
364
+ owner_key_value = owner.read_attribute(reflection.active_record_primary_key)
365
+ return [] if owner_key_value.nil?
366
+
367
+ what = "dependent: #{options[:dependent].inspect} of #{owner.class.name}##{reflection.name}"
368
+ conditions = { reflection.foreign_key => owner_key_value }
369
+ conditions[reflection.type] = owner.class.polymorphic_name if reflection.type
370
+ default_scope = lyra_target_default_scope
371
+ Lyra::PurposeBoundReads.internal(INTERNAL) do
372
+ relation = Lyra::Projections::DeferredRelation.new(reflection.klass, conditions: conditions)
373
+ relation = Lyra::Projections::DefaultScopes.narrow(
374
+ relation, default_scope, "#{what} (#{Lyra::Projections::DefaultScopes.description(reflection.klass)})"
375
+ )
376
+ begin
377
+ relation = lyra_apply_reflection_scope(relation)
378
+ rescue Lyra::Projections::UnsupportedQuery
379
+ raise Lyra::Projections::UnsupportedQuery.new(reflection.klass, "#{what} (its scope)")
380
+ end
381
+ relation.records.to_a
382
+ end
383
+ end
384
+
385
+ # The target's default scopes as ActiveRecord applies them to the
386
+ # association (scope_for_association), less what the association's
387
+ # own scope unscopes; nil when they select every record.
388
+ def lyra_target_default_scope
389
+ Lyra::Projections::DefaultScopes.for_association(reflection.klass, reflection, owner)
390
+ end
218
391
  end
219
392
 
220
393
  # A has_many proxy's calculations and plucks (payment.refunds.sum(:amount),
221
394
  # order.line_items.pluck(:id)) ran ActiveRecord's SQL against the table,
222
395
  # which under ES-NoProj is empty, or failed asking the event-backed scope
223
396
  # for relation internals. They go to that scope, as reading does.
397
+ #
398
+ # update_all (author.articles.update_all(author_id: nil)) is Active
399
+ # Record's on the association's scope; it was the table's under
400
+ # ES-NoProj, and refused. It goes to the event-backed scope too, which
401
+ # records an event for each record the association selects
402
+ # (CachedRelation#update_all). delete_all and destroy_all go through
403
+ # the association (HasManyAssociationPatch#delete_or_nullify_all_records,
404
+ # and each record's destroy).
224
405
  module CollectionProxyPatch
225
- %i[calculate pluck ids pick exists?].each do |name|
406
+ %i[calculate pluck ids pick exists? update_all].each do |name|
226
407
  define_method(name) do |*args, **kwargs, &block|
227
408
  return super(*args, **kwargs, &block) unless lyra_events_only_association?
228
409
 
@@ -230,6 +411,31 @@ module Lyra
230
411
  end
231
412
  end
232
413
 
414
+ # first, last and take on a :through association not loaded: Active
415
+ # Record's query orders by the join's orders (the intermediate
416
+ # records' among them), which the event-backed scope holds in its
417
+ # records' order; the proxy's own query ordered by the target's
418
+ # primary key alone.
419
+ # length on a :through association not loaded counts what the
420
+ # event-backed scope holds, as size does, without loading the target:
421
+ # a target loaded where the join's order cannot be reproduced would
422
+ # list the records in another order (ThroughRelation).
423
+ def length
424
+ return super if @association.send(:find_from_target?) || !lyra_events_only_association?
425
+ return super unless scope.is_a?(ThroughRelation)
426
+
427
+ scope.length
428
+ end
429
+
430
+ %i[first last take].each do |name|
431
+ define_method(name) do |*args|
432
+ return super(*args) if @association.send(:find_from_target?) || !lyra_events_only_association?
433
+ return super(*args) unless scope.is_a?(ThroughRelation)
434
+
435
+ scope.public_send(name, *args)
436
+ end
437
+ end
438
+
233
439
  private
234
440
 
235
441
  def lyra_events_only_association?
@@ -270,6 +476,454 @@ module Lyra
270
476
  end
271
477
  end
272
478
 
479
+ # The relation a :through association's targets come in under ES-NoProj
480
+ # (AssociationInterceptor.through_scoped).
481
+ #
482
+ # - Active Record's unscope(where: :col) on such an association drops
483
+ # the predicate on +col+ from every table of the join, the
484
+ # intermediate tables' default scopes included:
485
+ # owner.tags.unscope(where: :deleted_at) also follows a soft-deleted
486
+ # intermediate record. The intermediate records here were read through
487
+ # their default scopes already, so an unscope of a column one of them
488
+ # conditions is refused rather than answered short.
489
+ # - Where the join's order cannot be reproduced (+unordered_reason+), a
490
+ # read that depends on it (loading, first, last, take, a limit or
491
+ # offset, pluck and ids, which list in that order) is refused when
492
+ # there is more than one record; counts, sums, length and exists?
493
+ # answer, as they do not depend on it.
494
+ # - A bulk write writes each record once: the join may reach a record
495
+ # through two intermediate records, and SQL updates or deletes the row
496
+ # once.
497
+ class ThroughRelation < Lyra::Projections::CachedRelation
498
+ # Its own query methods come before CachedRelation's recording
499
+ # (Lyra::Projections::Rebuildable).
500
+ prepend Lyra::Projections::Rebuildable.wrappers
501
+
502
+ # Columns the intermediate models' default scopes condition.
503
+ attr_accessor :intermediate_columns
504
+ # Why the join's order cannot be reproduced, if it cannot.
505
+ attr_accessor :unordered_reason
506
+ # While the association's own scope is applied: the records are in
507
+ # the join's order already, which its order is part of.
508
+ attr_accessor :keep_order
509
+
510
+ ORDER_DEPENDENT = %i[to_a to_ary each records first last take second third [] find_each
511
+ find_in_batches in_batches limit offset pluck ids].freeze
512
+
513
+ ORDER_DEPENDENT.each do |name|
514
+ define_method(name) do |*args, **kwargs, &block|
515
+ refuse_join_order!
516
+ super(*args, **kwargs, &block)
517
+ end
518
+ end
519
+
520
+ %i[update_all delete_all touch_all destroy_all].each do |name|
521
+ define_method(name) do |*args, **kwargs, &block|
522
+ unique = @records.uniq
523
+ return super(*args, **kwargs, &block) if unique.size == @records.size
524
+
525
+ # The same selection, each record once: built again as this one is.
526
+ spawn_records(unique).inherit_rebuild(self).public_send(name, *args, **kwargs, &block)
527
+ end
528
+ end
529
+
530
+ def order(*args)
531
+ keep_order ? self : super
532
+ end
533
+
534
+ # As size: how many records, which does not depend on their order
535
+ # (they are loaded, and checked, as Active Record loads them).
536
+ def length
537
+ @order_free = true
538
+ to_a.length
539
+ ensure
540
+ @order_free = false
541
+ end
542
+
543
+ def reorder(*args)
544
+ relation = super
545
+ relation.unordered_reason = nil if relation.respond_to?(:unordered_reason=)
546
+ relation
547
+ end
548
+
549
+ def unscope(*args)
550
+ hit = through_unscoped_columns(args.flatten)
551
+ if hit.any?
552
+ raise Lyra::Projections::UnsupportedQuery.new(
553
+ model_class, "unscope(#{args.inspect[1..-2]}) on a :through association: Active Record's join drops " \
554
+ "#{hit.join(', ')} from the intermediate tables' default scopes too, and ES-NoProj read " \
555
+ "the intermediate records through them"
556
+ )
557
+ end
558
+ super
559
+ end
560
+
561
+ def spawn_records(records)
562
+ relation = super
563
+ if relation.is_a?(ThroughRelation)
564
+ relation.intermediate_columns = intermediate_columns
565
+ relation.unordered_reason = unordered_reason
566
+ relation.keep_order = keep_order
567
+ end
568
+ relation
569
+ end
570
+
571
+ # The records in the join's order as far as it is known, for Lyra's
572
+ # own use (AssociationInterceptor): no order check, no purpose check.
573
+ def join_records = @records.dup
574
+
575
+ private
576
+
577
+ def refuse_join_order!
578
+ return if @order_free || unordered_reason.nil? || @records.size < 2
579
+
580
+ raise Lyra::Projections::UnsupportedQuery.new(model_class, unordered_reason)
581
+ end
582
+
583
+ def through_unscoped_columns(args)
584
+ columns = Array(intermediate_columns)
585
+ return [] if columns.empty?
586
+
587
+ args.flat_map do |arg|
588
+ case arg
589
+ when :where, "where" then columns
590
+ when Hash then Array(arg[:where] || arg["where"]).map { |c| c.to_s.split(".").last.delete('"') } & columns
591
+ else []
592
+ end
593
+ end.uniq
594
+ end
595
+ end
596
+
597
+ class << self
598
+ # Whether reading a :through +reflection+'s join again runs none of
599
+ # the application's code: no association on the way has a scope (the
600
+ # default scopes are remembered: DefaultScopes.remembering).
601
+ def plain_through?(reflection)
602
+ return false unless reflection.scope.nil?
603
+
604
+ reflection.chain.all? { |step| step.scope.nil? }
605
+ rescue StandardError
606
+ false
607
+ end
608
+
609
+ # +relation+ narrowed by an association +scope+ evaluated with +owner+
610
+ # (a scope returning nil leaves it as it is, as in Active Record).
611
+ def scoped(relation, scope, owner)
612
+ result = scope.arity.zero? ? relation.instance_exec(&scope) : relation.instance_exec(owner, &scope)
613
+ result || relation
614
+ end
615
+
616
+ # Whether narrowing +before+ by a scope into +after+ ordered it.
617
+ def orders?(before, after)
618
+ after.respond_to?(:ordering) && after.ordering != before.ordering
619
+ end
620
+
621
+ # The key (and, for a polymorphic as: association, the owner's type)
622
+ # a direct +has_one+ of +owner+ reads its record by, as the has_many
623
+ # path reads its records. Without the type, an owner read a record
624
+ # of another type of owner with the same id.
625
+ def has_one_conditions(owner, has_one, owner_key_value)
626
+ conditions = { has_one.foreign_key => owner_key_value }
627
+ conditions[has_one.type] = owner.class.polymorphic_name if has_one.type
628
+ conditions
629
+ end
630
+
631
+ # The targets of a :through +reflection+ of +owner+, as Active
632
+ # Record's join reaches them, as [relation, ordered]: a
633
+ # ThroughRelation, and whether a scope on the way orders them.
634
+ #
635
+ # Every record on the way is read afresh, on an association object of
636
+ # its own (join_candidates: the join reads the tables), as Lyra's own
637
+ # read: the join loads no intermediate record, so none is checked
638
+ # here. The records returned are checked where they are returned
639
+ # (HasOneAssociationPatch, HasManyAssociationPatch#find_target), and a
640
+ # query on the relation is checked as a query through the
641
+ # intermediate models (joined_classes), as the join's SQL is. Through
642
+ # a has_one, every record it selects counts, not only the one it
643
+ # reads.
644
+ #
645
+ # The targets are read without their default scope and then narrowed
646
+ # by it as the join applies it (DefaultScopes.for_through: less what
647
+ # the source's scope or the association's own unscopes, so that
648
+ # -> { with_discarded } brings back a soft-deleted target), ordered as
649
+ # the join orders them (join_order), and narrowed by the association's
650
+ # own scope. In no order at all, they are in primary key order, as
651
+ # Active Record's first takes them.
652
+ #
653
+ # An intermediate model on the target's own table is aliased in the
654
+ # join, which applies no default scope to it (cat.grandchildren
655
+ # through a soft-deleted child): it is read without its default
656
+ # scope too.
657
+ #
658
+ # With source_type: (a polymorphic source), only the intermediate
659
+ # records whose type column holds that name are followed, as the
660
+ # join selects them (where(thing_type: "A")). A target reached
661
+ # through two intermediate records is there twice, as in the join.
662
+ def through_scoped(owner, reflection, head = reflection.klass)
663
+ klass = reflection.klass
664
+ source = reflection.source_reflection
665
+ through = reflection.through_reflection
666
+ pairs, inner_reason = Lyra::PurposeBoundReads.internal(INTERNAL) do
667
+ found, _, reason = aliased(through.klass, head) { join_candidates(owner, through, head) }
668
+ if (source_type = reflection.options[:source_type])
669
+ found = found.select { |record| record.read_attribute(source.foreign_type) == source_type.to_s }
670
+ end
671
+ reason = nil if found.size < 2
672
+ reached = without_default_scope(klass) do
673
+ found.flat_map { |record| join_candidates(record, source, head).first.compact.map { [_1, record] } }
674
+ end
675
+ [reached, reason]
676
+ end
677
+
678
+ default = Lyra::Projections::DefaultScopes.for_through(reflection, owner)
679
+ keys, reason = join_order(owner, reflection, head, default)
680
+ pairs = Lyra::PurposeBoundReads.internal(INTERNAL) { sort_pairs(pairs, keys, klass, through.klass) } if reason.nil?
681
+ targets = pairs.map(&:first)
682
+ targets = by_primary_key(klass, targets) if keys.empty?
683
+
684
+ base = ThroughRelation.new(klass, targets)
685
+ base.intermediate_columns = intermediate_columns(reflection, head)
686
+ base.unordered_reason = reason || inner_reason
687
+ # The intermediate models, joined as in the SQL: a query on the
688
+ # relation is checked as the join is (a right in force for them).
689
+ base.joined_classes = reflection.chain.drop(1).map(&:klass).uniq
690
+ relation = Lyra::Projections::DefaultScopes.narrow(base, default)
691
+ if reflection.scope
692
+ # The association's own unscope is the target table's alone in
693
+ # the join (DefaultScopes.for_through has taken it into account),
694
+ # and its order is in the records' order already.
695
+ relation.keep_order = true
696
+ columns = relation.intermediate_columns
697
+ relation.intermediate_columns = nil
698
+ relation = scoped(relation, reflection.scope, owner)
699
+ if relation.is_a?(ThroughRelation)
700
+ relation.intermediate_columns = columns
701
+ relation.keep_order = nil
702
+ end
703
+ end
704
+ [relation, keys.any? || !inner_reason.nil?]
705
+ end
706
+
707
+ # The records Active Record's join reaches through +reflection+ of
708
+ # +owner+, as [records, ordered, reason]: +ordered+ when a scope on
709
+ # the way (an association's, or a model's default scope) orders them,
710
+ # +reason+ when that order cannot be reproduced (ThroughRelation).
711
+ # +head+ is the model the whole join reads.
712
+ #
713
+ # - A :through association: through_scoped.
714
+ # - A has_one: every record it selects, in its order (the join does
715
+ # not stop at the first: has_one_candidates).
716
+ # - A belongs_to or has_many: what it reads.
717
+ def join_candidates(owner, reflection, head = reflection.klass)
718
+ if reflection.through_reflection?
719
+ relation, ordered = through_scoped(owner, reflection, head)
720
+ [relation.join_records, ordered, relation.unordered_reason]
721
+ elsif reflection.macro == :has_one
722
+ has_one_candidates(owner, reflection)
723
+ elsif reflection.collection?
724
+ relation = fresh_association(owner, reflection).scope
725
+ [relation.to_a, ordered?(relation), nil]
726
+ else
727
+ [Array(fresh_association(owner, reflection).reader), false, nil]
728
+ end
729
+ end
730
+
731
+ # Every record a direct +has_one+ of +owner+ selects, as
732
+ # [records, ordered]: those of the owner's (its type too, for as:),
733
+ # through the target's default scopes and then the has_one's own
734
+ # scope, in their order. A model read from events is read from the
735
+ # event store; any other with the association's own SQL, less the
736
+ # LIMIT 1 a has_one reads with (the join keeps every row).
737
+ def has_one_candidates(owner, has_one)
738
+ klass = has_one.klass
739
+ unless klass.respond_to?(:lyra_monitored) && klass.lyra_monitored
740
+ relation = fresh_association(owner, has_one).scope.except(:limit, :offset)
741
+ return [relation.to_a, ordered?(relation), nil]
742
+ end
743
+
744
+ owner_key_value = owner.read_attribute(has_one.active_record_primary_key)
745
+ return [[], false, nil] if owner_key_value.nil?
746
+
747
+ found = Lyra::Projections::EventStoreReader.where(klass, has_one_conditions(owner, has_one, owner_key_value))
748
+ relation = Lyra::Projections::DefaultScopes.narrow(
749
+ found, Lyra::Projections::DefaultScopes.for_association(klass, has_one, owner)
750
+ )
751
+ # A default scope that orders (order(:position)) orders the
752
+ # join too: its order first, then the association scope's.
753
+ ordered = orders?(found, relation)
754
+ if has_one.scope
755
+ scoped = scoped(relation, has_one.scope, owner)
756
+ ordered ||= orders?(relation, scoped)
757
+ relation = scoped
758
+ end
759
+ [relation.to_a, ordered, nil]
760
+ end
761
+
762
+ # A new association object for +reflection+ of +owner+: what it
763
+ # reads is read afresh (from the event store, or the table) and is
764
+ # not left loaded on the owner.
765
+ def fresh_association(owner, reflection)
766
+ reflection.association_class.new(owner, reflection)
767
+ end
768
+
769
+ private
770
+
771
+ # Whether +relation+ (Active Record's, or a CachedRelation) orders.
772
+ def ordered?(relation)
773
+ if relation.respond_to?(:ordering)
774
+ relation.ordering.any?
775
+ else
776
+ relation.respond_to?(:order_values) && relation.order_values.any?
777
+ end
778
+ end
779
+
780
+ # The order values of +scopes+ (association scope procs, in the
781
+ # order the join takes them) evaluated with +owner+ on Active
782
+ # Record's relation of +klass+, as AssociationScope evaluates them.
783
+ def order_values(klass, scopes, owner)
784
+ Lyra::Projections::DefaultScopes.built do
785
+ scopes.flat_map do |scope|
786
+ relation = klass.unscoped
787
+ item = relation.instance_exec(owner, &scope) || relation
788
+ item.respond_to?(:order_values) ? item.order_values : []
789
+ end
790
+ end
791
+ end
792
+
793
+ # The join's order for a :through +reflection+ of +owner+, as
794
+ # [keys, reason]: +keys+ is [side, order value] pairs, most
795
+ # significant first, side :target or :intermediate (the record of
796
+ # the step next to the target), or [:inner, nil] for steps further
797
+ # on, whose order the intermediate records came in already. Active
798
+ # Record's join orders by the target's default scope, the
799
+ # intermediate models' default scopes, then the association scopes on
800
+ # the target (its own, then the source's) and then the intermediate
801
+ # steps' scopes; an unscope(:order) in a scope on the target takes the
802
+ # default scopes' and the steps' orders off. +reason+, when the order
803
+ # cannot be reproduced here: an order on a table that is neither the
804
+ # target's nor the next step's, or a default scope ordering a step
805
+ # further on while an association scope orders too.
806
+ def join_order(owner, reflection, head, default)
807
+ klass = reflection.klass
808
+ first, *further = reflection.chain.drop(1)
809
+ unordered = unscopes_order?(klass, reflection.constraints, owner)
810
+ target_default = default ? default.order_values : []
811
+ first_default = unordered || same_table?(first.klass, head) ? [] : default_order_values(first.klass)
812
+ first_scope = unordered ? [] : order_values(first.klass, first.scopes, owner)
813
+ on_target = order_values(klass, reflection.constraints.reverse, owner)
814
+
815
+ keys = target_default.map { [:target, _1] } + first_default.map { [:intermediate, _1] } +
816
+ on_target.map { [side_of(_1, klass, first.klass), _1] } + first_scope.map { [:intermediate, _1] }
817
+ further_default = !unordered && further.any? { !same_table?(_1.klass, head) && default_orders?(_1.klass) }
818
+ further_scope = !unordered && further.any? { order_values(_1.klass, _1.scopes, owner).any? }
819
+ keys << [:inner, nil] if further_default || further_scope
820
+
821
+ reason =
822
+ if keys.any? { _1.first.nil? }
823
+ "#{reflection.name} (an order on a table other than the target's and the next step's)"
824
+ elsif further_default && (on_target.any? || first_scope.any?)
825
+ "#{reflection.name} (a default scope orders a step past the first, and an association scope orders too)"
826
+ end
827
+ reason &&= "#{reason}: ES-NoProj cannot order its records as Active Record's join does"
828
+ [keys, reason]
829
+ end
830
+
831
+ # +pairs+ ([target, intermediate record]) sorted by +keys+
832
+ # (join_order), stably, one key at a time from the least significant.
833
+ def sort_pairs(pairs, keys, klass, through_klass)
834
+ return pairs if pairs.size < 2
835
+
836
+ keys.reverse_each do |side, value|
837
+ next if side == :inner
838
+
839
+ index = side == :target ? 0 : 1
840
+ copies = pairs.map { |pair| pair[index].clone }
841
+ back = {}.compare_by_identity
842
+ copies.each_with_index { |copy, i| back[copy] = pairs[i] }
843
+ sorted = Lyra::Projections::CachedRelation.new(index.zero? ? klass : through_klass, copies).order(value).to_a
844
+ pairs = sorted.map { |copy| back[copy] }
845
+ end
846
+ pairs
847
+ end
848
+
849
+ # Which record an order value of a scope on the target reads: the
850
+ # target's (its table, or a string, which CachedRelation reads as
851
+ # the target's), the next step's, or neither (nil).
852
+ def side_of(value, klass, first_klass)
853
+ return :target unless value.is_a?(Arel::Nodes::Ordering) && value.expr.is_a?(Arel::Attributes::Attribute)
854
+
855
+ table = value.expr.relation.respond_to?(:name) ? value.expr.relation.name.to_s : nil
856
+ return :target if table == klass.table_name
857
+ return :intermediate if table == first_klass.table_name
858
+
859
+ nil
860
+ end
861
+
862
+ # Whether one of +scopes+ takes the order off (unscope(:order)).
863
+ def unscopes_order?(klass, scopes, owner)
864
+ Lyra::Projections::DefaultScopes.built do
865
+ scopes.any? do |scope|
866
+ relation = klass.unscoped
867
+ item = relation.instance_exec(owner, &scope) || relation
868
+ item.respond_to?(:unscope_values) && item.unscope_values.include?(:order)
869
+ end
870
+ end
871
+ end
872
+
873
+ def default_order_values(klass)
874
+ return [] unless Lyra::Projections::DefaultScopes.any?(klass)
875
+
876
+ Lyra::Projections::DefaultScopes.built { klass.scope_for_association }.order_values
877
+ end
878
+
879
+ # The columns the intermediate models' default scopes condition
880
+ # (ThroughRelation#unscope).
881
+ def intermediate_columns(reflection, head)
882
+ reflection.chain.drop(1).flat_map do |step|
883
+ next [] if same_table?(step.klass, head) || !Lyra::Projections::DefaultScopes.any?(step.klass)
884
+
885
+ scope = Lyra::Projections::DefaultScopes.built { step.klass.scope_for_association }
886
+ names = []
887
+ scope.where_clause.send(:predicates).each do |node|
888
+ Arel.fetch_attribute(node) { |attribute| names << attribute.name.to_s }
889
+ end
890
+ names
891
+ end.uniq
892
+ end
893
+
894
+ # +records+ in primary key order (left as they are if the keys do
895
+ # not compare).
896
+ def by_primary_key(klass, records)
897
+ key = klass.primary_key
898
+ return records unless key.is_a?(String)
899
+
900
+ records.sort_by { |record| record.read_attribute(key) }
901
+ rescue ArgumentError, NoMethodError
902
+ records
903
+ end
904
+
905
+ def default_orders?(klass)
906
+ Lyra::Projections::DefaultScopes.any?(klass) &&
907
+ Lyra::Projections::DefaultScopes.built { klass.scope_for_association }.order_values.any?
908
+ end
909
+
910
+ def same_table?(klass, head) = klass.table_name == head.table_name
911
+
912
+ # The block run with +klass+'s default scopes off when +klass+ is on
913
+ # +head+'s table, where the join aliases it and applies none.
914
+ def aliased(klass, head, &block)
915
+ same_table?(klass, head) ? without_default_scope(klass, &block) : yield
916
+ end
917
+
918
+ # The block run with +klass+'s default scopes off, as
919
+ # Model.unscoped { } turns them off: the source associations read
920
+ # every target, and through_scoped narrows them by the default
921
+ # scopes as the join applies them.
922
+ def without_default_scope(klass, &block)
923
+ Lyra::Projections::DefaultScopes.any?(klass) ? klass.unscoped(&block) : yield
924
+ end
925
+ end
926
+
273
927
  def self.install!
274
928
  return if @installed
275
929
 
@@ -278,6 +932,7 @@ module Lyra
278
932
  ActiveRecord::Associations::HasManyAssociation.prepend(HasManyAssociationPatch)
279
933
  ActiveRecord::Associations::Association.prepend(ScopeForCreatePatch)
280
934
  ActiveRecord::Associations::CollectionProxy.prepend(CollectionProxyPatch)
935
+ Lyra::Associations::Maintenance.install!
281
936
 
282
937
  @installed = true
283
938
  end