orfeas_lyra 0.9.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +2021 -17
  3. data/README.md +22 -9
  4. data/Rakefile +40 -1
  5. data/app/controllers/lyra/application_controller.rb +127 -0
  6. data/app/controllers/lyra/dashboard_controller.rb +33 -31
  7. data/app/controllers/lyra/flow_controller.rb +32 -34
  8. data/app/controllers/lyra/privacy_controller.rb +57 -18
  9. data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
  10. data/app/views/lyra/dashboard/index.html.erb +1 -0
  11. data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
  12. data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
  13. data/app/views/lyra/privacy/policy.html.erb +7 -0
  14. data/app/views/lyra/privacy/rights.html.erb +78 -0
  15. data/config/privacy_policies.rb +0 -3
  16. data/config/routes.rb +1 -0
  17. data/lib/lyra/access_log.rb +187 -6
  18. data/lib/lyra/advisory_lock.rb +236 -0
  19. data/lib/lyra/aggregate.rb +1 -1
  20. data/lib/lyra/associations/habtm_links.rb +382 -19
  21. data/lib/lyra/associations/maintenance.rb +229 -0
  22. data/lib/lyra/attribution.rb +48 -0
  23. data/lib/lyra/bypass_events.rb +446 -46
  24. data/lib/lyra/command_handler.rb +19 -14
  25. data/lib/lyra/configuration.rb +237 -10
  26. data/lib/lyra/consistency/read_your_writes.rb +4 -2
  27. data/lib/lyra/correlation.rb +121 -0
  28. data/lib/lyra/domain_events.rb +4 -0
  29. data/lib/lyra/dual_view.rb +53 -16
  30. data/lib/lyra/encrypted_attributes.rb +5 -0
  31. data/lib/lyra/engine.rb +67 -0
  32. data/lib/lyra/erasure.rb +1011 -106
  33. data/lib/lyra/event.rb +35 -1
  34. data/lib/lyra/event_data.rb +38 -0
  35. data/lib/lyra/event_flow.rb +31 -15
  36. data/lib/lyra/event_store_adapter.rb +28 -0
  37. data/lib/lyra/event_window.rb +58 -0
  38. data/lib/lyra/genesis.rb +548 -44
  39. data/lib/lyra/interceptors/association_interceptor.rb +689 -34
  40. data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
  41. data/lib/lyra/internal.rb +26 -0
  42. data/lib/lyra/mode_sync.rb +51 -3
  43. data/lib/lyra/mode_transition.rb +257 -30
  44. data/lib/lyra/model_selection.rb +35 -0
  45. data/lib/lyra/own_connection.rb +17 -2
  46. data/lib/lyra/privacy/adapters/pam.rb +25 -2
  47. data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
  48. data/lib/lyra/privacy/interface.rb +27 -7
  49. data/lib/lyra/privacy/subject_links.rb +143 -0
  50. data/lib/lyra/privacy/subject_records.rb +369 -0
  51. data/lib/lyra/projection.rb +7 -1
  52. data/lib/lyra/projections/async_checkpoint.rb +32 -0
  53. data/lib/lyra/projections/async_projection_job.rb +166 -23
  54. data/lib/lyra/projections/cached_joins.rb +33 -18
  55. data/lib/lyra/projections/cached_projection.rb +74 -33
  56. data/lib/lyra/projections/cached_relation.rb +3175 -321
  57. data/lib/lyra/projections/default_scopes.rb +254 -0
  58. data/lib/lyra/projections/deferred_relation.rb +23 -3
  59. data/lib/lyra/projections/event_store_reader.rb +47 -8
  60. data/lib/lyra/projections/foreign_keys.rb +92 -0
  61. data/lib/lyra/projections/lazy_projection.rb +165 -27
  62. data/lib/lyra/projections/model_projection.rb +165 -14
  63. data/lib/lyra/projections/rebuild.rb +34 -6
  64. data/lib/lyra/projections/reference_links.rb +463 -72
  65. data/lib/lyra/projections/typed_values.rb +629 -0
  66. data/lib/lyra/projections/unique_indexes.rb +595 -0
  67. data/lib/lyra/projections/uniqueness.rb +784 -0
  68. data/lib/lyra/projections/value_digest.rb +93 -0
  69. data/lib/lyra/purpose_bound_reads.rb +912 -77
  70. data/lib/lyra/repair.rb +119 -28
  71. data/lib/lyra/retention.rb +114 -8
  72. data/lib/lyra/rights/consent_store.rb +415 -0
  73. data/lib/lyra/rights/erasure_guard.rb +135 -0
  74. data/lib/lyra/rights/pending.rb +74 -0
  75. data/lib/lyra/rights/queries.rb +269 -0
  76. data/lib/lyra/rights/store.rb +512 -0
  77. data/lib/lyra/rights/subject_keys.rb +224 -0
  78. data/lib/lyra/rights.rb +281 -0
  79. data/lib/lyra/saves_without_effect.rb +69 -0
  80. data/lib/lyra/stream_tail.rb +319 -0
  81. data/lib/lyra/strict_data_access.rb +916 -106
  82. data/lib/lyra/temporal.rb +4 -2
  83. data/lib/lyra/touch_registry.rb +404 -0
  84. data/lib/lyra/verification/bypass_workflow.rb +452 -42
  85. data/lib/lyra/verification/rights_workflows.rb +170 -0
  86. data/lib/lyra/version.rb +1 -1
  87. data/lib/lyra/visualization/event_graph.rb +1 -1
  88. data/lib/lyra/visualization/timeline.rb +2 -2
  89. data/lib/lyra.rb +43 -2
  90. data/lib/tasks/lyra_mode.rake +110 -8
  91. data/lib/tasks/lyra_projections.rake +43 -2
  92. metadata +34 -4
  93. data/lib/lyra/associations/event_aware.rb +0 -224
@@ -6,6 +6,11 @@ module Lyra
6
6
  # events. It used to answer such queries anyway, unfiltered or partly
7
7
  # filtered; a wrong answer is worse than an error.
8
8
  class UnsupportedQuery < StandardError
9
+ # The query was refused at an Arel condition (where(arel_table[:qty]
10
+ # .eq(2))), which CachedRelation#_exec_scope can still evaluate by
11
+ # building the scope as Active Record builds it.
12
+ attr_accessor :arel_condition
13
+
9
14
  def initialize(model_class, what)
10
15
  name = model_class.respond_to?(:name) ? model_class.name : model_class.to_s
11
16
  super(
@@ -31,74 +36,810 @@ module Lyra
31
36
  # relation = CachedRelation.new(User, records)
32
37
  # relation.where(status: "active").order(:name).limit(10)
33
38
  #
34
- # What an ES-NoProj query returns to the application: records narrowed to
35
- # the selected columns, each checked against its policy for the purpose
36
- # in scope (Lyra::PurposeBoundReads). A query here loads every record of
37
- # the model and filters in memory, so the check is made on the records
38
- # returned, not on the ones built; only the outermost call checks.
39
+ # What an ES-NoProj query returns to the application, and what it uses
40
+ # (Lyra::PurposeBoundReads): the same rule as ActiveRecord's relations.
41
+ #
42
+ # A query here filters eagerly, so each relation keeps what its query
43
+ # named (query_terms: the conditions of where, where.not and where!, the
44
+ # order keys, a subquery's own terms; select and group are kept already)
45
+ # and the check is made when the answer is asked for:
46
+ #
47
+ # - a record load (to_a, first, find_by, each, length, ...) has its named
48
+ # attributes checked without the consent step, then returns records
49
+ # narrowed to the selected columns, each checked against its policy,
50
+ # consent included, for what it holds and what the query named;
51
+ # - an aggregate, exists?, ids, pluck, pick, a bulk write or in_batches is
52
+ # checked whole, subject Model$*.
53
+ #
54
+ # Only the outermost call checks: the relations a query builds inside
55
+ # itself (a group's aggregates, exists? through find_by, a subquery while
56
+ # a where filters) are its machinery, not deliveries.
39
57
  module CachedRelationDelivery
40
- %i[to_a to_ary [] first last second third take take! first! last! find_by find_by!].each do |name|
58
+ %i[to_a to_ary [] first last second third take take! first! last!].each do |name|
41
59
  define_method(name) do |*args, **kwargs, &block|
42
60
  deliver { super(*args, **kwargs, &block) }
43
61
  end
44
62
  end
45
63
 
64
+ # With a block (and no id), Enumerable's find over the records the
65
+ # relation loads, as Active Record does: each checked first.
46
66
  def find(*args, &block)
47
- return super if block
67
+ return deliver { @records.to_a }.find(*args, &block) if block
48
68
 
49
69
  deliver { super }
50
70
  end
51
71
 
52
- def each(&block)
72
+ # select with a block is Enumerable's over the loaded records, as in
73
+ # Active Record, and returns an Array: the block sees only records the
74
+ # purpose check delivered (narrowed to the selection, each checked,
75
+ # consent included). It used to see the cached records, unchecked.
76
+ def select(*args, &block)
53
77
  return super unless block
78
+
79
+ deliver { @records.to_a }.select(&block)
80
+ end
81
+
82
+ def find_by(*args, **kwargs, &block)
83
+ deliver(conditions_term(args.first || kwargs)) { super }
84
+ end
85
+
86
+ def find_by!(*args, **kwargs, &block)
87
+ deliver(conditions_term(args.first || kwargs)) { super }
88
+ end
89
+
90
+ # Without a block, an Enumerator over the checked records, as
91
+ # ActiveRecord's each over its loaded records.
92
+ def each(&block)
54
93
  return super if Thread.current[:lyra_cached_delivery]
94
+ return to_enum(:each) { size } unless block
95
+
96
+ deliver { @records.to_a }.each(&block)
97
+ end
98
+
99
+ # Batch by batch, as Active Record loads them (CachedRelation#batch_slices:
100
+ # by primary key, whatever the relation's order): each batch delivered
101
+ # (checked) in a checking scope of its own, so the subject's rights are
102
+ # read afresh before each (Lyra::Rights.batches). find_each runs
103
+ # through here.
104
+ def find_in_batches(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, cursor: model_class.primary_key,
105
+ order: :asc, &block)
106
+ return super unless block
107
+
108
+ slices = batch_slices(of: batch_size, start: start, finish: finish, error_on_ignore: error_on_ignore,
109
+ cursor: cursor, order: order)
110
+ unless Lyra::PurposeBoundReads.checking?
111
+ slices.each { |slice| block.call(deliver { slice }) }
112
+ return nil
113
+ end
114
+
115
+ Lyra::PurposeBoundReads.batch_scopes do |next_batch|
116
+ Lyra::PurposeBoundReads.once do
117
+ slices.each do |slice|
118
+ batch = deliver { slice }
119
+ Lyra::PurposeBoundReads.outside_once { block.call(batch) }
120
+ ensure
121
+ next_batch.call
122
+ end
123
+ end
124
+ end
125
+ nil
126
+ end
127
+
128
+ # As ActiveRecord: these load the records.
129
+ def length = deliver { @records.to_a }.length
130
+ def present? = deliver { @records.to_a }.present?
131
+ def blank? = deliver { @records.to_a }.blank?
132
+
133
+ # As ActiveRecord: these count or test for existence. With a block,
134
+ # they load the records.
135
+ %i[size empty? ids].each do |name|
136
+ define_method(name) { aggregated { super() } }
137
+ end
138
+
139
+ # Kaminari's: of a page, the records the relation selects before the
140
+ # window, counted as any count is.
141
+ def total_count
142
+ refuse_count_without_count!
143
+ return window_source.count if window_source
55
144
 
56
- deliver { @records }.each(&block)
145
+ aggregated { super() }
57
146
  end
58
147
 
59
- def find_in_batches(batch_size: 1000, &block)
60
- deliver { @records }.each_slice(batch_size, &block)
148
+ %i[any? none? one? many?].each do |name|
149
+ define_method(name) do |&block|
150
+ next deliver { @records.to_a }.public_send(name, &block) if block
151
+
152
+ aggregated { super() }
153
+ end
154
+ end
155
+
156
+ # A batch loaded with its records (query_relation) counts and
157
+ # aggregates by its query, as Active Record runs these as SQL whether
158
+ # the relation is loaded or not.
159
+ def count(column_name = nil, &block)
160
+ return deliver { @records.to_a }.count(&block) if block
161
+ return query_relation.count(column_name) if query_relation
162
+
163
+ all = column_name.nil? || [:all, "*", "all"].include?(column_name)
164
+ aggregated(all ? [] : [[:value, column_name]]) { super }
165
+ end
166
+
167
+ def sum(column_name = nil, &block)
168
+ return deliver { @records.to_a }.sum(&block) if block
169
+ return query_relation.sum(column_name) if query_relation
170
+ return windowed_aggregate(:sum, column_name) if window_source && group_columns.empty?
171
+
172
+ aggregated([[:value, column_name]]) { super }
173
+ end
174
+
175
+ %i[average minimum maximum].each do |name|
176
+ define_method(name) do |column_name|
177
+ return query_relation.public_send(name, column_name) if query_relation
178
+ return windowed_aggregate(name, column_name) if window_source && group_columns.empty?
179
+
180
+ aggregated([[:value, column_name]]) { super(column_name) }
181
+ end
182
+ end
183
+
184
+ def calculate(operation, column_name = nil)
185
+ return query_relation.calculate(operation, column_name) if query_relation
186
+
187
+ all = operation.to_s == "count" && (column_name.nil? || [:all, "*", "all"].include?(column_name))
188
+ aggregated(all ? [] : [[:value, column_name]]) { super }
189
+ end
190
+
191
+ # A Hash of conditions, or an SQL fragment with its binds: what they
192
+ # name is used (the fragment is not evaluated here; see UnsupportedQuery).
193
+ def exists?(conditions = :none)
194
+ return query_relation.exists?(conditions) if query_relation
195
+ if window_source && ![:none, nil, false].include?(conditions) && !window_number(limit_value || 1, "limit").zero?
196
+ return window_matching(conditions).exists?
197
+ end
198
+
199
+ terms = case conditions
200
+ when Hash then conditions_term(conditions)
201
+ when Array then conditions.first.is_a?(String) ? [[:value, conditions.first]] : []
202
+ else []
203
+ end
204
+ aggregated(terms) { super }
61
205
  end
62
206
 
63
207
  def pluck(*column_names)
64
- Lyra::PurposeBoundReads.check_columns(model_class, column_names)
208
+ aggregated([[:value, column_names]]) { super }
209
+ end
210
+
211
+ def pick(*column_names)
212
+ aggregated([[:value, column_names]]) { super }
213
+ end
214
+
215
+ # Bulk writes (method_missing below runs them): their filter is a use,
216
+ # and so is what update_all's SET expressions read.
217
+ # A batch loaded with its records writes with its query's
218
+ # (query_relation), as Active Record's update_all and delete_all run
219
+ # the relation's query; destroy_all destroys the records loaded.
220
+ %i[update_all delete_all touch_all destroy_all].each do |name|
221
+ define_method(name) do |*args, **kwargs, &block|
222
+ if name != :destroy_all && (query = query_relation)
223
+ return query.public_send(name, *args, **kwargs, &block)
224
+ end
225
+
226
+ unless Thread.current[:lyra_cached_delivery]
227
+ reads = name == :update_all ? Lyra::PurposeBoundReads::RelationCheck.set_reads(args.first || kwargs) : nil
228
+ query_check(:all, reads ? [[:value, reads]] : [])
229
+ end
230
+ super(*args, **kwargs, &block)
231
+ end
232
+ end
233
+
234
+ # Checked once for the relation; the application's block is its own
235
+ # code, checked each time.
236
+ def in_batches(*args, **kwargs, &block)
237
+ return super unless block && Lyra::PurposeBoundReads.checking?
238
+
239
+ Lyra::PurposeBoundReads.batch_scopes do |next_batch|
240
+ Lyra::PurposeBoundReads.once do
241
+ query_check(:all)
242
+ super(*args, **kwargs) do |batch|
243
+ Lyra::PurposeBoundReads.outside_once { block.call(batch) }
244
+ ensure
245
+ next_batch.call
246
+ end
247
+ end
248
+ end
249
+ end
250
+
251
+ # As ActiveRecord's records: the records the query loads, checked.
252
+ def records
253
+ deliver { @records.to_a }
254
+ end
255
+
256
+ # As ActiveRecord's: it loads the records (each shown is checked).
257
+ def inspect
258
+ shown = deliver { @records.to_a.first(11) }
259
+ entries = shown.first(10).map(&:inspect)
260
+ entries << "..." if shown.size > 10
261
+ "#<#{self.class.name} [#{entries.join(', ')}]>"
262
+ end
263
+
264
+ def to_s
265
+ inspect
266
+ end
267
+
268
+ # Filtering: what it names is recorded on the relation it returns.
269
+ # Relations evaluated while it filters (a subquery) are part of it.
270
+ def where(*args, **kwargs)
271
+ relation = inside_query { super }
272
+ conditions = args.empty? ? kwargs : args.first
273
+ return relation if conditions.blank? || conditions == :chain || !relation.is_a?(CachedRelation) || relation.equal?(self)
274
+
275
+ relation.continue_plan(self, [:where, conditions]) if conditions.is_a?(Hash)
276
+ relation.add_query_terms(conditions.is_a?(String) ? [[:value, conditions]] : conditions_term(conditions))
277
+ end
278
+
279
+ # It continues the plan: unscope and merge can take it off (they
280
+ # refused, after a where.not).
281
+ def not(conditions)
282
+ relation = inside_query { super }
283
+ relation.continue_plan(self, [:not, conditions]) if conditions.is_a?(Hash) && !relation.equal?(self)
284
+ relation.add_query_terms(conditions_term(conditions))
285
+ end
286
+
287
+ def order(*args)
288
+ relation = super
289
+ return relation if args.empty? || relation.equal?(self)
290
+
291
+ relation.continue_plan(self, [:order, args])
292
+ relation.add_query_terms([[:value, args]])
293
+ end
294
+
295
+ def reorder(*args)
296
+ relation = super
297
+ relation.continue_plan(self, [:reorder, args])
298
+ args.compact.empty? ? relation : relation.add_query_terms([[:value, args]])
299
+ end
300
+
301
+ def order!(*args)
65
302
  super
303
+ args.empty? ? self : add_query_terms!([[:value, args]])
304
+ end
305
+
306
+ # The conditions the two sides do not share are the OR (Active Record's
307
+ # or keeps the shared ones as they are): kept as or_terms, for merge.
308
+ def or(other)
309
+ relation = super
310
+ mine = query_terms
311
+ theirs = other.query_terms
312
+ common = mine & theirs
313
+ relation.add_query_terms(theirs)
314
+ relation.add_or_terms((mine - common) + (theirs - common))
66
315
  end
67
316
 
68
317
  private
69
318
 
70
- def deliver
71
- return yield if Thread.current[:lyra_cached_delivery]
319
+ # The records batched without loading them (CachedRelation#batch_records
320
+ # with counting: the sizes of the find_each and find_in_batches
321
+ # enumerators, in_batches' batches of keys) are those a right in force
322
+ # does not leave out, as a count's are, taken out before the limit: a
323
+ # restricted record was counted in the sizes. Batches that load their
324
+ # records keep them, and each is checked as it is delivered (a
325
+ # restricted record raises, as a load of it does in Active Record).
326
+ def batch_kept(records)
327
+ return records if Thread.current[:lyra_cached_delivery]
72
328
 
329
+ Lyra::PurposeBoundReads.rights_kept_records(model_class, lyra_named_fields, records) || records
330
+ end
331
+
332
+ def conditions_term(conditions)
333
+ conditions.present? ? [[:conditions, conditions]] : []
334
+ end
335
+
336
+ # Nested deliveries inside the block are the query's own work.
337
+ def inside_query
338
+ previous = Thread.current[:lyra_cached_delivery]
73
339
  Thread.current[:lyra_cached_delivery] = true
74
- result = yield
75
- Thread.current[:lyra_cached_delivery] = false
340
+ yield
341
+ ensure
342
+ Thread.current[:lyra_cached_delivery] = previous
343
+ end
344
+
345
+ def query_check(kind, extra_terms = [])
346
+ return unless Lyra::PurposeBoundReads.checking?
347
+
348
+ names = lyra_query_names(extra_terms)
349
+ named = names.result
350
+ Lyra::PurposeBoundReads.check_query(model_class, named, kind, names.whole, joined: joined_classes)
351
+ named
352
+ end
353
+
354
+ # An aggregate, exists?, pluck: checked whole, then run as one query,
355
+ # over the records a right in force does not leave out
356
+ # (Lyra::Rights::Queries; a record whose people cannot be read raises).
357
+ def aggregated(extra_terms = [])
358
+ return yield if Thread.current[:lyra_cached_delivery]
359
+ return yield unless Lyra::PurposeBoundReads.checking?
360
+
361
+ Lyra::PurposeBoundReads.once do
362
+ named = query_check(:all, extra_terms)
363
+ kept = window_source ? window_kept(named) : Lyra::PurposeBoundReads.rights_kept_records(model_class, named, @records)
364
+ kept ? inside_query { with_records(kept) { yield } } : inside_query { yield }
365
+ end
366
+ end
367
+
368
+ # A window's records under a right in force: the window taken from the
369
+ # records of the relation it was taken from that the right does not
370
+ # leave out, as the query's SQL leaves them out before its LIMIT and
371
+ # OFFSET (the window was taken first, and a record left out of it
372
+ # shortened it). nil when the right leaves out none.
373
+ def window_kept(named)
374
+ kept = Lyra::PurposeBoundReads.rights_kept_records(model_class, named, window_source.raw_records.to_a)
375
+ return nil unless kept
376
+
377
+ kept = kept.drop(offset_value.nil? ? 0 : window_number(offset_value, "offset"))
378
+ limit_value.nil? ? kept : kept.first(window_number(limit_value, "limit"))
379
+ end
380
+
381
+ # SUM, AVG, MIN or MAX of a relation with a limit or offset, as Active
382
+ # Record computes it: one query whose LIMIT and OFFSET apply to its one
383
+ # result row, not to the records. So the aggregate is over every record
384
+ # the relation selects before its window, or, when the window leaves
385
+ # out that row (a limit of 0, an offset), 0 for SUM and nil for the
386
+ # others. It used to be computed over the window's records. The records
387
+ # before the window are counted through their own relation, so that the
388
+ # purpose check and the rights in force apply as to any aggregate.
389
+ def windowed_aggregate(name, column_name)
390
+ if window_leaves_out_one_row?
391
+ aggregated([[:value, column_name]]) { name == :sum ? 0 : nil }
392
+ else
393
+ window_source.public_send(name, column_name)
394
+ end
395
+ end
396
+
397
+ # The block run with +records+ as this relation's records.
398
+ def with_records(records)
399
+ saved = @records
400
+ @records = records
401
+ yield
402
+ ensure
403
+ @records = saved
404
+ end
405
+
406
+ # A record load: its named attributes checked first, then each record.
407
+ def deliver(extra_terms = [])
408
+ return yield if Thread.current[:lyra_cached_delivery]
409
+
410
+ named = query_check(:load, extra_terms)
411
+ names = named ? named.fetch(model_class, []) : []
412
+ result = inside_query { yield }
76
413
  case result
77
- when Array then result.map { |record| delivered(record) }
78
- when ActiveRecord::Base then delivered(result)
414
+ when Array then result.map { |record| delivered(record, names) }
415
+ when ActiveRecord::Base then delivered(result, names)
79
416
  else result
80
417
  end
81
- ensure
82
- Thread.current[:lyra_cached_delivery] = false
83
418
  end
84
419
 
85
- def delivered(record)
420
+ def delivered(record, names)
86
421
  return record unless record.is_a?(ActiveRecord::Base)
87
422
 
88
423
  if selected_columns
424
+ whole = record
89
425
  values = record.attributes_before_type_cast.slice(*selected_columns)
90
- record = Lyra::PurposeBoundReads.internal { model_class.instantiate(values) }
426
+ record = Lyra::PurposeBoundReads.internal(INTERNAL) { model_class.instantiate(values) }
427
+ # Its people, read from the whole record before it was narrowed
428
+ # (the table under ES-NoProj has no row to read them from).
429
+ if Lyra::PurposeBoundReads.rights_in_force?
430
+ Lyra::PurposeBoundReads.internal(INTERNAL) { Lyra::Rights::SubjectKeys.remember(record, whole) }
431
+ end
91
432
  end
92
- Lyra::PurposeBoundReads.check(record)
433
+ Lyra::PurposeBoundReads.check(record, names)
93
434
  record
94
435
  end
95
436
  end
96
437
 
438
+ # A window (CachedRelation#limit, #offset, #page) is taken after the
439
+ # relation's conditions and order, as SQL takes LIMIT and OFFSET after
440
+ # WHERE and ORDER BY, whichever was called first. So a query method
441
+ # called on a window applies to the relation the window was taken from
442
+ # (its window_source), and the window is taken again from the result.
443
+ #
444
+ # How each relation was built, so that a write can build it again from
445
+ # the event store as it is then (CachedRelation#records_now), as Active
446
+ # Record's update_all and delete_all run their query when they are
447
+ # called. A relation holds the records it selected when it was built;
448
+ # writing those wrote a record destroyed since, missed one that matches
449
+ # since, and kept a window (limit, offset) that Active Record takes again.
450
+ #
451
+ # The recipe is [root, steps]: +root+ a Proc that reads the relation it
452
+ # started from afresh (every record of the model, an association's, a
453
+ # foreign-key lookup), set where such a relation is made
454
+ # (EventStoreReader, DeferredRelation, the :through associations), and
455
+ # +steps+ the query methods called on it since, [name, args, kwargs,
456
+ # block], each recorded by the wrapper below when it returns a relation
457
+ # of the same model. A relation made any other way (a copy, a batch's
458
+ # records, a relation of given records) has none, and a write on it
459
+ # writes the records it holds, those its stream does not say destroyed
460
+ # (CachedRelation#locked_records_now). A recipe belongs to the relation it
461
+ # was recorded on: a copy does not inherit it.
462
+ module Rebuildable
463
+ # The query methods: each returns the relation narrowed, ordered,
464
+ # windowed or otherwise derived from this one. Those that return the
465
+ # relation itself unchanged (lock, annotate, all) are no step; a bang
466
+ # method changes it in place, and is one.
467
+ STEPS = %i[create_with create_with! distinct eager_load except extending extending! group in_order_of includes
468
+ joins left_joins left_outer_joins limit limit! merge merge! narrowed_by none not offset only or order
469
+ order! padding page per preload readonly references reorder reselect reverse_order reverse_order!
470
+ rewhere select unscope unscope! unscoped where where! where_associated where_missing
471
+ without_count].freeze
472
+ LOADS = %i[to_a to_ary each records length present? blank? []].freeze
473
+ QUERY_FINDERS = %i[find find_by find_by! find_sole_by find_or_create_by find_or_create_by! find_or_initialize_by
474
+ create_or_find_by create_or_find_by!].freeze
475
+
476
+ # A module of wrappers for +klass+ (CachedRelation and its subclasses,
477
+ # whose own overrides come before CachedRelation's in the lookup).
478
+ def self.wrappers
479
+ Module.new do
480
+ STEPS.each do |name|
481
+ bang = name.end_with?("!")
482
+ define_method(name) do |*args, **kwargs, &block|
483
+ # A batch loaded with its records (CachedRelation#query_relation):
484
+ # its query is another relation's, as Active Record's
485
+ # load_records keeps the relation's query.
486
+ if !bang && (query = query_relation)
487
+ return query.public_send(name, *args, **kwargs, &block)
488
+ end
489
+ # As Active Record's: a loaded relation cannot be changed in
490
+ # place (where! changed its records).
491
+ raise ActiveRecord::UnmodifiableRelation if bang && loaded_records?
492
+
493
+ recipe = rebuild_recipe
494
+ # Copied before the call: what the caller changes afterwards
495
+ # (codes << "a2") does not reach the relation, nor its write.
496
+ step = Rebuildable.step(name, args, kwargs, block) if recipe
497
+ result = super(*args, **kwargs, &block)
498
+ Rebuildable.record(self, result, recipe, step)
499
+ result
500
+ end
501
+ end
502
+
503
+ # The finders that run their own query on a loaded relation in
504
+ # Active Record (where(...).take, where(...).sole): a batch loaded
505
+ # with its records (query_relation) finds by its query, not among
506
+ # the records it holds. find with a block is Enumerable's, over them.
507
+ QUERY_FINDERS.each do |name|
508
+ define_method(name) do |*args, **kwargs, &block|
509
+ if (name != :find || block.nil?) && (query = query_relation)
510
+ return query.public_send(name, *args, **kwargs, &block)
511
+ end
512
+
513
+ super(*args, **kwargs, &block)
514
+ end
515
+ end
516
+
517
+ # What loads an Active Record relation: after it, its destroy_all
518
+ # and update work on the records loaded (CachedRelation#loaded_records?).
519
+ LOADS.each do |name|
520
+ define_method(name) do |*args, **kwargs, &block|
521
+ result = super(*args, **kwargs, &block)
522
+ @loaded_records = true unless Thread.current[:lyra_cached_delivery] || (name == :each && block.nil?)
523
+ result
524
+ end
525
+ end
526
+
527
+ # A model's scopes and class methods, called on the relation. They
528
+ # are the application's code, run once, when the relation is built,
529
+ # as Active Record runs them: they are no step. What they build is
530
+ # recorded by the query methods they call (the conditions as they
531
+ # evaluated them, Time.now or the ids they plucked included); a
532
+ # result built otherwise has no recipe.
533
+ def method_missing(name, *args, **kwargs, &block)
534
+ result = super
535
+ if result.is_a?(CachedRelation) && !result.equal?(self)
536
+ result.instance_variable_set(:@loaded_records, nil)
537
+ end
538
+ result
539
+ end
540
+ end
541
+ end
542
+
543
+ # Stands for a value that cannot be copied safely: the recipe is
544
+ # dropped, and a write on the relation writes the records it holds.
545
+ UNSAFE = Object.new.freeze
546
+
547
+ # Steps that do not change which records are selected (methods added,
548
+ # associations to preload, attributes for new records): a relation
549
+ # built again without them selects the same records, so they are not
550
+ # recorded, a block included (extending { ... }).
551
+ NEUTRAL = %i[extending extending! readonly includes preload eager_load references create_with
552
+ create_with! strict_loading].freeze
553
+
554
+ # Steps that take conditions or orders off (unscope, rewhere, except,
555
+ # only) rebuild the relation from its plan (CachedRelation#replay_plan):
556
+ # from where it started (read again: a Proc, or a relation with its own
557
+ # recipe) through the query methods of the plan, each recorded as it
558
+ # runs. The relation they return has that recipe, which is the one
559
+ # kept: recording the step itself would replay it on a relation built
560
+ # again, which has no plan, and the write fell back to the records held
561
+ # when the relation was built.
562
+ PLAN_STEPS = %i[unscope unscope! rewhere except only].freeze
563
+
564
+ # [name, args, kwargs], the arguments deep-copied and frozen; nil when
565
+ # one cannot be copied (an object of the application's), or a block was
566
+ # given (the application's code, which a replay would run again).
567
+ # NEUTRAL: :neutral.
568
+ def self.step(name, args, kwargs, block)
569
+ return :neutral if NEUTRAL.include?(name)
570
+ return nil if block
571
+
572
+ args = copy(args)
573
+ kwargs = copy(kwargs)
574
+ return nil if args.equal?(UNSAFE) || kwargs.equal?(UNSAFE)
575
+
576
+ [name, args, kwargs]
577
+ end
578
+
579
+ # A copy of +value+ that changes to the caller's objects do not reach
580
+ # (frozen with +freeze+), or UNSAFE. Immutable values, classes, records,
581
+ # Arel nodes and relations (a CachedRelation is built again from its
582
+ # own recipe; an Active Record relation is copied) are kept as they are.
583
+ def self.copy(value, freeze: true)
584
+ case value
585
+ when nil, true, false, Symbol, Numeric, Module, Date, Time, ActiveSupport::TimeWithZone,
586
+ ActiveSupport::Duration, ActiveRecord::Base, CachedRelation
587
+ value
588
+ when String
589
+ freeze ? (value.frozen? ? value : value.dup.freeze) : value.dup
590
+ when Array
591
+ copied = []
592
+ value.each do |item|
593
+ item = copy(item, freeze: freeze)
594
+ return UNSAFE if item.equal?(UNSAFE)
595
+
596
+ copied << item
597
+ end
598
+ freeze ? copied.freeze : copied
599
+ when Hash
600
+ copied = {}
601
+ value.each do |key, item|
602
+ key = copy(key, freeze: true)
603
+ item = copy(item, freeze: freeze)
604
+ return UNSAFE if key.equal?(UNSAFE) || item.equal?(UNSAFE)
605
+
606
+ copied[key] = item
607
+ end
608
+ freeze ? copied.freeze : copied
609
+ when Set
610
+ items = copy(value.to_a, freeze: freeze)
611
+ return UNSAFE if items.equal?(UNSAFE)
612
+
613
+ freeze ? Set.new(items).freeze : Set.new(items)
614
+ when Range
615
+ first = copy(value.begin, freeze: true)
616
+ last = copy(value.end, freeze: true)
617
+ return UNSAFE if first.equal?(UNSAFE) || last.equal?(UNSAFE)
618
+
619
+ Range.new(first, last, value.exclude_end?)
620
+ when ActiveRecord::Relation
621
+ value.spawn
622
+ when Arel::Nodes::Node, Arel::Attributes::Attribute
623
+ value
624
+ else
625
+ UNSAFE
626
+ end
627
+ end
628
+
629
+ # +result+ of +step+ called on +receiver+ (whose recipe was +recipe+
630
+ # before the call) gets +recipe+ and the step (none if the step could
631
+ # not be recorded). A result that is the receiver itself is a step only
632
+ # for a bang method; a new relation whose receiver had no recipe keeps
633
+ # what it was given when made.
634
+ def self.record(receiver, result, recipe, step)
635
+ return unless result.is_a?(CachedRelation) && result.model_class == receiver.model_class
636
+
637
+ if step == :neutral
638
+ result.instance_variable_set(:@loaded_records, nil) unless result.equal?(receiver)
639
+ result.rebuild_recipe = recipe if recipe
640
+ return
641
+ end
642
+ if step && PLAN_STEPS.include?(step.first)
643
+ result.instance_variable_set(:@loaded_records, nil) unless result.equal?(receiver)
644
+ return
645
+ end
646
+
647
+ if result.equal?(receiver)
648
+ name = step ? step.first : nil
649
+ return if name && !name.to_s.end_with?("!")
650
+
651
+ result.rebuild_recipe = recipe && step && [recipe[0], recipe[1] + [step]]
652
+ else
653
+ # A new relation has loaded nothing yet (a copy holds the flag).
654
+ result.instance_variable_set(:@loaded_records, nil)
655
+ return unless recipe
656
+
657
+ result.rebuild_recipe = step && [recipe[0], recipe[1] + [step]]
658
+ end
659
+ end
660
+
661
+ # The relation +recipe+ builds now, or nil when a step does not give a
662
+ # relation. Each step gets a fresh copy of its arguments; a relation
663
+ # among them (merge, or, a subquery) is built again too. A first step
664
+ # that names primary keys reads only those records (keyed).
665
+ def self.rebuild(recipe)
666
+ root, steps = recipe
667
+ relation = keyed(root, steps.first) || root.call
668
+ steps.each do |name, args, kwargs|
669
+ return nil unless relation.is_a?(CachedRelation)
670
+
671
+ args = copy(args, freeze: false).map { fresh(_1) }
672
+ kwargs = copy(kwargs, freeze: false).transform_values { fresh(_1) }
673
+ relation = relation.public_send(name, *args, **kwargs)
674
+ end
675
+ relation.is_a?(CachedRelation) ? relation : nil
676
+ end
677
+
678
+ def self.fresh(arg)
679
+ case arg
680
+ when CachedRelation
681
+ recipe = arg.rebuild_recipe
682
+ (recipe && rebuild(recipe)) || arg
683
+ when Hash
684
+ arg.any? { |_, value| value.is_a?(CachedRelation) } ? arg.transform_values { fresh(_1) } : arg
685
+ else arg
686
+ end
687
+ end
688
+
689
+ # Every record of a model, read again (EventStoreReader.relation).
690
+ AllRecords = Struct.new(:model_class) do
691
+ def call = EventStoreReader.relation(model_class)
692
+ end
693
+
694
+ # The records of a model with these primary keys, read again
695
+ # (EventStoreReader.keyed).
696
+ Keyed = Struct.new(:model_class, :ids) do
697
+ def call = EventStoreReader.keyed(model_class, ids)
698
+ end
699
+
700
+ # When +root+ is every record of the model and +first+ a where that
701
+ # names primary keys (where(id: 5), where(id: [1, 2])), the records
702
+ # with those keys only: the step that follows selects the same records
703
+ # from them as from every record, at the cost of the records written,
704
+ # not of the model's size.
705
+ def self.keyed(root, first)
706
+ return nil unless root.is_a?(AllRecords) && first && first[0] == :where
707
+
708
+ args = first[1]
709
+ conditions = args.empty? ? first[2] : (args.size == 1 && args.first.is_a?(Hash) ? args.first : nil)
710
+ return nil unless conditions.is_a?(Hash)
711
+
712
+ key = root.model_class.primary_key
713
+ value = conditions.key?(key) ? conditions[key] : conditions[key.to_sym]
714
+ values = Array(value)
715
+ return nil if value.nil? || values.empty? || !values.all? { _1.is_a?(Integer) || _1.is_a?(String) }
716
+
717
+ Keyed.new(root.model_class, values).call
718
+ end
719
+ end
720
+
721
+ module CachedRelationWindow
722
+ %i[not where_missing where_associated order reorder reverse_order distinct uniq reselect group joins left_joins
723
+ left_outer_joins create_with rewhere in_order_of].each do |name|
724
+ define_method(name) do |*args, **kwargs, &block|
725
+ return super(*args, **kwargs, &block) unless window_source
726
+
727
+ rewindow(window_source.public_send(name, *args, **kwargs, &block))
728
+ end
729
+ end
730
+
731
+ # where with no conditions is the chain (where.not, where.missing) of
732
+ # this relation, which comes back here.
733
+ def where(*args, **kwargs, &block)
734
+ return super if window_source.nil? || (args.empty? && kwargs.empty?) || args.first == :chain
735
+
736
+ rewindow(window_source.where(*args, **kwargs, &block))
737
+ end
738
+
739
+ def select(*args, &block)
740
+ return super if window_source.nil? || block
741
+
742
+ rewindow(window_source.select(*args))
743
+ end
744
+
745
+ # merge of an Array is of the loaded records (the window's). A limit or
746
+ # offset the other relation has (an event-store relation's window, or
747
+ # an Active Record relation's limit_value and offset_value) replaces
748
+ # this one's, as Active Record's merge takes them; the rest of it is
749
+ # merged into the relation this window was taken from. An Active Record
750
+ # relation's limit used to be taken inside this window.
751
+ def merge(other, *rest)
752
+ relation = other.is_a?(CachedRelation) || other.is_a?(ActiveRecord::Relation)
753
+ window = relation ? merged_window(other) : nil
754
+ return super if window.nil? && (window_source.nil? || !relation)
755
+ return rewindow(window_source.merge(other, *rest)) if window.nil?
756
+
757
+ limit, offset, unwindowed = window
758
+ (window_source || self).merge(unwindowed)
759
+ .windowed_by(limit.nil? ? limit_value : limit, offset.nil? ? offset_value : offset,
760
+ pagination: @pagination)
761
+ end
762
+
763
+ # Kaminari's Model.page extends the relation with its modules:
764
+ # CachedRelation answers what they define itself (as a page); any
765
+ # other module extends the relation the window was taken from.
766
+ def extending(*modules, &block)
767
+ modules = modules.flatten
768
+ modules << Module.new(&block) if block
769
+ kaminari, others = modules.partition { CachedRelation.kaminari_module?(_1) }
770
+ relation =
771
+ if others.empty? then clone
772
+ elsif window_source then rewindow(window_source.extending(*others))
773
+ else super(*others)
774
+ end
775
+ relation.instance_variable_set(:@pagination, @pagination || {}) if kaminari.any?
776
+ relation
777
+ end
778
+
779
+ # unscope(:limit) and unscope(:offset) take the window off; anything
780
+ # else is taken off the relation the window was taken from.
781
+ def unscope(*args)
782
+ return super unless window_source
783
+
784
+ window_args, rest = args.flatten.partition { |arg| %w[limit offset].include?(arg.to_s) && !arg.is_a?(Hash) }
785
+ dropped = window_args.map(&:to_s)
786
+ source = rest.empty? ? window_source : window_source.unscope(*rest)
787
+ source.windowed_by(dropped.include?("limit") ? nil : limit_value, dropped.include?("offset") ? nil : offset_value,
788
+ pagination: @pagination)
789
+ end
790
+
791
+ def only(*keeps)
792
+ return super unless window_source
793
+
794
+ kept = keeps.flatten.map(&:to_sym)
795
+ window_source.only(*(kept - %i[limit offset]))
796
+ .windowed_by(kept.include?(:limit) ? limit_value : nil, kept.include?(:offset) ? offset_value : nil,
797
+ pagination: @pagination)
798
+ end
799
+
800
+ # As Active Record's: two relations with the same window are one query
801
+ # with both conditions (OR) and that window; with different windows,
802
+ # an ArgumentError.
803
+ def or(other)
804
+ theirs = other.is_a?(CachedRelation) ? [other.limit_value, other.offset_value] : nil
805
+ mine = [limit_value, offset_value]
806
+ return super if theirs.nil? || (window_source.nil? && theirs == mine)
807
+ unless theirs == mine
808
+ incompatible = []
809
+ incompatible << :limit unless theirs&.first == mine.first
810
+ incompatible << :offset unless theirs&.last == mine.last
811
+ raise ArgumentError, "Relation passed to #or must be structurally compatible. " \
812
+ "Incompatible values: #{incompatible.inspect}"
813
+ end
814
+
815
+ rewindow(window_source.or(other.window_source))
816
+ end
817
+
818
+ private
819
+
820
+ # [limit, offset, the other relation without them] when +other+ has a
821
+ # limit or offset, else nil.
822
+ def merged_window(other)
823
+ if other.is_a?(CachedRelation)
824
+ return other.window_source && [other.limit_value, other.offset_value, other.window_source]
825
+ end
826
+ return nil if other.limit_value.nil? && other.offset_value.nil?
827
+
828
+ [other.limit_value, other.offset_value, DefaultScopes.built { other.unscope(:limit, :offset) }]
829
+ end
830
+
831
+ def rewindow(relation)
832
+ return relation unless relation.is_a?(CachedRelation)
833
+
834
+ relation.windowed_by(limit_value, offset_value, pagination: @pagination)
835
+ end
836
+ end
837
+
97
838
  class CachedRelation
98
839
  include Enumerable
99
840
  include CachedJoins
100
841
 
101
- attr_reader :model_class, :records
842
+ attr_reader :model_class
102
843
  # Columns a select(...) asked for: the records returned are narrowed to
103
844
  # them, as SQL would return them, so that a purpose-bound read checks
104
845
  # what the query selected (Lyra::PurposeBoundReads). Carried through
@@ -123,6 +864,152 @@ module Lyra
123
864
 
124
865
  def group_columns = @group_columns || []
125
866
 
867
+ # What the query this relation answers named, as [kind, term] pairs
868
+ # (CachedRelationDelivery records them): [:conditions, hash] or
869
+ # [:value, columns or an SQL fragment]. Carried through every relation
870
+ # derived from this one, for the purpose check.
871
+ attr_writer :query_terms
872
+
873
+ def query_terms = @query_terms || []
874
+
875
+ # The order the records are in, as [{ column => :asc/:desc }]: an
876
+ # order adds to it and reorder replaces it, as in ActiveRecord (a
877
+ # default scope's order comes first). Carried through every relation
878
+ # derived from this one.
879
+ attr_writer :ordering
880
+
881
+ def ordering = @ordering || []
882
+
883
+ # The default scope (an ActiveRecord relation) these records were
884
+ # narrowed by (DefaultScopes.narrow), carried through every relation
885
+ # derived from this one.
886
+ attr_accessor :default_scope_applied
887
+
888
+ # How the records were selected, while that was conditions and orders
889
+ # only: [base, steps]. +base+ is the relation (or a Proc building it)
890
+ # the selection started from; +steps+ are [:scope, relation, what] (a
891
+ # default scope, or an ActiveRecord relation merged in), [:where, hash],
892
+ # [:order, args] and [:reorder, args]. unscope and rewhere replay it
893
+ # without what they take off, as ActiveRecord rebuilds its query.
894
+ # where, order, reorder and merge carry it on; any other query method
895
+ # (limit, where.not, an SQL fragment, a join) leaves it behind, and an
896
+ # unscope after it that would change the answer is refused.
897
+ attr_accessor :plan
898
+
899
+ # This relation's plan: +source+'s plus +step+ (nothing if +source+
900
+ # has none).
901
+ def continue_plan(source, step)
902
+ @plan = source.plan && [source.plan[0], source.plan[1] + [step]]
903
+ self
904
+ end
905
+
906
+ # A limit, offset or page was applied: what the relation holds is a
907
+ # window of its selection.
908
+ attr_writer :windowed
909
+
910
+ def windowed? = @windowed ? true : false
911
+
912
+ # Attributes create_with gives a record built through the relation.
913
+ attr_writer :create_with_attributes
914
+
915
+ def create_with_attributes = @create_with_attributes || {}
916
+
917
+ # Set on a relation whose records' locks are held and whose streams
918
+ # were read live in the current transaction (update_counters' groups):
919
+ # its bulk write does not take or read them again.
920
+ attr_writer :live_checked
921
+ protected :live_checked=
922
+
923
+ # How to build this relation again from the event store (Rebuildable),
924
+ # or nil. Only the relation it was set on has it: a copy (clone, dup,
925
+ # spawn) holds the same ivar, pointing at the original.
926
+ def rebuild_recipe
927
+ @rebuild && @rebuild.first.equal?(self) ? @rebuild.last : nil
928
+ end
929
+
930
+ def rebuild_recipe=(recipe)
931
+ @rebuild = recipe && [self, recipe]
932
+ end
933
+
934
+ # Whether the application has loaded the records (to_a, each, ...), as
935
+ # Active Record's loaded?: its destroy_all and update then work on
936
+ # those records, not on the query run again.
937
+ def loaded_records? = @loaded_records ? true : false
938
+
939
+ # Active Record's loaded? (and loaded): true once the records are
940
+ # loaded (load, to_a, each, records, ...); before, false on a relation
941
+ # made new (Model.all) and nil on one derived from another (where,
942
+ # order, ...; Rebuildable.record) or reset, as Active Record answers.
943
+ def loaded = @loaded_records ? true : @loaded_records
944
+ alias_method :loaded?, :loaded
945
+
946
+ # As Active Record's load: the records loaded, as any read delivers
947
+ # them (the purpose check, each record checked, a restricted one
948
+ # raising), each given to the block, and the relation returned. A
949
+ # relation loaded already is not loaded again (nor the block called),
950
+ # as in Active Record. It was refused (UnsupportedQuery).
951
+ def load(&block)
952
+ unless loaded_records?
953
+ loaded = records
954
+ loaded.each(&block) if block
955
+ @loaded_records = true
956
+ end
957
+ self
958
+ end
959
+
960
+ # As Active Record's reset: the relation forgets its records and is
961
+ # read again from the event store (Rebuildable), as a relation built at
962
+ # this moment is; it is no longer loaded. A relation that cannot be
963
+ # built again (one of given records) is refused: Active Record runs
964
+ # its query again, and the records it holds are not that query's.
965
+ def reset
966
+ recipe = rebuild_recipe
967
+ fresh = recipe && evaluate_relation(recipe)
968
+ unless fresh
969
+ raise UnsupportedQuery.new(model_class, "reset of a relation that cannot be read again from the event store")
970
+ end
971
+
972
+ replace_with!(fresh)
973
+ @loaded_records = nil
974
+ self
975
+ end
976
+
977
+ # As Active Record's reload: reset, then load. It was refused.
978
+ def reload
979
+ reset
980
+ load
981
+ end
982
+
983
+ # The relation whose query this one's loaded records were selected by
984
+ # (Active Record's load_records): an in_batches batch loaded with its
985
+ # records (load: true, or a batch of a loaded relation). Its query
986
+ # methods, its aggregates (count, sum, exists?, ...) and its
987
+ # update_all, delete_all and touch_all are that relation's, as Active
988
+ # Record runs them as the relation's SQL; its reads (to_a, size, ids,
989
+ # pluck, first, ...) and destroy_all and update are of the records
990
+ # loaded. nil for any other relation. A Proc builds it each time it is
991
+ # asked for (an in_batches(load: true) batch's query, read again from
992
+ # the event store, as Active Record runs it each time).
993
+ def query_relation
994
+ @query_relation.is_a?(Proc) ? @query_relation.call : @query_relation
995
+ end
996
+
997
+ # Loaded with its records, its query +query+'s (a relation, or a Proc
998
+ # building it).
999
+ def loaded_with(query)
1000
+ @loaded_records = true
1001
+ @query_relation = query
1002
+ self
1003
+ end
1004
+ protected :loaded_with
1005
+
1006
+ # This relation selects what +other+ does (a copy made of it): it is
1007
+ # built again as +other+ is.
1008
+ def inherit_rebuild(other)
1009
+ self.rebuild_recipe = other.rebuild_recipe
1010
+ self
1011
+ end
1012
+
126
1013
  def initialize(model_class, records = [])
127
1014
  @model_class = model_class
128
1015
  @records = records.to_a
@@ -134,10 +1021,69 @@ module Lyra
134
1021
  relation.selected_columns = selected_columns
135
1022
  relation.create_attributes = create_attributes
136
1023
  relation.group_columns = group_columns
1024
+ relation.query_terms = query_terms
1025
+ relation.joined_classes = joined_classes
1026
+ relation.ordering = ordering
1027
+ relation.default_scope_applied = default_scope_applied
1028
+ relation.windowed = windowed?
1029
+ relation.create_with_attributes = create_with_attributes
1030
+ relation.instance_variable_set(:@negated_columns, @negated_columns) if @negated_columns
1031
+ relation.instance_variable_set(:@or_terms, @or_terms) if @or_terms
1032
+ relation.instance_variable_set(:@reordering, @reordering) if @reordering
1033
+ relation.instance_variable_set(:@distinct, @distinct) if @distinct
137
1034
  relation.extending!(*extensions) if extensions.any?
138
1035
  relation
139
1036
  end
140
1037
 
1038
+ # The models joined into this relation's query, kept when a later call
1039
+ # (group, order) builds a relation without the joined rows: a column
1040
+ # named without its table may be theirs.
1041
+ attr_writer :joined_classes
1042
+
1043
+ def joined_classes
1044
+ ((@joined_classes || []) + joined_associations.values.map { _1.reflection.klass }).uniq
1045
+ end
1046
+
1047
+ # This relation, with +terms+ added to what its query named.
1048
+ def add_query_terms(terms)
1049
+ @query_terms = query_terms + terms if terms.any?
1050
+ self
1051
+ end
1052
+ alias_method :add_query_terms!, :add_query_terms
1053
+
1054
+ # The attribute names this relation's query names (and +extra_terms+),
1055
+ # by model, as Lyra::PurposeBoundReads.named_fields_of gives them for
1056
+ # an ActiveRecord relation: its conditions (a condition on a joined
1057
+ # table counts for that table's model), order keys, group and select
1058
+ # columns, and a subquery's own.
1059
+ def lyra_named_fields(extra_terms = [])
1060
+ lyra_query_names(extra_terms).result
1061
+ end
1062
+
1063
+ # The same as a Lyra::PurposeBoundReads::NamedFields, with what this
1064
+ # relation's subqueries name (#whole).
1065
+ def lyra_query_names(extra_terms = [])
1066
+ named = Lyra::PurposeBoundReads::NamedFields.new
1067
+ named.joined(model_class, joined_classes)
1068
+ (query_terms + extra_terms).each do |kind, term|
1069
+ kind == :conditions ? named.conditions(term, model_class) : named.value(term, model_class)
1070
+ end
1071
+ named.value(group_columns, model_class).value(selected_columns, model_class)
1072
+ end
1073
+
1074
+ # A relation over the records the block selects, keeping this one's
1075
+ # selection (Model.lyra_processable under ES-NoProj).
1076
+ def lyra_select_records(&block)
1077
+ spawn_records(@records.select(&block))
1078
+ end
1079
+
1080
+ # The records as they are, for Lyra's own relations (records is the
1081
+ # application's, checked).
1082
+ def raw_records
1083
+ @records
1084
+ end
1085
+ protected :raw_records
1086
+
141
1087
  # =========================================================================
142
1088
  # Enumerable / Array-like interface
143
1089
  # =========================================================================
@@ -244,39 +1190,141 @@ module Lyra
244
1190
  last || raise(ActiveRecord::RecordNotFound.new("Couldn't find #{model_class.name}", model_class))
245
1191
  end
246
1192
 
247
- def find(id)
248
- record = @records.find { |r| r.id.to_s == id.to_s }
1193
+ # As Active Record's where(id).take (and find_by's where(...).take):
1194
+ # on a window, the condition applies before it and the record is the
1195
+ # first after its offset (take's limit replaces the window's).
1196
+ # find(id), or find(id, id, ...) and find([ids]) as Active Record's
1197
+ # find_some: the records with those ids, all of them or RecordNotFound
1198
+ # (on a window, as many as its limit and offset leave; in an ordered
1199
+ # relation, in its order, else in the order of the ids). Several ids
1200
+ # used to be an ArgumentError.
1201
+ def find(*args)
1202
+ expects_array = args.first.is_a?(Array)
1203
+ return [] if expects_array && args.first.empty?
1204
+
1205
+ ids = args.flatten.compact.uniq
1206
+ case ids.size
1207
+ when 0
1208
+ raise ActiveRecord::RecordNotFound.new("Couldn't find #{model_class.name} without an ID", model_class)
1209
+ when 1
1210
+ record = find_one(ids.first)
1211
+ expects_array ? [record] : record
1212
+ else
1213
+ find_some(ids)
1214
+ end
1215
+ end
1216
+
1217
+ def find_one(id)
1218
+ record = window_first(@records) { |r| r.id.to_s == id.to_s }
249
1219
  record || raise(ActiveRecord::RecordNotFound.new(
250
1220
  "Couldn't find #{model_class.name} with '#{model_class.primary_key}'=#{id}",
251
1221
  model_class, model_class.primary_key, id
252
1222
  ))
253
1223
  end
1224
+ private :find_one
1225
+
1226
+ def find_some(ids)
1227
+ wanted = ids.map(&:to_s)
1228
+ source = (window_source || self).raw_records.select { |record| wanted.include?(record.id.to_s) }
1229
+ offset = offset_value.nil? ? 0 : window_number(offset_value, "offset")
1230
+ limit = limit_value.nil? ? nil : window_number(limit_value, "limit")
1231
+ if ordering.any?
1232
+ result = source.drop(offset)
1233
+ result = result.first(limit) if limit
1234
+ expected = limit && ids.size > limit ? limit : ids.size
1235
+ expected = ids.size - offset if offset_value && ids.size - offset < expected
1236
+ else
1237
+ sliced = ids.slice(offset, limit || ids.size) || []
1238
+ by_id = source.to_h { [_1.id.to_s, _1] }
1239
+ result = sliced.filter_map { by_id[_1.to_s] }
1240
+ expected = sliced.size
1241
+ end
1242
+ return result if result.size == expected
1243
+
1244
+ raise ActiveRecord::RecordNotFound.new(
1245
+ "Couldn't find all #{model_class.name.pluralize} with '#{model_class.primary_key}': (#{ids.join(', ')}) " \
1246
+ "(found #{result.size} results, but was looking for #{expected}).",
1247
+ model_class, model_class.primary_key, ids
1248
+ )
1249
+ end
1250
+ private :find_some
254
1251
 
255
1252
  def find_by(attributes)
256
- @records.find do |record|
257
- attributes.all? { |key, value| matches_value?(record, key, value) }
258
- end
1253
+ tests = condition_tests(model_class, attributes)
1254
+ window_first(@records) { |record| true.equal?(matches_all?(record, tests)) }
259
1255
  end
260
1256
 
1257
+ # The first of +records+ the block accepts; on a window, the first that
1258
+ # the relation it was taken from has after the window's offset.
1259
+ def window_first(records, &block)
1260
+ return records.find(&block) unless window_source
1261
+
1262
+ window_source.raw_records.select(&block).drop(window_number(offset_value || 0, "offset")).first
1263
+ end
1264
+ private :window_first
1265
+
261
1266
  def find_by!(attributes)
262
1267
  find_by(attributes) || raise(ActiveRecord::RecordNotFound.new(
263
1268
  "Couldn't find #{model_class.name}", model_class
264
1269
  ))
265
1270
  end
266
1271
 
267
- def exists?(conditions = nil)
1272
+ # As ActiveRecord's: the one record, RecordNotFound when there is none
1273
+ # and SoleRecordExceeded when there are more.
1274
+ def sole
1275
+ found, undesired = first(2)
1276
+ raise ActiveRecord::RecordNotFound.new("Couldn't find #{model_class.name}", model_class) if found.nil?
1277
+ raise ActiveRecord::SoleRecordExceeded.new(model_class) if undesired
1278
+
1279
+ found
1280
+ end
1281
+
1282
+ def find_sole_by(attributes)
1283
+ where(attributes).sole
1284
+ end
1285
+
1286
+ # As Active Record: with no argument, whether the relation has a record;
1287
+ # exists?(false) and exists?(nil) are false (they used to answer as
1288
+ # exists? with no argument). On a window, a limit of 0 has none, and a
1289
+ # condition applies before the window (Active Record's exists? keeps
1290
+ # only the offset).
1291
+ def exists?(conditions = :none)
1292
+ return false if conditions.nil? || conditions == false
1293
+ return false if !limit_value.nil? && window_number(limit_value, "limit").zero?
1294
+ return window_matching(conditions).exists? if window_source && conditions != :none
1295
+
268
1296
  case conditions
269
- when nil, false
1297
+ when :none
270
1298
  @records.any?
1299
+ when nil, false
1300
+ false
271
1301
  when Integer, String
272
1302
  @records.any? { |r| r.id.to_s == conditions.to_s }
273
1303
  when Hash
274
1304
  find_by(conditions).present?
1305
+ when Array
1306
+ # exists?(["title = ?", x]): a fragment, as where takes it (or refuses it).
1307
+ where(*conditions).any?
275
1308
  else
276
- @records.any?
1309
+ # It used to answer whether any record exists, ignoring the condition.
1310
+ raise UnsupportedQuery.new(model_class, "exists?(#{conditions.inspect[0, 60]})")
277
1311
  end
278
1312
  end
279
1313
 
1314
+ # The records exists?(conditions) asks for on a window: those of the
1315
+ # relation it was taken from that satisfy them, after its offset.
1316
+ def window_matching(conditions)
1317
+ matching =
1318
+ case conditions
1319
+ when Hash then window_source.where(conditions)
1320
+ when Integer, String then window_source.where(model_class.primary_key => conditions)
1321
+ when Array then window_source.where(*conditions)
1322
+ else raise UnsupportedQuery.new(model_class, "exists?(#{conditions.inspect[0, 60]})")
1323
+ end
1324
+ offset_value.nil? ? matching : matching.offset(offset_value)
1325
+ end
1326
+ protected :window_matching
1327
+
280
1328
  # =========================================================================
281
1329
  # Query methods (return new CachedRelation for chaining)
282
1330
  # =========================================================================
@@ -298,8 +1346,10 @@ module Lyra
298
1346
 
299
1347
  own, joined = split_joined_conditions(conditions)
300
1348
  own = AssociationConditions.rewrite(model_class, own)
1349
+ tests = condition_tests(model_class, own)
1350
+ joined = compile_joined(joined)
301
1351
  relation = filter_rows do |record, joins|
302
- own.all? { |key, value| matches_value?(record, key, value) } && joined_match(joins, joined) == true
1352
+ true.equal?(matches_all?(record, tests)) && (joined.empty? || true.equal?(joined_match(joins, joined)))
303
1353
  end
304
1354
  scalars = own.select { |key, value| model_class.column_names.include?(key.to_s) && !value.is_a?(Array) && !value.is_a?(Range) && !value.is_a?(Hash) }
305
1355
  relation.create_attributes = create_attributes.merge(scalars.transform_keys(&:to_s)) if scalars.any?
@@ -341,16 +1391,18 @@ module Lyra
341
1391
  # A predicate on one record. NULL on either side is unknown, so false.
342
1392
  def fragment_test(column, op, value)
343
1393
  if op.end_with?("like")
1394
+ TypedValues.fragment_value(model_class, column, nil) # refuses a pattern on a ciphertext
344
1395
  regexp = like_regexp(value.to_s, case_insensitive: op == "ilike")
345
1396
  return ->(record) { (v = record.read_attribute(column)) && regexp.match?(v.to_s) }
346
1397
  end
347
1398
 
348
- bound = model_class.type_for_attribute(column).cast(value)
1399
+ bound = TypedValues.fragment_value(model_class, column, value)
1400
+ read = TypedValues.comparable_reader(model_class, column)
349
1401
  lambda do |record|
350
- v = record.read_attribute(column)
1402
+ v = read.call(record)
351
1403
  next false if v.nil? || bound.nil?
352
1404
 
353
- cmp = v <=> bound
1405
+ cmp = TypedValues.compare(v, bound)
354
1406
  next false if cmp.nil?
355
1407
 
356
1408
  case op
@@ -383,10 +1435,14 @@ module Lyra
383
1435
  check_hash_conditions!(conditions, "where.not(#{conditions.inspect[0, 60]})")
384
1436
  own, joined = split_joined_conditions(conditions)
385
1437
  own = AssociationConditions.rewrite(model_class, own)
386
- filter_rows do |record, joins|
387
- own_match = own.all? { |key, value| matches_value?(record, key, value) }
388
- combine_and(own_match, joined_match(joins, joined)) == false
1438
+ tests = condition_tests(model_class, own)
1439
+ joined = compile_joined(joined)
1440
+ relation = filter_rows do |record, joins|
1441
+ false.equal?(combine_and(matches_all?(record, tests), joined.empty? ? true : joined_match(joins, joined)))
389
1442
  end
1443
+ negated = conditions.flat_map { |key, value| condition_column_names(key, value) }
1444
+ relation.instance_variable_set(:@negated_columns, negated_columns | negated)
1445
+ relation
390
1446
  end
391
1447
 
392
1448
  # where.missing(:assoc): records with no partner (a left join).
@@ -401,37 +1457,127 @@ module Lyra
401
1457
  joins(*associations)
402
1458
  end
403
1459
 
404
- def order(*args)
405
- return self if args.empty?
1460
+ # As Active Record's excluding (and without): where.not(primary key:
1461
+ # the ids of the records given, and of the relations given), so a
1462
+ # write on it selects at write time as any where.not does
1463
+ # (Rebuildable). A record or relation of another model is Active
1464
+ # Record's ArgumentError. It used to be Enumerable's excluding, which
1465
+ # returned an Array.
1466
+ def excluding(*records)
1467
+ relations, records = records.partition { _1.is_a?(CachedRelation) || _1.is_a?(ActiveRecord::Relation) }
1468
+ records = records.flatten(1).compact
1469
+ unless records.all?(model_class) &&
1470
+ relations.all? { (_1.is_a?(CachedRelation) ? _1.model_class : _1.model) == model_class }
1471
+ raise ArgumentError, "You must only pass a single or collection of #{model_class.name} objects to ##{__callee__}."
1472
+ end
406
1473
 
407
- keys = order_keys(args)
408
- sorted = @records.sort do |a, b|
409
- compare_for_order(a, b, keys)
1474
+ ids = records.map { |record| record.is_a?(model_class) ? record.id : record } + relations.flat_map(&:ids)
1475
+ self.not(model_class.primary_key => ids)
1476
+ end
1477
+ alias_method :without, :excluding
1478
+
1479
+ # An order by a value's position in a list (in_order_of): the record's
1480
+ # position, or +otherwise+ for a value not in it.
1481
+ PositionOrder = Struct.new(:column, :positions, :type, :otherwise) do
1482
+ def sort_reader
1483
+ ->(record) { positions.fetch(type.serialize(record.read_attribute(column)), otherwise) }
1484
+ end
1485
+
1486
+ def to_s = "in_order_of(#{column})"
1487
+ end
1488
+
1489
+ # As Active Record's in_order_of: the records whose +column+ has one of
1490
+ # +values+ (unless filter: false), ordered by its position among them
1491
+ # (a CASE in Active Record's ORDER BY), after any order the relation has
1492
+ # (Active Record's order! adds it), records not among them last. The
1493
+ # values are cast as Active Record casts them (an enum's label to its
1494
+ # value; one the column cannot hold is left out; nil is a position,
1495
+ # IS NULL); none left is none. A column that is not an attribute of the
1496
+ # model (an SQL expression, another table's) is refused. It used to be
1497
+ # Enumerable's in_order_of, which returned an Array.
1498
+ def in_order_of(column, values, filter: true)
1499
+ model_class.disallow_raw_sql!([column], permit: model_class.adapter_class.column_name_with_order_matcher)
1500
+ return none if values.empty?
1501
+ unless (column.is_a?(Symbol) || column.is_a?(String)) && !column.is_a?(Arel::Nodes::SqlLiteral)
1502
+ raise UnsupportedQuery.new(model_class, "in_order_of(#{column.inspect[0, 60]})")
410
1503
  end
411
1504
 
412
- spawn_records(sorted)
1505
+ # The model's own table in front of the column ("items.code") is
1506
+ # the column, as Active Record's arel_column reads it.
1507
+ column = column.to_s.delete_prefix("#{model_class.table_name}.") if column.is_a?(String)
1508
+ TypedValues.sort_reader(model_class, column) # refuses what is not an attribute, or is encrypted
1509
+ name = TypedValues.attribute_name(model_class, column)
1510
+ type = model_class.type_for_attribute(name)
1511
+ positions = {}
1512
+ kept = []
1513
+ values.each do |value|
1514
+ serialized = value.nil? || !type.serializable?(value) ? nil : type.serialize(value)
1515
+ next if !value.nil? && serialized.nil?
1516
+
1517
+ kept << value
1518
+ positions[serialized] = positions.size unless positions.key?(serialized)
1519
+ end
1520
+ return none if positions.empty?
1521
+
1522
+ # The filter is a where of the column (IS NULL for nil), so that
1523
+ # rewhere, unscope and except take it off as in Active Record; the
1524
+ # order is a step of the plan (:position), which unscope(:order)
1525
+ # takes off. It was a filter they did not see.
1526
+ relation = filter ? where(name => kept) : self
1527
+ key = PositionOrder.new(name, positions, type, positions.size)
1528
+ ordered = relation.send(:sorted_rows, relation.ordering + [{ key => :asc }])
1529
+ ordered.continue_plan(relation, [:position, key]).add_query_terms([[:value, [name]]])
1530
+ end
1531
+
1532
+ # As in ActiveRecord, an order adds to the one the relation has (a
1533
+ # default scope's order, an earlier order call); reorder replaces it.
1534
+ def order(*args)
1535
+ return self if args.empty?
1536
+
1537
+ sorted_by(ordering + order_keys(args))
413
1538
  end
414
1539
 
415
- # Order arguments as [{ column => :asc/:desc }]: symbols, hashes,
416
- # "col [ASC|DESC], ..." strings and Arel orderings of a column. Anything
417
- # else is refused; an unreadable ordering used to compare as equal and
418
- # leave the result silently unordered.
1540
+ def sorted_by(keys)
1541
+ relation = spawn_records(sort_records(@records, keys))
1542
+ relation.ordering = keys
1543
+ relation
1544
+ end
1545
+ private :sorted_by
1546
+
1547
+ # Order arguments as [{ column => :asc/:desc }]: symbols, hashes of a
1548
+ # column and asc or desc, "col [ASC|DESC], ..." strings (with this
1549
+ # model's table, if any, in front of the column) and Arel orderings of a
1550
+ # column of this model's table. Anything else is refused; an unreadable
1551
+ # ordering used to compare as equal and leave the result silently
1552
+ # unordered, and another table's column ("posts.title", an Arel
1553
+ # attribute of another table) or a direction other than asc or desc
1554
+ # ("desc nulls last") ordered by this model's column of that name,
1555
+ # ascending.
419
1556
  def order_keys(args)
420
1557
  args.flatten.flat_map do |arg|
421
1558
  case arg
422
1559
  when Symbol then [{ arg => :asc }]
423
- when Hash then arg.map { |column, direction| { column.to_sym => direction.to_s.downcase.to_sym } }
1560
+ when Hash
1561
+ arg.map do |column, direction|
1562
+ direction = direction.to_s.downcase
1563
+ raise UnsupportedQuery.new(model_class, "order(#{arg.inspect[0, 60]})") unless %w[asc desc].include?(direction)
1564
+
1565
+ { column.to_sym => direction.to_sym }
1566
+ end
424
1567
  when String
425
1568
  arg.split(",").map do |part|
426
- match = part.strip.match(/\A"?(?:\w+"?\.)?"?(\w+)"?(?:\s+(asc|desc))?\z/i)
427
- raise UnsupportedQuery.new(model_class, "order(#{arg.inspect})") unless match && model_class.column_names.include?(match[1])
1569
+ match = part.strip.match(/\A(?:"?(\w+)"?\.)?"?(\w+)"?(?:\s+(asc|desc))?\z/i)
1570
+ unless match && (match[1].nil? || match[1] == model_class.table_name.to_s) && model_class.column_names.include?(match[2])
1571
+ raise UnsupportedQuery.new(model_class, "order(#{arg.inspect})")
1572
+ end
428
1573
 
429
- { match[1].to_sym => (match[2] || "asc").downcase.to_sym }
1574
+ { match[2].to_sym => (match[3] || "asc").downcase.to_sym }
430
1575
  end
431
1576
  when Arel::Nodes::Ordering
432
- raise UnsupportedQuery.new(model_class, "order(#{arg.to_sql})") unless arg.expr.respond_to?(:name)
1577
+ column = own_column(arg.expr)
1578
+ raise UnsupportedQuery.new(model_class, "order(#{arg.to_sql})") unless column
433
1579
 
434
- [{ arg.expr.name.to_sym => arg.descending? ? :desc : :asc }]
1580
+ [{ column => arg.descending? ? :desc : :asc }]
435
1581
  else raise UnsupportedQuery.new(model_class, "order(#{arg.inspect[0, 60]})")
436
1582
  end
437
1583
  end
@@ -444,77 +1590,347 @@ module Lyra
444
1590
  raise UnsupportedQuery.new(model_class, "or(#{other.class})")
445
1591
  end
446
1592
 
447
- spawn_records((@records + other.records).uniq { |record| record.id })
1593
+ # Each record once, in the relation's order (one query's ORDER BY),
1594
+ # or else by primary key, as the table returns an unordered query's
1595
+ # rows. It used to be this relation's records, then the other's.
1596
+ union = spawn_records((@records + other.raw_records.to_a).uniq { |record| record.id })
1597
+ union.send(:sorted_by, ordering.any? ? ordering : [{ model_class.primary_key.to_sym => :asc }]).tap do |sorted|
1598
+ sorted.ordering = ordering
1599
+ end
1600
+ end
1601
+
1602
+ # The query terms that are an OR's alternatives (Delivery#or): their
1603
+ # columns are not conditions merge can replace one at a time.
1604
+ def or_terms = @or_terms || {}.compare_by_identity
1605
+
1606
+ def add_or_terms(terms)
1607
+ @or_terms = or_terms.merge(terms.to_h { [_1, true] }.compare_by_identity) if terms.any?
1608
+ self
448
1609
  end
449
1610
 
450
1611
  def reorder(*args)
451
- order(*args)
1612
+ keys = args.compact.empty? ? [] : order_keys(args)
1613
+ relation = keys.any? ? sorted_by(keys) : spawn_records(@records)
1614
+ relation.ordering = keys
1615
+ relation.instance_variable_set(:@reordering, true)
1616
+ relation
452
1617
  end
453
1618
 
1619
+ # Whether reorder made the order (Active Record's reordering_value):
1620
+ # merging such a relation replaces the order.
1621
+ def reordering? = @reordering ? true : false
1622
+
1623
+ def distinct? = @distinct ? true : false
1624
+
454
1625
  def reverse_order
455
- spawn_records(@records.reverse)
1626
+ relation = spawn_records(@records.reverse)
1627
+ relation.ordering = ordering.map { |key| key.transform_values { |direction| direction == :desc ? :asc : :desc } }
1628
+ relation
456
1629
  end
457
1630
 
458
- def limit(count)
459
- spawn_records(@records.first(count))
1631
+ # =========================================================================
1632
+ # Windows: limit, offset and pagination (Kaminari's page and per)
1633
+ # =========================================================================
1634
+
1635
+ # As SQL's LIMIT and OFFSET: a window of what the relation selects,
1636
+ # taken after its conditions and order, whenever they were given. The
1637
+ # relation keeps the one it was taken from (window_source), and the
1638
+ # query methods called on it after the window (where, order, a scope,
1639
+ # merge, ...) are applied to that one, the window then taken again
1640
+ # (CachedRelationWindow): Model.limit(2).where(...) is the first two
1641
+ # records that satisfy the condition, as in Active Record. They used to
1642
+ # filter the two records the limit had kept. A second limit or offset
1643
+ # replaces the first, as in Active Record (limit(1).limit(3) is three
1644
+ # records; limit(2).offset(1) is records 2 and 3); they used to apply to
1645
+ # the window.
1646
+ def limit(value)
1647
+ window_base.windowed_by(value, offset_value, pagination: @pagination)
1648
+ end
1649
+
1650
+ def offset(value)
1651
+ window_base.windowed_by(limit_value, value, pagination: @pagination)
1652
+ end
1653
+
1654
+ # The values limit and offset were given, as Active Record's (nil when
1655
+ # there is none). limit_value used to be the page size, nil after a
1656
+ # plain limit.
1657
+ attr_reader :limit_value, :offset_value
1658
+
1659
+ # The relation the window was taken from, or nil.
1660
+ attr_reader :window_source
1661
+
1662
+ # Whether Kaminari's page made this relation (its per and totals apply).
1663
+ # @pagination holds what Kaminari keeps on such a relation (:per, the
1664
+ # size per was last given; :max_per_page; :padding), carried to every
1665
+ # relation derived from it, as Active Record's spawn carries them.
1666
+ def paged? = !@pagination.nil?
1667
+
1668
+ # +limit+ and +offset+ of this relation's records, as a relation that
1669
+ # keeps this one as its window_source (nil both: a copy of this one).
1670
+ def windowed_by(limit, offset, pagination: nil)
1671
+ if limit.nil? && offset.nil?
1672
+ return clone.tap { _1.instance_variable_set(:@pagination, pagination&.dup) }
1673
+ end
1674
+
1675
+ start = offset.nil? ? 0 : window_number(offset, "offset")
1676
+ count = limit.nil? ? nil : window_number(limit, "limit")
1677
+ relation =
1678
+ if @joins
1679
+ rows = @rows.drop(start)
1680
+ with_rows(count ? rows.first(count) : rows, @joins)
1681
+ else
1682
+ records = @records.to_a.drop(start)
1683
+ spawn_records(count ? records.first(count) : records)
1684
+ end
1685
+ relation.windowed = true
1686
+ relation.instance_variable_set(:@window_source, self)
1687
+ relation.instance_variable_set(:@limit_value, limit)
1688
+ relation.instance_variable_set(:@offset_value, offset)
1689
+ relation.instance_variable_set(:@pagination, pagination&.dup)
1690
+ relation
460
1691
  end
1692
+ protected :windowed_by
1693
+
1694
+ # The relation a new window is taken from: the one this window was
1695
+ # taken from, or a copy of this relation (a later bang method on this
1696
+ # one must not change the window's).
1697
+ def window_base = window_source || clone
1698
+ private :window_base
461
1699
 
462
- def offset(count)
463
- spawn_records(@records.drop(count))
1700
+ # The window leaves out the one row an aggregate query returns (Active
1701
+ # Record's SUM, AVG, MIN, MAX of a windowed relation).
1702
+ def window_leaves_out_one_row?
1703
+ (!limit_value.nil? && window_number(limit_value, "limit").zero?) ||
1704
+ (!offset_value.nil? && window_number(offset_value, "offset").positive?)
464
1705
  end
1706
+ private :window_leaves_out_one_row?
1707
+
1708
+ # A limit or offset as SQL takes it: an Integer, or a String of one.
1709
+ def window_number(value, what)
1710
+ Integer(value)
1711
+ rescue ArgumentError, TypeError
1712
+ raise UnsupportedQuery.new(model_class, "#{what}(#{value.inspect})")
1713
+ end
1714
+ private :window_number
465
1715
 
466
1716
  # =========================================================================
467
- # Pagination support (Kaminari/WillPaginate compatibility)
1717
+ # Kaminari 1.2.2's pagination, as its page and PageScopeMethods give it
1718
+ # on an Active Record relation. Without Kaminari, the model's sizes are
1719
+ # Kaminari's defaults (25 a page, no maximum).
468
1720
  # =========================================================================
469
1721
 
470
- def page(num)
471
- @current_page = [num.to_i, 1].max
472
- @per_page ||= 25
473
- self
1722
+ # page(n): limit(size).offset(size * (n - 1)), size the model's
1723
+ # paginates_per (default_per_page), or its max_paginates_per when that
1724
+ # is smaller; a page number below 1, nil or one that is not a number is
1725
+ # page 1. page(n) without per used to be refused (and before that gave
1726
+ # every record); the maximum was not applied.
1727
+ def page(num = nil)
1728
+ size = max_per_page && default_per_page > max_per_page ? max_per_page : default_per_page
1729
+ number = num.to_i - 1
1730
+ window_base.windowed_by(size, size * (number.negative? ? 0 : number), pagination: {})
474
1731
  end
475
1732
 
476
- def per(num)
477
- @per_page = num.to_i
478
- paginated_records
1733
+ # On a page, Kaminari's per: nil, a negative or a non-numeric size keeps
1734
+ # the page as it is; 0 is limit(0); else the page of that size with the
1735
+ # same number, at most max_per_page (the argument, else what
1736
+ # max_paginates_per set on the relation, else the model's). Outside a
1737
+ # page, the relation as it is.
1738
+ def per(num, max_per_page: nil)
1739
+ return self unless paged?
1740
+
1741
+ max_per_page ||= @pagination[:max_per_page] || self.max_per_page
1742
+ pagination = @pagination.merge(per: (num || default_per_page).to_i)
1743
+ size = num.to_i
1744
+ return with_pagination(pagination) if size.negative? || !num.to_s.match?(/\A\d/)
1745
+ return paginated(size, offset_value, pagination) if size.zero?
1746
+
1747
+ size = max_per_page if max_per_page && max_per_page < size
1748
+ paginated(size, window_number(offset_value, "offset") / window_number(limit_value, "limit") * size, pagination)
1749
+ end
1750
+
1751
+ # On a page, Kaminari's: the page again with at most +new_max_per_page+
1752
+ # records, kept on the relation (the model's maximum is not changed; on
1753
+ # a relation that is not a page, Active Record hands it to the model).
1754
+ def max_paginates_per(new_max_per_page)
1755
+ return model_class.max_paginates_per(new_max_per_page) unless paged?
1756
+
1757
+ with_pagination(@pagination.merge(max_per_page: new_max_per_page))
1758
+ .per(@pagination[:per] || default_per_page, max_per_page: new_max_per_page)
1759
+ end
1760
+
1761
+ # Kaminari's padding(n): the page n records further on, its current
1762
+ # page and total pages counted without them.
1763
+ def padding(num)
1764
+ num = num.to_i
1765
+ raise ArgumentError, "padding must not be negative" if num.negative?
1766
+
1767
+ paginated(limit_value, window_number(offset_value || 0, "offset") + num, @pagination.merge(padding: num))
479
1768
  end
480
1769
 
1770
+ # Kaminari's total_pages: the records the relation selects before the
1771
+ # window (less the padding), by the page size, at most max_pages. 1 for
1772
+ # a relation that is not a page.
481
1773
  def total_pages
482
- return 1 if @per_page.nil? || @per_page <= 0
483
- (@records.size.to_f / @per_page).ceil
1774
+ return 1 unless paged?
1775
+
1776
+ count = [total_count - @pagination.fetch(:padding, 0), 0].max
1777
+ size = window_number(limit_value, "limit")
1778
+ raise zero_per_page("The number of total pages was incalculable. Perhaps you called .per(0)?") if size.zero?
1779
+
1780
+ pages = (count.to_f / size).ceil
1781
+ max_pages && max_pages < pages ? max_pages : pages
484
1782
  end
485
1783
 
1784
+ # Kaminari's current_page: (offset - padding) / page size + 1; 1 for a
1785
+ # relation that is not a page.
486
1786
  def current_page
487
- @current_page || 1
1787
+ return 1 unless paged?
1788
+
1789
+ size = window_number(limit_value, "limit")
1790
+ raise zero_per_page("Current page was incalculable. Perhaps you called .per(0)?") if size.zero?
1791
+
1792
+ ([window_number(offset_value || 0, "offset") - @pagination.fetch(:padding, 0), 0].max / size) + 1
1793
+ end
1794
+
1795
+ def next_page = (current_page + 1 unless last_page? || out_of_range?)
1796
+ def prev_page = (current_page - 1 unless first_page? || out_of_range?)
1797
+ def first_page? = current_page == 1
1798
+ # Of a page without_count, Kaminari's: the last page when it has
1799
+ # records and no record follows them; out of range when it has none.
1800
+ def last_page? = without_count? ? (!out_of_range? && !next_record?) : current_page == total_pages
1801
+ def out_of_range? = without_count? ? to_a.empty? : current_page > total_pages
1802
+
1803
+ # Kaminari's entry_name: the model's human name, pluralized but for a
1804
+ # count of 1 (page_entries_info's "Displaying items 1 - 2").
1805
+ def entry_name(options = {})
1806
+ refuse_outside_page!(:entry_name)
1807
+ default = options[:count] == 1 ? model_class.model_name.human : model_class.model_name.human.pluralize
1808
+ model_class.model_name.human(options.reverse_merge(default: default))
488
1809
  end
489
1810
 
1811
+ # Kaminari's without_count: a page that does not count its relation's
1812
+ # records (total_count raises, as Kaminari's does; last_page? and
1813
+ # out_of_range? look at the page itself).
1814
+ def without_count
1815
+ refuse_outside_page!(:without_count)
1816
+ with_pagination(@pagination.merge(without_count: true))
1817
+ end
1818
+
1819
+ def without_count? = paged? && @pagination[:without_count] ? true : false
1820
+
1821
+ # Kaminari's deprecated current_per_page: limit_value, with its warning.
1822
+ def current_per_page
1823
+ refuse_outside_page!(:current_per_page)
1824
+ ActiveSupport::Deprecation.warn "#current_per_page is deprecated and will be removed in the next major " \
1825
+ "version. Please use #limit_value instead."
1826
+ limit_value
1827
+ end
1828
+
1829
+ # What only a Kaminari page answers: on another relation, as on Active
1830
+ # Record's, there is no such method.
1831
+ PAGE_METHODS = %i[entry_name without_count current_per_page].freeze
1832
+
1833
+ def respond_to?(name, include_all = false)
1834
+ PAGE_METHODS.include?(name.to_sym) ? paged? : super
1835
+ end
1836
+
1837
+ # The model's Kaminari sizes, as an Active Record relation hands them
1838
+ # to its model; Kaminari's defaults without Kaminari.
1839
+ def default_per_page
1840
+ return model_class.default_per_page if model_class.respond_to?(:default_per_page)
1841
+
1842
+ kaminari_config ? kaminari_config.default_per_page : 25
1843
+ end
1844
+
1845
+ def max_per_page
1846
+ return model_class.max_per_page if model_class.respond_to?(:max_per_page)
1847
+
1848
+ kaminari_config&.max_per_page
1849
+ end
1850
+
1851
+ def max_pages
1852
+ return model_class.max_pages if model_class.respond_to?(:max_pages)
1853
+
1854
+ kaminari_config&.max_pages
1855
+ end
1856
+
1857
+ # The records the relation selects before the window (the window's own
1858
+ # used to be counted). CachedRelationDelivery counts them through
1859
+ # window_source, so that the purpose check and the rights in force
1860
+ # apply to the count as to any count.
490
1861
  def total_count
491
- @records.size
1862
+ refuse_count_without_count!
1863
+ (window_source || self).raw_records.size
1864
+ end
1865
+
1866
+ # Whether +mod+ is (or includes) one of Kaminari's relation modules,
1867
+ # which its Model.page extends a relation with. CachedRelation answers
1868
+ # what they define itself; extending it with them would replace those
1869
+ # answers with ones that run Active Record's queries.
1870
+ def self.kaminari_module?(mod)
1871
+ return false unless defined?(::Kaminari::PageScopeMethods)
1872
+
1873
+ kaminari = [::Kaminari::PageScopeMethods]
1874
+ kaminari << ::Kaminari::ActiveRecordRelationMethods if defined?(::Kaminari::ActiveRecordRelationMethods)
1875
+ kaminari.any? { |known| mod == known || mod.include?(known) }
1876
+ end
1877
+
1878
+ private
1879
+
1880
+ def paginated(limit, offset, pagination)
1881
+ window_base.windowed_by(limit, offset, pagination: pagination)
1882
+ end
1883
+
1884
+ def with_pagination(pagination)
1885
+ relation = clone
1886
+ relation.instance_variable_set(:@pagination, pagination)
1887
+ relation
1888
+ end
1889
+
1890
+ def kaminari_config
1891
+ ::Kaminari.config if defined?(::Kaminari) && ::Kaminari.respond_to?(:config)
492
1892
  end
493
1893
 
494
- def limit_value
495
- @per_page
1894
+ def refuse_outside_page!(name)
1895
+ raise NoMethodError.new("undefined method '#{name}' for an instance of #{self.class}", name) unless paged?
496
1896
  end
497
1897
 
498
- def offset_value
499
- return 0 unless @current_page && @per_page
500
- (@current_page - 1) * @per_page
1898
+ def refuse_count_without_count!
1899
+ return unless without_count?
1900
+
1901
+ raise "This scope is marked as a non-count paginable scope and can't be used in combination " \
1902
+ "with `#paginate' or `#page_entries_info'. Use #link_to_next_page or #link_to_previous_page instead."
501
1903
  end
502
1904
 
503
- private def paginated_records
504
- return self unless @current_page && @per_page
1905
+ # Whether a record follows the page (its relation has more than offset
1906
+ # plus limit records).
1907
+ def next_record?
1908
+ return false if limit_value.nil?
505
1909
 
506
- start_idx = (@current_page - 1) * @per_page
507
- paginated = @records[start_idx, @per_page] || []
1910
+ start = offset_value.nil? ? 0 : window_number(offset_value, "offset")
1911
+ (window_source || self).raw_records.size > start + window_number(limit_value, "limit")
1912
+ end
508
1913
 
509
- result = spawn_records(paginated)
510
- result.instance_variable_set(:@current_page, @current_page)
511
- result.instance_variable_set(:@per_page, @per_page)
512
- result.instance_variable_set(:@total_records, @records.size)
513
- result
1914
+ def zero_per_page(message)
1915
+ defined?(::Kaminari::ZeroPerPageOperation) ? ::Kaminari::ZeroPerPageOperation.new(message) : ZeroDivisionError.new(message)
514
1916
  end
515
1917
 
516
- def distinct
517
- spawn_records(@records.uniq)
1918
+ public
1919
+
1920
+ # Each record once (a joined relation's rows repeat it). Distinct
1921
+ # values of selected columns (SELECT DISTINCT name) are other rows than
1922
+ # the records: refused, in either order (they were the records).
1923
+ def distinct(value = true)
1924
+ unless value
1925
+ relation = spawn_records(@records)
1926
+ relation.instance_variable_set(:@distinct, nil)
1927
+ return relation
1928
+ end
1929
+ raise UnsupportedQuery.new(model_class, "distinct of selected columns (SELECT DISTINCT)") if selected_columns
1930
+
1931
+ relation = spawn_records(@records.uniq)
1932
+ relation.instance_variable_set(:@distinct, true)
1933
+ relation
518
1934
  end
519
1935
 
520
1936
  def uniq
@@ -557,9 +1973,207 @@ module Lyra
557
1973
  spawn_records([])
558
1974
  end
559
1975
 
1976
+ # As Model.unscoped: under ES-NoProj every record of a monitored model,
1977
+ # from the event store, with no default scope.
560
1978
  def unscoped
561
- self
1979
+ return self unless events_only? && model_class.respond_to?(:lyra_monitored) && model_class.lyra_monitored
1980
+
1981
+ EventStoreReader.relation(model_class)
1982
+ end
1983
+
1984
+ # unscope(where: :deleted_at), unscope(:where), unscope(:order), and the
1985
+ # scopes made of them (paranoia's with_deleted, discard's
1986
+ # with_discarded). While the relation was selected by conditions and
1987
+ # orders only (its plan), the selection is replayed without what is
1988
+ # taken off, the default scope included, as ActiveRecord rebuilds its
1989
+ # query: Model.with_discarded, Model.where(...).with_discarded,
1990
+ # author.books.where(...).with_discarded. After another query method
1991
+ # (a limit, where.not, an SQL fragment) an unscope that would change
1992
+ # the answer is refused rather than answered short: the records it
1993
+ # would bring back are no longer in the relation.
1994
+ def unscope(*args)
1995
+ args = args.flatten
1996
+ return self if args.empty?
1997
+
1998
+ relation =
1999
+ if plan
2000
+ base, steps = plan
2001
+ replay_plan(base, unscoped_steps(base, steps, args))
2002
+ else
2003
+ unscope_without_plan(args)
2004
+ end
2005
+ unscope_select_and_group(relation, args)
2006
+ end
2007
+
2008
+ # unscope(:select) and unscope(:group): the columns of every record,
2009
+ # and no grouping. They used to be left on the relation.
2010
+ def unscope_select_and_group(relation, args)
2011
+ names = args.reject { _1.is_a?(Hash) }.map(&:to_s)
2012
+ drop_select = names.include?("select") && relation.selected_columns
2013
+ drop_group = names.include?("group") && relation.group_columns.any?
2014
+ return relation unless drop_select || drop_group
2015
+
2016
+ relation = relation.clone.inherit_rebuild(relation) if relation.equal?(self)
2017
+ relation.selected_columns = nil if drop_select
2018
+ relation.group_columns = [] if drop_group
2019
+ relation
562
2020
  end
2021
+ private :unscope_select_and_group
2022
+
2023
+ # As ActiveRecord's: the conditions on these columns are replaced
2024
+ # (unscope, then where), the default scope's included.
2025
+ def rewhere(conditions)
2026
+ raise UnsupportedQuery.new(model_class, "rewhere(#{conditions.inspect[0, 60]})") unless conditions.is_a?(Hash)
2027
+ return self if conditions.empty?
2028
+
2029
+ unscope(where: conditions.keys).where(conditions)
2030
+ end
2031
+
2032
+ def unscope!(*args)
2033
+ replace_with!(unscope(*args))
2034
+ end
2035
+
2036
+ # The plan's base with +steps+ applied.
2037
+ def replay_plan(base, steps)
2038
+ relation = base.is_a?(Proc) ? base.call : base
2039
+ default = nil
2040
+ steps.each do |kind, arg, what|
2041
+ relation =
2042
+ case kind
2043
+ when :scope
2044
+ default = arg if what == :default
2045
+ relation.narrowed_by(arg, what == :default ? DefaultScopes.description(model_class) : what)
2046
+ when :where then relation.where(arg)
2047
+ when :not then relation.where.not(arg)
2048
+ when :merge then relation.merge(arg)
2049
+ when :order then relation.order(*arg)
2050
+ when :reorder then relation.reorder(*arg)
2051
+ when :position then relation.send(:sorted_rows, relation.ordering + [{ arg => :asc }])
2052
+ end
2053
+ end
2054
+ relation = relation.spawn.inherit_rebuild(relation) if relation.equal?(base)
2055
+ relation.plan = [base, steps]
2056
+ relation.default_scope_applied = default
2057
+ # What the plan does not hold (select, group, create_with) stays as
2058
+ # this relation has it, as Active Record keeps it on an unscope(where:).
2059
+ relation.selected_columns = selected_columns
2060
+ relation.group_columns = group_columns
2061
+ relation.create_with_attributes = create_with_attributes
2062
+ relation
2063
+ end
2064
+ private :replay_plan
2065
+
2066
+ # +steps+ without what unscope(*args) takes off. A condition the base
2067
+ # itself was read with (an association's foreign key) cannot be taken
2068
+ # off here, and is refused.
2069
+ def unscoped_steps(base, steps, args)
2070
+ args.each do |arg|
2071
+ where_columns, other = unscope_target(arg)
2072
+ if where_columns
2073
+ check_base_conditions!(base, where_columns, args)
2074
+ steps = steps.filter_map do |kind, value, what|
2075
+ next [kind, value, what] unless kind == :where || kind == :not
2076
+
2077
+ kept = where_columns == :all ? {} : value.reject { |key, condition| unscoped_key?(key, condition, where_columns, args) }
2078
+ kept.empty? ? nil : [kind, kept, what]
2079
+ end
2080
+ elsif other == :order
2081
+ steps = steps.reject { |kind, *| kind == :order || kind == :reorder || kind == :position }
2082
+ end
2083
+ steps = steps.map do |kind, value, what|
2084
+ case kind
2085
+ when :scope then [kind, DefaultScopes.built { value.unscope(arg) }, what]
2086
+ when :merge then [kind, value.unscope(arg), what]
2087
+ else [kind, value, what]
2088
+ end
2089
+ end
2090
+ end
2091
+ steps
2092
+ end
2093
+ private :unscoped_steps
2094
+
2095
+ # Whether unscope(where: +columns+) takes off the condition on +key+:
2096
+ # one on a column of them, or on a belongs_to association whose key is
2097
+ # one of them (where(owner: ann) is a condition on owner_id; it was
2098
+ # kept). A condition on several columns at once that names one of them
2099
+ # is refused.
2100
+ def unscoped_key?(key, condition, columns, args)
2101
+ names = key.is_a?(Array) ? key.map { column_of(_1) } : [column_of(key), *condition_column_names(key, condition)]
2102
+ hit = (names & columns).any?
2103
+ if hit && key.is_a?(Array)
2104
+ raise UnsupportedQuery.new(model_class, "unscope(#{args.inspect[1..-2]}) of a condition on several columns at once")
2105
+ end
2106
+
2107
+ hit
2108
+ end
2109
+ private :unscoped_key?
2110
+
2111
+ # [columns, nil] for a where unscope (:all for every condition), or
2112
+ # [nil, the clause] for another.
2113
+ def unscope_target(arg)
2114
+ case arg
2115
+ when :where, "where" then [:all, nil]
2116
+ when Hash
2117
+ raise UnsupportedQuery.new(model_class, "unscope(#{arg.inspect})") unless arg.keys.map(&:to_sym) == [:where]
2118
+
2119
+ [Array(arg.values.first).map { |column| column_of(column) }, nil]
2120
+ else [nil, arg.to_sym]
2121
+ end
2122
+ end
2123
+ private :unscope_target
2124
+
2125
+ def column_of(key) = key.to_s.split(".").last.delete('"')
2126
+ private :column_of
2127
+
2128
+ def check_base_conditions!(base, columns, args)
2129
+ return if base.is_a?(Proc)
2130
+
2131
+ named = base.query_terms.flat_map do |kind, term|
2132
+ next [] unless kind == :conditions && term.is_a?(Hash)
2133
+
2134
+ term.flat_map { |key, value| key.is_a?(Array) ? key.map { column_of(_1) } : [column_of(key), *condition_column_names(key, value)] }
2135
+ end
2136
+ # An SQL fragment of a where (where("qty > ?", 2)) names its columns
2137
+ # in its text; unscope(:where) used to leave it on the relation.
2138
+ fragments = base.query_terms.filter_map { |kind, term| term if kind == :value && term.is_a?(String) }
2139
+ named += fragments.flat_map { |fragment| fragment.scan(/\w+/) & model_class.column_names }
2140
+ named += ["an SQL fragment"] if columns == :all && fragments.any?
2141
+ hit = columns == :all ? named.uniq : named & columns
2142
+ return if hit.empty?
2143
+
2144
+ raise UnsupportedQuery.new(model_class, "unscope(#{args.inspect[1..-2]}) of #{hit.join(', ')}, which already " \
2145
+ "selected the relation's records (an association's key, or a condition " \
2146
+ "before a limit, where.not or SQL fragment)")
2147
+ end
2148
+ private :check_base_conditions!
2149
+
2150
+ # Without a plan: what the unscope would change must not be in the
2151
+ # relation already. An order can be dropped (records are kept in key
2152
+ # order, as an unordered query may return them).
2153
+ def unscope_without_plan(args)
2154
+ relation = self
2155
+ args.each do |arg|
2156
+ where_columns, other = unscope_target(arg)
2157
+ if default_scope_applied
2158
+ rest = DefaultScopes.built { default_scope_applied.unscope(arg) }
2159
+ if rest.where_clause != default_scope_applied.where_clause || rest.order_values != default_scope_applied.order_values
2160
+ raise UnsupportedQuery.new(model_class, "unscope(#{args.inspect[1..-2]}) after other query methods " \
2161
+ "(the records the default scope leaves out are not in the relation " \
2162
+ "any more: unscope first, as in #{model_class.name}.unscope(...).where(...))")
2163
+ end
2164
+ end
2165
+ if where_columns
2166
+ check_base_conditions!(self, where_columns, args)
2167
+ elsif other == :order && ordering.any?
2168
+ relation = relation.reorder(model_class.primary_key => :asc)
2169
+ relation.ordering = []
2170
+ elsif %i[limit offset].include?(other) && windowed?
2171
+ raise UnsupportedQuery.new(model_class, "unscope(:#{other}) after a #{other}")
2172
+ end
2173
+ end
2174
+ relation
2175
+ end
2176
+ private :unscope_without_plan
563
2177
 
564
2178
  def readonly(value = true)
565
2179
  self
@@ -622,14 +2236,15 @@ module Lyra
622
2236
  end
623
2237
 
624
2238
  def scope_for_create
625
- create_attributes
2239
+ create_attributes.merge(create_with_attributes)
626
2240
  end
627
2241
 
628
2242
  # Records created through the relation take its equality conditions
629
- # (an association's foreign key), as on ActiveRecord; they are created
630
- # through the model, so under ES-NoProj the write is an event.
2243
+ # (an association's foreign key) and create_with's attributes, as on
2244
+ # ActiveRecord; they are created through the model, so under ES-NoProj
2245
+ # the write is an event.
631
2246
  def new(attributes = nil, &block)
632
- model_class.new(create_attributes.merge((attributes || {}).to_h.transform_keys(&:to_s)), &block)
2247
+ model_class.new(scope_for_create.merge((attributes || {}).to_h.transform_keys(&:to_s)), &block)
633
2248
  end
634
2249
  alias_method :build, :new
635
2250
 
@@ -645,16 +2260,177 @@ module Lyra
645
2260
  find_by(attributes) || new(attributes, &block)
646
2261
  end
647
2262
 
2263
+ # As Active Record's (8.1): the record found, or created as
2264
+ # create_or_find_by creates it, so a record a unique index of the table
2265
+ # would conflict with is found (and, outside the relation, not found:
2266
+ # RecordNotFound) instead of created again. They created it.
648
2267
  def find_or_create_by(attributes, &block)
649
- find_by(attributes) || create(attributes, &block)
2268
+ find_by(attributes) || create_or_find_by(attributes, &block)
650
2269
  end
651
2270
 
652
2271
  def find_or_create_by!(attributes, &block)
653
- find_by(attributes) || create!(attributes, &block)
2272
+ find_by(attributes) || create_or_find_by!(attributes, &block)
2273
+ end
2274
+
2275
+ # As ActiveRecord's: the first record the relation selects, or one
2276
+ # built or created with its equality conditions. They used to run on
2277
+ # a bare relation of the model, creating records without them.
2278
+ def first_or_initialize(attributes = nil, &block)
2279
+ first || new(attributes, &block)
2280
+ end
2281
+
2282
+ def first_or_create(attributes = nil, &block)
2283
+ first || create(attributes, &block)
654
2284
  end
655
2285
 
2286
+ def first_or_create!(attributes = nil, &block)
2287
+ first || create!(attributes, &block)
2288
+ end
2289
+
2290
+ # As Active Record's: create, in a savepoint, and when a unique index of
2291
+ # the table refuses the record (ActiveRecord::RecordNotUnique, which
2292
+ # ES-NoProj raises from the event store: Lyra::Projections
2293
+ # ::UniqueIndexes), the record the relation finds with +attributes+
2294
+ # (ActiveRecord::RecordNotFound when none: the value was taken by a
2295
+ # record the attributes do not select). It looked for a conflict on an
2296
+ # index made of the attributes alone before, and a value the block set
2297
+ # was created twice.
2298
+ def create_or_find_by(attributes, &block)
2299
+ create_or_find(attributes) { create(attributes, &block) }
2300
+ end
2301
+
2302
+ def create_or_find_by!(attributes, &block)
2303
+ create_or_find(attributes) { create!(attributes, &block) }
2304
+ end
2305
+
2306
+ def create_or_find(attributes)
2307
+ record = nil
2308
+ model_class.transaction(requires_new: true) do
2309
+ record = yield
2310
+ record._last_transaction_return_status || raise(ActiveRecord::Rollback)
2311
+ end
2312
+ record
2313
+ rescue ActiveRecord::RecordNotUnique
2314
+ model_class.with_connection(&:transaction_open?) ? rewhere(attributes).take! : find_by!(attributes)
2315
+ end
2316
+ private :create_or_find
2317
+
2318
+ def create_with(attributes)
2319
+ relation = spawn_records(@records)
2320
+ relation.plan = plan
2321
+ relation.create_with_attributes = attributes.nil? ? {} : create_with_attributes.merge(attributes.to_h.transform_keys(&:to_s))
2322
+ relation
2323
+ end
2324
+
2325
+ def create_with!(attributes)
2326
+ replace_with!(create_with(attributes))
2327
+ end
2328
+
2329
+ # As ActiveRecord's relation update: each selected record updated
2330
+ # (callbacks and validations run; under ES-NoProj each is an event),
2331
+ # or, given an id (or ids), Active Record's model.update(id, ...): the
2332
+ # model's record with that id, found and written through the model's
2333
+ # own read and write paths (its default scope, the purpose check, an
2334
+ # event), whatever this relation selects. It used to look for the id in
2335
+ # this relation only (RecordNotFound outside it), which Active Record
2336
+ # does not: an application's code behaves as it does in Monitor.
2337
+ #
2338
+ # The records are those the relation selects when update is called, as
2339
+ # Active Record loads them then (records_now): a record destroyed
2340
+ # since the relation was built is not updated (it raised
2341
+ # RecordDestroyedError), and one that matches since is. Once the
2342
+ # application has loaded the records (loaded_records?), Active Record
2343
+ # updates those: one destroyed since takes the attributes in memory
2344
+ # and writes nothing, as Active Record's update of a row already gone
2345
+ # reaches none.
2346
+ def update(id = :all, attributes)
2347
+ return model_class.update(id, attributes) unless id == :all
2348
+
2349
+ update_each(attributes) { |record| record.update(attributes) }
2350
+ end
2351
+
2352
+ def update!(id = :all, attributes)
2353
+ return model_class.update!(id, attributes) unless id == :all
2354
+
2355
+ update_each(attributes) { |record| record.update!(attributes) }
2356
+ end
2357
+
2358
+ def update_each(attributes)
2359
+ return deliver { records_now }.each { |record| yield record } unless loaded_records?
2360
+
2361
+ loaded = records
2362
+ checks = events_only? && Lyra.config.lock_aggregate_before_append
2363
+ dead = checks ? Lyra::StreamTail.destroyed_streams(loaded.map { record_stream(_1) }).to_set : Set.new
2364
+ loaded.each { |record| dead.include?(record_stream(record)) ? record.assign_attributes(attributes) : yield(record) }
2365
+ end
2366
+ private :update_each
2367
+
2368
+ # As ActiveRecord's: each selected record's counters moved by the
2369
+ # given amounts (nil counting as 0), and touch: the timestamps, in
2370
+ # one update_all per set of current values, so each record gets the
2371
+ # event an update_all writes. It used to bump every record of the
2372
+ # model.
2373
+ # Nothing to change (no counter, no touch) is Active Record's
2374
+ # update_all's ArgumentError; it used to return 0.
2375
+ def update_counters(counters)
2376
+ counters = counters.dup
2377
+ touch = counters.delete(:touch)
2378
+ counters = counters.transform_keys(&:to_s)
2379
+ unknown = counters.keys - model_class.column_names
2380
+ raise UnsupportedQuery.new(model_class, "update_counters(#{unknown.join(', ')})") if unknown.any?
2381
+
2382
+ touched = {}
2383
+ if touch
2384
+ names = Array.wrap(touch == true ? nil : touch)
2385
+ options = names.extract_options!
2386
+ touched = model_class.touch_attributes_with_time(*names, **options)
2387
+ end
2388
+ raise ArgumentError, "Empty list of attributes to change" if counters.empty? && touched.empty?
2389
+ return 0 if @records.empty? && rebuild_recipe.nil?
2390
+
2391
+ model_class.transaction do
2392
+ # In the events-only store, the records the relation selects now,
2393
+ # every one's lock taken first, in one statement, and its counters
2394
+ # as its stream has them then (locked_records_now): the values the
2395
+ # relation holds may be stale, and an increment of them lost a
2396
+ # concurrent write's. Taking the locks group by group, in the order
2397
+ # of the values, deadlocked two writers.
2398
+ records = events_only? ? locked_records_now(current: true) : @records
2399
+ groups = records.group_by { |record| counters.keys.map { |column| record.read_attribute(column) } }
2400
+ groups.sum do |values, group|
2401
+ updates = counters.keys.zip(values).to_h { |column, value| [column, (value || 0) + counters[column]] }
2402
+ batch = spawn_records(group)
2403
+ batch.live_checked = true
2404
+ batch.update_all(updates.merge(touched))
2405
+ end
2406
+ end
2407
+ end
2408
+
2409
+ # There is no SQL: the relation is answered from the event store.
2410
+ def to_sql
2411
+ raise UnsupportedQuery.new(model_class, "to_sql (the relation is answered from the event store, not by SQL)")
2412
+ end
2413
+
2414
+ # Hints for the database, which ES-NoProj reads do not reach. A lock
2415
+ # takes no row lock: writes take Lyra's aggregate lock.
2416
+ %i[lock annotate optimizer_hints strict_loading].each do |name|
2417
+ define_method(name) { |*_args| self }
2418
+ end
2419
+
2420
+ # As ActiveRecord defines them: where, then the bulk write. They went
2421
+ # through the scope branch of method_missing and were refused.
2422
+ def delete_by(*args)
2423
+ where(*args).delete_all
2424
+ end
2425
+
2426
+ def destroy_by(*args)
2427
+ where(*args).destroy_all
2428
+ end
2429
+
2430
+ # In place, as Active Record's: it used to return a limited copy and
2431
+ # leave the relation as it was.
656
2432
  def limit!(value)
657
- spawn_records(@records.first(value))
2433
+ replace_with!(limit(value))
658
2434
  end
659
2435
 
660
2436
  def values
@@ -674,6 +2450,8 @@ module Lyra
674
2450
  def extending!(*modules, &block)
675
2451
  modules = modules.flatten
676
2452
  modules << Module.new(&block) if block
2453
+ kaminari, modules = modules.partition { CachedRelation.kaminari_module?(_1) }
2454
+ @pagination ||= {} if kaminari.any?
677
2455
  return self if modules.empty?
678
2456
 
679
2457
  @extensions = extensions + modules
@@ -689,57 +2467,116 @@ module Lyra
689
2467
  spawn_records(@records.dup)
690
2468
  end
691
2469
 
692
- def merge(other, *rest)
693
- # For association scopes, just return self since we already have filtered records
694
- self
695
- end
2470
+ # As ActiveRecord's merge: the records both select, in this relation's
2471
+ # order and then the other's. Where the other's conditions name a
2472
+ # column this relation's conditions name too, the other's replace this
2473
+ # one's, as Active Record's merge replaces them (where(status: "a")
2474
+ # .merge(where(status: "b")) is the "b" records); this relation's are
2475
+ # taken off as unscope takes them off, or the merge is refused.
2476
+ #
2477
+ # An ActiveRecord relation of the model is evaluated
2478
+ # (CachedRelation#narrowed_by: its where clause, order and limit; what
2479
+ # it unscopes is taken off this one first); a relation read from the
2480
+ # event store is intersected with this one (joined on either side, the
2481
+ # rows of the join). Another model's relation raises UnsupportedQuery.
2482
+ #
2483
+ # As Active Record: the arguments after the first are its options, and
2484
+ # none changes the answer (merge(other, rewhere: true) is merge(other);
2485
+ # they used to be merged as more relations); nil or false is an
2486
+ # ArgumentError (it used to return this relation); an Array is the
2487
+ # loaded records it has too; a Proc is evaluated on the relation.
2488
+ def merge(other, *_options)
2489
+ case other
2490
+ when nil, false then raise ArgumentError, "invalid argument: #{other.inspect}."
2491
+ when Array then records & other
2492
+ when CachedRelation then merge_events_relation(other)
2493
+ when ActiveRecord::Relation then merge_active_record_relation(other)
2494
+ when Hash then raise UnsupportedQuery.new(model_class, "merge of a Hash (#{other.keys.join(', ')})")
2495
+ else
2496
+ return clone.instance_exec(&other) if other.respond_to?(:to_proc)
696
2497
 
697
- def merge!(other, *rest)
698
- self
2498
+ raise ArgumentError, "#{other.inspect} is not an ActiveRecord::Relation"
2499
+ end
699
2500
  end
700
2501
 
701
- def bind_attribute(name, value)
702
- self
703
- end
2502
+ def merge!(other, *options)
2503
+ raise ArgumentError, "#{other.inspect} is not an ActiveRecord::Relation" if other.is_a?(Array)
704
2504
 
705
- def rewhere(conditions)
706
- where(conditions)
2505
+ merged = merge(other, *options)
2506
+ merged.is_a?(CachedRelation) ? replace_with!(merged) : merged
707
2507
  end
708
2508
 
709
- def except(*skips)
2509
+ def bind_attribute(name, value)
710
2510
  self
711
2511
  end
712
2512
 
2513
+ def except(*skips)
2514
+ unscope(*skips)
2515
+ end
2516
+
2517
+ # As Active Record's: the relation with only the parts named, the rest
2518
+ # taken off (only(:where) keeps the conditions and drops the order,
2519
+ # limit, select and group; only(:order) keeps the order of every record;
2520
+ # only() keeps nothing). The conditions and order are taken off as
2521
+ # unscope takes them off (replaying the relation's plan, or refusing
2522
+ # what it cannot take back); the window by CachedRelationWindow. A
2523
+ # joined relation without its joins is refused (its records are the
2524
+ # join's rows). only used to be refused, except only(:joins,
2525
+ # :left_outer_joins), which returned the relation unchanged.
713
2526
  def only(*keeps)
714
- self
715
- end
2527
+ kept = keeps.flatten.map(&:to_sym)
2528
+ if joined_associations.any? && (kept & %i[joins left_outer_joins]).empty?
2529
+ raise UnsupportedQuery.new(model_class, "only(#{kept.map(&:inspect).join(', ')}) of a joined relation " \
2530
+ "(its records are the rows of the join)")
2531
+ end
716
2532
 
717
- def where!(conditions)
718
- # In-place where modification (for AR internal use)
719
- # Filter records and update @records directly
720
- return self if conditions.nil?
2533
+ drops = []
2534
+ drops << :where unless kept.include?(:where)
2535
+ drops << :order unless kept.include?(:order)
2536
+ drops << :select unless kept.include?(:select)
2537
+ drops << :group unless kept.include?(:group)
2538
+ relation = unscope(*drops)
2539
+ relation = relation.clone.inherit_rebuild(relation) if relation.equal?(self)
2540
+ relation.create_with_attributes = {} unless kept.include?(:create_with)
2541
+ relation
2542
+ end
721
2543
 
722
- @records = @records.select do |record|
723
- case conditions
724
- when Hash
725
- conditions.all? { |key, value| matches_value?(record, key, value) }
726
- else
727
- true
728
- end
2544
+ def where!(conditions, *rest)
2545
+ # In-place where (ActiveRecord's internals build relations with it).
2546
+ # Hash conditions only: anything else used to be ignored, keeping
2547
+ # every record.
2548
+ return self if conditions.blank?
2549
+ unless conditions.is_a?(Hash) && rest.empty?
2550
+ raise UnsupportedQuery.new(model_class, "where!(#{[conditions, *rest].inspect[1..-2][0, 60]})")
729
2551
  end
730
- self
2552
+
2553
+ replace_with!(where(conditions))
731
2554
  end
732
2555
 
2556
+ # In place, as order (its plan kept, and before a window).
733
2557
  def order!(*args)
734
- # In-place order modification
735
2558
  return self if args.empty?
736
- keys = order_keys(args)
737
- @records = @records.sort { |a, b| compare_for_order(a, b, keys) }
738
- self
2559
+
2560
+ replace_with!(order(*args))
2561
+ end
2562
+
2563
+ # In place, as reverse_order. Active Record's last asks it of the
2564
+ # relation it builds (an association's last, through its scope).
2565
+ def reverse_order!
2566
+ replace_with!(reverse_order)
739
2567
  end
740
2568
 
2569
+ # As Active Record's: the selected columns replaced. It used to do
2570
+ # nothing, and the records came back with the columns of the first
2571
+ # select only.
741
2572
  def reselect(*args)
742
- self
2573
+ raise ArgumentError, "The method .reselect() must contain arguments." if args.blank?
2574
+ raise UnsupportedQuery.new(model_class, "reselect of a distinct relation (SELECT DISTINCT)") if distinct?
2575
+
2576
+ relation = spawn_records(@records)
2577
+ relation.plan = plan
2578
+ relation.selected_columns = select_names(args)
2579
+ relation
743
2580
  end
744
2581
 
745
2582
  def select(*args, &block)
@@ -748,13 +2585,24 @@ module Lyra
748
2585
  spawn_records(@records.select(&block))
749
2586
  else
750
2587
  # AR select (column selection): the records stay whole, for
751
- # filtering; the ones returned are narrowed to these columns.
2588
+ # filtering; the ones returned are narrowed to these columns. A
2589
+ # second select adds its columns, as Active Record's does (it used
2590
+ # to replace the first's).
2591
+ raise ArgumentError, "Call `select' with at least one field." if args.blank?
2592
+ raise UnsupportedQuery.new(model_class, "select of a distinct relation (SELECT DISTINCT)") if distinct?
2593
+
752
2594
  relation = spawn_records(@records)
753
- relation.selected_columns = args.flatten.map { |name| name.to_s.split(".").last.delete('"') }
2595
+ relation.plan = plan
2596
+ relation.selected_columns = ((selected_columns || []) + select_names(args)).uniq
754
2597
  relation
755
2598
  end
756
2599
  end
757
2600
 
2601
+ def select_names(args)
2602
+ args.flatten.compact.map { |name| name.to_s.split(".").last.delete('"') }
2603
+ end
2604
+ private :select_names
2605
+
758
2606
  # =========================================================================
759
2607
  # Aggregations
760
2608
  # =========================================================================
@@ -778,6 +2626,7 @@ module Lyra
778
2626
  # stock location this way.
779
2627
  def group(*columns)
780
2628
  relation = spawn_records(@records)
2629
+ relation.plan = plan
781
2630
  relation.group_columns = group_columns + columns.flatten.map { |c| c.to_s.split(".").last.delete('"') }
782
2631
  relation
783
2632
  end
@@ -815,7 +2664,14 @@ module Lyra
815
2664
  aggregate_values(column_name).max
816
2665
  end
817
2666
 
2667
+ # A window of a grouped query is a window of its groups (GROUP BY,
2668
+ # then LIMIT): refused, rather than grouping the window's records.
818
2669
  private def aggregate_by_group(operation, column_name)
2670
+ if window_source
2671
+ raise UnsupportedQuery.new(model_class, "#{operation} of a group with a limit or offset " \
2672
+ "(the window would apply to the groups)")
2673
+ end
2674
+
819
2675
  @records.group_by do |record|
820
2676
  values = group_columns.map { |column| record.read_attribute(column) }
821
2677
  values.size == 1 ? values.first : values
@@ -861,26 +2717,446 @@ module Lyra
861
2717
  end
862
2718
 
863
2719
  # =========================================================================
864
- # Batching (simplified - all records are in memory)
2720
+ # Batching, as Active Record's: by primary key (or the cursor columns),
2721
+ # ascending or as order: says, from start: to finish:, the relation's
2722
+ # own order ignored (logged, or ArgumentError with error_on_ignore:).
2723
+ # A limit and an offset are taken as Active Record takes them: the
2724
+ # batches stop after limit records, and each batch's query skips the
2725
+ # offset's records after the last batch (Active Record applies OFFSET
2726
+ # to every batch's query). They used to follow the relation's order,
2727
+ # and an offset was refused.
865
2728
  # =========================================================================
866
2729
 
867
- def find_each(batch_size: 1000, &block)
868
- each(&block)
2730
+ def find_each(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, cursor: model_class.primary_key,
2731
+ order: :asc, &block)
2732
+ options = { start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore,
2733
+ cursor: cursor, order: order }
2734
+ unless block
2735
+ return enum_for(:find_each, **options) do
2736
+ batch_records(start: start, finish: finish, cursor: cursor, order: order, counting: true).size
2737
+ end
2738
+ end
2739
+
2740
+ find_in_batches(**options) { |records| records.each(&block) }
2741
+ end
2742
+
2743
+ def find_in_batches(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, cursor: model_class.primary_key,
2744
+ order: :asc)
2745
+ options = { start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore,
2746
+ cursor: cursor, order: order }
2747
+ unless block_given?
2748
+ return to_enum(:find_in_batches, **options) do
2749
+ total = batch_records(start: start, finish: finish, cursor: cursor, order: order, counting: true).size
2750
+ (total - 1).div(batch_size) + 1
2751
+ end
2752
+ end
2753
+
2754
+ batch_slices(of: batch_size, start: start, finish: finish, error_on_ignore: error_on_ignore,
2755
+ cursor: cursor, order: order).each { |slice| yield slice }
2756
+ nil
869
2757
  end
870
2758
 
871
- def find_in_batches(batch_size: 1000)
872
- @records.each_slice(batch_size) do |batch|
2759
+ # Each batch a relation: of the records of the batch, in the relation's
2760
+ # order, with its limit and offset (Active Record's rewhere(cursor =>
2761
+ # values)), or with load: true the batch's records, in the batch's
2762
+ # order. Without a block, Active Record's BatchEnumerator over this
2763
+ # relation (each, each_record, update_all, delete_all, ...); it used to
2764
+ # raise LocalJumpError.
2765
+ #
2766
+ # On a loaded relation (loaded?), as Active Record's: the records
2767
+ # loaded, in slices, each batch this relation loaded with its slice
2768
+ # (query_relation: its query methods, aggregates and update_all,
2769
+ # delete_all and touch_all are this relation's).
2770
+ def in_batches(of: 1000, start: nil, finish: nil, load: false, error_on_ignore: nil, cursor: model_class.primary_key,
2771
+ order: :asc, use_ranges: nil, &block)
2772
+ cursor = Array(cursor).map(&:to_s)
2773
+ check_batch_options!(cursor, start, finish, order)
2774
+ ignored_order!(error_on_ignore) if ordering.any?
2775
+ unless block
2776
+ return ActiveRecord::Batches::BatchEnumerator.new(of: of, start: start, finish: finish, relation: self,
2777
+ cursor: cursor, order: order, use_ranges: use_ranges)
2778
+ end
2779
+
2780
+ recipe = rebuild_recipe
2781
+ if loaded_records?
2782
+ loaded_batch_slices(of, start, finish, cursor, order).each do |slice|
2783
+ batch = spawn_records(slice)
2784
+ batch.rebuild_recipe = recipe
2785
+ batch.loaded_with(self)
2786
+ yield batch
2787
+ end
2788
+ return nil
2789
+ end
2790
+
2791
+ ranges = !load && (use_ranges || (use_ranges.nil? && batch_scope_empty?))
2792
+ refuse_ranges_window! if ranges
2793
+ refuse_unordered_offset!("in_batches") if !load && unordered_offset?
2794
+ each_batch(of: of, start: start, finish: finish, cursor: cursor, order: order, counting: !load) do |slice, source, range|
2795
+ if load
2796
+ batch = source.spawn_records(slice)
2797
+ batch_recipe(batch, recipe, slice)
2798
+ # Active Record's rewhere(cursor => values), loaded with the
2799
+ # batch's records: its query, built when first asked for. Its
2800
+ # records are read in the cursor's order; its query, with an
2801
+ # offset and no order, is refused (unordered_offset?).
2802
+ query = batch.rebuild_recipe
2803
+ batch.loaded_with(lambda {
2804
+ refuse_unordered_offset!("a query of an in_batches(load: true) batch") if unordered_offset?
2805
+ fresh = query && evaluate_relation(query)
2806
+ next fresh if fresh
2807
+
2808
+ keyed = source.send(:batch_of, slice)
2809
+ keyed = keyed.windowed_by(limit_value, offset_value) unless limit_value.nil? && offset_value.nil?
2810
+ batch_recipe(keyed, recipe, slice)
2811
+ keyed
2812
+ })
2813
+ else
2814
+ keys = ranges ? range : slice
2815
+ batch = source.send(:batch_of, keys)
2816
+ batch = batch.windowed_by(limit_value, offset_value) unless ranges || (limit_value.nil? && offset_value.nil?)
2817
+ batch_recipe(batch, recipe, keys)
2818
+ end
873
2819
  yield batch
874
2820
  end
2821
+ nil
2822
+ end
2823
+
2824
+ # Whether this relation has an offset and no order: Active Record's
2825
+ # in_batches batch is its rewhere(cursor => values) with the OFFSET and
2826
+ # no ORDER BY, so which of the batch's records it skips is the order
2827
+ # the database's plan reads them in (an index scan, or the table's
2828
+ # physical order after an update), which nothing here can follow.
2829
+ def unordered_offset?
2830
+ ordering.empty? && !offset_value.nil? && window_number(offset_value, "offset").positive?
2831
+ end
2832
+ private :unordered_offset?
2833
+
2834
+ def refuse_unordered_offset!(what)
2835
+ raise UnsupportedQuery.new(model_class, "#{what} of a relation with an offset and no order (Active Record " \
2836
+ "skips the offset's records of each batch in the order the database " \
2837
+ "reads them; give the relation an order)")
2838
+ end
2839
+ private :refuse_unordered_offset!
2840
+
2841
+ # This relation narrowed to +records+ (a batch).
2842
+ def batch_of(records)
2843
+ members = records.to_h { |record| [record, true] }.compare_by_identity
2844
+ filter_rows { |record, _joins| members.key?(record) }
2845
+ end
2846
+ private :batch_of
2847
+
2848
+ # Each batch is Active Record's where(primary key => the keys of
2849
+ # +records+, those the batch's query selected) on this relation: a
2850
+ # write on it selects them again (Rebuildable), so a record of the
2851
+ # batch destroyed since, or no longer matching, is not written. On a
2852
+ # relation with a limit or offset, the condition is applied before
2853
+ # them, as Active Record's rewhere is (the offset skips records of the
2854
+ # batch again).
2855
+ #
2856
+ # A relation that starts from every record of the model starts from
2857
+ # the batch's keys instead (Rebuildable::Keyed): the steps then select
2858
+ # among them what they select among every record, the window included
2859
+ # (it is taken after the condition on the keys), at the cost of the
2860
+ # batch's records, not of the model's size.
2861
+ def batch_recipe(batch, recipe, records)
2862
+ return unless recipe
2863
+
2864
+ ids = records.map { record_id(_1) }
2865
+ keys = { model_class.primary_key => ids }
2866
+ root = recipe[0].is_a?(Rebuildable::AllRecords) ? Rebuildable::Keyed.new(model_class, Rebuildable.copy(ids)) : recipe[0]
2867
+ batch.rebuild_recipe = [root, recipe[1] + [[:where, [Rebuildable.copy(keys)], {}.freeze]]]
2868
+ end
2869
+ private :batch_recipe
2870
+
2871
+ # Active Record's empty_scope for batches: a relation of the model's
2872
+ # every record (to_sql is unscoped.all's).
2873
+ def batch_scope_empty?
2874
+ window_source.nil? && query_terms.empty? && ordering.empty? && joined_associations.empty? &&
2875
+ default_scope_applied.nil? && selected_columns.nil? && group_columns.empty?
2876
+ end
2877
+ private :batch_scope_empty?
2878
+
2879
+ # Batches as ranges of the cursor (use_ranges: true) on a relation with
2880
+ # an offset: Active Record reads each range with the OFFSET and no
2881
+ # ORDER BY, so which records it skips is the table's physical order.
2882
+ # A limit of 0 makes its OFFSET -1, which the database rejects.
2883
+ def refuse_ranges_window!
2884
+ if !offset_value.nil? && !window_number(offset_value, "offset").zero?
2885
+ raise UnsupportedQuery.new(model_class, "in_batches(use_ranges: true) of a relation with an offset (Active " \
2886
+ "Record skips records in the table's physical order)")
2887
+ end
2888
+ return if limit_value.nil? || window_number(limit_value, "limit").positive?
2889
+
2890
+ raise UnsupportedQuery.new(model_class, "in_batches(use_ranges: true) of a relation with limit(0) (Active " \
2891
+ "Record's query has OFFSET -1)")
2892
+ end
2893
+ private :refuse_ranges_window!
2894
+
2895
+ private
2896
+
2897
+ # The batches' records, as Active Record forms them: after its checks
2898
+ # of the options (ArgumentError as Active Record's), a lazy enumerator
2899
+ # of the slices, each read when the loop reaches it (each_batch). On a
2900
+ # loaded relation, the loaded records in slices.
2901
+ def batch_slices(of:, start:, finish:, error_on_ignore:, cursor:, order:, counting: false)
2902
+ cursor = Array(cursor).map(&:to_s)
2903
+ check_batch_options!(cursor, start, finish, order)
2904
+ ignored_order!(error_on_ignore) if ordering.any?
2905
+ return loaded_batch_slices(of, start, finish, cursor, order).lazy if loaded_records?
2906
+
2907
+ enum_for(:each_batch, of: of, start: start, finish: finish, cursor: cursor, order: order, counting: counting)
2908
+ .lazy.map { |slice, _, _| slice }
2909
+ end
2910
+
2911
+ # Active Record's batches of a loaded relation: its records (the
2912
+ # window's), from +start+ to +finish+, in cursor order, in slices of
2913
+ # +of+ (or of the limit, if smaller). Nothing is read again.
2914
+ def loaded_batch_slices(of, start, finish, cursor, order)
2915
+ directions = cursor.zip(Array(order)).map { |_column, direction| (direction || :asc).to_sym }
2916
+ records = @records.to_a
2917
+ records = records.select { |record| within_batch_bound?(record, cursor, directions, start, finish) } if start || finish
2918
+ records = sort_records(records, cursor.zip(directions).map { |column, direction| { column.to_sym => direction } })
2919
+ limit = limit_value.nil? ? nil : window_number(limit_value, "limit")
2920
+ size = limit && limit < of ? limit : of
2921
+ size.positive? ? records.each_slice(size).to_a : []
2922
+ end
2923
+
2924
+ # The records batched, counted (the sizes of the find_each and
2925
+ # find_in_batches enumerators): those the relation selects before its
2926
+ # window, from +start+ to +finish+, in cursor order (+counting+: those
2927
+ # a right in force does not leave out), then its offset and limit, as
2928
+ # Active Record counts the relation with the bounds' conditions.
2929
+ def batch_records(start:, finish:, cursor:, order:, counting: false)
2930
+ cursor = Array(cursor).map(&:to_s)
2931
+ source = window_source || self
2932
+ records = source.send(:batch_candidates, start: start, finish: finish, cursor: cursor, order: order)
2933
+ records = batch_kept(records) if counting
2934
+ records = records.drop(window_number(offset_value, "offset")) unless offset_value.nil?
2935
+ limit_value.nil? ? records : records.first(window_number(limit_value, "limit"))
2936
+ end
2937
+
2938
+ # The records this relation (no window) selects, from +start+ to
2939
+ # +finish+, in cursor order.
2940
+ def batch_candidates(start:, finish:, cursor:, order:)
2941
+ directions = cursor.zip(Array(order)).map { |_column, direction| (direction || :asc).to_sym }
2942
+ records = raw_records.to_a
2943
+ records = records.select { |record| within_batch_bound?(record, cursor, directions, start, finish) } if start || finish
2944
+ sort_records(records, cursor.zip(directions).map { |column, direction| { column.to_sym => direction } })
2945
+ end
2946
+
2947
+ # The records a count of the batches keeps: all of them, here.
2948
+ def batch_kept(records) = records
2949
+
2950
+ # Active Record's batch loop (batch_on_unloaded_relation): each batch
2951
+ # read when the loop reaches it, the records after the last batch's
2952
+ # last cursor values, past the offset's records, the batch's size of
2953
+ # them: +of+, or what is left of the limit when that is less. It ends
2954
+ # with an empty batch, one smaller than +of+ (or the limit), or when the
2955
+ # limit's records were given. Yields [the batch's records, the relation
2956
+ # they were selected from (its window left out), the records of the
2957
+ # batch's range of the cursor (those a right leaves out included)].
2958
+ #
2959
+ # A relation that can be built again (Rebuildable) is read from the
2960
+ # event store for each batch, as Active Record queries each, so a record
2961
+ # inserted, changed or deleted during the loop is seen as Active Record
2962
+ # sees it; any other gives the records it holds.
2963
+ def each_batch(of:, start:, finish:, cursor:, order:, counting:)
2964
+ offset = offset_value.nil? ? 0 : window_number(offset_value, "offset")
2965
+ limit = limit_value.nil? ? nil : window_number(limit_value, "limit")
2966
+ batch_limit = limit && limit < of ? limit : of
2967
+ remaining = limit
2968
+ reader = rebuild_recipe ? live_batch_reader(start, finish, cursor, order, counting) :
2969
+ held_batch_reader(start, finish, cursor, order, counting)
2970
+ last = nil
2971
+ loop do
2972
+ size = remaining && remaining < batch_limit ? remaining : batch_limit
2973
+ source, slice, range = reader.call(last, offset, size)
2974
+ break if slice.empty?
2975
+
2976
+ if cursor.any? { |column| slice.last.read_attribute(column).nil? }
2977
+ raise ArgumentError, "Not all of the batch cursor columns were included in the custom select clause " \
2978
+ "or some columns contain nil."
2979
+ end
2980
+ yield slice, source, range
2981
+ break if slice.size < batch_limit
2982
+
2983
+ if remaining
2984
+ remaining -= slice.size
2985
+ break unless remaining.positive?
2986
+ end
2987
+ last = slice.last
2988
+ end
2989
+ end
2990
+
2991
+ # Reads each batch from the relation built again (keyed_batch_now, or
2992
+ # whole_batch_now): the records after +last+'s cursor values, those a
2993
+ # right leaves out taken out when +counting+, past +offset+ of them,
2994
+ # the next +size+.
2995
+ def live_batch_reader(start, finish, cursor, order, counting)
2996
+ directions = cursor.zip(Array(order)).map { |_column, direction| (direction || :asc).to_sym }
2997
+ recipe = rebuild_recipe
2998
+ lambda do |last, offset, size|
2999
+ bound = last && cursor.map { |column| last.read_attribute(column) }
3000
+ source, every, kept = keyed_batch_now(recipe, bound, offset + size, start, finish, cursor, order, counting) ||
3001
+ whole_batch_now(recipe, bound, start, finish, cursor, order, directions, counting)
3002
+ slice = kept.drop(offset).first(size)
3003
+ upto = slice.empty? ? -1 : every.index { _1.equal?(slice.last) }
3004
+ [source, slice, every.first(upto + 1)]
3005
+ end
3006
+ end
3007
+
3008
+ # The same over the records this relation holds (it cannot be built
3009
+ # again), sorted once.
3010
+ def held_batch_reader(start, finish, cursor, order, counting)
3011
+ source = window_source || self
3012
+ every = source.send(:batch_candidates, start: start, finish: finish, cursor: cursor, order: order)
3013
+ kept = counting ? batch_kept(every) : every
3014
+ every_at = every.each_with_index.to_h.compare_by_identity
3015
+ kept_at = kept.each_with_index.to_h.compare_by_identity
3016
+ lambda do |last, offset, size|
3017
+ from = last ? kept_at.fetch(last) + 1 : 0
3018
+ slice = kept[from + offset, size] || []
3019
+ range = slice.empty? ? [] : every[(last ? every_at.fetch(last) + 1 : 0)..every_at.fetch(slice.last)]
3020
+ [source, slice, range]
3021
+ end
3022
+ end
3023
+
3024
+ # A batch from the relation built again whole: [the relation (its
3025
+ # window left out), its records after +bound+ in cursor order, the same
3026
+ # less those a right leaves out when +counting+]. One that cannot be
3027
+ # built again (an UnsupportedQuery now) gives the records it holds,
3028
+ # every time: the batches it gave before.
3029
+ def whole_batch_now(recipe, bound, start, finish, cursor, order, directions, counting)
3030
+ fresh = evaluate_relation(recipe) || self
3031
+ fresh = fresh.window_source || fresh
3032
+ every = fresh.send(:batch_candidates, start: start, finish: finish, cursor: cursor, order: order)
3033
+ every = every.select { |record| after_cursor?(record, cursor, directions, bound) } if bound
3034
+ [fresh, every, counting ? fresh.send(:batch_kept, every) : every]
3035
+ end
3036
+
3037
+ # The same, read by key, where the relation starts from every record
3038
+ # of the model and the cursor is its integer primary key (Active
3039
+ # Record's usual batches): the keys of the model's records after
3040
+ # +bound+ are listed (one query), and the relation's steps applied to
3041
+ # the records of the next keys, a chunk at a time, until +need+ are
3042
+ # selected (the offset's and the batch's). So a batch costs about its
3043
+ # own records, not the model's size. Of what the steps select from a
3044
+ # chunk, only the chunk's records are taken: a step that brings in
3045
+ # records of its own (an or, whose other relation is read whole) keeps
3046
+ # or leaves out each of the chunk's as it would among every record (an
3047
+ # or brought in the other relation's records whatever the chunk and
3048
+ # the cursor, so a batch repeated records, and a loop without a limit
3049
+ # never ended). Steps that decide a record by the others (distinct,
3050
+ # group) are read whole. A window (limit, offset) is taken by the
3051
+ # batch loop, from the relation the chunk's window was taken from. nil
3052
+ # where this does not apply.
3053
+ KEYED_WHOLE_STEPS = %i[distinct uniq group].freeze
3054
+
3055
+ def keyed_batch_now(recipe, bound, need, start, finish, cursor, order, counting)
3056
+ pk = model_class.primary_key
3057
+ return nil unless recipe[0].is_a?(Rebuildable::AllRecords) && cursor == [pk]
3058
+ return nil if recipe[1].any? { |name, *| KEYED_WHOLE_STEPS.include?(name) }
3059
+
3060
+ keys = CachedProjection.stream_versions_for(model_class).keys
3061
+ return nil unless keys.all? { _1.match?(/\A\d+\z/) }
3062
+
3063
+ keys = keys.map(&:to_i).sort
3064
+ keys.reverse! if Array(order).first.to_s == "desc"
3065
+ if bound
3066
+ after = Integer(bound.first)
3067
+ keys = Array(order).first.to_s == "desc" ? keys.select { _1 < after } : keys.select { _1 > after }
3068
+ end
3069
+ every = []
3070
+ kept = []
3071
+ keys.each_slice([need, KEYED_CHUNK].max) do |chunk|
3072
+ part_relation = evaluate_relation([Rebuildable::Keyed.new(model_class, chunk), recipe[1]])
3073
+ return nil unless part_relation
3074
+
3075
+ part_relation = part_relation.window_source || part_relation
3076
+ in_chunk = chunk.to_set
3077
+ part = part_relation.send(:batch_candidates, start: start, finish: finish, cursor: cursor, order: order)
3078
+ .select { |record| in_chunk.include?(record_id(record)) }
3079
+ every.concat(part)
3080
+ kept.concat(counting ? part_relation.send(:batch_kept, part) : part)
3081
+ break if kept.size >= need
3082
+ end
3083
+ # The batch's relation holds them in the relation's order, as one
3084
+ # built whole does (by primary key without an order).
3085
+ held = ordering.any? ? sort_records(every, ordering) : every.sort_by { record_id(_1) }
3086
+ source = spawn_records(held)
3087
+ source.windowed = false
3088
+ [source, every, kept]
3089
+ rescue ArgumentError, TypeError
3090
+ nil
3091
+ end
3092
+
3093
+ # Keys read at a time by keyed_batch_now.
3094
+ KEYED_CHUNK = 500
3095
+
3096
+ # Whether +record+'s cursor values come after +last+ in the cursor's
3097
+ # directions.
3098
+ def after_cursor?(record, cursor, directions, last)
3099
+ cursor.zip(directions, last).each do |column, direction, bound|
3100
+ compared = TypedValues.compare(record.read_attribute(column), bound) || 0
3101
+ compared = -compared if direction == :desc
3102
+ return compared.positive? unless compared.zero?
3103
+ end
3104
+ false
3105
+ end
3106
+
3107
+ # The relation +recipe+ builds now, as Lyra's own read, or nil.
3108
+ def evaluate_relation(recipe)
3109
+ Lyra::PurposeBoundReads.internal(INTERNAL) { model_class.uncached { Rebuildable.rebuild(recipe) } }
3110
+ rescue UnsupportedQuery
3111
+ nil
3112
+ end
3113
+
3114
+ # Active Record's start: and finish: conditions: the cursor's values
3115
+ # at or after +start+ and at or before +finish+, in the cursor's
3116
+ # directions (a record with a NULL among them is outside, as SQL's
3117
+ # comparison is unknown).
3118
+ def within_batch_bound?(record, cursor, directions, start, finish)
3119
+ values = cursor.map { |column| record.read_attribute(column) }
3120
+ return false if values.any?(&:nil?)
3121
+
3122
+ compare = lambda do |bound|
3123
+ cursor.zip(values, Array(bound), directions).each do |column, value, limit, direction|
3124
+ limit = model_class.type_for_attribute(column).cast(limit)
3125
+ compared = TypedValues.compare(value, limit) || 0
3126
+ compared = -compared if direction == :desc
3127
+ return compared unless compared.zero?
3128
+ end
3129
+ 0
3130
+ end
3131
+ (start.nil? || compare.call(start) >= 0) && (finish.nil? || compare.call(finish) <= 0)
875
3132
  end
876
3133
 
877
- def in_batches(of: 1000)
878
- @records.each_slice(of) do |batch|
879
- yield spawn_records(batch)
3134
+ def check_batch_options!(cursor, start, finish, order)
3135
+ raise ArgumentError, ":start must contain one value per cursor column" if start && Array(start).size != cursor.size
3136
+ raise ArgumentError, ":finish must contain one value per cursor column" if finish && Array(finish).size != cursor.size
3137
+ if (Array(model_class.primary_key) - cursor).any?
3138
+ indexes = model_class.with_connection { |connection| connection.schema_cache.indexes(model_class.table_name) }
3139
+ unless indexes.any? { |index| index.unique && index.where.nil? && (Array(index.columns) - cursor).empty? }
3140
+ raise ArgumentError, ":cursor must include a primary key or other unique column(s)"
3141
+ end
880
3142
  end
3143
+ return unless (Array(order) - %i[asc desc]).any?
3144
+
3145
+ raise ArgumentError, ":order must be :asc or :desc or an array consisting of :asc or :desc, got #{order.inspect}"
3146
+ end
3147
+
3148
+ def ignored_order!(error_on_ignore)
3149
+ raise_error = error_on_ignore.nil? ? ActiveRecord.error_on_ignored_order : error_on_ignore
3150
+ raise ArgumentError, ActiveRecord::Batches::ORDER_IGNORE_MESSAGE if raise_error
3151
+
3152
+ model_class.logger&.warn(ActiveRecord::Batches::ORDER_IGNORE_MESSAGE) if model_class.respond_to?(:logger)
881
3153
  end
882
3154
 
3155
+ public
3156
+
883
3157
  prepend CachedRelationDelivery
3158
+ prepend CachedRelationWindow
3159
+ prepend Rebuildable.wrappers
884
3160
 
885
3161
  # =========================================================================
886
3162
  # Inspection
@@ -898,15 +3174,45 @@ module Lyra
898
3174
  # Scope support - AR calls _exec_scope for named scopes
899
3175
  # =========================================================================
900
3176
 
901
- # Called by AR when executing scopes
902
- # Rails passes scope arguments and the scope body block
3177
+ # A scope called on the model (Model.active is all._exec_scope): its
3178
+ # body runs on this relation, as Active Record runs it, and its value
3179
+ # is the scope's: nil or false is the relation, anything else what the
3180
+ # body returned, as Active Record's (a scope whose body returns a
3181
+ # count, a record or an Array returns it; it used to return the
3182
+ # relation). An Active Record relation of the model it returns is
3183
+ # answered from the event store, every record narrowed by it (it is the
3184
+ # scope's value in Active Record, whatever relation the scope was
3185
+ # called on); it used to be ignored and this relation returned.
3186
+ #
3187
+ # A body whose where holds an Arel condition (where(arel_table[:qty]
3188
+ # .eq(2))) is built as Active Record builds it, and this relation is
3189
+ # narrowed by what it built (its where clause, order and limit), as a
3190
+ # scope called on a relation is (named_scope): the same scope used to be
3191
+ # refused when called on the model and answered on a relation.
3192
+ #
3193
+ # The body runs with no current scope for the model, as Active
3194
+ # Record's _exec_scope runs it (_scoping(nil)): a body that reads the
3195
+ # model itself (Model.where(...)) reads every record, not the relation
3196
+ # the scope was called on; it used to read that relation.
903
3197
  def _exec_scope(*args, &block)
904
- # Execute the scope block in our context
905
- # The block typically calls methods like `where`, `order`, etc.
906
- result = instance_exec(*args, &block)
3198
+ result =
3199
+ begin
3200
+ self.class.scoped(model_class, nil) { instance_exec(*args, &block) }
3201
+ rescue UnsupportedQuery => e
3202
+ raise unless e.arel_condition
907
3203
 
908
- # Return the result if it's a CachedRelation, otherwise self
909
- result.is_a?(CachedRelation) ? result : self
3204
+ built = built_past_reads("a scope") { model_class.unscoped._exec_scope(*args, &block) }
3205
+ raise unless built.is_a?(ActiveRecord::Relation) && built.model == model_class
3206
+
3207
+ return narrowed_by_scope(built, "a scope")
3208
+ end
3209
+
3210
+ case result
3211
+ when nil, false then self
3212
+ when ActiveRecord::Relation
3213
+ result.model == model_class ? EventStoreReader.relation(model_class).narrowed_by(result, "a scope") : result
3214
+ else result
3215
+ end
910
3216
  end
911
3217
 
912
3218
  # The relation a +scoping+ block is running under, per model class, as
@@ -914,10 +3220,29 @@ module Lyra
914
3220
  # reads (all, where, first, last) answer from that relation instead of
915
3221
  # from every record of the model. Scope bodies reached through an
916
3222
  # association (order.payments.completed) run that way.
3223
+ # Kept as Active Record keeps its current scope
3224
+ # (ActiveSupport::IsolatedExecutionState: per thread, or per fiber under
3225
+ # fiber isolation): a fiber or an enumerator inside a scoping block sees
3226
+ # the scope, as it does in Active Record. It was fiber-local
3227
+ # (Thread.current[]), and they read every record.
917
3228
  SCOPES_KEY = :lyra_cached_relation_scopes
918
3229
 
919
3230
  def self.scoped_for(model_class)
920
- (Thread.current[SCOPES_KEY] || {})[model_class]
3231
+ (ActiveSupport::IsolatedExecutionState[SCOPES_KEY] || {})[model_class]
3232
+ end
3233
+
3234
+ # The block run with +relation+ (nil: none) as +model_class+'s current
3235
+ # scope.
3236
+ def self.scoped(model_class, relation)
3237
+ scopes = (ActiveSupport::IsolatedExecutionState[SCOPES_KEY] ||= {})
3238
+ had_previous = scopes.key?(model_class)
3239
+ previous = scopes[model_class]
3240
+ relation ? scopes[model_class] = relation : scopes.delete(model_class)
3241
+ begin
3242
+ yield
3243
+ ensure
3244
+ had_previous ? scopes[model_class] = previous : scopes.delete(model_class)
3245
+ end
921
3246
  end
922
3247
 
923
3248
  # Runs the block with this relation as the model's current scope and
@@ -927,15 +3252,7 @@ module Lyra
927
3252
  def scoping(skip_inherited_scope = false, full = nil, all_queries: nil)
928
3253
  return self unless block_given?
929
3254
 
930
- scopes = (Thread.current[SCOPES_KEY] ||= {})
931
- had_previous = scopes.key?(model_class)
932
- previous = scopes[model_class]
933
- scopes[model_class] = self
934
- begin
935
- yield self
936
- ensure
937
- had_previous ? scopes[model_class] = previous : scopes.delete(model_class)
938
- end
3255
+ self.class.scoped(model_class, self) { yield self }
939
3256
  end
940
3257
 
941
3258
  def _scoping(skip_inherited_scope = false, full = nil, all_queries: nil, &block)
@@ -961,7 +3278,12 @@ module Lyra
961
3278
  # in the table, silently, regardless of any #where this relation was
962
3279
  # built from. Getting this wrong doesn't raise; it destroys data the
963
3280
  # caller never selected.
964
- BULK_MUTATION_METHODS = %i[delete_all destroy_all update_all].freeze
3281
+ # touch_all is update_all of the timestamp columns (as Rails defines
3282
+ # it). Left out, it went through the scope branch, which touched every
3283
+ # row of the table (an unscoped touch_all, publishing an event for each)
3284
+ # and then raised UnsupportedQuery: records the caller never selected
3285
+ # changed in the log, which ES-NoProj reads as the truth.
3286
+ BULK_MUTATION_METHODS = %i[delete_all destroy_all update_all touch_all].freeze
965
3287
 
966
3288
  # Inserts and upserts aren't scopes either: they run at once and return
967
3289
  # an ActiveRecord::Result. Through the scope branch below, the write ran
@@ -982,45 +3304,153 @@ module Lyra
982
3304
  return model_class.public_send(method_name)
983
3305
  end
984
3306
 
985
- # Try to delegate to model class scopes
986
- if model_class.respond_to?(method_name)
987
- # Execute the scope on a bare unscoped relation to get the where conditions
988
- # This works because scopes add where clauses to the relation
3307
+ # Active Record's own class methods and relation methods that this
3308
+ # class does not evaluate (sole, find_sole_by, invert_where,
3309
+ # find_by_sql, a bang query method, ...): refused before anything
3310
+ # runs. They used to run on a bare relation of the model, past this
3311
+ # relation's conditions: update and update_counters wrote to every
3312
+ # record, first_or_create! created one without the conditions.
3313
+ if active_record_method?(method_name)
3314
+ return model_class.public_send(method_name, *args, **kwargs, &block) if MODEL_READERS.include?(method_name)
3315
+
3316
+ raise UnsupportedQuery.new(model_class, "#{method_name} on a relation")
3317
+ end
3318
+
3319
+ return super unless model_class.respond_to?(method_name)
3320
+
3321
+ # A named scope: its relation is built as ActiveRecord builds it
3322
+ # (past Lyra's reads; building runs no query) and this relation is
3323
+ # narrowed by it.
3324
+ return named_scope(method_name, *args, **kwargs, &block) if named_scope?(method_name)
3325
+
3326
+ # Any other class method of the model runs with this relation as the
3327
+ # model's current scope, as ActiveRecord runs it on a relation: its
3328
+ # reads and writes (where, update_all) act on this relation's
3329
+ # records. That holds only where the model's reads answer from the
3330
+ # scope (a monitored model under ES-NoProj); elsewhere they would
3331
+ # read the table, and the call is refused before it runs.
3332
+ unless events_only? && model_class.respond_to?(:lyra_monitored) && model_class.lyra_monitored &&
3333
+ !Thread.current[:lyra_bypass_read_override]
3334
+ raise UnsupportedQuery.new(model_class, "#{method_name}, which is not a scope")
3335
+ end
3336
+
3337
+ result =
989
3338
  begin
990
- # Temporarily bypass Lyra's read overrides so scope calls go through AR
991
- # Without this, scope lambdas that call `where(...)` would hit our override
992
- # and return CachedRelation instead of building AR conditions
993
- Thread.current[:lyra_bypass_read_override] = true
994
- base_relation = model_class.unscoped
995
- scope_result = base_relation.public_send(method_name, *args, **kwargs, &block)
996
- rescue Lyra::StrictDataAccessViolation
997
- # Don't swallow strict data access violations - these are intentional framework errors
998
- raise
999
- rescue => e
1000
- # Answering anyway (as this used to, unfiltered) would be wrong.
1001
- raise UnsupportedQuery.new(model_class, "#{method_name} (it raised #{e.class})")
1002
- ensure
1003
- Thread.current[:lyra_bypass_read_override] = nil
1004
- end
3339
+ scope_copy.scoping { model_class.public_send(method_name, *args, **kwargs, &block) }
3340
+ rescue NoMethodError => e
3341
+ raise unless e.receiver.is_a?(CachedRelation)
1005
3342
 
1006
- unless scope_result.is_a?(ActiveRecord::Relation)
1007
- raise UnsupportedQuery.new(model_class, "#{method_name}, which is not a scope")
3343
+ raise UnsupportedQuery.new(model_class, "#{method_name} (it called #{e.name}, which is not evaluated here)")
1008
3344
  end
3345
+ result.is_a?(ActiveRecord::Relation) ? narrowed_by(result, method_name.to_s) : result
3346
+ end
3347
+
3348
+ # Readers of the model a scope body may ask its relation for.
3349
+ MODEL_READERS = %i[arel_table table_name primary_key column_names columns_hash attribute_names connection
3350
+ sanitize_sql sanitize_sql_like sanitize_sql_array predicate_builder type_for_attribute
3351
+ name base_class].freeze
1009
3352
 
1010
- extra = unsupported_clauses(scope_result) - %w[order limit]
1011
- raise UnsupportedQuery.new(model_class, "scope #{method_name} (it uses #{extra.join(', ')})") if extra.any?
3353
+ def active_record_method?(method_name)
3354
+ unless model_class.respond_to?(method_name)
3355
+ return ActiveRecord::Relation.method_defined?(method_name) || ActiveRecord::Relation.private_method_defined?(method_name)
3356
+ end
1012
3357
 
1013
- where_hash = extract_where_conditions(scope_result, strict: true)
1014
- raise UnsupportedQuery.new(model_class, "scope #{method_name}") if where_hash.nil?
3358
+ path = model_class.method(method_name).source_location&.first
3359
+ path.nil? || (DefaultScopes.active_record_path?(path) && !path.end_with?("/scoping/named.rb"))
3360
+ end
3361
+ private :active_record_method?
3362
+
3363
+ # The copy of this relation a scope body or class method runs on (as
3364
+ # Active Record's all hands it current_scope.clone). It selects what
3365
+ # this one does, and is built again as this one is (Rebuildable): the
3366
+ # query methods the body calls on it are recorded on what it returns,
3367
+ # with the values the body used, so a scope chain keeps its recipe and
3368
+ # a write never runs the body again.
3369
+ def scope_copy
3370
+ clone.inherit_rebuild(self)
3371
+ end
3372
+ private :scope_copy
3373
+
3374
+ def named_scope?(method_name)
3375
+ model_class.method(method_name).source_location&.first.to_s.end_with?("/active_record/scoping/named.rb")
3376
+ end
3377
+ private :named_scope?
3378
+
3379
+ def named_scope(method_name, *args, **kwargs, &block)
3380
+ # As Active Record runs a scope called on a relation: the model's
3381
+ # scope, with this relation as its current scope, so that its body
3382
+ # runs on this relation (_exec_scope), as it runs when the scope is
3383
+ # called on the model. A scope answers the same on the model and on a
3384
+ # relation, and one whose body returns a value (a count, a record)
3385
+ # returns it; it used to be refused on a relation. The current scope
3386
+ # is a copy of this relation, as Active Record's all hands the body
3387
+ # current_scope.clone: a bang method in the body (limit!) changes the
3388
+ # copy, not the relation the scope was called on.
3389
+ if events_only? && model_class.respond_to?(:lyra_monitored) && model_class.lyra_monitored &&
3390
+ !Thread.current[:lyra_bypass_read_override]
3391
+ begin
3392
+ return scope_copy.scoping { model_class.public_send(method_name, *args, **kwargs, &block) }
3393
+ rescue NoMethodError => e
3394
+ raise unless e.receiver.is_a?(CachedRelation)
1015
3395
 
1016
- # A scope's where, then its order and limit, as SQL applies them.
1017
- relation = where_hash.empty? ? self : where(where_hash)
1018
- relation = relation.order(*scope_result.order_values) if scope_result.order_values.any?
1019
- relation = relation.limit(scope_result.limit_value) if scope_result.limit_value
1020
- relation
1021
- else
1022
- super
3396
+ raise UnsupportedQuery.new(model_class, "scope #{method_name} (it called #{e.name}, which is not evaluated here)")
3397
+ end
3398
+ end
3399
+
3400
+ # Elsewhere (Lyra's own reads), built past Lyra's read overrides:
3401
+ # scope lambdas that call where build ActiveRecord conditions instead
3402
+ # of reading the event store, and this relation is narrowed by them.
3403
+ scope_result = built_past_reads(method_name) { model_class.unscoped.public_send(method_name, *args, **kwargs, &block) }
3404
+ unless scope_result.is_a?(ActiveRecord::Relation) && scope_result.model == model_class
3405
+ raise UnsupportedQuery.new(model_class, "#{method_name}, which is not a scope")
1023
3406
  end
3407
+
3408
+ narrowed_by_scope(scope_result, "scope #{method_name}")
3409
+ end
3410
+ private :named_scope
3411
+
3412
+ # The block's value, built with Lyra's reads bypassed (an Active Record
3413
+ # relation of the model), or UnsupportedQuery if it raised.
3414
+ def built_past_reads(what)
3415
+ Thread.current[:lyra_bypass_read_override] = true
3416
+ yield
3417
+ rescue Lyra::StrictDataAccessViolation
3418
+ raise
3419
+ rescue => e
3420
+ # Answering anyway (as this used to, unfiltered) would be wrong.
3421
+ raise UnsupportedQuery.new(model_class, "#{what} (it raised #{e.class})")
3422
+ ensure
3423
+ Thread.current[:lyra_bypass_read_override] = nil
3424
+ end
3425
+ private :built_past_reads
3426
+
3427
+ # This relation narrowed by +scope+ (an Active Record relation of the
3428
+ # model a scope built). A scope that unscopes (with_discarded) takes the
3429
+ # default scope off first, as it does on ActiveRecord's relation.
3430
+ def narrowed_by_scope(scope, what)
3431
+ base = scope.unscope_values.any? ? unscope(*scope.unscope_values) : self
3432
+ narrowed = base.narrowed_by(scope, what)
3433
+ narrowed = narrowed.spawn if narrowed.equal?(base)
3434
+ narrowed.continue_plan(base, [:scope, scope, what])
3435
+ end
3436
+ private :narrowed_by_scope
3437
+
3438
+ # This relation narrowed as an ActiveRecord relation of the same model
3439
+ # selects: its where clause, then its order and limit, as SQL applies
3440
+ # them. Anything else it holds (joins, an SQL fragment, a where.not)
3441
+ # raises UnsupportedQuery naming +what+, rather than answer with records
3442
+ # the relation does not select.
3443
+ def narrowed_by(relation, what)
3444
+ extra = unsupported_clauses(relation) - %w[order limit]
3445
+ raise UnsupportedQuery.new(model_class, "#{what} (it uses #{extra.join(', ')})") if extra.any?
3446
+
3447
+ conditions = extract_where_conditions(relation)
3448
+ raise UnsupportedQuery.new(model_class, what) if conditions.nil?
3449
+
3450
+ narrowed = conditions.reduce(self) { |narrowing, hash| narrowing.where(hash) }
3451
+ narrowed = narrowed.order(*relation.order_values) if relation.order_values.any?
3452
+ narrowed = narrowed.limit(relation.limit_value) if relation.limit_value
3453
+ narrowed
1024
3454
  end
1025
3455
 
1026
3456
  # Clauses of a scope's relation, other than its where clause, that this
@@ -1039,67 +3469,78 @@ module Lyra
1039
3469
  }.select { |_, used| used }.keys
1040
3470
  end
1041
3471
 
1042
- # Hash conditions only; with strict: true, returns nil as soon as one
1043
- # predicate is something it cannot read, so the caller refuses rather
1044
- # than filter by a subset of the conditions.
1045
- def extract_where_conditions(relation, strict: false)
1046
- # Try to extract hash conditions from the relation's where clause
1047
- return {} unless relation.respond_to?(:where_clause)
3472
+ # The relation's where clause as hash conditions on this model's own
3473
+ # columns, or nil as soon as one predicate is something else, so that the
3474
+ # caller refuses rather than filter by a subset of the conditions: an
3475
+ # equality or IN whose left side is not a column of this model's table (an
3476
+ # SQL literal, a function, another table's column) or whose right side is
3477
+ # not a value (another column, an SQL literal, a subquery), a NOT IN, and
3478
+ # any other kind of predicate. An equality or IN whose left side was not a
3479
+ # column used to be skipped: a scope holding
3480
+ # where(Arel.sql("lower(name)").eq("pear")) answered with every record its
3481
+ # other conditions selected.
3482
+ #
3483
+ # A list of hashes, applied one after another (AND): a column named twice
3484
+ # (where(qty: 1).where(qty: 2)) starts a new hash; the second condition used
3485
+ # to replace the first.
3486
+ def extract_where_conditions(relation)
3487
+ return [] unless relation.respond_to?(:where_clause)
1048
3488
 
1049
3489
  where_clause = relation.where_clause
1050
- return {} if where_clause.empty?
3490
+ return [] if where_clause.empty?
1051
3491
 
1052
- # Rails 7+ stores conditions in predicates
1053
- # Try to convert Arel predicates to a hash
1054
- conditions = {}
3492
+ conditions = [{}]
1055
3493
  where_clause.send(:predicates).each do |predicate|
1056
- case predicate
1057
- when Arel::Nodes::Equality
1058
- # Simple equality: column = value
1059
- if predicate.left.respond_to?(:name)
1060
- column = predicate.left.name.to_sym
1061
- value = extract_predicate_value(predicate.right)
1062
- conditions[column] = value
1063
- end
1064
- when Arel::Nodes::In
1065
- # IN clause: column IN (values)
1066
- if predicate.left.respond_to?(:name)
1067
- column = predicate.left.name.to_sym
1068
- values = predicate.right.map { |v| extract_predicate_value(v) }
1069
- conditions[column] = values
1070
- end
1071
- when Arel::Nodes::HomogeneousIn
1072
- # IN clause over an array of same-type scalars -- Rails' own
1073
- # optimization of `where(column: [v1, v2, ...])`, and what
1074
- # `where(id: [...])` compiles to whenever the array has more than
1075
- # one element. Without this branch, any such condition falls
1076
- # through unrecognized, and this method silently omits it from
1077
- # `conditions` -- collapsing the scope to unfiltered if it was
1078
- # the only condition.
1079
- if predicate.type == :in && predicate.attribute.respond_to?(:name)
1080
- conditions[predicate.attribute.name.to_sym] = predicate.values
1081
- elsif strict
3494
+ column, value =
3495
+ case predicate
3496
+ when Arel::Nodes::Equality
3497
+ [own_column(predicate.left), predicate_value(predicate.right)]
3498
+ when Arel::Nodes::In
3499
+ # column IN (values); a subquery is not a list of values.
3500
+ return nil unless predicate.right.is_a?(Array)
3501
+
3502
+ values = predicate.right.map { |v| predicate_value(v) }
3503
+ [own_column(predicate.left), values.any? { UNREADABLE_VALUE.equal?(_1) } ? UNREADABLE_VALUE : values]
3504
+ when Arel::Nodes::HomogeneousIn
3505
+ # IN over an array of same-type scalars: Rails' own form of
3506
+ # where(column: [v1, v2, ...]). Its NOT IN form (where.not) is refused.
3507
+ return nil unless predicate.type == :in
3508
+
3509
+ [own_column(predicate.attribute), predicate.values]
3510
+ else
1082
3511
  return nil
1083
3512
  end
1084
- else
1085
- return nil if strict
1086
- end
3513
+ return nil if column.nil? || UNREADABLE_VALUE.equal?(value)
3514
+
3515
+ conditions << {} if conditions.last.key?(column)
3516
+ conditions.last[column] = value
1087
3517
  end
3518
+ conditions.reject(&:empty?)
3519
+ rescue StandardError
3520
+ nil
3521
+ end
1088
3522
 
1089
- conditions
1090
- rescue => e
1091
- return nil if strict
3523
+ # +node+'s column name, when it is a column of this model's table.
3524
+ def own_column(node)
3525
+ return nil unless node.is_a?(Arel::Attributes::Attribute)
3526
+ return nil unless node.relation.respond_to?(:name) && node.relation.name.to_s == model_class.table_name.to_s
1092
3527
 
1093
- Rails.logger.debug("Lyra::CachedRelation: Could not extract where conditions - #{e.message}")
1094
- {}
3528
+ node.name.to_sym
1095
3529
  end
1096
3530
 
3531
+ # A predicate's right side that is not a value (predicate_value).
3532
+ UNREADABLE_VALUE = Object.new.freeze
3533
+
1097
3534
  # Conditions must be a hash: of this model's own attributes, or of a
1098
3535
  # table joined before (see CachedJoins). A SQL fragment or an Arel node
1099
3536
  # cannot be evaluated against cached records, and ignoring it would
1100
3537
  # widen the answer.
1101
3538
  def check_hash_conditions!(conditions, what)
1102
- raise UnsupportedQuery.new(model_class, what) unless conditions.is_a?(Hash)
3539
+ return if conditions.is_a?(Hash)
3540
+
3541
+ error = UnsupportedQuery.new(model_class, what)
3542
+ error.arel_condition = conditions.is_a?(Arel::Nodes::Node)
3543
+ raise error
1103
3544
  end
1104
3545
 
1105
3546
  # Keep the rows the block accepts, as a joined relation if this is one.
@@ -1108,19 +3549,232 @@ module Lyra
1108
3549
  @joins ? with_rows(rows, @joins) : spawn_records(rows.map(&:first))
1109
3550
  end
1110
3551
 
1111
- def extract_predicate_value(node)
3552
+ # The value on a predicate's right side, or UNREADABLE_VALUE when it is not
3553
+ # one: another column, an SQL literal, a function, a subquery. Such a right
3554
+ # side used to be taken as the value itself, matching nothing.
3555
+ def predicate_value(node)
1112
3556
  case node
1113
- when Arel::Nodes::Casted
1114
- node.value
3557
+ when Arel::Nodes::Casted, Arel::Nodes::Quoted then node.value
1115
3558
  when Arel::Nodes::BindParam
1116
3559
  # Rails 7+ bind params
1117
- node.value.value_before_type_cast
1118
- when NilClass
1119
- nil
3560
+ node.value.respond_to?(:value_before_type_cast) ? node.value.value_before_type_cast : UNREADABLE_VALUE
3561
+ when NilClass then nil
3562
+ when Arel::Nodes::SqlLiteral, Arel::Nodes::Node, Arel::Attributes::Attribute then UNREADABLE_VALUE
3563
+ else node.respond_to?(:value) ? node.value : node
3564
+ end
3565
+ end
3566
+
3567
+ # This relation's state taken from +other+ (the bang methods).
3568
+ def replace_with!(other)
3569
+ %i[@records @ordering @query_terms @create_attributes @create_with_attributes @plan @default_scope_applied
3570
+ @selected_columns @group_columns @windowed @rows @joins @joined_classes @window_source @limit_value
3571
+ @offset_value @pagination @negated_columns @or_terms @reordering @distinct @query_relation].each do |ivar|
3572
+ instance_variable_set(ivar, other.instance_variable_get(ivar))
3573
+ end
3574
+ # It is built again as +other+ is (Rebuildable).
3575
+ self.rebuild_recipe = other.rebuild_recipe
3576
+ self
3577
+ end
3578
+ protected :replace_with!
3579
+
3580
+ def merge_events_relation(other)
3581
+ unless other.model_class == model_class
3582
+ raise UnsupportedQuery.new(model_class, "merge of a #{other.model_class.name} relation")
3583
+ end
3584
+ if other.window_source
3585
+ return merge_events_relation(other.window_source).windowed_by(other.limit_value, other.offset_value)
3586
+ end
3587
+ raise UnsupportedQuery.new(model_class, "merge of a relation with a limit or offset") if other.windowed?
3588
+
3589
+ if (distinct? || other.distinct?) && (selected_columns || other.selected_columns)
3590
+ raise UnsupportedQuery.new(model_class, "merge of a relation with distinct selected columns (SELECT DISTINCT)")
3591
+ end
3592
+
3593
+ base = replacing(other.condition_columns, other.negated_columns)
3594
+ merged = base.send(:intersect_rows, other)
3595
+ merged.add_query_terms(other.query_terms)
3596
+ merged.create_attributes = base.create_attributes.merge(other.create_attributes)
3597
+ merged.create_with_attributes = base.create_with_attributes.merge(other.create_with_attributes)
3598
+ # The other's order follows this one's, or replaces it if the other
3599
+ # reorders; its group and select columns are added, and its distinct
3600
+ # taken, as Active Record's merge takes them (they were dropped).
3601
+ if other.reordering?
3602
+ merged = merged.send(:sorted_rows, other.ordering)
3603
+ merged.instance_variable_set(:@reordering, true)
3604
+ elsif other.ordering.any?
3605
+ merged = merged.send(:sorted_rows, base.ordering + other.ordering)
3606
+ end
3607
+ merged.group_columns = (base.group_columns + other.group_columns).uniq
3608
+ merged.selected_columns = ((base.selected_columns || []) + other.selected_columns).uniq if other.selected_columns
3609
+ merged = merged.distinct if other.distinct?
3610
+ # Its plan: this one's, then the merge (unscope, rewhere and a later
3611
+ # merge replay it), while both have one.
3612
+ if base.plan && other.plan && base.joined_associations.empty? && other.joined_associations.empty?
3613
+ merged.plan = [base.plan[0], base.plan[1] + [[:merge, other]]]
3614
+ end
3615
+ merged
3616
+ end
3617
+ private :merge_events_relation
3618
+
3619
+ # The rows of this relation whose records +other+ selects: this
3620
+ # relation's rows (with their partners, if it is joined), or, if only
3621
+ # +other+ is joined, +other+'s rows of this relation's records, in this
3622
+ # relation's order. Both joined to the same associations: the rows both
3623
+ # have. Joined to different ones, the rows would be those of both
3624
+ # joins: refused.
3625
+ def intersect_rows(other)
3626
+ pk = model_class.primary_key
3627
+ key = ->(record) { record&.read_attribute(record.class.primary_key) }
3628
+ mine = joined_associations
3629
+ theirs = other.joined_associations
3630
+ if theirs.empty?
3631
+ keys = other.raw_records.to_a.to_h { |record| [record.read_attribute(pk), true] }
3632
+ filter_rows { |record, _joins| keys.key?(record.read_attribute(pk)) }
3633
+ elsif mine.empty?
3634
+ by_record = other.joined_rows.group_by { |record, _joins| record.read_attribute(pk) }
3635
+ rows = @records.flat_map do |record|
3636
+ by_record.fetch(record.read_attribute(pk), []).map { |_partner_record, joins| [record, joins] }
3637
+ end
3638
+ with_rows(rows, theirs)
3639
+ elsif mine.transform_values(&:kind) == theirs.transform_values(&:kind)
3640
+ keys = other.joined_rows.to_h { |record, joins| [[record.read_attribute(pk), joins.transform_values(&key)], true] }
3641
+ filter_rows { |record, joins| keys.key?([record.read_attribute(pk), joins.transform_values(&key)]) }
1120
3642
  else
1121
- node.respond_to?(:value) ? node.value : node
3643
+ raise UnsupportedQuery.new(model_class, "merge of relations joined to different associations " \
3644
+ "(#{mine.keys.join(', ')}; #{theirs.keys.join(', ')})")
3645
+ end
3646
+ end
3647
+ private :intersect_rows
3648
+
3649
+ # sorted_by, keeping a joined relation's rows (each record's rows in
3650
+ # the order they had).
3651
+ def sorted_rows(keys)
3652
+ return sorted_by(keys) unless @joins
3653
+
3654
+ position = sort_records(@rows.map(&:first).uniq, keys).each_with_index.to_h.compare_by_identity
3655
+ rows = @rows.each_with_index.sort_by { |(record, _joins), index| [position[record], index] }.map(&:first)
3656
+ relation = with_rows(rows, @joins)
3657
+ relation.ordering = keys
3658
+ relation
3659
+ end
3660
+ private :sorted_rows
3661
+
3662
+ def merge_active_record_relation(other)
3663
+ unless other.model == model_class
3664
+ raise UnsupportedQuery.new(model_class, "merge of a #{other.model.name} relation")
3665
+ end
3666
+
3667
+ base = other.unscope_values.any? ? unscope(*other.unscope_values) : self
3668
+ if (others = extract_where_conditions(other))
3669
+ columns = Hash.new { |hash, column| hash[column] = [] }
3670
+ others.each { |conditions| conditions.each { |column, value| columns[column.to_s] << value } }
3671
+ base = base.replacing(columns, Set.new)
3672
+ end
3673
+ merged = base.narrowed_by(other, "merge(#{other.class.name})")
3674
+ merged = merged.reorder(*other.order_values) if other.reordering_value
3675
+ merged = merged.spawn if merged.equal?(self)
3676
+ merged.continue_plan(base, [:scope, other, "merge(#{other.class.name})"])
3677
+ merged.create_with_attributes = create_with_attributes.merge(other.create_with_value.stringify_keys)
3678
+ merged
3679
+ end
3680
+ private :merge_active_record_relation
3681
+
3682
+ # The columns of this model this relation's conditions name (a where,
3683
+ # where.not, a scope's, a default scope's, an association's key), each
3684
+ # with the values they name: Active Record's merge replaces them where
3685
+ # the other relation names them too. A tuple condition (where([:a, :b]
3686
+ # => ...)) names each of its columns with TUPLE; a condition on a
3687
+ # joined table is not this model's.
3688
+ #
3689
+ # An OR's alternatives (or_terms) over one column are one condition on
3690
+ # it that merge would replace whole (TUPLE, refused); over several
3691
+ # columns, Active Record's merge replaces none of them, and they are
3692
+ # left out.
3693
+ def condition_columns
3694
+ alternatives = []
3695
+ columns = query_terms.each_with_object(Hash.new { |hash, column| hash[column] = [] }) do |pair, found|
3696
+ kind, term = pair
3697
+ next unless kind == :conditions && term.is_a?(Hash)
3698
+
3699
+ term.each do |key, value|
3700
+ names = key.is_a?(Array) ? key.map { column_of(_1) } : condition_column_names(key, value)
3701
+ next alternatives.concat(names) if or_terms.key?(pair)
3702
+
3703
+ names.each { |column| found[column] << (key.is_a?(Array) ? TUPLE : value) }
3704
+ end
3705
+ end
3706
+ columns[alternatives.first] << TUPLE if alternatives.uniq.size == 1
3707
+ columns
3708
+ end
3709
+ protected :condition_columns
3710
+
3711
+ # What condition_columns gives a tuple condition's columns: a condition
3712
+ # merge cannot take off one column at a time.
3713
+ TUPLE = Object.new.freeze
3714
+
3715
+ # The columns of this model a where.not of this relation names.
3716
+ def negated_columns = @negated_columns || Set.new
3717
+ protected :negated_columns
3718
+
3719
+ # A condition on a joined table (where(users: { name: ... }),
3720
+ # where("users.name" => ...)) names "table.column".
3721
+ def condition_column_names(key, value)
3722
+ return key.map { column_of(_1) } if key.is_a?(Array)
3723
+ if value.is_a?(Hash) && !model_class.column_names.include?(key.to_s)
3724
+ joined = model_class.respond_to?(:reflect_on_association) && model_class.reflect_on_association(key)
3725
+ table = joined && !joined.polymorphic? ? joined.klass.table_name : key.to_s
3726
+ return value.keys.map { "#{table}.#{column_of(_1)}" }
3727
+ end
3728
+
3729
+ table, column = key.to_s.include?(".") ? key.to_s.split(".", 2).map { _1.delete('"') } : [nil, key.to_s]
3730
+ return ["#{table}.#{column}"] if table && table != model_class.table_name.to_s
3731
+
3732
+ reflection = model_class.respond_to?(:reflect_on_association) && !model_class.column_names.include?(column) &&
3733
+ model_class.reflect_on_association(column)
3734
+ return [column] unless reflection && reflection.macro == :belongs_to
3735
+
3736
+ [reflection.foreign_key.to_s, (reflection.foreign_type.to_s if reflection.polymorphic?)].compact
3737
+ end
3738
+ private :condition_column_names
3739
+
3740
+ # This relation without its conditions on the columns +columns+ (as
3741
+ # condition_columns gives them) names with other values, or that either
3742
+ # side negates: the other's conditions replace them (merge). Where the
3743
+ # two name the same values, the conditions are the same, and stay.
3744
+ def replacing(columns, negated)
3745
+ mine = condition_columns
3746
+ tuple = ->(column) { mine[column].any? { TUPLE.equal?(_1) } || columns[column].any? { TUPLE.equal?(_1) } }
3747
+ replaced = (mine.keys & columns.keys).reject do |column|
3748
+ !tuple.call(column) && !negated_columns.include?(column) && !negated.include?(column) &&
3749
+ mine[column].uniq.to_set == columns[column].uniq.to_set
1122
3750
  end
3751
+ return self if replaced.empty?
3752
+
3753
+ if (tupled = replaced.select(&tuple)).any?
3754
+ error = UnsupportedQuery.new(model_class, "merge of a relation with a condition on several columns at once, " \
3755
+ "or an OR of conditions on one (#{tupled.join(', ')}), which Active " \
3756
+ "Record's merge would replace")
3757
+ error.instance_variable_set(:@lyra_merge_refusal, true)
3758
+ raise error
3759
+ end
3760
+ # A joined table's condition is part of the join's rows, which no
3761
+ # plan replays: refused (it used to be kept with the other's, both).
3762
+ if (joined = replaced.select { _1.include?(".") }).any?
3763
+ error = UnsupportedQuery.new(model_class, "merge of a relation whose conditions on a joined table " \
3764
+ "(#{joined.join(', ')}) replace this relation's, as Active Record's " \
3765
+ "merge replaces them")
3766
+ error.instance_variable_set(:@lyra_merge_refusal, true)
3767
+ raise error
3768
+ end
3769
+ unscope(where: replaced)
3770
+ rescue UnsupportedQuery => e
3771
+ raise if replaced.nil? || e.instance_variable_get(:@lyra_merge_refusal)
3772
+
3773
+ raise UnsupportedQuery.new(model_class, "merge of a relation whose conditions on #{replaced.join(', ')} replace " \
3774
+ "this relation's, as Active Record's merge replaces them, where they " \
3775
+ "cannot be taken off (#{e.message[/evaluate (.*?) on /, 1]})")
1123
3776
  end
3777
+ protected :replacing
1124
3778
 
1125
3779
  private
1126
3780
 
@@ -1149,7 +3803,18 @@ module Lyra
1149
3803
  # WHERE at all" for delete_all/update_all -- the same class of bug this
1150
3804
  # method exists to avoid.
1151
3805
  def bulk_mutate(method_name, *args, **kwargs)
1152
- return 0 if @records.empty?
3806
+ # In the events-only store a relation that can be built again is
3807
+ # (records_now): it may select records now that it did not hold.
3808
+ if @records.empty? && !(events_only? && rebuild_recipe)
3809
+ return method_name == :destroy_all ? [] : 0
3810
+ end
3811
+ if method_name == :touch_all && events_only?
3812
+ # One time for every row, as Rails's touch_all gives: update_all of
3813
+ # the touched columns. With a table, touch_all itself runs below, on
3814
+ # the selected rows only, and publishes through the relation.
3815
+ updates = model_class.touch_attributes_with_time(*args, time: kwargs[:time])
3816
+ return bulk_mutate_as(:update_all, "touch_all", updates)
3817
+ end
1153
3818
  return event_sourced_bulk_mutate(method_name, *args, **kwargs) if events_only?
1154
3819
 
1155
3820
  ids = @records.map { |r| r.public_send(model_class.primary_key) }
@@ -1179,24 +3844,235 @@ module Lyra
1179
3844
  Lyra.event_sourcing_mode? && Lyra.config.projection_mode == :disabled
1180
3845
  end
1181
3846
 
3847
+ # A bulk write writes the records the relation selects when it is
3848
+ # called, as Active Record's runs its query then (records_now,
3849
+ # locked_records_now): a record destroyed since the relation was built
3850
+ # (by another relation's delete_all, a destroy, an erasure) is not
3851
+ # written, one that matches since is, and a window is taken again.
3852
+ # Writing the records held appended an update after a destroy and
3853
+ # brought the record back, an erased one included.
1182
3854
  def event_sourced_bulk_mutate(method_name, *args, **kwargs)
1183
- return @records.each(&:destroy) if method_name == :destroy_all
3855
+ if method_name == :destroy_all
3856
+ destroyed = destroy_records_now
3857
+ forget_destroyed(destroyed)
3858
+ return destroyed
3859
+ end
1184
3860
 
1185
3861
  # One transaction: a failed publish part-way leaves none of them.
1186
3862
  model_class.transaction { publish_bulk_events(method_name, *args, **kwargs) }
1187
3863
  end
1188
3864
 
3865
+ # The records the relation selects now, each destroyed in a transaction
3866
+ # of its own, as Active Record's destroy_all loads the records and
3867
+ # destroys each; one destroyed in between (its stream read under its
3868
+ # lock first) is left out of the writes and of the records returned.
3869
+ # The record's destroy then checks again, as every destroy does: its
3870
+ # before_destroy callbacks run in between, and may have destroyed it. A
3871
+ # destroy that returns false rolls back what its callbacks wrote, as on
3872
+ # its own, and its record is returned, as Active Record returns it.
3873
+ # Inside a transaction already open the locks add up until it ends, so
3874
+ # the limit applies to all of them there. The records are delivered as
3875
+ # a load delivers them (the purpose check, rights included): Active
3876
+ # Record loads them, and a restricted one raises
3877
+ # RestrictedProcessingError before any is destroyed; it was destroyed.
3878
+ def destroy_records_now
3879
+ targets = deliver { loaded_records? ? @records.to_a : records_now(skip_destroyed: false) }
3880
+ checks = Lyra.config.lock_aggregate_before_append
3881
+ if checks && model_class.with_connection(&:transaction_open?)
3882
+ Lyra::BypassEvents.check_lock_limit!(model_class, targets.size, "destroy_all")
3883
+ end
3884
+ targets.select do |record|
3885
+ live = false
3886
+ model_class.transaction do
3887
+ next if checks && Lyra::BypassEvents.destroyed_ids(model_class, [record_id(record)], write: "destroy_all").any?
3888
+
3889
+ live = true
3890
+ raise ActiveRecord::Rollback unless record.destroy
3891
+ end
3892
+ live
3893
+ end
3894
+ end
3895
+
3896
+ # As Active Record's destroy_all, which resets its relation: the
3897
+ # relation is read again from the event store (Rebuildable) and is no
3898
+ # longer loaded, so it no longer holds the records destroyed (it did,
3899
+ # loaded? still true). One that cannot be built again keeps those of
3900
+ # its records that were not destroyed.
3901
+ def forget_destroyed(destroyed)
3902
+ recipe = rebuild_recipe
3903
+ fresh = recipe && evaluate_relation(recipe)
3904
+ if fresh
3905
+ replace_with!(fresh)
3906
+ elsif !@joins
3907
+ gone = destroyed.select(&:destroyed?).to_set { record_id(_1) }
3908
+ @records = @records.reject { gone.include?(record_id(_1)) }
3909
+ end
3910
+ @loaded_records = nil
3911
+ end
3912
+
3913
+ # The records this relation selects now: built again from the event
3914
+ # store (Rebuildable) when it can be, as Lyra's own read; else the
3915
+ # records it holds, less those whose streams now end in a destroy
3916
+ # (+skip_destroyed+, read without a lock: the write each is given
3917
+ # takes its own). Not in the events-only store, the records held.
3918
+ def records_now(skip_destroyed: true)
3919
+ return @records.to_a unless events_only?
3920
+
3921
+ recipe = rebuild_recipe
3922
+ rebuilt = recipe && evaluate(recipe)
3923
+ return rebuilt if rebuilt
3924
+
3925
+ records = @records.to_a
3926
+ return records unless skip_destroyed && Lyra.config.lock_aggregate_before_append
3927
+
3928
+ dead = Lyra::StreamTail.destroyed_streams(records.map { record_stream(_1) }).to_set
3929
+ records.reject { |record| dead.include?(record_stream(record)) }
3930
+ end
3931
+
3932
+ # The records a bulk write writes: those this relation selects now,
3933
+ # each one's aggregate lock taken (and held until the transaction
3934
+ # ends) before it is read, so none can be destroyed, or changed, before
3935
+ # the write. Built again (Rebuildable): the locks of the records it
3936
+ # holds are taken (all or none, Lyra::AdvisoryLock.xact_lock_all), and
3937
+ # the selection read; if it holds a record not locked yet (one that
3938
+ # matches since), that one is locked and the selection read again, up
3939
+ # to REEVALUATIONS times, after which the records not locked are left
3940
+ # out, as if they had matched after the write. So the usual write reads
3941
+ # its selection once. A record destroyed by then is not in the
3942
+ # selection. A relation that cannot be built again writes the records
3943
+ # it holds, those whose streams do not end in a destroy, read under the
3944
+ # locks (Lyra::BypassEvents.destroyed_ids); +current+: with their values
3945
+ # as the event store has them then.
3946
+ #
3947
+ # More records than config.bulk_write_lock_limit are refused
3948
+ # (Lyra::BulkWriteLockLimitError) before any lock is taken or event
3949
+ # written. With lock_aggregate_before_append off, as for a single
3950
+ # record's write, nothing is locked or checked.
3951
+ def locked_records_now(current: false)
3952
+ return @records.to_a if @live_checked
3953
+
3954
+ recipe = rebuild_recipe
3955
+ locking = Lyra.config.lock_aggregate_before_append
3956
+ if recipe && !locking
3957
+ records = evaluate(recipe)
3958
+ return records if records
3959
+ elsif recipe
3960
+ locked = Set.new
3961
+ lock = lambda do |ids|
3962
+ Lyra::BypassEvents.check_lock_limit!(model_class, locked.size + ids.size)
3963
+ Lyra::BypassEvents.lock_streams!(model_class, ids)
3964
+ locked.merge(ids)
3965
+ end
3966
+ # The records held, unless an association's were never read.
3967
+ unread = defined?(DeferredRelation) && @records.is_a?(DeferredRelation::Records) && !@records.loaded?
3968
+ held = unread ? [] : @records.map { record_id(_1) }.uniq
3969
+ lock.call(held) if held.any?
3970
+ records = nil
3971
+ REEVALUATIONS.times do
3972
+ records = evaluate(recipe)
3973
+ break unless records
3974
+
3975
+ missing = records.map { record_id(_1) }.uniq.reject { locked.include?(_1) }
3976
+ return records if missing.empty?
3977
+
3978
+ lock.call(missing)
3979
+ end
3980
+ # Still a record not locked: read once more, under every lock
3981
+ # taken, and leave out those not locked.
3982
+ records &&= evaluate(recipe)
3983
+ return records.select { locked.include?(record_id(_1)) } if records
3984
+ end
3985
+
3986
+ records = @records.to_a
3987
+ if locking
3988
+ dead = Lyra::BypassEvents.destroyed_ids(model_class, records.map { record_id(_1) }).to_set
3989
+ records = records.reject { dead.include?(record_id(_1)) } if dead.any?
3990
+ end
3991
+ current ? current_records(records) : records
3992
+ end
3993
+
3994
+ # How many times a write reads its selection again for records that
3995
+ # match since it was read (locked_records_now).
3996
+ REEVALUATIONS = 4
3997
+
3998
+ # The records +recipe+ selects now, each once, read past the query
3999
+ # cache, as Lyra's own read; nil when it cannot be built again.
4000
+ def evaluate(recipe)
4001
+ relation =
4002
+ begin
4003
+ Lyra::PurposeBoundReads.internal(INTERNAL) { model_class.uncached { Rebuildable.rebuild(recipe) } }
4004
+ rescue UnsupportedQuery
4005
+ nil
4006
+ end
4007
+ relation && relation.raw_records.to_a.uniq { record_id(_1) }
4008
+ end
4009
+
4010
+ # +records+ as their streams have them now, in the same order (read
4011
+ # under their locks, none of them destroyed).
4012
+ def current_records(records)
4013
+ return records if records.empty?
4014
+
4015
+ EventStoreReader.find_many(model_class, records.map { record_id(_1) })
4016
+ end
4017
+
4018
+ def record_id(record) = record.public_send(model_class.primary_key)
4019
+
4020
+ def record_stream(record)
4021
+ Lyra.stream_name(model_class, record_id(record))
4022
+ end
4023
+
4024
+ # +method_name+ applied with +updates+, its events naming +source+
4025
+ # (touch_all, which is update_all of the timestamps).
4026
+ def bulk_mutate_as(method_name, source, updates)
4027
+ @bulk_source = source
4028
+ bulk_mutate(method_name, updates)
4029
+ ensure
4030
+ @bulk_source = nil
4031
+ end
4032
+
4033
+ # Rails' own write on an event-backed association scope, this relation
4034
+ # marked by its association code (Lyra::Associations::Maintenance): a
4035
+ # has_many :through's dependent: :nullify of its through records, the
4036
+ # dependent_association bypass, as on a table relation
4037
+ # (StrictDataAccessRelation#update_all), or the delete of its through
4038
+ # records when records are removed from it. Allowed in strict mode;
4039
+ # the nullify's events are named so. The same write made by the
4040
+ # application is an update_all or delete_all like any other.
1189
4041
  def publish_bulk_events(method_name, *args, **kwargs)
4042
+ kind = case method_name
4043
+ when :update_all then :nullify
4044
+ when :delete_all then :unlink
4045
+ end
4046
+ unless kind && Lyra::Associations::Maintenance.claim(kind, self)
4047
+ return publish_bulk_events_now(method_name, *args, **kwargs)
4048
+ end
4049
+
4050
+ previous = @bulk_source
4051
+ # As AssociationInterceptor writes a has_many's nullify.
4052
+ @bulk_source ||= "dependent_association" if kind == :nullify
4053
+ begin
4054
+ Lyra.without_strict_access { publish_bulk_events_now(method_name, *args, **kwargs) }
4055
+ ensure
4056
+ @bulk_source = previous
4057
+ end
4058
+ end
4059
+
4060
+ def publish_bulk_events_now(method_name, *args, **kwargs)
1190
4061
  if Lyra.config.strict_data_access && !Thread.current[:lyra_bypass_strict_access]
1191
4062
  raise Lyra::StrictDataAccessViolation.new(method_name, model_class)
1192
4063
  end
1193
4064
 
1194
4065
  pk = model_class.primary_key
4066
+ # The table's unique indexes, as its UPDATE checks them
4067
+ # (UniqueIndexes.bulk_update): the records' tuples locked after the
4068
+ # records and checked before any event.
4069
+ unique = UniqueIndexes.bulk_update(model_class, args.first || kwargs) if method_name == :update_all
4070
+ records = locked_records_now
1195
4071
  case method_name
1196
4072
  when :delete_all
1197
- @records.each do |record|
4073
+ records.each do |record|
1198
4074
  Lyra::BypassEvents.publish(model_class, record.public_send(pk), :destroyed,
1199
- attributes: record.attributes.except(*Lyra::BypassEvents::TIMESTAMP_COLUMNS),
4075
+ attributes: record.attributes,
1200
4076
  source: "delete_all")
1201
4077
  end
1202
4078
  when :update_all
@@ -1206,8 +4082,9 @@ module Lyra
1206
4082
  "with projections disabled; a SQL fragment can't be applied to event-stream records"
1207
4083
  end
1208
4084
 
4085
+ unique&.check!(records)
1209
4086
  updates = updates.transform_keys(&:to_s)
1210
- @records.each do |record|
4087
+ records.each do |record|
1211
4088
  changes = updates.each_with_object({}) do |(column, new_value), acc|
1212
4089
  old_value = record.read_attribute(column)
1213
4090
  acc[column] = [old_value, new_value] unless old_value == new_value
@@ -1216,100 +4093,77 @@ module Lyra
1216
4093
 
1217
4094
  Lyra::BypassEvents.publish(model_class, record.public_send(pk), :updated,
1218
4095
  attributes: changes.transform_values(&:last), changes: changes,
1219
- source: "update_all")
4096
+ source: @bulk_source || "update_all")
1220
4097
  end
1221
4098
  end
1222
4099
 
1223
4100
  # Rows left in the table from an earlier mode must not disagree with
1224
4101
  # the stream, so apply the same write to them without re-publishing.
1225
- ids = @records.map { |r| r.public_send(pk) }
1226
- Lyra.projection_write do
4102
+ return 0 if records.empty?
4103
+
4104
+ ids = records.map { |r| r.public_send(pk) }
4105
+ Lyra.projection_write(INTERNAL) do
1227
4106
  Thread.current[:lyra_bypass_read_override] = true
1228
4107
  model_class.unscoped.where(pk => ids).public_send(method_name, *args, **kwargs)
1229
4108
  ensure
1230
4109
  Thread.current[:lyra_bypass_read_override] = nil
1231
4110
  end
1232
- @records.size
1233
- end
1234
-
1235
- def matches_value?(record, key, value)
1236
- record_value = record.send(key)
1237
- # A record, or a relation of records, stands for its primary key, as in
1238
- # ActiveRecord: where(variant_id: [variant]) matched nothing, and
1239
- # Solidus's stock check found no stock under ES-NoProj.
1240
- value = value.to_a if value.is_a?(CachedRelation) || value.is_a?(ActiveRecord::Relation)
1241
- value = value.id if value.is_a?(ActiveRecord::Base)
1242
- value = value.map { |v| v.is_a?(ActiveRecord::Base) ? v.id : v } if value.is_a?(Array)
1243
-
1244
- case value
1245
- when Array
1246
- # Handle type coercion for arrays (e.g., array of string IDs vs integer column)
1247
- value.any? { |v| values_match?(record_value, v) }
1248
- when Range
1249
- value.cover?(record_value)
1250
- when nil
1251
- record_value.nil?
1252
- when Regexp
1253
- record_value.to_s.match?(value)
1254
- else
1255
- values_match?(record_value, value)
1256
- end
1257
- rescue NoMethodError
1258
- # ActiveRecord would reject a condition on an unknown column; treating
1259
- # it as "no match" (as this used to) silently empties the answer.
1260
- raise UnsupportedQuery.new(model_class, "a condition on unknown attribute #{key}")
1261
- end
1262
-
1263
- # Compare values with type coercion for common AR patterns
1264
- def values_match?(record_value, query_value)
1265
- return true if record_value == query_value
1266
-
1267
- # Handle string/integer coercion (common with params)
1268
- if record_value.is_a?(Integer) && query_value.is_a?(String)
1269
- record_value == query_value.to_i
1270
- elsif record_value.is_a?(String) && query_value.is_a?(Integer)
1271
- record_value.to_i == query_value
1272
- # Handle boolean string coercion
1273
- elsif record_value.in?([true, false]) && query_value.is_a?(String)
1274
- record_value == ActiveModel::Type::Boolean.new.cast(query_value)
1275
- else
1276
- false
1277
- end
1278
- end
1279
-
1280
- def compare_for_order(a, b, order_args)
1281
- order_args.each do |arg|
1282
- result = compare_single_order(a, b, arg)
1283
- return result unless result == 0
4111
+ records.size
4112
+ end
4113
+
4114
+ # The conditions of a hash on +klass+'s records, compiled once: each
4115
+ # value bound as Active Record binds it (Lyra::Projections::TypedValues).
4116
+ def condition_tests(klass, conditions)
4117
+ conditions.map { |key, value| TypedValues.condition(klass, key, value) }
4118
+ end
4119
+
4120
+ # Whether +record+ satisfies every one of +tests+ (condition_tests), as
4121
+ # SQL decides it on the table: true, false or nil (unknown, a NULL
4122
+ # compared).
4123
+ def matches_all?(record, tests)
4124
+ TypedValues.all_hold(tests, TypedValues::RecordReader, record)
4125
+ rescue ActiveModel::MissingAttributeError => e
4126
+ # A record narrowed by select lacks the column a condition reads.
4127
+ raise UnsupportedQuery.new(model_class, "a condition on a column not selected (#{e.message[0, 80]})")
4128
+ end
4129
+
4130
+ # +records+ in the order +keys+ ({ column => :asc/:desc }, or a
4131
+ # column) gives, ties in the order they had. Each record's sort values
4132
+ # are read once, not on every comparison.
4133
+ def sort_records(records, keys)
4134
+ plan = keys.flat_map do |key|
4135
+ pairs = key.is_a?(Hash) ? key.to_a : [[key, :asc]]
4136
+ pairs.map do |column, direction|
4137
+ reader = column.is_a?(PositionOrder) ? column.sort_reader : TypedValues.sort_reader(model_class, column)
4138
+ [reader, direction.to_s.downcase == "desc"]
4139
+ end
1284
4140
  end
1285
- 0
1286
- end
1287
-
1288
- def compare_single_order(a, b, arg)
1289
- case arg
1290
- when Symbol
1291
- compare_values(a.send(arg), b.send(arg))
1292
- when Hash
1293
- arg.each do |column, direction|
1294
- val_a = a.send(column)
1295
- val_b = b.send(column)
1296
- result = compare_values(val_a, val_b)
1297
- result = -result if direction.to_s.downcase == "desc"
1298
- return result unless result == 0
4141
+ return records.dup if plan.empty?
4142
+
4143
+ readers = plan.map(&:first)
4144
+ descending = plan.map(&:last)
4145
+ decorated = records.each_with_index.map { |record, index| [readers.map { _1.call(record) }, index, record] }
4146
+ decorated.sort! do |(a, index_a, _), (b, index_b, _)|
4147
+ result = index_a <=> index_b
4148
+ a.each_index do |i|
4149
+ compared = compare_values(a[i], b[i])
4150
+ next if compared.zero?
4151
+
4152
+ result = descending[i] ? -compared : compared
4153
+ break
1299
4154
  end
1300
- 0
1301
- else
1302
- 0
4155
+ result
1303
4156
  end
1304
- rescue NoMethodError
1305
- 0
4157
+ decorated.map(&:last)
1306
4158
  end
1307
4159
 
4160
+ # NULLs last ascending (first descending), as PostgreSQL orders them;
4161
+ # booleans false before true.
1308
4162
  def compare_values(a, b)
1309
4163
  return 0 if a.nil? && b.nil?
1310
4164
  return 1 if a.nil?
1311
4165
  return -1 if b.nil?
1312
- a <=> b || 0
4166
+ TypedValues.compare(a, b) || 0
1313
4167
  end
1314
4168
 
1315
4169
  # =========================================================================