orfeas_lyra 0.9.3 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +1923 -17
  3. data/README.md +15 -8
  4. data/Rakefile +40 -1
  5. data/app/controllers/lyra/application_controller.rb +108 -0
  6. data/app/controllers/lyra/dashboard_controller.rb +17 -15
  7. data/app/controllers/lyra/flow_controller.rb +30 -33
  8. data/app/controllers/lyra/privacy_controller.rb +57 -18
  9. data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
  10. data/app/views/lyra/dashboard/index.html.erb +1 -0
  11. data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
  12. data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
  13. data/app/views/lyra/privacy/policy.html.erb +7 -0
  14. data/app/views/lyra/privacy/rights.html.erb +78 -0
  15. data/config/privacy_policies.rb +0 -3
  16. data/config/routes.rb +1 -0
  17. data/lib/lyra/access_log.rb +187 -6
  18. data/lib/lyra/advisory_lock.rb +236 -0
  19. data/lib/lyra/aggregate.rb +1 -1
  20. data/lib/lyra/associations/habtm_links.rb +382 -19
  21. data/lib/lyra/associations/maintenance.rb +229 -0
  22. data/lib/lyra/attribution.rb +48 -0
  23. data/lib/lyra/bypass_events.rb +426 -45
  24. data/lib/lyra/command_handler.rb +19 -14
  25. data/lib/lyra/configuration.rb +237 -10
  26. data/lib/lyra/consistency/read_your_writes.rb +4 -2
  27. data/lib/lyra/correlation.rb +121 -0
  28. data/lib/lyra/domain_events.rb +4 -0
  29. data/lib/lyra/dual_view.rb +53 -16
  30. data/lib/lyra/encrypted_attributes.rb +5 -0
  31. data/lib/lyra/engine.rb +67 -0
  32. data/lib/lyra/erasure.rb +1002 -116
  33. data/lib/lyra/event.rb +35 -1
  34. data/lib/lyra/event_data.rb +38 -0
  35. data/lib/lyra/event_flow.rb +31 -15
  36. data/lib/lyra/event_store_adapter.rb +28 -0
  37. data/lib/lyra/event_window.rb +58 -0
  38. data/lib/lyra/genesis.rb +548 -44
  39. data/lib/lyra/interceptors/association_interceptor.rb +689 -34
  40. data/lib/lyra/interceptors/crud_interceptor.rb +547 -95
  41. data/lib/lyra/internal.rb +26 -0
  42. data/lib/lyra/mode_sync.rb +51 -3
  43. data/lib/lyra/mode_transition.rb +245 -34
  44. data/lib/lyra/model_selection.rb +35 -0
  45. data/lib/lyra/own_connection.rb +17 -2
  46. data/lib/lyra/privacy/adapters/pam.rb +25 -2
  47. data/lib/lyra/privacy/gdpr_compliance.rb +264 -36
  48. data/lib/lyra/privacy/interface.rb +27 -7
  49. data/lib/lyra/privacy/subject_links.rb +143 -0
  50. data/lib/lyra/privacy/subject_records.rb +369 -0
  51. data/lib/lyra/projection.rb +7 -1
  52. data/lib/lyra/projections/async_checkpoint.rb +32 -0
  53. data/lib/lyra/projections/async_projection_job.rb +166 -23
  54. data/lib/lyra/projections/cached_joins.rb +33 -18
  55. data/lib/lyra/projections/cached_projection.rb +74 -33
  56. data/lib/lyra/projections/cached_relation.rb +3175 -321
  57. data/lib/lyra/projections/default_scopes.rb +254 -0
  58. data/lib/lyra/projections/deferred_relation.rb +23 -3
  59. data/lib/lyra/projections/event_store_reader.rb +47 -8
  60. data/lib/lyra/projections/foreign_keys.rb +92 -0
  61. data/lib/lyra/projections/lazy_projection.rb +165 -27
  62. data/lib/lyra/projections/model_projection.rb +165 -14
  63. data/lib/lyra/projections/rebuild.rb +34 -6
  64. data/lib/lyra/projections/reference_links.rb +463 -72
  65. data/lib/lyra/projections/typed_values.rb +629 -0
  66. data/lib/lyra/projections/unique_indexes.rb +595 -0
  67. data/lib/lyra/projections/uniqueness.rb +784 -0
  68. data/lib/lyra/projections/value_digest.rb +93 -0
  69. data/lib/lyra/purpose_bound_reads.rb +912 -77
  70. data/lib/lyra/repair.rb +119 -28
  71. data/lib/lyra/retention.rb +114 -8
  72. data/lib/lyra/rights/consent_store.rb +415 -0
  73. data/lib/lyra/rights/erasure_guard.rb +135 -0
  74. data/lib/lyra/rights/pending.rb +74 -0
  75. data/lib/lyra/rights/queries.rb +269 -0
  76. data/lib/lyra/rights/store.rb +512 -0
  77. data/lib/lyra/rights/subject_keys.rb +224 -0
  78. data/lib/lyra/rights.rb +281 -0
  79. data/lib/lyra/saves_without_effect.rb +69 -0
  80. data/lib/lyra/stream_tail.rb +319 -0
  81. data/lib/lyra/strict_data_access.rb +906 -107
  82. data/lib/lyra/temporal.rb +4 -2
  83. data/lib/lyra/touch_registry.rb +404 -0
  84. data/lib/lyra/verification/bypass_workflow.rb +452 -42
  85. data/lib/lyra/verification/rights_workflows.rb +170 -0
  86. data/lib/lyra/version.rb +1 -1
  87. data/lib/lyra/visualization/event_graph.rb +1 -1
  88. data/lib/lyra/visualization/timeline.rb +2 -2
  89. data/lib/lyra.rb +43 -2
  90. data/lib/tasks/lyra_mode.rake +107 -8
  91. data/lib/tasks/lyra_projections.rake +43 -2
  92. metadata +34 -4
  93. data/lib/lyra/associations/event_aware.rb +0 -224
@@ -9,15 +9,18 @@ module Lyra
9
9
  # so item.tags << tag wrote the join row and the log never learnt of it. A
10
10
  # rebuild lost the association, and DualView did not compare it.
11
11
  #
12
- # Every insert into a join table goes through that class's _create_record
13
- # (tags << tag, tag_ids=, tags.create, an owner saved with new tags), and
14
- # every delete through a delete_all on one of its relations (tags.delete,
15
- # tags.clear, tag_ids=, destroying the owner). Each is recorded as an
16
- # AssociationLinked or AssociationUnlinked event:
12
+ # Every insert Rails makes into a join table goes through that class's
13
+ # _create_record (tags << tag, tag_ids=, tags.create, an owner saved with
14
+ # new tags), one row at a time, and every delete through a delete_all on
15
+ # one of its relations (tags.delete, tags.clear, tag_ids=, destroying the
16
+ # owner). Each is recorded as an AssociationLinked or AssociationUnlinked
17
+ # event:
17
18
  # - appended to the owner's stream, with the association, the join table
18
19
  # and the join rows inserted or deleted ({ "item_id" => 1, "tag_id" => 5 });
19
20
  # - linked into the stream of each associated record whose model Lyra
20
- # monitors, so either side's stream tells its links.
21
+ # monitors, so either side's stream tells its links. When only the
22
+ # associated records' model is monitored, the event is appended to the
23
+ # first of their streams and linked into the others (publish).
21
24
  # The events have no row operation, so replaying a record's state ignores
22
25
  # them. DualView compares each HABTM association's links with the join
23
26
  # table (differences), Rebuild restores join tables from them (rebuild),
@@ -25,10 +28,37 @@ module Lyra
25
28
  #
26
29
  # The join table is read as a set of pairs: a pair linked twice is one
27
30
  # link, and unlinking it removes it.
31
+ #
32
+ # Deleting join rows is Rails' association maintenance when the
33
+ # association does it (Lyra::Associations::Maintenance), allowed in
34
+ # strict mode as dependent: :nullify is; the application's own delete_all
35
+ # on the join model, or on a model it declared on the join table
36
+ # (join_model_of), is refused there like any bulk write, and recorded as
37
+ # an unlink where it runs. Its insert_all, insert_all! and upsert_all
38
+ # (insert, insert!, upsert) of join rows likewise: refused in strict
39
+ # mode, recorded as the links the rows written make where they run
40
+ # (insert_all). Before 1.0.0 these went unguarded and unrecorded.
41
+ #
42
+ # The links of a has_many :through over a model Lyra does not monitor
43
+ # are not recorded, from either side: that model's rows are the
44
+ # application's own, written through its callbacks or Rails' delete_all
45
+ # of them, and its table is no join table here (join_model_of) unless a
46
+ # has_and_belongs_to_many whose links Lyra records names it. Monitored,
47
+ # the through model records its rows as any model does.
28
48
  module HabtmLinks
29
49
  LINKED = "linked"
30
50
  UNLINKED = "unlinked"
31
51
 
52
+ # Records has_and_belongs_to_many declarations (HabtmLinks.declared!).
53
+ # Prepended to ActiveRecord::Associations::ClassMethods (install!).
54
+ module Declarations
55
+ def has_and_belongs_to_many(name, scope = nil, **options, &extension)
56
+ result = super
57
+ HabtmLinks.declared!(self, name)
58
+ result
59
+ end
60
+ end
61
+
32
62
  class << self
33
63
  # Whether +klass+ is the join model of a has_and_belongs_to_many.
34
64
  def join_model?(klass)
@@ -36,6 +66,112 @@ module Lyra
36
66
  klass.name.to_s.start_with?("HABTM_")
37
67
  end
38
68
 
69
+ # The has_and_belongs_to_many join model whose rows +klass+ writes:
70
+ # +klass+ itself when it is one, or, for a model the application
71
+ # declared on a join table whose links Lyra records (and does not
72
+ # monitor itself), the join model of an association on that table:
73
+ # a monitored model's has_and_belongs_to_many, or an unmonitored
74
+ # model's whose associated records' model is monitored (its links
75
+ # are recorded on their streams: publish). Nil otherwise. The latter
76
+ # were not recognised: a model on the join table of an unmonitored
77
+ # owner, whose monitored side had no association back onto the
78
+ # table, deleted the rows with neither the strict guard nor an
79
+ # unlink.
80
+ #
81
+ # A table that is no has_and_belongs_to_many's join table is not
82
+ # one, whatever links its rows make: a has_many :through over a model
83
+ # Lyra does not monitor writes that model's rows through its
84
+ # callbacks, or by Rails' delete_all of them (Maintenance), and Lyra
85
+ # records neither, the through model being an ordinary model of the
86
+ # application's. It is recorded only when that model is monitored
87
+ # (its rows' own events).
88
+ #
89
+ # Every delete_all, insert_all and upsert_all asks
90
+ # (StrictDataAccessRelation), so the answer for a model is kept, until
91
+ # the monitored models change (Configuration#monitor_model, a new
92
+ # configuration, a code reload: reset_join_models!) or a
93
+ # has_and_belongs_to_many is declared (Declarations).
94
+ def join_model_of(klass)
95
+ return klass if join_model?(klass)
96
+ return nil if klass.respond_to?(:lyra_monitored) && klass.lyra_monitored
97
+ return nil unless klass.respond_to?(:table_name) && !klass.abstract_class?
98
+
99
+ found = join_cache[:models].compute_if_absent(klass) { find_join_model(klass.table_name) || false }
100
+ found || nil
101
+ end
102
+
103
+ # The associations on +join_model+'s table whose owners a row of it
104
+ # can name besides its own: those whose owner's column is neither of
105
+ # its two (the inverse side's owner column is its associated
106
+ # records'), whose links are recorded now. Kept as join_model_of's
107
+ # answers are.
108
+ def other_sides(join_model)
109
+ left, right = columns(join_model)
110
+ found = join_cache[:others].compute_if_absent(join_model) do
111
+ join_models_on(join_model.table_name).reject { [left, right].include?(columns(_1).first) }
112
+ end
113
+ found.select { recorded?(_1) }
114
+ end
115
+
116
+ # Forget the join models join_model_of found.
117
+ def reset_join_models!
118
+ @join_models_of = nil
119
+ end
120
+
121
+ # A has_and_belongs_to_many +name+ was declared on +owner+
122
+ # (Declarations): its join model is kept, by owner and association
123
+ # (a class reloaded replaces its own), so join_model_of finds the
124
+ # tables of unmonitored owners too.
125
+ def declared!(owner, name)
126
+ join_model = owner.const_get("HABTM_#{name.to_s.camelize}", false)
127
+ @declared ||= Concurrent::Map.new
128
+ @declared[[owner.name || owner.object_id, name.to_s]] = join_model
129
+ @declarations = @declarations.to_i + 1
130
+ rescue NameError
131
+ nil
132
+ end
133
+
134
+ # Record has_and_belongs_to_many declarations from now on
135
+ # (Maintenance.install!), and those of the models already loaded (an
136
+ # initializer that loaded models before Active Record's load hooks
137
+ # ran Lyra's): declared_already!.
138
+ def install!
139
+ unless ActiveRecord::Associations::ClassMethods.ancestors.include?(Declarations)
140
+ ActiveRecord::Associations::ClassMethods.prepend(Declarations)
141
+ end
142
+ declared_already!
143
+ end
144
+
145
+ # Record the has_and_belongs_to_many of every model loaded now, as
146
+ # declared! records a new one: the classes its name stands for now
147
+ # (one reloaded since, still among Active Record's descendants, would
148
+ # replace the current one's). Returns how many.
149
+ def declared_already!
150
+ ActiveRecord::Base.descendants.sum do |model|
151
+ next 0 if model.abstract_class? || model.name.nil? || join_model?(model)
152
+ next 0 unless model.name.safe_constantize.equal?(model)
153
+
154
+ model.reflect_on_all_associations(:has_and_belongs_to_many).count do |reflection|
155
+ declared!(model, reflection.name)
156
+ end
157
+ rescue StandardError
158
+ 0
159
+ end
160
+ end
161
+
162
+ # Whether deleting +join_model+'s rows is a write strict data access
163
+ # guards: either side's model is monitored (the owner's, or the
164
+ # associated records'). Recording the unlink on the monitored side's
165
+ # stream when only that side is monitored is left to the owner's
166
+ # (recorded?).
167
+ def guarded?(join_model)
168
+ [join_model.left_model, join_model.right_reflection.klass].any? do |side|
169
+ side.respond_to?(:lyra_monitored) && side.lyra_monitored
170
+ end
171
+ rescue NameError
172
+ false
173
+ end
174
+
39
175
  # Around the INSERT of a join row (WriteHooks#_create_record).
40
176
  def create(record)
41
177
  join_model = record.class
@@ -51,18 +187,95 @@ module Lyra
51
187
  end
52
188
 
53
189
  # Around a delete_all on a join model's relation
54
- # (StrictDataAccessRelation#delete_all).
55
- def delete_all(relation)
56
- join_model = relation.klass
190
+ # (StrictDataAccessRelation#delete_all): the unlink of its owner, on
191
+ # the owner's stream, linked into each associated record's.
192
+ #
193
+ # A row can name the owners of other associations on the table too,
194
+ # in columns of their own (a table two owners share, each with its
195
+ # column, where the application filled both on one row): each of
196
+ # those owners gets the unlink Rails' own from its side records, as
197
+ # delete_all_on_table records it (other_sides). It was recorded for
198
+ # the side Rails deleted from only, and the other owner's stream kept
199
+ # a link whose row was gone. A row naming no other owner is recorded
200
+ # as before, and the inverse side of the association (its owner is
201
+ # this side's associated record) is not another owner.
202
+ def delete_all(relation, join_model = relation.klass)
57
203
  return yield unless recorded?(join_model)
58
204
 
59
205
  left, right = columns(join_model)
206
+ others = other_sides(join_model)
207
+ read = ([left, right] + others.flat_map { columns(_1) }).uniq
60
208
  Lyra::BypassEvents.atomically(join_model.left_model) do
61
- pairs = relation.pluck(left, right)
209
+ rows = relation.pluck(*read).map { |values| read.zip(Array(values)).to_h }
62
210
  result = yield
63
- pairs.group_by(&:first).each do |owner_id, owned|
64
- publish(join_model, UNLINKED, owner_id, owned.map { |l, r| { left => l, right => r } })
211
+ rows.group_by { _1[left] }.each do |owner_id, owned|
212
+ publish(join_model, UNLINKED, owner_id, owned.map { |row| { left => row[left], right => row[right] } })
65
213
  end
214
+ publish_sides(others, rows, UNLINKED) if others.any?
215
+ result
216
+ end
217
+ end
218
+
219
+ # Around a delete_all on a relation of a model the application
220
+ # declared on a join table (StrictDataAccessRelation#delete_all), its
221
+ # own or Rails' through it (a has_many :through over it, its
222
+ # dependent: :delete_all). Several monitored models'
223
+ # has_and_belongs_to_many can share the table: both sides of one
224
+ # association, a self-referential one, or two owners on columns of
225
+ # their own. Each of them whose model is recorded is a side
226
+ # (recorded_sides): the rows deleted are read first, by every side's
227
+ # columns, and each owner a row names in its side's foreign key gets
228
+ # the AssociationUnlinked Rails' own unlink from that side records,
229
+ # on its stream, linked into each associated record's stream, unless
230
+ # that record has an unlink of its own for the same rows (the inverse
231
+ # side of the association is recorded too). A side whose model is
232
+ # not monitored records nothing. It recorded the first model's side
233
+ # only, whichever owners the rows named.
234
+ def delete_all_on_table(relation)
235
+ sides = recorded_sides(relation.klass.table_name)
236
+ return yield if sides.empty?
237
+
238
+ columns = sides.flat_map { columns(_1) }.uniq
239
+ Lyra::BypassEvents.atomically(sides.first.left_model) do
240
+ rows = relation.pluck(*columns).map { |values| columns.zip(Array(values)).to_h }
241
+ result = yield
242
+ publish_sides(sides, rows, UNLINKED)
243
+ result
244
+ end
245
+ end
246
+
247
+ # Around the application's insert_all, insert_all! or upsert_all on a
248
+ # join model, or on a model it declared on a join table
249
+ # (StrictDataAccessRelation#join_insert; the block runs the write,
250
+ # given the RETURNING to ask for). The links the rows written make
251
+ # are recorded as Rails' own link from each side records them, as
252
+ # delete_all and delete_all_on_table record unlinks: a join model's
253
+ # rows link its owners (its side, and the other owners a row names:
254
+ # other_sides); an application model's rows
255
+ # link every monitored owner they name, of every side whose links are
256
+ # recorded (recorded_sides), each on its stream, linked into the
257
+ # associated records' streams that have none of their own. The rows
258
+ # written are read back by RETURNING (with the columns each side
259
+ # needs: Lyra::BypassEvents.with_returning), the caller getting the
260
+ # result it asked for: the values the association's scope added (an
261
+ # owner's item_tags.insert_all(...) sets its key) are in the event,
262
+ # and a row ON CONFLICT DO NOTHING skipped records nothing.
263
+ #
264
+ # An upsert_all (+conflict+: the columns of its ON CONFLICT) reads the
265
+ # rows its keys name first: a row it updated whose pair a side reads
266
+ # is unchanged records nothing; one whose pair changed is that side's
267
+ # unlink of the old pair and link of the new.
268
+ def insert_all(relation, attributes, returning:, conflict: nil)
269
+ klass = relation.klass
270
+ sides = join_model?(klass) ? [klass].select { recorded?(_1) } + other_sides(klass) : recorded_sides(klass.table_name)
271
+ return yield(returning) if sides.empty?
272
+
273
+ columns = sides.flat_map { columns(_1) }.uniq
274
+ keys = Array(conflict)
275
+ Lyra::BypassEvents.atomically(sides.first.left_model) do
276
+ before = keys.empty? ? {} : rows_by_key(relation, attributes, keys, columns)
277
+ result, written = Lyra::BypassEvents.with_returning(klass, returning, (keys + columns).uniq) { |ret| yield ret }
278
+ publish_inserts(sides, Array(written), before, keys)
66
279
  result
67
280
  end
68
281
  end
@@ -85,7 +298,7 @@ module Lyra
85
298
  # Remove the rows of +model_class+'s join tables (before the model's
86
299
  # table is cleared, which their foreign keys may reference).
87
300
  def clear(model_class)
88
- join_models(model_class).each_value { |join_model| Lyra.projection_write { join_model.unscoped.delete_all } }
301
+ join_models(model_class).each_value { |join_model| Lyra.projection_write(INTERNAL) { join_model.unscoped.delete_all } }
89
302
  end
90
303
 
91
304
  # Bring +model_class+'s join tables to what the link events say, in
@@ -119,8 +332,138 @@ module Lyra
119
332
 
120
333
  private
121
334
 
335
+ # What join_model_of and other_sides found, until the monitored
336
+ # models change or a has_and_belongs_to_many is declared.
337
+ def join_cache
338
+ config = Lyra.config
339
+ cache = @join_models_of
340
+ return cache if cache && cache[:config].equal?(config) && cache[:count] == config.monitored_models.size &&
341
+ cache[:declarations] == @declarations
342
+
343
+ @join_models_of = { config: config, count: config.monitored_models.size, declarations: @declarations,
344
+ models: Concurrent::Map.new, others: Concurrent::Map.new }
345
+ end
346
+
347
+ # The join model of a monitored model's has_and_belongs_to_many on
348
+ # +table+, or nil.
349
+ def find_join_model(table) = join_models_on(table).first
350
+
351
+ # The join models of the has_and_belongs_to_many on +table+ whose
352
+ # links Lyra records, each once (an STI subclass inherits its
353
+ # parent's): the monitored models' first, then those of unmonitored
354
+ # owners whose associated records' model is monitored
355
+ # (declared_join_models).
356
+ def join_models_on(table)
357
+ (Lyra.config.monitored_models.flat_map { join_models(_1).values } + declared_join_models)
358
+ .select { _1.table_name == table }.uniq
359
+ end
360
+
361
+ # The join models of the has_and_belongs_to_many declared on models
362
+ # Lyra does not monitor, linking to records of a model it does
363
+ # (declared!). A class since reloaded or removed is left out.
364
+ def declared_join_models
365
+ return [] unless @declared
366
+
367
+ monitored = Lyra.config.monitored_models
368
+ @declared.values.select do |join_model|
369
+ owner = join_model.left_model
370
+ next false unless owner.name && owner.name.safe_constantize.equal?(owner)
371
+ next false if monitored.any? { owner <= _1 }
372
+
373
+ target = join_model.right_reflection.klass
374
+ monitored.any? { target <= _1 }
375
+ rescue NameError
376
+ false
377
+ end
378
+ end
379
+
380
+ # The sides of +table+ whose unlinks are recorded now (recorded?).
381
+ def recorded_sides(table) = join_models_on(table).select { recorded?(_1) }
382
+
383
+ # Each side's links or unlinks (+operation+) of +rows+ (hashes by
384
+ # column; or a callable giving a side's rows), as publish records
385
+ # them, then linked into the associated records' streams that have
386
+ # none of their own for those rows.
387
+ def publish_sides(sides, rows, operation)
388
+ rows_of = rows.respond_to?(:call) ? rows : ->(_) { rows }
389
+ covered = Set.new
390
+ events = sides.flat_map do |join_model|
391
+ left, right = columns(join_model)
392
+ rows_of.call(join_model).select { _1[left] }.group_by { _1[left] }.map do |owner_id, owned|
393
+ pairs = owned.map { |row| { left => row[left], right => row[right] } }.uniq
394
+ event = build_event(join_model, operation, owner_id, pairs, name: association_name(join_model))
395
+ if owner_recorded?(join_model)
396
+ Lyra.append_events(event, stream_name: Lyra.stream_name(join_model.left_model, owner_id))
397
+ covered << [Lyra.stream_name(join_model.left_model, owner_id), left]
398
+ end
399
+ [event, join_model, owner_id, pairs]
400
+ end
401
+ end
402
+ events.each do |event, join_model, owner_id, pairs|
403
+ link_into_targets(event, join_model, owner_id, pairs, covered: covered,
404
+ append_first: !owner_recorded?(join_model))
405
+ end
406
+ rescue => e
407
+ Lyra::BypassEvents.publish_failed!(e, "has_and_belongs_to_many #{operation}")
408
+ end
409
+
410
+ # The rows an upsert's +keys+ name, before it (the attributes given,
411
+ # with the values the relation's scope adds), as hashes of +keys+ and
412
+ # +columns+, by their keys' values. Lyra's own read.
413
+ def rows_by_key(relation, attributes, keys, columns)
414
+ klass = relation.klass
415
+ scope = relation.scope_for_create.transform_keys(&:to_s)
416
+ types = keys.map { klass.type_for_attribute(_1) }
417
+ tuples = Array(attributes).filter_map do |row|
418
+ row = scope.merge(row.to_h.transform_keys(&:to_s))
419
+ next unless keys.all? { row.key?(_1) && !row[_1].nil? }
420
+
421
+ keys.each_with_index.map { |key, i| types[i].cast(row[key]) }
422
+ end.uniq
423
+ return {} if tuples.empty?
424
+
425
+ wanted = tuples.to_set
426
+ read = (keys + columns).uniq
427
+ found = Lyra::PurposeBoundReads.internal(INTERNAL) do
428
+ klass.unscoped.where(keys.first => tuples.map(&:first).uniq).pluck(*read)
429
+ end
430
+ found.each_with_object({}) do |values, acc|
431
+ row = read.zip(Array(values)).to_h
432
+ tuple = row.values_at(*keys)
433
+ acc[tuple] = row if wanted.include?(tuple)
434
+ end
435
+ end
436
+
437
+ # The links and unlinks the rows an insert or upsert wrote make
438
+ # (insert_all): every row an insert wrote, and every row an upsert
439
+ # inserted, links its pair; a row an upsert updated (found in
440
+ # +before+ by its +keys+) unlinks and links only where the side's
441
+ # pair changed.
442
+ def publish_inserts(sides, written, before, keys)
443
+ return publish_sides(sides, written, LINKED) if before.empty?
444
+
445
+ old_of = ->(row) { before[row.values_at(*keys)] }
446
+ changed = lambda do |join_model, row|
447
+ old = old_of.(row)
448
+ old && old.values_at(*columns(join_model)) != row.values_at(*columns(join_model))
449
+ end
450
+ publish_sides(sides, ->(side) { written.select { changed.(side, _1) }.map(&old_of) }, UNLINKED)
451
+ publish_sides(sides, ->(side) { written.select { old_of.(_1).nil? || changed.(side, _1) } }, LINKED)
452
+ end
453
+
454
+ # Whether +join_model+'s links are recorded: on its owner's stream
455
+ # when the owner's model is monitored, else on the associated
456
+ # records' streams when theirs is (owner_recorded?, targets_recorded?).
122
457
  def recorded?(join_model)
123
- join_model?(join_model) && Lyra::BypassEvents.enabled_for?(join_model.left_model)
458
+ join_model?(join_model) && (owner_recorded?(join_model) || targets_recorded?(join_model))
459
+ end
460
+
461
+ def owner_recorded?(join_model) = Lyra::BypassEvents.enabled_for?(join_model.left_model)
462
+
463
+ def targets_recorded?(join_model)
464
+ Lyra::BypassEvents.enabled_for?(join_model.right_reflection.klass)
465
+ rescue NameError
466
+ false
124
467
  end
125
468
 
126
469
  # The join model of each of +model_class+'s HABTM associations, by name.
@@ -150,11 +493,21 @@ module Lyra
150
493
  [reflection.foreign_key.to_s, reflection.association_foreign_key.to_s]
151
494
  end
152
495
 
496
+ # The event on the owner's stream, linked into each associated
497
+ # record's; when the owner's model is not monitored, on the
498
+ # associated records' streams only (the first one's, linked into the
499
+ # others'), as a link made from the other side would be seen there.
500
+ # Before, a link or unlink made from a side Lyra did not monitor was
501
+ # not recorded at all, though the other side's records were.
153
502
  def publish(join_model, operation, owner_id, rows)
154
503
  owner = join_model.left_model
155
504
  event = build_event(join_model, operation, owner_id, rows, name: association_name(join_model))
156
- Lyra.append_events(event, stream_name: Lyra.stream_name(owner, owner_id))
157
- link_into_targets(event, join_model, owner_id, rows)
505
+ if owner_recorded?(join_model)
506
+ Lyra.append_events(event, stream_name: Lyra.stream_name(owner, owner_id))
507
+ link_into_targets(event, join_model, owner_id, rows)
508
+ else
509
+ link_into_targets(event, join_model, owner_id, rows, append_first: true)
510
+ end
158
511
  rescue => e
159
512
  Lyra::BypassEvents.publish_failed!(e, "has_and_belongs_to_many #{operation}")
160
513
  end
@@ -183,7 +536,11 @@ module Lyra
183
536
  # monitored and the stream exists. An empty stream is left alone: a
184
537
  # stream that held only links would look imported to Genesis, and the
185
538
  # record would never get its Imported event.
186
- def link_into_targets(event, join_model, owner_id, rows)
539
+ # +covered+: [stream, column] pairs whose stream has an event of its
540
+ # own for the rows naming it in that column (publish_sides).
541
+ # +append_first+: the event is in no stream yet (its owner's model is
542
+ # not monitored); it is appended to the first stream, then linked.
543
+ def link_into_targets(event, join_model, owner_id, rows, covered: nil, append_first: false)
187
544
  target = join_model.right_reflection.klass
188
545
  return unless target.respond_to?(:lyra_monitored) && target.lyra_monitored
189
546
 
@@ -194,9 +551,15 @@ module Lyra
194
551
  rows.map { |row| row[right] }.compact.uniq.each do |target_id|
195
552
  stream = Lyra.stream_name(target, target_id)
196
553
  next if stream == own_stream
554
+ next if covered&.include?([stream, right])
197
555
  next unless store.read.stream(stream).limit(1).to_a.any?
198
556
 
199
- store.link([event.event_id], stream_name: stream)
557
+ if append_first
558
+ Lyra.append_events(event, stream_name: stream)
559
+ append_first = false
560
+ else
561
+ store.link([event.event_id], stream_name: stream)
562
+ end
200
563
  end
201
564
  end
202
565
 
@@ -229,7 +592,7 @@ module Lyra
229
592
  def apply(join_model, event)
230
593
  left, right = columns(join_model)
231
594
  rows = Array(value(event.data, :rows)).map { |row| row.to_h.transform_keys(&:to_s).slice(left, right) }
232
- Lyra.projection_write do
595
+ Lyra.projection_write(INTERNAL) do
233
596
  rows.each do |row|
234
597
  existing = join_model.unscoped.where(row)
235
598
  if value(event.data, :operation).to_s == LINKED