orfeas_lyra 0.9.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +2021 -17
- data/README.md +22 -9
- data/Rakefile +40 -1
- data/app/controllers/lyra/application_controller.rb +127 -0
- data/app/controllers/lyra/dashboard_controller.rb +33 -31
- data/app/controllers/lyra/flow_controller.rb +32 -34
- data/app/controllers/lyra/privacy_controller.rb +57 -18
- data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
- data/app/views/lyra/dashboard/index.html.erb +1 -0
- data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
- data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
- data/app/views/lyra/privacy/policy.html.erb +7 -0
- data/app/views/lyra/privacy/rights.html.erb +78 -0
- data/config/privacy_policies.rb +0 -3
- data/config/routes.rb +1 -0
- data/lib/lyra/access_log.rb +187 -6
- data/lib/lyra/advisory_lock.rb +236 -0
- data/lib/lyra/aggregate.rb +1 -1
- data/lib/lyra/associations/habtm_links.rb +382 -19
- data/lib/lyra/associations/maintenance.rb +229 -0
- data/lib/lyra/attribution.rb +48 -0
- data/lib/lyra/bypass_events.rb +446 -46
- data/lib/lyra/command_handler.rb +19 -14
- data/lib/lyra/configuration.rb +237 -10
- data/lib/lyra/consistency/read_your_writes.rb +4 -2
- data/lib/lyra/correlation.rb +121 -0
- data/lib/lyra/domain_events.rb +4 -0
- data/lib/lyra/dual_view.rb +53 -16
- data/lib/lyra/encrypted_attributes.rb +5 -0
- data/lib/lyra/engine.rb +67 -0
- data/lib/lyra/erasure.rb +1011 -106
- data/lib/lyra/event.rb +35 -1
- data/lib/lyra/event_data.rb +38 -0
- data/lib/lyra/event_flow.rb +31 -15
- data/lib/lyra/event_store_adapter.rb +28 -0
- data/lib/lyra/event_window.rb +58 -0
- data/lib/lyra/genesis.rb +548 -44
- data/lib/lyra/interceptors/association_interceptor.rb +689 -34
- data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
- data/lib/lyra/internal.rb +26 -0
- data/lib/lyra/mode_sync.rb +51 -3
- data/lib/lyra/mode_transition.rb +257 -30
- data/lib/lyra/model_selection.rb +35 -0
- data/lib/lyra/own_connection.rb +17 -2
- data/lib/lyra/privacy/adapters/pam.rb +25 -2
- data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
- data/lib/lyra/privacy/interface.rb +27 -7
- data/lib/lyra/privacy/subject_links.rb +143 -0
- data/lib/lyra/privacy/subject_records.rb +369 -0
- data/lib/lyra/projection.rb +7 -1
- data/lib/lyra/projections/async_checkpoint.rb +32 -0
- data/lib/lyra/projections/async_projection_job.rb +166 -23
- data/lib/lyra/projections/cached_joins.rb +33 -18
- data/lib/lyra/projections/cached_projection.rb +74 -33
- data/lib/lyra/projections/cached_relation.rb +3175 -321
- data/lib/lyra/projections/default_scopes.rb +254 -0
- data/lib/lyra/projections/deferred_relation.rb +23 -3
- data/lib/lyra/projections/event_store_reader.rb +47 -8
- data/lib/lyra/projections/foreign_keys.rb +92 -0
- data/lib/lyra/projections/lazy_projection.rb +165 -27
- data/lib/lyra/projections/model_projection.rb +165 -14
- data/lib/lyra/projections/rebuild.rb +34 -6
- data/lib/lyra/projections/reference_links.rb +463 -72
- data/lib/lyra/projections/typed_values.rb +629 -0
- data/lib/lyra/projections/unique_indexes.rb +595 -0
- data/lib/lyra/projections/uniqueness.rb +784 -0
- data/lib/lyra/projections/value_digest.rb +93 -0
- data/lib/lyra/purpose_bound_reads.rb +912 -77
- data/lib/lyra/repair.rb +119 -28
- data/lib/lyra/retention.rb +114 -8
- data/lib/lyra/rights/consent_store.rb +415 -0
- data/lib/lyra/rights/erasure_guard.rb +135 -0
- data/lib/lyra/rights/pending.rb +74 -0
- data/lib/lyra/rights/queries.rb +269 -0
- data/lib/lyra/rights/store.rb +512 -0
- data/lib/lyra/rights/subject_keys.rb +224 -0
- data/lib/lyra/rights.rb +281 -0
- data/lib/lyra/saves_without_effect.rb +69 -0
- data/lib/lyra/stream_tail.rb +319 -0
- data/lib/lyra/strict_data_access.rb +916 -106
- data/lib/lyra/temporal.rb +4 -2
- data/lib/lyra/touch_registry.rb +404 -0
- data/lib/lyra/verification/bypass_workflow.rb +452 -42
- data/lib/lyra/verification/rights_workflows.rb +170 -0
- data/lib/lyra/version.rb +1 -1
- data/lib/lyra/visualization/event_graph.rb +1 -1
- data/lib/lyra/visualization/timeline.rb +2 -2
- data/lib/lyra.rb +43 -2
- data/lib/tasks/lyra_mode.rake +110 -8
- data/lib/tasks/lyra_projections.rake +43 -2
- metadata +34 -4
- data/lib/lyra/associations/event_aware.rb +0 -224
|
@@ -9,15 +9,18 @@ module Lyra
|
|
|
9
9
|
# so item.tags << tag wrote the join row and the log never learnt of it. A
|
|
10
10
|
# rebuild lost the association, and DualView did not compare it.
|
|
11
11
|
#
|
|
12
|
-
# Every insert into a join table goes through that class's
|
|
13
|
-
# (tags << tag, tag_ids=, tags.create, an owner saved with
|
|
14
|
-
# every delete through a delete_all on
|
|
15
|
-
# tags.clear, tag_ids=, destroying the
|
|
16
|
-
# AssociationLinked or AssociationUnlinked
|
|
12
|
+
# Every insert Rails makes into a join table goes through that class's
|
|
13
|
+
# _create_record (tags << tag, tag_ids=, tags.create, an owner saved with
|
|
14
|
+
# new tags), one row at a time, and every delete through a delete_all on
|
|
15
|
+
# one of its relations (tags.delete, tags.clear, tag_ids=, destroying the
|
|
16
|
+
# owner). Each is recorded as an AssociationLinked or AssociationUnlinked
|
|
17
|
+
# event:
|
|
17
18
|
# - appended to the owner's stream, with the association, the join table
|
|
18
19
|
# and the join rows inserted or deleted ({ "item_id" => 1, "tag_id" => 5 });
|
|
19
20
|
# - linked into the stream of each associated record whose model Lyra
|
|
20
|
-
# monitors, so either side's stream tells its links.
|
|
21
|
+
# monitors, so either side's stream tells its links. When only the
|
|
22
|
+
# associated records' model is monitored, the event is appended to the
|
|
23
|
+
# first of their streams and linked into the others (publish).
|
|
21
24
|
# The events have no row operation, so replaying a record's state ignores
|
|
22
25
|
# them. DualView compares each HABTM association's links with the join
|
|
23
26
|
# table (differences), Rebuild restores join tables from them (rebuild),
|
|
@@ -25,10 +28,37 @@ module Lyra
|
|
|
25
28
|
#
|
|
26
29
|
# The join table is read as a set of pairs: a pair linked twice is one
|
|
27
30
|
# link, and unlinking it removes it.
|
|
31
|
+
#
|
|
32
|
+
# Deleting join rows is Rails' association maintenance when the
|
|
33
|
+
# association does it (Lyra::Associations::Maintenance), allowed in
|
|
34
|
+
# strict mode as dependent: :nullify is; the application's own delete_all
|
|
35
|
+
# on the join model, or on a model it declared on the join table
|
|
36
|
+
# (join_model_of), is refused there like any bulk write, and recorded as
|
|
37
|
+
# an unlink where it runs. Its insert_all, insert_all! and upsert_all
|
|
38
|
+
# (insert, insert!, upsert) of join rows likewise: refused in strict
|
|
39
|
+
# mode, recorded as the links the rows written make where they run
|
|
40
|
+
# (insert_all). Before 1.0.0 these went unguarded and unrecorded.
|
|
41
|
+
#
|
|
42
|
+
# The links of a has_many :through over a model Lyra does not monitor
|
|
43
|
+
# are not recorded, from either side: that model's rows are the
|
|
44
|
+
# application's own, written through its callbacks or Rails' delete_all
|
|
45
|
+
# of them, and its table is no join table here (join_model_of) unless a
|
|
46
|
+
# has_and_belongs_to_many whose links Lyra records names it. Monitored,
|
|
47
|
+
# the through model records its rows as any model does.
|
|
28
48
|
module HabtmLinks
|
|
29
49
|
LINKED = "linked"
|
|
30
50
|
UNLINKED = "unlinked"
|
|
31
51
|
|
|
52
|
+
# Records has_and_belongs_to_many declarations (HabtmLinks.declared!).
|
|
53
|
+
# Prepended to ActiveRecord::Associations::ClassMethods (install!).
|
|
54
|
+
module Declarations
|
|
55
|
+
def has_and_belongs_to_many(name, scope = nil, **options, &extension)
|
|
56
|
+
result = super
|
|
57
|
+
HabtmLinks.declared!(self, name)
|
|
58
|
+
result
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
32
62
|
class << self
|
|
33
63
|
# Whether +klass+ is the join model of a has_and_belongs_to_many.
|
|
34
64
|
def join_model?(klass)
|
|
@@ -36,6 +66,112 @@ module Lyra
|
|
|
36
66
|
klass.name.to_s.start_with?("HABTM_")
|
|
37
67
|
end
|
|
38
68
|
|
|
69
|
+
# The has_and_belongs_to_many join model whose rows +klass+ writes:
|
|
70
|
+
# +klass+ itself when it is one, or, for a model the application
|
|
71
|
+
# declared on a join table whose links Lyra records (and does not
|
|
72
|
+
# monitor itself), the join model of an association on that table:
|
|
73
|
+
# a monitored model's has_and_belongs_to_many, or an unmonitored
|
|
74
|
+
# model's whose associated records' model is monitored (its links
|
|
75
|
+
# are recorded on their streams: publish). Nil otherwise. The latter
|
|
76
|
+
# were not recognised: a model on the join table of an unmonitored
|
|
77
|
+
# owner, whose monitored side had no association back onto the
|
|
78
|
+
# table, deleted the rows with neither the strict guard nor an
|
|
79
|
+
# unlink.
|
|
80
|
+
#
|
|
81
|
+
# A table that is no has_and_belongs_to_many's join table is not
|
|
82
|
+
# one, whatever links its rows make: a has_many :through over a model
|
|
83
|
+
# Lyra does not monitor writes that model's rows through its
|
|
84
|
+
# callbacks, or by Rails' delete_all of them (Maintenance), and Lyra
|
|
85
|
+
# records neither, the through model being an ordinary model of the
|
|
86
|
+
# application's. It is recorded only when that model is monitored
|
|
87
|
+
# (its rows' own events).
|
|
88
|
+
#
|
|
89
|
+
# Every delete_all, insert_all and upsert_all asks
|
|
90
|
+
# (StrictDataAccessRelation), so the answer for a model is kept, until
|
|
91
|
+
# the monitored models change (Configuration#monitor_model, a new
|
|
92
|
+
# configuration, a code reload: reset_join_models!) or a
|
|
93
|
+
# has_and_belongs_to_many is declared (Declarations).
|
|
94
|
+
def join_model_of(klass)
|
|
95
|
+
return klass if join_model?(klass)
|
|
96
|
+
return nil if klass.respond_to?(:lyra_monitored) && klass.lyra_monitored
|
|
97
|
+
return nil unless klass.respond_to?(:table_name) && !klass.abstract_class?
|
|
98
|
+
|
|
99
|
+
found = join_cache[:models].compute_if_absent(klass) { find_join_model(klass.table_name) || false }
|
|
100
|
+
found || nil
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# The associations on +join_model+'s table whose owners a row of it
|
|
104
|
+
# can name besides its own: those whose owner's column is neither of
|
|
105
|
+
# its two (the inverse side's owner column is its associated
|
|
106
|
+
# records'), whose links are recorded now. Kept as join_model_of's
|
|
107
|
+
# answers are.
|
|
108
|
+
def other_sides(join_model)
|
|
109
|
+
left, right = columns(join_model)
|
|
110
|
+
found = join_cache[:others].compute_if_absent(join_model) do
|
|
111
|
+
join_models_on(join_model.table_name).reject { [left, right].include?(columns(_1).first) }
|
|
112
|
+
end
|
|
113
|
+
found.select { recorded?(_1) }
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# Forget the join models join_model_of found.
|
|
117
|
+
def reset_join_models!
|
|
118
|
+
@join_models_of = nil
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# A has_and_belongs_to_many +name+ was declared on +owner+
|
|
122
|
+
# (Declarations): its join model is kept, by owner and association
|
|
123
|
+
# (a class reloaded replaces its own), so join_model_of finds the
|
|
124
|
+
# tables of unmonitored owners too.
|
|
125
|
+
def declared!(owner, name)
|
|
126
|
+
join_model = owner.const_get("HABTM_#{name.to_s.camelize}", false)
|
|
127
|
+
@declared ||= Concurrent::Map.new
|
|
128
|
+
@declared[[owner.name || owner.object_id, name.to_s]] = join_model
|
|
129
|
+
@declarations = @declarations.to_i + 1
|
|
130
|
+
rescue NameError
|
|
131
|
+
nil
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# Record has_and_belongs_to_many declarations from now on
|
|
135
|
+
# (Maintenance.install!), and those of the models already loaded (an
|
|
136
|
+
# initializer that loaded models before Active Record's load hooks
|
|
137
|
+
# ran Lyra's): declared_already!.
|
|
138
|
+
def install!
|
|
139
|
+
unless ActiveRecord::Associations::ClassMethods.ancestors.include?(Declarations)
|
|
140
|
+
ActiveRecord::Associations::ClassMethods.prepend(Declarations)
|
|
141
|
+
end
|
|
142
|
+
declared_already!
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
# Record the has_and_belongs_to_many of every model loaded now, as
|
|
146
|
+
# declared! records a new one: the classes its name stands for now
|
|
147
|
+
# (one reloaded since, still among Active Record's descendants, would
|
|
148
|
+
# replace the current one's). Returns how many.
|
|
149
|
+
def declared_already!
|
|
150
|
+
ActiveRecord::Base.descendants.sum do |model|
|
|
151
|
+
next 0 if model.abstract_class? || model.name.nil? || join_model?(model)
|
|
152
|
+
next 0 unless model.name.safe_constantize.equal?(model)
|
|
153
|
+
|
|
154
|
+
model.reflect_on_all_associations(:has_and_belongs_to_many).count do |reflection|
|
|
155
|
+
declared!(model, reflection.name)
|
|
156
|
+
end
|
|
157
|
+
rescue StandardError
|
|
158
|
+
0
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
# Whether deleting +join_model+'s rows is a write strict data access
|
|
163
|
+
# guards: either side's model is monitored (the owner's, or the
|
|
164
|
+
# associated records'). Recording the unlink on the monitored side's
|
|
165
|
+
# stream when only that side is monitored is left to the owner's
|
|
166
|
+
# (recorded?).
|
|
167
|
+
def guarded?(join_model)
|
|
168
|
+
[join_model.left_model, join_model.right_reflection.klass].any? do |side|
|
|
169
|
+
side.respond_to?(:lyra_monitored) && side.lyra_monitored
|
|
170
|
+
end
|
|
171
|
+
rescue NameError
|
|
172
|
+
false
|
|
173
|
+
end
|
|
174
|
+
|
|
39
175
|
# Around the INSERT of a join row (WriteHooks#_create_record).
|
|
40
176
|
def create(record)
|
|
41
177
|
join_model = record.class
|
|
@@ -51,18 +187,95 @@ module Lyra
|
|
|
51
187
|
end
|
|
52
188
|
|
|
53
189
|
# Around a delete_all on a join model's relation
|
|
54
|
-
# (StrictDataAccessRelation#delete_all)
|
|
55
|
-
|
|
56
|
-
|
|
190
|
+
# (StrictDataAccessRelation#delete_all): the unlink of its owner, on
|
|
191
|
+
# the owner's stream, linked into each associated record's.
|
|
192
|
+
#
|
|
193
|
+
# A row can name the owners of other associations on the table too,
|
|
194
|
+
# in columns of their own (a table two owners share, each with its
|
|
195
|
+
# column, where the application filled both on one row): each of
|
|
196
|
+
# those owners gets the unlink Rails' own from its side records, as
|
|
197
|
+
# delete_all_on_table records it (other_sides). It was recorded for
|
|
198
|
+
# the side Rails deleted from only, and the other owner's stream kept
|
|
199
|
+
# a link whose row was gone. A row naming no other owner is recorded
|
|
200
|
+
# as before, and the inverse side of the association (its owner is
|
|
201
|
+
# this side's associated record) is not another owner.
|
|
202
|
+
def delete_all(relation, join_model = relation.klass)
|
|
57
203
|
return yield unless recorded?(join_model)
|
|
58
204
|
|
|
59
205
|
left, right = columns(join_model)
|
|
206
|
+
others = other_sides(join_model)
|
|
207
|
+
read = ([left, right] + others.flat_map { columns(_1) }).uniq
|
|
60
208
|
Lyra::BypassEvents.atomically(join_model.left_model) do
|
|
61
|
-
|
|
209
|
+
rows = relation.pluck(*read).map { |values| read.zip(Array(values)).to_h }
|
|
62
210
|
result = yield
|
|
63
|
-
|
|
64
|
-
publish(join_model, UNLINKED, owner_id, owned.map { |
|
|
211
|
+
rows.group_by { _1[left] }.each do |owner_id, owned|
|
|
212
|
+
publish(join_model, UNLINKED, owner_id, owned.map { |row| { left => row[left], right => row[right] } })
|
|
65
213
|
end
|
|
214
|
+
publish_sides(others, rows, UNLINKED) if others.any?
|
|
215
|
+
result
|
|
216
|
+
end
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
# Around a delete_all on a relation of a model the application
|
|
220
|
+
# declared on a join table (StrictDataAccessRelation#delete_all), its
|
|
221
|
+
# own or Rails' through it (a has_many :through over it, its
|
|
222
|
+
# dependent: :delete_all). Several monitored models'
|
|
223
|
+
# has_and_belongs_to_many can share the table: both sides of one
|
|
224
|
+
# association, a self-referential one, or two owners on columns of
|
|
225
|
+
# their own. Each of them whose model is recorded is a side
|
|
226
|
+
# (recorded_sides): the rows deleted are read first, by every side's
|
|
227
|
+
# columns, and each owner a row names in its side's foreign key gets
|
|
228
|
+
# the AssociationUnlinked Rails' own unlink from that side records,
|
|
229
|
+
# on its stream, linked into each associated record's stream, unless
|
|
230
|
+
# that record has an unlink of its own for the same rows (the inverse
|
|
231
|
+
# side of the association is recorded too). A side whose model is
|
|
232
|
+
# not monitored records nothing. It recorded the first model's side
|
|
233
|
+
# only, whichever owners the rows named.
|
|
234
|
+
def delete_all_on_table(relation)
|
|
235
|
+
sides = recorded_sides(relation.klass.table_name)
|
|
236
|
+
return yield if sides.empty?
|
|
237
|
+
|
|
238
|
+
columns = sides.flat_map { columns(_1) }.uniq
|
|
239
|
+
Lyra::BypassEvents.atomically(sides.first.left_model) do
|
|
240
|
+
rows = relation.pluck(*columns).map { |values| columns.zip(Array(values)).to_h }
|
|
241
|
+
result = yield
|
|
242
|
+
publish_sides(sides, rows, UNLINKED)
|
|
243
|
+
result
|
|
244
|
+
end
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
# Around the application's insert_all, insert_all! or upsert_all on a
|
|
248
|
+
# join model, or on a model it declared on a join table
|
|
249
|
+
# (StrictDataAccessRelation#join_insert; the block runs the write,
|
|
250
|
+
# given the RETURNING to ask for). The links the rows written make
|
|
251
|
+
# are recorded as Rails' own link from each side records them, as
|
|
252
|
+
# delete_all and delete_all_on_table record unlinks: a join model's
|
|
253
|
+
# rows link its owners (its side, and the other owners a row names:
|
|
254
|
+
# other_sides); an application model's rows
|
|
255
|
+
# link every monitored owner they name, of every side whose links are
|
|
256
|
+
# recorded (recorded_sides), each on its stream, linked into the
|
|
257
|
+
# associated records' streams that have none of their own. The rows
|
|
258
|
+
# written are read back by RETURNING (with the columns each side
|
|
259
|
+
# needs: Lyra::BypassEvents.with_returning), the caller getting the
|
|
260
|
+
# result it asked for: the values the association's scope added (an
|
|
261
|
+
# owner's item_tags.insert_all(...) sets its key) are in the event,
|
|
262
|
+
# and a row ON CONFLICT DO NOTHING skipped records nothing.
|
|
263
|
+
#
|
|
264
|
+
# An upsert_all (+conflict+: the columns of its ON CONFLICT) reads the
|
|
265
|
+
# rows its keys name first: a row it updated whose pair a side reads
|
|
266
|
+
# is unchanged records nothing; one whose pair changed is that side's
|
|
267
|
+
# unlink of the old pair and link of the new.
|
|
268
|
+
def insert_all(relation, attributes, returning:, conflict: nil)
|
|
269
|
+
klass = relation.klass
|
|
270
|
+
sides = join_model?(klass) ? [klass].select { recorded?(_1) } + other_sides(klass) : recorded_sides(klass.table_name)
|
|
271
|
+
return yield(returning) if sides.empty?
|
|
272
|
+
|
|
273
|
+
columns = sides.flat_map { columns(_1) }.uniq
|
|
274
|
+
keys = Array(conflict)
|
|
275
|
+
Lyra::BypassEvents.atomically(sides.first.left_model) do
|
|
276
|
+
before = keys.empty? ? {} : rows_by_key(relation, attributes, keys, columns)
|
|
277
|
+
result, written = Lyra::BypassEvents.with_returning(klass, returning, (keys + columns).uniq) { |ret| yield ret }
|
|
278
|
+
publish_inserts(sides, Array(written), before, keys)
|
|
66
279
|
result
|
|
67
280
|
end
|
|
68
281
|
end
|
|
@@ -85,7 +298,7 @@ module Lyra
|
|
|
85
298
|
# Remove the rows of +model_class+'s join tables (before the model's
|
|
86
299
|
# table is cleared, which their foreign keys may reference).
|
|
87
300
|
def clear(model_class)
|
|
88
|
-
join_models(model_class).each_value { |join_model| Lyra.projection_write { join_model.unscoped.delete_all } }
|
|
301
|
+
join_models(model_class).each_value { |join_model| Lyra.projection_write(INTERNAL) { join_model.unscoped.delete_all } }
|
|
89
302
|
end
|
|
90
303
|
|
|
91
304
|
# Bring +model_class+'s join tables to what the link events say, in
|
|
@@ -119,8 +332,138 @@ module Lyra
|
|
|
119
332
|
|
|
120
333
|
private
|
|
121
334
|
|
|
335
|
+
# What join_model_of and other_sides found, until the monitored
|
|
336
|
+
# models change or a has_and_belongs_to_many is declared.
|
|
337
|
+
def join_cache
|
|
338
|
+
config = Lyra.config
|
|
339
|
+
cache = @join_models_of
|
|
340
|
+
return cache if cache && cache[:config].equal?(config) && cache[:count] == config.monitored_models.size &&
|
|
341
|
+
cache[:declarations] == @declarations
|
|
342
|
+
|
|
343
|
+
@join_models_of = { config: config, count: config.monitored_models.size, declarations: @declarations,
|
|
344
|
+
models: Concurrent::Map.new, others: Concurrent::Map.new }
|
|
345
|
+
end
|
|
346
|
+
|
|
347
|
+
# The join model of a monitored model's has_and_belongs_to_many on
|
|
348
|
+
# +table+, or nil.
|
|
349
|
+
def find_join_model(table) = join_models_on(table).first
|
|
350
|
+
|
|
351
|
+
# The join models of the has_and_belongs_to_many on +table+ whose
|
|
352
|
+
# links Lyra records, each once (an STI subclass inherits its
|
|
353
|
+
# parent's): the monitored models' first, then those of unmonitored
|
|
354
|
+
# owners whose associated records' model is monitored
|
|
355
|
+
# (declared_join_models).
|
|
356
|
+
def join_models_on(table)
|
|
357
|
+
(Lyra.config.monitored_models.flat_map { join_models(_1).values } + declared_join_models)
|
|
358
|
+
.select { _1.table_name == table }.uniq
|
|
359
|
+
end
|
|
360
|
+
|
|
361
|
+
# The join models of the has_and_belongs_to_many declared on models
|
|
362
|
+
# Lyra does not monitor, linking to records of a model it does
|
|
363
|
+
# (declared!). A class since reloaded or removed is left out.
|
|
364
|
+
def declared_join_models
|
|
365
|
+
return [] unless @declared
|
|
366
|
+
|
|
367
|
+
monitored = Lyra.config.monitored_models
|
|
368
|
+
@declared.values.select do |join_model|
|
|
369
|
+
owner = join_model.left_model
|
|
370
|
+
next false unless owner.name && owner.name.safe_constantize.equal?(owner)
|
|
371
|
+
next false if monitored.any? { owner <= _1 }
|
|
372
|
+
|
|
373
|
+
target = join_model.right_reflection.klass
|
|
374
|
+
monitored.any? { target <= _1 }
|
|
375
|
+
rescue NameError
|
|
376
|
+
false
|
|
377
|
+
end
|
|
378
|
+
end
|
|
379
|
+
|
|
380
|
+
# The sides of +table+ whose unlinks are recorded now (recorded?).
|
|
381
|
+
def recorded_sides(table) = join_models_on(table).select { recorded?(_1) }
|
|
382
|
+
|
|
383
|
+
# Each side's links or unlinks (+operation+) of +rows+ (hashes by
|
|
384
|
+
# column; or a callable giving a side's rows), as publish records
|
|
385
|
+
# them, then linked into the associated records' streams that have
|
|
386
|
+
# none of their own for those rows.
|
|
387
|
+
def publish_sides(sides, rows, operation)
|
|
388
|
+
rows_of = rows.respond_to?(:call) ? rows : ->(_) { rows }
|
|
389
|
+
covered = Set.new
|
|
390
|
+
events = sides.flat_map do |join_model|
|
|
391
|
+
left, right = columns(join_model)
|
|
392
|
+
rows_of.call(join_model).select { _1[left] }.group_by { _1[left] }.map do |owner_id, owned|
|
|
393
|
+
pairs = owned.map { |row| { left => row[left], right => row[right] } }.uniq
|
|
394
|
+
event = build_event(join_model, operation, owner_id, pairs, name: association_name(join_model))
|
|
395
|
+
if owner_recorded?(join_model)
|
|
396
|
+
Lyra.append_events(event, stream_name: Lyra.stream_name(join_model.left_model, owner_id))
|
|
397
|
+
covered << [Lyra.stream_name(join_model.left_model, owner_id), left]
|
|
398
|
+
end
|
|
399
|
+
[event, join_model, owner_id, pairs]
|
|
400
|
+
end
|
|
401
|
+
end
|
|
402
|
+
events.each do |event, join_model, owner_id, pairs|
|
|
403
|
+
link_into_targets(event, join_model, owner_id, pairs, covered: covered,
|
|
404
|
+
append_first: !owner_recorded?(join_model))
|
|
405
|
+
end
|
|
406
|
+
rescue => e
|
|
407
|
+
Lyra::BypassEvents.publish_failed!(e, "has_and_belongs_to_many #{operation}")
|
|
408
|
+
end
|
|
409
|
+
|
|
410
|
+
# The rows an upsert's +keys+ name, before it (the attributes given,
|
|
411
|
+
# with the values the relation's scope adds), as hashes of +keys+ and
|
|
412
|
+
# +columns+, by their keys' values. Lyra's own read.
|
|
413
|
+
def rows_by_key(relation, attributes, keys, columns)
|
|
414
|
+
klass = relation.klass
|
|
415
|
+
scope = relation.scope_for_create.transform_keys(&:to_s)
|
|
416
|
+
types = keys.map { klass.type_for_attribute(_1) }
|
|
417
|
+
tuples = Array(attributes).filter_map do |row|
|
|
418
|
+
row = scope.merge(row.to_h.transform_keys(&:to_s))
|
|
419
|
+
next unless keys.all? { row.key?(_1) && !row[_1].nil? }
|
|
420
|
+
|
|
421
|
+
keys.each_with_index.map { |key, i| types[i].cast(row[key]) }
|
|
422
|
+
end.uniq
|
|
423
|
+
return {} if tuples.empty?
|
|
424
|
+
|
|
425
|
+
wanted = tuples.to_set
|
|
426
|
+
read = (keys + columns).uniq
|
|
427
|
+
found = Lyra::PurposeBoundReads.internal(INTERNAL) do
|
|
428
|
+
klass.unscoped.where(keys.first => tuples.map(&:first).uniq).pluck(*read)
|
|
429
|
+
end
|
|
430
|
+
found.each_with_object({}) do |values, acc|
|
|
431
|
+
row = read.zip(Array(values)).to_h
|
|
432
|
+
tuple = row.values_at(*keys)
|
|
433
|
+
acc[tuple] = row if wanted.include?(tuple)
|
|
434
|
+
end
|
|
435
|
+
end
|
|
436
|
+
|
|
437
|
+
# The links and unlinks the rows an insert or upsert wrote make
|
|
438
|
+
# (insert_all): every row an insert wrote, and every row an upsert
|
|
439
|
+
# inserted, links its pair; a row an upsert updated (found in
|
|
440
|
+
# +before+ by its +keys+) unlinks and links only where the side's
|
|
441
|
+
# pair changed.
|
|
442
|
+
def publish_inserts(sides, written, before, keys)
|
|
443
|
+
return publish_sides(sides, written, LINKED) if before.empty?
|
|
444
|
+
|
|
445
|
+
old_of = ->(row) { before[row.values_at(*keys)] }
|
|
446
|
+
changed = lambda do |join_model, row|
|
|
447
|
+
old = old_of.(row)
|
|
448
|
+
old && old.values_at(*columns(join_model)) != row.values_at(*columns(join_model))
|
|
449
|
+
end
|
|
450
|
+
publish_sides(sides, ->(side) { written.select { changed.(side, _1) }.map(&old_of) }, UNLINKED)
|
|
451
|
+
publish_sides(sides, ->(side) { written.select { old_of.(_1).nil? || changed.(side, _1) } }, LINKED)
|
|
452
|
+
end
|
|
453
|
+
|
|
454
|
+
# Whether +join_model+'s links are recorded: on its owner's stream
|
|
455
|
+
# when the owner's model is monitored, else on the associated
|
|
456
|
+
# records' streams when theirs is (owner_recorded?, targets_recorded?).
|
|
122
457
|
def recorded?(join_model)
|
|
123
|
-
join_model?(join_model) &&
|
|
458
|
+
join_model?(join_model) && (owner_recorded?(join_model) || targets_recorded?(join_model))
|
|
459
|
+
end
|
|
460
|
+
|
|
461
|
+
def owner_recorded?(join_model) = Lyra::BypassEvents.enabled_for?(join_model.left_model)
|
|
462
|
+
|
|
463
|
+
def targets_recorded?(join_model)
|
|
464
|
+
Lyra::BypassEvents.enabled_for?(join_model.right_reflection.klass)
|
|
465
|
+
rescue NameError
|
|
466
|
+
false
|
|
124
467
|
end
|
|
125
468
|
|
|
126
469
|
# The join model of each of +model_class+'s HABTM associations, by name.
|
|
@@ -150,11 +493,21 @@ module Lyra
|
|
|
150
493
|
[reflection.foreign_key.to_s, reflection.association_foreign_key.to_s]
|
|
151
494
|
end
|
|
152
495
|
|
|
496
|
+
# The event on the owner's stream, linked into each associated
|
|
497
|
+
# record's; when the owner's model is not monitored, on the
|
|
498
|
+
# associated records' streams only (the first one's, linked into the
|
|
499
|
+
# others'), as a link made from the other side would be seen there.
|
|
500
|
+
# Before, a link or unlink made from a side Lyra did not monitor was
|
|
501
|
+
# not recorded at all, though the other side's records were.
|
|
153
502
|
def publish(join_model, operation, owner_id, rows)
|
|
154
503
|
owner = join_model.left_model
|
|
155
504
|
event = build_event(join_model, operation, owner_id, rows, name: association_name(join_model))
|
|
156
|
-
|
|
157
|
-
|
|
505
|
+
if owner_recorded?(join_model)
|
|
506
|
+
Lyra.append_events(event, stream_name: Lyra.stream_name(owner, owner_id))
|
|
507
|
+
link_into_targets(event, join_model, owner_id, rows)
|
|
508
|
+
else
|
|
509
|
+
link_into_targets(event, join_model, owner_id, rows, append_first: true)
|
|
510
|
+
end
|
|
158
511
|
rescue => e
|
|
159
512
|
Lyra::BypassEvents.publish_failed!(e, "has_and_belongs_to_many #{operation}")
|
|
160
513
|
end
|
|
@@ -183,7 +536,11 @@ module Lyra
|
|
|
183
536
|
# monitored and the stream exists. An empty stream is left alone: a
|
|
184
537
|
# stream that held only links would look imported to Genesis, and the
|
|
185
538
|
# record would never get its Imported event.
|
|
186
|
-
|
|
539
|
+
# +covered+: [stream, column] pairs whose stream has an event of its
|
|
540
|
+
# own for the rows naming it in that column (publish_sides).
|
|
541
|
+
# +append_first+: the event is in no stream yet (its owner's model is
|
|
542
|
+
# not monitored); it is appended to the first stream, then linked.
|
|
543
|
+
def link_into_targets(event, join_model, owner_id, rows, covered: nil, append_first: false)
|
|
187
544
|
target = join_model.right_reflection.klass
|
|
188
545
|
return unless target.respond_to?(:lyra_monitored) && target.lyra_monitored
|
|
189
546
|
|
|
@@ -194,9 +551,15 @@ module Lyra
|
|
|
194
551
|
rows.map { |row| row[right] }.compact.uniq.each do |target_id|
|
|
195
552
|
stream = Lyra.stream_name(target, target_id)
|
|
196
553
|
next if stream == own_stream
|
|
554
|
+
next if covered&.include?([stream, right])
|
|
197
555
|
next unless store.read.stream(stream).limit(1).to_a.any?
|
|
198
556
|
|
|
199
|
-
|
|
557
|
+
if append_first
|
|
558
|
+
Lyra.append_events(event, stream_name: stream)
|
|
559
|
+
append_first = false
|
|
560
|
+
else
|
|
561
|
+
store.link([event.event_id], stream_name: stream)
|
|
562
|
+
end
|
|
200
563
|
end
|
|
201
564
|
end
|
|
202
565
|
|
|
@@ -229,7 +592,7 @@ module Lyra
|
|
|
229
592
|
def apply(join_model, event)
|
|
230
593
|
left, right = columns(join_model)
|
|
231
594
|
rows = Array(value(event.data, :rows)).map { |row| row.to_h.transform_keys(&:to_s).slice(left, right) }
|
|
232
|
-
Lyra.projection_write do
|
|
595
|
+
Lyra.projection_write(INTERNAL) do
|
|
233
596
|
rows.each do |row|
|
|
234
597
|
existing = join_model.unscoped.where(row)
|
|
235
598
|
if value(event.data, :operation).to_s == LINKED
|