orfeas_lyra 0.9.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +2021 -17
  3. data/README.md +22 -9
  4. data/Rakefile +40 -1
  5. data/app/controllers/lyra/application_controller.rb +127 -0
  6. data/app/controllers/lyra/dashboard_controller.rb +33 -31
  7. data/app/controllers/lyra/flow_controller.rb +32 -34
  8. data/app/controllers/lyra/privacy_controller.rb +57 -18
  9. data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
  10. data/app/views/lyra/dashboard/index.html.erb +1 -0
  11. data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
  12. data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
  13. data/app/views/lyra/privacy/policy.html.erb +7 -0
  14. data/app/views/lyra/privacy/rights.html.erb +78 -0
  15. data/config/privacy_policies.rb +0 -3
  16. data/config/routes.rb +1 -0
  17. data/lib/lyra/access_log.rb +187 -6
  18. data/lib/lyra/advisory_lock.rb +236 -0
  19. data/lib/lyra/aggregate.rb +1 -1
  20. data/lib/lyra/associations/habtm_links.rb +382 -19
  21. data/lib/lyra/associations/maintenance.rb +229 -0
  22. data/lib/lyra/attribution.rb +48 -0
  23. data/lib/lyra/bypass_events.rb +446 -46
  24. data/lib/lyra/command_handler.rb +19 -14
  25. data/lib/lyra/configuration.rb +237 -10
  26. data/lib/lyra/consistency/read_your_writes.rb +4 -2
  27. data/lib/lyra/correlation.rb +121 -0
  28. data/lib/lyra/domain_events.rb +4 -0
  29. data/lib/lyra/dual_view.rb +53 -16
  30. data/lib/lyra/encrypted_attributes.rb +5 -0
  31. data/lib/lyra/engine.rb +67 -0
  32. data/lib/lyra/erasure.rb +1011 -106
  33. data/lib/lyra/event.rb +35 -1
  34. data/lib/lyra/event_data.rb +38 -0
  35. data/lib/lyra/event_flow.rb +31 -15
  36. data/lib/lyra/event_store_adapter.rb +28 -0
  37. data/lib/lyra/event_window.rb +58 -0
  38. data/lib/lyra/genesis.rb +548 -44
  39. data/lib/lyra/interceptors/association_interceptor.rb +689 -34
  40. data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
  41. data/lib/lyra/internal.rb +26 -0
  42. data/lib/lyra/mode_sync.rb +51 -3
  43. data/lib/lyra/mode_transition.rb +257 -30
  44. data/lib/lyra/model_selection.rb +35 -0
  45. data/lib/lyra/own_connection.rb +17 -2
  46. data/lib/lyra/privacy/adapters/pam.rb +25 -2
  47. data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
  48. data/lib/lyra/privacy/interface.rb +27 -7
  49. data/lib/lyra/privacy/subject_links.rb +143 -0
  50. data/lib/lyra/privacy/subject_records.rb +369 -0
  51. data/lib/lyra/projection.rb +7 -1
  52. data/lib/lyra/projections/async_checkpoint.rb +32 -0
  53. data/lib/lyra/projections/async_projection_job.rb +166 -23
  54. data/lib/lyra/projections/cached_joins.rb +33 -18
  55. data/lib/lyra/projections/cached_projection.rb +74 -33
  56. data/lib/lyra/projections/cached_relation.rb +3175 -321
  57. data/lib/lyra/projections/default_scopes.rb +254 -0
  58. data/lib/lyra/projections/deferred_relation.rb +23 -3
  59. data/lib/lyra/projections/event_store_reader.rb +47 -8
  60. data/lib/lyra/projections/foreign_keys.rb +92 -0
  61. data/lib/lyra/projections/lazy_projection.rb +165 -27
  62. data/lib/lyra/projections/model_projection.rb +165 -14
  63. data/lib/lyra/projections/rebuild.rb +34 -6
  64. data/lib/lyra/projections/reference_links.rb +463 -72
  65. data/lib/lyra/projections/typed_values.rb +629 -0
  66. data/lib/lyra/projections/unique_indexes.rb +595 -0
  67. data/lib/lyra/projections/uniqueness.rb +784 -0
  68. data/lib/lyra/projections/value_digest.rb +93 -0
  69. data/lib/lyra/purpose_bound_reads.rb +912 -77
  70. data/lib/lyra/repair.rb +119 -28
  71. data/lib/lyra/retention.rb +114 -8
  72. data/lib/lyra/rights/consent_store.rb +415 -0
  73. data/lib/lyra/rights/erasure_guard.rb +135 -0
  74. data/lib/lyra/rights/pending.rb +74 -0
  75. data/lib/lyra/rights/queries.rb +269 -0
  76. data/lib/lyra/rights/store.rb +512 -0
  77. data/lib/lyra/rights/subject_keys.rb +224 -0
  78. data/lib/lyra/rights.rb +281 -0
  79. data/lib/lyra/saves_without_effect.rb +69 -0
  80. data/lib/lyra/stream_tail.rb +319 -0
  81. data/lib/lyra/strict_data_access.rb +916 -106
  82. data/lib/lyra/temporal.rb +4 -2
  83. data/lib/lyra/touch_registry.rb +404 -0
  84. data/lib/lyra/verification/bypass_workflow.rb +452 -42
  85. data/lib/lyra/verification/rights_workflows.rb +170 -0
  86. data/lib/lyra/version.rb +1 -1
  87. data/lib/lyra/visualization/event_graph.rb +1 -1
  88. data/lib/lyra/visualization/timeline.rb +2 -2
  89. data/lib/lyra.rb +43 -2
  90. data/lib/tasks/lyra_mode.rake +110 -8
  91. data/lib/tasks/lyra_projections.rake +43 -2
  92. metadata +34 -4
  93. data/lib/lyra/associations/event_aware.rb +0 -224
@@ -0,0 +1,319 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Lyra
4
+ # Whether a record's stream says it was destroyed: the question
5
+ # CrudInterceptor#lyra_lock_aggregate_unless_destroyed! asks, under the lock
6
+ # on the aggregate, before every update and destroy that the record's row
7
+ # lock does not cover (ES-NoProj, ES-Async, ES-Lazy).
8
+ #
9
+ # The record was destroyed when the stream's last replayed event (the last
10
+ # one Lyra::Event.operation_of gives an operation) is a destroy. Events that
11
+ # are not replayed can follow a destroy: an additional domain event
12
+ # (also:, replay: false), an ErasureApplied, a has_and_belongs_to_many link
13
+ # (AssociationLinked) linked into the stream. Before, the check read only
14
+ # the last event, through Rails Event Store, and let a stale copy write the
15
+ # record again after any of them.
16
+ #
17
+ # The operation is recorded in the event's data, not in its type: a
18
+ # destroy can be stored under any name (custom event names, domain events).
19
+ # So the type alone cannot answer. The stream's events are read backward
20
+ # with a query of their id, type and stored data, deserializing nothing,
21
+ # and skipping in SQL the types Lyra never replays (NEVER_REPLAYED: links
22
+ # and erasure records, which a popular record's stream can hold by the
23
+ # hundred). The first event read decides at once when it is positively a
24
+ # replayed create, import or update:
25
+ #
26
+ # - its type ends in Created, Updated or Imported, so operation_of gives it
27
+ # an operation (the recorded one, else the name's);
28
+ # - its data holds no destroy's operation ("operation" followed by destroy
29
+ # or deleted, or by a YAML tag, anchor or alias), so the recorded one is
30
+ # not a destroy;
31
+ # - its data holds no "replay" followed by false.
32
+ #
33
+ # That is the hot path, one query: an update after a create or an update.
34
+ # Data stored in a json or jsonb column comes back as a Hash, and
35
+ # operation_of decides from it directly. Any other event is read through
36
+ # Rails Event Store, by its id, and operation_of decides; one that is not
37
+ # replayed (an additional domain event) sends the check to the event
38
+ # before it. The query reads one event, then pages of PAGE.
39
+ #
40
+ # The shortcut needs RubyEventStore::ActiveRecord::EventRepository (whose
41
+ # models give the tables and the connection), RES's default mapper (one
42
+ # that transforms records on load could change the type or the data that
43
+ # operation_of sees) and a serializer that writes the operation as text
44
+ # next to its key: YAML, JSON or Lyra::EventSerializer. The query is tried
45
+ # once, in the first check, in a savepoint where a transaction is open, so
46
+ # that a failure cannot abort the write. With any other repository, mapper
47
+ # or serializer, or a query that fails (a Rails Event Store whose internals
48
+ # or tables changed), every check reads the stream through Rails Event
49
+ # Store.
50
+ module StreamTail
51
+ # A possible destroy's recorded operation, as YAML (":operation:
52
+ # :destroyed", "operation: !ruby/string:... destroyed", "operation: &1
53
+ # ...", "operation: *1"), JSON ('"operation":"destroyed"') and
54
+ # Lyra::EventSerializer write it: every value Lyra::Event.recorded_operation
55
+ # maps to :destroyed contains "destroy" or "deleted". A match elsewhere in
56
+ # the data (a column named operation, say) only sends the check to the
57
+ # full read.
58
+ MAY_BE_DESTROY = /operation[^0-9A-Za-z!&*]{1,8}(?:destroy|deleted|[!&*])/
59
+ # An additional domain event, never replayed (Lyra::DomainEvents).
60
+ NOT_REPLAYED = /replay[^0-9A-Za-z]{1,8}false/
61
+ # Types operation_of always gives an operation, when their data records
62
+ # none (Lyra::Event.operation_from_name), Destroyed apart.
63
+ REPLAYED_TYPE = /(?:Created|Updated|Imported)\z/
64
+ # Lyra's own events that are never replayed: their data records no
65
+ # operation Lyra::Event.recorded_operation knows ("linked", "unlinked",
66
+ # or none) and their names end in none of operation_from_name's
67
+ # suffixes. No model may store a write under these names
68
+ # (Lyra::DomainEvents, ModelConfig's event_mapping refuse them).
69
+ NEVER_REPLAYED = %w[
70
+ Lyra::Events::AssociationLinked Lyra::Events::AssociationUnlinked Lyra::Events::ErasureApplied
71
+ ].freeze
72
+ # Events read per page after the first.
73
+ PAGE = 50
74
+
75
+ # Where the events are read from.
76
+ Source = Struct.new(:stream_klass, :select_from)
77
+ # What operation_of needs of an event: its type and its data.
78
+ Tail = Struct.new(:event_type, :data)
79
+
80
+ CONNECTION_ERRORS = [
81
+ ActiveRecord::ConnectionNotEstablished, ActiveRecord::ConnectionTimeoutError,
82
+ (ActiveRecord::ConnectionFailed if defined?(ActiveRecord::ConnectionFailed))
83
+ ].compact.freeze
84
+ RESOLVING = Mutex.new
85
+
86
+ class << self
87
+ # True when the last replayed event of +stream_name+ is a destroy. False
88
+ # for an empty stream, or one with no replayed event.
89
+ def destroyed?(stream_name, store = Lyra.config.event_store)
90
+ source = source_for(store)
91
+ return destroyed_by_reading?(stream_name, store) unless source
92
+
93
+ before = nil
94
+ limit = 1
95
+ loop do
96
+ rows = event_rows(source, stream_name, before: before, limit: limit)
97
+ rows.each do |_, event_id, type, data|
98
+ operation = operation_of_row(store, event_id, type, data)
99
+ return operation == :destroyed if operation
100
+ end
101
+ return false if rows.size < limit
102
+
103
+ before = rows.last.first
104
+ limit = PAGE
105
+ end
106
+ end
107
+
108
+ # Of +stream_names+, those whose last replayed event is a destroy, as
109
+ # destroyed? answers for each. On PostgreSQL with the shortcut's source,
110
+ # the last event of every stream is read in one query (per BATCH
111
+ # streams); a stream whose last event does not decide on its own (not
112
+ # positively a replayed create, import or update, nor data that
113
+ # operation_of decides) is asked with destroyed? as before. Elsewhere,
114
+ # destroyed? for each.
115
+ def destroyed_streams(stream_names, store = Lyra.config.event_store)
116
+ names = stream_names.uniq
117
+ return [] if names.empty?
118
+
119
+ source = source_for(store)
120
+ batched = source && source.stream_klass.with_connection { _1.adapter_name.match?(/postgres/i) }
121
+ return names.select { destroyed?(_1, store) } unless batched
122
+
123
+ destroyed = []
124
+ names.each_slice(BATCH) do |slice|
125
+ last = last_rows(source, slice)
126
+ slice.each do |name|
127
+ row = last[name]
128
+ next unless row # no replayed-kind event: not destroyed, as destroyed? answers
129
+
130
+ _, event_id, type, data = row
131
+ operation = data.is_a?(Hash) ? Lyra::Event.operation_of(Tail.new(type, data)) : nil
132
+ operation ||= :updated if replayed_non_destroy?(type, data)
133
+ decided = operation ? operation == :destroyed : destroyed?(name, store)
134
+ destroyed << name if decided
135
+ end
136
+ end
137
+ destroyed
138
+ end
139
+
140
+ # Streams read per query by destroyed_streams.
141
+ BATCH = 500
142
+
143
+ # { stream => [link id, event_id, event_type, stored data] } of the
144
+ # last event of each of +names+ that has one, the NEVER_REPLAYED types
145
+ # left out.
146
+ def last_rows(source, names)
147
+ source.stream_klass.with_connection do |connection|
148
+ list = names.map { connection.quote(_1) }.join(", ")
149
+ sql = source.select_from.sub(/\ASELECT /, "SELECT DISTINCT ON (s.stream) s.stream, ") +
150
+ " WHERE s.stream IN (#{list}) ORDER BY s.stream, s.id DESC"
151
+ rows = connection.uncached { connection.select_all(sql, "Lyra stream tail") }.cast_values
152
+ rows.to_h { |stream, *rest| [stream, rest] }
153
+ end
154
+ end
155
+ private :last_rows
156
+
157
+ # The answer read through Rails Event Store: the stream backward, from
158
+ # its last event, to the first replayed event.
159
+ def destroyed_by_reading?(stream_name, store = Lyra.config.event_store)
160
+ before = nil
161
+ limit = 1
162
+ loop do
163
+ spec = store.read.stream(stream_name).backward.limit(limit)
164
+ spec = spec.from(before) if before
165
+ events = spec.to_a
166
+ events.each do |event|
167
+ operation = Lyra::Event.operation_of(event)
168
+ return operation == :destroyed if operation
169
+ end
170
+ return false if events.size < limit
171
+
172
+ before = events.last.event_id
173
+ limit = PAGE
174
+ end
175
+ end
176
+
177
+ # [[link id, event_id, event_type, stored data], ...] for the stream's
178
+ # events before link +before+ (all, if nil), newest first, the
179
+ # NEVER_REPLAYED types left out. Newest is what Rails Event Store's
180
+ # backward read returns first: the link with the highest id (Lyra
181
+ # appends with expected_version :any, which records no position).
182
+ def event_rows(source, stream_name, before: nil, limit: 1)
183
+ source.stream_klass.with_connection do |connection|
184
+ sql = +"#{source.select_from} WHERE s.stream = #{connection.quote(stream_name)}"
185
+ sql << " AND s.id < #{Integer(before)}" if before
186
+ sql << " ORDER BY s.id DESC LIMIT #{Integer(limit)}"
187
+ # Read under the lock the caller holds, never from the query cache.
188
+ connection.uncached { connection.select_all(sql, "Lyra stream tail") }.cast_values
189
+ end
190
+ end
191
+
192
+ # The operation the event has when replayed, or nil (not replayed).
193
+ def operation_of_row(store, event_id, type, data)
194
+ return Lyra::Event.operation_of(Tail.new(type, data)) if data.is_a?(Hash)
195
+ # Positively replayed, and not a destroy: which one does not matter.
196
+ return :updated if replayed_non_destroy?(type, data)
197
+
198
+ Lyra::Event.operation_of(store.read.event!(event_id))
199
+ end
200
+
201
+ # True when the stored event is positively a replayed create, import
202
+ # or update (see the module comment).
203
+ def replayed_non_destroy?(type, data)
204
+ return false unless data.is_a?(String) && REPLAYED_TYPE.match?(type)
205
+
206
+ data = data.b unless data.encoding == Encoding::BINARY
207
+ !MAY_BE_DESTROY.match?(data) && !NOT_REPLAYED.match?(data)
208
+ end
209
+
210
+ # Whether a write may be stored under +event_name+ (a generated
211
+ # Lyra::Events name, or a class's name): not under one of
212
+ # NEVER_REPLAYED's.
213
+ def reserved_name?(event_name)
214
+ name = event_name.to_s
215
+ NEVER_REPLAYED.include?(name) || NEVER_REPLAYED.include?("Lyra::Events::#{name.gsub('::', '')}")
216
+ end
217
+
218
+ # The Source for +store+, or nil when the shortcut does not apply.
219
+ # Resolved once per event store, by one thread at a time. A store that
220
+ # could not be resolved for want of a connection is tried again on the
221
+ # next check; any other failure is final for that store, and logged
222
+ # once.
223
+ def source_for(store)
224
+ cached = @source
225
+ return cached.last if cached && cached.first.equal?(store)
226
+
227
+ RESOLVING.synchronize do
228
+ cached = @source
229
+ next cached.last if cached && cached.first.equal?(store)
230
+
231
+ source = resolve(store)
232
+ @source = [store, source]
233
+ source
234
+ rescue *CONNECTION_ERRORS => e
235
+ warn_once(:connection, e)
236
+ nil
237
+ rescue StandardError => e
238
+ warn_once(:resolve, e)
239
+ @source = [store, nil]
240
+ nil
241
+ end
242
+ end
243
+
244
+ def reset!
245
+ @source = nil
246
+ @warned = nil
247
+ end
248
+
249
+ private
250
+
251
+ def resolve(store)
252
+ repository = active_record_repository(store)
253
+ return nil unless repository && default_mapper?(store) &&
254
+ literal_serializer?(repository.instance_variable_get(:@serializer))
255
+
256
+ # The repository's own models: their tables and their connection,
257
+ # which is the write's (the default models inherit ActiveRecord::Base).
258
+ event_klass = repository.send(:event_klass)
259
+ stream_klass = repository.send(:stream_klass)
260
+ select_from = stream_klass.with_connection do |connection|
261
+ select_from(connection, stream_klass, event_klass).tap { probe!(connection, _1) }
262
+ end
263
+ Source.new(stream_klass, select_from.freeze)
264
+ end
265
+
266
+ def select_from(connection, stream_klass, event_klass)
267
+ skipped = NEVER_REPLAYED.map { connection.quote(_1) }.join(", ")
268
+ "SELECT s.id, e.event_id, e.event_type, e.data FROM #{connection.quote_table_name(stream_klass.table_name)} s " \
269
+ "JOIN #{connection.quote_table_name(event_klass.table_name)} e ON e.event_id = s.event_id " \
270
+ "AND e.event_type NOT IN (#{skipped})"
271
+ end
272
+
273
+ # Run the query once on the write's connection, in a savepoint when a
274
+ # transaction is open, so that its failure cannot abort the write.
275
+ # Raises if the query fails.
276
+ def probe!(connection, select_from)
277
+ probe = -> { connection.select_all("#{select_from} WHERE s.stream = '' LIMIT 0", "Lyra stream tail probe") }
278
+ connection.transaction_open? ? connection.transaction(requires_new: true) { probe.call } : probe.call
279
+ end
280
+
281
+ # The ActiveRecord::EventRepository behind a Rails Event Store client
282
+ # (RailsEventStore::Client wraps it in an InstrumentedRepository), or nil.
283
+ def active_record_repository(store)
284
+ return nil unless defined?(RubyEventStore::ActiveRecord::EventRepository)
285
+
286
+ repository = store.instance_variable_get(:@repository)
287
+ while defined?(RubyEventStore::InstrumentedRepository) && repository.is_a?(RubyEventStore::InstrumentedRepository)
288
+ repository = repository.instance_variable_get(:@repository)
289
+ end
290
+ repository.is_a?(RubyEventStore::ActiveRecord::EventRepository) ? repository : nil
291
+ end
292
+
293
+ # True when the client's mapper is RES's default, under the wrappers
294
+ # Rails Event Store adds (InstrumentedBatchMapper, BatchMapper,
295
+ # InstrumentedMapper).
296
+ def default_mapper?(store)
297
+ mapper = store.instance_variable_get(:@mapper)
298
+ wrappers = [RubyEventStore::Mappers::InstrumentedBatchMapper, RubyEventStore::Mappers::BatchMapper,
299
+ RubyEventStore::Mappers::InstrumentedMapper]
300
+ mapper = mapper.instance_variable_get(:@mapper) while wrappers.any? { mapper.instance_of?(_1) }
301
+ mapper.instance_of?(RubyEventStore::Mappers::Default)
302
+ end
303
+
304
+ def literal_serializer?(serializer)
305
+ serializer.equal?(::JSON) || serializer.equal?(Lyra::EventSerializer) ||
306
+ serializer.equal?(RubyEventStore::Serializers::YAML)
307
+ end
308
+
309
+ def warn_once(kind, error)
310
+ @warned ||= {}
311
+ return if @warned[kind]
312
+
313
+ @warned[kind] = true
314
+ message = "Lyra: the destroyed check reads through Rails Event Store - #{error.class}: #{error.message}"
315
+ defined?(Rails) && Rails.logger ? Rails.logger.warn(message) : Kernel.warn(message)
316
+ end
317
+ end
318
+ end
319
+ end