orfeas_lyra 0.9.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +2021 -17
  3. data/README.md +22 -9
  4. data/Rakefile +40 -1
  5. data/app/controllers/lyra/application_controller.rb +127 -0
  6. data/app/controllers/lyra/dashboard_controller.rb +33 -31
  7. data/app/controllers/lyra/flow_controller.rb +32 -34
  8. data/app/controllers/lyra/privacy_controller.rb +57 -18
  9. data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
  10. data/app/views/lyra/dashboard/index.html.erb +1 -0
  11. data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
  12. data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
  13. data/app/views/lyra/privacy/policy.html.erb +7 -0
  14. data/app/views/lyra/privacy/rights.html.erb +78 -0
  15. data/config/privacy_policies.rb +0 -3
  16. data/config/routes.rb +1 -0
  17. data/lib/lyra/access_log.rb +187 -6
  18. data/lib/lyra/advisory_lock.rb +236 -0
  19. data/lib/lyra/aggregate.rb +1 -1
  20. data/lib/lyra/associations/habtm_links.rb +382 -19
  21. data/lib/lyra/associations/maintenance.rb +229 -0
  22. data/lib/lyra/attribution.rb +48 -0
  23. data/lib/lyra/bypass_events.rb +446 -46
  24. data/lib/lyra/command_handler.rb +19 -14
  25. data/lib/lyra/configuration.rb +237 -10
  26. data/lib/lyra/consistency/read_your_writes.rb +4 -2
  27. data/lib/lyra/correlation.rb +121 -0
  28. data/lib/lyra/domain_events.rb +4 -0
  29. data/lib/lyra/dual_view.rb +53 -16
  30. data/lib/lyra/encrypted_attributes.rb +5 -0
  31. data/lib/lyra/engine.rb +67 -0
  32. data/lib/lyra/erasure.rb +1011 -106
  33. data/lib/lyra/event.rb +35 -1
  34. data/lib/lyra/event_data.rb +38 -0
  35. data/lib/lyra/event_flow.rb +31 -15
  36. data/lib/lyra/event_store_adapter.rb +28 -0
  37. data/lib/lyra/event_window.rb +58 -0
  38. data/lib/lyra/genesis.rb +548 -44
  39. data/lib/lyra/interceptors/association_interceptor.rb +689 -34
  40. data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
  41. data/lib/lyra/internal.rb +26 -0
  42. data/lib/lyra/mode_sync.rb +51 -3
  43. data/lib/lyra/mode_transition.rb +257 -30
  44. data/lib/lyra/model_selection.rb +35 -0
  45. data/lib/lyra/own_connection.rb +17 -2
  46. data/lib/lyra/privacy/adapters/pam.rb +25 -2
  47. data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
  48. data/lib/lyra/privacy/interface.rb +27 -7
  49. data/lib/lyra/privacy/subject_links.rb +143 -0
  50. data/lib/lyra/privacy/subject_records.rb +369 -0
  51. data/lib/lyra/projection.rb +7 -1
  52. data/lib/lyra/projections/async_checkpoint.rb +32 -0
  53. data/lib/lyra/projections/async_projection_job.rb +166 -23
  54. data/lib/lyra/projections/cached_joins.rb +33 -18
  55. data/lib/lyra/projections/cached_projection.rb +74 -33
  56. data/lib/lyra/projections/cached_relation.rb +3175 -321
  57. data/lib/lyra/projections/default_scopes.rb +254 -0
  58. data/lib/lyra/projections/deferred_relation.rb +23 -3
  59. data/lib/lyra/projections/event_store_reader.rb +47 -8
  60. data/lib/lyra/projections/foreign_keys.rb +92 -0
  61. data/lib/lyra/projections/lazy_projection.rb +165 -27
  62. data/lib/lyra/projections/model_projection.rb +165 -14
  63. data/lib/lyra/projections/rebuild.rb +34 -6
  64. data/lib/lyra/projections/reference_links.rb +463 -72
  65. data/lib/lyra/projections/typed_values.rb +629 -0
  66. data/lib/lyra/projections/unique_indexes.rb +595 -0
  67. data/lib/lyra/projections/uniqueness.rb +784 -0
  68. data/lib/lyra/projections/value_digest.rb +93 -0
  69. data/lib/lyra/purpose_bound_reads.rb +912 -77
  70. data/lib/lyra/repair.rb +119 -28
  71. data/lib/lyra/retention.rb +114 -8
  72. data/lib/lyra/rights/consent_store.rb +415 -0
  73. data/lib/lyra/rights/erasure_guard.rb +135 -0
  74. data/lib/lyra/rights/pending.rb +74 -0
  75. data/lib/lyra/rights/queries.rb +269 -0
  76. data/lib/lyra/rights/store.rb +512 -0
  77. data/lib/lyra/rights/subject_keys.rb +224 -0
  78. data/lib/lyra/rights.rb +281 -0
  79. data/lib/lyra/saves_without_effect.rb +69 -0
  80. data/lib/lyra/stream_tail.rb +319 -0
  81. data/lib/lyra/strict_data_access.rb +916 -106
  82. data/lib/lyra/temporal.rb +4 -2
  83. data/lib/lyra/touch_registry.rb +404 -0
  84. data/lib/lyra/verification/bypass_workflow.rb +452 -42
  85. data/lib/lyra/verification/rights_workflows.rb +170 -0
  86. data/lib/lyra/version.rb +1 -1
  87. data/lib/lyra/visualization/event_graph.rb +1 -1
  88. data/lib/lyra/visualization/timeline.rb +2 -2
  89. data/lib/lyra.rb +43 -2
  90. data/lib/tasks/lyra_mode.rake +110 -8
  91. data/lib/tasks/lyra_projections.rake +43 -2
  92. metadata +34 -4
  93. data/lib/lyra/associations/event_aware.rb +0 -224
data/lib/lyra/erasure.rb CHANGED
@@ -14,7 +14,10 @@ module Lyra
14
14
  #
15
15
  # Personal fields are those the record's policy declares plus those the
16
16
  # events' privacy stamps (config.annotate_privacy) list, or exactly the
17
- # +fields:+ given. In the record's events a value is erased wherever it
17
+ # +fields:+ given, which must be among those unless +any_fields: true+
18
+ # (from 0.9.3: an erasure rewrites history in place, so naming any column,
19
+ # a payment's status, say, would rewrite non-personal history under an
20
+ # Art. 17 reason). In the record's events a value is erased wherever it
18
21
  # appears: under the attribute's own name (attributes, both sides of each
19
22
  # change, payload keys), and under any other name, matched by value (a
20
23
  # domain-event payload that copies the email as payer_email).
@@ -24,12 +27,50 @@ module Lyra
24
27
  # the rows of those records (monitored models) get the value replaced
25
28
  # where they hold it, with an ErasureApplied event in their streams, so
26
29
  # every stream still replays to its row. The search is a text match on the
27
- # stored events, then an exact match on each candidate's data.
30
+ # stored events, then an exact match on each candidate's data. From 1.0.0
31
+ # the tables of monitored models with a policy are searched too (not in
32
+ # ES-NoProj), so a row whose events lack the value is found; a row with no
33
+ # stream is erased and then imported (Genesis), so its first event never
34
+ # holds the value. Only the person's own values are searched for: a third
35
+ # party's value nested under a personal key in the person's data is erased
36
+ # there only. Holders are counted by record; the person's linked records
37
+ # (belongs_to the erased record) do not count toward max_copies
38
+ # (config.erasure_max_copies), and each value left as shared is reported
39
+ # by field and count (Result#shared). Lyra's own record streams
40
+ # (kept_stream?: rights and consent, access logs, exports) are never
41
+ # rewritten. A dry run (dry_run: true) reports all of it and writes nothing.
42
+ #
43
+ # Each row gets the rule its own events get: the record's row the key and
44
+ # value rule, another record's row the value rule only (a third party's
45
+ # email under an "email" key there is not this person's). Every column is
46
+ # scrubbed; from 1.0.0 a json, jsonb or array column, or a serialized
47
+ # attribute, is recursed into and written back whole. A text column that
48
+ # holds JSON is a string in its events, and is compared as one. A hash or
49
+ # array under a personal field's name is replaced whole. Encrypted
50
+ # attributes are compared in clear, in the events as in the row. A record
51
+ # whose row still holds a value (as a JSON key, inside a longer string) is
52
+ # reported in Result#left_behind, not in Result#copies.
28
53
  #
29
54
  # The replacement is nil where the column allows it, "erased:<id>" for a
30
55
  # string column that does not, and the column's default otherwise; a value
31
56
  # with no column of its own (a payload key) gets the origin field's.
32
57
  #
58
+ # The subject's access log (Lyra::AccessLog, the stream
59
+ # Lyra::DataAccess$<Model>$<id>) holds who read the record, when and under
60
+ # which purpose, never a value. It may be evidence the application must
61
+ # keep (Art. 5(2), a security investigation) or the subject's data to
62
+ # erase with the rest; Lyra cannot know which. From 1.0.0 erasure keeps it
63
+ # unless config.erase_access_log or +access_log: true+ says otherwise; then
64
+ # its events are deleted, from every stream they are in and from the
65
+ # events table (unlinking the stream alone would leave them). The
66
+ # configuration applies only once the record is fully erased: when this
67
+ # erasure, with any earlier ones, has covered every personal field. A
68
+ # field-level erasure (a retention period for one field) leaves a record
69
+ # whose other data is still processed, and its log with it. An explicit
70
+ # +access_log: true+ deletes the log whatever was erased. Either way
71
+ # ErasureApplied records access_log ("erased" or "kept") and
72
+ # access_log_entries (how many it held), so the decision is visible.
73
+ #
33
74
  # Crypto-shredding, which would leave the log untouched, remains future
34
75
  # work.
35
76
  module Erasure
@@ -38,7 +79,8 @@ module Lyra
38
79
  class Unsupported < StandardError; end
39
80
 
40
81
  # Other records holding a value, beyond which it is taken to be shared by
41
- # many people (a city, a placeholder) rather than a copy of this one's.
82
+ # many people (a city, a placeholder) rather than a copy of this one's: the
83
+ # default of config.erasure_max_copies (from 1.0.0).
42
84
  MAX_COPIES = 10
43
85
 
44
86
  # The PAM types searched for in other records: direct identifiers, whose
@@ -49,84 +91,372 @@ module Lyra
49
91
  IDENTIFYING_TYPES = %i[email phone identifier ssn credit_card financial payment_token ip_address
50
92
  credential token].freeze
51
93
 
94
+ # +left_behind+: the records whose row still holds an erased value after
95
+ # the erasure, as { record: "Model id", columns: [...] } (never the value):
96
+ # a value held as a JSON key, inside a longer string, or in a key column
97
+ # (a primary or foreign key is never rewritten). The erased record itself
98
+ # (its replayed state in ES-NoProj) and, with everywhere, the other
99
+ # records reached; those are not in +copies+. A record whose key holds an
100
+ # erased value is named "Model <key withheld>".
101
+ # +not_searched+: with everywhere, the encrypted columns of monitored
102
+ # models ("Model.column"), which the search of the log cannot read.
103
+ # +key_withheld+: the erased record's own key holds an erased value.
104
+ # +shared+ (from 1.0.0): with everywhere, each value left as shared, as
105
+ # { field:, type:, holders: } (the personal field it was found under, its
106
+ # PAM type and how many other records hold it), never the value;
107
+ # +shared_values+ is their count, an Integer as before.
108
+ # +rows_without_events+ (from 1.0.0): with everywhere, the records whose
109
+ # row held an erased value that their events did not (written by raw SQL,
110
+ # say), found by searching the tables directly and erased in the row only:
111
+ # their row already differed from their replay, and lyra:repair brings
112
+ # the log in line. A row with no stream at all is imported instead (its
113
+ # Imported event holds the erased values) and is not listed here.
114
+ # +lifting+ (from 1.0.0, with PAM DSL's rights): the ids of the
115
+ # restrictions of processing the erasure lifted (lifting:), or, in a dry
116
+ # run, would lift; [] when none.
117
+ # +restricted_copies+ (from 1.0.0, with everywhere and PAM DSL's rights):
118
+ # the other records holding a copy that a restriction of processing
119
+ # covers, as { record: "Model id", columns: [...] }, never the value. They
120
+ # are not modified: the copy stays until the holder's restriction is
121
+ # lifted, and an erasure then (everywhere again) takes it.
52
122
  Result = Struct.new(:model, :id, :fields, :events_rewritten, :row_erased, :copies, :shared_values,
53
- keyword_init: true)
123
+ :access_log, :access_log_entries, :left_behind, :not_searched, :key_withheld,
124
+ :shared, :rows_without_events, :dry_run, :lifting, :restricted_copies, keyword_init: true)
125
+
126
+ WITHHELD = "<key withheld>"
127
+
128
+ # Lyra's own records of how a person's rights were handled (kept_stream?):
129
+ # restrictions and objections (Lyra::Rights::STREAM), consent
130
+ # (Lyra::Rights::CONSENT_STREAM), the access log and its erasures, the
131
+ # subject-access exports, each an index stream and its per-subject
132
+ # streams ("<name>$<subject key>"). Evidence the controller keeps
133
+ # (Art. 5(2)); everywhere never rewrites their events nor counts them as
134
+ # holders, even when a subject key is an erased value. The record's own
135
+ # access log is erased, or kept, by access_log:.
136
+ def self.kept_streams
137
+ names = []
138
+ names += [Lyra::Rights::STREAM, Lyra::Rights::CONSENT_STREAM] if defined?(Lyra::Rights::STREAM)
139
+ names += [Lyra::AccessLog::STREAM, Lyra::AccessLog::ERASURES, Lyra::AccessLog::EXPORTS] if defined?(Lyra::AccessLog::STREAM)
140
+ names
141
+ end
142
+
143
+ def self.kept_stream?(name)
144
+ kept_streams.any? { |kept| name == kept || name.start_with?("#{kept}$") }
145
+ end
146
+
147
+ # The replacement's own form: an earlier erasure's marker is no one's value.
148
+ REPLACEMENT = /\Aerased:/
149
+
150
+ # An event's envelope entries that name the record, never scrubbed.
151
+ ENVELOPE_KEYS = %w[model_class model_id operation timestamp replay].freeze
54
152
 
55
153
  class << self
56
- def erase!(model, id, reason:, fields: nil, erased_by: nil, everywhere: false, max_copies: MAX_COPIES)
57
- stream = Lyra.stream_name(model, id)
58
- events = Lyra.config.event_store.read.stream(stream).to_a
59
- personal = (fields || personal_fields(model, events)).map(&:to_s).uniq
60
- raise Unsupported, "#{model.name} #{id}: no personal fields to erase" if personal.empty?
154
+ # +max_copies+: nil takes config.erasure_max_copies (10 by default); else a
155
+ # positive Integer (ArgumentError before anything is read or written).
156
+ # +dry_run+ (from 1.0.0): do everything in a transaction and roll it back,
157
+ # so the result says what the erasure would do (fields, events, copies,
158
+ # rows without events, values left as shared and their holders, what would
159
+ # be left behind) and nothing is written. A value left as shared cannot be
160
+ # found again once the person's own values are erased, so the threshold is
161
+ # chosen with a dry run first.
162
+ def erase!(model, id, reason:, fields: nil, erased_by: nil, everywhere: false, max_copies: nil,
163
+ any_fields: false, access_log: nil, access_log_subjects: [], dry_run: false, lifting: nil)
164
+ max_copies = Lyra.config.erasure_max_copies if max_copies.nil?
165
+ unless max_copies.is_a?(Integer) && max_copies.positive?
166
+ raise ArgumentError, "max_copies must be a positive Integer (got #{max_copies.inspect})"
167
+ end
168
+
169
+ # A restriction of processing (Art. 18) refuses the erasure (dry run
170
+ # included) unless +lifting:+ names it, notice recorded: then it is lifted
171
+ # and the record erased in one transaction; a dry run lifts nothing and
172
+ # reports the restrictions it would lift in Result#lifting
173
+ # (Lyra::Rights::ErasureGuard).
174
+ lifts = if defined?(Rights::ErasureGuard)
175
+ Rights::ErasureGuard.check!(model, id, fields: fields, subjects: access_log_subjects, lifting: lifting)
176
+ else
177
+ []
178
+ end
179
+ if lifts.any?
180
+ return model.transaction(requires_new: true) do
181
+ Rights::ErasureGuard.lift!(lifts, by: erased_by) unless dry_run
182
+ Rights::ErasureGuard.lifted(lifts) do
183
+ erase!(model, id, reason: reason, fields: fields, erased_by: erased_by, everywhere: everywhere,
184
+ max_copies: max_copies, any_fields: any_fields, access_log: access_log,
185
+ access_log_subjects: access_log_subjects, dry_run: dry_run)
186
+ end.tap { |result| result.lifting = lifts.map(&:id) }
187
+ end
188
+ end
61
189
 
62
- replacements = personal.to_h { |field| [field, replacement(model, id, field)] }
190
+ stream = Lyra.stream_name(model, id)
191
+ # The record's access stream, and those of the ids the application used
192
+ # as this person's subject (validate_access!(…, subject: user_id)).
193
+ access_streams = [stream, *Array(access_log_subjects).map { |s| Lyra::AccessLog.erasable_key(s) }]
194
+ .uniq.map { |key| Lyra::AccessLog.stream_for(key) }
195
+ personal = []
63
196
  rewritten = []
64
197
  row_erased = false
65
198
  copies = []
66
- shared = 0
199
+ shared = []
200
+ rows_only = []
201
+ restricted_copies = []
202
+ left = []
203
+ not_searched = []
204
+ withheld = false
205
+ log_outcome = nil
206
+ log_entries = 0
207
+ touched = Thread.current[:lyra_erasure_touched] = []
67
208
 
68
- PurposeBoundReads.internal do
69
- model.transaction do
70
- originals = original_values(model, id, events, personal)
209
+ PurposeBoundReads.internal(INTERNAL) do
210
+ # A savepoint for a dry run, so its rollback is real inside a caller's transaction.
211
+ model.transaction(requires_new: dry_run) do
212
+ # The locks first, the stream second: a write that is half-way between
213
+ # its row and its event (a Monitor update appends its event in the
214
+ # transaction that writes the row) must finish before the stream is
215
+ # read, or its event, committed in the window, keeps the value the
216
+ # erasure was asked to remove (0.9.3 and earlier read the stream first).
217
+ lock_record(model, id, stream)
218
+ events = Lyra.config.event_store.read.stream(stream).to_a
219
+ declared = personal_fields(model, events)
220
+ personal = (fields || declared).map(&:to_s).uniq
221
+ undeclared = any_fields || fields.nil? ? [] : personal - declared
222
+ if undeclared.any?
223
+ raise Unsupported, "#{model.name} #{id}: #{undeclared.join(', ')} not declared personal by its policy " \
224
+ "or the events' privacy stamps; pass any_fields: true to erase them anyway"
225
+ end
226
+ raise Unsupported, "#{model.name} #{id}: no personal fields to erase" if personal.empty?
227
+
228
+ replacements = personal.to_h { |field| [field, replacement(model, id, field)] }
229
+ erase_log = access_log.nil? ? Lyra.config.erase_access_log && complete?(declared, personal, events) : access_log
230
+ originals, own = original_values(model, id, events, personal)
231
+ # The origin as it was, for the links of its records (a belongs_to
232
+ # with primary_key: names one of its columns, which may be erased).
233
+ origin_state = current_state(model, id)
71
234
  resolve = ->(key, value) { resolve_in(model, id, replacements, originals, key, value) }
72
235
  rewritten = events.filter_map { |event| rewrite(event, resolve) }
73
- Lyra.config.event_store.overwrite(rewritten) if rewritten.any?
74
- row_erased = erase_row(model, id, replacements)
75
- record_erasure(model, id, stream, personal, reason, erased_by)
236
+ overwrite(rewritten) if rewritten.any?
237
+ row_erased, row_copies = erase_row(model, id, replacements, resolve)
238
+ # What the record's own row still holds (a JSON key, inside a
239
+ # longer string), reported as for other records.
240
+ left = [own_left_behind(model, id, originals.keys)].compact
241
+ withheld = withheld?(id, originals.keys)
242
+ log_outcome, log_entries = access_log!(access_streams, erase_log)
243
+ # A key column is never rewritten, so it is not named as erased.
244
+ record_erasure(model, id, stream, (personal | row_copies) - key_columns(model), reason, erased_by,
245
+ any_fields: any_fields, requested: fields, beyond_policy: any_fields ? personal - declared : [],
246
+ access_log: log_outcome, access_log_entries: log_entries,
247
+ access_log_subjects: access_streams.size - 1)
76
248
  if everywhere
77
- copies, shared = erase_copies(model, id, stream, originals, replacements, reason, erased_by, max_copies)
249
+ # Only the person's own values are searched for elsewhere (from
250
+ # 1.0.0): a third party's email nested under an "email" key in
251
+ # the person's data is erased here, but is not the person's.
252
+ copies, shared, others, rows_only, restricted_copies =
253
+ erase_copies(model, id, stream, origin_state, originals.slice(*own), replacements, reason, erased_by,
254
+ max_copies)
255
+ left += others
256
+ not_searched = encrypted_columns
78
257
  end
258
+ raise ActiveRecord::Rollback if dry_run
79
259
  end
80
260
  end
81
261
  forget_cached(model, id)
262
+ # A dry run's reads may have cached the erased state: drop it.
263
+ touched.each { |m, i| forget_cached(m, i) } if dry_run
82
264
 
83
265
  Result.new(model: model.name, id: id, fields: personal, events_rewritten: rewritten.size,
84
- row_erased: row_erased, copies: copies, shared_values: shared)
266
+ row_erased: row_erased, copies: copies, shared_values: shared.size, shared: shared,
267
+ rows_without_events: rows_only, left_behind: left,
268
+ not_searched: not_searched, key_withheld: withheld,
269
+ access_log: log_outcome, access_log_entries: log_entries, dry_run: dry_run, lifting: [],
270
+ restricted_copies: restricted_copies)
271
+ ensure
272
+ Thread.current[:lyra_erasure_touched] = nil
273
+ end
274
+
275
+ # The subject's access log alone: for a record whose personal fields
276
+ # were all erased before (Retention's hard delete, when an earlier
277
+ # erasure kept the log). Records an ErasureApplied with no fields when
278
+ # the log held entries. Returns how many it deleted.
279
+ def erase_access_log!(model, id, reason:, erased_by: nil)
280
+ stream = Lyra.stream_name(model, id)
281
+ model.transaction do
282
+ _, entries = access_log!([Lyra::AccessLog.stream_for(stream)], true)
283
+ if entries.positive?
284
+ record_erasure(model, id, stream, [], reason, erased_by, access_log: "erased", access_log_entries: entries)
285
+ end
286
+ entries
287
+ end
85
288
  end
86
289
 
87
290
  private
88
291
 
292
+ # Rewrite +events+ in place, and their value-stream links with them
293
+ # (Projections::ReferenceLinks.relink_values): a link named by a digest
294
+ # of an erased unique value is deleted, one of its replacement added.
295
+ def overwrite(events)
296
+ Lyra.config.event_store.overwrite(events)
297
+ Lyra::Projections::ReferenceLinks.relink_values(events)
298
+ end
299
+
300
+ # Whether this erasure, with the ones before it (their ErasureApplied
301
+ # events), covers every personal field of the record.
302
+ def complete?(declared, personal, events)
303
+ earlier = events.select { |event| event.is_a?(Lyra::Events::ErasureApplied) }
304
+ .flat_map { |event| Array(event.data[:fields]).map(&:to_s) }
305
+ (declared - personal - earlier).empty?
306
+ end
307
+
89
308
  def personal_fields(model, events)
90
309
  declared = Lyra::Privacy.policy_for(model).declared_fields.map(&:to_s)
91
310
  stamped = events.flat_map { |event| Lyra::Privacy.stamp_of(event)&.dig("fields")&.keys || [] }
92
311
  declared | stamped
93
312
  end
94
313
 
314
+ # A structured column (json, jsonb, an array, a serialized attribute)
315
+ # that is NOT NULL gets its own default, read as the database stores it
316
+ # (an int[] default '{}' is [], a jsonb default '{}' is {}), never a
317
+ # string marker.
95
318
  def replacement(model, id, field)
96
319
  column = model.columns_hash[field]
97
320
  return nil if column.nil? || column.null
98
- return "erased:#{id}" if %i[string text].include?(column.type)
99
- return model.type_for_attribute(field).cast(column.default) unless column.default.nil?
321
+
322
+ structured = structured?(model, field)
323
+ return "erased:#{id}" if %i[string text].include?(column.type) && !structured
324
+ unless column.default.nil?
325
+ type = model.type_for_attribute(field)
326
+ return structured ? type.deserialize(column.default) : type.cast(column.default)
327
+ end
100
328
 
101
329
  raise Unsupported, "#{model.name}.#{field} is NOT NULL with no default; name a replacement by hand"
102
330
  end
103
331
 
104
332
  # value => the personal field it was found under, for every non-empty
105
- # string the record's row or events held in a personal field.
333
+ # string the record's row or events held in a personal field. A plain
334
+ # value, or the strings of an array (an array column, a change's [old,
335
+ # new]), is the person's. Inside a hash (a jsonb declared personal) only
336
+ # the strings that are identifying for the field's type are: an email for
337
+ # :email, a phone number for :phone; a label ("home", "active") is not,
338
+ # or it would be erased wherever it appears. A string under a nested key
339
+ # that is itself a personal field counts as that field's. An earlier
340
+ # erasure's replacement ("erased:<id>") is no one's value.
341
+ #
342
+ # Returns [found, own]. +own+ are the values of the person's own fields:
343
+ # those at the top of a personal field, in the row and in the events
344
+ # (an attribute, either side of a change, a payload entry of one of the
345
+ # record's own CRUD events), with the strings inside a personal structure
346
+ # taken by the rule above. Only those are searched for in other records
347
+ # (everywhere). A value found only deeper, under a personal key nested
348
+ # in the person's data ({"manager" => {"email" => …}} in a payload), is
349
+ # someone else's: it is erased in this record by the key rule, as its
350
+ # events are, but not searched for elsewhere, where it would erase that
351
+ # person's own records (from 1.0.0).
106
352
  def original_values(model, id, events, personal)
107
353
  found = {}
354
+ own = []
355
+ policy = Lyra::Privacy.policy_for(model)
356
+ types = personal.to_h { |field| [field, policy.annotation(field)&.type&.to_sym] }
108
357
  row = model.unscoped.find_by(model.primary_key => id) unless events_only?
109
- personal.each { |field| remember(found, row&.read_attribute(field), field) } if row
110
- events.each { |event| collect(event.data, personal, found) }
111
- found
358
+ if row
359
+ personal.each do |field|
360
+ value = row.read_attribute(field)
361
+ personal_strings(value, types[field]).each do |v|
362
+ remember(found, v, field)
363
+ own << v
364
+ end
365
+ collect(value, personal, found, types) if value.is_a?(Hash) || value.is_a?(Array)
366
+ end
367
+ end
368
+ events.each do |event|
369
+ data = opened(event.data)
370
+ collect(data, personal, found, types)
371
+ own.concat(top_level_values(data, personal, types, model, id))
372
+ end
373
+ [found, own.uniq & found.keys]
374
+ end
375
+
376
+ # The strings of +data+'s personal fields at the top of the record: in a
377
+ # CRUD envelope its attributes, each side of its changes and its
378
+ # payload's entries, when the envelope is the record's own; none otherwise.
379
+ def top_level_values(data, personal, types, model, id)
380
+ return [] unless own_envelope?(data, model, id)
381
+
382
+ sections = %w[attributes changes payload].map { |key| data[key] || data[key.to_sym] }
383
+ sections.grep(Hash).flat_map do |section|
384
+ section.flat_map { |key, value| personal.include?(key.to_s) ? personal_strings(value, types[key.to_s]) : [] }
385
+ end
386
+ end
387
+
388
+ # Whether +data+ is a CRUD envelope of the record +model+ +id+ itself
389
+ # (Monitor's, Hijack's, a domain event named after its write). Any other
390
+ # event in its stream (an invitation naming a friend's email) holds
391
+ # values the key rule erases there, but they are not the person's own
392
+ # and are not searched for elsewhere (from 1.0.0).
393
+ def own_envelope?(data, model, id)
394
+ return false unless data.is_a?(Hash) && Lyra::Event.recorded_operation(data)
395
+
396
+ owner = envelope_model(data)
397
+ record = data[:model_id] || data["model_id"]
398
+ owner && Lyra.stream_owner_name(owner) == Lyra.stream_owner_name(model) && record.to_s == id.to_s
112
399
  end
113
400
 
114
- def collect(value, personal, found, under: nil)
401
+ def personal_strings(value, type)
402
+ case value
403
+ when String then [value]
404
+ when Array then value.flat_map { |inner| personal_strings(inner, type) }
405
+ when Hash then strings_in(value).select { |inner| identifying_string?(inner, type) }
406
+ else []
407
+ end
408
+ end
409
+
410
+ EMAIL = /\A[^@\s]+@[^@\s]+\.[^@\s]+\z/
411
+
412
+ # Whether +value+, found inside a personal structure of +type+, is
413
+ # itself an identifier of that type.
414
+ def identifying_string?(value, type)
415
+ digits = value.count("0-9")
416
+ case type
417
+ when :email then value.match?(EMAIL)
418
+ when :phone then value.match?(%r{\A\+?[\d\s().\-/]+\z}) && digits.between?(7, 15)
419
+ when :ip_address then ip_address?(value)
420
+ when :credit_card then value.match?(/\A[\d\s-]+\z/) && digits.between?(12, 19)
421
+ when :financial
422
+ value.match?(/\A[A-Z]{2}\d{2}[A-Z0-9 ]{10,32}\z/) || (value.match?(/\A[\d\s-]+\z/) && digits >= 6)
423
+ when :ssn, :identifier, :payment_token, :token, :credential, :online_identifier
424
+ !value.match?(/\s/) && value.length >= 6 && digits.positive?
425
+ else false
426
+ end
427
+ end
428
+
429
+ def ip_address?(value)
430
+ require "ipaddr"
431
+ IPAddr.new(value) && value.match?(/[.:]/)
432
+ rescue IPAddr::Error
433
+ false
434
+ end
435
+
436
+ def strings_in(value)
437
+ case value
438
+ when Hash then value.values.flat_map { |inner| strings_in(inner) }
439
+ when Array then value.flat_map { |inner| strings_in(inner) }
440
+ when String then [value]
441
+ else []
442
+ end
443
+ end
444
+
445
+ def collect(value, personal, found, types)
115
446
  case value
116
447
  when Hash
117
448
  value.each do |key, inner|
118
449
  if personal.include?(key.to_s)
119
- Array(inner).each { |v| remember(found, v, key.to_s) }
120
- else
121
- collect(inner, personal, found, under: key.to_s)
450
+ personal_strings(inner, types[key.to_s]).each { |v| remember(found, v, key.to_s) }
122
451
  end
452
+ collect(inner, personal, found, types)
123
453
  end
124
- when Array then value.each { |inner| collect(inner, personal, found, under: under) }
454
+ when Array then value.each { |inner| collect(inner, personal, found, types) }
125
455
  end
126
456
  end
127
457
 
128
458
  def remember(found, value, field)
129
- found[value] ||= field if value.is_a?(String) && !value.empty?
459
+ found[value] ||= field if value.is_a?(String) && !value.empty? && !value.match?(REPLACEMENT)
130
460
  end
131
461
 
132
462
  # The replacement for +value+ found under +key+ in +model+ +id+'s data,
@@ -139,24 +469,129 @@ module Lyra
139
469
  replacement_for(model, id, key, replacements[originals[value]])
140
470
  end
141
471
 
142
- # A copy held in +model+'s own column gets that column's replacement
143
- # (so the row and its events agree); elsewhere the origin field's.
472
+ # A copy held in +model+'s own scalar column gets that column's
473
+ # replacement (so the row and its events agree); elsewhere, inside a
474
+ # structured column included, the origin field's. A structured
475
+ # replacement ([] or {}) never stands for one value inside another: nil.
144
476
  def replacement_for(model, id, key, fallback)
145
- model.columns_hash.key?(key) ? replacement(model, id, key) : fallback
477
+ fallback = scalar(fallback)
478
+ return fallback unless model.columns_hash.key?(key) && !structured?(model, key)
479
+
480
+ replacement(model, id, key)
146
481
  rescue Unsupported
147
482
  fallback
148
483
  end
149
484
 
485
+ def scalar(value) = value.is_a?(Hash) || value.is_a?(Array) ? nil : value
486
+
150
487
  # The event with every personal value replaced, or nil if it carried none.
151
488
  def rewrite(event, resolve)
152
- data, changed = scrub(event.data, resolve)
489
+ data, changed = scrub_event(event.data, resolve)
153
490
  return nil unless changed
154
491
 
155
492
  event.class.new(event_id: event.event_id, data: data, metadata: event.metadata.to_h)
156
493
  end
157
494
 
495
+ # An event's data. A CRUD envelope (an event that records an operation:
496
+ # Monitor's, Hijack's, a domain event named after a write, a bypass
497
+ # event) is scrubbed as its row is: each of its attributes as a column,
498
+ # each side of each change ([old, new]) as a column, the domain payload
499
+ # entry by entry. Its own entries are never replaced whole, whatever a
500
+ # policy calls its fields. An encrypted attribute's ciphertext is
501
+ # compared in clear, as the row's value is, and a replaced value sealed
502
+ # again. Any other event gets the key rule at its top level too.
503
+ def scrub_event(data, resolve)
504
+ return scrub(data, resolve) unless data.is_a?(Hash) && Lyra::Event.recorded_operation(data)
505
+
506
+ model = envelope_model(data)
507
+ encrypted = model ? Lyra::EncryptedAttributes.names(model) : []
508
+ keys = model ? key_columns(model) : []
509
+ changed = false
510
+ copy = data.to_h do |key, value|
511
+ new_value, did = case key.to_s
512
+ when *ENVELOPE_KEYS then [value, false]
513
+ when "attributes"
514
+ scrub_columns(value, resolve, keys) { |c, v| scrub_column(model, encrypted, c, v, resolve) }
515
+ when "changes"
516
+ scrub_columns(value, resolve, keys) { |c, v| scrub_change(model, encrypted, c, v, resolve) }
517
+ when "payload" then scrub(value, resolve)
518
+ else scrub(value, resolve, key: key.to_s)
519
+ end
520
+ changed ||= did
521
+ [key, new_value]
522
+ end
523
+ [copy, changed]
524
+ end
525
+
526
+ # +keys+: the record's key columns, never rewritten (scrub_row).
527
+ def scrub_columns(values, resolve, keys)
528
+ return scrub(values, resolve) unless values.is_a?(Hash)
529
+
530
+ changed = false
531
+ copy = values.to_h do |column, value|
532
+ next [column, value] if keys.include?(column.to_s)
533
+
534
+ new_value, did = yield(column.to_s, value)
535
+ changed ||= did
536
+ [column, new_value]
537
+ end
538
+ [copy, changed]
539
+ end
540
+
541
+ def scrub_change(model, encrypted, column, change, resolve)
542
+ return scrub_column(model, encrypted, column, change, resolve) unless change.is_a?(Array)
543
+
544
+ results = change.map { |side| scrub_column(model, encrypted, column, side, resolve) }
545
+ [results.map(&:first), results.any?(&:last)]
546
+ end
547
+
548
+ # One column's value in an event, as the row's is scrubbed.
549
+ def scrub_column(model, encrypted, column, value, resolve)
550
+ return scrub(value, resolve, key: column) unless encrypted.include?(column)
551
+
552
+ clear = Lyra::EncryptedAttributes.open(model, { column => value })[column]
553
+ new_value, did = scrub(clear, resolve, key: column)
554
+ did ? [Lyra::EncryptedAttributes.seal_value(model, column, new_value), true] : [value, false]
555
+ end
556
+
557
+ # +data+ with an envelope's encrypted attributes in clear, to read
558
+ # values from (never written back).
559
+ def opened(data)
560
+ return data unless data.is_a?(Hash) && Lyra::Event.recorded_operation(data)
561
+
562
+ model = envelope_model(data)
563
+ encrypted = model ? Lyra::EncryptedAttributes.names(model) : []
564
+ return data if encrypted.empty?
565
+
566
+ open = ->(column, value) { Lyra::EncryptedAttributes.open(model, { column.to_s => value })[column.to_s] }
567
+ data.to_h do |key, value|
568
+ next [key, value] unless value.is_a?(Hash) && %w[attributes changes].include?(key.to_s)
569
+
570
+ [key, value.to_h do |column, inner|
571
+ [column, key.to_s == "changes" && inner.is_a?(Array) ? inner.map { open.(column, _1) } : open.(column, inner)]
572
+ end]
573
+ end
574
+ end
575
+
576
+ def envelope_model(data)
577
+ name = data[:model_class] || data["model_class"]
578
+ return nil if name.nil?
579
+
580
+ model = Lyra.config.monitored_model_named(name) || name.to_s.safe_constantize
581
+ model.is_a?(Class) && model < ActiveRecord::Base ? model : nil
582
+ end
583
+
158
584
  # [copy of +value+ with personal values replaced, whether any was].
585
+ # A hash or an array under a personal field's own name (a json or array
586
+ # column declared personal) is replaced whole, by the key rule, in the
587
+ # row and in its events alike; one under any other name is recursed
588
+ # into.
159
589
  def scrub(value, resolve, key: nil)
590
+ if key && (value.is_a?(Hash) || value.is_a?(Array))
591
+ replaced, changed = scrub_value(value, resolve, key)
592
+ return [replaced, true] if changed
593
+ end
594
+
160
595
  case value
161
596
  when Hash
162
597
  changed = false
@@ -170,89 +605,379 @@ module Lyra
170
605
  results = value.map { |inner| scrub(inner, resolve, key: key) }
171
606
  [results.map(&:first), results.any?(&:last)]
172
607
  else
173
- replaced = resolve.call(key, value)
174
- replaced == :keep || replaced == value ? [value, false] : [replaced, true]
608
+ scrub_value(value, resolve, key)
175
609
  end
176
610
  end
177
611
 
178
- def erase_row(model, id, replacements)
179
- return false if events_only?
612
+ def scrub_value(value, resolve, key)
613
+ replaced = resolve.call(key, value)
614
+ replaced == :keep || replaced == value ? [value, false] : [replaced, true]
615
+ end
616
+
617
+ # The record's aggregate advisory lock, then its row: the locks a writer
618
+ # holds (a Monitor or Hijack write keeps the row until it commits, with its
619
+ # event appended in the same transaction; the other modes take the advisory
620
+ # lock), in Lyra's order, the aggregate lock first (Lyra::AdvisoryLock).
621
+ # Waiting here is what lets an in-flight write finish before the stream is
622
+ # read. Taken the other way round (row, then advisory lock), it deadlocked
623
+ # against a write that skips callbacks, which takes them in this order.
624
+ def lock_record(model, id, stream)
625
+ model.with_connection do |connection|
626
+ Lyra::AdvisoryLock.xact_lock(connection, "lyra_aggregate/#{stream}", purpose: "erasure")
627
+ end
628
+ model.unscoped.lock.find_by(model.primary_key => id) unless events_only?
629
+ end
630
+
631
+ # The record's row, every column of it scrubbed with the rule its events
632
+ # got (+resolve+: the origin's key and value rule). That covers the
633
+ # erased fields, any other column that holds one of their values (a
634
+ # username that is the email), and, from 1.0.0, values inside a json,
635
+ # jsonb or array column: 0.9.3 and earlier left those in the row while
636
+ # the events lost them, and the row no longer replayed to its events.
637
+ # Returns [erased?, the changed columns other than the erased fields].
638
+ def erase_row(model, id, replacements, resolve)
639
+ return [false, []] if events_only?
180
640
 
181
641
  row = model.unscoped.lock.find_by(model.primary_key => id)
182
- return false unless row
642
+ return [false, []] unless row
643
+
644
+ columns = scrub_row(model, row, resolve)
645
+ write_with_lock(model, id, row, columns)
646
+ [true, columns.keys - replacements.keys]
647
+ end
183
648
 
184
- columns = replacements.slice(*row.attribute_names)
185
- write_columns(model, id, columns)
186
- true
649
+ # The row's columns whose value +resolve+ changes => their new value.
650
+ # Each value is scrubbed as the attribute's type gives it, which is how
651
+ # its events hold it: a hash or an array (json, jsonb, hstore, a
652
+ # PostgreSQL array, a serialized attribute, a text column the model casts
653
+ # as json) is recursed into and written back whole; anything else is one
654
+ # value, so a text column that holds JSON is compared as a string, as in
655
+ # its events. Only the table's columns: a virtual attribute (attribute :x)
656
+ # has none to write. Never a key column (key_columns). An encrypted
657
+ # column is read in clear, as its events are compared, and Active Record
658
+ # encrypts the replacement as it writes it.
659
+ def scrub_row(model, row, resolve)
660
+ keys = key_columns(model)
661
+ row.attributes.slice(*(model.column_names - keys)).each_with_object({}) do |(column, value), changed|
662
+ new_value, did = scrub(value, resolve, key: column)
663
+ changed[column] = new_value if did
664
+ end
187
665
  end
188
666
 
189
- # everywhere: events of other streams holding the erased values, and
190
- # the rows of the monitored records they belong to. Returns [records
191
- # erased, values left as shared]. A value held by more than
192
- # +max_copies+ other records is shared by many people, not a copy of
193
- # this one's: erasing it everywhere erased everyone's (a replay's
194
- # placeholder address took all 1,954 addresses with it). It is left,
195
- # and reported.
196
- def erase_copies(origin, origin_id, origin_stream, originals, replacements, reason, erased_by, max_copies)
197
- return [[], 0] if originals.empty?
667
+ # The primary key and the foreign keys (and polymorphic types): never
668
+ # rewritten. The record's stream is named after its key, and a foreign
669
+ # key is another record's. One that holds an erased value is reported
670
+ # (left_behind), never erased.
671
+ def key_columns(model)
672
+ keys = [model.primary_key.to_s]
673
+ model.reflect_on_all_associations(:belongs_to).each do |reflection|
674
+ keys << reflection.foreign_key.to_s
675
+ keys << reflection.foreign_type.to_s if reflection.polymorphic?
676
+ end
677
+ keys
678
+ end
679
+
680
+ # Whether the attribute's type gives a structure (a hash or an array).
681
+ def structured?(model, column)
682
+ definition = model.columns_hash[column]
683
+ return false unless definition
684
+ return true if definition.respond_to?(:array?) && definition.array?
685
+
686
+ type = model.type_for_attribute(column)
687
+ %i[json jsonb hstore].include?(type.type) || type.is_a?(ActiveRecord::Type::Serialized)
688
+ end
689
+
690
+ # The columns of +model+ +id+'s row that still hold one of +values+
691
+ # anywhere: as a JSON key, or inside a longer string (a text column
692
+ # holding JSON). The erasure replaces whole values only, so those are
693
+ # left; this is what tells them apart from an erased copy.
694
+ # In ES-NoProj, where there is no row, the record's replayed state.
695
+ def columns_holding(model, id, values)
696
+ return [] if values.empty?
697
+
698
+ if events_only?
699
+ state = Lyra::DualView.new(model, id).event_sourced_state[:state] || {}
700
+ return state.filter_map { |column, value| column.to_s if holds?(value, values) }
701
+ end
702
+
703
+ row = model.unscoped.find_by(model.primary_key => id)
704
+ return [] unless row
705
+
706
+ row.attributes.slice(*model.column_names).filter_map do |column, value|
707
+ # citext compares without case, so a copy differing in case is the value there.
708
+ fold = model.columns_hash[column]&.type == :citext
709
+ column if holds?(value, values, fold: fold)
710
+ end
711
+ end
712
+
713
+ # The record's own row (or replayed state), as an entry of left_behind,
714
+ # or nil when it holds none of +values+.
715
+ def own_left_behind(model, id, values)
716
+ held = columns_holding(model, id, values)
717
+ { record: label(model, id, values), columns: held } if held.any?
718
+ end
719
+
720
+ # "Model id", or "Model <key withheld>" when the key holds one of
721
+ # +values+: a result, a task's output and an ErasureApplied reason never
722
+ # print an erased value.
723
+ def label(model, id, values) = "#{model.name} #{withheld?(id, values) ? WITHHELD : id}"
724
+
725
+ def withheld?(id, values) = holds?(id.to_s, values)
726
+
727
+ def holds?(value, values, fold: false)
728
+ case value
729
+ when Hash then value.any? { |key, inner| holds?(key.to_s, values, fold: fold) || holds?(inner, values, fold: fold) }
730
+ when Array then value.any? { |inner| holds?(inner, values, fold: fold) }
731
+ when String
732
+ fold ? values.any? { |v| value.downcase.include?(v.downcase) } : values.any? { |v| value.include?(v) }
733
+ else false
734
+ end
735
+ end
736
+
737
+ # With everywhere: the encrypted columns of the monitored models. The
738
+ # search is a text match on the stored events, which hold their
739
+ # ciphertext, so a copy there cannot be found; they are named, never
740
+ # searched (decrypting the whole log is not done).
741
+ def encrypted_columns
742
+ Lyra.config.monitored_models.flat_map do |model|
743
+ Lyra::EncryptedAttributes.names(model).map { |column| "#{model.name}.#{column}" }
744
+ end.uniq
745
+ end
746
+
747
+ # The erasure is a write, and moves the optimistic lock column as any
748
+ # write does: a copy loaded before it fails with StaleObjectError
749
+ # instead of saving the erased values back. The move is recorded as an
750
+ # ordinary update of that column alone (source lyra_erasure), so every
751
+ # replay (DualView, the mode checks, ES-NoProj reads, rebuilds and
752
+ # projections) gives the row's value. ErasureApplied is not replayed,
753
+ # and the rewritten events keep their place in history, so neither can
754
+ # carry it. A model without optimistic locking gets no such event.
755
+ def write_with_lock(model, id, row, columns)
756
+ return write_columns(model, id, columns) if columns.empty? || !model.locking_enabled?
757
+
758
+ lock = model.locking_column
759
+ # In ES-Async the row can trail the log: move from the later of the
760
+ # two, so the log's version sequence never repeats.
761
+ was = [row.read_attribute(lock), logged_version(model, id, lock)].compact.max
762
+ moved = was.to_i + 1
763
+ write_columns(model, id, columns.merge(lock => moved))
764
+ data = Lyra::EventData.build(model, id, :updated, changes: { lock => [was, moved] })
765
+ metadata = { source: SOURCE, lock_moved_by_erasure: true, correlation_id: Lyra::Correlation.current_id }.compact
766
+ event = Lyra::BypassEvents.event_class_for(model, :updated)
767
+ .new(data: data, metadata: Lyra::Privacy.stamp(model, data, metadata))
768
+ Lyra.append_events(event, stream_name: Lyra.stream_name(model, id))
769
+ end
770
+
771
+ # The version the record's stream last recorded, or nil.
772
+ def logged_version(model, id, lock)
773
+ Lyra.config.event_store.read.stream(Lyra.stream_name(model, id)).backward.each do |event|
774
+ version = Lyra::OptimisticLocking.version_in(event.data, lock.to_s)
775
+ return version unless version.nil?
776
+ end
777
+ nil
778
+ end
779
+
780
+ # everywhere: the records other than the erased one that hold its
781
+ # values, found through their events (events_mentioning, one text search
782
+ # of the log for every value) and, from 1.0.0, by searching the tables of
783
+ # monitored models with a policy directly (row_candidates: a row can hold
784
+ # a value its events do not). Returns [records erased, values left as
785
+ # shared ({ field:, type:, holders: }), records left behind, records
786
+ # erased in the row only].
787
+ #
788
+ # A value held by more than +max_copies+ other records is shared by many
789
+ # people, not a copy of this one's: erasing it everywhere erased
790
+ # everyone's (a replay's placeholder address took all 1,954 addresses
791
+ # with it). It is left, and reported. Holders are counted by record:
792
+ # through their events (an exact match on each candidate event) and
793
+ # their rows (the table search's match) alike. The person's linked
794
+ # records (Lyra::Privacy::SubjectLinks: a belongs_to to the erased
795
+ # record's model naming it) do not count (from 1.0.0). A record is
796
+ # listed as erased only if its row, when it has one, no longer holds the
797
+ # value; one whose row still holds it (as a JSON key, inside a longer
798
+ # string, in a key column, in another case in a citext column) is listed
799
+ # as left behind, with the columns, never the value.
800
+ def erase_copies(origin, origin_id, origin_stream, origin_state, originals, replacements, reason, erased_by,
801
+ max_copies)
802
+ none = [[], [], [], [], []]
803
+ return none if originals.empty?
198
804
 
199
805
  policy = Lyra::Privacy.policy_for(origin)
200
- originals = originals.select { |_value, field| IDENTIFYING_TYPES.include?(policy.annotation(field)&.type&.to_sym) }
201
- shared = originals.keys.select { |value| holders(value, origin_stream) > max_copies }
202
- originals = originals.except(*shared)
203
- return [[], shared.size] if originals.empty?
204
-
205
- candidates = events_mentioning(originals.keys).reject { |event| streams_of(event).include?(origin_stream) }
206
- by_record = candidates.group_by { |event| record_of(event) }
207
- by_record.filter_map do |(model, id), events|
208
- resolve = lambda do |key, value|
209
- next :keep unless value.is_a?(String) && originals.key?(value)
210
-
211
- model ? replacement_for(model, id, key, replacements[originals[value]]) : replacements[originals[value]]
212
- end
213
- rewritten = events.filter_map { |event| rewrite(event, resolve) }
806
+ type_of = ->(field) { policy.annotation(field)&.type&.to_sym }
807
+ originals = originals.select { |_value, field| IDENTIFYING_TYPES.include?(type_of.(field)) }
808
+ return none if originals.empty?
809
+
810
+ values = originals.keys
811
+ candidates = events_mentioning(values)
812
+ streams = event_streams(candidates.map(&:event_id))
813
+ candidates = candidates.reject do |event|
814
+ names = streams.fetch(event.event_id, [])
815
+ names.include?(origin_stream) || names.any? { |name| kept_stream?(name) }
816
+ end
817
+ rows = events_only? ? {} : row_candidates(values, origin_stream)
818
+
819
+ holders = values.to_h { |value| [value, Set.new] }
820
+ candidates.each do |event|
821
+ data = opened(event.data)
822
+ holder = holder_of(streams.fetch(event.event_id, []))
823
+ values.each { |value| holders[value] << holder if holds_whole?(data, value) }
824
+ end
825
+ rows.each { |stream, row| row[:values].each { |value| holders[value] << stream } }
826
+ linked = nil
827
+ shared = values.filter_map do |value|
828
+ next if holders[value].size <= max_copies
829
+
830
+ linked ||= linked_streams(origin, origin_id, origin_state)
831
+ count = (holders[value] - linked).size
832
+ [value, count] if count > max_copies
833
+ end
834
+ report = shared.map { |value, count| { field: originals[value], type: type_of.(originals[value]), holders: count } }
835
+ originals = originals.except(*shared.map(&:first))
836
+ return [[], report, [], [], []] if originals.empty?
837
+
838
+ values = originals.keys
839
+ # The candidates for the values erased: a text match without case, as
840
+ # the search was (a citext copy differing in case is reported).
841
+ candidates = candidates.select { |event| holds?(opened(event.data), values, fold: true) }
842
+ by_record = candidates.group_by { |event| record_in(streams.fetch(event.event_id, [])) }
843
+ copy_reason = "#{reason} (copies of #{label(origin, origin_id, values)})"
844
+ copies = []
845
+ left = []
846
+ rows_only = []
847
+
848
+ # Events in no monitored record's stream: their events only.
849
+ by_record.each do |(model, _id), events|
850
+ next if model
851
+
852
+ rewritten = events.filter_map { |event| rewrite(event, holder_resolve(nil, nil, originals, replacements)) }
214
853
  next if rewritten.empty?
215
854
 
216
- Lyra.config.event_store.overwrite(rewritten)
217
- next "#{rewritten.size} events" unless model
855
+ overwrite(rewritten)
856
+ copies << "#{rewritten.size} events"
857
+ end
218
858
 
219
- copied = erase_copied_columns(model, id, originals, replacements)
220
- record_erasure(model, id, Lyra.stream_name(model, id), copied,
221
- "#{reason} (copies of #{origin.name} #{origin_id})", erased_by)
222
- forget_cached(model, id)
223
- "#{model.name} #{id}"
224
- end.then { |copies| [copies, shared.size] }
859
+ targets = by_record.keys.filter_map { |model, id| [Lyra.stream_name(model, id), model, id] if model }
860
+ rows.each { |stream, row| targets << [stream, row[:model], row[:id]] if row[:values].intersect?(values) }
861
+ restricted = []
862
+ targets.uniq(&:first).each do |stream, model, id|
863
+ # A holder under a restriction of processing (Art. 18) keeps its
864
+ # copy until the restriction is lifted: reported, never modified.
865
+ if restricted_holder?(model, id)
866
+ restricted << { record: label(model, id, values), columns: columns_holding(model, id, values) }
867
+ next
868
+ end
869
+
870
+ erased, row_only = erase_holder(model, id, stream, originals, replacements, copy_reason, erased_by)
871
+ held = columns_holding(model, id, values)
872
+ name = label(model, id, values)
873
+ left << { record: name, columns: held } if held.any?
874
+ copies << name if erased && held.empty?
875
+ rows_only << name if row_only
876
+ end
877
+ [copies, report, left, rows_only, restricted]
225
878
  end
226
879
 
227
- # How many streams other than +origin_stream+ hold +value+ (text match).
228
- def holders(value, origin_stream)
229
- ActiveRecord::Base.with_connection do |conn|
230
- conn.select_value(<<~SQL.squish).to_i
231
- SELECT count(DISTINCT s.stream) FROM event_store_events e
232
- JOIN event_store_events_in_streams s ON s.event_id = e.event_id
233
- WHERE convert_from(e.data, 'UTF8') LIKE #{conn.quote("%#{ActiveRecord::Base.sanitize_sql_like(value)}%")}
234
- AND s.stream <> #{conn.quote(origin_stream)}
235
- SQL
880
+ # Whether a restriction in force covers the holder +model+ +id+ (with
881
+ # PAM DSL's rights; Lyra::Rights::ErasureGuard).
882
+ def restricted_holder?(model, id)
883
+ return false unless defined?(Rights::ErasureGuard) && PamDsl.rights_store.any_active?
884
+
885
+ record = Rights::SubjectKeys.find_record(model, id)
886
+ record ? Rights::ErasureGuard.covering_record(record).any? : false
887
+ end
888
+
889
+ # The value rule only, for a record other than the erased one: a copy
890
+ # of an erased value is replaced, a third party's value under a
891
+ # personal key is kept.
892
+ def holder_resolve(model, id, originals, replacements)
893
+ lambda do |key, value|
894
+ next :keep unless value.is_a?(String) && originals.key?(value)
895
+
896
+ fallback = replacements[originals[value]]
897
+ model ? replacement_for(model, id, key, fallback) : scalar(fallback)
236
898
  end
237
899
  end
238
900
 
239
- def events_mentioning(values)
901
+ # One record holding a copy, under its locks (row, then advisory lock),
902
+ # its stream read again: a copy written since the search is in it. Its
903
+ # events are rewritten by the value rule, and its row is scrubbed by the
904
+ # same rule whether or not its events held the value whole (from 1.0.0:
905
+ # a row given the value by raw SQL whose events mention it only inside a
906
+ # longer string, or not at all). When only the row held it, its
907
+ # ErasureApplied says row_only: its row already differed from its
908
+ # replay. A row with no stream is erased and then imported
909
+ # (erase_and_import). Returns [erased?, erased in the row only?].
910
+ def erase_holder(model, id, stream, originals, replacements, reason, erased_by)
911
+ lock_record(model, id, stream)
912
+ events = Lyra.config.event_store.read.stream(stream).to_a
913
+ if events.empty? && !events_only?
914
+ # No Genesis run elsewhere imports the row as it was meanwhile.
915
+ Lyra::Genesis.hold_lock!(model)
916
+ events = Lyra.config.event_store.read.stream(stream).to_a
917
+ end
918
+ resolve = holder_resolve(model, id, originals, replacements)
919
+ return erase_and_import(model, id, stream, resolve, reason, erased_by) if events.empty? && !events_only?
920
+
921
+ rewritten = events.filter_map { |event| rewrite(event, resolve) }
922
+ overwrite(rewritten) if rewritten.any?
923
+ copied = erase_copied_columns(model, id, resolve)
924
+ return [false, false] if copied.empty? && rewritten.empty?
925
+
926
+ row_only = rewritten.empty?
927
+ record_erasure(model, id, stream, copied, reason, erased_by, row_only: row_only)
928
+ forget_cached(model, id)
929
+ [true, row_only]
930
+ end
931
+
932
+ # A row with no stream: erased, then imported the Genesis way, so its
933
+ # Imported event holds the erased values, never the personal ones (an
934
+ # ErasureApplied alone would start a stream Genesis never imports: it
935
+ # imports only rows with no stream), then its ErasureApplied. The lock
936
+ # column moves as with any erasure; the Imported event holds its new
937
+ # value. If the row was imported meanwhile (by a Genesis run on a
938
+ # connection the lock did not hold back), that stream is read again and
939
+ # rewritten instead, and no import is claimed.
940
+ def erase_and_import(model, id, stream, resolve, reason, erased_by)
941
+ row = model.unscoped.lock.find_by(model.primary_key => id)
942
+ return [false, false] unless row
943
+
944
+ columns = scrub_row(model, row, resolve)
945
+ return [false, false] if columns.empty?
946
+
947
+ lock = model.locking_enabled? ? model.locking_column : nil
948
+ write_columns(model, id, lock ? columns.merge(lock => row.read_attribute(lock).to_i + 1) : columns)
949
+ imported = Lyra::Genesis.import_one(model, id)
950
+ unless imported
951
+ rewritten = Lyra.config.event_store.read.stream(stream).to_a.filter_map { |event| rewrite(event, resolve) }
952
+ overwrite(rewritten) if rewritten.any?
953
+ end
954
+ record_erasure(model, id, stream, columns.keys, reason, erased_by, imported: imported)
955
+ forget_cached(model, id)
956
+ [true, false]
957
+ end
958
+
959
+ # event_id => the streams the event is in, in one query.
960
+ def event_streams(event_ids)
961
+ return {} if event_ids.empty?
962
+
240
963
  ActiveRecord::Base.with_connection do |conn|
241
- clauses = values.map do |value|
242
- "convert_from(data, 'UTF8') LIKE #{conn.quote("%#{ActiveRecord::Base.sanitize_sql_like(value)}%")}"
243
- end
244
- ids = conn.select_values("SELECT event_id FROM event_store_events WHERE #{clauses.join(' OR ')}")
245
- ids.filter_map { |event_id| Lyra.config.event_store.read.event(event_id) rescue nil }
964
+ event_ids.each_slice(1_000).flat_map do |ids|
965
+ conn.select_rows("SELECT event_id, stream FROM event_store_events_in_streams " \
966
+ "WHERE event_id IN (#{ids.map { |i| conn.quote(i) }.join(', ')})")
967
+ end.group_by(&:first).transform_values { |rows| rows.map(&:last) }
246
968
  end
247
969
  end
248
970
 
249
- def streams_of(event)
250
- Lyra.config.event_store.streams_of(event.event_id).map(&:name)
971
+ # The holder an event counts for: the monitored record whose stream it is
972
+ # in (so a reference link, Model.fk$id, adds none), else its stream.
973
+ def holder_of(names)
974
+ model, id = record_in(names)
975
+ model ? Lyra.stream_name(model, id) : names.min
251
976
  end
252
977
 
253
- # [model, id] when the event is in a monitored record's stream, else [nil, nil].
254
- def record_of(event)
255
- streams_of(event).each do |name|
978
+ # [model, id] of the monitored record whose stream is among +names+, else [nil, nil].
979
+ def record_in(names)
980
+ names.each do |name|
256
981
  model_name, id = name.split("$", 2)
257
982
  model = Lyra.config.monitored_model_named(model_name)
258
983
  return [model, id] if model && id
@@ -260,20 +985,170 @@ module Lyra
260
985
  [nil, nil]
261
986
  end
262
987
 
263
- # The columns of +model+ +id+'s row that hold an erased value, replaced.
264
- # Returns the column names.
265
- def erase_copied_columns(model, id, originals, replacements)
988
+ # "%"…"%": +value+ as a JSON string, quotes included, for a LIKE on a
989
+ # json or jsonb column as text: it matches the value whole, as a value
990
+ # or a key, not inside a longer one.
991
+ def json_pattern(value)
992
+ "%#{ActiveRecord::Base.sanitize_sql_like(JSON.generate(value))}%"
993
+ end
994
+
995
+ # The streams of the person's linked records, by the rule of the
996
+ # subject-access copy (Lyra::Privacy::SubjectLinks): the records of
997
+ # monitored models with a belongs_to to +origin+'s model (any
998
+ # association: an owner, an assignee, a manager), a polymorphic one with
999
+ # its type column naming it, or a <model>_id column no association owns,
1000
+ # whose foreign key holds the origin's id, or the column the belongs_to
1001
+ # names as its primary_key (+origin_state+: the origin as it was before
1002
+ # this erasure). Records of the origin's own model are not linked. They
1003
+ # are taken as the person's own for the threshold only (they do not
1004
+ # count toward max_copies); they are erased as any holder is. Each query
1005
+ # runs in a savepoint: one that fails finds none.
1006
+ def linked_streams(origin, origin_id, origin_state)
1007
+ links = Lyra::Privacy::SubjectLinks.new(Lyra.stream_owner_name(origin))
1008
+ links.linked_models.each_with_object(Set.new) do |model, linked|
1009
+ links.for(model).each do |link|
1010
+ target = link_target(model, link, origin, origin_id, origin_state)
1011
+ next if target.nil?
1012
+
1013
+ scope = model.unscoped.where(link.fk => target)
1014
+ scope = scope.where(link.type_column => link.type_value) if link.type_column
1015
+ ids = model.transaction(requires_new: true) { scope.pluck(model.primary_key) }
1016
+ ids.each { |id| linked << Lyra.stream_name(model, id) }
1017
+ rescue StandardError
1018
+ next
1019
+ end
1020
+ end
1021
+ end
1022
+
1023
+ # What +link+'s foreign key holds for the origin: its id, or the value of
1024
+ # the column a belongs_to names with primary_key:.
1025
+ def link_target(model, link, origin, origin_id, origin_state)
1026
+ reflection = model.reflect_on_all_associations(:belongs_to)
1027
+ .find { |r| !r.polymorphic? && r.foreign_key.to_s == link.fk }
1028
+ column = reflection&.association_primary_key.to_s
1029
+ return origin_id if column.empty? || column == origin.primary_key.to_s
1030
+
1031
+ origin_state[column]
1032
+ end
1033
+
1034
+ # The origin's attributes as they are now (its replayed state in
1035
+ # ES-NoProj), keyed by column name.
1036
+ def current_state(model, id)
1037
+ if events_only?
1038
+ state = Lyra::DualView.new(model, id).event_sourced_state[:state] || {}
1039
+ return state.transform_keys(&:to_s)
1040
+ end
1041
+
1042
+ model.unscoped.find_by(model.primary_key => id)&.attributes || {}
1043
+ end
1044
+
1045
+ # From 1.0.0, with everywhere: the rows of monitored models with a
1046
+ # policy that may hold one of +values+, searched in the tables
1047
+ # themselves, so a row whose events lack the value (raw SQL, a write
1048
+ # before monitoring, a write Lyra could not intercept) is found too.
1049
+ # One query per table, which returns the keys and, per value, whether
1050
+ # the row matched; no row is loaded here. String and text columns by
1051
+ # equality (a citext column without case), json and jsonb columns as text
1052
+ # by a LIKE on the value as a JSON string (a value or a key), a text
1053
+ # column the model reads as a structure (serialized, attribute :json) by
1054
+ # a LIKE on the bare value, string arrays with = ANY, hstore values with
1055
+ # = ANY(avals(…)). Encrypted columns are not searched (not_searched). The
1056
+ # erasure itself matches exactly, and reports what it cannot erase.
1057
+ # Returns { stream => { model:, id:, values: [matched] } } without the
1058
+ # origin's own row.
1059
+ def row_candidates(values, origin_stream)
1060
+ searched_models.each_with_object({}) do |model, found|
1061
+ model.with_connection do |conn|
1062
+ encrypted = Lyra::EncryptedAttributes.names(model)
1063
+ columns = model.columns.reject { |column| encrypted.include?(column.name) }
1064
+ conditions = values.map do |value|
1065
+ clauses = columns.filter_map { |column| search_clause(conn, model, column, value) }
1066
+ clauses.empty? ? "FALSE" : "(#{clauses.join(' OR ')})"
1067
+ end
1068
+ next if conditions.all?("FALSE")
1069
+
1070
+ flags = conditions.each_with_index.map { |condition, i| "#{condition} AS v#{i}" }
1071
+ conn.select_rows(<<~SQL.squish).each do |id, *matched|
1072
+ SELECT #{conn.quote_column_name(model.primary_key)}, #{flags.join(', ')}
1073
+ FROM #{conn.quote_table_name(model.table_name)}
1074
+ WHERE #{conditions.join(' OR ')}
1075
+ SQL
1076
+ stream = Lyra.stream_name(model, id)
1077
+ next if stream == origin_stream
1078
+
1079
+ held = values.select.with_index { |_value, i| [true, "t"].include?(matched[i]) }
1080
+ found[stream] = { model: model, id: id, values: held }
1081
+ end
1082
+ end
1083
+ end
1084
+ end
1085
+
1086
+ # The monitored models with a policy, one per table, as the class that
1087
+ # owns it (the STI base class, so no type condition narrows the rows):
1088
+ # the tables of models without a policy, or not monitored, are not
1089
+ # searched.
1090
+ def searched_models
1091
+ Lyra.config.monitored_models
1092
+ .select { |model| Lyra::Privacy.policy_for(model).loaded? }
1093
+ .map { |model| model.respond_to?(:base_class) ? model.base_class : model }
1094
+ .uniq(&:table_name)
1095
+ .select(&:table_exists?)
1096
+ end
1097
+
1098
+ TEXT_TYPES = %i[string text citext].freeze
1099
+
1100
+ # The condition +column+ holds +value+, or nil for a column not searched.
1101
+ def search_clause(conn, model, column, value)
1102
+ name = conn.quote_column_name(column.name)
1103
+ quoted = conn.quote(value)
1104
+ if column.respond_to?(:array?) && column.array?
1105
+ "#{quoted} = ANY(#{name})" if TEXT_TYPES.include?(column.type)
1106
+ elsif %i[json jsonb].include?(column.type)
1107
+ "#{name}::text LIKE #{conn.quote(json_pattern(value))}"
1108
+ elsif column.type == :hstore
1109
+ "#{quoted} = ANY(avals(#{name}))"
1110
+ elsif TEXT_TYPES.include?(column.type) && structured?(model, column.name)
1111
+ "#{name} LIKE #{conn.quote("%#{ActiveRecord::Base.sanitize_sql_like(value)}%")}"
1112
+ elsif TEXT_TYPES.include?(column.type)
1113
+ "#{name} = #{quoted}"
1114
+ end
1115
+ end
1116
+
1117
+ def holds_whole?(value, wanted)
1118
+ case value
1119
+ when Hash then value.any? { |_key, inner| holds_whole?(inner, wanted) }
1120
+ when Array then value.any? { |inner| holds_whole?(inner, wanted) }
1121
+ when String then value == wanted
1122
+ else false
1123
+ end
1124
+ end
1125
+
1126
+ # The events whose stored data mentions one of +values+: one text search
1127
+ # of the log for all of them, without case (a citext column holds a copy
1128
+ # that differs in case; the match that erases is exact, and such a copy
1129
+ # is reported). Read in batches.
1130
+ def events_mentioning(values)
1131
+ ActiveRecord::Base.with_connection do |conn|
1132
+ clauses = values.map do |value|
1133
+ "convert_from(data, 'UTF8') ILIKE #{conn.quote("%#{ActiveRecord::Base.sanitize_sql_like(value)}%")}"
1134
+ end
1135
+ ids = conn.select_values("SELECT event_id FROM event_store_events WHERE #{clauses.join(' OR ')}")
1136
+ ids.each_slice(1_000).flat_map { |batch| Lyra.config.event_store.read.events(batch).to_a }
1137
+ end
1138
+ end
1139
+
1140
+ # The columns of +model+ +id+'s row that hold an erased value, replaced
1141
+ # by the rule its events got (+resolve+: the value rule only, so a third
1142
+ # party's email under an "email" key is kept), inside json, jsonb and
1143
+ # array columns too (scrub_row). Returns the column names.
1144
+ def erase_copied_columns(model, id, resolve)
266
1145
  return [] if events_only?
267
1146
 
268
1147
  row = model.unscoped.lock.find_by(model.primary_key => id)
269
1148
  return [] unless row
270
1149
 
271
- columns = row.attributes.each_with_object({}) do |(column, value), acc|
272
- next unless value.is_a?(String) && originals.key?(value)
273
-
274
- acc[column] = replacement_for(model, id, column, replacements[originals[value]])
275
- end
276
- write_columns(model, id, columns)
1150
+ columns = scrub_row(model, row, resolve)
1151
+ write_with_lock(model, id, row, columns)
277
1152
  columns.keys
278
1153
  end
279
1154
 
@@ -284,21 +1159,51 @@ module Lyra
284
1159
  def write_columns(model, id, columns)
285
1160
  return if columns.empty?
286
1161
 
287
- Lyra.projection_write { model.unscoped.where(model.primary_key => id).update_all(columns) }
1162
+ Lyra.projection_write(INTERNAL) { model.unscoped.where(model.primary_key => id).update_all(columns) }
288
1163
  end
289
1164
 
290
1165
  def events_only?
291
1166
  Lyra.config.event_sourcing_mode? && Lyra.config.projection_mode == :disabled
292
1167
  end
293
1168
 
294
- def record_erasure(model, id, stream, fields, reason, erased_by)
295
- data = { model_class: model.name, model_id: id, fields: fields, reason: reason.to_s, erased_at: Time.current }
1169
+ # The subject's access streams: their events deleted when +erase+, else
1170
+ # kept. Returns ["erased" or "kept", how many entries they held].
1171
+ def access_log!(access_streams, erase)
1172
+ if erase
1173
+ ["erased", access_streams.sum { |s| Lyra::AccessLog.delete_stream!(s) }]
1174
+ else
1175
+ ["kept", access_streams.sum { |s| Lyra::AccessLog.count_stream(s) }]
1176
+ end
1177
+ end
1178
+
1179
+ # +access_log+ and +access_log_entries+: what happened to the subject's
1180
+ # access log ("erased" or "kept", and how many entries it held).
1181
+ # +any_fields+: the request named columns the policy does not declare personal
1182
+ # (ANY_FIELDS=1); +requested+ is what it named (nil: the declared ones);
1183
+ # +beyond_policy+ the columns erased only because of it. The operating
1184
+ # system's user and the host are recorded beside the label in erased_by.
1185
+ # +row_only+: the value was in the row, not in the record's events, and
1186
+ # only the row was erased; +imported+: the row had no stream, and was
1187
+ # imported after it was erased (both from 1.0.0, with everywhere).
1188
+ def record_erasure(model, id, stream, fields, reason, erased_by, any_fields: false, requested: nil, beyond_policy: [],
1189
+ access_log: nil, access_log_entries: nil, access_log_subjects: 0, row_only: false, imported: false)
1190
+ data = { model_class: model.name, model_id: id, fields: fields, reason: reason.to_s, erased_at: Time.current,
1191
+ any_fields: any_fields, requested_fields: requested&.map(&:to_s), beyond_policy: beyond_policy }
1192
+ data.merge!(access_log: access_log, access_log_entries: access_log_entries) if access_log
1193
+ data[:access_log_subjects] = access_log_subjects if access_log_subjects.positive?
1194
+ data[:row_only] = true if row_only
1195
+ data[:imported] = true if imported
1196
+ who = Lyra::Attribution.of
296
1197
  metadata = {
297
1198
  source: SOURCE,
298
1199
  erased_by: erased_by&.to_s || (defined?(::Current) && ::Current.respond_to?(:user) ? ::Current.user&.id : nil),
1200
+ os_user: who[:os_user],
1201
+ host: who[:host],
1202
+ pid: who[:pid],
299
1203
  correlation_id: Lyra::Correlation.current_id
300
1204
  }.compact
301
1205
  Lyra.append_events(Lyra::Events::ErasureApplied.new(data: data, metadata: metadata), stream_name: stream)
1206
+ Thread.current[:lyra_erasure_touched]&.push([model, id])
302
1207
  end
303
1208
 
304
1209
  def forget_cached(model, id)