orfeas_lyra 0.9.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +2021 -17
- data/README.md +22 -9
- data/Rakefile +40 -1
- data/app/controllers/lyra/application_controller.rb +127 -0
- data/app/controllers/lyra/dashboard_controller.rb +33 -31
- data/app/controllers/lyra/flow_controller.rb +32 -34
- data/app/controllers/lyra/privacy_controller.rb +57 -18
- data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
- data/app/views/lyra/dashboard/index.html.erb +1 -0
- data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
- data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
- data/app/views/lyra/privacy/policy.html.erb +7 -0
- data/app/views/lyra/privacy/rights.html.erb +78 -0
- data/config/privacy_policies.rb +0 -3
- data/config/routes.rb +1 -0
- data/lib/lyra/access_log.rb +187 -6
- data/lib/lyra/advisory_lock.rb +236 -0
- data/lib/lyra/aggregate.rb +1 -1
- data/lib/lyra/associations/habtm_links.rb +382 -19
- data/lib/lyra/associations/maintenance.rb +229 -0
- data/lib/lyra/attribution.rb +48 -0
- data/lib/lyra/bypass_events.rb +446 -46
- data/lib/lyra/command_handler.rb +19 -14
- data/lib/lyra/configuration.rb +237 -10
- data/lib/lyra/consistency/read_your_writes.rb +4 -2
- data/lib/lyra/correlation.rb +121 -0
- data/lib/lyra/domain_events.rb +4 -0
- data/lib/lyra/dual_view.rb +53 -16
- data/lib/lyra/encrypted_attributes.rb +5 -0
- data/lib/lyra/engine.rb +67 -0
- data/lib/lyra/erasure.rb +1011 -106
- data/lib/lyra/event.rb +35 -1
- data/lib/lyra/event_data.rb +38 -0
- data/lib/lyra/event_flow.rb +31 -15
- data/lib/lyra/event_store_adapter.rb +28 -0
- data/lib/lyra/event_window.rb +58 -0
- data/lib/lyra/genesis.rb +548 -44
- data/lib/lyra/interceptors/association_interceptor.rb +689 -34
- data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
- data/lib/lyra/internal.rb +26 -0
- data/lib/lyra/mode_sync.rb +51 -3
- data/lib/lyra/mode_transition.rb +257 -30
- data/lib/lyra/model_selection.rb +35 -0
- data/lib/lyra/own_connection.rb +17 -2
- data/lib/lyra/privacy/adapters/pam.rb +25 -2
- data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
- data/lib/lyra/privacy/interface.rb +27 -7
- data/lib/lyra/privacy/subject_links.rb +143 -0
- data/lib/lyra/privacy/subject_records.rb +369 -0
- data/lib/lyra/projection.rb +7 -1
- data/lib/lyra/projections/async_checkpoint.rb +32 -0
- data/lib/lyra/projections/async_projection_job.rb +166 -23
- data/lib/lyra/projections/cached_joins.rb +33 -18
- data/lib/lyra/projections/cached_projection.rb +74 -33
- data/lib/lyra/projections/cached_relation.rb +3175 -321
- data/lib/lyra/projections/default_scopes.rb +254 -0
- data/lib/lyra/projections/deferred_relation.rb +23 -3
- data/lib/lyra/projections/event_store_reader.rb +47 -8
- data/lib/lyra/projections/foreign_keys.rb +92 -0
- data/lib/lyra/projections/lazy_projection.rb +165 -27
- data/lib/lyra/projections/model_projection.rb +165 -14
- data/lib/lyra/projections/rebuild.rb +34 -6
- data/lib/lyra/projections/reference_links.rb +463 -72
- data/lib/lyra/projections/typed_values.rb +629 -0
- data/lib/lyra/projections/unique_indexes.rb +595 -0
- data/lib/lyra/projections/uniqueness.rb +784 -0
- data/lib/lyra/projections/value_digest.rb +93 -0
- data/lib/lyra/purpose_bound_reads.rb +912 -77
- data/lib/lyra/repair.rb +119 -28
- data/lib/lyra/retention.rb +114 -8
- data/lib/lyra/rights/consent_store.rb +415 -0
- data/lib/lyra/rights/erasure_guard.rb +135 -0
- data/lib/lyra/rights/pending.rb +74 -0
- data/lib/lyra/rights/queries.rb +269 -0
- data/lib/lyra/rights/store.rb +512 -0
- data/lib/lyra/rights/subject_keys.rb +224 -0
- data/lib/lyra/rights.rb +281 -0
- data/lib/lyra/saves_without_effect.rb +69 -0
- data/lib/lyra/stream_tail.rb +319 -0
- data/lib/lyra/strict_data_access.rb +916 -106
- data/lib/lyra/temporal.rb +4 -2
- data/lib/lyra/touch_registry.rb +404 -0
- data/lib/lyra/verification/bypass_workflow.rb +452 -42
- data/lib/lyra/verification/rights_workflows.rb +170 -0
- data/lib/lyra/version.rb +1 -1
- data/lib/lyra/visualization/event_graph.rb +1 -1
- data/lib/lyra/visualization/timeline.rb +2 -2
- data/lib/lyra.rb +43 -2
- data/lib/tasks/lyra_mode.rake +110 -8
- data/lib/tasks/lyra_projections.rake +43 -2
- metadata +34 -4
- data/lib/lyra/associations/event_aware.rb +0 -224
data/lib/lyra/erasure.rb
CHANGED
|
@@ -14,7 +14,10 @@ module Lyra
|
|
|
14
14
|
#
|
|
15
15
|
# Personal fields are those the record's policy declares plus those the
|
|
16
16
|
# events' privacy stamps (config.annotate_privacy) list, or exactly the
|
|
17
|
-
# +fields:+ given
|
|
17
|
+
# +fields:+ given, which must be among those unless +any_fields: true+
|
|
18
|
+
# (from 0.9.3: an erasure rewrites history in place, so naming any column,
|
|
19
|
+
# a payment's status, say, would rewrite non-personal history under an
|
|
20
|
+
# Art. 17 reason). In the record's events a value is erased wherever it
|
|
18
21
|
# appears: under the attribute's own name (attributes, both sides of each
|
|
19
22
|
# change, payload keys), and under any other name, matched by value (a
|
|
20
23
|
# domain-event payload that copies the email as payer_email).
|
|
@@ -24,12 +27,50 @@ module Lyra
|
|
|
24
27
|
# the rows of those records (monitored models) get the value replaced
|
|
25
28
|
# where they hold it, with an ErasureApplied event in their streams, so
|
|
26
29
|
# every stream still replays to its row. The search is a text match on the
|
|
27
|
-
# stored events, then an exact match on each candidate's data.
|
|
30
|
+
# stored events, then an exact match on each candidate's data. From 1.0.0
|
|
31
|
+
# the tables of monitored models with a policy are searched too (not in
|
|
32
|
+
# ES-NoProj), so a row whose events lack the value is found; a row with no
|
|
33
|
+
# stream is erased and then imported (Genesis), so its first event never
|
|
34
|
+
# holds the value. Only the person's own values are searched for: a third
|
|
35
|
+
# party's value nested under a personal key in the person's data is erased
|
|
36
|
+
# there only. Holders are counted by record; the person's linked records
|
|
37
|
+
# (belongs_to the erased record) do not count toward max_copies
|
|
38
|
+
# (config.erasure_max_copies), and each value left as shared is reported
|
|
39
|
+
# by field and count (Result#shared). Lyra's own record streams
|
|
40
|
+
# (kept_stream?: rights and consent, access logs, exports) are never
|
|
41
|
+
# rewritten. A dry run (dry_run: true) reports all of it and writes nothing.
|
|
42
|
+
#
|
|
43
|
+
# Each row gets the rule its own events get: the record's row the key and
|
|
44
|
+
# value rule, another record's row the value rule only (a third party's
|
|
45
|
+
# email under an "email" key there is not this person's). Every column is
|
|
46
|
+
# scrubbed; from 1.0.0 a json, jsonb or array column, or a serialized
|
|
47
|
+
# attribute, is recursed into and written back whole. A text column that
|
|
48
|
+
# holds JSON is a string in its events, and is compared as one. A hash or
|
|
49
|
+
# array under a personal field's name is replaced whole. Encrypted
|
|
50
|
+
# attributes are compared in clear, in the events as in the row. A record
|
|
51
|
+
# whose row still holds a value (as a JSON key, inside a longer string) is
|
|
52
|
+
# reported in Result#left_behind, not in Result#copies.
|
|
28
53
|
#
|
|
29
54
|
# The replacement is nil where the column allows it, "erased:<id>" for a
|
|
30
55
|
# string column that does not, and the column's default otherwise; a value
|
|
31
56
|
# with no column of its own (a payload key) gets the origin field's.
|
|
32
57
|
#
|
|
58
|
+
# The subject's access log (Lyra::AccessLog, the stream
|
|
59
|
+
# Lyra::DataAccess$<Model>$<id>) holds who read the record, when and under
|
|
60
|
+
# which purpose, never a value. It may be evidence the application must
|
|
61
|
+
# keep (Art. 5(2), a security investigation) or the subject's data to
|
|
62
|
+
# erase with the rest; Lyra cannot know which. From 1.0.0 erasure keeps it
|
|
63
|
+
# unless config.erase_access_log or +access_log: true+ says otherwise; then
|
|
64
|
+
# its events are deleted, from every stream they are in and from the
|
|
65
|
+
# events table (unlinking the stream alone would leave them). The
|
|
66
|
+
# configuration applies only once the record is fully erased: when this
|
|
67
|
+
# erasure, with any earlier ones, has covered every personal field. A
|
|
68
|
+
# field-level erasure (a retention period for one field) leaves a record
|
|
69
|
+
# whose other data is still processed, and its log with it. An explicit
|
|
70
|
+
# +access_log: true+ deletes the log whatever was erased. Either way
|
|
71
|
+
# ErasureApplied records access_log ("erased" or "kept") and
|
|
72
|
+
# access_log_entries (how many it held), so the decision is visible.
|
|
73
|
+
#
|
|
33
74
|
# Crypto-shredding, which would leave the log untouched, remains future
|
|
34
75
|
# work.
|
|
35
76
|
module Erasure
|
|
@@ -38,7 +79,8 @@ module Lyra
|
|
|
38
79
|
class Unsupported < StandardError; end
|
|
39
80
|
|
|
40
81
|
# Other records holding a value, beyond which it is taken to be shared by
|
|
41
|
-
# many people (a city, a placeholder) rather than a copy of this one's
|
|
82
|
+
# many people (a city, a placeholder) rather than a copy of this one's: the
|
|
83
|
+
# default of config.erasure_max_copies (from 1.0.0).
|
|
42
84
|
MAX_COPIES = 10
|
|
43
85
|
|
|
44
86
|
# The PAM types searched for in other records: direct identifiers, whose
|
|
@@ -49,84 +91,372 @@ module Lyra
|
|
|
49
91
|
IDENTIFYING_TYPES = %i[email phone identifier ssn credit_card financial payment_token ip_address
|
|
50
92
|
credential token].freeze
|
|
51
93
|
|
|
94
|
+
# +left_behind+: the records whose row still holds an erased value after
|
|
95
|
+
# the erasure, as { record: "Model id", columns: [...] } (never the value):
|
|
96
|
+
# a value held as a JSON key, inside a longer string, or in a key column
|
|
97
|
+
# (a primary or foreign key is never rewritten). The erased record itself
|
|
98
|
+
# (its replayed state in ES-NoProj) and, with everywhere, the other
|
|
99
|
+
# records reached; those are not in +copies+. A record whose key holds an
|
|
100
|
+
# erased value is named "Model <key withheld>".
|
|
101
|
+
# +not_searched+: with everywhere, the encrypted columns of monitored
|
|
102
|
+
# models ("Model.column"), which the search of the log cannot read.
|
|
103
|
+
# +key_withheld+: the erased record's own key holds an erased value.
|
|
104
|
+
# +shared+ (from 1.0.0): with everywhere, each value left as shared, as
|
|
105
|
+
# { field:, type:, holders: } (the personal field it was found under, its
|
|
106
|
+
# PAM type and how many other records hold it), never the value;
|
|
107
|
+
# +shared_values+ is their count, an Integer as before.
|
|
108
|
+
# +rows_without_events+ (from 1.0.0): with everywhere, the records whose
|
|
109
|
+
# row held an erased value that their events did not (written by raw SQL,
|
|
110
|
+
# say), found by searching the tables directly and erased in the row only:
|
|
111
|
+
# their row already differed from their replay, and lyra:repair brings
|
|
112
|
+
# the log in line. A row with no stream at all is imported instead (its
|
|
113
|
+
# Imported event holds the erased values) and is not listed here.
|
|
114
|
+
# +lifting+ (from 1.0.0, with PAM DSL's rights): the ids of the
|
|
115
|
+
# restrictions of processing the erasure lifted (lifting:), or, in a dry
|
|
116
|
+
# run, would lift; [] when none.
|
|
117
|
+
# +restricted_copies+ (from 1.0.0, with everywhere and PAM DSL's rights):
|
|
118
|
+
# the other records holding a copy that a restriction of processing
|
|
119
|
+
# covers, as { record: "Model id", columns: [...] }, never the value. They
|
|
120
|
+
# are not modified: the copy stays until the holder's restriction is
|
|
121
|
+
# lifted, and an erasure then (everywhere again) takes it.
|
|
52
122
|
Result = Struct.new(:model, :id, :fields, :events_rewritten, :row_erased, :copies, :shared_values,
|
|
53
|
-
|
|
123
|
+
:access_log, :access_log_entries, :left_behind, :not_searched, :key_withheld,
|
|
124
|
+
:shared, :rows_without_events, :dry_run, :lifting, :restricted_copies, keyword_init: true)
|
|
125
|
+
|
|
126
|
+
WITHHELD = "<key withheld>"
|
|
127
|
+
|
|
128
|
+
# Lyra's own records of how a person's rights were handled (kept_stream?):
|
|
129
|
+
# restrictions and objections (Lyra::Rights::STREAM), consent
|
|
130
|
+
# (Lyra::Rights::CONSENT_STREAM), the access log and its erasures, the
|
|
131
|
+
# subject-access exports, each an index stream and its per-subject
|
|
132
|
+
# streams ("<name>$<subject key>"). Evidence the controller keeps
|
|
133
|
+
# (Art. 5(2)); everywhere never rewrites their events nor counts them as
|
|
134
|
+
# holders, even when a subject key is an erased value. The record's own
|
|
135
|
+
# access log is erased, or kept, by access_log:.
|
|
136
|
+
def self.kept_streams
|
|
137
|
+
names = []
|
|
138
|
+
names += [Lyra::Rights::STREAM, Lyra::Rights::CONSENT_STREAM] if defined?(Lyra::Rights::STREAM)
|
|
139
|
+
names += [Lyra::AccessLog::STREAM, Lyra::AccessLog::ERASURES, Lyra::AccessLog::EXPORTS] if defined?(Lyra::AccessLog::STREAM)
|
|
140
|
+
names
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def self.kept_stream?(name)
|
|
144
|
+
kept_streams.any? { |kept| name == kept || name.start_with?("#{kept}$") }
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# The replacement's own form: an earlier erasure's marker is no one's value.
|
|
148
|
+
REPLACEMENT = /\Aerased:/
|
|
149
|
+
|
|
150
|
+
# An event's envelope entries that name the record, never scrubbed.
|
|
151
|
+
ENVELOPE_KEYS = %w[model_class model_id operation timestamp replay].freeze
|
|
54
152
|
|
|
55
153
|
class << self
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
154
|
+
# +max_copies+: nil takes config.erasure_max_copies (10 by default); else a
|
|
155
|
+
# positive Integer (ArgumentError before anything is read or written).
|
|
156
|
+
# +dry_run+ (from 1.0.0): do everything in a transaction and roll it back,
|
|
157
|
+
# so the result says what the erasure would do (fields, events, copies,
|
|
158
|
+
# rows without events, values left as shared and their holders, what would
|
|
159
|
+
# be left behind) and nothing is written. A value left as shared cannot be
|
|
160
|
+
# found again once the person's own values are erased, so the threshold is
|
|
161
|
+
# chosen with a dry run first.
|
|
162
|
+
def erase!(model, id, reason:, fields: nil, erased_by: nil, everywhere: false, max_copies: nil,
|
|
163
|
+
any_fields: false, access_log: nil, access_log_subjects: [], dry_run: false, lifting: nil)
|
|
164
|
+
max_copies = Lyra.config.erasure_max_copies if max_copies.nil?
|
|
165
|
+
unless max_copies.is_a?(Integer) && max_copies.positive?
|
|
166
|
+
raise ArgumentError, "max_copies must be a positive Integer (got #{max_copies.inspect})"
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# A restriction of processing (Art. 18) refuses the erasure (dry run
|
|
170
|
+
# included) unless +lifting:+ names it, notice recorded: then it is lifted
|
|
171
|
+
# and the record erased in one transaction; a dry run lifts nothing and
|
|
172
|
+
# reports the restrictions it would lift in Result#lifting
|
|
173
|
+
# (Lyra::Rights::ErasureGuard).
|
|
174
|
+
lifts = if defined?(Rights::ErasureGuard)
|
|
175
|
+
Rights::ErasureGuard.check!(model, id, fields: fields, subjects: access_log_subjects, lifting: lifting)
|
|
176
|
+
else
|
|
177
|
+
[]
|
|
178
|
+
end
|
|
179
|
+
if lifts.any?
|
|
180
|
+
return model.transaction(requires_new: true) do
|
|
181
|
+
Rights::ErasureGuard.lift!(lifts, by: erased_by) unless dry_run
|
|
182
|
+
Rights::ErasureGuard.lifted(lifts) do
|
|
183
|
+
erase!(model, id, reason: reason, fields: fields, erased_by: erased_by, everywhere: everywhere,
|
|
184
|
+
max_copies: max_copies, any_fields: any_fields, access_log: access_log,
|
|
185
|
+
access_log_subjects: access_log_subjects, dry_run: dry_run)
|
|
186
|
+
end.tap { |result| result.lifting = lifts.map(&:id) }
|
|
187
|
+
end
|
|
188
|
+
end
|
|
61
189
|
|
|
62
|
-
|
|
190
|
+
stream = Lyra.stream_name(model, id)
|
|
191
|
+
# The record's access stream, and those of the ids the application used
|
|
192
|
+
# as this person's subject (validate_access!(…, subject: user_id)).
|
|
193
|
+
access_streams = [stream, *Array(access_log_subjects).map { |s| Lyra::AccessLog.erasable_key(s) }]
|
|
194
|
+
.uniq.map { |key| Lyra::AccessLog.stream_for(key) }
|
|
195
|
+
personal = []
|
|
63
196
|
rewritten = []
|
|
64
197
|
row_erased = false
|
|
65
198
|
copies = []
|
|
66
|
-
shared =
|
|
199
|
+
shared = []
|
|
200
|
+
rows_only = []
|
|
201
|
+
restricted_copies = []
|
|
202
|
+
left = []
|
|
203
|
+
not_searched = []
|
|
204
|
+
withheld = false
|
|
205
|
+
log_outcome = nil
|
|
206
|
+
log_entries = 0
|
|
207
|
+
touched = Thread.current[:lyra_erasure_touched] = []
|
|
67
208
|
|
|
68
|
-
PurposeBoundReads.internal do
|
|
69
|
-
|
|
70
|
-
|
|
209
|
+
PurposeBoundReads.internal(INTERNAL) do
|
|
210
|
+
# A savepoint for a dry run, so its rollback is real inside a caller's transaction.
|
|
211
|
+
model.transaction(requires_new: dry_run) do
|
|
212
|
+
# The locks first, the stream second: a write that is half-way between
|
|
213
|
+
# its row and its event (a Monitor update appends its event in the
|
|
214
|
+
# transaction that writes the row) must finish before the stream is
|
|
215
|
+
# read, or its event, committed in the window, keeps the value the
|
|
216
|
+
# erasure was asked to remove (0.9.3 and earlier read the stream first).
|
|
217
|
+
lock_record(model, id, stream)
|
|
218
|
+
events = Lyra.config.event_store.read.stream(stream).to_a
|
|
219
|
+
declared = personal_fields(model, events)
|
|
220
|
+
personal = (fields || declared).map(&:to_s).uniq
|
|
221
|
+
undeclared = any_fields || fields.nil? ? [] : personal - declared
|
|
222
|
+
if undeclared.any?
|
|
223
|
+
raise Unsupported, "#{model.name} #{id}: #{undeclared.join(', ')} not declared personal by its policy " \
|
|
224
|
+
"or the events' privacy stamps; pass any_fields: true to erase them anyway"
|
|
225
|
+
end
|
|
226
|
+
raise Unsupported, "#{model.name} #{id}: no personal fields to erase" if personal.empty?
|
|
227
|
+
|
|
228
|
+
replacements = personal.to_h { |field| [field, replacement(model, id, field)] }
|
|
229
|
+
erase_log = access_log.nil? ? Lyra.config.erase_access_log && complete?(declared, personal, events) : access_log
|
|
230
|
+
originals, own = original_values(model, id, events, personal)
|
|
231
|
+
# The origin as it was, for the links of its records (a belongs_to
|
|
232
|
+
# with primary_key: names one of its columns, which may be erased).
|
|
233
|
+
origin_state = current_state(model, id)
|
|
71
234
|
resolve = ->(key, value) { resolve_in(model, id, replacements, originals, key, value) }
|
|
72
235
|
rewritten = events.filter_map { |event| rewrite(event, resolve) }
|
|
73
|
-
|
|
74
|
-
row_erased = erase_row(model, id, replacements)
|
|
75
|
-
|
|
236
|
+
overwrite(rewritten) if rewritten.any?
|
|
237
|
+
row_erased, row_copies = erase_row(model, id, replacements, resolve)
|
|
238
|
+
# What the record's own row still holds (a JSON key, inside a
|
|
239
|
+
# longer string), reported as for other records.
|
|
240
|
+
left = [own_left_behind(model, id, originals.keys)].compact
|
|
241
|
+
withheld = withheld?(id, originals.keys)
|
|
242
|
+
log_outcome, log_entries = access_log!(access_streams, erase_log)
|
|
243
|
+
# A key column is never rewritten, so it is not named as erased.
|
|
244
|
+
record_erasure(model, id, stream, (personal | row_copies) - key_columns(model), reason, erased_by,
|
|
245
|
+
any_fields: any_fields, requested: fields, beyond_policy: any_fields ? personal - declared : [],
|
|
246
|
+
access_log: log_outcome, access_log_entries: log_entries,
|
|
247
|
+
access_log_subjects: access_streams.size - 1)
|
|
76
248
|
if everywhere
|
|
77
|
-
|
|
249
|
+
# Only the person's own values are searched for elsewhere (from
|
|
250
|
+
# 1.0.0): a third party's email nested under an "email" key in
|
|
251
|
+
# the person's data is erased here, but is not the person's.
|
|
252
|
+
copies, shared, others, rows_only, restricted_copies =
|
|
253
|
+
erase_copies(model, id, stream, origin_state, originals.slice(*own), replacements, reason, erased_by,
|
|
254
|
+
max_copies)
|
|
255
|
+
left += others
|
|
256
|
+
not_searched = encrypted_columns
|
|
78
257
|
end
|
|
258
|
+
raise ActiveRecord::Rollback if dry_run
|
|
79
259
|
end
|
|
80
260
|
end
|
|
81
261
|
forget_cached(model, id)
|
|
262
|
+
# A dry run's reads may have cached the erased state: drop it.
|
|
263
|
+
touched.each { |m, i| forget_cached(m, i) } if dry_run
|
|
82
264
|
|
|
83
265
|
Result.new(model: model.name, id: id, fields: personal, events_rewritten: rewritten.size,
|
|
84
|
-
row_erased: row_erased, copies: copies, shared_values: shared
|
|
266
|
+
row_erased: row_erased, copies: copies, shared_values: shared.size, shared: shared,
|
|
267
|
+
rows_without_events: rows_only, left_behind: left,
|
|
268
|
+
not_searched: not_searched, key_withheld: withheld,
|
|
269
|
+
access_log: log_outcome, access_log_entries: log_entries, dry_run: dry_run, lifting: [],
|
|
270
|
+
restricted_copies: restricted_copies)
|
|
271
|
+
ensure
|
|
272
|
+
Thread.current[:lyra_erasure_touched] = nil
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
# The subject's access log alone: for a record whose personal fields
|
|
276
|
+
# were all erased before (Retention's hard delete, when an earlier
|
|
277
|
+
# erasure kept the log). Records an ErasureApplied with no fields when
|
|
278
|
+
# the log held entries. Returns how many it deleted.
|
|
279
|
+
def erase_access_log!(model, id, reason:, erased_by: nil)
|
|
280
|
+
stream = Lyra.stream_name(model, id)
|
|
281
|
+
model.transaction do
|
|
282
|
+
_, entries = access_log!([Lyra::AccessLog.stream_for(stream)], true)
|
|
283
|
+
if entries.positive?
|
|
284
|
+
record_erasure(model, id, stream, [], reason, erased_by, access_log: "erased", access_log_entries: entries)
|
|
285
|
+
end
|
|
286
|
+
entries
|
|
287
|
+
end
|
|
85
288
|
end
|
|
86
289
|
|
|
87
290
|
private
|
|
88
291
|
|
|
292
|
+
# Rewrite +events+ in place, and their value-stream links with them
|
|
293
|
+
# (Projections::ReferenceLinks.relink_values): a link named by a digest
|
|
294
|
+
# of an erased unique value is deleted, one of its replacement added.
|
|
295
|
+
def overwrite(events)
|
|
296
|
+
Lyra.config.event_store.overwrite(events)
|
|
297
|
+
Lyra::Projections::ReferenceLinks.relink_values(events)
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
# Whether this erasure, with the ones before it (their ErasureApplied
|
|
301
|
+
# events), covers every personal field of the record.
|
|
302
|
+
def complete?(declared, personal, events)
|
|
303
|
+
earlier = events.select { |event| event.is_a?(Lyra::Events::ErasureApplied) }
|
|
304
|
+
.flat_map { |event| Array(event.data[:fields]).map(&:to_s) }
|
|
305
|
+
(declared - personal - earlier).empty?
|
|
306
|
+
end
|
|
307
|
+
|
|
89
308
|
def personal_fields(model, events)
|
|
90
309
|
declared = Lyra::Privacy.policy_for(model).declared_fields.map(&:to_s)
|
|
91
310
|
stamped = events.flat_map { |event| Lyra::Privacy.stamp_of(event)&.dig("fields")&.keys || [] }
|
|
92
311
|
declared | stamped
|
|
93
312
|
end
|
|
94
313
|
|
|
314
|
+
# A structured column (json, jsonb, an array, a serialized attribute)
|
|
315
|
+
# that is NOT NULL gets its own default, read as the database stores it
|
|
316
|
+
# (an int[] default '{}' is [], a jsonb default '{}' is {}), never a
|
|
317
|
+
# string marker.
|
|
95
318
|
def replacement(model, id, field)
|
|
96
319
|
column = model.columns_hash[field]
|
|
97
320
|
return nil if column.nil? || column.null
|
|
98
|
-
|
|
99
|
-
|
|
321
|
+
|
|
322
|
+
structured = structured?(model, field)
|
|
323
|
+
return "erased:#{id}" if %i[string text].include?(column.type) && !structured
|
|
324
|
+
unless column.default.nil?
|
|
325
|
+
type = model.type_for_attribute(field)
|
|
326
|
+
return structured ? type.deserialize(column.default) : type.cast(column.default)
|
|
327
|
+
end
|
|
100
328
|
|
|
101
329
|
raise Unsupported, "#{model.name}.#{field} is NOT NULL with no default; name a replacement by hand"
|
|
102
330
|
end
|
|
103
331
|
|
|
104
332
|
# value => the personal field it was found under, for every non-empty
|
|
105
|
-
# string the record's row or events held in a personal field.
|
|
333
|
+
# string the record's row or events held in a personal field. A plain
|
|
334
|
+
# value, or the strings of an array (an array column, a change's [old,
|
|
335
|
+
# new]), is the person's. Inside a hash (a jsonb declared personal) only
|
|
336
|
+
# the strings that are identifying for the field's type are: an email for
|
|
337
|
+
# :email, a phone number for :phone; a label ("home", "active") is not,
|
|
338
|
+
# or it would be erased wherever it appears. A string under a nested key
|
|
339
|
+
# that is itself a personal field counts as that field's. An earlier
|
|
340
|
+
# erasure's replacement ("erased:<id>") is no one's value.
|
|
341
|
+
#
|
|
342
|
+
# Returns [found, own]. +own+ are the values of the person's own fields:
|
|
343
|
+
# those at the top of a personal field, in the row and in the events
|
|
344
|
+
# (an attribute, either side of a change, a payload entry of one of the
|
|
345
|
+
# record's own CRUD events), with the strings inside a personal structure
|
|
346
|
+
# taken by the rule above. Only those are searched for in other records
|
|
347
|
+
# (everywhere). A value found only deeper, under a personal key nested
|
|
348
|
+
# in the person's data ({"manager" => {"email" => …}} in a payload), is
|
|
349
|
+
# someone else's: it is erased in this record by the key rule, as its
|
|
350
|
+
# events are, but not searched for elsewhere, where it would erase that
|
|
351
|
+
# person's own records (from 1.0.0).
|
|
106
352
|
def original_values(model, id, events, personal)
|
|
107
353
|
found = {}
|
|
354
|
+
own = []
|
|
355
|
+
policy = Lyra::Privacy.policy_for(model)
|
|
356
|
+
types = personal.to_h { |field| [field, policy.annotation(field)&.type&.to_sym] }
|
|
108
357
|
row = model.unscoped.find_by(model.primary_key => id) unless events_only?
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
358
|
+
if row
|
|
359
|
+
personal.each do |field|
|
|
360
|
+
value = row.read_attribute(field)
|
|
361
|
+
personal_strings(value, types[field]).each do |v|
|
|
362
|
+
remember(found, v, field)
|
|
363
|
+
own << v
|
|
364
|
+
end
|
|
365
|
+
collect(value, personal, found, types) if value.is_a?(Hash) || value.is_a?(Array)
|
|
366
|
+
end
|
|
367
|
+
end
|
|
368
|
+
events.each do |event|
|
|
369
|
+
data = opened(event.data)
|
|
370
|
+
collect(data, personal, found, types)
|
|
371
|
+
own.concat(top_level_values(data, personal, types, model, id))
|
|
372
|
+
end
|
|
373
|
+
[found, own.uniq & found.keys]
|
|
374
|
+
end
|
|
375
|
+
|
|
376
|
+
# The strings of +data+'s personal fields at the top of the record: in a
|
|
377
|
+
# CRUD envelope its attributes, each side of its changes and its
|
|
378
|
+
# payload's entries, when the envelope is the record's own; none otherwise.
|
|
379
|
+
def top_level_values(data, personal, types, model, id)
|
|
380
|
+
return [] unless own_envelope?(data, model, id)
|
|
381
|
+
|
|
382
|
+
sections = %w[attributes changes payload].map { |key| data[key] || data[key.to_sym] }
|
|
383
|
+
sections.grep(Hash).flat_map do |section|
|
|
384
|
+
section.flat_map { |key, value| personal.include?(key.to_s) ? personal_strings(value, types[key.to_s]) : [] }
|
|
385
|
+
end
|
|
386
|
+
end
|
|
387
|
+
|
|
388
|
+
# Whether +data+ is a CRUD envelope of the record +model+ +id+ itself
|
|
389
|
+
# (Monitor's, Hijack's, a domain event named after its write). Any other
|
|
390
|
+
# event in its stream (an invitation naming a friend's email) holds
|
|
391
|
+
# values the key rule erases there, but they are not the person's own
|
|
392
|
+
# and are not searched for elsewhere (from 1.0.0).
|
|
393
|
+
def own_envelope?(data, model, id)
|
|
394
|
+
return false unless data.is_a?(Hash) && Lyra::Event.recorded_operation(data)
|
|
395
|
+
|
|
396
|
+
owner = envelope_model(data)
|
|
397
|
+
record = data[:model_id] || data["model_id"]
|
|
398
|
+
owner && Lyra.stream_owner_name(owner) == Lyra.stream_owner_name(model) && record.to_s == id.to_s
|
|
112
399
|
end
|
|
113
400
|
|
|
114
|
-
def
|
|
401
|
+
def personal_strings(value, type)
|
|
402
|
+
case value
|
|
403
|
+
when String then [value]
|
|
404
|
+
when Array then value.flat_map { |inner| personal_strings(inner, type) }
|
|
405
|
+
when Hash then strings_in(value).select { |inner| identifying_string?(inner, type) }
|
|
406
|
+
else []
|
|
407
|
+
end
|
|
408
|
+
end
|
|
409
|
+
|
|
410
|
+
EMAIL = /\A[^@\s]+@[^@\s]+\.[^@\s]+\z/
|
|
411
|
+
|
|
412
|
+
# Whether +value+, found inside a personal structure of +type+, is
|
|
413
|
+
# itself an identifier of that type.
|
|
414
|
+
def identifying_string?(value, type)
|
|
415
|
+
digits = value.count("0-9")
|
|
416
|
+
case type
|
|
417
|
+
when :email then value.match?(EMAIL)
|
|
418
|
+
when :phone then value.match?(%r{\A\+?[\d\s().\-/]+\z}) && digits.between?(7, 15)
|
|
419
|
+
when :ip_address then ip_address?(value)
|
|
420
|
+
when :credit_card then value.match?(/\A[\d\s-]+\z/) && digits.between?(12, 19)
|
|
421
|
+
when :financial
|
|
422
|
+
value.match?(/\A[A-Z]{2}\d{2}[A-Z0-9 ]{10,32}\z/) || (value.match?(/\A[\d\s-]+\z/) && digits >= 6)
|
|
423
|
+
when :ssn, :identifier, :payment_token, :token, :credential, :online_identifier
|
|
424
|
+
!value.match?(/\s/) && value.length >= 6 && digits.positive?
|
|
425
|
+
else false
|
|
426
|
+
end
|
|
427
|
+
end
|
|
428
|
+
|
|
429
|
+
def ip_address?(value)
|
|
430
|
+
require "ipaddr"
|
|
431
|
+
IPAddr.new(value) && value.match?(/[.:]/)
|
|
432
|
+
rescue IPAddr::Error
|
|
433
|
+
false
|
|
434
|
+
end
|
|
435
|
+
|
|
436
|
+
def strings_in(value)
|
|
437
|
+
case value
|
|
438
|
+
when Hash then value.values.flat_map { |inner| strings_in(inner) }
|
|
439
|
+
when Array then value.flat_map { |inner| strings_in(inner) }
|
|
440
|
+
when String then [value]
|
|
441
|
+
else []
|
|
442
|
+
end
|
|
443
|
+
end
|
|
444
|
+
|
|
445
|
+
def collect(value, personal, found, types)
|
|
115
446
|
case value
|
|
116
447
|
when Hash
|
|
117
448
|
value.each do |key, inner|
|
|
118
449
|
if personal.include?(key.to_s)
|
|
119
|
-
|
|
120
|
-
else
|
|
121
|
-
collect(inner, personal, found, under: key.to_s)
|
|
450
|
+
personal_strings(inner, types[key.to_s]).each { |v| remember(found, v, key.to_s) }
|
|
122
451
|
end
|
|
452
|
+
collect(inner, personal, found, types)
|
|
123
453
|
end
|
|
124
|
-
when Array then value.each { |inner| collect(inner, personal, found,
|
|
454
|
+
when Array then value.each { |inner| collect(inner, personal, found, types) }
|
|
125
455
|
end
|
|
126
456
|
end
|
|
127
457
|
|
|
128
458
|
def remember(found, value, field)
|
|
129
|
-
found[value] ||= field if value.is_a?(String) && !value.empty?
|
|
459
|
+
found[value] ||= field if value.is_a?(String) && !value.empty? && !value.match?(REPLACEMENT)
|
|
130
460
|
end
|
|
131
461
|
|
|
132
462
|
# The replacement for +value+ found under +key+ in +model+ +id+'s data,
|
|
@@ -139,24 +469,129 @@ module Lyra
|
|
|
139
469
|
replacement_for(model, id, key, replacements[originals[value]])
|
|
140
470
|
end
|
|
141
471
|
|
|
142
|
-
# A copy held in +model+'s own column gets that column's
|
|
143
|
-
# (so the row and its events agree); elsewhere
|
|
472
|
+
# A copy held in +model+'s own scalar column gets that column's
|
|
473
|
+
# replacement (so the row and its events agree); elsewhere, inside a
|
|
474
|
+
# structured column included, the origin field's. A structured
|
|
475
|
+
# replacement ([] or {}) never stands for one value inside another: nil.
|
|
144
476
|
def replacement_for(model, id, key, fallback)
|
|
145
|
-
|
|
477
|
+
fallback = scalar(fallback)
|
|
478
|
+
return fallback unless model.columns_hash.key?(key) && !structured?(model, key)
|
|
479
|
+
|
|
480
|
+
replacement(model, id, key)
|
|
146
481
|
rescue Unsupported
|
|
147
482
|
fallback
|
|
148
483
|
end
|
|
149
484
|
|
|
485
|
+
def scalar(value) = value.is_a?(Hash) || value.is_a?(Array) ? nil : value
|
|
486
|
+
|
|
150
487
|
# The event with every personal value replaced, or nil if it carried none.
|
|
151
488
|
def rewrite(event, resolve)
|
|
152
|
-
data, changed =
|
|
489
|
+
data, changed = scrub_event(event.data, resolve)
|
|
153
490
|
return nil unless changed
|
|
154
491
|
|
|
155
492
|
event.class.new(event_id: event.event_id, data: data, metadata: event.metadata.to_h)
|
|
156
493
|
end
|
|
157
494
|
|
|
495
|
+
# An event's data. A CRUD envelope (an event that records an operation:
|
|
496
|
+
# Monitor's, Hijack's, a domain event named after a write, a bypass
|
|
497
|
+
# event) is scrubbed as its row is: each of its attributes as a column,
|
|
498
|
+
# each side of each change ([old, new]) as a column, the domain payload
|
|
499
|
+
# entry by entry. Its own entries are never replaced whole, whatever a
|
|
500
|
+
# policy calls its fields. An encrypted attribute's ciphertext is
|
|
501
|
+
# compared in clear, as the row's value is, and a replaced value sealed
|
|
502
|
+
# again. Any other event gets the key rule at its top level too.
|
|
503
|
+
def scrub_event(data, resolve)
|
|
504
|
+
return scrub(data, resolve) unless data.is_a?(Hash) && Lyra::Event.recorded_operation(data)
|
|
505
|
+
|
|
506
|
+
model = envelope_model(data)
|
|
507
|
+
encrypted = model ? Lyra::EncryptedAttributes.names(model) : []
|
|
508
|
+
keys = model ? key_columns(model) : []
|
|
509
|
+
changed = false
|
|
510
|
+
copy = data.to_h do |key, value|
|
|
511
|
+
new_value, did = case key.to_s
|
|
512
|
+
when *ENVELOPE_KEYS then [value, false]
|
|
513
|
+
when "attributes"
|
|
514
|
+
scrub_columns(value, resolve, keys) { |c, v| scrub_column(model, encrypted, c, v, resolve) }
|
|
515
|
+
when "changes"
|
|
516
|
+
scrub_columns(value, resolve, keys) { |c, v| scrub_change(model, encrypted, c, v, resolve) }
|
|
517
|
+
when "payload" then scrub(value, resolve)
|
|
518
|
+
else scrub(value, resolve, key: key.to_s)
|
|
519
|
+
end
|
|
520
|
+
changed ||= did
|
|
521
|
+
[key, new_value]
|
|
522
|
+
end
|
|
523
|
+
[copy, changed]
|
|
524
|
+
end
|
|
525
|
+
|
|
526
|
+
# +keys+: the record's key columns, never rewritten (scrub_row).
|
|
527
|
+
def scrub_columns(values, resolve, keys)
|
|
528
|
+
return scrub(values, resolve) unless values.is_a?(Hash)
|
|
529
|
+
|
|
530
|
+
changed = false
|
|
531
|
+
copy = values.to_h do |column, value|
|
|
532
|
+
next [column, value] if keys.include?(column.to_s)
|
|
533
|
+
|
|
534
|
+
new_value, did = yield(column.to_s, value)
|
|
535
|
+
changed ||= did
|
|
536
|
+
[column, new_value]
|
|
537
|
+
end
|
|
538
|
+
[copy, changed]
|
|
539
|
+
end
|
|
540
|
+
|
|
541
|
+
def scrub_change(model, encrypted, column, change, resolve)
|
|
542
|
+
return scrub_column(model, encrypted, column, change, resolve) unless change.is_a?(Array)
|
|
543
|
+
|
|
544
|
+
results = change.map { |side| scrub_column(model, encrypted, column, side, resolve) }
|
|
545
|
+
[results.map(&:first), results.any?(&:last)]
|
|
546
|
+
end
|
|
547
|
+
|
|
548
|
+
# One column's value in an event, as the row's is scrubbed.
|
|
549
|
+
def scrub_column(model, encrypted, column, value, resolve)
|
|
550
|
+
return scrub(value, resolve, key: column) unless encrypted.include?(column)
|
|
551
|
+
|
|
552
|
+
clear = Lyra::EncryptedAttributes.open(model, { column => value })[column]
|
|
553
|
+
new_value, did = scrub(clear, resolve, key: column)
|
|
554
|
+
did ? [Lyra::EncryptedAttributes.seal_value(model, column, new_value), true] : [value, false]
|
|
555
|
+
end
|
|
556
|
+
|
|
557
|
+
# +data+ with an envelope's encrypted attributes in clear, to read
|
|
558
|
+
# values from (never written back).
|
|
559
|
+
def opened(data)
|
|
560
|
+
return data unless data.is_a?(Hash) && Lyra::Event.recorded_operation(data)
|
|
561
|
+
|
|
562
|
+
model = envelope_model(data)
|
|
563
|
+
encrypted = model ? Lyra::EncryptedAttributes.names(model) : []
|
|
564
|
+
return data if encrypted.empty?
|
|
565
|
+
|
|
566
|
+
open = ->(column, value) { Lyra::EncryptedAttributes.open(model, { column.to_s => value })[column.to_s] }
|
|
567
|
+
data.to_h do |key, value|
|
|
568
|
+
next [key, value] unless value.is_a?(Hash) && %w[attributes changes].include?(key.to_s)
|
|
569
|
+
|
|
570
|
+
[key, value.to_h do |column, inner|
|
|
571
|
+
[column, key.to_s == "changes" && inner.is_a?(Array) ? inner.map { open.(column, _1) } : open.(column, inner)]
|
|
572
|
+
end]
|
|
573
|
+
end
|
|
574
|
+
end
|
|
575
|
+
|
|
576
|
+
def envelope_model(data)
|
|
577
|
+
name = data[:model_class] || data["model_class"]
|
|
578
|
+
return nil if name.nil?
|
|
579
|
+
|
|
580
|
+
model = Lyra.config.monitored_model_named(name) || name.to_s.safe_constantize
|
|
581
|
+
model.is_a?(Class) && model < ActiveRecord::Base ? model : nil
|
|
582
|
+
end
|
|
583
|
+
|
|
158
584
|
# [copy of +value+ with personal values replaced, whether any was].
|
|
585
|
+
# A hash or an array under a personal field's own name (a json or array
|
|
586
|
+
# column declared personal) is replaced whole, by the key rule, in the
|
|
587
|
+
# row and in its events alike; one under any other name is recursed
|
|
588
|
+
# into.
|
|
159
589
|
def scrub(value, resolve, key: nil)
|
|
590
|
+
if key && (value.is_a?(Hash) || value.is_a?(Array))
|
|
591
|
+
replaced, changed = scrub_value(value, resolve, key)
|
|
592
|
+
return [replaced, true] if changed
|
|
593
|
+
end
|
|
594
|
+
|
|
160
595
|
case value
|
|
161
596
|
when Hash
|
|
162
597
|
changed = false
|
|
@@ -170,89 +605,379 @@ module Lyra
|
|
|
170
605
|
results = value.map { |inner| scrub(inner, resolve, key: key) }
|
|
171
606
|
[results.map(&:first), results.any?(&:last)]
|
|
172
607
|
else
|
|
173
|
-
|
|
174
|
-
replaced == :keep || replaced == value ? [value, false] : [replaced, true]
|
|
608
|
+
scrub_value(value, resolve, key)
|
|
175
609
|
end
|
|
176
610
|
end
|
|
177
611
|
|
|
178
|
-
def
|
|
179
|
-
|
|
612
|
+
def scrub_value(value, resolve, key)
|
|
613
|
+
replaced = resolve.call(key, value)
|
|
614
|
+
replaced == :keep || replaced == value ? [value, false] : [replaced, true]
|
|
615
|
+
end
|
|
616
|
+
|
|
617
|
+
# The record's aggregate advisory lock, then its row: the locks a writer
|
|
618
|
+
# holds (a Monitor or Hijack write keeps the row until it commits, with its
|
|
619
|
+
# event appended in the same transaction; the other modes take the advisory
|
|
620
|
+
# lock), in Lyra's order, the aggregate lock first (Lyra::AdvisoryLock).
|
|
621
|
+
# Waiting here is what lets an in-flight write finish before the stream is
|
|
622
|
+
# read. Taken the other way round (row, then advisory lock), it deadlocked
|
|
623
|
+
# against a write that skips callbacks, which takes them in this order.
|
|
624
|
+
def lock_record(model, id, stream)
|
|
625
|
+
model.with_connection do |connection|
|
|
626
|
+
Lyra::AdvisoryLock.xact_lock(connection, "lyra_aggregate/#{stream}", purpose: "erasure")
|
|
627
|
+
end
|
|
628
|
+
model.unscoped.lock.find_by(model.primary_key => id) unless events_only?
|
|
629
|
+
end
|
|
630
|
+
|
|
631
|
+
# The record's row, every column of it scrubbed with the rule its events
|
|
632
|
+
# got (+resolve+: the origin's key and value rule). That covers the
|
|
633
|
+
# erased fields, any other column that holds one of their values (a
|
|
634
|
+
# username that is the email), and, from 1.0.0, values inside a json,
|
|
635
|
+
# jsonb or array column: 0.9.3 and earlier left those in the row while
|
|
636
|
+
# the events lost them, and the row no longer replayed to its events.
|
|
637
|
+
# Returns [erased?, the changed columns other than the erased fields].
|
|
638
|
+
def erase_row(model, id, replacements, resolve)
|
|
639
|
+
return [false, []] if events_only?
|
|
180
640
|
|
|
181
641
|
row = model.unscoped.lock.find_by(model.primary_key => id)
|
|
182
|
-
return false unless row
|
|
642
|
+
return [false, []] unless row
|
|
643
|
+
|
|
644
|
+
columns = scrub_row(model, row, resolve)
|
|
645
|
+
write_with_lock(model, id, row, columns)
|
|
646
|
+
[true, columns.keys - replacements.keys]
|
|
647
|
+
end
|
|
183
648
|
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
649
|
+
# The row's columns whose value +resolve+ changes => their new value.
|
|
650
|
+
# Each value is scrubbed as the attribute's type gives it, which is how
|
|
651
|
+
# its events hold it: a hash or an array (json, jsonb, hstore, a
|
|
652
|
+
# PostgreSQL array, a serialized attribute, a text column the model casts
|
|
653
|
+
# as json) is recursed into and written back whole; anything else is one
|
|
654
|
+
# value, so a text column that holds JSON is compared as a string, as in
|
|
655
|
+
# its events. Only the table's columns: a virtual attribute (attribute :x)
|
|
656
|
+
# has none to write. Never a key column (key_columns). An encrypted
|
|
657
|
+
# column is read in clear, as its events are compared, and Active Record
|
|
658
|
+
# encrypts the replacement as it writes it.
|
|
659
|
+
def scrub_row(model, row, resolve)
|
|
660
|
+
keys = key_columns(model)
|
|
661
|
+
row.attributes.slice(*(model.column_names - keys)).each_with_object({}) do |(column, value), changed|
|
|
662
|
+
new_value, did = scrub(value, resolve, key: column)
|
|
663
|
+
changed[column] = new_value if did
|
|
664
|
+
end
|
|
187
665
|
end
|
|
188
666
|
|
|
189
|
-
#
|
|
190
|
-
#
|
|
191
|
-
#
|
|
192
|
-
#
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
667
|
+
# The primary key and the foreign keys (and polymorphic types): never
|
|
668
|
+
# rewritten. The record's stream is named after its key, and a foreign
|
|
669
|
+
# key is another record's. One that holds an erased value is reported
|
|
670
|
+
# (left_behind), never erased.
|
|
671
|
+
def key_columns(model)
|
|
672
|
+
keys = [model.primary_key.to_s]
|
|
673
|
+
model.reflect_on_all_associations(:belongs_to).each do |reflection|
|
|
674
|
+
keys << reflection.foreign_key.to_s
|
|
675
|
+
keys << reflection.foreign_type.to_s if reflection.polymorphic?
|
|
676
|
+
end
|
|
677
|
+
keys
|
|
678
|
+
end
|
|
679
|
+
|
|
680
|
+
# Whether the attribute's type gives a structure (a hash or an array).
|
|
681
|
+
def structured?(model, column)
|
|
682
|
+
definition = model.columns_hash[column]
|
|
683
|
+
return false unless definition
|
|
684
|
+
return true if definition.respond_to?(:array?) && definition.array?
|
|
685
|
+
|
|
686
|
+
type = model.type_for_attribute(column)
|
|
687
|
+
%i[json jsonb hstore].include?(type.type) || type.is_a?(ActiveRecord::Type::Serialized)
|
|
688
|
+
end
|
|
689
|
+
|
|
690
|
+
# The columns of +model+ +id+'s row that still hold one of +values+
|
|
691
|
+
# anywhere: as a JSON key, or inside a longer string (a text column
|
|
692
|
+
# holding JSON). The erasure replaces whole values only, so those are
|
|
693
|
+
# left; this is what tells them apart from an erased copy.
|
|
694
|
+
# In ES-NoProj, where there is no row, the record's replayed state.
|
|
695
|
+
def columns_holding(model, id, values)
|
|
696
|
+
return [] if values.empty?
|
|
697
|
+
|
|
698
|
+
if events_only?
|
|
699
|
+
state = Lyra::DualView.new(model, id).event_sourced_state[:state] || {}
|
|
700
|
+
return state.filter_map { |column, value| column.to_s if holds?(value, values) }
|
|
701
|
+
end
|
|
702
|
+
|
|
703
|
+
row = model.unscoped.find_by(model.primary_key => id)
|
|
704
|
+
return [] unless row
|
|
705
|
+
|
|
706
|
+
row.attributes.slice(*model.column_names).filter_map do |column, value|
|
|
707
|
+
# citext compares without case, so a copy differing in case is the value there.
|
|
708
|
+
fold = model.columns_hash[column]&.type == :citext
|
|
709
|
+
column if holds?(value, values, fold: fold)
|
|
710
|
+
end
|
|
711
|
+
end
|
|
712
|
+
|
|
713
|
+
# The record's own row (or replayed state), as an entry of left_behind,
|
|
714
|
+
# or nil when it holds none of +values+.
|
|
715
|
+
def own_left_behind(model, id, values)
|
|
716
|
+
held = columns_holding(model, id, values)
|
|
717
|
+
{ record: label(model, id, values), columns: held } if held.any?
|
|
718
|
+
end
|
|
719
|
+
|
|
720
|
+
# "Model id", or "Model <key withheld>" when the key holds one of
|
|
721
|
+
# +values+: a result, a task's output and an ErasureApplied reason never
|
|
722
|
+
# print an erased value.
|
|
723
|
+
def label(model, id, values) = "#{model.name} #{withheld?(id, values) ? WITHHELD : id}"
|
|
724
|
+
|
|
725
|
+
def withheld?(id, values) = holds?(id.to_s, values)
|
|
726
|
+
|
|
727
|
+
def holds?(value, values, fold: false)
|
|
728
|
+
case value
|
|
729
|
+
when Hash then value.any? { |key, inner| holds?(key.to_s, values, fold: fold) || holds?(inner, values, fold: fold) }
|
|
730
|
+
when Array then value.any? { |inner| holds?(inner, values, fold: fold) }
|
|
731
|
+
when String
|
|
732
|
+
fold ? values.any? { |v| value.downcase.include?(v.downcase) } : values.any? { |v| value.include?(v) }
|
|
733
|
+
else false
|
|
734
|
+
end
|
|
735
|
+
end
|
|
736
|
+
|
|
737
|
+
# With everywhere: the encrypted columns of the monitored models. The
|
|
738
|
+
# search is a text match on the stored events, which hold their
|
|
739
|
+
# ciphertext, so a copy there cannot be found; they are named, never
|
|
740
|
+
# searched (decrypting the whole log is not done).
|
|
741
|
+
def encrypted_columns
|
|
742
|
+
Lyra.config.monitored_models.flat_map do |model|
|
|
743
|
+
Lyra::EncryptedAttributes.names(model).map { |column| "#{model.name}.#{column}" }
|
|
744
|
+
end.uniq
|
|
745
|
+
end
|
|
746
|
+
|
|
747
|
+
# The erasure is a write, and moves the optimistic lock column as any
|
|
748
|
+
# write does: a copy loaded before it fails with StaleObjectError
|
|
749
|
+
# instead of saving the erased values back. The move is recorded as an
|
|
750
|
+
# ordinary update of that column alone (source lyra_erasure), so every
|
|
751
|
+
# replay (DualView, the mode checks, ES-NoProj reads, rebuilds and
|
|
752
|
+
# projections) gives the row's value. ErasureApplied is not replayed,
|
|
753
|
+
# and the rewritten events keep their place in history, so neither can
|
|
754
|
+
# carry it. A model without optimistic locking gets no such event.
|
|
755
|
+
def write_with_lock(model, id, row, columns)
|
|
756
|
+
return write_columns(model, id, columns) if columns.empty? || !model.locking_enabled?
|
|
757
|
+
|
|
758
|
+
lock = model.locking_column
|
|
759
|
+
# In ES-Async the row can trail the log: move from the later of the
|
|
760
|
+
# two, so the log's version sequence never repeats.
|
|
761
|
+
was = [row.read_attribute(lock), logged_version(model, id, lock)].compact.max
|
|
762
|
+
moved = was.to_i + 1
|
|
763
|
+
write_columns(model, id, columns.merge(lock => moved))
|
|
764
|
+
data = Lyra::EventData.build(model, id, :updated, changes: { lock => [was, moved] })
|
|
765
|
+
metadata = { source: SOURCE, lock_moved_by_erasure: true, correlation_id: Lyra::Correlation.current_id }.compact
|
|
766
|
+
event = Lyra::BypassEvents.event_class_for(model, :updated)
|
|
767
|
+
.new(data: data, metadata: Lyra::Privacy.stamp(model, data, metadata))
|
|
768
|
+
Lyra.append_events(event, stream_name: Lyra.stream_name(model, id))
|
|
769
|
+
end
|
|
770
|
+
|
|
771
|
+
# The version the record's stream last recorded, or nil.
|
|
772
|
+
def logged_version(model, id, lock)
|
|
773
|
+
Lyra.config.event_store.read.stream(Lyra.stream_name(model, id)).backward.each do |event|
|
|
774
|
+
version = Lyra::OptimisticLocking.version_in(event.data, lock.to_s)
|
|
775
|
+
return version unless version.nil?
|
|
776
|
+
end
|
|
777
|
+
nil
|
|
778
|
+
end
|
|
779
|
+
|
|
780
|
+
# everywhere: the records other than the erased one that hold its
|
|
781
|
+
# values, found through their events (events_mentioning, one text search
|
|
782
|
+
# of the log for every value) and, from 1.0.0, by searching the tables of
|
|
783
|
+
# monitored models with a policy directly (row_candidates: a row can hold
|
|
784
|
+
# a value its events do not). Returns [records erased, values left as
|
|
785
|
+
# shared ({ field:, type:, holders: }), records left behind, records
|
|
786
|
+
# erased in the row only].
|
|
787
|
+
#
|
|
788
|
+
# A value held by more than +max_copies+ other records is shared by many
|
|
789
|
+
# people, not a copy of this one's: erasing it everywhere erased
|
|
790
|
+
# everyone's (a replay's placeholder address took all 1,954 addresses
|
|
791
|
+
# with it). It is left, and reported. Holders are counted by record:
|
|
792
|
+
# through their events (an exact match on each candidate event) and
|
|
793
|
+
# their rows (the table search's match) alike. The person's linked
|
|
794
|
+
# records (Lyra::Privacy::SubjectLinks: a belongs_to to the erased
|
|
795
|
+
# record's model naming it) do not count (from 1.0.0). A record is
|
|
796
|
+
# listed as erased only if its row, when it has one, no longer holds the
|
|
797
|
+
# value; one whose row still holds it (as a JSON key, inside a longer
|
|
798
|
+
# string, in a key column, in another case in a citext column) is listed
|
|
799
|
+
# as left behind, with the columns, never the value.
|
|
800
|
+
def erase_copies(origin, origin_id, origin_stream, origin_state, originals, replacements, reason, erased_by,
|
|
801
|
+
max_copies)
|
|
802
|
+
none = [[], [], [], [], []]
|
|
803
|
+
return none if originals.empty?
|
|
198
804
|
|
|
199
805
|
policy = Lyra::Privacy.policy_for(origin)
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
candidates = events_mentioning(
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
806
|
+
type_of = ->(field) { policy.annotation(field)&.type&.to_sym }
|
|
807
|
+
originals = originals.select { |_value, field| IDENTIFYING_TYPES.include?(type_of.(field)) }
|
|
808
|
+
return none if originals.empty?
|
|
809
|
+
|
|
810
|
+
values = originals.keys
|
|
811
|
+
candidates = events_mentioning(values)
|
|
812
|
+
streams = event_streams(candidates.map(&:event_id))
|
|
813
|
+
candidates = candidates.reject do |event|
|
|
814
|
+
names = streams.fetch(event.event_id, [])
|
|
815
|
+
names.include?(origin_stream) || names.any? { |name| kept_stream?(name) }
|
|
816
|
+
end
|
|
817
|
+
rows = events_only? ? {} : row_candidates(values, origin_stream)
|
|
818
|
+
|
|
819
|
+
holders = values.to_h { |value| [value, Set.new] }
|
|
820
|
+
candidates.each do |event|
|
|
821
|
+
data = opened(event.data)
|
|
822
|
+
holder = holder_of(streams.fetch(event.event_id, []))
|
|
823
|
+
values.each { |value| holders[value] << holder if holds_whole?(data, value) }
|
|
824
|
+
end
|
|
825
|
+
rows.each { |stream, row| row[:values].each { |value| holders[value] << stream } }
|
|
826
|
+
linked = nil
|
|
827
|
+
shared = values.filter_map do |value|
|
|
828
|
+
next if holders[value].size <= max_copies
|
|
829
|
+
|
|
830
|
+
linked ||= linked_streams(origin, origin_id, origin_state)
|
|
831
|
+
count = (holders[value] - linked).size
|
|
832
|
+
[value, count] if count > max_copies
|
|
833
|
+
end
|
|
834
|
+
report = shared.map { |value, count| { field: originals[value], type: type_of.(originals[value]), holders: count } }
|
|
835
|
+
originals = originals.except(*shared.map(&:first))
|
|
836
|
+
return [[], report, [], [], []] if originals.empty?
|
|
837
|
+
|
|
838
|
+
values = originals.keys
|
|
839
|
+
# The candidates for the values erased: a text match without case, as
|
|
840
|
+
# the search was (a citext copy differing in case is reported).
|
|
841
|
+
candidates = candidates.select { |event| holds?(opened(event.data), values, fold: true) }
|
|
842
|
+
by_record = candidates.group_by { |event| record_in(streams.fetch(event.event_id, [])) }
|
|
843
|
+
copy_reason = "#{reason} (copies of #{label(origin, origin_id, values)})"
|
|
844
|
+
copies = []
|
|
845
|
+
left = []
|
|
846
|
+
rows_only = []
|
|
847
|
+
|
|
848
|
+
# Events in no monitored record's stream: their events only.
|
|
849
|
+
by_record.each do |(model, _id), events|
|
|
850
|
+
next if model
|
|
851
|
+
|
|
852
|
+
rewritten = events.filter_map { |event| rewrite(event, holder_resolve(nil, nil, originals, replacements)) }
|
|
214
853
|
next if rewritten.empty?
|
|
215
854
|
|
|
216
|
-
|
|
217
|
-
|
|
855
|
+
overwrite(rewritten)
|
|
856
|
+
copies << "#{rewritten.size} events"
|
|
857
|
+
end
|
|
218
858
|
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
859
|
+
targets = by_record.keys.filter_map { |model, id| [Lyra.stream_name(model, id), model, id] if model }
|
|
860
|
+
rows.each { |stream, row| targets << [stream, row[:model], row[:id]] if row[:values].intersect?(values) }
|
|
861
|
+
restricted = []
|
|
862
|
+
targets.uniq(&:first).each do |stream, model, id|
|
|
863
|
+
# A holder under a restriction of processing (Art. 18) keeps its
|
|
864
|
+
# copy until the restriction is lifted: reported, never modified.
|
|
865
|
+
if restricted_holder?(model, id)
|
|
866
|
+
restricted << { record: label(model, id, values), columns: columns_holding(model, id, values) }
|
|
867
|
+
next
|
|
868
|
+
end
|
|
869
|
+
|
|
870
|
+
erased, row_only = erase_holder(model, id, stream, originals, replacements, copy_reason, erased_by)
|
|
871
|
+
held = columns_holding(model, id, values)
|
|
872
|
+
name = label(model, id, values)
|
|
873
|
+
left << { record: name, columns: held } if held.any?
|
|
874
|
+
copies << name if erased && held.empty?
|
|
875
|
+
rows_only << name if row_only
|
|
876
|
+
end
|
|
877
|
+
[copies, report, left, rows_only, restricted]
|
|
225
878
|
end
|
|
226
879
|
|
|
227
|
-
#
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
880
|
+
# Whether a restriction in force covers the holder +model+ +id+ (with
|
|
881
|
+
# PAM DSL's rights; Lyra::Rights::ErasureGuard).
|
|
882
|
+
def restricted_holder?(model, id)
|
|
883
|
+
return false unless defined?(Rights::ErasureGuard) && PamDsl.rights_store.any_active?
|
|
884
|
+
|
|
885
|
+
record = Rights::SubjectKeys.find_record(model, id)
|
|
886
|
+
record ? Rights::ErasureGuard.covering_record(record).any? : false
|
|
887
|
+
end
|
|
888
|
+
|
|
889
|
+
# The value rule only, for a record other than the erased one: a copy
|
|
890
|
+
# of an erased value is replaced, a third party's value under a
|
|
891
|
+
# personal key is kept.
|
|
892
|
+
def holder_resolve(model, id, originals, replacements)
|
|
893
|
+
lambda do |key, value|
|
|
894
|
+
next :keep unless value.is_a?(String) && originals.key?(value)
|
|
895
|
+
|
|
896
|
+
fallback = replacements[originals[value]]
|
|
897
|
+
model ? replacement_for(model, id, key, fallback) : scalar(fallback)
|
|
236
898
|
end
|
|
237
899
|
end
|
|
238
900
|
|
|
239
|
-
|
|
901
|
+
# One record holding a copy, under its locks (row, then advisory lock),
|
|
902
|
+
# its stream read again: a copy written since the search is in it. Its
|
|
903
|
+
# events are rewritten by the value rule, and its row is scrubbed by the
|
|
904
|
+
# same rule whether or not its events held the value whole (from 1.0.0:
|
|
905
|
+
# a row given the value by raw SQL whose events mention it only inside a
|
|
906
|
+
# longer string, or not at all). When only the row held it, its
|
|
907
|
+
# ErasureApplied says row_only: its row already differed from its
|
|
908
|
+
# replay. A row with no stream is erased and then imported
|
|
909
|
+
# (erase_and_import). Returns [erased?, erased in the row only?].
|
|
910
|
+
def erase_holder(model, id, stream, originals, replacements, reason, erased_by)
|
|
911
|
+
lock_record(model, id, stream)
|
|
912
|
+
events = Lyra.config.event_store.read.stream(stream).to_a
|
|
913
|
+
if events.empty? && !events_only?
|
|
914
|
+
# No Genesis run elsewhere imports the row as it was meanwhile.
|
|
915
|
+
Lyra::Genesis.hold_lock!(model)
|
|
916
|
+
events = Lyra.config.event_store.read.stream(stream).to_a
|
|
917
|
+
end
|
|
918
|
+
resolve = holder_resolve(model, id, originals, replacements)
|
|
919
|
+
return erase_and_import(model, id, stream, resolve, reason, erased_by) if events.empty? && !events_only?
|
|
920
|
+
|
|
921
|
+
rewritten = events.filter_map { |event| rewrite(event, resolve) }
|
|
922
|
+
overwrite(rewritten) if rewritten.any?
|
|
923
|
+
copied = erase_copied_columns(model, id, resolve)
|
|
924
|
+
return [false, false] if copied.empty? && rewritten.empty?
|
|
925
|
+
|
|
926
|
+
row_only = rewritten.empty?
|
|
927
|
+
record_erasure(model, id, stream, copied, reason, erased_by, row_only: row_only)
|
|
928
|
+
forget_cached(model, id)
|
|
929
|
+
[true, row_only]
|
|
930
|
+
end
|
|
931
|
+
|
|
932
|
+
# A row with no stream: erased, then imported the Genesis way, so its
|
|
933
|
+
# Imported event holds the erased values, never the personal ones (an
|
|
934
|
+
# ErasureApplied alone would start a stream Genesis never imports: it
|
|
935
|
+
# imports only rows with no stream), then its ErasureApplied. The lock
|
|
936
|
+
# column moves as with any erasure; the Imported event holds its new
|
|
937
|
+
# value. If the row was imported meanwhile (by a Genesis run on a
|
|
938
|
+
# connection the lock did not hold back), that stream is read again and
|
|
939
|
+
# rewritten instead, and no import is claimed.
|
|
940
|
+
def erase_and_import(model, id, stream, resolve, reason, erased_by)
|
|
941
|
+
row = model.unscoped.lock.find_by(model.primary_key => id)
|
|
942
|
+
return [false, false] unless row
|
|
943
|
+
|
|
944
|
+
columns = scrub_row(model, row, resolve)
|
|
945
|
+
return [false, false] if columns.empty?
|
|
946
|
+
|
|
947
|
+
lock = model.locking_enabled? ? model.locking_column : nil
|
|
948
|
+
write_columns(model, id, lock ? columns.merge(lock => row.read_attribute(lock).to_i + 1) : columns)
|
|
949
|
+
imported = Lyra::Genesis.import_one(model, id)
|
|
950
|
+
unless imported
|
|
951
|
+
rewritten = Lyra.config.event_store.read.stream(stream).to_a.filter_map { |event| rewrite(event, resolve) }
|
|
952
|
+
overwrite(rewritten) if rewritten.any?
|
|
953
|
+
end
|
|
954
|
+
record_erasure(model, id, stream, columns.keys, reason, erased_by, imported: imported)
|
|
955
|
+
forget_cached(model, id)
|
|
956
|
+
[true, false]
|
|
957
|
+
end
|
|
958
|
+
|
|
959
|
+
# event_id => the streams the event is in, in one query.
|
|
960
|
+
def event_streams(event_ids)
|
|
961
|
+
return {} if event_ids.empty?
|
|
962
|
+
|
|
240
963
|
ActiveRecord::Base.with_connection do |conn|
|
|
241
|
-
|
|
242
|
-
"
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
ids.filter_map { |event_id| Lyra.config.event_store.read.event(event_id) rescue nil }
|
|
964
|
+
event_ids.each_slice(1_000).flat_map do |ids|
|
|
965
|
+
conn.select_rows("SELECT event_id, stream FROM event_store_events_in_streams " \
|
|
966
|
+
"WHERE event_id IN (#{ids.map { |i| conn.quote(i) }.join(', ')})")
|
|
967
|
+
end.group_by(&:first).transform_values { |rows| rows.map(&:last) }
|
|
246
968
|
end
|
|
247
969
|
end
|
|
248
970
|
|
|
249
|
-
|
|
250
|
-
|
|
971
|
+
# The holder an event counts for: the monitored record whose stream it is
|
|
972
|
+
# in (so a reference link, Model.fk$id, adds none), else its stream.
|
|
973
|
+
def holder_of(names)
|
|
974
|
+
model, id = record_in(names)
|
|
975
|
+
model ? Lyra.stream_name(model, id) : names.min
|
|
251
976
|
end
|
|
252
977
|
|
|
253
|
-
# [model, id]
|
|
254
|
-
def
|
|
255
|
-
|
|
978
|
+
# [model, id] of the monitored record whose stream is among +names+, else [nil, nil].
|
|
979
|
+
def record_in(names)
|
|
980
|
+
names.each do |name|
|
|
256
981
|
model_name, id = name.split("$", 2)
|
|
257
982
|
model = Lyra.config.monitored_model_named(model_name)
|
|
258
983
|
return [model, id] if model && id
|
|
@@ -260,20 +985,170 @@ module Lyra
|
|
|
260
985
|
[nil, nil]
|
|
261
986
|
end
|
|
262
987
|
|
|
263
|
-
#
|
|
264
|
-
#
|
|
265
|
-
|
|
988
|
+
# "%"…"%": +value+ as a JSON string, quotes included, for a LIKE on a
|
|
989
|
+
# json or jsonb column as text: it matches the value whole, as a value
|
|
990
|
+
# or a key, not inside a longer one.
|
|
991
|
+
def json_pattern(value)
|
|
992
|
+
"%#{ActiveRecord::Base.sanitize_sql_like(JSON.generate(value))}%"
|
|
993
|
+
end
|
|
994
|
+
|
|
995
|
+
# The streams of the person's linked records, by the rule of the
|
|
996
|
+
# subject-access copy (Lyra::Privacy::SubjectLinks): the records of
|
|
997
|
+
# monitored models with a belongs_to to +origin+'s model (any
|
|
998
|
+
# association: an owner, an assignee, a manager), a polymorphic one with
|
|
999
|
+
# its type column naming it, or a <model>_id column no association owns,
|
|
1000
|
+
# whose foreign key holds the origin's id, or the column the belongs_to
|
|
1001
|
+
# names as its primary_key (+origin_state+: the origin as it was before
|
|
1002
|
+
# this erasure). Records of the origin's own model are not linked. They
|
|
1003
|
+
# are taken as the person's own for the threshold only (they do not
|
|
1004
|
+
# count toward max_copies); they are erased as any holder is. Each query
|
|
1005
|
+
# runs in a savepoint: one that fails finds none.
|
|
1006
|
+
def linked_streams(origin, origin_id, origin_state)
|
|
1007
|
+
links = Lyra::Privacy::SubjectLinks.new(Lyra.stream_owner_name(origin))
|
|
1008
|
+
links.linked_models.each_with_object(Set.new) do |model, linked|
|
|
1009
|
+
links.for(model).each do |link|
|
|
1010
|
+
target = link_target(model, link, origin, origin_id, origin_state)
|
|
1011
|
+
next if target.nil?
|
|
1012
|
+
|
|
1013
|
+
scope = model.unscoped.where(link.fk => target)
|
|
1014
|
+
scope = scope.where(link.type_column => link.type_value) if link.type_column
|
|
1015
|
+
ids = model.transaction(requires_new: true) { scope.pluck(model.primary_key) }
|
|
1016
|
+
ids.each { |id| linked << Lyra.stream_name(model, id) }
|
|
1017
|
+
rescue StandardError
|
|
1018
|
+
next
|
|
1019
|
+
end
|
|
1020
|
+
end
|
|
1021
|
+
end
|
|
1022
|
+
|
|
1023
|
+
# What +link+'s foreign key holds for the origin: its id, or the value of
|
|
1024
|
+
# the column a belongs_to names with primary_key:.
|
|
1025
|
+
def link_target(model, link, origin, origin_id, origin_state)
|
|
1026
|
+
reflection = model.reflect_on_all_associations(:belongs_to)
|
|
1027
|
+
.find { |r| !r.polymorphic? && r.foreign_key.to_s == link.fk }
|
|
1028
|
+
column = reflection&.association_primary_key.to_s
|
|
1029
|
+
return origin_id if column.empty? || column == origin.primary_key.to_s
|
|
1030
|
+
|
|
1031
|
+
origin_state[column]
|
|
1032
|
+
end
|
|
1033
|
+
|
|
1034
|
+
# The origin's attributes as they are now (its replayed state in
|
|
1035
|
+
# ES-NoProj), keyed by column name.
|
|
1036
|
+
def current_state(model, id)
|
|
1037
|
+
if events_only?
|
|
1038
|
+
state = Lyra::DualView.new(model, id).event_sourced_state[:state] || {}
|
|
1039
|
+
return state.transform_keys(&:to_s)
|
|
1040
|
+
end
|
|
1041
|
+
|
|
1042
|
+
model.unscoped.find_by(model.primary_key => id)&.attributes || {}
|
|
1043
|
+
end
|
|
1044
|
+
|
|
1045
|
+
# From 1.0.0, with everywhere: the rows of monitored models with a
|
|
1046
|
+
# policy that may hold one of +values+, searched in the tables
|
|
1047
|
+
# themselves, so a row whose events lack the value (raw SQL, a write
|
|
1048
|
+
# before monitoring, a write Lyra could not intercept) is found too.
|
|
1049
|
+
# One query per table, which returns the keys and, per value, whether
|
|
1050
|
+
# the row matched; no row is loaded here. String and text columns by
|
|
1051
|
+
# equality (a citext column without case), json and jsonb columns as text
|
|
1052
|
+
# by a LIKE on the value as a JSON string (a value or a key), a text
|
|
1053
|
+
# column the model reads as a structure (serialized, attribute :json) by
|
|
1054
|
+
# a LIKE on the bare value, string arrays with = ANY, hstore values with
|
|
1055
|
+
# = ANY(avals(…)). Encrypted columns are not searched (not_searched). The
|
|
1056
|
+
# erasure itself matches exactly, and reports what it cannot erase.
|
|
1057
|
+
# Returns { stream => { model:, id:, values: [matched] } } without the
|
|
1058
|
+
# origin's own row.
|
|
1059
|
+
def row_candidates(values, origin_stream)
|
|
1060
|
+
searched_models.each_with_object({}) do |model, found|
|
|
1061
|
+
model.with_connection do |conn|
|
|
1062
|
+
encrypted = Lyra::EncryptedAttributes.names(model)
|
|
1063
|
+
columns = model.columns.reject { |column| encrypted.include?(column.name) }
|
|
1064
|
+
conditions = values.map do |value|
|
|
1065
|
+
clauses = columns.filter_map { |column| search_clause(conn, model, column, value) }
|
|
1066
|
+
clauses.empty? ? "FALSE" : "(#{clauses.join(' OR ')})"
|
|
1067
|
+
end
|
|
1068
|
+
next if conditions.all?("FALSE")
|
|
1069
|
+
|
|
1070
|
+
flags = conditions.each_with_index.map { |condition, i| "#{condition} AS v#{i}" }
|
|
1071
|
+
conn.select_rows(<<~SQL.squish).each do |id, *matched|
|
|
1072
|
+
SELECT #{conn.quote_column_name(model.primary_key)}, #{flags.join(', ')}
|
|
1073
|
+
FROM #{conn.quote_table_name(model.table_name)}
|
|
1074
|
+
WHERE #{conditions.join(' OR ')}
|
|
1075
|
+
SQL
|
|
1076
|
+
stream = Lyra.stream_name(model, id)
|
|
1077
|
+
next if stream == origin_stream
|
|
1078
|
+
|
|
1079
|
+
held = values.select.with_index { |_value, i| [true, "t"].include?(matched[i]) }
|
|
1080
|
+
found[stream] = { model: model, id: id, values: held }
|
|
1081
|
+
end
|
|
1082
|
+
end
|
|
1083
|
+
end
|
|
1084
|
+
end
|
|
1085
|
+
|
|
1086
|
+
# The monitored models with a policy, one per table, as the class that
|
|
1087
|
+
# owns it (the STI base class, so no type condition narrows the rows):
|
|
1088
|
+
# the tables of models without a policy, or not monitored, are not
|
|
1089
|
+
# searched.
|
|
1090
|
+
def searched_models
|
|
1091
|
+
Lyra.config.monitored_models
|
|
1092
|
+
.select { |model| Lyra::Privacy.policy_for(model).loaded? }
|
|
1093
|
+
.map { |model| model.respond_to?(:base_class) ? model.base_class : model }
|
|
1094
|
+
.uniq(&:table_name)
|
|
1095
|
+
.select(&:table_exists?)
|
|
1096
|
+
end
|
|
1097
|
+
|
|
1098
|
+
TEXT_TYPES = %i[string text citext].freeze
|
|
1099
|
+
|
|
1100
|
+
# The condition +column+ holds +value+, or nil for a column not searched.
|
|
1101
|
+
def search_clause(conn, model, column, value)
|
|
1102
|
+
name = conn.quote_column_name(column.name)
|
|
1103
|
+
quoted = conn.quote(value)
|
|
1104
|
+
if column.respond_to?(:array?) && column.array?
|
|
1105
|
+
"#{quoted} = ANY(#{name})" if TEXT_TYPES.include?(column.type)
|
|
1106
|
+
elsif %i[json jsonb].include?(column.type)
|
|
1107
|
+
"#{name}::text LIKE #{conn.quote(json_pattern(value))}"
|
|
1108
|
+
elsif column.type == :hstore
|
|
1109
|
+
"#{quoted} = ANY(avals(#{name}))"
|
|
1110
|
+
elsif TEXT_TYPES.include?(column.type) && structured?(model, column.name)
|
|
1111
|
+
"#{name} LIKE #{conn.quote("%#{ActiveRecord::Base.sanitize_sql_like(value)}%")}"
|
|
1112
|
+
elsif TEXT_TYPES.include?(column.type)
|
|
1113
|
+
"#{name} = #{quoted}"
|
|
1114
|
+
end
|
|
1115
|
+
end
|
|
1116
|
+
|
|
1117
|
+
def holds_whole?(value, wanted)
|
|
1118
|
+
case value
|
|
1119
|
+
when Hash then value.any? { |_key, inner| holds_whole?(inner, wanted) }
|
|
1120
|
+
when Array then value.any? { |inner| holds_whole?(inner, wanted) }
|
|
1121
|
+
when String then value == wanted
|
|
1122
|
+
else false
|
|
1123
|
+
end
|
|
1124
|
+
end
|
|
1125
|
+
|
|
1126
|
+
# The events whose stored data mentions one of +values+: one text search
|
|
1127
|
+
# of the log for all of them, without case (a citext column holds a copy
|
|
1128
|
+
# that differs in case; the match that erases is exact, and such a copy
|
|
1129
|
+
# is reported). Read in batches.
|
|
1130
|
+
def events_mentioning(values)
|
|
1131
|
+
ActiveRecord::Base.with_connection do |conn|
|
|
1132
|
+
clauses = values.map do |value|
|
|
1133
|
+
"convert_from(data, 'UTF8') ILIKE #{conn.quote("%#{ActiveRecord::Base.sanitize_sql_like(value)}%")}"
|
|
1134
|
+
end
|
|
1135
|
+
ids = conn.select_values("SELECT event_id FROM event_store_events WHERE #{clauses.join(' OR ')}")
|
|
1136
|
+
ids.each_slice(1_000).flat_map { |batch| Lyra.config.event_store.read.events(batch).to_a }
|
|
1137
|
+
end
|
|
1138
|
+
end
|
|
1139
|
+
|
|
1140
|
+
# The columns of +model+ +id+'s row that hold an erased value, replaced
|
|
1141
|
+
# by the rule its events got (+resolve+: the value rule only, so a third
|
|
1142
|
+
# party's email under an "email" key is kept), inside json, jsonb and
|
|
1143
|
+
# array columns too (scrub_row). Returns the column names.
|
|
1144
|
+
def erase_copied_columns(model, id, resolve)
|
|
266
1145
|
return [] if events_only?
|
|
267
1146
|
|
|
268
1147
|
row = model.unscoped.lock.find_by(model.primary_key => id)
|
|
269
1148
|
return [] unless row
|
|
270
1149
|
|
|
271
|
-
columns =
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
acc[column] = replacement_for(model, id, column, replacements[originals[value]])
|
|
275
|
-
end
|
|
276
|
-
write_columns(model, id, columns)
|
|
1150
|
+
columns = scrub_row(model, row, resolve)
|
|
1151
|
+
write_with_lock(model, id, row, columns)
|
|
277
1152
|
columns.keys
|
|
278
1153
|
end
|
|
279
1154
|
|
|
@@ -284,21 +1159,51 @@ module Lyra
|
|
|
284
1159
|
def write_columns(model, id, columns)
|
|
285
1160
|
return if columns.empty?
|
|
286
1161
|
|
|
287
|
-
Lyra.projection_write { model.unscoped.where(model.primary_key => id).update_all(columns) }
|
|
1162
|
+
Lyra.projection_write(INTERNAL) { model.unscoped.where(model.primary_key => id).update_all(columns) }
|
|
288
1163
|
end
|
|
289
1164
|
|
|
290
1165
|
def events_only?
|
|
291
1166
|
Lyra.config.event_sourcing_mode? && Lyra.config.projection_mode == :disabled
|
|
292
1167
|
end
|
|
293
1168
|
|
|
294
|
-
|
|
295
|
-
|
|
1169
|
+
# The subject's access streams: their events deleted when +erase+, else
|
|
1170
|
+
# kept. Returns ["erased" or "kept", how many entries they held].
|
|
1171
|
+
def access_log!(access_streams, erase)
|
|
1172
|
+
if erase
|
|
1173
|
+
["erased", access_streams.sum { |s| Lyra::AccessLog.delete_stream!(s) }]
|
|
1174
|
+
else
|
|
1175
|
+
["kept", access_streams.sum { |s| Lyra::AccessLog.count_stream(s) }]
|
|
1176
|
+
end
|
|
1177
|
+
end
|
|
1178
|
+
|
|
1179
|
+
# +access_log+ and +access_log_entries+: what happened to the subject's
|
|
1180
|
+
# access log ("erased" or "kept", and how many entries it held).
|
|
1181
|
+
# +any_fields+: the request named columns the policy does not declare personal
|
|
1182
|
+
# (ANY_FIELDS=1); +requested+ is what it named (nil: the declared ones);
|
|
1183
|
+
# +beyond_policy+ the columns erased only because of it. The operating
|
|
1184
|
+
# system's user and the host are recorded beside the label in erased_by.
|
|
1185
|
+
# +row_only+: the value was in the row, not in the record's events, and
|
|
1186
|
+
# only the row was erased; +imported+: the row had no stream, and was
|
|
1187
|
+
# imported after it was erased (both from 1.0.0, with everywhere).
|
|
1188
|
+
def record_erasure(model, id, stream, fields, reason, erased_by, any_fields: false, requested: nil, beyond_policy: [],
|
|
1189
|
+
access_log: nil, access_log_entries: nil, access_log_subjects: 0, row_only: false, imported: false)
|
|
1190
|
+
data = { model_class: model.name, model_id: id, fields: fields, reason: reason.to_s, erased_at: Time.current,
|
|
1191
|
+
any_fields: any_fields, requested_fields: requested&.map(&:to_s), beyond_policy: beyond_policy }
|
|
1192
|
+
data.merge!(access_log: access_log, access_log_entries: access_log_entries) if access_log
|
|
1193
|
+
data[:access_log_subjects] = access_log_subjects if access_log_subjects.positive?
|
|
1194
|
+
data[:row_only] = true if row_only
|
|
1195
|
+
data[:imported] = true if imported
|
|
1196
|
+
who = Lyra::Attribution.of
|
|
296
1197
|
metadata = {
|
|
297
1198
|
source: SOURCE,
|
|
298
1199
|
erased_by: erased_by&.to_s || (defined?(::Current) && ::Current.respond_to?(:user) ? ::Current.user&.id : nil),
|
|
1200
|
+
os_user: who[:os_user],
|
|
1201
|
+
host: who[:host],
|
|
1202
|
+
pid: who[:pid],
|
|
299
1203
|
correlation_id: Lyra::Correlation.current_id
|
|
300
1204
|
}.compact
|
|
301
1205
|
Lyra.append_events(Lyra::Events::ErasureApplied.new(data: data, metadata: metadata), stream_name: stream)
|
|
1206
|
+
Thread.current[:lyra_erasure_touched]&.push([model, id])
|
|
302
1207
|
end
|
|
303
1208
|
|
|
304
1209
|
def forget_cached(model, id)
|