orfeas_lyra 0.9.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +2021 -17
- data/README.md +22 -9
- data/Rakefile +40 -1
- data/app/controllers/lyra/application_controller.rb +127 -0
- data/app/controllers/lyra/dashboard_controller.rb +33 -31
- data/app/controllers/lyra/flow_controller.rb +32 -34
- data/app/controllers/lyra/privacy_controller.rb +57 -18
- data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
- data/app/views/lyra/dashboard/index.html.erb +1 -0
- data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
- data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
- data/app/views/lyra/privacy/policy.html.erb +7 -0
- data/app/views/lyra/privacy/rights.html.erb +78 -0
- data/config/privacy_policies.rb +0 -3
- data/config/routes.rb +1 -0
- data/lib/lyra/access_log.rb +187 -6
- data/lib/lyra/advisory_lock.rb +236 -0
- data/lib/lyra/aggregate.rb +1 -1
- data/lib/lyra/associations/habtm_links.rb +382 -19
- data/lib/lyra/associations/maintenance.rb +229 -0
- data/lib/lyra/attribution.rb +48 -0
- data/lib/lyra/bypass_events.rb +446 -46
- data/lib/lyra/command_handler.rb +19 -14
- data/lib/lyra/configuration.rb +237 -10
- data/lib/lyra/consistency/read_your_writes.rb +4 -2
- data/lib/lyra/correlation.rb +121 -0
- data/lib/lyra/domain_events.rb +4 -0
- data/lib/lyra/dual_view.rb +53 -16
- data/lib/lyra/encrypted_attributes.rb +5 -0
- data/lib/lyra/engine.rb +67 -0
- data/lib/lyra/erasure.rb +1011 -106
- data/lib/lyra/event.rb +35 -1
- data/lib/lyra/event_data.rb +38 -0
- data/lib/lyra/event_flow.rb +31 -15
- data/lib/lyra/event_store_adapter.rb +28 -0
- data/lib/lyra/event_window.rb +58 -0
- data/lib/lyra/genesis.rb +548 -44
- data/lib/lyra/interceptors/association_interceptor.rb +689 -34
- data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
- data/lib/lyra/internal.rb +26 -0
- data/lib/lyra/mode_sync.rb +51 -3
- data/lib/lyra/mode_transition.rb +257 -30
- data/lib/lyra/model_selection.rb +35 -0
- data/lib/lyra/own_connection.rb +17 -2
- data/lib/lyra/privacy/adapters/pam.rb +25 -2
- data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
- data/lib/lyra/privacy/interface.rb +27 -7
- data/lib/lyra/privacy/subject_links.rb +143 -0
- data/lib/lyra/privacy/subject_records.rb +369 -0
- data/lib/lyra/projection.rb +7 -1
- data/lib/lyra/projections/async_checkpoint.rb +32 -0
- data/lib/lyra/projections/async_projection_job.rb +166 -23
- data/lib/lyra/projections/cached_joins.rb +33 -18
- data/lib/lyra/projections/cached_projection.rb +74 -33
- data/lib/lyra/projections/cached_relation.rb +3175 -321
- data/lib/lyra/projections/default_scopes.rb +254 -0
- data/lib/lyra/projections/deferred_relation.rb +23 -3
- data/lib/lyra/projections/event_store_reader.rb +47 -8
- data/lib/lyra/projections/foreign_keys.rb +92 -0
- data/lib/lyra/projections/lazy_projection.rb +165 -27
- data/lib/lyra/projections/model_projection.rb +165 -14
- data/lib/lyra/projections/rebuild.rb +34 -6
- data/lib/lyra/projections/reference_links.rb +463 -72
- data/lib/lyra/projections/typed_values.rb +629 -0
- data/lib/lyra/projections/unique_indexes.rb +595 -0
- data/lib/lyra/projections/uniqueness.rb +784 -0
- data/lib/lyra/projections/value_digest.rb +93 -0
- data/lib/lyra/purpose_bound_reads.rb +912 -77
- data/lib/lyra/repair.rb +119 -28
- data/lib/lyra/retention.rb +114 -8
- data/lib/lyra/rights/consent_store.rb +415 -0
- data/lib/lyra/rights/erasure_guard.rb +135 -0
- data/lib/lyra/rights/pending.rb +74 -0
- data/lib/lyra/rights/queries.rb +269 -0
- data/lib/lyra/rights/store.rb +512 -0
- data/lib/lyra/rights/subject_keys.rb +224 -0
- data/lib/lyra/rights.rb +281 -0
- data/lib/lyra/saves_without_effect.rb +69 -0
- data/lib/lyra/stream_tail.rb +319 -0
- data/lib/lyra/strict_data_access.rb +916 -106
- data/lib/lyra/temporal.rb +4 -2
- data/lib/lyra/touch_registry.rb +404 -0
- data/lib/lyra/verification/bypass_workflow.rb +452 -42
- data/lib/lyra/verification/rights_workflows.rb +170 -0
- data/lib/lyra/version.rb +1 -1
- data/lib/lyra/visualization/event_graph.rb +1 -1
- data/lib/lyra/visualization/timeline.rb +2 -2
- data/lib/lyra.rb +43 -2
- data/lib/tasks/lyra_mode.rake +110 -8
- data/lib/tasks/lyra_projections.rake +43 -2
- metadata +34 -4
- data/lib/lyra/associations/event_aware.rb +0 -224
|
@@ -50,7 +50,7 @@ module Lyra
|
|
|
50
50
|
return nil unless pam_field
|
|
51
51
|
|
|
52
52
|
Annotation.new(field: pam_field.name, type: pam_field.type, sensitivity: pam_field.sensitivity,
|
|
53
|
-
sensitive: pam_field.sensitive?, purposes: pam_field
|
|
53
|
+
sensitive: pam_field.sensitive?, purposes: admitting_purposes(pam_field), source: :policy,
|
|
54
54
|
transformations: pam_field.transformations.keys)
|
|
55
55
|
end
|
|
56
56
|
|
|
@@ -62,8 +62,21 @@ module Lyra
|
|
|
62
62
|
pam_policy.validate_access!(fields, purpose, subject: subject)
|
|
63
63
|
end
|
|
64
64
|
|
|
65
|
+
def validate_fields!(fields, purpose, subject: nil)
|
|
66
|
+
pam_policy.validate_fields!(fields, purpose, subject: subject)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def field_violations(fields, purpose)
|
|
70
|
+
pam_policy.field_violations(fields, purpose)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def validate_rights!(fields, purpose, subject:)
|
|
74
|
+
pam_policy.validate_rights!(fields, purpose, subject: subject)
|
|
75
|
+
end
|
|
76
|
+
|
|
65
77
|
def allowed_purposes(field)
|
|
66
|
-
find_field(field)
|
|
78
|
+
pam_field = find_field(field)
|
|
79
|
+
pam_field ? admitting_purposes(pam_field) : []
|
|
67
80
|
end
|
|
68
81
|
|
|
69
82
|
def consent_required?(purpose)
|
|
@@ -112,6 +125,16 @@ module Lyra
|
|
|
112
125
|
|
|
113
126
|
private
|
|
114
127
|
|
|
128
|
+
# The purposes the policy admits +pam_field+ for (Policy#allowed?):
|
|
129
|
+
# the field's own allow_for and every purpose that lists it in
|
|
130
|
+
# requires or optionally. Before 1.0.0 the annotation (and so the
|
|
131
|
+
# privacy stamp) carried allow_for alone, and a policy that lists its
|
|
132
|
+
# fields on the purpose side stamped every field with no purpose.
|
|
133
|
+
def admitting_purposes(pam_field)
|
|
134
|
+
listed = pam_policy.purposes.each_value.select { |purpose| pam_policy.allowed?(pam_field.name, purpose.name) }
|
|
135
|
+
(pam_field.purposes | listed.map(&:name))
|
|
136
|
+
end
|
|
137
|
+
|
|
115
138
|
def find_field(field)
|
|
116
139
|
pam_policy.get_field(field.to_sym)
|
|
117
140
|
rescue PamDsl::InvalidFieldError
|
|
@@ -22,10 +22,16 @@ module Lyra
|
|
|
22
22
|
#
|
|
23
23
|
# @param subject_id [Object] The data subject's identifier
|
|
24
24
|
# @param subject_type [String] The type/class of the subject (default: 'User')
|
|
25
|
+
# @param actor_label [Proc] ->(actor, subject) { label }: how the copy names who acted
|
|
26
|
+
# (default "you", "staff", "staff or you" or "system"; PamDsl::SubjectAccess::DEFAULT_ACTOR_LABEL)
|
|
27
|
+
# @param access_log_subjects [Array] the keys the application passed as this subject to the
|
|
28
|
+
# access log besides its records (validate_access!(..., subject: 7) writes Lyra::DataAccess$7);
|
|
29
|
+
# default: the bare subject id. [] reads none, where bare ids of several types share the log.
|
|
25
30
|
#
|
|
26
|
-
def initialize(subject_id:, subject_type: 'User')
|
|
31
|
+
def initialize(subject_id:, subject_type: 'User', actor_label: nil, access_log_subjects: nil)
|
|
27
32
|
@subject_id = subject_id
|
|
28
33
|
@subject_type = subject_type
|
|
34
|
+
@access_log_subjects = access_log_subjects.nil? ? [subject_id.to_s] : Array(access_log_subjects).map(&:to_s)
|
|
29
35
|
|
|
30
36
|
@pam_compliance = PamDsl::GDPRCompliance.new(
|
|
31
37
|
subject_id: subject_id,
|
|
@@ -37,15 +43,45 @@ module Lyra
|
|
|
37
43
|
model_class_extractor: method(:extract_model_class),
|
|
38
44
|
model_id_extractor: method(:extract_model_id),
|
|
39
45
|
changes_extractor: method(:extract_changes),
|
|
40
|
-
retention_policy:
|
|
46
|
+
retention_policy: retention_policy,
|
|
47
|
+
policy_resolver: method(:policy_for_model),
|
|
48
|
+
records_reader: method(:read_subject_records),
|
|
49
|
+
access_reader: method(:read_access_log),
|
|
50
|
+
export_recorder: Lyra::AccessLog.method(:record_export),
|
|
51
|
+
actor_label: actor_label,
|
|
52
|
+
rights_keys: method(:rights_keys)
|
|
41
53
|
)
|
|
42
54
|
end
|
|
43
55
|
|
|
44
56
|
# Delegate all GDPR methods to the PAM DSL implementation
|
|
45
57
|
|
|
46
|
-
# Right to Access (Article 15)
|
|
47
|
-
|
|
48
|
-
|
|
58
|
+
# Right to Access (Article 15): the subject's copy. Beside the event
|
|
59
|
+
# summaries it always gave, +records+ (each of the subject's records
|
|
60
|
+
# whole, with its history), +accesses+ (the access log; nil when
|
|
61
|
+
# config.record_access_events is off), +processing_context+, +notice+
|
|
62
|
+
# (the policy's access_notice) and +gaps+. See PamDsl::SubjectAccess
|
|
63
|
+
# and Lyra::Privacy::SubjectRecords. From 1.0.0 +restrictions+ and
|
|
64
|
+
# +objections+ list the subject's requests of Arts. 18 and 21: those
|
|
65
|
+
# recorded under the subject's key (EnfAuthor$7), and under the key of
|
|
66
|
+
# one of the subject's records that names no other person (a contested
|
|
67
|
+
# invoice). A record shared with another person is left out: its
|
|
68
|
+
# restriction may be the other person's request.
|
|
69
|
+
#
|
|
70
|
+
# An explicit call answers a request and is recorded, in the stream
|
|
71
|
+
# Lyra::SubjectAccessExports$<Type>$<id> (Lyra::AccessLog.record_export),
|
|
72
|
+
# unless config.record_subject_access_exports is false. record: false is
|
|
73
|
+
# for a view of the data that answers no request: full_report and the
|
|
74
|
+
# dashboard's subject and GDPR routes pass it, so a view is not on record
|
|
75
|
+
# as an answered request.
|
|
76
|
+
#
|
|
77
|
+
# history: false and accesses: false are for a request the subject
|
|
78
|
+
# has narrowed (recital 63); disclose_actors: true adds who acted (ids,
|
|
79
|
+
# IP addresses) to the role labels, after the controller has weighed
|
|
80
|
+
# the staff's rights; verified_by: names who verified the subject's
|
|
81
|
+
# identity, for the record of the export.
|
|
82
|
+
def data_export(history: true, accesses: true, disclose_actors: false, verified_by: nil, record: true)
|
|
83
|
+
pam_compliance.data_export(history: history, accesses: accesses, disclose_actors: disclose_actors,
|
|
84
|
+
verified_by: verified_by, record: record)
|
|
49
85
|
end
|
|
50
86
|
|
|
51
87
|
# Right to be Forgotten (Article 17)
|
|
@@ -78,39 +114,247 @@ module Lyra
|
|
|
78
114
|
pam_compliance.consent_audit
|
|
79
115
|
end
|
|
80
116
|
|
|
81
|
-
# Full GDPR Compliance Report
|
|
117
|
+
# Full GDPR Compliance Report. Its data_export is not recorded as an
|
|
118
|
+
# answered request (see data_export).
|
|
82
119
|
def full_report
|
|
83
120
|
pam_compliance.full_report
|
|
84
121
|
end
|
|
85
122
|
|
|
86
123
|
private
|
|
87
124
|
|
|
125
|
+
# The PAM policy that governs +model_name+'s model (its privacy_policy,
|
|
126
|
+
# else config.privacy_policy), or nil. A subject's records span models
|
|
127
|
+
# with different policies; before 1.0.0 the wrapper passed none, and the
|
|
128
|
+
# register's legal basis read "Not specified".
|
|
129
|
+
def policy_for_model(model_name)
|
|
130
|
+
model = Lyra.config.monitored_model_named(model_name) || model_name.to_s.safe_constantize
|
|
131
|
+
policy = model.is_a?(Class) ? Lyra::Privacy.policy_for(model) : Lyra::Privacy.policy(Lyra.config.privacy_policy)
|
|
132
|
+
policy.respond_to?(:pam_policy) ? policy.pam_policy : nil
|
|
133
|
+
rescue StandardError
|
|
134
|
+
nil
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# The keys the subject's rights are recorded under (see data_export):
|
|
138
|
+
# the subject's own, and those of their records that have requests
|
|
139
|
+
# (one lookup in lyra_rights) and name this subject alone.
|
|
140
|
+
def rights_keys(records)
|
|
141
|
+
model = Lyra.config.monitored_model_named(@subject_type.to_s) || @subject_type.to_s.safe_constantize
|
|
142
|
+
person = active_record_model?(model) ? Lyra.stream_name(model, @subject_id) : "#{@subject_type}$#{@subject_id}"
|
|
143
|
+
candidates = records.filter_map do |raw|
|
|
144
|
+
record_model = Lyra.config.monitored_model_named(raw[:model].to_s) || raw[:model].to_s.safe_constantize
|
|
145
|
+
next unless active_record_model?(record_model) && !raw[:id].nil?
|
|
146
|
+
|
|
147
|
+
key = Lyra.stream_name(record_model, raw[:id])
|
|
148
|
+
[key, record_model, raw[:id]] unless key == person
|
|
149
|
+
end
|
|
150
|
+
store = PamDsl.rights_store
|
|
151
|
+
keys = candidates.map(&:first)
|
|
152
|
+
with_requests = store.respond_to?(:keys_with_requests) ? store.keys_with_requests(keys) : keys
|
|
153
|
+
own = candidates.select { |key, _, _| with_requests.include?(key) }.filter_map do |key, record_model, id|
|
|
154
|
+
# Its row, or under ES-NoProj its state from its events.
|
|
155
|
+
row = Lyra::Rights::SubjectKeys.find_record(record_model, id)
|
|
156
|
+
key if row && Lyra::Rights::SubjectKeys.person_keys(row) == [person]
|
|
157
|
+
end
|
|
158
|
+
[person, *own]
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
def active_record_model?(model)
|
|
162
|
+
model.is_a?(Class) && model < ActiveRecord::Base
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def read_subject_records(subject_id, subject_type, events, history: true)
|
|
166
|
+
SubjectRecords.new(subject_id, subject_type, events, history: history).call
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
ACCESS_LOG_OFF = "access log not enabled (accesses lists earlier exports only)"
|
|
170
|
+
|
|
171
|
+
# The access log's entries, oldest first: for each of the subject's
|
|
172
|
+
# records, those within the periods the record was the subject's (a
|
|
173
|
+
# moved record's accesses while it was another person's are theirs);
|
|
174
|
+
# for the subject's own key and the keys the application passed as the
|
|
175
|
+
# subject (access_log_subjects:, the bare id by default); and the
|
|
176
|
+
# subject's earlier exports, which have a stream of their own. With
|
|
177
|
+
# the log off, only the exports, and the gap that says so.
|
|
178
|
+
def read_access_log(subject_id, subject_type, records)
|
|
179
|
+
subject = Lyra::AccessLog.subject_key_for(subject_type, subject_id)
|
|
180
|
+
exports = Lyra::AccessLog.exports_for(subject).map { |event| access_entry(subject, event) }
|
|
181
|
+
return { entries: exports, gap: ACCESS_LOG_OFF } unless Lyra.config.record_access_events
|
|
182
|
+
|
|
183
|
+
periods = records.to_h { |record| ["#{record[:model]}$#{record[:id]}", record[:periods]] }
|
|
184
|
+
([subject] + @access_log_subjects).each { |key| periods[key] = nil unless periods.key?(key) }
|
|
185
|
+
entries = periods.flat_map do |key, record_periods|
|
|
186
|
+
Lyra::AccessLog.for(key).map { |event| access_entry(key, event) }
|
|
187
|
+
.select { |entry| within?(record_periods, entry[:at]) }
|
|
188
|
+
end
|
|
189
|
+
(entries + exports).sort_by { |entry| entry[:at] || Time.at(0) }
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
# Whether +at+ falls in one of +periods+ (nil: the whole record).
|
|
193
|
+
def within?(periods, at)
|
|
194
|
+
return true if periods.nil? || at.nil?
|
|
195
|
+
|
|
196
|
+
periods.any? { |period| (period[:from].nil? || at >= period[:from]) && (period[:to].nil? || at <= period[:to]) }
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
def access_entry(key, event)
|
|
200
|
+
data = event.data.to_h.transform_keys(&:to_sym)
|
|
201
|
+
metadata = event.metadata.to_h.transform_keys(&:to_sym)
|
|
202
|
+
exported = event.event_type.to_s.end_with?("SubjectAccessExported")
|
|
203
|
+
{
|
|
204
|
+
at: data[:accessed_at] || data[:exported_at] || metadata[:timestamp],
|
|
205
|
+
record: key,
|
|
206
|
+
policy: data[:policy],
|
|
207
|
+
purpose: exported ? nil : data[:purpose],
|
|
208
|
+
legal_basis: data[:legal_basis],
|
|
209
|
+
fields: data[:fields],
|
|
210
|
+
outcome: exported ? "exported" : data[:outcome],
|
|
211
|
+
sections: data[:sections],
|
|
212
|
+
actor: metadata.slice(:user_id, :ip_address, :user_type).compact
|
|
213
|
+
}
|
|
214
|
+
end
|
|
215
|
+
|
|
88
216
|
# Read all events related to the subject from Lyra's event store
|
|
217
|
+
# Every event of the subject's own record and of the records that refer
|
|
218
|
+
# to them (SubjectLinks: a belongs_to to the subject's class, or
|
|
219
|
+
# <type>_id), in log order. A related record is followed through its
|
|
220
|
+
# stream: its links are tracked from event to event, and an event belongs
|
|
221
|
+
# to the subject when the record named them before or after it. An event
|
|
222
|
+
# carries only what it changed (Lyra::EventData, 1.0.0), so matching the
|
|
223
|
+
# foreign key in each event alone kept a related record's create and lost
|
|
224
|
+
# its later updates and its destroy.
|
|
225
|
+
#
|
|
226
|
+
# The event that moves a record between people is given with the
|
|
227
|
+
# subject's side only: moving in, its new values; moving away, the
|
|
228
|
+
# links' old values and nothing of the new owner's (from 1.0.0; the
|
|
229
|
+
# reports built on these events gave both sides).
|
|
89
230
|
def read_subject_events(subject_id, subject_type)
|
|
90
|
-
|
|
91
|
-
|
|
231
|
+
links = SubjectLinks.new(subject_type)
|
|
232
|
+
own = links.subject_model ? Lyra.stream_owner_name(links.subject_model) : subject_type.to_s
|
|
233
|
+
values = {} # stream => the link columns' last known values
|
|
234
|
+
matching = []
|
|
235
|
+
each_event do |event|
|
|
236
|
+
relates = event_relates_to_subject?(event, subject_id, subject_type)
|
|
237
|
+
model_id = extract_model_id(event)
|
|
238
|
+
model = linked_model(extract_model_class(event))
|
|
239
|
+
name = model.is_a?(Class) ? Lyra.stream_owner_name(model) : model.to_s
|
|
240
|
+
if model_id.nil? || (name == own && model_id.to_s == subject_id.to_s) || links.for(model).empty?
|
|
241
|
+
matching << event if relates
|
|
242
|
+
next
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
key = "#{name}$#{model_id}"
|
|
246
|
+
classified, values[key] = SubjectRecords.classify([event], links: links, model: model, subject_id: subject_id,
|
|
247
|
+
initial: values[key] || {})
|
|
248
|
+
case classified.first.last
|
|
249
|
+
when :inside then matching << event
|
|
250
|
+
when :in, :away then matching << subject_side(event, classified.first.last, links.columns(model))
|
|
251
|
+
else matching << event if relates
|
|
252
|
+
end
|
|
92
253
|
end
|
|
254
|
+
matching
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
def linked_model(name)
|
|
258
|
+
model = Lyra.config.monitored_model_named(name) || name.to_s.safe_constantize
|
|
259
|
+
model.is_a?(Class) && model < ActiveRecord::Base ? model : name.to_s
|
|
260
|
+
rescue StandardError
|
|
261
|
+
name.to_s
|
|
93
262
|
end
|
|
94
263
|
|
|
95
|
-
#
|
|
264
|
+
# A transition event with the subject's side only: an update moving the
|
|
265
|
+
# record in keeps its new values, one moving it away the links' old
|
|
266
|
+
# values. A create or import moving it in is the subject's whole.
|
|
267
|
+
def subject_side(event, kind, link_columns)
|
|
268
|
+
data = event.data.to_h
|
|
269
|
+
changes = data[:changes] || data["changes"]
|
|
270
|
+
return event unless changes.is_a?(Hash) && Lyra::Event.operation_of(event) == :updated
|
|
271
|
+
|
|
272
|
+
side =
|
|
273
|
+
if kind == :in
|
|
274
|
+
changes.to_h { |field, change| [field, [nil, change.is_a?(Array) ? change.last : change]] }
|
|
275
|
+
else
|
|
276
|
+
changes.select { |field, _| link_columns.include?(field.to_s) }
|
|
277
|
+
.to_h { |field, change| [field, [change.is_a?(Array) ? change.first : nil, nil]] }
|
|
278
|
+
end
|
|
279
|
+
event.class.new(event_id: event.event_id, metadata: event.metadata.to_h,
|
|
280
|
+
data: data.except(:changes, "changes").merge(changes: side))
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
EVENT_BATCH = 1_000
|
|
284
|
+
|
|
285
|
+
# Every event in the log, read in batches: a report that must see all of it
|
|
286
|
+
# (a subject's events, retention) holds a batch at a time, not the whole log.
|
|
287
|
+
# SaveWithoutEffect markers hold no one's data and record no change
|
|
288
|
+
# (Lyra::SavesWithoutEffect): passed over.
|
|
289
|
+
def each_event(batch_size = EVENT_BATCH)
|
|
290
|
+
Lyra.config.event_store.read.in_batches(batch_size).each do |event|
|
|
291
|
+
yield event unless Lyra::SavesWithoutEffect.marker?(event)
|
|
292
|
+
end
|
|
293
|
+
end
|
|
294
|
+
|
|
295
|
+
# Whether an event holds the subject's data: an event of the subject's
|
|
296
|
+
# own record, or of a record that refers to the subject through its
|
|
297
|
+
# foreign key (a Registration's user_id, for subject type User).
|
|
298
|
+
#
|
|
299
|
+
# Who *made* a change is not whose data it is: before 0.9.3 an event
|
|
300
|
+
# whose metadata named the subject as its actor (metadata[:user_id])
|
|
301
|
+
# was exported too, so a staff member's export held every record they
|
|
302
|
+
# had edited, other people's data, while a subject's own records written
|
|
303
|
+
# by staff could be missed.
|
|
96
304
|
def event_relates_to_subject?(event, subject_id, subject_type)
|
|
97
|
-
metadata = event.respond_to?(:metadata) ? event.metadata : {}
|
|
98
305
|
data = event.respond_to?(:data) ? event.data : {}
|
|
306
|
+
id = subject_id.to_s
|
|
307
|
+
return true if extract_model_class(event).to_s == subject_type.to_s && extract_model_id(event).to_s == id
|
|
308
|
+
|
|
309
|
+
explicit = data[:subject_id] || data["subject_id"]
|
|
310
|
+
if explicit
|
|
311
|
+
type = data[:subject_type] || data["subject_type"]
|
|
312
|
+
return explicit.to_s == id && (type.nil? || type.to_s == subject_type.to_s)
|
|
313
|
+
end
|
|
314
|
+
|
|
315
|
+
attributes = extract_attributes(event)
|
|
316
|
+
value = attributes[subject_foreign_key(subject_type)] || attributes[subject_foreign_key(subject_type).to_sym]
|
|
317
|
+
!value.nil? && value.to_s == id
|
|
318
|
+
end
|
|
99
319
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
(extract_model_class(event) == subject_type && extract_model_id(event) == subject_id)
|
|
320
|
+
# The foreign key a record uses to refer to a subject of +subject_type+.
|
|
321
|
+
def subject_foreign_key(subject_type)
|
|
322
|
+
"#{subject_type.to_s.demodulize.underscore}_id"
|
|
104
323
|
end
|
|
105
324
|
|
|
106
325
|
# Lyra-specific extractors
|
|
107
326
|
|
|
327
|
+
# The fields the event gives or changes (Lyra::Event.fields_of): an
|
|
328
|
+
# update holds its changes only, in every mode.
|
|
329
|
+
# Encrypted attributes are opened (the log holds their ciphertext), so
|
|
330
|
+
# the export's inventory and lineage give the values, as its records do.
|
|
108
331
|
def extract_attributes(event)
|
|
332
|
+
return opened(event, Lyra::Event.fields_of(event)) if event.respond_to?(:data)
|
|
109
333
|
return event.attributes if event.respond_to?(:attributes) && !event.attributes.is_a?(Method)
|
|
110
|
-
return event.data[:attributes] || event.data["attributes"] || {} if event.respond_to?(:data)
|
|
111
334
|
{}
|
|
112
335
|
end
|
|
113
336
|
|
|
337
|
+
# The model whose encrypted attributes +event+ may hold, or nil.
|
|
338
|
+
def encrypted_model(event)
|
|
339
|
+
model = linked_model(extract_model_class(event))
|
|
340
|
+
model.is_a?(Class) && Lyra::EncryptedAttributes.names(model).any? ? model : nil
|
|
341
|
+
end
|
|
342
|
+
|
|
343
|
+
def opened(event, values)
|
|
344
|
+
model = encrypted_model(event)
|
|
345
|
+
model && values.is_a?(Hash) ? Lyra::EncryptedAttributes.open(model, values) : values
|
|
346
|
+
end
|
|
347
|
+
|
|
348
|
+
def opened_changes(event, changes)
|
|
349
|
+
model = encrypted_model(event)
|
|
350
|
+
return changes unless model && changes.is_a?(Hash)
|
|
351
|
+
|
|
352
|
+
changes.to_h do |field, change|
|
|
353
|
+
opened_change = Array(change).map { |value| Lyra::EncryptedAttributes.open(model, { field => value })[field] }
|
|
354
|
+
[field, change.is_a?(Array) ? opened_change : opened_change.first]
|
|
355
|
+
end
|
|
356
|
+
end
|
|
357
|
+
|
|
114
358
|
def extract_timestamp(event)
|
|
115
359
|
return event.timestamp if event.respond_to?(:timestamp) && event.timestamp
|
|
116
360
|
return event.data[:timestamp] || event.data["timestamp"] if event.respond_to?(:data)
|
|
@@ -138,23 +382,43 @@ module Lyra
|
|
|
138
382
|
def extract_changes(event)
|
|
139
383
|
if event.respond_to?(:changes) && !event.changes.is_a?(Method)
|
|
140
384
|
begin
|
|
141
|
-
return event.changes unless event.method(:changes).owner.to_s.include?('ActiveRecord')
|
|
385
|
+
return opened_changes(event, event.changes) unless event.method(:changes).owner.to_s.include?('ActiveRecord')
|
|
142
386
|
rescue
|
|
143
387
|
return event.changes
|
|
144
388
|
end
|
|
145
389
|
end
|
|
146
|
-
return event.data[:changes] || event.data["changes"] || {} if event.respond_to?(:data)
|
|
390
|
+
return opened_changes(event, event.data[:changes] || event.data["changes"] || {}) if event.respond_to?(:data)
|
|
147
391
|
{}
|
|
148
392
|
end
|
|
149
393
|
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
394
|
+
# The retention periods the compliance check applies, as PAM's check
|
|
395
|
+
# expects them ({ "Model" => { duration: d }, default: { duration: d } }):
|
|
396
|
+
# each monitored model's period from its privacy policy's retention block,
|
|
397
|
+
# over config.retention_policy, which may be a single duration (7.years) or
|
|
398
|
+
# such a hash.
|
|
399
|
+
#
|
|
400
|
+
# Before 0.9.3 the policies' retention blocks were ignored in favour of
|
|
401
|
+
# hard-coded periods for models named Payment, Invoice, Student and
|
|
402
|
+
# Enrollment, and a single duration in config.retention_policy raised
|
|
403
|
+
# TypeError.
|
|
404
|
+
def retention_policy
|
|
405
|
+
configured = Lyra.config.retention_policy
|
|
406
|
+
policy =
|
|
407
|
+
case configured
|
|
408
|
+
when nil then { default: { duration: 7.years } }
|
|
409
|
+
when Hash then configured.transform_values { |v| v.is_a?(Hash) ? v : { duration: v } }
|
|
410
|
+
else { default: { duration: configured } }
|
|
411
|
+
end
|
|
412
|
+
|
|
413
|
+
Lyra.config.monitored_models.each do |model|
|
|
414
|
+
duration = Lyra::Privacy.policy_for(model).retention_for(model)
|
|
415
|
+
# A Duration, or a plain number of seconds (Numeric; a Duration also
|
|
416
|
+
# answers is_a?(Numeric) for its value).
|
|
417
|
+
policy[model.name] = { duration: duration } if duration.is_a?(ActiveSupport::Duration) || duration.is_a?(Numeric)
|
|
418
|
+
rescue StandardError
|
|
419
|
+
next
|
|
420
|
+
end
|
|
421
|
+
policy
|
|
158
422
|
end
|
|
159
423
|
end
|
|
160
424
|
|
|
@@ -175,34 +439,36 @@ module Lyra
|
|
|
175
439
|
# Returns nil for models without defined retention (infinite/manual)
|
|
176
440
|
def retention_compliance_with_policy(policy_name)
|
|
177
441
|
integration = PolicyIntegration.new(policy_name)
|
|
178
|
-
|
|
442
|
+
now = Time.current
|
|
443
|
+
totals = Hash.new(0)
|
|
444
|
+
expired_ids = Hash.new { |hash, key| hash[key] = [] }
|
|
445
|
+
durations = {}
|
|
446
|
+
|
|
447
|
+
# One pass over the log, keeping counts and the ids of expired events, not the events.
|
|
448
|
+
each_event do |event|
|
|
449
|
+
model_class = extract_model_class(event)
|
|
450
|
+
totals[model_class] += 1
|
|
451
|
+
duration = durations.fetch(model_class) { durations[model_class] = integration.retention_duration(model_class) }
|
|
452
|
+
next if duration.nil?
|
|
453
|
+
|
|
454
|
+
timestamp = extract_timestamp(event)
|
|
455
|
+
next unless timestamp && timestamp < (now - duration)
|
|
456
|
+
|
|
457
|
+
expired_ids[model_class] << (event.respond_to?(:event_id) ? event.event_id : event.object_id)
|
|
458
|
+
end
|
|
179
459
|
|
|
180
|
-
|
|
181
|
-
retention_duration =
|
|
460
|
+
totals.map do |model_class, total|
|
|
461
|
+
retention_duration = durations[model_class]
|
|
182
462
|
|
|
183
463
|
# nil means infinite/manual retention - always compliant
|
|
184
464
|
if retention_duration.nil?
|
|
185
|
-
{
|
|
186
|
-
|
|
187
|
-
total_events: model_events.count,
|
|
188
|
-
expired_events: 0,
|
|
189
|
-
retention_period: nil,
|
|
190
|
-
compliance_status: :manual,
|
|
191
|
-
expired_event_ids: []
|
|
192
|
-
}
|
|
465
|
+
{ model_class: model_class, total_events: total, expired_events: 0, retention_period: nil,
|
|
466
|
+
compliance_status: :manual, expired_event_ids: [] }
|
|
193
467
|
else
|
|
194
|
-
expired =
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
{
|
|
199
|
-
model_class: model_class,
|
|
200
|
-
total_events: model_events.count,
|
|
201
|
-
expired_events: expired.count,
|
|
202
|
-
retention_period: retention_duration,
|
|
203
|
-
compliance_status: expired.empty? ? :compliant : :requires_action,
|
|
204
|
-
expired_event_ids: expired.map(&:event_id)
|
|
205
|
-
}
|
|
468
|
+
expired = expired_ids[model_class]
|
|
469
|
+
{ model_class: model_class, total_events: total, expired_events: expired.size,
|
|
470
|
+
retention_period: retention_duration, compliance_status: expired.empty? ? :compliant : :requires_action,
|
|
471
|
+
expired_event_ids: expired }
|
|
206
472
|
end
|
|
207
473
|
end
|
|
208
474
|
end
|
|
@@ -57,6 +57,23 @@ module Lyra
|
|
|
57
57
|
true
|
|
58
58
|
end
|
|
59
59
|
|
|
60
|
+
# Def. 1 without its consent step, for a use whose subjects are not
|
|
61
|
+
# known yet (a query before it runs). +subject+ only labels the record.
|
|
62
|
+
def validate_fields!(_fields, _purpose, subject: nil)
|
|
63
|
+
true
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# The violations validate_fields! would find, without enforcing them.
|
|
67
|
+
def field_violations(_fields, _purpose)
|
|
68
|
+
[]
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# Condition 6 alone (the subject's restrictions and objections, Arts.
|
|
72
|
+
# 18 and 21), for a read that names none of the declared fields.
|
|
73
|
+
def validate_rights!(_fields, _purpose, subject:)
|
|
74
|
+
true
|
|
75
|
+
end
|
|
76
|
+
|
|
60
77
|
def allowed_purposes(_field)
|
|
61
78
|
[]
|
|
62
79
|
end
|
|
@@ -194,9 +211,10 @@ module Lyra
|
|
|
194
211
|
# the data was written, so reclassifying a field later does not rewrite
|
|
195
212
|
# the history, and it can be read without the policy or PAM.
|
|
196
213
|
#
|
|
197
|
-
# A create (or import
|
|
198
|
-
#
|
|
199
|
-
# attribute,
|
|
214
|
+
# A create (or import) annotates the attributes it carries; an update
|
|
215
|
+
# only the ones it changed. A destroy carries none, so it, an event with
|
|
216
|
+
# no declared attribute, and a model without a loaded policy are left
|
|
217
|
+
# unstamped.
|
|
200
218
|
#
|
|
201
219
|
# @return [Hash] the metadata, with metadata[:privacy] added when there
|
|
202
220
|
# is something to annotate
|
|
@@ -208,10 +226,12 @@ module Lyra
|
|
|
208
226
|
|
|
209
227
|
attributes = data[:attributes] || data["attributes"] || {}
|
|
210
228
|
changes = data[:changes] || data["changes"] || {}
|
|
211
|
-
# Every declared attribute the event carries:
|
|
212
|
-
# and
|
|
213
|
-
#
|
|
214
|
-
#
|
|
229
|
+
# Every declared attribute the event carries: a create's (or import's)
|
|
230
|
+
# attributes and an update's changes. Every mode now builds the same
|
|
231
|
+
# shape (Lyra::EventData), so a Monitor update carries only its changes,
|
|
232
|
+
# not the whole row as before 1.0.0, when stamping only the changes left
|
|
233
|
+
# 28,374 of the Olist replay's events carrying personal data
|
|
234
|
+
# unannotated, against Theorem 3's premise.
|
|
215
235
|
touched = attributes.keys | changes.keys
|
|
216
236
|
|
|
217
237
|
fields = touched.each_with_object({}) do |field, acc|
|