orfeas_lyra 0.9.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +2021 -17
  3. data/README.md +22 -9
  4. data/Rakefile +40 -1
  5. data/app/controllers/lyra/application_controller.rb +127 -0
  6. data/app/controllers/lyra/dashboard_controller.rb +33 -31
  7. data/app/controllers/lyra/flow_controller.rb +32 -34
  8. data/app/controllers/lyra/privacy_controller.rb +57 -18
  9. data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
  10. data/app/views/lyra/dashboard/index.html.erb +1 -0
  11. data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
  12. data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
  13. data/app/views/lyra/privacy/policy.html.erb +7 -0
  14. data/app/views/lyra/privacy/rights.html.erb +78 -0
  15. data/config/privacy_policies.rb +0 -3
  16. data/config/routes.rb +1 -0
  17. data/lib/lyra/access_log.rb +187 -6
  18. data/lib/lyra/advisory_lock.rb +236 -0
  19. data/lib/lyra/aggregate.rb +1 -1
  20. data/lib/lyra/associations/habtm_links.rb +382 -19
  21. data/lib/lyra/associations/maintenance.rb +229 -0
  22. data/lib/lyra/attribution.rb +48 -0
  23. data/lib/lyra/bypass_events.rb +446 -46
  24. data/lib/lyra/command_handler.rb +19 -14
  25. data/lib/lyra/configuration.rb +237 -10
  26. data/lib/lyra/consistency/read_your_writes.rb +4 -2
  27. data/lib/lyra/correlation.rb +121 -0
  28. data/lib/lyra/domain_events.rb +4 -0
  29. data/lib/lyra/dual_view.rb +53 -16
  30. data/lib/lyra/encrypted_attributes.rb +5 -0
  31. data/lib/lyra/engine.rb +67 -0
  32. data/lib/lyra/erasure.rb +1011 -106
  33. data/lib/lyra/event.rb +35 -1
  34. data/lib/lyra/event_data.rb +38 -0
  35. data/lib/lyra/event_flow.rb +31 -15
  36. data/lib/lyra/event_store_adapter.rb +28 -0
  37. data/lib/lyra/event_window.rb +58 -0
  38. data/lib/lyra/genesis.rb +548 -44
  39. data/lib/lyra/interceptors/association_interceptor.rb +689 -34
  40. data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
  41. data/lib/lyra/internal.rb +26 -0
  42. data/lib/lyra/mode_sync.rb +51 -3
  43. data/lib/lyra/mode_transition.rb +257 -30
  44. data/lib/lyra/model_selection.rb +35 -0
  45. data/lib/lyra/own_connection.rb +17 -2
  46. data/lib/lyra/privacy/adapters/pam.rb +25 -2
  47. data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
  48. data/lib/lyra/privacy/interface.rb +27 -7
  49. data/lib/lyra/privacy/subject_links.rb +143 -0
  50. data/lib/lyra/privacy/subject_records.rb +369 -0
  51. data/lib/lyra/projection.rb +7 -1
  52. data/lib/lyra/projections/async_checkpoint.rb +32 -0
  53. data/lib/lyra/projections/async_projection_job.rb +166 -23
  54. data/lib/lyra/projections/cached_joins.rb +33 -18
  55. data/lib/lyra/projections/cached_projection.rb +74 -33
  56. data/lib/lyra/projections/cached_relation.rb +3175 -321
  57. data/lib/lyra/projections/default_scopes.rb +254 -0
  58. data/lib/lyra/projections/deferred_relation.rb +23 -3
  59. data/lib/lyra/projections/event_store_reader.rb +47 -8
  60. data/lib/lyra/projections/foreign_keys.rb +92 -0
  61. data/lib/lyra/projections/lazy_projection.rb +165 -27
  62. data/lib/lyra/projections/model_projection.rb +165 -14
  63. data/lib/lyra/projections/rebuild.rb +34 -6
  64. data/lib/lyra/projections/reference_links.rb +463 -72
  65. data/lib/lyra/projections/typed_values.rb +629 -0
  66. data/lib/lyra/projections/unique_indexes.rb +595 -0
  67. data/lib/lyra/projections/uniqueness.rb +784 -0
  68. data/lib/lyra/projections/value_digest.rb +93 -0
  69. data/lib/lyra/purpose_bound_reads.rb +912 -77
  70. data/lib/lyra/repair.rb +119 -28
  71. data/lib/lyra/retention.rb +114 -8
  72. data/lib/lyra/rights/consent_store.rb +415 -0
  73. data/lib/lyra/rights/erasure_guard.rb +135 -0
  74. data/lib/lyra/rights/pending.rb +74 -0
  75. data/lib/lyra/rights/queries.rb +269 -0
  76. data/lib/lyra/rights/store.rb +512 -0
  77. data/lib/lyra/rights/subject_keys.rb +224 -0
  78. data/lib/lyra/rights.rb +281 -0
  79. data/lib/lyra/saves_without_effect.rb +69 -0
  80. data/lib/lyra/stream_tail.rb +319 -0
  81. data/lib/lyra/strict_data_access.rb +916 -106
  82. data/lib/lyra/temporal.rb +4 -2
  83. data/lib/lyra/touch_registry.rb +404 -0
  84. data/lib/lyra/verification/bypass_workflow.rb +452 -42
  85. data/lib/lyra/verification/rights_workflows.rb +170 -0
  86. data/lib/lyra/version.rb +1 -1
  87. data/lib/lyra/visualization/event_graph.rb +1 -1
  88. data/lib/lyra/visualization/timeline.rb +2 -2
  89. data/lib/lyra.rb +43 -2
  90. data/lib/tasks/lyra_mode.rake +110 -8
  91. data/lib/tasks/lyra_projections.rake +43 -2
  92. metadata +34 -4
  93. data/lib/lyra/associations/event_aware.rb +0 -224
@@ -50,7 +50,7 @@ module Lyra
50
50
  return nil unless pam_field
51
51
 
52
52
  Annotation.new(field: pam_field.name, type: pam_field.type, sensitivity: pam_field.sensitivity,
53
- sensitive: pam_field.sensitive?, purposes: pam_field.purposes, source: :policy,
53
+ sensitive: pam_field.sensitive?, purposes: admitting_purposes(pam_field), source: :policy,
54
54
  transformations: pam_field.transformations.keys)
55
55
  end
56
56
 
@@ -62,8 +62,21 @@ module Lyra
62
62
  pam_policy.validate_access!(fields, purpose, subject: subject)
63
63
  end
64
64
 
65
+ def validate_fields!(fields, purpose, subject: nil)
66
+ pam_policy.validate_fields!(fields, purpose, subject: subject)
67
+ end
68
+
69
+ def field_violations(fields, purpose)
70
+ pam_policy.field_violations(fields, purpose)
71
+ end
72
+
73
+ def validate_rights!(fields, purpose, subject:)
74
+ pam_policy.validate_rights!(fields, purpose, subject: subject)
75
+ end
76
+
65
77
  def allowed_purposes(field)
66
- find_field(field)&.purposes || []
78
+ pam_field = find_field(field)
79
+ pam_field ? admitting_purposes(pam_field) : []
67
80
  end
68
81
 
69
82
  def consent_required?(purpose)
@@ -112,6 +125,16 @@ module Lyra
112
125
 
113
126
  private
114
127
 
128
+ # The purposes the policy admits +pam_field+ for (Policy#allowed?):
129
+ # the field's own allow_for and every purpose that lists it in
130
+ # requires or optionally. Before 1.0.0 the annotation (and so the
131
+ # privacy stamp) carried allow_for alone, and a policy that lists its
132
+ # fields on the purpose side stamped every field with no purpose.
133
+ def admitting_purposes(pam_field)
134
+ listed = pam_policy.purposes.each_value.select { |purpose| pam_policy.allowed?(pam_field.name, purpose.name) }
135
+ (pam_field.purposes | listed.map(&:name))
136
+ end
137
+
115
138
  def find_field(field)
116
139
  pam_policy.get_field(field.to_sym)
117
140
  rescue PamDsl::InvalidFieldError
@@ -22,10 +22,16 @@ module Lyra
22
22
  #
23
23
  # @param subject_id [Object] The data subject's identifier
24
24
  # @param subject_type [String] The type/class of the subject (default: 'User')
25
+ # @param actor_label [Proc] ->(actor, subject) { label }: how the copy names who acted
26
+ # (default "you", "staff", "staff or you" or "system"; PamDsl::SubjectAccess::DEFAULT_ACTOR_LABEL)
27
+ # @param access_log_subjects [Array] the keys the application passed as this subject to the
28
+ # access log besides its records (validate_access!(..., subject: 7) writes Lyra::DataAccess$7);
29
+ # default: the bare subject id. [] reads none, where bare ids of several types share the log.
25
30
  #
26
- def initialize(subject_id:, subject_type: 'User')
31
+ def initialize(subject_id:, subject_type: 'User', actor_label: nil, access_log_subjects: nil)
27
32
  @subject_id = subject_id
28
33
  @subject_type = subject_type
34
+ @access_log_subjects = access_log_subjects.nil? ? [subject_id.to_s] : Array(access_log_subjects).map(&:to_s)
29
35
 
30
36
  @pam_compliance = PamDsl::GDPRCompliance.new(
31
37
  subject_id: subject_id,
@@ -37,15 +43,45 @@ module Lyra
37
43
  model_class_extractor: method(:extract_model_class),
38
44
  model_id_extractor: method(:extract_model_id),
39
45
  changes_extractor: method(:extract_changes),
40
- retention_policy: Lyra.config.retention_policy || default_retention_policy
46
+ retention_policy: retention_policy,
47
+ policy_resolver: method(:policy_for_model),
48
+ records_reader: method(:read_subject_records),
49
+ access_reader: method(:read_access_log),
50
+ export_recorder: Lyra::AccessLog.method(:record_export),
51
+ actor_label: actor_label,
52
+ rights_keys: method(:rights_keys)
41
53
  )
42
54
  end
43
55
 
44
56
  # Delegate all GDPR methods to the PAM DSL implementation
45
57
 
46
- # Right to Access (Article 15)
47
- def data_export
48
- pam_compliance.data_export
58
+ # Right to Access (Article 15): the subject's copy. Beside the event
59
+ # summaries it always gave, +records+ (each of the subject's records
60
+ # whole, with its history), +accesses+ (the access log; nil when
61
+ # config.record_access_events is off), +processing_context+, +notice+
62
+ # (the policy's access_notice) and +gaps+. See PamDsl::SubjectAccess
63
+ # and Lyra::Privacy::SubjectRecords. From 1.0.0 +restrictions+ and
64
+ # +objections+ list the subject's requests of Arts. 18 and 21: those
65
+ # recorded under the subject's key (EnfAuthor$7), and under the key of
66
+ # one of the subject's records that names no other person (a contested
67
+ # invoice). A record shared with another person is left out: its
68
+ # restriction may be the other person's request.
69
+ #
70
+ # An explicit call answers a request and is recorded, in the stream
71
+ # Lyra::SubjectAccessExports$<Type>$<id> (Lyra::AccessLog.record_export),
72
+ # unless config.record_subject_access_exports is false. record: false is
73
+ # for a view of the data that answers no request: full_report and the
74
+ # dashboard's subject and GDPR routes pass it, so a view is not on record
75
+ # as an answered request.
76
+ #
77
+ # history: false and accesses: false are for a request the subject
78
+ # has narrowed (recital 63); disclose_actors: true adds who acted (ids,
79
+ # IP addresses) to the role labels, after the controller has weighed
80
+ # the staff's rights; verified_by: names who verified the subject's
81
+ # identity, for the record of the export.
82
+ def data_export(history: true, accesses: true, disclose_actors: false, verified_by: nil, record: true)
83
+ pam_compliance.data_export(history: history, accesses: accesses, disclose_actors: disclose_actors,
84
+ verified_by: verified_by, record: record)
49
85
  end
50
86
 
51
87
  # Right to be Forgotten (Article 17)
@@ -78,39 +114,247 @@ module Lyra
78
114
  pam_compliance.consent_audit
79
115
  end
80
116
 
81
- # Full GDPR Compliance Report
117
+ # Full GDPR Compliance Report. Its data_export is not recorded as an
118
+ # answered request (see data_export).
82
119
  def full_report
83
120
  pam_compliance.full_report
84
121
  end
85
122
 
86
123
  private
87
124
 
125
+ # The PAM policy that governs +model_name+'s model (its privacy_policy,
126
+ # else config.privacy_policy), or nil. A subject's records span models
127
+ # with different policies; before 1.0.0 the wrapper passed none, and the
128
+ # register's legal basis read "Not specified".
129
+ def policy_for_model(model_name)
130
+ model = Lyra.config.monitored_model_named(model_name) || model_name.to_s.safe_constantize
131
+ policy = model.is_a?(Class) ? Lyra::Privacy.policy_for(model) : Lyra::Privacy.policy(Lyra.config.privacy_policy)
132
+ policy.respond_to?(:pam_policy) ? policy.pam_policy : nil
133
+ rescue StandardError
134
+ nil
135
+ end
136
+
137
+ # The keys the subject's rights are recorded under (see data_export):
138
+ # the subject's own, and those of their records that have requests
139
+ # (one lookup in lyra_rights) and name this subject alone.
140
+ def rights_keys(records)
141
+ model = Lyra.config.monitored_model_named(@subject_type.to_s) || @subject_type.to_s.safe_constantize
142
+ person = active_record_model?(model) ? Lyra.stream_name(model, @subject_id) : "#{@subject_type}$#{@subject_id}"
143
+ candidates = records.filter_map do |raw|
144
+ record_model = Lyra.config.monitored_model_named(raw[:model].to_s) || raw[:model].to_s.safe_constantize
145
+ next unless active_record_model?(record_model) && !raw[:id].nil?
146
+
147
+ key = Lyra.stream_name(record_model, raw[:id])
148
+ [key, record_model, raw[:id]] unless key == person
149
+ end
150
+ store = PamDsl.rights_store
151
+ keys = candidates.map(&:first)
152
+ with_requests = store.respond_to?(:keys_with_requests) ? store.keys_with_requests(keys) : keys
153
+ own = candidates.select { |key, _, _| with_requests.include?(key) }.filter_map do |key, record_model, id|
154
+ # Its row, or under ES-NoProj its state from its events.
155
+ row = Lyra::Rights::SubjectKeys.find_record(record_model, id)
156
+ key if row && Lyra::Rights::SubjectKeys.person_keys(row) == [person]
157
+ end
158
+ [person, *own]
159
+ end
160
+
161
+ def active_record_model?(model)
162
+ model.is_a?(Class) && model < ActiveRecord::Base
163
+ end
164
+
165
+ def read_subject_records(subject_id, subject_type, events, history: true)
166
+ SubjectRecords.new(subject_id, subject_type, events, history: history).call
167
+ end
168
+
169
+ ACCESS_LOG_OFF = "access log not enabled (accesses lists earlier exports only)"
170
+
171
+ # The access log's entries, oldest first: for each of the subject's
172
+ # records, those within the periods the record was the subject's (a
173
+ # moved record's accesses while it was another person's are theirs);
174
+ # for the subject's own key and the keys the application passed as the
175
+ # subject (access_log_subjects:, the bare id by default); and the
176
+ # subject's earlier exports, which have a stream of their own. With
177
+ # the log off, only the exports, and the gap that says so.
178
+ def read_access_log(subject_id, subject_type, records)
179
+ subject = Lyra::AccessLog.subject_key_for(subject_type, subject_id)
180
+ exports = Lyra::AccessLog.exports_for(subject).map { |event| access_entry(subject, event) }
181
+ return { entries: exports, gap: ACCESS_LOG_OFF } unless Lyra.config.record_access_events
182
+
183
+ periods = records.to_h { |record| ["#{record[:model]}$#{record[:id]}", record[:periods]] }
184
+ ([subject] + @access_log_subjects).each { |key| periods[key] = nil unless periods.key?(key) }
185
+ entries = periods.flat_map do |key, record_periods|
186
+ Lyra::AccessLog.for(key).map { |event| access_entry(key, event) }
187
+ .select { |entry| within?(record_periods, entry[:at]) }
188
+ end
189
+ (entries + exports).sort_by { |entry| entry[:at] || Time.at(0) }
190
+ end
191
+
192
+ # Whether +at+ falls in one of +periods+ (nil: the whole record).
193
+ def within?(periods, at)
194
+ return true if periods.nil? || at.nil?
195
+
196
+ periods.any? { |period| (period[:from].nil? || at >= period[:from]) && (period[:to].nil? || at <= period[:to]) }
197
+ end
198
+
199
+ def access_entry(key, event)
200
+ data = event.data.to_h.transform_keys(&:to_sym)
201
+ metadata = event.metadata.to_h.transform_keys(&:to_sym)
202
+ exported = event.event_type.to_s.end_with?("SubjectAccessExported")
203
+ {
204
+ at: data[:accessed_at] || data[:exported_at] || metadata[:timestamp],
205
+ record: key,
206
+ policy: data[:policy],
207
+ purpose: exported ? nil : data[:purpose],
208
+ legal_basis: data[:legal_basis],
209
+ fields: data[:fields],
210
+ outcome: exported ? "exported" : data[:outcome],
211
+ sections: data[:sections],
212
+ actor: metadata.slice(:user_id, :ip_address, :user_type).compact
213
+ }
214
+ end
215
+
88
216
  # Read all events related to the subject from Lyra's event store
217
+ # Every event of the subject's own record and of the records that refer
218
+ # to them (SubjectLinks: a belongs_to to the subject's class, or
219
+ # <type>_id), in log order. A related record is followed through its
220
+ # stream: its links are tracked from event to event, and an event belongs
221
+ # to the subject when the record named them before or after it. An event
222
+ # carries only what it changed (Lyra::EventData, 1.0.0), so matching the
223
+ # foreign key in each event alone kept a related record's create and lost
224
+ # its later updates and its destroy.
225
+ #
226
+ # The event that moves a record between people is given with the
227
+ # subject's side only: moving in, its new values; moving away, the
228
+ # links' old values and nothing of the new owner's (from 1.0.0; the
229
+ # reports built on these events gave both sides).
89
230
  def read_subject_events(subject_id, subject_type)
90
- Lyra.config.event_store.read.to_a.select do |event|
91
- event_relates_to_subject?(event, subject_id, subject_type)
231
+ links = SubjectLinks.new(subject_type)
232
+ own = links.subject_model ? Lyra.stream_owner_name(links.subject_model) : subject_type.to_s
233
+ values = {} # stream => the link columns' last known values
234
+ matching = []
235
+ each_event do |event|
236
+ relates = event_relates_to_subject?(event, subject_id, subject_type)
237
+ model_id = extract_model_id(event)
238
+ model = linked_model(extract_model_class(event))
239
+ name = model.is_a?(Class) ? Lyra.stream_owner_name(model) : model.to_s
240
+ if model_id.nil? || (name == own && model_id.to_s == subject_id.to_s) || links.for(model).empty?
241
+ matching << event if relates
242
+ next
243
+ end
244
+
245
+ key = "#{name}$#{model_id}"
246
+ classified, values[key] = SubjectRecords.classify([event], links: links, model: model, subject_id: subject_id,
247
+ initial: values[key] || {})
248
+ case classified.first.last
249
+ when :inside then matching << event
250
+ when :in, :away then matching << subject_side(event, classified.first.last, links.columns(model))
251
+ else matching << event if relates
252
+ end
92
253
  end
254
+ matching
255
+ end
256
+
257
+ def linked_model(name)
258
+ model = Lyra.config.monitored_model_named(name) || name.to_s.safe_constantize
259
+ model.is_a?(Class) && model < ActiveRecord::Base ? model : name.to_s
260
+ rescue StandardError
261
+ name.to_s
93
262
  end
94
263
 
95
- # Check if an event relates to the subject
264
+ # A transition event with the subject's side only: an update moving the
265
+ # record in keeps its new values, one moving it away the links' old
266
+ # values. A create or import moving it in is the subject's whole.
267
+ def subject_side(event, kind, link_columns)
268
+ data = event.data.to_h
269
+ changes = data[:changes] || data["changes"]
270
+ return event unless changes.is_a?(Hash) && Lyra::Event.operation_of(event) == :updated
271
+
272
+ side =
273
+ if kind == :in
274
+ changes.to_h { |field, change| [field, [nil, change.is_a?(Array) ? change.last : change]] }
275
+ else
276
+ changes.select { |field, _| link_columns.include?(field.to_s) }
277
+ .to_h { |field, change| [field, [change.is_a?(Array) ? change.first : nil, nil]] }
278
+ end
279
+ event.class.new(event_id: event.event_id, metadata: event.metadata.to_h,
280
+ data: data.except(:changes, "changes").merge(changes: side))
281
+ end
282
+
283
+ EVENT_BATCH = 1_000
284
+
285
+ # Every event in the log, read in batches: a report that must see all of it
286
+ # (a subject's events, retention) holds a batch at a time, not the whole log.
287
+ # SaveWithoutEffect markers hold no one's data and record no change
288
+ # (Lyra::SavesWithoutEffect): passed over.
289
+ def each_event(batch_size = EVENT_BATCH)
290
+ Lyra.config.event_store.read.in_batches(batch_size).each do |event|
291
+ yield event unless Lyra::SavesWithoutEffect.marker?(event)
292
+ end
293
+ end
294
+
295
+ # Whether an event holds the subject's data: an event of the subject's
296
+ # own record, or of a record that refers to the subject through its
297
+ # foreign key (a Registration's user_id, for subject type User).
298
+ #
299
+ # Who *made* a change is not whose data it is: before 0.9.3 an event
300
+ # whose metadata named the subject as its actor (metadata[:user_id])
301
+ # was exported too, so a staff member's export held every record they
302
+ # had edited, other people's data, while a subject's own records written
303
+ # by staff could be missed.
96
304
  def event_relates_to_subject?(event, subject_id, subject_type)
97
- metadata = event.respond_to?(:metadata) ? event.metadata : {}
98
305
  data = event.respond_to?(:data) ? event.data : {}
306
+ id = subject_id.to_s
307
+ return true if extract_model_class(event).to_s == subject_type.to_s && extract_model_id(event).to_s == id
308
+
309
+ explicit = data[:subject_id] || data["subject_id"]
310
+ if explicit
311
+ type = data[:subject_type] || data["subject_type"]
312
+ return explicit.to_s == id && (type.nil? || type.to_s == subject_type.to_s)
313
+ end
314
+
315
+ attributes = extract_attributes(event)
316
+ value = attributes[subject_foreign_key(subject_type)] || attributes[subject_foreign_key(subject_type).to_sym]
317
+ !value.nil? && value.to_s == id
318
+ end
99
319
 
100
- metadata[:user_id] == subject_id ||
101
- data[:user_id] == subject_id ||
102
- data[:subject_id] == subject_id ||
103
- (extract_model_class(event) == subject_type && extract_model_id(event) == subject_id)
320
+ # The foreign key a record uses to refer to a subject of +subject_type+.
321
+ def subject_foreign_key(subject_type)
322
+ "#{subject_type.to_s.demodulize.underscore}_id"
104
323
  end
105
324
 
106
325
  # Lyra-specific extractors
107
326
 
327
+ # The fields the event gives or changes (Lyra::Event.fields_of): an
328
+ # update holds its changes only, in every mode.
329
+ # Encrypted attributes are opened (the log holds their ciphertext), so
330
+ # the export's inventory and lineage give the values, as its records do.
108
331
  def extract_attributes(event)
332
+ return opened(event, Lyra::Event.fields_of(event)) if event.respond_to?(:data)
109
333
  return event.attributes if event.respond_to?(:attributes) && !event.attributes.is_a?(Method)
110
- return event.data[:attributes] || event.data["attributes"] || {} if event.respond_to?(:data)
111
334
  {}
112
335
  end
113
336
 
337
+ # The model whose encrypted attributes +event+ may hold, or nil.
338
+ def encrypted_model(event)
339
+ model = linked_model(extract_model_class(event))
340
+ model.is_a?(Class) && Lyra::EncryptedAttributes.names(model).any? ? model : nil
341
+ end
342
+
343
+ def opened(event, values)
344
+ model = encrypted_model(event)
345
+ model && values.is_a?(Hash) ? Lyra::EncryptedAttributes.open(model, values) : values
346
+ end
347
+
348
+ def opened_changes(event, changes)
349
+ model = encrypted_model(event)
350
+ return changes unless model && changes.is_a?(Hash)
351
+
352
+ changes.to_h do |field, change|
353
+ opened_change = Array(change).map { |value| Lyra::EncryptedAttributes.open(model, { field => value })[field] }
354
+ [field, change.is_a?(Array) ? opened_change : opened_change.first]
355
+ end
356
+ end
357
+
114
358
  def extract_timestamp(event)
115
359
  return event.timestamp if event.respond_to?(:timestamp) && event.timestamp
116
360
  return event.data[:timestamp] || event.data["timestamp"] if event.respond_to?(:data)
@@ -138,23 +382,43 @@ module Lyra
138
382
  def extract_changes(event)
139
383
  if event.respond_to?(:changes) && !event.changes.is_a?(Method)
140
384
  begin
141
- return event.changes unless event.method(:changes).owner.to_s.include?('ActiveRecord')
385
+ return opened_changes(event, event.changes) unless event.method(:changes).owner.to_s.include?('ActiveRecord')
142
386
  rescue
143
387
  return event.changes
144
388
  end
145
389
  end
146
- return event.data[:changes] || event.data["changes"] || {} if event.respond_to?(:data)
390
+ return opened_changes(event, event.data[:changes] || event.data["changes"] || {}) if event.respond_to?(:data)
147
391
  {}
148
392
  end
149
393
 
150
- def default_retention_policy
151
- {
152
- default: { duration: 7.years },
153
- 'Payment' => { duration: 10.years },
154
- 'Invoice' => { duration: 10.years },
155
- 'Student' => { duration: 10.years },
156
- 'Enrollment' => { duration: 10.years }
157
- }
394
+ # The retention periods the compliance check applies, as PAM's check
395
+ # expects them ({ "Model" => { duration: d }, default: { duration: d } }):
396
+ # each monitored model's period from its privacy policy's retention block,
397
+ # over config.retention_policy, which may be a single duration (7.years) or
398
+ # such a hash.
399
+ #
400
+ # Before 0.9.3 the policies' retention blocks were ignored in favour of
401
+ # hard-coded periods for models named Payment, Invoice, Student and
402
+ # Enrollment, and a single duration in config.retention_policy raised
403
+ # TypeError.
404
+ def retention_policy
405
+ configured = Lyra.config.retention_policy
406
+ policy =
407
+ case configured
408
+ when nil then { default: { duration: 7.years } }
409
+ when Hash then configured.transform_values { |v| v.is_a?(Hash) ? v : { duration: v } }
410
+ else { default: { duration: configured } }
411
+ end
412
+
413
+ Lyra.config.monitored_models.each do |model|
414
+ duration = Lyra::Privacy.policy_for(model).retention_for(model)
415
+ # A Duration, or a plain number of seconds (Numeric; a Duration also
416
+ # answers is_a?(Numeric) for its value).
417
+ policy[model.name] = { duration: duration } if duration.is_a?(ActiveSupport::Duration) || duration.is_a?(Numeric)
418
+ rescue StandardError
419
+ next
420
+ end
421
+ policy
158
422
  end
159
423
  end
160
424
 
@@ -175,34 +439,36 @@ module Lyra
175
439
  # Returns nil for models without defined retention (infinite/manual)
176
440
  def retention_compliance_with_policy(policy_name)
177
441
  integration = PolicyIntegration.new(policy_name)
178
- events = collect_all_events
442
+ now = Time.current
443
+ totals = Hash.new(0)
444
+ expired_ids = Hash.new { |hash, key| hash[key] = [] }
445
+ durations = {}
446
+
447
+ # One pass over the log, keeping counts and the ids of expired events, not the events.
448
+ each_event do |event|
449
+ model_class = extract_model_class(event)
450
+ totals[model_class] += 1
451
+ duration = durations.fetch(model_class) { durations[model_class] = integration.retention_duration(model_class) }
452
+ next if duration.nil?
453
+
454
+ timestamp = extract_timestamp(event)
455
+ next unless timestamp && timestamp < (now - duration)
456
+
457
+ expired_ids[model_class] << (event.respond_to?(:event_id) ? event.event_id : event.object_id)
458
+ end
179
459
 
180
- events.group_by { |e| e.model_class }.map do |model_class, model_events|
181
- retention_duration = integration.retention_duration(model_class)
460
+ totals.map do |model_class, total|
461
+ retention_duration = durations[model_class]
182
462
 
183
463
  # nil means infinite/manual retention - always compliant
184
464
  if retention_duration.nil?
185
- {
186
- model_class: model_class,
187
- total_events: model_events.count,
188
- expired_events: 0,
189
- retention_period: nil,
190
- compliance_status: :manual,
191
- expired_event_ids: []
192
- }
465
+ { model_class: model_class, total_events: total, expired_events: 0, retention_period: nil,
466
+ compliance_status: :manual, expired_event_ids: [] }
193
467
  else
194
- expired = model_events.select do |event|
195
- event.timestamp && event.timestamp < (Time.current - retention_duration)
196
- end
197
-
198
- {
199
- model_class: model_class,
200
- total_events: model_events.count,
201
- expired_events: expired.count,
202
- retention_period: retention_duration,
203
- compliance_status: expired.empty? ? :compliant : :requires_action,
204
- expired_event_ids: expired.map(&:event_id)
205
- }
468
+ expired = expired_ids[model_class]
469
+ { model_class: model_class, total_events: total, expired_events: expired.size,
470
+ retention_period: retention_duration, compliance_status: expired.empty? ? :compliant : :requires_action,
471
+ expired_event_ids: expired }
206
472
  end
207
473
  end
208
474
  end
@@ -57,6 +57,23 @@ module Lyra
57
57
  true
58
58
  end
59
59
 
60
+ # Def. 1 without its consent step, for a use whose subjects are not
61
+ # known yet (a query before it runs). +subject+ only labels the record.
62
+ def validate_fields!(_fields, _purpose, subject: nil)
63
+ true
64
+ end
65
+
66
+ # The violations validate_fields! would find, without enforcing them.
67
+ def field_violations(_fields, _purpose)
68
+ []
69
+ end
70
+
71
+ # Condition 6 alone (the subject's restrictions and objections, Arts.
72
+ # 18 and 21), for a read that names none of the declared fields.
73
+ def validate_rights!(_fields, _purpose, subject:)
74
+ true
75
+ end
76
+
60
77
  def allowed_purposes(_field)
61
78
  []
62
79
  end
@@ -194,9 +211,10 @@ module Lyra
194
211
  # the data was written, so reclassifying a field later does not rewrite
195
212
  # the history, and it can be read without the policy or PAM.
196
213
  #
197
- # A create (or import, or destroy) annotates the attributes it carries;
198
- # an update only the ones it changed. An event with no declared
199
- # attribute, or a model without a loaded policy, is left unstamped.
214
+ # A create (or import) annotates the attributes it carries; an update
215
+ # only the ones it changed. A destroy carries none, so it, an event with
216
+ # no declared attribute, and a model without a loaded policy are left
217
+ # unstamped.
200
218
  #
201
219
  # @return [Hash] the metadata, with metadata[:privacy] added when there
202
220
  # is something to annotate
@@ -208,10 +226,12 @@ module Lyra
208
226
 
209
227
  attributes = data[:attributes] || data["attributes"] || {}
210
228
  changes = data[:changes] || data["changes"] || {}
211
- # Every declared attribute the event carries: an update's own changes
212
- # and, in Monitor, the whole row it also carries. Stamping only the
213
- # changes left 28,374 of the Olist replay's events carrying personal
214
- # data unannotated, against Theorem 3's premise.
229
+ # Every declared attribute the event carries: a create's (or import's)
230
+ # attributes and an update's changes. Every mode now builds the same
231
+ # shape (Lyra::EventData), so a Monitor update carries only its changes,
232
+ # not the whole row as before 1.0.0, when stamping only the changes left
233
+ # 28,374 of the Olist replay's events carrying personal data
234
+ # unannotated, against Theorem 3's premise.
215
235
  touched = attributes.keys | changes.keys
216
236
 
217
237
  fields = touched.each_with_object({}) do |field, acc|