orfeas_lyra 0.9.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +2021 -17
  3. data/README.md +22 -9
  4. data/Rakefile +40 -1
  5. data/app/controllers/lyra/application_controller.rb +127 -0
  6. data/app/controllers/lyra/dashboard_controller.rb +33 -31
  7. data/app/controllers/lyra/flow_controller.rb +32 -34
  8. data/app/controllers/lyra/privacy_controller.rb +57 -18
  9. data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
  10. data/app/views/lyra/dashboard/index.html.erb +1 -0
  11. data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
  12. data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
  13. data/app/views/lyra/privacy/policy.html.erb +7 -0
  14. data/app/views/lyra/privacy/rights.html.erb +78 -0
  15. data/config/privacy_policies.rb +0 -3
  16. data/config/routes.rb +1 -0
  17. data/lib/lyra/access_log.rb +187 -6
  18. data/lib/lyra/advisory_lock.rb +236 -0
  19. data/lib/lyra/aggregate.rb +1 -1
  20. data/lib/lyra/associations/habtm_links.rb +382 -19
  21. data/lib/lyra/associations/maintenance.rb +229 -0
  22. data/lib/lyra/attribution.rb +48 -0
  23. data/lib/lyra/bypass_events.rb +446 -46
  24. data/lib/lyra/command_handler.rb +19 -14
  25. data/lib/lyra/configuration.rb +237 -10
  26. data/lib/lyra/consistency/read_your_writes.rb +4 -2
  27. data/lib/lyra/correlation.rb +121 -0
  28. data/lib/lyra/domain_events.rb +4 -0
  29. data/lib/lyra/dual_view.rb +53 -16
  30. data/lib/lyra/encrypted_attributes.rb +5 -0
  31. data/lib/lyra/engine.rb +67 -0
  32. data/lib/lyra/erasure.rb +1011 -106
  33. data/lib/lyra/event.rb +35 -1
  34. data/lib/lyra/event_data.rb +38 -0
  35. data/lib/lyra/event_flow.rb +31 -15
  36. data/lib/lyra/event_store_adapter.rb +28 -0
  37. data/lib/lyra/event_window.rb +58 -0
  38. data/lib/lyra/genesis.rb +548 -44
  39. data/lib/lyra/interceptors/association_interceptor.rb +689 -34
  40. data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
  41. data/lib/lyra/internal.rb +26 -0
  42. data/lib/lyra/mode_sync.rb +51 -3
  43. data/lib/lyra/mode_transition.rb +257 -30
  44. data/lib/lyra/model_selection.rb +35 -0
  45. data/lib/lyra/own_connection.rb +17 -2
  46. data/lib/lyra/privacy/adapters/pam.rb +25 -2
  47. data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
  48. data/lib/lyra/privacy/interface.rb +27 -7
  49. data/lib/lyra/privacy/subject_links.rb +143 -0
  50. data/lib/lyra/privacy/subject_records.rb +369 -0
  51. data/lib/lyra/projection.rb +7 -1
  52. data/lib/lyra/projections/async_checkpoint.rb +32 -0
  53. data/lib/lyra/projections/async_projection_job.rb +166 -23
  54. data/lib/lyra/projections/cached_joins.rb +33 -18
  55. data/lib/lyra/projections/cached_projection.rb +74 -33
  56. data/lib/lyra/projections/cached_relation.rb +3175 -321
  57. data/lib/lyra/projections/default_scopes.rb +254 -0
  58. data/lib/lyra/projections/deferred_relation.rb +23 -3
  59. data/lib/lyra/projections/event_store_reader.rb +47 -8
  60. data/lib/lyra/projections/foreign_keys.rb +92 -0
  61. data/lib/lyra/projections/lazy_projection.rb +165 -27
  62. data/lib/lyra/projections/model_projection.rb +165 -14
  63. data/lib/lyra/projections/rebuild.rb +34 -6
  64. data/lib/lyra/projections/reference_links.rb +463 -72
  65. data/lib/lyra/projections/typed_values.rb +629 -0
  66. data/lib/lyra/projections/unique_indexes.rb +595 -0
  67. data/lib/lyra/projections/uniqueness.rb +784 -0
  68. data/lib/lyra/projections/value_digest.rb +93 -0
  69. data/lib/lyra/purpose_bound_reads.rb +912 -77
  70. data/lib/lyra/repair.rb +119 -28
  71. data/lib/lyra/retention.rb +114 -8
  72. data/lib/lyra/rights/consent_store.rb +415 -0
  73. data/lib/lyra/rights/erasure_guard.rb +135 -0
  74. data/lib/lyra/rights/pending.rb +74 -0
  75. data/lib/lyra/rights/queries.rb +269 -0
  76. data/lib/lyra/rights/store.rb +512 -0
  77. data/lib/lyra/rights/subject_keys.rb +224 -0
  78. data/lib/lyra/rights.rb +281 -0
  79. data/lib/lyra/saves_without_effect.rb +69 -0
  80. data/lib/lyra/stream_tail.rb +319 -0
  81. data/lib/lyra/strict_data_access.rb +916 -106
  82. data/lib/lyra/temporal.rb +4 -2
  83. data/lib/lyra/touch_registry.rb +404 -0
  84. data/lib/lyra/verification/bypass_workflow.rb +452 -42
  85. data/lib/lyra/verification/rights_workflows.rb +170 -0
  86. data/lib/lyra/version.rb +1 -1
  87. data/lib/lyra/visualization/event_graph.rb +1 -1
  88. data/lib/lyra/visualization/timeline.rb +2 -2
  89. data/lib/lyra.rb +43 -2
  90. data/lib/tasks/lyra_mode.rake +110 -8
  91. data/lib/tasks/lyra_projections.rake +43 -2
  92. metadata +34 -4
  93. data/lib/lyra/associations/event_aware.rb +0 -224
@@ -12,9 +12,11 @@ module Lyra
12
12
  subject_type: subject_type
13
13
  )
14
14
 
15
+ # A view of the data, not an answered Art. 15 request: not recorded as an
16
+ # export (the application's explicit data_export call is).
15
17
  render json: {
16
18
  subject: { type: subject_type, id: subject_id },
17
- data_export: compliance.data_export,
19
+ data_export: compliance.data_export(record: false),
18
20
  generated_at: Time.current
19
21
  }
20
22
  end
@@ -31,7 +33,7 @@ module Lyra
31
33
 
32
34
  render json: {
33
35
  subject: { type: subject_type, id: subject_id },
34
- right_to_access: compliance.data_export,
36
+ right_to_access: compliance.data_export(record: false), # a view, not recorded as an export
35
37
  right_to_be_forgotten: compliance.right_to_be_forgotten_report,
36
38
  rectification_history: compliance.rectification_history,
37
39
  processing_activities: compliance.processing_activities,
@@ -70,7 +72,9 @@ module Lyra
70
72
  subject_type = params[:subject_type]
71
73
  subject_id = params[:subject_id]
72
74
 
73
- flow = EventFlow.new(subject_id: subject_id, subject_type: subject_type)
75
+ subject_model = Lyra.config.monitored_model_named(subject_type.to_s)
76
+ refuse_restricted_record!(subject_model, subject_id) if subject_model
77
+ flow = EventFlow.new(subject_id: subject_id, subject_type: subject_type, event_filter: method(:rights_visible))
74
78
  data = flow.privacy_impact_analysis
75
79
 
76
80
  render json: data
@@ -81,7 +85,7 @@ module Lyra
81
85
  field_name = params[:field_name]
82
86
  model_class = params[:model_class]
83
87
 
84
- flow = EventFlow.new
88
+ flow = EventFlow.new(event_filter: method(:rights_visible))
85
89
  lineage = flow.data_lineage(field_name, model_class)
86
90
 
87
91
  render json: lineage
@@ -95,14 +99,35 @@ module Lyra
95
99
  Rails.application.config.pam_dsl.default_policy : nil
96
100
  end
97
101
 
102
+ # GET /lyra/privacy/rights
103
+ # The data subjects' requests still open (PamDsl.rights.open_requests):
104
+ # restrictions not yet lifted (Art. 18) and objections not yet decided
105
+ # (Art. 21), oldest first, each with its age against the month Art. 12(3)
106
+ # gives the controller to act on a request. It reads Lyra's rights table,
107
+ # not a monitored model's records, under the dashboard's purpose rules,
108
+ # and is logged as a dashboard read. Names, grounds and states only:
109
+ # never a value of the subject's data.
110
+ def rights
111
+ now = Time.current
112
+ requests = PamDsl.rights.open_requests
113
+ log_dashboard_read("privacy/rights", {}, returned: requests.size)
114
+ @requests = requests.map { |request| open_request_row(request, now) }
115
+
116
+ respond_to do |format|
117
+ format.html
118
+ format.json { render json: { open_requests: @requests, generated_at: now } }
119
+ end
120
+ end
121
+
98
122
  # GET /lyra/privacy/pii_detection
99
123
  def pii_detection
100
- # Detect PII across all events
101
- events = Lyra.config.event_store.read.to_a
124
+ # Detect PII across the most recent events (bounded: recent_events)
125
+ events = recent_events
126
+ log_dashboard_read("privacy/pii_detection", {}, returned: events.size)
102
127
  @pii_inventory = extract_pii_from_events(events)
103
128
 
104
129
  @total_events = events.count
105
- @events_with_pii = events.count { |e| Privacy::PIIDetector.detect(event_attributes(e)).any? }
130
+ @events_with_pii = events.count { |e| Privacy::PIIDetector.detect(event_fields(e)).any? }
106
131
  @pii_categories = @pii_inventory.keys
107
132
  @sensitive_pii = @pii_inventory.select { |k, _|
108
133
  [:ssn, :credit_card, :health, :biometric].include?(k)
@@ -130,6 +155,26 @@ module Lyra
130
155
 
131
156
  private
132
157
 
158
+ # One open request, for the rights page. Only an objection still awaiting
159
+ # the controller's decision has a deadline: one month from its receipt
160
+ # (Art. 12(3); an extension under its second sentence is the
161
+ # controller's to record and tell the subject of). A restriction in
162
+ # force is the controller's action on the request already (Lyra records
163
+ # it when it is applied), so it is listed with its age and no deadline:
164
+ # it lasts until it is lifted, with the Art. 18(3) notice.
165
+ def open_request_row(request, now)
166
+ restriction = request.kind == :restriction
167
+ deadline = restriction ? nil : (request.requested_at.to_datetime >> 1).to_time
168
+ {
169
+ id: request.id, kind: request.kind, subject: request.subject_key, article: request.article,
170
+ ground: restriction ? request.ground : nil, fields: restriction ? request.fields : [],
171
+ policy: restriction ? nil : request.policy, purpose: restriction ? nil : request.purpose,
172
+ state: request.state, requested_at: request.requested_at,
173
+ age_days: ((now - request.requested_at) / 86_400).floor, deadline: deadline,
174
+ overdue: deadline ? now > deadline : false
175
+ }
176
+ end
177
+
133
178
  def require_pam_dsl
134
179
  unless defined?(PAM_DSL_AVAILABLE) && PAM_DSL_AVAILABLE
135
180
  respond_to do |format|
@@ -143,16 +188,10 @@ module Lyra
143
188
  defined?(PAM_DSL_AVAILABLE) && PAM_DSL_AVAILABLE
144
189
  end
145
190
 
146
- def event_attributes(event)
147
- # First try Lyra::Event accessor method
148
- if event.respond_to?(:attributes)
149
- attrs = event.attributes
150
- return attrs if attrs.is_a?(Hash)
151
- end
152
- # Fall back to data hash for RubyEventStore events
153
- data = event.respond_to?(:data) ? event.data : nil
154
- return {} unless data
155
- data[:attributes] || data["attributes"] || {}
191
+ # The fields an event gives or changes, whatever mode wrote it
192
+ # (Lyra::Event.fields_of): what personal data is detected in.
193
+ def event_fields(event)
194
+ Lyra::Event.fields_of(event)
156
195
  end
157
196
 
158
197
  def event_model_class(event)
@@ -165,7 +204,7 @@ module Lyra
165
204
  def extract_pii_from_events(events)
166
205
  inventory = Hash.new { |h, k| h[k] = [] }
167
206
  events.each do |event|
168
- pii = Privacy::PIIDetector.detect(event_attributes(event))
207
+ pii = Privacy::PIIDetector.detect(event_fields(event))
169
208
  next unless pii.is_a?(Hash)
170
209
  pii.each do |field, info|
171
210
  next unless info.is_a?(Hash) && info[:type]
@@ -48,8 +48,8 @@
48
48
  <%
49
49
  # DualView.find_discrepancies returns only the records that differ, each
50
50
  # a DualView#compare result: differences maps a field to
51
- # { crud:, event_sourced: }, or is { exists_mismatch: true } when the row
52
- # and the log disagree on whether the record exists.
51
+ # { crud:, event_sourced: }; exists: { crud:, event_sourced: } is the one
52
+ # entry when the row and the log disagree on whether the record exists.
53
53
  mismatches = @discrepancies.size
54
54
  total_records = [@records_checked.to_i, mismatches].max
55
55
  matches = total_records - mismatches
@@ -95,7 +95,7 @@
95
95
  <span class="status mismatch">Mismatch</span>
96
96
  </div>
97
97
  <div class="discrepancy-body">
98
- <% if differences[:exists_mismatch] %>
98
+ <% if differences.key?(:exists) %>
99
99
  <p class="existence-note">
100
100
  <% if crud_view[:exists] %>
101
101
  The row exists, but the event log does not describe this record
@@ -112,7 +112,7 @@
112
112
  <p class="source-label">Database State</p>
113
113
  <h4>Current Values</h4>
114
114
  <ul class="field-list">
115
- <% if differences[:exists_mismatch] %>
115
+ <% if differences.key?(:exists) %>
116
116
  <li>
117
117
  <span class="field-name">row</span>
118
118
  <span class="field-value field-mismatch"><%= crud_view[:exists] ? "present" : "absent" %></span>
@@ -131,7 +131,7 @@
131
131
  <p class="source-label">Event-Sourced State</p>
132
132
  <h4>Projected Values</h4>
133
133
  <ul class="field-list">
134
- <% if differences[:exists_mismatch] %>
134
+ <% if differences.key?(:exists) %>
135
135
  <li>
136
136
  <span class="field-name">events</span>
137
137
  <span class="field-value field-mismatch"><%= es_view[:events_count].to_i %></span>
@@ -101,6 +101,7 @@
101
101
  <div class="section-grid">
102
102
  <%= link_to "PII Detection", pii_detection_path %>
103
103
  <%= link_to "Privacy Policy (PAM)", privacy_policy_path %>
104
+ <%= link_to "Rights Requests (Arts. 18, 21)", privacy_rights_path %>
104
105
  </div>
105
106
  <% end %>
106
107
 
@@ -42,7 +42,7 @@
42
42
  <div class="summary-cards">
43
43
  <div class="summary-card">
44
44
  <div class="value"><%= @total_events %></div>
45
- <div class="label">Total Events</div>
45
+ <div class="label">Total Events<%= " (most recent #{@window[:limit]})" if @window && @window[:truncated] %></div>
46
46
  </div>
47
47
  <% (@operations_summary || {}).each do |op, count| %>
48
48
  <div class="summary-card">
@@ -59,7 +59,7 @@
59
59
  <div class="summary-cards">
60
60
  <div class="summary-card">
61
61
  <div class="value"><%= @total_events %></div>
62
- <div class="label">Total Events</div>
62
+ <div class="label">Total Events<%= " (most recent #{@window[:limit]})" if @window && @window[:truncated] %></div>
63
63
  </div>
64
64
  <div class="summary-card">
65
65
  <div class="value"><%= @events_with_pii %></div>
@@ -132,6 +132,13 @@
132
132
  <strong>Required fields:</strong> <%= purpose.required_fields.join(', ') %>
133
133
  </div>
134
134
  <% end %>
135
+ <% if purpose.respond_to?(:objectable) %>
136
+ <div class="required-fields">
137
+ <strong>Right to object:</strong> <%= purpose.objectable || "none" %>;
138
+ <strong>continues under a restriction:</strong>
139
+ <%= purpose.restriction_exceptions.any? ? purpose.restriction_exceptions.map { |e| e.to_s.tr('_', ' ') }.join(', ') : "no (no Art. 18(2) exception declared)" %>
140
+ </div>
141
+ <% end %>
135
142
  </div>
136
143
  <% end %>
137
144
  </div>
@@ -0,0 +1,78 @@
1
+ <style>
2
+ .lyra-rights { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; max-width: 1200px; margin: 0 auto; padding: 20px; }
3
+ .lyra-rights h1 { color: #1a1a2e; border-bottom: 2px solid #4a4e69; padding-bottom: 10px; }
4
+ .lyra-rights .back-link { display: inline-block; margin-bottom: 20px; color: #4a4e69; text-decoration: none; }
5
+ .lyra-rights .back-link:hover { text-decoration: underline; }
6
+ .lyra-rights .note { color: #6c757d; font-size: 13px; margin-bottom: 20px; }
7
+ .lyra-rights table { width: 100%; border-collapse: collapse; background: #fff; border: 1px solid #dee2e6; border-radius: 8px; }
8
+ .lyra-rights th { background: #f8f9fa; padding: 10px 12px; text-align: left; border-bottom: 1px solid #dee2e6; font-weight: 600; font-size: 13px; }
9
+ .lyra-rights td { padding: 10px 12px; border-bottom: 1px solid #eee; font-size: 13px; vertical-align: top; }
10
+ .lyra-rights tr:last-child td { border-bottom: none; }
11
+ .lyra-rights .overdue { color: #721c24; background: #f8d7da; padding: 2px 8px; border-radius: 3px; font-weight: 600; }
12
+ .lyra-rights .due { color: #155724; background: #d4edda; padding: 2px 8px; border-radius: 3px; }
13
+ .lyra-rights .empty-state { text-align: center; padding: 40px; color: #6c757d; background: #f8f9fa; border: 1px solid #dee2e6; border-radius: 8px; }
14
+ </style>
15
+
16
+ <div class="lyra-rights">
17
+ <%= link_to "&larr; Back to Dashboard".html_safe, dashboard_path, class: "back-link" %>
18
+
19
+ <h1>Open rights requests</h1>
20
+ <p class="note">
21
+ Restrictions of processing not yet lifted (Art. 18) and objections not yet decided (Art. 21), oldest first.
22
+ An objection awaiting a decision is to be answered within one month of its receipt (Art. 12(3)): that is
23
+ its deadline, without an extension, which the controller records and tells the subject of. A restriction
24
+ listed here is in force: recording it was the controller's action on the request, so it has no deadline;
25
+ it lasts until it is lifted, after the subject is told (Art. 18(3)).
26
+ </p>
27
+
28
+ <% if @requests.empty? %>
29
+ <div class="empty-state">No open requests.</div>
30
+ <% else %>
31
+ <table>
32
+ <thead>
33
+ <tr>
34
+ <th>Request</th>
35
+ <th>Subject</th>
36
+ <th>Article</th>
37
+ <th>What</th>
38
+ <th>State</th>
39
+ <th>Received</th>
40
+ <th>Age</th>
41
+ <th>Deadline</th>
42
+ </tr>
43
+ </thead>
44
+ <tbody>
45
+ <% @requests.each do |request| %>
46
+ <tr>
47
+ <td><%= request[:kind] %><br><code><%= request[:id] %></code></td>
48
+ <td><%= request[:subject] %></td>
49
+ <td><%= request[:article] %></td>
50
+ <td>
51
+ <% if request[:kind] == :restriction %>
52
+ <%= request[:ground].to_s.tr("_", " ") %>:
53
+ <%= request[:fields].any? ? request[:fields].join(", ") : "all of the subject's data" %>
54
+ <% else %>
55
+ purpose <%= request[:purpose] %> of policy <%= request[:policy] %>
56
+ <% end %>
57
+ </td>
58
+ <td><%= request[:state].to_s.tr("_", " ") %></td>
59
+ <td><%= request[:requested_at].strftime("%Y-%m-%d %H:%M") %></td>
60
+ <td><%= request[:age_days] %> days</td>
61
+ <td>
62
+ <% if request[:deadline] %>
63
+ <%= request[:deadline].strftime("%Y-%m-%d") %>
64
+ <% if request[:overdue] %>
65
+ <span class="overdue">overdue</span>
66
+ <% else %>
67
+ <span class="due">due</span>
68
+ <% end %>
69
+ <% else %>
70
+ in force (no deadline)
71
+ <% end %>
72
+ </td>
73
+ </tr>
74
+ <% end %>
75
+ </tbody>
76
+ </table>
77
+ <% end %>
78
+ </div>
@@ -181,7 +181,6 @@ PamDsl.define_policy :university_system do
181
181
  consent do
182
182
  for_purpose :marketing do
183
183
  required!
184
- granular!
185
184
  withdrawable!
186
185
  expires_in 2.years
187
186
  describe "We'll send you information about upcoming events, programs, and opportunities"
@@ -189,7 +188,6 @@ PamDsl.define_policy :university_system do
189
188
 
190
189
  for_purpose :analytics do
191
190
  required! false
192
- granular!
193
191
  withdrawable!
194
192
  describe "Help us improve our services by sharing anonymous usage data"
195
193
  end
@@ -266,7 +264,6 @@ PamDsl.define_policy :ecommerce do
266
264
  consent do
267
265
  for_purpose :marketing do
268
266
  required!
269
- granular!
270
267
  withdrawable!
271
268
  expires_in 1.year
272
269
  end
data/config/routes.rb CHANGED
@@ -20,6 +20,7 @@ Lyra::Engine.routes.draw do
20
20
  get 'privacy/data_lineage/:field_name', to: 'privacy#data_lineage', as: :data_lineage
21
21
  get 'privacy/pii_detection', to: 'privacy#pii_detection', as: :pii_detection
22
22
  get 'privacy/policy', to: 'privacy#policy', as: :privacy_policy
23
+ get 'privacy/rights', to: 'privacy#rights', as: :privacy_rights
23
24
 
24
25
  # Configuration and projections routes
25
26
  get 'config/projections', to: 'dashboard#projections', as: :projections
@@ -17,22 +17,48 @@ module Lyra
17
17
  #
18
18
  # Who accessed goes in the metadata: user_id (Current.user) and
19
19
  # ip_address (Current.ip_address) when the application keeps them in
20
- # Current, request and correlation ids, and whatever
20
+ # Current, request and correlation ids, the cause from outside every write
21
+ # scope (a write in scope may roll back; the entry stays), and whatever
21
22
  # config.access_metadata_proc returns for the access (a console, a job,
22
23
  # an API token: anything Current does not know). Without either, an
23
24
  # access carries no user: set one.
24
25
  #
25
- # Every access is recorded, with no sampling. An access that cannot be
26
- # recorded does not go ahead: the store's error propagates from
27
- # validate_access!. Off by default, and nothing is recorded in disabled
26
+ # Every access is recorded, with no sampling, on a connection of its own
27
+ # (Lyra::OwnConnection), so that it stays recorded when the reader's
28
+ # transaction rolls back: the read has already returned its value by then.
29
+ # Inside a transaction that costs a thread and a second connection per
30
+ # access, so size the pool for it: with every thread of the application
31
+ # (Puma's, Sidekiq's) inside a transaction, a read that finds no free
32
+ # connection waits the pool's checkout_timeout (5 s by default) and is
33
+ # then recorded inline, in the reader's transaction, and lost if that rolls
34
+ # back. A pool of twice the threads avoids the wait.
35
+ # An access that cannot be recorded does not go ahead: the store's error
36
+ # propagates from validate_access!. Off by default, and nothing is recorded in disabled
28
37
  # mode: reads can outnumber writes by orders of magnitude, so its cost is
29
38
  # measured separately (LYRA_RECORD_ACCESS=1 in the Aegean testbed).
30
39
  #
31
40
  # The Article 30 record of processing does not depend on it: PAM's
32
41
  # reporter generates that from the declared policy. The access log is
33
42
  # the evidence of what was actually accessed.
43
+ #
44
+ # Erasure (from 1.0.0). A subject's log can be deleted: with the record
45
+ # (Lyra::Erasure.erase!, config.erase_access_log or access_log: true), or
46
+ # on its own with AccessLog.erase!, which also takes the bare ids an
47
+ # application passes as subjects. Its events are deleted from every stream
48
+ # they are in and from the events table, and an AccessLogErased event in
49
+ # Lyra::AccessLogErasures records which subject, how many entries, why and
50
+ # who. A model-wide entry (Lyra::DataAccess$<Model>$*, written by a pluck)
51
+ # names no subject: it is the record of a read of everyone's data, and no
52
+ # one person's erasure removes it.
34
53
  module AccessLog
35
54
  STREAM = "Lyra::DataAccess"
55
+ ERASURES = "Lyra::AccessLogErasures"
56
+ # Each Art. 15 export, per subject (record_export); not part of the
57
+ # access log, so erasing the log leaves it.
58
+ EXPORTS = "Lyra::SubjectAccessExports"
59
+ DELETE_BATCH = 1000
60
+
61
+ class ModelWideSubject < ArgumentError; end
36
62
 
37
63
  class << self
38
64
  # PamDsl.access_recorder protocol.
@@ -44,7 +70,74 @@ module Lyra
44
70
  event_class = access.outcome == :denied ? Lyra::Events::DataAccessDenied : Lyra::Events::DataAccessed
45
71
  subject = subject_key(access.subject)
46
72
  event = event_class.new(data: data_for(access, subject), metadata: metadata(access))
47
- Lyra.append_events(event, stream_name: stream_for(subject))
73
+ # Outside a transaction this just runs; inside one it runs on a second
74
+ # connection, so a rollback of the reader's transaction keeps the entry.
75
+ Lyra::OwnConnection.run { Lyra.append_events(event, stream_name: stream_for(subject)) }
76
+ end
77
+
78
+ # A read through the dashboard (a flow or privacy route): who, which route, the
79
+ # ids it was scoped to and how much came back. Written when
80
+ # config.record_access_events is on, to Lyra::DataAccess$Dashboard.
81
+ def dashboard_read(route:, scope: {}, returned: nil, truncated: nil)
82
+ return unless record?(nil)
83
+
84
+ data = { route: route.to_s, scope: scope.transform_values(&:to_s), returned: returned, truncated: truncated,
85
+ accessed_at: Time.current }.compact
86
+ event = Lyra::Events::DashboardRead.new(data: data, metadata: metadata_for_dashboard)
87
+ # On its own connection, as every access entry (call).
88
+ Lyra::OwnConnection.run { Lyra.append_events(event, stream_name: stream_for("Dashboard")) }
89
+ end
90
+
91
+ # Records that a data subject's Art. 15 copy was made
92
+ # (GDPRCompliance#data_export, an explicit call): a SubjectAccessExported
93
+ # event with which sections it gave, how many records, who verified the
94
+ # subject and who exported (the metadata, as for any access). Never a
95
+ # value. On unless config.record_subject_access_exports is false;
96
+ # nothing in disabled mode. Written on its own connection, as every
97
+ # access entry.
98
+ #
99
+ # It goes to a stream of its own, Lyra::SubjectAccessExports$<subject>,
100
+ # not to the subject's access stream: erasing the access log (erase!,
101
+ # Lyra::Erasure with access_log:) deletes that stream, and the evidence
102
+ # that a request was answered must outlive it, as the record of an
103
+ # erasure does (Lyra::AccessLogErasures).
104
+ def record_export(export)
105
+ return if !Lyra.config.record_subject_access_exports || Lyra.config.disabled_mode?
106
+
107
+ subject = subject_key_for(export[:subject_type], export[:subject_id])
108
+ data = {
109
+ subject: subject,
110
+ sections: Array(export[:sections]).map(&:to_s),
111
+ records: export[:records],
112
+ gaps: export[:gaps],
113
+ disclose_actors: export[:disclose_actors] ? true : false,
114
+ verified_by: export[:verified_by]&.to_s,
115
+ exported_at: export[:generated_at] || Time.current
116
+ }.compact
117
+ event = Lyra::Events::SubjectAccessExported.new(data: data, metadata: metadata_for_export)
118
+ Lyra::OwnConnection.run { Lyra.append_events(event, stream_name: exports_stream_for(subject)) }
119
+ end
120
+
121
+ # The recorded exports of +subject+ (a key such as "Registration$5"),
122
+ # oldest first.
123
+ def exports_for(subject)
124
+ Lyra.config.event_store.read.stream(exports_stream_for(subject)).to_a
125
+ end
126
+
127
+ def exports_stream_for(subject)
128
+ "#{EXPORTS}$#{subject}"
129
+ end
130
+
131
+ # The access-log key of a subject given by type and id: the record's
132
+ # ("Registration$5", the STI base class's name) when the type is a
133
+ # model, "<type>$<id>" otherwise.
134
+ def subject_key_for(subject_type, subject_id)
135
+ model = subject_type.to_s.safe_constantize
136
+ if model.is_a?(Class) && model < ActiveRecord::Base
137
+ Lyra.stream_name(model, subject_id)
138
+ else
139
+ "#{subject_type}$#{subject_id}"
140
+ end
48
141
  end
49
142
 
50
143
  def stream_for(subject)
@@ -65,6 +158,63 @@ module Lyra
65
158
  Lyra.config.event_store.read.stream(stream_for(subject_key(subject))).to_a
66
159
  end
67
160
 
161
+ # Deletes +subject+'s access log (a record, or an id the application
162
+ # passed as the subject) and records that it did, with the reason and
163
+ # who. Returns how many entries it deleted. Refuses a model-wide key.
164
+ def erase!(subject, reason:, erased_by: nil)
165
+ key = erasable_key(subject)
166
+ Lyra::PurposeBoundReads.internal(INTERNAL) do
167
+ ActiveRecord::Base.transaction do
168
+ entries = delete_stream!(stream_for(key))
169
+ # Beside the claimed label, what the operating system says
170
+ # (Lyra::Attribution), as for an ErasureApplied.
171
+ who = { os_user: Lyra::Attribution.os_user, host: Lyra::Attribution.host, pid: Process.pid }
172
+ erasure = Lyra::Events::AccessLogErased.new(
173
+ data: { subject: key, entries: entries, reason: reason.to_s, erased_at: Time.current },
174
+ metadata: { source: "lyra_access_log", erased_by: erased_by&.to_s, os_user: who[:os_user],
175
+ host: who[:host], pid: who[:pid], correlation_id: Lyra::Correlation.current_id }.compact
176
+ )
177
+ Lyra.append_events(erasure, stream_name: ERASURES)
178
+ entries
179
+ end
180
+ end
181
+ end
182
+
183
+ # The subject key for +subject+, refusing nil and model-wide keys.
184
+ def erasable_key(subject)
185
+ key = subject_key(subject)
186
+ raise ArgumentError, "no subject" if key.nil? || key.empty?
187
+ if key.end_with?("$*")
188
+ raise ModelWideSubject, "#{key} is a model-wide entry (a pluck): it names no subject, and erasing it " \
189
+ "would erase the record of reads of everyone's data"
190
+ end
191
+
192
+ key
193
+ end
194
+
195
+ # How many entries +stream+ holds.
196
+ def count_stream(stream)
197
+ ActiveRecord::Base.with_connection do |conn|
198
+ conn.select_value("SELECT count(*) FROM event_store_events_in_streams WHERE stream = #{conn.quote(stream)}").to_i
199
+ end
200
+ end
201
+
202
+ # Deletes +stream+'s events from every stream they are in and from the
203
+ # events table (RES's delete_stream only unlinks). Returns how many.
204
+ def delete_stream!(stream)
205
+ ActiveRecord::Base.with_connection do |conn|
206
+ ids = conn.select_values(
207
+ "SELECT event_id FROM event_store_events_in_streams WHERE stream = #{conn.quote(stream)}"
208
+ ).uniq
209
+ ids.each_slice(DELETE_BATCH) do |slice|
210
+ list = slice.map { |event_id| conn.quote(event_id) }.join(", ")
211
+ conn.execute("DELETE FROM event_store_events_in_streams WHERE event_id IN (#{list})")
212
+ conn.execute("DELETE FROM event_store_events WHERE event_id IN (#{list})")
213
+ end
214
+ ids.size
215
+ end
216
+ end
217
+
68
218
  private
69
219
 
70
220
  def data_for(access, subject)
@@ -86,13 +236,39 @@ module Lyra
86
236
  def metadata(access)
87
237
  {
88
238
  user_id: current(:user)&.id,
239
+ user_type: Lyra::Attribution.actor_type(current(:user)),
89
240
  ip_address: current(:ip_address),
90
241
  request_id: current(:request_id),
91
242
  correlation_id: Lyra::Correlation.current_id,
92
- causation_id: Lyra::Causation.current_id
243
+ # Not a write in scope: the entry commits on its own connection and
244
+ # stays when that write rolls back, so it would name no event. The
245
+ # correlation still ties the read to the writes around it.
246
+ causation_id: Lyra::Causation.outside_write_scopes
93
247
  }.merge(custom_metadata(access)).merge(source: "lyra_access_log").compact
94
248
  end
95
249
 
250
+ def metadata_for_export
251
+ {
252
+ user_id: current(:user)&.id,
253
+ user_type: Lyra::Attribution.actor_type(current(:user)),
254
+ ip_address: current(:ip_address),
255
+ request_id: current(:request_id),
256
+ correlation_id: Lyra::Correlation.current_id,
257
+ source: "lyra_subject_access"
258
+ }.compact
259
+ end
260
+
261
+ def metadata_for_dashboard
262
+ {
263
+ user_id: current(:user)&.id,
264
+ user_type: Lyra::Attribution.actor_type(current(:user)),
265
+ ip_address: current(:ip_address),
266
+ request_id: current(:request_id),
267
+ correlation_id: Lyra::Correlation.current_id,
268
+ source: "lyra_dashboard"
269
+ }.compact
270
+ end
271
+
96
272
  def current(attribute)
97
273
  defined?(::Current) && ::Current.respond_to?(attribute) ? ::Current.public_send(attribute) : nil
98
274
  end
@@ -115,6 +291,11 @@ module Lyra
115
291
  module Events
116
292
  class DataAccessed < Lyra::Event; end
117
293
  class DataAccessDenied < Lyra::Event; end
294
+ class DashboardRead < Lyra::Event; end
295
+ # A data subject's Art. 15 copy was made (AccessLog.record_export).
296
+ class SubjectAccessExported < Lyra::Event; end
297
+ # A subject's access log was deleted (AccessLog.erase!).
298
+ class AccessLogErased < Lyra::Event; end
118
299
  end
119
300
  end
120
301