orfeas_lyra 0.9.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +2021 -17
- data/README.md +22 -9
- data/Rakefile +40 -1
- data/app/controllers/lyra/application_controller.rb +127 -0
- data/app/controllers/lyra/dashboard_controller.rb +33 -31
- data/app/controllers/lyra/flow_controller.rb +32 -34
- data/app/controllers/lyra/privacy_controller.rb +57 -18
- data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
- data/app/views/lyra/dashboard/index.html.erb +1 -0
- data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
- data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
- data/app/views/lyra/privacy/policy.html.erb +7 -0
- data/app/views/lyra/privacy/rights.html.erb +78 -0
- data/config/privacy_policies.rb +0 -3
- data/config/routes.rb +1 -0
- data/lib/lyra/access_log.rb +187 -6
- data/lib/lyra/advisory_lock.rb +236 -0
- data/lib/lyra/aggregate.rb +1 -1
- data/lib/lyra/associations/habtm_links.rb +382 -19
- data/lib/lyra/associations/maintenance.rb +229 -0
- data/lib/lyra/attribution.rb +48 -0
- data/lib/lyra/bypass_events.rb +446 -46
- data/lib/lyra/command_handler.rb +19 -14
- data/lib/lyra/configuration.rb +237 -10
- data/lib/lyra/consistency/read_your_writes.rb +4 -2
- data/lib/lyra/correlation.rb +121 -0
- data/lib/lyra/domain_events.rb +4 -0
- data/lib/lyra/dual_view.rb +53 -16
- data/lib/lyra/encrypted_attributes.rb +5 -0
- data/lib/lyra/engine.rb +67 -0
- data/lib/lyra/erasure.rb +1011 -106
- data/lib/lyra/event.rb +35 -1
- data/lib/lyra/event_data.rb +38 -0
- data/lib/lyra/event_flow.rb +31 -15
- data/lib/lyra/event_store_adapter.rb +28 -0
- data/lib/lyra/event_window.rb +58 -0
- data/lib/lyra/genesis.rb +548 -44
- data/lib/lyra/interceptors/association_interceptor.rb +689 -34
- data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
- data/lib/lyra/internal.rb +26 -0
- data/lib/lyra/mode_sync.rb +51 -3
- data/lib/lyra/mode_transition.rb +257 -30
- data/lib/lyra/model_selection.rb +35 -0
- data/lib/lyra/own_connection.rb +17 -2
- data/lib/lyra/privacy/adapters/pam.rb +25 -2
- data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
- data/lib/lyra/privacy/interface.rb +27 -7
- data/lib/lyra/privacy/subject_links.rb +143 -0
- data/lib/lyra/privacy/subject_records.rb +369 -0
- data/lib/lyra/projection.rb +7 -1
- data/lib/lyra/projections/async_checkpoint.rb +32 -0
- data/lib/lyra/projections/async_projection_job.rb +166 -23
- data/lib/lyra/projections/cached_joins.rb +33 -18
- data/lib/lyra/projections/cached_projection.rb +74 -33
- data/lib/lyra/projections/cached_relation.rb +3175 -321
- data/lib/lyra/projections/default_scopes.rb +254 -0
- data/lib/lyra/projections/deferred_relation.rb +23 -3
- data/lib/lyra/projections/event_store_reader.rb +47 -8
- data/lib/lyra/projections/foreign_keys.rb +92 -0
- data/lib/lyra/projections/lazy_projection.rb +165 -27
- data/lib/lyra/projections/model_projection.rb +165 -14
- data/lib/lyra/projections/rebuild.rb +34 -6
- data/lib/lyra/projections/reference_links.rb +463 -72
- data/lib/lyra/projections/typed_values.rb +629 -0
- data/lib/lyra/projections/unique_indexes.rb +595 -0
- data/lib/lyra/projections/uniqueness.rb +784 -0
- data/lib/lyra/projections/value_digest.rb +93 -0
- data/lib/lyra/purpose_bound_reads.rb +912 -77
- data/lib/lyra/repair.rb +119 -28
- data/lib/lyra/retention.rb +114 -8
- data/lib/lyra/rights/consent_store.rb +415 -0
- data/lib/lyra/rights/erasure_guard.rb +135 -0
- data/lib/lyra/rights/pending.rb +74 -0
- data/lib/lyra/rights/queries.rb +269 -0
- data/lib/lyra/rights/store.rb +512 -0
- data/lib/lyra/rights/subject_keys.rb +224 -0
- data/lib/lyra/rights.rb +281 -0
- data/lib/lyra/saves_without_effect.rb +69 -0
- data/lib/lyra/stream_tail.rb +319 -0
- data/lib/lyra/strict_data_access.rb +916 -106
- data/lib/lyra/temporal.rb +4 -2
- data/lib/lyra/touch_registry.rb +404 -0
- data/lib/lyra/verification/bypass_workflow.rb +452 -42
- data/lib/lyra/verification/rights_workflows.rb +170 -0
- data/lib/lyra/version.rb +1 -1
- data/lib/lyra/visualization/event_graph.rb +1 -1
- data/lib/lyra/visualization/timeline.rb +2 -2
- data/lib/lyra.rb +43 -2
- data/lib/tasks/lyra_mode.rake +110 -8
- data/lib/tasks/lyra_projections.rake +43 -2
- metadata +34 -4
- data/lib/lyra/associations/event_aware.rb +0 -224
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Lyra
|
|
4
|
+
module Verification
|
|
5
|
+
# The data lifecycle of a personal value (the thesis's data-lifecycle
|
|
6
|
+
# CPN, D_0..D_3) with the Restricted place D_R of Art. 18, as a PetriFlow
|
|
7
|
+
# net. A specification checked by exploration, as the CRUD nets are, not
|
|
8
|
+
# runtime machinery: the runtime is PAM DSL's PamDsl::Rights.project, and
|
|
9
|
+
# a test matches its transitions against this net's.
|
|
10
|
+
#
|
|
11
|
+
# Places: Active (D_0), Retained (D_1), Deleted (D_2), Anonymized (D_3),
|
|
12
|
+
# and D_R split by where the data came from (the token keeps its origin)
|
|
13
|
+
# and by whether the Art. 18(3) notice is recorded:
|
|
14
|
+
#
|
|
15
|
+
# restricted / restricted_retained (code state :active)
|
|
16
|
+
# lift_notified / lift_notified_retained (code state :lift_notified)
|
|
17
|
+
#
|
|
18
|
+
# Transitions: d1 deactivate (D_0 -> D_1), d2 delete (D_1 -> D_2), d3
|
|
19
|
+
# anonymize (D_1 -> D_3), d4 erase on request (D_0 -> D_2); r1 restrict
|
|
20
|
+
# (D_0 or D_1 -> D_R), the notice (D_R -> D_R notified), r2 lift (back to
|
|
21
|
+
# the origin). d2, d3 and d4 are not enabled from D_R: Deleted and
|
|
22
|
+
# Anonymized are reached from Restricted only through the notice and the
|
|
23
|
+
# lift (Lyra::Erasure.erase!(..., lifting:) is the lift then d4).
|
|
24
|
+
class DataLifecycleWorkflow < PetriFlow::Workflow
|
|
25
|
+
workflow_name "Data Lifecycle with Restriction (Art. 18)"
|
|
26
|
+
|
|
27
|
+
places :active, :retained, :deleted, :anonymized,
|
|
28
|
+
:restricted, :lift_notified, :restricted_retained, :lift_notified_retained
|
|
29
|
+
|
|
30
|
+
initial_place :active
|
|
31
|
+
terminal_places :deleted, :anonymized
|
|
32
|
+
|
|
33
|
+
transition :deactivate, from: :active, to: :retained, trigger: "d1: active use ends"
|
|
34
|
+
transition :delete, from: :retained, to: :deleted, trigger: "d2: retention expired, :hard_delete"
|
|
35
|
+
transition :anonymize, from: :retained, to: :anonymized, trigger: "d3: retention expired, :anonymize"
|
|
36
|
+
transition :erase, from: :active, to: :deleted, trigger: "d4: Art. 17 request"
|
|
37
|
+
|
|
38
|
+
transition :restrict, from: :active, to: :restricted, trigger: "r1: ProcessingRestricted"
|
|
39
|
+
transition :restrict, from: :retained, to: :restricted_retained, trigger: "r1: ProcessingRestricted"
|
|
40
|
+
transition :notify_lift, from: :restricted, to: :lift_notified, trigger: "RestrictionLiftNotified (Art. 18(3))"
|
|
41
|
+
transition :notify_lift, from: :restricted_retained, to: :lift_notified_retained,
|
|
42
|
+
trigger: "RestrictionLiftNotified (Art. 18(3))"
|
|
43
|
+
transition :lift, from: :lift_notified, to: :active, trigger: "r2: RestrictionLifted"
|
|
44
|
+
transition :lift, from: :lift_notified_retained, to: :retained, trigger: "r2: RestrictionLifted"
|
|
45
|
+
|
|
46
|
+
# The PAM DSL event each restriction transition records.
|
|
47
|
+
EVENTS = { restrict: "ProcessingRestricted", notify_lift: "RestrictionLiftNotified",
|
|
48
|
+
lift: "RestrictionLifted" }.freeze
|
|
49
|
+
|
|
50
|
+
# The state of the restriction (PamDsl::Rights::Restriction#state) a
|
|
51
|
+
# token in a D_R place stands for.
|
|
52
|
+
RESTRICTION_STATES = { restricted: :active, restricted_retained: :active, lift_notified: :lift_notified,
|
|
53
|
+
lift_notified_retained: :lift_notified }.freeze
|
|
54
|
+
|
|
55
|
+
class << self
|
|
56
|
+
# The restriction's steps the net allows, as the code's state
|
|
57
|
+
# machine reads them: { [state before, event type] => state after },
|
|
58
|
+
# :none before a restriction is recorded and :lifted after it ends.
|
|
59
|
+
def restriction_steps
|
|
60
|
+
defined_transitions.filter_map do |t|
|
|
61
|
+
event = EVENTS[t[:name]]
|
|
62
|
+
next unless event
|
|
63
|
+
|
|
64
|
+
[[RESTRICTION_STATES.fetch(t[:from].first, :none), event], RESTRICTION_STATES.fetch(t[:to].first, :lifted)]
|
|
65
|
+
end.to_h
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# The ids of the lift transitions (r2).
|
|
69
|
+
def lift_transitions
|
|
70
|
+
defined_transitions.select { |t| t[:name] == :lift }.map { |t| t[:id] }
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# An objection (Art. 21) to one purpose, as a PetriFlow net in the style
|
|
76
|
+
# of the consent net: received (pending, with its Art. 18(1)(d)
|
|
77
|
+
# restriction of the purpose), then upheld (the purpose stops for the
|
|
78
|
+
# subject), overridden (compelling grounds, with the Art. 18(3) notice),
|
|
79
|
+
# rejected (no right to object applies) or withdrawn by the subject (a
|
|
80
|
+
# pending or an upheld objection). Matched against
|
|
81
|
+
# PamDsl::Rights.project by a test.
|
|
82
|
+
class ObjectionWorkflow < PetriFlow::Workflow
|
|
83
|
+
workflow_name "Objection (Art. 21)"
|
|
84
|
+
|
|
85
|
+
places :none, :pending, :upheld, :overridden, :rejected, :withdrawn
|
|
86
|
+
|
|
87
|
+
initial_place :none
|
|
88
|
+
terminal_places :overridden, :rejected, :withdrawn
|
|
89
|
+
|
|
90
|
+
transition :receive, from: :none, to: :pending, trigger: "ObjectionReceived (with ProcessingRestricted, 18(1)(d))"
|
|
91
|
+
transition :uphold, from: :pending, to: :upheld, trigger: "ObjectionUpheld (21(3): at once for direct marketing)"
|
|
92
|
+
transition :override, from: :pending, to: :overridden, trigger: "ObjectionOverridden (21(1), notice 18(3))"
|
|
93
|
+
transition :reject, from: :pending, to: :rejected, trigger: "ObjectionRejected (no right to object)"
|
|
94
|
+
transition :withdraw, from: :pending, to: :withdrawn, trigger: "ObjectionWithdrawn"
|
|
95
|
+
transition :withdraw, from: :upheld, to: :withdrawn, trigger: "ObjectionWithdrawn"
|
|
96
|
+
|
|
97
|
+
EVENTS = { receive: "ObjectionReceived", uphold: "ObjectionUpheld", override: "ObjectionOverridden",
|
|
98
|
+
reject: "ObjectionRejected", withdraw: "ObjectionWithdrawn" }.freeze
|
|
99
|
+
|
|
100
|
+
# { [state before, event type] => state after }, the places being the
|
|
101
|
+
# code's objection states (:none before one is recorded).
|
|
102
|
+
def self.steps
|
|
103
|
+
defined_transitions.to_h { |t| [[t[:from].first, EVENTS.fetch(t[:name])], t[:to].first] }
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# An objection under Art. 21(1) or 21(6) together with the Art. 18(1)(d)
|
|
108
|
+
# restriction it creates: two tokens, one per request, so the net shows
|
|
109
|
+
# how the decision ends the restriction. Upheld or withdrawn, the
|
|
110
|
+
# restriction is lifted without a notice (processing does not resume, or
|
|
111
|
+
# the subject asked); overridden, only after the Art. 18(3) notice. A
|
|
112
|
+
# notice before the decision is not possible.
|
|
113
|
+
class ObjectionRestrictionWorkflow < PetriFlow::Workflow
|
|
114
|
+
workflow_name "Objection and its Art. 18(1)(d) restriction"
|
|
115
|
+
|
|
116
|
+
places :o_none, :o_pending, :o_upheld, :o_overridden, :o_withdrawn,
|
|
117
|
+
:r_none, :r_active, :r_lift_notified, :r_lifted
|
|
118
|
+
|
|
119
|
+
initial_place :o_none
|
|
120
|
+
terminal_places :r_lifted
|
|
121
|
+
|
|
122
|
+
transition :receive, from: :o_none, to: :o_pending, trigger: "ObjectionReceived"
|
|
123
|
+
transition :restrict, from: %i[o_pending r_none], to: %i[o_pending r_active],
|
|
124
|
+
trigger: "ProcessingRestricted (objection_id)"
|
|
125
|
+
transition :uphold, from: %i[o_pending r_active], to: %i[o_upheld r_active], trigger: "ObjectionUpheld"
|
|
126
|
+
transition :override, from: %i[o_pending r_active], to: %i[o_overridden r_active], trigger: "ObjectionOverridden"
|
|
127
|
+
transition :withdraw, from: %i[o_pending r_active], to: %i[o_withdrawn r_active], trigger: "ObjectionWithdrawn"
|
|
128
|
+
transition :withdraw, from: %i[o_upheld r_active], to: %i[o_withdrawn r_active], trigger: "ObjectionWithdrawn"
|
|
129
|
+
# (a name of its own: PetriFlow names a transition by its first input place)
|
|
130
|
+
transition :withdraw_lifted, from: %i[o_upheld r_lifted], to: %i[o_withdrawn r_lifted], trigger: "ObjectionWithdrawn"
|
|
131
|
+
transition :lift, from: %i[o_upheld r_active], to: %i[o_upheld r_lifted], trigger: "RestrictionLifted (no notice)"
|
|
132
|
+
transition :lift, from: %i[o_withdrawn r_active], to: %i[o_withdrawn r_lifted], trigger: "RestrictionLifted (no notice)"
|
|
133
|
+
transition :notify_lift, from: %i[o_overridden r_active], to: %i[o_overridden r_lift_notified],
|
|
134
|
+
trigger: "RestrictionLiftNotified (Art. 18(3))"
|
|
135
|
+
transition :lift, from: %i[o_overridden r_lift_notified], to: %i[o_overridden r_lifted], trigger: "RestrictionLifted"
|
|
136
|
+
|
|
137
|
+
EVENTS = { receive: "ObjectionReceived", restrict: "ProcessingRestricted", uphold: "ObjectionUpheld",
|
|
138
|
+
override: "ObjectionOverridden", withdraw: "ObjectionWithdrawn", withdraw_lifted: "ObjectionWithdrawn",
|
|
139
|
+
notify_lift: "RestrictionLiftNotified",
|
|
140
|
+
lift: "RestrictionLifted" }.freeze
|
|
141
|
+
|
|
142
|
+
# The initial marking: no objection and no restriction yet.
|
|
143
|
+
def initial_marking
|
|
144
|
+
PetriFlow::Core::Marking.new(self.class.defined_places.to_h { |p| [p, %i[o_none r_none].include?(p) ? 1 : 0] })
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# { [[objection state, restriction state], event type] => [states after] }
|
|
148
|
+
# over the reachable markings, the code's states named as the places
|
|
149
|
+
# without their prefix.
|
|
150
|
+
def steps
|
|
151
|
+
analyzer = PetriFlow::Verification::ReachabilityAnalyzer.new(net, initial_marking)
|
|
152
|
+
analyzer.analyze
|
|
153
|
+
by_key = analyzer.reachable_markings.to_h { |m| [analyzer.send(:marking_to_key, m), states_of(m)] }
|
|
154
|
+
ids = self.class.defined_transitions.to_h { |t| [t[:id], EVENTS.fetch(t[:name])] }
|
|
155
|
+
analyzer.state_graph.flat_map do |key, edges|
|
|
156
|
+
edges.map { |transition, after| [[by_key.fetch(key), ids.fetch(transition)], by_key.fetch(after)] }
|
|
157
|
+
end.to_h
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
private
|
|
161
|
+
|
|
162
|
+
def states_of(marking)
|
|
163
|
+
%w[o r].map do |prefix|
|
|
164
|
+
place = self.class.defined_places.find { |p| p.to_s.start_with?("#{prefix}_") && marking.tokens_at(p).positive? }
|
|
165
|
+
place.to_s.delete_prefix("#{prefix}_").to_sym
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
end
|
|
169
|
+
end
|
|
170
|
+
end
|
data/lib/lyra/version.rb
CHANGED
|
@@ -166,7 +166,7 @@ module Lyra
|
|
|
166
166
|
def detect_pii?(event)
|
|
167
167
|
return false unless Lyra.privacy_features_available?
|
|
168
168
|
|
|
169
|
-
Lyra::Privacy::PIIDetector.detect(
|
|
169
|
+
Lyra::Privacy::PIIDetector.detect(Lyra::Event.fields_of(event)).any?
|
|
170
170
|
rescue
|
|
171
171
|
false
|
|
172
172
|
end
|
|
@@ -188,7 +188,7 @@ module Lyra
|
|
|
188
188
|
def has_pii?(event)
|
|
189
189
|
return false unless Lyra.privacy_features_available?
|
|
190
190
|
|
|
191
|
-
Lyra::Privacy::PIIDetector.detect(
|
|
191
|
+
Lyra::Privacy::PIIDetector.detect(Lyra::Event.fields_of(event)).any?
|
|
192
192
|
end
|
|
193
193
|
|
|
194
194
|
def build_links(events)
|
|
@@ -287,7 +287,7 @@ module Lyra
|
|
|
287
287
|
|
|
288
288
|
def has_pii?(event)
|
|
289
289
|
return false unless Lyra.privacy_features_available?
|
|
290
|
-
Lyra::Privacy::PIIDetector.detect(
|
|
290
|
+
Lyra::Privacy::PIIDetector.detect(Lyra::Event.fields_of(event)).any?
|
|
291
291
|
end
|
|
292
292
|
|
|
293
293
|
def h(text)
|
data/lib/lyra.rb
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
require "lyra/version"
|
|
2
|
+
require "lyra/internal"
|
|
3
|
+
require "lyra/attribution"
|
|
2
4
|
require "rails_event_store"
|
|
3
5
|
# Lyra builds on these at load time (jobs, ActiveRecord errors); require them
|
|
4
6
|
# rather than rely on the host application having loaded rails/all first.
|
|
@@ -34,20 +36,25 @@ require "lyra/model_selection"
|
|
|
34
36
|
require "lyra/configuration"
|
|
35
37
|
require "lyra/correlation"
|
|
36
38
|
require "lyra/event"
|
|
39
|
+
require "lyra/event_data"
|
|
40
|
+
require "lyra/saves_without_effect"
|
|
37
41
|
require "lyra/encrypted_attributes"
|
|
38
42
|
require "lyra/optimistic_locking"
|
|
39
43
|
require "lyra/event_store_adapter"
|
|
40
44
|
require "lyra/event_serializer"
|
|
45
|
+
require "lyra/stream_tail"
|
|
41
46
|
require "lyra/projection"
|
|
42
47
|
require "lyra/temporal"
|
|
43
48
|
require "lyra/domain_events"
|
|
44
49
|
require "lyra/mode_transition"
|
|
45
50
|
require "lyra/mode_sync"
|
|
51
|
+
require "lyra/touch_registry"
|
|
46
52
|
require "lyra/dual_view_sampler"
|
|
47
53
|
require "lyra/command"
|
|
48
54
|
require "lyra/aggregate"
|
|
49
55
|
require "lyra/command_handler"
|
|
50
56
|
require "lyra/dual_view"
|
|
57
|
+
require "lyra/event_window"
|
|
51
58
|
require "lyra/event_flow"
|
|
52
59
|
require "lyra/event_analyzer"
|
|
53
60
|
require "lyra/id_generator"
|
|
@@ -65,13 +72,19 @@ require "lyra/projections/rebuild"
|
|
|
65
72
|
require "lyra/sti_stream_migration"
|
|
66
73
|
require "lyra/projections/async_checkpoint"
|
|
67
74
|
require "lyra/projections/async_projection_job"
|
|
75
|
+
require "lyra/projections/typed_values"
|
|
76
|
+
require "lyra/projections/value_digest"
|
|
68
77
|
require "lyra/projections/cached_projection"
|
|
69
78
|
require "lyra/projections/reference_links"
|
|
70
79
|
require "lyra/projections/cached_joins"
|
|
71
80
|
require "lyra/projections/cached_relation"
|
|
72
81
|
require "lyra/projections/deferred_relation"
|
|
82
|
+
require "lyra/projections/default_scopes"
|
|
83
|
+
require "lyra/projections/foreign_keys"
|
|
73
84
|
require "lyra/projections/association_conditions"
|
|
74
85
|
require "lyra/projections/event_store_reader"
|
|
86
|
+
require "lyra/projections/uniqueness"
|
|
87
|
+
require "lyra/projections/unique_indexes"
|
|
75
88
|
require "lyra/projections/lazy_projection"
|
|
76
89
|
require "lyra/genesis"
|
|
77
90
|
require "lyra/repair"
|
|
@@ -79,9 +92,9 @@ require "lyra/purpose_bound_reads"
|
|
|
79
92
|
require "lyra/erasure"
|
|
80
93
|
require "lyra/retention"
|
|
81
94
|
|
|
82
|
-
#
|
|
83
|
-
require "lyra/associations/event_aware"
|
|
95
|
+
# has_and_belongs_to_many links as events
|
|
84
96
|
require "lyra/associations/habtm_links"
|
|
97
|
+
require "lyra/associations/maintenance"
|
|
85
98
|
|
|
86
99
|
# Consistency helpers
|
|
87
100
|
require "lyra/consistency/read_your_writes"
|
|
@@ -96,7 +109,10 @@ if PAM_DSL_AVAILABLE
|
|
|
96
109
|
require "lyra/privacy/adapters/pam"
|
|
97
110
|
require "lyra/privacy/pii_masker"
|
|
98
111
|
require "lyra/privacy/gdpr_compliance"
|
|
112
|
+
require "lyra/privacy/subject_links"
|
|
113
|
+
require "lyra/privacy/subject_records"
|
|
99
114
|
require "lyra/access_log"
|
|
115
|
+
require "lyra/rights"
|
|
100
116
|
end
|
|
101
117
|
|
|
102
118
|
# Schema validation
|
|
@@ -117,6 +133,7 @@ if PETRI_FLOW_AVAILABLE
|
|
|
117
133
|
require "lyra/verification/crud_lifecycle_workflow"
|
|
118
134
|
require "lyra/verification/workflow_generator"
|
|
119
135
|
require "lyra/verification/bypass_workflow"
|
|
136
|
+
require "lyra/verification/rights_workflows"
|
|
120
137
|
end
|
|
121
138
|
|
|
122
139
|
module Lyra
|
|
@@ -166,6 +183,29 @@ module Lyra
|
|
|
166
183
|
model_class.name
|
|
167
184
|
end
|
|
168
185
|
|
|
186
|
+
# The column that holds a record's key: the model's single primary key,
|
|
187
|
+
# whatever it is called ("id" unless the model names another). Composite
|
|
188
|
+
# primary keys are refused at enrolment (Lyra::ModelSelection).
|
|
189
|
+
def self.key_column(model_class)
|
|
190
|
+
key = model_class.primary_key if model_class.respond_to?(:primary_key)
|
|
191
|
+
key.is_a?(String) && !key.empty? ? key : "id"
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
# A record's key as its row holds it: +id+ (from a stream name, a string)
|
|
195
|
+
# as an integer when the key column is an integer one.
|
|
196
|
+
def self.key_value(model_class, id)
|
|
197
|
+
return id unless id.to_s.match?(/\A\d+\z/)
|
|
198
|
+
|
|
199
|
+
key = key_column(model_class)
|
|
200
|
+
type = begin
|
|
201
|
+
model_class.columns_hash[key]&.type if model_class.respond_to?(:columns_hash)
|
|
202
|
+
rescue StandardError
|
|
203
|
+
nil # no table to read the column from: an "id" is taken to be an integer, as before 1.0.0
|
|
204
|
+
end
|
|
205
|
+
integer = type ? %i[integer bigint].include?(type) : key == "id"
|
|
206
|
+
integer ? id.to_i : id
|
|
207
|
+
end
|
|
208
|
+
|
|
169
209
|
# Delegate event_store to configuration
|
|
170
210
|
def self.event_store
|
|
171
211
|
config.event_store
|
|
@@ -228,6 +268,7 @@ module Lyra
|
|
|
228
268
|
next
|
|
229
269
|
end
|
|
230
270
|
failures << "#{model.name}: no primary key" if model.primary_key.blank?
|
|
271
|
+
failures << "#{model.name}: composite primary key (not supported)" if model.primary_key.is_a?(Array)
|
|
231
272
|
end
|
|
232
273
|
|
|
233
274
|
raise MappingVerificationError, "Lyra mapping verification failed: #{failures.join('; ')}" if failures.any?
|
data/lib/tasks/lyra_mode.rake
CHANGED
|
@@ -29,6 +29,8 @@ namespace :lyra do
|
|
|
29
29
|
report.discrepancies.first(20).each { puts " #{_1}" }
|
|
30
30
|
puts(report.clean? ? "Clean: certified for #{Lyra.config.mode_transition_certificate_ttl}s." : "Not certified.")
|
|
31
31
|
exit 1 unless report.clean?
|
|
32
|
+
rescue Lyra::ModeTransition::Refused => e
|
|
33
|
+
abort e.message
|
|
32
34
|
end
|
|
33
35
|
|
|
34
36
|
desc "Show the mode the application runs in and the last one applied"
|
|
@@ -36,44 +38,140 @@ namespace :lyra do
|
|
|
36
38
|
puts "configured: #{Lyra::ModeTransition.current}"
|
|
37
39
|
puts "last applied: #{Lyra::ModeTransition.last_applied || '(none recorded)'}"
|
|
38
40
|
puts "gate: #{Lyra::ModeTransition.gate_enabled? ? 'on' : 'off'}"
|
|
41
|
+
live = Lyra::TouchRegistry.live_processes
|
|
42
|
+
puts "touches left out (config.record_touches = false): #{live.empty? ? 'by no live process' : "#{live.size} live process(es)"}"
|
|
43
|
+
live.each do |p|
|
|
44
|
+
puts " #{p[:host]} pid #{p[:pid]} (Lyra #{p[:version]}), since #{p[:since]&.utc&.iso8601}, " \
|
|
45
|
+
"last seen #{p[:last_seen]&.utc&.iso8601}"
|
|
46
|
+
end
|
|
47
|
+
puts " #{Lyra::TouchRegistry::STEPS}" if live.any?
|
|
39
48
|
end
|
|
40
49
|
end
|
|
41
50
|
|
|
42
51
|
desc "Bring the event log back in line with the tables after Monitor lost events " \
|
|
43
|
-
"(DRY_RUN=1 lists them; MODELS=User,Order limits it
|
|
52
|
+
"(DRY_RUN=1 lists them; MODELS=User,Order limits it; UNREPRESENTABLE=skip repairs the rest when rows hold " \
|
|
53
|
+
"values their model cannot represent, default config.genesis_unrepresentable, :refuse)"
|
|
44
54
|
task repair: :environment do
|
|
45
55
|
models = ENV["MODELS"] ? ENV["MODELS"].split(",").map { _1.strip.constantize } : lyra_monitored_models.call
|
|
46
|
-
result = Lyra::Repair.run(models: models, dry_run: ENV["DRY_RUN"].present
|
|
56
|
+
result = Lyra::Repair.run(models: models, dry_run: ENV["DRY_RUN"].present?, operator: ENV["OPERATOR"],
|
|
57
|
+
unrepresentable: ENV["UNREPRESENTABLE"].presence || Lyra.config.genesis_unrepresentable)
|
|
47
58
|
puts result.summary
|
|
48
59
|
(ENV["DRY_RUN"].present? ? result.found : result.remaining).first(20).each { puts " #{_1}" }
|
|
60
|
+
if result.unrepresentable.any?
|
|
61
|
+
puts "Values their model cannot represent (#{ENV['DRY_RUN'].present? ? 'would not be repaired' : 'not repaired'}):"
|
|
62
|
+
result.unrepresentable.each { puts " #{_1}" }
|
|
63
|
+
end
|
|
49
64
|
exit 1 if result.remaining.any? && ENV["DRY_RUN"].blank?
|
|
50
65
|
rescue Lyra::Repair::Refused => e
|
|
51
66
|
abort e.message
|
|
52
67
|
end
|
|
53
68
|
desc "Erase one record's personal data from its row and its events (Art. 17) " \
|
|
54
|
-
"(MODEL=Registration ID=5 REASON='Art. 17 request #12' [FIELDS=email,phone]
|
|
55
|
-
"[
|
|
69
|
+
"(MODEL=Registration ID=5 REASON='Art. 17 request #12' [OPERATOR=jdoe] [FIELDS=email,phone] " \
|
|
70
|
+
"[ANY_FIELDS=1: FIELDS may name columns the policy does not declare personal] [EVERYWHERE=1] " \
|
|
71
|
+
"[MAX_COPIES=n: with EVERYWHERE, a value held by more other records is left as shared; default config.erasure_max_copies, 10] " \
|
|
72
|
+
"[DRY_RUN=1: report what would be erased and the values left as shared, write nothing] " \
|
|
73
|
+
"[ACCESS_LOG=1|0: delete or keep the record's access log; default config.erase_access_log] " \
|
|
74
|
+
"[ACCESS_LOG_SUBJECTS=42,user-42: ids the application used as this person's subject, whose logs go too])"
|
|
56
75
|
task erase: :environment do
|
|
57
76
|
model = ENV.fetch("MODEL").constantize
|
|
58
|
-
max_copies = Lyra
|
|
77
|
+
max_copies = Lyra.config.erasure_max_copies
|
|
59
78
|
if ENV["MAX_COPIES"].present?
|
|
60
79
|
max_copies = Integer(ENV["MAX_COPIES"], 10, exception: false)
|
|
61
80
|
abort "MAX_COPIES must be a positive integer (got #{ENV['MAX_COPIES'].inspect})" unless max_copies&.positive?
|
|
62
81
|
end
|
|
82
|
+
access_log = case ENV["ACCESS_LOG"].to_s.strip.downcase
|
|
83
|
+
when "" then nil
|
|
84
|
+
when "1", "true", "yes", "erase" then true
|
|
85
|
+
when "0", "false", "no", "keep" then false
|
|
86
|
+
else abort "ACCESS_LOG must be 1 (erase) or 0 (keep) (got #{ENV['ACCESS_LOG'].inspect})"
|
|
87
|
+
end
|
|
88
|
+
dry_run = ENV["DRY_RUN"].present?
|
|
63
89
|
result = Lyra::Erasure.erase!(model, ENV.fetch("ID"), reason: ENV.fetch("REASON"),
|
|
64
90
|
fields: ENV["FIELDS"]&.split(",")&.map(&:strip),
|
|
65
|
-
everywhere: ENV["EVERYWHERE"].present?, max_copies: max_copies
|
|
66
|
-
|
|
91
|
+
everywhere: ENV["EVERYWHERE"].present?, max_copies: max_copies,
|
|
92
|
+
any_fields: ENV["ANY_FIELDS"].present?, access_log: access_log,
|
|
93
|
+
access_log_subjects: ENV["ACCESS_LOG_SUBJECTS"].to_s.split(",").map(&:strip).reject(&:empty?),
|
|
94
|
+
erased_by: Lyra::ModeTransition.actor(ENV["OPERATOR"]), dry_run: dry_run,
|
|
95
|
+
lifting: ENV["LIFTING"].to_s.split(",").map(&:strip).reject(&:empty?).presence)
|
|
96
|
+
puts "Dry run: nothing was written. This is what the erasure would do." if dry_run
|
|
97
|
+
# A key that holds an erased value is never printed.
|
|
98
|
+
shown_id = result.key_withheld ? Lyra::Erasure::WITHHELD : result.id
|
|
99
|
+
puts "Erased #{result.fields.join(', ')} of #{result.model} #{shown_id}: " \
|
|
67
100
|
"#{result.events_rewritten} events rewritten, row #{result.row_erased ? 'erased' : 'absent'}."
|
|
101
|
+
if result.access_log
|
|
102
|
+
puts "Access log #{result.access_log}: #{result.access_log_entries} entries."
|
|
103
|
+
end
|
|
68
104
|
puts "Copies erased: #{result.copies.join(', ')}" if result.copies.any?
|
|
105
|
+
lifted = Array(result.lifting)
|
|
106
|
+
puts "Restrictions #{dry_run ? 'it would lift' : 'lifted'} (Art. 18(3) notice recorded): #{lifted.join(', ')}" if lifted.any?
|
|
107
|
+
kept = Array(result.restricted_copies)
|
|
108
|
+
if kept.any?
|
|
109
|
+
puts "Copies kept under a restriction of processing (Art. 18), not modified; each stays until its holder's " \
|
|
110
|
+
"restriction is lifted, then erase again with EVERYWHERE=1: " +
|
|
111
|
+
kept.map { |entry| "#{entry[:record]} (#{entry[:columns].join(', ')})" }.join("; ")
|
|
112
|
+
end
|
|
113
|
+
left = Array(result.left_behind)
|
|
114
|
+
if left.any?
|
|
115
|
+
puts "Left behind (the value is still in these columns, as a JSON key, inside a longer string or in a key): " +
|
|
116
|
+
left.map { |entry| "#{entry[:record]} (#{entry[:columns].join(', ')})" }.join("; ")
|
|
117
|
+
end
|
|
118
|
+
not_searched = Array(result.not_searched)
|
|
119
|
+
if not_searched.any?
|
|
120
|
+
puts "Not searched (encrypted, the log holds their ciphertext): #{not_searched.join(', ')}"
|
|
121
|
+
end
|
|
69
122
|
if result.shared_values.positive?
|
|
70
123
|
puts "Values left as shared (held by more than #{max_copies} other records): #{result.shared_values}"
|
|
124
|
+
# By field and count, never the value. Once the person's own values are
|
|
125
|
+
# erased, a later run cannot find these copies: choose MAX_COPIES with a
|
|
126
|
+
# dry run first.
|
|
127
|
+
Array(result.shared).each do |entry|
|
|
128
|
+
hint = if dry_run
|
|
129
|
+
"check them, and if they are this person's, erase with MAX_COPIES=#{entry[:holders]}"
|
|
130
|
+
else
|
|
131
|
+
"left in place; this record's values are now erased, so a later run cannot find them: " \
|
|
132
|
+
"next time run with DRY_RUN=1 first and choose MAX_COPIES"
|
|
133
|
+
end
|
|
134
|
+
puts " #{entry[:field]} (#{entry[:type]}): held by #{entry[:holders]} other records; #{hint}"
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
rows_only = Array(result.rows_without_events)
|
|
138
|
+
if rows_only.any?
|
|
139
|
+
remedy = if Lyra.config.monitor_mode? || Lyra.config.disabled_mode?
|
|
140
|
+
"their rows differed from their events before the erasure: run bin/rails lyra:repair " \
|
|
141
|
+
"to record the rows' state in the log"
|
|
142
|
+
else
|
|
143
|
+
"the events are authoritative in #{Lyra::ModeTransition.current}, and these rows differ from " \
|
|
144
|
+
"them: rebuild their tables from the events (bin/rails lyra:projections:rebuild MODEL=...)"
|
|
145
|
+
end
|
|
146
|
+
puts "Rows without events (erased in the row only): #{rows_only.join(', ')}; #{remedy}"
|
|
71
147
|
end
|
|
72
148
|
rescue KeyError => e
|
|
73
149
|
abort "#{e.message}: MODEL, ID and REASON are required"
|
|
74
|
-
rescue Lyra::Erasure::Unsupported => e
|
|
150
|
+
rescue Lyra::Erasure::Unsupported, Lyra::AccessLog::ModelWideSubject => e
|
|
151
|
+
abort e.message
|
|
152
|
+
end
|
|
153
|
+
desc "Delete one subject's access log (SUBJECT=Registration$5 or an id the application passes as the " \
|
|
154
|
+
"subject, e.g. SUBJECT=42; REASON='Art. 17 request #12' [OPERATOR=jdoe]). Model-wide entries " \
|
|
155
|
+
"(Registration$*) are refused: they name no subject"
|
|
156
|
+
task erase_access_log: :environment do
|
|
157
|
+
subject = ENV.fetch("SUBJECT")
|
|
158
|
+
entries = Lyra::AccessLog.erase!(subject, reason: ENV.fetch("REASON"),
|
|
159
|
+
erased_by: Lyra::ModeTransition.actor(ENV["OPERATOR"]))
|
|
160
|
+
puts "Access log of #{subject} erased: #{entries} entries (recorded in #{Lyra::AccessLog::ERASURES})."
|
|
161
|
+
rescue KeyError => e
|
|
162
|
+
abort "#{e.message}: SUBJECT and REASON are required"
|
|
163
|
+
rescue ArgumentError => e
|
|
75
164
|
abort e.message
|
|
76
165
|
end
|
|
166
|
+
namespace :rights do
|
|
167
|
+
desc "Rebuild lyra_rights and lyra_consents (the subjects' restrictions, objections and consent) from " \
|
|
168
|
+
"their event streams, Lyra::Rights and Lyra::Consent"
|
|
169
|
+
task rebuild: :environment do
|
|
170
|
+
requests, consents = Lyra::Rights.rebuild!
|
|
171
|
+
puts "Rebuilt #{requests} restrictions and objections, and #{consents} consent records."
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
|
|
77
175
|
namespace :retention do
|
|
78
176
|
desc "Apply the privacy policy's retention rules (config.retention_executor; DRY_RUN=1 lists what " \
|
|
79
177
|
"would happen, executor on or off; MODELS=Registration,Order limits it)"
|
|
@@ -82,6 +180,10 @@ namespace :lyra do
|
|
|
82
180
|
result = Lyra::Retention.apply!(models: models, dry_run: ENV["DRY_RUN"].present?)
|
|
83
181
|
puts result.summary
|
|
84
182
|
result.actions.first(50).each { puts " #{_1}" }
|
|
183
|
+
if result.restricted.any?
|
|
184
|
+
puts "Kept under a restriction of processing (Art. 18), acted on once it is lifted:"
|
|
185
|
+
result.restricted.each { puts " #{_1}" }
|
|
186
|
+
end
|
|
85
187
|
rescue Lyra::Retention::Disabled => e
|
|
86
188
|
abort e.message
|
|
87
189
|
end
|
|
@@ -32,11 +32,35 @@ namespace :lyra do
|
|
|
32
32
|
)
|
|
33
33
|
end
|
|
34
34
|
puts "Done."
|
|
35
|
+
rescue Lyra::Projections::Rebuild::Refused => e
|
|
36
|
+
abort e.message
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
desc "Link every event of the monitored models (MODEL=Namespace::Model[,Other] for specific models) " \
|
|
41
|
+
"into the value streams ES-NoProj checks unique indexes against, from the first event: after a " \
|
|
42
|
+
"rolling deploy in which processes running an earlier Lyra, or reading the indexes before a migration " \
|
|
43
|
+
"added one, wrote (Lyra::Projections::ReferenceLinks.relink_values!)"
|
|
44
|
+
task link_values: :environment do
|
|
45
|
+
models =
|
|
46
|
+
if ENV["MODEL"]
|
|
47
|
+
ENV["MODEL"].split(",").map { |name| name.strip.constantize }
|
|
48
|
+
else
|
|
49
|
+
Rails.application.eager_load! unless Rails.application.config.eager_load
|
|
50
|
+
Lyra.config.monitored_models
|
|
51
|
+
end
|
|
52
|
+
models.map { _1.respond_to?(:base_class) ? _1.base_class : _1 }.uniq.each do |model|
|
|
53
|
+
next if Lyra::Projections::UniqueIndexes.columns(model).empty?
|
|
54
|
+
|
|
55
|
+
linked = Lyra::Projections::ReferenceLinks.relink_values!(model)
|
|
56
|
+
puts "#{model.name}: #{linked} value links (#{Lyra::Projections::UniqueIndexes.columns(model).join(', ')})"
|
|
35
57
|
end
|
|
36
58
|
end
|
|
37
59
|
|
|
38
60
|
desc "Give rows that predate Lyra their Imported events, ahead of first use " \
|
|
39
|
-
"(MODEL=Namespace::Model[,Other] for specific models, or all monitored models
|
|
61
|
+
"(MODEL=Namespace::Model[,Other] for specific models, or all monitored models; " \
|
|
62
|
+
"UNREPRESENTABLE=skip imports the rest of a model holding rows whose values it cannot represent, " \
|
|
63
|
+
"default config.genesis_unrepresentable, :refuse)"
|
|
40
64
|
task genesis: :environment do
|
|
41
65
|
models =
|
|
42
66
|
if ENV["MODEL"]
|
|
@@ -53,8 +77,25 @@ namespace :lyra do
|
|
|
53
77
|
exit 1
|
|
54
78
|
end
|
|
55
79
|
|
|
80
|
+
# A model holding rows it cannot represent is refused (nothing of it
|
|
81
|
+
# imported) and listed, the other models imported, and the task exits 1.
|
|
82
|
+
# With UNREPRESENTABLE=skip those rows are listed and left out.
|
|
83
|
+
unrepresentable = ENV["UNREPRESENTABLE"].presence || Lyra.config.genesis_unrepresentable
|
|
84
|
+
refused = false
|
|
56
85
|
models.each do |model_class|
|
|
57
|
-
|
|
86
|
+
result = Lyra::Genesis.import(model_class, unrepresentable: unrepresentable)
|
|
87
|
+
skipped = result.skipped.positive? ? ", #{result.skipped} skipped (values the model cannot represent)" : ""
|
|
88
|
+
puts format(" %-30s %6d rows imported%s", model_class.name, result.imported, skipped)
|
|
89
|
+
result.unrepresentable.each { puts " #{_1}" }
|
|
90
|
+
rescue Lyra::Genesis::Refused => e
|
|
91
|
+
puts format(" %-30s refused: %d rows hold values the model cannot represent; nothing imported", model_class.name, e.rows)
|
|
92
|
+
e.values.each { puts " #{_1}" }
|
|
93
|
+
puts " (and #{e.found - e.values.size} more)" if e.found > e.values.size
|
|
94
|
+
refused = true
|
|
95
|
+
end
|
|
96
|
+
if refused
|
|
97
|
+
puts "Not every model was imported: fix the rows listed or their models, or skip those rows with UNREPRESENTABLE=skip."
|
|
98
|
+
exit 1
|
|
58
99
|
end
|
|
59
100
|
puts "Done."
|
|
60
101
|
end
|