orfeas_lyra 0.9.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +2021 -17
  3. data/README.md +22 -9
  4. data/Rakefile +40 -1
  5. data/app/controllers/lyra/application_controller.rb +127 -0
  6. data/app/controllers/lyra/dashboard_controller.rb +33 -31
  7. data/app/controllers/lyra/flow_controller.rb +32 -34
  8. data/app/controllers/lyra/privacy_controller.rb +57 -18
  9. data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
  10. data/app/views/lyra/dashboard/index.html.erb +1 -0
  11. data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
  12. data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
  13. data/app/views/lyra/privacy/policy.html.erb +7 -0
  14. data/app/views/lyra/privacy/rights.html.erb +78 -0
  15. data/config/privacy_policies.rb +0 -3
  16. data/config/routes.rb +1 -0
  17. data/lib/lyra/access_log.rb +187 -6
  18. data/lib/lyra/advisory_lock.rb +236 -0
  19. data/lib/lyra/aggregate.rb +1 -1
  20. data/lib/lyra/associations/habtm_links.rb +382 -19
  21. data/lib/lyra/associations/maintenance.rb +229 -0
  22. data/lib/lyra/attribution.rb +48 -0
  23. data/lib/lyra/bypass_events.rb +446 -46
  24. data/lib/lyra/command_handler.rb +19 -14
  25. data/lib/lyra/configuration.rb +237 -10
  26. data/lib/lyra/consistency/read_your_writes.rb +4 -2
  27. data/lib/lyra/correlation.rb +121 -0
  28. data/lib/lyra/domain_events.rb +4 -0
  29. data/lib/lyra/dual_view.rb +53 -16
  30. data/lib/lyra/encrypted_attributes.rb +5 -0
  31. data/lib/lyra/engine.rb +67 -0
  32. data/lib/lyra/erasure.rb +1011 -106
  33. data/lib/lyra/event.rb +35 -1
  34. data/lib/lyra/event_data.rb +38 -0
  35. data/lib/lyra/event_flow.rb +31 -15
  36. data/lib/lyra/event_store_adapter.rb +28 -0
  37. data/lib/lyra/event_window.rb +58 -0
  38. data/lib/lyra/genesis.rb +548 -44
  39. data/lib/lyra/interceptors/association_interceptor.rb +689 -34
  40. data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
  41. data/lib/lyra/internal.rb +26 -0
  42. data/lib/lyra/mode_sync.rb +51 -3
  43. data/lib/lyra/mode_transition.rb +257 -30
  44. data/lib/lyra/model_selection.rb +35 -0
  45. data/lib/lyra/own_connection.rb +17 -2
  46. data/lib/lyra/privacy/adapters/pam.rb +25 -2
  47. data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
  48. data/lib/lyra/privacy/interface.rb +27 -7
  49. data/lib/lyra/privacy/subject_links.rb +143 -0
  50. data/lib/lyra/privacy/subject_records.rb +369 -0
  51. data/lib/lyra/projection.rb +7 -1
  52. data/lib/lyra/projections/async_checkpoint.rb +32 -0
  53. data/lib/lyra/projections/async_projection_job.rb +166 -23
  54. data/lib/lyra/projections/cached_joins.rb +33 -18
  55. data/lib/lyra/projections/cached_projection.rb +74 -33
  56. data/lib/lyra/projections/cached_relation.rb +3175 -321
  57. data/lib/lyra/projections/default_scopes.rb +254 -0
  58. data/lib/lyra/projections/deferred_relation.rb +23 -3
  59. data/lib/lyra/projections/event_store_reader.rb +47 -8
  60. data/lib/lyra/projections/foreign_keys.rb +92 -0
  61. data/lib/lyra/projections/lazy_projection.rb +165 -27
  62. data/lib/lyra/projections/model_projection.rb +165 -14
  63. data/lib/lyra/projections/rebuild.rb +34 -6
  64. data/lib/lyra/projections/reference_links.rb +463 -72
  65. data/lib/lyra/projections/typed_values.rb +629 -0
  66. data/lib/lyra/projections/unique_indexes.rb +595 -0
  67. data/lib/lyra/projections/uniqueness.rb +784 -0
  68. data/lib/lyra/projections/value_digest.rb +93 -0
  69. data/lib/lyra/purpose_bound_reads.rb +912 -77
  70. data/lib/lyra/repair.rb +119 -28
  71. data/lib/lyra/retention.rb +114 -8
  72. data/lib/lyra/rights/consent_store.rb +415 -0
  73. data/lib/lyra/rights/erasure_guard.rb +135 -0
  74. data/lib/lyra/rights/pending.rb +74 -0
  75. data/lib/lyra/rights/queries.rb +269 -0
  76. data/lib/lyra/rights/store.rb +512 -0
  77. data/lib/lyra/rights/subject_keys.rb +224 -0
  78. data/lib/lyra/rights.rb +281 -0
  79. data/lib/lyra/saves_without_effect.rb +69 -0
  80. data/lib/lyra/stream_tail.rb +319 -0
  81. data/lib/lyra/strict_data_access.rb +916 -106
  82. data/lib/lyra/temporal.rb +4 -2
  83. data/lib/lyra/touch_registry.rb +404 -0
  84. data/lib/lyra/verification/bypass_workflow.rb +452 -42
  85. data/lib/lyra/verification/rights_workflows.rb +170 -0
  86. data/lib/lyra/version.rb +1 -1
  87. data/lib/lyra/visualization/event_graph.rb +1 -1
  88. data/lib/lyra/visualization/timeline.rb +2 -2
  89. data/lib/lyra.rb +43 -2
  90. data/lib/tasks/lyra_mode.rake +110 -8
  91. data/lib/tasks/lyra_projections.rake +43 -2
  92. metadata +34 -4
  93. data/lib/lyra/associations/event_aware.rb +0 -224
@@ -0,0 +1,170 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Lyra
4
+ module Verification
5
+ # The data lifecycle of a personal value (the thesis's data-lifecycle
6
+ # CPN, D_0..D_3) with the Restricted place D_R of Art. 18, as a PetriFlow
7
+ # net. A specification checked by exploration, as the CRUD nets are, not
8
+ # runtime machinery: the runtime is PAM DSL's PamDsl::Rights.project, and
9
+ # a test matches its transitions against this net's.
10
+ #
11
+ # Places: Active (D_0), Retained (D_1), Deleted (D_2), Anonymized (D_3),
12
+ # and D_R split by where the data came from (the token keeps its origin)
13
+ # and by whether the Art. 18(3) notice is recorded:
14
+ #
15
+ # restricted / restricted_retained (code state :active)
16
+ # lift_notified / lift_notified_retained (code state :lift_notified)
17
+ #
18
+ # Transitions: d1 deactivate (D_0 -> D_1), d2 delete (D_1 -> D_2), d3
19
+ # anonymize (D_1 -> D_3), d4 erase on request (D_0 -> D_2); r1 restrict
20
+ # (D_0 or D_1 -> D_R), the notice (D_R -> D_R notified), r2 lift (back to
21
+ # the origin). d2, d3 and d4 are not enabled from D_R: Deleted and
22
+ # Anonymized are reached from Restricted only through the notice and the
23
+ # lift (Lyra::Erasure.erase!(..., lifting:) is the lift then d4).
24
+ class DataLifecycleWorkflow < PetriFlow::Workflow
25
+ workflow_name "Data Lifecycle with Restriction (Art. 18)"
26
+
27
+ places :active, :retained, :deleted, :anonymized,
28
+ :restricted, :lift_notified, :restricted_retained, :lift_notified_retained
29
+
30
+ initial_place :active
31
+ terminal_places :deleted, :anonymized
32
+
33
+ transition :deactivate, from: :active, to: :retained, trigger: "d1: active use ends"
34
+ transition :delete, from: :retained, to: :deleted, trigger: "d2: retention expired, :hard_delete"
35
+ transition :anonymize, from: :retained, to: :anonymized, trigger: "d3: retention expired, :anonymize"
36
+ transition :erase, from: :active, to: :deleted, trigger: "d4: Art. 17 request"
37
+
38
+ transition :restrict, from: :active, to: :restricted, trigger: "r1: ProcessingRestricted"
39
+ transition :restrict, from: :retained, to: :restricted_retained, trigger: "r1: ProcessingRestricted"
40
+ transition :notify_lift, from: :restricted, to: :lift_notified, trigger: "RestrictionLiftNotified (Art. 18(3))"
41
+ transition :notify_lift, from: :restricted_retained, to: :lift_notified_retained,
42
+ trigger: "RestrictionLiftNotified (Art. 18(3))"
43
+ transition :lift, from: :lift_notified, to: :active, trigger: "r2: RestrictionLifted"
44
+ transition :lift, from: :lift_notified_retained, to: :retained, trigger: "r2: RestrictionLifted"
45
+
46
+ # The PAM DSL event each restriction transition records.
47
+ EVENTS = { restrict: "ProcessingRestricted", notify_lift: "RestrictionLiftNotified",
48
+ lift: "RestrictionLifted" }.freeze
49
+
50
+ # The state of the restriction (PamDsl::Rights::Restriction#state) a
51
+ # token in a D_R place stands for.
52
+ RESTRICTION_STATES = { restricted: :active, restricted_retained: :active, lift_notified: :lift_notified,
53
+ lift_notified_retained: :lift_notified }.freeze
54
+
55
+ class << self
56
+ # The restriction's steps the net allows, as the code's state
57
+ # machine reads them: { [state before, event type] => state after },
58
+ # :none before a restriction is recorded and :lifted after it ends.
59
+ def restriction_steps
60
+ defined_transitions.filter_map do |t|
61
+ event = EVENTS[t[:name]]
62
+ next unless event
63
+
64
+ [[RESTRICTION_STATES.fetch(t[:from].first, :none), event], RESTRICTION_STATES.fetch(t[:to].first, :lifted)]
65
+ end.to_h
66
+ end
67
+
68
+ # The ids of the lift transitions (r2).
69
+ def lift_transitions
70
+ defined_transitions.select { |t| t[:name] == :lift }.map { |t| t[:id] }
71
+ end
72
+ end
73
+ end
74
+
75
+ # An objection (Art. 21) to one purpose, as a PetriFlow net in the style
76
+ # of the consent net: received (pending, with its Art. 18(1)(d)
77
+ # restriction of the purpose), then upheld (the purpose stops for the
78
+ # subject), overridden (compelling grounds, with the Art. 18(3) notice),
79
+ # rejected (no right to object applies) or withdrawn by the subject (a
80
+ # pending or an upheld objection). Matched against
81
+ # PamDsl::Rights.project by a test.
82
+ class ObjectionWorkflow < PetriFlow::Workflow
83
+ workflow_name "Objection (Art. 21)"
84
+
85
+ places :none, :pending, :upheld, :overridden, :rejected, :withdrawn
86
+
87
+ initial_place :none
88
+ terminal_places :overridden, :rejected, :withdrawn
89
+
90
+ transition :receive, from: :none, to: :pending, trigger: "ObjectionReceived (with ProcessingRestricted, 18(1)(d))"
91
+ transition :uphold, from: :pending, to: :upheld, trigger: "ObjectionUpheld (21(3): at once for direct marketing)"
92
+ transition :override, from: :pending, to: :overridden, trigger: "ObjectionOverridden (21(1), notice 18(3))"
93
+ transition :reject, from: :pending, to: :rejected, trigger: "ObjectionRejected (no right to object)"
94
+ transition :withdraw, from: :pending, to: :withdrawn, trigger: "ObjectionWithdrawn"
95
+ transition :withdraw, from: :upheld, to: :withdrawn, trigger: "ObjectionWithdrawn"
96
+
97
+ EVENTS = { receive: "ObjectionReceived", uphold: "ObjectionUpheld", override: "ObjectionOverridden",
98
+ reject: "ObjectionRejected", withdraw: "ObjectionWithdrawn" }.freeze
99
+
100
+ # { [state before, event type] => state after }, the places being the
101
+ # code's objection states (:none before one is recorded).
102
+ def self.steps
103
+ defined_transitions.to_h { |t| [[t[:from].first, EVENTS.fetch(t[:name])], t[:to].first] }
104
+ end
105
+ end
106
+
107
+ # An objection under Art. 21(1) or 21(6) together with the Art. 18(1)(d)
108
+ # restriction it creates: two tokens, one per request, so the net shows
109
+ # how the decision ends the restriction. Upheld or withdrawn, the
110
+ # restriction is lifted without a notice (processing does not resume, or
111
+ # the subject asked); overridden, only after the Art. 18(3) notice. A
112
+ # notice before the decision is not possible.
113
+ class ObjectionRestrictionWorkflow < PetriFlow::Workflow
114
+ workflow_name "Objection and its Art. 18(1)(d) restriction"
115
+
116
+ places :o_none, :o_pending, :o_upheld, :o_overridden, :o_withdrawn,
117
+ :r_none, :r_active, :r_lift_notified, :r_lifted
118
+
119
+ initial_place :o_none
120
+ terminal_places :r_lifted
121
+
122
+ transition :receive, from: :o_none, to: :o_pending, trigger: "ObjectionReceived"
123
+ transition :restrict, from: %i[o_pending r_none], to: %i[o_pending r_active],
124
+ trigger: "ProcessingRestricted (objection_id)"
125
+ transition :uphold, from: %i[o_pending r_active], to: %i[o_upheld r_active], trigger: "ObjectionUpheld"
126
+ transition :override, from: %i[o_pending r_active], to: %i[o_overridden r_active], trigger: "ObjectionOverridden"
127
+ transition :withdraw, from: %i[o_pending r_active], to: %i[o_withdrawn r_active], trigger: "ObjectionWithdrawn"
128
+ transition :withdraw, from: %i[o_upheld r_active], to: %i[o_withdrawn r_active], trigger: "ObjectionWithdrawn"
129
+ # (a name of its own: PetriFlow names a transition by its first input place)
130
+ transition :withdraw_lifted, from: %i[o_upheld r_lifted], to: %i[o_withdrawn r_lifted], trigger: "ObjectionWithdrawn"
131
+ transition :lift, from: %i[o_upheld r_active], to: %i[o_upheld r_lifted], trigger: "RestrictionLifted (no notice)"
132
+ transition :lift, from: %i[o_withdrawn r_active], to: %i[o_withdrawn r_lifted], trigger: "RestrictionLifted (no notice)"
133
+ transition :notify_lift, from: %i[o_overridden r_active], to: %i[o_overridden r_lift_notified],
134
+ trigger: "RestrictionLiftNotified (Art. 18(3))"
135
+ transition :lift, from: %i[o_overridden r_lift_notified], to: %i[o_overridden r_lifted], trigger: "RestrictionLifted"
136
+
137
+ EVENTS = { receive: "ObjectionReceived", restrict: "ProcessingRestricted", uphold: "ObjectionUpheld",
138
+ override: "ObjectionOverridden", withdraw: "ObjectionWithdrawn", withdraw_lifted: "ObjectionWithdrawn",
139
+ notify_lift: "RestrictionLiftNotified",
140
+ lift: "RestrictionLifted" }.freeze
141
+
142
+ # The initial marking: no objection and no restriction yet.
143
+ def initial_marking
144
+ PetriFlow::Core::Marking.new(self.class.defined_places.to_h { |p| [p, %i[o_none r_none].include?(p) ? 1 : 0] })
145
+ end
146
+
147
+ # { [[objection state, restriction state], event type] => [states after] }
148
+ # over the reachable markings, the code's states named as the places
149
+ # without their prefix.
150
+ def steps
151
+ analyzer = PetriFlow::Verification::ReachabilityAnalyzer.new(net, initial_marking)
152
+ analyzer.analyze
153
+ by_key = analyzer.reachable_markings.to_h { |m| [analyzer.send(:marking_to_key, m), states_of(m)] }
154
+ ids = self.class.defined_transitions.to_h { |t| [t[:id], EVENTS.fetch(t[:name])] }
155
+ analyzer.state_graph.flat_map do |key, edges|
156
+ edges.map { |transition, after| [[by_key.fetch(key), ids.fetch(transition)], by_key.fetch(after)] }
157
+ end.to_h
158
+ end
159
+
160
+ private
161
+
162
+ def states_of(marking)
163
+ %w[o r].map do |prefix|
164
+ place = self.class.defined_places.find { |p| p.to_s.start_with?("#{prefix}_") && marking.tokens_at(p).positive? }
165
+ place.to_s.delete_prefix("#{prefix}_").to_sym
166
+ end
167
+ end
168
+ end
169
+ end
170
+ end
data/lib/lyra/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Lyra
2
- VERSION = "0.9.2"
2
+ VERSION = "1.0.0"
3
3
  end
@@ -166,7 +166,7 @@ module Lyra
166
166
  def detect_pii?(event)
167
167
  return false unless Lyra.privacy_features_available?
168
168
 
169
- Lyra::Privacy::PIIDetector.detect(event_attributes(event)).any?
169
+ Lyra::Privacy::PIIDetector.detect(Lyra::Event.fields_of(event)).any?
170
170
  rescue
171
171
  false
172
172
  end
@@ -188,7 +188,7 @@ module Lyra
188
188
  def has_pii?(event)
189
189
  return false unless Lyra.privacy_features_available?
190
190
 
191
- Lyra::Privacy::PIIDetector.detect(event_attributes(event)).any?
191
+ Lyra::Privacy::PIIDetector.detect(Lyra::Event.fields_of(event)).any?
192
192
  end
193
193
 
194
194
  def build_links(events)
@@ -287,7 +287,7 @@ module Lyra
287
287
 
288
288
  def has_pii?(event)
289
289
  return false unless Lyra.privacy_features_available?
290
- Lyra::Privacy::PIIDetector.detect(event_attributes(event)).any?
290
+ Lyra::Privacy::PIIDetector.detect(Lyra::Event.fields_of(event)).any?
291
291
  end
292
292
 
293
293
  def h(text)
data/lib/lyra.rb CHANGED
@@ -1,4 +1,6 @@
1
1
  require "lyra/version"
2
+ require "lyra/internal"
3
+ require "lyra/attribution"
2
4
  require "rails_event_store"
3
5
  # Lyra builds on these at load time (jobs, ActiveRecord errors); require them
4
6
  # rather than rely on the host application having loaded rails/all first.
@@ -34,20 +36,25 @@ require "lyra/model_selection"
34
36
  require "lyra/configuration"
35
37
  require "lyra/correlation"
36
38
  require "lyra/event"
39
+ require "lyra/event_data"
40
+ require "lyra/saves_without_effect"
37
41
  require "lyra/encrypted_attributes"
38
42
  require "lyra/optimistic_locking"
39
43
  require "lyra/event_store_adapter"
40
44
  require "lyra/event_serializer"
45
+ require "lyra/stream_tail"
41
46
  require "lyra/projection"
42
47
  require "lyra/temporal"
43
48
  require "lyra/domain_events"
44
49
  require "lyra/mode_transition"
45
50
  require "lyra/mode_sync"
51
+ require "lyra/touch_registry"
46
52
  require "lyra/dual_view_sampler"
47
53
  require "lyra/command"
48
54
  require "lyra/aggregate"
49
55
  require "lyra/command_handler"
50
56
  require "lyra/dual_view"
57
+ require "lyra/event_window"
51
58
  require "lyra/event_flow"
52
59
  require "lyra/event_analyzer"
53
60
  require "lyra/id_generator"
@@ -65,13 +72,19 @@ require "lyra/projections/rebuild"
65
72
  require "lyra/sti_stream_migration"
66
73
  require "lyra/projections/async_checkpoint"
67
74
  require "lyra/projections/async_projection_job"
75
+ require "lyra/projections/typed_values"
76
+ require "lyra/projections/value_digest"
68
77
  require "lyra/projections/cached_projection"
69
78
  require "lyra/projections/reference_links"
70
79
  require "lyra/projections/cached_joins"
71
80
  require "lyra/projections/cached_relation"
72
81
  require "lyra/projections/deferred_relation"
82
+ require "lyra/projections/default_scopes"
83
+ require "lyra/projections/foreign_keys"
73
84
  require "lyra/projections/association_conditions"
74
85
  require "lyra/projections/event_store_reader"
86
+ require "lyra/projections/uniqueness"
87
+ require "lyra/projections/unique_indexes"
75
88
  require "lyra/projections/lazy_projection"
76
89
  require "lyra/genesis"
77
90
  require "lyra/repair"
@@ -79,9 +92,9 @@ require "lyra/purpose_bound_reads"
79
92
  require "lyra/erasure"
80
93
  require "lyra/retention"
81
94
 
82
- # Event-aware associations
83
- require "lyra/associations/event_aware"
95
+ # has_and_belongs_to_many links as events
84
96
  require "lyra/associations/habtm_links"
97
+ require "lyra/associations/maintenance"
85
98
 
86
99
  # Consistency helpers
87
100
  require "lyra/consistency/read_your_writes"
@@ -96,7 +109,10 @@ if PAM_DSL_AVAILABLE
96
109
  require "lyra/privacy/adapters/pam"
97
110
  require "lyra/privacy/pii_masker"
98
111
  require "lyra/privacy/gdpr_compliance"
112
+ require "lyra/privacy/subject_links"
113
+ require "lyra/privacy/subject_records"
99
114
  require "lyra/access_log"
115
+ require "lyra/rights"
100
116
  end
101
117
 
102
118
  # Schema validation
@@ -117,6 +133,7 @@ if PETRI_FLOW_AVAILABLE
117
133
  require "lyra/verification/crud_lifecycle_workflow"
118
134
  require "lyra/verification/workflow_generator"
119
135
  require "lyra/verification/bypass_workflow"
136
+ require "lyra/verification/rights_workflows"
120
137
  end
121
138
 
122
139
  module Lyra
@@ -166,6 +183,29 @@ module Lyra
166
183
  model_class.name
167
184
  end
168
185
 
186
+ # The column that holds a record's key: the model's single primary key,
187
+ # whatever it is called ("id" unless the model names another). Composite
188
+ # primary keys are refused at enrolment (Lyra::ModelSelection).
189
+ def self.key_column(model_class)
190
+ key = model_class.primary_key if model_class.respond_to?(:primary_key)
191
+ key.is_a?(String) && !key.empty? ? key : "id"
192
+ end
193
+
194
+ # A record's key as its row holds it: +id+ (from a stream name, a string)
195
+ # as an integer when the key column is an integer one.
196
+ def self.key_value(model_class, id)
197
+ return id unless id.to_s.match?(/\A\d+\z/)
198
+
199
+ key = key_column(model_class)
200
+ type = begin
201
+ model_class.columns_hash[key]&.type if model_class.respond_to?(:columns_hash)
202
+ rescue StandardError
203
+ nil # no table to read the column from: an "id" is taken to be an integer, as before 1.0.0
204
+ end
205
+ integer = type ? %i[integer bigint].include?(type) : key == "id"
206
+ integer ? id.to_i : id
207
+ end
208
+
169
209
  # Delegate event_store to configuration
170
210
  def self.event_store
171
211
  config.event_store
@@ -228,6 +268,7 @@ module Lyra
228
268
  next
229
269
  end
230
270
  failures << "#{model.name}: no primary key" if model.primary_key.blank?
271
+ failures << "#{model.name}: composite primary key (not supported)" if model.primary_key.is_a?(Array)
231
272
  end
232
273
 
233
274
  raise MappingVerificationError, "Lyra mapping verification failed: #{failures.join('; ')}" if failures.any?
@@ -29,6 +29,8 @@ namespace :lyra do
29
29
  report.discrepancies.first(20).each { puts " #{_1}" }
30
30
  puts(report.clean? ? "Clean: certified for #{Lyra.config.mode_transition_certificate_ttl}s." : "Not certified.")
31
31
  exit 1 unless report.clean?
32
+ rescue Lyra::ModeTransition::Refused => e
33
+ abort e.message
32
34
  end
33
35
 
34
36
  desc "Show the mode the application runs in and the last one applied"
@@ -36,44 +38,140 @@ namespace :lyra do
36
38
  puts "configured: #{Lyra::ModeTransition.current}"
37
39
  puts "last applied: #{Lyra::ModeTransition.last_applied || '(none recorded)'}"
38
40
  puts "gate: #{Lyra::ModeTransition.gate_enabled? ? 'on' : 'off'}"
41
+ live = Lyra::TouchRegistry.live_processes
42
+ puts "touches left out (config.record_touches = false): #{live.empty? ? 'by no live process' : "#{live.size} live process(es)"}"
43
+ live.each do |p|
44
+ puts " #{p[:host]} pid #{p[:pid]} (Lyra #{p[:version]}), since #{p[:since]&.utc&.iso8601}, " \
45
+ "last seen #{p[:last_seen]&.utc&.iso8601}"
46
+ end
47
+ puts " #{Lyra::TouchRegistry::STEPS}" if live.any?
39
48
  end
40
49
  end
41
50
 
42
51
  desc "Bring the event log back in line with the tables after Monitor lost events " \
43
- "(DRY_RUN=1 lists them; MODELS=User,Order limits it)"
52
+ "(DRY_RUN=1 lists them; MODELS=User,Order limits it; UNREPRESENTABLE=skip repairs the rest when rows hold " \
53
+ "values their model cannot represent, default config.genesis_unrepresentable, :refuse)"
44
54
  task repair: :environment do
45
55
  models = ENV["MODELS"] ? ENV["MODELS"].split(",").map { _1.strip.constantize } : lyra_monitored_models.call
46
- result = Lyra::Repair.run(models: models, dry_run: ENV["DRY_RUN"].present?)
56
+ result = Lyra::Repair.run(models: models, dry_run: ENV["DRY_RUN"].present?, operator: ENV["OPERATOR"],
57
+ unrepresentable: ENV["UNREPRESENTABLE"].presence || Lyra.config.genesis_unrepresentable)
47
58
  puts result.summary
48
59
  (ENV["DRY_RUN"].present? ? result.found : result.remaining).first(20).each { puts " #{_1}" }
60
+ if result.unrepresentable.any?
61
+ puts "Values their model cannot represent (#{ENV['DRY_RUN'].present? ? 'would not be repaired' : 'not repaired'}):"
62
+ result.unrepresentable.each { puts " #{_1}" }
63
+ end
49
64
  exit 1 if result.remaining.any? && ENV["DRY_RUN"].blank?
50
65
  rescue Lyra::Repair::Refused => e
51
66
  abort e.message
52
67
  end
53
68
  desc "Erase one record's personal data from its row and its events (Art. 17) " \
54
- "(MODEL=Registration ID=5 REASON='Art. 17 request #12' [FIELDS=email,phone] [EVERYWHERE=1] " \
55
- "[MAX_COPIES=10: with EVERYWHERE, a value held by more other records is left as shared])"
69
+ "(MODEL=Registration ID=5 REASON='Art. 17 request #12' [OPERATOR=jdoe] [FIELDS=email,phone] " \
70
+ "[ANY_FIELDS=1: FIELDS may name columns the policy does not declare personal] [EVERYWHERE=1] " \
71
+ "[MAX_COPIES=n: with EVERYWHERE, a value held by more other records is left as shared; default config.erasure_max_copies, 10] " \
72
+ "[DRY_RUN=1: report what would be erased and the values left as shared, write nothing] " \
73
+ "[ACCESS_LOG=1|0: delete or keep the record's access log; default config.erase_access_log] " \
74
+ "[ACCESS_LOG_SUBJECTS=42,user-42: ids the application used as this person's subject, whose logs go too])"
56
75
  task erase: :environment do
57
76
  model = ENV.fetch("MODEL").constantize
58
- max_copies = Lyra::Erasure::MAX_COPIES
77
+ max_copies = Lyra.config.erasure_max_copies
59
78
  if ENV["MAX_COPIES"].present?
60
79
  max_copies = Integer(ENV["MAX_COPIES"], 10, exception: false)
61
80
  abort "MAX_COPIES must be a positive integer (got #{ENV['MAX_COPIES'].inspect})" unless max_copies&.positive?
62
81
  end
82
+ access_log = case ENV["ACCESS_LOG"].to_s.strip.downcase
83
+ when "" then nil
84
+ when "1", "true", "yes", "erase" then true
85
+ when "0", "false", "no", "keep" then false
86
+ else abort "ACCESS_LOG must be 1 (erase) or 0 (keep) (got #{ENV['ACCESS_LOG'].inspect})"
87
+ end
88
+ dry_run = ENV["DRY_RUN"].present?
63
89
  result = Lyra::Erasure.erase!(model, ENV.fetch("ID"), reason: ENV.fetch("REASON"),
64
90
  fields: ENV["FIELDS"]&.split(",")&.map(&:strip),
65
- everywhere: ENV["EVERYWHERE"].present?, max_copies: max_copies)
66
- puts "Erased #{result.fields.join(', ')} of #{result.model} #{result.id}: " \
91
+ everywhere: ENV["EVERYWHERE"].present?, max_copies: max_copies,
92
+ any_fields: ENV["ANY_FIELDS"].present?, access_log: access_log,
93
+ access_log_subjects: ENV["ACCESS_LOG_SUBJECTS"].to_s.split(",").map(&:strip).reject(&:empty?),
94
+ erased_by: Lyra::ModeTransition.actor(ENV["OPERATOR"]), dry_run: dry_run,
95
+ lifting: ENV["LIFTING"].to_s.split(",").map(&:strip).reject(&:empty?).presence)
96
+ puts "Dry run: nothing was written. This is what the erasure would do." if dry_run
97
+ # A key that holds an erased value is never printed.
98
+ shown_id = result.key_withheld ? Lyra::Erasure::WITHHELD : result.id
99
+ puts "Erased #{result.fields.join(', ')} of #{result.model} #{shown_id}: " \
67
100
  "#{result.events_rewritten} events rewritten, row #{result.row_erased ? 'erased' : 'absent'}."
101
+ if result.access_log
102
+ puts "Access log #{result.access_log}: #{result.access_log_entries} entries."
103
+ end
68
104
  puts "Copies erased: #{result.copies.join(', ')}" if result.copies.any?
105
+ lifted = Array(result.lifting)
106
+ puts "Restrictions #{dry_run ? 'it would lift' : 'lifted'} (Art. 18(3) notice recorded): #{lifted.join(', ')}" if lifted.any?
107
+ kept = Array(result.restricted_copies)
108
+ if kept.any?
109
+ puts "Copies kept under a restriction of processing (Art. 18), not modified; each stays until its holder's " \
110
+ "restriction is lifted, then erase again with EVERYWHERE=1: " +
111
+ kept.map { |entry| "#{entry[:record]} (#{entry[:columns].join(', ')})" }.join("; ")
112
+ end
113
+ left = Array(result.left_behind)
114
+ if left.any?
115
+ puts "Left behind (the value is still in these columns, as a JSON key, inside a longer string or in a key): " +
116
+ left.map { |entry| "#{entry[:record]} (#{entry[:columns].join(', ')})" }.join("; ")
117
+ end
118
+ not_searched = Array(result.not_searched)
119
+ if not_searched.any?
120
+ puts "Not searched (encrypted, the log holds their ciphertext): #{not_searched.join(', ')}"
121
+ end
69
122
  if result.shared_values.positive?
70
123
  puts "Values left as shared (held by more than #{max_copies} other records): #{result.shared_values}"
124
+ # By field and count, never the value. Once the person's own values are
125
+ # erased, a later run cannot find these copies: choose MAX_COPIES with a
126
+ # dry run first.
127
+ Array(result.shared).each do |entry|
128
+ hint = if dry_run
129
+ "check them, and if they are this person's, erase with MAX_COPIES=#{entry[:holders]}"
130
+ else
131
+ "left in place; this record's values are now erased, so a later run cannot find them: " \
132
+ "next time run with DRY_RUN=1 first and choose MAX_COPIES"
133
+ end
134
+ puts " #{entry[:field]} (#{entry[:type]}): held by #{entry[:holders]} other records; #{hint}"
135
+ end
136
+ end
137
+ rows_only = Array(result.rows_without_events)
138
+ if rows_only.any?
139
+ remedy = if Lyra.config.monitor_mode? || Lyra.config.disabled_mode?
140
+ "their rows differed from their events before the erasure: run bin/rails lyra:repair " \
141
+ "to record the rows' state in the log"
142
+ else
143
+ "the events are authoritative in #{Lyra::ModeTransition.current}, and these rows differ from " \
144
+ "them: rebuild their tables from the events (bin/rails lyra:projections:rebuild MODEL=...)"
145
+ end
146
+ puts "Rows without events (erased in the row only): #{rows_only.join(', ')}; #{remedy}"
71
147
  end
72
148
  rescue KeyError => e
73
149
  abort "#{e.message}: MODEL, ID and REASON are required"
74
- rescue Lyra::Erasure::Unsupported => e
150
+ rescue Lyra::Erasure::Unsupported, Lyra::AccessLog::ModelWideSubject => e
151
+ abort e.message
152
+ end
153
+ desc "Delete one subject's access log (SUBJECT=Registration$5 or an id the application passes as the " \
154
+ "subject, e.g. SUBJECT=42; REASON='Art. 17 request #12' [OPERATOR=jdoe]). Model-wide entries " \
155
+ "(Registration$*) are refused: they name no subject"
156
+ task erase_access_log: :environment do
157
+ subject = ENV.fetch("SUBJECT")
158
+ entries = Lyra::AccessLog.erase!(subject, reason: ENV.fetch("REASON"),
159
+ erased_by: Lyra::ModeTransition.actor(ENV["OPERATOR"]))
160
+ puts "Access log of #{subject} erased: #{entries} entries (recorded in #{Lyra::AccessLog::ERASURES})."
161
+ rescue KeyError => e
162
+ abort "#{e.message}: SUBJECT and REASON are required"
163
+ rescue ArgumentError => e
75
164
  abort e.message
76
165
  end
166
+ namespace :rights do
167
+ desc "Rebuild lyra_rights and lyra_consents (the subjects' restrictions, objections and consent) from " \
168
+ "their event streams, Lyra::Rights and Lyra::Consent"
169
+ task rebuild: :environment do
170
+ requests, consents = Lyra::Rights.rebuild!
171
+ puts "Rebuilt #{requests} restrictions and objections, and #{consents} consent records."
172
+ end
173
+ end
174
+
77
175
  namespace :retention do
78
176
  desc "Apply the privacy policy's retention rules (config.retention_executor; DRY_RUN=1 lists what " \
79
177
  "would happen, executor on or off; MODELS=Registration,Order limits it)"
@@ -82,6 +180,10 @@ namespace :lyra do
82
180
  result = Lyra::Retention.apply!(models: models, dry_run: ENV["DRY_RUN"].present?)
83
181
  puts result.summary
84
182
  result.actions.first(50).each { puts " #{_1}" }
183
+ if result.restricted.any?
184
+ puts "Kept under a restriction of processing (Art. 18), acted on once it is lifted:"
185
+ result.restricted.each { puts " #{_1}" }
186
+ end
85
187
  rescue Lyra::Retention::Disabled => e
86
188
  abort e.message
87
189
  end
@@ -32,11 +32,35 @@ namespace :lyra do
32
32
  )
33
33
  end
34
34
  puts "Done."
35
+ rescue Lyra::Projections::Rebuild::Refused => e
36
+ abort e.message
37
+ end
38
+ end
39
+
40
+ desc "Link every event of the monitored models (MODEL=Namespace::Model[,Other] for specific models) " \
41
+ "into the value streams ES-NoProj checks unique indexes against, from the first event: after a " \
42
+ "rolling deploy in which processes running an earlier Lyra, or reading the indexes before a migration " \
43
+ "added one, wrote (Lyra::Projections::ReferenceLinks.relink_values!)"
44
+ task link_values: :environment do
45
+ models =
46
+ if ENV["MODEL"]
47
+ ENV["MODEL"].split(",").map { |name| name.strip.constantize }
48
+ else
49
+ Rails.application.eager_load! unless Rails.application.config.eager_load
50
+ Lyra.config.monitored_models
51
+ end
52
+ models.map { _1.respond_to?(:base_class) ? _1.base_class : _1 }.uniq.each do |model|
53
+ next if Lyra::Projections::UniqueIndexes.columns(model).empty?
54
+
55
+ linked = Lyra::Projections::ReferenceLinks.relink_values!(model)
56
+ puts "#{model.name}: #{linked} value links (#{Lyra::Projections::UniqueIndexes.columns(model).join(', ')})"
35
57
  end
36
58
  end
37
59
 
38
60
  desc "Give rows that predate Lyra their Imported events, ahead of first use " \
39
- "(MODEL=Namespace::Model[,Other] for specific models, or all monitored models)"
61
+ "(MODEL=Namespace::Model[,Other] for specific models, or all monitored models; " \
62
+ "UNREPRESENTABLE=skip imports the rest of a model holding rows whose values it cannot represent, " \
63
+ "default config.genesis_unrepresentable, :refuse)"
40
64
  task genesis: :environment do
41
65
  models =
42
66
  if ENV["MODEL"]
@@ -53,8 +77,25 @@ namespace :lyra do
53
77
  exit 1
54
78
  end
55
79
 
80
+ # A model holding rows it cannot represent is refused (nothing of it
81
+ # imported) and listed, the other models imported, and the task exits 1.
82
+ # With UNREPRESENTABLE=skip those rows are listed and left out.
83
+ unrepresentable = ENV["UNREPRESENTABLE"].presence || Lyra.config.genesis_unrepresentable
84
+ refused = false
56
85
  models.each do |model_class|
57
- puts format(" %-30s %6d rows imported", model_class.name, Lyra::Genesis.import_all(model_class))
86
+ result = Lyra::Genesis.import(model_class, unrepresentable: unrepresentable)
87
+ skipped = result.skipped.positive? ? ", #{result.skipped} skipped (values the model cannot represent)" : ""
88
+ puts format(" %-30s %6d rows imported%s", model_class.name, result.imported, skipped)
89
+ result.unrepresentable.each { puts " #{_1}" }
90
+ rescue Lyra::Genesis::Refused => e
91
+ puts format(" %-30s refused: %d rows hold values the model cannot represent; nothing imported", model_class.name, e.rows)
92
+ e.values.each { puts " #{_1}" }
93
+ puts " (and #{e.found - e.values.size} more)" if e.found > e.values.size
94
+ refused = true
95
+ end
96
+ if refused
97
+ puts "Not every model was imported: fix the rows listed or their models, or skip those rows with UNREPRESENTABLE=skip."
98
+ exit 1
58
99
  end
59
100
  puts "Done."
60
101
  end