orfeas_lyra 0.9.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +2021 -17
  3. data/README.md +22 -9
  4. data/Rakefile +40 -1
  5. data/app/controllers/lyra/application_controller.rb +127 -0
  6. data/app/controllers/lyra/dashboard_controller.rb +33 -31
  7. data/app/controllers/lyra/flow_controller.rb +32 -34
  8. data/app/controllers/lyra/privacy_controller.rb +57 -18
  9. data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
  10. data/app/views/lyra/dashboard/index.html.erb +1 -0
  11. data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
  12. data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
  13. data/app/views/lyra/privacy/policy.html.erb +7 -0
  14. data/app/views/lyra/privacy/rights.html.erb +78 -0
  15. data/config/privacy_policies.rb +0 -3
  16. data/config/routes.rb +1 -0
  17. data/lib/lyra/access_log.rb +187 -6
  18. data/lib/lyra/advisory_lock.rb +236 -0
  19. data/lib/lyra/aggregate.rb +1 -1
  20. data/lib/lyra/associations/habtm_links.rb +382 -19
  21. data/lib/lyra/associations/maintenance.rb +229 -0
  22. data/lib/lyra/attribution.rb +48 -0
  23. data/lib/lyra/bypass_events.rb +446 -46
  24. data/lib/lyra/command_handler.rb +19 -14
  25. data/lib/lyra/configuration.rb +237 -10
  26. data/lib/lyra/consistency/read_your_writes.rb +4 -2
  27. data/lib/lyra/correlation.rb +121 -0
  28. data/lib/lyra/domain_events.rb +4 -0
  29. data/lib/lyra/dual_view.rb +53 -16
  30. data/lib/lyra/encrypted_attributes.rb +5 -0
  31. data/lib/lyra/engine.rb +67 -0
  32. data/lib/lyra/erasure.rb +1011 -106
  33. data/lib/lyra/event.rb +35 -1
  34. data/lib/lyra/event_data.rb +38 -0
  35. data/lib/lyra/event_flow.rb +31 -15
  36. data/lib/lyra/event_store_adapter.rb +28 -0
  37. data/lib/lyra/event_window.rb +58 -0
  38. data/lib/lyra/genesis.rb +548 -44
  39. data/lib/lyra/interceptors/association_interceptor.rb +689 -34
  40. data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
  41. data/lib/lyra/internal.rb +26 -0
  42. data/lib/lyra/mode_sync.rb +51 -3
  43. data/lib/lyra/mode_transition.rb +257 -30
  44. data/lib/lyra/model_selection.rb +35 -0
  45. data/lib/lyra/own_connection.rb +17 -2
  46. data/lib/lyra/privacy/adapters/pam.rb +25 -2
  47. data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
  48. data/lib/lyra/privacy/interface.rb +27 -7
  49. data/lib/lyra/privacy/subject_links.rb +143 -0
  50. data/lib/lyra/privacy/subject_records.rb +369 -0
  51. data/lib/lyra/projection.rb +7 -1
  52. data/lib/lyra/projections/async_checkpoint.rb +32 -0
  53. data/lib/lyra/projections/async_projection_job.rb +166 -23
  54. data/lib/lyra/projections/cached_joins.rb +33 -18
  55. data/lib/lyra/projections/cached_projection.rb +74 -33
  56. data/lib/lyra/projections/cached_relation.rb +3175 -321
  57. data/lib/lyra/projections/default_scopes.rb +254 -0
  58. data/lib/lyra/projections/deferred_relation.rb +23 -3
  59. data/lib/lyra/projections/event_store_reader.rb +47 -8
  60. data/lib/lyra/projections/foreign_keys.rb +92 -0
  61. data/lib/lyra/projections/lazy_projection.rb +165 -27
  62. data/lib/lyra/projections/model_projection.rb +165 -14
  63. data/lib/lyra/projections/rebuild.rb +34 -6
  64. data/lib/lyra/projections/reference_links.rb +463 -72
  65. data/lib/lyra/projections/typed_values.rb +629 -0
  66. data/lib/lyra/projections/unique_indexes.rb +595 -0
  67. data/lib/lyra/projections/uniqueness.rb +784 -0
  68. data/lib/lyra/projections/value_digest.rb +93 -0
  69. data/lib/lyra/purpose_bound_reads.rb +912 -77
  70. data/lib/lyra/repair.rb +119 -28
  71. data/lib/lyra/retention.rb +114 -8
  72. data/lib/lyra/rights/consent_store.rb +415 -0
  73. data/lib/lyra/rights/erasure_guard.rb +135 -0
  74. data/lib/lyra/rights/pending.rb +74 -0
  75. data/lib/lyra/rights/queries.rb +269 -0
  76. data/lib/lyra/rights/store.rb +512 -0
  77. data/lib/lyra/rights/subject_keys.rb +224 -0
  78. data/lib/lyra/rights.rb +281 -0
  79. data/lib/lyra/saves_without_effect.rb +69 -0
  80. data/lib/lyra/stream_tail.rb +319 -0
  81. data/lib/lyra/strict_data_access.rb +916 -106
  82. data/lib/lyra/temporal.rb +4 -2
  83. data/lib/lyra/touch_registry.rb +404 -0
  84. data/lib/lyra/verification/bypass_workflow.rb +452 -42
  85. data/lib/lyra/verification/rights_workflows.rb +170 -0
  86. data/lib/lyra/version.rb +1 -1
  87. data/lib/lyra/visualization/event_graph.rb +1 -1
  88. data/lib/lyra/visualization/timeline.rb +2 -2
  89. data/lib/lyra.rb +43 -2
  90. data/lib/tasks/lyra_mode.rake +110 -8
  91. data/lib/tasks/lyra_projections.rake +43 -2
  92. metadata +34 -4
  93. data/lib/lyra/associations/event_aware.rb +0 -224
@@ -0,0 +1,269 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Lyra
4
+ module Rights
5
+ # The subject's rights in queries over many people (Lyra::PurposeBoundReads).
6
+ #
7
+ # Condition 6 of Def. 1 is per subject, so a query whose subject is the
8
+ # whole model ("Model$*": an aggregate, pluck, ids, exists?) cannot be
9
+ # checked against one person's restriction. Refusing it would let one
10
+ # person's request stop everyone's statistics; running it as it is would
11
+ # process the restricted person's data. Such a query leaves the restricted
12
+ # subjects out: under a purpose, when a request in force would refuse a
13
+ # read of a subject's record (PamDsl::Policy#rights_violations: a
14
+ # restriction of all of the subject's data, or of a field the query names,
15
+ # and no Art. 18(2) exception of the purpose that applies; a pending
16
+ # objection's restriction of this purpose; an upheld objection to it), the
17
+ # query runs with that subject's rows excluded:
18
+ #
19
+ # - a record key (EnfArticle$7): the record's own key column NOT IN (...);
20
+ # - a person key (Payer$7) the model links to (Rights::SubjectKeys: a
21
+ # belongs_to, a polymorphic belongs_to, a <type>_id column of a
22
+ # monitored model): (payer_id IS NULL OR payer_id NOT IN (...)), and for
23
+ # a polymorphic link (owner_id IS NULL OR owner_type IS NULL OR
24
+ # owner_type NOT IN ('Payer', <its STI subclasses>) OR owner_id NOT IN
25
+ # (...)), so rows that name no one are kept, as SQL's NOT IN alone would
26
+ # not keep them. The type column may hold an STI subclass's name; the
27
+ # subclasses are those loaded (eager loading, as in production).
28
+ #
29
+ # ES-NoProj's relations (Projections::CachedRelation) hold their records
30
+ # in memory: a record is left out when any of its keys (its own, and its
31
+ # people's, by the same resolver) is refused; a record whose links cannot
32
+ # be read raises Rights::UnresolvedSubjectError (fail closed).
33
+ #
34
+ # A join, subquery or CTE that reaches a model with such a subject is
35
+ # refused rather than filtered (refuse_reaching!): the excluded rows would
36
+ # have to be traced through the join.
37
+ #
38
+ # In audit mode nothing is left out: the query runs whole and the log says
39
+ # how many restricted subjects it may include. With nothing in force
40
+ # anywhere, none of this does any work beyond the store's any_active? (a
41
+ # clock read, once its scope has read the rights).
42
+ #
43
+ # Model.lyra_processable gives a batch job the records the purpose in
44
+ # scope may process: the same exclusion, for every field the policy
45
+ # declares unless fields: names the ones the job reads, in every
46
+ # enforcement mode.
47
+ module Queries
48
+ # The subjects a query on +model+ leaves out: +own+ (ids of the model's
49
+ # own records), +links+ ([[column, ids]]), +polymorphic+
50
+ # ([[id column, type column, type, ids]]) and +keys+ (the subject keys).
51
+ Exclusion = Struct.new(:model, :own, :links, :polymorphic, :keys, keyword_init: true) do
52
+ # The condition the rows kept satisfy.
53
+ def arel
54
+ table = model.arel_table
55
+ conditions = []
56
+ conditions << table[Lyra.key_column(model)].not_in(own) if own.any?
57
+ links.each { |column, ids| conditions << table[column].eq(nil).or(table[column].not_in(ids)) }
58
+ polymorphic.each do |column, type_column, type, ids|
59
+ conditions << table[column].eq(nil).or(table[type_column].eq(nil))
60
+ .or(table[type_column].not_in(Queries.type_names(type)))
61
+ .or(table[column].not_in(ids))
62
+ end
63
+ conditions.reduce { |all, condition| all.and(condition) }
64
+ end
65
+
66
+ # The condition as SQL, compiled once (a list of hundreds of ids is
67
+ # costly to build and compile on every query).
68
+ def sql
69
+ @sql ||= Arel.sql("(#{arel.to_sql})")
70
+ end
71
+
72
+ def keeps?(record)
73
+ !SubjectKeys.keys_for(record).intersect?(keys)
74
+ end
75
+ end
76
+
77
+ class << self
78
+ # The names a polymorphic type column may hold for the owner +type+:
79
+ # its own and its STI subclasses'.
80
+ def type_names(type)
81
+ klass = type.to_s.safe_constantize
82
+ return [type.to_s] unless klass.is_a?(Class) && klass < ActiveRecord::Base
83
+
84
+ ([klass] + klass.descendants).map(&:name).compact.uniq
85
+ end
86
+
87
+ # Is any request in force anywhere (PamDsl's store)?
88
+ def in_force?
89
+ PamDsl.rights_store.any_active?
90
+ end
91
+
92
+ # The Exclusion for a query on +model+ naming +field_names+ under
93
+ # +purpose+ (default: the one in scope), or nil when no subject is
94
+ # left out. +audit+: in audit mode, log and return nil.
95
+ def exclusion(model, field_names, purpose: PurposeBoundReads.current, audit: true)
96
+ return nil unless purpose && in_force?
97
+
98
+ policy = pam_policy(model)
99
+ return nil unless policy
100
+
101
+ refused = refused_keys(model, policy, purpose, field_names)
102
+ return nil if refused.empty?
103
+
104
+ if audit && policy.enforcement_mode == :audit
105
+ PamDsl.logger.warn("Lyra: #{model.name} query under purpose '#{purpose}' includes the rows of " \
106
+ "#{refused.size} restricted subject#{'s' unless refused.size == 1} (audit mode: " \
107
+ "not left out)")
108
+ return nil
109
+ end
110
+
111
+ built(model, refused)
112
+ end
113
+
114
+ # +relation+ (an Active Record relation) leaving out the restricted
115
+ # subjects, or nil when there are none.
116
+ def filter(relation, field_names)
117
+ exclusion = exclusion(relation.klass, field_names)
118
+ exclusion && relation.where(exclusion.sql)
119
+ end
120
+
121
+ # The records of +records+ a query on +model+ keeps, or nil when it
122
+ # keeps them all.
123
+ def kept_records(model, field_names, records)
124
+ exclusion = exclusion(model, field_names)
125
+ exclusion && records.select { |record| exclusion.keeps?(record) }
126
+ end
127
+
128
+ # Model.lyra_processable: +relation+ without the records the purpose
129
+ # in scope may not process (for +fields+, default every declared
130
+ # field). Any relation with no purpose in scope.
131
+ def processable(relation, fields = nil)
132
+ model = relation.respond_to?(:model_class) ? relation.model_class : relation.klass
133
+ policy = PurposeBoundReads.current && pam_policy(model)
134
+ return relation unless policy
135
+
136
+ fields = fields.nil? ? policy.fields.keys : Array(fields).map(&:to_sym)
137
+ exclusion = exclusion(model, fields, audit: false)
138
+ return relation unless exclusion
139
+
140
+ if relation.is_a?(Projections::CachedRelation)
141
+ relation.lyra_select_records { |record| exclusion.keeps?(record) }
142
+ else
143
+ relation.where(exclusion.sql)
144
+ end
145
+ end
146
+
147
+ # A join, subquery or CTE reaching +model+ (naming +field_names+ of
148
+ # it) under the purpose in scope: refused when it would reach a
149
+ # subject left out of a query on +model+, as Condition 6 refuses a
150
+ # read (strict mode raises, audit mode logs).
151
+ def refuse_reaching!(model, field_names)
152
+ purpose = PurposeBoundReads.current
153
+ return unless purpose && in_force?
154
+
155
+ policy = pam_policy(model)
156
+ return unless policy
157
+
158
+ key = refused_keys(model, policy, purpose, field_names).first
159
+ policy.validate_rights!(field_names, purpose, subject: key) if key
160
+ end
161
+
162
+ private
163
+
164
+ def pam_policy(model)
165
+ return nil unless model.respond_to?(:lyra_monitored) && model.lyra_monitored
166
+
167
+ policy = Lyra::Privacy.policy_for(model)
168
+ policy.respond_to?(:pam_policy) ? policy.pam_policy : nil
169
+ end
170
+
171
+ CACHE_MUTEX = Mutex.new
172
+
173
+ # The keys in force that a query on +model+ can reach, and whose
174
+ # requests would refuse it. Remembered per (model, policy, purpose,
175
+ # fields) for as long as the store's snapshot is the same object
176
+ # (Store#snapshot_token): a change to the rights in force, here or
177
+ # read from another process, starts afresh. Not remembered for a
178
+ # purpose with the :consent exception (consent can change without
179
+ # the rights changing), nor while this connection's transaction has
180
+ # rights of its own.
181
+ def refused_keys(model, policy, purpose, field_names)
182
+ fields = Array(field_names).map(&:to_sym).uniq.sort
183
+ store = PamDsl.rights_store
184
+ token = store.respond_to?(:snapshot_token) ? store.snapshot_token : nil
185
+ declared = policy.purposes[purpose.to_sym]
186
+ return compute_refused(model, policy, purpose, fields) if token.nil? || declared&.continues_under_restriction?(:consent)
187
+
188
+ cache_key = [model, policy.object_id, purpose.to_sym, fields]
189
+ cache = @refused_cache
190
+ unless cache && cache[:token].equal?(token)
191
+ cache = CACHE_MUTEX.synchronize { @refused_cache = { token: token, entries: {} } }
192
+ end
193
+ cached = cache[:entries][cache_key]
194
+ return cached if cached
195
+
196
+ refused = compute_refused(model, policy, purpose, fields).freeze
197
+ CACHE_MUTEX.synchronize { cache[:entries][cache_key] = refused }
198
+ refused
199
+ end
200
+
201
+ # build, remembered with the refused keys it was built from (a frozen
202
+ # array refused_keys remembers, so the same object while the snapshot
203
+ # is).
204
+ def built(model, refused)
205
+ cache = @refused_cache
206
+ return build(model, refused) unless refused.frozen? && cache
207
+
208
+ cache_key = [:built, model, refused.object_id]
209
+ cached = cache[:entries][cache_key]
210
+ return cached if cached
211
+
212
+ exclusion = build(model, refused)
213
+ return nil unless exclusion
214
+
215
+ exclusion.sql
216
+ CACHE_MUTEX.synchronize { cache[:entries][cache_key] = exclusion }
217
+ end
218
+
219
+ # One pass over the requests in force: those that could cover the
220
+ # access, by subject, each subject then decided on its own requests
221
+ # (the :consent exception asks its consent).
222
+ def compute_refused(model, policy, purpose, fields)
223
+ declared = policy.purposes[purpose.to_sym] || PamDsl::Purpose.new(purpose.to_sym)
224
+ types = reachable_types(model)
225
+ requests = PamDsl::Rights.in_force_requests(PamDsl.rights_store)
226
+ candidates = PamDsl::Rights.applicable(requests, policy, declared, fields).map(&:subject_key).uniq
227
+ return [] if candidates.empty?
228
+
229
+ by_key = requests.group_by(&:subject_key)
230
+ candidates.select do |key|
231
+ type, = Rights.split_key(key)
232
+ type && (types.nil? || types.include?(type)) &&
233
+ PamDsl::Rights.violations_among(by_key.fetch(key, []), policy, declared, fields, key).any?
234
+ end
235
+ end
236
+
237
+ # The subject types whose keys a row of +model+ can name: its own,
238
+ # and its links' (nil when a polymorphic link can name any).
239
+ def reachable_types(model)
240
+ plan = SubjectKeys.links(model)
241
+ return nil if plan.polymorphic.any?
242
+
243
+ [Lyra.stream_owner_name(model), *plan.associations.map(&:last), *plan.convention.map(&:last)].uniq
244
+ end
245
+
246
+ def build(model, keys)
247
+ own_type = Lyra.stream_owner_name(model)
248
+ plan = SubjectKeys.links(model)
249
+ own = []
250
+ by_type = Hash.new { |hash, type| hash[type] = [] }
251
+ keys.each do |key|
252
+ type, id = Rights.split_key(key)
253
+ type == own_type ? own << id : by_type[type] << id
254
+ end
255
+ links = (plan.associations + plan.convention).filter_map do |column, type|
256
+ [column, by_type[type]] if type != own_type && by_type.key?(type)
257
+ end
258
+ polymorphic = plan.polymorphic.flat_map do |column, type_column|
259
+ by_type.filter_map { |type, ids| [column, type_column, type, ids] unless type == own_type }
260
+ end
261
+ # Keys of types no column of the model holds name none of its rows.
262
+ return nil if own.empty? && links.empty? && polymorphic.empty?
263
+
264
+ Exclusion.new(model: model, own: own, links: links, polymorphic: polymorphic, keys: keys)
265
+ end
266
+ end
267
+ end
268
+ end
269
+ end