orfeas_lyra 0.9.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +2021 -17
- data/README.md +22 -9
- data/Rakefile +40 -1
- data/app/controllers/lyra/application_controller.rb +127 -0
- data/app/controllers/lyra/dashboard_controller.rb +33 -31
- data/app/controllers/lyra/flow_controller.rb +32 -34
- data/app/controllers/lyra/privacy_controller.rb +57 -18
- data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
- data/app/views/lyra/dashboard/index.html.erb +1 -0
- data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
- data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
- data/app/views/lyra/privacy/policy.html.erb +7 -0
- data/app/views/lyra/privacy/rights.html.erb +78 -0
- data/config/privacy_policies.rb +0 -3
- data/config/routes.rb +1 -0
- data/lib/lyra/access_log.rb +187 -6
- data/lib/lyra/advisory_lock.rb +236 -0
- data/lib/lyra/aggregate.rb +1 -1
- data/lib/lyra/associations/habtm_links.rb +382 -19
- data/lib/lyra/associations/maintenance.rb +229 -0
- data/lib/lyra/attribution.rb +48 -0
- data/lib/lyra/bypass_events.rb +446 -46
- data/lib/lyra/command_handler.rb +19 -14
- data/lib/lyra/configuration.rb +237 -10
- data/lib/lyra/consistency/read_your_writes.rb +4 -2
- data/lib/lyra/correlation.rb +121 -0
- data/lib/lyra/domain_events.rb +4 -0
- data/lib/lyra/dual_view.rb +53 -16
- data/lib/lyra/encrypted_attributes.rb +5 -0
- data/lib/lyra/engine.rb +67 -0
- data/lib/lyra/erasure.rb +1011 -106
- data/lib/lyra/event.rb +35 -1
- data/lib/lyra/event_data.rb +38 -0
- data/lib/lyra/event_flow.rb +31 -15
- data/lib/lyra/event_store_adapter.rb +28 -0
- data/lib/lyra/event_window.rb +58 -0
- data/lib/lyra/genesis.rb +548 -44
- data/lib/lyra/interceptors/association_interceptor.rb +689 -34
- data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
- data/lib/lyra/internal.rb +26 -0
- data/lib/lyra/mode_sync.rb +51 -3
- data/lib/lyra/mode_transition.rb +257 -30
- data/lib/lyra/model_selection.rb +35 -0
- data/lib/lyra/own_connection.rb +17 -2
- data/lib/lyra/privacy/adapters/pam.rb +25 -2
- data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
- data/lib/lyra/privacy/interface.rb +27 -7
- data/lib/lyra/privacy/subject_links.rb +143 -0
- data/lib/lyra/privacy/subject_records.rb +369 -0
- data/lib/lyra/projection.rb +7 -1
- data/lib/lyra/projections/async_checkpoint.rb +32 -0
- data/lib/lyra/projections/async_projection_job.rb +166 -23
- data/lib/lyra/projections/cached_joins.rb +33 -18
- data/lib/lyra/projections/cached_projection.rb +74 -33
- data/lib/lyra/projections/cached_relation.rb +3175 -321
- data/lib/lyra/projections/default_scopes.rb +254 -0
- data/lib/lyra/projections/deferred_relation.rb +23 -3
- data/lib/lyra/projections/event_store_reader.rb +47 -8
- data/lib/lyra/projections/foreign_keys.rb +92 -0
- data/lib/lyra/projections/lazy_projection.rb +165 -27
- data/lib/lyra/projections/model_projection.rb +165 -14
- data/lib/lyra/projections/rebuild.rb +34 -6
- data/lib/lyra/projections/reference_links.rb +463 -72
- data/lib/lyra/projections/typed_values.rb +629 -0
- data/lib/lyra/projections/unique_indexes.rb +595 -0
- data/lib/lyra/projections/uniqueness.rb +784 -0
- data/lib/lyra/projections/value_digest.rb +93 -0
- data/lib/lyra/purpose_bound_reads.rb +912 -77
- data/lib/lyra/repair.rb +119 -28
- data/lib/lyra/retention.rb +114 -8
- data/lib/lyra/rights/consent_store.rb +415 -0
- data/lib/lyra/rights/erasure_guard.rb +135 -0
- data/lib/lyra/rights/pending.rb +74 -0
- data/lib/lyra/rights/queries.rb +269 -0
- data/lib/lyra/rights/store.rb +512 -0
- data/lib/lyra/rights/subject_keys.rb +224 -0
- data/lib/lyra/rights.rb +281 -0
- data/lib/lyra/saves_without_effect.rb +69 -0
- data/lib/lyra/stream_tail.rb +319 -0
- data/lib/lyra/strict_data_access.rb +916 -106
- data/lib/lyra/temporal.rb +4 -2
- data/lib/lyra/touch_registry.rb +404 -0
- data/lib/lyra/verification/bypass_workflow.rb +452 -42
- data/lib/lyra/verification/rights_workflows.rb +170 -0
- data/lib/lyra/version.rb +1 -1
- data/lib/lyra/visualization/event_graph.rb +1 -1
- data/lib/lyra/visualization/timeline.rb +2 -2
- data/lib/lyra.rb +43 -2
- data/lib/tasks/lyra_mode.rake +110 -8
- data/lib/tasks/lyra_projections.rake +43 -2
- metadata +34 -4
- data/lib/lyra/associations/event_aware.rb +0 -224
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Lyra
|
|
4
|
+
module Rights
|
|
5
|
+
# The subject's rights in queries over many people (Lyra::PurposeBoundReads).
|
|
6
|
+
#
|
|
7
|
+
# Condition 6 of Def. 1 is per subject, so a query whose subject is the
|
|
8
|
+
# whole model ("Model$*": an aggregate, pluck, ids, exists?) cannot be
|
|
9
|
+
# checked against one person's restriction. Refusing it would let one
|
|
10
|
+
# person's request stop everyone's statistics; running it as it is would
|
|
11
|
+
# process the restricted person's data. Such a query leaves the restricted
|
|
12
|
+
# subjects out: under a purpose, when a request in force would refuse a
|
|
13
|
+
# read of a subject's record (PamDsl::Policy#rights_violations: a
|
|
14
|
+
# restriction of all of the subject's data, or of a field the query names,
|
|
15
|
+
# and no Art. 18(2) exception of the purpose that applies; a pending
|
|
16
|
+
# objection's restriction of this purpose; an upheld objection to it), the
|
|
17
|
+
# query runs with that subject's rows excluded:
|
|
18
|
+
#
|
|
19
|
+
# - a record key (EnfArticle$7): the record's own key column NOT IN (...);
|
|
20
|
+
# - a person key (Payer$7) the model links to (Rights::SubjectKeys: a
|
|
21
|
+
# belongs_to, a polymorphic belongs_to, a <type>_id column of a
|
|
22
|
+
# monitored model): (payer_id IS NULL OR payer_id NOT IN (...)), and for
|
|
23
|
+
# a polymorphic link (owner_id IS NULL OR owner_type IS NULL OR
|
|
24
|
+
# owner_type NOT IN ('Payer', <its STI subclasses>) OR owner_id NOT IN
|
|
25
|
+
# (...)), so rows that name no one are kept, as SQL's NOT IN alone would
|
|
26
|
+
# not keep them. The type column may hold an STI subclass's name; the
|
|
27
|
+
# subclasses are those loaded (eager loading, as in production).
|
|
28
|
+
#
|
|
29
|
+
# ES-NoProj's relations (Projections::CachedRelation) hold their records
|
|
30
|
+
# in memory: a record is left out when any of its keys (its own, and its
|
|
31
|
+
# people's, by the same resolver) is refused; a record whose links cannot
|
|
32
|
+
# be read raises Rights::UnresolvedSubjectError (fail closed).
|
|
33
|
+
#
|
|
34
|
+
# A join, subquery or CTE that reaches a model with such a subject is
|
|
35
|
+
# refused rather than filtered (refuse_reaching!): the excluded rows would
|
|
36
|
+
# have to be traced through the join.
|
|
37
|
+
#
|
|
38
|
+
# In audit mode nothing is left out: the query runs whole and the log says
|
|
39
|
+
# how many restricted subjects it may include. With nothing in force
|
|
40
|
+
# anywhere, none of this does any work beyond the store's any_active? (a
|
|
41
|
+
# clock read, once its scope has read the rights).
|
|
42
|
+
#
|
|
43
|
+
# Model.lyra_processable gives a batch job the records the purpose in
|
|
44
|
+
# scope may process: the same exclusion, for every field the policy
|
|
45
|
+
# declares unless fields: names the ones the job reads, in every
|
|
46
|
+
# enforcement mode.
|
|
47
|
+
module Queries
|
|
48
|
+
# The subjects a query on +model+ leaves out: +own+ (ids of the model's
|
|
49
|
+
# own records), +links+ ([[column, ids]]), +polymorphic+
|
|
50
|
+
# ([[id column, type column, type, ids]]) and +keys+ (the subject keys).
|
|
51
|
+
Exclusion = Struct.new(:model, :own, :links, :polymorphic, :keys, keyword_init: true) do
|
|
52
|
+
# The condition the rows kept satisfy.
|
|
53
|
+
def arel
|
|
54
|
+
table = model.arel_table
|
|
55
|
+
conditions = []
|
|
56
|
+
conditions << table[Lyra.key_column(model)].not_in(own) if own.any?
|
|
57
|
+
links.each { |column, ids| conditions << table[column].eq(nil).or(table[column].not_in(ids)) }
|
|
58
|
+
polymorphic.each do |column, type_column, type, ids|
|
|
59
|
+
conditions << table[column].eq(nil).or(table[type_column].eq(nil))
|
|
60
|
+
.or(table[type_column].not_in(Queries.type_names(type)))
|
|
61
|
+
.or(table[column].not_in(ids))
|
|
62
|
+
end
|
|
63
|
+
conditions.reduce { |all, condition| all.and(condition) }
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# The condition as SQL, compiled once (a list of hundreds of ids is
|
|
67
|
+
# costly to build and compile on every query).
|
|
68
|
+
def sql
|
|
69
|
+
@sql ||= Arel.sql("(#{arel.to_sql})")
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def keeps?(record)
|
|
73
|
+
!SubjectKeys.keys_for(record).intersect?(keys)
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
class << self
|
|
78
|
+
# The names a polymorphic type column may hold for the owner +type+:
|
|
79
|
+
# its own and its STI subclasses'.
|
|
80
|
+
def type_names(type)
|
|
81
|
+
klass = type.to_s.safe_constantize
|
|
82
|
+
return [type.to_s] unless klass.is_a?(Class) && klass < ActiveRecord::Base
|
|
83
|
+
|
|
84
|
+
([klass] + klass.descendants).map(&:name).compact.uniq
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Is any request in force anywhere (PamDsl's store)?
|
|
88
|
+
def in_force?
|
|
89
|
+
PamDsl.rights_store.any_active?
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# The Exclusion for a query on +model+ naming +field_names+ under
|
|
93
|
+
# +purpose+ (default: the one in scope), or nil when no subject is
|
|
94
|
+
# left out. +audit+: in audit mode, log and return nil.
|
|
95
|
+
def exclusion(model, field_names, purpose: PurposeBoundReads.current, audit: true)
|
|
96
|
+
return nil unless purpose && in_force?
|
|
97
|
+
|
|
98
|
+
policy = pam_policy(model)
|
|
99
|
+
return nil unless policy
|
|
100
|
+
|
|
101
|
+
refused = refused_keys(model, policy, purpose, field_names)
|
|
102
|
+
return nil if refused.empty?
|
|
103
|
+
|
|
104
|
+
if audit && policy.enforcement_mode == :audit
|
|
105
|
+
PamDsl.logger.warn("Lyra: #{model.name} query under purpose '#{purpose}' includes the rows of " \
|
|
106
|
+
"#{refused.size} restricted subject#{'s' unless refused.size == 1} (audit mode: " \
|
|
107
|
+
"not left out)")
|
|
108
|
+
return nil
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
built(model, refused)
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# +relation+ (an Active Record relation) leaving out the restricted
|
|
115
|
+
# subjects, or nil when there are none.
|
|
116
|
+
def filter(relation, field_names)
|
|
117
|
+
exclusion = exclusion(relation.klass, field_names)
|
|
118
|
+
exclusion && relation.where(exclusion.sql)
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# The records of +records+ a query on +model+ keeps, or nil when it
|
|
122
|
+
# keeps them all.
|
|
123
|
+
def kept_records(model, field_names, records)
|
|
124
|
+
exclusion = exclusion(model, field_names)
|
|
125
|
+
exclusion && records.select { |record| exclusion.keeps?(record) }
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# Model.lyra_processable: +relation+ without the records the purpose
|
|
129
|
+
# in scope may not process (for +fields+, default every declared
|
|
130
|
+
# field). Any relation with no purpose in scope.
|
|
131
|
+
def processable(relation, fields = nil)
|
|
132
|
+
model = relation.respond_to?(:model_class) ? relation.model_class : relation.klass
|
|
133
|
+
policy = PurposeBoundReads.current && pam_policy(model)
|
|
134
|
+
return relation unless policy
|
|
135
|
+
|
|
136
|
+
fields = fields.nil? ? policy.fields.keys : Array(fields).map(&:to_sym)
|
|
137
|
+
exclusion = exclusion(model, fields, audit: false)
|
|
138
|
+
return relation unless exclusion
|
|
139
|
+
|
|
140
|
+
if relation.is_a?(Projections::CachedRelation)
|
|
141
|
+
relation.lyra_select_records { |record| exclusion.keeps?(record) }
|
|
142
|
+
else
|
|
143
|
+
relation.where(exclusion.sql)
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# A join, subquery or CTE reaching +model+ (naming +field_names+ of
|
|
148
|
+
# it) under the purpose in scope: refused when it would reach a
|
|
149
|
+
# subject left out of a query on +model+, as Condition 6 refuses a
|
|
150
|
+
# read (strict mode raises, audit mode logs).
|
|
151
|
+
def refuse_reaching!(model, field_names)
|
|
152
|
+
purpose = PurposeBoundReads.current
|
|
153
|
+
return unless purpose && in_force?
|
|
154
|
+
|
|
155
|
+
policy = pam_policy(model)
|
|
156
|
+
return unless policy
|
|
157
|
+
|
|
158
|
+
key = refused_keys(model, policy, purpose, field_names).first
|
|
159
|
+
policy.validate_rights!(field_names, purpose, subject: key) if key
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
private
|
|
163
|
+
|
|
164
|
+
def pam_policy(model)
|
|
165
|
+
return nil unless model.respond_to?(:lyra_monitored) && model.lyra_monitored
|
|
166
|
+
|
|
167
|
+
policy = Lyra::Privacy.policy_for(model)
|
|
168
|
+
policy.respond_to?(:pam_policy) ? policy.pam_policy : nil
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
CACHE_MUTEX = Mutex.new
|
|
172
|
+
|
|
173
|
+
# The keys in force that a query on +model+ can reach, and whose
|
|
174
|
+
# requests would refuse it. Remembered per (model, policy, purpose,
|
|
175
|
+
# fields) for as long as the store's snapshot is the same object
|
|
176
|
+
# (Store#snapshot_token): a change to the rights in force, here or
|
|
177
|
+
# read from another process, starts afresh. Not remembered for a
|
|
178
|
+
# purpose with the :consent exception (consent can change without
|
|
179
|
+
# the rights changing), nor while this connection's transaction has
|
|
180
|
+
# rights of its own.
|
|
181
|
+
def refused_keys(model, policy, purpose, field_names)
|
|
182
|
+
fields = Array(field_names).map(&:to_sym).uniq.sort
|
|
183
|
+
store = PamDsl.rights_store
|
|
184
|
+
token = store.respond_to?(:snapshot_token) ? store.snapshot_token : nil
|
|
185
|
+
declared = policy.purposes[purpose.to_sym]
|
|
186
|
+
return compute_refused(model, policy, purpose, fields) if token.nil? || declared&.continues_under_restriction?(:consent)
|
|
187
|
+
|
|
188
|
+
cache_key = [model, policy.object_id, purpose.to_sym, fields]
|
|
189
|
+
cache = @refused_cache
|
|
190
|
+
unless cache && cache[:token].equal?(token)
|
|
191
|
+
cache = CACHE_MUTEX.synchronize { @refused_cache = { token: token, entries: {} } }
|
|
192
|
+
end
|
|
193
|
+
cached = cache[:entries][cache_key]
|
|
194
|
+
return cached if cached
|
|
195
|
+
|
|
196
|
+
refused = compute_refused(model, policy, purpose, fields).freeze
|
|
197
|
+
CACHE_MUTEX.synchronize { cache[:entries][cache_key] = refused }
|
|
198
|
+
refused
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
# build, remembered with the refused keys it was built from (a frozen
|
|
202
|
+
# array refused_keys remembers, so the same object while the snapshot
|
|
203
|
+
# is).
|
|
204
|
+
def built(model, refused)
|
|
205
|
+
cache = @refused_cache
|
|
206
|
+
return build(model, refused) unless refused.frozen? && cache
|
|
207
|
+
|
|
208
|
+
cache_key = [:built, model, refused.object_id]
|
|
209
|
+
cached = cache[:entries][cache_key]
|
|
210
|
+
return cached if cached
|
|
211
|
+
|
|
212
|
+
exclusion = build(model, refused)
|
|
213
|
+
return nil unless exclusion
|
|
214
|
+
|
|
215
|
+
exclusion.sql
|
|
216
|
+
CACHE_MUTEX.synchronize { cache[:entries][cache_key] = exclusion }
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
# One pass over the requests in force: those that could cover the
|
|
220
|
+
# access, by subject, each subject then decided on its own requests
|
|
221
|
+
# (the :consent exception asks its consent).
|
|
222
|
+
def compute_refused(model, policy, purpose, fields)
|
|
223
|
+
declared = policy.purposes[purpose.to_sym] || PamDsl::Purpose.new(purpose.to_sym)
|
|
224
|
+
types = reachable_types(model)
|
|
225
|
+
requests = PamDsl::Rights.in_force_requests(PamDsl.rights_store)
|
|
226
|
+
candidates = PamDsl::Rights.applicable(requests, policy, declared, fields).map(&:subject_key).uniq
|
|
227
|
+
return [] if candidates.empty?
|
|
228
|
+
|
|
229
|
+
by_key = requests.group_by(&:subject_key)
|
|
230
|
+
candidates.select do |key|
|
|
231
|
+
type, = Rights.split_key(key)
|
|
232
|
+
type && (types.nil? || types.include?(type)) &&
|
|
233
|
+
PamDsl::Rights.violations_among(by_key.fetch(key, []), policy, declared, fields, key).any?
|
|
234
|
+
end
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
# The subject types whose keys a row of +model+ can name: its own,
|
|
238
|
+
# and its links' (nil when a polymorphic link can name any).
|
|
239
|
+
def reachable_types(model)
|
|
240
|
+
plan = SubjectKeys.links(model)
|
|
241
|
+
return nil if plan.polymorphic.any?
|
|
242
|
+
|
|
243
|
+
[Lyra.stream_owner_name(model), *plan.associations.map(&:last), *plan.convention.map(&:last)].uniq
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
def build(model, keys)
|
|
247
|
+
own_type = Lyra.stream_owner_name(model)
|
|
248
|
+
plan = SubjectKeys.links(model)
|
|
249
|
+
own = []
|
|
250
|
+
by_type = Hash.new { |hash, type| hash[type] = [] }
|
|
251
|
+
keys.each do |key|
|
|
252
|
+
type, id = Rights.split_key(key)
|
|
253
|
+
type == own_type ? own << id : by_type[type] << id
|
|
254
|
+
end
|
|
255
|
+
links = (plan.associations + plan.convention).filter_map do |column, type|
|
|
256
|
+
[column, by_type[type]] if type != own_type && by_type.key?(type)
|
|
257
|
+
end
|
|
258
|
+
polymorphic = plan.polymorphic.flat_map do |column, type_column|
|
|
259
|
+
by_type.filter_map { |type, ids| [column, type_column, type, ids] unless type == own_type }
|
|
260
|
+
end
|
|
261
|
+
# Keys of types no column of the model holds name none of its rows.
|
|
262
|
+
return nil if own.empty? && links.empty? && polymorphic.empty?
|
|
263
|
+
|
|
264
|
+
Exclusion.new(model: model, own: own, links: links, polymorphic: polymorphic, keys: keys)
|
|
265
|
+
end
|
|
266
|
+
end
|
|
267
|
+
end
|
|
268
|
+
end
|
|
269
|
+
end
|