orfeas_lyra 0.9.3 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +1923 -17
- data/README.md +15 -8
- data/Rakefile +40 -1
- data/app/controllers/lyra/application_controller.rb +108 -0
- data/app/controllers/lyra/dashboard_controller.rb +17 -15
- data/app/controllers/lyra/flow_controller.rb +30 -33
- data/app/controllers/lyra/privacy_controller.rb +57 -18
- data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
- data/app/views/lyra/dashboard/index.html.erb +1 -0
- data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
- data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
- data/app/views/lyra/privacy/policy.html.erb +7 -0
- data/app/views/lyra/privacy/rights.html.erb +78 -0
- data/config/privacy_policies.rb +0 -3
- data/config/routes.rb +1 -0
- data/lib/lyra/access_log.rb +187 -6
- data/lib/lyra/advisory_lock.rb +236 -0
- data/lib/lyra/aggregate.rb +1 -1
- data/lib/lyra/associations/habtm_links.rb +382 -19
- data/lib/lyra/associations/maintenance.rb +229 -0
- data/lib/lyra/attribution.rb +48 -0
- data/lib/lyra/bypass_events.rb +426 -45
- data/lib/lyra/command_handler.rb +19 -14
- data/lib/lyra/configuration.rb +237 -10
- data/lib/lyra/consistency/read_your_writes.rb +4 -2
- data/lib/lyra/correlation.rb +121 -0
- data/lib/lyra/domain_events.rb +4 -0
- data/lib/lyra/dual_view.rb +53 -16
- data/lib/lyra/encrypted_attributes.rb +5 -0
- data/lib/lyra/engine.rb +67 -0
- data/lib/lyra/erasure.rb +1002 -116
- data/lib/lyra/event.rb +35 -1
- data/lib/lyra/event_data.rb +38 -0
- data/lib/lyra/event_flow.rb +31 -15
- data/lib/lyra/event_store_adapter.rb +28 -0
- data/lib/lyra/event_window.rb +58 -0
- data/lib/lyra/genesis.rb +548 -44
- data/lib/lyra/interceptors/association_interceptor.rb +689 -34
- data/lib/lyra/interceptors/crud_interceptor.rb +547 -95
- data/lib/lyra/internal.rb +26 -0
- data/lib/lyra/mode_sync.rb +51 -3
- data/lib/lyra/mode_transition.rb +245 -34
- data/lib/lyra/model_selection.rb +35 -0
- data/lib/lyra/own_connection.rb +17 -2
- data/lib/lyra/privacy/adapters/pam.rb +25 -2
- data/lib/lyra/privacy/gdpr_compliance.rb +264 -36
- data/lib/lyra/privacy/interface.rb +27 -7
- data/lib/lyra/privacy/subject_links.rb +143 -0
- data/lib/lyra/privacy/subject_records.rb +369 -0
- data/lib/lyra/projection.rb +7 -1
- data/lib/lyra/projections/async_checkpoint.rb +32 -0
- data/lib/lyra/projections/async_projection_job.rb +166 -23
- data/lib/lyra/projections/cached_joins.rb +33 -18
- data/lib/lyra/projections/cached_projection.rb +74 -33
- data/lib/lyra/projections/cached_relation.rb +3175 -321
- data/lib/lyra/projections/default_scopes.rb +254 -0
- data/lib/lyra/projections/deferred_relation.rb +23 -3
- data/lib/lyra/projections/event_store_reader.rb +47 -8
- data/lib/lyra/projections/foreign_keys.rb +92 -0
- data/lib/lyra/projections/lazy_projection.rb +165 -27
- data/lib/lyra/projections/model_projection.rb +165 -14
- data/lib/lyra/projections/rebuild.rb +34 -6
- data/lib/lyra/projections/reference_links.rb +463 -72
- data/lib/lyra/projections/typed_values.rb +629 -0
- data/lib/lyra/projections/unique_indexes.rb +595 -0
- data/lib/lyra/projections/uniqueness.rb +784 -0
- data/lib/lyra/projections/value_digest.rb +93 -0
- data/lib/lyra/purpose_bound_reads.rb +912 -77
- data/lib/lyra/repair.rb +119 -28
- data/lib/lyra/retention.rb +114 -8
- data/lib/lyra/rights/consent_store.rb +415 -0
- data/lib/lyra/rights/erasure_guard.rb +135 -0
- data/lib/lyra/rights/pending.rb +74 -0
- data/lib/lyra/rights/queries.rb +269 -0
- data/lib/lyra/rights/store.rb +512 -0
- data/lib/lyra/rights/subject_keys.rb +224 -0
- data/lib/lyra/rights.rb +281 -0
- data/lib/lyra/saves_without_effect.rb +69 -0
- data/lib/lyra/stream_tail.rb +319 -0
- data/lib/lyra/strict_data_access.rb +906 -107
- data/lib/lyra/temporal.rb +4 -2
- data/lib/lyra/touch_registry.rb +404 -0
- data/lib/lyra/verification/bypass_workflow.rb +452 -42
- data/lib/lyra/verification/rights_workflows.rb +170 -0
- data/lib/lyra/version.rb +1 -1
- data/lib/lyra/visualization/event_graph.rb +1 -1
- data/lib/lyra/visualization/timeline.rb +2 -2
- data/lib/lyra.rb +43 -2
- data/lib/tasks/lyra_mode.rake +107 -8
- data/lib/tasks/lyra_projections.rake +43 -2
- metadata +34 -4
- data/lib/lyra/associations/event_aware.rb +0 -224
data/README.md
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# Lyra
|
|
2
2
|
|
|
3
|
+
[](https://github.com/mpantel/lyra-engine/actions/workflows/test.yml)
|
|
4
|
+
|
|
3
5
|
**CRUD to Event Sourcing Transformation Engine**
|
|
4
6
|
|
|
5
7
|
*Part of ORFEAS (Object-Relational Framework for Event-Aware Systems)*
|
|
@@ -35,10 +37,14 @@ seven configurations:
|
|
|
35
37
|
`delete`, `update_all`, `delete_all`, `insert_all`, `insert_all!`, `upsert_all`,
|
|
36
38
|
and the updates of `dependent: :nullify`) on a monitored model publish bypass
|
|
37
39
|
events; with `config.strict_data_access = true` they raise instead (except
|
|
38
|
-
`touch` and the `dependent: :nullify`
|
|
40
|
+
`touch` and Rails' own association writes, such as the `dependent: :nullify`
|
|
41
|
+
updates or removing a `has_and_belongs_to_many` link; the same writes made by
|
|
42
|
+
the application raise).
|
|
39
43
|
- Each record has its own stream, named `"Model$id"` (for example `"Order$42"`).
|
|
40
44
|
- **ES-NoProj** writes no rows; it answers a query exactly from the events or
|
|
41
|
-
raises `Lyra::Projections::UnsupportedQuery`.
|
|
45
|
+
raises `Lyra::Projections::UnsupportedQuery`. Since it writes no rows to
|
|
46
|
+
monitored tables, the switch into it is refused while an unmonitored table
|
|
47
|
+
has a database foreign key into a monitored one.
|
|
42
48
|
- **Switching modes after boot is gated**: `Lyra::ModeTransition.to!` (and
|
|
43
49
|
`config.enable_hijack!`, `enable_monitor!`, `enable_event_sourcing!`) checks
|
|
44
50
|
that rows and events agree before a switch that changes the authoritative
|
|
@@ -48,18 +54,19 @@ seven configurations:
|
|
|
48
54
|
|
|
49
55
|
## Requirements and installation
|
|
50
56
|
|
|
51
|
-
- Ruby >= 4.0, Rails
|
|
57
|
+
- Ruby >= 4.0, Rails 8.1 (`>= 8.1, < 8.2`; tested on 8.1.4.1), `rails_event_store` ~> 3.0
|
|
58
|
+
(from `lyra.gemspec`). CI tests Ruby 4.0 on PostgreSQL 18.7.
|
|
52
59
|
- PostgreSQL. Lyra depends on the `pg` gem; the advisory locks behind Genesis
|
|
53
60
|
and ES-Lazy, and id reservation in Hijack mode, need PostgreSQL.
|
|
54
61
|
|
|
55
|
-
The gems are on RubyGems
|
|
56
|
-
`
|
|
62
|
+
The gems are on RubyGems, all three at 1.0.0 (22 November 2026):
|
|
63
|
+
`orfeas_lyra`, `orfeas_pam_dsl` and `orfeas_petri_flow`. Their names differ from the files to require:
|
|
57
64
|
|
|
58
65
|
```ruby
|
|
59
66
|
# Gemfile
|
|
60
|
-
gem "orfeas_lyra", "~> 0
|
|
61
|
-
gem "orfeas_pam_dsl", "~> 0
|
|
62
|
-
gem "orfeas_petri_flow", "~> 0
|
|
67
|
+
gem "orfeas_lyra", "~> 1.0", require: "lyra"
|
|
68
|
+
gem "orfeas_pam_dsl", "~> 1.0", require: "pam_dsl" # optional, privacy features
|
|
69
|
+
gem "orfeas_petri_flow", "~> 1.0", require: "petri_flow" # optional, verification
|
|
63
70
|
```
|
|
64
71
|
|
|
65
72
|
Create the event store tables with the RailsEventStore 3 generator
|
data/Rakefile
CHANGED
|
@@ -42,13 +42,52 @@ namespace :test do
|
|
|
42
42
|
Rake::Task["test"].invoke
|
|
43
43
|
|
|
44
44
|
puts "\n=== Running PAM DSL tests ==="
|
|
45
|
-
|
|
45
|
+
# Twice: with ActiveSupport, then with the ActiveSupport-free polyfill (test:both).
|
|
46
|
+
Dir.chdir("gems/pam_dsl") { system("rake test:both") || raise("PAM DSL tests failed") }
|
|
46
47
|
|
|
47
48
|
puts "\n=== Running PetriFlow tests ==="
|
|
48
49
|
Dir.chdir("gems/petri_flow") { system("rake test") || raise("PetriFlow tests failed") }
|
|
49
50
|
|
|
50
51
|
puts "\n=== All tests passed! ==="
|
|
51
52
|
end
|
|
53
|
+
|
|
54
|
+
# The Solidus case study's specs, in its own bundle: spec/olist (the Olist
|
|
55
|
+
# replay in every mode) and the rest of its suite, less the JavaScript
|
|
56
|
+
# system examples, which need a Chrome (bin/rspec-chrome-docker runs them;
|
|
57
|
+
# SOLIDUS_SPECS_JS=1 includes them here). On a database of their own on the
|
|
58
|
+
# Lyra test server (SOLIDUS_DATABASE_URL), dropped and loaded from
|
|
59
|
+
# db/schema.rb first. SOLIDUS_SPECS names spec files or directories (default:
|
|
60
|
+
# every spec). The system examples rebuild app/assets/builds/tailwind.css,
|
|
61
|
+
# which is committed: it is put back as it was, so that the tree stays clean
|
|
62
|
+
# (the benchmark campaign refuses, and its reports record, a dirty tree).
|
|
63
|
+
desc "Run the Solidus case study's specs (the Olist replay and the rest, without JavaScript)"
|
|
64
|
+
task :solidus do
|
|
65
|
+
dir = File.expand_path("examples/solidus_case_study", __dir__)
|
|
66
|
+
abort "The Solidus case study is not in this checkout." unless File.directory?(dir)
|
|
67
|
+
|
|
68
|
+
env = {
|
|
69
|
+
"RAILS_ENV" => "test",
|
|
70
|
+
"DISABLE_DATABASE_ENVIRONMENT_CHECK" => "1",
|
|
71
|
+
"BUNDLE_GEMFILE" => File.join(dir, "Gemfile"),
|
|
72
|
+
"DATABASE_URL" => ENV.fetch("SOLIDUS_DATABASE_URL",
|
|
73
|
+
"postgres://postgres:postgres@localhost:5433/solidus_case_study_test")
|
|
74
|
+
}
|
|
75
|
+
css = File.join(dir, "app/assets/builds/tailwind.css")
|
|
76
|
+
committed = File.exist?(css) ? File.binread(css) : nil
|
|
77
|
+
args = ENV.fetch("SOLIDUS_SPECS", "").split
|
|
78
|
+
args = ["--tag", "~js", *args] unless ENV["SOLIDUS_SPECS_JS"] == "1"
|
|
79
|
+
begin
|
|
80
|
+
Bundler.with_unbundled_env do
|
|
81
|
+
Dir.chdir(dir) do
|
|
82
|
+
system(env, "bundle exec rails db:drop db:create db:schema:load") ||
|
|
83
|
+
raise("Solidus case study: the test database could not be prepared")
|
|
84
|
+
system(env, "bundle", "exec", "rspec", *args) || raise("Solidus case study specs failed")
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
ensure
|
|
88
|
+
File.binwrite(css, committed) if committed && File.binread(css) != committed
|
|
89
|
+
end
|
|
90
|
+
end
|
|
52
91
|
end
|
|
53
92
|
|
|
54
93
|
task default: :test
|
|
@@ -13,6 +13,19 @@ module Lyra
|
|
|
13
13
|
class UnknownModel < StandardError; end
|
|
14
14
|
rescue_from(UnknownModel) { head :not_found }
|
|
15
15
|
|
|
16
|
+
# The dashboard reads as the application does, under a purpose: the one the
|
|
17
|
+
# request names (?purpose=), else config.dashboard_purpose; with neither it
|
|
18
|
+
# reads without one, as config.reads_without_purpose decides. A read the
|
|
19
|
+
# policy refuses answers 403 and says how to name a purpose.
|
|
20
|
+
around_action :with_dashboard_purpose
|
|
21
|
+
READ_REFUSALS = [Lyra::PurposeBoundReads::PurposeRequiredError] +
|
|
22
|
+
%w[PurposeFieldMismatchError UndeclaredPurposeError ConsentRequiredError UndeterminedBasisError
|
|
23
|
+
SpecialCategoryViolationError CriminalDataViolationError InvalidFieldError
|
|
24
|
+
RestrictedProcessingError ObjectedProcessingError]
|
|
25
|
+
.filter_map { |name| PamDsl.const_get(name) if defined?(PamDsl) && PamDsl.const_defined?(name) } +
|
|
26
|
+
(defined?(Lyra::Rights::UnresolvedSubjectError) ? [Lyra::Rights::UnresolvedSubjectError] : [])
|
|
27
|
+
rescue_from(*READ_REFUSALS) { |error| refuse_read(error) }
|
|
28
|
+
|
|
16
29
|
helper_method :pam_dsl_available?, :user_tracking_configured?, :petri_flow_available?
|
|
17
30
|
|
|
18
31
|
# Check if PAM DSL is available for privacy features
|
|
@@ -32,6 +45,23 @@ module Lyra
|
|
|
32
45
|
|
|
33
46
|
private
|
|
34
47
|
|
|
48
|
+
def with_dashboard_purpose(&block)
|
|
49
|
+
purpose = params[:purpose].presence || Lyra.config.dashboard_purpose
|
|
50
|
+
return yield unless purpose
|
|
51
|
+
|
|
52
|
+
Lyra::PurposeBoundReads.with_purpose(purpose, &block)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def refuse_read(error)
|
|
56
|
+
message = "The dashboard read was refused: #{error.message}. Name a purpose the policy allows for it " \
|
|
57
|
+
"(?purpose=...), or set config.dashboard_purpose."
|
|
58
|
+
if request.format.json?
|
|
59
|
+
render json: { error: message }, status: :forbidden
|
|
60
|
+
else
|
|
61
|
+
render plain: message, status: :forbidden
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
35
65
|
# config.dashboard_authorization is evaluated with instance_exec on this
|
|
36
66
|
# controller, so its body may call the controller's methods directly,
|
|
37
67
|
# private ones included (request, session, cookies, params, and any
|
|
@@ -69,6 +99,84 @@ module Lyra
|
|
|
69
99
|
Lyra.config.monitored_model_named(name) || raise(UnknownModel, name.to_s)
|
|
70
100
|
end
|
|
71
101
|
|
|
102
|
+
# The most recent events, newest first, at most this request's ?limit=
|
|
103
|
+
# (default and ceiling: config.dashboard_max_events), access-log entries
|
|
104
|
+
# left out (Lyra::EventWindow): a dashboard request does not read the whole
|
|
105
|
+
# log. @window says what was read, for the response.
|
|
106
|
+
def recent_events
|
|
107
|
+
max = Lyra.config.dashboard_max_events
|
|
108
|
+
limit = bounded_param(:limit, default: max, max: max)
|
|
109
|
+
events, @window = Lyra::EventWindow.recent(limit)
|
|
110
|
+
rights_visible(events)
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# The subject's rights (Arts. 18 and 21) in the routes that show events
|
|
114
|
+
# or replayed state, under the dashboard's purpose: a route about one
|
|
115
|
+
# record is refused (403, as a read of the record is) when a restriction
|
|
116
|
+
# or upheld objection covers it; a route over many records leaves out
|
|
117
|
+
# the events of the records they cover, and says how many it left out in
|
|
118
|
+
# the X-Lyra-Rights-Withheld header. With no purpose, nothing in force, or
|
|
119
|
+
# the policy in audit mode, every event is shown.
|
|
120
|
+
def refuse_restricted_record!(model_class, id)
|
|
121
|
+
return unless rights_checked?
|
|
122
|
+
|
|
123
|
+
record = Lyra::Rights::SubjectKeys.find_record(model_class, id)
|
|
124
|
+
return unless record
|
|
125
|
+
|
|
126
|
+
policy = Lyra::Privacy.policy_for(model_class)
|
|
127
|
+
policy.validate_rights!(policy.declared_fields, Lyra::PurposeBoundReads.current, subject: record)
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def rights_visible(events)
|
|
131
|
+
return events unless rights_checked?
|
|
132
|
+
|
|
133
|
+
verdicts = {}
|
|
134
|
+
kept = events.reject do |event|
|
|
135
|
+
model, id = event_record(event)
|
|
136
|
+
next false unless model
|
|
137
|
+
|
|
138
|
+
verdicts.fetch([model, id]) { verdicts[[model, id]] = restricted_record?(model, id) }
|
|
139
|
+
end
|
|
140
|
+
withheld = events.size - kept.size
|
|
141
|
+
response.headers["X-Lyra-Rights-Withheld"] = withheld.to_s if withheld.positive?
|
|
142
|
+
kept
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
def rights_checked?
|
|
146
|
+
defined?(Lyra::Rights::Queries) && Lyra::PurposeBoundReads.current && Lyra::Rights::Queries.in_force?
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# The monitored model and id an event is about, or nil.
|
|
150
|
+
def event_record(event)
|
|
151
|
+
data = event.respond_to?(:data) && event.data.is_a?(Hash) ? event.data : {}
|
|
152
|
+
name = data[:model_class] || data["model_class"]
|
|
153
|
+
id = data[:model_id] || data["model_id"]
|
|
154
|
+
model = name && Lyra.config.monitored_model_named(name.to_s)
|
|
155
|
+
model && !id.nil? ? [model, id.to_s] : nil
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def restricted_record?(model, id)
|
|
159
|
+
policy = Lyra::Privacy.policy_for(model)
|
|
160
|
+
pam = policy.respond_to?(:pam_policy) ? policy.pam_policy : nil
|
|
161
|
+
return false unless pam && pam.enforcement_mode == :strict
|
|
162
|
+
|
|
163
|
+
record = Lyra::Rights::SubjectKeys.find_record(model, id)
|
|
164
|
+
return false unless record
|
|
165
|
+
|
|
166
|
+
pam.rights_violations(pam.fields.keys, Lyra::PurposeBoundReads.current, subject: record).any?
|
|
167
|
+
rescue Lyra::Rights::UnresolvedSubjectError
|
|
168
|
+
true
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
# Record a dashboard read in the access log (when config.record_access_events
|
|
172
|
+
# is on): which route, for what, how much came back, never a value.
|
|
173
|
+
def log_dashboard_read(route, scope = {}, returned: nil)
|
|
174
|
+
# The access log loads with PAM DSL; without it there is nothing to record to.
|
|
175
|
+
return unless defined?(Lyra::AccessLog)
|
|
176
|
+
|
|
177
|
+
Lyra::AccessLog.dashboard_read(route: route, scope: scope, returned: returned, truncated: @window&.dig(:truncated))
|
|
178
|
+
end
|
|
179
|
+
|
|
72
180
|
# An integer parameter, clamped to 1..max; the default when absent. Bounds
|
|
73
181
|
# the work a single dashboard request can ask for.
|
|
74
182
|
def bounded_param(key, default:, max:)
|
|
@@ -21,6 +21,7 @@ module Lyra
|
|
|
21
21
|
model_class = monitored_model!(params[:model_class])
|
|
22
22
|
model_id = params[:id]
|
|
23
23
|
|
|
24
|
+
# Under the request's purpose (ApplicationController#with_dashboard_purpose).
|
|
24
25
|
@comparison = DualView.new(model_class, model_id).compare
|
|
25
26
|
@audit_trail = DualView.new(model_class, model_id).audit_trail
|
|
26
27
|
@analysis = StateAnalyzer.analyze(model_class, model_id)
|
|
@@ -125,7 +126,7 @@ module Lyra
|
|
|
125
126
|
end
|
|
126
127
|
|
|
127
128
|
if @selected_model.present? && @selected_id.present?
|
|
128
|
-
@record = @model_class.find_by(
|
|
129
|
+
@record = @model_class.find_by(Lyra.key_column(@model_class) => @selected_id)
|
|
129
130
|
@audit_entries = AuditProjection.audit_trail(@model_class, @selected_id)
|
|
130
131
|
end
|
|
131
132
|
end
|
|
@@ -134,7 +135,7 @@ module Lyra
|
|
|
134
135
|
def audit_trail_for_record
|
|
135
136
|
@model_class = monitored_model!(params[:model_class])
|
|
136
137
|
@model_id = params[:id]
|
|
137
|
-
@record = @model_class.find_by(
|
|
138
|
+
@record = @model_class.find_by(Lyra.key_column(@model_class) => @model_id)
|
|
138
139
|
@audit_entries = AuditProjection.audit_trail(@model_class, @model_id)
|
|
139
140
|
@monitored_models = Lyra.config.monitored_models
|
|
140
141
|
|
|
@@ -171,6 +172,7 @@ module Lyra
|
|
|
171
172
|
model_class = params[:model_class]
|
|
172
173
|
model_id = params[:id]
|
|
173
174
|
depth = bounded_param(:depth, default: 1, max: 3) # How many related entities to include
|
|
175
|
+
refuse_restricted_record!(monitored_model!(model_class), model_id)
|
|
174
176
|
|
|
175
177
|
events = fetch_entity_events(model_class, model_id, depth: depth)
|
|
176
178
|
graph = Visualization::EventGraph.new(events)
|
|
@@ -190,7 +192,7 @@ module Lyra
|
|
|
190
192
|
models = model_filter.present? ? [monitored_model!(model_filter)] : Lyra.config.monitored_models
|
|
191
193
|
|
|
192
194
|
models.each do |model_class|
|
|
193
|
-
model_class.order(updated_at: :desc).limit(limit).each do |record|
|
|
195
|
+
model_class.lyra_processable.order(updated_at: :desc).limit(limit).each do |record|
|
|
194
196
|
stream_name = "#{model_class.name}$#{record.id}"
|
|
195
197
|
begin
|
|
196
198
|
event_count = Lyra.config.event_store.read.stream(stream_name).count
|
|
@@ -399,7 +401,7 @@ module Lyra
|
|
|
399
401
|
def collect_events_for_model(model_class)
|
|
400
402
|
# Collect all events for a model class
|
|
401
403
|
events = []
|
|
402
|
-
model_class.pluck(:id).each do |id|
|
|
404
|
+
model_class.lyra_processable.pluck(:id).each do |id|
|
|
403
405
|
stream_name = "#{model_class.name}$#{id}"
|
|
404
406
|
begin
|
|
405
407
|
events.concat(Lyra.config.event_store.read.stream(stream_name).to_a)
|
|
@@ -416,7 +418,7 @@ module Lyra
|
|
|
416
418
|
model_class.column_names.include?(field.to_s)
|
|
417
419
|
end
|
|
418
420
|
|
|
419
|
-
scope = model_class.order(updated_at: :desc)
|
|
421
|
+
scope = model_class.lyra_processable.order(updated_at: :desc)
|
|
420
422
|
|
|
421
423
|
# Apply search if provided
|
|
422
424
|
if search_query.present? && display_fields.any?
|
|
@@ -460,7 +462,7 @@ module Lyra
|
|
|
460
462
|
def fetch_recent_events(limit: 100)
|
|
461
463
|
events = []
|
|
462
464
|
Lyra.config.monitored_models.each do |model_class|
|
|
463
|
-
model_class.order(updated_at: :desc).limit(limit / Lyra.config.monitored_models.size).each do |record|
|
|
465
|
+
model_class.lyra_processable.order(updated_at: :desc).limit(limit / Lyra.config.monitored_models.size).each do |record|
|
|
464
466
|
stream_name = "#{model_class.name}$#{record.id}"
|
|
465
467
|
begin
|
|
466
468
|
stream_events = Lyra.config.event_store.read.stream(stream_name).to_a
|
|
@@ -470,7 +472,7 @@ module Lyra
|
|
|
470
472
|
end
|
|
471
473
|
end
|
|
472
474
|
end
|
|
473
|
-
events.sort_by { |e| event_timestamp(e) }.last(limit)
|
|
475
|
+
rights_visible(events.sort_by { |e| event_timestamp(e) }.last(limit))
|
|
474
476
|
end
|
|
475
477
|
|
|
476
478
|
def fetch_filtered_events(model_class: nil, operation: nil, limit: 50)
|
|
@@ -478,7 +480,7 @@ module Lyra
|
|
|
478
480
|
models = model_class.present? ? [monitored_model!(model_class)] : Lyra.config.monitored_models
|
|
479
481
|
|
|
480
482
|
models.each do |klass|
|
|
481
|
-
klass.order(updated_at: :desc).limit(limit).each do |record|
|
|
483
|
+
klass.lyra_processable.order(updated_at: :desc).limit(limit).each do |record|
|
|
482
484
|
stream_name = "#{klass.name}$#{record.id}"
|
|
483
485
|
begin
|
|
484
486
|
stream_events = Lyra.config.event_store.read.stream(stream_name).to_a
|
|
@@ -497,7 +499,7 @@ module Lyra
|
|
|
497
499
|
end
|
|
498
500
|
end
|
|
499
501
|
|
|
500
|
-
events.sort_by { |e| event_timestamp(e) }.last(limit)
|
|
502
|
+
rights_visible(events.sort_by { |e| event_timestamp(e) }.last(limit))
|
|
501
503
|
end
|
|
502
504
|
|
|
503
505
|
def fetch_entity_events(model_class, model_id, depth: 1)
|
|
@@ -516,7 +518,7 @@ module Lyra
|
|
|
516
518
|
# Find events sharing correlation IDs
|
|
517
519
|
correlation_ids.each do |corr_id|
|
|
518
520
|
Lyra.config.monitored_models.each do |klass|
|
|
519
|
-
klass.order(updated_at: :desc).limit(20).each do |record|
|
|
521
|
+
klass.lyra_processable.order(updated_at: :desc).limit(20).each do |record|
|
|
520
522
|
other_stream = "#{klass.name}$#{record.id}"
|
|
521
523
|
next if other_stream == stream_name
|
|
522
524
|
|
|
@@ -535,7 +537,7 @@ module Lyra
|
|
|
535
537
|
# Stream doesn't exist
|
|
536
538
|
end
|
|
537
539
|
|
|
538
|
-
events.uniq { |e| e.event_id }.sort_by { |e| event_timestamp(e) }
|
|
540
|
+
rights_visible(events.uniq { |e| e.event_id }.sort_by { |e| event_timestamp(e) })
|
|
539
541
|
end
|
|
540
542
|
|
|
541
543
|
def record_display_name(record)
|
|
@@ -559,7 +561,7 @@ module Lyra
|
|
|
559
561
|
events = []
|
|
560
562
|
|
|
561
563
|
Lyra.config.monitored_models.each do |model_class|
|
|
562
|
-
model_class.where("updated_at >= ?", cutoff).find_each do |record|
|
|
564
|
+
model_class.lyra_processable.where("updated_at >= ?", cutoff).find_each do |record|
|
|
563
565
|
stream_name = "#{model_class.name}$#{record.id}"
|
|
564
566
|
begin
|
|
565
567
|
stream_events = Lyra.config.event_store.read.stream(stream_name).to_a
|
|
@@ -570,14 +572,14 @@ module Lyra
|
|
|
570
572
|
end
|
|
571
573
|
end
|
|
572
574
|
|
|
573
|
-
events.sort_by { |e| event_timestamp(e) }
|
|
575
|
+
rights_visible(events.sort_by { |e| event_timestamp(e) })
|
|
574
576
|
end
|
|
575
577
|
|
|
576
578
|
def fetch_events_for_model(model_class, days: 7)
|
|
577
579
|
cutoff = days.days.ago
|
|
578
580
|
events = []
|
|
579
581
|
|
|
580
|
-
model_class.where("updated_at >= ?", cutoff).find_each do |record|
|
|
582
|
+
model_class.lyra_processable.where("updated_at >= ?", cutoff).find_each do |record|
|
|
581
583
|
stream_name = "#{model_class.name}$#{record.id}"
|
|
582
584
|
begin
|
|
583
585
|
stream_events = Lyra.config.event_store.read.stream(stream_name).to_a
|
|
@@ -587,7 +589,7 @@ module Lyra
|
|
|
587
589
|
end
|
|
588
590
|
end
|
|
589
591
|
|
|
590
|
-
events.sort_by { |e| event_timestamp(e) }
|
|
592
|
+
rights_visible(events.sort_by { |e| event_timestamp(e) })
|
|
591
593
|
end
|
|
592
594
|
|
|
593
595
|
def event_timestamp(event)
|
|
@@ -5,9 +5,12 @@ module Lyra
|
|
|
5
5
|
subject_id = params[:subject_id]
|
|
6
6
|
subject_type = params[:subject_type]
|
|
7
7
|
|
|
8
|
+
subject_model = subject_type && Lyra.config.monitored_model_named(subject_type)
|
|
9
|
+
refuse_restricted_record!(subject_model, subject_id) if subject_model && subject_id
|
|
8
10
|
flow = EventFlow.new(
|
|
9
11
|
subject_id: subject_id,
|
|
10
|
-
subject_type: subject_type
|
|
12
|
+
subject_type: subject_type,
|
|
13
|
+
event_filter: method(:rights_visible)
|
|
11
14
|
)
|
|
12
15
|
|
|
13
16
|
@flow_data = flow.flow_data
|
|
@@ -41,6 +44,7 @@ module Lyra
|
|
|
41
44
|
def event_chain
|
|
42
45
|
model_class = monitored_model!(params[:model_class])
|
|
43
46
|
model_id = params[:model_id]
|
|
47
|
+
refuse_restricted_record!(model_class, model_id)
|
|
44
48
|
|
|
45
49
|
flow = EventFlow.new
|
|
46
50
|
chain = flow.reconstruct_state_chain(model_class.name, model_id)
|
|
@@ -63,8 +67,9 @@ module Lyra
|
|
|
63
67
|
format.json { render json: mapping }
|
|
64
68
|
end
|
|
65
69
|
else
|
|
66
|
-
#
|
|
67
|
-
events =
|
|
70
|
+
# Summary of the CRUD operations in the most recent events (bounded: recent_events)
|
|
71
|
+
events = recent_events
|
|
72
|
+
log_dashboard_read("flow/crud_mapping", {}, returned: events.size)
|
|
68
73
|
@models_summary = events.group_by { |e| event_model_class(e) }.transform_values do |model_events|
|
|
69
74
|
{
|
|
70
75
|
total: model_events.count,
|
|
@@ -80,7 +85,8 @@ module Lyra
|
|
|
80
85
|
render json: {
|
|
81
86
|
total_events: @total_events,
|
|
82
87
|
operations: @operations_summary,
|
|
83
|
-
models: @models_summary
|
|
88
|
+
models: @models_summary,
|
|
89
|
+
window: @window
|
|
84
90
|
}
|
|
85
91
|
end
|
|
86
92
|
end
|
|
@@ -91,6 +97,7 @@ module Lyra
|
|
|
91
97
|
def visualization
|
|
92
98
|
model_class = monitored_model!(params[:model_class])
|
|
93
99
|
model_id = params[:model_id]
|
|
100
|
+
refuse_restricted_record!(model_class, model_id)
|
|
94
101
|
format = params[:format] || 'json'
|
|
95
102
|
|
|
96
103
|
stream_name = "#{model_class.name}$#{model_id}"
|
|
@@ -116,9 +123,12 @@ module Lyra
|
|
|
116
123
|
def correlation
|
|
117
124
|
correlation_id = params[:correlation_id]
|
|
118
125
|
|
|
119
|
-
events
|
|
126
|
+
# Among the most recent events (bounded: recent_events); a correlation older
|
|
127
|
+
# than the window is not found, and window says how far back it looked.
|
|
128
|
+
events = recent_events.select do |event|
|
|
120
129
|
event.metadata[:correlation_id] == correlation_id
|
|
121
130
|
end
|
|
131
|
+
log_dashboard_read("flow/correlation", { correlation_id: correlation_id }, returned: events.size)
|
|
122
132
|
|
|
123
133
|
flow = EventFlow.new
|
|
124
134
|
timeline = flow.build_timeline(events)
|
|
@@ -129,7 +139,8 @@ module Lyra
|
|
|
129
139
|
started_at: events.min_by(&:timestamp)&.timestamp,
|
|
130
140
|
completed_at: events.max_by(&:timestamp)&.timestamp,
|
|
131
141
|
events: timeline,
|
|
132
|
-
|
|
142
|
+
window: @window,
|
|
143
|
+
privacy_impact: pam_dsl_available? ? events.count { |e| Privacy::PIIDetector.detect(event_fields(e)).any? } : nil
|
|
133
144
|
}
|
|
134
145
|
end
|
|
135
146
|
|
|
@@ -137,24 +148,30 @@ module Lyra
|
|
|
137
148
|
def user_actions
|
|
138
149
|
user_id = params[:user_id]
|
|
139
150
|
|
|
140
|
-
events =
|
|
151
|
+
# Among the most recent events, and at most ?actions= of them listed (default 100,
|
|
152
|
+
# at most 500): one staff member's whole activity is not one request's answer.
|
|
153
|
+
events = recent_events.select do |event|
|
|
141
154
|
event.metadata[:user_id] == user_id.to_i
|
|
142
155
|
end
|
|
156
|
+
log_dashboard_read("flow/user_actions", { user_id: user_id }, returned: events.size)
|
|
143
157
|
|
|
144
158
|
actions = events.group_by { |e| e.metadata[:action_id] }
|
|
159
|
+
listed = bounded_param(:actions, default: 100, max: 500)
|
|
145
160
|
|
|
146
161
|
render json: {
|
|
147
162
|
user_id: user_id,
|
|
148
163
|
total_actions: actions.count,
|
|
149
164
|
total_events: events.count,
|
|
150
|
-
|
|
165
|
+
actions_listed: [actions.count, listed].min,
|
|
166
|
+
window: @window,
|
|
167
|
+
actions: actions.first(listed).map do |action_id, action_events|
|
|
151
168
|
{
|
|
152
169
|
action_id: action_id,
|
|
153
170
|
user_action: action_events.first.metadata[:user_action],
|
|
154
171
|
timestamp: event_timestamp(action_events.min_by { |e| event_timestamp(e) }),
|
|
155
172
|
events_count: action_events.count,
|
|
156
173
|
models_affected: action_events.map { |e| event_model_class(e) }.uniq,
|
|
157
|
-
pii_affected: pam_dsl_available? ? action_events.any? { |e| Privacy::PIIDetector.detect(
|
|
174
|
+
pii_affected: pam_dsl_available? ? action_events.any? { |e| Privacy::PIIDetector.detect(event_fields(e)).any? } : nil
|
|
158
175
|
}
|
|
159
176
|
end
|
|
160
177
|
}
|
|
@@ -196,30 +213,10 @@ module Lyra
|
|
|
196
213
|
data[:timestamp] || data["timestamp"] || event.metadata[:timestamp]
|
|
197
214
|
end
|
|
198
215
|
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
return attrs if attrs.is_a?(Hash)
|
|
204
|
-
end
|
|
205
|
-
# Fall back to data hash for RubyEventStore events
|
|
206
|
-
data = event.respond_to?(:data) ? event.data : nil
|
|
207
|
-
return {} unless data
|
|
208
|
-
data[:attributes] || data["attributes"] || {}
|
|
209
|
-
end
|
|
210
|
-
|
|
211
|
-
def load_events(subject_id, subject_type)
|
|
212
|
-
events = Lyra.config.event_store.read.to_a
|
|
213
|
-
|
|
214
|
-
if subject_id && subject_type
|
|
215
|
-
# The subject's own record. What a user did is user_actions; their
|
|
216
|
-
# edits of other records are not part of their timeline.
|
|
217
|
-
events = events.select do |event|
|
|
218
|
-
event_model_class(event).to_s == subject_type.to_s && event_model_id(event).to_s == subject_id.to_s
|
|
219
|
-
end
|
|
220
|
-
end
|
|
221
|
-
|
|
222
|
-
events
|
|
216
|
+
# The fields an event gives or changes, whatever mode wrote it
|
|
217
|
+
# (Lyra::Event.fields_of): what personal data is detected in.
|
|
218
|
+
def event_fields(event)
|
|
219
|
+
Lyra::Event.fields_of(event)
|
|
223
220
|
end
|
|
224
221
|
end
|
|
225
222
|
end
|
|
@@ -12,9 +12,11 @@ module Lyra
|
|
|
12
12
|
subject_type: subject_type
|
|
13
13
|
)
|
|
14
14
|
|
|
15
|
+
# A view of the data, not an answered Art. 15 request: not recorded as an
|
|
16
|
+
# export (the application's explicit data_export call is).
|
|
15
17
|
render json: {
|
|
16
18
|
subject: { type: subject_type, id: subject_id },
|
|
17
|
-
data_export: compliance.data_export,
|
|
19
|
+
data_export: compliance.data_export(record: false),
|
|
18
20
|
generated_at: Time.current
|
|
19
21
|
}
|
|
20
22
|
end
|
|
@@ -31,7 +33,7 @@ module Lyra
|
|
|
31
33
|
|
|
32
34
|
render json: {
|
|
33
35
|
subject: { type: subject_type, id: subject_id },
|
|
34
|
-
right_to_access: compliance.data_export,
|
|
36
|
+
right_to_access: compliance.data_export(record: false), # a view, not recorded as an export
|
|
35
37
|
right_to_be_forgotten: compliance.right_to_be_forgotten_report,
|
|
36
38
|
rectification_history: compliance.rectification_history,
|
|
37
39
|
processing_activities: compliance.processing_activities,
|
|
@@ -70,7 +72,9 @@ module Lyra
|
|
|
70
72
|
subject_type = params[:subject_type]
|
|
71
73
|
subject_id = params[:subject_id]
|
|
72
74
|
|
|
73
|
-
|
|
75
|
+
subject_model = Lyra.config.monitored_model_named(subject_type.to_s)
|
|
76
|
+
refuse_restricted_record!(subject_model, subject_id) if subject_model
|
|
77
|
+
flow = EventFlow.new(subject_id: subject_id, subject_type: subject_type, event_filter: method(:rights_visible))
|
|
74
78
|
data = flow.privacy_impact_analysis
|
|
75
79
|
|
|
76
80
|
render json: data
|
|
@@ -81,7 +85,7 @@ module Lyra
|
|
|
81
85
|
field_name = params[:field_name]
|
|
82
86
|
model_class = params[:model_class]
|
|
83
87
|
|
|
84
|
-
flow = EventFlow.new
|
|
88
|
+
flow = EventFlow.new(event_filter: method(:rights_visible))
|
|
85
89
|
lineage = flow.data_lineage(field_name, model_class)
|
|
86
90
|
|
|
87
91
|
render json: lineage
|
|
@@ -95,14 +99,35 @@ module Lyra
|
|
|
95
99
|
Rails.application.config.pam_dsl.default_policy : nil
|
|
96
100
|
end
|
|
97
101
|
|
|
102
|
+
# GET /lyra/privacy/rights
|
|
103
|
+
# The data subjects' requests still open (PamDsl.rights.open_requests):
|
|
104
|
+
# restrictions not yet lifted (Art. 18) and objections not yet decided
|
|
105
|
+
# (Art. 21), oldest first, each with its age against the month Art. 12(3)
|
|
106
|
+
# gives the controller to act on a request. It reads Lyra's rights table,
|
|
107
|
+
# not a monitored model's records, under the dashboard's purpose rules,
|
|
108
|
+
# and is logged as a dashboard read. Names, grounds and states only:
|
|
109
|
+
# never a value of the subject's data.
|
|
110
|
+
def rights
|
|
111
|
+
now = Time.current
|
|
112
|
+
requests = PamDsl.rights.open_requests
|
|
113
|
+
log_dashboard_read("privacy/rights", {}, returned: requests.size)
|
|
114
|
+
@requests = requests.map { |request| open_request_row(request, now) }
|
|
115
|
+
|
|
116
|
+
respond_to do |format|
|
|
117
|
+
format.html
|
|
118
|
+
format.json { render json: { open_requests: @requests, generated_at: now } }
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
|
|
98
122
|
# GET /lyra/privacy/pii_detection
|
|
99
123
|
def pii_detection
|
|
100
|
-
# Detect PII across
|
|
101
|
-
events =
|
|
124
|
+
# Detect PII across the most recent events (bounded: recent_events)
|
|
125
|
+
events = recent_events
|
|
126
|
+
log_dashboard_read("privacy/pii_detection", {}, returned: events.size)
|
|
102
127
|
@pii_inventory = extract_pii_from_events(events)
|
|
103
128
|
|
|
104
129
|
@total_events = events.count
|
|
105
|
-
@events_with_pii = events.count { |e| Privacy::PIIDetector.detect(
|
|
130
|
+
@events_with_pii = events.count { |e| Privacy::PIIDetector.detect(event_fields(e)).any? }
|
|
106
131
|
@pii_categories = @pii_inventory.keys
|
|
107
132
|
@sensitive_pii = @pii_inventory.select { |k, _|
|
|
108
133
|
[:ssn, :credit_card, :health, :biometric].include?(k)
|
|
@@ -130,6 +155,26 @@ module Lyra
|
|
|
130
155
|
|
|
131
156
|
private
|
|
132
157
|
|
|
158
|
+
# One open request, for the rights page. Only an objection still awaiting
|
|
159
|
+
# the controller's decision has a deadline: one month from its receipt
|
|
160
|
+
# (Art. 12(3); an extension under its second sentence is the
|
|
161
|
+
# controller's to record and tell the subject of). A restriction in
|
|
162
|
+
# force is the controller's action on the request already (Lyra records
|
|
163
|
+
# it when it is applied), so it is listed with its age and no deadline:
|
|
164
|
+
# it lasts until it is lifted, with the Art. 18(3) notice.
|
|
165
|
+
def open_request_row(request, now)
|
|
166
|
+
restriction = request.kind == :restriction
|
|
167
|
+
deadline = restriction ? nil : (request.requested_at.to_datetime >> 1).to_time
|
|
168
|
+
{
|
|
169
|
+
id: request.id, kind: request.kind, subject: request.subject_key, article: request.article,
|
|
170
|
+
ground: restriction ? request.ground : nil, fields: restriction ? request.fields : [],
|
|
171
|
+
policy: restriction ? nil : request.policy, purpose: restriction ? nil : request.purpose,
|
|
172
|
+
state: request.state, requested_at: request.requested_at,
|
|
173
|
+
age_days: ((now - request.requested_at) / 86_400).floor, deadline: deadline,
|
|
174
|
+
overdue: deadline ? now > deadline : false
|
|
175
|
+
}
|
|
176
|
+
end
|
|
177
|
+
|
|
133
178
|
def require_pam_dsl
|
|
134
179
|
unless defined?(PAM_DSL_AVAILABLE) && PAM_DSL_AVAILABLE
|
|
135
180
|
respond_to do |format|
|
|
@@ -143,16 +188,10 @@ module Lyra
|
|
|
143
188
|
defined?(PAM_DSL_AVAILABLE) && PAM_DSL_AVAILABLE
|
|
144
189
|
end
|
|
145
190
|
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
return attrs if attrs.is_a?(Hash)
|
|
151
|
-
end
|
|
152
|
-
# Fall back to data hash for RubyEventStore events
|
|
153
|
-
data = event.respond_to?(:data) ? event.data : nil
|
|
154
|
-
return {} unless data
|
|
155
|
-
data[:attributes] || data["attributes"] || {}
|
|
191
|
+
# The fields an event gives or changes, whatever mode wrote it
|
|
192
|
+
# (Lyra::Event.fields_of): what personal data is detected in.
|
|
193
|
+
def event_fields(event)
|
|
194
|
+
Lyra::Event.fields_of(event)
|
|
156
195
|
end
|
|
157
196
|
|
|
158
197
|
def event_model_class(event)
|
|
@@ -165,7 +204,7 @@ module Lyra
|
|
|
165
204
|
def extract_pii_from_events(events)
|
|
166
205
|
inventory = Hash.new { |h, k| h[k] = [] }
|
|
167
206
|
events.each do |event|
|
|
168
|
-
pii = Privacy::PIIDetector.detect(
|
|
207
|
+
pii = Privacy::PIIDetector.detect(event_fields(event))
|
|
169
208
|
next unless pii.is_a?(Hash)
|
|
170
209
|
pii.each do |field, info|
|
|
171
210
|
next unless info.is_a?(Hash) && info[:type]
|