orfeas_lyra 0.9.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +2021 -17
- data/README.md +22 -9
- data/Rakefile +40 -1
- data/app/controllers/lyra/application_controller.rb +127 -0
- data/app/controllers/lyra/dashboard_controller.rb +33 -31
- data/app/controllers/lyra/flow_controller.rb +32 -34
- data/app/controllers/lyra/privacy_controller.rb +57 -18
- data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
- data/app/views/lyra/dashboard/index.html.erb +1 -0
- data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
- data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
- data/app/views/lyra/privacy/policy.html.erb +7 -0
- data/app/views/lyra/privacy/rights.html.erb +78 -0
- data/config/privacy_policies.rb +0 -3
- data/config/routes.rb +1 -0
- data/lib/lyra/access_log.rb +187 -6
- data/lib/lyra/advisory_lock.rb +236 -0
- data/lib/lyra/aggregate.rb +1 -1
- data/lib/lyra/associations/habtm_links.rb +382 -19
- data/lib/lyra/associations/maintenance.rb +229 -0
- data/lib/lyra/attribution.rb +48 -0
- data/lib/lyra/bypass_events.rb +446 -46
- data/lib/lyra/command_handler.rb +19 -14
- data/lib/lyra/configuration.rb +237 -10
- data/lib/lyra/consistency/read_your_writes.rb +4 -2
- data/lib/lyra/correlation.rb +121 -0
- data/lib/lyra/domain_events.rb +4 -0
- data/lib/lyra/dual_view.rb +53 -16
- data/lib/lyra/encrypted_attributes.rb +5 -0
- data/lib/lyra/engine.rb +67 -0
- data/lib/lyra/erasure.rb +1011 -106
- data/lib/lyra/event.rb +35 -1
- data/lib/lyra/event_data.rb +38 -0
- data/lib/lyra/event_flow.rb +31 -15
- data/lib/lyra/event_store_adapter.rb +28 -0
- data/lib/lyra/event_window.rb +58 -0
- data/lib/lyra/genesis.rb +548 -44
- data/lib/lyra/interceptors/association_interceptor.rb +689 -34
- data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
- data/lib/lyra/internal.rb +26 -0
- data/lib/lyra/mode_sync.rb +51 -3
- data/lib/lyra/mode_transition.rb +257 -30
- data/lib/lyra/model_selection.rb +35 -0
- data/lib/lyra/own_connection.rb +17 -2
- data/lib/lyra/privacy/adapters/pam.rb +25 -2
- data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
- data/lib/lyra/privacy/interface.rb +27 -7
- data/lib/lyra/privacy/subject_links.rb +143 -0
- data/lib/lyra/privacy/subject_records.rb +369 -0
- data/lib/lyra/projection.rb +7 -1
- data/lib/lyra/projections/async_checkpoint.rb +32 -0
- data/lib/lyra/projections/async_projection_job.rb +166 -23
- data/lib/lyra/projections/cached_joins.rb +33 -18
- data/lib/lyra/projections/cached_projection.rb +74 -33
- data/lib/lyra/projections/cached_relation.rb +3175 -321
- data/lib/lyra/projections/default_scopes.rb +254 -0
- data/lib/lyra/projections/deferred_relation.rb +23 -3
- data/lib/lyra/projections/event_store_reader.rb +47 -8
- data/lib/lyra/projections/foreign_keys.rb +92 -0
- data/lib/lyra/projections/lazy_projection.rb +165 -27
- data/lib/lyra/projections/model_projection.rb +165 -14
- data/lib/lyra/projections/rebuild.rb +34 -6
- data/lib/lyra/projections/reference_links.rb +463 -72
- data/lib/lyra/projections/typed_values.rb +629 -0
- data/lib/lyra/projections/unique_indexes.rb +595 -0
- data/lib/lyra/projections/uniqueness.rb +784 -0
- data/lib/lyra/projections/value_digest.rb +93 -0
- data/lib/lyra/purpose_bound_reads.rb +912 -77
- data/lib/lyra/repair.rb +119 -28
- data/lib/lyra/retention.rb +114 -8
- data/lib/lyra/rights/consent_store.rb +415 -0
- data/lib/lyra/rights/erasure_guard.rb +135 -0
- data/lib/lyra/rights/pending.rb +74 -0
- data/lib/lyra/rights/queries.rb +269 -0
- data/lib/lyra/rights/store.rb +512 -0
- data/lib/lyra/rights/subject_keys.rb +224 -0
- data/lib/lyra/rights.rb +281 -0
- data/lib/lyra/saves_without_effect.rb +69 -0
- data/lib/lyra/stream_tail.rb +319 -0
- data/lib/lyra/strict_data_access.rb +916 -106
- data/lib/lyra/temporal.rb +4 -2
- data/lib/lyra/touch_registry.rb +404 -0
- data/lib/lyra/verification/bypass_workflow.rb +452 -42
- data/lib/lyra/verification/rights_workflows.rb +170 -0
- data/lib/lyra/version.rb +1 -1
- data/lib/lyra/visualization/event_graph.rb +1 -1
- data/lib/lyra/visualization/timeline.rb +2 -2
- data/lib/lyra.rb +43 -2
- data/lib/tasks/lyra_mode.rake +110 -8
- data/lib/tasks/lyra_projections.rake +43 -2
- metadata +34 -4
- data/lib/lyra/associations/event_aware.rb +0 -224
data/README.md
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# Lyra
|
|
2
2
|
|
|
3
|
+
[](https://github.com/mpantel/lyra-engine/actions/workflows/test.yml)
|
|
4
|
+
|
|
3
5
|
**CRUD to Event Sourcing Transformation Engine**
|
|
4
6
|
|
|
5
7
|
*Part of ORFEAS (Object-Relational Framework for Event-Aware Systems)*
|
|
@@ -35,10 +37,14 @@ seven configurations:
|
|
|
35
37
|
`delete`, `update_all`, `delete_all`, `insert_all`, `insert_all!`, `upsert_all`,
|
|
36
38
|
and the updates of `dependent: :nullify`) on a monitored model publish bypass
|
|
37
39
|
events; with `config.strict_data_access = true` they raise instead (except
|
|
38
|
-
`touch` and the `dependent: :nullify`
|
|
40
|
+
`touch` and Rails' own association writes, such as the `dependent: :nullify`
|
|
41
|
+
updates or removing a `has_and_belongs_to_many` link; the same writes made by
|
|
42
|
+
the application raise).
|
|
39
43
|
- Each record has its own stream, named `"Model$id"` (for example `"Order$42"`).
|
|
40
44
|
- **ES-NoProj** writes no rows; it answers a query exactly from the events or
|
|
41
|
-
raises `Lyra::Projections::UnsupportedQuery`.
|
|
45
|
+
raises `Lyra::Projections::UnsupportedQuery`. Since it writes no rows to
|
|
46
|
+
monitored tables, the switch into it is refused while an unmonitored table
|
|
47
|
+
has a database foreign key into a monitored one.
|
|
42
48
|
- **Switching modes after boot is gated**: `Lyra::ModeTransition.to!` (and
|
|
43
49
|
`config.enable_hijack!`, `enable_monitor!`, `enable_event_sourcing!`) checks
|
|
44
50
|
that rows and events agree before a switch that changes the authoritative
|
|
@@ -48,18 +54,19 @@ seven configurations:
|
|
|
48
54
|
|
|
49
55
|
## Requirements and installation
|
|
50
56
|
|
|
51
|
-
- Ruby >= 4.0, Rails
|
|
57
|
+
- Ruby >= 4.0, Rails 8.1 (`>= 8.1, < 8.2`; tested on 8.1.4.1), `rails_event_store` ~> 3.0
|
|
58
|
+
(from `lyra.gemspec`). CI tests Ruby 4.0 on PostgreSQL 18.7.
|
|
52
59
|
- PostgreSQL. Lyra depends on the `pg` gem; the advisory locks behind Genesis
|
|
53
60
|
and ES-Lazy, and id reservation in Hijack mode, need PostgreSQL.
|
|
54
61
|
|
|
55
|
-
The gems are on RubyGems, at 0.
|
|
56
|
-
files to require:
|
|
62
|
+
The gems are on RubyGems, all three at 1.0.0 (22 November 2026):
|
|
63
|
+
`orfeas_lyra`, `orfeas_pam_dsl` and `orfeas_petri_flow`. Their names differ from the files to require:
|
|
57
64
|
|
|
58
65
|
```ruby
|
|
59
66
|
# Gemfile
|
|
60
|
-
gem "orfeas_lyra", "~> 0
|
|
61
|
-
gem "orfeas_pam_dsl", "~> 0
|
|
62
|
-
gem "orfeas_petri_flow", "~> 0
|
|
67
|
+
gem "orfeas_lyra", "~> 1.0", require: "lyra"
|
|
68
|
+
gem "orfeas_pam_dsl", "~> 1.0", require: "pam_dsl" # optional, privacy features
|
|
69
|
+
gem "orfeas_petri_flow", "~> 1.0", require: "petri_flow" # optional, verification
|
|
63
70
|
```
|
|
64
71
|
|
|
65
72
|
Create the event store tables with the RailsEventStore 3 generator
|
|
@@ -89,7 +96,13 @@ Beyond the modes above (see the [API Reference](docs/API_REFERENCE.md) for each)
|
|
|
89
96
|
- **Dashboard**: `mount Lyra::Engine, at: "/lyra"`. Outside development and
|
|
90
97
|
test it answers 403 until `config.dashboard_authorization` (a proc run on the
|
|
91
98
|
controller, e.g. `->(controller) { controller.current_user&.admin? }`) says
|
|
92
|
-
who may use it.
|
|
99
|
+
who may use it. The dashboard is a development tool. In production, whoever
|
|
100
|
+
the rule admits reads every data subject's records and history, with no
|
|
101
|
+
declared purpose and no access-log entry, and the rule covers the whole
|
|
102
|
+
dashboard, not single pages. Bringing it to production is a future feature:
|
|
103
|
+
per-page and per-role authorization, a declared purpose and an access-log
|
|
104
|
+
entry for every subject shown or exported, masking by sensitivity, retention
|
|
105
|
+
for what it records, and a security review of its routes.
|
|
93
106
|
|
|
94
107
|
---
|
|
95
108
|
|
data/Rakefile
CHANGED
|
@@ -42,13 +42,52 @@ namespace :test do
|
|
|
42
42
|
Rake::Task["test"].invoke
|
|
43
43
|
|
|
44
44
|
puts "\n=== Running PAM DSL tests ==="
|
|
45
|
-
|
|
45
|
+
# Twice: with ActiveSupport, then with the ActiveSupport-free polyfill (test:both).
|
|
46
|
+
Dir.chdir("gems/pam_dsl") { system("rake test:both") || raise("PAM DSL tests failed") }
|
|
46
47
|
|
|
47
48
|
puts "\n=== Running PetriFlow tests ==="
|
|
48
49
|
Dir.chdir("gems/petri_flow") { system("rake test") || raise("PetriFlow tests failed") }
|
|
49
50
|
|
|
50
51
|
puts "\n=== All tests passed! ==="
|
|
51
52
|
end
|
|
53
|
+
|
|
54
|
+
# The Solidus case study's specs, in its own bundle: spec/olist (the Olist
|
|
55
|
+
# replay in every mode) and the rest of its suite, less the JavaScript
|
|
56
|
+
# system examples, which need a Chrome (bin/rspec-chrome-docker runs them;
|
|
57
|
+
# SOLIDUS_SPECS_JS=1 includes them here). On a database of their own on the
|
|
58
|
+
# Lyra test server (SOLIDUS_DATABASE_URL), dropped and loaded from
|
|
59
|
+
# db/schema.rb first. SOLIDUS_SPECS names spec files or directories (default:
|
|
60
|
+
# every spec). The system examples rebuild app/assets/builds/tailwind.css,
|
|
61
|
+
# which is committed: it is put back as it was, so that the tree stays clean
|
|
62
|
+
# (the benchmark campaign refuses, and its reports record, a dirty tree).
|
|
63
|
+
desc "Run the Solidus case study's specs (the Olist replay and the rest, without JavaScript)"
|
|
64
|
+
task :solidus do
|
|
65
|
+
dir = File.expand_path("examples/solidus_case_study", __dir__)
|
|
66
|
+
abort "The Solidus case study is not in this checkout." unless File.directory?(dir)
|
|
67
|
+
|
|
68
|
+
env = {
|
|
69
|
+
"RAILS_ENV" => "test",
|
|
70
|
+
"DISABLE_DATABASE_ENVIRONMENT_CHECK" => "1",
|
|
71
|
+
"BUNDLE_GEMFILE" => File.join(dir, "Gemfile"),
|
|
72
|
+
"DATABASE_URL" => ENV.fetch("SOLIDUS_DATABASE_URL",
|
|
73
|
+
"postgres://postgres:postgres@localhost:5433/solidus_case_study_test")
|
|
74
|
+
}
|
|
75
|
+
css = File.join(dir, "app/assets/builds/tailwind.css")
|
|
76
|
+
committed = File.exist?(css) ? File.binread(css) : nil
|
|
77
|
+
args = ENV.fetch("SOLIDUS_SPECS", "").split
|
|
78
|
+
args = ["--tag", "~js", *args] unless ENV["SOLIDUS_SPECS_JS"] == "1"
|
|
79
|
+
begin
|
|
80
|
+
Bundler.with_unbundled_env do
|
|
81
|
+
Dir.chdir(dir) do
|
|
82
|
+
system(env, "bundle exec rails db:drop db:create db:schema:load") ||
|
|
83
|
+
raise("Solidus case study: the test database could not be prepared")
|
|
84
|
+
system(env, "bundle", "exec", "rspec", *args) || raise("Solidus case study specs failed")
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
ensure
|
|
88
|
+
File.binwrite(css, committed) if committed && File.binread(css) != committed
|
|
89
|
+
end
|
|
90
|
+
end
|
|
52
91
|
end
|
|
53
92
|
|
|
54
93
|
task default: :test
|
|
@@ -9,6 +9,23 @@ module Lyra
|
|
|
9
9
|
# Lyra::Configuration#dashboard_authorization.
|
|
10
10
|
before_action :authorize_lyra_dashboard!
|
|
11
11
|
|
|
12
|
+
# A model_class parameter that names no monitored model answers 404.
|
|
13
|
+
class UnknownModel < StandardError; end
|
|
14
|
+
rescue_from(UnknownModel) { head :not_found }
|
|
15
|
+
|
|
16
|
+
# The dashboard reads as the application does, under a purpose: the one the
|
|
17
|
+
# request names (?purpose=), else config.dashboard_purpose; with neither it
|
|
18
|
+
# reads without one, as config.reads_without_purpose decides. A read the
|
|
19
|
+
# policy refuses answers 403 and says how to name a purpose.
|
|
20
|
+
around_action :with_dashboard_purpose
|
|
21
|
+
READ_REFUSALS = [Lyra::PurposeBoundReads::PurposeRequiredError] +
|
|
22
|
+
%w[PurposeFieldMismatchError UndeclaredPurposeError ConsentRequiredError UndeterminedBasisError
|
|
23
|
+
SpecialCategoryViolationError CriminalDataViolationError InvalidFieldError
|
|
24
|
+
RestrictedProcessingError ObjectedProcessingError]
|
|
25
|
+
.filter_map { |name| PamDsl.const_get(name) if defined?(PamDsl) && PamDsl.const_defined?(name) } +
|
|
26
|
+
(defined?(Lyra::Rights::UnresolvedSubjectError) ? [Lyra::Rights::UnresolvedSubjectError] : [])
|
|
27
|
+
rescue_from(*READ_REFUSALS) { |error| refuse_read(error) }
|
|
28
|
+
|
|
12
29
|
helper_method :pam_dsl_available?, :user_tracking_configured?, :petri_flow_available?
|
|
13
30
|
|
|
14
31
|
# Check if PAM DSL is available for privacy features
|
|
@@ -28,6 +45,23 @@ module Lyra
|
|
|
28
45
|
|
|
29
46
|
private
|
|
30
47
|
|
|
48
|
+
def with_dashboard_purpose(&block)
|
|
49
|
+
purpose = params[:purpose].presence || Lyra.config.dashboard_purpose
|
|
50
|
+
return yield unless purpose
|
|
51
|
+
|
|
52
|
+
Lyra::PurposeBoundReads.with_purpose(purpose, &block)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def refuse_read(error)
|
|
56
|
+
message = "The dashboard read was refused: #{error.message}. Name a purpose the policy allows for it " \
|
|
57
|
+
"(?purpose=...), or set config.dashboard_purpose."
|
|
58
|
+
if request.format.json?
|
|
59
|
+
render json: { error: message }, status: :forbidden
|
|
60
|
+
else
|
|
61
|
+
render plain: message, status: :forbidden
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
31
65
|
# config.dashboard_authorization is evaluated with instance_exec on this
|
|
32
66
|
# controller, so its body may call the controller's methods directly,
|
|
33
67
|
# private ones included (request, session, cookies, params, and any
|
|
@@ -57,6 +91,99 @@ module Lyra
|
|
|
57
91
|
head :forbidden unless allowed
|
|
58
92
|
end
|
|
59
93
|
|
|
94
|
+
# The monitored model a request names, or 404. The routes that take a model
|
|
95
|
+
# read its records and return their attributes, so only models Lyra monitors
|
|
96
|
+
# may be named: resolving any constant (as 0.9.2 and earlier did) let a
|
|
97
|
+
# dashboard user read any table, unmonitored ones included.
|
|
98
|
+
def monitored_model!(name)
|
|
99
|
+
Lyra.config.monitored_model_named(name) || raise(UnknownModel, name.to_s)
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# The most recent events, newest first, at most this request's ?limit=
|
|
103
|
+
# (default and ceiling: config.dashboard_max_events), access-log entries
|
|
104
|
+
# left out (Lyra::EventWindow): a dashboard request does not read the whole
|
|
105
|
+
# log. @window says what was read, for the response.
|
|
106
|
+
def recent_events
|
|
107
|
+
max = Lyra.config.dashboard_max_events
|
|
108
|
+
limit = bounded_param(:limit, default: max, max: max)
|
|
109
|
+
events, @window = Lyra::EventWindow.recent(limit)
|
|
110
|
+
rights_visible(events)
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# The subject's rights (Arts. 18 and 21) in the routes that show events
|
|
114
|
+
# or replayed state, under the dashboard's purpose: a route about one
|
|
115
|
+
# record is refused (403, as a read of the record is) when a restriction
|
|
116
|
+
# or upheld objection covers it; a route over many records leaves out
|
|
117
|
+
# the events of the records they cover, and says how many it left out in
|
|
118
|
+
# the X-Lyra-Rights-Withheld header. With no purpose, nothing in force, or
|
|
119
|
+
# the policy in audit mode, every event is shown.
|
|
120
|
+
def refuse_restricted_record!(model_class, id)
|
|
121
|
+
return unless rights_checked?
|
|
122
|
+
|
|
123
|
+
record = Lyra::Rights::SubjectKeys.find_record(model_class, id)
|
|
124
|
+
return unless record
|
|
125
|
+
|
|
126
|
+
policy = Lyra::Privacy.policy_for(model_class)
|
|
127
|
+
policy.validate_rights!(policy.declared_fields, Lyra::PurposeBoundReads.current, subject: record)
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def rights_visible(events)
|
|
131
|
+
return events unless rights_checked?
|
|
132
|
+
|
|
133
|
+
verdicts = {}
|
|
134
|
+
kept = events.reject do |event|
|
|
135
|
+
model, id = event_record(event)
|
|
136
|
+
next false unless model
|
|
137
|
+
|
|
138
|
+
verdicts.fetch([model, id]) { verdicts[[model, id]] = restricted_record?(model, id) }
|
|
139
|
+
end
|
|
140
|
+
withheld = events.size - kept.size
|
|
141
|
+
response.headers["X-Lyra-Rights-Withheld"] = withheld.to_s if withheld.positive?
|
|
142
|
+
kept
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
def rights_checked?
|
|
146
|
+
defined?(Lyra::Rights::Queries) && Lyra::PurposeBoundReads.current && Lyra::Rights::Queries.in_force?
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# The monitored model and id an event is about, or nil.
|
|
150
|
+
def event_record(event)
|
|
151
|
+
data = event.respond_to?(:data) && event.data.is_a?(Hash) ? event.data : {}
|
|
152
|
+
name = data[:model_class] || data["model_class"]
|
|
153
|
+
id = data[:model_id] || data["model_id"]
|
|
154
|
+
model = name && Lyra.config.monitored_model_named(name.to_s)
|
|
155
|
+
model && !id.nil? ? [model, id.to_s] : nil
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def restricted_record?(model, id)
|
|
159
|
+
policy = Lyra::Privacy.policy_for(model)
|
|
160
|
+
pam = policy.respond_to?(:pam_policy) ? policy.pam_policy : nil
|
|
161
|
+
return false unless pam && pam.enforcement_mode == :strict
|
|
162
|
+
|
|
163
|
+
record = Lyra::Rights::SubjectKeys.find_record(model, id)
|
|
164
|
+
return false unless record
|
|
165
|
+
|
|
166
|
+
pam.rights_violations(pam.fields.keys, Lyra::PurposeBoundReads.current, subject: record).any?
|
|
167
|
+
rescue Lyra::Rights::UnresolvedSubjectError
|
|
168
|
+
true
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
# Record a dashboard read in the access log (when config.record_access_events
|
|
172
|
+
# is on): which route, for what, how much came back, never a value.
|
|
173
|
+
def log_dashboard_read(route, scope = {}, returned: nil)
|
|
174
|
+
# The access log loads with PAM DSL; without it there is nothing to record to.
|
|
175
|
+
return unless defined?(Lyra::AccessLog)
|
|
176
|
+
|
|
177
|
+
Lyra::AccessLog.dashboard_read(route: route, scope: scope, returned: returned, truncated: @window&.dig(:truncated))
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
# An integer parameter, clamped to 1..max; the default when absent. Bounds
|
|
181
|
+
# the work a single dashboard request can ask for.
|
|
182
|
+
def bounded_param(key, default:, max:)
|
|
183
|
+
value = params[key].presence&.to_i || default
|
|
184
|
+
value.clamp(1, max)
|
|
185
|
+
end
|
|
186
|
+
|
|
60
187
|
def lyra_dashboard_open_by_default?
|
|
61
188
|
return true if Rails.env.development? || Rails.env.test?
|
|
62
189
|
|
|
@@ -11,16 +11,17 @@ module Lyra
|
|
|
11
11
|
|
|
12
12
|
# GET /lyra/dashboard/model/:model_class
|
|
13
13
|
def model_overview
|
|
14
|
-
@model_class = params[:model_class]
|
|
14
|
+
@model_class = monitored_model!(params[:model_class])
|
|
15
15
|
@records_count = @model_class.count
|
|
16
16
|
@events_count = count_events_for_model(@model_class)
|
|
17
17
|
end
|
|
18
18
|
|
|
19
19
|
# GET /lyra/dashboard/compare/:model_class/:id
|
|
20
20
|
def compare
|
|
21
|
-
model_class = params[:model_class]
|
|
21
|
+
model_class = monitored_model!(params[:model_class])
|
|
22
22
|
model_id = params[:id]
|
|
23
23
|
|
|
24
|
+
# Under the request's purpose (ApplicationController#with_dashboard_purpose).
|
|
24
25
|
@comparison = DualView.new(model_class, model_id).compare
|
|
25
26
|
@audit_trail = DualView.new(model_class, model_id).audit_trail
|
|
26
27
|
@analysis = StateAnalyzer.analyze(model_class, model_id)
|
|
@@ -34,7 +35,7 @@ module Lyra
|
|
|
34
35
|
|
|
35
36
|
# GET /lyra/dashboard/discrepancies/:model_class
|
|
36
37
|
def discrepancies
|
|
37
|
-
@model_class = params[:model_class]
|
|
38
|
+
@model_class = monitored_model!(params[:model_class])
|
|
38
39
|
@discrepancies = DualView.find_discrepancies(@model_class)
|
|
39
40
|
# find_discrepancies returns only the records that differ
|
|
40
41
|
@records_checked = @model_class.count
|
|
@@ -120,21 +121,21 @@ module Lyra
|
|
|
120
121
|
|
|
121
122
|
# If a model is selected, load available records for the picker
|
|
122
123
|
if @selected_model.present?
|
|
123
|
-
@model_class = @selected_model
|
|
124
|
+
@model_class = monitored_model!(@selected_model)
|
|
124
125
|
@available_records = fetch_available_records(@model_class, @search_query)
|
|
125
126
|
end
|
|
126
127
|
|
|
127
128
|
if @selected_model.present? && @selected_id.present?
|
|
128
|
-
@record = @model_class.find_by(
|
|
129
|
+
@record = @model_class.find_by(Lyra.key_column(@model_class) => @selected_id)
|
|
129
130
|
@audit_entries = AuditProjection.audit_trail(@model_class, @selected_id)
|
|
130
131
|
end
|
|
131
132
|
end
|
|
132
133
|
|
|
133
134
|
# GET /lyra/dashboard/audit_trail/:model_class/:id
|
|
134
135
|
def audit_trail_for_record
|
|
135
|
-
@model_class = params[:model_class]
|
|
136
|
+
@model_class = monitored_model!(params[:model_class])
|
|
136
137
|
@model_id = params[:id]
|
|
137
|
-
@record = @model_class.find_by(
|
|
138
|
+
@record = @model_class.find_by(Lyra.key_column(@model_class) => @model_id)
|
|
138
139
|
@audit_entries = AuditProjection.audit_trail(@model_class, @model_id)
|
|
139
140
|
@monitored_models = Lyra.config.monitored_models
|
|
140
141
|
|
|
@@ -146,7 +147,7 @@ module Lyra
|
|
|
146
147
|
|
|
147
148
|
# GET /lyra/dashboard/visualizations/timeline
|
|
148
149
|
def timeline
|
|
149
|
-
events = fetch_recent_events(limit:
|
|
150
|
+
events = fetch_recent_events(limit: bounded_param(:limit, default: 100, max: 1000))
|
|
150
151
|
timeline = Visualization::Timeline.new(events)
|
|
151
152
|
|
|
152
153
|
render json: timeline.to_data
|
|
@@ -158,7 +159,7 @@ module Lyra
|
|
|
158
159
|
events = fetch_filtered_events(
|
|
159
160
|
model_class: params[:model_class],
|
|
160
161
|
operation: params[:operation],
|
|
161
|
-
limit:
|
|
162
|
+
limit: bounded_param(:limit, default: 50, max: 500)
|
|
162
163
|
)
|
|
163
164
|
graph = Visualization::EventGraph.new(events)
|
|
164
165
|
|
|
@@ -170,7 +171,8 @@ module Lyra
|
|
|
170
171
|
def entity_graph
|
|
171
172
|
model_class = params[:model_class]
|
|
172
173
|
model_id = params[:id]
|
|
173
|
-
depth =
|
|
174
|
+
depth = bounded_param(:depth, default: 1, max: 3) # How many related entities to include
|
|
175
|
+
refuse_restricted_record!(monitored_model!(model_class), model_id)
|
|
174
176
|
|
|
175
177
|
events = fetch_entity_events(model_class, model_id, depth: depth)
|
|
176
178
|
graph = Visualization::EventGraph.new(events)
|
|
@@ -184,13 +186,13 @@ module Lyra
|
|
|
184
186
|
# Returns list of entities with event counts for the picker UI
|
|
185
187
|
def event_list
|
|
186
188
|
model_filter = params[:model_class]
|
|
187
|
-
limit =
|
|
189
|
+
limit = bounded_param(:limit, default: 20, max: 100)
|
|
188
190
|
|
|
189
191
|
entities = []
|
|
190
|
-
models = model_filter.present? ? [model_filter
|
|
192
|
+
models = model_filter.present? ? [monitored_model!(model_filter)] : Lyra.config.monitored_models
|
|
191
193
|
|
|
192
194
|
models.each do |model_class|
|
|
193
|
-
model_class.order(updated_at: :desc).limit(limit).each do |record|
|
|
195
|
+
model_class.lyra_processable.order(updated_at: :desc).limit(limit).each do |record|
|
|
194
196
|
stream_name = "#{model_class.name}$#{record.id}"
|
|
195
197
|
begin
|
|
196
198
|
event_count = Lyra.config.event_store.read.stream(stream_name).count
|
|
@@ -224,7 +226,7 @@ module Lyra
|
|
|
224
226
|
|
|
225
227
|
# GET /lyra/dashboard/visualizations/heatmap
|
|
226
228
|
def heatmap
|
|
227
|
-
events = fetch_events_for_period(days:
|
|
229
|
+
events = fetch_events_for_period(days: bounded_param(:days, default: 7, max: 90))
|
|
228
230
|
heatmap = Visualization::ActivityHeatmap.new(events)
|
|
229
231
|
|
|
230
232
|
render json: heatmap.to_data
|
|
@@ -232,7 +234,7 @@ module Lyra
|
|
|
232
234
|
|
|
233
235
|
# GET /lyra/visualizations/event_graph (HTML view)
|
|
234
236
|
def event_graph_view
|
|
235
|
-
@limit =
|
|
237
|
+
@limit = bounded_param(:limit, default: 50, max: 500)
|
|
236
238
|
events = fetch_recent_events(limit: @limit)
|
|
237
239
|
graph = Visualization::EventGraph.new(events)
|
|
238
240
|
@graph_data = graph.to_data
|
|
@@ -241,7 +243,7 @@ module Lyra
|
|
|
241
243
|
|
|
242
244
|
# GET /lyra/visualizations/heatmap (HTML view)
|
|
243
245
|
def heatmap_view
|
|
244
|
-
@days =
|
|
246
|
+
@days = bounded_param(:days, default: 7, max: 90)
|
|
245
247
|
events = fetch_events_for_period(days: @days)
|
|
246
248
|
heatmap = Visualization::ActivityHeatmap.new(events)
|
|
247
249
|
@heatmap_data = heatmap.to_data
|
|
@@ -362,8 +364,8 @@ module Lyra
|
|
|
362
364
|
|
|
363
365
|
# GET /lyra/dashboard/visualizations/model_heatmap/:model_class
|
|
364
366
|
def model_heatmap
|
|
365
|
-
model_class = params[:model_class]
|
|
366
|
-
events = fetch_events_for_model(model_class, days:
|
|
367
|
+
model_class = monitored_model!(params[:model_class])
|
|
368
|
+
events = fetch_events_for_model(model_class, days: bounded_param(:days, default: 7, max: 90))
|
|
367
369
|
heatmap = Visualization::ActivityHeatmap.new(events)
|
|
368
370
|
|
|
369
371
|
render json: {
|
|
@@ -399,7 +401,7 @@ module Lyra
|
|
|
399
401
|
def collect_events_for_model(model_class)
|
|
400
402
|
# Collect all events for a model class
|
|
401
403
|
events = []
|
|
402
|
-
model_class.pluck(:id).each do |id|
|
|
404
|
+
model_class.lyra_processable.pluck(:id).each do |id|
|
|
403
405
|
stream_name = "#{model_class.name}$#{id}"
|
|
404
406
|
begin
|
|
405
407
|
events.concat(Lyra.config.event_store.read.stream(stream_name).to_a)
|
|
@@ -416,7 +418,7 @@ module Lyra
|
|
|
416
418
|
model_class.column_names.include?(field.to_s)
|
|
417
419
|
end
|
|
418
420
|
|
|
419
|
-
scope = model_class.order(updated_at: :desc)
|
|
421
|
+
scope = model_class.lyra_processable.order(updated_at: :desc)
|
|
420
422
|
|
|
421
423
|
# Apply search if provided
|
|
422
424
|
if search_query.present? && display_fields.any?
|
|
@@ -460,7 +462,7 @@ module Lyra
|
|
|
460
462
|
def fetch_recent_events(limit: 100)
|
|
461
463
|
events = []
|
|
462
464
|
Lyra.config.monitored_models.each do |model_class|
|
|
463
|
-
model_class.order(updated_at: :desc).limit(limit / Lyra.config.monitored_models.size).each do |record|
|
|
465
|
+
model_class.lyra_processable.order(updated_at: :desc).limit(limit / Lyra.config.monitored_models.size).each do |record|
|
|
464
466
|
stream_name = "#{model_class.name}$#{record.id}"
|
|
465
467
|
begin
|
|
466
468
|
stream_events = Lyra.config.event_store.read.stream(stream_name).to_a
|
|
@@ -470,15 +472,15 @@ module Lyra
|
|
|
470
472
|
end
|
|
471
473
|
end
|
|
472
474
|
end
|
|
473
|
-
events.sort_by { |e| event_timestamp(e) }.last(limit)
|
|
475
|
+
rights_visible(events.sort_by { |e| event_timestamp(e) }.last(limit))
|
|
474
476
|
end
|
|
475
477
|
|
|
476
478
|
def fetch_filtered_events(model_class: nil, operation: nil, limit: 50)
|
|
477
479
|
events = []
|
|
478
|
-
models = model_class.present? ? [model_class
|
|
480
|
+
models = model_class.present? ? [monitored_model!(model_class)] : Lyra.config.monitored_models
|
|
479
481
|
|
|
480
482
|
models.each do |klass|
|
|
481
|
-
klass.order(updated_at: :desc).limit(limit).each do |record|
|
|
483
|
+
klass.lyra_processable.order(updated_at: :desc).limit(limit).each do |record|
|
|
482
484
|
stream_name = "#{klass.name}$#{record.id}"
|
|
483
485
|
begin
|
|
484
486
|
stream_events = Lyra.config.event_store.read.stream(stream_name).to_a
|
|
@@ -497,7 +499,7 @@ module Lyra
|
|
|
497
499
|
end
|
|
498
500
|
end
|
|
499
501
|
|
|
500
|
-
events.sort_by { |e| event_timestamp(e) }.last(limit)
|
|
502
|
+
rights_visible(events.sort_by { |e| event_timestamp(e) }.last(limit))
|
|
501
503
|
end
|
|
502
504
|
|
|
503
505
|
def fetch_entity_events(model_class, model_id, depth: 1)
|
|
@@ -516,7 +518,7 @@ module Lyra
|
|
|
516
518
|
# Find events sharing correlation IDs
|
|
517
519
|
correlation_ids.each do |corr_id|
|
|
518
520
|
Lyra.config.monitored_models.each do |klass|
|
|
519
|
-
klass.order(updated_at: :desc).limit(20).each do |record|
|
|
521
|
+
klass.lyra_processable.order(updated_at: :desc).limit(20).each do |record|
|
|
520
522
|
other_stream = "#{klass.name}$#{record.id}"
|
|
521
523
|
next if other_stream == stream_name
|
|
522
524
|
|
|
@@ -535,7 +537,7 @@ module Lyra
|
|
|
535
537
|
# Stream doesn't exist
|
|
536
538
|
end
|
|
537
539
|
|
|
538
|
-
events.uniq { |e| e.event_id }.sort_by { |e| event_timestamp(e) }
|
|
540
|
+
rights_visible(events.uniq { |e| e.event_id }.sort_by { |e| event_timestamp(e) })
|
|
539
541
|
end
|
|
540
542
|
|
|
541
543
|
def record_display_name(record)
|
|
@@ -559,7 +561,7 @@ module Lyra
|
|
|
559
561
|
events = []
|
|
560
562
|
|
|
561
563
|
Lyra.config.monitored_models.each do |model_class|
|
|
562
|
-
model_class.where("updated_at >= ?", cutoff).find_each do |record|
|
|
564
|
+
model_class.lyra_processable.where("updated_at >= ?", cutoff).find_each do |record|
|
|
563
565
|
stream_name = "#{model_class.name}$#{record.id}"
|
|
564
566
|
begin
|
|
565
567
|
stream_events = Lyra.config.event_store.read.stream(stream_name).to_a
|
|
@@ -570,14 +572,14 @@ module Lyra
|
|
|
570
572
|
end
|
|
571
573
|
end
|
|
572
574
|
|
|
573
|
-
events.sort_by { |e| event_timestamp(e) }
|
|
575
|
+
rights_visible(events.sort_by { |e| event_timestamp(e) })
|
|
574
576
|
end
|
|
575
577
|
|
|
576
578
|
def fetch_events_for_model(model_class, days: 7)
|
|
577
579
|
cutoff = days.days.ago
|
|
578
580
|
events = []
|
|
579
581
|
|
|
580
|
-
model_class.where("updated_at >= ?", cutoff).find_each do |record|
|
|
582
|
+
model_class.lyra_processable.where("updated_at >= ?", cutoff).find_each do |record|
|
|
581
583
|
stream_name = "#{model_class.name}$#{record.id}"
|
|
582
584
|
begin
|
|
583
585
|
stream_events = Lyra.config.event_store.read.stream(stream_name).to_a
|
|
@@ -587,7 +589,7 @@ module Lyra
|
|
|
587
589
|
end
|
|
588
590
|
end
|
|
589
591
|
|
|
590
|
-
events.sort_by { |e| event_timestamp(e) }
|
|
592
|
+
rights_visible(events.sort_by { |e| event_timestamp(e) })
|
|
591
593
|
end
|
|
592
594
|
|
|
593
595
|
def event_timestamp(event)
|
|
@@ -5,9 +5,12 @@ module Lyra
|
|
|
5
5
|
subject_id = params[:subject_id]
|
|
6
6
|
subject_type = params[:subject_type]
|
|
7
7
|
|
|
8
|
+
subject_model = subject_type && Lyra.config.monitored_model_named(subject_type)
|
|
9
|
+
refuse_restricted_record!(subject_model, subject_id) if subject_model && subject_id
|
|
8
10
|
flow = EventFlow.new(
|
|
9
11
|
subject_id: subject_id,
|
|
10
|
-
subject_type: subject_type
|
|
12
|
+
subject_type: subject_type,
|
|
13
|
+
event_filter: method(:rights_visible)
|
|
11
14
|
)
|
|
12
15
|
|
|
13
16
|
@flow_data = flow.flow_data
|
|
@@ -39,8 +42,9 @@ module Lyra
|
|
|
39
42
|
|
|
40
43
|
# GET /lyra/flow/event_chain/:model_class/:model_id
|
|
41
44
|
def event_chain
|
|
42
|
-
model_class = params[:model_class]
|
|
45
|
+
model_class = monitored_model!(params[:model_class])
|
|
43
46
|
model_id = params[:model_id]
|
|
47
|
+
refuse_restricted_record!(model_class, model_id)
|
|
44
48
|
|
|
45
49
|
flow = EventFlow.new
|
|
46
50
|
chain = flow.reconstruct_state_chain(model_class.name, model_id)
|
|
@@ -63,8 +67,9 @@ module Lyra
|
|
|
63
67
|
format.json { render json: mapping }
|
|
64
68
|
end
|
|
65
69
|
else
|
|
66
|
-
#
|
|
67
|
-
events =
|
|
70
|
+
# Summary of the CRUD operations in the most recent events (bounded: recent_events)
|
|
71
|
+
events = recent_events
|
|
72
|
+
log_dashboard_read("flow/crud_mapping", {}, returned: events.size)
|
|
68
73
|
@models_summary = events.group_by { |e| event_model_class(e) }.transform_values do |model_events|
|
|
69
74
|
{
|
|
70
75
|
total: model_events.count,
|
|
@@ -80,7 +85,8 @@ module Lyra
|
|
|
80
85
|
render json: {
|
|
81
86
|
total_events: @total_events,
|
|
82
87
|
operations: @operations_summary,
|
|
83
|
-
models: @models_summary
|
|
88
|
+
models: @models_summary,
|
|
89
|
+
window: @window
|
|
84
90
|
}
|
|
85
91
|
end
|
|
86
92
|
end
|
|
@@ -89,8 +95,9 @@ module Lyra
|
|
|
89
95
|
|
|
90
96
|
# GET /lyra/flow/visualization/:model_class/:model_id
|
|
91
97
|
def visualization
|
|
92
|
-
model_class = params[:model_class]
|
|
98
|
+
model_class = monitored_model!(params[:model_class])
|
|
93
99
|
model_id = params[:model_id]
|
|
100
|
+
refuse_restricted_record!(model_class, model_id)
|
|
94
101
|
format = params[:format] || 'json'
|
|
95
102
|
|
|
96
103
|
stream_name = "#{model_class.name}$#{model_id}"
|
|
@@ -116,9 +123,12 @@ module Lyra
|
|
|
116
123
|
def correlation
|
|
117
124
|
correlation_id = params[:correlation_id]
|
|
118
125
|
|
|
119
|
-
events
|
|
126
|
+
# Among the most recent events (bounded: recent_events); a correlation older
|
|
127
|
+
# than the window is not found, and window says how far back it looked.
|
|
128
|
+
events = recent_events.select do |event|
|
|
120
129
|
event.metadata[:correlation_id] == correlation_id
|
|
121
130
|
end
|
|
131
|
+
log_dashboard_read("flow/correlation", { correlation_id: correlation_id }, returned: events.size)
|
|
122
132
|
|
|
123
133
|
flow = EventFlow.new
|
|
124
134
|
timeline = flow.build_timeline(events)
|
|
@@ -129,7 +139,8 @@ module Lyra
|
|
|
129
139
|
started_at: events.min_by(&:timestamp)&.timestamp,
|
|
130
140
|
completed_at: events.max_by(&:timestamp)&.timestamp,
|
|
131
141
|
events: timeline,
|
|
132
|
-
|
|
142
|
+
window: @window,
|
|
143
|
+
privacy_impact: pam_dsl_available? ? events.count { |e| Privacy::PIIDetector.detect(event_fields(e)).any? } : nil
|
|
133
144
|
}
|
|
134
145
|
end
|
|
135
146
|
|
|
@@ -137,24 +148,30 @@ module Lyra
|
|
|
137
148
|
def user_actions
|
|
138
149
|
user_id = params[:user_id]
|
|
139
150
|
|
|
140
|
-
events =
|
|
151
|
+
# Among the most recent events, and at most ?actions= of them listed (default 100,
|
|
152
|
+
# at most 500): one staff member's whole activity is not one request's answer.
|
|
153
|
+
events = recent_events.select do |event|
|
|
141
154
|
event.metadata[:user_id] == user_id.to_i
|
|
142
155
|
end
|
|
156
|
+
log_dashboard_read("flow/user_actions", { user_id: user_id }, returned: events.size)
|
|
143
157
|
|
|
144
158
|
actions = events.group_by { |e| e.metadata[:action_id] }
|
|
159
|
+
listed = bounded_param(:actions, default: 100, max: 500)
|
|
145
160
|
|
|
146
161
|
render json: {
|
|
147
162
|
user_id: user_id,
|
|
148
163
|
total_actions: actions.count,
|
|
149
164
|
total_events: events.count,
|
|
150
|
-
|
|
165
|
+
actions_listed: [actions.count, listed].min,
|
|
166
|
+
window: @window,
|
|
167
|
+
actions: actions.first(listed).map do |action_id, action_events|
|
|
151
168
|
{
|
|
152
169
|
action_id: action_id,
|
|
153
170
|
user_action: action_events.first.metadata[:user_action],
|
|
154
171
|
timestamp: event_timestamp(action_events.min_by { |e| event_timestamp(e) }),
|
|
155
172
|
events_count: action_events.count,
|
|
156
173
|
models_affected: action_events.map { |e| event_model_class(e) }.uniq,
|
|
157
|
-
pii_affected: pam_dsl_available? ? action_events.any? { |e| Privacy::PIIDetector.detect(
|
|
174
|
+
pii_affected: pam_dsl_available? ? action_events.any? { |e| Privacy::PIIDetector.detect(event_fields(e)).any? } : nil
|
|
158
175
|
}
|
|
159
176
|
end
|
|
160
177
|
}
|
|
@@ -196,29 +213,10 @@ module Lyra
|
|
|
196
213
|
data[:timestamp] || data["timestamp"] || event.metadata[:timestamp]
|
|
197
214
|
end
|
|
198
215
|
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
return attrs if attrs.is_a?(Hash)
|
|
204
|
-
end
|
|
205
|
-
# Fall back to data hash for RubyEventStore events
|
|
206
|
-
data = event.respond_to?(:data) ? event.data : nil
|
|
207
|
-
return {} unless data
|
|
208
|
-
data[:attributes] || data["attributes"] || {}
|
|
209
|
-
end
|
|
210
|
-
|
|
211
|
-
def load_events(subject_id, subject_type)
|
|
212
|
-
events = Lyra.config.event_store.read.to_a
|
|
213
|
-
|
|
214
|
-
if subject_id && subject_type
|
|
215
|
-
events = events.select do |event|
|
|
216
|
-
event.metadata[:user_id] == subject_id.to_i ||
|
|
217
|
-
(event_model_class(event) == subject_type && event_model_id(event) == subject_id.to_i)
|
|
218
|
-
end
|
|
219
|
-
end
|
|
220
|
-
|
|
221
|
-
events
|
|
216
|
+
# The fields an event gives or changes, whatever mode wrote it
|
|
217
|
+
# (Lyra::Event.fields_of): what personal data is detected in.
|
|
218
|
+
def event_fields(event)
|
|
219
|
+
Lyra::Event.fields_of(event)
|
|
222
220
|
end
|
|
223
221
|
end
|
|
224
222
|
end
|