orfeas_lyra 0.9.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +2021 -17
- data/README.md +22 -9
- data/Rakefile +40 -1
- data/app/controllers/lyra/application_controller.rb +127 -0
- data/app/controllers/lyra/dashboard_controller.rb +33 -31
- data/app/controllers/lyra/flow_controller.rb +32 -34
- data/app/controllers/lyra/privacy_controller.rb +57 -18
- data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
- data/app/views/lyra/dashboard/index.html.erb +1 -0
- data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
- data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
- data/app/views/lyra/privacy/policy.html.erb +7 -0
- data/app/views/lyra/privacy/rights.html.erb +78 -0
- data/config/privacy_policies.rb +0 -3
- data/config/routes.rb +1 -0
- data/lib/lyra/access_log.rb +187 -6
- data/lib/lyra/advisory_lock.rb +236 -0
- data/lib/lyra/aggregate.rb +1 -1
- data/lib/lyra/associations/habtm_links.rb +382 -19
- data/lib/lyra/associations/maintenance.rb +229 -0
- data/lib/lyra/attribution.rb +48 -0
- data/lib/lyra/bypass_events.rb +446 -46
- data/lib/lyra/command_handler.rb +19 -14
- data/lib/lyra/configuration.rb +237 -10
- data/lib/lyra/consistency/read_your_writes.rb +4 -2
- data/lib/lyra/correlation.rb +121 -0
- data/lib/lyra/domain_events.rb +4 -0
- data/lib/lyra/dual_view.rb +53 -16
- data/lib/lyra/encrypted_attributes.rb +5 -0
- data/lib/lyra/engine.rb +67 -0
- data/lib/lyra/erasure.rb +1011 -106
- data/lib/lyra/event.rb +35 -1
- data/lib/lyra/event_data.rb +38 -0
- data/lib/lyra/event_flow.rb +31 -15
- data/lib/lyra/event_store_adapter.rb +28 -0
- data/lib/lyra/event_window.rb +58 -0
- data/lib/lyra/genesis.rb +548 -44
- data/lib/lyra/interceptors/association_interceptor.rb +689 -34
- data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
- data/lib/lyra/internal.rb +26 -0
- data/lib/lyra/mode_sync.rb +51 -3
- data/lib/lyra/mode_transition.rb +257 -30
- data/lib/lyra/model_selection.rb +35 -0
- data/lib/lyra/own_connection.rb +17 -2
- data/lib/lyra/privacy/adapters/pam.rb +25 -2
- data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
- data/lib/lyra/privacy/interface.rb +27 -7
- data/lib/lyra/privacy/subject_links.rb +143 -0
- data/lib/lyra/privacy/subject_records.rb +369 -0
- data/lib/lyra/projection.rb +7 -1
- data/lib/lyra/projections/async_checkpoint.rb +32 -0
- data/lib/lyra/projections/async_projection_job.rb +166 -23
- data/lib/lyra/projections/cached_joins.rb +33 -18
- data/lib/lyra/projections/cached_projection.rb +74 -33
- data/lib/lyra/projections/cached_relation.rb +3175 -321
- data/lib/lyra/projections/default_scopes.rb +254 -0
- data/lib/lyra/projections/deferred_relation.rb +23 -3
- data/lib/lyra/projections/event_store_reader.rb +47 -8
- data/lib/lyra/projections/foreign_keys.rb +92 -0
- data/lib/lyra/projections/lazy_projection.rb +165 -27
- data/lib/lyra/projections/model_projection.rb +165 -14
- data/lib/lyra/projections/rebuild.rb +34 -6
- data/lib/lyra/projections/reference_links.rb +463 -72
- data/lib/lyra/projections/typed_values.rb +629 -0
- data/lib/lyra/projections/unique_indexes.rb +595 -0
- data/lib/lyra/projections/uniqueness.rb +784 -0
- data/lib/lyra/projections/value_digest.rb +93 -0
- data/lib/lyra/purpose_bound_reads.rb +912 -77
- data/lib/lyra/repair.rb +119 -28
- data/lib/lyra/retention.rb +114 -8
- data/lib/lyra/rights/consent_store.rb +415 -0
- data/lib/lyra/rights/erasure_guard.rb +135 -0
- data/lib/lyra/rights/pending.rb +74 -0
- data/lib/lyra/rights/queries.rb +269 -0
- data/lib/lyra/rights/store.rb +512 -0
- data/lib/lyra/rights/subject_keys.rb +224 -0
- data/lib/lyra/rights.rb +281 -0
- data/lib/lyra/saves_without_effect.rb +69 -0
- data/lib/lyra/stream_tail.rb +319 -0
- data/lib/lyra/strict_data_access.rb +916 -106
- data/lib/lyra/temporal.rb +4 -2
- data/lib/lyra/touch_registry.rb +404 -0
- data/lib/lyra/verification/bypass_workflow.rb +452 -42
- data/lib/lyra/verification/rights_workflows.rb +170 -0
- data/lib/lyra/version.rb +1 -1
- data/lib/lyra/visualization/event_graph.rb +1 -1
- data/lib/lyra/visualization/timeline.rb +2 -2
- data/lib/lyra.rb +43 -2
- data/lib/tasks/lyra_mode.rake +110 -8
- data/lib/tasks/lyra_projections.rake +43 -2
- metadata +34 -4
- data/lib/lyra/associations/event_aware.rb +0 -224
data/lib/lyra/event.rb
CHANGED
|
@@ -18,12 +18,46 @@ module Lyra
|
|
|
18
18
|
data = event.data.is_a?(Hash) ? event.data : {}
|
|
19
19
|
return nil if data[:replay] == false || data["replay"] == false
|
|
20
20
|
|
|
21
|
+
recorded_operation(data) || operation_from_name(event)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# The operation the data records, or nil.
|
|
25
|
+
def self.recorded_operation(data)
|
|
21
26
|
case (data[:operation] || data["operation"]).to_s
|
|
22
27
|
when "created", "create" then :created
|
|
23
28
|
when "imported" then :imported
|
|
24
29
|
when "updated", "update" then :updated
|
|
25
30
|
when "destroyed", "destroy", "deleted" then :destroyed
|
|
26
|
-
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# The attributes a create or an import gives its record: the event's
|
|
35
|
+
# attributes, every column, created_at and updated_at included. They are
|
|
36
|
+
# ordinary columns from 1.0.0, in every mode and every event; logs written
|
|
37
|
+
# by earlier versions, whose Monitor creates left the two out, are not
|
|
38
|
+
# supported (CHANGELOG, 1.0.0, Breaking).
|
|
39
|
+
def self.created_attributes(data)
|
|
40
|
+
data = {} unless data.is_a?(Hash)
|
|
41
|
+
data[:attributes] || data["attributes"] || {}
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# The fields an event gives or changes, as name => value: a create's or an
|
|
45
|
+
# import's attributes, an update's changes as their new values, and for a
|
|
46
|
+
# destroy whatever attributes the event holds (none from Lyra::EventData).
|
|
47
|
+
# What the
|
|
48
|
+
# privacy views and reports detect personal data in, whatever mode wrote
|
|
49
|
+
# the event.
|
|
50
|
+
def self.fields_of(event)
|
|
51
|
+
data = event.respond_to?(:data) && event.data.is_a?(Hash) ? event.data : nil
|
|
52
|
+
return(event.respond_to?(:attributes) && event.attributes.is_a?(Hash) ? event.attributes : {}) unless data
|
|
53
|
+
|
|
54
|
+
case recorded_operation(data)
|
|
55
|
+
when :created, :imported
|
|
56
|
+
created_attributes(data)
|
|
57
|
+
when :updated
|
|
58
|
+
(data[:changes] || data["changes"] || {}).to_h { |field, change| [field, change.is_a?(Array) ? change.last : change] }
|
|
59
|
+
else
|
|
60
|
+
data[:attributes] || data["attributes"] || {}
|
|
27
61
|
end
|
|
28
62
|
end
|
|
29
63
|
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Lyra
|
|
4
|
+
# The data payload of the event a write produces: one shape, in every mode
|
|
5
|
+
# and for every writer of CRUD events (Monitor's callbacks, the command
|
|
6
|
+
# handler of Hijack and event sourcing, bypass writes, Genesis and repair).
|
|
7
|
+
#
|
|
8
|
+
# operation attributes changes
|
|
9
|
+
# created, imported the record's attributes {}
|
|
10
|
+
# updated {} the delta, [old, new] pairs
|
|
11
|
+
# destroyed {} {}
|
|
12
|
+
#
|
|
13
|
+
# Replay reads only a create's attributes and an update's changes, so state is
|
|
14
|
+
# derived the same way whichever mode wrote the event. The record is not
|
|
15
|
+
# repeated on an update, nor kept on a destroy: its state is in the events
|
|
16
|
+
# before, and every copy of personal data in the log is a copy the erasure
|
|
17
|
+
# executor must rewrite.
|
|
18
|
+
#
|
|
19
|
+
# Logs written before 1.0.0, in the earlier per-mode shapes, are not
|
|
20
|
+
# supported (CHANGELOG, 1.0.0, Breaking).
|
|
21
|
+
module EventData
|
|
22
|
+
CREATES = %i[created imported].freeze
|
|
23
|
+
|
|
24
|
+
module_function
|
|
25
|
+
|
|
26
|
+
def build(model_class, model_id, operation, attributes: {}, changes: {}, timestamp: Time.current)
|
|
27
|
+
kind = operation.to_sym
|
|
28
|
+
{
|
|
29
|
+
model_class: model_class.respond_to?(:name) ? model_class.name : model_class.to_s,
|
|
30
|
+
model_id: model_id,
|
|
31
|
+
operation: operation,
|
|
32
|
+
attributes: CREATES.include?(kind) ? attributes : {},
|
|
33
|
+
changes: kind == :updated ? changes : {},
|
|
34
|
+
timestamp: timestamp
|
|
35
|
+
}
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
data/lib/lyra/event_flow.rb
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
module Lyra
|
|
2
2
|
# Visualizes event flows and chains
|
|
3
3
|
class EventFlow
|
|
4
|
-
attr_reader :subject_id, :subject_type, :time_range
|
|
4
|
+
attr_reader :subject_id, :subject_type, :time_range, :window
|
|
5
5
|
|
|
6
|
-
|
|
6
|
+
# +event_filter+: a callable given the events read, returning those to
|
|
7
|
+
# show (the dashboard leaves out restricted subjects' events).
|
|
8
|
+
def initialize(subject_id: nil, subject_type: nil, time_range: nil, event_filter: nil)
|
|
9
|
+
@event_filter = event_filter
|
|
7
10
|
@subject_id = subject_id
|
|
8
11
|
@subject_type = subject_type
|
|
9
12
|
@time_range = time_range || (30.days.ago..Time.current)
|
|
@@ -18,7 +21,8 @@ module Lyra
|
|
|
18
21
|
timeline: build_timeline(events),
|
|
19
22
|
flows: build_flows(grouped),
|
|
20
23
|
statistics: calculate_statistics(events),
|
|
21
|
-
privacy_impact: analyze_privacy_impact(events)
|
|
24
|
+
privacy_impact: analyze_privacy_impact(events),
|
|
25
|
+
window: window
|
|
22
26
|
}
|
|
23
27
|
end
|
|
24
28
|
|
|
@@ -226,24 +230,36 @@ module Lyra
|
|
|
226
230
|
event.data[:changes] || event.data["changes"] || {}
|
|
227
231
|
end
|
|
228
232
|
|
|
233
|
+
# A bounded read, never the whole log: with a subject, the newest
|
|
234
|
+
# config.dashboard_max_events events of that record's stream (what a user
|
|
235
|
+
# did to other records is the user_actions route, not their timeline);
|
|
236
|
+
# without one, the newest that many events of the log (Lyra::EventWindow).
|
|
237
|
+
# Oldest first. @window says what was read.
|
|
229
238
|
def load_events
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
# Filter by subject if specified
|
|
233
|
-
if subject_id && subject_type
|
|
234
|
-
events = events.select { |e| event_relates_to_subject?(e) }
|
|
235
|
-
end
|
|
239
|
+
max = Lyra.config.dashboard_max_events
|
|
240
|
+
events = subject_id && subject_type ? subject_stream_events(max) : recent_events(max)
|
|
236
241
|
|
|
237
242
|
# Filter by time range
|
|
238
243
|
events = events.select { |e| time_range.cover?(parse_timestamp(event_timestamp(e))) }
|
|
239
244
|
|
|
240
|
-
events
|
|
245
|
+
@event_filter ? @event_filter.call(events) : events
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
def subject_stream_events(max)
|
|
249
|
+
model = Lyra.config.monitored_model_named(subject_type)
|
|
250
|
+
unless model
|
|
251
|
+
@window = { limit: max, returned: 0, truncated: false }
|
|
252
|
+
return []
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
newest = Lyra.config.event_store.read.stream(Lyra.stream_name(model, subject_id)).backward.limit(max + 1).to_a
|
|
256
|
+
@window = { limit: max, returned: [newest.size, max].min, truncated: newest.size > max }
|
|
257
|
+
newest.first(max).reverse
|
|
241
258
|
end
|
|
242
259
|
|
|
243
|
-
def
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
(event_model_class(event) == subject_type && event_model_id(event) == subject_id)
|
|
260
|
+
def recent_events(max)
|
|
261
|
+
newest, @window = Lyra::EventWindow.recent(max)
|
|
262
|
+
newest.reverse
|
|
247
263
|
end
|
|
248
264
|
|
|
249
265
|
def group_by_correlation(events)
|
|
@@ -305,7 +321,7 @@ module Lyra
|
|
|
305
321
|
def detect_pii(event)
|
|
306
322
|
return {} unless Lyra.privacy_features_available?
|
|
307
323
|
|
|
308
|
-
Lyra::Privacy::PIIDetector.detect(
|
|
324
|
+
Lyra::Privacy::PIIDetector.detect(Lyra::Event.fields_of(event))
|
|
309
325
|
end
|
|
310
326
|
|
|
311
327
|
def has_pii?(event)
|
|
@@ -65,4 +65,32 @@ module Lyra
|
|
|
65
65
|
end
|
|
66
66
|
private_class_method :create_own_table_on
|
|
67
67
|
|
|
68
|
+
# Add to Lyra's own table +name+ the +columns+ (name => [type, options]) it
|
|
69
|
+
# lacks, as when an earlier version created it. Like create_own_table, outside
|
|
70
|
+
# the caller's transaction (on a connection of its own when one is open), so a
|
|
71
|
+
# rollback of the caller's work cannot take a column back while Lyra goes on
|
|
72
|
+
# as if it were there; and safely when two processes add the same column.
|
|
73
|
+
def self.add_own_columns(connection, name, columns)
|
|
74
|
+
missing = columns.reject { |column, _| connection.column_exists?(name, column) }
|
|
75
|
+
return if missing.empty?
|
|
76
|
+
return add_own_columns_on(connection, name, missing) unless connection.transaction_open?
|
|
77
|
+
|
|
78
|
+
own = connection.pool.db_config.new_connection
|
|
79
|
+
begin
|
|
80
|
+
add_own_columns_on(own, name, missing)
|
|
81
|
+
ensure
|
|
82
|
+
own.disconnect!
|
|
83
|
+
end
|
|
84
|
+
connection.schema_cache.clear_data_source_cache!(name.to_s)
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def self.add_own_columns_on(connection, name, columns)
|
|
88
|
+
columns.each do |column, (type, options)|
|
|
89
|
+
connection.add_column(name, column, type, **(options || {}), if_not_exists: true)
|
|
90
|
+
rescue ActiveRecord::StatementInvalid
|
|
91
|
+
raise unless connection.column_exists?(name, column)
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
private_class_method :add_own_columns_on
|
|
95
|
+
|
|
68
96
|
end
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Lyra
|
|
4
|
+
# The newest events of the log, for a reader that must not read all of it:
|
|
5
|
+
# the dashboard's routes and Lyra::EventFlow. At most +limit+ events, newest
|
|
6
|
+
# first, and a window that says what was read.
|
|
7
|
+
#
|
|
8
|
+
# The access log's own entries are skipped (ACCESS_LOG_TYPES).
|
|
9
|
+
# With config.record_access_events on, every read adds one, so they would
|
|
10
|
+
# fill the window and push out the events it is for. Skipping them costs a
|
|
11
|
+
# scan, so the scan stops after SCAN_FACTOR x limit events: a log that is
|
|
12
|
+
# mostly access entries still bounds the work, and the window then says it
|
|
13
|
+
# stopped early. SaveWithoutEffect markers (Lyra::SavesWithoutEffect), which
|
|
14
|
+
# record no change, are skipped too.
|
|
15
|
+
module EventWindow
|
|
16
|
+
SCAN_FACTOR = 10
|
|
17
|
+
# Lyra::AccessLog's event types, named here because the access log loads
|
|
18
|
+
# only with PAM DSL.
|
|
19
|
+
ACCESS_LOG_TYPES = %w[
|
|
20
|
+
Lyra::Events::DataAccessed Lyra::Events::DataAccessDenied
|
|
21
|
+
Lyra::Events::DashboardRead Lyra::Events::AccessLogErased
|
|
22
|
+
].freeze
|
|
23
|
+
SKIPPED_TYPES = (ACCESS_LOG_TYPES + [Lyra::SavesWithoutEffect::TYPE]).freeze
|
|
24
|
+
BATCH = 1_000
|
|
25
|
+
|
|
26
|
+
module_function
|
|
27
|
+
|
|
28
|
+
# [events, window]. window: limit, returned, truncated (older events may
|
|
29
|
+
# exist that were not read) and skipped (access-log entries and markers
|
|
30
|
+
# passed over).
|
|
31
|
+
def recent(limit)
|
|
32
|
+
cap = limit * SCAN_FACTOR
|
|
33
|
+
events = []
|
|
34
|
+
scanned = 0
|
|
35
|
+
skipped = 0
|
|
36
|
+
truncated = false
|
|
37
|
+
|
|
38
|
+
Lyra.config.event_store.read.backward.limit(cap + 1).in_batches([limit + 1, BATCH].min).each do |event|
|
|
39
|
+
scanned += 1
|
|
40
|
+
if scanned > cap
|
|
41
|
+
truncated = true
|
|
42
|
+
break
|
|
43
|
+
end
|
|
44
|
+
if SKIPPED_TYPES.include?(event.event_type)
|
|
45
|
+
skipped += 1
|
|
46
|
+
next
|
|
47
|
+
end
|
|
48
|
+
if events.size == limit
|
|
49
|
+
truncated = true
|
|
50
|
+
break
|
|
51
|
+
end
|
|
52
|
+
events << event
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
[events, { limit: limit, returned: events.size, truncated: truncated, skipped: skipped }]
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|