orfeas_lyra 0.9.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +2021 -17
  3. data/README.md +22 -9
  4. data/Rakefile +40 -1
  5. data/app/controllers/lyra/application_controller.rb +127 -0
  6. data/app/controllers/lyra/dashboard_controller.rb +33 -31
  7. data/app/controllers/lyra/flow_controller.rb +32 -34
  8. data/app/controllers/lyra/privacy_controller.rb +57 -18
  9. data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
  10. data/app/views/lyra/dashboard/index.html.erb +1 -0
  11. data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
  12. data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
  13. data/app/views/lyra/privacy/policy.html.erb +7 -0
  14. data/app/views/lyra/privacy/rights.html.erb +78 -0
  15. data/config/privacy_policies.rb +0 -3
  16. data/config/routes.rb +1 -0
  17. data/lib/lyra/access_log.rb +187 -6
  18. data/lib/lyra/advisory_lock.rb +236 -0
  19. data/lib/lyra/aggregate.rb +1 -1
  20. data/lib/lyra/associations/habtm_links.rb +382 -19
  21. data/lib/lyra/associations/maintenance.rb +229 -0
  22. data/lib/lyra/attribution.rb +48 -0
  23. data/lib/lyra/bypass_events.rb +446 -46
  24. data/lib/lyra/command_handler.rb +19 -14
  25. data/lib/lyra/configuration.rb +237 -10
  26. data/lib/lyra/consistency/read_your_writes.rb +4 -2
  27. data/lib/lyra/correlation.rb +121 -0
  28. data/lib/lyra/domain_events.rb +4 -0
  29. data/lib/lyra/dual_view.rb +53 -16
  30. data/lib/lyra/encrypted_attributes.rb +5 -0
  31. data/lib/lyra/engine.rb +67 -0
  32. data/lib/lyra/erasure.rb +1011 -106
  33. data/lib/lyra/event.rb +35 -1
  34. data/lib/lyra/event_data.rb +38 -0
  35. data/lib/lyra/event_flow.rb +31 -15
  36. data/lib/lyra/event_store_adapter.rb +28 -0
  37. data/lib/lyra/event_window.rb +58 -0
  38. data/lib/lyra/genesis.rb +548 -44
  39. data/lib/lyra/interceptors/association_interceptor.rb +689 -34
  40. data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
  41. data/lib/lyra/internal.rb +26 -0
  42. data/lib/lyra/mode_sync.rb +51 -3
  43. data/lib/lyra/mode_transition.rb +257 -30
  44. data/lib/lyra/model_selection.rb +35 -0
  45. data/lib/lyra/own_connection.rb +17 -2
  46. data/lib/lyra/privacy/adapters/pam.rb +25 -2
  47. data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
  48. data/lib/lyra/privacy/interface.rb +27 -7
  49. data/lib/lyra/privacy/subject_links.rb +143 -0
  50. data/lib/lyra/privacy/subject_records.rb +369 -0
  51. data/lib/lyra/projection.rb +7 -1
  52. data/lib/lyra/projections/async_checkpoint.rb +32 -0
  53. data/lib/lyra/projections/async_projection_job.rb +166 -23
  54. data/lib/lyra/projections/cached_joins.rb +33 -18
  55. data/lib/lyra/projections/cached_projection.rb +74 -33
  56. data/lib/lyra/projections/cached_relation.rb +3175 -321
  57. data/lib/lyra/projections/default_scopes.rb +254 -0
  58. data/lib/lyra/projections/deferred_relation.rb +23 -3
  59. data/lib/lyra/projections/event_store_reader.rb +47 -8
  60. data/lib/lyra/projections/foreign_keys.rb +92 -0
  61. data/lib/lyra/projections/lazy_projection.rb +165 -27
  62. data/lib/lyra/projections/model_projection.rb +165 -14
  63. data/lib/lyra/projections/rebuild.rb +34 -6
  64. data/lib/lyra/projections/reference_links.rb +463 -72
  65. data/lib/lyra/projections/typed_values.rb +629 -0
  66. data/lib/lyra/projections/unique_indexes.rb +595 -0
  67. data/lib/lyra/projections/uniqueness.rb +784 -0
  68. data/lib/lyra/projections/value_digest.rb +93 -0
  69. data/lib/lyra/purpose_bound_reads.rb +912 -77
  70. data/lib/lyra/repair.rb +119 -28
  71. data/lib/lyra/retention.rb +114 -8
  72. data/lib/lyra/rights/consent_store.rb +415 -0
  73. data/lib/lyra/rights/erasure_guard.rb +135 -0
  74. data/lib/lyra/rights/pending.rb +74 -0
  75. data/lib/lyra/rights/queries.rb +269 -0
  76. data/lib/lyra/rights/store.rb +512 -0
  77. data/lib/lyra/rights/subject_keys.rb +224 -0
  78. data/lib/lyra/rights.rb +281 -0
  79. data/lib/lyra/saves_without_effect.rb +69 -0
  80. data/lib/lyra/stream_tail.rb +319 -0
  81. data/lib/lyra/strict_data_access.rb +916 -106
  82. data/lib/lyra/temporal.rb +4 -2
  83. data/lib/lyra/touch_registry.rb +404 -0
  84. data/lib/lyra/verification/bypass_workflow.rb +452 -42
  85. data/lib/lyra/verification/rights_workflows.rb +170 -0
  86. data/lib/lyra/version.rb +1 -1
  87. data/lib/lyra/visualization/event_graph.rb +1 -1
  88. data/lib/lyra/visualization/timeline.rb +2 -2
  89. data/lib/lyra.rb +43 -2
  90. data/lib/tasks/lyra_mode.rake +110 -8
  91. data/lib/tasks/lyra_projections.rake +43 -2
  92. metadata +34 -4
  93. data/lib/lyra/associations/event_aware.rb +0 -224
data/lib/lyra/event.rb CHANGED
@@ -18,12 +18,46 @@ module Lyra
18
18
  data = event.data.is_a?(Hash) ? event.data : {}
19
19
  return nil if data[:replay] == false || data["replay"] == false
20
20
 
21
+ recorded_operation(data) || operation_from_name(event)
22
+ end
23
+
24
+ # The operation the data records, or nil.
25
+ def self.recorded_operation(data)
21
26
  case (data[:operation] || data["operation"]).to_s
22
27
  when "created", "create" then :created
23
28
  when "imported" then :imported
24
29
  when "updated", "update" then :updated
25
30
  when "destroyed", "destroy", "deleted" then :destroyed
26
- when "" then operation_from_name(event)
31
+ end
32
+ end
33
+
34
+ # The attributes a create or an import gives its record: the event's
35
+ # attributes, every column, created_at and updated_at included. They are
36
+ # ordinary columns from 1.0.0, in every mode and every event; logs written
37
+ # by earlier versions, whose Monitor creates left the two out, are not
38
+ # supported (CHANGELOG, 1.0.0, Breaking).
39
+ def self.created_attributes(data)
40
+ data = {} unless data.is_a?(Hash)
41
+ data[:attributes] || data["attributes"] || {}
42
+ end
43
+
44
+ # The fields an event gives or changes, as name => value: a create's or an
45
+ # import's attributes, an update's changes as their new values, and for a
46
+ # destroy whatever attributes the event holds (none from Lyra::EventData).
47
+ # What the
48
+ # privacy views and reports detect personal data in, whatever mode wrote
49
+ # the event.
50
+ def self.fields_of(event)
51
+ data = event.respond_to?(:data) && event.data.is_a?(Hash) ? event.data : nil
52
+ return(event.respond_to?(:attributes) && event.attributes.is_a?(Hash) ? event.attributes : {}) unless data
53
+
54
+ case recorded_operation(data)
55
+ when :created, :imported
56
+ created_attributes(data)
57
+ when :updated
58
+ (data[:changes] || data["changes"] || {}).to_h { |field, change| [field, change.is_a?(Array) ? change.last : change] }
59
+ else
60
+ data[:attributes] || data["attributes"] || {}
27
61
  end
28
62
  end
29
63
 
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Lyra
4
+ # The data payload of the event a write produces: one shape, in every mode
5
+ # and for every writer of CRUD events (Monitor's callbacks, the command
6
+ # handler of Hijack and event sourcing, bypass writes, Genesis and repair).
7
+ #
8
+ # operation attributes changes
9
+ # created, imported the record's attributes {}
10
+ # updated {} the delta, [old, new] pairs
11
+ # destroyed {} {}
12
+ #
13
+ # Replay reads only a create's attributes and an update's changes, so state is
14
+ # derived the same way whichever mode wrote the event. The record is not
15
+ # repeated on an update, nor kept on a destroy: its state is in the events
16
+ # before, and every copy of personal data in the log is a copy the erasure
17
+ # executor must rewrite.
18
+ #
19
+ # Logs written before 1.0.0, in the earlier per-mode shapes, are not
20
+ # supported (CHANGELOG, 1.0.0, Breaking).
21
+ module EventData
22
+ CREATES = %i[created imported].freeze
23
+
24
+ module_function
25
+
26
+ def build(model_class, model_id, operation, attributes: {}, changes: {}, timestamp: Time.current)
27
+ kind = operation.to_sym
28
+ {
29
+ model_class: model_class.respond_to?(:name) ? model_class.name : model_class.to_s,
30
+ model_id: model_id,
31
+ operation: operation,
32
+ attributes: CREATES.include?(kind) ? attributes : {},
33
+ changes: kind == :updated ? changes : {},
34
+ timestamp: timestamp
35
+ }
36
+ end
37
+ end
38
+ end
@@ -1,9 +1,12 @@
1
1
  module Lyra
2
2
  # Visualizes event flows and chains
3
3
  class EventFlow
4
- attr_reader :subject_id, :subject_type, :time_range
4
+ attr_reader :subject_id, :subject_type, :time_range, :window
5
5
 
6
- def initialize(subject_id: nil, subject_type: nil, time_range: nil)
6
+ # +event_filter+: a callable given the events read, returning those to
7
+ # show (the dashboard leaves out restricted subjects' events).
8
+ def initialize(subject_id: nil, subject_type: nil, time_range: nil, event_filter: nil)
9
+ @event_filter = event_filter
7
10
  @subject_id = subject_id
8
11
  @subject_type = subject_type
9
12
  @time_range = time_range || (30.days.ago..Time.current)
@@ -18,7 +21,8 @@ module Lyra
18
21
  timeline: build_timeline(events),
19
22
  flows: build_flows(grouped),
20
23
  statistics: calculate_statistics(events),
21
- privacy_impact: analyze_privacy_impact(events)
24
+ privacy_impact: analyze_privacy_impact(events),
25
+ window: window
22
26
  }
23
27
  end
24
28
 
@@ -226,24 +230,36 @@ module Lyra
226
230
  event.data[:changes] || event.data["changes"] || {}
227
231
  end
228
232
 
233
+ # A bounded read, never the whole log: with a subject, the newest
234
+ # config.dashboard_max_events events of that record's stream (what a user
235
+ # did to other records is the user_actions route, not their timeline);
236
+ # without one, the newest that many events of the log (Lyra::EventWindow).
237
+ # Oldest first. @window says what was read.
229
238
  def load_events
230
- events = Lyra.config.event_store.read.to_a
231
-
232
- # Filter by subject if specified
233
- if subject_id && subject_type
234
- events = events.select { |e| event_relates_to_subject?(e) }
235
- end
239
+ max = Lyra.config.dashboard_max_events
240
+ events = subject_id && subject_type ? subject_stream_events(max) : recent_events(max)
236
241
 
237
242
  # Filter by time range
238
243
  events = events.select { |e| time_range.cover?(parse_timestamp(event_timestamp(e))) }
239
244
 
240
- events
245
+ @event_filter ? @event_filter.call(events) : events
246
+ end
247
+
248
+ def subject_stream_events(max)
249
+ model = Lyra.config.monitored_model_named(subject_type)
250
+ unless model
251
+ @window = { limit: max, returned: 0, truncated: false }
252
+ return []
253
+ end
254
+
255
+ newest = Lyra.config.event_store.read.stream(Lyra.stream_name(model, subject_id)).backward.limit(max + 1).to_a
256
+ @window = { limit: max, returned: [newest.size, max].min, truncated: newest.size > max }
257
+ newest.first(max).reverse
241
258
  end
242
259
 
243
- def event_relates_to_subject?(event)
244
- event.metadata[:user_id] == subject_id ||
245
- event.data[:user_id] == subject_id ||
246
- (event_model_class(event) == subject_type && event_model_id(event) == subject_id)
260
+ def recent_events(max)
261
+ newest, @window = Lyra::EventWindow.recent(max)
262
+ newest.reverse
247
263
  end
248
264
 
249
265
  def group_by_correlation(events)
@@ -305,7 +321,7 @@ module Lyra
305
321
  def detect_pii(event)
306
322
  return {} unless Lyra.privacy_features_available?
307
323
 
308
- Lyra::Privacy::PIIDetector.detect(event_attributes(event))
324
+ Lyra::Privacy::PIIDetector.detect(Lyra::Event.fields_of(event))
309
325
  end
310
326
 
311
327
  def has_pii?(event)
@@ -65,4 +65,32 @@ module Lyra
65
65
  end
66
66
  private_class_method :create_own_table_on
67
67
 
68
+ # Add to Lyra's own table +name+ the +columns+ (name => [type, options]) it
69
+ # lacks, as when an earlier version created it. Like create_own_table, outside
70
+ # the caller's transaction (on a connection of its own when one is open), so a
71
+ # rollback of the caller's work cannot take a column back while Lyra goes on
72
+ # as if it were there; and safely when two processes add the same column.
73
+ def self.add_own_columns(connection, name, columns)
74
+ missing = columns.reject { |column, _| connection.column_exists?(name, column) }
75
+ return if missing.empty?
76
+ return add_own_columns_on(connection, name, missing) unless connection.transaction_open?
77
+
78
+ own = connection.pool.db_config.new_connection
79
+ begin
80
+ add_own_columns_on(own, name, missing)
81
+ ensure
82
+ own.disconnect!
83
+ end
84
+ connection.schema_cache.clear_data_source_cache!(name.to_s)
85
+ end
86
+
87
+ def self.add_own_columns_on(connection, name, columns)
88
+ columns.each do |column, (type, options)|
89
+ connection.add_column(name, column, type, **(options || {}), if_not_exists: true)
90
+ rescue ActiveRecord::StatementInvalid
91
+ raise unless connection.column_exists?(name, column)
92
+ end
93
+ end
94
+ private_class_method :add_own_columns_on
95
+
68
96
  end
@@ -0,0 +1,58 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Lyra
4
+ # The newest events of the log, for a reader that must not read all of it:
5
+ # the dashboard's routes and Lyra::EventFlow. At most +limit+ events, newest
6
+ # first, and a window that says what was read.
7
+ #
8
+ # The access log's own entries are skipped (ACCESS_LOG_TYPES).
9
+ # With config.record_access_events on, every read adds one, so they would
10
+ # fill the window and push out the events it is for. Skipping them costs a
11
+ # scan, so the scan stops after SCAN_FACTOR x limit events: a log that is
12
+ # mostly access entries still bounds the work, and the window then says it
13
+ # stopped early. SaveWithoutEffect markers (Lyra::SavesWithoutEffect), which
14
+ # record no change, are skipped too.
15
+ module EventWindow
16
+ SCAN_FACTOR = 10
17
+ # Lyra::AccessLog's event types, named here because the access log loads
18
+ # only with PAM DSL.
19
+ ACCESS_LOG_TYPES = %w[
20
+ Lyra::Events::DataAccessed Lyra::Events::DataAccessDenied
21
+ Lyra::Events::DashboardRead Lyra::Events::AccessLogErased
22
+ ].freeze
23
+ SKIPPED_TYPES = (ACCESS_LOG_TYPES + [Lyra::SavesWithoutEffect::TYPE]).freeze
24
+ BATCH = 1_000
25
+
26
+ module_function
27
+
28
+ # [events, window]. window: limit, returned, truncated (older events may
29
+ # exist that were not read) and skipped (access-log entries and markers
30
+ # passed over).
31
+ def recent(limit)
32
+ cap = limit * SCAN_FACTOR
33
+ events = []
34
+ scanned = 0
35
+ skipped = 0
36
+ truncated = false
37
+
38
+ Lyra.config.event_store.read.backward.limit(cap + 1).in_batches([limit + 1, BATCH].min).each do |event|
39
+ scanned += 1
40
+ if scanned > cap
41
+ truncated = true
42
+ break
43
+ end
44
+ if SKIPPED_TYPES.include?(event.event_type)
45
+ skipped += 1
46
+ next
47
+ end
48
+ if events.size == limit
49
+ truncated = true
50
+ break
51
+ end
52
+ events << event
53
+ end
54
+
55
+ [events, { limit: limit, returned: events.size, truncated: truncated, skipped: skipped }]
56
+ end
57
+ end
58
+ end