orfeas_lyra 0.9.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +2021 -17
  3. data/README.md +22 -9
  4. data/Rakefile +40 -1
  5. data/app/controllers/lyra/application_controller.rb +127 -0
  6. data/app/controllers/lyra/dashboard_controller.rb +33 -31
  7. data/app/controllers/lyra/flow_controller.rb +32 -34
  8. data/app/controllers/lyra/privacy_controller.rb +57 -18
  9. data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
  10. data/app/views/lyra/dashboard/index.html.erb +1 -0
  11. data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
  12. data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
  13. data/app/views/lyra/privacy/policy.html.erb +7 -0
  14. data/app/views/lyra/privacy/rights.html.erb +78 -0
  15. data/config/privacy_policies.rb +0 -3
  16. data/config/routes.rb +1 -0
  17. data/lib/lyra/access_log.rb +187 -6
  18. data/lib/lyra/advisory_lock.rb +236 -0
  19. data/lib/lyra/aggregate.rb +1 -1
  20. data/lib/lyra/associations/habtm_links.rb +382 -19
  21. data/lib/lyra/associations/maintenance.rb +229 -0
  22. data/lib/lyra/attribution.rb +48 -0
  23. data/lib/lyra/bypass_events.rb +446 -46
  24. data/lib/lyra/command_handler.rb +19 -14
  25. data/lib/lyra/configuration.rb +237 -10
  26. data/lib/lyra/consistency/read_your_writes.rb +4 -2
  27. data/lib/lyra/correlation.rb +121 -0
  28. data/lib/lyra/domain_events.rb +4 -0
  29. data/lib/lyra/dual_view.rb +53 -16
  30. data/lib/lyra/encrypted_attributes.rb +5 -0
  31. data/lib/lyra/engine.rb +67 -0
  32. data/lib/lyra/erasure.rb +1011 -106
  33. data/lib/lyra/event.rb +35 -1
  34. data/lib/lyra/event_data.rb +38 -0
  35. data/lib/lyra/event_flow.rb +31 -15
  36. data/lib/lyra/event_store_adapter.rb +28 -0
  37. data/lib/lyra/event_window.rb +58 -0
  38. data/lib/lyra/genesis.rb +548 -44
  39. data/lib/lyra/interceptors/association_interceptor.rb +689 -34
  40. data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
  41. data/lib/lyra/internal.rb +26 -0
  42. data/lib/lyra/mode_sync.rb +51 -3
  43. data/lib/lyra/mode_transition.rb +257 -30
  44. data/lib/lyra/model_selection.rb +35 -0
  45. data/lib/lyra/own_connection.rb +17 -2
  46. data/lib/lyra/privacy/adapters/pam.rb +25 -2
  47. data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
  48. data/lib/lyra/privacy/interface.rb +27 -7
  49. data/lib/lyra/privacy/subject_links.rb +143 -0
  50. data/lib/lyra/privacy/subject_records.rb +369 -0
  51. data/lib/lyra/projection.rb +7 -1
  52. data/lib/lyra/projections/async_checkpoint.rb +32 -0
  53. data/lib/lyra/projections/async_projection_job.rb +166 -23
  54. data/lib/lyra/projections/cached_joins.rb +33 -18
  55. data/lib/lyra/projections/cached_projection.rb +74 -33
  56. data/lib/lyra/projections/cached_relation.rb +3175 -321
  57. data/lib/lyra/projections/default_scopes.rb +254 -0
  58. data/lib/lyra/projections/deferred_relation.rb +23 -3
  59. data/lib/lyra/projections/event_store_reader.rb +47 -8
  60. data/lib/lyra/projections/foreign_keys.rb +92 -0
  61. data/lib/lyra/projections/lazy_projection.rb +165 -27
  62. data/lib/lyra/projections/model_projection.rb +165 -14
  63. data/lib/lyra/projections/rebuild.rb +34 -6
  64. data/lib/lyra/projections/reference_links.rb +463 -72
  65. data/lib/lyra/projections/typed_values.rb +629 -0
  66. data/lib/lyra/projections/unique_indexes.rb +595 -0
  67. data/lib/lyra/projections/uniqueness.rb +784 -0
  68. data/lib/lyra/projections/value_digest.rb +93 -0
  69. data/lib/lyra/purpose_bound_reads.rb +912 -77
  70. data/lib/lyra/repair.rb +119 -28
  71. data/lib/lyra/retention.rb +114 -8
  72. data/lib/lyra/rights/consent_store.rb +415 -0
  73. data/lib/lyra/rights/erasure_guard.rb +135 -0
  74. data/lib/lyra/rights/pending.rb +74 -0
  75. data/lib/lyra/rights/queries.rb +269 -0
  76. data/lib/lyra/rights/store.rb +512 -0
  77. data/lib/lyra/rights/subject_keys.rb +224 -0
  78. data/lib/lyra/rights.rb +281 -0
  79. data/lib/lyra/saves_without_effect.rb +69 -0
  80. data/lib/lyra/stream_tail.rb +319 -0
  81. data/lib/lyra/strict_data_access.rb +916 -106
  82. data/lib/lyra/temporal.rb +4 -2
  83. data/lib/lyra/touch_registry.rb +404 -0
  84. data/lib/lyra/verification/bypass_workflow.rb +452 -42
  85. data/lib/lyra/verification/rights_workflows.rb +170 -0
  86. data/lib/lyra/version.rb +1 -1
  87. data/lib/lyra/visualization/event_graph.rb +1 -1
  88. data/lib/lyra/visualization/timeline.rb +2 -2
  89. data/lib/lyra.rb +43 -2
  90. data/lib/tasks/lyra_mode.rake +110 -8
  91. data/lib/tasks/lyra_projections.rake +43 -2
  92. metadata +34 -4
  93. data/lib/lyra/associations/event_aware.rb +0 -224
@@ -0,0 +1,229 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Lyra
4
+ module Associations
5
+ # The callback-bypassing writes Rails makes to keep an association, which
6
+ # strict data access lets through (StrictDataAccess,
7
+ # StrictDataAccessRelation, CachedRelation), as against the same writes
8
+ # made by the application:
9
+ #
10
+ # - :nullify: a has_many's update_all clearing the children's foreign key
11
+ # (and, for a polymorphic as: association, their type), on
12
+ # dependent: :nullify when the owner is destroyed and when children are
13
+ # removed from the collection (delete, delete_all, clear, assigning a
14
+ # new set or new ids), whatever the key is called; a has_many
15
+ # :through's update_all clearing the source key of its through records
16
+ # (dependent: :nullify); a has_one's update_columns clearing its
17
+ # record's key (dependent: :nullify);
18
+ # - :unlink: the delete_all of the rows that link an owner to its
19
+ # records: has_and_belongs_to_many join rows (delete, delete_all,
20
+ # clear, destroy, destroy_all, assigning, destroying the owner),
21
+ # recorded as AssociationUnlinked (HabtmLinks), the same rows through a
22
+ # model the application declared on the join table (HabtmLinks
23
+ # .join_model_of), and a has_many :through's through records when
24
+ # records are removed from it (delete, clear, assigning: the
25
+ # through records are deleted, not destroyed, unless the association
26
+ # says dependent: :destroy).
27
+ #
28
+ # A has_many's dependent: :delete_all is not maintenance: it deletes the
29
+ # owner's children, past their callbacks, and strict mode refuses it.
30
+ #
31
+ # Rails' association code marks the one relation (or record) it is about
32
+ # to write, just before the write, once the records it needs are loaded
33
+ # (marking); the write on that very object takes the mark (claim), once,
34
+ # and is let through, with its events. Any other write in between (the
35
+ # application's after_find, after_initialize or destroy callbacks, Lyra's
36
+ # own reads materialising records) is on another object, and the mark is
37
+ # not its. The same write made by the application
38
+ # (Model.where(...).update_all(fk: nil), Item::HABTM_Tags.delete_all) has
39
+ # no mark: it is an ordinary bulk write, refused in strict mode. Before
40
+ # 1.0.0 an update_all setting one *_id column to nil was taken for a
41
+ # nullify by its shape alone, and application code could bypass strict
42
+ # mode with it; and Rails' own writes of another shape (a polymorphic or
43
+ # custom foreign key, a has_many :through's removal) were refused.
44
+ module Maintenance
45
+ KEY = :lyra_association_maintenance
46
+ # Rails versions whose HasManyThroughAssociation#delete_records
47
+ # HasManyThroughAssociationPatch reproduces, checked line by line
48
+ # (8.0.5.1, 8.1.4.1).
49
+ VERIFIED_THROUGH_DELETE = Gem::Requirement.new(">= 8.0", "< 8.2.a")
50
+
51
+ class << self
52
+ # Run Rails' association write in the block, marked as +kind+ for
53
+ # +target+, the relation or record it writes.
54
+ def marking(kind, target)
55
+ previous = Thread.current[KEY]
56
+ Thread.current[KEY] = [kind, target]
57
+ yield
58
+ ensure
59
+ Thread.current[KEY] = previous
60
+ end
61
+
62
+ # Whether the write about to be made on +receiver+ is Rails'
63
+ # association write marked +kind+. The mark is taken: Rails makes
64
+ # one write of the object it marks. A mark on a model class (the
65
+ # coarser fallback, HasManyThroughAssociationFallback) is taken by
66
+ # the first write of that model's records.
67
+ def claim(kind, receiver)
68
+ mark = Thread.current[KEY]
69
+ return false unless mark && mark[0] == kind && marked?(mark[1], receiver)
70
+
71
+ Thread.current[KEY] = nil
72
+ true
73
+ end
74
+
75
+ # The has_many :through patch for Active Record +version+: the copy
76
+ # of delete_records where it was verified, else the coarser
77
+ # fallback, which marks around Rails' own method.
78
+ def through_patch(version = ActiveRecord.version)
79
+ if VERIFIED_THROUGH_DELETE.satisfied_by?(Gem::Version.new(version.to_s))
80
+ HasManyThroughAssociationPatch
81
+ else
82
+ HasManyThroughAssociationFallback
83
+ end
84
+ end
85
+
86
+ # Said once, when the fallback is installed.
87
+ def warn_coarse_through!(version = ActiveRecord.version)
88
+ return if @warned_coarse_through
89
+
90
+ @warned_coarse_through = true
91
+ message = "Lyra: Active Record #{version} is not one whose has_many :through delete_records Lyra " \
92
+ "has checked (#{VERIFIED_THROUGH_DELETE}); strict data access tells its nullify and " \
93
+ "through-record deletes from the application's by model there, not by the relation written, " \
94
+ "so a write of the same model made during them (a callback) is let through too"
95
+ defined?(Rails) && Rails.logger ? Rails.logger.warn(message) : Kernel.warn(message)
96
+ end
97
+
98
+ private
99
+
100
+ def marked?(target, receiver)
101
+ return true if target.equal?(receiver)
102
+ return false unless target.is_a?(Class)
103
+
104
+ model = if receiver.respond_to?(:model_class) then receiver.model_class
105
+ elsif receiver.respond_to?(:klass) then receiver.klass
106
+ else receiver.class
107
+ end
108
+ model.is_a?(Class) && model <= target
109
+ end
110
+
111
+ public
112
+
113
+ # The kind of a has_many's delete_count(+method+) on +klass+'s
114
+ # records: a nullify, or the unlink of join rows; nil for a
115
+ # dependent: :delete_all of the children.
116
+ def has_many_kind(method, klass)
117
+ return :nullify unless method == :delete_all
118
+
119
+ HabtmLinks.join_model_of(klass) ? :unlink : nil
120
+ end
121
+ end
122
+
123
+ # has_many: Rails' delete_count is the write of dependent: :nullify and
124
+ # :delete_all, and of removing records from the collection (delete,
125
+ # clear, replace): update_all of the owner's key, or delete_all, on the
126
+ # scope it is given.
127
+ module HasManyAssociationPatch
128
+ private
129
+
130
+ def delete_count(method, scope)
131
+ kind = Maintenance.has_many_kind(method, scope.klass)
132
+ return super unless kind
133
+
134
+ Maintenance.marking(kind, scope) { super }
135
+ end
136
+ end
137
+
138
+ # has_many :through: removing records deletes, nullifies or destroys
139
+ # their through records, by the through association's scope. Active
140
+ # Record's HasManyThroughAssociation#delete_records (Rails 8.0, 8.1),
141
+ # with the relation it writes marked just before the write: the scope is
142
+ # built, and the records a :destroy loads have run their callbacks, by
143
+ # then. A :destroy of through records with a primary key goes through
144
+ # their callbacks (destroy_all) and is no bypass.
145
+ module HasManyThroughAssociationPatch
146
+ private
147
+
148
+ def delete_records(records, method)
149
+ ensure_not_nested
150
+
151
+ scope = through_association.scope
152
+ scope.where! construct_join_attributes(*records)
153
+ scope = scope.where(through_scope_attributes)
154
+
155
+ case method
156
+ when :destroy
157
+ if scope.model.primary_key
158
+ count = scope.destroy_all.count(&:destroyed?)
159
+ else
160
+ scope.each(&:_run_destroy_callbacks)
161
+ count = Maintenance.marking(:unlink, scope) { scope.delete_all }
162
+ end
163
+ when :nullify
164
+ count = Maintenance.marking(:nullify, scope) { scope.update_all(source_reflection.foreign_key => nil) }
165
+ else
166
+ count = Maintenance.marking(:unlink, scope) { scope.delete_all }
167
+ end
168
+
169
+ delete_through_records(records)
170
+
171
+ if source_reflection.options[:counter_cache] && method != :destroy
172
+ counter = source_reflection.counter_cache_column
173
+ klass.decrement_counter counter, records.map(&:id)
174
+ end
175
+
176
+ if through_reflection.collection? && update_through_counter?(method)
177
+ update_counter(-count, through_reflection)
178
+ else
179
+ update_counter(-count)
180
+ end
181
+
182
+ count
183
+ end
184
+ end
185
+
186
+ # has_many :through on an Active Record not checked
187
+ # (VERIFIED_THROUGH_DELETE): Rails' own delete_records, with the
188
+ # through model marked for the whole of it. Coarser: a write of the
189
+ # through model made meanwhile (a callback of a :destroy) is taken for
190
+ # Rails'.
191
+ module HasManyThroughAssociationFallback
192
+ private
193
+
194
+ def delete_records(records, method)
195
+ return super if method == :destroy && through_reflection.klass.primary_key
196
+
197
+ kind = method == :nullify ? :nullify : :unlink
198
+ Maintenance.marking(kind, through_reflection.klass) { super }
199
+ end
200
+ end
201
+
202
+ # has_one dependent: :nullify clears its record's key with
203
+ # update_columns. The record is loaded first (its after_find and
204
+ # after_initialize run before the mark), then marked.
205
+ module HasOneAssociationPatch
206
+ def delete(method = options[:dependent])
207
+ return super unless method == :nullify
208
+
209
+ record = load_target
210
+ return super unless record
211
+
212
+ Maintenance.marking(:nullify, record) { super }
213
+ end
214
+ end
215
+
216
+ def self.install!
217
+ return if @installed
218
+
219
+ ActiveRecord::Associations::HasManyAssociation.prepend(HasManyAssociationPatch)
220
+ through = through_patch
221
+ warn_coarse_through! unless through == HasManyThroughAssociationPatch
222
+ ActiveRecord::Associations::HasManyThroughAssociation.prepend(through)
223
+ ActiveRecord::Associations::HasOneAssociation.prepend(HasOneAssociationPatch)
224
+ HabtmLinks.install!
225
+ @installed = true
226
+ end
227
+ end
228
+ end
229
+ end
@@ -0,0 +1,48 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "etc"
4
+ require "socket"
5
+
6
+ module Lyra
7
+ # Who made a change that no record's callbacks attribute: a mode switch, an
8
+ # erasure, a repair. The actor is a label the operator or the application
9
+ # supplies (OPERATOR=..., Current.user, LYRA_OPERATOR): nothing checks it, so it
10
+ # can name anyone. Next to it Lyra records what the operating system says: the
11
+ # user the process runs as, the host and the process id. They do not make the
12
+ # label true, but a reader can compare the two, and OS and SSH audit logs
13
+ # (which Lyra cannot replace) can be matched against the host and the time.
14
+ module Attribution
15
+ module_function
16
+
17
+ # { actor:, os_user:, host:, pid: }
18
+ def of(operator = nil)
19
+ { actor: Lyra::ModeTransition.actor(operator), os_user: os_user, host: host, pid: Process.pid }
20
+ end
21
+
22
+ # The user the process runs as (by uid, not by the USER variable, which any
23
+ # caller can set).
24
+ def os_user
25
+ Etc.getpwuid(Process.uid)&.name || Process.uid.to_s
26
+ rescue StandardError
27
+ Process.uid.to_s
28
+ end
29
+
30
+ def host
31
+ Socket.gethostname
32
+ end
33
+
34
+ # The class of the user who acted (Current.user), as events record it
35
+ # beside user_id from 1.0.0 (user_type): the table's owner class for an
36
+ # Active Record user (the STI base class), the class name otherwise, nil
37
+ # for an anonymous class. An id alone cannot tell a staff User 7 from the
38
+ # data subject Person 7; the Art. 15 copy compares both.
39
+ def actor_type(user)
40
+ return nil if user.nil?
41
+
42
+ klass = user.class
43
+ klass.respond_to?(:base_class) ? Lyra.stream_owner_name(klass) : klass.name
44
+ rescue StandardError
45
+ nil
46
+ end
47
+ end
48
+ end