orfeas_lyra 0.9.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +2021 -17
  3. data/README.md +22 -9
  4. data/Rakefile +40 -1
  5. data/app/controllers/lyra/application_controller.rb +127 -0
  6. data/app/controllers/lyra/dashboard_controller.rb +33 -31
  7. data/app/controllers/lyra/flow_controller.rb +32 -34
  8. data/app/controllers/lyra/privacy_controller.rb +57 -18
  9. data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
  10. data/app/views/lyra/dashboard/index.html.erb +1 -0
  11. data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
  12. data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
  13. data/app/views/lyra/privacy/policy.html.erb +7 -0
  14. data/app/views/lyra/privacy/rights.html.erb +78 -0
  15. data/config/privacy_policies.rb +0 -3
  16. data/config/routes.rb +1 -0
  17. data/lib/lyra/access_log.rb +187 -6
  18. data/lib/lyra/advisory_lock.rb +236 -0
  19. data/lib/lyra/aggregate.rb +1 -1
  20. data/lib/lyra/associations/habtm_links.rb +382 -19
  21. data/lib/lyra/associations/maintenance.rb +229 -0
  22. data/lib/lyra/attribution.rb +48 -0
  23. data/lib/lyra/bypass_events.rb +446 -46
  24. data/lib/lyra/command_handler.rb +19 -14
  25. data/lib/lyra/configuration.rb +237 -10
  26. data/lib/lyra/consistency/read_your_writes.rb +4 -2
  27. data/lib/lyra/correlation.rb +121 -0
  28. data/lib/lyra/domain_events.rb +4 -0
  29. data/lib/lyra/dual_view.rb +53 -16
  30. data/lib/lyra/encrypted_attributes.rb +5 -0
  31. data/lib/lyra/engine.rb +67 -0
  32. data/lib/lyra/erasure.rb +1011 -106
  33. data/lib/lyra/event.rb +35 -1
  34. data/lib/lyra/event_data.rb +38 -0
  35. data/lib/lyra/event_flow.rb +31 -15
  36. data/lib/lyra/event_store_adapter.rb +28 -0
  37. data/lib/lyra/event_window.rb +58 -0
  38. data/lib/lyra/genesis.rb +548 -44
  39. data/lib/lyra/interceptors/association_interceptor.rb +689 -34
  40. data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
  41. data/lib/lyra/internal.rb +26 -0
  42. data/lib/lyra/mode_sync.rb +51 -3
  43. data/lib/lyra/mode_transition.rb +257 -30
  44. data/lib/lyra/model_selection.rb +35 -0
  45. data/lib/lyra/own_connection.rb +17 -2
  46. data/lib/lyra/privacy/adapters/pam.rb +25 -2
  47. data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
  48. data/lib/lyra/privacy/interface.rb +27 -7
  49. data/lib/lyra/privacy/subject_links.rb +143 -0
  50. data/lib/lyra/privacy/subject_records.rb +369 -0
  51. data/lib/lyra/projection.rb +7 -1
  52. data/lib/lyra/projections/async_checkpoint.rb +32 -0
  53. data/lib/lyra/projections/async_projection_job.rb +166 -23
  54. data/lib/lyra/projections/cached_joins.rb +33 -18
  55. data/lib/lyra/projections/cached_projection.rb +74 -33
  56. data/lib/lyra/projections/cached_relation.rb +3175 -321
  57. data/lib/lyra/projections/default_scopes.rb +254 -0
  58. data/lib/lyra/projections/deferred_relation.rb +23 -3
  59. data/lib/lyra/projections/event_store_reader.rb +47 -8
  60. data/lib/lyra/projections/foreign_keys.rb +92 -0
  61. data/lib/lyra/projections/lazy_projection.rb +165 -27
  62. data/lib/lyra/projections/model_projection.rb +165 -14
  63. data/lib/lyra/projections/rebuild.rb +34 -6
  64. data/lib/lyra/projections/reference_links.rb +463 -72
  65. data/lib/lyra/projections/typed_values.rb +629 -0
  66. data/lib/lyra/projections/unique_indexes.rb +595 -0
  67. data/lib/lyra/projections/uniqueness.rb +784 -0
  68. data/lib/lyra/projections/value_digest.rb +93 -0
  69. data/lib/lyra/purpose_bound_reads.rb +912 -77
  70. data/lib/lyra/repair.rb +119 -28
  71. data/lib/lyra/retention.rb +114 -8
  72. data/lib/lyra/rights/consent_store.rb +415 -0
  73. data/lib/lyra/rights/erasure_guard.rb +135 -0
  74. data/lib/lyra/rights/pending.rb +74 -0
  75. data/lib/lyra/rights/queries.rb +269 -0
  76. data/lib/lyra/rights/store.rb +512 -0
  77. data/lib/lyra/rights/subject_keys.rb +224 -0
  78. data/lib/lyra/rights.rb +281 -0
  79. data/lib/lyra/saves_without_effect.rb +69 -0
  80. data/lib/lyra/stream_tail.rb +319 -0
  81. data/lib/lyra/strict_data_access.rb +916 -106
  82. data/lib/lyra/temporal.rb +4 -2
  83. data/lib/lyra/touch_registry.rb +404 -0
  84. data/lib/lyra/verification/bypass_workflow.rb +452 -42
  85. data/lib/lyra/verification/rights_workflows.rb +170 -0
  86. data/lib/lyra/version.rb +1 -1
  87. data/lib/lyra/visualization/event_graph.rb +1 -1
  88. data/lib/lyra/visualization/timeline.rb +2 -2
  89. data/lib/lyra.rb +43 -2
  90. data/lib/tasks/lyra_mode.rake +110 -8
  91. data/lib/tasks/lyra_projections.rake +43 -2
  92. metadata +34 -4
  93. data/lib/lyra/associations/event_aware.rb +0 -224
@@ -0,0 +1,143 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Lyra
4
+ module Privacy
5
+ # How a monitored model's records refer to a data subject, for the
6
+ # Art. 15 copy and the GDPR helpers: each belongs_to whose class is the
7
+ # subject's model (belongs_to :owner, class_name: "Person"), each
8
+ # polymorphic belongs_to (with its type column naming the subject's
9
+ # class), and the <type>_id column by convention (person_id for Person).
10
+ # A column an association owns links only to that association's class
11
+ # (a polymorphic one's, only with its type column naming the subject's
12
+ # class): before 1.0.0's fix the convention also read it, so owner_id of
13
+ # belongs_to :owner, class_name: "Person" named a model called Owner too,
14
+ # and a polymorphic owner_id named Owner whatever owner_type said. The
15
+ # convention reads only a column no association owns, and only for a
16
+ # subject type that is a monitored model or names no model at all (not
17
+ # for a model Lyra does not monitor). The subject's own model is not
18
+ # linked to itself: a person record pointing at another person (a
19
+ # manager) is that other person's record.
20
+ #
21
+ # A record names the subject while one of its links holds the subject's
22
+ # id (and, for a polymorphic link, the subject's class name).
23
+ class SubjectLinks
24
+ Link = Struct.new(:fk, :type_column, :type_value, keyword_init: true)
25
+
26
+ attr_reader :subject_type
27
+
28
+ def initialize(subject_type)
29
+ @subject_type = subject_type.to_s
30
+ @convention = "#{@subject_type.demodulize.underscore}_id"
31
+ @cache = {}
32
+ end
33
+
34
+ # The subject's model, if the subject type names an Active Record class.
35
+ def subject_model
36
+ return @subject_model if defined?(@subject_model)
37
+
38
+ model = Lyra.config.monitored_model_named(@subject_type) || @subject_type.safe_constantize
39
+ @subject_model = model.is_a?(Class) && model < ActiveRecord::Base ? model : nil
40
+ end
41
+
42
+ # The links of +model+ (a class or a model name) to the subject.
43
+ def for(model)
44
+ key = model.is_a?(Class) ? model.name : model.to_s
45
+ @cache[key] ||= model.is_a?(Class) ? class_links(model) : name_links(key)
46
+ end
47
+
48
+ # The columns the links read.
49
+ def columns(model)
50
+ self.for(model).flat_map { |link| [link.fk, link.type_column] }.compact.uniq
51
+ end
52
+
53
+ # Whether +values+ (column => value) name the subject through a link.
54
+ def names?(model, values, subject_id)
55
+ self.for(model).any? do |link|
56
+ value = values[link.fk]
57
+ next false if value.nil? || value.to_s != subject_id.to_s
58
+ next true unless link.type_column
59
+
60
+ values[link.type_column].to_s == link.type_value
61
+ end
62
+ end
63
+
64
+ # The monitored models (one per table) with at least one link and a table.
65
+ def linked_models
66
+ Lyra.config.monitored_models.map { |m| m.respond_to?(:base_class) ? m.base_class : m }.uniq.select do |model|
67
+ self.for(model).any? && model.table_exists?
68
+ rescue StandardError
69
+ false
70
+ end
71
+ end
72
+
73
+ # The rows of +model+ that name the subject now, one query per link.
74
+ def rows(model, subject_id)
75
+ self.for(model).flat_map do |link|
76
+ scope = model.unscoped.where(link.fk => subject_id)
77
+ scope = scope.where(link.type_column => link.type_value) if link.type_column
78
+ scope.to_a
79
+ end.uniq(&:id)
80
+ end
81
+
82
+ private
83
+
84
+ def own_model?(model)
85
+ subject_model && Lyra.stream_owner_name(model) == Lyra.stream_owner_name(subject_model)
86
+ end
87
+
88
+ def class_links(model)
89
+ return [] if own_model?(model)
90
+
91
+ reflections = model.reflect_on_all_associations(:belongs_to)
92
+ links = reflections.filter_map { |reflection| link_for(reflection) }
93
+ owned = reflections.map { |reflection| reflection.foreign_key.to_s }
94
+ columns = safe_columns(model)
95
+ if convention? && columns.include?(@convention) && !owned.include?(@convention)
96
+ links << Link.new(fk: @convention)
97
+ end
98
+ links.select { |link| columns.empty? || columns.include?(link.fk) }.uniq
99
+ end
100
+
101
+ def name_links(name)
102
+ return [] if name == @subject_type || !convention?
103
+
104
+ [Link.new(fk: @convention)]
105
+ end
106
+
107
+ # The <type>_id convention holds for a subject type that is a monitored
108
+ # model, or that names no Active Record class at all (a subject the
109
+ # application identifies by type and id only); not for a model Lyra
110
+ # does not monitor.
111
+ def convention?
112
+ return true if subject_model.nil?
113
+
114
+ Lyra.config.monitored_models.any? do |model|
115
+ Lyra.stream_owner_name(model) == Lyra.stream_owner_name(subject_model)
116
+ end
117
+ end
118
+
119
+ def link_for(reflection)
120
+ if reflection.polymorphic?
121
+ return nil unless subject_model
122
+
123
+ Link.new(fk: reflection.foreign_key.to_s, type_column: reflection.foreign_type.to_s,
124
+ type_value: subject_model.polymorphic_name)
125
+ else
126
+ target = reflection.class_name.to_s.safe_constantize
127
+ return nil unless target.is_a?(Class) && subject_model
128
+ return nil unless Lyra.stream_owner_name(target) == Lyra.stream_owner_name(subject_model)
129
+
130
+ Link.new(fk: reflection.foreign_key.to_s)
131
+ end
132
+ rescue StandardError
133
+ nil
134
+ end
135
+
136
+ def safe_columns(model)
137
+ model.table_exists? ? model.column_names : []
138
+ rescue StandardError
139
+ []
140
+ end
141
+ end
142
+ end
143
+ end
@@ -0,0 +1,369 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Lyra
4
+ module Privacy
5
+ # The data subject's records for the Art. 15 copy (from 1.0.0), as
6
+ # PamDsl::GDPRCompliance's records_reader gives them: which records are
7
+ # the subject's, their values now, and their stored history.
8
+ #
9
+ # Selection: a record is the subject's by what it is, not by which of its
10
+ # events repeat the foreign key.
11
+ # - Rows: the subject's own row (the model named by the subject type),
12
+ # and, in each monitored model that links to the subject
13
+ # (Lyra::Privacy::SubjectLinks: a belongs_to to the subject's class,
14
+ # polymorphic included, or the <type>_id column), the rows that name
15
+ # them, by indexed queries. A row that pre-dates monitoring, never
16
+ # imported, is found this way.
17
+ # - Events: every record the subject's events touch (the GDPR helpers'
18
+ # one batched pass over the log), so a destroyed record, a record of an
19
+ # events-only model (ES-NoProj) and one that has since moved to someone
20
+ # else are found by their create, import or move.
21
+ #
22
+ # Periods. A related record is the subject's while one of its links
23
+ # names them. Each event of its stream is inside a period, outside it,
24
+ # or a transition: the event that moves the record in (its new values are
25
+ # the subject's, not the old ones) or away (its old values are the
26
+ # subject's, not the new ones). The history gives events inside the
27
+ # periods whole, a transition only the subject's side of it (the link's
28
+ # new value on a move away reads "moved to another person"), and nothing
29
+ # outside; +periods+ lists the periods when they are not the whole stream.
30
+ #
31
+ # Values: the row's, read as Lyra (encrypted columns come back in
32
+ # plaintext as Active Record decrypts them), +state+ :current; or, with
33
+ # no row or in ES-NoProj and ES-Async, the state replayed from the
34
+ # stream: :replayed, or :destroyed with the last state before the
35
+ # destroy; or, for a record whose links no longer name the subject,
36
+ # :reassigned with the state just before it moved away. A value last set
37
+ # outside the subject's periods (by or for another person) is not given:
38
+ # its field is listed in +set_outside_periods+. Encrypted values in the
39
+ # log are opened.
40
+ #
41
+ # History: each change as { at:, operation:, field:, from:, to:, actor: },
42
+ # a create or an import giving each attribute (from nil), a destroy one
43
+ # entry with no field, any other event in a period (ErasureApplied, a
44
+ # domain event) one entry naming it, with no values.
45
+ class SubjectRecords
46
+ MOVED_AWAY = "moved to another person"
47
+
48
+ # Each event of a related record's stream with its kind (:inside, :in,
49
+ # :away or :outside the subject's periods), and the link columns'
50
+ # values after the last event. +initial+ holds the columns' values
51
+ # before the stream, where known (a row the stream never changed); a
52
+ # change of a column gives its value before the change.
53
+ def self.classify(events, links:, model:, subject_id:, initial: {})
54
+ columns = links.columns(model)
55
+ values = initial.dup
56
+ touched = []
57
+ classified = events.map do |event|
58
+ data = event.data.is_a?(Hash) ? event.data : {}
59
+ operation = Lyra::Event.operation_of(event)
60
+ changes = operation == :updated ? (data[:changes] || data["changes"] || {}) : {}
61
+ columns.each do |column|
62
+ next if touched.include?(column) || !change_for(changes, column)
63
+
64
+ values[column] = Array(change_for(changes, column)).first
65
+ end
66
+ # Before its create or import the record named no one.
67
+ before = %i[created imported].include?(operation) ? false : links.names?(model, values, subject_id)
68
+ case operation
69
+ when :created, :imported
70
+ attributes = Lyra::Event.created_attributes(data)
71
+ columns.each { |column| values[column] = value_for(attributes, column) }
72
+ touched |= columns
73
+ when :updated
74
+ columns.each do |column|
75
+ change = change_for(changes, column)
76
+ next unless change
77
+
78
+ values[column] = change.is_a?(Array) ? change.last : change
79
+ touched |= [column]
80
+ end
81
+ end
82
+ after = links.names?(model, values, subject_id)
83
+ [event, before ? (after ? :inside : :away) : (after ? :in : :outside)]
84
+ end
85
+ [classified, values]
86
+ end
87
+
88
+ def self.change_for(changes, column)
89
+ changes.key?(column) ? changes[column] : changes[column.to_sym]
90
+ end
91
+
92
+ def self.value_for(hash, column)
93
+ hash.key?(column) ? hash[column] : hash[column.to_sym]
94
+ end
95
+
96
+ def initialize(subject_id, subject_type, events, history: true)
97
+ @subject_id = subject_id
98
+ @subject_type = subject_type.to_s
99
+ @events = events
100
+ @history = history
101
+ @links = SubjectLinks.new(@subject_type)
102
+ end
103
+
104
+ def call
105
+ candidates.filter_map { |candidate| collect(*candidate) }
106
+ end
107
+
108
+ private
109
+
110
+ # [model class or name, id, row or nil], each record once.
111
+ def candidates
112
+ found = {}
113
+ add = lambda do |model, id, row = nil|
114
+ entry = (found[stream_for(model, id)] ||= [model, id, nil])
115
+ entry[2] ||= row
116
+ end
117
+
118
+ Lyra::PurposeBoundReads.internal(INTERNAL) do
119
+ Lyra::Projections::LazyProjection.catch_up! if lazy?
120
+ unless events_only?
121
+ own_row&.then { |row| add.call(own_model, row.id, row) }
122
+ @links.linked_models.each do |model|
123
+ @links.rows(model, @subject_id).each { |row| add.call(model, row.id, row) }
124
+ end
125
+ end
126
+ @events.each do |event|
127
+ id = event_model_id(event)
128
+ next if id.nil?
129
+
130
+ model = model_named(event_model_class(event))
131
+ add.call(model, id) if model
132
+ end
133
+ # A record the events found, in a model with a table: its row, if any.
134
+ found.each do |key, (model, id, row)|
135
+ next if row || events_only? || !table?(model)
136
+
137
+ found[key][2] = model.unscoped.find_by(model.primary_key => id)
138
+ end
139
+ end
140
+ found.values
141
+ end
142
+
143
+ def collect(model, id, row)
144
+ events = read_stream(stream_for(model, id))
145
+ own = own_record?(model, id)
146
+ if own
147
+ classified = events.map { |event| [event, :inside] }
148
+ names_now = true
149
+ else
150
+ initial = row ? @links.columns(model).to_h { |column| [column, row_value(row, column)] } : {}
151
+ classified, final = self.class.classify(events, links: @links, model: model, subject_id: @subject_id,
152
+ initial: initial)
153
+ names_now = @links.names?(model, final, @subject_id)
154
+ end
155
+ kept = classified.reject { |_event, kind| kind == :outside }
156
+ return nil if kept.empty? && !(row && names_now)
157
+
158
+ replay = replay(model, classified)
159
+ state, as_of, attributes, mine = state_of(model, row, replay, kept, own || names_now)
160
+ {
161
+ model: model_name(model), id: id, state: state, as_of: as_of, attributes: attributes,
162
+ set_outside_periods: outside_fields(model, attributes, mine),
163
+ history_from: kept.first && timestamp(kept.first.first),
164
+ periods: own ? nil : periods(classified),
165
+ history: @history ? kept.flat_map { |event, kind| history_entries(model, event, kind) } : nil
166
+ }
167
+ end
168
+
169
+ # The record's state: the row (:current) unless the log is the truth
170
+ # here, else the replayed state; :reassigned when the record no longer
171
+ # names the subject. With it, which fields the subject's periods set.
172
+ def state_of(model, row, replay, kept, names_now)
173
+ if !names_now && kept.any?
174
+ return [:reassigned, replay[:away_at], open(model, replay[:away_state]), replay[:away_mine]]
175
+ end
176
+ if row && !(replay_preferred? && replay[:any])
177
+ return [:current, Time.current, open(model, row.attributes), replay[:mine]]
178
+ end
179
+ return [:current, Time.current, {}, {}] unless replay[:any]
180
+
181
+ [replay[:destroyed] ? :destroyed : :replayed, replay[:last_at], open(model, replay[:state]), replay[:mine]]
182
+ end
183
+
184
+ # The stream replayed, each field remembering whether the subject's
185
+ # periods set it; and the state just before the last move away.
186
+ def replay(_model, classified)
187
+ state = {}
188
+ mine = {}
189
+ result = { any: false, destroyed: false }
190
+ classified.each do |event, kind|
191
+ operation = Lyra::Event.operation_of(event)
192
+ next unless operation
193
+
194
+ data = event.data.is_a?(Hash) ? event.data : {}
195
+ if kind == :away
196
+ result.merge!(away_state: state.dup, away_mine: mine.dup, away_at: timestamp(event))
197
+ end
198
+ ours = %i[inside in].include?(kind)
199
+ case operation
200
+ when :created, :imported
201
+ state = Lyra::Event.created_attributes(data).to_h.transform_keys(&:to_s)
202
+ mine = state.keys.to_h { |field| [field, ours] }
203
+ when :updated
204
+ (data[:changes] || data["changes"] || {}).each do |field, change|
205
+ state[field.to_s] = change.is_a?(Array) ? change.last : change
206
+ mine[field.to_s] = ours
207
+ end
208
+ end
209
+ result.merge!(any: true, destroyed: operation == :destroyed, last_at: timestamp(event))
210
+ end
211
+ result.merge(state: state, mine: mine)
212
+ end
213
+
214
+ # The fields whose value the subject's periods did not set; the
215
+ # record's key and links are never among them.
216
+ def outside_fields(model, attributes, mine)
217
+ keep = [primary_key(model), *@links.columns(model)].compact
218
+ attributes.keys.select { |field| mine[field] == false && !keep.include?(field) }
219
+ end
220
+
221
+ def periods(classified)
222
+ kinds = classified.map(&:last)
223
+ throughout = !kinds.include?(:away) && !kinds.include?(:outside) &&
224
+ kinds.each_with_index.none? { |kind, i| kind == :in && i.positive? }
225
+ return nil if throughout
226
+
227
+ classified.each_with_object([]) do |(event, kind), periods|
228
+ at = timestamp(event)
229
+ case kind
230
+ when :in then periods << { from: at, to: nil }
231
+ when :inside, :away
232
+ periods << { from: nil, to: nil } if periods.empty? || periods.last[:to]
233
+ periods.last[:to] = at if kind == :away
234
+ end
235
+ end
236
+ end
237
+
238
+ def history_entries(model, event, kind)
239
+ data = event.data.is_a?(Hash) ? event.data : {}
240
+ at = timestamp(event)
241
+ actor = actor_of(event)
242
+ operation = Lyra::Event.operation_of(event)
243
+ links = @links.columns(model)
244
+ case operation
245
+ when :created, :imported
246
+ open(model, Lyra::Event.created_attributes(data).to_h).map do |field, value|
247
+ { at: at, operation: operation, field: field.to_s, from: nil, to: value, actor: actor }
248
+ end
249
+ when :updated
250
+ (data[:changes] || data["changes"] || {}).map do |field, change|
251
+ from, to = change.is_a?(Array) ? change : [nil, change]
252
+ entry = { at: at, operation: operation, field: field.to_s, actor: actor }
253
+ case kind
254
+ when :in then entry[:to] = open_value(model, field, to)
255
+ when :away
256
+ entry[:from] = open_value(model, field, from)
257
+ entry[:to] = MOVED_AWAY if links.include?(field.to_s)
258
+ else
259
+ entry[:from] = open_value(model, field, from)
260
+ entry[:to] = open_value(model, field, to)
261
+ end
262
+ entry
263
+ end
264
+ when :destroyed
265
+ [{ at: at, operation: operation, field: nil, from: nil, to: nil, actor: actor }]
266
+ else
267
+ [{ at: at, operation: nil, event: event.event_type.to_s.demodulize, field: nil, from: nil, to: nil, actor: actor }]
268
+ end
269
+ end
270
+
271
+ # Who wrote the event: the ids its metadata records (Lyra::Event#user_id
272
+ # reads both places Lyra has kept them).
273
+ def actor_of(event)
274
+ metadata = event.metadata.to_h
275
+ nested = event.data.is_a?(Hash) ? (event.data[:metadata] || event.data["metadata"] || {}) : {}
276
+ %i[user_id ip_address user_type].to_h do |key|
277
+ [key, nested[key] || nested[key.to_s] || metadata[key] || metadata[key.to_s]]
278
+ end.compact
279
+ end
280
+
281
+ def open(model, attributes)
282
+ attributes = attributes.to_h.transform_keys(&:to_s)
283
+ model.is_a?(Class) ? Lyra::EncryptedAttributes.open(model, attributes) : attributes
284
+ end
285
+
286
+ def open_value(model, field, value)
287
+ open(model, { field.to_s => value })[field.to_s]
288
+ end
289
+
290
+ def timestamp(event)
291
+ data = event.data.is_a?(Hash) ? event.data : {}
292
+ data[:timestamp] || data["timestamp"] || event.metadata[:timestamp]
293
+ end
294
+
295
+ def read_stream(stream)
296
+ Lyra.config.event_store.read.stream(stream).to_a
297
+ rescue RubyEventStore::EventNotFoundInStream
298
+ []
299
+ end
300
+
301
+ def row_value(row, column)
302
+ row.has_attribute?(column) ? row[column] : nil
303
+ end
304
+
305
+ def primary_key(model)
306
+ model.is_a?(Class) ? model.primary_key.to_s : "id"
307
+ rescue StandardError
308
+ "id"
309
+ end
310
+
311
+ # The subject's own model, if the subject type names one with a table.
312
+ def own_model
313
+ model = @links.subject_model
314
+ model && table?(model) ? model : nil
315
+ end
316
+
317
+ def own_row
318
+ own_model&.unscoped&.find_by(own_model.primary_key => @subject_id)
319
+ end
320
+
321
+ def own_record?(model, id)
322
+ model_name(model) == model_name(@links.subject_model || @subject_type) && id.to_s == @subject_id.to_s
323
+ end
324
+
325
+ def model_named(name)
326
+ return nil if name.nil? || name.to_s.empty?
327
+
328
+ model = Lyra.config.monitored_model_named(name) || name.to_s.safe_constantize
329
+ model.is_a?(Class) && model < ActiveRecord::Base ? model : name.to_s
330
+ end
331
+
332
+ def model_name(model)
333
+ model.is_a?(Class) ? Lyra.stream_owner_name(model) : model.to_s
334
+ end
335
+
336
+ def stream_for(model, id)
337
+ model.is_a?(Class) ? Lyra.stream_name(model, id) : "#{model}$#{id}"
338
+ end
339
+
340
+ def event_model_class(event)
341
+ event.data.is_a?(Hash) ? (event.data[:model_class] || event.data["model_class"]) : nil
342
+ end
343
+
344
+ def event_model_id(event)
345
+ event.data.is_a?(Hash) ? (event.data[:model_id] || event.data["model_id"]) : nil
346
+ end
347
+
348
+ def table?(model)
349
+ model.is_a?(Class) && model.table_exists?
350
+ rescue StandardError
351
+ false
352
+ end
353
+
354
+ def events_only?
355
+ Lyra.config.event_sourcing_mode? && Lyra.config.projection_mode == :disabled
356
+ end
357
+
358
+ def lazy?
359
+ Lyra.config.event_sourcing_mode? && Lyra.config.projection_mode == :lazy
360
+ end
361
+
362
+ # ES-NoProj has no rows, and ES-Async's may not have caught up yet: the
363
+ # log is read for their values.
364
+ def replay_preferred?
365
+ Lyra.config.event_sourcing_mode? && %i[disabled async].include?(Lyra.config.projection_mode)
366
+ end
367
+ end
368
+ end
369
+ end
@@ -50,7 +50,7 @@ module Lyra
50
50
  events.each do |event|
51
51
  case event_operation(event)
52
52
  when :created, :imported
53
- state = event_attributes(event)
53
+ state = event_created_attributes(event)
54
54
  when :updated
55
55
  state.merge!(event_changes(event).transform_values { |v| v.last })
56
56
  when :destroyed
@@ -72,6 +72,12 @@ module Lyra
72
72
  op.is_a?(String) ? op.to_sym : op
73
73
  end
74
74
 
75
+ # What a create or import gives its record (Lyra::Event.created_attributes).
76
+ def event_created_attributes(event)
77
+ return event.attributes if event.respond_to?(:attributes) && !event.attributes.is_a?(Hash)
78
+ Lyra::Event.created_attributes(event.data)
79
+ end
80
+
75
81
  def event_attributes(event)
76
82
  return event.attributes if event.respond_to?(:attributes) && !event.attributes.is_a?(Hash)
77
83
  event.data[:attributes] || event.data["attributes"] || {}
@@ -29,6 +29,38 @@ module Lyra
29
29
  SQL
30
30
  end
31
31
 
32
+ # write for each [stream, event id] of +pairs+, in one statement.
33
+ def write_all(connection, pairs)
34
+ pairs = pairs.to_h.to_a
35
+ return if pairs.empty?
36
+
37
+ ensure_table!(connection)
38
+ values = pairs.map { |stream, event_id| "(#{connection.quote(stream)}, #{connection.quote(event_id)}, CURRENT_TIMESTAMP)" }
39
+ connection.execute(<<~SQL.squish)
40
+ INSERT INTO #{TABLE} (stream, event_id, updated_at) VALUES #{values.join(', ')}
41
+ ON CONFLICT (stream) DO UPDATE SET event_id = EXCLUDED.event_id, updated_at = EXCLUDED.updated_at
42
+ SQL
43
+ end
44
+
45
+ # Of +streams+, those with an event past their checkpoint (in the
46
+ # stream's own order), or with events and no checkpoint: the streams
47
+ # whose rows a projection job has still to bring up to date. One
48
+ # query. A stream with no event is not behind.
49
+ def behind(connection, streams)
50
+ return [] if streams.empty?
51
+
52
+ ensure_table!(connection)
53
+ list = streams.map { connection.quote(_1) }.join(", ")
54
+ connection.uncached do
55
+ connection.select_values(<<~SQL.squish, "Lyra ES-Async rows behind")
56
+ SELECT DISTINCT s.stream FROM event_store_events_in_streams s
57
+ LEFT JOIN #{TABLE} c ON c.stream = s.stream
58
+ LEFT JOIN event_store_events_in_streams cs ON cs.stream = c.stream AND cs.event_id = c.event_id
59
+ WHERE s.stream IN (#{list}) AND (cs.id IS NULL OR s.id > cs.id)
60
+ SQL
61
+ end
62
+ end
63
+
32
64
  # Forget every checkpoint (tests; after the tables were truncated).
33
65
  def reset!(connection = nil)
34
66
  return ActiveRecord::Base.with_connection { |c| reset!(c) } unless connection