orfeas_lyra 0.9.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +2021 -17
  3. data/README.md +22 -9
  4. data/Rakefile +40 -1
  5. data/app/controllers/lyra/application_controller.rb +127 -0
  6. data/app/controllers/lyra/dashboard_controller.rb +33 -31
  7. data/app/controllers/lyra/flow_controller.rb +32 -34
  8. data/app/controllers/lyra/privacy_controller.rb +57 -18
  9. data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
  10. data/app/views/lyra/dashboard/index.html.erb +1 -0
  11. data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
  12. data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
  13. data/app/views/lyra/privacy/policy.html.erb +7 -0
  14. data/app/views/lyra/privacy/rights.html.erb +78 -0
  15. data/config/privacy_policies.rb +0 -3
  16. data/config/routes.rb +1 -0
  17. data/lib/lyra/access_log.rb +187 -6
  18. data/lib/lyra/advisory_lock.rb +236 -0
  19. data/lib/lyra/aggregate.rb +1 -1
  20. data/lib/lyra/associations/habtm_links.rb +382 -19
  21. data/lib/lyra/associations/maintenance.rb +229 -0
  22. data/lib/lyra/attribution.rb +48 -0
  23. data/lib/lyra/bypass_events.rb +446 -46
  24. data/lib/lyra/command_handler.rb +19 -14
  25. data/lib/lyra/configuration.rb +237 -10
  26. data/lib/lyra/consistency/read_your_writes.rb +4 -2
  27. data/lib/lyra/correlation.rb +121 -0
  28. data/lib/lyra/domain_events.rb +4 -0
  29. data/lib/lyra/dual_view.rb +53 -16
  30. data/lib/lyra/encrypted_attributes.rb +5 -0
  31. data/lib/lyra/engine.rb +67 -0
  32. data/lib/lyra/erasure.rb +1011 -106
  33. data/lib/lyra/event.rb +35 -1
  34. data/lib/lyra/event_data.rb +38 -0
  35. data/lib/lyra/event_flow.rb +31 -15
  36. data/lib/lyra/event_store_adapter.rb +28 -0
  37. data/lib/lyra/event_window.rb +58 -0
  38. data/lib/lyra/genesis.rb +548 -44
  39. data/lib/lyra/interceptors/association_interceptor.rb +689 -34
  40. data/lib/lyra/interceptors/crud_interceptor.rb +578 -91
  41. data/lib/lyra/internal.rb +26 -0
  42. data/lib/lyra/mode_sync.rb +51 -3
  43. data/lib/lyra/mode_transition.rb +257 -30
  44. data/lib/lyra/model_selection.rb +35 -0
  45. data/lib/lyra/own_connection.rb +17 -2
  46. data/lib/lyra/privacy/adapters/pam.rb +25 -2
  47. data/lib/lyra/privacy/gdpr_compliance.rb +314 -48
  48. data/lib/lyra/privacy/interface.rb +27 -7
  49. data/lib/lyra/privacy/subject_links.rb +143 -0
  50. data/lib/lyra/privacy/subject_records.rb +369 -0
  51. data/lib/lyra/projection.rb +7 -1
  52. data/lib/lyra/projections/async_checkpoint.rb +32 -0
  53. data/lib/lyra/projections/async_projection_job.rb +166 -23
  54. data/lib/lyra/projections/cached_joins.rb +33 -18
  55. data/lib/lyra/projections/cached_projection.rb +74 -33
  56. data/lib/lyra/projections/cached_relation.rb +3175 -321
  57. data/lib/lyra/projections/default_scopes.rb +254 -0
  58. data/lib/lyra/projections/deferred_relation.rb +23 -3
  59. data/lib/lyra/projections/event_store_reader.rb +47 -8
  60. data/lib/lyra/projections/foreign_keys.rb +92 -0
  61. data/lib/lyra/projections/lazy_projection.rb +165 -27
  62. data/lib/lyra/projections/model_projection.rb +165 -14
  63. data/lib/lyra/projections/rebuild.rb +34 -6
  64. data/lib/lyra/projections/reference_links.rb +463 -72
  65. data/lib/lyra/projections/typed_values.rb +629 -0
  66. data/lib/lyra/projections/unique_indexes.rb +595 -0
  67. data/lib/lyra/projections/uniqueness.rb +784 -0
  68. data/lib/lyra/projections/value_digest.rb +93 -0
  69. data/lib/lyra/purpose_bound_reads.rb +912 -77
  70. data/lib/lyra/repair.rb +119 -28
  71. data/lib/lyra/retention.rb +114 -8
  72. data/lib/lyra/rights/consent_store.rb +415 -0
  73. data/lib/lyra/rights/erasure_guard.rb +135 -0
  74. data/lib/lyra/rights/pending.rb +74 -0
  75. data/lib/lyra/rights/queries.rb +269 -0
  76. data/lib/lyra/rights/store.rb +512 -0
  77. data/lib/lyra/rights/subject_keys.rb +224 -0
  78. data/lib/lyra/rights.rb +281 -0
  79. data/lib/lyra/saves_without_effect.rb +69 -0
  80. data/lib/lyra/stream_tail.rb +319 -0
  81. data/lib/lyra/strict_data_access.rb +916 -106
  82. data/lib/lyra/temporal.rb +4 -2
  83. data/lib/lyra/touch_registry.rb +404 -0
  84. data/lib/lyra/verification/bypass_workflow.rb +452 -42
  85. data/lib/lyra/verification/rights_workflows.rb +170 -0
  86. data/lib/lyra/version.rb +1 -1
  87. data/lib/lyra/visualization/event_graph.rb +1 -1
  88. data/lib/lyra/visualization/timeline.rb +2 -2
  89. data/lib/lyra.rb +43 -2
  90. data/lib/tasks/lyra_mode.rake +110 -8
  91. data/lib/tasks/lyra_projections.rake +43 -2
  92. metadata +34 -4
  93. data/lib/lyra/associations/event_aware.rb +0 -224
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Lyra
4
+ # What only Lyra's own call sites hold. Lyra.projection_write and
5
+ # PurposeBoundReads.internal switch off the checks that keep every write in
6
+ # the log and every read under its purpose; they are for Lyra's projections,
7
+ # replays and tasks, and application code that called them could change a row
8
+ # with no event or read a declared field under :deny.
9
+ #
10
+ # The constant is private, so only code written inside `module Lyra` can name it.
11
+ # That keeps application code from calling them by accident or by following the
12
+ # documentation; it is not a barrier against code that is out to defeat it
13
+ # (Lyra.const_get reaches it), which no Ruby code could provide.
14
+ #
15
+ # Application code that needs to write without events (seeding, fixtures, a
16
+ # table wipe whose history is not wanted) uses Lyra.unrecorded_write, which asks
17
+ # for a reason and records that it happened.
18
+ INTERNAL = Object.new.freeze
19
+ private_constant :INTERNAL
20
+
21
+ # Lyra's internal switches called from outside Lyra.
22
+ class InternalApiError < StandardError; end
23
+
24
+ # Lyra.unrecorded_write while config.unrecorded_writes is :deny.
25
+ class UnrecordedWriteDenied < StandardError; end
26
+ end
@@ -13,7 +13,13 @@ module Lyra
13
13
  # Each process remembers the latest applied row it has seen. At most every
14
14
  # config.mode_sync_interval seconds (default 5) it looks for a newer one,
15
15
  # and adopts its mode: the process that recorded it already passed the
16
- # gate, so the gate is not run again. It looks:
16
+ # gate, so the gate is not run again; but a switch the gate checks (an
17
+ # escalation, or leaving a lagging mode) is adopted only if the
18
+ # row names the certificate of the clean check that cleared it (or the operator
19
+ # overrode the gate, which the row says), so one INSERT cannot move every
20
+ # process to a mode whose stores were never compared (0.9.3 adopted any row;
21
+ # ModeTransition.adoption_refusal).
22
+ # It looks:
17
23
  # - at the start of each web request (Middleware);
18
24
  # - before each background job (ActiveJob before_perform);
19
25
  # - before each write to a monitored model, which covers consoles and
@@ -44,6 +50,11 @@ module Lyra
44
50
  # Cheap when nothing is due: a clock read. Returns the label adopted, or
45
51
  # nil.
46
52
  def maybe_sync!
53
+ # A process that leaves touches out keeps its registry entry fresh from
54
+ # here, outside any transaction (nothing with config.record_touches true).
55
+ if Lyra.config.record_touches == false && !ActiveRecord::Base.with_connection(&:transaction_open?)
56
+ TouchRegistry.heartbeat_if_due
57
+ end
47
58
  return nil unless enabled?
48
59
  return nil if due_in.positive?
49
60
  return nil if ActiveRecord::Base.with_connection(&:transaction_open?)
@@ -60,11 +71,48 @@ module Lyra
60
71
  return nil unless row
61
72
  return nil if @last_seen_id && row[:id] <= @last_seen_id
62
73
 
63
- @last_seen_id = row[:id]
64
- return nil if row[:config] == ModeTransition.current
74
+ if row[:config] == ModeTransition.current
75
+ @last_seen_id = row[:id]
76
+ return nil
77
+ end
78
+
79
+ # The row counts as seen once it is decided: adopted, or refused (logged
80
+ # once). An error while deciding leaves it unseen, so the next check tries
81
+ # again rather than leave this process in its mode for good.
82
+ # The application runs in a mode whose log rebuilds rows, and this
83
+ # process leaves touches out: it must not go on that way. It refuses
84
+ # every touch it would leave out from now on (TouchRegistry.fence!),
85
+ # and each check raises, until it runs with record_touches on.
86
+ if Lyra.config.record_touches == false && TouchRegistry.authoritative?(row[:config])
87
+ TouchRegistry.fence!(row[:config])
88
+ message = TouchRegistry.process_refusal(row[:config])
89
+ Rails.logger.error(message)
90
+ raise ModeTransition::Refused, message
91
+ end
92
+
93
+ # Not while another process leaves touches out of the log: the row
94
+ # stays unseen, and this process adopts it once that process's entry
95
+ # has ended or expired (as a fenced process retries).
96
+ if TouchRegistry.authoritative?(row[:config]) &&
97
+ (touches = TouchRegistry.gate_refusal(ModeTransition.current, row[:config]))
98
+ if @touch_refused_id != row[:id]
99
+ @touch_refused_id = row[:id]
100
+ Rails.logger.error("Lyra: did not adopt the application's mode #{row[:config]} yet; this process stays " \
101
+ "in #{ModeTransition.current} and tries again: #{touches}")
102
+ end
103
+ return nil
104
+ end
105
+
106
+ if (refusal = ModeTransition.adoption_refusal(row))
107
+ @last_seen_id = row[:id]
108
+ Rails.logger.error("Lyra: did not adopt the application's mode #{row[:config]}; this process stays in " \
109
+ "#{ModeTransition.current}: #{refusal}")
110
+ return nil
111
+ end
65
112
 
66
113
  previous = ModeTransition.current
67
114
  ModeTransition.apply_label(row[:config])
115
+ @last_seen_id = row[:id]
68
116
  Rails.logger.warn("Lyra: adopted the application's mode #{row[:config]} (was #{previous}), switched elsewhere")
69
117
  row[:config]
70
118
  end
@@ -9,13 +9,17 @@ module Lyra
9
9
  # - Escalation, from Disabled or Monitor to Hijack or event sourcing: the
10
10
  # events become authoritative, so they must reproduce every row. Rows
11
11
  # that predate Lyra are imported first (Lyra::Genesis), or each would
12
- # count as a discrepancy.
12
+ # count as a discrepancy. A row holding a value its model cannot
13
+ # represent is not imported, and refuses the switch (from 1.0.0).
13
14
  # - Leaving a mode whose tables lag the log (ES-NoProj, ES-Lazy, ES-Async)
14
15
  # for one that reads them: the tables must catch up first. ES-Lazy
15
16
  # catches up on read; ES-NoProj needs a Rebuild (rebuild: true).
16
17
  # Every other switch (Monitor to Disabled, ES-Sync to ES-NoProj, ...) keeps
17
18
  # the authoritative store as it is and passes freely. Entering ES-NoProj
18
- # also links the events' foreign keys (Lyra::Projections::ReferenceLinks).
19
+ # also links the events' foreign keys (Lyra::Projections::ReferenceLinks),
20
+ # and is refused for a default scope ES-NoProj cannot evaluate and for a
21
+ # database foreign key from an unmonitored table into a monitored one
22
+ # (Lyra::Projections::ForeignKeys); force: true overrides both.
19
23
  #
20
24
  # A check compares every row with the state replayed from its stream, and
21
25
  # every stream with its row, in batches, reading the real table (never the
@@ -55,13 +59,18 @@ module Lyra
55
59
  def to_s = "#{model} #{id}: #{problem}#{" (#{details})" if details}"
56
60
  end
57
61
 
62
+ # +unrepresentable+: rows Genesis would not import, holding values their
63
+ # model cannot represent (each listed value is also a discrepancy, up to
64
+ # Lyra::Genesis::LISTED per model).
58
65
  Report = Struct.new(:from, :to, :checked, :rechecked, :imported, :rebuilt, :discrepancies,
59
- :position, :started_at, :finished_at, keyword_init: true) do
66
+ :position, :started_at, :finished_at, :certificate_id, :unrepresentable,
67
+ keyword_init: true) do
60
68
  def clean? = discrepancies.empty?
61
69
 
62
70
  def summary
63
71
  "#{from} -> #{to}: #{checked} records checked, #{rechecked} re-checked, " \
64
- "#{imported} imported, #{discrepancies.size} discrepancies"
72
+ "#{imported} imported, #{discrepancies.size} discrepancies" \
73
+ "#{", #{unrepresentable} rows with values their model cannot represent" if unrepresentable.to_i.positive?}"
65
74
  end
66
75
  end
67
76
 
@@ -99,6 +108,11 @@ module Lyra
99
108
  from = current
100
109
  to = label(mode, projection_mode)
101
110
  report = nil
111
+ # Not overridden by force: no mode whose log rebuilds rows may be entered
112
+ # while touches are left out of it, here or in another process.
113
+ refuse_unrecorded_touches!(from, to)
114
+ refuse_unevaluable_default_scopes!(from, to) unless force
115
+ refuse_foreign_keys_into_event_only_tables!(from, to) unless force
102
116
 
103
117
  if gate_enabled? && gate_required?(from, to) && !force
104
118
  report = certified_check(from, to) || check(to: to, from: from, rebuild: rebuild)
@@ -109,8 +123,17 @@ module Lyra
109
123
  end
110
124
 
111
125
  link_references if to == "event_sourcing/disabled" && from != to
112
- apply(mode, projection_mode)
113
- id = record_applied(to)
126
+ # The registry's last word and the switch's record, under the registry's
127
+ # lock: a process registering to leave touches out either sees this
128
+ # switch (and records its touch) or is seen here (and the switch is
129
+ # refused). TouchRegistry.serialize.
130
+ id = TouchRegistry.serialize(to) do
131
+ refuse_unrecorded_touches!(from, to)
132
+ apply(mode, projection_mode)
133
+ # forced: the gate was required and the operator overrode it.
134
+ record_applied(to, from: from, forced: force && gate_enabled? && gate_required?(from, to),
135
+ certificate_id: report&.certificate_id)
136
+ end
114
137
  ModeSync.seen!(id)
115
138
  announce(from, to)
116
139
  report
@@ -124,49 +147,133 @@ module Lyra
124
147
  apply(mode, projection_mode)
125
148
  end
126
149
 
127
- # The application-wide mode: { id:, config: } of the latest applied row.
150
+ # The application-wide mode: the latest applied row, as { id:, config:, from:,
151
+ # forced:, certificate_id:, at: }. from and certificate_id are nil and forced
152
+ # false in a row an earlier version wrote.
128
153
  def latest_applied
129
154
  return nil unless table_ready?
130
155
 
131
- row = sql(:select_one, "SELECT id, to_config FROM #{TABLE} WHERE kind = 'applied' ORDER BY id DESC LIMIT 1")
132
- row && { id: row["id"].to_i, config: row["to_config"] }
156
+ row = sql(:select_one,
157
+ "SELECT id, to_config, from_config, forced, certificate_id, created_at FROM #{TABLE} " \
158
+ "WHERE kind = 'applied' ORDER BY id DESC LIMIT 1")
159
+ return nil unless row
160
+
161
+ at = row["created_at"].is_a?(String) ? Time.zone.parse(row["created_at"]) : row["created_at"]
162
+ { id: row["id"].to_i, config: row["to_config"], from: row["from_config"].presence,
163
+ forced: row["forced"] == true || row["forced"] == "t", certificate_id: row["certificate_id"]&.to_i, at: at }
164
+ end
165
+
166
+ # Why ModeSync must not adopt +row+ (latest_applied's hash), or nil when it may.
167
+ #
168
+ # With the gate off nothing is certified, and a switch is adopted as recorded.
169
+ # With it on, a switch the gate checks (gate_required?: an escalation from
170
+ # Disabled or Monitor, or leaving a lagging mode for one that reads the
171
+ # tables) must have passed the gate: the row names the certificate of
172
+ # the clean check that cleared it, one for the row's own from -> to, recorded
173
+ # before the row and named by no earlier switch. The switcher checked the
174
+ # certificate's age when it switched, so no clock is compared here. A row
175
+ # recorded with the gate overridden (forced) is adopted: the operator meant
176
+ # it, and it is on record.
177
+ #
178
+ # Whether the switch is gated is judged from the row's from and from where the
179
+ # switch before it left the application, so a row cannot name a from that
180
+ # makes it ungated. A row an earlier version wrote has no from and names no
181
+ # certificate; it is judged from this process's mode.
182
+ #
183
+ # This is a check against mistakes and naive writes, not against anyone who
184
+ # can write the table: such a writer can also write the certificate.
185
+ def adoption_refusal(row, from: current)
186
+ return nil unless gate_enabled?
187
+
188
+ to = row[:config]
189
+ from = row[:from] || from
190
+ previous = previous_applied(row[:id])
191
+ return nil unless gate_required?(from, to) || (previous && gate_required?(previous, to))
192
+ return nil if row[:forced]
193
+ return nil if names_its_certificate?(row, from)
194
+
195
+ named = row[:certificate_id] ? "certificate #{row[:certificate_id]}, which does not clear it" : "no certificate"
196
+ "no clean check certifies #{from} -> #{to} for this switch (row #{row[:id]} names #{named}); " \
197
+ "only a gated switch records both"
198
+ end
199
+
200
+ # Where the applied row before +id+ left the application, or nil.
201
+ def previous_applied(id)
202
+ sql(:select_value,
203
+ "SELECT to_config FROM #{TABLE} WHERE kind = 'applied' AND id < #{Integer(id)} ORDER BY id DESC LIMIT 1")
204
+ end
205
+
206
+ def names_its_certificate?(row, from)
207
+ return false unless row[:certificate_id]
208
+
209
+ cert = Integer(row[:certificate_id])
210
+ id = Integer(row[:id])
211
+ sql(:select_value,
212
+ "SELECT 1 FROM #{TABLE} c WHERE c.id = #{cert} AND c.kind = 'certificate' AND c.from_config = #{q(from)} " \
213
+ "AND c.to_config = #{q(row[:config])} AND c.id < #{id} AND NOT EXISTS (SELECT 1 FROM #{TABLE} a " \
214
+ "WHERE a.kind = 'applied' AND a.certificate_id = #{cert} AND a.id < #{id})").present?
133
215
  end
134
216
 
135
217
  # Compare every monitored record's row with its events. Returns a
136
218
  # Report; stores a certificate when it is clean.
137
219
  def check(to:, from: current, models: Lyra.config.monitored_models, rebuild: false, batch_size: 1_000)
220
+ # With record_touches false DualView takes timestamp-only differences
221
+ # as agreement, and a process that leaves touches out moves rows its
222
+ # events do not show: no certificate may come of either.
223
+ refuse_unrecorded_touches!(from, to)
138
224
  started_at = Time.current
139
225
  position = event_position
140
226
  imported = 0
141
227
  rebuilt = 0
142
-
228
+ discrepancies = []
229
+ unrepresentable = 0
230
+
231
+ # A row its model cannot represent refuses the model's import (nothing
232
+ # of the model is imported), whatever config.genesis_unrepresentable
233
+ # says: a skipped row would have no events. Each listed value is a
234
+ # discrepancy; the model's rows are not compared (each would be "row
235
+ # but no events").
236
+ refused = []
143
237
  if !%w[disabled monitor].include?(to) && %w[disabled monitor].include?(from)
144
- models.each { |model| imported += Lyra::Genesis.import_all(model) }
238
+ models.each do |model|
239
+ imported += Lyra::Genesis.import_all(model, unrepresentable: :refuse)
240
+ rescue Lyra::Genesis::Refused => e
241
+ refused << model
242
+ unrepresentable += e.rows
243
+ e.values.each do |value|
244
+ discrepancies << Discrepancy.new(model: value.model, id: value.id, problem: "a value the model cannot represent",
245
+ details: value.details)
246
+ end
247
+ end
145
248
  end
146
249
  if LAGGING.include?(from)
147
250
  Lyra::Projections::LazyProjection.catch_up!(force: true) if from == "event_sourcing/lazy"
148
251
  rebuilt = models.sum { |model| Lyra::Projections::Rebuild.rebuild(model)[:records] } if rebuild
149
252
  end
150
253
 
151
- discrepancies = []
152
254
  checked = 0
153
- models.each do |model|
255
+ (models - refused).each do |model|
154
256
  ids_to_check(model).each_slice(batch_size) do |ids|
155
257
  ids.each { |id| (problem = compare(model, id)) && discrepancies << problem }
156
258
  checked += ids.size
157
259
  end
158
260
  end
159
261
 
160
- changed = changed_since(models, position, started_at)
262
+ changed = changed_since(models - refused, position, started_at)
161
263
  changed.each do |model, ids|
162
264
  discrepancies.reject! { |d| d.model == model.name && ids.include?(d.id.to_s) }
163
265
  ids.each { |id| (problem = compare(model, id)) && discrepancies << problem }
164
266
  end
165
267
 
166
268
  report = Report.new(from: from, to: to, checked: checked, rechecked: changed.values.sum(&:size),
167
- imported: imported, rebuilt: rebuilt, discrepancies: discrepancies,
269
+ imported: imported, rebuilt: rebuilt, discrepancies: discrepancies, unrepresentable: unrepresentable,
168
270
  position: event_position, started_at: started_at, finished_at: Time.current)
169
- certify(report) if report.clean?
271
+ # A process may have begun leaving touches out while the check ran: the
272
+ # last look and the certificate under the registry's lock.
273
+ TouchRegistry.serialize(to) do
274
+ refuse_unrecorded_touches!(from, to)
275
+ certify(report) if report.clean?
276
+ end
170
277
  report
171
278
  end
172
279
 
@@ -178,14 +285,18 @@ module Lyra
178
285
 
179
286
  to = current
180
287
  last = last_applied
288
+ if TouchRegistry.authoritative?(to) && (touches = TouchRegistry.gate_refusal(last || to, to))
289
+ raise Refused, touches
290
+ end
181
291
  if last.nil? || last == to || !gate_required?(last, to)
182
- ModeSync.seen!(last == to ? latest_applied&.dig(:id) : record_applied(to))
292
+ ModeSync.seen!(last == to ? latest_applied&.dig(:id) : record_applied(to, from: last))
183
293
  return
184
294
  end
185
295
 
186
296
  report = certified_check(last, to)
187
297
  if report&.clean? || ENV["LYRA_FORCE_MODE_TRANSITION"].present?
188
- ModeSync.seen!(record_applied(to))
298
+ ModeSync.seen!(record_applied(to, from: last, forced: !report&.clean?,
299
+ certificate_id: (report.certificate_id if report&.clean?)))
189
300
  return
190
301
  end
191
302
 
@@ -194,6 +305,33 @@ module Lyra
194
305
  "or set LYRA_FORCE_MODE_TRANSITION=1.", report)
195
306
  end
196
307
 
308
+ # At boot, in ES-NoProj set by configuration (config.projection_mode =
309
+ # :disabled in an initializer) rather than reached by a switch: what
310
+ # the switch into ES-NoProj would refuse, as one warning each, for the
311
+ # models enrolled by then (with eager loading, all of them): a database
312
+ # foreign key into an event-only table, and a default scope ES-NoProj
313
+ # cannot evaluate. Never stops the boot: config.mode = ... is the raw,
314
+ # ungated setter, and a model enrolled later is not seen here.
315
+ # Returns the warnings.
316
+ def check_es_noproj!(models: Lyra.config.monitored_models)
317
+ return [] unless Lyra.config.event_sourcing_mode? && Lyra.config.projection_mode == :disabled
318
+
319
+ foreign_keys = Lyra::Projections::ForeignKeys
320
+ messages = foreign_keys.blocking(models).map do |fk|
321
+ "Lyra: in ES-NoProj every write to #{fk.from_table} that sets #{fk.column} will fail. " \
322
+ "#{foreign_keys.description([fk])} #{foreign_keys.remedy([fk])} " \
323
+ "The switch into ES-NoProj (Lyra::ModeTransition.to!) refuses this."
324
+ end
325
+ messages += Lyra::Projections::DefaultScopes.unevaluable(models).map do |model, reason|
326
+ "Lyra: in ES-NoProj every read of #{model.name} will be refused: ES-NoProj cannot evaluate its default " \
327
+ "scope from the event store (#{reason}). Write the default scope as hash conditions and column " \
328
+ "orderings, or use another projection mode."
329
+ end
330
+ logger = defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
331
+ messages.each { |message| logger ? logger.warn(message) : Kernel.warn(message) }
332
+ messages
333
+ end
334
+
197
335
  def last_applied
198
336
  return nil unless table_ready?
199
337
 
@@ -203,6 +341,9 @@ module Lyra
203
341
  # Every id of +model+ with a row or a stream (Lyra::Repair).
204
342
  def record_ids(model) = ids_to_check(model)
205
343
 
344
+ # Whether the table can be used (created on first use).
345
+ def table_available? = table_ready?
346
+
206
347
  # nil if the record's row and events agree, else a Discrepancy
207
348
  # (Lyra::Repair).
208
349
  def discrepancy(model, id) = compare(model, id)
@@ -221,6 +362,53 @@ module Lyra
221
362
  Lyra.config.monitored_models.each { |model| Lyra::Projections::ReferenceLinks.catch_up(model, force: true) }
222
363
  end
223
364
 
365
+ # A mode whose log rebuilds rows is refused while touches are left out
366
+ # of the log: by this process (config.record_touches false,
367
+ # Configuration#touch_refusal) or by any live process
368
+ # (TouchRegistry.gate_refusal).
369
+ def refuse_unrecorded_touches!(from, to)
370
+ return unless TouchRegistry.authoritative?(to)
371
+
372
+ if (own = Lyra.config.touch_refusal(to))
373
+ raise Refused, "Lyra refuses #{from} -> #{to}: #{own}"
374
+ end
375
+ others = TouchRegistry.gate_refusal(from, to)
376
+ raise Refused, others if others
377
+ end
378
+
379
+ # ES-NoProj reads through each model's default scopes from the event
380
+ # store, and refuses a read whose default scope it cannot evaluate (an
381
+ # SQL string, a join, an ordering by an expression): every read of
382
+ # such a model would fail. The switch into ES-NoProj names them first.
383
+ # force: true switches anyway.
384
+ def refuse_unevaluable_default_scopes!(from, to)
385
+ return unless to == "event_sourcing/disabled" && from != to
386
+
387
+ refused = Lyra::Projections::DefaultScopes.unevaluable(Lyra.config.monitored_models)
388
+ return if refused.empty?
389
+
390
+ raise Refused.new("Lyra refuses #{from} -> #{to}: ES-NoProj cannot evaluate the default scope of " \
391
+ "#{refused.map { |model, _| model.name }.join(', ')} from the event store, and would refuse " \
392
+ "every read of #{refused.size == 1 ? 'that model' : 'those models'}. Write the default scope " \
393
+ "as hash conditions and column orderings, or use another projection mode. " \
394
+ "(#{refused.first.last})")
395
+ end
396
+
397
+ # ES-NoProj leaves the tables of monitored models empty, so a database
398
+ # foreign key into one from a table no monitored model writes refuses
399
+ # every write there that references a record
400
+ # (Lyra::Projections::ForeignKeys). The switch into ES-NoProj names
401
+ # each such constraint first. force: true switches anyway.
402
+ def refuse_foreign_keys_into_event_only_tables!(from, to)
403
+ return unless to == "event_sourcing/disabled" && from != to
404
+
405
+ blocking = Lyra::Projections::ForeignKeys.blocking(Lyra.config.monitored_models)
406
+ return if blocking.empty?
407
+
408
+ raise Refused.new("Lyra refuses #{from} -> #{to}: #{Lyra::Projections::ForeignKeys.description(blocking)} " +
409
+ Lyra::Projections::ForeignKeys.remedy(blocking))
410
+ end
411
+
224
412
  def apply(mode, projection_mode)
225
413
  Lyra.config.mode = mode.to_sym
226
414
  Lyra.config.hijack_enabled = mode.to_sym == :hijack
@@ -228,12 +416,14 @@ module Lyra
228
416
  end
229
417
 
230
418
  # A fresh certificate for the switch, re-checked for what changed since.
419
+ # Only one issued after every period in which a process left touches out
420
+ # (its latest heartbeat or its end, TouchRegistry.certificate_floor_sql).
231
421
  def certified_check(from, to)
232
422
  return nil unless table_ready?
233
423
 
234
424
  row = sql(:select_one,
235
- "SELECT position, created_at FROM #{TABLE} WHERE kind = 'certificate' AND from_config = #{q(from)} " \
236
- "AND to_config = #{q(to)} ORDER BY id DESC LIMIT 1"
425
+ "SELECT id, position, created_at FROM #{TABLE} WHERE kind = 'certificate' AND from_config = #{q(from)} " \
426
+ "AND to_config = #{q(to)} AND id > #{TouchRegistry.certificate_floor_sql} ORDER BY id DESC LIMIT 1"
237
427
  )
238
428
  return nil unless row
239
429
 
@@ -245,7 +435,7 @@ module Lyra
245
435
  discrepancies = changed.flat_map { |model, ids| ids.filter_map { |id| compare(model, id) } }
246
436
  Report.new(from: from, to: to, checked: 0, rechecked: changed.values.sum(&:size), imported: 0, rebuilt: 0,
247
437
  discrepancies: discrepancies, position: event_position, started_at: certified_at,
248
- finished_at: Time.current)
438
+ finished_at: Time.current, certificate_id: row["id"].to_i)
249
439
  end
250
440
 
251
441
  # Every id that has a row or a stream.
@@ -280,7 +470,7 @@ module Lyra
280
470
  next Discrepancy.new(model: model.name, id: id, problem: "row of a destroyed record") if gone
281
471
 
282
472
  differences = Lyra::DualView.new(model, id).compare[:differences]
283
- next nil if differences[:no_differences]
473
+ next nil if differences.empty?
284
474
 
285
475
  Discrepancy.new(model: model.name, id: id, problem: "row differs from its events",
286
476
  details: differences.keys.first(5).join(", "))
@@ -306,24 +496,47 @@ module Lyra
306
496
  end
307
497
  end
308
498
 
499
+ # Store a clean check's certificate; returns its id, which the report keeps
500
+ # and the switch it clears names.
309
501
  def certify(report)
310
502
  return unless table_ready?
311
503
 
312
- sql(:execute,
504
+ report.certificate_id = sql(:select_value,
313
505
  "INSERT INTO #{TABLE} (kind, from_config, to_config, position, checked, created_at) VALUES " \
314
506
  "('certificate', #{q(report.from)}, #{q(report.to)}, #{Integer(report.position)}, #{Integer(report.checked)}, " \
315
- "#{q(report.finished_at.utc)})"
316
- )
507
+ "#{q(report.finished_at.utc)}) RETURNING id"
508
+ ).to_i
317
509
  end
318
510
 
319
511
  # Record +to+ as the application-wide mode; returns the row's id.
320
- def record_applied(to)
512
+ def record_applied(to, from: nil, forced: false, certificate_id: nil)
321
513
  return nil unless table_ready?
322
514
 
515
+ who = Lyra::Attribution.of
323
516
  sql(:select_value,
324
- "INSERT INTO #{TABLE} (kind, from_config, to_config, position, checked, created_at) VALUES " \
325
- "('applied', NULL, #{q(to)}, #{Integer(event_position)}, 0, #{q(Time.current.utc)}) RETURNING id"
326
- ).to_i
517
+ "INSERT INTO #{TABLE} (kind, from_config, to_config, position, checked, created_at, actor, os_user, host, pid, forced, " \
518
+ "certificate_id) VALUES " \
519
+ "('applied', #{q(from)}, #{q(to)}, #{Integer(event_position)}, 0, #{q(Time.current.utc)}, #{q(who[:actor])}, " \
520
+ "#{q(who[:os_user])}, #{q(who[:host])}, #{Integer(who[:pid])}, #{forced ? "TRUE" : "FALSE"}, " \
521
+ "#{certificate_id ? Integer(certificate_id) : "NULL"}) RETURNING id"
522
+ ).to_i.tap do |id|
523
+ Rails.logger&.warn("[Lyra] Mode switched to #{to} by #{who[:actor]} (os user #{who[:os_user]} on #{who[:host]}, pid #{who[:pid]}; " \
524
+ "lyra_mode_transitions #{id})")
525
+ end
526
+ end
527
+
528
+ # Who is making a change: the given operator, else the signed-in user
529
+ # (Current.user), else LYRA_OPERATOR, else the operating-system user and
530
+ # process of a task or console. A label nobody checks: Lyra::Attribution
531
+ # records the operating system's user and the host beside it. Recorded with
532
+ # every mode switch, every erasure (0.9.3) and every repair (1.0.0).
533
+ public def actor(operator = nil)
534
+ return operator.to_s if operator.present?
535
+
536
+ user = defined?(::Current) && ::Current.respond_to?(:user) ? ::Current.user : nil
537
+ return "user:#{user.respond_to?(:id) ? user.id : user}" if user
538
+
539
+ ENV["LYRA_OPERATOR"].presence || ["process", ENV["USER"], Process.pid].compact.join(":")
327
540
  end
328
541
 
329
542
  # Say what a switch reaches: with ModeSync, every process within its
@@ -358,8 +571,22 @@ module Lyra
358
571
  t.bigint :position, null: false, default: 0
359
572
  t.integer :checked, null: false, default: 0
360
573
  t.datetime :created_at, null: false
574
+ t.string :actor
575
+ t.string :os_user
576
+ t.string :host
577
+ t.integer :pid
578
+ t.boolean :forced, null: false, default: false
579
+ t.bigint :certificate_id
361
580
  end
362
581
  end
582
+ # Columns later versions added: actor (0.9.3); what the operating system
583
+ # says, whether the gate was overridden, and the certificate that cleared
584
+ # a switch (1.0.0). Added outside the caller's transaction, so a rollback
585
+ # cannot take one back, and safely when two processes add one at once.
586
+ Lyra.add_own_columns(connection, TABLE,
587
+ actor: [:string], os_user: [:string], host: [:string], pid: [:integer],
588
+ forced: [:boolean, { null: false, default: false }], certificate_id: [:bigint],
589
+ started_at: [:datetime], seen_at: [:datetime], ended_at: [:datetime])
363
590
  end
364
591
  @table_ready = true
365
592
  rescue ActiveRecord::NoDatabaseError, ActiveRecord::ConnectionNotEstablished
@@ -369,7 +596,7 @@ module Lyra
369
596
  def bypass
370
597
  previous = Thread.current[:lyra_bypass_read_override]
371
598
  Thread.current[:lyra_bypass_read_override] = true
372
- Lyra::PurposeBoundReads.internal { yield }
599
+ Lyra::PurposeBoundReads.internal(INTERNAL) { yield }
373
600
  ensure
374
601
  Thread.current[:lyra_bypass_read_override] = previous
375
602
  end
@@ -69,6 +69,7 @@ module Lyra
69
69
  warn_about_mode(chosen.size) if spec.all
70
70
  log(chosen.keys, spec)
71
71
  chosen.map do |model, options|
72
+ refuse_composite_key!(model, resolve: true)
72
73
  model.monitor_with_lyra(options) unless model.respond_to?(:lyra_monitored) && model.lyra_monitored
73
74
  model
74
75
  end
@@ -102,8 +103,42 @@ module Lyra
102
103
  false # no database yet (assets:precompile), or a model that cannot reflect
103
104
  end
104
105
 
106
+ # Refuse (ArgumentError) a model whose primary key is composite
107
+ # (self.primary_key = [:a, :b], or one its table declares): a record is
108
+ # one stream named by its key, and every lookup by key is by one column.
109
+ # A single primary key of any name and type is supported. Checks the key
110
+ # the model declared or has already read; with +resolve+, reads it from
111
+ # the schema now when the table exists (config.models does, once the
112
+ # application has loaded; monitor_with_lyra in a class body does not
113
+ # touch the database).
114
+ def refuse_composite_key!(model, resolve: false)
115
+ key = known_primary_key(model)
116
+ key = model.primary_key if key.nil? && resolve && table_present?(model)
117
+ return unless key.is_a?(Array)
118
+
119
+ raise ArgumentError, "#{model.name || model} has a composite primary key (#{key.join(', ')}), which Lyra does " \
120
+ "not support: a monitored model needs a single-column primary key"
121
+ end
122
+
105
123
  private
106
124
 
125
+ # The primary key +model+ declared or has already read from the schema,
126
+ # without reading it now; nil when it is not known yet.
127
+ def known_primary_key(model)
128
+ return nil unless model.instance_variable_defined?(:@primary_key)
129
+
130
+ # Unread, Active Record holds a BasicObject placeholder here, which
131
+ # has no is_a?.
132
+ key = model.instance_variable_get(:@primary_key)
133
+ Array === key || String === key ? key : nil
134
+ end
135
+
136
+ def table_present?(model)
137
+ model.table_exists?
138
+ rescue StandardError
139
+ false
140
+ end
141
+
107
142
  def resolve(name)
108
143
  model = begin
109
144
  name.constantize
@@ -25,7 +25,13 @@ module Lyra
25
25
  # Runs the block, on a connection of its own when +model_class+'s
26
26
  # connection has a transaction open. Yields +inline+; returns the block's
27
27
  # value.
28
- def run(model_class = ActiveRecord::Base, &block)
28
+ #
29
+ # +wait_for_connection+ false: with no connection free in the pool
30
+ # (none idle, and the pool at its size), run inline at once rather
31
+ # than wait up to the checkout timeout, holding the caller's
32
+ # transaction and its locks (ReferenceLinks.catch_up, which does not
33
+ # wait inline either).
34
+ def run(model_class = ActiveRecord::Base, wait_for_connection: true, &block)
29
35
  where = model_class.with_connection do |connection|
30
36
  next :here unless connection.transaction_open?
31
37
 
@@ -35,10 +41,19 @@ module Lyra
35
41
  return yield(true) if where == :inline
36
42
 
37
43
  pool = model_class.connection_pool
44
+ unless wait_for_connection
45
+ stat = pool.stat
46
+ return yield(true) unless stat[:idle].to_i.positive? || stat[:connections].to_i < stat[:size].to_i
47
+ end
38
48
  begin
39
49
  # prevent_permanent_checkout: whatever the block leases is returned
40
50
  # with the connection, not left to the thread when it ends.
41
- Thread.new { pool.with_connection(prevent_permanent_checkout: true) { block.call(false) } }.value
51
+ # The block's error is raised here, by value; the thread does not
52
+ # also report it (a refused Genesis import, say).
53
+ Thread.new do
54
+ Thread.current.report_on_exception = false
55
+ pool.with_connection(prevent_permanent_checkout: true) { block.call(false) }
56
+ end.value
42
57
  rescue ActiveRecord::ConnectionTimeoutError
43
58
  yield(true)
44
59
  end