orfeas_lyra 0.9.3 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +1923 -17
  3. data/README.md +15 -8
  4. data/Rakefile +40 -1
  5. data/app/controllers/lyra/application_controller.rb +108 -0
  6. data/app/controllers/lyra/dashboard_controller.rb +17 -15
  7. data/app/controllers/lyra/flow_controller.rb +30 -33
  8. data/app/controllers/lyra/privacy_controller.rb +57 -18
  9. data/app/views/lyra/dashboard/discrepancies.html.erb +5 -5
  10. data/app/views/lyra/dashboard/index.html.erb +1 -0
  11. data/app/views/lyra/flow/crud_mapping.html.erb +1 -1
  12. data/app/views/lyra/privacy/pii_detection.html.erb +1 -1
  13. data/app/views/lyra/privacy/policy.html.erb +7 -0
  14. data/app/views/lyra/privacy/rights.html.erb +78 -0
  15. data/config/privacy_policies.rb +0 -3
  16. data/config/routes.rb +1 -0
  17. data/lib/lyra/access_log.rb +187 -6
  18. data/lib/lyra/advisory_lock.rb +236 -0
  19. data/lib/lyra/aggregate.rb +1 -1
  20. data/lib/lyra/associations/habtm_links.rb +382 -19
  21. data/lib/lyra/associations/maintenance.rb +229 -0
  22. data/lib/lyra/attribution.rb +48 -0
  23. data/lib/lyra/bypass_events.rb +426 -45
  24. data/lib/lyra/command_handler.rb +19 -14
  25. data/lib/lyra/configuration.rb +237 -10
  26. data/lib/lyra/consistency/read_your_writes.rb +4 -2
  27. data/lib/lyra/correlation.rb +121 -0
  28. data/lib/lyra/domain_events.rb +4 -0
  29. data/lib/lyra/dual_view.rb +53 -16
  30. data/lib/lyra/encrypted_attributes.rb +5 -0
  31. data/lib/lyra/engine.rb +67 -0
  32. data/lib/lyra/erasure.rb +1002 -116
  33. data/lib/lyra/event.rb +35 -1
  34. data/lib/lyra/event_data.rb +38 -0
  35. data/lib/lyra/event_flow.rb +31 -15
  36. data/lib/lyra/event_store_adapter.rb +28 -0
  37. data/lib/lyra/event_window.rb +58 -0
  38. data/lib/lyra/genesis.rb +548 -44
  39. data/lib/lyra/interceptors/association_interceptor.rb +689 -34
  40. data/lib/lyra/interceptors/crud_interceptor.rb +547 -95
  41. data/lib/lyra/internal.rb +26 -0
  42. data/lib/lyra/mode_sync.rb +51 -3
  43. data/lib/lyra/mode_transition.rb +245 -34
  44. data/lib/lyra/model_selection.rb +35 -0
  45. data/lib/lyra/own_connection.rb +17 -2
  46. data/lib/lyra/privacy/adapters/pam.rb +25 -2
  47. data/lib/lyra/privacy/gdpr_compliance.rb +264 -36
  48. data/lib/lyra/privacy/interface.rb +27 -7
  49. data/lib/lyra/privacy/subject_links.rb +143 -0
  50. data/lib/lyra/privacy/subject_records.rb +369 -0
  51. data/lib/lyra/projection.rb +7 -1
  52. data/lib/lyra/projections/async_checkpoint.rb +32 -0
  53. data/lib/lyra/projections/async_projection_job.rb +166 -23
  54. data/lib/lyra/projections/cached_joins.rb +33 -18
  55. data/lib/lyra/projections/cached_projection.rb +74 -33
  56. data/lib/lyra/projections/cached_relation.rb +3175 -321
  57. data/lib/lyra/projections/default_scopes.rb +254 -0
  58. data/lib/lyra/projections/deferred_relation.rb +23 -3
  59. data/lib/lyra/projections/event_store_reader.rb +47 -8
  60. data/lib/lyra/projections/foreign_keys.rb +92 -0
  61. data/lib/lyra/projections/lazy_projection.rb +165 -27
  62. data/lib/lyra/projections/model_projection.rb +165 -14
  63. data/lib/lyra/projections/rebuild.rb +34 -6
  64. data/lib/lyra/projections/reference_links.rb +463 -72
  65. data/lib/lyra/projections/typed_values.rb +629 -0
  66. data/lib/lyra/projections/unique_indexes.rb +595 -0
  67. data/lib/lyra/projections/uniqueness.rb +784 -0
  68. data/lib/lyra/projections/value_digest.rb +93 -0
  69. data/lib/lyra/purpose_bound_reads.rb +912 -77
  70. data/lib/lyra/repair.rb +119 -28
  71. data/lib/lyra/retention.rb +114 -8
  72. data/lib/lyra/rights/consent_store.rb +415 -0
  73. data/lib/lyra/rights/erasure_guard.rb +135 -0
  74. data/lib/lyra/rights/pending.rb +74 -0
  75. data/lib/lyra/rights/queries.rb +269 -0
  76. data/lib/lyra/rights/store.rb +512 -0
  77. data/lib/lyra/rights/subject_keys.rb +224 -0
  78. data/lib/lyra/rights.rb +281 -0
  79. data/lib/lyra/saves_without_effect.rb +69 -0
  80. data/lib/lyra/stream_tail.rb +319 -0
  81. data/lib/lyra/strict_data_access.rb +906 -107
  82. data/lib/lyra/temporal.rb +4 -2
  83. data/lib/lyra/touch_registry.rb +404 -0
  84. data/lib/lyra/verification/bypass_workflow.rb +452 -42
  85. data/lib/lyra/verification/rights_workflows.rb +170 -0
  86. data/lib/lyra/version.rb +1 -1
  87. data/lib/lyra/visualization/event_graph.rb +1 -1
  88. data/lib/lyra/visualization/timeline.rb +2 -2
  89. data/lib/lyra.rb +43 -2
  90. data/lib/tasks/lyra_mode.rake +107 -8
  91. data/lib/tasks/lyra_projections.rake +43 -2
  92. metadata +34 -4
  93. data/lib/lyra/associations/event_aware.rb +0 -224
@@ -6,30 +6,236 @@ module Lyra
6
6
  #
7
7
  # The CRUD nets only cover writes that enter through a callback, so
8
8
  # Mapping Completeness says nothing about update_all and friends. This
9
- # net covers them, as StrictDataAccess and CachedRelation implement them:
9
+ # net covers them, as StrictDataAccess, BypassEvents and CachedRelation
10
+ # implement them in Lyra 1.0.0. It follows one record through one write.
10
11
  #
11
- # - configuration, fixed once at the start: strict_data_access on or off,
12
- # and a table-backed store or an events-only one (event sourcing with
13
- # projections disabled);
14
- # - update_column(s): write, then event (built from the persisted
15
- # values, so "assign, then update_columns" is still seen);
16
- # - delete: event, then write;
12
+ # Configuration, fixed once at the start: strict_data_access on or off,
13
+ # and one of three stores, given by two places that mirror
14
+ # BypassEvents.locks_rows? and BypassEvents.checks_streams?:
15
+ #
16
+ # - table_backed: the rows are in a table, which bulk writes lock and
17
+ # write. ES-Sync, Hijack and Monitor delete the row with the destroy,
18
+ # so a write to a destroyed record reaches no row: SQL matches none,
19
+ # and delete, which locks the row first, finds none and logs nothing;
20
+ # - table_backed with streams_checked: a table that lags the event store
21
+ # (ES-Async, whose rows wait for the projection job, and ES-Lazy, whose
22
+ # rows are brought up to date under the lock). Every write but
23
+ # insert_all reads the record's stream under its aggregate lock first:
24
+ # update_column(s), touch, delete, the bulk writes and dependent:
25
+ # :nullify leave out a record whose stream ends in a destroy; ES-Lazy's
26
+ # upsert_all refuses one whose row is still in the table, and
27
+ # ES-Async's insert_all, insert_all! and upsert_all bring the rows of
28
+ # the keys they write up to date with their streams first;
29
+ # - events_only with streams_checked: event sourcing with projections
30
+ # disabled (ES-NoProj). The events are the write; destroyed records are
31
+ # left out, and a bulk write selects its records when it is called.
32
+ #
33
+ # Behaviour classes (METHODS), each after the strict guard where it has
34
+ # one:
35
+ #
36
+ # - update_column(s) (column): write, then event (built from the
37
+ # persisted values, so "assign, then update_columns" is still seen);
38
+ # a value a unique index refuses raises ActiveRecord::RecordNotUnique
39
+ # and writes nothing (refuse_unique_column: the table's UPDATE, in
40
+ # ES-Sync after the value's lock; or, where the table lags the event
41
+ # store or there is none (ES-Async, ES-Lazy, ES-NoProj), the event
42
+ # store, read under the value's lock after the stream check:
43
+ # Lyra::Projections::UniqueIndexes);
44
+ # - delete (instance): event, then write;
17
45
  # - touch: always allowed (Rails calls it itself for belongs_to ...
18
46
  # touch: true), write, then event;
19
- # - bulk methods (update_all, delete_all, insert_all(!)): write, then one
20
- # event per row; in the events-only store the events are the write;
21
- # - dependent: :nullify: always allowed, events then write;
22
- # - upsert_all: write, then a created or updated event per row (existing
23
- # rows found by the conflict key first); rejected in the events-only
24
- # store, where ON CONFLICT would check the table, not the stream.
47
+ # - update_all, delete_all, touch_all (bulk): write, then one event per
48
+ # row; in the events-only store the events are the write. A write
49
+ # that would hold more locks than config.bulk_write_lock_limit raises
50
+ # BulkWriteLockLimitError before it writes or records anything: the
51
+ # records' aggregate locks where the streams are checked, and in the
52
+ # event-sourcing modes, ES-Sync included, the locks of the unique
53
+ # values an update_all sets (UniqueIndexes::BulkUpdate#check!).
54
+ # Rails's update_counters,
55
+ # increment_counter, decrement_counter, increment! and decrement! reach
56
+ # update_all (in ES-NoProj through CrudInterceptor's class-level
57
+ # overrides, ROUTED). An update_all whose values a unique index
58
+ # refuses (two records given one value, or a value another record
59
+ # holds) raises ActiveRecord::RecordNotUnique and writes nothing
60
+ # (refuse_unique_bulk; in ES-NoProj, ES-Async and ES-Lazy checked
61
+ # from the event store before any write or event,
62
+ # UniqueIndexes::BulkUpdate);
63
+ # - insert_all(!) (insert): write, then a created event per row, in
64
+ # every store (ES-NoProj sends inserts to the table relation too); a
65
+ # new row has nothing for Genesis to admit. Where the streams are
66
+ # checked, the rows are first made to conflict as Monitor's table
67
+ # would (bring_insert_keys_up_to_date): ES-NoProj reads the keys the
68
+ # rows give, and the values of the model's other unique indexes, from
69
+ # the event store under their locks; ES-Async locks the keys and
70
+ # brings their rows up to date with their streams
71
+ # (AsyncProjectionJob.project_pending), so a record whose destroy was
72
+ # pending is inserted again and one whose create was pending
73
+ # conflicts. Those locks count against bulk_write_lock_limit, with, in
74
+ # ES-Sync, ES-Async and ES-Lazy, the unique values' (below). Rows
75
+ # that give no key, and ES-Lazy (caught up when the write began),
76
+ # insert as given (insert_as_given). In ES-Async and ES-Lazy the
77
+ # values of the model's unique indexes are then looked up in the event
78
+ # store under their locks (a record whose create is not projected yet
79
+ # holds its value: StrictDataAccessRelation#projected_unique_rows),
80
+ # and in ES-Sync locked. A conflict: insert_all writes nothing and
81
+ # records nothing (ON CONFLICT DO NOTHING, or left out of the insert:
82
+ # unchanged), insert_all! (and a conflict on an index other than
83
+ # insert_all's unique_by) raises ActiveRecord::RecordNotUnique and
84
+ # writes nothing;
85
+ # - upsert_all (upsert): write, then a created or updated event per row
86
+ # (existing rows found by the conflict key first); rejected in the
87
+ # events-only store, where ON CONFLICT would check the table, not the
88
+ # stream. Where the streams are checked, the rows it conflicts with are
89
+ # locked (within bulk_write_lock_limit) and their streams read
90
+ # (bring_upsert_rows_up_to_date): in ES-Async, with the keys the rows
91
+ # give, their rows are brought up to date with their streams, so a
92
+ # destroyed record whose row was still in the table is inserted again
93
+ # (a Created after the Destroyed), as Monitor inserts it; in ES-Lazy a
94
+ # destroy committed after the write's catch-up refuses the upsert with
95
+ # RecordDestroyedError before anything is written, since ON CONFLICT
96
+ # would update that row and bring the record back. A value of a unique
97
+ # index other than its conflict columns that another record holds, or
98
+ # (ES-Async, ES-Lazy) a value of its conflict columns whose holder's
99
+ # row is not in the table, raises ActiveRecord::RecordNotUnique and
100
+ # writes nothing (refuse_unique_upsert: the table's statement, or in
101
+ # ES-Async and ES-Lazy the event store, before the write). In ES-Sync,
102
+ # ES-Async and ES-Lazy the unique values' locks count against
103
+ # bulk_write_lock_limit with the rows' (refuse_lock_limit_upsert);
104
+ # - Rails' association nullify: always allowed, events then write;
105
+ # limited and checked as the bulk writes are where the streams are
106
+ # checked. Only the write Rails' association code marks
107
+ # (Lyra::Associations::Maintenance: a has_many's dependent: :nullify
108
+ # and the removal of its children, a has_many :through's
109
+ # dependent: :nullify); the same update_all made by the application
110
+ # (Model.where(...).update_all(fk: nil)) is a bulk write, guarded. A
111
+ # has_one's dependent: :nullify is Rails' update_columns of the record
112
+ # it marked: the column path, with no strict guard
113
+ # (request_association_column); removing records from a has_many
114
+ # :through whose through model is monitored deletes the through
115
+ # records it marked: the bulk path, with no strict guard
116
+ # (request_through_delete);
117
+ # - has_and_belongs_to_many links (link): deleting join rows
118
+ # (Lyra::Associations::HabtmLinks). When Rails' association code
119
+ # deletes the rows it marked (delete, delete_all, clear, destroy,
120
+ # destroy_all, assigning records or ids, destroying the owner, and the
121
+ # same through a model the application declared on the join table)
122
+ # it is always allowed (request_unlink); the application's own
123
+ # delete_all on the join model, or on its model of the join table, is
124
+ # guarded as a bulk write is (request_join), when either side's model
125
+ # is monitored. Either way: write, then an AssociationUnlinked event
126
+ # (publish_unlink), wherever a side is monitored: on the owner's
127
+ # stream, linked into the associated records'; when only the
128
+ # associated records' model is monitored, on their streams; through
129
+ # the application's model of a join table several associations share,
130
+ # and for a row naming another association's owner too, on the stream
131
+ # of every monitored owner the rows name, as Rails' unlink from that
132
+ # side records it. The application's insert_all, insert_all! and
133
+ # upsert_all of join rows (insert, insert!, upsert) are guarded the
134
+ # same way (request_join_insert); allowed: write, then an
135
+ # AssociationLinked event for each row written (publish_link; an
136
+ # upsert that moved a row to another pair also unlinks the old one),
137
+ # recorded as Rails' own link from each side records it. A row ON
138
+ # CONFLICT skipped, or an upsert that left every pair as it was,
139
+ # records nothing (skip_conflict_link); insert_all! of a pair taken
140
+ # raises ActiveRecord::RecordNotUnique (refuse_conflict_link). A table
141
+ # is a join table when a has_and_belongs_to_many whose links Lyra
142
+ # records names it, the owner's model monitored or not
143
+ # (HabtmLinks.join_model_of). A write no monitored side records is not
144
+ # Lyra's: neither guarded nor modelled. The join rows live in a table
145
+ # in every configuration (ES-NoProj's too), and have no stream of
146
+ # their own: no Genesis, no stream check, no lock limit. (Rails'
147
+ # join-row inserts go through the join model's callbacks, one row at
148
+ # a time, and are recorded as AssociationLinked the same way; they are
149
+ # no bypass.)
25
150
  #
26
- # With strict mode on, each method is rejected unless it runs inside
27
- # Lyra.without_strict_access (allow_*_in_bypass_block).
151
+ # With strict mode on, each guarded method is rejected unless it runs
152
+ # inside Lyra.without_strict_access (allow_*_in_bypass_block).
153
+ #
154
+ # The record the write reaches is one of three (a choice of the data,
155
+ # not of the configuration):
156
+ #
157
+ # - live, and admitted: the write and its events, as above;
158
+ # - a row Genesis could not import (config.genesis_unrepresentable :skip
159
+ # left it without a stream, or :refuse refused the model's import):
160
+ # Lyra::Genesis::Refused before any write or event (admit_all!), where
161
+ # Genesis runs (the event-sourcing modes; Hijack and Monitor with
162
+ # genesis: true). insert_all creates new rows, none of which is asked,
163
+ # nor is a join row (link); but where it reads keys or unique values
164
+ # from the event store (ES-NoProj, ES-Async, ES-Lazy), the model's
165
+ # first use refuses it when the model's import was refused
166
+ # (genesis_unrepresentable :refuse; refuse_genesis_insert);
167
+ # - destroyed: its stream ends in a destroy (or, table-backed, its row
168
+ # is gone). No write and no event (unchanged): Active Record's write of
169
+ # a row already gone reaches none, and an event would bring the record
170
+ # back or log a second destroy. upsert_all and insert_all have no such
171
+ # ending: a destroyed record whose row is gone (in ES-Async, once its
172
+ # row is brought up to date) is inserted again (a new Created, as
173
+ # create!(id:) gives), and in ES-Lazy one whose row is still in the
174
+ # table refuses the upsert (above). insert_all's unchanged ending is a
175
+ # key or a unique value a live record holds (in ES-NoProj, as the
176
+ # event store holds it, read under its lock; the table's row of any
177
+ # other, left by a destroyed record, is deleted and the row inserted).
28
178
  #
29
179
  # The property checked by .coverage is Bypass Coverage: every reachable
30
- # marking in which nothing can fire is either a rejection or a store
31
- # change with an event logged. A store change without one is a silent
32
- # write.
180
+ # marking in which nothing can fire ends in exactly one way: rejected
181
+ # (nothing written, nothing logged), a store change with an event
182
+ # logged, or unchanged with neither. A store change without an event is
183
+ # a silent write; an event for a write that changed nothing (unchanged)
184
+ # is the destroyed record brought back; a rejection beside a write or an
185
+ # event is a write the refusal did not take back.
186
+ #
187
+ # What the net leaves out, by design:
188
+ #
189
+ # - config.record_touches true, the default, is modelled. With it false
190
+ # (Monitor only), a touch or touch_all whose only changes are the
191
+ # update timestamps ends at store_changed with no event;
192
+ # - the append is modelled as succeeding. Where events are required
193
+ # (Hijack, the event-sourcing modes, Monitor with
194
+ # monitor_append_failure :fail_write) a failed one rolls the write
195
+ # back; in Monitor otherwise it is logged and the write stands without
196
+ # its event. So in Monitor with genesis: true, delete and dependent:
197
+ # :nullify, which ask Genesis inside the publish, write a row Genesis
198
+ # skipped (genesis_unrepresentable :skip) without an event, and log
199
+ # it: it has no stream to add to, and the other children of a nullify
200
+ # keep theirs. update_column(s), touch, the bulk writes and upsert_all
201
+ # ask before they write, and are refused there too. A link or unlink
202
+ # asks Genesis for the associated records' model inside its publish
203
+ # (HabtmLinks.link_into_targets), a refusal handled as a failed append;
204
+ # - config.lock_aggregate_before_append on, the default, is modelled.
205
+ # With it off no stream is checked and no record's lock limited
206
+ # (checks_streams? is false; the unique values' locks still are):
207
+ # ES-Async and ES-Lazy write as table_backed, and an ES-NoProj
208
+ # write to a destroyed record logs an event after the destroy, as a
209
+ # write through callbacks does there with the lock off;
210
+ # - a has_many :through over a model Lyra does not monitor: its through
211
+ # rows are that model's, written through its callbacks or by Rails'
212
+ # delete_all of them, and no link of either side is recorded, so none
213
+ # of its writes is modelled (its table is no join table:
214
+ # HabtmLinks.join_model_of). Monitored, the through model's rows are
215
+ # any monitored model's (request_through_delete, the bulk path);
216
+ # - a model whose primary key has several columns, which Lyra does not
217
+ # support (refused at enrolment where the key is known): enrolled
218
+ # before Active Record read it, its bulk inserts are refused in the
219
+ # event-sourcing modes before anything is written, and Genesis refuses
220
+ # the model on its first use there; Monitor and Hijack record them;
221
+ # - ES-NoProj's create and update through callbacks are not bypass
222
+ # writes (their unique indexes are checked from the event store as
223
+ # update_column(s)' and update_all's are, Lyra::Projections
224
+ # ::UniqueIndexes);
225
+ # - Lyra.unrecorded_write, the application's write without events
226
+ # (seeding, fixtures, a wipe): inside it each write here is past the
227
+ # strict guard and records no event for its rows, only the block's
228
+ # UnrecordedWrite (reason, caller) in Lyra::UnrecordedWrites, or is
229
+ # refused with the block (config.unrecorded_writes :deny); Lyra's own
230
+ # projection writes (Lyra.projection_write) record nothing;
231
+ # - a projection's refusal (Lyra::Projections::ProjectionBehind,
232
+ # DuplicateInLog) is not a bypass write's outcome: an ES-Async job or
233
+ # an ES-Lazy catch-up meets it after the write committed. Inside one,
234
+ # only where ES-Async brings rows up to date before writing
235
+ # (AsyncProjectionJob.project_pending, UniqueIndexes
236
+ # .bring_rows_up_to_date), it is an ActiveRecord::RecordNotUnique
237
+ # raised before the write, which writes and records nothing: rejected,
238
+ # as refuse_conflict_insert and refuse_unique_* end;
33
239
  class BypassWorkflow < PetriFlow::Workflow
34
240
  # Every callback-bypassing method Lyra overrides, and the behaviour
35
241
  # class that models it. Kept equal to the overrides in
@@ -41,39 +247,56 @@ module Lyra
41
247
  delete: :instance,
42
248
  update_all: :bulk,
43
249
  delete_all: :bulk,
44
- insert_all: :bulk,
45
- insert_all!: :bulk,
250
+ insert_all: :insert,
251
+ insert_all!: :insert,
46
252
  upsert_all: :upsert,
47
- touch: :touch
253
+ touch: :touch,
254
+ touch_all: :bulk
255
+ }.freeze
256
+
257
+ # Class-level methods Lyra overrides elsewhere (ES-NoProj's
258
+ # CrudInterceptor) that write through a method of METHODS. Kept equal
259
+ # to those overrides by the same test.
260
+ ROUTED = {
261
+ update_counters: :update_all,
262
+ increment_counter: :update_all,
263
+ decrement_counter: :update_all
48
264
  }.freeze
49
265
 
50
266
  workflow_name "Callback-Bypassing Writes"
51
267
 
52
268
  places :idle, :ready,
53
- :strict_on, :strict_off, :table_backed, :events_only,
269
+ :strict_on, :strict_off, :table_backed, :events_only, :streams_checked,
54
270
  :instance_requested, :instance_allowed, :instance_event_first,
55
271
  :column_requested, :column_allowed, :column_event_pending,
56
272
  :touch_requested, :touch_event_pending,
57
273
  :bulk_requested, :bulk_allowed, :bulk_event_pending,
58
- :upsert_requested, :upsert_allowed, :upsert_event_pending,
274
+ :insert_requested, :insert_allowed, :insert_ready, :insert_event_pending,
275
+ :upsert_requested, :upsert_allowed, :upsert_ready, :upsert_event_pending,
59
276
  :nullify_allowed, :nullify_event_first,
60
- :rejected, :event_logged, :store_changed
277
+ :join_requested, :link_allowed, :link_event_pending,
278
+ :join_insert_requested, :join_insert_allowed, :linked_event_pending,
279
+ :rejected, :event_logged, :store_changed, :unchanged
61
280
 
62
281
  initial_place :idle
63
- terminal_places :rejected, :store_changed
282
+ terminal_places :rejected, :store_changed, :unchanged
64
283
 
65
284
  # Configuration
66
285
  transition :configure_strict_table, from: :idle, to: [:strict_on, :table_backed, :ready],
67
- trigger: "strict_data_access = true, projections enabled"
68
- transition :configure_strict_events, from: :idle, to: [:strict_on, :events_only, :ready],
286
+ trigger: "strict_data_access = true; ES-Sync, Hijack or Monitor"
287
+ transition :configure_strict_lagging, from: :idle, to: [:strict_on, :table_backed, :streams_checked, :ready],
288
+ trigger: "strict_data_access = true; ES-Async or ES-Lazy (BypassEvents.checks_streams?)"
289
+ transition :configure_strict_events, from: :idle, to: [:strict_on, :events_only, :streams_checked, :ready],
69
290
  trigger: "strict_data_access = true, event sourcing with projections disabled"
70
291
  transition :configure_permissive_table, from: :idle, to: [:strict_off, :table_backed, :ready],
71
- trigger: "strict_data_access = false (default), projections enabled"
72
- transition :configure_permissive_events, from: :idle, to: [:strict_off, :events_only, :ready],
292
+ trigger: "strict_data_access = false (default); ES-Sync, Hijack or Monitor"
293
+ transition :configure_permissive_lagging, from: :idle, to: [:strict_off, :table_backed, :streams_checked, :ready],
294
+ trigger: "strict_data_access = false; ES-Async or ES-Lazy (BypassEvents.checks_streams?)"
295
+ transition :configure_permissive_events, from: :idle, to: [:strict_off, :events_only, :streams_checked, :ready],
73
296
  trigger: "strict_data_access = false, event sourcing with projections disabled"
74
297
 
75
298
  # The strict guard, shared by every overridable method
76
- %i[instance column bulk upsert].each do |kind|
299
+ %i[instance column bulk insert upsert].each do |kind|
77
300
  transition :"request_#{kind}", from: :ready, to: :"#{kind}_requested",
78
301
  trigger: "a #{kind} bypass method is called"
79
302
  transition :"reject_#{kind}", from: [:"#{kind}_requested", :strict_on], to: [:rejected, :strict_on],
@@ -89,6 +312,13 @@ module Lyra
89
312
  # (StrictDataAccess#with_bypass_event)
90
313
  transition :write_column, from: :column_allowed, to: [:store_changed, :column_event_pending],
91
314
  trigger: "super (SQL), persisted values read first"
315
+ # A value a unique index refuses: the table's UPDATE raises (ES-Sync:
316
+ # under the value's lock); in ES-NoProj, ES-Async and ES-Lazy the
317
+ # event store says it is taken, before anything is written or
318
+ # recorded (Lyra::Projections::UniqueIndexes).
319
+ transition :refuse_unique_column, from: :column_allowed, to: :rejected,
320
+ trigger: "raise ActiveRecord::RecordNotUnique (a unique index: the table's UPDATE, or in ES-NoProj, " \
321
+ "ES-Async and ES-Lazy a value a live record holds in the event store, UniqueIndexes)"
92
322
  transition :publish_column_event, from: :column_event_pending, to: :event_logged,
93
323
  trigger: "BypassEvents.publish"
94
324
 
@@ -106,37 +336,217 @@ module Lyra
106
336
  transition :write_instance, from: :instance_event_first, to: :store_changed,
107
337
  trigger: "super (SQL)"
108
338
 
109
- # update_all, delete_all, insert_all: write, then an event per row
339
+ # update_all, delete_all, touch_all: write, then an event per row
110
340
  # (StrictDataAccessRelation)...
111
341
  transition :write_bulk, from: [:bulk_allowed, :table_backed], to: [:store_changed, :bulk_event_pending, :table_backed],
112
- trigger: "super (SQL), rows snapshotted first"
342
+ trigger: "super (SQL) on the rows snapshotted, and locked, first"
113
343
  transition :publish_bulk_events, from: :bulk_event_pending, to: :event_logged,
114
- trigger: "BypassEvents.publish_updates / publish_destroys / publish_upserts"
344
+ trigger: "BypassEvents.publish_updates / publish_destroys"
115
345
  # ...or, with the stream as the only store, the events are the write
116
346
  # (CachedRelation#event_sourced_bulk_mutate)
117
347
  transition :publish_bulk_as_write, from: [:bulk_allowed, :events_only], to: [:event_logged, :store_changed, :events_only],
118
- trigger: "BypassEvents.publish per record"
348
+ trigger: "BypassEvents.publish per record selected now (locked_records_now)"
349
+ # A value a unique index refuses (update_all; update_counters reaches
350
+ # it): the table's UPDATE raises; in ES-NoProj, ES-Async and ES-Lazy
351
+ # the records' tuples are checked against each other and the event
352
+ # store before any write or event (UniqueIndexes::BulkUpdate).
353
+ transition :refuse_unique_bulk, from: :bulk_allowed, to: :rejected,
354
+ trigger: "raise ActiveRecord::RecordNotUnique (a unique index: the table's UPDATE, or in ES-NoProj, " \
355
+ "ES-Async and ES-Lazy the values the records would hold, UniqueIndexes::BulkUpdate)"
119
356
 
120
- # dependent: :nullify, always allowed (StrictDataAccessRelation#update_all)
357
+ # insert_all(!): the rows made to conflict as Monitor's would where
358
+ # the streams are checked, or as given...
359
+ transition :insert_as_given, from: :insert_allowed, to: :insert_ready,
360
+ trigger: "the rows as given: a table that does not lag, ES-Lazy (caught up when the write began), " \
361
+ "rows that give no key"
362
+ transition :bring_insert_keys_up_to_date, from: [:insert_allowed, :streams_checked],
363
+ to: [:insert_ready, :streams_checked],
364
+ trigger: "the keys and unique values the rows give, locked: ES-NoProj reads them from the event store " \
365
+ "(events_only_inserts); ES-Async brings their rows up to date with their streams " \
366
+ "(AsyncProjectionJob.project_pending); ES-Async and ES-Lazy read the unique values " \
367
+ "from the event store (projected_unique_rows)"
368
+ transition :refuse_lock_limit_insert, from: :insert_allowed, to: :rejected,
369
+ trigger: "raise BulkWriteLockLimitError (the keys locked where the streams are checked, and in ES-Sync, " \
370
+ "ES-Async and ES-Lazy the unique values: projected_unique_rows)"
371
+ transition :refuse_genesis_insert, from: [:insert_allowed, :streams_checked], to: [:rejected, :streams_checked],
372
+ trigger: "raise Genesis::Refused (Genesis.first_use, before the keys or unique values are read " \
373
+ "from the event store: the model's import refused, genesis_unrepresentable :refuse)"
374
+ # ...then write, then a created event per row, in every store
375
+ # (StrictDataAccessRelation; CachedRelation#insert_through_table)
376
+ transition :write_insert, from: :insert_ready, to: [:store_changed, :insert_event_pending],
377
+ trigger: "super (SQL), primary keys returned"
378
+ transition :publish_insert_events, from: :insert_event_pending, to: :event_logged,
379
+ trigger: "BypassEvents.publish_upserts (created)"
380
+ # A key or a unique value taken: insert_all skips the row, insert_all!
381
+ # fails its statement. Neither writes or records anything. In
382
+ # ES-NoProj the event store says which are taken
383
+ # (StrictDataAccessRelation#events_only_inserts, under their locks),
384
+ # and in ES-Async and ES-Lazy which unique values are
385
+ # (projected_unique_rows): the conflict is found before anything is
386
+ # written.
387
+ transition :skip_conflict_insert, from: :insert_ready, to: :unchanged,
388
+ trigger: "insert_all: ON CONFLICT DO NOTHING, no key returned " \
389
+ "(ES-NoProj: a key or unique value the event store holds, left out of the insert)"
390
+ transition :refuse_conflict_insert, from: :insert_ready, to: :rejected,
391
+ trigger: "insert_all!, or a conflict on an index other than insert_all's unique_by: " \
392
+ "raise ActiveRecord::RecordNotUnique"
393
+
394
+ # Rails' association nullify, always allowed (StrictDataAccessRelation
395
+ # #update_all; in ES-NoProj AssociationInterceptor and CachedRelation,
396
+ # through the bulk write)
121
397
  transition :request_nullify, from: :ready, to: :nullify_allowed,
122
- trigger: "parent destroyed with dependent: :nullify"
398
+ trigger: "Rails' association code nullifies the relation it marked " \
399
+ "(Lyra::Associations::Maintenance: dependent: :nullify, children removed); " \
400
+ "the application's update_all(fk: nil) is request_bulk"
123
401
  transition :publish_nullify_events, from: :nullify_allowed, to: [:event_logged, :nullify_event_first],
124
402
  trigger: "publish_nullify_events"
125
403
  transition :write_nullify, from: :nullify_event_first, to: :store_changed,
126
404
  trigger: "super (SQL)"
127
405
 
406
+ # has_one dependent: :nullify: Rails' update_columns of the record it
407
+ # marked, the column path without the strict guard (StrictDataAccess
408
+ # #update_columns).
409
+ transition :request_association_column, from: :ready, to: :column_allowed,
410
+ trigger: "has_one dependent: :nullify: update_columns of the record Rails marked"
411
+
412
+ # Removing records from a has_many :through whose through model is
413
+ # monitored: Rails deletes the through records it marked, the bulk
414
+ # path without the strict guard (StrictDataAccessRelation#delete_all,
415
+ # CachedRelation#publish_bulk_events).
416
+ transition :request_through_delete, from: :ready, to: :bulk_allowed,
417
+ trigger: "has_many :through: delete_all of the through records Rails marked"
418
+
419
+ # has_and_belongs_to_many links. Rails' own deletes of the join rows
420
+ # it marked are never refused; the application's delete_all on the
421
+ # join model (or its model of the join table) is guarded as a bulk
422
+ # write is. Then the rows are read, deleted, and each owner's unlink
423
+ # logged (HabtmLinks.delete_all), in every configuration.
424
+ transition :request_unlink, from: :ready, to: :link_allowed,
425
+ trigger: "Rails' association code deletes the join rows it marked " \
426
+ "(delete, clear, destroy, assigning, the owner destroyed)"
427
+ transition :request_join, from: :ready, to: :join_requested,
428
+ trigger: "the application's delete_all on a join model or its model of the join table"
429
+ transition :reject_join, from: [:join_requested, :strict_on], to: [:rejected, :strict_on],
430
+ trigger: "raise StrictDataAccessViolation"
431
+ transition :allow_join, from: [:join_requested, :strict_off], to: [:link_allowed, :strict_off],
432
+ trigger: "strict mode off"
433
+ transition :allow_join_in_bypass_block, from: [:join_requested, :strict_on], to: [:link_allowed, :strict_on],
434
+ trigger: "inside Lyra.without_strict_access"
435
+ transition :write_unlink, from: :link_allowed, to: [:store_changed, :link_event_pending],
436
+ trigger: "super (SQL), the pairs read first"
437
+ transition :publish_unlink, from: :link_event_pending, to: :event_logged,
438
+ trigger: "AssociationUnlinked on each monitored side's streams, every owner a row names " \
439
+ "(HabtmLinks.publish, publish_sides)"
440
+
441
+ # The application's insert_all, insert_all!, upsert_all of join rows,
442
+ # guarded as its delete_all of them is; the rows written read back
443
+ # (RETURNING) and each side's link logged (HabtmLinks.insert_all), in
444
+ # every configuration.
445
+ transition :request_join_insert, from: :ready, to: :join_insert_requested,
446
+ trigger: "the application's insert_all, insert_all! or upsert_all on a join model " \
447
+ "or its model of the join table"
448
+ transition :reject_join_insert, from: [:join_insert_requested, :strict_on], to: [:rejected, :strict_on],
449
+ trigger: "raise StrictDataAccessViolation"
450
+ transition :allow_join_insert, from: [:join_insert_requested, :strict_off], to: [:join_insert_allowed, :strict_off],
451
+ trigger: "strict mode off"
452
+ transition :allow_join_insert_in_bypass_block, from: [:join_insert_requested, :strict_on],
453
+ to: [:join_insert_allowed, :strict_on],
454
+ trigger: "inside Lyra.without_strict_access"
455
+ transition :write_link, from: :join_insert_allowed, to: [:store_changed, :linked_event_pending],
456
+ trigger: "super (SQL), the rows written returned (an upsert's rows read by its keys first)"
457
+ transition :publish_link, from: :linked_event_pending, to: :event_logged,
458
+ trigger: "AssociationLinked on each monitored side's streams, every owner a row names " \
459
+ "(an upsert that moved a row: the old pair's AssociationUnlinked too; HabtmLinks.publish_sides)"
460
+ transition :skip_conflict_link, from: :join_insert_allowed, to: :unchanged,
461
+ trigger: "ON CONFLICT DO NOTHING skipped every row, or an upsert left every pair as it was"
462
+ transition :refuse_conflict_link, from: :join_insert_allowed, to: :rejected,
463
+ trigger: "insert_all! of a pair taken: raise ActiveRecord::RecordNotUnique"
464
+
128
465
  # upsert_all: snapshot by conflict key, write, then created/updated events
129
- transition :write_upsert, from: [:upsert_allowed, :table_backed], to: [:store_changed, :upsert_event_pending, :table_backed],
466
+ transition :upsert_as_given, from: [:upsert_allowed, :table_backed], to: [:upsert_ready, :table_backed],
467
+ trigger: "the rows found by conflict key, as the table has them (ES-Sync, Hijack, Monitor)"
468
+ # Where the streams are checked, the rows it conflicts with (and, in
469
+ # ES-Async, the keys the rows give) are locked and their streams read
470
+ # first (StrictDataAccessRelation#lock_upserted_rows).
471
+ transition :bring_upsert_rows_up_to_date, from: [:upsert_allowed, :table_backed, :streams_checked],
472
+ to: [:upsert_ready, :table_backed, :streams_checked],
473
+ trigger: "their rows brought up to date with their streams and read again " \
474
+ "(ES-Async: AsyncProjectionJob.project_pending; ES-Lazy: LazyProjection.refresh_records)"
475
+ transition :write_upsert, from: [:upsert_ready, :table_backed], to: [:store_changed, :upsert_event_pending, :table_backed],
130
476
  trigger: "super (SQL), existing rows snapshotted by conflict key first"
131
477
  transition :publish_upsert_events, from: :upsert_event_pending, to: :event_logged,
132
478
  trigger: "BypassEvents.publish_upserts"
479
+ # A value of a unique index other than the conflict columns that
480
+ # another record holds: the table's statement raises; in ES-Async and
481
+ # ES-Lazy the event store says it is taken (a record not yet projected
482
+ # holds it), or a value of the conflict columns is held by a record
483
+ # whose row is not in the table, before anything is written or
484
+ # recorded (StrictDataAccessRelation#projected_unique_rows).
485
+ transition :refuse_unique_upsert, from: :upsert_ready, to: :rejected,
486
+ trigger: "raise ActiveRecord::RecordNotUnique (a unique index: the table's statement, or in ES-Async " \
487
+ "and ES-Lazy a value a live record holds in the event store, or a conflict-column holder " \
488
+ "whose row is not in the table, projected_unique_rows)"
133
489
  transition :reject_upsert_events_only, from: [:upsert_allowed, :events_only], to: [:rejected, :events_only],
134
- trigger: "raise ArgumentError (ON CONFLICT would check the table, not the stream)"
490
+ trigger: "raise UnsupportedQuery (ON CONFLICT would check the table, not the stream)"
491
+ transition :refuse_lock_limit_upsert, from: [:upsert_allowed, :table_backed], to: [:rejected, :table_backed],
492
+ trigger: "raise BulkWriteLockLimitError (the rows locked where the streams are checked, and in ES-Sync, " \
493
+ "ES-Async and ES-Lazy the unique values: projected_unique_rows)"
494
+ transition :refuse_destroyed_upsert, from: [:upsert_allowed, :table_backed, :streams_checked],
495
+ to: [:rejected, :table_backed, :streams_checked],
496
+ trigger: "ES-Lazy: raise RecordDestroyedError (a destroy committed after the write's catch-up, " \
497
+ "its row still in the table)"
498
+
499
+ # A row Genesis could not import: refused before any write or event.
500
+ # upsert_all asks after the events-only rejection (its snapshot by
501
+ # conflict key admits the rows), so only where there is a table.
502
+ {
503
+ column: :column_allowed, touch: :touch_requested, instance: :instance_allowed,
504
+ bulk: :bulk_allowed, nullify: :nullify_allowed
505
+ }.each do |kind, at|
506
+ transition :"refuse_genesis_#{kind}", from: at, to: :rejected,
507
+ trigger: "raise Genesis::Refused (Genesis.admit_all!)"
508
+ end
509
+ transition :refuse_genesis_upsert, from: [:upsert_allowed, :table_backed], to: [:rejected, :table_backed],
510
+ trigger: "raise Genesis::Refused (Genesis.admit_all!)"
511
+
512
+ # More locks than config.bulk_write_lock_limit: refused before any
513
+ # write or event (BypassEvents.check_lock_limit!). The records'
514
+ # aggregate locks where the streams are checked; for update_all,
515
+ # insert_all(!) and upsert_all in ES-Sync, ES-Async and ES-Lazy also
516
+ # the unique values' (UniqueIndexes::BulkUpdate#check!,
517
+ # projected_unique_rows), so ES-Sync, which shares table_backed with
518
+ # Hijack and Monitor, reaches the bulk, insert and upsert refusals.
519
+ transition :refuse_lock_limit_bulk, from: :bulk_allowed, to: :rejected,
520
+ trigger: "raise BulkWriteLockLimitError (the records locked where the streams are checked, and in " \
521
+ "ES-Sync an update_all's unique values: UniqueIndexes::BulkUpdate#check!)"
522
+ transition :refuse_lock_limit_nullify, from: [:nullify_allowed, :streams_checked], to: [:rejected, :streams_checked],
523
+ trigger: "raise BulkWriteLockLimitError"
524
+
525
+ # A record destroyed since it was read: no write, no event. Where the
526
+ # streams are checked, its stream ends in a destroy, read under its
527
+ # aggregate lock (BypassEvents.destroyed_ids / locked_snapshot,
528
+ # CachedRelation#locked_records_now; a row still in the table is not
529
+ # written: BypassEvents.unwritten); table-backed, its row is gone
530
+ # (SQL matches none; delete locks the row first and finds none).
531
+ {
532
+ column: :column_allowed, touch: :touch_requested, instance: :instance_allowed,
533
+ bulk: :bulk_allowed, nullify: :nullify_allowed
534
+ }.each do |kind, at|
535
+ transition :"skip_destroyed_#{kind}", from: at, to: :unchanged,
536
+ trigger: "the record's stream ends in a destroy, or its row is gone"
537
+ end
538
+
539
+ ENDING_PLACES = %i[rejected store_changed event_logged unchanged].freeze
540
+ # The outcome places a covered dead marking holds, exactly.
541
+ ENDINGS = [%i[rejected], %i[event_logged store_changed], %i[unchanged]].freeze
135
542
 
136
543
  # Bypass Coverage over every reachable dead marking of the workflow.
137
544
  #
138
545
  # @return [Hash] covered: true when there is no silent write;
139
- # silent_writes: the offending dead markings, as place => tokens hashes.
546
+ # dead_markings: how many there are; unchanged: how many of them
547
+ # end with no write and no event; silent_writes: the offending dead
548
+ # markings (any other combination of ENDING_PLACES, none included),
549
+ # as place => tokens hashes.
140
550
  def self.coverage(workflow = new)
141
551
  analyzer = PetriFlow::Verification::ReachabilityAnalyzer.new(workflow.net, workflow.initial_marking)
142
552
  analyzer.analyze
@@ -145,13 +555,13 @@ module Lyra
145
555
  marked_places(marking, workflow) if workflow.net.deadlocked?
146
556
  end
147
557
 
148
- silent = dead.reject do |marked|
149
- marked[:rejected] || (marked[:store_changed] && marked[:event_logged])
150
- end
558
+ endings = dead.map { |marked| marked.slice(*ENDING_PLACES).keys.sort }
559
+ silent = dead.reject.with_index { |_, i| ENDINGS.include?(endings[i]) }
151
560
 
152
561
  {
153
562
  covered: silent.empty?,
154
563
  dead_markings: dead.size,
564
+ unchanged: endings.count(%i[unchanged]),
155
565
  silent_writes: silent
156
566
  }
157
567
  ensure