@serfab/cadre-core 0.13.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/README.md +20 -6
  2. package/dist/bootstrap-peer-store-file.js +2 -2
  3. package/dist/bootstrap-peer-store.d.ts +32 -12
  4. package/dist/bootstrap-peer-store.js +41 -15
  5. package/dist/bootstrap-peer-store.js.map +1 -1
  6. package/dist/cadre-node.d.ts +558 -80
  7. package/dist/cadre-node.js +1060 -170
  8. package/dist/cadre-node.js.map +1 -1
  9. package/dist/control-authorization.d.ts +2 -2
  10. package/dist/control-authorization.js +1 -0
  11. package/dist/control-authorization.js.map +1 -1
  12. package/dist/control-cohort.d.ts +13 -26
  13. package/dist/control-cohort.js +0 -26
  14. package/dist/control-cohort.js.map +1 -1
  15. package/dist/control-database.d.ts +299 -44
  16. package/dist/control-database.js +469 -69
  17. package/dist/control-database.js.map +1 -1
  18. package/dist/control-read-retry.d.ts +40 -11
  19. package/dist/control-read-retry.js +48 -15
  20. package/dist/control-read-retry.js.map +1 -1
  21. package/dist/control-retry.d.ts +58 -3
  22. package/dist/control-retry.js +58 -3
  23. package/dist/control-retry.js.map +1 -1
  24. package/dist/control-schema.d.ts +1 -1
  25. package/dist/control-schema.js +103 -19
  26. package/dist/control-schema.js.map +1 -1
  27. package/dist/control-write-retry.d.ts +80 -11
  28. package/dist/control-write-retry.js +168 -29
  29. package/dist/control-write-retry.js.map +1 -1
  30. package/dist/enrolled-machine-store.js +2 -2
  31. package/dist/index.d.ts +14 -7
  32. package/dist/index.js +33 -6
  33. package/dist/index.js.map +1 -1
  34. package/dist/node-local-snapshot.d.ts +15 -7
  35. package/dist/node-local-snapshot.js +23 -8
  36. package/dist/node-local-snapshot.js.map +1 -1
  37. package/dist/peer-dial.d.ts +143 -0
  38. package/dist/peer-dial.js +176 -0
  39. package/dist/peer-dial.js.map +1 -0
  40. package/dist/relay-addrs.d.ts +90 -52
  41. package/dist/relay-addrs.js +91 -59
  42. package/dist/relay-addrs.js.map +1 -1
  43. package/dist/relay-reservation.d.ts +71 -3
  44. package/dist/relay-reservation.js +116 -8
  45. package/dist/relay-reservation.js.map +1 -1
  46. package/dist/seed-bootstrap.d.ts +30 -1
  47. package/dist/seed-bootstrap.js +62 -17
  48. package/dist/seed-bootstrap.js.map +1 -1
  49. package/dist/storage-scope.d.ts +62 -0
  50. package/dist/storage-scope.js +75 -0
  51. package/dist/storage-scope.js.map +1 -0
  52. package/dist/strand-database.d.ts +34 -16
  53. package/dist/strand-database.js +38 -24
  54. package/dist/strand-database.js.map +1 -1
  55. package/dist/strand-first-sync-gate.d.ts +152 -0
  56. package/dist/strand-first-sync-gate.js +208 -0
  57. package/dist/strand-first-sync-gate.js.map +1 -0
  58. package/dist/strand-formation-manager.d.ts +87 -5
  59. package/dist/strand-formation-manager.js +109 -5
  60. package/dist/strand-formation-manager.js.map +1 -1
  61. package/dist/strand-formation-protocol.d.ts +21 -1
  62. package/dist/strand-formation-protocol.js +30 -0
  63. package/dist/strand-formation-protocol.js.map +1 -1
  64. package/dist/strand-instance-manager.d.ts +243 -6
  65. package/dist/strand-instance-manager.js +470 -37
  66. package/dist/strand-instance-manager.js.map +1 -1
  67. package/dist/strand-member-key.d.ts +20 -12
  68. package/dist/strand-member-key.js +20 -12
  69. package/dist/strand-member-key.js.map +1 -1
  70. package/dist/strand-member-registry.js +3 -0
  71. package/dist/strand-member-registry.js.map +1 -1
  72. package/dist/strand-membership-reconciler.d.ts +361 -0
  73. package/dist/strand-membership-reconciler.js +548 -0
  74. package/dist/strand-membership-reconciler.js.map +1 -0
  75. package/dist/strand-membership-writer.d.ts +131 -11
  76. package/dist/strand-membership-writer.js +396 -134
  77. package/dist/strand-membership-writer.js.map +1 -1
  78. package/dist/strand-network-config.d.ts +46 -14
  79. package/dist/strand-network-config.js +79 -23
  80. package/dist/strand-network-config.js.map +1 -1
  81. package/dist/strand-solicitation.d.ts +10 -1
  82. package/dist/strand-solicitation.js +5 -0
  83. package/dist/strand-solicitation.js.map +1 -1
  84. package/dist/strand-wake-protocol.d.ts +11 -7
  85. package/dist/strand-wake-protocol.js +20 -52
  86. package/dist/strand-wake-protocol.js.map +1 -1
  87. package/dist/strand-watcher.js +6 -0
  88. package/dist/strand-watcher.js.map +1 -1
  89. package/dist/timed-step.d.ts +12 -0
  90. package/dist/timed-step.js +28 -0
  91. package/dist/timed-step.js.map +1 -0
  92. package/dist/timeout-scheduler.d.ts +19 -0
  93. package/dist/timeout-scheduler.js +22 -0
  94. package/dist/timeout-scheduler.js.map +1 -0
  95. package/dist/types.d.ts +262 -30
  96. package/dist/types.js.map +1 -1
  97. package/package.json +8 -8
  98. package/src/bootstrap-peer-store-file.ts +2 -2
  99. package/src/bootstrap-peer-store.ts +69 -27
  100. package/src/cadre-node.ts +7301 -6353
  101. package/src/control-authorization.ts +2 -1
  102. package/src/control-cohort.ts +122 -130
  103. package/src/control-database.ts +537 -69
  104. package/src/control-read-retry.ts +202 -167
  105. package/src/control-retry.ts +297 -190
  106. package/src/control-schema.ts +103 -19
  107. package/src/control-write-retry.ts +525 -383
  108. package/src/enrolled-machine-store.ts +2 -2
  109. package/src/index.ts +83 -6
  110. package/src/node-local-snapshot.ts +25 -8
  111. package/src/peer-dial.ts +215 -0
  112. package/src/relay-addrs.ts +458 -436
  113. package/src/relay-reservation.ts +951 -828
  114. package/src/seed-bootstrap.ts +76 -18
  115. package/src/storage-scope.ts +78 -0
  116. package/src/strand-database.ts +50 -27
  117. package/src/strand-first-sync-gate.ts +258 -0
  118. package/src/strand-formation-manager.ts +140 -5
  119. package/src/strand-formation-protocol.ts +43 -1
  120. package/src/strand-instance-manager.ts +577 -37
  121. package/src/strand-member-key.ts +20 -12
  122. package/src/strand-member-registry.ts +3 -0
  123. package/src/strand-membership-reconciler.ts +680 -0
  124. package/src/strand-membership-writer.ts +527 -189
  125. package/src/strand-network-config.ts +102 -24
  126. package/src/strand-solicitation.ts +16 -1
  127. package/src/strand-wake-protocol.ts +21 -66
  128. package/src/strand-watcher.ts +6 -0
  129. package/src/timed-step.ts +28 -0
  130. package/src/timeout-scheduler.ts +28 -0
  131. package/src/types.ts +1789 -1554
@@ -1,383 +1,525 @@
1
- import { chainMessages, retryControlOperation } from './control-retry.js';
2
- import type { ControlRetryOptions } from './control-retry.js';
3
-
4
- /**
5
- * Bounded retry for TRANSIENT control-write failures — the classifier and loop behind
6
- * `ControlDatabase.lockedWithRetry`, the single funnel every local control write passes
7
- * through.
8
- *
9
- * Why this exists: the control database replicates every block to the whole party, so a
10
- * three-machine party commits control writes unanimously (`ceil(3 × 0.75) = 3` approvals) —
11
- * and no `superMajorityThreshold` can lower that bar without violating Optimystic's
12
- * partition-safety condition at the shipped admission fraction (the arithmetic lives in
13
- * `docs/architecture.md` → "Replication cluster size"). Cadre therefore accepts unanimity
14
- * and absorbs the transient half of its cost here: a write that failed because the cohort
15
- * did not ANSWER (a stream reset mid-connection-formation, zero approvals) is re-presented
16
- * a moment later, which the degraded-cohort-member scenario measured as safe (a failed
17
- * write rolls back, nothing half-commits) and effective (the next write commits in ~1 s).
18
- *
19
- * A write somebody actually REJECTED is never retried — re-presenting it would re-present
20
- * a spent signature against a cohort that already said no.
21
- *
22
- * TWO policies ship from here, and the split is deliberate: the default
23
- * ({@link isRetriableControlWriteFailure}, {@link CONTROL_WRITE_ATTEMPTS}) sits under every
24
- * control write, and {@link SCHEMA_INIT_RETRY_POLICY} is opted into by
25
- * `ControlDatabase.loadSchema` alone. The widened one absorbs one extra failure class whose
26
- * re-run safety holds for schema DDL and NOT for writes in general; anything added to the default
27
- * list must be safe for all ~19 callers.
28
- *
29
- * Lives outside `control-database.ts` so a spec can drive the classifier against real engine
30
- * errors without importing the whole database class. The loop itself lives in
31
- * `control-retry.ts`, shared with the READ policy (`control-read-retry.ts`) — this module
32
- * owns the write policies: their classifiers, attempt counts, backoff and budget.
33
- */
34
-
35
- /**
36
- * Attempts allowed per control write: the first, plus two retries of a transiently-failed
37
- * cluster commit. Bounded so a genuinely unreachable cohort terminates in the transactor's
38
- * own error rather than spinning; two retries covers the observed failure (a stream still
39
- * forming when the write fired) with margin.
40
- */
41
- export const CONTROL_WRITE_ATTEMPTS = 3;
42
-
43
- /**
44
- * Attempts allowed for {@link SCHEMA_INIT_RETRY_POLICY}, the one call site that also absorbs a
45
- * self-coordination refusal.
46
- *
47
- * Two more than {@link CONTROL_WRITE_ATTEMPTS} because what this retry buys is wall-clock for a
48
- * cold-starting node's bootstrap dials to land, and because the refusal is cheap to re-provoke
49
- * relative to a cluster round-trip: `findCoordinator` already spends its own bounded wait before
50
- * raising (3 attempts, 500 ms apart, `db-p2p/src/libp2p-key-network.ts` ~422) and then decides
51
- * LOCALLY that it may not elect itself, so no peer is contacted and no cohort deadline is
52
- * consumed.
53
- *
54
- * Note that the ~1 s `findCoordinator` spends internally is real, on top of this policy's own
55
- * backoff — which is exactly why {@link CONTROL_WRITE_RETRY_BUDGET_MS} and not this count is what
56
- * actually terminates the loop in the worst case.
57
- */
58
- export const SCHEMA_INIT_ATTEMPTS = 5;
59
-
60
- /**
61
- * Elapsed-time ceiling on the whole retry loop, measured from the START of the first
62
- * attempt and checked after a failed attempt BEFORE sleeping.
63
- *
64
- * This budget — not the attempt count — is what makes the policy safe to sit under every
65
- * control write. A transient failure (stream reset while a connection is still forming)
66
- * surfaces in well under a second, so it is retried and the loop adds at most ~2.2 s. A
67
- * genuinely silent cohort member fails at ~20 s (two 10 s `ClusterClient` response-deadline
68
- * attempts, measured in `control-write-degraded-cohort-member.integration.ts`), which
69
- * already exceeds this budget when attempt 1 returns — so that case is surfaced immediately
70
- * and retry adds ZERO latency to the case where it cannot help.
71
- */
72
- export const CONTROL_WRITE_RETRY_BUDGET_MS = 10_000;
73
-
74
- /**
75
- * Backoff before retry N, jittered ±50% (so the first sleep is 125–375 ms — the jitter can
76
- * never floor a delay to zero). Each single sleep is additionally capped at the LARGEST
77
- * delay in the list, so a `stop()` racing a backoff is never held up by more than ~1 s.
78
- * There is no `AbortSignal` at this seam to cancel a sleep outright; the cap is the
79
- * substitute, deliberately not plumbing one through every write path.
80
- */
81
- const CONTROL_WRITE_RETRY_DELAYS_MS: readonly number[] = [250, 1_000];
82
-
83
- /**
84
- * Backoff for {@link SCHEMA_INIT_RETRY_POLICY}, jittered and capped the same way. Worst case
85
- * ~4.6 s of sleep across four retries (375 + 750 + 1500 + 2000, each capped at the largest base),
86
- * which stays inside {@link CONTROL_WRITE_RETRY_BUDGET_MS} — the budget still terminates the loop
87
- * first if any single attempt is itself slow.
88
- *
89
- * Sized against measured facts, not guessed:
90
- *
91
- * - the self-coordination guard clears the moment the node has ONE connection again (both
92
- * branches that raise `grace-period-not-elapsed` require `getConnections().length === 0`), not
93
- * when the 30 s grace period expires — so this does NOT need to cover the grace period, only
94
- * the gap until a bootstrap dial completes;
95
- * - the guard's 30 s `gracePeriodMs` cannot be tuned from this repo anyway: no caller in either
96
- * repo passes a `SelfCoordinationConfig`, so the default is always in force;
97
- * - a node that has NEVER connected is waved through as a bootstrap node (network high-water
98
- * mark of 1), so this retry only ever delays a node that HAS seen peers. One whose peers are
99
- * gone for good still fails, ~4.6 s later — which is why the budget stays at the shared
100
- * ceiling rather than stretching to cover the full 30 s window.
101
- */
102
- const SCHEMA_INIT_RETRY_DELAYS_MS: readonly number[] = [250, 500, 1_000, 2_000];
103
-
104
- /** The network transactor's aggregate, raised when some cohort peers gave no usable answer. */
105
- const TRANSACTOR_AGGREGATE = /Some peers did not complete:/;
106
-
107
- /**
108
- * How the aggregate's per-batch details name a SINGLE-block batch — the shape both the block
109
- * read (`get`) and phase 1 (`pend`) format, and the only shape safe to re-present.
110
- */
111
- const SINGLE_BLOCK_BATCH_TOKEN = '[block:';
112
-
113
- /**
114
- * How the aggregate's per-batch details name a MULTI-block batch — the shape phase 2
115
- * (`commitBlocks`) formats, and the marker of an outcome too indeterminate to retry.
116
- */
117
- const COMMIT_BATCH_TOKEN = '[blocks:';
118
-
119
- /**
120
- * The cluster coordinator's super-majority shortfall (`db-p2p/src/repo/cluster-coordinator.ts`),
121
- * matched ONLY with a ZERO rejection count. The same message with a non-zero count means a
122
- * member actually voted no (that branch carries `membership-not-admitted` rejections), and
123
- * retrying it would re-present a spent signature to a cohort that already refused it. This one
124
- * is raised while collecting PROMISES, before any commit, so re-presenting it is safe. A
125
- * decisive rejection is a different error entirely (`ValidatorRejectionError`,
126
- * `Transaction rejected by validators`), which this classifier also never matches.
127
- */
128
- const SUPER_MAJORITY_SHORTFALL_UNANSWERED =
129
- /Failed to get super-majority: \d+\/\d+ approvals \(needed \d+, 0 rejections\)/;
130
-
131
- /**
132
- * Is this a transactor aggregate from a phase where nothing can have committed yet?
133
- *
134
- * The transactor raises `Some peers did not complete:` from three sites
135
- * (`db-core/src/transactor/network-transactor.ts`), and only two of them are safe to
136
- * re-present:
137
- *
138
- * - `get` (a block read, which a write body also performs) and `pend` (phase 1) fail before
139
- * anything commits, so the write is known not to have landed;
140
- * - `commitBlocks` (phase 2) is REACHABLE here and NOT safe. `commitBlock` throws the
141
- * aggregate when a header/tail commit got NO response at all — precisely the transient
142
- * class this retry targets — and that escapes `commit()`, survives `TransactorSource`'s
143
- * cancel-and-rethrow (cancel reaches PENDING actions only; a peer that already committed
144
- * stays committed) and `Collection.syncInternal` (which retries StaleFailure RETURN values,
145
- * not throws), reaching this funnel QuereusError-wrapped. The tail commit is one batch to
146
- * one coordinator running consensus internally, so a no-response there is INDETERMINATE:
147
- * the commit may have completed with only the response lost. Re-running the write body over
148
- * a write that landed turns a success into a constraint failure (e.g.
149
- * `UNIQUE constraint failed: CadrePeer.PeerId`). Surfacing the transient error instead lets
150
- * the caller re-read committed state and decide. Commit-phase messages are vetoed by
151
- * {@link reportsIndeterminateCommit}, not here.
152
- *
153
- * The `cause` chain does not discriminate the phases — a commit-phase aggregate carries the
154
- * same stream-reset/dial error a pend-phase one does. The per-batch DETAIL text does: `get`
155
- * and `pend` format `<peerId>[block:<id>](<status>)`, `commitBlocks` formats
156
- * `<peerId>[blocks:<count>](<status>)`. `[block:` cannot occur inside `[blocks:`, so the two
157
- * tokens are disjoint and separate the phases.
158
- *
159
- * NOTE: the discriminator is a formatting detail of another repo. If Optimystic ever reformats
160
- * those per-batch details, this fails CLOSED — the aggregate stops matching and control writes
161
- * simply stop being retried, silently losing the absorption rather than doing anything unsafe.
162
- * The guard against that is live: `control-write-degraded-cohort-member.integration.ts` asserts
163
- * this classifier against the real degraded-cohort failure object and asserts the `[block:`
164
- * token on the live aggregate; if it reddens there, an upstream reformat is the first thing to
165
- * check.
166
- *
167
- * An aggregate whose details came out EMPTY (possible when `formatBatchStatuses` has no
168
- * batches to format) carries neither token, matches nothing here, and is not retried — an
169
- * unattributable failure is not a proven non-commit.
170
- *
171
- * Exported for the READ classifier (`control-read-retry.ts`), which reuses it verbatim: a
172
- * `get` batch formats the same single-block `[block:` token, and a read can never produce
173
- * the commit-phase `[blocks:` shape at all.
174
- */
175
- export function isUncommittedTransactorAggregate(message: string): boolean {
176
- return TRANSACTOR_AGGREGATE.test(message) && message.includes(SINGLE_BLOCK_BATCH_TOKEN);
177
- }
178
-
179
- /**
180
- * Does ANY message in the failure's `cause` chain describe a commit-phase batch — i.e. an
181
- * outcome nobody can call committed or not?
182
- *
183
- * Vetoes the whole chain rather than the one message carrying the token, and vetoes it
184
- * regardless of which matcher would otherwise have claimed it. The narrower per-message rule
185
- * would already catch every shape the transactor is known to build (a commit-phase aggregate
186
- * emits `[blocks:` for every batch it formats, and embeds its cause's message inline as
187
- * `root: …`, so both tokens land in one string) — but "known to build" is an argument about
188
- * another repo's error assembly, and this way an indeterminate commit anywhere in the chain
189
- * costs a retry we could have had instead of risking a re-run over a write that landed.
190
- */
191
- function reportsIndeterminateCommit(messages: readonly string[]): boolean {
192
- return messages.some(message => message.includes(COMMIT_BATCH_TOKEN));
193
- }
194
-
195
- /** The coordinator's shortfall, raised pre-commit, with nobody having voted no. */
196
- function isUnansweredSuperMajorityShortfall(message: string): boolean {
197
- return SUPER_MAJORITY_SHORTFALL_UNANSWERED.test(message);
198
- }
199
-
200
- /**
201
- * Optimystic refusing to let a node elect ITSELF coordinator because its last connection dropped
202
- * less than the guard's grace period ago (`Libp2pKeyPeerNetwork.shouldAllowSelfCoordination` →
203
- * `findCoordinator`, `db-p2p/src/libp2p-key-network.ts`).
204
- *
205
- * Matched ONLY with the `grace-period-not-elapsed` reason. The same sentence is raised with three
206
- * other reasons and none of them is worth a retry: `disabled` is static configuration, and
207
- * `partition-detected` / `suspicious-shrinkage` describe a network condition that will not resolve
208
- * inside a few seconds of backoff. The grace-period one WILL — it clears as soon as the node holds
209
- * one connection again, which a cold-starting node dialling its bootstrap peers is actively
210
- * working on.
211
- *
212
- * NOTE: matched by TEXT, not by type, and that is forced rather than chosen.
213
- * `FindCoordinatorError` (`code: 'SELF_COORDINATION_BLOCKED'`) is exported from `@optimystic/db-p2p`
214
- * and this package depends on it, but the error OBJECT does not survive the trip:
215
- * `OptimysticVirtualTable.initialize` catches and rethrows as `new Error(message)` with no `cause`
216
- * (`quereus-plugin-optimystic/src/optimystic-module.ts`), so only the text reaches this repo. Like
217
- * every matcher here this fails CLOSED — a rewording upstream stops the retry engaging, it never
218
- * makes it unsafe.
219
- */
220
- const SELF_COORDINATION_GRACE_REFUSAL =
221
- /Self-coordination blocked: grace-period-not-elapsed\. No coordinator available for key\./;
222
-
223
- /** Self-coordination refused because the node's last connection dropped moments ago. */
224
- function isSelfCoordinationGraceRefusal(message: string): boolean {
225
- return SELF_COORDINATION_GRACE_REFUSAL.test(message);
226
- }
227
-
228
- /**
229
- * The error messages that mean "the cohort did not answer AND nothing committed", and nothing
230
- * else. The observed one-shot failure was `registerSelf()` racing a connection still forming:
231
- * a read/pend-phase transactor aggregate with `cause=The stream has been reset`.
232
- *
233
- * Deliberately NOT matched: every constraint / authorization failure
234
- * (`CHECK constraint failed:`, `UNIQUE constraint failed:`) — those reach the same funnel
235
- * and must be retried zero times: the cohort REFUSED the write, so re-presenting it
236
- * re-presents a spent signature and can never commit.
237
- */
238
- const RETRIABLE_CONTROL_WRITE_MATCHERS: readonly ((message: string) => boolean)[] = [
239
- isUncommittedTransactorAggregate,
240
- isUnansweredSuperMajorityShortfall,
241
- ];
242
-
243
- /**
244
- * The above PLUS the self-coordination grace refusal — the classifier for schema init ONLY
245
- * ({@link SCHEMA_INIT_RETRY_POLICY}), deliberately not folded into the list above.
246
- *
247
- * Why this class is safe HERE and not everywhere: a write refused at coordinator SELECTION never
248
- * reached a peer, so re-presenting it is a proven non-commit. But `findCoordinator` is also
249
- * reached from `NetworkTransactor.commitBlock` during PHASE 2 (`resolveCoordinator`,
250
- * `db-core/src/transactor/network-transactor.ts`), and `commit()` commits the header block before
251
- * the rest — so a general control write refused there can be refused AFTER something already
252
- * committed. Worse, that error carries no `[blocks:` batch token, so
253
- * {@link reportsIndeterminateCommit} would not veto it, and re-running an insert body over a write
254
- * that landed is exactly the `UNIQUE constraint failed: CadrePeer.PeerId` failure that veto exists
255
- * to prevent.
256
- *
257
- * Schema init does not have that problem: `apply schema` is a diff rather than a replay and a
258
- * failed `create table` leaves the catalog clean, so a re-run emits only the tables that did not
259
- * land (the full argument is at the `loadSchema` call site in `control-database.ts`).
260
- */
261
- const RETRIABLE_SCHEMA_INIT_MATCHERS: readonly ((message: string) => boolean)[] = [
262
- ...RETRIABLE_CONTROL_WRITE_MATCHERS,
263
- isSelfCoordinationGraceRefusal,
264
- ];
265
-
266
- /**
267
- * Did this control write fail because the cluster cohort did not ANSWER — i.e. is
268
- * re-presenting the SAME signed write a moment later the right response?
269
- *
270
- * Classifies by MESSAGE, walking the `cause` chain with the shared {@link chainMessages}
271
- * (Quereus' own `unwrapError` underneath) —
272
- * the failure surface is not recognisable by type. The transactor and the coordinator both throw bare `Error`s, and by
273
- * the time one reaches `ControlDatabase` it is wrapped in a `QuereusError` (the real chain
274
- * is `QuereusError` → `Error` → `Error`), so the match must work at any depth. Anything
275
- * that is not an `Error` is never retried.
276
- *
277
- * {@link reportsIndeterminateCommit} vetoes the whole chain before any matcher runs, so a
278
- * failure that reports a commit-phase batch anywhere is never retried on the strength of some
279
- * other level looking transient.
280
- *
281
- * NOTE: this depends on engine/transactor error TEXT. The messages producible without a
282
- * network are driven from the REAL engine in `control-formation-seat-budget.spec.ts`, so
283
- * a rewording reddens a spec rather than silently disabling the retry. The two RETRIABLE
284
- * messages need a real multi-node cluster to produce; the degraded-cohort scenario
285
- * (`control-write-degraded-cohort-member.integration.ts`) asserts this classifier against
286
- * both LIVE — the super-majority shortfall from a real silent member, and the transactor
287
- * aggregate from a real stream reset, which the retry there absorbs end to end.
288
- */
289
- export function isRetriableControlWriteFailure(error: unknown): boolean {
290
- return matchesRetriableMessage(error, RETRIABLE_CONTROL_WRITE_MATCHERS);
291
- }
292
-
293
- /**
294
- * {@link isRetriableControlWriteFailure} widened by exactly one class — the self-coordination
295
- * grace refusal ({@link RETRIABLE_SCHEMA_INIT_MATCHERS}) — for `ControlDatabase.loadSchema`.
296
- *
297
- * Opt-in per call site, via {@link SCHEMA_INIT_RETRY_POLICY}. Every other control write keeps the
298
- * default classifier untouched.
299
- */
300
- export function isRetriableSchemaInitFailure(error: unknown): boolean {
301
- return matchesRetriableMessage(error, RETRIABLE_SCHEMA_INIT_MATCHERS);
302
- }
303
-
304
- /**
305
- * Shared body of both classifiers: walk the `cause` chain, let
306
- * {@link reportsIndeterminateCommit} veto it, then ask `matchers`. The veto runs BEFORE any
307
- * matcher for every policy — a chain reporting a commit-phase batch is never retried, however
308
- * transient some other level looks.
309
- */
310
- function matchesRetriableMessage(
311
- error: unknown,
312
- matchers: readonly ((message: string) => boolean)[]
313
- ): boolean {
314
- if (!(error instanceof Error)) {
315
- return false;
316
- }
317
- const messages = chainMessages(error);
318
- if (reportsIndeterminateCommit(messages)) {
319
- return false;
320
- }
321
- return messages.some(message => matchers.some(matches => matches(message)));
322
- }
323
-
324
- /**
325
- * Policy and pacing overrides for {@link retryControlWrite}. Production callers pass nothing
326
- * (or a named policy such as {@link SCHEMA_INIT_RETRY_POLICY}); specs inject a recorded
327
- * `sleep`, a shorter delay list, or a fake clock so no test ever waits out a real backoff.
328
- *
329
- * Every field defaults to the shipped control-write policy ({@link CONTROL_WRITE_ATTEMPTS},
330
- * {@link CONTROL_WRITE_RETRY_DELAYS_MS}, {@link isRetriableControlWriteFailure}), so an
331
- * omitted field is byte-identical to the behaviour before per-call-site policies existed.
332
- * The field shapes and semantics live on the shared {@link ControlRetryOptions}.
333
- */
334
- export type ControlWriteRetryOptions = ControlRetryOptions;
335
-
336
- /**
337
- * The one non-default retry policy: `ControlDatabase.loadSchema`'s distributed DDL, which
338
- * additionally absorbs optimystic's self-coordination grace refusal and gets more attempts over a
339
- * longer backoff to do it (see {@link SCHEMA_INIT_ATTEMPTS} and
340
- * {@link SCHEMA_INIT_RETRY_DELAYS_MS}). Still bounded by the shared
341
- * {@link CONTROL_WRITE_RETRY_BUDGET_MS}.
342
- *
343
- * NOTE: exactly one call site opts in today, and the re-run safety this policy assumes is that
344
- * site's, not a general property — `apply schema` is a diff and a failed `create table` leaves the
345
- * catalog clean. A second opt-in must re-derive that argument for its own write body first; if this
346
- * policy ever grows a third consumer, rename it for what the callers share rather than widening it
347
- * by default.
348
- */
349
- export const SCHEMA_INIT_RETRY_POLICY: Readonly<ControlWriteRetryOptions> = {
350
- attempts: SCHEMA_INIT_ATTEMPTS,
351
- delaysMs: SCHEMA_INIT_RETRY_DELAYS_MS,
352
- isRetriable: isRetriableSchemaInitFailure,
353
- };
354
-
355
- /**
356
- * Run `attempt` up to {@link CONTROL_WRITE_ATTEMPTS} times (or `options.attempts`), retrying only
357
- * failures the policy's classifier — {@link isRetriableControlWriteFailure} by default —
358
- * calls transient, and only while {@link CONTROL_WRITE_RETRY_BUDGET_MS} has not elapsed.
359
- *
360
- * `attempt` must be atomic and re-runnable: a failed cluster write rolls back (nothing is
361
- * half-applied), and re-running the body re-runs its reads too — `ControlDatabase`'s
362
- * locked write bodies all satisfy this (see the contract on
363
- * `ControlDatabase.withWriteLock`). The backoff sleeps happen with NO lock held; the caller
364
- * takes and releases its lock inside `attempt`.
365
- *
366
- * On exhaustion (attempts or budget) the LAST error is rethrown unchanged — never wrapped,
367
- * so the exact messages downstream code and the degraded-cohort-member scenario assert on
368
- * survive. A non-retriable failure propagates from the attempt that raised it.
369
- */
370
- export function retryControlWrite<T>(
371
- attempt: () => Promise<T>,
372
- options: ControlWriteRetryOptions = {}
373
- ): Promise<T> {
374
- return retryControlOperation(attempt, {
375
- attempts: CONTROL_WRITE_ATTEMPTS,
376
- delaysMs: CONTROL_WRITE_RETRY_DELAYS_MS,
377
- budgetMs: CONTROL_WRITE_RETRY_BUDGET_MS,
378
- isRetriable: isRetriableControlWriteFailure,
379
- // The default prefix, named anyway: the degraded-cohort scenario asserts on
380
- // `Control write [<label>] …` lines byte-for-byte.
381
- logPrefix: 'Control write',
382
- }, options);
383
- }
1
+ import { CoordinatorPartialCommitError, SyncRetryExhaustedError, TornActionError } from '@optimystic/db-core';
2
+ import { PartialCommitError } from '@optimystic/quereus-plugin-optimystic';
3
+ import { causeChain, chainMessages, retryControlOperation } from './control-retry.js';
4
+ import type { ControlRetryOptions } from './control-retry.js';
5
+
6
+ /**
7
+ * Bounded retry for TRANSIENT control-write failures — the classifier and loop behind
8
+ * `ControlDatabase.lockedWithRetry`, the single funnel every local control write passes
9
+ * through.
10
+ *
11
+ * Why this exists: the control database replicates every block to the whole party, so a
12
+ * three-machine party commits control writes unanimously (`ceil(3 × 0.75) = 3` approvals) —
13
+ * and no `superMajorityThreshold` can lower that bar without violating Optimystic's
14
+ * partition-safety condition at the shipped admission fraction (the arithmetic lives in
15
+ * `docs/architecture.md` → "Replication cluster size"). Cadre therefore accepts unanimity
16
+ * and absorbs the transient half of its cost here: a write that failed because the cohort
17
+ * did not ANSWER (a stream reset mid-connection-formation, zero approvals) is re-presented
18
+ * a moment later, which the degraded-cohort-member scenario measured as safe (a failed
19
+ * write rolls back, nothing half-commits) and effective (the next write commits in ~1 s).
20
+ *
21
+ * A write somebody actually REJECTED is never retried on the strength of the rejection itself —
22
+ * no matcher here claims a rejection message, at any phase, because re-presenting one would
23
+ * re-present a spent signature against a cohort that already said no. Rejections are re-presented
24
+ * anyway when they arrive inside the transactor's promise-phase `[block:` aggregate, which
25
+ * {@link isUncommittedTransactorAggregate} claims on the WRAPPER alone, whatever the cause inside
26
+ * says. Safe because nothing has committed at that phase, and deliberately kept — one of the
27
+ * remaining promise-phase rejections is exactly the kind a re-presentation fixes. See the
28
+ * accepted-tradeoff `NOTE:` there for which, and why no blanket rejection veto is wanted.
29
+ *
30
+ * A failure that says some of the write may be STORED is a different matter, and never
31
+ * re-presented whatever text it carries: {@link reportsPossiblyStoredWrite} vetoes it by type
32
+ * before any matcher runs. The one half-landed write that IS re-presented is a torn write the
33
+ * library marks final — not saved and unable to land ({@link isFinalTornWrite}).
34
+ *
35
+ * TWO policies ship from here, and the split is deliberate: the default
36
+ * ({@link isRetriableControlWriteFailure}, {@link CONTROL_WRITE_ATTEMPTS}) sits under every
37
+ * control write, and {@link SCHEMA_INIT_RETRY_POLICY} is opted into by
38
+ * `ControlDatabase.loadSchema` alone. The widened one absorbs one extra failure class whose
39
+ * re-run safety holds for schema DDL and NOT for writes in general; anything added to the default
40
+ * list must be safe for all ~19 callers.
41
+ *
42
+ * Lives outside `control-database.ts` so a spec can drive the classifier against real engine
43
+ * errors without importing the whole database class. The loop itself lives in
44
+ * `control-retry.ts`, shared with the READ policy (`control-read-retry.ts`) — this module
45
+ * owns the write policies: their classifiers, attempt counts, backoff and budget.
46
+ */
47
+
48
+ /**
49
+ * Attempts allowed per control write: the first, plus two retries of a transiently-failed
50
+ * cluster commit. Bounded so a genuinely unreachable cohort terminates in the transactor's
51
+ * own error rather than spinning; two retries covers the observed failure (a stream still
52
+ * forming when the write fired) with margin.
53
+ */
54
+ export const CONTROL_WRITE_ATTEMPTS = 3;
55
+
56
+ /**
57
+ * Attempts allowed for {@link SCHEMA_INIT_RETRY_POLICY}, the one call site that also absorbs a
58
+ * self-coordination refusal.
59
+ *
60
+ * Two more than {@link CONTROL_WRITE_ATTEMPTS} because what this retry buys is wall-clock for a
61
+ * cold-starting node's bootstrap dials to land, and because the refusal is cheap to re-provoke
62
+ * relative to a cluster round-trip: `findCoordinator` already spends its own bounded wait before
63
+ * raising (3 attempts, 500 ms apart, `db-p2p/src/libp2p-key-network.ts` ~422) and then decides
64
+ * LOCALLY that it may not elect itself, so no peer is contacted and no cohort deadline is
65
+ * consumed.
66
+ *
67
+ * Note that the ~1 s `findCoordinator` spends internally is real, on top of this policy's own
68
+ * backoff — which is exactly why {@link CONTROL_WRITE_RETRY_BUDGET_MS} and not this count is what
69
+ * actually terminates the loop in the worst case.
70
+ */
71
+ export const SCHEMA_INIT_ATTEMPTS = 5;
72
+
73
+ /**
74
+ * Elapsed-time ceiling on the whole retry loop, measured from the START of the first
75
+ * attempt and checked after a failed attempt BEFORE sleeping.
76
+ *
77
+ * This budget — not the attempt count — is what makes the policy safe to sit under every
78
+ * control write. A transient failure (stream reset while a connection is still forming)
79
+ * surfaces in well under a second, so it is retried and the loop adds at most ~2.2 s. A
80
+ * genuinely silent cohort member fails at ~20 s (two 10 s `ClusterClient` response-deadline
81
+ * attempts, measured in `control-write-degraded-cohort-member.integration.ts`), which
82
+ * already exceeds this budget when attempt 1 returns — so that case is surfaced immediately
83
+ * and retry adds ZERO latency to the case where it cannot help.
84
+ *
85
+ * NOTE: a failed commit attempt now also pays a cancel discharge before its error returns
86
+ * (optimystic `TransactorSource.transact`'s catch, added by upstream
87
+ * `1-a-failed-attempt-must-discharge-its-own-pend`) — bounded by six rounds and
88
+ * `abortOrCancelTimeoutMs`, which every collection this repo opens sets to 5 s
89
+ * (`../optimystic/packages/quereus-plugin-optimystic/src/optimystic-adapter/collection-factory.ts`).
90
+ * Two failed attempts whose cancels each run their full budget would consume this whole 10 s
91
+ * ceiling and cut the three-attempt policy to two. Not observed — every measured round of the
92
+ * transient-reset case committed on attempt 3 of 3 — so this is a condition to watch, not work
93
+ * to do: if that case ever starts failing with `failed after 2/3 attempt(s)`, the cancel
94
+ * discharge is where the time went.
95
+ */
96
+ export const CONTROL_WRITE_RETRY_BUDGET_MS = 10_000;
97
+
98
+ /**
99
+ * Backoff before retry N, jittered ±50% (so the first sleep is 125–375 ms — the jitter can
100
+ * never floor a delay to zero). Each single sleep is additionally capped at the LARGEST
101
+ * delay in the list, so a `stop()` racing a backoff is never held up by more than ~1 s.
102
+ * There is no `AbortSignal` at this seam to cancel a sleep outright; the cap is the
103
+ * substitute, deliberately not plumbing one through every write path.
104
+ */
105
+ const CONTROL_WRITE_RETRY_DELAYS_MS: readonly number[] = [250, 1_000];
106
+
107
+ /**
108
+ * Backoff for {@link SCHEMA_INIT_RETRY_POLICY}, jittered and capped the same way. Worst case
109
+ * ~4.6 s of sleep across four retries (375 + 750 + 1500 + 2000, each capped at the largest base),
110
+ * which stays inside {@link CONTROL_WRITE_RETRY_BUDGET_MS} — the budget still terminates the loop
111
+ * first if any single attempt is itself slow.
112
+ *
113
+ * Sized against measured facts, not guessed:
114
+ *
115
+ * - the self-coordination guard clears the moment the node has ONE connection again (both
116
+ * branches that raise `grace-period-not-elapsed` require `getConnections().length === 0`), not
117
+ * when the 30 s grace period expires — so this does NOT need to cover the grace period, only
118
+ * the gap until a bootstrap dial completes;
119
+ * - the guard's 30 s `gracePeriodMs` cannot be tuned from this repo anyway: no caller in either
120
+ * repo passes a `SelfCoordinationConfig`, so the default is always in force;
121
+ * - a node that has NEVER connected is waved through as a bootstrap node (network high-water
122
+ * mark of 1), so this retry only ever delays a node that HAS seen peers. One whose peers are
123
+ * gone for good still fails, ~4.6 s later — which is why the budget stays at the shared
124
+ * ceiling rather than stretching to cover the full 30 s window.
125
+ */
126
+ const SCHEMA_INIT_RETRY_DELAYS_MS: readonly number[] = [250, 500, 1_000, 2_000];
127
+
128
+ /** The network transactor's aggregate, raised when some cohort peers gave no usable answer. */
129
+ const TRANSACTOR_AGGREGATE = /Some peers did not complete:/;
130
+
131
+ /**
132
+ * How the aggregate's per-batch details name a SINGLE-block batch — the shape both the block
133
+ * read (`get`) and phase 1 (`pend`) format, and the only shape safe to re-present.
134
+ */
135
+ const SINGLE_BLOCK_BATCH_TOKEN = '[block:';
136
+
137
+ /**
138
+ * How the aggregate's per-batch details name a MULTI-block batch — the shape phase 2
139
+ * (`commitBlocks`) formats, and the marker of an outcome too indeterminate to retry.
140
+ */
141
+ const COMMIT_BATCH_TOKEN = '[blocks:';
142
+
143
+ /**
144
+ * The cluster coordinator's super-majority shortfall (`db-p2p/src/repo/cluster-coordinator.ts`),
145
+ * matched ONLY with a ZERO rejection count. The same message with a non-zero count means a
146
+ * member actually voted no (that branch carries `membership-not-admitted` rejections), and
147
+ * retrying it would re-present a spent signature to a cohort that already refused it. This one
148
+ * is raised while collecting PROMISES, before any commit, so re-presenting it is safe.
149
+ *
150
+ * This regex itself never matches a decisive rejection (`ValidatorRejectionError`, `Transaction
151
+ * rejected by validators`) — that text carries no `Failed to get super-majority:` prefix. But the
152
+ * composite classifier is not this matcher alone: a rejection raised while promises are still
153
+ * being collected reaches this repo wrapped in the transactor's `Some peers did not complete:
154
+ * …[block:…]` aggregate, and {@link isUncommittedTransactorAggregate} matches that wrapper on its
155
+ * own, whatever the cause inside it says — see the accepted-tradeoff `NOTE:` there.
156
+ */
157
+ const SUPER_MAJORITY_SHORTFALL_UNANSWERED =
158
+ /Failed to get super-majority: \d+\/\d+ approvals \(needed \d+, 0 rejections\)/;
159
+
160
+ /**
161
+ * Is this a transactor aggregate from a phase where nothing can have committed yet?
162
+ *
163
+ * The transactor raises `Some peers did not complete:` from three sites
164
+ * (`db-core/src/transactor/network-transactor.ts`), and only two of them are safe to
165
+ * re-present:
166
+ *
167
+ * - `get` (a block read, which a write body also performs) and `pend` (phase 1) fail before
168
+ * anything commits, so the write is known not to have landed;
169
+ * - `commitBlocks` (phase 2) is REACHABLE here and NOT safe. `commitBlock` throws the
170
+ * aggregate when a header/tail commit got NO response at all — precisely the transient
171
+ * class this retry targets — and that escapes `commit()`, survives `TransactorSource`'s
172
+ * cancel-and-rethrow (cancel reaches PENDING actions only; a peer that already committed
173
+ * stays committed) and `Collection.syncInternal` (which retries StaleFailure RETURN values,
174
+ * not throws), reaching this funnel QuereusError-wrapped. The tail commit is one batch to
175
+ * one coordinator running consensus internally, so a no-response there is INDETERMINATE:
176
+ * the commit may have completed with only the response lost. Re-running the write body over
177
+ * a write that landed turns a success into a constraint failure (e.g.
178
+ * `UNIQUE constraint failed: CadrePeer.PeerId`). Surfacing the transient error instead lets
179
+ * the caller re-read committed state and decide. Commit-phase messages are vetoed by
180
+ * {@link reportsIndeterminateCommit}, not here.
181
+ *
182
+ * The `cause` chain does not discriminate the phases — a commit-phase aggregate carries the
183
+ * same stream-reset/dial error a pend-phase one does. The per-batch DETAIL text does: `get`
184
+ * and `pend` format `<peerId>[block:<id>](<status>)`, `commitBlocks` formats
185
+ * `<peerId>[blocks:<count>](<status>)`. `[block:` cannot occur inside `[blocks:`, so the two
186
+ * tokens are disjoint and separate the phases.
187
+ *
188
+ * NOTE: the discriminator is a formatting detail of another repo. If Optimystic ever reformats
189
+ * those per-batch details, this fails CLOSED — the aggregate stops matching and control writes
190
+ * simply stop being retried, silently losing the absorption rather than doing anything unsafe.
191
+ * The guard against that is live: `control-write-degraded-cohort-member.integration.ts` asserts
192
+ * this classifier against the real degraded-cohort failure object and asserts the `[block:`
193
+ * token on the live aggregate; if it reddens there, an upstream reformat is the first thing to
194
+ * check.
195
+ *
196
+ * NOTE: the discriminator is the prefix AND the token within ONE message — never relax it to
197
+ * "contains `[block:`". Beyond the `get` and `pend` sites above, Optimystic renders `[block:` in
198
+ * one further place — a third formatter, outside the aggregate's three sites — `dischargeCancel`
199
+ * (`Cancel of action <id> did not discharge <n> block(s): …`), which does not raise the
200
+ * `Some peers did not complete:` prefix; only the conjunction keeps a cancel fault from
201
+ * classifying as a retriable get/pend (checked against optimystic `c56c2bd4`, 2026-09-16).
202
+ *
203
+ * NOTE: accepted tradeoff — a REJECTION raised while promises are still being collected reaches
204
+ * this repo wrapped in this same aggregate, and this matcher claims it on the wrapper alone,
205
+ * whatever the cause inside says. That re-presents such a rejection up to two more times inside
206
+ * the 10 s budget. Kept deliberately, and re-decided on 2026-09-17 against the current upstream
207
+ * behaviour rather than inherited:
208
+ *
209
+ * - Contention is no longer a rejection at all. "Another write holds this block right now"
210
+ * (`pending conflict`) is now a `held` verdict that counts toward neither approvals nor
211
+ * rejections (`validatePendOperations`,
212
+ * `../optimystic/packages/db-p2p/src/cluster/cluster-repo.ts`), so the rejection this tradeoff
213
+ * was originally written about does not arrive any more. It reaches this funnel as
214
+ * `SyncRetryExhaustedError` instead, which {@link reportsPossiblyStoredWrite} declines by type
215
+ * — upstream documents resubmitting after it as unsafe (the attempt that spent the budget may
216
+ * have left its log entry standing), and its collection sync has already spent ten retries on
217
+ * the race. The torn-write rule beside it: a `TornActionError` is re-presented only when
218
+ * `final` is true; any other torn write is declined.
219
+ * - What remains rejectable at the promise phase is stale revision, block-unavailable,
220
+ * membership-not-admitted, and a configured validator's refusal. Only the first is worth a
221
+ * retry — and it is worth one: this loop re-runs the WHOLE write body, reads included
222
+ * (`ControlDatabase.withWriteLock`'s contract), so attempt 2 presents a pend against the
223
+ * revision that made attempt 1 stale. A blanket "decline any chain reporting a validator
224
+ * rejection" rule would remove that retry to save two wasted attempts on the other three.
225
+ *
226
+ * Revisit if a re-presentation ever becomes expensive relative to what it buys — a rejection class
227
+ * that costs a full cohort round-trip to re-present, or a write body whose reads are no longer
228
+ * cheap — or when upstream offers a typed refusal surface to classify on instead of this text
229
+ * (`../optimystic/tickets/backlog/debt-a-downstream-repo-classifies-retries-by-parsing-our-error-text`).
230
+ * Contention is no longer the reason to revisit.
231
+ *
232
+ * An aggregate whose details came out EMPTY (possible when `formatBatchStatuses` has no
233
+ * batches to format) carries neither token, matches nothing here, and is not retried — an
234
+ * unattributable failure is not a proven non-commit.
235
+ *
236
+ * Exported for the READ classifier (`control-read-retry.ts`), which reuses it verbatim: a
237
+ * `get` batch formats the same single-block `[block:` token, and a read can never produce
238
+ * the commit-phase `[blocks:` shape at all.
239
+ */
240
+ export function isUncommittedTransactorAggregate(message: string): boolean {
241
+ return TRANSACTOR_AGGREGATE.test(message) && message.includes(SINGLE_BLOCK_BATCH_TOKEN);
242
+ }
243
+
244
+ /**
245
+ * Does ANY message in the failure's `cause` chain describe a commit-phase batch — i.e. an
246
+ * outcome nobody can call committed or not?
247
+ *
248
+ * Vetoes the whole chain rather than the one message carrying the token, and vetoes it
249
+ * regardless of which matcher would otherwise have claimed it. The narrower per-message rule
250
+ * would already catch every shape the transactor is known to build (a commit-phase aggregate
251
+ * emits `[blocks:` for every batch it formats, and embeds its cause's message inline as
252
+ * `root: …`, so both tokens land in one string) — but "known to build" is an argument about
253
+ * another repo's error assembly, and this way an indeterminate commit anywhere in the chain
254
+ * costs a retry we could have had instead of risking a re-run over a write that landed.
255
+ */
256
+ function reportsIndeterminateCommit(messages: readonly string[]): boolean {
257
+ return messages.some(message => message.includes(COMMIT_BATCH_TOKEN));
258
+ }
259
+
260
+ /**
261
+ * Does any link of the `cause` chain say some of this write may be stored, or may still be?
262
+ * Such a failure is never re-presented — a re-run could store the write twice.
263
+ *
264
+ * - `SyncRetryExhaustedError` (and its `SyncRevisionStalledError` subclass): upstream documents
265
+ * resubmitting as unsafe, because the attempt that spent the budget may have left its log
266
+ * entry standing.
267
+ * - `TornActionError` without `final: true`: the write's log entry is stored and nobody could
268
+ * establish that the rest can never land — it may be saved already, or land later.
269
+ * - `CoordinatorPartialCommitError` / `PartialCommitError`: another collection or tree of the
270
+ * same transaction is already durable, and re-running the whole body would apply it twice.
271
+ *
272
+ * Every one of these embeds text from elsewhere in its message (a responder's refusal, the
273
+ * underlying failure), which can carry the pend-phase aggregate's prefix and `[block:` token —
274
+ * so this veto runs before any text matcher.
275
+ *
276
+ * Matched by TYPE: each survives the bridge's and Quereus' rewraps on `cause`. An error built by a
277
+ * second loaded copy of `@optimystic/db-core` (or of the plugin, for `PartialCommitError`) fails
278
+ * `instanceof`, and then this veto does not fire — falling back to the text classifiers, which is
279
+ * how these failures were classified before the veto existed. The asymmetry with
280
+ * {@link isFinalTornWrite} is deliberate: there, a missed `instanceof` means no retry, which is the
281
+ * safe side. No text fallback parses `TornActionError`'s closing sentence, since upstream says its
282
+ * wording is for log lines only.
283
+ *
284
+ * NOTE: the plugin is registered through its `/plugin` entry (`control-database.ts`) but
285
+ * `PartialCommitError` is imported from its root entry; the two are one class only because the
286
+ * plugin's build emits both entries over one shared chunk. If upstream ever bundles the entries
287
+ * separately, this `instanceof` stops matching with every spec still green — import the class from
288
+ * the entry the plugin is registered through.
289
+ */
290
+ function reportsPossiblyStoredWrite(links: readonly Error[]): boolean {
291
+ return links.some(link =>
292
+ link instanceof SyncRetryExhaustedError
293
+ || (link instanceof TornActionError && link.final !== true)
294
+ || link instanceof CoordinatorPartialCommitError
295
+ || link instanceof PartialCommitError);
296
+ }
297
+
298
+ /**
299
+ * A torn write the library marks FINAL: not saved, unable to land, its pending records confirmed
300
+ * cancelled — so submitting it again stores it once.
301
+ *
302
+ * Worth re-presenting, not merely safe to: upstream raises it mostly when a rival holds the
303
+ * revision this write's log entry claimed, which is contention, and the loop re-runs the write
304
+ * body's reads, so the next attempt builds on the rival's revision — the same argument that keeps
305
+ * stale-revision rejections retried (the accepted-tradeoff `NOTE:` on
306
+ * {@link isUncommittedTransactorAggregate}).
307
+ *
308
+ * NOTE: `CoordinatorStaleLossError` (db-core; "nothing durably committed, safe to re-drive") is not
309
+ * claimed, so it falls to the text matchers; it escapes only after the coordinator's own retry budget
310
+ * ran out. If control writes are seen abandoned on it, claim it here by type beside this one.
311
+ */
312
+ function isFinalTornWrite(link: Error): boolean {
313
+ return link instanceof TornActionError && link.final === true;
314
+ }
315
+
316
+ /** The coordinator's shortfall, raised pre-commit, with nobody having voted no. */
317
+ function isUnansweredSuperMajorityShortfall(message: string): boolean {
318
+ return SUPER_MAJORITY_SHORTFALL_UNANSWERED.test(message);
319
+ }
320
+
321
+ /**
322
+ * Optimystic refusing to let a node elect ITSELF coordinator because its last connection dropped
323
+ * less than the guard's grace period ago (`Libp2pKeyPeerNetwork.shouldAllowSelfCoordination` →
324
+ * `findCoordinator`, `db-p2p/src/libp2p-key-network.ts`).
325
+ *
326
+ * Matched ONLY with the `grace-period-not-elapsed` reason. The same sentence is raised with three
327
+ * other reasons and none of them is worth a retry: `disabled` is static configuration, and
328
+ * `partition-detected` / `suspicious-shrinkage` describe a network condition that will not resolve
329
+ * inside a few seconds of backoff. The grace-period one WILL — it clears as soon as the node holds
330
+ * one connection again, which a cold-starting node dialling its bootstrap peers is actively
331
+ * working on.
332
+ *
333
+ * NOTE: matched by TEXT, not by type. `FindCoordinatorError` (`code: 'SELF_COORDINATION_BLOCKED'`)
334
+ * is exported from `@optimystic/db-p2p`, and when this matcher was written the error OBJECT did not
335
+ * survive the trip: `OptimysticVirtualTable.initialize` rethrew as `new Error(message)` with no
336
+ * `cause`. It now rethrows through `rewrapAsQueryError`, which keeps `cause`
337
+ * (`quereus-plugin-optimystic/src/optimystic-module.ts`); whether every Quereus wrapper above it on
338
+ * the schema-init path keeps it too has not been checked, and the text still arrives either way.
339
+ * Like every matcher here this fails CLOSED — a rewording upstream stops the retry engaging, it
340
+ * never makes it unsafe.
341
+ */
342
+ const SELF_COORDINATION_GRACE_REFUSAL =
343
+ /Self-coordination blocked: grace-period-not-elapsed\. No coordinator available for key\./;
344
+
345
+ /** Self-coordination refused because the node's last connection dropped moments ago. */
346
+ function isSelfCoordinationGraceRefusal(message: string): boolean {
347
+ return SELF_COORDINATION_GRACE_REFUSAL.test(message);
348
+ }
349
+
350
+ /**
351
+ * The error messages that mean "the cohort did not answer AND nothing committed", and nothing
352
+ * else. The observed one-shot failure was `registerSelf()` racing a connection still forming:
353
+ * a read/pend-phase transactor aggregate with `cause=The stream has been reset`.
354
+ *
355
+ * Deliberately NOT matched: every constraint / authorization failure
356
+ * (`CHECK constraint failed:`, `UNIQUE constraint failed:`) — those reach the same funnel
357
+ * and must be retried zero times: the cohort REFUSED the write, so re-presenting it
358
+ * re-presents a spent signature and can never commit.
359
+ */
360
+ const RETRIABLE_CONTROL_WRITE_MATCHERS: readonly ((message: string) => boolean)[] = [
361
+ isUncommittedTransactorAggregate,
362
+ isUnansweredSuperMajorityShortfall,
363
+ ];
364
+
365
+ /**
366
+ * The above PLUS the self-coordination grace refusal — the classifier for schema init ONLY
367
+ * ({@link SCHEMA_INIT_RETRY_POLICY}), deliberately not folded into the list above.
368
+ *
369
+ * Why this class is safe HERE and not everywhere: a write refused at coordinator SELECTION never
370
+ * reached a peer, so re-presenting it is a proven non-commit. But `findCoordinator` is also
371
+ * reached from `NetworkTransactor.commitBlock` during PHASE 2 (`resolveCoordinator`,
372
+ * `db-core/src/transactor/network-transactor.ts`), and `commit()` commits the header block before
373
+ * the rest — so a general control write refused there can be refused AFTER something already
374
+ * committed. Worse, that error carries no `[blocks:` batch token, so
375
+ * {@link reportsIndeterminateCommit} would not veto it, and re-running an insert body over a write
376
+ * that landed is exactly the `UNIQUE constraint failed: CadrePeer.PeerId` failure that veto exists
377
+ * to prevent.
378
+ *
379
+ * Schema init does not have that problem: `apply schema` is a diff rather than a replay and a
380
+ * failed `create table` leaves the catalog clean, so a re-run emits only the tables that did not
381
+ * land (the full argument is at the `loadSchema` call site in `control-database.ts`).
382
+ */
383
+ const RETRIABLE_SCHEMA_INIT_MATCHERS: readonly ((message: string) => boolean)[] = [
384
+ ...RETRIABLE_CONTROL_WRITE_MATCHERS,
385
+ isSelfCoordinationGraceRefusal,
386
+ ];
387
+
388
+ /**
389
+ * Did this control write fail because the cluster cohort did not ANSWER — i.e. is
390
+ * re-presenting the SAME signed write a moment later the right response?
391
+ *
392
+ * Classifies the transient cases by MESSAGE, walking the `cause` chain with the shared
393
+ * {@link chainMessages} (Quereus' own `unwrapError` underneath) — the transactor and the
394
+ * coordinator both throw bare `Error`s, not recognisable by type. By the time one reaches
395
+ * `ControlDatabase` it is wrapped in a `QuereusError` (the real chain is `QuereusError` → `Error`
396
+ * → `Error`), so the match must work at any depth. Anything that is not an `Error` is never
397
+ * retried.
398
+ *
399
+ * Two vetoes run over the whole chain before any matcher: {@link reportsPossiblyStoredWrite}
400
+ * (by type — a spent sync budget, a non-final torn write, a partial commit) and
401
+ * {@link reportsIndeterminateCommit} (by text — a commit-phase batch). A chain either veto flags,
402
+ * at any level, is never retried on the strength of some other level looking transient. The one
403
+ * typed failure claimed as retriable is a final torn write ({@link isFinalTornWrite}).
404
+ *
405
+ * NOTE: this depends on engine/transactor error TEXT. The messages producible without a
406
+ * network are driven from the REAL engine in `control-formation-seat-budget.spec.ts`, so
407
+ * a rewording reddens a spec rather than silently disabling the retry. The two RETRIABLE
408
+ * messages need a real multi-node cluster to produce; the degraded-cohort scenario
409
+ * (`control-write-degraded-cohort-member.integration.ts`) asserts this classifier against
410
+ * both LIVE — the super-majority shortfall from a real silent member, and the transactor
411
+ * aggregate from a real stream reset, which the retry there absorbs end to end.
412
+ */
413
+ export function isRetriableControlWriteFailure(error: unknown): boolean {
414
+ return matchesRetriableFailure(error, RETRIABLE_CONTROL_WRITE_MATCHERS);
415
+ }
416
+
417
+ /**
418
+ * {@link isRetriableControlWriteFailure} widened by exactly one class — the self-coordination
419
+ * grace refusal ({@link RETRIABLE_SCHEMA_INIT_MATCHERS}) — for `ControlDatabase.loadSchema`.
420
+ *
421
+ * Opt-in per call site, via {@link SCHEMA_INIT_RETRY_POLICY}. Every other control write keeps the
422
+ * default classifier untouched.
423
+ */
424
+ export function isRetriableSchemaInitFailure(error: unknown): boolean {
425
+ return matchesRetriableFailure(error, RETRIABLE_SCHEMA_INIT_MATCHERS);
426
+ }
427
+
428
+ /**
429
+ * Shared body of both classifiers: walk the `cause` chain, let {@link reportsPossiblyStoredWrite}
430
+ * and then {@link reportsIndeterminateCommit} veto it, then claim a final torn write
431
+ * ({@link isFinalTornWrite}), then ask `matchers`. The vetoes run BEFORE any matcher for every
432
+ * policy — a chain reporting a possibly-stored write or a commit-phase batch is never retried,
433
+ * however transient some other level looks, and a final torn write carried by a partial commit is
434
+ * still declined because a sibling collection landed.
435
+ */
436
+ function matchesRetriableFailure(
437
+ error: unknown,
438
+ matchers: readonly ((message: string) => boolean)[]
439
+ ): boolean {
440
+ if (!(error instanceof Error)) {
441
+ return false;
442
+ }
443
+ const links = causeChain(error);
444
+ if (reportsPossiblyStoredWrite(links)) {
445
+ return false;
446
+ }
447
+ const messages = chainMessages(error);
448
+ if (reportsIndeterminateCommit(messages)) {
449
+ return false;
450
+ }
451
+ if (links.some(isFinalTornWrite)) {
452
+ return true;
453
+ }
454
+ return messages.some(message => matchers.some(matches => matches(message)));
455
+ }
456
+
457
+ /**
458
+ * Policy and pacing overrides for {@link retryControlWrite}. Production callers pass nothing
459
+ * (or a named policy such as {@link SCHEMA_INIT_RETRY_POLICY}); specs inject a recorded
460
+ * `sleep`, a shorter delay list, or a fake clock so no test ever waits out a real backoff.
461
+ *
462
+ * Every field defaults to the shipped control-write policy ({@link CONTROL_WRITE_ATTEMPTS},
463
+ * {@link CONTROL_WRITE_RETRY_DELAYS_MS}, {@link isRetriableControlWriteFailure}), so an
464
+ * omitted field is byte-identical to the behaviour before per-call-site policies existed.
465
+ * The field shapes and semantics live on the shared {@link ControlRetryOptions}.
466
+ */
467
+ export type ControlWriteRetryOptions = ControlRetryOptions;
468
+
469
+ /**
470
+ * The one non-default retry policy: `ControlDatabase.loadSchema`'s distributed DDL, which
471
+ * additionally absorbs optimystic's self-coordination grace refusal and gets more attempts over a
472
+ * longer backoff to do it (see {@link SCHEMA_INIT_ATTEMPTS} and
473
+ * {@link SCHEMA_INIT_RETRY_DELAYS_MS}). Still bounded by the shared
474
+ * {@link CONTROL_WRITE_RETRY_BUDGET_MS}.
475
+ *
476
+ * NOTE: exactly one call site opts in today, and the re-run safety this policy assumes is that
477
+ * site's, not a general property — `apply schema` is a diff and a failed `create table` leaves the
478
+ * catalog clean. A second opt-in must re-derive that argument for its own write body first; if this
479
+ * policy ever grows a third consumer, rename it for what the callers share rather than widening it
480
+ * by default.
481
+ */
482
+ export const SCHEMA_INIT_RETRY_POLICY: Readonly<ControlWriteRetryOptions> = {
483
+ attempts: SCHEMA_INIT_ATTEMPTS,
484
+ delaysMs: SCHEMA_INIT_RETRY_DELAYS_MS,
485
+ isRetriable: isRetriableSchemaInitFailure,
486
+ };
487
+
488
+ /**
489
+ * Run `attempt` up to {@link CONTROL_WRITE_ATTEMPTS} times (or `options.attempts`), retrying only
490
+ * failures the policy's classifier — {@link isRetriableControlWriteFailure} by default —
491
+ * calls transient, and only while {@link CONTROL_WRITE_RETRY_BUDGET_MS} has not elapsed.
492
+ *
493
+ * `attempt` must be atomic and re-runnable: a failed cluster write rolls back (nothing is
494
+ * half-applied), and re-running the body re-runs its reads too — `ControlDatabase`'s
495
+ * locked write bodies all satisfy this (see the contract on
496
+ * `ControlDatabase.withWriteLock`). The backoff sleeps happen with NO lock held; the caller
497
+ * takes and releases its lock inside `attempt`.
498
+ *
499
+ * On exhaustion (attempts or budget) the LAST error is rethrown unchanged — never wrapped,
500
+ * so the exact messages downstream code and the degraded-cohort-member scenario assert on
501
+ * survive. A non-retriable failure propagates from the attempt that raised it.
502
+ *
503
+ * Either way — declined or exhausted — an `options.onAbandon` observer is notified exactly
504
+ * once before the rethrow (a `ControlRetryAbandonment`). Only WRITES pass one: an abandoned
505
+ * read throws to a caller that is awaiting it, so nothing is lost silently there
506
+ * (`control-read-retry.ts` says so at its own seam), while an abandoned write may have no
507
+ * caller at all — a background self-record republish is `void`-ed with a `debug` catch, so
508
+ * its failure reaches nobody unless something is listening here.
509
+ * `ControlDatabase.setControlWriteAbandonedListener` is the only thing that passes one in
510
+ * production; `retryControlWrite` itself neither supplies nor requires an observer.
511
+ */
512
+ export function retryControlWrite<T>(
513
+ attempt: () => Promise<T>,
514
+ options: ControlWriteRetryOptions = {}
515
+ ): Promise<T> {
516
+ return retryControlOperation(attempt, {
517
+ attempts: CONTROL_WRITE_ATTEMPTS,
518
+ delaysMs: CONTROL_WRITE_RETRY_DELAYS_MS,
519
+ budgetMs: CONTROL_WRITE_RETRY_BUDGET_MS,
520
+ isRetriable: isRetriableControlWriteFailure,
521
+ // The default prefix, named anyway: the degraded-cohort scenario asserts on
522
+ // `Control write [<label>] …` lines byte-for-byte.
523
+ logPrefix: 'Control write',
524
+ }, options);
525
+ }