@serfab/cadre-core 1.9.0 → 1.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/dist/cadre-node.d.ts +148 -44
  2. package/dist/cadre-node.js +363 -139
  3. package/dist/cadre-node.js.map +1 -1
  4. package/dist/control-authorization.d.ts +2 -2
  5. package/dist/control-authorization.js +1 -0
  6. package/dist/control-authorization.js.map +1 -1
  7. package/dist/control-cohort.d.ts +1 -1
  8. package/dist/control-cohort.js +1 -1
  9. package/dist/control-database.d.ts +105 -14
  10. package/dist/control-database.js +242 -73
  11. package/dist/control-database.js.map +1 -1
  12. package/dist/control-formation-recorder.d.ts +6 -6
  13. package/dist/control-formation-recorder.js +7 -8
  14. package/dist/control-formation-recorder.js.map +1 -1
  15. package/dist/control-schema.d.ts +1 -1
  16. package/dist/control-schema.js +1066 -979
  17. package/dist/control-schema.js.map +1 -1
  18. package/dist/control-write-retry.d.ts +21 -15
  19. package/dist/control-write-retry.js +21 -15
  20. package/dist/control-write-retry.js.map +1 -1
  21. package/dist/diagnostics/connection-path.d.ts +1 -18
  22. package/dist/diagnostics/connection-path.js +1 -20
  23. package/dist/diagnostics/connection-path.js.map +1 -1
  24. package/dist/identity-key.d.ts +0 -30
  25. package/dist/identity-key.js +7 -40
  26. package/dist/identity-key.js.map +1 -1
  27. package/dist/index.d.ts +8 -5
  28. package/dist/index.js +12 -6
  29. package/dist/index.js.map +1 -1
  30. package/dist/invitation-bootstrap.d.ts +32 -0
  31. package/dist/invitation-bootstrap.js +30 -0
  32. package/dist/invitation-bootstrap.js.map +1 -0
  33. package/dist/link-budget.d.ts +204 -119
  34. package/dist/link-budget.js +208 -120
  35. package/dist/link-budget.js.map +1 -1
  36. package/dist/membership-connection-gater.d.ts +3 -3
  37. package/dist/membership-connection-gater.js +3 -3
  38. package/dist/peer-dial.d.ts +18 -14
  39. package/dist/peer-dial.js +18 -14
  40. package/dist/peer-dial.js.map +1 -1
  41. package/dist/peer-join-backfill.d.ts +3 -3
  42. package/dist/pending-join-runner.d.ts +186 -0
  43. package/dist/pending-join-runner.js +649 -0
  44. package/dist/pending-join-runner.js.map +1 -0
  45. package/dist/push-fanout.d.ts +1 -1
  46. package/dist/push-fanout.js +1 -1
  47. package/dist/relay-stun.d.ts +24 -0
  48. package/dist/relay-stun.js +51 -0
  49. package/dist/relay-stun.js.map +1 -0
  50. package/dist/seed-bootstrap.d.ts +2 -2
  51. package/dist/seed-bootstrap.js +5 -5
  52. package/dist/strand-addr-protocol.js +2 -2
  53. package/dist/strand-formation-deadlines.d.ts +5 -5
  54. package/dist/strand-formation-deadlines.js +5 -5
  55. package/dist/strand-formation-manager.d.ts +18 -4
  56. package/dist/strand-formation-manager.js +48 -33
  57. package/dist/strand-formation-manager.js.map +1 -1
  58. package/dist/strand-formation-protocol.d.ts +71 -14
  59. package/dist/strand-formation-protocol.js +222 -47
  60. package/dist/strand-formation-protocol.js.map +1 -1
  61. package/dist/strand-formation-rejection.d.ts +94 -0
  62. package/dist/strand-formation-rejection.js +77 -0
  63. package/dist/strand-formation-rejection.js.map +1 -0
  64. package/dist/strand-instance-manager.js +4 -1
  65. package/dist/strand-instance-manager.js.map +1 -1
  66. package/dist/strand-membership-writer.js +32 -32
  67. package/dist/strand-solicitation.d.ts +11 -3
  68. package/dist/strand-solicitation.js +9 -0
  69. package/dist/strand-solicitation.js.map +1 -1
  70. package/dist/types.d.ts +103 -17
  71. package/dist/types.js +5 -9
  72. package/dist/types.js.map +1 -1
  73. package/package.json +22 -22
  74. package/src/cadre-node.ts +8654 -8400
  75. package/src/control-authorization.ts +2 -1
  76. package/src/control-cohort.ts +128 -128
  77. package/src/control-database.ts +303 -82
  78. package/src/control-formation-recorder.ts +353 -354
  79. package/src/control-read-retry.ts +216 -216
  80. package/src/control-schema.ts +1078 -991
  81. package/src/control-write-retry.ts +567 -561
  82. package/src/diagnostics/connection-path.ts +1 -29
  83. package/src/formation-approval.ts +610 -610
  84. package/src/hibernation-manager.ts +638 -638
  85. package/src/identity-key.ts +7 -62
  86. package/src/index.ts +644 -627
  87. package/src/invitation-bootstrap.ts +40 -0
  88. package/src/joined-strand-store.ts +498 -498
  89. package/src/link-budget.ts +684 -583
  90. package/src/membership-connection-gater.ts +570 -570
  91. package/src/node-local-snapshot.ts +298 -298
  92. package/src/peer-dial.ts +315 -311
  93. package/src/peer-join-backfill.ts +710 -710
  94. package/src/pending-join-runner.ts +735 -0
  95. package/src/push-fanout.ts +1 -1
  96. package/src/relay-reservation.ts +1121 -1121
  97. package/src/relay-server.ts +98 -98
  98. package/src/relay-stun.ts +58 -0
  99. package/src/seed-bootstrap.ts +1448 -1448
  100. package/src/strand-addr-protocol.ts +547 -547
  101. package/src/strand-database.ts +287 -287
  102. package/src/strand-first-sync-gate.ts +391 -391
  103. package/src/strand-formation-deadlines.ts +227 -227
  104. package/src/strand-formation-manager.ts +725 -700
  105. package/src/strand-formation-protocol.ts +1036 -802
  106. package/src/strand-formation-rejection.ts +129 -0
  107. package/src/strand-fret-addrs.ts +145 -145
  108. package/src/strand-instance-manager.ts +1789 -1786
  109. package/src/strand-membership-writer.ts +1974 -1974
  110. package/src/strand-network-state-file.ts +40 -40
  111. package/src/strand-network-state.ts +243 -243
  112. package/src/strand-revocation-enforcer.ts +654 -654
  113. package/src/strand-solicitation.ts +532 -519
  114. package/src/strand-wake-protocol.ts +401 -401
  115. package/src/types.ts +2222 -2140
  116. package/dist/diagnostics/webrtc-turn-tracker.d.ts +0 -50
  117. package/dist/diagnostics/webrtc-turn-tracker.js +0 -196
  118. package/dist/diagnostics/webrtc-turn-tracker.js.map +0 -1
  119. package/src/diagnostics/webrtc-turn-tracker.ts +0 -222
@@ -1,216 +1,216 @@
1
- import { BlockUnavailableError } from '@optimystic/db-core';
2
- import { causeChain, chainMessages, retryControlOperation } from './control-retry.js';
3
- import type { ControlRetryOptions } from './control-retry.js';
4
- import { isUncommittedTransactorAggregate } from './control-write-retry.js';
5
-
6
- /**
7
- * Bounded retry for TRANSIENT control-read failures — the classifier and policy behind
8
- * `ControlDatabase.readRows`, the single funnel every control read passes through.
9
- *
10
- * Why this exists: control WRITES have absorbed transient cluster failures since
11
- * `control-write-retry.ts` landed, but a read had no second chance — one stream reset
12
- * during a scan ended the read outright (measured: a rigged single `eval` failure killed
13
- * `queryRevokedStamps` after exactly one attempt while the same injection against `exec`
14
- * was absorbed). This module gives reads the same bounded re-presentation with a SHORTER
15
- * deadline and a NARROWER failure set, because the tightest caller deadline over a control
16
- * read is the inbound admission gate's 2 s fail-open timeout
17
- * (`ADMISSION_DECISION_TIMEOUT_MS`, `link-budget.ts`) — a read budget that
18
- * does not fit inside it with headroom spends its retries after the gate has already
19
- * admitted.
20
- *
21
- * This is deliberately NOT "the write classifier minus its commit veto" — the retriable
22
- * set differs in both directions (see {@link isRetriableControlReadFailure}). The loop
23
- * lives in `control-retry.ts`, shared with the write policy; this module owns only the
24
- * read policy. And like the write side, the retry classifies by TEXT. That was forced when
25
- * it was written — optimystic's scan path used to rethrow `new Error('Query failed: ' +
26
- * message)` with no `cause` — and is no longer: `OptimysticVirtualTable` now rethrows
27
- * through `rewrapAsQueryError` (`quereus-plugin-optimystic/src/optimystic-module.ts`),
28
- * which keeps the typed `BlockUnavailableError` on `cause`, and Quereus wraps that as
29
- * `Error during query on table '<T>': …`, again preserving `cause`
30
- * (`quereus/src/runtime/emit/scan.ts`). The text matchers still work because every wrap
31
- * embeds the inner message. Every matcher fails CLOSED — an upstream rewording stops the
32
- * retry engaging, it never makes it unsafe. {@link isCohortUnreachableRead}, which is not a
33
- * retry classifier, matches by type.
34
- */
35
-
36
- /**
37
- * Attempts allowed per control read: the first, plus two retries. Attempts are cheap here —
38
- * the observed transient failure (the transactor's read-phase aggregate off a stream still
39
- * forming) surfaces in ~25 ms — and {@link CONTROL_READ_RETRY_BUDGET_MS} is what terminates
40
- * the loop when an attempt is slow instead.
41
- */
42
- export const CONTROL_READ_ATTEMPTS = 3;
43
-
44
- /**
45
- * Backoff before read retry N, jittered ±50% and capped at the largest base by the shared
46
- * loop — so worst case 150 ms + 400 ms = 550 ms of total sleep (the second delay's
47
- * upside jitter is clipped by the cap). Much shorter than the write list because the
48
- * whole loop must fit under {@link CONTROL_READ_RETRY_BUDGET_MS}; the spec pins that
49
- * relationship rather than the number.
50
- */
51
- export const CONTROL_READ_RETRY_DELAYS_MS: readonly number[] = [100, 400];
52
-
53
- /**
54
- * Elapsed-time ceiling on the whole read-retry loop, checked after a failed attempt BEFORE
55
- * sleeping — so one slow attempt (e.g. a `cohort-unreachable` read that burned the
56
- * transactor's own deadline) terminates the loop rather than compounding.
57
- *
58
- * 1500 ms, sized against `ADMISSION_DECISION_TIMEOUT_MS` (2000 ms): the inbound admission
59
- * gate reads the control DB and is FAIL-OPEN on both throw and timeout, so a read that
60
- * outlives the gate's deadline spends its retries after the gate has already admitted an
61
- * unplaced peer. Under the deadline, a read that succeeds on its second attempt lets the
62
- * gate make the real decision instead. `control-read-retry.spec.ts` asserts the
63
- * relationship so a future edit to either constant reddens instead of silently
64
- * reintroducing the fail-open admit. NOTE: `queryCadrePeers` / `queryPeerRecord` each
65
- * issue TWO network reads (the revoked-stamp filter plus the row scan) and the budget is
66
- * per read, so a membership read can spend up to two budgets back to back — still inside
67
- * 2 s only because each stays well under it. Do not raise these numbers without
68
- * re-checking against `ADMISSION_DECISION_TIMEOUT_MS`.
69
- *
70
- * It cuts the loop off by design, so it does not grow with the link or with the number that
71
- * dominates a slow attempt: an attempt whose cohort consult finds a silent peer costs
72
- * `clusterPolicy.cohortQueryTimeoutMs`, two link round trips at the declared link
73
- * (`cohortReadDeadlineMs` in `link-budget.ts`, 7 000 ms at the default declaration; the plugin's
74
- * `COHORT_READ_DEADLINE_MS` is the same number). That is more than four times this budget, so
75
- * such an attempt is never retried and the caller gets its error unchanged. That is deliberate:
76
- * the caller this budget is sized for, the admission gate, has
77
- * already taken its fail-open answer by then, and a retry would only spend time after it. What
78
- * the budget still buys is the fast failures it was built for: the ~25 ms transactor read-phase
79
- * aggregate off a stream still forming, and a `cohort-unreachable` read during bring-up, which
80
- * fails fast because there is no connection to ask. Both retry twice well inside 1500 ms. See
81
- * `docs/cadre-consistency.md` → "Deadlines Over Optimystic's Reads and Commits".
82
- */
83
- // eslint-disable-next-line no-restricted-syntax -- cuts off by design: a slow attempt is not retried, because the admission gate this budget is sized for has already failed open by then; see docs/cadre-consistency.md → "Deadlines Over Optimystic's Reads and Commits"
84
- export const CONTROL_READ_RETRY_BUDGET_MS = 1_500;
85
-
86
- /**
87
- * The block-unavailability reasons a repeat read can actually improve on, matched against
88
- * upstream's exact template `Block <id> is unavailable (<reason>): …`
89
- * (`BlockUnavailableError`, `db-core/src/network/struct.ts`):
90
- *
91
- * - **`peers-unreachable`** — part of the cohort answered and part could not be asked, and
92
- * OTHER coordinators are reachable, so asking one of them can still settle it. Upstream's
93
- * own definition invites the second attempt.
94
- * - **`cohort-unreachable`** — the marginal member of the set. Upstream says the answer
95
- * will not improve until this node's connectivity does — but during bring-up connectivity
96
- * DOES improve within a second (the boot-time failure shape recorded in
97
- * `tickets/blocked/control-read-over-fresh-edge-stream-resets`), and the cost of being
98
- * wrong is bounded by {@link CONTROL_READ_RETRY_BUDGET_MS}, which is the whole reason it
99
- * is safe to include.
100
- *
101
- * Deliberately NOT matched:
102
- *
103
- * - **`claimed-elsewhere`** — a cohort peer positively claims the block exists and nobody
104
- * could corroborate or acquire it. MEASURED not to clear: reissuing the same call every
105
- * second for 60 s returned the identical error every time (2026-08-20, recorded in the
106
- * fix ticket). Retrying spends the whole budget and fails anyway. The measured cause (a
107
- * block only one machine held) was fixed upstream by the solo-commit proof in
108
- * `@optimystic/db-p2p` 0.28.0 (`tickets/complete/block-held-by-only-one-machine-is-unreadable`),
109
- * so a `claimed-elsewhere` that surfaces now is a new defect — this retry must not hide it.
110
- * - **`unmaterializable`** — records are held locally but cannot be reassembled. A local
111
- * data problem; a second read reads the same records.
112
- * - **`Block <id> may be stale`** (`BlockPossiblyStaleError`) — about currency, not
113
- * existence: a cohort claim no reachable coordinator could confirm or refute will not
114
- * resolve inside a few hundred milliseconds of backoff.
115
- * - Anything unmatched. The default is no retry.
116
- */
117
- const RETRIABLE_BLOCK_UNAVAILABLE =
118
- /Block \S+ is unavailable \((?:peers-unreachable|cohort-unreachable)\)/;
119
-
120
- /** A block-unavailable read failure whose reason a repeat read can improve on. */
121
- function isRetriableBlockUnavailable(message: string): boolean {
122
- return RETRIABLE_BLOCK_UNAVAILABLE.test(message);
123
- }
124
-
125
- /**
126
- * Pacing overrides for {@link retryControlRead} — the read twin of
127
- * `ControlWriteRetryOptions`. Every field defaults to the shipped read policy
128
- * ({@link CONTROL_READ_ATTEMPTS}, {@link CONTROL_READ_RETRY_DELAYS_MS},
129
- * {@link isRetriableControlReadFailure}); field shapes and semantics live on the shared
130
- * {@link ControlRetryOptions}.
131
- */
132
- export type ControlReadRetryOptions = ControlRetryOptions;
133
-
134
- /**
135
- * Did this control read fail in a way a repeat read can improve on?
136
- *
137
- * Two classes, both meaning "the cluster could not be ASKED properly just now":
138
- *
139
- * - the transactor's read-phase aggregate (`Some peers did not complete: …[block:…]`),
140
- * reused verbatim from the write classifier — a `get` batch formats the single-block
141
- * `[block:` token, and a read can never produce the commit-phase `[blocks:` token, so
142
- * the write side's indeterminate-commit veto has nothing to veto here (a read commits
143
- * nothing and re-running it is always safe);
144
- * - the block-unavailability reasons in {@link RETRIABLE_BLOCK_UNAVAILABLE} (and only
145
- * those — see its comment for the measured exclusions).
146
- *
147
- * Classifies by MESSAGE, walking the `cause` chain with the shared `chainMessages` (why
148
- * text, in the module comment). Anything that is not an `Error` is never retried.
149
- */
150
- export function isRetriableControlReadFailure(error: unknown): boolean {
151
- if (!(error instanceof Error)) {
152
- return false;
153
- }
154
- return chainMessages(error).some(message =>
155
- isUncommittedTransactorAggregate(message) || isRetriableBlockUnavailable(message));
156
- }
157
-
158
- /**
159
- * Did this control read fail because no cohort member other than the answering node could
160
- * be asked about a block nobody here holds — a `BlockUnavailableError` with reason
161
- * `cohort-unreachable` anywhere on the `cause` chain?
162
- *
163
- * Not a retry classifier. It answers "is there a better answer to wait for?", and is used
164
- * by exactly one reader, `ControlDatabase.queryRevokedStamps`, to read an isolated node's
165
- * never-received `Revocation` block as "no revocations known" (the reasoning is in the NOTE
166
- * there). Upstream names this the one reason a caller may treat permissively
167
- * (`db-core/src/transactor/network-transactor.ts`). The other reasons do NOT match:
168
- * `peers-unreachable` means part of the cohort answered, and `claimed-elsewhere` /
169
- * `unmaterializable` mean the block is known to exist, so an empty answer could hide a
170
- * real revocation.
171
- *
172
- * Matched by TYPE, since the typed error survives upstream's rewraps (module comment).
173
- * Fails CLOSED: a non-`Error`, a text-only copy of the message, and an error built by a
174
- * second loaded copy of `@optimystic/db-core` (which `instanceof` cannot recognise) all
175
- * answer false, and the reader throws as it did before this existed.
176
- */
177
- export function isCohortUnreachableRead(error: unknown): boolean {
178
- return causeChain(error).some(link =>
179
- link instanceof BlockUnavailableError && link.reason === 'cohort-unreachable');
180
- }
181
-
182
- /**
183
- * Run `attempt` — a full drain of one control read — up to
184
- * {@link CONTROL_READ_ATTEMPTS} times, retrying only failures
185
- * {@link isRetriableControlReadFailure} calls transient, and only while
186
- * {@link CONTROL_READ_RETRY_BUDGET_MS} has not elapsed.
187
- *
188
- * Reads are idempotent, so `attempt` needs no atomicity contract — but it must be a
189
- * COLLECTING drain, not a live iterator: a half-consumed iterator cannot be retried
190
- * without re-yielding rows the caller already saw, which is why
191
- * `ControlDatabase.readRows` materializes before this loop ever sees a failure.
192
- *
193
- * On exhaustion the LAST error is rethrown unchanged. Log lines carry the
194
- * `Control read [<label>] …` prefix so a read's retries can be attributed among the
195
- * several reads in flight concurrently in a real party.
196
- *
197
- * No reader passes an `onAbandon` observer, deliberately, though the shared options type
198
- * offers one ({@link ControlRetryOptions.onAbandon}) and the loop would honour it. That hook
199
- * exists because an abandoned WRITE can have no caller at all — the background self-record
200
- * republish is `void`-ed with a `debug`-only catch, so giving up on it is invisible. Every
201
- * control read is awaited by the caller that issued it, so an abandoned read surfaces as that
202
- * caller's rejection; there is nothing here to lose silently. A fire-and-forget read would be
203
- * the reason to start passing one.
204
- */
205
- export function retryControlRead<T>(
206
- attempt: () => Promise<T>,
207
- options: ControlReadRetryOptions = {}
208
- ): Promise<T> {
209
- return retryControlOperation(attempt, {
210
- attempts: CONTROL_READ_ATTEMPTS,
211
- delaysMs: CONTROL_READ_RETRY_DELAYS_MS,
212
- budgetMs: CONTROL_READ_RETRY_BUDGET_MS,
213
- isRetriable: isRetriableControlReadFailure,
214
- logPrefix: 'Control read',
215
- }, options);
216
- }
1
+ import { BlockUnavailableError } from '@optimystic/db-core';
2
+ import { causeChain, chainMessages, retryControlOperation } from './control-retry.js';
3
+ import type { ControlRetryOptions } from './control-retry.js';
4
+ import { isUncommittedTransactorAggregate } from './control-write-retry.js';
5
+
6
+ /**
7
+ * Bounded retry for TRANSIENT control-read failures — the classifier and policy behind
8
+ * `ControlDatabase.readRows`, the single funnel every control read passes through.
9
+ *
10
+ * Why this exists: control WRITES have absorbed transient cluster failures since
11
+ * `control-write-retry.ts` landed, but a read had no second chance — one stream reset
12
+ * during a scan ended the read outright (measured: a rigged single `eval` failure killed
13
+ * `queryRevokedStamps` after exactly one attempt while the same injection against `exec`
14
+ * was absorbed). This module gives reads the same bounded re-presentation with a SHORTER
15
+ * deadline and a NARROWER failure set, because the tightest caller deadline over a control
16
+ * read is the inbound admission gate's 2 s fail-open timeout
17
+ * (`ADMISSION_DECISION_TIMEOUT_MS`, `link-budget.ts`) — a read budget that
18
+ * does not fit inside it with headroom spends its retries after the gate has already
19
+ * admitted.
20
+ *
21
+ * This is deliberately NOT "the write classifier minus its commit veto" — the retriable
22
+ * set differs in both directions (see {@link isRetriableControlReadFailure}). The loop
23
+ * lives in `control-retry.ts`, shared with the write policy; this module owns only the
24
+ * read policy. And like the write side, the retry classifies by TEXT. That was forced when
25
+ * it was written — optimystic's scan path used to rethrow `new Error('Query failed: ' +
26
+ * message)` with no `cause` — and is no longer: `OptimysticVirtualTable` now rethrows
27
+ * through `rewrapAsQueryError` (`quereus-plugin-optimystic/src/optimystic-module.ts`),
28
+ * which keeps the typed `BlockUnavailableError` on `cause`, and Quereus wraps that as
29
+ * `Error during query on table '<T>': …`, again preserving `cause`
30
+ * (`quereus/src/runtime/emit/scan.ts`). The text matchers still work because every wrap
31
+ * embeds the inner message. Every matcher fails CLOSED — an upstream rewording stops the
32
+ * retry engaging, it never makes it unsafe. {@link isCohortUnreachableRead}, which is not a
33
+ * retry classifier, matches by type.
34
+ */
35
+
36
+ /**
37
+ * Attempts allowed per control read: the first, plus two retries. Attempts are cheap here —
38
+ * the observed transient failure (the transactor's read-phase aggregate off a stream still
39
+ * forming) surfaces in ~25 ms — and {@link CONTROL_READ_RETRY_BUDGET_MS} is what terminates
40
+ * the loop when an attempt is slow instead.
41
+ */
42
+ export const CONTROL_READ_ATTEMPTS = 3;
43
+
44
+ /**
45
+ * Backoff before read retry N, jittered ±50% and capped at the largest base by the shared
46
+ * loop — so worst case 150 ms + 400 ms = 550 ms of total sleep (the second delay's
47
+ * upside jitter is clipped by the cap). Much shorter than the write list because the
48
+ * whole loop must fit under {@link CONTROL_READ_RETRY_BUDGET_MS}; the spec pins that
49
+ * relationship rather than the number.
50
+ */
51
+ export const CONTROL_READ_RETRY_DELAYS_MS: readonly number[] = [100, 400];
52
+
53
+ /**
54
+ * Elapsed-time ceiling on the whole read-retry loop, checked after a failed attempt BEFORE
55
+ * sleeping — so one slow attempt (e.g. a `cohort-unreachable` read that burned the
56
+ * transactor's own deadline) terminates the loop rather than compounding.
57
+ *
58
+ * 1500 ms, sized against `ADMISSION_DECISION_TIMEOUT_MS` (2000 ms): the inbound admission
59
+ * gate reads the control DB and is FAIL-OPEN on both throw and timeout, so a read that
60
+ * outlives the gate's deadline spends its retries after the gate has already admitted an
61
+ * unplaced peer. Under the deadline, a read that succeeds on its second attempt lets the
62
+ * gate make the real decision instead. `control-read-retry.spec.ts` asserts the
63
+ * relationship so a future edit to either constant reddens instead of silently
64
+ * reintroducing the fail-open admit. NOTE: `queryCadrePeers` / `queryPeerRecord` each
65
+ * issue TWO network reads (the revoked-stamp filter plus the row scan) and the budget is
66
+ * per read, so a membership read can spend up to two budgets back to back — still inside
67
+ * 2 s only because each stays well under it. Do not raise these numbers without
68
+ * re-checking against `ADMISSION_DECISION_TIMEOUT_MS`.
69
+ *
70
+ * It cuts the loop off by design, so it does not grow with the link or with the number that
71
+ * dominates a slow attempt: an attempt whose cohort consult finds a silent peer costs
72
+ * `clusterPolicy.cohortQueryTimeoutMs`, two link round trips at the declared link
73
+ * (`cohortReadDeadlineMs` in `link-budget.ts`, 7 000 ms at the default declaration; the plugin's
74
+ * `COHORT_READ_DEADLINE_MS` is the same number). That is more than four times this budget, so
75
+ * such an attempt is never retried and the caller gets its error unchanged. That is deliberate:
76
+ * the caller this budget is sized for, the admission gate, has
77
+ * already taken its fail-open answer by then, and a retry would only spend time after it. What
78
+ * the budget still buys is the fast failures it was built for: the ~25 ms transactor read-phase
79
+ * aggregate off a stream still forming, and a `cohort-unreachable` read during bring-up, which
80
+ * fails fast because there is no connection to ask. Both retry twice well inside 1500 ms. See
81
+ * `docs/cadre-consistency.md` → "Deadlines Over Optimystic's Reads and Commits".
82
+ */
83
+ // eslint-disable-next-line no-restricted-syntax -- cuts off by design: a slow attempt is not retried, because the admission gate this budget is sized for has already failed open by then; see docs/cadre-consistency.md → "Deadlines Over Optimystic's Reads and Commits"
84
+ export const CONTROL_READ_RETRY_BUDGET_MS = 1_500;
85
+
86
+ /**
87
+ * The block-unavailability reasons a repeat read can actually improve on, matched against
88
+ * upstream's exact template `Block <id> is unavailable (<reason>): …`
89
+ * (`BlockUnavailableError`, `db-core/src/network/struct.ts`):
90
+ *
91
+ * - **`peers-unreachable`** — part of the cohort answered and part could not be asked, and
92
+ * OTHER coordinators are reachable, so asking one of them can still settle it. Upstream's
93
+ * own definition invites the second attempt.
94
+ * - **`cohort-unreachable`** — the marginal member of the set. Upstream says the answer
95
+ * will not improve until this node's connectivity does — but during bring-up connectivity
96
+ * DOES improve within a second (the boot-time failure shape recorded in
97
+ * `tickets/blocked/control-read-over-fresh-edge-stream-resets`), and the cost of being
98
+ * wrong is bounded by {@link CONTROL_READ_RETRY_BUDGET_MS}, which is the whole reason it
99
+ * is safe to include.
100
+ *
101
+ * Deliberately NOT matched:
102
+ *
103
+ * - **`claimed-elsewhere`** — a cohort peer positively claims the block exists and nobody
104
+ * could corroborate or acquire it. MEASURED not to clear: reissuing the same call every
105
+ * second for 60 s returned the identical error every time (2026-08-20, recorded in the
106
+ * fix ticket). Retrying spends the whole budget and fails anyway. The measured cause (a
107
+ * block only one machine held) was fixed upstream by the solo-commit proof in
108
+ * `@optimystic/db-p2p` 0.28.0 (`tickets/complete/block-held-by-only-one-machine-is-unreadable`),
109
+ * so a `claimed-elsewhere` that surfaces now is a new defect — this retry must not hide it.
110
+ * - **`unmaterializable`** — records are held locally but cannot be reassembled. A local
111
+ * data problem; a second read reads the same records.
112
+ * - **`Block <id> may be stale`** (`BlockPossiblyStaleError`) — about currency, not
113
+ * existence: a cohort claim no reachable coordinator could confirm or refute will not
114
+ * resolve inside a few hundred milliseconds of backoff.
115
+ * - Anything unmatched. The default is no retry.
116
+ */
117
+ const RETRIABLE_BLOCK_UNAVAILABLE =
118
+ /Block \S+ is unavailable \((?:peers-unreachable|cohort-unreachable)\)/;
119
+
120
+ /** A block-unavailable read failure whose reason a repeat read can improve on. */
121
+ function isRetriableBlockUnavailable(message: string): boolean {
122
+ return RETRIABLE_BLOCK_UNAVAILABLE.test(message);
123
+ }
124
+
125
+ /**
126
+ * Pacing overrides for {@link retryControlRead} — the read twin of
127
+ * `ControlWriteRetryOptions`. Every field defaults to the shipped read policy
128
+ * ({@link CONTROL_READ_ATTEMPTS}, {@link CONTROL_READ_RETRY_DELAYS_MS},
129
+ * {@link isRetriableControlReadFailure}); field shapes and semantics live on the shared
130
+ * {@link ControlRetryOptions}.
131
+ */
132
+ export type ControlReadRetryOptions = ControlRetryOptions;
133
+
134
+ /**
135
+ * Did this control read fail in a way a repeat read can improve on?
136
+ *
137
+ * Two classes, both meaning "the cluster could not be ASKED properly just now":
138
+ *
139
+ * - the transactor's read-phase aggregate (`Some peers did not complete: …[block:…]`),
140
+ * reused verbatim from the write classifier — a `get` batch formats the single-block
141
+ * `[block:` token, and a read can never produce the commit-phase `[blocks:` token, so
142
+ * the write side's indeterminate-commit veto has nothing to veto here (a read commits
143
+ * nothing and re-running it is always safe);
144
+ * - the block-unavailability reasons in {@link RETRIABLE_BLOCK_UNAVAILABLE} (and only
145
+ * those — see its comment for the measured exclusions).
146
+ *
147
+ * Classifies by MESSAGE, walking the `cause` chain with the shared `chainMessages` (why
148
+ * text, in the module comment). Anything that is not an `Error` is never retried.
149
+ */
150
+ export function isRetriableControlReadFailure(error: unknown): boolean {
151
+ if (!(error instanceof Error)) {
152
+ return false;
153
+ }
154
+ return chainMessages(error).some(message =>
155
+ isUncommittedTransactorAggregate(message) || isRetriableBlockUnavailable(message));
156
+ }
157
+
158
+ /**
159
+ * Did this control read fail because no cohort member other than the answering node could
160
+ * be asked about a block nobody here holds — a `BlockUnavailableError` with reason
161
+ * `cohort-unreachable` anywhere on the `cause` chain?
162
+ *
163
+ * Not a retry classifier. It answers "is there a better answer to wait for?", and is used
164
+ * by exactly one reader, `ControlDatabase.queryRevokedStamps`, to read an isolated node's
165
+ * never-received `Revocation` block as "no revocations known" (the reasoning is in the NOTE
166
+ * there). Upstream names this the one reason a caller may treat permissively
167
+ * (`db-core/src/transactor/network-transactor.ts`). The other reasons do NOT match:
168
+ * `peers-unreachable` means part of the cohort answered, and `claimed-elsewhere` /
169
+ * `unmaterializable` mean the block is known to exist, so an empty answer could hide a
170
+ * real revocation.
171
+ *
172
+ * Matched by TYPE, since the typed error survives upstream's rewraps (module comment).
173
+ * Fails CLOSED: a non-`Error`, a text-only copy of the message, and an error built by a
174
+ * second loaded copy of `@optimystic/db-core` (which `instanceof` cannot recognise) all
175
+ * answer false, and the reader throws as it did before this existed.
176
+ */
177
+ export function isCohortUnreachableRead(error: unknown): boolean {
178
+ return causeChain(error).some(link =>
179
+ link instanceof BlockUnavailableError && link.reason === 'cohort-unreachable');
180
+ }
181
+
182
+ /**
183
+ * Run `attempt` — a full drain of one control read — up to
184
+ * {@link CONTROL_READ_ATTEMPTS} times, retrying only failures
185
+ * {@link isRetriableControlReadFailure} calls transient, and only while
186
+ * {@link CONTROL_READ_RETRY_BUDGET_MS} has not elapsed.
187
+ *
188
+ * Reads are idempotent, so `attempt` needs no atomicity contract — but it must be a
189
+ * COLLECTING drain, not a live iterator: a half-consumed iterator cannot be retried
190
+ * without re-yielding rows the caller already saw, which is why
191
+ * `ControlDatabase.readRows` materializes before this loop ever sees a failure.
192
+ *
193
+ * On exhaustion the LAST error is rethrown unchanged. Log lines carry the
194
+ * `Control read [<label>] …` prefix so a read's retries can be attributed among the
195
+ * several reads in flight concurrently in a real party.
196
+ *
197
+ * No reader passes an `onAbandon` observer, deliberately, though the shared options type
198
+ * offers one ({@link ControlRetryOptions.onAbandon}) and the loop would honour it. That hook
199
+ * exists because an abandoned WRITE can have no caller at all — the background self-record
200
+ * republish is `void`-ed with a `debug`-only catch, so giving up on it is invisible. Every
201
+ * control read is awaited by the caller that issued it, so an abandoned read surfaces as that
202
+ * caller's rejection; there is nothing here to lose silently. A fire-and-forget read would be
203
+ * the reason to start passing one.
204
+ */
205
+ export function retryControlRead<T>(
206
+ attempt: () => Promise<T>,
207
+ options: ControlReadRetryOptions = {}
208
+ ): Promise<T> {
209
+ return retryControlOperation(attempt, {
210
+ attempts: CONTROL_READ_ATTEMPTS,
211
+ delaysMs: CONTROL_READ_RETRY_DELAYS_MS,
212
+ budgetMs: CONTROL_READ_RETRY_BUDGET_MS,
213
+ isRetriable: isRetriableControlReadFailure,
214
+ logPrefix: 'Control read',
215
+ }, options);
216
+ }