@serfab/cadre-core 1.10.0 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cadre-node.d.ts +0 -27
- package/dist/cadre-node.js +2 -94
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-schema.js +1066 -1066
- package/dist/diagnostics/connection-path.d.ts +1 -18
- package/dist/diagnostics/connection-path.js +1 -20
- package/dist/diagnostics/connection-path.js.map +1 -1
- package/dist/identity-key.d.ts +0 -30
- package/dist/identity-key.js +7 -40
- package/dist/identity-key.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.js +6 -4
- package/dist/index.js.map +1 -1
- package/dist/relay-stun.d.ts +24 -0
- package/dist/relay-stun.js +51 -0
- package/dist/relay-stun.js.map +1 -0
- package/dist/seed-bootstrap.js +4 -4
- package/dist/strand-membership-writer.js +32 -32
- package/package.json +2 -2
- package/src/cadre-node.ts +8654 -8753
- package/src/control-authorization.ts +117 -117
- package/src/control-cohort.ts +128 -128
- package/src/control-formation-recorder.ts +353 -353
- package/src/control-read-retry.ts +216 -216
- package/src/control-schema.ts +1078 -1078
- package/src/control-write-retry.ts +567 -567
- package/src/diagnostics/connection-path.ts +1 -29
- package/src/formation-approval.ts +610 -610
- package/src/hibernation-manager.ts +638 -638
- package/src/identity-key.ts +7 -62
- package/src/index.ts +644 -647
- package/src/invitation-bootstrap.ts +40 -40
- package/src/joined-strand-store.ts +498 -498
- package/src/link-budget.ts +684 -684
- package/src/membership-connection-gater.ts +570 -570
- package/src/node-local-snapshot.ts +298 -298
- package/src/peer-dial.ts +315 -315
- package/src/peer-join-backfill.ts +710 -710
- package/src/pending-join-runner.ts +735 -735
- package/src/relay-reservation.ts +1121 -1121
- package/src/relay-server.ts +98 -98
- package/src/relay-stun.ts +58 -0
- package/src/seed-bootstrap.ts +1448 -1448
- package/src/strand-addr-protocol.ts +547 -547
- package/src/strand-database.ts +287 -287
- package/src/strand-first-sync-gate.ts +391 -391
- package/src/strand-formation-deadlines.ts +227 -227
- package/src/strand-formation-manager.ts +725 -725
- package/src/strand-fret-addrs.ts +145 -145
- package/src/strand-instance-manager.ts +1789 -1789
- package/src/strand-membership-writer.ts +1974 -1974
- package/src/strand-network-state-file.ts +40 -40
- package/src/strand-network-state.ts +243 -243
- package/src/strand-revocation-enforcer.ts +654 -654
- package/src/strand-solicitation.ts +532 -532
- package/src/strand-wake-protocol.ts +401 -401
- package/src/types.ts +2222 -2222
- package/dist/diagnostics/webrtc-turn-tracker.d.ts +0 -50
- package/dist/diagnostics/webrtc-turn-tracker.js +0 -196
- package/dist/diagnostics/webrtc-turn-tracker.js.map +0 -1
- package/src/diagnostics/webrtc-turn-tracker.ts +0 -222
|
@@ -1,216 +1,216 @@
|
|
|
1
|
-
import { BlockUnavailableError } from '@optimystic/db-core';
|
|
2
|
-
import { causeChain, chainMessages, retryControlOperation } from './control-retry.js';
|
|
3
|
-
import type { ControlRetryOptions } from './control-retry.js';
|
|
4
|
-
import { isUncommittedTransactorAggregate } from './control-write-retry.js';
|
|
5
|
-
|
|
6
|
-
/**
|
|
7
|
-
* Bounded retry for TRANSIENT control-read failures — the classifier and policy behind
|
|
8
|
-
* `ControlDatabase.readRows`, the single funnel every control read passes through.
|
|
9
|
-
*
|
|
10
|
-
* Why this exists: control WRITES have absorbed transient cluster failures since
|
|
11
|
-
* `control-write-retry.ts` landed, but a read had no second chance — one stream reset
|
|
12
|
-
* during a scan ended the read outright (measured: a rigged single `eval` failure killed
|
|
13
|
-
* `queryRevokedStamps` after exactly one attempt while the same injection against `exec`
|
|
14
|
-
* was absorbed). This module gives reads the same bounded re-presentation with a SHORTER
|
|
15
|
-
* deadline and a NARROWER failure set, because the tightest caller deadline over a control
|
|
16
|
-
* read is the inbound admission gate's 2 s fail-open timeout
|
|
17
|
-
* (`ADMISSION_DECISION_TIMEOUT_MS`, `link-budget.ts`) — a read budget that
|
|
18
|
-
* does not fit inside it with headroom spends its retries after the gate has already
|
|
19
|
-
* admitted.
|
|
20
|
-
*
|
|
21
|
-
* This is deliberately NOT "the write classifier minus its commit veto" — the retriable
|
|
22
|
-
* set differs in both directions (see {@link isRetriableControlReadFailure}). The loop
|
|
23
|
-
* lives in `control-retry.ts`, shared with the write policy; this module owns only the
|
|
24
|
-
* read policy. And like the write side, the retry classifies by TEXT. That was forced when
|
|
25
|
-
* it was written — optimystic's scan path used to rethrow `new Error('Query failed: ' +
|
|
26
|
-
* message)` with no `cause` — and is no longer: `OptimysticVirtualTable` now rethrows
|
|
27
|
-
* through `rewrapAsQueryError` (`quereus-plugin-optimystic/src/optimystic-module.ts`),
|
|
28
|
-
* which keeps the typed `BlockUnavailableError` on `cause`, and Quereus wraps that as
|
|
29
|
-
* `Error during query on table '<T>': …`, again preserving `cause`
|
|
30
|
-
* (`quereus/src/runtime/emit/scan.ts`). The text matchers still work because every wrap
|
|
31
|
-
* embeds the inner message. Every matcher fails CLOSED — an upstream rewording stops the
|
|
32
|
-
* retry engaging, it never makes it unsafe. {@link isCohortUnreachableRead}, which is not a
|
|
33
|
-
* retry classifier, matches by type.
|
|
34
|
-
*/
|
|
35
|
-
|
|
36
|
-
/**
|
|
37
|
-
* Attempts allowed per control read: the first, plus two retries. Attempts are cheap here —
|
|
38
|
-
* the observed transient failure (the transactor's read-phase aggregate off a stream still
|
|
39
|
-
* forming) surfaces in ~25 ms — and {@link CONTROL_READ_RETRY_BUDGET_MS} is what terminates
|
|
40
|
-
* the loop when an attempt is slow instead.
|
|
41
|
-
*/
|
|
42
|
-
export const CONTROL_READ_ATTEMPTS = 3;
|
|
43
|
-
|
|
44
|
-
/**
|
|
45
|
-
* Backoff before read retry N, jittered ±50% and capped at the largest base by the shared
|
|
46
|
-
* loop — so worst case 150 ms + 400 ms = 550 ms of total sleep (the second delay's
|
|
47
|
-
* upside jitter is clipped by the cap). Much shorter than the write list because the
|
|
48
|
-
* whole loop must fit under {@link CONTROL_READ_RETRY_BUDGET_MS}; the spec pins that
|
|
49
|
-
* relationship rather than the number.
|
|
50
|
-
*/
|
|
51
|
-
export const CONTROL_READ_RETRY_DELAYS_MS: readonly number[] = [100, 400];
|
|
52
|
-
|
|
53
|
-
/**
|
|
54
|
-
* Elapsed-time ceiling on the whole read-retry loop, checked after a failed attempt BEFORE
|
|
55
|
-
* sleeping — so one slow attempt (e.g. a `cohort-unreachable` read that burned the
|
|
56
|
-
* transactor's own deadline) terminates the loop rather than compounding.
|
|
57
|
-
*
|
|
58
|
-
* 1500 ms, sized against `ADMISSION_DECISION_TIMEOUT_MS` (2000 ms): the inbound admission
|
|
59
|
-
* gate reads the control DB and is FAIL-OPEN on both throw and timeout, so a read that
|
|
60
|
-
* outlives the gate's deadline spends its retries after the gate has already admitted an
|
|
61
|
-
* unplaced peer. Under the deadline, a read that succeeds on its second attempt lets the
|
|
62
|
-
* gate make the real decision instead. `control-read-retry.spec.ts` asserts the
|
|
63
|
-
* relationship so a future edit to either constant reddens instead of silently
|
|
64
|
-
* reintroducing the fail-open admit. NOTE: `queryCadrePeers` / `queryPeerRecord` each
|
|
65
|
-
* issue TWO network reads (the revoked-stamp filter plus the row scan) and the budget is
|
|
66
|
-
* per read, so a membership read can spend up to two budgets back to back — still inside
|
|
67
|
-
* 2 s only because each stays well under it. Do not raise these numbers without
|
|
68
|
-
* re-checking against `ADMISSION_DECISION_TIMEOUT_MS`.
|
|
69
|
-
*
|
|
70
|
-
* It cuts the loop off by design, so it does not grow with the link or with the number that
|
|
71
|
-
* dominates a slow attempt: an attempt whose cohort consult finds a silent peer costs
|
|
72
|
-
* `clusterPolicy.cohortQueryTimeoutMs`, two link round trips at the declared link
|
|
73
|
-
* (`cohortReadDeadlineMs` in `link-budget.ts`, 7 000 ms at the default declaration; the plugin's
|
|
74
|
-
* `COHORT_READ_DEADLINE_MS` is the same number). That is more than four times this budget, so
|
|
75
|
-
* such an attempt is never retried and the caller gets its error unchanged. That is deliberate:
|
|
76
|
-
* the caller this budget is sized for, the admission gate, has
|
|
77
|
-
* already taken its fail-open answer by then, and a retry would only spend time after it. What
|
|
78
|
-
* the budget still buys is the fast failures it was built for: the ~25 ms transactor read-phase
|
|
79
|
-
* aggregate off a stream still forming, and a `cohort-unreachable` read during bring-up, which
|
|
80
|
-
* fails fast because there is no connection to ask. Both retry twice well inside 1500 ms. See
|
|
81
|
-
* `docs/cadre-consistency.md` → "Deadlines Over Optimystic's Reads and Commits".
|
|
82
|
-
*/
|
|
83
|
-
// eslint-disable-next-line no-restricted-syntax -- cuts off by design: a slow attempt is not retried, because the admission gate this budget is sized for has already failed open by then; see docs/cadre-consistency.md → "Deadlines Over Optimystic's Reads and Commits"
|
|
84
|
-
export const CONTROL_READ_RETRY_BUDGET_MS = 1_500;
|
|
85
|
-
|
|
86
|
-
/**
|
|
87
|
-
* The block-unavailability reasons a repeat read can actually improve on, matched against
|
|
88
|
-
* upstream's exact template `Block <id> is unavailable (<reason>): …`
|
|
89
|
-
* (`BlockUnavailableError`, `db-core/src/network/struct.ts`):
|
|
90
|
-
*
|
|
91
|
-
* - **`peers-unreachable`** — part of the cohort answered and part could not be asked, and
|
|
92
|
-
* OTHER coordinators are reachable, so asking one of them can still settle it. Upstream's
|
|
93
|
-
* own definition invites the second attempt.
|
|
94
|
-
* - **`cohort-unreachable`** — the marginal member of the set. Upstream says the answer
|
|
95
|
-
* will not improve until this node's connectivity does — but during bring-up connectivity
|
|
96
|
-
* DOES improve within a second (the boot-time failure shape recorded in
|
|
97
|
-
* `tickets/blocked/control-read-over-fresh-edge-stream-resets`), and the cost of being
|
|
98
|
-
* wrong is bounded by {@link CONTROL_READ_RETRY_BUDGET_MS}, which is the whole reason it
|
|
99
|
-
* is safe to include.
|
|
100
|
-
*
|
|
101
|
-
* Deliberately NOT matched:
|
|
102
|
-
*
|
|
103
|
-
* - **`claimed-elsewhere`** — a cohort peer positively claims the block exists and nobody
|
|
104
|
-
* could corroborate or acquire it. MEASURED not to clear: reissuing the same call every
|
|
105
|
-
* second for 60 s returned the identical error every time (2026-08-20, recorded in the
|
|
106
|
-
* fix ticket). Retrying spends the whole budget and fails anyway. The measured cause (a
|
|
107
|
-
* block only one machine held) was fixed upstream by the solo-commit proof in
|
|
108
|
-
* `@optimystic/db-p2p` 0.28.0 (`tickets/complete/block-held-by-only-one-machine-is-unreadable`),
|
|
109
|
-
* so a `claimed-elsewhere` that surfaces now is a new defect — this retry must not hide it.
|
|
110
|
-
* - **`unmaterializable`** — records are held locally but cannot be reassembled. A local
|
|
111
|
-
* data problem; a second read reads the same records.
|
|
112
|
-
* - **`Block <id> may be stale`** (`BlockPossiblyStaleError`) — about currency, not
|
|
113
|
-
* existence: a cohort claim no reachable coordinator could confirm or refute will not
|
|
114
|
-
* resolve inside a few hundred milliseconds of backoff.
|
|
115
|
-
* - Anything unmatched. The default is no retry.
|
|
116
|
-
*/
|
|
117
|
-
const RETRIABLE_BLOCK_UNAVAILABLE =
|
|
118
|
-
/Block \S+ is unavailable \((?:peers-unreachable|cohort-unreachable)\)/;
|
|
119
|
-
|
|
120
|
-
/** A block-unavailable read failure whose reason a repeat read can improve on. */
|
|
121
|
-
function isRetriableBlockUnavailable(message: string): boolean {
|
|
122
|
-
return RETRIABLE_BLOCK_UNAVAILABLE.test(message);
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
/**
|
|
126
|
-
* Pacing overrides for {@link retryControlRead} — the read twin of
|
|
127
|
-
* `ControlWriteRetryOptions`. Every field defaults to the shipped read policy
|
|
128
|
-
* ({@link CONTROL_READ_ATTEMPTS}, {@link CONTROL_READ_RETRY_DELAYS_MS},
|
|
129
|
-
* {@link isRetriableControlReadFailure}); field shapes and semantics live on the shared
|
|
130
|
-
* {@link ControlRetryOptions}.
|
|
131
|
-
*/
|
|
132
|
-
export type ControlReadRetryOptions = ControlRetryOptions;
|
|
133
|
-
|
|
134
|
-
/**
|
|
135
|
-
* Did this control read fail in a way a repeat read can improve on?
|
|
136
|
-
*
|
|
137
|
-
* Two classes, both meaning "the cluster could not be ASKED properly just now":
|
|
138
|
-
*
|
|
139
|
-
* - the transactor's read-phase aggregate (`Some peers did not complete: …[block:…]`),
|
|
140
|
-
* reused verbatim from the write classifier — a `get` batch formats the single-block
|
|
141
|
-
* `[block:` token, and a read can never produce the commit-phase `[blocks:` token, so
|
|
142
|
-
* the write side's indeterminate-commit veto has nothing to veto here (a read commits
|
|
143
|
-
* nothing and re-running it is always safe);
|
|
144
|
-
* - the block-unavailability reasons in {@link RETRIABLE_BLOCK_UNAVAILABLE} (and only
|
|
145
|
-
* those — see its comment for the measured exclusions).
|
|
146
|
-
*
|
|
147
|
-
* Classifies by MESSAGE, walking the `cause` chain with the shared `chainMessages` (why
|
|
148
|
-
* text, in the module comment). Anything that is not an `Error` is never retried.
|
|
149
|
-
*/
|
|
150
|
-
export function isRetriableControlReadFailure(error: unknown): boolean {
|
|
151
|
-
if (!(error instanceof Error)) {
|
|
152
|
-
return false;
|
|
153
|
-
}
|
|
154
|
-
return chainMessages(error).some(message =>
|
|
155
|
-
isUncommittedTransactorAggregate(message) || isRetriableBlockUnavailable(message));
|
|
156
|
-
}
|
|
157
|
-
|
|
158
|
-
/**
|
|
159
|
-
* Did this control read fail because no cohort member other than the answering node could
|
|
160
|
-
* be asked about a block nobody here holds — a `BlockUnavailableError` with reason
|
|
161
|
-
* `cohort-unreachable` anywhere on the `cause` chain?
|
|
162
|
-
*
|
|
163
|
-
* Not a retry classifier. It answers "is there a better answer to wait for?", and is used
|
|
164
|
-
* by exactly one reader, `ControlDatabase.queryRevokedStamps`, to read an isolated node's
|
|
165
|
-
* never-received `Revocation` block as "no revocations known" (the reasoning is in the NOTE
|
|
166
|
-
* there). Upstream names this the one reason a caller may treat permissively
|
|
167
|
-
* (`db-core/src/transactor/network-transactor.ts`). The other reasons do NOT match:
|
|
168
|
-
* `peers-unreachable` means part of the cohort answered, and `claimed-elsewhere` /
|
|
169
|
-
* `unmaterializable` mean the block is known to exist, so an empty answer could hide a
|
|
170
|
-
* real revocation.
|
|
171
|
-
*
|
|
172
|
-
* Matched by TYPE, since the typed error survives upstream's rewraps (module comment).
|
|
173
|
-
* Fails CLOSED: a non-`Error`, a text-only copy of the message, and an error built by a
|
|
174
|
-
* second loaded copy of `@optimystic/db-core` (which `instanceof` cannot recognise) all
|
|
175
|
-
* answer false, and the reader throws as it did before this existed.
|
|
176
|
-
*/
|
|
177
|
-
export function isCohortUnreachableRead(error: unknown): boolean {
|
|
178
|
-
return causeChain(error).some(link =>
|
|
179
|
-
link instanceof BlockUnavailableError && link.reason === 'cohort-unreachable');
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
/**
|
|
183
|
-
* Run `attempt` — a full drain of one control read — up to
|
|
184
|
-
* {@link CONTROL_READ_ATTEMPTS} times, retrying only failures
|
|
185
|
-
* {@link isRetriableControlReadFailure} calls transient, and only while
|
|
186
|
-
* {@link CONTROL_READ_RETRY_BUDGET_MS} has not elapsed.
|
|
187
|
-
*
|
|
188
|
-
* Reads are idempotent, so `attempt` needs no atomicity contract — but it must be a
|
|
189
|
-
* COLLECTING drain, not a live iterator: a half-consumed iterator cannot be retried
|
|
190
|
-
* without re-yielding rows the caller already saw, which is why
|
|
191
|
-
* `ControlDatabase.readRows` materializes before this loop ever sees a failure.
|
|
192
|
-
*
|
|
193
|
-
* On exhaustion the LAST error is rethrown unchanged. Log lines carry the
|
|
194
|
-
* `Control read [<label>] …` prefix so a read's retries can be attributed among the
|
|
195
|
-
* several reads in flight concurrently in a real party.
|
|
196
|
-
*
|
|
197
|
-
* No reader passes an `onAbandon` observer, deliberately, though the shared options type
|
|
198
|
-
* offers one ({@link ControlRetryOptions.onAbandon}) and the loop would honour it. That hook
|
|
199
|
-
* exists because an abandoned WRITE can have no caller at all — the background self-record
|
|
200
|
-
* republish is `void`-ed with a `debug`-only catch, so giving up on it is invisible. Every
|
|
201
|
-
* control read is awaited by the caller that issued it, so an abandoned read surfaces as that
|
|
202
|
-
* caller's rejection; there is nothing here to lose silently. A fire-and-forget read would be
|
|
203
|
-
* the reason to start passing one.
|
|
204
|
-
*/
|
|
205
|
-
export function retryControlRead<T>(
|
|
206
|
-
attempt: () => Promise<T>,
|
|
207
|
-
options: ControlReadRetryOptions = {}
|
|
208
|
-
): Promise<T> {
|
|
209
|
-
return retryControlOperation(attempt, {
|
|
210
|
-
attempts: CONTROL_READ_ATTEMPTS,
|
|
211
|
-
delaysMs: CONTROL_READ_RETRY_DELAYS_MS,
|
|
212
|
-
budgetMs: CONTROL_READ_RETRY_BUDGET_MS,
|
|
213
|
-
isRetriable: isRetriableControlReadFailure,
|
|
214
|
-
logPrefix: 'Control read',
|
|
215
|
-
}, options);
|
|
216
|
-
}
|
|
1
|
+
import { BlockUnavailableError } from '@optimystic/db-core';
|
|
2
|
+
import { causeChain, chainMessages, retryControlOperation } from './control-retry.js';
|
|
3
|
+
import type { ControlRetryOptions } from './control-retry.js';
|
|
4
|
+
import { isUncommittedTransactorAggregate } from './control-write-retry.js';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Bounded retry for TRANSIENT control-read failures — the classifier and policy behind
|
|
8
|
+
* `ControlDatabase.readRows`, the single funnel every control read passes through.
|
|
9
|
+
*
|
|
10
|
+
* Why this exists: control WRITES have absorbed transient cluster failures since
|
|
11
|
+
* `control-write-retry.ts` landed, but a read had no second chance — one stream reset
|
|
12
|
+
* during a scan ended the read outright (measured: a rigged single `eval` failure killed
|
|
13
|
+
* `queryRevokedStamps` after exactly one attempt while the same injection against `exec`
|
|
14
|
+
* was absorbed). This module gives reads the same bounded re-presentation with a SHORTER
|
|
15
|
+
* deadline and a NARROWER failure set, because the tightest caller deadline over a control
|
|
16
|
+
* read is the inbound admission gate's 2 s fail-open timeout
|
|
17
|
+
* (`ADMISSION_DECISION_TIMEOUT_MS`, `link-budget.ts`) — a read budget that
|
|
18
|
+
* does not fit inside it with headroom spends its retries after the gate has already
|
|
19
|
+
* admitted.
|
|
20
|
+
*
|
|
21
|
+
* This is deliberately NOT "the write classifier minus its commit veto" — the retriable
|
|
22
|
+
* set differs in both directions (see {@link isRetriableControlReadFailure}). The loop
|
|
23
|
+
* lives in `control-retry.ts`, shared with the write policy; this module owns only the
|
|
24
|
+
* read policy. And like the write side, the retry classifies by TEXT. That was forced when
|
|
25
|
+
* it was written — optimystic's scan path used to rethrow `new Error('Query failed: ' +
|
|
26
|
+
* message)` with no `cause` — and is no longer: `OptimysticVirtualTable` now rethrows
|
|
27
|
+
* through `rewrapAsQueryError` (`quereus-plugin-optimystic/src/optimystic-module.ts`),
|
|
28
|
+
* which keeps the typed `BlockUnavailableError` on `cause`, and Quereus wraps that as
|
|
29
|
+
* `Error during query on table '<T>': …`, again preserving `cause`
|
|
30
|
+
* (`quereus/src/runtime/emit/scan.ts`). The text matchers still work because every wrap
|
|
31
|
+
* embeds the inner message. Every matcher fails CLOSED — an upstream rewording stops the
|
|
32
|
+
* retry engaging, it never makes it unsafe. {@link isCohortUnreachableRead}, which is not a
|
|
33
|
+
* retry classifier, matches by type.
|
|
34
|
+
*/
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Attempts allowed per control read: the first, plus two retries. Attempts are cheap here —
|
|
38
|
+
* the observed transient failure (the transactor's read-phase aggregate off a stream still
|
|
39
|
+
* forming) surfaces in ~25 ms — and {@link CONTROL_READ_RETRY_BUDGET_MS} is what terminates
|
|
40
|
+
* the loop when an attempt is slow instead.
|
|
41
|
+
*/
|
|
42
|
+
export const CONTROL_READ_ATTEMPTS = 3;
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Backoff before read retry N, jittered ±50% and capped at the largest base by the shared
|
|
46
|
+
* loop — so worst case 150 ms + 400 ms = 550 ms of total sleep (the second delay's
|
|
47
|
+
* upside jitter is clipped by the cap). Much shorter than the write list because the
|
|
48
|
+
* whole loop must fit under {@link CONTROL_READ_RETRY_BUDGET_MS}; the spec pins that
|
|
49
|
+
* relationship rather than the number.
|
|
50
|
+
*/
|
|
51
|
+
export const CONTROL_READ_RETRY_DELAYS_MS: readonly number[] = [100, 400];
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Elapsed-time ceiling on the whole read-retry loop, checked after a failed attempt BEFORE
|
|
55
|
+
* sleeping — so one slow attempt (e.g. a `cohort-unreachable` read that burned the
|
|
56
|
+
* transactor's own deadline) terminates the loop rather than compounding.
|
|
57
|
+
*
|
|
58
|
+
* 1500 ms, sized against `ADMISSION_DECISION_TIMEOUT_MS` (2000 ms): the inbound admission
|
|
59
|
+
* gate reads the control DB and is FAIL-OPEN on both throw and timeout, so a read that
|
|
60
|
+
* outlives the gate's deadline spends its retries after the gate has already admitted an
|
|
61
|
+
* unplaced peer. Under the deadline, a read that succeeds on its second attempt lets the
|
|
62
|
+
* gate make the real decision instead. `control-read-retry.spec.ts` asserts the
|
|
63
|
+
* relationship so a future edit to either constant reddens instead of silently
|
|
64
|
+
* reintroducing the fail-open admit. NOTE: `queryCadrePeers` / `queryPeerRecord` each
|
|
65
|
+
* issue TWO network reads (the revoked-stamp filter plus the row scan) and the budget is
|
|
66
|
+
* per read, so a membership read can spend up to two budgets back to back — still inside
|
|
67
|
+
* 2 s only because each stays well under it. Do not raise these numbers without
|
|
68
|
+
* re-checking against `ADMISSION_DECISION_TIMEOUT_MS`.
|
|
69
|
+
*
|
|
70
|
+
* It cuts the loop off by design, so it does not grow with the link or with the number that
|
|
71
|
+
* dominates a slow attempt: an attempt whose cohort consult finds a silent peer costs
|
|
72
|
+
* `clusterPolicy.cohortQueryTimeoutMs`, two link round trips at the declared link
|
|
73
|
+
* (`cohortReadDeadlineMs` in `link-budget.ts`, 7 000 ms at the default declaration; the plugin's
|
|
74
|
+
* `COHORT_READ_DEADLINE_MS` is the same number). That is more than four times this budget, so
|
|
75
|
+
* such an attempt is never retried and the caller gets its error unchanged. That is deliberate:
|
|
76
|
+
* the caller this budget is sized for, the admission gate, has
|
|
77
|
+
* already taken its fail-open answer by then, and a retry would only spend time after it. What
|
|
78
|
+
* the budget still buys is the fast failures it was built for: the ~25 ms transactor read-phase
|
|
79
|
+
* aggregate off a stream still forming, and a `cohort-unreachable` read during bring-up, which
|
|
80
|
+
* fails fast because there is no connection to ask. Both retry twice well inside 1500 ms. See
|
|
81
|
+
* `docs/cadre-consistency.md` → "Deadlines Over Optimystic's Reads and Commits".
|
|
82
|
+
*/
|
|
83
|
+
// eslint-disable-next-line no-restricted-syntax -- cuts off by design: a slow attempt is not retried, because the admission gate this budget is sized for has already failed open by then; see docs/cadre-consistency.md → "Deadlines Over Optimystic's Reads and Commits"
|
|
84
|
+
export const CONTROL_READ_RETRY_BUDGET_MS = 1_500;
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* The block-unavailability reasons a repeat read can actually improve on, matched against
|
|
88
|
+
* upstream's exact template `Block <id> is unavailable (<reason>): …`
|
|
89
|
+
* (`BlockUnavailableError`, `db-core/src/network/struct.ts`):
|
|
90
|
+
*
|
|
91
|
+
* - **`peers-unreachable`** — part of the cohort answered and part could not be asked, and
|
|
92
|
+
* OTHER coordinators are reachable, so asking one of them can still settle it. Upstream's
|
|
93
|
+
* own definition invites the second attempt.
|
|
94
|
+
* - **`cohort-unreachable`** — the marginal member of the set. Upstream says the answer
|
|
95
|
+
* will not improve until this node's connectivity does — but during bring-up connectivity
|
|
96
|
+
* DOES improve within a second (the boot-time failure shape recorded in
|
|
97
|
+
* `tickets/blocked/control-read-over-fresh-edge-stream-resets`), and the cost of being
|
|
98
|
+
* wrong is bounded by {@link CONTROL_READ_RETRY_BUDGET_MS}, which is the whole reason it
|
|
99
|
+
* is safe to include.
|
|
100
|
+
*
|
|
101
|
+
* Deliberately NOT matched:
|
|
102
|
+
*
|
|
103
|
+
* - **`claimed-elsewhere`** — a cohort peer positively claims the block exists and nobody
|
|
104
|
+
* could corroborate or acquire it. MEASURED not to clear: reissuing the same call every
|
|
105
|
+
* second for 60 s returned the identical error every time (2026-08-20, recorded in the
|
|
106
|
+
* fix ticket). Retrying spends the whole budget and fails anyway. The measured cause (a
|
|
107
|
+
* block only one machine held) was fixed upstream by the solo-commit proof in
|
|
108
|
+
* `@optimystic/db-p2p` 0.28.0 (`tickets/complete/block-held-by-only-one-machine-is-unreadable`),
|
|
109
|
+
* so a `claimed-elsewhere` that surfaces now is a new defect — this retry must not hide it.
|
|
110
|
+
* - **`unmaterializable`** — records are held locally but cannot be reassembled. A local
|
|
111
|
+
* data problem; a second read reads the same records.
|
|
112
|
+
* - **`Block <id> may be stale`** (`BlockPossiblyStaleError`) — about currency, not
|
|
113
|
+
* existence: a cohort claim no reachable coordinator could confirm or refute will not
|
|
114
|
+
* resolve inside a few hundred milliseconds of backoff.
|
|
115
|
+
* - Anything unmatched. The default is no retry.
|
|
116
|
+
*/
|
|
117
|
+
const RETRIABLE_BLOCK_UNAVAILABLE =
|
|
118
|
+
/Block \S+ is unavailable \((?:peers-unreachable|cohort-unreachable)\)/;
|
|
119
|
+
|
|
120
|
+
/** A block-unavailable read failure whose reason a repeat read can improve on. */
|
|
121
|
+
function isRetriableBlockUnavailable(message: string): boolean {
|
|
122
|
+
return RETRIABLE_BLOCK_UNAVAILABLE.test(message);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Pacing overrides for {@link retryControlRead} — the read twin of
|
|
127
|
+
* `ControlWriteRetryOptions`. Every field defaults to the shipped read policy
|
|
128
|
+
* ({@link CONTROL_READ_ATTEMPTS}, {@link CONTROL_READ_RETRY_DELAYS_MS},
|
|
129
|
+
* {@link isRetriableControlReadFailure}); field shapes and semantics live on the shared
|
|
130
|
+
* {@link ControlRetryOptions}.
|
|
131
|
+
*/
|
|
132
|
+
export type ControlReadRetryOptions = ControlRetryOptions;
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Did this control read fail in a way a repeat read can improve on?
|
|
136
|
+
*
|
|
137
|
+
* Two classes, both meaning "the cluster could not be ASKED properly just now":
|
|
138
|
+
*
|
|
139
|
+
* - the transactor's read-phase aggregate (`Some peers did not complete: …[block:…]`),
|
|
140
|
+
* reused verbatim from the write classifier — a `get` batch formats the single-block
|
|
141
|
+
* `[block:` token, and a read can never produce the commit-phase `[blocks:` token, so
|
|
142
|
+
* the write side's indeterminate-commit veto has nothing to veto here (a read commits
|
|
143
|
+
* nothing and re-running it is always safe);
|
|
144
|
+
* - the block-unavailability reasons in {@link RETRIABLE_BLOCK_UNAVAILABLE} (and only
|
|
145
|
+
* those — see its comment for the measured exclusions).
|
|
146
|
+
*
|
|
147
|
+
* Classifies by MESSAGE, walking the `cause` chain with the shared `chainMessages` (why
|
|
148
|
+
* text, in the module comment). Anything that is not an `Error` is never retried.
|
|
149
|
+
*/
|
|
150
|
+
export function isRetriableControlReadFailure(error: unknown): boolean {
|
|
151
|
+
if (!(error instanceof Error)) {
|
|
152
|
+
return false;
|
|
153
|
+
}
|
|
154
|
+
return chainMessages(error).some(message =>
|
|
155
|
+
isUncommittedTransactorAggregate(message) || isRetriableBlockUnavailable(message));
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Did this control read fail because no cohort member other than the answering node could
|
|
160
|
+
* be asked about a block nobody here holds — a `BlockUnavailableError` with reason
|
|
161
|
+
* `cohort-unreachable` anywhere on the `cause` chain?
|
|
162
|
+
*
|
|
163
|
+
* Not a retry classifier. It answers "is there a better answer to wait for?", and is used
|
|
164
|
+
* by exactly one reader, `ControlDatabase.queryRevokedStamps`, to read an isolated node's
|
|
165
|
+
* never-received `Revocation` block as "no revocations known" (the reasoning is in the NOTE
|
|
166
|
+
* there). Upstream names this the one reason a caller may treat permissively
|
|
167
|
+
* (`db-core/src/transactor/network-transactor.ts`). The other reasons do NOT match:
|
|
168
|
+
* `peers-unreachable` means part of the cohort answered, and `claimed-elsewhere` /
|
|
169
|
+
* `unmaterializable` mean the block is known to exist, so an empty answer could hide a
|
|
170
|
+
* real revocation.
|
|
171
|
+
*
|
|
172
|
+
* Matched by TYPE, since the typed error survives upstream's rewraps (module comment).
|
|
173
|
+
* Fails CLOSED: a non-`Error`, a text-only copy of the message, and an error built by a
|
|
174
|
+
* second loaded copy of `@optimystic/db-core` (which `instanceof` cannot recognise) all
|
|
175
|
+
* answer false, and the reader throws as it did before this existed.
|
|
176
|
+
*/
|
|
177
|
+
export function isCohortUnreachableRead(error: unknown): boolean {
|
|
178
|
+
return causeChain(error).some(link =>
|
|
179
|
+
link instanceof BlockUnavailableError && link.reason === 'cohort-unreachable');
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Run `attempt` — a full drain of one control read — up to
|
|
184
|
+
* {@link CONTROL_READ_ATTEMPTS} times, retrying only failures
|
|
185
|
+
* {@link isRetriableControlReadFailure} calls transient, and only while
|
|
186
|
+
* {@link CONTROL_READ_RETRY_BUDGET_MS} has not elapsed.
|
|
187
|
+
*
|
|
188
|
+
* Reads are idempotent, so `attempt` needs no atomicity contract — but it must be a
|
|
189
|
+
* COLLECTING drain, not a live iterator: a half-consumed iterator cannot be retried
|
|
190
|
+
* without re-yielding rows the caller already saw, which is why
|
|
191
|
+
* `ControlDatabase.readRows` materializes before this loop ever sees a failure.
|
|
192
|
+
*
|
|
193
|
+
* On exhaustion the LAST error is rethrown unchanged. Log lines carry the
|
|
194
|
+
* `Control read [<label>] …` prefix so a read's retries can be attributed among the
|
|
195
|
+
* several reads in flight concurrently in a real party.
|
|
196
|
+
*
|
|
197
|
+
* No reader passes an `onAbandon` observer, deliberately, though the shared options type
|
|
198
|
+
* offers one ({@link ControlRetryOptions.onAbandon}) and the loop would honour it. That hook
|
|
199
|
+
* exists because an abandoned WRITE can have no caller at all — the background self-record
|
|
200
|
+
* republish is `void`-ed with a `debug`-only catch, so giving up on it is invisible. Every
|
|
201
|
+
* control read is awaited by the caller that issued it, so an abandoned read surfaces as that
|
|
202
|
+
* caller's rejection; there is nothing here to lose silently. A fire-and-forget read would be
|
|
203
|
+
* the reason to start passing one.
|
|
204
|
+
*/
|
|
205
|
+
export function retryControlRead<T>(
|
|
206
|
+
attempt: () => Promise<T>,
|
|
207
|
+
options: ControlReadRetryOptions = {}
|
|
208
|
+
): Promise<T> {
|
|
209
|
+
return retryControlOperation(attempt, {
|
|
210
|
+
attempts: CONTROL_READ_ATTEMPTS,
|
|
211
|
+
delaysMs: CONTROL_READ_RETRY_DELAYS_MS,
|
|
212
|
+
budgetMs: CONTROL_READ_RETRY_BUDGET_MS,
|
|
213
|
+
isRetriable: isRetriableControlReadFailure,
|
|
214
|
+
logPrefix: 'Control read',
|
|
215
|
+
}, options);
|
|
216
|
+
}
|