@serfab/cadre-core 1.18.0 → 1.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cadre-node.d.ts +65 -2
- package/dist/cadre-node.js +120 -4
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-read-retry.d.ts +19 -16
- package/dist/control-read-retry.js +22 -54
- package/dist/control-read-retry.js.map +1 -1
- package/dist/control-retry.d.ts +50 -6
- package/dist/control-retry.js +53 -6
- package/dist/control-retry.js.map +1 -1
- package/dist/control-write-retry.d.ts +24 -16
- package/dist/control-write-retry.js +58 -30
- package/dist/control-write-retry.js.map +1 -1
- package/dist/edge-node/config.d.ts +8 -2
- package/dist/edge-node/config.js +5 -1
- package/dist/edge-node/config.js.map +1 -1
- package/dist/edge-node/edge-join.js +2 -0
- package/dist/edge-node/edge-join.js.map +1 -1
- package/dist/edge-node/edge-node.d.ts +10 -0
- package/dist/edge-node/edge-node.js +52 -8
- package/dist/edge-node/edge-node.js.map +1 -1
- package/dist/edge-node/options.d.ts +11 -2
- package/dist/edge-node/options.js +11 -4
- package/dist/edge-node/options.js.map +1 -1
- package/dist/edge-node/platform.d.ts +15 -3
- package/dist/edge-node/strands.js +3 -2
- package/dist/edge-node/strands.js.map +1 -1
- package/dist/index.d.ts +5 -3
- package/dist/index.js +7 -2
- package/dist/index.js.map +1 -1
- package/dist/peer-join-backfill.d.ts +8 -11
- package/dist/peer-join-backfill.js +22 -35
- package/dist/peer-join-backfill.js.map +1 -1
- package/dist/self-address-update.d.ts +13 -0
- package/dist/self-address-update.js +19 -0
- package/dist/self-address-update.js.map +1 -0
- package/dist/strand-database.d.ts +56 -0
- package/dist/strand-database.js +87 -0
- package/dist/strand-database.js.map +1 -1
- package/dist/strand-instance-manager.d.ts +67 -2
- package/dist/strand-instance-manager.js +105 -4
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-membership-reconciler.d.ts +13 -1
- package/dist/strand-membership-reconciler.js +23 -14
- package/dist/strand-membership-reconciler.js.map +1 -1
- package/dist/strand-membership-writer.d.ts +16 -2
- package/dist/strand-membership-writer.js +54 -8
- package/dist/strand-membership-writer.js.map +1 -1
- package/dist/strand-offline.d.ts +131 -0
- package/dist/strand-offline.js +142 -0
- package/dist/strand-offline.js.map +1 -0
- package/dist/types.d.ts +35 -0
- package/dist/types.js.map +1 -1
- package/package.json +8 -8
- package/src/cadre-node.ts +11456 -11326
- package/src/control-read-retry.ts +22 -58
- package/src/control-retry.ts +54 -6
- package/src/control-write-retry.ts +58 -30
- package/src/edge-node/config.ts +11 -2
- package/src/edge-node/edge-join.ts +1 -0
- package/src/edge-node/edge-node.ts +55 -6
- package/src/edge-node/options.ts +20 -4
- package/src/edge-node/platform.ts +15 -3
- package/src/edge-node/strands.ts +3 -2
- package/src/index.ts +917 -899
- package/src/peer-join-backfill.ts +33 -43
- package/src/self-address-update.ts +19 -0
- package/src/strand-database.ts +466 -352
- package/src/strand-instance-manager.ts +2210 -2092
- package/src/strand-membership-reconciler.ts +29 -14
- package/src/strand-membership-writer.ts +61 -14
- package/src/strand-offline.ts +247 -0
- package/src/types.ts +2960 -2926
|
@@ -17,16 +17,14 @@ import type { ControlRetryOptions } from './control-retry.js';
|
|
|
17
17
|
* This is deliberately NOT "the write classifier minus its commit veto" — the retriable
|
|
18
18
|
* set differs in both directions (see {@link isRetriableControlReadFailure}). The loop
|
|
19
19
|
* lives in `control-retry.ts`, shared with the write policy; this module owns only the
|
|
20
|
-
* read policy.
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
* which keeps the typed `BlockUnavailableError` on `cause`, and Quereus wraps that as
|
|
20
|
+
* read policy. A block-unavailability failure is matched by TYPE: `OptimysticVirtualTable`
|
|
21
|
+
* rethrows a scan failure through `rewrapAsQueryError`
|
|
22
|
+
* (`quereus-plugin-optimystic/src/optimystic-module.ts`), which keeps the typed
|
|
23
|
+
* `BlockUnavailableError` on `cause`, and Quereus wraps that as
|
|
25
24
|
* `Error during query on table '<T>': …`, again preserving `cause`
|
|
26
|
-
* (`quereus/src/runtime/emit/scan.ts`). The
|
|
27
|
-
*
|
|
28
|
-
* retry engaging, it never makes it unsafe.
|
|
29
|
-
* retry classifier, matches by type.
|
|
25
|
+
* (`quereus/src/runtime/emit/scan.ts`). The transactor's aggregate is matched by TEXT, since
|
|
26
|
+
* upstream raises it as a plain `Error`. Every matcher fails CLOSED — an upstream rewording or
|
|
27
|
+
* a lost `cause` stops the retry engaging, it never makes it unsafe.
|
|
30
28
|
*/
|
|
31
29
|
/**
|
|
32
30
|
* Attempts allowed per control read: the first, plus two retries. Attempts are cheap here —
|
|
@@ -92,11 +90,13 @@ export type ControlReadRetryOptions = ControlRetryOptions;
|
|
|
92
90
|
* `[block:` token, and a read can never produce the commit-phase `[blocks:` token, so
|
|
93
91
|
* the write side's indeterminate-commit veto has nothing to veto here (a read commits
|
|
94
92
|
* nothing and re-running it is always safe);
|
|
95
|
-
* -
|
|
96
|
-
*
|
|
93
|
+
* - a block read the cohort could not answer ({@link reportsUnreachableCohort}, the
|
|
94
|
+
* predicate the write classifier shares — see it for the reasons and the measured
|
|
95
|
+
* exclusions).
|
|
97
96
|
*
|
|
98
|
-
*
|
|
99
|
-
*
|
|
97
|
+
* The aggregate is matched by message at any depth of the `cause` chain (`chainMessages`),
|
|
98
|
+
* the unavailability by type on the same chain. Anything that is not an `Error` is never
|
|
99
|
+
* retried, and anything unmatched is not retried either.
|
|
100
100
|
*/
|
|
101
101
|
export declare function isRetriableControlReadFailure(error: unknown): boolean;
|
|
102
102
|
/**
|
|
@@ -109,9 +109,12 @@ export declare function isRetriableControlReadFailure(error: unknown): boolean;
|
|
|
109
109
|
* never-received `Revocation` block as "no revocations known" (the reasoning is in the NOTE
|
|
110
110
|
* there). Upstream names this the one reason a caller may treat permissively
|
|
111
111
|
* (`db-core/src/transactor/network-transactor.ts`). The other reasons do NOT match:
|
|
112
|
-
* `peers-unreachable` means part of the cohort answered, and
|
|
113
|
-
* `unmaterializable` mean the block is known to exist, so an empty
|
|
114
|
-
* real revocation.
|
|
112
|
+
* `peers-unreachable` means part of the cohort answered (or no coordinator did), and
|
|
113
|
+
* `claimed-elsewhere` / `unmaterializable` mean the block is known to exist, so an empty
|
|
114
|
+
* answer could hide a real revocation. NOTE: narrower than upstream's same-named
|
|
115
|
+
* `isCohortUnreachableRead` (`@optimystic/db-core`), which also matches `peers-unreachable`
|
|
116
|
+
* and is what the retry classifiers use ({@link reportsUnreachableCohort}); do not swap one
|
|
117
|
+
* for the other.
|
|
115
118
|
*
|
|
116
119
|
* Matched by TYPE, since the typed error survives upstream's rewraps (module comment).
|
|
117
120
|
* Fails CLOSED: a non-`Error`, a text-only copy of the message, and an error built by a
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { BlockUnavailableError } from '@optimystic/db-core';
|
|
2
|
-
import { causeChain, chainMessages, retryControlOperation } from './control-retry.js';
|
|
2
|
+
import { causeChain, chainMessages, reportsUnreachableCohort, retryControlOperation } from './control-retry.js';
|
|
3
3
|
import { isUncommittedTransactorAggregate } from './control-write-retry.js';
|
|
4
4
|
/**
|
|
5
5
|
* Bounded retry for TRANSIENT control-read failures — the classifier and policy behind
|
|
@@ -19,16 +19,14 @@ import { isUncommittedTransactorAggregate } from './control-write-retry.js';
|
|
|
19
19
|
* This is deliberately NOT "the write classifier minus its commit veto" — the retriable
|
|
20
20
|
* set differs in both directions (see {@link isRetriableControlReadFailure}). The loop
|
|
21
21
|
* lives in `control-retry.ts`, shared with the write policy; this module owns only the
|
|
22
|
-
* read policy.
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
* which keeps the typed `BlockUnavailableError` on `cause`, and Quereus wraps that as
|
|
22
|
+
* read policy. A block-unavailability failure is matched by TYPE: `OptimysticVirtualTable`
|
|
23
|
+
* rethrows a scan failure through `rewrapAsQueryError`
|
|
24
|
+
* (`quereus-plugin-optimystic/src/optimystic-module.ts`), which keeps the typed
|
|
25
|
+
* `BlockUnavailableError` on `cause`, and Quereus wraps that as
|
|
27
26
|
* `Error during query on table '<T>': …`, again preserving `cause`
|
|
28
|
-
* (`quereus/src/runtime/emit/scan.ts`). The
|
|
29
|
-
*
|
|
30
|
-
* retry engaging, it never makes it unsafe.
|
|
31
|
-
* retry classifier, matches by type.
|
|
27
|
+
* (`quereus/src/runtime/emit/scan.ts`). The transactor's aggregate is matched by TEXT, since
|
|
28
|
+
* upstream raises it as a plain `Error`. Every matcher fails CLOSED — an upstream rewording or
|
|
29
|
+
* a lost `cause` stops the retry engaging, it never makes it unsafe.
|
|
32
30
|
*/
|
|
33
31
|
/**
|
|
34
32
|
* Attempts allowed per control read: the first, plus two retries. Attempts are cheap here —
|
|
@@ -77,42 +75,6 @@ export const CONTROL_READ_RETRY_DELAYS_MS = [100, 400];
|
|
|
77
75
|
*/
|
|
78
76
|
// eslint-disable-next-line no-restricted-syntax -- cuts off by design: a slow attempt is not retried, because the admission gate this budget is sized for has already failed open by then; see docs/cadre-consistency.md → "Deadlines Over Optimystic's Reads and Commits"
|
|
79
77
|
export const CONTROL_READ_RETRY_BUDGET_MS = 1500;
|
|
80
|
-
/**
|
|
81
|
-
* The block-unavailability reasons a repeat read can actually improve on, matched against
|
|
82
|
-
* upstream's exact template `Block <id> is unavailable (<reason>): …`
|
|
83
|
-
* (`BlockUnavailableError`, `db-core/src/network/struct.ts`):
|
|
84
|
-
*
|
|
85
|
-
* - **`peers-unreachable`** — part of the cohort answered and part could not be asked, and
|
|
86
|
-
* OTHER coordinators are reachable, so asking one of them can still settle it. Upstream's
|
|
87
|
-
* own definition invites the second attempt.
|
|
88
|
-
* - **`cohort-unreachable`** — the marginal member of the set. Upstream says the answer
|
|
89
|
-
* will not improve until this node's connectivity does — but during bring-up connectivity
|
|
90
|
-
* DOES improve within a second (the boot-time failure shape recorded in
|
|
91
|
-
* `tickets/blocked/control-read-over-fresh-edge-stream-resets`), and the cost of being
|
|
92
|
-
* wrong is bounded by {@link CONTROL_READ_RETRY_BUDGET_MS}, which is the whole reason it
|
|
93
|
-
* is safe to include.
|
|
94
|
-
*
|
|
95
|
-
* Deliberately NOT matched:
|
|
96
|
-
*
|
|
97
|
-
* - **`claimed-elsewhere`** — a cohort peer positively claims the block exists and nobody
|
|
98
|
-
* could corroborate or acquire it. MEASURED not to clear: reissuing the same call every
|
|
99
|
-
* second for 60 s returned the identical error every time (2026-08-20, recorded in the
|
|
100
|
-
* fix ticket). Retrying spends the whole budget and fails anyway. The measured cause (a
|
|
101
|
-
* block only one machine held) was fixed upstream by the solo-commit proof in
|
|
102
|
-
* `@optimystic/db-p2p` 0.28.0 (`tickets/complete/block-held-by-only-one-machine-is-unreadable`),
|
|
103
|
-
* so a `claimed-elsewhere` that surfaces now is a new defect — this retry must not hide it.
|
|
104
|
-
* - **`unmaterializable`** — records are held locally but cannot be reassembled. A local
|
|
105
|
-
* data problem; a second read reads the same records.
|
|
106
|
-
* - **`Block <id> may be stale`** (`BlockPossiblyStaleError`) — about currency, not
|
|
107
|
-
* existence: a cohort claim no reachable coordinator could confirm or refute will not
|
|
108
|
-
* resolve inside a few hundred milliseconds of backoff.
|
|
109
|
-
* - Anything unmatched. The default is no retry.
|
|
110
|
-
*/
|
|
111
|
-
const RETRIABLE_BLOCK_UNAVAILABLE = /Block \S+ is unavailable \((?:peers-unreachable|cohort-unreachable)\)/;
|
|
112
|
-
/** A block-unavailable read failure whose reason a repeat read can improve on. */
|
|
113
|
-
function isRetriableBlockUnavailable(message) {
|
|
114
|
-
return RETRIABLE_BLOCK_UNAVAILABLE.test(message);
|
|
115
|
-
}
|
|
116
78
|
/**
|
|
117
79
|
* Did this control read fail in a way a repeat read can improve on?
|
|
118
80
|
*
|
|
@@ -123,17 +85,20 @@ function isRetriableBlockUnavailable(message) {
|
|
|
123
85
|
* `[block:` token, and a read can never produce the commit-phase `[blocks:` token, so
|
|
124
86
|
* the write side's indeterminate-commit veto has nothing to veto here (a read commits
|
|
125
87
|
* nothing and re-running it is always safe);
|
|
126
|
-
* -
|
|
127
|
-
*
|
|
88
|
+
* - a block read the cohort could not answer ({@link reportsUnreachableCohort}, the
|
|
89
|
+
* predicate the write classifier shares — see it for the reasons and the measured
|
|
90
|
+
* exclusions).
|
|
128
91
|
*
|
|
129
|
-
*
|
|
130
|
-
*
|
|
92
|
+
* The aggregate is matched by message at any depth of the `cause` chain (`chainMessages`),
|
|
93
|
+
* the unavailability by type on the same chain. Anything that is not an `Error` is never
|
|
94
|
+
* retried, and anything unmatched is not retried either.
|
|
131
95
|
*/
|
|
132
96
|
export function isRetriableControlReadFailure(error) {
|
|
133
97
|
if (!(error instanceof Error)) {
|
|
134
98
|
return false;
|
|
135
99
|
}
|
|
136
|
-
return
|
|
100
|
+
return reportsUnreachableCohort(causeChain(error))
|
|
101
|
+
|| chainMessages(error).some(isUncommittedTransactorAggregate);
|
|
137
102
|
}
|
|
138
103
|
/**
|
|
139
104
|
* Did this control read fail because no cohort member other than the answering node could
|
|
@@ -145,9 +110,12 @@ export function isRetriableControlReadFailure(error) {
|
|
|
145
110
|
* never-received `Revocation` block as "no revocations known" (the reasoning is in the NOTE
|
|
146
111
|
* there). Upstream names this the one reason a caller may treat permissively
|
|
147
112
|
* (`db-core/src/transactor/network-transactor.ts`). The other reasons do NOT match:
|
|
148
|
-
* `peers-unreachable` means part of the cohort answered, and
|
|
149
|
-
* `unmaterializable` mean the block is known to exist, so an empty
|
|
150
|
-
* real revocation.
|
|
113
|
+
* `peers-unreachable` means part of the cohort answered (or no coordinator did), and
|
|
114
|
+
* `claimed-elsewhere` / `unmaterializable` mean the block is known to exist, so an empty
|
|
115
|
+
* answer could hide a real revocation. NOTE: narrower than upstream's same-named
|
|
116
|
+
* `isCohortUnreachableRead` (`@optimystic/db-core`), which also matches `peers-unreachable`
|
|
117
|
+
* and is what the retry classifiers use ({@link reportsUnreachableCohort}); do not swap one
|
|
118
|
+
* for the other.
|
|
151
119
|
*
|
|
152
120
|
* Matched by TYPE, since the typed error survives upstream's rewraps (module comment).
|
|
153
121
|
* Fails CLOSED: a non-`Error`, a text-only copy of the message, and an error built by a
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"control-read-retry.js","sourceRoot":"","sources":["../src/control-read-retry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"control-read-retry.js","sourceRoot":"","sources":["../src/control-read-retry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,wBAAwB,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAEhH,OAAO,EAAE,gCAAgC,EAAE,MAAM,0BAA0B,CAAC;AAE5E;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH;;;;;GAKG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC;AAEvC;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,4BAA4B,GAAsB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;AAE1E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,2QAA2Q;AAC3Q,MAAM,CAAC,MAAM,4BAA4B,GAAG,IAAK,CAAC;AAWlD;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,6BAA6B,CAAC,KAAc;IAC3D,IAAI,CAAC,CAAC,KAAK,YAAY,KAAK,CAAC,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC;IACd,CAAC;IACD,OAAO,wBAAwB,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;WAC9C,aAAa,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;AACjE,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,UAAU,uBAAuB,CAAC,KAAc;IACrD,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACpC,IAAI,YAAY,qBAAqB,IAAI,IAAI,CAAC,MAAM,KAAK,oBAAoB,CAAC,CAAC;AACjF,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,UAAU,gBAAgB,CAC/B,OAAyB,EACzB,UAAmC,EAAE;IAErC,OAAO,qBAAqB,CAAC,OAAO,EAAE;QACrC,QAAQ,EAAE,qBAAqB;QAC/B,QAAQ,EAAE,4BAA4B;QACtC,QAAQ,EAAE,4BAA4B;QACtC,WAAW,EAAE,6BAA6B;QAC1C,SAAS,EAAE,cAAc;KACzB,EAAE,OAAO,CAAC,CAAC;AACb,CAAC"}
|
package/dist/control-retry.d.ts
CHANGED
|
@@ -130,10 +130,10 @@ export interface ControlRetryOptions {
|
|
|
130
130
|
*/
|
|
131
131
|
export declare function retryControlOperation<T>(attempt: () => Promise<T>, policy: ControlRetryPolicy, options?: ControlRetryOptions): Promise<T>;
|
|
132
132
|
/**
|
|
133
|
-
* Every message in the failure's `cause` chain — the
|
|
134
|
-
*
|
|
135
|
-
* wrap on the way out of optimystic and
|
|
136
|
-
* the deepest failure is visible at every level.
|
|
133
|
+
* Every message in the failure's `cause` chain — the substrate both retry classifiers match
|
|
134
|
+
* their text matchers against (the transactor's aggregate and the coordinator's shortfall are
|
|
135
|
+
* plain `Error`s, recognisable only by text); every wrap on the way out of optimystic and
|
|
136
|
+
* Quereus embeds the inner message, so the text of the deepest failure is visible at every level.
|
|
137
137
|
*
|
|
138
138
|
* `unwrapError` declares its `message` as `string`, but it follows `.cause` without checking
|
|
139
139
|
* what that holds — a chain link that is not an `Error` (a stream rejected with a bare string
|
|
@@ -149,8 +149,52 @@ export declare function retryControlOperation<T>(attempt: () => Promise<T>, poli
|
|
|
149
149
|
export declare function chainMessages(error: Error): string[];
|
|
150
150
|
/**
|
|
151
151
|
* `error` and every `cause` below it, outermost first — the substrate for the classifiers that
|
|
152
|
-
* match by TYPE rather than text (the read side's
|
|
153
|
-
* possibly-stored veto). Stops at a non-`Error` link (it can carry no further `cause` worth
|
|
152
|
+
* match by TYPE rather than text ({@link reportsUnreachableCohort}, the read side's isolated-node
|
|
153
|
+
* check, the write side's possibly-stored veto). Stops at a non-`Error` link (it can carry no further `cause` worth
|
|
154
154
|
* trusting) and at a repeat, so a cyclic chain terminates.
|
|
155
155
|
*/
|
|
156
156
|
export declare function causeChain(error: unknown): Error[];
|
|
157
|
+
/**
|
|
158
|
+
* Does any link of the `cause` chain report a block read that failed because the cohort could not
|
|
159
|
+
* be asked — a `BlockUnavailableError` whose reason is `peers-unreachable` or `cohort-unreachable`?
|
|
160
|
+
* The one block-unavailability test both retry classifiers share, so the read and write policies
|
|
161
|
+
* cannot disagree on which reasons a repeat can improve on:
|
|
162
|
+
*
|
|
163
|
+
* - **`peers-unreachable`** — part of the cohort answered and part could not be asked, or no
|
|
164
|
+
* coordinator answered the read at all (`NetworkTransactor.get` flags such a block rather than
|
|
165
|
+
* failing the whole read). Another attempt can reach one that answers; upstream's own definition
|
|
166
|
+
* invites the second attempt.
|
|
167
|
+
* - **`cohort-unreachable`** — the marginal member of the set. Upstream says the answer will not
|
|
168
|
+
* improve until this node's connectivity does — but during bring-up connectivity DOES improve
|
|
169
|
+
* within a second (the boot-time failure shape recorded in
|
|
170
|
+
* `tickets/blocked/control-read-over-fresh-edge-stream-resets`), and each policy's elapsed budget
|
|
171
|
+
* bounds the cost of being wrong, which is the whole reason it is safe to include.
|
|
172
|
+
*
|
|
173
|
+
* Deliberately NOT matched:
|
|
174
|
+
*
|
|
175
|
+
* - **`claimed-elsewhere`** — a cohort peer positively claims the block exists and nobody could
|
|
176
|
+
* corroborate or acquire it. MEASURED not to clear: reissuing the same call every second for 60 s
|
|
177
|
+
* returned the identical error every time (2026-08-20, recorded in the fix ticket). Retrying spends
|
|
178
|
+
* the whole budget and fails anyway. The measured cause (a block only one machine held) was fixed
|
|
179
|
+
* upstream by the solo-commit proof in `@optimystic/db-p2p` 0.28.0
|
|
180
|
+
* (`tickets/complete/block-held-by-only-one-machine-is-unreadable`), so a `claimed-elsewhere` that
|
|
181
|
+
* surfaces now is a new defect — a retry must not hide it.
|
|
182
|
+
* - **`unmaterializable`** — records are held locally but cannot be reassembled. A local data
|
|
183
|
+
* problem; a second read reads the same records.
|
|
184
|
+
* - **`BlockPossiblyStaleError`** (`Block <id> may be stale`) — about currency, not existence: a
|
|
185
|
+
* cohort claim no reachable coordinator could confirm or refute will not resolve inside a few
|
|
186
|
+
* hundred milliseconds of backoff.
|
|
187
|
+
*
|
|
188
|
+
* The reason set is upstream's (`isCohortUnreachableRead`, `db-core/src/transaction/commit-failure-class.ts`),
|
|
189
|
+
* the rule its commit-failure sort and the plugin's offline reads use for "the cohort could not be
|
|
190
|
+
* reached". NOTE: a reason upstream adds to that set is therefore retried here with no change in
|
|
191
|
+
* this repo — the right default for a set defined as unreachability; if it ever grows, re-check it
|
|
192
|
+
* against the exclusions above.
|
|
193
|
+
*
|
|
194
|
+
* Matched by TYPE. Upstream builds this error only on the node doing the read, from the
|
|
195
|
+
* `unavailable` flag a get result carries (`TransactorSource.tryGet`, `NetworkTransactor.getStatus`),
|
|
196
|
+
* never from a peer's error, and every rewrap on the way to either funnel keeps `cause`. NOTE: an
|
|
197
|
+
* error built by a second loaded copy of `@optimystic/db-core`, or one rebuilt as a plain `Error`
|
|
198
|
+
* from its text, fails `instanceof` and is not retried — the safe side.
|
|
199
|
+
*/
|
|
200
|
+
export declare function reportsUnreachableCohort(links: readonly Error[]): boolean;
|
package/dist/control-retry.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import debug from 'debug';
|
|
2
|
+
import { isCohortUnreachableRead } from '@optimystic/db-core';
|
|
2
3
|
import { unwrapError } from '@quereus/quereus';
|
|
3
4
|
const log = debug('sereus:cadre:control-db');
|
|
4
5
|
/**
|
|
@@ -100,10 +101,10 @@ function notifyAbandoned(listener, prefix, tag, abandonment) {
|
|
|
100
101
|
}
|
|
101
102
|
}
|
|
102
103
|
/**
|
|
103
|
-
* Every message in the failure's `cause` chain — the
|
|
104
|
-
*
|
|
105
|
-
* wrap on the way out of optimystic and
|
|
106
|
-
* the deepest failure is visible at every level.
|
|
104
|
+
* Every message in the failure's `cause` chain — the substrate both retry classifiers match
|
|
105
|
+
* their text matchers against (the transactor's aggregate and the coordinator's shortfall are
|
|
106
|
+
* plain `Error`s, recognisable only by text); every wrap on the way out of optimystic and
|
|
107
|
+
* Quereus embeds the inner message, so the text of the deepest failure is visible at every level.
|
|
107
108
|
*
|
|
108
109
|
* `unwrapError` declares its `message` as `string`, but it follows `.cause` without checking
|
|
109
110
|
* what that holds — a chain link that is not an `Error` (a stream rejected with a bare string
|
|
@@ -123,8 +124,8 @@ export function chainMessages(error) {
|
|
|
123
124
|
}
|
|
124
125
|
/**
|
|
125
126
|
* `error` and every `cause` below it, outermost first — the substrate for the classifiers that
|
|
126
|
-
* match by TYPE rather than text (the read side's
|
|
127
|
-
* possibly-stored veto). Stops at a non-`Error` link (it can carry no further `cause` worth
|
|
127
|
+
* match by TYPE rather than text ({@link reportsUnreachableCohort}, the read side's isolated-node
|
|
128
|
+
* check, the write side's possibly-stored veto). Stops at a non-`Error` link (it can carry no further `cause` worth
|
|
128
129
|
* trusting) and at a repeat, so a cyclic chain terminates.
|
|
129
130
|
*/
|
|
130
131
|
export function causeChain(error) {
|
|
@@ -136,6 +137,52 @@ export function causeChain(error) {
|
|
|
136
137
|
}
|
|
137
138
|
return links;
|
|
138
139
|
}
|
|
140
|
+
/**
|
|
141
|
+
* Does any link of the `cause` chain report a block read that failed because the cohort could not
|
|
142
|
+
* be asked — a `BlockUnavailableError` whose reason is `peers-unreachable` or `cohort-unreachable`?
|
|
143
|
+
* The one block-unavailability test both retry classifiers share, so the read and write policies
|
|
144
|
+
* cannot disagree on which reasons a repeat can improve on:
|
|
145
|
+
*
|
|
146
|
+
* - **`peers-unreachable`** — part of the cohort answered and part could not be asked, or no
|
|
147
|
+
* coordinator answered the read at all (`NetworkTransactor.get` flags such a block rather than
|
|
148
|
+
* failing the whole read). Another attempt can reach one that answers; upstream's own definition
|
|
149
|
+
* invites the second attempt.
|
|
150
|
+
* - **`cohort-unreachable`** — the marginal member of the set. Upstream says the answer will not
|
|
151
|
+
* improve until this node's connectivity does — but during bring-up connectivity DOES improve
|
|
152
|
+
* within a second (the boot-time failure shape recorded in
|
|
153
|
+
* `tickets/blocked/control-read-over-fresh-edge-stream-resets`), and each policy's elapsed budget
|
|
154
|
+
* bounds the cost of being wrong, which is the whole reason it is safe to include.
|
|
155
|
+
*
|
|
156
|
+
* Deliberately NOT matched:
|
|
157
|
+
*
|
|
158
|
+
* - **`claimed-elsewhere`** — a cohort peer positively claims the block exists and nobody could
|
|
159
|
+
* corroborate or acquire it. MEASURED not to clear: reissuing the same call every second for 60 s
|
|
160
|
+
* returned the identical error every time (2026-08-20, recorded in the fix ticket). Retrying spends
|
|
161
|
+
* the whole budget and fails anyway. The measured cause (a block only one machine held) was fixed
|
|
162
|
+
* upstream by the solo-commit proof in `@optimystic/db-p2p` 0.28.0
|
|
163
|
+
* (`tickets/complete/block-held-by-only-one-machine-is-unreadable`), so a `claimed-elsewhere` that
|
|
164
|
+
* surfaces now is a new defect — a retry must not hide it.
|
|
165
|
+
* - **`unmaterializable`** — records are held locally but cannot be reassembled. A local data
|
|
166
|
+
* problem; a second read reads the same records.
|
|
167
|
+
* - **`BlockPossiblyStaleError`** (`Block <id> may be stale`) — about currency, not existence: a
|
|
168
|
+
* cohort claim no reachable coordinator could confirm or refute will not resolve inside a few
|
|
169
|
+
* hundred milliseconds of backoff.
|
|
170
|
+
*
|
|
171
|
+
* The reason set is upstream's (`isCohortUnreachableRead`, `db-core/src/transaction/commit-failure-class.ts`),
|
|
172
|
+
* the rule its commit-failure sort and the plugin's offline reads use for "the cohort could not be
|
|
173
|
+
* reached". NOTE: a reason upstream adds to that set is therefore retried here with no change in
|
|
174
|
+
* this repo — the right default for a set defined as unreachability; if it ever grows, re-check it
|
|
175
|
+
* against the exclusions above.
|
|
176
|
+
*
|
|
177
|
+
* Matched by TYPE. Upstream builds this error only on the node doing the read, from the
|
|
178
|
+
* `unavailable` flag a get result carries (`TransactorSource.tryGet`, `NetworkTransactor.getStatus`),
|
|
179
|
+
* never from a peer's error, and every rewrap on the way to either funnel keeps `cause`. NOTE: an
|
|
180
|
+
* error built by a second loaded copy of `@optimystic/db-core`, or one rebuilt as a plain `Error`
|
|
181
|
+
* from its text, fails `instanceof` and is not retried — the safe side.
|
|
182
|
+
*/
|
|
183
|
+
export function reportsUnreachableCohort(links) {
|
|
184
|
+
return links.some(link => isCohortUnreachableRead(link));
|
|
185
|
+
}
|
|
139
186
|
/**
|
|
140
187
|
* Backoff for the retry that follows attempt `attemptNumber`: the matching base delay
|
|
141
188
|
* (last entry repeats), jittered ±50%, then capped at the list's largest base so no single
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"control-retry.js","sourceRoot":"","sources":["../src/control-retry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAE/C,MAAM,GAAG,GAAG,KAAK,CAAC,yBAAyB,CAAC,CAAC;AAwH7C;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAC1C,OAAyB,EACzB,MAA0B,EAC1B,UAA+B,EAAE;IAEjC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC;IAClE,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC;IACnD,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,CAAC;IAC9D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,YAAY,CAAC;IAC5C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;IACpC,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,IAAI,eAAe,CAAC;IACnD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IACpC,mFAAmF;IACnF,qCAAqC;IACrC,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACvD,MAAM,KAAK,GAAG,GAAG,EAAE,CAAC;IACpB,IAAI,SAAkB,CAAC;IACvB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,uFAAuF;IACvF,iDAAiD;IACjD,IAAI,WAAW,GAA8B,UAAU,CAAC;IACxD,KAAK,IAAI,aAAa,GAAG,CAAC,EAAE,aAAa,IAAI,QAAQ,EAAE,aAAa,EAAE,EAAE,CAAC;QACxE,YAAY,GAAG,aAAa,CAAC;QAC7B,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,MAAM,OAAO,EAAE,CAAC;YAC/B,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;gBACvB,GAAG,CAAC,iCAAiC,EAAE,MAAM,EAAE,GAAG,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;YAC9E,CAAC;YACD,OAAO,MAAM,CAAC;QACf,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;gBACzB,gFAAgF;gBAChF,mEAAmE;gBACnE,yEAAyE;gBACzE,GAAG,CAAC,oEAAoE,EACvE,MAAM,EAAE,GAAG,EAAE,aAAa,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;gBAC9C,eAAe,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE;oBACvC,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAChE,YAAY,EAAE,eAAe,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,KAAK;oBACjE,MAAM,EAAE,UAAU,EAAE,KAAK;iBACzB,CAAC,CAAC;gBACH,MAAM,KAAK,CAAC;YACb,CAAC;YACD,SAAS,GAAG,KAAK,CAAC;YAClB,IAAI,aAAa,KAAK,QAAQ,EAAE,CAAC;gBAChC,MAAM;YACP,CAAC;YACD,MAAM,OAAO,GAAG,GAAG,EAAE,GAAG,KAAK,CAAC;YAC9B,IAAI,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;gBAChC,WAAW,GAAG,QAAQ,CAAC;gBACvB,MAAM;YACP,CAAC;YACD,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;YACnD,GAAG,CAAC,gEAAgE,EACnE,MAAM,EAAE,GAAG,EAAE,aAAa,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;YACrD,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC;QACpB,CAAC;IACF,CAAC;IACD,GAAG,CAAC,wCAAwC,EAAE,MAAM,EAAE,GAAG,EAAE,YAAY,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;IAC9F,eAAe,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE;QACvC,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAChE,YAAY,EAAE,eAAe,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,KAAK;QACjE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,SAAS;KACrC,CAAC,CAAC;IACH,MAAM,SAAS,CAAC;AACjB,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,eAAe,CACvB,QAAiD,EACjD,MAAc,EACd,GAAW,EACX,WAAoC;IAEpC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACf,OAAO;IACR,CAAC;IACD,IAAI,CAAC;QACJ,QAAQ,CAAC,WAAW,CAAC,CAAC;IACvB,CAAC;IAAC,OAAO,aAAa,EAAE,CAAC;QACxB,GAAG,CAAC,8EAA8E,EACjF,MAAM,EAAE,GAAG,EAAE,aAAa,CAAC,CAAC;IAC9B,CAAC;AACF,CAAC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,UAAU,aAAa,CAAC,KAAY;IACzC,OAAO,WAAW,CAAC,KAAK,CAAC;SACvB,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC;SAC7B,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC;AAClD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,UAAU,CAAC,KAAc;IACxC,MAAM,KAAK,GAAY,EAAE,CAAC;IAC1B,IAAI,OAAO,GAAY,KAAK,CAAC;IAC7B,OAAO,OAAO,YAAY,KAAK,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QAC7D,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACpB,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC;IACzB,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,SAAS,aAAa,CAAC,MAAyB,EAAE,aAAqB;IACtE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,CAAC;IACV,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,aAAa,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAE,CAAC;IACrE,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9C,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,YAAY,CAAC,EAAU;IAC/B,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AACxD,CAAC"}
|
|
1
|
+
{"version":3,"file":"control-retry.js","sourceRoot":"","sources":["../src/control-retry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAE/C,MAAM,GAAG,GAAG,KAAK,CAAC,yBAAyB,CAAC,CAAC;AAwH7C;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAC1C,OAAyB,EACzB,MAA0B,EAC1B,UAA+B,EAAE;IAEjC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC;IAClE,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC;IACnD,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,CAAC;IAC9D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,YAAY,CAAC;IAC5C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;IACpC,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,IAAI,eAAe,CAAC;IACnD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IACpC,mFAAmF;IACnF,qCAAqC;IACrC,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACvD,MAAM,KAAK,GAAG,GAAG,EAAE,CAAC;IACpB,IAAI,SAAkB,CAAC;IACvB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,uFAAuF;IACvF,iDAAiD;IACjD,IAAI,WAAW,GAA8B,UAAU,CAAC;IACxD,KAAK,IAAI,aAAa,GAAG,CAAC,EAAE,aAAa,IAAI,QAAQ,EAAE,aAAa,EAAE,EAAE,CAAC;QACxE,YAAY,GAAG,aAAa,CAAC;QAC7B,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,MAAM,OAAO,EAAE,CAAC;YAC/B,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;gBACvB,GAAG,CAAC,iCAAiC,EAAE,MAAM,EAAE,GAAG,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;YAC9E,CAAC;YACD,OAAO,MAAM,CAAC;QACf,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;gBACzB,gFAAgF;gBAChF,mEAAmE;gBACnE,yEAAyE;gBACzE,GAAG,CAAC,oEAAoE,EACvE,MAAM,EAAE,GAAG,EAAE,aAAa,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;gBAC9C,eAAe,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE;oBACvC,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAChE,YAAY,EAAE,eAAe,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,KAAK;oBACjE,MAAM,EAAE,UAAU,EAAE,KAAK;iBACzB,CAAC,CAAC;gBACH,MAAM,KAAK,CAAC;YACb,CAAC;YACD,SAAS,GAAG,KAAK,CAAC;YAClB,IAAI,aAAa,KAAK,QAAQ,EAAE,CAAC;gBAChC,MAAM;YACP,CAAC;YACD,MAAM,OAAO,GAAG,GAAG,EAAE,GAAG,KAAK,CAAC;YAC9B,IAAI,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;gBAChC,WAAW,GAAG,QAAQ,CAAC;gBACvB,MAAM;YACP,CAAC;YACD,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;YACnD,GAAG,CAAC,gEAAgE,EACnE,MAAM,EAAE,GAAG,EAAE,aAAa,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;YACrD,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC;QACpB,CAAC;IACF,CAAC;IACD,GAAG,CAAC,wCAAwC,EAAE,MAAM,EAAE,GAAG,EAAE,YAAY,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;IAC9F,eAAe,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE;QACvC,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAChE,YAAY,EAAE,eAAe,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,KAAK;QACjE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,SAAS;KACrC,CAAC,CAAC;IACH,MAAM,SAAS,CAAC;AACjB,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,eAAe,CACvB,QAAiD,EACjD,MAAc,EACd,GAAW,EACX,WAAoC;IAEpC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACf,OAAO;IACR,CAAC;IACD,IAAI,CAAC;QACJ,QAAQ,CAAC,WAAW,CAAC,CAAC;IACvB,CAAC;IAAC,OAAO,aAAa,EAAE,CAAC;QACxB,GAAG,CAAC,8EAA8E,EACjF,MAAM,EAAE,GAAG,EAAE,aAAa,CAAC,CAAC;IAC9B,CAAC;AACF,CAAC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,UAAU,aAAa,CAAC,KAAY;IACzC,OAAO,WAAW,CAAC,KAAK,CAAC;SACvB,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC;SAC7B,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC;AAClD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,UAAU,CAAC,KAAc;IACxC,MAAM,KAAK,GAAY,EAAE,CAAC;IAC1B,IAAI,OAAO,GAAY,KAAK,CAAC;IAC7B,OAAO,OAAO,YAAY,KAAK,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QAC7D,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACpB,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC;IACzB,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,MAAM,UAAU,wBAAwB,CAAC,KAAuB;IAC/D,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAC;AAC1D,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,SAAS,aAAa,CAAC,MAAyB,EAAE,aAAqB;IACtE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,CAAC;IACV,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,aAAa,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAE,CAAC;IACrE,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9C,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,YAAY,CAAC,EAAU;IAC/B,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AACxD,CAAC"}
|
|
@@ -112,7 +112,9 @@ export declare const CONTROL_WRITE_RETRY_BUDGET_MS = 10000;
|
|
|
112
112
|
* re-present:
|
|
113
113
|
*
|
|
114
114
|
* - `get` (a block read, which a write body also performs) and `pend` (phase 1) fail before
|
|
115
|
-
* anything commits, so the write is known not to have landed
|
|
115
|
+
* anything commits, so the write is known not to have landed. `get` raises it only when a
|
|
116
|
+
* coordinator answered and left a block out; a block nobody answered for fails typed
|
|
117
|
+
* instead, which {@link matchesRetriableFailure} claims ({@link reportsUnreachableCohort});
|
|
116
118
|
* - `commitBlocks` (phase 2) is REACHABLE here and NOT safe. `commitBlock` throws the
|
|
117
119
|
* aggregate when a header/tail commit got NO response at all — precisely the transient
|
|
118
120
|
* class this retry targets — and that escapes `commit()`, survives `TransactorSource`'s
|
|
@@ -135,10 +137,12 @@ export declare const CONTROL_WRITE_RETRY_BUDGET_MS = 10000;
|
|
|
135
137
|
* NOTE: the discriminator is a formatting detail of another repo. If Optimystic ever reformats
|
|
136
138
|
* those per-batch details, this fails CLOSED — the aggregate stops matching and control writes
|
|
137
139
|
* simply stop being retried, silently losing the absorption rather than doing anything unsafe.
|
|
138
|
-
* The guard
|
|
139
|
-
*
|
|
140
|
-
*
|
|
141
|
-
*
|
|
140
|
+
* The guard is `control-write-degraded-cohort-member.integration.ts`, which asserts the `[block:`
|
|
141
|
+
* token on any live aggregate a degraded case produces; if it reddens there, an upstream reformat
|
|
142
|
+
* is the first thing to check. NOTE: on Optimystic 1.20.0 none of its cases produced this form
|
|
143
|
+
* (2026-10-28: the injected reset failed the write's read typed, and a pend's shortfall arrives on
|
|
144
|
+
* its own), so the guard can pass without seeing it; if the form is ever needed live again, inject
|
|
145
|
+
* the reset at the pend's batch rather than the read's.
|
|
142
146
|
*
|
|
143
147
|
* NOTE: the discriminator is the prefix AND the token within ONE message — never relax it to
|
|
144
148
|
* "contains `[block:`". Beyond the `get` and `pend` sites above, Optimystic renders `[block:` in
|
|
@@ -163,12 +167,14 @@ export declare const CONTROL_WRITE_RETRY_BUDGET_MS = 10000;
|
|
|
163
167
|
* have left its log entry standing), and its collection sync has already spent ten retries on
|
|
164
168
|
* the race. The torn-write rule beside it: a `TornActionError` is re-presented only when
|
|
165
169
|
* `final` is true; any other torn write is declined.
|
|
166
|
-
* - What remains rejectable at the promise phase is stale revision,
|
|
167
|
-
*
|
|
168
|
-
*
|
|
170
|
+
* - What remains rejectable at the promise phase is stale revision, membership-not-admitted, an
|
|
171
|
+
* expired transaction, and a configured validator's refusal (`ClusterMember.evaluatePromise`).
|
|
172
|
+
* A block whose revision a member cannot establish is approved, not refused: the member logs
|
|
173
|
+
* `validation-block-unavailable` and votes on (`judgeStaleRevisions`). Stale revision is worth
|
|
174
|
+
* a retry: this loop re-runs the WHOLE write body, reads included
|
|
169
175
|
* (`ControlDatabase.withWriteLock`'s contract), so attempt 2 presents a pend against the
|
|
170
176
|
* revision that made attempt 1 stale. A blanket "decline any chain reporting a validator
|
|
171
|
-
* rejection" rule would remove that retry to save two wasted attempts on the
|
|
177
|
+
* rejection" rule would remove that retry to save two wasted attempts on the others.
|
|
172
178
|
*
|
|
173
179
|
* Revisit if a re-presentation ever becomes expensive relative to what it buys — a rejection class
|
|
174
180
|
* that costs a full cohort round-trip to re-present, or a write body whose reads are no longer
|
|
@@ -199,16 +205,18 @@ export declare function isUncommittedTransactorAggregate(message: string): boole
|
|
|
199
205
|
* Two vetoes run over the whole chain before any matcher: {@link reportsPossiblyStoredWrite}
|
|
200
206
|
* (by type — a spent sync budget, a non-final torn write, a partial commit) and
|
|
201
207
|
* {@link reportsIndeterminateCommit} (by text — a commit-phase batch). A chain either veto flags,
|
|
202
|
-
* at any level, is never retried on the strength of some other level looking transient.
|
|
203
|
-
*
|
|
208
|
+
* at any level, is never retried on the strength of some other level looking transient. Two typed
|
|
209
|
+
* failures are claimed as retriable: a block read the cohort could not answer
|
|
210
|
+
* ({@link reportsUnreachableCohort}, shared with the read classifier) and a final torn write
|
|
211
|
+
* ({@link isFinalTornWrite}).
|
|
204
212
|
*
|
|
205
213
|
* NOTE: this depends on engine/transactor error TEXT. The messages producible without a
|
|
206
214
|
* network are driven from the REAL engine in `control-formation-seat-budget.spec.ts`, so
|
|
207
|
-
* a rewording reddens a spec rather than silently disabling the retry. The
|
|
208
|
-
*
|
|
209
|
-
* (`control-write-degraded-cohort-member.integration.ts`) asserts this classifier against
|
|
210
|
-
*
|
|
211
|
-
*
|
|
215
|
+
* a rewording reddens a spec rather than silently disabling the retry. The RETRIABLE failures
|
|
216
|
+
* need a real multi-node cluster to produce; the degraded-cohort scenario
|
|
217
|
+
* (`control-write-degraded-cohort-member.integration.ts`) asserts this classifier against the
|
|
218
|
+
* super-majority shortfall from a real silent member, and absorbs a real stream reset end to end
|
|
219
|
+
* (on Optimystic 1.20.0 the reset fails the write's read as an unanswered block, claimed by type).
|
|
212
220
|
*/
|
|
213
221
|
export declare function isRetriableControlWriteFailure(error: unknown): boolean;
|
|
214
222
|
/**
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { CoordinatorPartialCommitError, SyncRetryExhaustedError, TornActionError } from '@optimystic/db-core';
|
|
2
2
|
import { ConcurrentModificationError, PartialCommitError } from '@optimystic/quereus-plugin-optimystic';
|
|
3
|
-
import { causeChain, chainMessages, retryControlOperation } from './control-retry.js';
|
|
3
|
+
import { causeChain, chainMessages, reportsUnreachableCohort, retryControlOperation } from './control-retry.js';
|
|
4
4
|
/**
|
|
5
5
|
* Bounded retry for TRANSIENT control-write failures — the classifier and loop behind
|
|
6
6
|
* `ControlDatabase.lockedWithRetry`, the single funnel every local control write passes
|
|
@@ -148,11 +148,17 @@ const SINGLE_BLOCK_BATCH_TOKEN = '[block:';
|
|
|
148
148
|
*/
|
|
149
149
|
const COMMIT_BATCH_TOKEN = '[blocks:';
|
|
150
150
|
/**
|
|
151
|
-
* The cluster coordinator's super-majority shortfall (`db-
|
|
152
|
-
*
|
|
153
|
-
*
|
|
154
|
-
*
|
|
155
|
-
*
|
|
151
|
+
* The cluster coordinator's super-majority shortfall (`PromiseShortfallError`, db-core), matched
|
|
152
|
+
* ONLY with a ZERO rejection count. The same message with a non-zero count means a member actually
|
|
153
|
+
* voted no (that branch carries `membership-not-admitted` rejections), and retrying it would
|
|
154
|
+
* re-present a spent signature to a cohort that already refused it. A pend's shortfall reaches this
|
|
155
|
+
* funnel as that error on its own, before any commit, so re-presenting it is safe; a commit's
|
|
156
|
+
* arrives inside the `[blocks:` aggregate, which {@link reportsIndeterminateCommit} vetoes (both
|
|
157
|
+
* seen live in `control-write-degraded-cohort-member`).
|
|
158
|
+
*
|
|
159
|
+
* NOTE: matched by TEXT although the error is typed. Upstream keeps this sentence byte for byte as a
|
|
160
|
+
* downstream contract, so the regex is stable; moving to `instanceof PromiseShortfallError` with
|
|
161
|
+
* `rejections === 0` is backlog `debt-control-write-retry-matches-the-shortfall-by-type`.
|
|
156
162
|
*
|
|
157
163
|
* This regex itself never matches a decisive rejection (`ValidatorRejectionError`, `Transaction
|
|
158
164
|
* rejected by validators`) — that text carries no `Failed to get super-majority:` prefix. But the
|
|
@@ -170,7 +176,9 @@ const SUPER_MAJORITY_SHORTFALL_UNANSWERED = /Failed to get super-majority: \d+\/
|
|
|
170
176
|
* re-present:
|
|
171
177
|
*
|
|
172
178
|
* - `get` (a block read, which a write body also performs) and `pend` (phase 1) fail before
|
|
173
|
-
* anything commits, so the write is known not to have landed
|
|
179
|
+
* anything commits, so the write is known not to have landed. `get` raises it only when a
|
|
180
|
+
* coordinator answered and left a block out; a block nobody answered for fails typed
|
|
181
|
+
* instead, which {@link matchesRetriableFailure} claims ({@link reportsUnreachableCohort});
|
|
174
182
|
* - `commitBlocks` (phase 2) is REACHABLE here and NOT safe. `commitBlock` throws the
|
|
175
183
|
* aggregate when a header/tail commit got NO response at all — precisely the transient
|
|
176
184
|
* class this retry targets — and that escapes `commit()`, survives `TransactorSource`'s
|
|
@@ -193,10 +201,12 @@ const SUPER_MAJORITY_SHORTFALL_UNANSWERED = /Failed to get super-majority: \d+\/
|
|
|
193
201
|
* NOTE: the discriminator is a formatting detail of another repo. If Optimystic ever reformats
|
|
194
202
|
* those per-batch details, this fails CLOSED — the aggregate stops matching and control writes
|
|
195
203
|
* simply stop being retried, silently losing the absorption rather than doing anything unsafe.
|
|
196
|
-
* The guard
|
|
197
|
-
*
|
|
198
|
-
*
|
|
199
|
-
*
|
|
204
|
+
* The guard is `control-write-degraded-cohort-member.integration.ts`, which asserts the `[block:`
|
|
205
|
+
* token on any live aggregate a degraded case produces; if it reddens there, an upstream reformat
|
|
206
|
+
* is the first thing to check. NOTE: on Optimystic 1.20.0 none of its cases produced this form
|
|
207
|
+
* (2026-10-28: the injected reset failed the write's read typed, and a pend's shortfall arrives on
|
|
208
|
+
* its own), so the guard can pass without seeing it; if the form is ever needed live again, inject
|
|
209
|
+
* the reset at the pend's batch rather than the read's.
|
|
200
210
|
*
|
|
201
211
|
* NOTE: the discriminator is the prefix AND the token within ONE message — never relax it to
|
|
202
212
|
* "contains `[block:`". Beyond the `get` and `pend` sites above, Optimystic renders `[block:` in
|
|
@@ -221,12 +231,14 @@ const SUPER_MAJORITY_SHORTFALL_UNANSWERED = /Failed to get super-majority: \d+\/
|
|
|
221
231
|
* have left its log entry standing), and its collection sync has already spent ten retries on
|
|
222
232
|
* the race. The torn-write rule beside it: a `TornActionError` is re-presented only when
|
|
223
233
|
* `final` is true; any other torn write is declined.
|
|
224
|
-
* - What remains rejectable at the promise phase is stale revision,
|
|
225
|
-
*
|
|
226
|
-
*
|
|
234
|
+
* - What remains rejectable at the promise phase is stale revision, membership-not-admitted, an
|
|
235
|
+
* expired transaction, and a configured validator's refusal (`ClusterMember.evaluatePromise`).
|
|
236
|
+
* A block whose revision a member cannot establish is approved, not refused: the member logs
|
|
237
|
+
* `validation-block-unavailable` and votes on (`judgeStaleRevisions`). Stale revision is worth
|
|
238
|
+
* a retry: this loop re-runs the WHOLE write body, reads included
|
|
227
239
|
* (`ControlDatabase.withWriteLock`'s contract), so attempt 2 presents a pend against the
|
|
228
240
|
* revision that made attempt 1 stale. A blanket "decline any chain reporting a validator
|
|
229
|
-
* rejection" rule would remove that retry to save two wasted attempts on the
|
|
241
|
+
* rejection" rule would remove that retry to save two wasted attempts on the others.
|
|
230
242
|
*
|
|
231
243
|
* Revisit if a re-presentation ever becomes expensive relative to what it buys — a rejection class
|
|
232
244
|
* that costs a full cohort round-trip to re-present, or a write body whose reads are no longer
|
|
@@ -381,7 +393,9 @@ function isSelfCoordinationGraceRefusal(message) {
|
|
|
381
393
|
/**
|
|
382
394
|
* The error messages that mean "the cohort did not answer AND nothing committed", and nothing
|
|
383
395
|
* else. The observed one-shot failure was `registerSelf()` racing a connection still forming:
|
|
384
|
-
* a read/pend-phase transactor aggregate with `cause=The stream has been reset`.
|
|
396
|
+
* a read/pend-phase transactor aggregate with `cause=The stream has been reset`. When none of
|
|
397
|
+
* the cohort answers the read at all, that failure arrives typed instead, and is claimed by
|
|
398
|
+
* type in {@link matchesRetriableFailure} ({@link reportsUnreachableCohort}), not here.
|
|
385
399
|
*
|
|
386
400
|
* Deliberately NOT matched: every constraint / authorization failure
|
|
387
401
|
* (`CHECK constraint failed:`, `UNIQUE constraint failed:`) — those reach the same funnel
|
|
@@ -446,16 +460,18 @@ const RETRIABLE_SCHEMA_INIT_MATCHERS = [
|
|
|
446
460
|
* Two vetoes run over the whole chain before any matcher: {@link reportsPossiblyStoredWrite}
|
|
447
461
|
* (by type — a spent sync budget, a non-final torn write, a partial commit) and
|
|
448
462
|
* {@link reportsIndeterminateCommit} (by text — a commit-phase batch). A chain either veto flags,
|
|
449
|
-
* at any level, is never retried on the strength of some other level looking transient.
|
|
450
|
-
*
|
|
463
|
+
* at any level, is never retried on the strength of some other level looking transient. Two typed
|
|
464
|
+
* failures are claimed as retriable: a block read the cohort could not answer
|
|
465
|
+
* ({@link reportsUnreachableCohort}, shared with the read classifier) and a final torn write
|
|
466
|
+
* ({@link isFinalTornWrite}).
|
|
451
467
|
*
|
|
452
468
|
* NOTE: this depends on engine/transactor error TEXT. The messages producible without a
|
|
453
469
|
* network are driven from the REAL engine in `control-formation-seat-budget.spec.ts`, so
|
|
454
|
-
* a rewording reddens a spec rather than silently disabling the retry. The
|
|
455
|
-
*
|
|
456
|
-
* (`control-write-degraded-cohort-member.integration.ts`) asserts this classifier against
|
|
457
|
-
*
|
|
458
|
-
*
|
|
470
|
+
* a rewording reddens a spec rather than silently disabling the retry. The RETRIABLE failures
|
|
471
|
+
* need a real multi-node cluster to produce; the degraded-cohort scenario
|
|
472
|
+
* (`control-write-degraded-cohort-member.integration.ts`) asserts this classifier against the
|
|
473
|
+
* super-majority shortfall from a real silent member, and absorbs a real stream reset end to end
|
|
474
|
+
* (on Optimystic 1.20.0 the reset fails the write's read as an unanswered block, claimed by type).
|
|
459
475
|
*/
|
|
460
476
|
export function isRetriableControlWriteFailure(error) {
|
|
461
477
|
return matchesRetriableFailure(error, RETRIABLE_CONTROL_WRITE_MATCHERS);
|
|
@@ -483,12 +499,13 @@ export function isRetriableLostUpdateFailure(error) {
|
|
|
483
499
|
/**
|
|
484
500
|
* Shared body of the three classifiers: walk the `cause` chain, let
|
|
485
501
|
* {@link reportsPossiblyStoredWrite} and then {@link reportsIndeterminateCommit} veto it, then claim
|
|
486
|
-
* a
|
|
487
|
-
*
|
|
488
|
-
*
|
|
489
|
-
*
|
|
490
|
-
*
|
|
491
|
-
*
|
|
502
|
+
* a block read the cohort could not answer ({@link reportsUnreachableCohort}), a final torn write
|
|
503
|
+
* ({@link isFinalTornWrite}) or any link `claimsLink` names, then ask `matchers`. The vetoes run
|
|
504
|
+
* BEFORE any claim for every policy — a chain reporting a possibly-stored write or a commit-phase
|
|
505
|
+
* batch is never retried, however transient some other level looks, and a final torn write or a
|
|
506
|
+
* lost update carried by a partial commit is still declined because a sibling collection landed.
|
|
507
|
+
* `claimsLink` is the one typed claim a policy may add; it sits here beside the shared typed claims
|
|
508
|
+
* so the vetoes stay in one place above all of them.
|
|
492
509
|
*/
|
|
493
510
|
function matchesRetriableFailure(error, matchers, claimsLink) {
|
|
494
511
|
if (!(error instanceof Error)) {
|
|
@@ -502,7 +519,18 @@ function matchesRetriableFailure(error, matchers, claimsLink) {
|
|
|
502
519
|
if (reportsIndeterminateCommit(messages)) {
|
|
503
520
|
return false;
|
|
504
521
|
}
|
|
505
|
-
|
|
522
|
+
// NOTE: an unanswered read can fail AFTER this write's log entry is stored: the refresh between
|
|
523
|
+
// commit attempts reads the collection's log ends, and finishing the write's own entry
|
|
524
|
+
// (`Collection.completeOwnEntry`) asks `getStatus` when no attempt is retained at the entry's
|
|
525
|
+
// revision (a forked lineage — both commit paths retain the failed attempt before refreshing).
|
|
526
|
+
// Either throw escapes the refresh loop unwrapped. Claimed anyway, because the `[block:`
|
|
527
|
+
// aggregate those same reads raise is already claimed by the text matcher, so the typed form
|
|
528
|
+
// adds no new case: the re-run is a new transaction whose body re-reads the blocks, so it diffs
|
|
529
|
+
// to nothing or fails on `UNIQUE` where the earlier write landed, and writes it once where it did
|
|
530
|
+
// not. A refresh that already saved a sibling collection arrives as
|
|
531
|
+
// `CoordinatorPartialCommitError`, which the veto above declines.
|
|
532
|
+
if (reportsUnreachableCohort(links)
|
|
533
|
+
|| links.some(link => isFinalTornWrite(link) || claimsLink?.(link) === true)) {
|
|
506
534
|
return true;
|
|
507
535
|
}
|
|
508
536
|
return messages.some(message => matchers.some(matches => matches(message)));
|