@serfab/cadre-core 1.16.0 → 1.17.0

package/src/control-write-retry.ts

@@ -1,567 +1,569 @@
-import { CoordinatorPartialCommitError, SyncRetryExhaustedError, TornActionError } from '@optimystic/db-core';
-import { PartialCommitError } from '@optimystic/quereus-plugin-optimystic';
-import { causeChain, chainMessages, retryControlOperation } from './control-retry.js';
-import type { ControlRetryOptions } from './control-retry.js';
-
-/**
- * Bounded retry for TRANSIENT control-write failures — the classifier and loop behind
- * `ControlDatabase.lockedWithRetry`, the single funnel every local control write passes
- * through.
- *
- * Why this exists: the control database replicates every block to the whole party, so a
- * three-machine party commits control writes unanimously (`ceil(3 × 0.75) = 3` approvals) —
- * and no `superMajorityThreshold` can lower that bar without violating Optimystic's
- * partition-safety condition at the shipped admission fraction (the arithmetic lives in
- * `docs/architecture.md` → "Replication cluster size"). Cadre therefore accepts unanimity
- * and absorbs the transient half of its cost here: a write that failed because the cohort
- * did not ANSWER (a stream reset mid-connection-formation, zero approvals) is re-presented
- * a moment later, which the degraded-cohort-member scenario measured as safe (a failed
- * write rolls back, nothing half-commits) and effective (the next write commits in ~1 s).
- *
- * A write somebody actually REJECTED is never retried on the strength of the rejection itself —
- * no matcher here claims a rejection message, at any phase, because re-presenting one would
- * re-present a spent signature against a cohort that already said no. Rejections are re-presented
- * anyway when they arrive inside the transactor's promise-phase `[block:` aggregate, which
- * {@link isUncommittedTransactorAggregate} claims on the WRAPPER alone, whatever the cause inside
- * says. Safe because nothing has committed at that phase, and deliberately kept — one of the
- * remaining promise-phase rejections is exactly the kind a re-presentation fixes. See the
- * accepted-tradeoff `NOTE:` there for which, and why no blanket rejection veto is wanted.
- *
- * A failure that says some of the write may be STORED is a different matter, and never
- * re-presented whatever text it carries: {@link reportsPossiblyStoredWrite} vetoes it by type
- * before any matcher runs. The one half-landed write that IS re-presented is a torn write the
- * library marks final — not saved and unable to land ({@link isFinalTornWrite}).
- *
- * TWO policies ship from here, and the split is deliberate: the default
- * ({@link isRetriableControlWriteFailure}, {@link CONTROL_WRITE_ATTEMPTS}) sits under every
- * control write, and {@link SCHEMA_INIT_RETRY_POLICY} is opted into by
- * `ControlDatabase.loadSchema` alone. The widened one absorbs one extra failure class whose
- * re-run safety holds for schema DDL and NOT for writes in general; anything added to the default
- * list must be safe for all ~19 callers.
- *
- * Lives outside `control-database.ts` so a spec can drive the classifier against real engine
- * errors without importing the whole database class. The loop itself lives in
- * `control-retry.ts`, shared with the READ policy (`control-read-retry.ts`) — this module
- * owns the write policies: their classifiers, attempt counts, backoff and budget.
- */
-
-/**
- * Attempts allowed per control write: the first, plus two retries of a transiently-failed
- * cluster commit. Bounded so a genuinely unreachable cohort terminates in the transactor's
- * own error rather than spinning; two retries covers the observed failure (a stream still
- * forming when the write fired) with margin.
- */
-export const CONTROL_WRITE_ATTEMPTS = 3;
-
-/**
- * Attempts allowed for {@link SCHEMA_INIT_RETRY_POLICY}, the one call site that also absorbs a
- * self-coordination refusal.
- *
- * Two more than {@link CONTROL_WRITE_ATTEMPTS} because what this retry buys is wall-clock for a
- * cold-starting node's bootstrap dials to land, and because the refusal is cheap to re-provoke
- * relative to a cluster round-trip: `findCoordinator` already spends its own bounded wait before
- * raising (3 attempts, 500 ms apart, `db-p2p/src/libp2p-key-network.ts` ~422) and then decides
- * LOCALLY that it may not elect itself, so no peer is contacted and no cohort deadline is
- * consumed.
- *
- * Note that the ~1 s `findCoordinator` spends internally is real, on top of this policy's own
- * backoff — which is exactly why {@link CONTROL_WRITE_RETRY_BUDGET_MS} and not this count is what
- * actually terminates the loop in the worst case.
- */
-export const SCHEMA_INIT_ATTEMPTS = 5;
-
-/**
- * Elapsed-time ceiling on the whole retry loop, measured from the START of the first
- * attempt and checked after a failed attempt BEFORE sleeping.
- *
- * This budget — not the attempt count — is what makes the policy safe to sit under every
- * control write. A transient failure (stream reset while a connection is still forming)
- * surfaces in well under a second, so it is retried and the loop adds at most ~2.2 s. A
- * genuinely silent cohort member costs ~21 s per consensus round (two `ClusterClient`
- * response-deadline attempts, 10.5 s each at the default declared link): one pend round plus
- * the cancel discharge's three rounds, about 84 s for one write in
- * `control-write-degraded-cohort-member.integration.ts`, which already exceeds this budget
- * when attempt 1 returns — so that case is surfaced immediately and retry adds ZERO latency
- * to the case where it cannot help.
- *
- * It cuts the loop off by design, so it does not grow with the link: its job is to end the
- * retries of slow attempts, and the caller gets the last attempt's error unchanged. An attempt
- * whose read phase consulted a silent peer ends at about one per-peer read deadline
- * (`cohortReadDeadlineMs` in `link-budget.ts`: 7 s at the default declared link, 5 s before it
- * was derived, 1 s at Optimystic's own default), so it still gets one retry inside this budget
- * where it once got two; an attempt that runs into the `ClusterClient` response deadline
- * (10.5 s at the default declared link) still gets none. See `docs/cadre-consistency.md` →
- * "Deadlines Over Optimystic's Reads and Commits".
- *
- * NOTE: a failed commit attempt also pays a cancel discharge before its error returns
- * (optimystic `TransactorSource.transact`'s catch) — bounded by six rounds and
- * `abortOrCancelTimeoutMs`, which every collection this repo opens sets to the larger of 5 s
- * and the RPC dial deadline: 42.5 s at the default declared link (`optimysticDialLimits` in
- * `link-budget.ts`;
- * `../optimystic/packages/quereus-plugin-optimystic/src/optimystic-adapter/collection-factory.ts`).
- * One cancel that runs its full budget therefore ends the retries on its own, cutting the
- * three-attempt policy to however many attempts came before it. A cancel only runs that long
- * when a peer stays silent, and that attempt has already spent its pend round past this
- * budget, so the cut loses nothing there. After a transient fault the cancel is either
- * answered or gives up after its six rounds against a dead transport, in about a second
- * either way (optimystic `NetworkTransactor.MAX_CANCEL_ROUNDS`). Not observed — every
- * measured round of the transient-reset case committed on attempt 3 of 3 — so this is a
- * condition to watch, not work to do: if that case ever starts failing with
- * `failed after 2/3 attempt(s)`, the cancel discharge is where the time went.
- */
-// eslint-disable-next-line no-restricted-syntax -- cuts off by design: it ends the retries of slow attempts rather than waiting them out; see docs/cadre-consistency.md → "Deadlines Over Optimystic's Reads and Commits"
-export const CONTROL_WRITE_RETRY_BUDGET_MS = 10_000;
-
-/**
- * Backoff before retry N, jittered ±50% (so the first sleep is 125–375 ms — the jitter can
- * never floor a delay to zero). Each single sleep is additionally capped at the LARGEST
- * delay in the list, so a `stop()` racing a backoff is never held up by more than ~1 s.
- * There is no `AbortSignal` at this seam to cancel a sleep outright; the cap is the
- * substitute, deliberately not plumbing one through every write path.
- */
-const CONTROL_WRITE_RETRY_DELAYS_MS: readonly number[] = [250, 1_000];
-
-/**
- * Backoff for {@link SCHEMA_INIT_RETRY_POLICY}, jittered and capped the same way. Worst case
- * ~4.6 s of sleep across four retries (375 + 750 + 1500 + 2000, each capped at the largest base),
- * which stays inside {@link CONTROL_WRITE_RETRY_BUDGET_MS} — the budget still terminates the loop
- * first if any single attempt is itself slow.
- *
- * Sized against measured facts, not guessed:
- *
- *  - the self-coordination guard clears the moment the node has ONE connection again (both
- *    branches that raise `grace-period-not-elapsed` require `getConnections().length === 0`), not
- *    when the 30 s grace period expires — so this does NOT need to cover the grace period, only
- *    the gap until a bootstrap dial completes;
- *  - the guard's 30 s `gracePeriodMs` cannot be tuned from this repo anyway: no caller in either
- *    repo passes a `SelfCoordinationConfig`, so the default is always in force;
- *  - a node that has NEVER connected is waved through as a bootstrap node (network high-water
- *    mark of 1), so this retry only ever delays a node that HAS seen peers. One whose peers are
- *    gone for good still fails, ~4.6 s later — which is why the budget stays at the shared
- *    ceiling rather than stretching to cover the full 30 s window.
- */
-const SCHEMA_INIT_RETRY_DELAYS_MS: readonly number[] = [250, 500, 1_000, 2_000];
-
-/** The network transactor's aggregate, raised when some cohort peers gave no usable answer. */
-const TRANSACTOR_AGGREGATE = /Some peers did not complete:/;
-
-/**
- * How the aggregate's per-batch details name a SINGLE-block batch — the shape both the block
- * read (`get`) and phase 1 (`pend`) format, and the only shape safe to re-present.
- */
-const SINGLE_BLOCK_BATCH_TOKEN = '[block:';
-
-/**
- * How the aggregate's per-batch details name a MULTI-block batch — the shape phase 2
- * (`commitBlocks`) formats, and the marker of an outcome too indeterminate to retry.
- */
-const COMMIT_BATCH_TOKEN = '[blocks:';
-
-/**
- * The cluster coordinator's super-majority shortfall (`db-p2p/src/repo/cluster-coordinator.ts`),
- * matched ONLY with a ZERO rejection count. The same message with a non-zero count means a
- * member actually voted no (that branch carries `membership-not-admitted` rejections), and
- * retrying it would re-present a spent signature to a cohort that already refused it. This one
- * is raised while collecting PROMISES, before any commit, so re-presenting it is safe.
- *
- * This regex itself never matches a decisive rejection (`ValidatorRejectionError`, `Transaction
- * rejected by validators`) — that text carries no `Failed to get super-majority:` prefix. But the
- * composite classifier is not this matcher alone: a rejection raised while promises are still
- * being collected reaches this repo wrapped in the transactor's `Some peers did not complete:
- * …[block:…]` aggregate, and {@link isUncommittedTransactorAggregate} matches that wrapper on its
- * own, whatever the cause inside it says — see the accepted-tradeoff `NOTE:` there.
- */
-const SUPER_MAJORITY_SHORTFALL_UNANSWERED =
-	/Failed to get super-majority: \d+\/\d+ approvals \(needed \d+, 0 rejections\)/;
-
-/**
- * Is this a transactor aggregate from a phase where nothing can have committed yet?
- *
- * The transactor raises `Some peers did not complete:` from three sites
- * (`db-core/src/transactor/network-transactor.ts`), and only two of them are safe to
- * re-present:
- *
- * - `get` (a block read, which a write body also performs) and `pend` (phase 1) fail before
- *   anything commits, so the write is known not to have landed;
- * - `commitBlocks` (phase 2) is REACHABLE here and NOT safe. `commitBlock` throws the
- *   aggregate when a header/tail commit got NO response at all — precisely the transient
- *   class this retry targets — and that escapes `commit()`, survives `TransactorSource`'s
- *   cancel-and-rethrow (cancel reaches PENDING actions only; a peer that already committed
- *   stays committed) and `Collection.syncInternal` (which retries StaleFailure RETURN values,
- *   not throws), reaching this funnel QuereusError-wrapped. The tail commit is one batch to
- *   one coordinator running consensus internally, so a no-response there is INDETERMINATE:
- *   the commit may have completed with only the response lost. Re-running the write body over
- *   a write that landed turns a success into a constraint failure (e.g.
- *   `UNIQUE constraint failed: CadrePeer.PeerId`). Surfacing the transient error instead lets
- *   the caller re-read committed state and decide. Commit-phase messages are vetoed by
- *   {@link reportsIndeterminateCommit}, not here.
- *
- * The `cause` chain does not discriminate the phases — a commit-phase aggregate carries the
- * same stream-reset/dial error a pend-phase one does. The per-batch DETAIL text does: `get`
- * and `pend` format `<peerId>[block:<id>](<status>)`, `commitBlocks` formats
- * `<peerId>[blocks:<count>](<status>)`. `[block:` cannot occur inside `[blocks:`, so the two
- * tokens are disjoint and separate the phases.
- *
- * NOTE: the discriminator is a formatting detail of another repo. If Optimystic ever reformats
- * those per-batch details, this fails CLOSED — the aggregate stops matching and control writes
- * simply stop being retried, silently losing the absorption rather than doing anything unsafe.
- * The guard against that is live: `control-write-degraded-cohort-member.integration.ts` asserts
- * this classifier against the real degraded-cohort failure object and asserts the `[block:`
- * token on the live aggregate; if it reddens there, an upstream reformat is the first thing to
- * check.
- *
- * NOTE: the discriminator is the prefix AND the token within ONE message — never relax it to
- * "contains `[block:`". Beyond the `get` and `pend` sites above, Optimystic renders `[block:` in
- * one further place — a third formatter, outside the aggregate's three sites — `dischargeCancel`
- * (`Cancel of action <id> did not discharge <n> block(s): …`), which does not raise the
- * `Some peers did not complete:` prefix; only the conjunction keeps a cancel fault from
- * classifying as a retriable get/pend (checked against optimystic `c56c2bd4`, 2026-09-16).
- *
- * NOTE: accepted tradeoff — a REJECTION raised while promises are still being collected reaches
- * this repo wrapped in this same aggregate, and this matcher claims it on the wrapper alone,
- * whatever the cause inside says. That re-presents such a rejection up to two more times inside
- * the 10 s budget. Kept deliberately, and re-decided on 2026-09-17 against the current upstream
- * behaviour rather than inherited:
- *
- *  - Contention is no longer a rejection at all. "Another write holds this block right now"
- *    (`pending conflict`) is now a `held` verdict that counts toward neither approvals nor
- *    rejections (`validatePendOperations`,
- *    `../optimystic/packages/db-p2p/src/cluster/cluster-repo.ts`), so the rejection this tradeoff
- *    was originally written about does not arrive any more. It reaches this funnel as
- *    `SyncRetryExhaustedError` instead, which {@link reportsPossiblyStoredWrite} declines by type
- *    — upstream documents resubmitting after it as unsafe (the attempt that spent the budget may
- *    have left its log entry standing), and its collection sync has already spent ten retries on
- *    the race. The torn-write rule beside it: a `TornActionError` is re-presented only when
- *    `final` is true; any other torn write is declined.
- *  - What remains rejectable at the promise phase is stale revision, block-unavailable,
- *    membership-not-admitted, and a configured validator's refusal. Only the first is worth a
- *    retry — and it is worth one: this loop re-runs the WHOLE write body, reads included
- *    (`ControlDatabase.withWriteLock`'s contract), so attempt 2 presents a pend against the
- *    revision that made attempt 1 stale. A blanket "decline any chain reporting a validator
- *    rejection" rule would remove that retry to save two wasted attempts on the other three.
- *
- * Revisit if a re-presentation ever becomes expensive relative to what it buys — a rejection class
- * that costs a full cohort round-trip to re-present, or a write body whose reads are no longer
- * cheap — or when upstream offers a typed refusal surface to classify on instead of this text
- * (`../optimystic/tickets/backlog/debt-a-downstream-repo-classifies-retries-by-parsing-our-error-text`).
- * Contention is no longer the reason to revisit.
- *
- * An aggregate whose details came out EMPTY (possible when `formatBatchStatuses` has no
- * batches to format) carries neither token, matches nothing here, and is not retried — an
- * unattributable failure is not a proven non-commit.
- *
- * Exported for the READ classifier (`control-read-retry.ts`), which reuses it verbatim: a
- * `get` batch formats the same single-block `[block:` token, and a read can never produce
- * the commit-phase `[blocks:` shape at all.
- */
-export function isUncommittedTransactorAggregate(message: string): boolean {
-	return TRANSACTOR_AGGREGATE.test(message) && message.includes(SINGLE_BLOCK_BATCH_TOKEN);
-}
-
-/**
- * Does ANY message in the failure's `cause` chain describe a commit-phase batch — i.e. an
- * outcome nobody can call committed or not?
- *
- * Vetoes the whole chain rather than the one message carrying the token, and vetoes it
- * regardless of which matcher would otherwise have claimed it. The narrower per-message rule
- * would already catch every shape the transactor is known to build (a commit-phase aggregate
- * emits `[blocks:` for every batch it formats, and embeds its cause's message inline as
- * `root: …`, so both tokens land in one string) — but "known to build" is an argument about
- * another repo's error assembly, and this way an indeterminate commit anywhere in the chain
- * costs a retry we could have had instead of risking a re-run over a write that landed.
- */
-function reportsIndeterminateCommit(messages: readonly string[]): boolean {
-	return messages.some(message => message.includes(COMMIT_BATCH_TOKEN));
-}
-
-/**
- * Does any link of the `cause` chain say some of this write may be stored, or may still be?
- * Such a failure is never re-presented — a re-run could store the write twice.
- *
- * - `SyncRetryExhaustedError` (and its `SyncRevisionStalledError` subclass): upstream documents
- *   resubmitting as unsafe, because the attempt that spent the budget may have left its log
- *   entry standing.
- * - `TornActionError` without `final: true`: the write's log entry is stored and nobody could
- *   establish that the rest can never land — it may be saved already, or land later.
- * - `CoordinatorPartialCommitError` / `PartialCommitError`: another collection or tree of the
- *   same transaction is already durable, and re-running the whole body would apply it twice.
- *
- * Every one of these embeds text from elsewhere in its message (a responder's refusal, the
- * underlying failure), which can carry the pend-phase aggregate's prefix and `[block:` token —
- * so this veto runs before any text matcher.
- *
- * Matched by TYPE: each survives the bridge's and Quereus' rewraps on `cause`. An error built by a
- * second loaded copy of `@optimystic/db-core` (or of the plugin, for `PartialCommitError`) fails
- * `instanceof`, and then this veto does not fire — falling back to the text classifiers, which is
- * how these failures were classified before the veto existed. The asymmetry with
- * {@link isFinalTornWrite} is deliberate: there, a missed `instanceof` means no retry, which is the
- * safe side. No text fallback parses `TornActionError`'s closing sentence, since upstream says its
- * wording is for log lines only.
- *
- * NOTE: the plugin is registered through its `/plugin` entry (`control-database.ts`) but
- * `PartialCommitError` is imported from its root entry; the two are one class only because the
- * plugin's build emits both entries over one shared chunk. If upstream ever bundles the entries
- * separately, this `instanceof` stops matching with every spec still green — import the class from
- * the entry the plugin is registered through.
- */
-function reportsPossiblyStoredWrite(links: readonly Error[]): boolean {
-	return links.some(link =>
-		link instanceof SyncRetryExhaustedError
-		|| (link instanceof TornActionError && link.final !== true)
-		|| link instanceof CoordinatorPartialCommitError
-		|| link instanceof PartialCommitError);
-}
-
-/**
- * A torn write the library marks FINAL: not saved, unable to land, its pending records confirmed
- * cancelled — so submitting it again stores it once.
- *
- * Worth re-presenting, not merely safe to: upstream raises it mostly when a rival holds the
- * revision this write's log entry claimed, which is contention, and the loop re-runs the write
- * body's reads, so the next attempt builds on the rival's revision — the same argument that keeps
- * stale-revision rejections retried (the accepted-tradeoff `NOTE:` on
- * {@link isUncommittedTransactorAggregate}).
- *
- * NOTE: `CoordinatorStaleLossError` (db-core; "nothing durably committed, safe to re-drive") is not
- * claimed, so it falls to the text matchers; it escapes only after the coordinator's own retry budget
- * ran out. Seen abandoning a control write on 2026-09-18 (`control-delete-while-alone-convergence`,
- * under load): a node's own storage refused its own pend as a `stale conflict` for the ~14 s the
- * coordinator spent re-driving, then gave up. Still not claimed here — the failing attempt already
- * ran past {@link CONTROL_WRITE_RETRY_BUDGET_MS}, so a retry from this loop would never get to run,
- * and the refusal came from the node's OWN storage disagreeing with its own revision view, which does
- * not change while the node is alone; re-driving the same write body again would hit the same
- * refusal. The cause (a commit torn by a sibling that stopped mid-commit, leaving the node with a
- * revision view its own storage disputes) is `tickets/blocked/forked-control-collection-sync-livelocks.md`
- * → "Second trigger".
- */
-function isFinalTornWrite(link: Error): boolean {
-	return link instanceof TornActionError && link.final === true;
-}
-
-/** The coordinator's shortfall, raised pre-commit, with nobody having voted no. */
-function isUnansweredSuperMajorityShortfall(message: string): boolean {
-	return SUPER_MAJORITY_SHORTFALL_UNANSWERED.test(message);
-}
-
-/**
- * Optimystic refusing to let a node elect ITSELF coordinator because its last connection dropped
- * less than the guard's grace period ago (`Libp2pKeyPeerNetwork.shouldAllowSelfCoordination` →
- * `findCoordinator`, `db-p2p/src/libp2p-key-network.ts`).
- *
- * Matched ONLY with the `grace-period-not-elapsed` reason. The same sentence is raised with three
- * other reasons and none of them is worth a retry: `disabled` is static configuration, and
- * `partition-detected` / `suspicious-shrinkage` describe a network condition that will not resolve
- * inside a few seconds of backoff. The grace-period one WILL — it clears as soon as the node holds
- * one connection again, which a cold-starting node dialling its bootstrap peers is actively
- * working on.
- *
- * NOTE: matched by TEXT, not by type. `FindCoordinatorError` (`code: 'SELF_COORDINATION_BLOCKED'`)
- * is exported from `@optimystic/db-p2p`, and when this matcher was written the error OBJECT did not
- * survive the trip: `OptimysticVirtualTable.initialize` rethrew as `new Error(message)` with no
- * `cause`. It now rethrows through `rewrapAsQueryError`, which keeps `cause`
- * (`quereus-plugin-optimystic/src/optimystic-module.ts`); whether every Quereus wrapper above it on
- * the schema-init path keeps it too has not been checked, and the text still arrives either way.
- * Like every matcher here this fails CLOSED — a rewording upstream stops the retry engaging, it
- * never makes it unsafe.
- */
-const SELF_COORDINATION_GRACE_REFUSAL =
-	/Self-coordination blocked: grace-period-not-elapsed\. No coordinator available for key\./;
-
-/** Self-coordination refused because the node's last connection dropped moments ago. */
-function isSelfCoordinationGraceRefusal(message: string): boolean {
-	return SELF_COORDINATION_GRACE_REFUSAL.test(message);
-}
-
-/**
- * The error messages that mean "the cohort did not answer AND nothing committed", and nothing
- * else. The observed one-shot failure was `registerSelf()` racing a connection still forming:
- * a read/pend-phase transactor aggregate with `cause=The stream has been reset`.
- *
- * Deliberately NOT matched: every constraint / authorization failure
- * (`CHECK constraint failed:`, `UNIQUE constraint failed:`) — those reach the same funnel
- * and must be retried zero times: the cohort REFUSED the write, so re-presenting it
- * re-presents a spent signature and can never commit.
- */
-const RETRIABLE_CONTROL_WRITE_MATCHERS: readonly ((message: string) => boolean)[] = [
-	isUncommittedTransactorAggregate,
-	isUnansweredSuperMajorityShortfall,
-];
-
-/**
- * The above PLUS the self-coordination grace refusal — the classifier for schema init ONLY
- * ({@link SCHEMA_INIT_RETRY_POLICY}), deliberately not folded into the list above.
- *
- * Why this class is safe HERE and not everywhere: a write refused at coordinator SELECTION never
- * reached a peer, so re-presenting it is a proven non-commit. But `findCoordinator` is also
- * reached from `NetworkTransactor.commitBlock` during PHASE 2 (`resolveCoordinator`,
- * `db-core/src/transactor/network-transactor.ts`), and `commit()` commits the header block before
- * the rest — so a general control write refused there can be refused AFTER something already
- * committed. Worse, that error carries no `[blocks:` batch token, so
- * {@link reportsIndeterminateCommit} would not veto it, and re-running an insert body over a write
- * that landed is exactly the `UNIQUE constraint failed: CadrePeer.PeerId` failure that veto exists
- * to prevent.
- *
- * Schema init does not have that problem: `apply schema` is a diff rather than a replay, and a
- * failed apply is taken back whole — Quereus unwinds every step it ran and verifies the catalog
- * against a pre-apply fingerprint — so a re-run emits exactly the DDL the live catalog is missing.
- * The full argument, including the plugin-side dependency that unwind carries, is at the
- * `loadSchema` call site in `control-database.ts`.
- *
- * RETRIED DELIBERATELY, not by oversight: the failures Quereus cannot take back. When the unwind
- * does not complete — an undo statement threw, the post-unwind catalog did not match the pre-apply
- * fingerprint, that catalog could not be re-collected to check, or a step the differ marks
- * irreversible because it discards data (dropping a table or a column, narrowing a column's type —
- * none of which a `CadreControl` diff generates today) poisoned the journal before it ran, leaving
- * nothing unwound — Quereus keeps the failing step's own message, APPENDS the reason the
- * schema could not be restored, and carries the original as `cause`. So a transient cause
- * underneath still matches here, and neither veto ({@link reportsPossiblyStoredWrite},
- * {@link reportsIndeterminateCommit}) looks for that reason text. Retrying over a partially migrated
- * schema is safe for the same reason the ordinary re-run is: the plugin commits whatever its write
- * batch holds, so storage and the catalog are left in step either way, and the next apply diffs
- * against what is really there. Vetoing it instead would turn a healable transient outage into a
- * dead start.
- */
-const RETRIABLE_SCHEMA_INIT_MATCHERS: readonly ((message: string) => boolean)[] = [
-	...RETRIABLE_CONTROL_WRITE_MATCHERS,
-	isSelfCoordinationGraceRefusal,
-];
-
-/**
- * Did this control write fail because the cluster cohort did not ANSWER — i.e. is
- * re-presenting the SAME signed write a moment later the right response?
- *
- * Classifies the transient cases by MESSAGE, walking the `cause` chain with the shared
- * {@link chainMessages} (Quereus' own `unwrapError` underneath) — the transactor and the
- * coordinator both throw bare `Error`s, not recognisable by type. By the time one reaches
- * `ControlDatabase` it is wrapped in a `QuereusError` (the real chain is `QuereusError` → `Error`
- * → `Error`), so the match must work at any depth. Anything that is not an `Error` is never
- * retried.
- *
- * Two vetoes run over the whole chain before any matcher: {@link reportsPossiblyStoredWrite}
- * (by type — a spent sync budget, a non-final torn write, a partial commit) and
- * {@link reportsIndeterminateCommit} (by text — a commit-phase batch). A chain either veto flags,
- * at any level, is never retried on the strength of some other level looking transient. The one
- * typed failure claimed as retriable is a final torn write ({@link isFinalTornWrite}).
- *
- * NOTE: this depends on engine/transactor error TEXT. The messages producible without a
- * network are driven from the REAL engine in `control-formation-seat-budget.spec.ts`, so
- * a rewording reddens a spec rather than silently disabling the retry. The two RETRIABLE
- * messages need a real multi-node cluster to produce; the degraded-cohort scenario
- * (`control-write-degraded-cohort-member.integration.ts`) asserts this classifier against
- * both LIVE — the super-majority shortfall from a real silent member, and the transactor
- * aggregate from a real stream reset, which the retry there absorbs end to end.
- */
-export function isRetriableControlWriteFailure(error: unknown): boolean {
-	return matchesRetriableFailure(error, RETRIABLE_CONTROL_WRITE_MATCHERS);
-}
-
-/**
- * {@link isRetriableControlWriteFailure} widened by exactly one class — the self-coordination
- * grace refusal ({@link RETRIABLE_SCHEMA_INIT_MATCHERS}) — for `ControlDatabase.loadSchema`.
- *
- * Opt-in per call site, via {@link SCHEMA_INIT_RETRY_POLICY}. Every other control write keeps the
- * default classifier untouched.
- */
-export function isRetriableSchemaInitFailure(error: unknown): boolean {
-	return matchesRetriableFailure(error, RETRIABLE_SCHEMA_INIT_MATCHERS);
-}
-
-/**
- * Shared body of both classifiers: walk the `cause` chain, let {@link reportsPossiblyStoredWrite}
- * and then {@link reportsIndeterminateCommit} veto it, then claim a final torn write
- * ({@link isFinalTornWrite}), then ask `matchers`. The vetoes run BEFORE any matcher for every
- * policy — a chain reporting a possibly-stored write or a commit-phase batch is never retried,
- * however transient some other level looks, and a final torn write carried by a partial commit is
- * still declined because a sibling collection landed.
- */
-function matchesRetriableFailure(
-	error: unknown,
-	matchers: readonly ((message: string) => boolean)[]
-): boolean {
-	if (!(error instanceof Error)) {
-		return false;
-	}
-	const links = causeChain(error);
-	if (reportsPossiblyStoredWrite(links)) {
-		return false;
-	}
-	const messages = chainMessages(error);
-	if (reportsIndeterminateCommit(messages)) {
-		return false;
-	}
-	if (links.some(isFinalTornWrite)) {
-		return true;
-	}
-	return messages.some(message => matchers.some(matches => matches(message)));
-}
-
-/**
- * Policy and pacing overrides for {@link retryControlWrite}. Production callers pass nothing
- * (or a named policy such as {@link SCHEMA_INIT_RETRY_POLICY}); specs inject a recorded
- * `sleep`, a shorter delay list, or a fake clock so no test ever waits out a real backoff.
- *
- * Every field defaults to the shipped control-write policy ({@link CONTROL_WRITE_ATTEMPTS},
- * {@link CONTROL_WRITE_RETRY_DELAYS_MS}, {@link isRetriableControlWriteFailure}), so an
- * omitted field is byte-identical to the behaviour before per-call-site policies existed.
- * The field shapes and semantics live on the shared {@link ControlRetryOptions}.
- */
-export type ControlWriteRetryOptions = ControlRetryOptions;
-
-/**
- * The one non-default retry policy: `ControlDatabase.loadSchema`'s distributed DDL, which
- * additionally absorbs optimystic's self-coordination grace refusal and gets more attempts over a
- * longer backoff to do it (see {@link SCHEMA_INIT_ATTEMPTS} and
- * {@link SCHEMA_INIT_RETRY_DELAYS_MS}). Still bounded by the shared
- * {@link CONTROL_WRITE_RETRY_BUDGET_MS}.
- *
- * NOTE: exactly one call site opts in today, and the re-run safety this policy assumes is that
- * site's, not a general property — `apply schema` is a diff, and a failed apply is unwound whole
- * and verified against the pre-apply catalog (full argument at the `loadSchema` call site in
- * `control-database.ts`). A second opt-in must re-derive that argument for its own write body
- * first; if this policy ever grows a third consumer, rename it for what the callers share rather
- * than widening it by default.
- */
-export const SCHEMA_INIT_RETRY_POLICY: Readonly<ControlWriteRetryOptions> = {
-	attempts: SCHEMA_INIT_ATTEMPTS,
-	delaysMs: SCHEMA_INIT_RETRY_DELAYS_MS,
-	isRetriable: isRetriableSchemaInitFailure,
-};
-
-/**
- * Run `attempt` up to {@link CONTROL_WRITE_ATTEMPTS} times (or `options.attempts`), retrying only
- * failures the policy's classifier — {@link isRetriableControlWriteFailure} by default —
- * calls transient, and only while {@link CONTROL_WRITE_RETRY_BUDGET_MS} has not elapsed.
- *
- * `attempt` must be atomic and re-runnable: a failed cluster write rolls back (nothing is
- * half-applied), and re-running the body re-runs its reads too — `ControlDatabase`'s
- * locked write bodies all satisfy this (see the contract on
- * `ControlDatabase.withWriteLock`). The backoff sleeps happen with NO lock held; the caller
- * takes and releases its lock inside `attempt`.
- *
- * On exhaustion (attempts or budget) the LAST error is rethrown unchanged — never wrapped,
- * so the exact messages downstream code and the degraded-cohort-member scenario assert on
- * survive. A non-retriable failure propagates from the attempt that raised it.
- *
- * Either way — declined or exhausted — an `options.onAbandon` observer is notified exactly
- * once before the rethrow (a `ControlRetryAbandonment`). Only WRITES pass one: an abandoned
- * read throws to a caller that is awaiting it, so nothing is lost silently there
- * (`control-read-retry.ts` says so at its own seam), while an abandoned write may have no
- * caller at all — a background self-record republish is `void`-ed with a `debug` catch, so
- * its failure reaches nobody unless something is listening here.
- * `ControlDatabase.setControlWriteAbandonedListener` is the only thing that passes one in
- * production; `retryControlWrite` itself neither supplies nor requires an observer.
- */
-export function retryControlWrite<T>(
-	attempt: () => Promise<T>,
-	options: ControlWriteRetryOptions = {}
-): Promise<T> {
-	return retryControlOperation(attempt, {
-		attempts: CONTROL_WRITE_ATTEMPTS,
-		delaysMs: CONTROL_WRITE_RETRY_DELAYS_MS,
-		budgetMs: CONTROL_WRITE_RETRY_BUDGET_MS,
-		isRetriable: isRetriableControlWriteFailure,
-		// The default prefix, named anyway: the degraded-cohort scenario asserts on
-		// `Control write [<label>] …` lines byte-for-byte.
-		logPrefix: 'Control write',
-	}, options);
-}
+import { CoordinatorPartialCommitError, SyncRetryExhaustedError, TornActionError } from '@optimystic/db-core';
+import { PartialCommitError } from '@optimystic/quereus-plugin-optimystic';
+import { causeChain, chainMessages, retryControlOperation } from './control-retry.js';
+import type { ControlRetryOptions } from './control-retry.js';
+
+/**
+ * Bounded retry for TRANSIENT control-write failures — the classifier and loop behind
+ * `ControlDatabase.lockedWithRetry`, the single funnel every local control write passes
+ * through.
+ *
+ * Why this exists: the control database replicates every block to the whole party, so a
+ * three-machine party commits control writes unanimously (`ceil(3 × 0.75) = 3` approvals) —
+ * and no `superMajorityThreshold` can lower that bar without violating Optimystic's
+ * partition-safety condition at the shipped admission fraction (the arithmetic lives in
+ * `docs/architecture.md` → "Replication cluster size"). Cadre therefore accepts unanimity
+ * and absorbs the transient half of its cost here: a write that failed because the cohort
+ * did not ANSWER (a stream reset mid-connection-formation, zero approvals) is re-presented
+ * a moment later, which the degraded-cohort-member scenario measured as safe (a failed
+ * write rolls back, nothing half-commits) and effective (the next write commits in ~1 s).
+ *
+ * A write somebody actually REJECTED is never retried on the strength of the rejection itself —
+ * no matcher here claims a rejection message, at any phase, because re-presenting one would
+ * re-present a spent signature against a cohort that already said no. Rejections are re-presented
+ * anyway when they arrive inside the transactor's promise-phase `[block:` aggregate, which
+ * {@link isUncommittedTransactorAggregate} claims on the WRAPPER alone, whatever the cause inside
+ * says. Safe because nothing has committed at that phase, and deliberately kept — one of the
+ * remaining promise-phase rejections is exactly the kind a re-presentation fixes. See the
+ * accepted-tradeoff `NOTE:` there for which, and why no blanket rejection veto is wanted.
+ *
+ * A failure that says some of the write may be STORED is a different matter, and never
+ * re-presented whatever text it carries: {@link reportsPossiblyStoredWrite} vetoes it by type
+ * before any matcher runs. The one half-landed write that IS re-presented is a torn write the
+ * library marks final — not saved and unable to land ({@link isFinalTornWrite}).
+ *
+ * TWO policies ship from here, and the split is deliberate: the default
+ * ({@link isRetriableControlWriteFailure}, {@link CONTROL_WRITE_ATTEMPTS}) sits under every
+ * control write, and {@link SCHEMA_INIT_RETRY_POLICY} is opted into by
+ * `ControlDatabase.applyControlSchema` alone. The widened one absorbs one extra failure class whose
+ * re-run safety holds for schema DDL and NOT for writes in general; anything added to the default
+ * list must be safe for all ~19 callers.
+ *
+ * Lives outside `control-database.ts` so a spec can drive the classifier against real engine
+ * errors without importing the whole database class. The loop itself lives in
+ * `control-retry.ts`, shared with the READ policy (`control-read-retry.ts`) — this module
+ * owns the write policies: their classifiers, attempt counts, backoff and budget.
+ */
+
+/**
+ * Attempts allowed per control write: the first, plus two retries of a transiently-failed
+ * cluster commit. Bounded so a genuinely unreachable cohort terminates in the transactor's
+ * own error rather than spinning; two retries covers the observed failure (a stream still
+ * forming when the write fired) with margin.
+ */
+export const CONTROL_WRITE_ATTEMPTS = 3;
+
+/**
+ * Attempts allowed for {@link SCHEMA_INIT_RETRY_POLICY}, the one call site that also absorbs a
+ * self-coordination refusal.
+ *
+ * Two more than {@link CONTROL_WRITE_ATTEMPTS} because what this retry buys is wall-clock for a
+ * cold-starting node's bootstrap dials to land, and because the refusal is cheap to re-provoke
+ * relative to a cluster round-trip: `findCoordinator` already spends its own bounded wait before
+ * raising (3 attempts, 500 ms apart, `db-p2p/src/libp2p-key-network.ts` ~422) and then decides
+ * LOCALLY that it may not elect itself, so no peer is contacted and no cohort deadline is
+ * consumed.
+ *
+ * Note that the ~1 s `findCoordinator` spends internally is real, on top of this policy's own
+ * backoff — which is exactly why {@link CONTROL_WRITE_RETRY_BUDGET_MS} and not this count is what
+ * actually terminates the loop in the worst case.
+ */
+export const SCHEMA_INIT_ATTEMPTS = 5;
+
+/**
+ * Elapsed-time ceiling on the whole retry loop, measured from the START of the first
+ * attempt and checked after a failed attempt BEFORE sleeping.
+ *
+ * This budget — not the attempt count — is what makes the policy safe to sit under every
+ * control write. A transient failure (stream reset while a connection is still forming)
+ * surfaces in well under a second, so it is retried and the loop adds at most ~2.2 s. A
+ * genuinely silent cohort member costs ~21 s per consensus round (two `ClusterClient`
+ * response-deadline attempts, 10.5 s each at the default declared link): one pend round plus
+ * the cancel discharge's three rounds, about 84 s for one write in
+ * `control-write-degraded-cohort-member.integration.ts`, which already exceeds this budget
+ * when attempt 1 returns — so that case is surfaced immediately and retry adds ZERO latency
+ * to the case where it cannot help.
+ *
+ * It cuts the loop off by design, so it does not grow with the link: its job is to end the
+ * retries of slow attempts, and the caller gets the last attempt's error unchanged. An attempt
+ * whose read phase consulted a silent peer ends at about one per-peer read deadline
+ * (`cohortReadDeadlineMs` in `link-budget.ts`: 7 s at the default declared link, 5 s before it
+ * was derived, 1 s at Optimystic's own default), so it still gets one retry inside this budget
+ * where it once got two; an attempt that runs into the `ClusterClient` response deadline
+ * (10.5 s at the default declared link) still gets none. See `docs/cadre-consistency.md` →
+ * "Deadlines Over Optimystic's Reads and Commits".
+ *
+ * NOTE: a failed commit attempt also pays a cancel discharge before its error returns
+ * (optimystic `TransactorSource.transact`'s catch) — bounded by six rounds and
+ * `abortOrCancelTimeoutMs`, which every collection this repo opens sets to the larger of 5 s
+ * and the RPC dial deadline: 42.5 s at the default declared link (`optimysticDialLimits` in
+ * `link-budget.ts`;
+ * `../optimystic/packages/quereus-plugin-optimystic/src/optimystic-adapter/collection-factory.ts`).
+ * One cancel that runs its full budget therefore ends the retries on its own, cutting the
+ * three-attempt policy to however many attempts came before it. A cancel only runs that long
+ * when a peer stays silent, and that attempt has already spent its pend round past this
+ * budget, so the cut loses nothing there. After a transient fault the cancel is either
+ * answered or gives up after its six rounds against a dead transport, in about a second
+ * either way (optimystic `NetworkTransactor.MAX_CANCEL_ROUNDS`). Not observed — every
+ * measured round of the transient-reset case committed on attempt 3 of 3 — so this is a
+ * condition to watch, not work to do: if that case ever starts failing with
+ * `failed after 2/3 attempt(s)`, the cancel discharge is where the time went.
+ */
+// eslint-disable-next-line no-restricted-syntax -- cuts off by design: it ends the retries of slow attempts rather than waiting them out; see docs/cadre-consistency.md → "Deadlines Over Optimystic's Reads and Commits"
+export const CONTROL_WRITE_RETRY_BUDGET_MS = 10_000;
+
+/**
+ * Backoff before retry N, jittered ±50% (so the first sleep is 125–375 ms — the jitter can
+ * never floor a delay to zero). Each single sleep is additionally capped at the LARGEST
+ * delay in the list, so a `stop()` racing a backoff is never held up by more than ~1 s.
+ * There is no `AbortSignal` at this seam to cancel a sleep outright; the cap is the
+ * substitute, deliberately not plumbing one through every write path.
+ */
+const CONTROL_WRITE_RETRY_DELAYS_MS: readonly number[] = [250, 1_000];
+
+/**
+ * Backoff for {@link SCHEMA_INIT_RETRY_POLICY}, jittered and capped the same way. Worst case
+ * ~4.6 s of sleep across four retries (375 + 750 + 1500 + 2000, each capped at the largest base),
+ * which stays inside {@link CONTROL_WRITE_RETRY_BUDGET_MS} — the budget still terminates the loop
+ * first if any single attempt is itself slow.
+ *
+ * Sized against measured facts, not guessed:
+ *
+ *  - the self-coordination guard clears the moment the node has ONE connection again (both
+ *    branches that raise `grace-period-not-elapsed` require `getConnections().length === 0`), not
+ *    when the 30 s grace period expires — so this does NOT need to cover the grace period, only
+ *    the gap until a bootstrap dial completes;
+ *  - the guard's 30 s `gracePeriodMs` cannot be tuned from this repo anyway: no caller in either
+ *    repo passes a `SelfCoordinationConfig`, so the default is always in force;
+ *  - a node that has NEVER connected is waved through as a bootstrap node (network high-water
+ *    mark of 1), so this retry only ever delays a node that HAS seen peers. One whose peers are
+ *    gone for good still fails, ~4.6 s later — which is why the budget stays at the shared
+ *    ceiling rather than stretching to cover the full 30 s window.
+ */
+const SCHEMA_INIT_RETRY_DELAYS_MS: readonly number[] = [250, 500, 1_000, 2_000];
+
+/** The network transactor's aggregate, raised when some cohort peers gave no usable answer. */
+const TRANSACTOR_AGGREGATE = /Some peers did not complete:/;
+
+/**
+ * How the aggregate's per-batch details name a SINGLE-block batch — the shape both the block
+ * read (`get`) and phase 1 (`pend`) format, and the only shape safe to re-present.
+ */
+const SINGLE_BLOCK_BATCH_TOKEN = '[block:';
+
+/**
+ * How the aggregate's per-batch details name a MULTI-block batch — the shape phase 2
+ * (`commitBlocks`) formats, and the marker of an outcome too indeterminate to retry.
+ */
+const COMMIT_BATCH_TOKEN = '[blocks:';
+
+/**
+ * The cluster coordinator's super-majority shortfall (`db-p2p/src/repo/cluster-coordinator.ts`),
+ * matched ONLY with a ZERO rejection count. The same message with a non-zero count means a
+ * member actually voted no (that branch carries `membership-not-admitted` rejections), and
+ * retrying it would re-present a spent signature to a cohort that already refused it. This one
+ * is raised while collecting PROMISES, before any commit, so re-presenting it is safe.
+ *
+ * This regex itself never matches a decisive rejection (`ValidatorRejectionError`, `Transaction
+ * rejected by validators`) — that text carries no `Failed to get super-majority:` prefix. But the
+ * composite classifier is not this matcher alone: a rejection raised while promises are still
+ * being collected reaches this repo wrapped in the transactor's `Some peers did not complete:
+ * …[block:…]` aggregate, and {@link isUncommittedTransactorAggregate} matches that wrapper on its
+ * own, whatever the cause inside it says — see the accepted-tradeoff `NOTE:` there.
+ */
+const SUPER_MAJORITY_SHORTFALL_UNANSWERED =
+	/Failed to get super-majority: \d+\/\d+ approvals \(needed \d+, 0 rejections\)/;
+
+/**
+ * Is this a transactor aggregate from a phase where nothing can have committed yet?
+ *
+ * The transactor raises `Some peers did not complete:` from three sites
+ * (`db-core/src/transactor/network-transactor.ts`), and only two of them are safe to
+ * re-present:
+ *
+ * - `get` (a block read, which a write body also performs) and `pend` (phase 1) fail before
+ *   anything commits, so the write is known not to have landed;
+ * - `commitBlocks` (phase 2) is REACHABLE here and NOT safe. `commitBlock` throws the
+ *   aggregate when a header/tail commit got NO response at all — precisely the transient
+ *   class this retry targets — and that escapes `commit()`, survives `TransactorSource`'s
+ *   cancel-and-rethrow (cancel reaches PENDING actions only; a peer that already committed
+ *   stays committed) and `Collection.syncInternal` (which retries StaleFailure RETURN values,
+ *   not throws), reaching this funnel QuereusError-wrapped. The tail commit is one batch to
+ *   one coordinator running consensus internally, so a no-response there is INDETERMINATE:
+ *   the commit may have completed with only the response lost. Re-running the write body over
+ *   a write that landed turns a success into a constraint failure (e.g.
+ *   `UNIQUE constraint failed: CadrePeer.PeerId`). Surfacing the transient error instead lets
+ *   the caller re-read committed state and decide. Commit-phase messages are vetoed by
+ *   {@link reportsIndeterminateCommit}, not here.
+ *
+ * The `cause` chain does not discriminate the phases — a commit-phase aggregate carries the
+ * same stream-reset/dial error a pend-phase one does. The per-batch DETAIL text does: `get`
+ * and `pend` format `<peerId>[block:<id>](<status>)`, `commitBlocks` formats
+ * `<peerId>[blocks:<count>](<status>)`. `[block:` cannot occur inside `[blocks:`, so the two
+ * tokens are disjoint and separate the phases.
+ *
+ * NOTE: the discriminator is a formatting detail of another repo. If Optimystic ever reformats
+ * those per-batch details, this fails CLOSED — the aggregate stops matching and control writes
+ * simply stop being retried, silently losing the absorption rather than doing anything unsafe.
+ * The guard against that is live: `control-write-degraded-cohort-member.integration.ts` asserts
+ * this classifier against the real degraded-cohort failure object and asserts the `[block:`
+ * token on the live aggregate; if it reddens there, an upstream reformat is the first thing to
+ * check.
+ *
+ * NOTE: the discriminator is the prefix AND the token within ONE message — never relax it to
+ * "contains `[block:`". Beyond the `get` and `pend` sites above, Optimystic renders `[block:` in
+ * one further place — a third formatter, outside the aggregate's three sites — `dischargeCancel`
+ * (`Cancel of action <id> did not discharge <n> block(s): …`), which does not raise the
+ * `Some peers did not complete:` prefix; only the conjunction keeps a cancel fault from
+ * classifying as a retriable get/pend (checked against optimystic `c56c2bd4`, 2026-09-16).
+ *
+ * NOTE: accepted tradeoff — a REJECTION raised while promises are still being collected reaches
+ * this repo wrapped in this same aggregate, and this matcher claims it on the wrapper alone,
+ * whatever the cause inside says. That re-presents such a rejection up to two more times inside
+ * the 10 s budget. Kept deliberately, and re-decided on 2026-09-17 against the current upstream
+ * behaviour rather than inherited:
+ *
+ *  - Contention is no longer a rejection at all. "Another write holds this block right now"
+ *    (`pending conflict`) is now a `held` verdict that counts toward neither approvals nor
+ *    rejections (`validatePendOperations`,
+ *    `../optimystic/packages/db-p2p/src/cluster/cluster-repo.ts`), so the rejection this tradeoff
+ *    was originally written about does not arrive any more. It reaches this funnel as
+ *    `SyncRetryExhaustedError` instead, which {@link reportsPossiblyStoredWrite} declines by type
+ *    — upstream documents resubmitting after it as unsafe (the attempt that spent the budget may
+ *    have left its log entry standing), and its collection sync has already spent ten retries on
+ *    the race. The torn-write rule beside it: a `TornActionError` is re-presented only when
+ *    `final` is true; any other torn write is declined.
+ *  - What remains rejectable at the promise phase is stale revision, block-unavailable,
+ *    membership-not-admitted, and a configured validator's refusal. Only the first is worth a
+ *    retry — and it is worth one: this loop re-runs the WHOLE write body, reads included
+ *    (`ControlDatabase.withWriteLock`'s contract), so attempt 2 presents a pend against the
+ *    revision that made attempt 1 stale. A blanket "decline any chain reporting a validator
+ *    rejection" rule would remove that retry to save two wasted attempts on the other three.
+ *
+ * Revisit if a re-presentation ever becomes expensive relative to what it buys — a rejection class
+ * that costs a full cohort round-trip to re-present, or a write body whose reads are no longer
+ * cheap — or when upstream offers a typed refusal surface to classify on instead of this text
+ * (`../optimystic/tickets/backlog/debt-a-downstream-repo-classifies-retries-by-parsing-our-error-text`).
+ * Contention is no longer the reason to revisit.
+ *
+ * An aggregate whose details came out EMPTY (possible when `formatBatchStatuses` has no
+ * batches to format) carries neither token, matches nothing here, and is not retried — an
+ * unattributable failure is not a proven non-commit.
+ *
+ * Exported for the READ classifier (`control-read-retry.ts`), which reuses it verbatim: a
+ * `get` batch formats the same single-block `[block:` token, and a read can never produce
+ * the commit-phase `[blocks:` shape at all.
+ */
+export function isUncommittedTransactorAggregate(message: string): boolean {
+	return TRANSACTOR_AGGREGATE.test(message) && message.includes(SINGLE_BLOCK_BATCH_TOKEN);
+}
+
+/**
+ * Does ANY message in the failure's `cause` chain describe a commit-phase batch — i.e. an
+ * outcome nobody can call committed or not?
+ *
+ * Vetoes the whole chain rather than the one message carrying the token, and vetoes it
+ * regardless of which matcher would otherwise have claimed it. The narrower per-message rule
+ * would already catch every shape the transactor is known to build (a commit-phase aggregate
+ * emits `[blocks:` for every batch it formats, and embeds its cause's message inline as
+ * `root: …`, so both tokens land in one string) — but "known to build" is an argument about
+ * another repo's error assembly, and this way an indeterminate commit anywhere in the chain
+ * costs a retry we could have had instead of risking a re-run over a write that landed.
+ */
+function reportsIndeterminateCommit(messages: readonly string[]): boolean {
+	return messages.some(message => message.includes(COMMIT_BATCH_TOKEN));
+}
+
+/**
+ * Does any link of the `cause` chain say some of this write may be stored, or may still be?
+ * Such a failure is never re-presented — a re-run could store the write twice.
+ *
+ * - `SyncRetryExhaustedError` (and its `SyncRevisionStalledError` subclass): upstream documents
+ *   resubmitting as unsafe, because the attempt that spent the budget may have left its log
+ *   entry standing.
+ * - `TornActionError` without `final: true`: the write's log entry is stored and nobody could
+ *   establish that the rest can never land — it may be saved already, or land later.
+ * - `CoordinatorPartialCommitError` / `PartialCommitError`: another collection or tree of the
+ *   same transaction is already durable, and re-running the whole body would apply it twice.
+ *
+ * Every one of these embeds text from elsewhere in its message (a responder's refusal, the
+ * underlying failure), which can carry the pend-phase aggregate's prefix and `[block:` token —
+ * so this veto runs before any text matcher.
+ *
+ * Matched by TYPE: each survives the bridge's and Quereus' rewraps on `cause`. An error built by a
+ * second loaded copy of `@optimystic/db-core` (or of the plugin, for `PartialCommitError`) fails
+ * `instanceof`, and then this veto does not fire — falling back to the text classifiers, which is
+ * how these failures were classified before the veto existed. The asymmetry with
+ * {@link isFinalTornWrite} is deliberate: there, a missed `instanceof` means no retry, which is the
+ * safe side. No text fallback parses `TornActionError`'s closing sentence, since upstream says its
+ * wording is for log lines only.
+ *
+ * NOTE: the plugin is registered through its `/plugin` entry (`control-database.ts`) but
+ * `PartialCommitError` is imported from its root entry; the two are one class only because the
+ * plugin's build emits both entries over one shared chunk. If upstream ever bundles the entries
+ * separately, this `instanceof` stops matching with every spec still green — import the class from
+ * the entry the plugin is registered through.
+ */
+function reportsPossiblyStoredWrite(links: readonly Error[]): boolean {
+	return links.some(link =>
+		link instanceof SyncRetryExhaustedError
+		|| (link instanceof TornActionError && link.final !== true)
+		|| link instanceof CoordinatorPartialCommitError
+		|| link instanceof PartialCommitError);
+}
+
+/**
+ * A torn write the library marks FINAL: not saved, unable to land, its pending records confirmed
+ * cancelled — so submitting it again stores it once.
+ *
+ * Worth re-presenting, not merely safe to: upstream raises it mostly when a rival holds the
+ * revision this write's log entry claimed, which is contention, and the loop re-runs the write
+ * body's reads, so the next attempt builds on the rival's revision — the same argument that keeps
+ * stale-revision rejections retried (the accepted-tradeoff `NOTE:` on
+ * {@link isUncommittedTransactorAggregate}).
+ *
+ * NOTE: `CoordinatorStaleLossError` (db-core; "nothing durably committed, safe to re-drive") is not
+ * claimed, so it falls to the text matchers; it escapes only after the coordinator's own retry budget
+ * ran out. Seen abandoning a control write on 2026-09-18 (`control-delete-while-alone-convergence`,
+ * under load): a node's own storage refused its own pend as a `stale conflict` for the ~14 s the
+ * coordinator spent re-driving, then gave up. Still not claimed here — the failing attempt already
+ * ran past {@link CONTROL_WRITE_RETRY_BUDGET_MS}, so a retry from this loop would never get to run,
+ * and the refusal came from the node's OWN storage disagreeing with its own revision view, which does
+ * not change while the node is alone; re-driving the same write body again would hit the same
+ * refusal. The cause (a commit torn by a sibling that stopped mid-commit, leaving the node with a
+ * revision view its own storage disputes) is `tickets/blocked/forked-control-collection-sync-livelocks.md`
+ * → "Second trigger".
+ */
+function isFinalTornWrite(link: Error): boolean {
+	return link instanceof TornActionError && link.final === true;
+}
+
+/** The coordinator's shortfall, raised pre-commit, with nobody having voted no. */
+function isUnansweredSuperMajorityShortfall(message: string): boolean {
+	return SUPER_MAJORITY_SHORTFALL_UNANSWERED.test(message);
+}
+
+/**
+ * Optimystic refusing to let a node elect ITSELF coordinator because its last connection dropped
+ * less than the guard's grace period ago (`Libp2pKeyPeerNetwork.shouldAllowSelfCoordination` →
+ * `findCoordinator`, `db-p2p/src/libp2p-key-network.ts`).
+ *
+ * Matched ONLY with the `grace-period-not-elapsed` reason. The same sentence is raised with three
+ * other reasons and none of them is worth a retry: `disabled` is static configuration, and
+ * `partition-detected` / `suspicious-shrinkage` describe a network condition that will not resolve
+ * inside a few seconds of backoff. The grace-period one WILL — it clears as soon as the node holds
+ * one connection again, which a cold-starting node dialling its bootstrap peers is actively
+ * working on.
+ *
+ * NOTE: matched by TEXT, not by type. `FindCoordinatorError` (`code: 'SELF_COORDINATION_BLOCKED'`)
+ * is exported from `@optimystic/db-p2p`, and when this matcher was written the error OBJECT did not
+ * survive the trip: `OptimysticVirtualTable.initialize` rethrew as `new Error(message)` with no
+ * `cause`. It now rethrows through `rewrapAsQueryError`, which keeps `cause`
+ * (`quereus-plugin-optimystic/src/optimystic-module.ts`); whether every Quereus wrapper above it on
+ * the schema-init path keeps it too has not been checked, and the text still arrives either way.
+ * Like every matcher here this fails CLOSED — a rewording upstream stops the retry engaging, it
+ * never makes it unsafe.
+ */
+const SELF_COORDINATION_GRACE_REFUSAL =
+	/Self-coordination blocked: grace-period-not-elapsed\. No coordinator available for key\./;
+
+/** Self-coordination refused because the node's last connection dropped moments ago. */
+function isSelfCoordinationGraceRefusal(message: string): boolean {
+	return SELF_COORDINATION_GRACE_REFUSAL.test(message);
+}
+
+/**
+ * The error messages that mean "the cohort did not answer AND nothing committed", and nothing
+ * else. The observed one-shot failure was `registerSelf()` racing a connection still forming:
+ * a read/pend-phase transactor aggregate with `cause=The stream has been reset`.
+ *
+ * Deliberately NOT matched: every constraint / authorization failure
+ * (`CHECK constraint failed:`, `UNIQUE constraint failed:`) — those reach the same funnel
+ * and must be retried zero times: the cohort REFUSED the write, so re-presenting it
+ * re-presents a spent signature and can never commit.
+ */
+const RETRIABLE_CONTROL_WRITE_MATCHERS: readonly ((message: string) => boolean)[] = [
+	isUncommittedTransactorAggregate,
+	isUnansweredSuperMajorityShortfall,
+];
+
+/**
+ * The above PLUS the self-coordination grace refusal — the classifier for schema init ONLY
+ * ({@link SCHEMA_INIT_RETRY_POLICY}), deliberately not folded into the list above.
+ *
+ * Why this class is safe HERE and not everywhere: a write refused at coordinator SELECTION never
+ * reached a peer, so re-presenting it is a proven non-commit. But `findCoordinator` is also
+ * reached from `NetworkTransactor.commitBlock` during PHASE 2 (`resolveCoordinator`,
+ * `db-core/src/transactor/network-transactor.ts`), and `commit()` commits the header block before
+ * the rest — so a general control write refused there can be refused AFTER something already
+ * committed. Worse, that error carries no `[blocks:` batch token, so
+ * {@link reportsIndeterminateCommit} would not veto it, and re-running an insert body over a write
+ * that landed is exactly the `UNIQUE constraint failed: CadrePeer.PeerId` failure that veto exists
+ * to prevent.
+ *
+ * Schema init does not have that problem: `apply schema` is a diff rather than a replay, and a
+ * failed apply is taken back whole — Quereus unwinds every step it ran and verifies the catalog
+ * against a pre-apply fingerprint — so a re-run emits exactly the DDL the live catalog is missing.
+ * The full argument, including the plugin-side dependency that unwind carries, is at the
+ * `applyControlSchema` call site in `control-database.ts`.
+ *
+ * RETRIED DELIBERATELY, not by oversight: the failures Quereus cannot take back. When the unwind
+ * does not complete — an undo statement threw, the post-unwind catalog did not match the pre-apply
+ * fingerprint, that catalog could not be re-collected to check, or a step the differ marks
+ * irreversible because it discards data (dropping a table or a column, narrowing a column's type —
+ * none of which reaches a `CadreControl` apply: a cold store's diff is all creates, and
+ * `refuseIncompatibleControlStore` in `control-schema-version.ts` refuses a warm store whose diff
+ * holds anything but creates and tag changes before the apply runs) poisoned the journal before it
+ * ran, leaving nothing unwound — Quereus keeps the failing step's own message, APPENDS the reason the
+ * schema could not be restored, and carries the original as `cause`. So a transient cause
+ * underneath still matches here, and neither veto ({@link reportsPossiblyStoredWrite},
+ * {@link reportsIndeterminateCommit}) looks for that reason text. Retrying over a partially migrated
+ * schema is safe for the same reason the ordinary re-run is: the plugin commits whatever its write
+ * batch holds, so storage and the catalog are left in step either way, and the next apply diffs
+ * against what is really there. Vetoing it instead would turn a healable transient outage into a
+ * dead start.
+ */
+const RETRIABLE_SCHEMA_INIT_MATCHERS: readonly ((message: string) => boolean)[] = [
+	...RETRIABLE_CONTROL_WRITE_MATCHERS,
+	isSelfCoordinationGraceRefusal,
+];
+
+/**
+ * Did this control write fail because the cluster cohort did not ANSWER — i.e. is
+ * re-presenting the SAME signed write a moment later the right response?
+ *
+ * Classifies the transient cases by MESSAGE, walking the `cause` chain with the shared
+ * {@link chainMessages} (Quereus' own `unwrapError` underneath) — the transactor and the
+ * coordinator both throw bare `Error`s, not recognisable by type. By the time one reaches
+ * `ControlDatabase` it is wrapped in a `QuereusError` (the real chain is `QuereusError` → `Error`
+ * → `Error`), so the match must work at any depth. Anything that is not an `Error` is never
+ * retried.
+ *
+ * Two vetoes run over the whole chain before any matcher: {@link reportsPossiblyStoredWrite}
+ * (by type — a spent sync budget, a non-final torn write, a partial commit) and
+ * {@link reportsIndeterminateCommit} (by text — a commit-phase batch). A chain either veto flags,
+ * at any level, is never retried on the strength of some other level looking transient. The one
+ * typed failure claimed as retriable is a final torn write ({@link isFinalTornWrite}).
+ *
+ * NOTE: this depends on engine/transactor error TEXT. The messages producible without a
+ * network are driven from the REAL engine in `control-formation-seat-budget.spec.ts`, so
+ * a rewording reddens a spec rather than silently disabling the retry. The two RETRIABLE
+ * messages need a real multi-node cluster to produce; the degraded-cohort scenario
+ * (`control-write-degraded-cohort-member.integration.ts`) asserts this classifier against
+ * both LIVE — the super-majority shortfall from a real silent member, and the transactor
+ * aggregate from a real stream reset, which the retry there absorbs end to end.
+ */
+export function isRetriableControlWriteFailure(error: unknown): boolean {
+	return matchesRetriableFailure(error, RETRIABLE_CONTROL_WRITE_MATCHERS);
+}
+
+/**
+ * {@link isRetriableControlWriteFailure} widened by exactly one class — the self-coordination
+ * grace refusal ({@link RETRIABLE_SCHEMA_INIT_MATCHERS}) — for `ControlDatabase.applyControlSchema`.
+ *
+ * Opt-in per call site, via {@link SCHEMA_INIT_RETRY_POLICY}. Every other control write keeps the
+ * default classifier untouched.
+ */
+export function isRetriableSchemaInitFailure(error: unknown): boolean {
+	return matchesRetriableFailure(error, RETRIABLE_SCHEMA_INIT_MATCHERS);
+}
+
+/**
+ * Shared body of both classifiers: walk the `cause` chain, let {@link reportsPossiblyStoredWrite}
+ * and then {@link reportsIndeterminateCommit} veto it, then claim a final torn write
+ * ({@link isFinalTornWrite}), then ask `matchers`. The vetoes run BEFORE any matcher for every
+ * policy — a chain reporting a possibly-stored write or a commit-phase batch is never retried,
+ * however transient some other level looks, and a final torn write carried by a partial commit is
+ * still declined because a sibling collection landed.
+ */
+function matchesRetriableFailure(
+	error: unknown,
+	matchers: readonly ((message: string) => boolean)[]
+): boolean {
+	if (!(error instanceof Error)) {
+		return false;
+	}
+	const links = causeChain(error);
+	if (reportsPossiblyStoredWrite(links)) {
+		return false;
+	}
+	const messages = chainMessages(error);
+	if (reportsIndeterminateCommit(messages)) {
+		return false;
+	}
+	if (links.some(isFinalTornWrite)) {
+		return true;
+	}
+	return messages.some(message => matchers.some(matches => matches(message)));
+}
+
+/**
+ * Policy and pacing overrides for {@link retryControlWrite}. Production callers pass nothing
+ * (or a named policy such as {@link SCHEMA_INIT_RETRY_POLICY}); specs inject a recorded
+ * `sleep`, a shorter delay list, or a fake clock so no test ever waits out a real backoff.
+ *
+ * Every field defaults to the shipped control-write policy ({@link CONTROL_WRITE_ATTEMPTS},
+ * {@link CONTROL_WRITE_RETRY_DELAYS_MS}, {@link isRetriableControlWriteFailure}), so an
+ * omitted field is byte-identical to the behaviour before per-call-site policies existed.
+ * The field shapes and semantics live on the shared {@link ControlRetryOptions}.
+ */
+export type ControlWriteRetryOptions = ControlRetryOptions;
+
+/**
+ * The one non-default retry policy: `ControlDatabase.applyControlSchema`'s distributed DDL, which
+ * additionally absorbs optimystic's self-coordination grace refusal and gets more attempts over a
+ * longer backoff to do it (see {@link SCHEMA_INIT_ATTEMPTS} and
+ * {@link SCHEMA_INIT_RETRY_DELAYS_MS}). Still bounded by the shared
+ * {@link CONTROL_WRITE_RETRY_BUDGET_MS}.
+ *
+ * NOTE: exactly one call site opts in today, and the re-run safety this policy assumes is that
+ * site's, not a general property — `apply schema` is a diff, and a failed apply is unwound whole
+ * and verified against the pre-apply catalog (full argument at the `applyControlSchema` call site
+ * in `control-database.ts`). A second opt-in must re-derive that argument for its own write body
+ * first; if this policy ever grows a third consumer, rename it for what the callers share rather
+ * than widening it by default.
+ */
+export const SCHEMA_INIT_RETRY_POLICY: Readonly<ControlWriteRetryOptions> = {
+	attempts: SCHEMA_INIT_ATTEMPTS,
+	delaysMs: SCHEMA_INIT_RETRY_DELAYS_MS,
+	isRetriable: isRetriableSchemaInitFailure,
+};
+
+/**
+ * Run `attempt` up to {@link CONTROL_WRITE_ATTEMPTS} times (or `options.attempts`), retrying only
+ * failures the policy's classifier — {@link isRetriableControlWriteFailure} by default —
+ * calls transient, and only while {@link CONTROL_WRITE_RETRY_BUDGET_MS} has not elapsed.
+ *
+ * `attempt` must be atomic and re-runnable: a failed cluster write rolls back (nothing is
+ * half-applied), and re-running the body re-runs its reads too — `ControlDatabase`'s
+ * locked write bodies all satisfy this (see the contract on
+ * `ControlDatabase.withWriteLock`). The backoff sleeps happen with NO lock held; the caller
+ * takes and releases its lock inside `attempt`.
+ *
+ * On exhaustion (attempts or budget) the LAST error is rethrown unchanged — never wrapped,
+ * so the exact messages downstream code and the degraded-cohort-member scenario assert on
+ * survive. A non-retriable failure propagates from the attempt that raised it.
+ *
+ * Either way — declined or exhausted — an `options.onAbandon` observer is notified exactly
+ * once before the rethrow (a `ControlRetryAbandonment`). Only WRITES pass one: an abandoned
+ * read throws to a caller that is awaiting it, so nothing is lost silently there
+ * (`control-read-retry.ts` says so at its own seam), while an abandoned write may have no
+ * caller at all — a background self-record republish is `void`-ed with a `debug` catch, so
+ * its failure reaches nobody unless something is listening here.
+ * `ControlDatabase.setControlWriteAbandonedListener` is the only thing that passes one in
+ * production; `retryControlWrite` itself neither supplies nor requires an observer.
+ */
+export function retryControlWrite<T>(
+	attempt: () => Promise<T>,
+	options: ControlWriteRetryOptions = {}
+): Promise<T> {
+	return retryControlOperation(attempt, {
+		attempts: CONTROL_WRITE_ATTEMPTS,
+		delaysMs: CONTROL_WRITE_RETRY_DELAYS_MS,
+		budgetMs: CONTROL_WRITE_RETRY_BUDGET_MS,
+		isRetriable: isRetriableControlWriteFailure,
+		// The default prefix, named anyway: the degraded-cohort scenario asserts on
+		// `Control write [<label>] …` lines byte-for-byte.
+		logPrefix: 'Control write',
+	}, options);
+}