@serfab/cadre-core 0.9.0 → 0.10.0

package/src/control-stream.ts

@@ -9,6 +9,12 @@
  * abort-on-timeout read-to-EOF helper so a misbehaving peer that opens a stream
  * and never half-closes its write end cannot pin a receiver forever.
  *
+ * The sender side of all three protocols is the same exchange — dial under a
+ * deadline, write one frame, half-close, read one response, reset the stream on
+ * any failure — so that shape lives here too, as {@link withDeadline} +
+ * {@link exchangeFrame}; each protocol supplies only its dial options, its
+ * request object, and its response decoder.
+ *
  * Dependency-free by design: it imports nothing from the protocol modules, so
  * the import graph stays acyclic (`control-stream` ← `seed-bootstrap` ←
  * `strand-wake-protocol`). In particular it returns RAW bytes from
@@ -37,10 +43,13 @@ export function writeFrame(stream: ControlStream, obj: unknown): void {
 
 /**
  * Reject if `op` does not settle within `ms`. On timeout, invoke `onTimeout`
- * (used by the sender to abort the in-flight dial/stream so it does not leak)
- * before rejecting. A throwing `onTimeout` is swallowed so it cannot mask the
- * timeout rejection. The rejection message is exactly `<label> timed out after
- * <ms>ms`, so callers fold any domain prefix into `label`.
+ * before rejecting — a hook for releasing whatever the op is parked on. A
+ * throwing `onTimeout` is swallowed so it cannot mask the timeout rejection. The
+ * rejection message is exactly `<label> timed out after <ms>ms`, so callers fold
+ * any domain prefix into `label`.
+ *
+ * Senders want {@link withDeadline}, which wires `onTimeout` to an
+ * `AbortController` so the in-flight dial/stream is cancelled rather than leaked.
  */
 export function withTimeout<T>(
   ms: number,
@@ -61,6 +70,74 @@ export function withTimeout<T>(
   });
 }
 
+/**
+ * Run `op` under a `withTimeout` deadline that also *cancels* it: the signal
+ * handed to `op` is aborted immediately before the timeout rejection, so a
+ * sender can pass it to `dialProtocol` (cancelling an in-flight connect) and to
+ * {@link exchangeFrame} (resetting a live stream) instead of leaking either.
+ */
+export function withDeadline<T>(
+  ms: number,
+  label: string,
+  op: (signal: AbortSignal) => Promise<T>,
+): Promise<T> {
+  const controller = new AbortController();
+  return withTimeout(ms, label, () => op(controller.signal), () => controller.abort());
+}
+
+/**
+ * Send one request frame on an open stream, half-close the write end, and read
+ * the response through `readResponse`.
+ *
+ * `signal` is the caller's deadline (see {@link withDeadline}): an abort resets
+ * the stream, releasing the otherwise unbounded response read. If the deadline
+ * already fired while the stream was being dialed, the freshly-opened stream is
+ * reset here rather than left dangling. Every reset — deadline, response-decode
+ * failure, or a `readResponse` that rejects without aborting (the size-cap path
+ * of {@link readStreamToEnd}) — goes through one idempotent `abort`, so the
+ * deadline listener and the error path cannot double-reset a stream.
+ *
+ * `readResponse` owns its own read bound, which is a backstop rather than the
+ * primary one: the caller's deadline started first and therefore fires first.
+ * It matters only when an abort does not release the read (e.g. a stream double
+ * whose `abort()` is a no-op).
+ */
+export async function exchangeFrame<T>(
+  stream: ControlStream,
+  signal: AbortSignal,
+  request: unknown,
+  readResponse: (stream: ControlStream) => Promise<T>,
+  abortMessage: string,
+): Promise<T> {
+  let reset = false;
+  const resetStream = (err: Error): void => {
+    if (reset) return;
+    reset = true;
+    stream.abort(err);
+  };
+
+  const abortErr = new Error(abortMessage);
+  const onAbort = (): void => resetStream(abortErr);
+  if (signal.aborted) {
+    onAbort();
+    throw abortErr;
+  }
+  signal.addEventListener('abort', onAbort, { once: true });
+
+  try {
+    writeFrame(stream, request);
+    // close() half-closes the write end (EOF) while the read end stays open for
+    // the response — the libp2p 3.x request/response pattern.
+    await stream.close();
+    return await readResponse(stream);
+  } catch (err) {
+    resetStream(err instanceof Error ? err : new Error(String(err)));
+    throw err;
+  } finally {
+    signal.removeEventListener('abort', onAbort);
+  }
+}
+
 /** Normalize a libp2p chunk (Uint8Array or Uint8ArrayList) to a flat Uint8Array. */
 function toBytes(chunk: Uint8Array | { subarray(): Uint8Array }): Uint8Array {
   return chunk instanceof Uint8Array ? chunk : chunk.subarray();

package/src/control-write-retry.ts

@@ -0,0 +1,473 @@
+import debug from 'debug';
+import { unwrapError } from '@quereus/quereus';
+
+const log = debug('sereus:cadre:control-db');
+
+/**
+ * Bounded retry for TRANSIENT control-write failures — the classifier and loop behind
+ * `ControlDatabase.lockedWithRetry`, the single funnel every local control write passes
+ * through.
+ *
+ * Why this exists: the control database replicates every block to the whole party, so a
+ * three-machine party commits control writes unanimously (`ceil(3 × 0.75) = 3` approvals) —
+ * and no `superMajorityThreshold` can lower that bar without violating Optimystic's
+ * partition-safety condition at the shipped admission fraction (the arithmetic lives in
+ * `docs/architecture.md` → "Replication cluster size"). Cadre therefore accepts unanimity
+ * and absorbs the transient half of its cost here: a write that failed because the cohort
+ * did not ANSWER (a stream reset mid-connection-formation, zero approvals) is re-presented
+ * a moment later, which the degraded-cohort-member scenario measured as safe (a failed
+ * write rolls back, nothing half-commits) and effective (the next write commits in ~1 s).
+ *
+ * A write somebody actually REJECTED is never retried — re-presenting it would re-present
+ * a spent signature against a cohort that already said no.
+ *
+ * TWO policies ship from here, and the split is deliberate: the default
+ * ({@link isRetriableControlWriteFailure}, {@link CONTROL_WRITE_ATTEMPTS}) sits under every
+ * control write, and {@link SCHEMA_INIT_RETRY_POLICY} is opted into by
+ * `ControlDatabase.loadSchema` alone. The widened one absorbs one extra failure class whose
+ * re-run safety holds for schema DDL and NOT for writes in general; anything added to the default
+ * list must be safe for all ~19 callers.
+ *
+ * Lives outside `control-database.ts` so a spec can drive the classifier against real engine
+ * errors without importing the whole database class.
+ */
+
+/**
+ * Attempts allowed per control write: the first, plus two retries of a transiently-failed
+ * cluster commit. Bounded so a genuinely unreachable cohort terminates in the transactor's
+ * own error rather than spinning; two retries covers the observed failure (a stream still
+ * forming when the write fired) with margin.
+ */
+export const CONTROL_WRITE_ATTEMPTS = 3;
+
+/**
+ * Attempts allowed for {@link SCHEMA_INIT_RETRY_POLICY}, the one call site that also absorbs a
+ * self-coordination refusal.
+ *
+ * Two more than {@link CONTROL_WRITE_ATTEMPTS} because what this retry buys is wall-clock for a
+ * cold-starting node's bootstrap dials to land, and because the refusal is cheap to re-provoke
+ * relative to a cluster round-trip: `findCoordinator` already spends its own bounded wait before
+ * raising (3 attempts, 500 ms apart, `db-p2p/src/libp2p-key-network.ts` ~422) and then decides
+ * LOCALLY that it may not elect itself, so no peer is contacted and no cohort deadline is
+ * consumed.
+ *
+ * Note that the ~1 s `findCoordinator` spends internally is real, on top of this policy's own
+ * backoff — which is exactly why {@link CONTROL_WRITE_RETRY_BUDGET_MS} and not this count is what
+ * actually terminates the loop in the worst case.
+ */
+export const SCHEMA_INIT_ATTEMPTS = 5;
+
+/**
+ * Elapsed-time ceiling on the whole retry loop, measured from the START of the first
+ * attempt and checked after a failed attempt BEFORE sleeping.
+ *
+ * This budget — not the attempt count — is what makes the policy safe to sit under every
+ * control write. A transient failure (stream reset while a connection is still forming)
+ * surfaces in well under a second, so it is retried and the loop adds at most ~2.2 s. A
+ * genuinely silent cohort member fails at ~20 s (two 10 s `ClusterClient` response-deadline
+ * attempts, measured in `control-write-degraded-cohort-member.integration.ts`), which
+ * already exceeds this budget when attempt 1 returns — so that case is surfaced immediately
+ * and retry adds ZERO latency to the case where it cannot help.
+ */
+export const CONTROL_WRITE_RETRY_BUDGET_MS = 10_000;
+
+/**
+ * Backoff before retry N, jittered ±50% (so the first sleep is 125–375 ms — the jitter can
+ * never floor a delay to zero). Each single sleep is additionally capped at the LARGEST
+ * delay in the list, so a `stop()` racing a backoff is never held up by more than ~1 s.
+ * There is no `AbortSignal` at this seam to cancel a sleep outright; the cap is the
+ * substitute, deliberately not plumbing one through every write path.
+ */
+const CONTROL_WRITE_RETRY_DELAYS_MS: readonly number[] = [250, 1_000];
+
+/**
+ * Backoff for {@link SCHEMA_INIT_RETRY_POLICY}, jittered and capped the same way. Worst case
+ * ~4.6 s of sleep across four retries (375 + 750 + 1500 + 2000, each capped at the largest base),
+ * which stays inside {@link CONTROL_WRITE_RETRY_BUDGET_MS} — the budget still terminates the loop
+ * first if any single attempt is itself slow.
+ *
+ * Sized against measured facts, not guessed:
+ *
+ *  - the self-coordination guard clears the moment the node has ONE connection again (both
+ *    branches that raise `grace-period-not-elapsed` require `getConnections().length === 0`), not
+ *    when the 30 s grace period expires — so this does NOT need to cover the grace period, only
+ *    the gap until a bootstrap dial completes;
+ *  - the guard's 30 s `gracePeriodMs` cannot be tuned from this repo anyway: no caller in either
+ *    repo passes a `SelfCoordinationConfig`, so the default is always in force;
+ *  - a node that has NEVER connected is waved through as a bootstrap node (network high-water
+ *    mark of 1), so this retry only ever delays a node that HAS seen peers. One whose peers are
+ *    gone for good still fails, ~4.6 s later — which is why the budget stays at the shared
+ *    ceiling rather than stretching to cover the full 30 s window.
+ */
+const SCHEMA_INIT_RETRY_DELAYS_MS: readonly number[] = [250, 500, 1_000, 2_000];
+
+/** The network transactor's aggregate, raised when some cohort peers gave no usable answer. */
+const TRANSACTOR_AGGREGATE = /Some peers did not complete:/;
+
+/**
+ * How the aggregate's per-batch details name a SINGLE-block batch — the shape both the block
+ * read (`get`) and phase 1 (`pend`) format, and the only shape safe to re-present.
+ */
+const SINGLE_BLOCK_BATCH_TOKEN = '[block:';
+
+/**
+ * How the aggregate's per-batch details name a MULTI-block batch — the shape phase 2
+ * (`commitBlocks`) formats, and the marker of an outcome too indeterminate to retry.
+ */
+const COMMIT_BATCH_TOKEN = '[blocks:';
+
+/**
+ * The cluster coordinator's super-majority shortfall (`db-p2p/src/repo/cluster-coordinator.ts`),
+ * matched ONLY with a ZERO rejection count. The same message with a non-zero count means a
+ * member actually voted no (that branch carries `membership-not-admitted` rejections), and
+ * retrying it would re-present a spent signature to a cohort that already refused it. This one
+ * is raised while collecting PROMISES, before any commit, so re-presenting it is safe. A
+ * decisive rejection is a different error entirely (`ValidatorRejectionError`,
+ * `Transaction rejected by validators`), which this classifier also never matches.
+ */
+const SUPER_MAJORITY_SHORTFALL_UNANSWERED =
+	/Failed to get super-majority: \d+\/\d+ approvals \(needed \d+, 0 rejections\)/;
+
+/**
+ * Is this a transactor aggregate from a phase where nothing can have committed yet?
+ *
+ * The transactor raises `Some peers did not complete:` from three sites
+ * (`db-core/src/transactor/network-transactor.ts`), and only two of them are safe to
+ * re-present:
+ *
+ * - `get` (a block read, which a write body also performs) and `pend` (phase 1) fail before
+ *   anything commits, so the write is known not to have landed;
+ * - `commitBlocks` (phase 2) is REACHABLE here and NOT safe. `commitBlock` throws the
+ *   aggregate when a header/tail commit got NO response at all — precisely the transient
+ *   class this retry targets — and that escapes `commit()`, survives `TransactorSource`'s
+ *   cancel-and-rethrow (cancel reaches PENDING actions only; a peer that already committed
+ *   stays committed) and `Collection.syncInternal` (which retries StaleFailure RETURN values,
+ *   not throws), reaching this funnel QuereusError-wrapped. The tail commit is one batch to
+ *   one coordinator running consensus internally, so a no-response there is INDETERMINATE:
+ *   the commit may have completed with only the response lost. Re-running the write body over
+ *   a write that landed turns a success into a constraint failure (e.g.
+ *   `UNIQUE constraint failed: CadrePeer.PeerId`). Surfacing the transient error instead lets
+ *   the caller re-read committed state and decide. Commit-phase messages are vetoed by
+ *   {@link reportsIndeterminateCommit}, not here.
+ *
+ * The `cause` chain does not discriminate the phases — a commit-phase aggregate carries the
+ * same stream-reset/dial error a pend-phase one does. The per-batch DETAIL text does: `get`
+ * and `pend` format `<peerId>[block:<id>](<status>)`, `commitBlocks` formats
+ * `<peerId>[blocks:<count>](<status>)`. `[block:` cannot occur inside `[blocks:`, so the two
+ * tokens are disjoint and separate the phases.
+ *
+ * NOTE: the discriminator is a formatting detail of another repo. If Optimystic ever reformats
+ * those per-batch details, this fails CLOSED — the aggregate stops matching and control writes
+ * simply stop being retried, silently losing the absorption rather than doing anything unsafe.
+ * The guard against that is a scenario asserting the classifier against a live cluster failure
+ * (`control-write-retry-scenario-coverage`); if that lands and later reddens here, an upstream
+ * reformat is the first thing to check.
+ *
+ * An aggregate whose details came out EMPTY (possible when `formatBatchStatuses` has no
+ * batches to format) carries neither token, matches nothing here, and is not retried — an
+ * unattributable failure is not a proven non-commit.
+ */
+function isUncommittedTransactorAggregate(message: string): boolean {
+	return TRANSACTOR_AGGREGATE.test(message) && message.includes(SINGLE_BLOCK_BATCH_TOKEN);
+}
+
+/**
+ * Does ANY message in the failure's `cause` chain describe a commit-phase batch — i.e. an
+ * outcome nobody can call committed or not?
+ *
+ * Vetoes the whole chain rather than the one message carrying the token, and vetoes it
+ * regardless of which matcher would otherwise have claimed it. The narrower per-message rule
+ * would already catch every shape the transactor is known to build (a commit-phase aggregate
+ * emits `[blocks:` for every batch it formats, and embeds its cause's message inline as
+ * `root: …`, so both tokens land in one string) — but "known to build" is an argument about
+ * another repo's error assembly, and this way an indeterminate commit anywhere in the chain
+ * costs a retry we could have had instead of risking a re-run over a write that landed.
+ */
+function reportsIndeterminateCommit(messages: readonly string[]): boolean {
+	return messages.some(message => message.includes(COMMIT_BATCH_TOKEN));
+}
+
+/** The coordinator's shortfall, raised pre-commit, with nobody having voted no. */
+function isUnansweredSuperMajorityShortfall(message: string): boolean {
+	return SUPER_MAJORITY_SHORTFALL_UNANSWERED.test(message);
+}
+
+/**
+ * Optimystic refusing to let a node elect ITSELF coordinator because its last connection dropped
+ * less than the guard's grace period ago (`Libp2pKeyPeerNetwork.shouldAllowSelfCoordination` →
+ * `findCoordinator`, `db-p2p/src/libp2p-key-network.ts`).
+ *
+ * Matched ONLY with the `grace-period-not-elapsed` reason. The same sentence is raised with three
+ * other reasons and none of them is worth a retry: `disabled` is static configuration, and
+ * `partition-detected` / `suspicious-shrinkage` describe a network condition that will not resolve
+ * inside a few seconds of backoff. The grace-period one WILL — it clears as soon as the node holds
+ * one connection again, which a cold-starting node dialling its bootstrap peers is actively
+ * working on.
+ *
+ * NOTE: matched by TEXT, not by type, and that is forced rather than chosen.
+ * `FindCoordinatorError` (`code: 'SELF_COORDINATION_BLOCKED'`) is exported from `@optimystic/db-p2p`
+ * and this package depends on it, but the error OBJECT does not survive the trip:
+ * `OptimysticVirtualTable.initialize` catches and rethrows as `new Error(message)` with no `cause`
+ * (`quereus-plugin-optimystic/src/optimystic-module.ts`), so only the text reaches this repo. Like
+ * every matcher here this fails CLOSED — a rewording upstream stops the retry engaging, it never
+ * makes it unsafe.
+ */
+const SELF_COORDINATION_GRACE_REFUSAL =
+	/Self-coordination blocked: grace-period-not-elapsed\. No coordinator available for key\./;
+
+/** Self-coordination refused because the node's last connection dropped moments ago. */
+function isSelfCoordinationGraceRefusal(message: string): boolean {
+	return SELF_COORDINATION_GRACE_REFUSAL.test(message);
+}
+
+/**
+ * Every message in the failure's `cause` chain.
+ *
+ * `unwrapError` declares its `message` as `string`, but it follows `.cause` without checking what
+ * that holds — a chain link that is not an `Error` (a stream rejected with a bare string reason,
+ * an `AbortSignal.reason` that is a plain object) yields `undefined` there, and calling
+ * `.includes` on it would throw a `TypeError` out of this classifier, INSIDE `retryControlWrite`'s
+ * catch, replacing the real control-write failure with a confusing one. Non-strings are dropped:
+ * a link nobody can read is a link that matches nothing, which is already the conservative answer.
+ *
+ * NOTE: a cause chain with a CYCLE would spin forever inside `unwrapError` itself. No error in
+ * this repo builds one; if a hang ever localises to a control-write failure path, look there.
+ */
+function chainMessages(error: Error): string[] {
+	return unwrapError(error)
+		.map(({ message }) => message)
+		.filter(message => typeof message === 'string');
+}
+
+/**
+ * The error messages that mean "the cohort did not answer AND nothing committed", and nothing
+ * else. The observed one-shot failure was `registerSelf()` racing a connection still forming:
+ * a read/pend-phase transactor aggregate with `cause=The stream has been reset`.
+ *
+ * Deliberately NOT matched: every constraint / authorization failure
+ * (`CHECK constraint failed:`, `UNIQUE constraint failed:`) — those reach the same funnel
+ * and must be retried zero times. In particular the lost-use-number messages
+ * (`isLostUseNumberRace` in `control-database.ts`) are constraint failures, so this
+ * classifier and that one are DISJOINT and the two retry loops can never multiply.
+ */
+const RETRIABLE_CONTROL_WRITE_MATCHERS: readonly ((message: string) => boolean)[] = [
+	isUncommittedTransactorAggregate,
+	isUnansweredSuperMajorityShortfall,
+];
+
+/**
+ * The above PLUS the self-coordination grace refusal — the classifier for schema init ONLY
+ * ({@link SCHEMA_INIT_RETRY_POLICY}), deliberately not folded into the list above.
+ *
+ * Why this class is safe HERE and not everywhere: a write refused at coordinator SELECTION never
+ * reached a peer, so re-presenting it is a proven non-commit. But `findCoordinator` is also
+ * reached from `NetworkTransactor.commitBlock` during PHASE 2 (`resolveCoordinator`,
+ * `db-core/src/transactor/network-transactor.ts`), and `commit()` commits the header block before
+ * the rest — so a general control write refused there can be refused AFTER something already
+ * committed. Worse, that error carries no `[blocks:` batch token, so
+ * {@link reportsIndeterminateCommit} would not veto it, and re-running an insert body over a write
+ * that landed is exactly the `UNIQUE constraint failed: CadrePeer.PeerId` failure that veto exists
+ * to prevent.
+ *
+ * Schema init does not have that problem: `apply schema` is a diff rather than a replay and a
+ * failed `create table` leaves the catalog clean, so a re-run emits only the tables that did not
+ * land (the full argument is at the `loadSchema` call site in `control-database.ts`).
+ */
+const RETRIABLE_SCHEMA_INIT_MATCHERS: readonly ((message: string) => boolean)[] = [
+	...RETRIABLE_CONTROL_WRITE_MATCHERS,
+	isSelfCoordinationGraceRefusal,
+];
+
+/**
+ * Did this control write fail because the cluster cohort did not ANSWER — i.e. is
+ * re-presenting the SAME signed write a moment later the right response?
+ *
+ * Classifies by MESSAGE, walking the `cause` chain with Quereus' own {@link unwrapError} —
+ * the exact shape of `isLostUseNumberRace`, for the same reason: neither surface is
+ * recognisable by type. The transactor and the coordinator both throw bare `Error`s, and by
+ * the time one reaches `ControlDatabase` it is wrapped in a `QuereusError` (the real chain
+ * is `QuereusError` → `Error` → `Error`), so the match must work at any depth. Anything
+ * that is not an `Error` is never retried.
+ *
+ * {@link reportsIndeterminateCommit} vetoes the whole chain before any matcher runs, so a
+ * failure that reports a commit-phase batch anywhere is never retried on the strength of some
+ * other level looking transient.
+ *
+ * NOTE: this depends on engine/transactor error TEXT. The messages producible without a
+ * network are driven from the REAL engine in `control-formation-use-number-retry.spec.ts`, so
+ * a rewording reddens a spec rather than silently disabling the retry. The two RETRIABLE
+ * messages need a real multi-node cluster to produce and are still literals in
+ * `control-write-retry.spec.ts`; `control-write-retry-scenario-coverage` closes that gap.
+ */
+export function isRetriableControlWriteFailure(error: unknown): boolean {
+	return matchesRetriableMessage(error, RETRIABLE_CONTROL_WRITE_MATCHERS);
+}
+
+/**
+ * {@link isRetriableControlWriteFailure} widened by exactly one class — the self-coordination
+ * grace refusal ({@link RETRIABLE_SCHEMA_INIT_MATCHERS}) — for `ControlDatabase.loadSchema`.
+ *
+ * Opt-in per call site, via {@link SCHEMA_INIT_RETRY_POLICY}. Every other control write keeps the
+ * default classifier untouched.
+ */
+export function isRetriableSchemaInitFailure(error: unknown): boolean {
+	return matchesRetriableMessage(error, RETRIABLE_SCHEMA_INIT_MATCHERS);
+}
+
+/**
+ * Shared body of both classifiers: walk the `cause` chain, let
+ * {@link reportsIndeterminateCommit} veto it, then ask `matchers`. The veto runs BEFORE any
+ * matcher for every policy — a chain reporting a commit-phase batch is never retried, however
+ * transient some other level looks.
+ */
+function matchesRetriableMessage(
+	error: unknown,
+	matchers: readonly ((message: string) => boolean)[]
+): boolean {
+	if (!(error instanceof Error)) {
+		return false;
+	}
+	const messages = chainMessages(error);
+	if (reportsIndeterminateCommit(messages)) {
+		return false;
+	}
+	return messages.some(message => matchers.some(matches => matches(message)));
+}
+
+/**
+ * Policy and pacing for {@link retryControlWrite}. Production callers pass nothing (or a named
+ * policy such as {@link SCHEMA_INIT_RETRY_POLICY}); specs inject a recorded `sleep`, a shorter
+ * delay list, or a fake clock so no test ever waits out a real backoff.
+ *
+ * Every field defaults to the shipped control-write policy, so an omitted field is byte-identical
+ * to the behaviour before per-call-site policies existed.
+ */
+export interface ControlWriteRetryOptions {
+	/**
+	 * Total attempts, first included. Default {@link CONTROL_WRITE_ATTEMPTS}, floored at 1 —
+	 * below that the loop would run the body zero times and rethrow a `lastError` nobody set.
+	 */
+	attempts?: number;
+	/** Backoff base before retry N (last entry repeats). Default {@link CONTROL_WRITE_RETRY_DELAYS_MS}. */
+	delaysMs?: readonly number[];
+	/**
+	 * Which failures are transient enough to re-present. Default
+	 * {@link isRetriableControlWriteFailure}; the only other shipped value is
+	 * {@link isRetriableSchemaInitFailure}, via {@link SCHEMA_INIT_RETRY_POLICY}.
+	 */
+	isRetriable?: (error: unknown) => boolean;
+	/** The sleep primitive. Default: `setTimeout`. */
+	sleep?: (ms: number) => Promise<void>;
+	/** Clock for the elapsed-budget check. Default: `Date.now`. */
+	now?: () => number;
+}
+
+/**
+ * The one non-default retry policy: `ControlDatabase.loadSchema`'s distributed DDL, which
+ * additionally absorbs optimystic's self-coordination grace refusal and gets more attempts over a
+ * longer backoff to do it (see {@link SCHEMA_INIT_ATTEMPTS} and
+ * {@link SCHEMA_INIT_RETRY_DELAYS_MS}). Still bounded by the shared
+ * {@link CONTROL_WRITE_RETRY_BUDGET_MS}.
+ *
+ * NOTE: exactly one call site opts in today, and the re-run safety this policy assumes is that
+ * site's, not a general property — `apply schema` is a diff and a failed `create table` leaves the
+ * catalog clean. A second opt-in must re-derive that argument for its own write body first; if this
+ * policy ever grows a third consumer, rename it for what the callers share rather than widening it
+ * by default.
+ */
+export const SCHEMA_INIT_RETRY_POLICY: Readonly<ControlWriteRetryOptions> = {
+	attempts: SCHEMA_INIT_ATTEMPTS,
+	delaysMs: SCHEMA_INIT_RETRY_DELAYS_MS,
+	isRetriable: isRetriableSchemaInitFailure,
+};
+
+/**
+ * Run `attempt` up to {@link CONTROL_WRITE_ATTEMPTS} times (or `options.attempts`), retrying only
+ * failures the policy's classifier — {@link isRetriableControlWriteFailure} by default —
+ * calls transient, and only while {@link CONTROL_WRITE_RETRY_BUDGET_MS} has not elapsed.
+ *
+ * `attempt` must be atomic and re-runnable: a failed cluster write rolls back (nothing is
+ * half-applied), and re-running the body re-runs its reads too — `ControlDatabase`'s
+ * locked write bodies all satisfy this (see the contract on
+ * `ControlDatabase.withWriteLock`). The backoff sleeps happen with NO lock held; the caller
+ * takes and releases its lock inside `attempt`.
+ *
+ * On exhaustion (attempts or budget) the LAST error is rethrown unchanged — never wrapped,
+ * so the exact messages downstream code and the degraded-cohort-member scenario assert on
+ * survive. A non-retriable failure propagates from the attempt that raised it.
+ */
+export async function retryControlWrite<T>(
+	attempt: () => Promise<T>,
+	options: ControlWriteRetryOptions = {}
+): Promise<T> {
+	const attempts = Math.max(1, options.attempts ?? CONTROL_WRITE_ATTEMPTS);
+	const delays = options.delaysMs ?? CONTROL_WRITE_RETRY_DELAYS_MS;
+	const isRetriable = options.isRetriable ?? isRetriableControlWriteFailure;
+	const sleep = options.sleep ?? defaultSleep;
+	const now = options.now ?? Date.now;
+	const start = now();
+	let lastError: unknown;
+	let attemptsMade = 0;
+	for (let attemptNumber = 1; attemptNumber <= attempts; attemptNumber++) {
+		attemptsMade = attemptNumber;
+		try {
+			const result = await attempt();
+			if (attemptNumber > 1) {
+				log('Control write committed on attempt %d/%d', attemptNumber, attempts);
+			}
+			return result;
+		} catch (error) {
+			if (!isRetriable(error)) {
+				// The ONLY trace that this funnel saw a failure and declined it. Without it the
+				// log is silent either way, so "the classifier vetoed this one" is
+				// indistinguishable from "the retry is not wired into this path at all".
+				// "here" is load-bearing: this fires for every non-transient failure, and some
+				// of them — the lost-use-number constraint failures — ARE retried by the OUTER
+				// loop (`ControlDatabase.withUseNumberRetry`) that this one nests inside.
+				log('Control write failed non-transiently on attempt %d/%d, not retried here: %s',
+					attemptNumber, attempts, error);
+				throw error;
+			}
+			lastError = error;
+			if (attemptNumber === attempts) {
+				break;
+			}
+			const elapsed = now() - start;
+			if (elapsed >= CONTROL_WRITE_RETRY_BUDGET_MS) {
+				break;
+			}
+			const delay = jitteredDelay(delays, attemptNumber);
+			log('Control write failed transiently (attempt %d/%d), retrying in %d ms: %s',
+				attemptNumber, attempts, delay, error);
+			await sleep(delay);
+		}
+	}
+	log('Control write failed after %d/%d attempt(s): %s', attemptsMade, attempts, lastError);
+	throw lastError;
+}
+
+/**
+ * Backoff for the retry that follows attempt `attemptNumber`: the matching base delay
+ * (last entry repeats), jittered ±50%, then capped at the list's largest base so no single
+ * sleep exceeds it (see {@link CONTROL_WRITE_RETRY_DELAYS_MS} for why). `Math.random` is
+ * fine here — the jitter only de-synchronizes concurrent retriers, nothing is derived
+ * from it.
+ *
+ * NOTE: the cap bites on the LAST delay, whose base IS the largest, so ~half of those sleeps
+ * land exactly on the cap rather than spread — de-synchronization is only partial there.
+ * Harmless while retriers are a handful of party nodes whose attempts already start seconds
+ * apart; if a party ever retries in a tight synchronized herd, raise the cap above the
+ * largest base instead of jittering about it.
+ */
+function jitteredDelay(delays: readonly number[], attemptNumber: number): number {
+	if (delays.length === 0) {
+		return 0;
+	}
+	const base = delays[Math.min(attemptNumber - 1, delays.length - 1)]!;
+	const jittered = base * (0.5 + Math.random());
+	return Math.min(jittered, Math.max(...delays));
+}
+
+function defaultSleep(ms: number): Promise<void> {
+	return new Promise(resolve => setTimeout(resolve, ms));
+}