@serfab/cadre-core 0.11.0 → 0.13.0

package/src/control-read-retry.ts

@@ -0,0 +1,167 @@
+import { chainMessages, retryControlOperation } from './control-retry.js';
+import type { ControlRetryOptions } from './control-retry.js';
+import { isUncommittedTransactorAggregate } from './control-write-retry.js';
+
+/**
+ * Bounded retry for TRANSIENT control-read failures — the classifier and policy behind
+ * `ControlDatabase.readRows`, the single funnel every control read passes through.
+ *
+ * Why this exists: control WRITES have absorbed transient cluster failures since
+ * `control-write-retry.ts` landed, but a read had no second chance — one stream reset
+ * during a scan ended the read outright (measured: a rigged single `eval` failure killed
+ * `queryRevokedStamps` after exactly one attempt while the same injection against `exec`
+ * was absorbed). This module gives reads the same bounded re-presentation with a SHORTER
+ * deadline and a NARROWER failure set, because the tightest caller deadline over a control
+ * read is the inbound admission gate's 2 s fail-open timeout
+ * (`ADMISSION_DECISION_TIMEOUT_MS`, `membership-connection-gater.ts`) — a read budget that
+ * does not fit inside it with headroom spends its retries after the gate has already
+ * admitted.
+ *
+ * This is deliberately NOT "the write classifier minus its commit veto" — the retriable
+ * set differs in both directions (see {@link isRetriableControlReadFailure}). The loop
+ * lives in `control-retry.ts`, shared with the write policy; this module owns only the
+ * read policy. And like the write side, classification is by TEXT: optimystic's
+ * `OptimysticVirtualTable` catches every scan-path error and rethrows
+ * `new Error('Query failed: ' + message)` with NO `cause`
+ * (`quereus-plugin-optimystic/src/optimystic-module.ts`), so the typed
+ * `BlockUnavailableError` and its `reason` field are destroyed before they reach this
+ * repo; Quereus then wraps that as `Error during query on table '<T>': …` preserving
+ * `cause` (`quereus/src/runtime/emit/scan.ts`). Every matcher fails CLOSED — an upstream
+ * rewording stops the retry engaging, it never makes it unsafe.
+ */
+
+/**
+ * Attempts allowed per control read: the first, plus two retries. Attempts are cheap here —
+ * the observed transient failure (the transactor's read-phase aggregate off a stream still
+ * forming) surfaces in ~25 ms — and {@link CONTROL_READ_RETRY_BUDGET_MS} is what terminates
+ * the loop when an attempt is slow instead.
+ */
+export const CONTROL_READ_ATTEMPTS = 3;
+
+/**
+ * Backoff before read retry N, jittered ±50% and capped at the largest base by the shared
+ * loop — so worst case 150 ms + 400 ms = 550 ms of total sleep (the second delay's
+ * upside jitter is clipped by the cap). Much shorter than the write list because the
+ * whole loop must fit under {@link CONTROL_READ_RETRY_BUDGET_MS}; the spec pins that
+ * relationship rather than the number.
+ */
+export const CONTROL_READ_RETRY_DELAYS_MS: readonly number[] = [100, 400];
+
+/**
+ * Elapsed-time ceiling on the whole read-retry loop, checked after a failed attempt BEFORE
+ * sleeping — so one slow attempt (e.g. a `cohort-unreachable` read that burned the
+ * transactor's own deadline) terminates the loop rather than compounding.
+ *
+ * 1500 ms, sized against `ADMISSION_DECISION_TIMEOUT_MS` (2000 ms): the inbound admission
+ * gate reads the control DB and is FAIL-OPEN on both throw and timeout, so a read that
+ * outlives the gate's deadline spends its retries after the gate has already admitted an
+ * unplaced peer. Under the deadline, a read that succeeds on its second attempt lets the
+ * gate make the real decision instead. `control-read-retry.spec.ts` asserts the
+ * relationship so a future edit to either constant reddens instead of silently
+ * reintroducing the fail-open admit. NOTE: `queryCadrePeers` / `queryPeerRecord` each
+ * issue TWO network reads (the revoked-stamp filter plus the row scan) and the budget is
+ * per read, so a membership read can spend up to two budgets back to back — still inside
+ * 2 s only because each stays well under it. Do not raise these numbers without
+ * re-checking against `ADMISSION_DECISION_TIMEOUT_MS`.
+ */
+export const CONTROL_READ_RETRY_BUDGET_MS = 1_500;
+
+/**
+ * The block-unavailability reasons a repeat read can actually improve on, matched against
+ * upstream's exact template `Block <id> is unavailable (<reason>): …`
+ * (`BlockUnavailableError`, `db-core/src/network/struct.ts`):
+ *
+ * - **`peers-unreachable`** — part of the cohort answered and part could not be asked, and
+ *   OTHER coordinators are reachable, so asking one of them can still settle it. Upstream's
+ *   own definition invites the second attempt.
+ * - **`cohort-unreachable`** — the marginal member of the set. Upstream says the answer
+ *   will not improve until this node's connectivity does — but during bring-up connectivity
+ *   DOES improve within a second (the boot-time failure shape recorded in
+ *   `tickets/blocked/control-read-over-fresh-edge-stream-resets`), and the cost of being
+ *   wrong is bounded by {@link CONTROL_READ_RETRY_BUDGET_MS}, which is the whole reason it
+ *   is safe to include.
+ *
+ * Deliberately NOT matched:
+ *
+ * - **`claimed-elsewhere`** — a cohort peer positively claims the block exists and nobody
+ *   could corroborate or acquire it. MEASURED not to clear: reissuing the same call every
+ *   second for 60 s returned the identical error every time (2026-08-20, recorded in the
+ *   fix ticket). Retrying spends the whole budget and fails anyway; the root cause is
+ *   upstream and tracked by `tickets/blocked/block-held-by-only-one-machine-is-unreadable`
+ *   — this retry must not paper over it.
+ * - **`unmaterializable`** — records are held locally but cannot be reassembled. A local
+ *   data problem; a second read reads the same records.
+ * - **`Block <id> may be stale`** (`BlockPossiblyStaleError`) — about currency, not
+ *   existence: a cohort claim no reachable coordinator could confirm or refute will not
+ *   resolve inside a few hundred milliseconds of backoff.
+ * - Anything unmatched. The default is no retry.
+ */
+const RETRIABLE_BLOCK_UNAVAILABLE =
+	/Block \S+ is unavailable \((?:peers-unreachable|cohort-unreachable)\)/;
+
+/** A block-unavailable read failure whose reason a repeat read can improve on. */
+function isRetriableBlockUnavailable(message: string): boolean {
+	return RETRIABLE_BLOCK_UNAVAILABLE.test(message);
+}
+
+/**
+ * Pacing overrides for {@link retryControlRead} — the read twin of
+ * `ControlWriteRetryOptions`. Every field defaults to the shipped read policy
+ * ({@link CONTROL_READ_ATTEMPTS}, {@link CONTROL_READ_RETRY_DELAYS_MS},
+ * {@link isRetriableControlReadFailure}); field shapes and semantics live on the shared
+ * {@link ControlRetryOptions}.
+ */
+export type ControlReadRetryOptions = ControlRetryOptions;
+
+/**
+ * Did this control read fail in a way a repeat read can improve on?
+ *
+ * Two classes, both meaning "the cluster could not be ASKED properly just now":
+ *
+ * - the transactor's read-phase aggregate (`Some peers did not complete: …[block:…]`),
+ *   reused verbatim from the write classifier — a `get` batch formats the single-block
+ *   `[block:` token, and a read can never produce the commit-phase `[blocks:` token, so
+ *   the write side's indeterminate-commit veto has nothing to veto here (a read commits
+ *   nothing and re-running it is always safe);
+ * - the block-unavailability reasons in {@link RETRIABLE_BLOCK_UNAVAILABLE} (and only
+ *   those — see its comment for the measured exclusions).
+ *
+ * Classifies by MESSAGE, walking the `cause` chain with the shared `chainMessages`,
+ * because only text survives the trip out of optimystic (see the module comment).
+ * Anything that is not an `Error` is never retried.
+ */
+export function isRetriableControlReadFailure(error: unknown): boolean {
+	if (!(error instanceof Error)) {
+		return false;
+	}
+	return chainMessages(error).some(message =>
+		isUncommittedTransactorAggregate(message) || isRetriableBlockUnavailable(message));
+}
+
+/**
+ * Run `attempt` — a full drain of one control read — up to
+ * {@link CONTROL_READ_ATTEMPTS} times, retrying only failures
+ * {@link isRetriableControlReadFailure} calls transient, and only while
+ * {@link CONTROL_READ_RETRY_BUDGET_MS} has not elapsed.
+ *
+ * Reads are idempotent, so `attempt` needs no atomicity contract — but it must be a
+ * COLLECTING drain, not a live iterator: a half-consumed iterator cannot be retried
+ * without re-yielding rows the caller already saw, which is why
+ * `ControlDatabase.readRows` materializes before this loop ever sees a failure.
+ *
+ * On exhaustion the LAST error is rethrown unchanged. Log lines carry the
+ * `Control read [<label>] …` prefix so a read's retries can be attributed among the
+ * several reads in flight concurrently in a real party.
+ */
+export function retryControlRead<T>(
+	attempt: () => Promise<T>,
+	options: ControlReadRetryOptions = {}
+): Promise<T> {
+	return retryControlOperation(attempt, {
+		attempts: CONTROL_READ_ATTEMPTS,
+		delaysMs: CONTROL_READ_RETRY_DELAYS_MS,
+		budgetMs: CONTROL_READ_RETRY_BUDGET_MS,
+		isRetriable: isRetriableControlReadFailure,
+		logPrefix: 'Control read',
+	}, options);
+}

package/src/control-retry.ts

@@ -0,0 +1,190 @@
+import debug from 'debug';
+import { unwrapError } from '@quereus/quereus';
+
+const log = debug('sereus:cadre:control-db');
+
+/**
+ * The bounded retry loop shared by the control plane's transient-failure policies:
+ * writes (`control-write-retry.ts`, behind `ControlDatabase.lockedWithRetry`) and reads
+ * (`control-read-retry.ts`, behind `ControlDatabase.readRows`). One loop, two policy
+ * modules — they differ in classifier, attempt count, backoff and elapsed budget, but the
+ * loop mechanics (jittered capped backoff, budget checked after a failed attempt BEFORE
+ * sleeping, last error rethrown unchanged) are deliberately identical so a fix to one
+ * cannot silently miss the other.
+ *
+ * The log prefix is part of the policy because the write funnel's lines are asserted
+ * byte-identically by `control-write-degraded-cohort-member.integration.ts` — extracting
+ * this loop must not move a single character of them.
+ */
+
+/** A retry policy: what to retry, how often, and under which elapsed-time ceiling. */
+export interface ControlRetryPolicy {
+	/** Total attempts, first included (floored at 1 by the loop). */
+	attempts: number;
+	/**
+	 * Backoff base before retry N (last entry repeats), jittered ±50% — so the jitter can
+	 * never floor a delay to zero — and each single sleep additionally capped at the
+	 * LARGEST delay in the list, so a `stop()` racing a backoff is never held up longer
+	 * than that. There is no `AbortSignal` at this seam to cancel a sleep outright; the
+	 * cap is the substitute, deliberately not plumbing one through every call path.
+	 */
+	delaysMs: readonly number[];
+	/**
+	 * Elapsed-time ceiling on the whole loop, measured from the START of the first attempt
+	 * and checked after a failed attempt BEFORE sleeping — so one slow attempt terminates
+	 * the loop rather than compounding, and retry adds ZERO latency to the case where it
+	 * cannot help.
+	 */
+	budgetMs: number;
+	/** Which failures are transient enough to re-present. */
+	isRetriable: (error: unknown) => boolean;
+	/**
+	 * Subject of every log line the loop emits (`<prefix>[ [<label>]] …`). Defaults to
+	 * `'Control write'`, which keeps the write funnel's lines byte-identical to what it
+	 * logged before this loop was shared out of `control-write-retry.ts`.
+	 */
+	logPrefix?: string;
+}
+
+/**
+ * Per-call overrides applied over a policy. Production callers pass at most `label`
+ * (usually via the policy module's wrapper); specs inject a recorded `sleep`, a shorter
+ * delay list, or a fake clock so no test ever waits out a real backoff.
+ */
+export interface ControlRetryOptions {
+	/**
+	 * Total attempts, first included. Default: the policy's, floored at 1 — below that
+	 * the loop would run the body zero times and rethrow a `lastError` nobody set.
+	 */
+	attempts?: number;
+	/** Backoff base before retry N (last entry repeats). Default: the policy's. */
+	delaysMs?: readonly number[];
+	/** Which failures are transient enough to re-present. Default: the policy's classifier. */
+	isRetriable?: (error: unknown) => boolean;
+	/**
+	 * Operation label stamped into every log line this loop emits (rendered as
+	 * `<prefix> [<label>] …`), and NOTHING else — no behavioural effect. The debug log is
+	 * the only surface where the loop's decisions are observable (the rethrown error is
+	 * unchanged and no attempt counter is exposed), and several operations retry
+	 * CONCURRENTLY in a real party, so an unlabelled line cannot be attributed. The
+	 * degraded-cohort scenario asserts on the write funnel's lines per-operation.
+	 */
+	label?: string;
+	/** The sleep primitive. Default: `setTimeout`. */
+	sleep?: (ms: number) => Promise<void>;
+	/** Clock for the elapsed-budget check. Default: `Date.now`. */
+	now?: () => number;
+}
+
+/**
+ * Run `attempt` up to the policy's attempt count, retrying only failures its classifier
+ * calls transient, and only while its elapsed budget has not run out.
+ *
+ * `attempt` must be safe to re-run — for writes that means atomic (a failed cluster write
+ * rolls back, nothing half-applies; the contract on `ControlDatabase.withWriteLock`), for
+ * reads it holds trivially. The backoff sleeps happen with NO lock held; a caller that
+ * locks takes and releases its lock inside `attempt`.
+ *
+ * On exhaustion (attempts or budget) the LAST error is rethrown unchanged — never
+ * wrapped, so the exact messages downstream code and the integration scenarios assert on
+ * survive. A non-retriable failure propagates from the attempt that raised it.
+ */
+export async function retryControlOperation<T>(
+	attempt: () => Promise<T>,
+	policy: ControlRetryPolicy,
+	options: ControlRetryOptions = {}
+): Promise<T> {
+	const attempts = Math.max(1, options.attempts ?? policy.attempts);
+	const delays = options.delaysMs ?? policy.delaysMs;
+	const isRetriable = options.isRetriable ?? policy.isRetriable;
+	const sleep = options.sleep ?? defaultSleep;
+	const now = options.now ?? Date.now;
+	const prefix = policy.logPrefix ?? 'Control write';
+	// Empty when unlabelled, so an unlabelled line is byte-identical to what the write
+	// loop logged before labels existed.
+	const tag = options.label ? ` [${options.label}]` : '';
+	const start = now();
+	let lastError: unknown;
+	let attemptsMade = 0;
+	for (let attemptNumber = 1; attemptNumber <= attempts; attemptNumber++) {
+		attemptsMade = attemptNumber;
+		try {
+			const result = await attempt();
+			if (attemptNumber > 1) {
+				log('%s%s committed on attempt %d/%d', prefix, tag, attemptNumber, attempts);
+			}
+			return result;
+		} catch (error) {
+			if (!isRetriable(error)) {
+				// The ONLY trace that this funnel saw a failure and declined it. Without it the
+				// log is silent either way, so "the classifier vetoed this one" is
+				// indistinguishable from "the retry is not wired into this path at all".
+				log('%s%s failed non-transiently on attempt %d/%d, not retried here: %s',
+					prefix, tag, attemptNumber, attempts, error);
+				throw error;
+			}
+			lastError = error;
+			if (attemptNumber === attempts) {
+				break;
+			}
+			const elapsed = now() - start;
+			if (elapsed >= policy.budgetMs) {
+				break;
+			}
+			const delay = jitteredDelay(delays, attemptNumber);
+			log('%s%s failed transiently (attempt %d/%d), retrying in %d ms: %s',
+				prefix, tag, attemptNumber, attempts, delay, error);
+			await sleep(delay);
+		}
+	}
+	log('%s%s failed after %d/%d attempt(s): %s', prefix, tag, attemptsMade, attempts, lastError);
+	throw lastError;
+}
+
+/**
+ * Every message in the failure's `cause` chain — the shared substrate both classifiers
+ * match against, since the failures they care about are not recognisable by type (the
+ * typed errors are destroyed on the way out of optimystic; only text survives).
+ *
+ * `unwrapError` declares its `message` as `string`, but it follows `.cause` without checking
+ * what that holds — a chain link that is not an `Error` (a stream rejected with a bare string
+ * reason, an `AbortSignal.reason` that is a plain object) yields `undefined` there, and
+ * calling `.includes` on it would throw a `TypeError` out of a classifier, INSIDE
+ * {@link retryControlOperation}'s catch, replacing the real failure with a confusing one.
+ * Non-strings are dropped: a link nobody can read is a link that matches nothing, which is
+ * already the conservative answer.
+ *
+ * NOTE: a cause chain with a CYCLE would spin forever inside `unwrapError` itself. No error
+ * in this repo builds one; if a hang ever localises to a control retry path, look there.
+ */
+export function chainMessages(error: Error): string[] {
+	return unwrapError(error)
+		.map(({ message }) => message)
+		.filter(message => typeof message === 'string');
+}
+
+/**
+ * Backoff for the retry that follows attempt `attemptNumber`: the matching base delay
+ * (last entry repeats), jittered ±50%, then capped at the list's largest base so no single
+ * sleep exceeds it (see {@link ControlRetryPolicy.delaysMs} for why). `Math.random` is
+ * fine here — the jitter only de-synchronizes concurrent retriers, nothing is derived
+ * from it.
+ *
+ * NOTE: the cap bites on the LAST delay, whose base IS the largest, so ~half of those sleeps
+ * land exactly on the cap rather than spread — de-synchronization is only partial there.
+ * Harmless while retriers are a handful of party nodes whose attempts already start seconds
+ * apart; if a party ever retries in a tight synchronized herd, raise the cap above the
+ * largest base instead of jittering about it.
+ */
+function jitteredDelay(delays: readonly number[], attemptNumber: number): number {
+	if (delays.length === 0) {
+		return 0;
+	}
+	const base = delays[Math.min(attemptNumber - 1, delays.length - 1)]!;
+	const jittered = base * (0.5 + Math.random());
+	return Math.min(jittered, Math.max(...delays));
+}
+
+function defaultSleep(ms: number): Promise<void> {
+	return new Promise(resolve => setTimeout(resolve, ms));
+}

package/src/control-schema.ts

@@ -142,6 +142,17 @@ declare schema CadreControl {
         Id text primary key,    -- UUID
         MemberPrivateKey text null unique,   -- Our private key as a member of this strand
         Type text, -- Types: 'o' = Open, 'c' = Closed -- Open can still control writes in the sApp, but only Closed controls reads
+        FounderOwnerKey text null,  -- ed25519 (base64url) owner key of the MACHINE that published this row
+                                    -- (== insert context.OwnerKey; in the reference model each machine's
+                                    -- owner key is the key behind its PeerId, so this names the founding
+                                    -- machine — the one machine that runs the strand's one-time founder
+                                    -- bootstrap). cadre-core derives "am I the founder?" at launch by
+                                    -- comparing it to the node's own owner key (cadre-node.ts
+                                    -- launchStrand). Null on a consent-seated strand — the consent branch
+                                    -- of AuthorizedInsert carries no signature, so there is no
+                                    -- trustworthy signer to record (see the NOTE on that branch below).
+                                    -- Provenance, not content: cadre-node.ts strandRowMismatches
+                                    -- deliberately excludes it from the identical-content comparison.
         StampId text not null unique,   -- single-use authorization nonce (anti-replay).
                                         -- \`unique\` holds over LIVE rows only; a removed row's stamp is retired
                                         -- permanently into Revocation (NotRevoked below).
@@ -172,8 +183,16 @@ declare schema CadreControl {
         -- Mirrors OwnerKey.NoUpdate / FormationInvite.Immutable.
         constraint NoUpdate check on update (false),
         constraint AuthorizedInsert check on insert (
-            -- Authorized by an owner signing over THIS row (Id, Type, MemberPrivateKey, StampId); single-use via unique StampId
-            exists (select 1 from OwnerKey A where A.Key = context.OwnerKey and verify(digest('CadreControl.Strand', 'add', new.Id, new.Type, coalesce(new.MemberPrivateKey, ''), new.StampId), context.Signature, A.Key, 'ed25519'))
+            -- Authorized by an owner signing over THIS row (Id, Type, MemberPrivateKey, StampId); single-use via unique StampId.
+            -- FounderOwnerKey persists WHO published the row, so the founding machine can later
+            -- recognise its own strand (founder derivation, cadre-node.ts launchStrand). The
+            -- equality pins the stored column to the VERIFIED signer — same shape as
+            -- CadrePeer.VouchOwner — so a writer cannot record a founder it is not. Deliberately
+            -- NOT added to the signed digest: the equality already binds it to the signature's
+            -- context.OwnerKey, and widening the digest would churn every existing
+            -- insertStrand field-order contract for nothing.
+            (exists (select 1 from OwnerKey A where A.Key = context.OwnerKey and verify(digest('CadreControl.Strand', 'add', new.Id, new.Type, coalesce(new.MemberPrivateKey, ''), new.StampId), context.Signature, A.Key, 'ed25519'))
+                and new.FounderOwnerKey = context.OwnerKey)
 
                 -- or authorized WITHOUT a signature by a redemption record naming THIS EXACT ROW: a
                 -- FormationUsage row carrying both this strand's Id and its one-off StampId
@@ -240,6 +259,15 @@ declare schema CadreControl {
                 or (
                     new.Type = 'o'
                     and new.MemberPrivateKey is null
+                    -- NOTE: a consent-seated strand records NO founder machine — this branch
+                    -- carries no signature, so there is no trustworthy signer to persist.
+                    -- Consequence: the restart-before-founding orphan shape (a relaunch cannot
+                    -- derive founder-ness from the row) persists for consent-formed strands
+                    -- only; the responder that provisions one must keep passing an explicit
+                    -- founder flag at launch. Revisit if consent-formed strands ever hit the
+                    -- empty-Header symptom in practice: FormationUsage names the redeeming
+                    -- peer (PeerKey), which is the candidate derivation source.
+                    and new.FounderOwnerKey is null
                     and exists (
                         select 1 from FormationUsage FU
                             where FU.StrandId = new.Id
@@ -487,7 +515,7 @@ declare schema CadreControl {
         ExpiresAt datetime null,
         TotalUses int null check (TotalUses >= 0),
         ValidationUrl text null,   -- Web hook - send disclosure, IP address...
-        StrandId text null,   -- Host strand this invite binds to (provision-then-record). Non-null => the responder records consent against this pre-existing strand; null => legacy responder-provisions path.
+        StrandId text null,   -- Host strand this invite binds to (provision-then-record). Non-null => the responder records consent against this pre-existing strand; null => unbound invite: the responder provisions a fresh open strand and atomically records its one consent row.
         StampId text not null unique,   -- single-use authorization nonce (anti-replay)
         constraint AuthorizedInsert check on insert (
             -- Authorized by an owner signing over THIS row
@@ -676,9 +704,25 @@ declare schema CadreControl {
 
     -- Token is no longer a primary-key prefix (the key is the redemption nonce), so the
     -- per-token filters — the cap count in Authorized above, and cadre-core's
-    -- countFormationUsage / isTokenUsed / hasOutstandingFormationInvite — get an index
-    -- rather than a table scan.
-    index FormationUsageByToken on FormationUsage (Token);
+    -- countFormationUsage / isTokenUsed / hasOutstandingFormationInvite — are served by a
+    -- table scan: a bare equality on a non-key column is a shape the optimystic vtab declines
+    -- to claim, so the engine applies the filter itself.
+    --
+    -- There WAS an \`index FormationUsageByToken on FormationUsage (Token)\` here, declared by
+    -- \`formation-unique-token-redesign\` purely to spare that scan. It was removed 2026-08-25
+    -- because the tripwire that change recorded ("an under-reporting descent would admit a
+    -- seat the invitation did not pay for — a failure mode a scan could not produce") fired:
+    -- a secondary-index descent on a second machine returns only the rows THAT machine wrote,
+    -- so a cap counted through it under-reports without bound across machines. Measured with
+    -- the engine's own \`index:seek\` trace — each machine's copy of the index sub-collection
+    -- sits at a different revision while their copies of the table itself agree, and the
+    -- refresh that runs immediately before the descent does not close the gap. Tracked in
+    -- \`tickets/blocked/secondary-index-seek-blind-to-sibling-rows.md\`, and upstream as
+    -- \`bug-index-subcollection-sits-one-revision-behind-on-the-sibling\`.
+    --
+    -- DO NOT re-declare it to speed the count up until that is fixed upstream. Re-adding this
+    -- one line is also how the reproducer comes back: it is what makes
+    -- \`strand-formation-concurrent-redemption\` fail.
 
     -- Append-only retirement record for the one-off StampId nonces of removed OwnerKey /
     -- CadrePeer / ValidationKey / Strand rows. Without it a removal was undoable: the add

package/src/control-write-retry.ts

@@ -1,7 +1,5 @@
-import debug from 'debug';
-import { unwrapError } from '@quereus/quereus';
-
-const log = debug('sereus:cadre:control-db');
+import { chainMessages, retryControlOperation } from './control-retry.js';
+import type { ControlRetryOptions } from './control-retry.js';
 
 /**
  * Bounded retry for TRANSIENT control-write failures — the classifier and loop behind
@@ -29,7 +27,9 @@ const log = debug('sereus:cadre:control-db');
  * list must be safe for all ~19 callers.
  *
  * Lives outside `control-database.ts` so a spec can drive the classifier against real engine
- * errors without importing the whole database class.
+ * errors without importing the whole database class. The loop itself lives in
+ * `control-retry.ts`, shared with the READ policy (`control-read-retry.ts`) — this module
+ * owns the write policies: their classifiers, attempt counts, backoff and budget.
  */
 
 /**
@@ -167,8 +167,12 @@ const SUPER_MAJORITY_SHORTFALL_UNANSWERED =
  * An aggregate whose details came out EMPTY (possible when `formatBatchStatuses` has no
  * batches to format) carries neither token, matches nothing here, and is not retried — an
  * unattributable failure is not a proven non-commit.
+ *
+ * Exported for the READ classifier (`control-read-retry.ts`), which reuses it verbatim: a
+ * `get` batch formats the same single-block `[block:` token, and a read can never produce
+ * the commit-phase `[blocks:` shape at all.
  */
-function isUncommittedTransactorAggregate(message: string): boolean {
+export function isUncommittedTransactorAggregate(message: string): boolean {
 	return TRANSACTOR_AGGREGATE.test(message) && message.includes(SINGLE_BLOCK_BATCH_TOKEN);
 }
 
@@ -221,25 +225,6 @@ function isSelfCoordinationGraceRefusal(message: string): boolean {
 	return SELF_COORDINATION_GRACE_REFUSAL.test(message);
 }
 
-/**
- * Every message in the failure's `cause` chain.
- *
- * `unwrapError` declares its `message` as `string`, but it follows `.cause` without checking what
- * that holds — a chain link that is not an `Error` (a stream rejected with a bare string reason,
- * an `AbortSignal.reason` that is a plain object) yields `undefined` there, and calling
- * `.includes` on it would throw a `TypeError` out of this classifier, INSIDE `retryControlWrite`'s
- * catch, replacing the real control-write failure with a confusing one. Non-strings are dropped:
- * a link nobody can read is a link that matches nothing, which is already the conservative answer.
- *
- * NOTE: a cause chain with a CYCLE would spin forever inside `unwrapError` itself. No error in
- * this repo builds one; if a hang ever localises to a control-write failure path, look there.
- */
-function chainMessages(error: Error): string[] {
-	return unwrapError(error)
-		.map(({ message }) => message)
-		.filter(message => typeof message === 'string');
-}
-
 /**
  * The error messages that mean "the cohort did not answer AND nothing committed", and nothing
  * else. The observed one-shot failure was `registerSelf()` racing a connection still forming:
@@ -282,7 +267,8 @@ const RETRIABLE_SCHEMA_INIT_MATCHERS: readonly ((message: string) => boolean)[]
  * Did this control write fail because the cluster cohort did not ANSWER — i.e. is
  * re-presenting the SAME signed write a moment later the right response?
  *
- * Classifies by MESSAGE, walking the `cause` chain with Quereus' own {@link unwrapError} —
+ * Classifies by MESSAGE, walking the `cause` chain with the shared {@link chainMessages}
+ * (Quereus' own `unwrapError` underneath) —
  * the failure surface is not recognisable by type. The transactor and the coordinator both throw bare `Error`s, and by
  * the time one reaches `ControlDatabase` it is wrapped in a `QuereusError` (the real chain
  * is `QuereusError` → `Error` → `Error`), so the match must work at any depth. Anything
@@ -336,41 +322,16 @@ function matchesRetriableMessage(
 }
 
 /**
- * Policy and pacing for {@link retryControlWrite}. Production callers pass nothing (or a named
- * policy such as {@link SCHEMA_INIT_RETRY_POLICY}); specs inject a recorded `sleep`, a shorter
- * delay list, or a fake clock so no test ever waits out a real backoff.
+ * Policy and pacing overrides for {@link retryControlWrite}. Production callers pass nothing
+ * (or a named policy such as {@link SCHEMA_INIT_RETRY_POLICY}); specs inject a recorded
+ * `sleep`, a shorter delay list, or a fake clock so no test ever waits out a real backoff.
  *
- * Every field defaults to the shipped control-write policy, so an omitted field is byte-identical
- * to the behaviour before per-call-site policies existed.
+ * Every field defaults to the shipped control-write policy ({@link CONTROL_WRITE_ATTEMPTS},
+ * {@link CONTROL_WRITE_RETRY_DELAYS_MS}, {@link isRetriableControlWriteFailure}), so an
+ * omitted field is byte-identical to the behaviour before per-call-site policies existed.
+ * The field shapes and semantics live on the shared {@link ControlRetryOptions}.
  */
-export interface ControlWriteRetryOptions {
-	/**
-	 * Total attempts, first included. Default {@link CONTROL_WRITE_ATTEMPTS}, floored at 1 —
-	 * below that the loop would run the body zero times and rethrow a `lastError` nobody set.
-	 */
-	attempts?: number;
-	/** Backoff base before retry N (last entry repeats). Default {@link CONTROL_WRITE_RETRY_DELAYS_MS}. */
-	delaysMs?: readonly number[];
-	/**
-	 * Which failures are transient enough to re-present. Default
-	 * {@link isRetriableControlWriteFailure}; the only other shipped value is
-	 * {@link isRetriableSchemaInitFailure}, via {@link SCHEMA_INIT_RETRY_POLICY}.
-	 */
-	isRetriable?: (error: unknown) => boolean;
-	/**
-	 * Operation label stamped into every log line this loop emits (rendered as
-	 * `Control write [<label>] …`), and NOTHING else — no behavioural effect. The debug
-	 * log is the only surface where the loop's decisions are observable (the rethrown
-	 * error is unchanged and no attempt counter is exposed), and several writes retry
-	 * CONCURRENTLY in a real party, so an unlabelled line cannot be attributed to a
-	 * write. The degraded-cohort scenario asserts on these lines per-operation.
-	 */
-	label?: string;
-	/** The sleep primitive. Default: `setTimeout`. */
-	sleep?: (ms: number) => Promise<void>;
-	/** Clock for the elapsed-budget check. Default: `Date.now`. */
-	now?: () => number;
-}
+export type ControlWriteRetryOptions = ControlRetryOptions;
 
 /**
  * The one non-default retry policy: `ControlDatabase.loadSchema`'s distributed DDL, which
@@ -406,78 +367,17 @@ export const SCHEMA_INIT_RETRY_POLICY: Readonly<ControlWriteRetryOptions> = {
  * so the exact messages downstream code and the degraded-cohort-member scenario assert on
  * survive. A non-retriable failure propagates from the attempt that raised it.
  */
-export async function retryControlWrite<T>(
+export function retryControlWrite<T>(
 	attempt: () => Promise<T>,
 	options: ControlWriteRetryOptions = {}
 ): Promise<T> {
-	const attempts = Math.max(1, options.attempts ?? CONTROL_WRITE_ATTEMPTS);
-	const delays = options.delaysMs ?? CONTROL_WRITE_RETRY_DELAYS_MS;
-	const isRetriable = options.isRetriable ?? isRetriableControlWriteFailure;
-	const sleep = options.sleep ?? defaultSleep;
-	const now = options.now ?? Date.now;
-	// Empty when unlabelled, so an unlabelled line is byte-identical to what this loop
-	// logged before labels existed.
-	const tag = options.label ? ` [${options.label}]` : '';
-	const start = now();
-	let lastError: unknown;
-	let attemptsMade = 0;
-	for (let attemptNumber = 1; attemptNumber <= attempts; attemptNumber++) {
-		attemptsMade = attemptNumber;
-		try {
-			const result = await attempt();
-			if (attemptNumber > 1) {
-				log('Control write%s committed on attempt %d/%d', tag, attemptNumber, attempts);
-			}
-			return result;
-		} catch (error) {
-			if (!isRetriable(error)) {
-				// The ONLY trace that this funnel saw a failure and declined it. Without it the
-				// log is silent either way, so "the classifier vetoed this one" is
-				// indistinguishable from "the retry is not wired into this path at all".
-				log('Control write%s failed non-transiently on attempt %d/%d, not retried here: %s',
-					tag, attemptNumber, attempts, error);
-				throw error;
-			}
-			lastError = error;
-			if (attemptNumber === attempts) {
-				break;
-			}
-			const elapsed = now() - start;
-			if (elapsed >= CONTROL_WRITE_RETRY_BUDGET_MS) {
-				break;
-			}
-			const delay = jitteredDelay(delays, attemptNumber);
-			log('Control write%s failed transiently (attempt %d/%d), retrying in %d ms: %s',
-				tag, attemptNumber, attempts, delay, error);
-			await sleep(delay);
-		}
-	}
-	log('Control write%s failed after %d/%d attempt(s): %s', tag, attemptsMade, attempts, lastError);
-	throw lastError;
-}
-
-/**
- * Backoff for the retry that follows attempt `attemptNumber`: the matching base delay
- * (last entry repeats), jittered ±50%, then capped at the list's largest base so no single
- * sleep exceeds it (see {@link CONTROL_WRITE_RETRY_DELAYS_MS} for why). `Math.random` is
- * fine here — the jitter only de-synchronizes concurrent retriers, nothing is derived
- * from it.
- *
- * NOTE: the cap bites on the LAST delay, whose base IS the largest, so ~half of those sleeps
- * land exactly on the cap rather than spread — de-synchronization is only partial there.
- * Harmless while retriers are a handful of party nodes whose attempts already start seconds
- * apart; if a party ever retries in a tight synchronized herd, raise the cap above the
- * largest base instead of jittering about it.
- */
-function jitteredDelay(delays: readonly number[], attemptNumber: number): number {
-	if (delays.length === 0) {
-		return 0;
-	}
-	const base = delays[Math.min(attemptNumber - 1, delays.length - 1)]!;
-	const jittered = base * (0.5 + Math.random());
-	return Math.min(jittered, Math.max(...delays));
-}
-
-function defaultSleep(ms: number): Promise<void> {
-	return new Promise(resolve => setTimeout(resolve, ms));
+	return retryControlOperation(attempt, {
+		attempts: CONTROL_WRITE_ATTEMPTS,
+		delaysMs: CONTROL_WRITE_RETRY_DELAYS_MS,
+		budgetMs: CONTROL_WRITE_RETRY_BUDGET_MS,
+		isRetriable: isRetriableControlWriteFailure,
+		// The default prefix, named anyway: the degraded-cohort scenario asserts on
+		// `Control write [<label>] …` lines byte-for-byte.
+		logPrefix: 'Control write',
+	}, options);
 }