@serfab/cadre-core 0.10.0 → 0.12.0

package/src/control-read-retry.ts

@@ -0,0 +1,167 @@
+import { chainMessages, retryControlOperation } from './control-retry.js';
+import type { ControlRetryOptions } from './control-retry.js';
+import { isUncommittedTransactorAggregate } from './control-write-retry.js';
+
+/**
+ * Bounded retry for TRANSIENT control-read failures — the classifier and policy behind
+ * `ControlDatabase.readRows`, the single funnel every control read passes through.
+ *
+ * Why this exists: control WRITES have absorbed transient cluster failures since
+ * `control-write-retry.ts` landed, but a read had no second chance — one stream reset
+ * during a scan ended the read outright (measured: a rigged single `eval` failure killed
+ * `queryRevokedStamps` after exactly one attempt while the same injection against `exec`
+ * was absorbed). This module gives reads the same bounded re-presentation with a SHORTER
+ * deadline and a NARROWER failure set, because the tightest caller deadline over a control
+ * read is the inbound admission gate's 2 s fail-open timeout
+ * (`ADMISSION_DECISION_TIMEOUT_MS`, `membership-connection-gater.ts`) — a read budget that
+ * does not fit inside it with headroom spends its retries after the gate has already
+ * admitted.
+ *
+ * This is deliberately NOT "the write classifier minus its commit veto" — the retriable
+ * set differs in both directions (see {@link isRetriableControlReadFailure}). The loop
+ * lives in `control-retry.ts`, shared with the write policy; this module owns only the
+ * read policy. And like the write side, classification is by TEXT: optimystic's
+ * `OptimysticVirtualTable` catches every scan-path error and rethrows
+ * `new Error('Query failed: ' + message)` with NO `cause`
+ * (`quereus-plugin-optimystic/src/optimystic-module.ts`), so the typed
+ * `BlockUnavailableError` and its `reason` field are destroyed before they reach this
+ * repo; Quereus then wraps that as `Error during query on table '<T>': …` preserving
+ * `cause` (`quereus/src/runtime/emit/scan.ts`). Every matcher fails CLOSED — an upstream
+ * rewording stops the retry engaging, it never makes it unsafe.
+ */
+
+/**
+ * Attempts allowed per control read: the first, plus two retries. Attempts are cheap here —
+ * the observed transient failure (the transactor's read-phase aggregate off a stream still
+ * forming) surfaces in ~25 ms — and {@link CONTROL_READ_RETRY_BUDGET_MS} is what terminates
+ * the loop when an attempt is slow instead.
+ */
+export const CONTROL_READ_ATTEMPTS = 3;
+
+/**
+ * Backoff before read retry N, jittered ±50% and capped at the largest base by the shared
+ * loop — so worst case 150 ms + 400 ms = 550 ms of total sleep (the second delay's
+ * upside jitter is clipped by the cap). Much shorter than the write list because the
+ * whole loop must fit under {@link CONTROL_READ_RETRY_BUDGET_MS}; the spec pins that
+ * relationship rather than the number.
+ */
+export const CONTROL_READ_RETRY_DELAYS_MS: readonly number[] = [100, 400];
+
+/**
+ * Elapsed-time ceiling on the whole read-retry loop, checked after a failed attempt BEFORE
+ * sleeping — so one slow attempt (e.g. a `cohort-unreachable` read that burned the
+ * transactor's own deadline) terminates the loop rather than compounding.
+ *
+ * 1500 ms, sized against `ADMISSION_DECISION_TIMEOUT_MS` (2000 ms): the inbound admission
+ * gate reads the control DB and is FAIL-OPEN on both throw and timeout, so a read that
+ * outlives the gate's deadline spends its retries after the gate has already admitted an
+ * unplaced peer. Under the deadline, a read that succeeds on its second attempt lets the
+ * gate make the real decision instead. `control-read-retry.spec.ts` asserts the
+ * relationship so a future edit to either constant reddens instead of silently
+ * reintroducing the fail-open admit. NOTE: `queryCadrePeers` / `queryPeerRecord` each
+ * issue TWO network reads (the revoked-stamp filter plus the row scan) and the budget is
+ * per read, so a membership read can spend up to two budgets back to back — still inside
+ * 2 s only because each stays well under it. Do not raise these numbers without
+ * re-checking against `ADMISSION_DECISION_TIMEOUT_MS`.
+ */
+export const CONTROL_READ_RETRY_BUDGET_MS = 1_500;
+
+/**
+ * The block-unavailability reasons a repeat read can actually improve on, matched against
+ * upstream's exact template `Block <id> is unavailable (<reason>): …`
+ * (`BlockUnavailableError`, `db-core/src/network/struct.ts`):
+ *
+ * - **`peers-unreachable`** — part of the cohort answered and part could not be asked, and
+ *   OTHER coordinators are reachable, so asking one of them can still settle it. Upstream's
+ *   own definition invites the second attempt.
+ * - **`cohort-unreachable`** — the marginal member of the set. Upstream says the answer
+ *   will not improve until this node's connectivity does — but during bring-up connectivity
+ *   DOES improve within a second (the boot-time failure shape recorded in
+ *   `tickets/blocked/control-read-over-fresh-edge-stream-resets`), and the cost of being
+ *   wrong is bounded by {@link CONTROL_READ_RETRY_BUDGET_MS}, which is the whole reason it
+ *   is safe to include.
+ *
+ * Deliberately NOT matched:
+ *
+ * - **`claimed-elsewhere`** — a cohort peer positively claims the block exists and nobody
+ *   could corroborate or acquire it. MEASURED not to clear: reissuing the same call every
+ *   second for 60 s returned the identical error every time (2026-08-20, recorded in the
+ *   fix ticket). Retrying spends the whole budget and fails anyway; the root cause is
+ *   upstream and tracked by `tickets/blocked/block-held-by-only-one-machine-is-unreadable`
+ *   — this retry must not paper over it.
+ * - **`unmaterializable`** — records are held locally but cannot be reassembled. A local
+ *   data problem; a second read reads the same records.
+ * - **`Block <id> may be stale`** (`BlockPossiblyStaleError`) — about currency, not
+ *   existence: a cohort claim no reachable coordinator could confirm or refute will not
+ *   resolve inside a few hundred milliseconds of backoff.
+ * - Anything unmatched. The default is no retry.
+ */
+const RETRIABLE_BLOCK_UNAVAILABLE =
+	/Block \S+ is unavailable \((?:peers-unreachable|cohort-unreachable)\)/;
+
+/** A block-unavailable read failure whose reason a repeat read can improve on. */
+function isRetriableBlockUnavailable(message: string): boolean {
+	return RETRIABLE_BLOCK_UNAVAILABLE.test(message);
+}
+
+/**
+ * Pacing overrides for {@link retryControlRead} — the read twin of
+ * `ControlWriteRetryOptions`. Every field defaults to the shipped read policy
+ * ({@link CONTROL_READ_ATTEMPTS}, {@link CONTROL_READ_RETRY_DELAYS_MS},
+ * {@link isRetriableControlReadFailure}); field shapes and semantics live on the shared
+ * {@link ControlRetryOptions}.
+ */
+export type ControlReadRetryOptions = ControlRetryOptions;
+
+/**
+ * Did this control read fail in a way a repeat read can improve on?
+ *
+ * Two classes, both meaning "the cluster could not be ASKED properly just now":
+ *
+ * - the transactor's read-phase aggregate (`Some peers did not complete: …[block:…]`),
+ *   reused verbatim from the write classifier — a `get` batch formats the single-block
+ *   `[block:` token, and a read can never produce the commit-phase `[blocks:` token, so
+ *   the write side's indeterminate-commit veto has nothing to veto here (a read commits
+ *   nothing and re-running it is always safe);
+ * - the block-unavailability reasons in {@link RETRIABLE_BLOCK_UNAVAILABLE} (and only
+ *   those — see its comment for the measured exclusions).
+ *
+ * Classifies by MESSAGE, walking the `cause` chain with the shared `chainMessages`,
+ * because only text survives the trip out of optimystic (see the module comment).
+ * Anything that is not an `Error` is never retried.
+ */
+export function isRetriableControlReadFailure(error: unknown): boolean {
+	if (!(error instanceof Error)) {
+		return false;
+	}
+	return chainMessages(error).some(message =>
+		isUncommittedTransactorAggregate(message) || isRetriableBlockUnavailable(message));
+}
+
+/**
+ * Run `attempt` — a full drain of one control read — up to
+ * {@link CONTROL_READ_ATTEMPTS} times, retrying only failures
+ * {@link isRetriableControlReadFailure} calls transient, and only while
+ * {@link CONTROL_READ_RETRY_BUDGET_MS} has not elapsed.
+ *
+ * Reads are idempotent, so `attempt` needs no atomicity contract — but it must be a
+ * COLLECTING drain, not a live iterator: a half-consumed iterator cannot be retried
+ * without re-yielding rows the caller already saw, which is why
+ * `ControlDatabase.readRows` materializes before this loop ever sees a failure.
+ *
+ * On exhaustion the LAST error is rethrown unchanged. Log lines carry the
+ * `Control read [<label>] …` prefix so a read's retries can be attributed among the
+ * several reads in flight concurrently in a real party.
+ */
+export function retryControlRead<T>(
+	attempt: () => Promise<T>,
+	options: ControlReadRetryOptions = {}
+): Promise<T> {
+	return retryControlOperation(attempt, {
+		attempts: CONTROL_READ_ATTEMPTS,
+		delaysMs: CONTROL_READ_RETRY_DELAYS_MS,
+		budgetMs: CONTROL_READ_RETRY_BUDGET_MS,
+		isRetriable: isRetriableControlReadFailure,
+		logPrefix: 'Control read',
+	}, options);
+}

package/src/control-retry.ts

@@ -0,0 +1,190 @@
+import debug from 'debug';
+import { unwrapError } from '@quereus/quereus';
+
+const log = debug('sereus:cadre:control-db');
+
+/**
+ * The bounded retry loop shared by the control plane's transient-failure policies:
+ * writes (`control-write-retry.ts`, behind `ControlDatabase.lockedWithRetry`) and reads
+ * (`control-read-retry.ts`, behind `ControlDatabase.readRows`). One loop, two policy
+ * modules — they differ in classifier, attempt count, backoff and elapsed budget, but the
+ * loop mechanics (jittered capped backoff, budget checked after a failed attempt BEFORE
+ * sleeping, last error rethrown unchanged) are deliberately identical so a fix to one
+ * cannot silently miss the other.
+ *
+ * The log prefix is part of the policy because the write funnel's lines are asserted
+ * byte-identically by `control-write-degraded-cohort-member.integration.ts` — extracting
+ * this loop must not move a single character of them.
+ */
+
+/** A retry policy: what to retry, how often, and under which elapsed-time ceiling. */
+export interface ControlRetryPolicy {
+	/** Total attempts, first included (floored at 1 by the loop). */
+	attempts: number;
+	/**
+	 * Backoff base before retry N (last entry repeats), jittered ±50% — so the jitter can
+	 * never floor a delay to zero — and each single sleep additionally capped at the
+	 * LARGEST delay in the list, so a `stop()` racing a backoff is never held up longer
+	 * than that. There is no `AbortSignal` at this seam to cancel a sleep outright; the
+	 * cap is the substitute, deliberately not plumbing one through every call path.
+	 */
+	delaysMs: readonly number[];
+	/**
+	 * Elapsed-time ceiling on the whole loop, measured from the START of the first attempt
+	 * and checked after a failed attempt BEFORE sleeping — so one slow attempt terminates
+	 * the loop rather than compounding, and retry adds ZERO latency to the case where it
+	 * cannot help.
+	 */
+	budgetMs: number;
+	/** Which failures are transient enough to re-present. */
+	isRetriable: (error: unknown) => boolean;
+	/**
+	 * Subject of every log line the loop emits (`<prefix>[ [<label>]] …`). Defaults to
+	 * `'Control write'`, which keeps the write funnel's lines byte-identical to what it
+	 * logged before this loop was shared out of `control-write-retry.ts`.
+	 */
+	logPrefix?: string;
+}
+
+/**
+ * Per-call overrides applied over a policy. Production callers pass at most `label`
+ * (usually via the policy module's wrapper); specs inject a recorded `sleep`, a shorter
+ * delay list, or a fake clock so no test ever waits out a real backoff.
+ */
+export interface ControlRetryOptions {
+	/**
+	 * Total attempts, first included. Default: the policy's, floored at 1 — below that
+	 * the loop would run the body zero times and rethrow a `lastError` nobody set.
+	 */
+	attempts?: number;
+	/** Backoff base before retry N (last entry repeats). Default: the policy's. */
+	delaysMs?: readonly number[];
+	/** Which failures are transient enough to re-present. Default: the policy's classifier. */
+	isRetriable?: (error: unknown) => boolean;
+	/**
+	 * Operation label stamped into every log line this loop emits (rendered as
+	 * `<prefix> [<label>] …`), and NOTHING else — no behavioural effect. The debug log is
+	 * the only surface where the loop's decisions are observable (the rethrown error is
+	 * unchanged and no attempt counter is exposed), and several operations retry
+	 * CONCURRENTLY in a real party, so an unlabelled line cannot be attributed. The
+	 * degraded-cohort scenario asserts on the write funnel's lines per-operation.
+	 */
+	label?: string;
+	/** The sleep primitive. Default: `setTimeout`. */
+	sleep?: (ms: number) => Promise<void>;
+	/** Clock for the elapsed-budget check. Default: `Date.now`. */
+	now?: () => number;
+}
+
+/**
+ * Run `attempt` up to the policy's attempt count, retrying only failures its classifier
+ * calls transient, and only while its elapsed budget has not run out.
+ *
+ * `attempt` must be safe to re-run — for writes that means atomic (a failed cluster write
+ * rolls back, nothing half-applies; the contract on `ControlDatabase.withWriteLock`), for
+ * reads it holds trivially. The backoff sleeps happen with NO lock held; a caller that
+ * locks takes and releases its lock inside `attempt`.
+ *
+ * On exhaustion (attempts or budget) the LAST error is rethrown unchanged — never
+ * wrapped, so the exact messages downstream code and the integration scenarios assert on
+ * survive. A non-retriable failure propagates from the attempt that raised it.
+ */
+export async function retryControlOperation<T>(
+	attempt: () => Promise<T>,
+	policy: ControlRetryPolicy,
+	options: ControlRetryOptions = {}
+): Promise<T> {
+	const attempts = Math.max(1, options.attempts ?? policy.attempts);
+	const delays = options.delaysMs ?? policy.delaysMs;
+	const isRetriable = options.isRetriable ?? policy.isRetriable;
+	const sleep = options.sleep ?? defaultSleep;
+	const now = options.now ?? Date.now;
+	const prefix = policy.logPrefix ?? 'Control write';
+	// Empty when unlabelled, so an unlabelled line is byte-identical to what the write
+	// loop logged before labels existed.
+	const tag = options.label ? ` [${options.label}]` : '';
+	const start = now();
+	let lastError: unknown;
+	let attemptsMade = 0;
+	for (let attemptNumber = 1; attemptNumber <= attempts; attemptNumber++) {
+		attemptsMade = attemptNumber;
+		try {
+			const result = await attempt();
+			if (attemptNumber > 1) {
+				log('%s%s committed on attempt %d/%d', prefix, tag, attemptNumber, attempts);
+			}
+			return result;
+		} catch (error) {
+			if (!isRetriable(error)) {
+				// The ONLY trace that this funnel saw a failure and declined it. Without it the
+				// log is silent either way, so "the classifier vetoed this one" is
+				// indistinguishable from "the retry is not wired into this path at all".
+				log('%s%s failed non-transiently on attempt %d/%d, not retried here: %s',
+					prefix, tag, attemptNumber, attempts, error);
+				throw error;
+			}
+			lastError = error;
+			if (attemptNumber === attempts) {
+				break;
+			}
+			const elapsed = now() - start;
+			if (elapsed >= policy.budgetMs) {
+				break;
+			}
+			const delay = jitteredDelay(delays, attemptNumber);
+			log('%s%s failed transiently (attempt %d/%d), retrying in %d ms: %s',
+				prefix, tag, attemptNumber, attempts, delay, error);
+			await sleep(delay);
+		}
+	}
+	log('%s%s failed after %d/%d attempt(s): %s', prefix, tag, attemptsMade, attempts, lastError);
+	throw lastError;
+}
+
+/**
+ * Every message in the failure's `cause` chain — the shared substrate both classifiers
+ * match against, since the failures they care about are not recognisable by type (the
+ * typed errors are destroyed on the way out of optimystic; only text survives).
+ *
+ * `unwrapError` declares its `message` as `string`, but it follows `.cause` without checking
+ * what that holds — a chain link that is not an `Error` (a stream rejected with a bare string
+ * reason, an `AbortSignal.reason` that is a plain object) yields `undefined` there, and
+ * calling `.includes` on it would throw a `TypeError` out of a classifier, INSIDE
+ * {@link retryControlOperation}'s catch, replacing the real failure with a confusing one.
+ * Non-strings are dropped: a link nobody can read is a link that matches nothing, which is
+ * already the conservative answer.
+ *
+ * NOTE: a cause chain with a CYCLE would spin forever inside `unwrapError` itself. No error
+ * in this repo builds one; if a hang ever localises to a control retry path, look there.
+ */
+export function chainMessages(error: Error): string[] {
+	return unwrapError(error)
+		.map(({ message }) => message)
+		.filter(message => typeof message === 'string');
+}
+
+/**
+ * Backoff for the retry that follows attempt `attemptNumber`: the matching base delay
+ * (last entry repeats), jittered ±50%, then capped at the list's largest base so no single
+ * sleep exceeds it (see {@link ControlRetryPolicy.delaysMs} for why). `Math.random` is
+ * fine here — the jitter only de-synchronizes concurrent retriers, nothing is derived
+ * from it.
+ *
+ * NOTE: the cap bites on the LAST delay, whose base IS the largest, so ~half of those sleeps
+ * land exactly on the cap rather than spread — de-synchronization is only partial there.
+ * Harmless while retriers are a handful of party nodes whose attempts already start seconds
+ * apart; if a party ever retries in a tight synchronized herd, raise the cap above the
+ * largest base instead of jittering about it.
+ */
+function jitteredDelay(delays: readonly number[], attemptNumber: number): number {
+	if (delays.length === 0) {
+		return 0;
+	}
+	const base = delays[Math.min(attemptNumber - 1, delays.length - 1)]!;
+	const jittered = base * (0.5 + Math.random());
+	return Math.min(jittered, Math.max(...delays));
+}
+
+function defaultSleep(ms: number): Promise<void> {
+	return new Promise(resolve => setTimeout(resolve, ms));
+}