@optimystic/db-p2p 0.16.3 → 0.18.0

package/src/repo/coordinator-repo.ts

@@ -1,7 +1,6 @@
-import type { PendRequest, ActionBlocks, IRepo, MessageOptions, CommitResult, GetBlockResults, PendResult, BlockGets, CommitRequest, RepoMessage, IKeyNetwork, ICluster, ClusterConsensusConfig, BlockId, ActionRev, ActionContext, ClusterRecord } from "@optimystic/db-core";
+import type { PendRequest, ActionBlocks, IRepo, MessageOptions, CommitResult, GetBlockResults, PendResult, StaleFailure, BlockGets, CommitRequest, RepoMessage, IKeyNetwork, ICluster, ClusterConsensusConfig, BlockId, ActionRev, ActionContext, ClusterRecord } from "@optimystic/db-core";
 import { LruMap, blockIdsForTransforms, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
-import { ClusterCoordinator } from "./cluster-coordinator.js";
-import type { ClusterClient } from "../cluster/client.js";
+import { ClusterCoordinator, ValidatorRejectionError } from "./cluster-coordinator.js";
 import type { PeerId } from "@libp2p/interface";
 import { peerIdFromString } from "@libp2p/peer-id";
 import type { FretService } from "p2p-fret";
@@ -9,10 +8,74 @@ import { createLogger } from '../logger.js';
 import type { IPeerReputation } from "../reputation/types.js";
 import { PenaltyReason } from "../reputation/types.js";
 import type { ITransactionStateStore } from "../cluster/i-transaction-state-store.js";
-import { selectQuorumRev, type RevClaim, type QuorumRev } from "../cluster/quorum-restore.js";
+import { quorumSize, corroboratorCapacity, selectQuorumRev, type RevClaim, type QuorumRev } from "../cluster/quorum-restore.js";
+import { RECONCILE_TIMEOUT_MS } from "../cluster/reconcile-block.js";
+import { isMissingBaseRevisionFailure, MISSING_BASE_REVISION_REASON } from "../storage/storage-repo.js";
+import type { ReconcileBlockCallback } from "../cluster/cluster-repo.js";
 
 const log = createLogger('coordinator-repo');
 
+/**
+ * Acquire a block's content for a cohort-corroborated revision, from the cohort, and persist it.
+ *
+ * Deliberately the SAME shape as the commit path's {@link ReconcileBlockCallback}, and in the live
+ * node the very same instance (`libp2p-node-base` passes its `reconcileBlock` to both): read-driven
+ * acquisition needs exactly what reconcile already provides — a per-peer-bounded archive fetch, a
+ * quorum vote on the target `(rev, actionId)`, a quorum vote on the *content* at that revision, and a
+ * persist through the monotonic, commit-latched `StorageRepo.saveReplicatedBlock` funnel. Reusing it
+ * is what keeps read-repair from being a weaker trust path than reconcile.
+ */
+export type AcquireBlockCallback = ReconcileBlockCallback;
+
+/** True when a freshly-read local revision is strictly ahead of the baseline the repair started from. */
+function isAdvanceOver(rev: number | undefined, baselineRev: number | undefined): boolean {
+	return typeof rev === 'number' && (baselineRev === undefined || rev > baselineRev);
+}
+
+/**
+ * Reject if `promise` has not settled within `ms`. The timer is cleared on either outcome, so no
+ * handle outlives the race (hence no `unref`, which does not exist off Node).
+ */
+function withDeadline<T>(promise: Promise<T>, ms: number, label: string): Promise<T> {
+	let timer: ReturnType<typeof setTimeout> | undefined;
+	const deadline = new Promise<never>((_, reject) => {
+		timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms);
+	});
+	return Promise.race([promise, deadline]).finally(() => {
+		if (timer !== undefined) clearTimeout(timer);
+	});
+}
+
+/**
+ * Resolve `undefined` if `promise` has not settled within `ms` — a slow peer is skipped, not an
+ * error. Same timer discipline as {@link withDeadline}: one repair pass races this once per cohort
+ * peer, so leaving the handles pending would keep the event loop alive for the full timeout after
+ * every peer has already answered.
+ */
+function withTimeout<T>(promise: Promise<T>, ms: number): Promise<T | undefined> {
+	let timer: ReturnType<typeof setTimeout> | undefined;
+	const expiry = new Promise<undefined>(resolve => {
+		timer = setTimeout(() => resolve(undefined), ms);
+	});
+	return Promise.race([promise, expiry]).finally(() => {
+		if (timer !== undefined) clearTimeout(timer);
+	});
+}
+
+/**
+ * What one round of polling the cohort learned about a block: the revision the OTHER
+ * cohort members corroborated, and what this node itself already holds. The two are kept
+ * apart on purpose — the local revision is the baseline being repaired, never evidence
+ * about the cluster (see {@link CoordinatorRepo.queryClusterForLatest}) — but the caller
+ * still needs it to tell whether the corroborated revision is actually an advance.
+ */
+interface ClusterLatestQuery {
+	/** Highest `(rev, actionId)` corroborated by peers other than this node, if any. */
+	corroborated?: ActionRev;
+	/** This node's own latest for the block, as answered by the callback's self short-circuit. */
+	local?: ActionRev;
+}
+
 /**
  * Extended cluster interface that includes the ability to check if a transaction was executed.
  * This is used by CoordinatorRepo to avoid duplicate execution.
@@ -36,12 +99,31 @@ interface CoordinatorRepoComponents {
 	 * Used for read-path cluster verification to discover unknown revisions.
 	 */
 	clusterLatestCallback?: ClusterLatestCallback;
+	/**
+	 * Optional callback that actually moves a block's bytes from the cohort into local storage once
+	 * {@link clusterLatestCallback} has established a corroborated revision this node lacks. Absent →
+	 * the read path can still *select* the right revision but converges only when the node already
+	 * holds the corroborated action as a promotable pending. See {@link AcquireBlockCallback}.
+	 */
+	acquireBlockFromCohort?: AcquireBlockCallback;
 }
 
+/**
+ * Consensus config for the coordinator side, plus the repair yardstick the read-repair path measures
+ * a (possibly shrunken) cohort view against. `repairCorroborationClusterSize` is deliberately its own
+ * field rather than an overload of {@link ClusterConsensusConfig.assumedClusterSize}: this same object
+ * also builds the `ClusterCoordinator`, and a field whose value silently differed from the cluster
+ * member's copy of it would be a trap. See `cluster/cluster-policy.ts` for why the two differ.
+ */
+export type CoordinatorRepoConfig = Partial<ClusterConsensusConfig> & {
+	clusterSize?: number;
+	repairCorroborationClusterSize?: number;
+};
+
 export function coordinatorRepo(
 	keyNetwork: IKeyNetwork,
-	createClusterClient: (peerId: PeerId) => ClusterClient,
-	cfg?: Partial<ClusterConsensusConfig> & { clusterSize?: number },
+	createClusterClient: (peerId: PeerId) => ICluster,
+	cfg?: CoordinatorRepoConfig,
 	fretService?: FretService,
 	reputation?: IPeerReputation,
 	stateStore?: ITransactionStateStore
@@ -56,7 +138,8 @@ export function coordinatorRepo(
 		fretService,
 		components.clusterLatestCallback,
 		reputation,
-		stateStore
+		stateStore,
+		components.acquireBlockFromCohort
 	);
 }
 
@@ -73,6 +156,13 @@ export class CoordinatorRepo implements IRepo {
 	private readonly readRepairSampleRate: number;
 	/** Simple-majority threshold from the consensus policy; drives the read-repair corroboration quorum. */
 	private readonly simpleMajorityThreshold: number;
+	/**
+	 * Yardstick the read-repair corroboration floor is measured against; the floor for
+	 * {@link corroboratorCapacity}. Resolved by `resolveClusterPolicy` for a real node; falls back to
+	 * `assumedClusterSize` and then `clusterSize` for direct constructors (see the constructor), so a
+	 * caller that has adopted neither field keeps today's behavior exactly.
+	 */
+	private readonly repairCorroborationClusterSize: number;
 	/** Resolved super-majority threshold the coordinator commits on (mirrors the value handed to ClusterCoordinator). */
 	private readonly superMajorityThreshold: number;
 	private readonly reputation?: IPeerReputation;
@@ -83,19 +173,21 @@ export class CoordinatorRepo implements IRepo {
 
 	constructor(
 		readonly keyNetwork: IKeyNetwork,
-		readonly createClusterClient: (peerId: PeerId) => ClusterClient,
+		readonly createClusterClient: (peerId: PeerId) => ICluster,
 		private readonly storageRepo: IRepo,
-		cfg?: Partial<ClusterConsensusConfig> & { clusterSize?: number },
+		cfg?: CoordinatorRepoConfig,
 		localCluster?: LocalClusterWithExecutionTracking,
 		localPeerId?: PeerId,
 		fretService?: FretService,
 		private readonly clusterLatestCallback?: ClusterLatestCallback,
 		reputation?: IPeerReputation,
-		stateStore?: ITransactionStateStore
+		stateStore?: ITransactionStateStore,
+		private readonly acquireBlockFromCohort?: AcquireBlockCallback
 	) {
 		this.localPeerId = localPeerId;
 		const policy: ClusterConsensusConfig & { clusterSize: number } = {
 			clusterSize: cfg?.clusterSize ?? 10,
+			assumedClusterSize: cfg?.assumedClusterSize,
 			superMajorityThreshold: cfg?.superMajorityThreshold ?? DEFAULT_SUPER_MAJORITY_THRESHOLD,
 			simpleMajorityThreshold: cfg?.simpleMajorityThreshold ?? 0.51,
 			minAbsoluteClusterSize: cfg?.minAbsoluteClusterSize ?? 3,
@@ -120,6 +212,15 @@ export class CoordinatorRepo implements IRepo {
 		this.readRepairSampleRate = policy.readRepairSampleRate!;
 		this.simpleMajorityThreshold = policy.simpleMajorityThreshold;
 		this.superMajorityThreshold = policy.superMajorityThreshold;
+		// Unlike the membership admission gate (which treats an absent assumedClusterSize as "unknown"
+		// and admits — refusing writes outright is unacceptable), this falls back to the replication
+		// factor and stays strict: the failure mode of getting this wrong is a block that goes
+		// unrepaired, degraded rather than dead, so there is no reason to relax it for a caller that
+		// has not adopted the new field. A real node is handed an explicit
+		// `repairCorroborationClusterSize` by `resolveClusterPolicy`; the `assumedClusterSize` middle
+		// term keeps direct constructors (embedders, existing tests) behaving as before.
+		this.repairCorroborationClusterSize =
+			cfg?.repairCorroborationClusterSize ?? policy.assumedClusterSize ?? policy.clusterSize;
 		this.reputation = reputation;
 		const localClusterRef = localCluster && localPeerId ? {
 			update: localCluster.update.bind(localCluster),
@@ -192,6 +293,13 @@ export class CoordinatorRepo implements IRepo {
 
 	async get(blockGets: BlockGets, options?: MessageOptions): Promise<GetBlockResults> {
 		// Soft proximity check — warn but still serve reads for graceful degradation
+		// NOTE: a soft-served read now also *acquires* the block durably (see restoreCorroborated), where
+		// before it could at most promote a pending this node already held. So a soft serve leaves behind
+		// a replica of a block this node is not responsible for, and nothing sweeps those: ring-shift
+		// sheds a keyspace RANGE, not "blocks outside my cohort". Fine while soft serves are what they
+		// are meant to be — a rare degradation during routing churn — since routing already placed this
+		// node near the block. If they ever become routine, gate acquisition (not the serve itself) on
+		// isResponsibleForBlock.
 		for (const blockId of blockGets.blockIds) {
 			if (!await this.isResponsibleForBlock(blockId)) {
 				log('proximity:get-warning', { blockId, msg: 'serving read for non-responsible block' });
@@ -205,14 +313,21 @@ export class CoordinatorRepo implements IRepo {
 		//   (a) Missing — block isn't present locally at all (legacy behavior).
 		//   (b) Stale-by-policy — block is present but read-repair policy says verify.
 		// Skip cluster fetch if this is already a sync request (to prevent recursive queries).
+		// A sync read is also never marked `unavailable` here — the consult it skips is the
+		// one whose failure the flag reports, and flagging would feed the recursion this
+		// bypass exists to prevent. (Storage-level 'unmaterializable' flags still pass
+		// through untouched; they report local state, not the consult.)
 		const skipClusterFetch = (options as any)?.skipClusterFetch;
 		// NOTE: NetworkTransactor.get treats an authoritative "absent" ({ state: {} })
 		// as final and no longer retries it (ticket txn-perf-authoritative-notfound),
-		// relying on this cluster reconciliation to have already run. If a coordinator
-		// is ever configured WITHOUT clusterLatestCallback, a missing block is answered
-		// from local state alone with no transactor-level retry to compensate. That is
-		// fine today (such a coordinator has no cluster to reconcile against), but keep
-		// this coupling in mind if a partial-cluster read path is added.
+		// relying on this cluster reconciliation to have already run. When the consult
+		// runs and FAILS, the entry is flagged `unavailable: 'peers-unreachable'` below,
+		// which re-enables the transactor-level retry against a different peer. If a
+		// coordinator is configured WITHOUT clusterLatestCallback, there is no cohort to
+		// consult and the local answer IS the whole truth — it stays authoritative, with
+		// no flag and no transactor-level retry to compensate. That is fine (such a
+		// coordinator has no cluster to reconcile against), but keep this coupling in
+		// mind if a partial-cluster read path is added.
 		if (this.clusterLatestCallback && !skipClusterFetch) {
 			for (const blockId of blockGets.blockIds) {
 				const localEntry = localResult[blockId];
@@ -231,7 +346,7 @@ export class CoordinatorRepo implements IRepo {
 				}
 
 				try {
-					await this.fetchBlockFromCluster(blockId, blockGets.context);
+					await this.fetchBlockFromCluster(blockId, blockGets.context, localRev);
 					const refreshed = await this.storageRepo.get({ blockIds: [blockId], context: blockGets.context }, options);
 					const newRev = refreshed[blockId]?.state?.latest?.rev;
 					if (refreshed[blockId]) {
@@ -246,6 +361,23 @@ export class CoordinatorRepo implements IRepo {
 					}
 				} catch (err) {
 					log('cluster-fetch:error', { blockId, error: (err as Error).message });
+					// The consult that was supposed to make this answer trustworthy did not run.
+					// Only a locally-MISSING block is downgraded to `unavailable`: a merely-stale
+					// block still has a real local answer and stays authoritative. A sharper flag
+					// already on the entry (storage's 'unmaterializable') is never overwritten.
+					// NOTE: a consult that runs but corroborates nothing (per-peer timeouts and
+					// "peer holds nothing" both surface as absent claims → no quorum) does NOT
+					// land here and stays an authoritative absent — the common new-collection
+					// probe against a healthy cohort takes exactly that path, and the callback
+					// contract cannot distinguish the two without counting responders.
+					if (isMissing) {
+						const entry = localResult[blockId];
+						if (!entry) {
+							localResult[blockId] = { state: {}, unavailable: 'peers-unreachable' };
+						} else if (entry.unavailable === undefined) {
+							entry.unavailable = 'peers-unreachable';
+						}
+					}
 				}
 			}
 		}
@@ -291,7 +423,12 @@ export class CoordinatorRepo implements IRepo {
 		this.lastSeenCommitMs.set(blockId, ts);
 	}
 
-	private async fetchBlockFromCluster(blockId: BlockId, context?: ActionContext): Promise<void> {
+	/**
+	 * One repair pass for a block: ask the cohort what it holds, and converge onto that if it is
+	 * ahead of `localRev` — the revision the caller's read already loaded, and the baseline every
+	 * decision below is measured against.
+	 */
+	private async fetchBlockFromCluster(blockId: BlockId, context?: ActionContext, localRev?: number): Promise<void> {
 		if (!this.clusterLatestCallback) return;
 
 		const blockIdBytes = new TextEncoder().encode(blockId);
@@ -312,40 +449,178 @@ export class CoordinatorRepo implements IRepo {
 			return;
 		}
 
-		const clusterLatest = await this.queryClusterForLatest(peerIds, blockId, context);
-		if (clusterLatest) {
-			// Found on cluster - trigger restoration to sync the block
-			await this.storageRepo.get({ blockIds: [blockId], context: { committed: [clusterLatest], rev: clusterLatest.rev } });
-			log('cluster-fetch:synced', { blockId, rev: clusterLatest.rev });
+		const { corroborated, local } = await this.queryClusterForLatest(peerIds, blockId, context);
+		// Nothing corroborated: keep local data AND stay eligible for repair — marking the
+		// block seen here would suppress the next attempt for the whole read-repair window.
+		if (!corroborated) return;
+
+		// The self answer is the sharper baseline (same storage, same context, read alongside the
+		// cohort's), but it exists only when `findCluster` returned this node. A soft serve for a
+		// block this node is no longer responsible for is absent from its own cohort view, so fall
+		// back to the revision the caller's read already loaded. Without the fallback both decisions
+		// below degrade to "any local revision is an advance", which restores backwards and reports
+		// a sync at the revision the pass started from.
+		const baselineRev = local?.rev ?? localRev;
+
+		// Never restore backwards. With this node's own claim excluded from the quorum, a
+		// cohort that lags behind the reader corroborates an OLDER revision; adopting it
+		// would be a regression, and logging it as a sync would be a lie. The cohort did
+		// answer, so the block is verified fresh — mark it seen.
+		// NOTE: in a cohort of two, that sole peer is the only corroborator, so a lying one can park
+		// the reader here — corroborating the revision it already holds — and re-arm the lazy window
+		// on every pass, hiding a real divergence. Bounded by `readRepairWindowMs` (10s default) and
+		// no worse than the peer simply staying silent. If two-member cohorts become a supported
+		// production topology rather than a dev convenience, stop re-arming the window on a
+		// corroboration that came from a single voter.
+		if (baselineRev !== undefined && corroborated.rev <= baselineRev) {
+			log('cluster-fetch:local-current', { blockId, localRev: baselineRev, clusterRev: corroborated.rev });
 			this.markBlocksSeen([blockId]);
+			return;
 		}
+
+		// Corroborated revision is ahead of ours — converge onto it.
+		const rev = await this.restoreCorroborated(blockId, corroborated, baselineRev, peerIds);
+
+		// Log the OUTCOME, not the attempt. Logging `synced` unconditionally reported hundreds of
+		// phantom convergences per run and made a real replication defect invisible for two debugging
+		// sessions.
+		if (rev !== undefined) {
+			log('cluster-fetch:synced', { blockId, rev });
+		} else {
+			log('cluster-fetch:not-restored', { blockId, localRev: baselineRev, clusterRev: corroborated.rev });
+		}
+		// The block is marked seen either way — the cohort DID answer, so its freshness was checked,
+		// which is what the read-repair window tracks. A failed convergence therefore waits out the
+		// window before retrying.
+		// NOTE: that damping covers only a block this node holds at an OLDER revision. A block entirely
+		// missing locally never consults the window (`get` triggers on `isMissing` before
+		// `shouldReadRepair`), so a persistently failing acquisition — e.g. a two-node deployment that
+		// never set `assumedClusterSize`, where the content quorum can never be met — re-fetches an
+		// archive on every read of that block. Correct, and self-limiting once the cohort can agree; if
+		// it ever shows as read amplification, gate the acquisition step (not the latest-query) on the
+		// same window rather than widening `isMissing`.
+		this.markBlocksSeen([blockId]);
 	}
 
 	/**
-	 * Query cluster peers for their latest revision and return the highest
-	 * revision corroborated by a quorum of distinct peers.
+	 * Bring this node up to the cohort-corroborated `corroborated`, returning the revision it holds
+	 * afterwards when that is an advance over `baselineRev`, else `undefined`.
+	 *
+	 * Two mechanisms, cheapest first:
+	 *  1. **Promote a local pending** — free, no network, and the only mechanism that existed before
+	 *     block acquisition. Covers the node that saw the pend and missed the commit broadcast.
+	 *  2. **Acquire the bytes from the cohort** ({@link AcquireBlockCallback}) — covers everything else,
+	 *     including a block this node has never seen at all.
+	 *
+	 * **Why acquisition is gated here and not on a plain local miss.** `BlockStorage.getBlock` returns
+	 * `undefined` for a block with no local metadata *without* consulting its restore callback, so that
+	 * an insert probing a fresh random block id for a collision does not cost a network fetch. That
+	 * remains true: this method runs only after {@link queryClusterForLatest} produced a quorum-
+	 * corroborated `(rev, actionId)`, which a genuinely non-existent block can never produce (no peer
+	 * claims it, so `selectQuorumRev` declines and `fetchBlockFromCluster` returns before reaching
+	 * here). The cost of a genuine absence is unchanged — the latest-query round trip that already
+	 * happened — while a block the cohort demonstrably holds is no longer thrown away.
+	 *
+	 * Cohort peer ids are passed straight through: the callback filters self out and caps its own
+	 * corroboration quorum by how many peers could answer at all.
+	 */
+	private async restoreCorroborated(
+		blockId: BlockId,
+		corroborated: ActionRev,
+		baselineRev: number | undefined,
+		cohortPeerIds: string[]
+	): Promise<number | undefined> {
+		const promoted = await this.promoteCorroborated(blockId, corroborated);
+		if (isAdvanceOver(promoted, baselineRev)) {
+			return promoted;
+		}
+
+		if (!this.acquireBlockFromCohort) {
+			return undefined;
+		}
+		try {
+			// Bounded: a stalled cohort peer must not hold up the caller's read. Persisting happens
+			// inside the callback via `saveReplicatedBlock`, which takes the per-block commit latch —
+			// safe to call from here because the read path holds no latch of its own (`StorageRepo.get`
+			// acquires and releases it around the promotion above, and nothing wraps this method).
+			// NOTE: `get` walks its block ids sequentially, so the bound is per block, not per call — a
+			// multi-block read that is missing N blocks against a wholly stalled cohort waits N × this.
+			// Acceptable today (the underlying per-peer archive fetch is itself 1s-bounded and runs the
+			// cohort in parallel, so the 5s is a stall ceiling, not a typical cost). If a cold reader
+			// batching a wide read ever times out above this layer, repair the block ids concurrently
+			// rather than shortening the bound.
+			await withDeadline(
+				this.acquireBlockFromCohort(blockId, corroborated, cohortPeerIds),
+				RECONCILE_TIMEOUT_MS,
+				`block acquisition for ${blockId}`
+			);
+		} catch (err) {
+			// Declines are cheap and retryable — nothing was persisted. Report and leave the block behind.
+			log('cluster-fetch:acquire-error', { blockId, rev: corroborated.rev, error: (err as Error).message });
+			return undefined;
+		}
+		const acquired = await this.readLocalRev(blockId);
+		return isAdvanceOver(acquired, baselineRev) ? acquired : undefined;
+	}
+
+	/**
+	 * Promote a corroborated action this node already holds as a local pending — the no-network half of
+	 * the repair. Returns the local revision afterwards.
+	 *
+	 * A pending-only block (metadata seeded by `savePendingTransaction`, no committed revision) asked
+	 * for a forward revision no promotion can reach used to throw out of `BlockStorage.ensureRevision`;
+	 * `StorageRepo.get` now reports it as an entry flagged `unavailable` instead (ticket
+	 * repo-reports-unavailable-vs-absent). On THIS path either shape is an absence, not a read failure —
+	 * acquisition is precisely the mechanism that can supply the revision — so both are logged as
+	 * `promote-unavailable` and stepped over rather than short-circuiting the caller.
+	 */
+	private async promoteCorroborated(blockId: BlockId, corroborated: ActionRev): Promise<number | undefined> {
+		try {
+			const entry = await this.readLocalEntry(blockId, { committed: [corroborated], rev: corroborated.rev });
+			if (entry?.unavailable !== undefined) {
+				log('cluster-fetch:promote-unavailable', { blockId, rev: corroborated.rev, error: entry.unavailable });
+				return undefined;
+			}
+			return entry?.state?.latest?.rev;
+		} catch (err) {
+			log('cluster-fetch:promote-unavailable', { blockId, rev: corroborated.rev, error: (err as Error).message });
+			return undefined;
+		}
+	}
+
+	/** This node's own answer for a block, optionally driving a promotion context through the read.
+	 *  Callers that care whether the answer is authoritative inspect `entry.unavailable`. */
+	private async readLocalEntry(blockId: BlockId, context?: ActionContext) {
+		const result = await this.storageRepo.get({ blockIds: [blockId], context });
+		return result[blockId];
+	}
+
+	/** This node's own `latest.rev` for a block, optionally driving a promotion context through the read. */
+	private async readLocalRev(blockId: BlockId, context?: ActionContext): Promise<number | undefined> {
+		return (await this.readLocalEntry(blockId, context))?.state?.latest?.rev;
+	}
+
+	/**
+	 * Query cluster peers for their latest revision and return the highest revision
+	 * corroborated by a quorum of distinct peers, alongside this node's own latest.
 	 *
 	 * Replaces the old "max rev any single peer reports" — which let one lying
 	 * peer over-reporting its revision steer restoration — with quorum
-	 * corroboration on the exact `(rev, actionId)` pair (see
-	 * {@link selectQuorumRev}). The local node's own latest is included as a
-	 * corroborating vote because `clusterLatestCallback` self-short-circuits to
-	 * local storage. Returns `undefined` (keep local, do not restore) when no
-	 * revision is corroborated.
+	 * corroboration on the exact `(rev, actionId)` pair (see {@link selectQuorumRev}).
+	 *
+	 * This node's own answer is split out of the claim set rather than counted in it:
+	 * `clusterLatestCallback` short-circuits self to local storage, so including it let a
+	 * reader whose only peer timed out "corroborate" the very revision it was trying to
+	 * repair. It is returned separately so the caller can compare, not vote.
 	 *
 	 * NOTE: the quorum is corroboration-of-a-claim, NOT Sybil-resistant cohort
-	 * membership — a peer minting fresh keypairs still casts a vote. Commit-cert
-	 * + membership anchoring is deferred to backlog
+	 * membership — a peer minting fresh keypairs still casts a vote, and the claims
+	 * themselves are bare assertions (a `BlockArchive` carries no commit certificate, so
+	 * there is nothing here to verify a `(rev, actionId)` against). Commit-cert +
+	 * membership anchoring is deferred to backlog
 	 * `debt-read-repair-commit-cert-verification`.
 	 */
-	private async queryClusterForLatest(peerIds: string[], blockId: BlockId, context?: ActionContext): Promise<ActionRev | undefined> {
-		// Add timeout wrapper to prevent hanging on unresponsive peers
-		const withTimeout = <T>(promise: Promise<T>, timeoutMs: number): Promise<T | undefined> =>
-			Promise.race([
-				promise,
-				new Promise<undefined>(resolve => setTimeout(() => resolve(undefined), timeoutMs))
-			]);
-
+	private async queryClusterForLatest(peerIds: string[], blockId: BlockId, context?: ActionContext): Promise<ClusterLatestQuery> {
 		// Query peers in parallel for their latest revision (with 1s timeout per peer),
 		// tagging each response with the peer that made it so votes stay distinct.
 		const latestResults = await Promise.allSettled(
@@ -356,18 +631,33 @@ export class CoordinatorRepo implements IRepo {
 			})
 		);
 
+		// NOTE: self-exclusion is keyed on `localPeerId`, which is optional for the single-node/test
+		// construction this class has always tolerated. Left unset, this node's own answer is counted
+		// as a peer claim again. Harmless today — the self answer can only ever corroborate the
+		// revision already held, so the pass declines as `local-current` — but if a future caller can
+		// make self report something the reader does not hold, make `localPeerId` required instead.
+		const selfId = this.localPeerId?.toString();
+		let local: ActionRev | undefined;
 		const claims: RevClaim[] = [];
 		for (const result of latestResults) {
-			if (result.status === 'fulfilled' && result.value.value) {
-				const { peerIdStr, value } = result.value;
-				claims.push({ peerId: peerIdStr, rev: value.rev, actionId: value.actionId });
+			if (result.status !== 'fulfilled' || !result.value.value) continue;
+			const { peerIdStr, value } = result.value;
+			if (peerIdStr === selfId) {
+				local = value;
+				continue;
 			}
+			claims.push({ peerId: peerIdStr, rev: value.rev, actionId: value.actionId });
 		}
 
-		const selected = selectQuorumRev(claims, this.simpleMajorityThreshold);
+		const capacity = corroboratorCapacity(peerIds.filter(id => id !== selfId).length, this.repairCorroborationClusterSize);
+		const selected = selectQuorumRev(claims, this.simpleMajorityThreshold, capacity);
 		if (!selected) {
-			log('cluster-fetch:no-quorum', { blockId, responders: claims.length });
-			return undefined;
+			log('cluster-fetch:no-quorum', {
+				blockId,
+				responders: claims.length,
+				required: quorumSize(claims.length, this.simpleMajorityThreshold, capacity)
+			});
+			return { local };
 		}
 
 		// Best-effort: penalize peers whose claim contradicts the corroborated pair
@@ -375,7 +665,7 @@ export class CoordinatorRepo implements IRepo {
 		// rev). A lower rev is just lag, never penalized. Never let this throw.
 		this.penalizeContradictingRevClaims(claims, selected, blockId);
 
-		return { actionId: selected.actionId, rev: selected.rev };
+		return { corroborated: { actionId: selected.actionId, rev: selected.rev }, local };
 	}
 
 	/**
@@ -445,10 +735,61 @@ export class CoordinatorRepo implements IRepo {
 			};
 		} catch (error) {
 			log('coordinator-repo:pend-error', { actionId: request.actionId, error: (error as Error).message });
+			const stale = await this.classifyStaleRejection(error, request, allBlockIds);
+			if (stale) return stale;
 			throw error;
 		}
 	}
 
+	/**
+	 * Decide whether a cluster validator rejection was an optimistic-concurrency loss — the block
+	 * already advanced past the requested revision — rather than a genuine validation fault.
+	 * A confirmed loss returns a {@link StaleFailure} carrying `conflict: true` so the caller
+	 * receives a non-success *response* that says plainly it is a lost race: network-transactor's
+	 * pend then takes its stale branch and both writers (`Collection.sync`, and the coordinator's
+	 * multi-collection pendPhase via `isConflictFailure`) retry, instead of a thrown error escaping
+	 * mid-batch (which splits multi-tree commits — see PartialCommitError).
+	 *
+	 * The failure carries no `missing` list: confirmation is a local re-read that reveals the
+	 * revision is taken but not which actions took it, and no consumer rebases from `missing`
+	 * anyway (it is only counted or logged). `conflict` conveys retryability directly instead.
+	 *
+	 * Confirmation is purely local: re-read the affected blocks from our own storage and require
+	 * `latest.rev >= request.rev`. The signed reject-reason text is never consulted — it is
+	 * free-form wire-visible prose and must not become control flow. Anything unconfirmed
+	 * (including read errors during confirmation) stays a throw, preserving fail-fast for
+	 * genuine validation faults.
+	 */
+	private async classifyStaleRejection(error: unknown, request: PendRequest, blockIds: BlockId[]): Promise<StaleFailure | undefined> {
+		if (!(error instanceof ValidatorRejectionError) || request.rev === undefined) return undefined;
+		let results: GetBlockResults;
+		try {
+			results = await this.storageRepo.get({ blockIds });
+		} catch (readError) {
+			log('coordinator-repo:pend-stale-classify-read-error', {
+				actionId: request.actionId,
+				error: (readError as Error).message
+			});
+			return undefined;
+		}
+		for (const blockId of blockIds) {
+			const latest = results[blockId]?.state.latest;
+			if (latest && latest.rev >= request.rev) {
+				log('coordinator-repo:pend-stale-classified', {
+					actionId: request.actionId,
+					blockId,
+					latestRev: latest.rev,
+					requestedRev: request.rev
+				});
+				return { success: false, conflict: true, reason: `stale revision: block ${blockId} at rev ${latest.rev}, requested rev ${request.rev}` };
+			}
+		}
+		// NOTE: conservative — when only remote members saw the newer revision (local storage still
+		// behind), staleness can't be confirmed locally and the rejection stays a throw. If that
+		// shows up in practice, extend confirmation with a quorum read; never trust the reject text.
+		return undefined;
+	}
+
 	async cancel(actionRef: ActionBlocks, options?: MessageOptions): Promise<void> {
 		const blockIds = actionRef.blockIds;
 		await this.verifyResponsibility(blockIds);
@@ -501,23 +842,32 @@ export class CoordinatorRepo implements IRepo {
 				this.markBlocksSeen(blockIds);
 				return { success: true };
 			}
-			// Local cluster didn't execute during consensus. Attempt a local commit,
-			// but tolerate failure (e.g., "pending action not found") when the cluster
-			// already reached consensus — this coordinator was likely picked for commit
-			// after missing the pend phase (unreachable during pend, fresh join, etc.).
-			// The cluster's majority is authoritative; this peer will catch up via sync.
+			// Local cluster didn't execute during consensus. Attempt a local commit, but tolerate
+			// local divergence when the cluster already reached consensus — this coordinator was
+			// likely picked for commit after missing the pend phase (unreachable during pend, fresh
+			// join, etc.). The cluster's majority is authoritative; this peer catches up via sync.
+			//
+			// Divergence reaches us in BOTH shapes and both must be tolerated identically:
+			//   - a THROW ("Pending action … not found"), when we never saw the pend;
+			//   - a RETURNED `success:false` carrying `missing-base-revision`, when we saw the pend
+			//     but not the revision that created the block (see StorageRepo.internalCommit).
+			// Only the throw was tolerated before the refusal existed. Reporting the refusal to the
+			// caller instead would surface a committed transaction as a stale loss: db-core's
+			// commitPhase treats any returned `success:false` as a permanent stale failure, so the
+			// client would retry an action the cluster already landed until it exhausted its budget.
 			try {
 				const result = await this.storageRepo.commit(request, options);
-				if (result.success) this.markBlocksSeen(blockIds);
+				if (result.success) {
+					this.markBlocksSeen(blockIds);
+					return result;
+				}
+				if (isMissingBaseRevisionFailure(result) && clusterReachedCommitConsensus(record)) {
+					return this.tolerateLocalCommitDivergence(request, blockIds, result.reason ?? MISSING_BASE_REVISION_REASON);
+				}
 				return result;
 			} catch (err) {
 				if (clusterReachedCommitConsensus(record)) {
-					log('coordinator-repo:commit-local-failed-cluster-succeeded', {
-						actionId: request.actionId,
-						error: (err as Error).message
-					});
-					this.markBlocksSeen(blockIds);
-					return { success: true };
+					return this.tolerateLocalCommitDivergence(request, blockIds, (err as Error).message);
 				}
 				throw err;
 			}
@@ -526,6 +876,17 @@ export class CoordinatorRepo implements IRepo {
 			throw error;
 		}
 	}
+
+	/**
+	 * Report success for a commit the cluster carried but this peer could not apply locally. The
+	 * blocks are marked seen so the read path treats them as freshness-checked; convergence comes
+	 * from replication (cohort reconcile, or read-driven acquisition), not from replay here.
+	 */
+	private tolerateLocalCommitDivergence(request: CommitRequest, blockIds: BlockId[], detail: string): CommitResult {
+		log('coordinator-repo:commit-local-failed-cluster-succeeded', { actionId: request.actionId, error: detail });
+		this.markBlocksSeen(blockIds);
+		return { success: true };
+	}
 }
 
 /** True if a simple majority of cluster peers signed an approving commit. */