@optimystic/db-p2p 0.17.0 → 0.18.0

package/src/repo/cluster-coordinator.ts

@@ -1,7 +1,6 @@
 import { peerIdFromString } from "@libp2p/peer-id";
-import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, ClusterConsensusConfig } from "@optimystic/db-core";
+import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, ClusterConsensusConfig, ICluster } from "@optimystic/db-core";
 import { CURRENT_MEMBERSHIP_VERSION, computeClusterMessageHash, membershipDigest } from "@optimystic/db-core";
-import { ClusterClient } from "../cluster/client.js";
 import { Pending } from "@optimystic/db-core";
 import type { PeerId } from "@libp2p/interface";
 import { createLogger, verbose } from '../logger.js'
@@ -13,6 +12,26 @@ import type { ITransactionStateStore } from "../cluster/i-transaction-state-stor
 
 const log = createLogger('cluster')
 
+/**
+ * Consensus refused a transaction: enough members voted reject that super-majority became
+ * impossible. A typed error (rather than a bare `Error`) so the repo layer above can distinguish
+ * "the cluster voted this down" from transport/availability failures WITHOUT string-matching the
+ * rejection reasons — those are free-form text that is part of each member's signed vote payload
+ * (see cluster-repo's `computeSigningPayload`), so their wording must never become control flow.
+ * `CoordinatorRepo.pend` uses this to decide whether a rejection is a retryable stale-revision
+ * loss (confirmed against local storage) or a genuine validation fault.
+ */
+export class ValidatorRejectionError extends Error {
+	constructor(
+		message: string,
+		/** Per-peer reject reasons, verbatim from the vote signatures (free-form, wire-visible). */
+		readonly rejectReasons: Record<string, string>
+	) {
+		super(message);
+		this.name = 'ValidatorRejectionError';
+	}
+}
+
 /** Cancel handle for an injected timer; cancels a not-yet-fired timer (safe no-op after fire/cancel). */
 export type TimerCancel = () => void;
 
@@ -76,7 +95,8 @@ export class ClusterCoordinator {
 
 	constructor(
 		private readonly keyNetwork: IKeyNetwork,
-		private readonly createClusterClient: (peerId: PeerId) => ClusterClient,
+		/** Factory for a per-peer cluster RPC handle; only `update` is ever called, hence `ICluster`. */
+		private readonly createClusterClient: (peerId: PeerId) => ICluster,
 		private readonly cfg: ClusterConsensusConfig & { clusterSize: number },
 		private readonly localCluster?: {
 			update: (record: ClusterRecord) => Promise<ClusterRecord>;
@@ -322,9 +342,11 @@ export class ClusterCoordinator {
 		// If more than (peerCount - superMajority) nodes reject, we can never reach super-majority
 		const maxAllowedRejections = peerCount - superMajority;
 		if (rejectionCount > maxAllowedRejections) {
-			const rejectReasons = Object.entries(promises)
+			const rejectReasonsByPeer = Object.fromEntries(Object.entries(promises)
 				.filter(([_, sig]) => sig.type === 'reject')
-				.map(([peerId, sig]) => `${peerId}: ${sig.rejectReason ?? 'unknown'}`)
+				.map(([peerId, sig]) => [peerId, sig.rejectReason ?? 'unknown']));
+			const rejectReasons = Object.entries(rejectReasonsByPeer)
+				.map(([peerId, reason]) => `${peerId}: ${reason}`)
 				.join('; ');
 			log('cluster-tx:rejected-by-validators', {
 				messageHash: record.messageHash,
@@ -334,7 +356,9 @@ export class ClusterCoordinator {
 				reasons: rejectReasons
 			});
 			this.updateTransactionRecord(promised.record, 'rejected-by-validators');
-			throw new Error(`Transaction rejected by validators (${rejectionCount}/${peerCount} rejected): ${rejectReasons}`);
+			throw new ValidatorRejectionError(
+				`Transaction rejected by validators (${rejectionCount}/${peerCount} rejected): ${rejectReasons}`,
+				rejectReasonsByPeer);
 		}
 
 		if (peerCount > 1 && approvalCount < superMajority) {

package/src/repo/coordinator-repo.ts

@@ -1,7 +1,6 @@
-import type { PendRequest, ActionBlocks, IRepo, MessageOptions, CommitResult, GetBlockResults, PendResult, BlockGets, CommitRequest, RepoMessage, IKeyNetwork, ICluster, ClusterConsensusConfig, BlockId, ActionRev, ActionContext, ClusterRecord } from "@optimystic/db-core";
+import type { PendRequest, ActionBlocks, IRepo, MessageOptions, CommitResult, GetBlockResults, PendResult, StaleFailure, BlockGets, CommitRequest, RepoMessage, IKeyNetwork, ICluster, ClusterConsensusConfig, BlockId, ActionRev, ActionContext, ClusterRecord } from "@optimystic/db-core";
 import { LruMap, blockIdsForTransforms, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
-import { ClusterCoordinator } from "./cluster-coordinator.js";
-import type { ClusterClient } from "../cluster/client.js";
+import { ClusterCoordinator, ValidatorRejectionError } from "./cluster-coordinator.js";
 import type { PeerId } from "@libp2p/interface";
 import { peerIdFromString } from "@libp2p/peer-id";
 import type { FretService } from "p2p-fret";
@@ -9,8 +8,9 @@ import { createLogger } from '../logger.js';
 import type { IPeerReputation } from "../reputation/types.js";
 import { PenaltyReason } from "../reputation/types.js";
 import type { ITransactionStateStore } from "../cluster/i-transaction-state-store.js";
-import { quorumSize, selectQuorumRev, type RevClaim, type QuorumRev } from "../cluster/quorum-restore.js";
+import { quorumSize, corroboratorCapacity, selectQuorumRev, type RevClaim, type QuorumRev } from "../cluster/quorum-restore.js";
 import { RECONCILE_TIMEOUT_MS } from "../cluster/reconcile-block.js";
+import { isMissingBaseRevisionFailure, MISSING_BASE_REVISION_REASON } from "../storage/storage-repo.js";
 import type { ReconcileBlockCallback } from "../cluster/cluster-repo.js";
 
 const log = createLogger('coordinator-repo');
@@ -28,8 +28,8 @@ const log = createLogger('coordinator-repo');
 export type AcquireBlockCallback = ReconcileBlockCallback;
 
 /** True when a freshly-read local revision is strictly ahead of the baseline the repair started from. */
-function isAdvanceOver(rev: number | undefined, baseline: ActionRev | undefined): boolean {
-	return typeof rev === 'number' && (baseline === undefined || rev > baseline.rev);
+function isAdvanceOver(rev: number | undefined, baselineRev: number | undefined): boolean {
+	return typeof rev === 'number' && (baselineRev === undefined || rev > baselineRev);
 }
 
 /**
@@ -46,6 +46,22 @@ function withDeadline<T>(promise: Promise<T>, ms: number, label: string): Promis
 	});
 }
 
+/**
+ * Resolve `undefined` if `promise` has not settled within `ms` — a slow peer is skipped, not an
+ * error. Same timer discipline as {@link withDeadline}: one repair pass races this once per cohort
+ * peer, so leaving the handles pending would keep the event loop alive for the full timeout after
+ * every peer has already answered.
+ */
+function withTimeout<T>(promise: Promise<T>, ms: number): Promise<T | undefined> {
+	let timer: ReturnType<typeof setTimeout> | undefined;
+	const expiry = new Promise<undefined>(resolve => {
+		timer = setTimeout(() => resolve(undefined), ms);
+	});
+	return Promise.race([promise, expiry]).finally(() => {
+		if (timer !== undefined) clearTimeout(timer);
+	});
+}
+
 /**
  * What one round of polling the cohort learned about a block: the revision the OTHER
  * cohort members corroborated, and what this node itself already holds. The two are kept
@@ -92,10 +108,22 @@ interface CoordinatorRepoComponents {
 	acquireBlockFromCohort?: AcquireBlockCallback;
 }
 
+/**
+ * Consensus config for the coordinator side, plus the repair yardstick the read-repair path measures
+ * a (possibly shrunken) cohort view against. `repairCorroborationClusterSize` is deliberately its own
+ * field rather than an overload of {@link ClusterConsensusConfig.assumedClusterSize}: this same object
+ * also builds the `ClusterCoordinator`, and a field whose value silently differed from the cluster
+ * member's copy of it would be a trap. See `cluster/cluster-policy.ts` for why the two differ.
+ */
+export type CoordinatorRepoConfig = Partial<ClusterConsensusConfig> & {
+	clusterSize?: number;
+	repairCorroborationClusterSize?: number;
+};
+
 export function coordinatorRepo(
 	keyNetwork: IKeyNetwork,
-	createClusterClient: (peerId: PeerId) => ClusterClient,
-	cfg?: Partial<ClusterConsensusConfig> & { clusterSize?: number },
+	createClusterClient: (peerId: PeerId) => ICluster,
+	cfg?: CoordinatorRepoConfig,
 	fretService?: FretService,
 	reputation?: IPeerReputation,
 	stateStore?: ITransactionStateStore
@@ -128,8 +156,13 @@ export class CoordinatorRepo implements IRepo {
 	private readonly readRepairSampleRate: number;
 	/** Simple-majority threshold from the consensus policy; drives the read-repair corroboration quorum. */
 	private readonly simpleMajorityThreshold: number;
-	/** Configured full cluster size; the operator's declaration of how many corroborators should exist. */
-	private readonly clusterSize: number;
+	/**
+	 * Yardstick the read-repair corroboration floor is measured against; the floor for
+	 * {@link corroboratorCapacity}. Resolved by `resolveClusterPolicy` for a real node; falls back to
+	 * `assumedClusterSize` and then `clusterSize` for direct constructors (see the constructor), so a
+	 * caller that has adopted neither field keeps today's behavior exactly.
+	 */
+	private readonly repairCorroborationClusterSize: number;
 	/** Resolved super-majority threshold the coordinator commits on (mirrors the value handed to ClusterCoordinator). */
 	private readonly superMajorityThreshold: number;
 	private readonly reputation?: IPeerReputation;
@@ -140,9 +173,9 @@ export class CoordinatorRepo implements IRepo {
 
 	constructor(
 		readonly keyNetwork: IKeyNetwork,
-		readonly createClusterClient: (peerId: PeerId) => ClusterClient,
+		readonly createClusterClient: (peerId: PeerId) => ICluster,
 		private readonly storageRepo: IRepo,
-		cfg?: Partial<ClusterConsensusConfig> & { clusterSize?: number },
+		cfg?: CoordinatorRepoConfig,
 		localCluster?: LocalClusterWithExecutionTracking,
 		localPeerId?: PeerId,
 		fretService?: FretService,
@@ -154,6 +187,7 @@ export class CoordinatorRepo implements IRepo {
 		this.localPeerId = localPeerId;
 		const policy: ClusterConsensusConfig & { clusterSize: number } = {
 			clusterSize: cfg?.clusterSize ?? 10,
+			assumedClusterSize: cfg?.assumedClusterSize,
 			superMajorityThreshold: cfg?.superMajorityThreshold ?? DEFAULT_SUPER_MAJORITY_THRESHOLD,
 			simpleMajorityThreshold: cfg?.simpleMajorityThreshold ?? 0.51,
 			minAbsoluteClusterSize: cfg?.minAbsoluteClusterSize ?? 3,
@@ -178,7 +212,15 @@ export class CoordinatorRepo implements IRepo {
 		this.readRepairSampleRate = policy.readRepairSampleRate!;
 		this.simpleMajorityThreshold = policy.simpleMajorityThreshold;
 		this.superMajorityThreshold = policy.superMajorityThreshold;
-		this.clusterSize = policy.clusterSize;
+		// Unlike the membership admission gate (which treats an absent assumedClusterSize as "unknown"
+		// and admits — refusing writes outright is unacceptable), this falls back to the replication
+		// factor and stays strict: the failure mode of getting this wrong is a block that goes
+		// unrepaired, degraded rather than dead, so there is no reason to relax it for a caller that
+		// has not adopted the new field. A real node is handed an explicit
+		// `repairCorroborationClusterSize` by `resolveClusterPolicy`; the `assumedClusterSize` middle
+		// term keeps direct constructors (embedders, existing tests) behaving as before.
+		this.repairCorroborationClusterSize =
+			cfg?.repairCorroborationClusterSize ?? policy.assumedClusterSize ?? policy.clusterSize;
 		this.reputation = reputation;
 		const localClusterRef = localCluster && localPeerId ? {
 			update: localCluster.update.bind(localCluster),
@@ -251,6 +293,13 @@ export class CoordinatorRepo implements IRepo {
 
 	async get(blockGets: BlockGets, options?: MessageOptions): Promise<GetBlockResults> {
 		// Soft proximity check — warn but still serve reads for graceful degradation
+		// NOTE: a soft-served read now also *acquires* the block durably (see restoreCorroborated), where
+		// before it could at most promote a pending this node already held. So a soft serve leaves behind
+		// a replica of a block this node is not responsible for, and nothing sweeps those: ring-shift
+		// sheds a keyspace RANGE, not "blocks outside my cohort". Fine while soft serves are what they
+		// are meant to be — a rare degradation during routing churn — since routing already placed this
+		// node near the block. If they ever become routine, gate acquisition (not the serve itself) on
+		// isResponsibleForBlock.
 		for (const blockId of blockGets.blockIds) {
 			if (!await this.isResponsibleForBlock(blockId)) {
 				log('proximity:get-warning', { blockId, msg: 'serving read for non-responsible block' });
@@ -264,14 +313,21 @@ export class CoordinatorRepo implements IRepo {
 		//   (a) Missing — block isn't present locally at all (legacy behavior).
 		//   (b) Stale-by-policy — block is present but read-repair policy says verify.
 		// Skip cluster fetch if this is already a sync request (to prevent recursive queries).
+		// A sync read is also never marked `unavailable` here — the consult it skips is the
+		// one whose failure the flag reports, and flagging would feed the recursion this
+		// bypass exists to prevent. (Storage-level 'unmaterializable' flags still pass
+		// through untouched; they report local state, not the consult.)
 		const skipClusterFetch = (options as any)?.skipClusterFetch;
 		// NOTE: NetworkTransactor.get treats an authoritative "absent" ({ state: {} })
 		// as final and no longer retries it (ticket txn-perf-authoritative-notfound),
-		// relying on this cluster reconciliation to have already run. If a coordinator
-		// is ever configured WITHOUT clusterLatestCallback, a missing block is answered
-		// from local state alone with no transactor-level retry to compensate. That is
-		// fine today (such a coordinator has no cluster to reconcile against), but keep
-		// this coupling in mind if a partial-cluster read path is added.
+		// relying on this cluster reconciliation to have already run. When the consult
+		// runs and FAILS, the entry is flagged `unavailable: 'peers-unreachable'` below,
+		// which re-enables the transactor-level retry against a different peer. If a
+		// coordinator is configured WITHOUT clusterLatestCallback, there is no cohort to
+		// consult and the local answer IS the whole truth — it stays authoritative, with
+		// no flag and no transactor-level retry to compensate. That is fine (such a
+		// coordinator has no cluster to reconcile against), but keep this coupling in
+		// mind if a partial-cluster read path is added.
 		if (this.clusterLatestCallback && !skipClusterFetch) {
 			for (const blockId of blockGets.blockIds) {
 				const localEntry = localResult[blockId];
@@ -290,7 +346,7 @@ export class CoordinatorRepo implements IRepo {
 				}
 
 				try {
-					await this.fetchBlockFromCluster(blockId, blockGets.context);
+					await this.fetchBlockFromCluster(blockId, blockGets.context, localRev);
 					const refreshed = await this.storageRepo.get({ blockIds: [blockId], context: blockGets.context }, options);
 					const newRev = refreshed[blockId]?.state?.latest?.rev;
 					if (refreshed[blockId]) {
@@ -305,6 +361,23 @@ export class CoordinatorRepo implements IRepo {
 					}
 				} catch (err) {
 					log('cluster-fetch:error', { blockId, error: (err as Error).message });
+					// The consult that was supposed to make this answer trustworthy did not run.
+					// Only a locally-MISSING block is downgraded to `unavailable`: a merely-stale
+					// block still has a real local answer and stays authoritative. A sharper flag
+					// already on the entry (storage's 'unmaterializable') is never overwritten.
+					// NOTE: a consult that runs but corroborates nothing (per-peer timeouts and
+					// "peer holds nothing" both surface as absent claims → no quorum) does NOT
+					// land here and stays an authoritative absent — the common new-collection
+					// probe against a healthy cohort takes exactly that path, and the callback
+					// contract cannot distinguish the two without counting responders.
+					if (isMissing) {
+						const entry = localResult[blockId];
+						if (!entry) {
+							localResult[blockId] = { state: {}, unavailable: 'peers-unreachable' };
+						} else if (entry.unavailable === undefined) {
+							entry.unavailable = 'peers-unreachable';
+						}
+					}
 				}
 			}
 		}
@@ -350,7 +423,12 @@ export class CoordinatorRepo implements IRepo {
 		this.lastSeenCommitMs.set(blockId, ts);
 	}
 
-	private async fetchBlockFromCluster(blockId: BlockId, context?: ActionContext): Promise<void> {
+	/**
+	 * One repair pass for a block: ask the cohort what it holds, and converge onto that if it is
+	 * ahead of `localRev` — the revision the caller's read already loaded, and the baseline every
+	 * decision below is measured against.
+	 */
+	private async fetchBlockFromCluster(blockId: BlockId, context?: ActionContext, localRev?: number): Promise<void> {
 		if (!this.clusterLatestCallback) return;
 
 		const blockIdBytes = new TextEncoder().encode(blockId);
@@ -376,18 +454,32 @@ export class CoordinatorRepo implements IRepo {
 		// block seen here would suppress the next attempt for the whole read-repair window.
 		if (!corroborated) return;
 
+		// The self answer is the sharper baseline (same storage, same context, read alongside the
+		// cohort's), but it exists only when `findCluster` returned this node. A soft serve for a
+		// block this node is no longer responsible for is absent from its own cohort view, so fall
+		// back to the revision the caller's read already loaded. Without the fallback both decisions
+		// below degrade to "any local revision is an advance", which restores backwards and reports
+		// a sync at the revision the pass started from.
+		const baselineRev = local?.rev ?? localRev;
+
 		// Never restore backwards. With this node's own claim excluded from the quorum, a
 		// cohort that lags behind the reader corroborates an OLDER revision; adopting it
 		// would be a regression, and logging it as a sync would be a lie. The cohort did
 		// answer, so the block is verified fresh — mark it seen.
-		if (local && corroborated.rev <= local.rev) {
-			log('cluster-fetch:local-current', { blockId, localRev: local.rev, clusterRev: corroborated.rev });
+		// NOTE: in a cohort of two, that sole peer is the only corroborator, so a lying one can park
+		// the reader here — corroborating the revision it already holds — and re-arm the lazy window
+		// on every pass, hiding a real divergence. Bounded by `readRepairWindowMs` (10s default) and
+		// no worse than the peer simply staying silent. If two-member cohorts become a supported
+		// production topology rather than a dev convenience, stop re-arming the window on a
+		// corroboration that came from a single voter.
+		if (baselineRev !== undefined && corroborated.rev <= baselineRev) {
+			log('cluster-fetch:local-current', { blockId, localRev: baselineRev, clusterRev: corroborated.rev });
 			this.markBlocksSeen([blockId]);
 			return;
 		}
 
 		// Corroborated revision is ahead of ours — converge onto it.
-		const rev = await this.restoreCorroborated(blockId, corroborated, local, peerIds);
+		const rev = await this.restoreCorroborated(blockId, corroborated, baselineRev, peerIds);
 
 		// Log the OUTCOME, not the attempt. Logging `synced` unconditionally reported hundreds of
 		// phantom convergences per run and made a real replication defect invisible for two debugging
@@ -395,15 +487,15 @@ export class CoordinatorRepo implements IRepo {
 		if (rev !== undefined) {
 			log('cluster-fetch:synced', { blockId, rev });
 		} else {
-			log('cluster-fetch:not-restored', { blockId, localRev: local?.rev, clusterRev: corroborated.rev });
+			log('cluster-fetch:not-restored', { blockId, localRev: baselineRev, clusterRev: corroborated.rev });
 		}
 		// The block is marked seen either way — the cohort DID answer, so its freshness was checked,
 		// which is what the read-repair window tracks. A failed convergence therefore waits out the
 		// window before retrying.
 		// NOTE: that damping covers only a block this node holds at an OLDER revision. A block entirely
 		// missing locally never consults the window (`get` triggers on `isMissing` before
-		// `shouldReadRepair`), so a persistently failing acquisition — e.g. a two-node deployment left
-		// at the default `clusterSize: 10`, where the content quorum can never be met — re-fetches an
+		// `shouldReadRepair`), so a persistently failing acquisition — e.g. a two-node deployment that
+		// never set `assumedClusterSize`, where the content quorum can never be met — re-fetches an
 		// archive on every read of that block. Correct, and self-limiting once the cohort can agree; if
 		// it ever shows as read amplification, gate the acquisition step (not the latest-query) on the
 		// same window rather than widening `isMissing`.
@@ -412,7 +504,7 @@ export class CoordinatorRepo implements IRepo {
 
 	/**
 	 * Bring this node up to the cohort-corroborated `corroborated`, returning the revision it holds
-	 * afterwards when that is an advance over `local`, else `undefined`.
+	 * afterwards when that is an advance over `baselineRev`, else `undefined`.
 	 *
 	 * Two mechanisms, cheapest first:
 	 *  1. **Promote a local pending** — free, no network, and the only mechanism that existed before
@@ -435,11 +527,11 @@ export class CoordinatorRepo implements IRepo {
 	private async restoreCorroborated(
 		blockId: BlockId,
 		corroborated: ActionRev,
-		local: ActionRev | undefined,
+		baselineRev: number | undefined,
 		cohortPeerIds: string[]
 	): Promise<number | undefined> {
 		const promoted = await this.promoteCorroborated(blockId, corroborated);
-		if (isAdvanceOver(promoted, local)) {
+		if (isAdvanceOver(promoted, baselineRev)) {
 			return promoted;
 		}
 
@@ -451,6 +543,12 @@ export class CoordinatorRepo implements IRepo {
 			// inside the callback via `saveReplicatedBlock`, which takes the per-block commit latch —
 			// safe to call from here because the read path holds no latch of its own (`StorageRepo.get`
 			// acquires and releases it around the promotion above, and nothing wraps this method).
+			// NOTE: `get` walks its block ids sequentially, so the bound is per block, not per call — a
+			// multi-block read that is missing N blocks against a wholly stalled cohort waits N × this.
+			// Acceptable today (the underlying per-peer archive fetch is itself 1s-bounded and runs the
+			// cohort in parallel, so the 5s is a stall ceiling, not a typical cost). If a cold reader
+			// batching a wide read ever times out above this layer, repair the block ids concurrently
+			// rather than shortening the bound.
 			await withDeadline(
 				this.acquireBlockFromCohort(blockId, corroborated, cohortPeerIds),
 				RECONCILE_TIMEOUT_MS,
@@ -462,7 +560,7 @@ export class CoordinatorRepo implements IRepo {
 			return undefined;
 		}
 		const acquired = await this.readLocalRev(blockId);
-		return isAdvanceOver(acquired, local) ? acquired : undefined;
+		return isAdvanceOver(acquired, baselineRev) ? acquired : undefined;
 	}
 
 	/**
@@ -470,41 +568,36 @@ export class CoordinatorRepo implements IRepo {
 	 * the repair. Returns the local revision afterwards.
 	 *
 	 * A pending-only block (metadata seeded by `savePendingTransaction`, no committed revision) asked
-	 * for a forward revision throws out of `BlockStorage.ensureRevision` when no restore can supply it.
-	 * On THIS path that is an absence, not a read failure — acquisition is precisely the mechanism that
-	 * can supply it — so the throw is logged and swallowed rather than short-circuiting the caller.
+	 * for a forward revision no promotion can reach used to throw out of `BlockStorage.ensureRevision`;
+	 * `StorageRepo.get` now reports it as an entry flagged `unavailable` instead (ticket
+	 * repo-reports-unavailable-vs-absent). On THIS path either shape is an absence, not a read failure —
+	 * acquisition is precisely the mechanism that can supply the revision — so both are logged as
+	 * `promote-unavailable` and stepped over rather than short-circuiting the caller.
 	 */
 	private async promoteCorroborated(blockId: BlockId, corroborated: ActionRev): Promise<number | undefined> {
 		try {
-			return await this.readLocalRev(blockId, { committed: [corroborated], rev: corroborated.rev });
+			const entry = await this.readLocalEntry(blockId, { committed: [corroborated], rev: corroborated.rev });
+			if (entry?.unavailable !== undefined) {
+				log('cluster-fetch:promote-unavailable', { blockId, rev: corroborated.rev, error: entry.unavailable });
+				return undefined;
+			}
+			return entry?.state?.latest?.rev;
 		} catch (err) {
 			log('cluster-fetch:promote-unavailable', { blockId, rev: corroborated.rev, error: (err as Error).message });
 			return undefined;
 		}
 	}
 
-	/** This node's own `latest.rev` for a block, optionally driving a promotion context through the read. */
-	private async readLocalRev(blockId: BlockId, context?: ActionContext): Promise<number | undefined> {
+	/** This node's own answer for a block, optionally driving a promotion context through the read.
+	 *  Callers that care whether the answer is authoritative inspect `entry.unavailable`. */
+	private async readLocalEntry(blockId: BlockId, context?: ActionContext) {
 		const result = await this.storageRepo.get({ blockIds: [blockId], context });
-		return result[blockId]?.state?.latest?.rev;
+		return result[blockId];
 	}
 
-	/**
-	 * How many peers other than this node could corroborate a claim about a block, given a
-	 * cohort view of `peerIds`. Deliberately the MAX of what we observe and what the
-	 * configured cluster size implies: the corroboration floor may only be relaxed for a
-	 * cohort that is genuinely small, never for one that merely *looks* small. `findCluster`
-	 * results are unauthenticated, so a partition — or an attacker with routing influence —
-	 * can shrink this node's view to itself plus one peer; measuring against the configured
-	 * size keeps that shrunken view from talking the requirement down to a single voter.
-	 * The escape hatch for a real two-node deployment is therefore to configure
-	 * `clusterSize: 2`, an explicit operator declaration, mirroring how
-	 * `allowUnvalidatedSmallCluster` gates the membership admission floor.
-	 */
-	private corroboratorCapacity(peerIds: string[]): number {
-		const selfId = this.localPeerId?.toString();
-		const observed = peerIds.filter(id => id !== selfId).length;
-		return Math.max(observed, this.clusterSize - 1);
+	/** This node's own `latest.rev` for a block, optionally driving a promotion context through the read. */
+	private async readLocalRev(blockId: BlockId, context?: ActionContext): Promise<number | undefined> {
+		return (await this.readLocalEntry(blockId, context))?.state?.latest?.rev;
 	}
 
 	/**
@@ -528,13 +621,6 @@ export class CoordinatorRepo implements IRepo {
 	 * `debt-read-repair-commit-cert-verification`.
 	 */
 	private async queryClusterForLatest(peerIds: string[], blockId: BlockId, context?: ActionContext): Promise<ClusterLatestQuery> {
-		// Add timeout wrapper to prevent hanging on unresponsive peers
-		const withTimeout = <T>(promise: Promise<T>, timeoutMs: number): Promise<T | undefined> =>
-			Promise.race([
-				promise,
-				new Promise<undefined>(resolve => setTimeout(() => resolve(undefined), timeoutMs))
-			]);
-
 		// Query peers in parallel for their latest revision (with 1s timeout per peer),
 		// tagging each response with the peer that made it so votes stay distinct.
 		const latestResults = await Promise.allSettled(
@@ -545,6 +631,11 @@ export class CoordinatorRepo implements IRepo {
 			})
 		);
 
+		// NOTE: self-exclusion is keyed on `localPeerId`, which is optional for the single-node/test
+		// construction this class has always tolerated. Left unset, this node's own answer is counted
+		// as a peer claim again. Harmless today — the self answer can only ever corroborate the
+		// revision already held, so the pass declines as `local-current` — but if a future caller can
+		// make self report something the reader does not hold, make `localPeerId` required instead.
 		const selfId = this.localPeerId?.toString();
 		let local: ActionRev | undefined;
 		const claims: RevClaim[] = [];
@@ -558,7 +649,7 @@ export class CoordinatorRepo implements IRepo {
 			claims.push({ peerId: peerIdStr, rev: value.rev, actionId: value.actionId });
 		}
 
-		const capacity = this.corroboratorCapacity(peerIds);
+		const capacity = corroboratorCapacity(peerIds.filter(id => id !== selfId).length, this.repairCorroborationClusterSize);
 		const selected = selectQuorumRev(claims, this.simpleMajorityThreshold, capacity);
 		if (!selected) {
 			log('cluster-fetch:no-quorum', {
@@ -644,10 +735,61 @@ export class CoordinatorRepo implements IRepo {
 			};
 		} catch (error) {
 			log('coordinator-repo:pend-error', { actionId: request.actionId, error: (error as Error).message });
+			const stale = await this.classifyStaleRejection(error, request, allBlockIds);
+			if (stale) return stale;
 			throw error;
 		}
 	}
 
+	/**
+	 * Decide whether a cluster validator rejection was an optimistic-concurrency loss — the block
+	 * already advanced past the requested revision — rather than a genuine validation fault.
+	 * A confirmed loss returns a {@link StaleFailure} carrying `conflict: true` so the caller
+	 * receives a non-success *response* that says plainly it is a lost race: network-transactor's
+	 * pend then takes its stale branch and both writers (`Collection.sync`, and the coordinator's
+	 * multi-collection pendPhase via `isConflictFailure`) retry, instead of a thrown error escaping
+	 * mid-batch (which splits multi-tree commits — see PartialCommitError).
+	 *
+	 * The failure carries no `missing` list: confirmation is a local re-read that reveals the
+	 * revision is taken but not which actions took it, and no consumer rebases from `missing`
+	 * anyway (it is only counted or logged). `conflict` conveys retryability directly instead.
+	 *
+	 * Confirmation is purely local: re-read the affected blocks from our own storage and require
+	 * `latest.rev >= request.rev`. The signed reject-reason text is never consulted — it is
+	 * free-form wire-visible prose and must not become control flow. Anything unconfirmed
+	 * (including read errors during confirmation) stays a throw, preserving fail-fast for
+	 * genuine validation faults.
+	 */
+	private async classifyStaleRejection(error: unknown, request: PendRequest, blockIds: BlockId[]): Promise<StaleFailure | undefined> {
+		if (!(error instanceof ValidatorRejectionError) || request.rev === undefined) return undefined;
+		let results: GetBlockResults;
+		try {
+			results = await this.storageRepo.get({ blockIds });
+		} catch (readError) {
+			log('coordinator-repo:pend-stale-classify-read-error', {
+				actionId: request.actionId,
+				error: (readError as Error).message
+			});
+			return undefined;
+		}
+		for (const blockId of blockIds) {
+			const latest = results[blockId]?.state.latest;
+			if (latest && latest.rev >= request.rev) {
+				log('coordinator-repo:pend-stale-classified', {
+					actionId: request.actionId,
+					blockId,
+					latestRev: latest.rev,
+					requestedRev: request.rev
+				});
+				return { success: false, conflict: true, reason: `stale revision: block ${blockId} at rev ${latest.rev}, requested rev ${request.rev}` };
+			}
+		}
+		// NOTE: conservative — when only remote members saw the newer revision (local storage still
+		// behind), staleness can't be confirmed locally and the rejection stays a throw. If that
+		// shows up in practice, extend confirmation with a quorum read; never trust the reject text.
+		return undefined;
+	}
+
 	async cancel(actionRef: ActionBlocks, options?: MessageOptions): Promise<void> {
 		const blockIds = actionRef.blockIds;
 		await this.verifyResponsibility(blockIds);
@@ -700,23 +842,32 @@ export class CoordinatorRepo implements IRepo {
 				this.markBlocksSeen(blockIds);
 				return { success: true };
 			}
-			// Local cluster didn't execute during consensus. Attempt a local commit,
-			// but tolerate failure (e.g., "pending action not found") when the cluster
-			// already reached consensus — this coordinator was likely picked for commit
-			// after missing the pend phase (unreachable during pend, fresh join, etc.).
-			// The cluster's majority is authoritative; this peer will catch up via sync.
+			// Local cluster didn't execute during consensus. Attempt a local commit, but tolerate
+			// local divergence when the cluster already reached consensus — this coordinator was
+			// likely picked for commit after missing the pend phase (unreachable during pend, fresh
+			// join, etc.). The cluster's majority is authoritative; this peer catches up via sync.
+			//
+			// Divergence reaches us in BOTH shapes and both must be tolerated identically:
+			//   - a THROW ("Pending action … not found"), when we never saw the pend;
+			//   - a RETURNED `success:false` carrying `missing-base-revision`, when we saw the pend
+			//     but not the revision that created the block (see StorageRepo.internalCommit).
+			// Only the throw was tolerated before the refusal existed. Reporting the refusal to the
+			// caller instead would surface a committed transaction as a stale loss: db-core's
+			// commitPhase treats any returned `success:false` as a permanent stale failure, so the
+			// client would retry an action the cluster already landed until it exhausted its budget.
 			try {
 				const result = await this.storageRepo.commit(request, options);
-				if (result.success) this.markBlocksSeen(blockIds);
+				if (result.success) {
+					this.markBlocksSeen(blockIds);
+					return result;
+				}
+				if (isMissingBaseRevisionFailure(result) && clusterReachedCommitConsensus(record)) {
+					return this.tolerateLocalCommitDivergence(request, blockIds, result.reason ?? MISSING_BASE_REVISION_REASON);
+				}
 				return result;
 			} catch (err) {
 				if (clusterReachedCommitConsensus(record)) {
-					log('coordinator-repo:commit-local-failed-cluster-succeeded', {
-						actionId: request.actionId,
-						error: (err as Error).message
-					});
-					this.markBlocksSeen(blockIds);
-					return { success: true };
+					return this.tolerateLocalCommitDivergence(request, blockIds, (err as Error).message);
 				}
 				throw err;
 			}
@@ -725,6 +876,17 @@ export class CoordinatorRepo implements IRepo {
 			throw error;
 		}
 	}
+
+	/**
+	 * Report success for a commit the cluster carried but this peer could not apply locally. The
+	 * blocks are marked seen so the read path treats them as freshness-checked; convergence comes
+	 * from replication (cohort reconcile, or read-driven acquisition), not from replay here.
+	 */
+	private tolerateLocalCommitDivergence(request: CommitRequest, blockIds: BlockId[], detail: string): CommitResult {
+		log('coordinator-repo:commit-local-failed-cluster-succeeded', { actionId: request.actionId, error: detail });
+		this.markBlocksSeen(blockIds);
+		return { success: true };
+	}
 }
 
 /** True if a simple majority of cluster peers signed an approving commit. */