@optimystic/db-p2p 0.18.0 → 0.19.0

package/src/libp2p-node-base.ts

@@ -802,12 +802,20 @@ export async function createLibp2pNodeBase(
 		options.transactionStateStore
 	);
 
-	// Create callback for querying cluster peers for their latest block revision
+	// Create callback for querying cluster peers for their latest block revision. Three-way
+	// contract (see ClusterLatestCallback): an ActionRev is the peer's claim, a resolved
+	// `undefined` is the peer answering "I hold nothing", and a REJECTION is silence — the
+	// coordinator counts it as "did not answer" and refuses to report an authoritative absent
+	// over it. Transport errors must therefore propagate, not collapse into `undefined` (that
+	// collapse let a slow two-node cohort report a missing block as authoritatively absent —
+	// ticket cluster-read-consult-cannot-report-unreachable).
 	const clusterLatestCallback: ClusterLatestCallback = async (peerId, blockId, context?) => {
 		// Self-read short-circuit: dialling self via SyncClient is a round trip
 		// with no remote on the other end, and on nodes without listen addresses
 		// (solo WebSocket-only, bare-RN, etc.) the self-dial can hang the dial
-		// queue. Read directly from the local storage repo instead.
+		// queue. Read directly from the local storage repo instead. The catch stays:
+		// a local storage error is not a cohort peer being unreachable, and the
+		// coordinator ignores a self rejection anyway.
 		if (peerId.equals(node.peerId)) {
 			try {
 				const result = await storageRepo.get({ blockIds: [blockId], context });
@@ -817,21 +825,22 @@ export async function createLibp2pNodeBase(
 			}
 		}
 		const syncClient = new SyncClient(peerId, keyNetwork, protocolPrefix);
-		try {
-			const response = await syncClient.requestBlock({ blockId, rev: undefined });
-			if (response.success && response.archive) {
-				const revisions = Object.keys(response.archive.revisions).map(Number);
-				if (revisions.length > 0) {
-					const maxRev = Math.max(...revisions);
-					const revisionData = response.archive.revisions[maxRev];
-					if (revisionData?.action) {
-						return { actionId: revisionData.action.actionId, rev: maxRev };
-					}
+		// No try/catch: a dial or protocol failure rejects through to the coordinator, whose
+		// per-peer deadline also bounds a hung request — slowness needs no race here.
+		const response = await syncClient.requestBlock({ blockId, rev: undefined });
+		if (response.success && response.archive) {
+			const revisions = Object.keys(response.archive.revisions).map(Number);
+			if (revisions.length > 0) {
+				const maxRev = Math.max(...revisions);
+				const revisionData = response.archive.revisions[maxRev];
+				if (revisionData?.action) {
+					return { actionId: revisionData.action.actionId, rev: maxRev };
 				}
 			}
-		} catch {
-			// Peer may be unreachable - return undefined to skip this peer
 		}
+		// The peer DID answer, without data: `success:false` is the sync service's "Block not
+		// found in local storage", and an archive with no usable revisions holds nothing either
+		// way. Both are absent claims, not silence.
 		return undefined;
 	};
 

package/src/repo/coordinator-repo.ts

@@ -1,5 +1,5 @@
 import type { PendRequest, ActionBlocks, IRepo, MessageOptions, CommitResult, GetBlockResults, PendResult, StaleFailure, BlockGets, CommitRequest, RepoMessage, IKeyNetwork, ICluster, ClusterConsensusConfig, BlockId, ActionRev, ActionContext, ClusterRecord } from "@optimystic/db-core";
-import { LruMap, blockIdsForTransforms, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
+import { LruMap, blockIdsForTransforms, highestStaleAt, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
 import { ClusterCoordinator, ValidatorRejectionError } from "./cluster-coordinator.js";
 import type { PeerId } from "@libp2p/interface";
 import { peerIdFromString } from "@libp2p/peer-id";
@@ -27,6 +27,9 @@ const log = createLogger('coordinator-repo');
  */
 export type AcquireBlockCallback = ReconcileBlockCallback;
 
+/** How long one cohort peer gets to answer the latest-revision consult before it counts as silent. */
+const LATEST_QUERY_TIMEOUT_MS = 1000;
+
 /** True when a freshly-read local revision is strictly ahead of the baseline the repair started from. */
 function isAdvanceOver(rev: number | undefined, baselineRev: number | undefined): boolean {
 	return typeof rev === 'number' && (baselineRev === undefined || rev > baselineRev);
@@ -46,22 +49,6 @@ function withDeadline<T>(promise: Promise<T>, ms: number, label: string): Promis
 	});
 }
 
-/**
- * Resolve `undefined` if `promise` has not settled within `ms` — a slow peer is skipped, not an
- * error. Same timer discipline as {@link withDeadline}: one repair pass races this once per cohort
- * peer, so leaving the handles pending would keep the event loop alive for the full timeout after
- * every peer has already answered.
- */
-function withTimeout<T>(promise: Promise<T>, ms: number): Promise<T | undefined> {
-	let timer: ReturnType<typeof setTimeout> | undefined;
-	const expiry = new Promise<undefined>(resolve => {
-		timer = setTimeout(() => resolve(undefined), ms);
-	});
-	return Promise.race([promise, expiry]).finally(() => {
-		if (timer !== undefined) clearTimeout(timer);
-	});
-}
-
 /**
  * What one round of polling the cohort learned about a block: the revision the OTHER
  * cohort members corroborated, and what this node itself already holds. The two are kept
@@ -74,6 +61,14 @@ interface ClusterLatestQuery {
 	corroborated?: ActionRev;
 	/** This node's own latest for the block, as answered by the callback's self short-circuit. */
 	local?: ActionRev;
+	/**
+	 * Cohort peers (self excluded) that never answered the consult — the callback rejected
+	 * (dial failure, protocol error) or blew the per-peer deadline. Silence, not evidence:
+	 * these are never counted as claims, but while this is non-empty a caller must not treat
+	 * "nothing corroborated" as an authoritative absence, because a silent peer could be the
+	 * sole holder.
+	 */
+	silent: string[];
 }
 
 /**
@@ -85,8 +80,18 @@ interface LocalClusterWithExecutionTracking extends ICluster {
 }
 
 /**
- * Callback to query a cluster peer for their latest revision of a block.
- * Returns the peer's latest ActionRev if they have the block, undefined otherwise.
+ * Callback to query a cluster peer for their latest revision of a block. Three-way contract:
+ *  - resolves an `ActionRev` — the peer answered and holds the block at that revision;
+ *  - resolves `undefined` — the peer answered and holds NOTHING (an absent claim);
+ *  - REJECTS — the peer could not be asked at all (dial failure, protocol error).
+ *
+ * The distinction between the last two is load-bearing: `queryClusterForLatest` counts a
+ * rejection as a SILENT peer, which stops `CoordinatorRepo.get` from reporting a locally-missing
+ * block as an authoritative absent, while a resolved `undefined` is a real answer that keeps the
+ * absent authoritative. Implementations must therefore let transport errors propagate rather than
+ * swallowing them into `undefined` (see the implementations in `libp2p-node-base` and the mesh
+ * harness's `silentPeers` failure knob). Slowness needs no handling here — the caller deadlines
+ * each query and treats expiry as silence.
  */
 export type ClusterLatestCallback = (peerId: PeerId, blockId: BlockId, context?: ActionContext) => Promise<ActionRev | undefined>;
 
@@ -321,7 +326,8 @@ export class CoordinatorRepo implements IRepo {
 		// NOTE: NetworkTransactor.get treats an authoritative "absent" ({ state: {} })
 		// as final and no longer retries it (ticket txn-perf-authoritative-notfound),
 		// relying on this cluster reconciliation to have already run. When the consult
-		// runs and FAILS, the entry is flagged `unavailable: 'peers-unreachable'` below,
+		// FAILS outright — or runs while part of the cohort stays SILENT and the block
+		// stays missing — the entry is flagged `unavailable: 'peers-unreachable'` below,
 		// which re-enables the transactor-level retry against a different peer. If a
 		// coordinator is configured WITHOUT clusterLatestCallback, there is no cohort to
 		// consult and the local answer IS the whole truth — it stays authoritative, with
@@ -346,7 +352,7 @@ export class CoordinatorRepo implements IRepo {
 				}
 
 				try {
-					await this.fetchBlockFromCluster(blockId, blockGets.context, localRev);
+					const { inconclusive } = await this.fetchBlockFromCluster(blockId, blockGets.context, localRev);
 					const refreshed = await this.storageRepo.get({ blockIds: [blockId], context: blockGets.context }, options);
 					const newRev = refreshed[blockId]?.state?.latest?.rev;
 					if (refreshed[blockId]) {
@@ -359,24 +365,20 @@ export class CoordinatorRepo implements IRepo {
 							log('cluster-tx:read-repair-noop', { blockId });
 						}
 					}
+					// The consult ran but came back INCONCLUSIVE (a silent cohort peer, or a
+					// corroborated revision this node could not acquire — see
+					// fetchBlockFromCluster). Either way the reader cannot rule the block out,
+					// so a still-missing block must not pose as an authoritative absent. When
+					// the whole cohort answers "holds nothing" the absent stays authoritative —
+					// the new-collection probe against a healthy cohort stays one round-trip.
+					if (isMissing && inconclusive) {
+						this.flagUnconfirmedAbsence(localResult, blockId);
+					}
 				} catch (err) {
 					log('cluster-fetch:error', { blockId, error: (err as Error).message });
 					// The consult that was supposed to make this answer trustworthy did not run.
-					// Only a locally-MISSING block is downgraded to `unavailable`: a merely-stale
-					// block still has a real local answer and stays authoritative. A sharper flag
-					// already on the entry (storage's 'unmaterializable') is never overwritten.
-					// NOTE: a consult that runs but corroborates nothing (per-peer timeouts and
-					// "peer holds nothing" both surface as absent claims → no quorum) does NOT
-					// land here and stays an authoritative absent — the common new-collection
-					// probe against a healthy cohort takes exactly that path, and the callback
-					// contract cannot distinguish the two without counting responders.
 					if (isMissing) {
-						const entry = localResult[blockId];
-						if (!entry) {
-							localResult[blockId] = { state: {}, unavailable: 'peers-unreachable' };
-						} else if (entry.unavailable === undefined) {
-							entry.unavailable = 'peers-unreachable';
-						}
+						this.flagUnconfirmedAbsence(localResult, blockId);
 					}
 				}
 			}
@@ -385,6 +387,22 @@ export class CoordinatorRepo implements IRepo {
 		return localResult;
 	}
 
+	/**
+	 * Downgrade an absence the coordinator could not confirm to `unavailable: 'peers-unreachable'` —
+	 * the flag `NetworkTransactor.get` retries against another peer instead of taking as final.
+	 *
+	 * No-op once the entry carries a real answer (the consult restored the block) or a sharper flag
+	 * (storage's `'unmaterializable'`), so callers only need to establish that the answer is a guess.
+	 */
+	private flagUnconfirmedAbsence(results: GetBlockResults, blockId: BlockId): void {
+		const entry = results[blockId];
+		if (!entry) {
+			results[blockId] = { state: {}, unavailable: 'peers-unreachable' };
+		} else if (!entry.state?.latest && entry.unavailable === undefined) {
+			entry.unavailable = 'peers-unreachable';
+		}
+	}
+
 	/** Decide whether the read-repair policy wants us to consult the cluster for a present-but-possibly-stale block. */
 	private shouldReadRepair(blockId: BlockId): boolean {
 		switch (this.readRepairMode) {
@@ -427,14 +445,22 @@ export class CoordinatorRepo implements IRepo {
 	 * One repair pass for a block: ask the cohort what it holds, and converge onto that if it is
 	 * ahead of `localRev` — the revision the caller's read already loaded, and the baseline every
 	 * decision below is measured against.
+	 *
+	 * Returns the one thing `get` needs beyond the storage side effects: whether the pass was
+	 * INCONCLUSIVE — it neither confirmed the cohort holds nothing nor left this node holding the
+	 * block. Two ways that happens: a cohort peer other than this node stayed SILENT (rejected
+	 * callback or per-peer deadline), or a revision WAS corroborated and the convergence onto it
+	 * failed. In both, `get` has learned that its local absence may be wrong, so it must not report
+	 * a still-missing block as an authoritative absent. Paths that consult nobody (no cohort,
+	 * solo-self) are conclusive: there, the local answer genuinely is the whole truth.
 	 */
-	private async fetchBlockFromCluster(blockId: BlockId, context?: ActionContext, localRev?: number): Promise<void> {
-		if (!this.clusterLatestCallback) return;
+	private async fetchBlockFromCluster(blockId: BlockId, context?: ActionContext, localRev?: number): Promise<{ inconclusive: boolean }> {
+		if (!this.clusterLatestCallback) return { inconclusive: false };
 
 		const blockIdBytes = new TextEncoder().encode(blockId);
 		const peers = await this.keyNetwork.findCluster(blockIdBytes);
 		const peerIds = peers ? Object.keys(peers) : [];
-		if (peerIds.length === 0) return;
+		if (peerIds.length === 0) return { inconclusive: false };
 
 		// Solo-cluster short-circuit: the only responsible peer is us. There is no
 		// remote to sync from, so skip the callback entirely. Querying ourselves
@@ -446,13 +472,17 @@ export class CoordinatorRepo implements IRepo {
 			&& peerIds[0] === this.localPeerId.toString()
 		) {
 			log('cluster-fetch:solo-self-skip', { blockId });
-			return;
+			return { inconclusive: false };
 		}
 
-		const { corroborated, local } = await this.queryClusterForLatest(peerIds, blockId, context);
+		const { corroborated, local, silent } = await this.queryClusterForLatest(peerIds, blockId, context);
+		// Any silence flags the WHOLE consult, not a fraction of it (fail-closed): one silent
+		// peer could be the sole holder, and the cost — an extra transactor-level retry against
+		// another coordinator — is paid only while a peer is actually unreachable.
+		const cohortSilent = silent.length > 0;
 		// Nothing corroborated: keep local data AND stay eligible for repair — marking the
 		// block seen here would suppress the next attempt for the whole read-repair window.
-		if (!corroborated) return;
+		if (!corroborated) return { inconclusive: cohortSilent };
 
 		// The self answer is the sharper baseline (same storage, same context, read alongside the
 		// cohort's), but it exists only when `findCluster` returned this node. A soft serve for a
@@ -475,7 +505,7 @@ export class CoordinatorRepo implements IRepo {
 		if (baselineRev !== undefined && corroborated.rev <= baselineRev) {
 			log('cluster-fetch:local-current', { blockId, localRev: baselineRev, clusterRev: corroborated.rev });
 			this.markBlocksSeen([blockId]);
-			return;
+			return { inconclusive: cohortSilent };
 		}
 
 		// Corroborated revision is ahead of ours — converge onto it.
@@ -489,6 +519,10 @@ export class CoordinatorRepo implements IRepo {
 		} else {
 			log('cluster-fetch:not-restored', { blockId, localRev: baselineRev, clusterRev: corroborated.rev });
 		}
+		// A corroborated revision this node failed to converge onto is inconclusive in its own right,
+		// even with the whole cohort answering: the reader has just been TOLD the block exists, so
+		// reporting it absent would be a lie of the same kind a silent peer causes (see `get`).
+		const inconclusive = cohortSilent || rev === undefined;
 		// The block is marked seen either way — the cohort DID answer, so its freshness was checked,
 		// which is what the read-repair window tracks. A failed convergence therefore waits out the
 		// window before retrying.
@@ -500,6 +534,7 @@ export class CoordinatorRepo implements IRepo {
 		// it ever shows as read amplification, gate the acquisition step (not the latest-query) on the
 		// same window rather than widening `isMissing`.
 		this.markBlocksSeen([blockId]);
+		return { inconclusive };
 	}
 
 	/**
@@ -621,13 +656,23 @@ export class CoordinatorRepo implements IRepo {
 	 * `debt-read-repair-commit-cert-verification`.
 	 */
 	private async queryClusterForLatest(peerIds: string[], blockId: BlockId, context?: ActionContext): Promise<ClusterLatestQuery> {
-		// Query peers in parallel for their latest revision (with 1s timeout per peer),
-		// tagging each response with the peer that made it so votes stay distinct.
+		// Query peers in parallel for their latest revision. Each query is DEADLINED (rejects), not
+		// raced-to-undefined: a peer that blows the deadline lands in the silent set below exactly
+		// like a dial failure, because a slow peer and a peer claiming "I hold nothing" must produce
+		// different answers (ticket cluster-read-consult-cannot-report-unreachable).
+		// NOTE: LATEST_QUERY_TIMEOUT_MS is a LAN-shaped budget. A cohort whose round trip honestly
+		// exceeds it now reads as permanently silent, which is safe (the read is flagged, not
+		// mis-reported) but makes every miss cost a transactor-level retry. If a WAN deployment shows
+		// steady `cluster-fetch:peers-silent` against healthy peers, raise this rather than softening
+		// the deadline back into an absent claim.
 		const latestResults = await Promise.allSettled(
 			peerIds.map(async peerIdStr => {
 				const peerId = peerIdFromString(peerIdStr);
-				const value = await withTimeout(this.clusterLatestCallback!(peerId, blockId, context), 1000);
-				return { peerIdStr, value };
+				return await withDeadline(
+					this.clusterLatestCallback!(peerId, blockId, context),
+					LATEST_QUERY_TIMEOUT_MS,
+					`latest query to ${peerIdStr}`
+				);
 			})
 		);
 
@@ -639,15 +684,30 @@ export class CoordinatorRepo implements IRepo {
 		const selfId = this.localPeerId?.toString();
 		let local: ActionRev | undefined;
 		const claims: RevClaim[] = [];
-		for (const result of latestResults) {
-			if (result.status !== 'fulfilled' || !result.value.value) continue;
-			const { peerIdStr, value } = result.value;
+		const silent: string[] = [];
+		// `allSettled` preserves input order, so results correlate to `peerIds` by index — a
+		// rejected entry carries no payload of its own, and its peer id is what `silent` records.
+		for (let i = 0; i < latestResults.length; i++) {
+			const result = latestResults[i]!;
+			const peerIdStr = peerIds[i]!;
+			if (result.status !== 'fulfilled') {
+				// Silence: the callback rejected or the deadline expired. Never a claim. Self is
+				// excluded — its short-circuit reads local storage, and a local read error is not a
+				// cohort peer being unreachable.
+				if (peerIdStr !== selfId) silent.push(peerIdStr);
+				continue;
+			}
+			const value = result.value;
 			if (peerIdStr === selfId) {
 				local = value;
 				continue;
 			}
+			if (!value) continue; // responded, holds nothing — an absent claim, not silence
 			claims.push({ peerId: peerIdStr, rev: value.rev, actionId: value.actionId });
 		}
+		if (silent.length > 0) {
+			log('cluster-fetch:peers-silent', { blockId, silent: silent.length, consulted: peerIds.length });
+		}
 
 		const capacity = corroboratorCapacity(peerIds.filter(id => id !== selfId).length, this.repairCorroborationClusterSize);
 		const selected = selectQuorumRev(claims, this.simpleMajorityThreshold, capacity);
@@ -657,7 +717,7 @@ export class CoordinatorRepo implements IRepo {
 				responders: claims.length,
 				required: quorumSize(claims.length, this.simpleMajorityThreshold, capacity)
 			});
-			return { local };
+			return { local, silent };
 		}
 
 		// Best-effort: penalize peers whose claim contradicts the corroborated pair
@@ -665,7 +725,7 @@ export class CoordinatorRepo implements IRepo {
 		// rev). A lower rev is just lag, never penalized. Never let this throw.
 		this.penalizeContradictingRevClaims(claims, selected, blockId);
 
-		return { corroborated: { actionId: selected.actionId, rev: selected.rev }, local };
+		return { corroborated: { actionId: selected.actionId, rev: selected.rev }, local, silent };
 	}
 
 	/**
@@ -761,7 +821,8 @@ export class CoordinatorRepo implements IRepo {
 	 * genuine validation faults.
 	 */
 	private async classifyStaleRejection(error: unknown, request: PendRequest, blockIds: BlockId[]): Promise<StaleFailure | undefined> {
-		if (!(error instanceof ValidatorRejectionError) || request.rev === undefined) return undefined;
+		const requestedRev = request.rev;
+		if (!(error instanceof ValidatorRejectionError) || requestedRev === undefined) return undefined;
 		let results: GetBlockResults;
 		try {
 			results = await this.storageRepo.get({ blockIds });
@@ -772,21 +833,36 @@ export class CoordinatorRepo implements IRepo {
 			});
 			return undefined;
 		}
-		for (const blockId of blockIds) {
+		// Scan EVERY block rather than stopping at the first confirmation: several of the request's
+		// blocks can be past the requested revision at different revisions, and it is the highest
+		// that the loser's next request has to clear (see `highestStaleAt`). Both the reported
+		// number and the reason prose name that block, so they never disagree.
+		const staleAt = highestStaleAt(blockIds.map(blockId => {
 			const latest = results[blockId]?.state.latest;
-			if (latest && latest.rev >= request.rev) {
-				log('coordinator-repo:pend-stale-classified', {
-					actionId: request.actionId,
-					blockId,
-					latestRev: latest.rev,
-					requestedRev: request.rev
-				});
-				return { success: false, conflict: true, reason: `stale revision: block ${blockId} at rev ${latest.rev}, requested rev ${request.rev}` };
-			}
+			return latest && latest.rev >= requestedRev ? { blockId, rev: latest.rev } : undefined;
+		}));
+		if (staleAt) {
+			log('coordinator-repo:pend-stale-classified', {
+				actionId: request.actionId,
+				blockId: staleAt.blockId,
+				latestRev: staleAt.rev,
+				requestedRev
+			});
+			return {
+				success: false,
+				conflict: true,
+				reason: `stale revision: block ${staleAt.blockId} at rev ${staleAt.rev}, requested rev ${requestedRev}`,
+				// The same fact as the reason prose, but as data. This is the ONLY place a losing
+				// writer can learn the revision it lost to, since this failure deliberately carries
+				// no `missing`. Confirmed-local: read out of our own storage just above.
+				staleAt
+			};
 		}
 		// NOTE: conservative — when only remote members saw the newer revision (local storage still
 		// behind), staleness can't be confirmed locally and the rejection stays a throw. If that
 		// shows up in practice, extend confirmation with a quorum read; never trust the reject text.
+		// `staleAt` is absent on this path for the same reason, and deliberately so — there is no
+		// confirmed number to report, and the field's contract forbids inferring one from that text.
 		return undefined;
 	}
 

package/src/storage/storage-repo.ts

@@ -3,11 +3,12 @@ import type {
 	ActionId, BlockGets, ActionPending, PendSuccess, ActionTransform, ActionTransforms,
 	GetBlockResult, IBlock, ActionRev, BlockUnavailableReason,
 	PendValidationHook,
-	CollectionId, IBlockChangeNotifier, CollectionChangeListener, CollectionChangeEvent
+	CollectionId, IBlockChangeNotifier, CollectionChangeListener, CollectionChangeEvent,
+	StaleFailure
 } from "@optimystic/db-core";
 import {
 	Latches, transformForBlockId, applyTransform, groupBy, concatTransform, emptyTransforms,
-	blockIdsForTransforms, transformsFromTransform
+	blockIdsForTransforms, transformsFromTransform, highestStaleAt
 } from "@optimystic/db-core";
 import { asyncIteratorToArray } from "../it-utility.js";
 import type { IBlockStorage } from "./i-block-storage.js";
@@ -374,6 +375,10 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 		log('pend actionId=%s blockIds=%d rev=%s', request.actionId, blockIds.length, request.rev);
 		const pendings: ActionPending[] = [];
 		const missing: ActionTransforms[] = [];
+		// Highest revision this node confirms holding among the blocks that are at or past the
+		// requested one — reported as StaleFailure.staleAt so a losing writer learns the number
+		// instead of parsing prose. Confirmed-local only: we read it from our own storage below.
+		let staleAt: StaleFailure['staleAt'];
 
 		// Potential race condition: A concurrent commit operation could complete
 		// between the conflict checks (latest.rev, listPendingTransactions) and the
@@ -394,6 +399,13 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 			if (request.rev !== undefined || transforms.insert) {
 				const latest = await blockStorage.getLatest();
 				if (latest && latest.rev >= (request.rev ?? 0)) {
+					// Only a real revision race yields a meaningful `staleAt`. When `request.rev` is
+					// undefined this same branch fires for an insert collision (the comparison degrades
+					// to `latest.rev >= 0`, true for any existing block), and reporting that block's
+					// revision would be a number that answers a question nobody asked.
+					if (request.rev !== undefined) {
+						staleAt = highestStaleAt([staleAt, { blockId, rev: latest.rev }]);
+					}
 					const transforms = await asyncIteratorToArray(blockStorage.listRevisions(request.rev ?? 0, latest.rev));
 					for (const actionRev of transforms) {
 						const transform = await blockStorage.getTransaction(actionRev.actionId);
@@ -415,7 +427,8 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 			return {
 				success: false,
 				conflict: true,
-				missing
+				missing,
+				...(staleAt === undefined ? {} : { staleAt })
 			};
 		}
 
@@ -502,6 +515,10 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 			//   - toCommit: latest.rev < request.rev or no latest yet → run internalCommit.
 			const toCommit: { blockId: BlockId, storage: IBlockStorage }[] = [];
 			const missedCommits: { blockId: BlockId, transforms: ActionTransform[] }[] = [];
+			// Highest revision among the blocks confirmed lost to a newer one — reported as
+			// StaleFailure.staleAt. The idempotent-retry `continue` below is a no-op, not a loss,
+			// so it never seeds this.
+			let staleAt: StaleFailure['staleAt'];
 			for (const entry of blockStorages) {
 				const { blockId, storage } = entry;
 				const latest = await storage.getLatest();
@@ -510,6 +527,7 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 						// Idempotent no-op for this block — already committed with this exact (actionId, rev).
 						continue;
 					}
+					staleAt = highestStaleAt([staleAt, { blockId, rev: latest.rev }]);
 					const transforms: ActionTransform[] = [];
 					for await (const actionRev of storage.listRevisions(request.rev, latest.rev)) {
 						const transform = await storage.getTransaction(actionRev.actionId);
@@ -532,7 +550,8 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 				log('commit:stale actionId=%s missed=%d', request.actionId, missedCommits.length);
 				return { // Return directly, locks will be released in finally
 					success: false,
-					missing: perBlockActionTransformsToPerAction(missedCommits)
+					missing: perBlockActionTransformsToPerAction(missedCommits),
+					...(staleAt === undefined ? {} : { staleAt })
 				};
 			}
 

package/src/testing/mesh-harness.ts

@@ -42,6 +42,13 @@ export interface MeshFailureConfig {
 	failingPeers?: Set<string>;
 	/** Make findCluster return empty (simulate DHT failure) */
 	findClusterFails?: boolean;
+	/**
+	 * Peers that are unreachable on the READ path: their latest-revision consult
+	 * (`ClusterLatestCallback`) REJECTS — silence the coordinator must count as "did not
+	 * answer", never as the peer claiming absence — and the reconcile/acquire transfer skips
+	 * them as a source. Distinct from `failingPeers`, which fails cluster (write) updates.
+	 */
+	silentPeers?: Set<string>;
 }
 
 class MockPeerNetwork implements IPeerNetwork {
@@ -118,10 +125,13 @@ export interface Mesh {
  *
  * `nodes` is captured by reference and is fully populated by the time either caller invokes it.
  */
-const makeReconcileBlock = (nodes: MeshNode[], selfPeerId: string, storageRepo: StorageRepo): ReconcileBlockCallback =>
+const makeReconcileBlock = (nodes: MeshNode[], selfPeerId: string, storageRepo: StorageRepo, failures: MeshFailureConfig): ReconcileBlockCallback =>
 	async (blockId, committed, cohortPeerIds) => {
 		for (const peerIdStr of cohortPeerIds) {
 			if (peerIdStr === selfPeerId) continue;
+			// A silent peer cannot serve bytes either — without this, a test that silenced a
+			// peer's consult could still accidentally converge THROUGH that peer.
+			if (failures.silentPeers?.has(peerIdStr)) continue;
 			const target = nodes.find(n => n.peerId.toString() === peerIdStr);
 			if (!target) continue;
 			const result = await target.storageRepo.get({ blockIds: [blockId] }, { skipClusterFetch: true } as any);
@@ -177,7 +187,7 @@ export async function createMesh(nodeCount: number, options: MeshOptions): Promi
 
 		// Active reconciliation: when a member commits a block it never pended (cohort drift),
 		// pull the committed revision from a sibling cohort node that holds it.
-		const reconcileBlock = makeReconcileBlock(nodes, peerId.toString(), storageRepo);
+		const reconcileBlock = makeReconcileBlock(nodes, peerId.toString(), storageRepo, failures);
 
 		const member = clusterMember({
 			storageRepo,
@@ -223,6 +233,12 @@ export async function createMesh(nodeCount: number, options: MeshOptions): Promi
 		// existed to expose. Transfer now happens where it does in production: through
 		// `acquireBlockFromCohort` below, gated on a corroborated revision.
 		const clusterLatestCallback: ClusterLatestCallback = async (peerId: PeerId, blockId: BlockId, context?): Promise<ActionRev | undefined> => {
+			// Silence: the peer never answers. REJECTS, mirroring what a dial failure does to the
+			// production callback — the coordinator must count this as "did not answer", never as
+			// an absent claim (a resolved `undefined` remains the peer answering "I hold nothing").
+			if (failures.silentPeers?.has(peerId.toString())) {
+				throw new Error(`Peer ${peerId.toString()} is silent`);
+			}
 			const target = nodes.find(n => n.peerId.equals(peerId));
 			if (!target) return undefined;
 			const result = await target.storageRepo.get(
@@ -265,7 +281,7 @@ export async function createMesh(nodeCount: number, options: MeshOptions): Promi
 			clusterLatestCallback,
 			// The read path's transfer mechanism — the same callback the member uses on the commit path,
 			// mirroring how `libp2p-node-base` shares one `reconcileBlock` between both.
-			acquireBlockFromCohort: makeReconcileBlock(nodes, node.peerId.toString(), node.storageRepo)
+			acquireBlockFromCohort: makeReconcileBlock(nodes, node.peerId.toString(), node.storageRepo, failures)
 		});
 	}