@optimystic/db-p2p 0.16.3 → 0.18.0

package/src/repo/service.ts

@@ -8,6 +8,7 @@ import { encodePeers, type RedirectPayload } from './redirect.js'
 import { MAX_BLOCK_MESSAGE_BYTES } from '../protocol-limits.js'
 import type { Uint8ArrayList } from 'uint8arraylist'
 import { createLogger } from '../logger.js'
+import { createInboundStreamAuthorization, type InboundStreamAuthorization, type InboundStreamAuthorizationInit } from '../inbound-authorization.js'
 
 const debugLog = createLogger('repo-service')
 
@@ -38,7 +39,7 @@ export type RepoServiceComponents = BaseComponents & {
 	libp2p?: Libp2p
 }
 
-export type RepoServiceInit = {
+export type RepoServiceInit = InboundStreamAuthorizationInit & {
 	protocol?: string,
 	protocolPrefix?: string,
 	maxInboundStreams?: number,
@@ -81,6 +82,8 @@ export class RepoService implements Startable {
 	 * self identity, and connection addrs through this explicitly-set reference.
 	 */
 	private libp2pRef: Libp2p | undefined
+	/** Optional embedder authorization gate; `undefined` (the default) means no check runs. */
+	private readonly authorization: InboundStreamAuthorization | undefined
 
 	constructor(components: RepoServiceComponents, init: RepoServiceInit = {}) {
 		this.components = components
@@ -92,6 +95,7 @@ export class RepoService implements Startable {
 		this.repo = components.repo
 		this.running = false
 		this.responsibilityK = init.responsibilityK ?? 1
+		this.authorization = createInboundStreamAuthorization(init, this.protocol, (msg, ...args) => this.log.error(msg, ...args))
 	}
 
 	readonly [Symbol.toStringTag] = '@libp2p/repo-service'
@@ -236,8 +240,8 @@ export class RepoService implements Startable {
 	/**
 	 * Handle incoming streams on the repo protocol
 	 */
-	private handleIncomingStream(stream: Stream, connection: Connection): void {
-		const peerId = connection.remotePeer
+	private handleIncomingStream(stream: Stream, connection?: Connection): void {
+		const peerId = connection?.remotePeer
 
 		const processStream = async function* (this: RepoService, source: AsyncIterable<Uint8ArrayList>) {
 			for await (const msg of source) {
@@ -278,6 +282,9 @@ export class RepoService implements Startable {
 
 		void (async () => {
 			try {
+				// Authorization runs before ANY decoding or execution. Guarded on the field so a
+				// node without a predicate keeps the original path untouched.
+				if (this.authorization && await this.authorization.deny(stream, peerId?.toString())) return
 				const responses = pipe(
 					stream,
 					(source) => lpDecode(source, { maxDataLength: MAX_BLOCK_MESSAGE_BYTES }),

package/src/storage/block-storage.ts

@@ -36,6 +36,17 @@ export class BlockStorage implements IBlockStorage {
 	async getBlock(rev?: number): Promise<{ block: IBlock, actionRev: ActionRev } | undefined> {
 		const meta = await this.storage.getMetadata(this.blockId);
 		if (!meta) {
+			// No metadata at all ⇒ this node has never seen the block, and reads report it absent
+			// WITHOUT consulting `restoreCallback`. That is deliberate, not an oversight: `restoreCallback`
+			// is reachable only from ensureRevision below, so a never-seen block is never fetched HERE.
+			// Attempting a fetch at this layer would turn every read of a genuinely non-existent block —
+			// the common case for an insert probing for a collision — into a network round trip, because
+			// storage cannot tell "nobody has this" from "I don't have this".
+			//
+			// The layer that CAN tell them apart makes that call instead: `CoordinatorRepo` acquires the
+			// block only once the cohort has corroborated a `(rev, actionId)` for it
+			// (`restoreCorroborated` → `acquireBlockFromCohort`), so an id no peer claims still costs
+			// nothing beyond the latest-query it already performed. Keep this early return as-is.
 			return undefined;
 		}
 

package/src/storage/storage-repo.ts

@@ -1,7 +1,7 @@
 import type {
 	IRepo, MessageOptions, BlockId, CommitRequest, CommitResult, GetBlockResults, PendRequest, PendResult, ActionBlocks,
 	ActionId, BlockGets, ActionPending, PendSuccess, ActionTransform, ActionTransforms,
-	GetBlockResult, IBlock, ActionRev,
+	GetBlockResult, IBlock, ActionRev, BlockUnavailableReason,
 	PendValidationHook,
 	CollectionId, IBlockChangeNotifier, CollectionChangeListener, CollectionChangeEvent
 } from "@optimystic/db-core";
@@ -44,6 +44,37 @@ export async function withBlockCommitLatch<T>(blockId: BlockId, fn: () => Promis
 	}
 }
 
+/**
+ * Stable, greppable prefix on the failure reason a commit carries when this node cannot materialize
+ * the revision it was asked to record. It is a STRING marker rather than only an error class because
+ * {@link StorageRepo.commit} reports per-block faults as `StaleFailure.reason` (a plain string that
+ * also crosses the wire), so the class identity is lost by the time a caller inspects the result.
+ */
+export const MISSING_BASE_REVISION_REASON = 'missing-base-revision';
+
+/**
+ * This node was asked to commit revision N of a block it holds no materializable base for, so
+ * applying the transform would materialize nothing while `latest` advanced to N — a block that is
+ * then unreadable locally, unservable to peers, and that rejects every later write (see
+ * {@link StorageRepo.internalCommit}). The commit is refused instead; the caller heals the node
+ * out-of-band (`ClusterMember` pulls the committed revision from a cohort peer) and retries.
+ */
+export class MissingBaseRevisionError extends Error {
+	constructor(readonly blockId: BlockId, readonly rev: number, detail: string) {
+		super(`${MISSING_BASE_REVISION_REASON}: block ${blockId} cannot materialize rev ${rev} — ${detail}`);
+		this.name = 'MissingBaseRevisionError';
+	}
+}
+
+/**
+ * True when a {@link CommitResult} failed because this node holds no materializable base for one of
+ * the committed blocks. Distinguishes that recoverable divergence (heal by fetching the block from a
+ * cohort peer) from a genuine storage fault, which must still propagate.
+ */
+export function isMissingBaseRevisionFailure(result: CommitResult): boolean {
+	return !result.success && (result.reason?.startsWith(MISSING_BASE_REVISION_REASON) ?? false);
+}
+
 export type StorageRepoOptions = {
 	/** Optional hook to validate transactions in PendRequests */
 	validatePend?: PendValidationHook;
@@ -155,6 +186,11 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 		const promotions: { collectionId: CollectionId, blockId: BlockId, actionId: ActionId, rev: number }[] = [];
 		const results = await Promise.all(distinctBlockIds.map(async (blockId) => {
 			const blockStorage = this.createBlockStorage(blockId);
+			// Set when this node KNOWS its answer for the block is a guess: the promotion
+			// below refused for a missing base, or getBlock() threw (truncated history /
+			// failed restore). An absent-reading block then reports `unavailable` instead of
+			// posing as an authoritative "never existed" — see BlockUnavailableReason.
+			let unavailable: BlockUnavailableReason | undefined;
 
 			// Ensure that all outstanding transactions in the context are committed.
 			// This promotes a landed-elsewhere pending via internalCommit, which mutates
@@ -185,14 +221,30 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 						// Sort a COPY: when `latest` is undefined, `missing` aliases the caller's
 						// `context.committed` array, and an in-place `.sort()` would reorder the shared
 						// request context under the caller's feet.
-						for (const { actionId, rev } of [...missing].sort((a, b) => a.rev - b.rev)) {
-							const pending = await blockStorage.getPendingTransaction(actionId);
-							if (pending) {
-								const collectionId = await this.internalCommit(blockId, actionId, rev, blockStorage);
-								if (collectionId !== undefined) {
-									promotions.push({ collectionId, blockId, actionId, rev });
+						try {
+							for (const { actionId, rev } of [...missing].sort((a, b) => a.rev - b.rev)) {
+								const pending = await blockStorage.getPendingTransaction(actionId);
+								if (pending) {
+									const collectionId = await this.internalCommit(blockId, actionId, rev, blockStorage);
+									if (collectionId !== undefined) {
+										promotions.push({ collectionId, blockId, actionId, rev });
+									}
 								}
 							}
+						} catch (err) {
+							// This node holds no materializable base for the block, so NO context revision
+							// can be promoted here (each builds on the one before). Leave `latest` where it
+							// is — the invariant internalCommit just enforced — and let the commit-path
+							// healing supply the content; a read must not fail for it. Every other fault
+							// still propagates.
+							if (!(err instanceof MissingBaseRevisionError)) {
+								throw err;
+							}
+							// This node holds records PROVING the block exists (a pending it could not
+							// promote); if the block then reads as absent below, the answer is a guess,
+							// not an authoritative "never existed".
+							unavailable = 'unmaterializable';
+							log('get:promote-skipped-missing-base blockId=%s rev=%d reason=%s', blockId, err.rev, err.message);
 						}
 					});
 				}
@@ -205,7 +257,26 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 			// (actionId, rev) self-heals it via storage.recover() in commit(). Not repaired lazily on
 			// the read path because get() holds no commit latch; if stale reads on unwritten blocks
 			// ever become a problem, add a latched lazy recover() here.
-			const blockRev = await blockStorage.getBlock(context?.rev);
+			//
+			// getBlock() THROWS when this node holds a `latest` it cannot materialize (truncated
+			// history: "Failed to find materialized block", or a failed restore). Caught PER BLOCK so
+			// one broken block cannot fail the whole batch's Promise.all and take healthy siblings
+			// down with it. The read still fails for THIS block — TransactorSource throws
+			// BlockUnavailableError on the flagged entry — so nothing is swallowed.
+			let blockRev: Awaited<ReturnType<IBlockStorage['getBlock']>>;
+			try {
+				blockRev = await blockStorage.getBlock(context?.rev);
+			} catch (err) {
+				// NOTE: the entry drops `state.latest`, which this node does know (getLatest() does not
+				// materialize, so it does not throw). Empty state is what makes CoordinatorRepo treat the
+				// block as missing and consult the cohort — exactly the repair this block needs. If a
+				// consumer ever needs the revision behind an unavailable answer (e.g. to ask the cohort
+				// for a specific rev instead of the whole block), carry `latest` here and widen the
+				// coordinator's consult trigger to `isMissing || unavailable` so repair still fires.
+				log('get:unmaterializable blockId=%s error=%s', blockId,
+					err instanceof Error ? err.message : String(err));
+				return [blockId, { state: {}, unavailable: 'unmaterializable' } as GetBlockResult];
+			}
 
 			// Include pending action if requested — handled first so a pending-only
 			// insert (no committed revision yet) can still be served by applying the
@@ -213,6 +284,8 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 			if (context?.actionId !== undefined) {
 				const pendingTransform = await blockStorage.getPendingTransaction(context.actionId);
 				if (!pendingTransform) {
+					// Caller-contract violation (the caller asserted a pending this repo never had, or
+					// cancelled) — an error, not an availability question. Deliberately NOT `unavailable`.
 					throw new Error(`Pending action ${context.actionId} not found`);
 				}
 				const block = applyTransform(blockRev?.block, pendingTransform);
@@ -221,12 +294,21 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 					state: {
 						latest: await blockStorage.getLatest(),
 						pendings: [context.actionId]
-					}
-				}];
+					},
+					// A pending applied to a missing base can materialize nothing (applyTransform drops
+					// updates with no block to apply them to) — that absence is a guess, and is flagged.
+					// A materialized block is a real answer regardless of the earlier refusal.
+					...(unavailable !== undefined && block === undefined ? { unavailable } : {})
+				} as GetBlockResult];
 			}
 
 			if (!blockRev) {
-				return [blockId, { state: {} } as GetBlockResult];
+				// `unavailable` distinguishes "never existed" (the common insert-probe case, no flag)
+				// from "this node cannot reconstruct it" (the promotion above refused for a missing
+				// base). A tombstoned block also lands here with meta.latest set, but it never enters
+				// the missing-base catch, so it stays an authoritative absent — keyed off the explicit
+				// flag, not off "no block".
+				return [blockId, { state: {}, ...(unavailable !== undefined ? { unavailable } : {}) } as GetBlockResult];
 			}
 
 			const pendings = await asyncIteratorToArray(blockStorage.listPendingTransactions());
@@ -279,6 +361,8 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 		if (this.validatePend && request.transaction && request.operationsHash) {
 			const validationResult = await this.validatePend(request.transaction, request.operationsHash);
 			if (!validationResult.valid) {
+				// Hard rejection: the transaction itself is invalid, so no `conflict` flag — a
+				// re-read and re-pend would fail the same way and only burn the retry budget.
 				return {
 					success: false,
 					reason: validationResult.reason ?? 'Transaction validation failed'
@@ -330,16 +414,18 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 			log('pend:stale actionId=%s missing=%d', request.actionId, missing.length);
 			return {
 				success: false,
+				conflict: true,
 				missing
 			};
 		}
 
 		if (pendings.length > 0) {
 			if (request.policy === 'f') {	// Fail on pending actions
-				return { success: false, pending: pendings };
+				return { success: false, conflict: true, pending: pendings };
 			} else if (request.policy === 'r') {	// Return populated pending actions
 				return {
 					success: false,
+					conflict: true,
 					pending: await Promise.all(pendings.map(async action => {
 						const blockStorage = this.createBlockStorage(action.blockId);
 						return {
@@ -639,14 +725,24 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 
 		// Get prior materialized block if it exists
 		const latest = await storage.getLatest();
-		const priorBlock = latest
-			? (await storage.getBlock(latest.rev))?.block
-			: undefined;
+		const priorBlock = await this.readCommitBase(blockId, actionId, rev, storage, latest);
 
 		// Apply transform and save materialized block
 		// applyTransform handles undefined priorBlock correctly for inserts
 		const newBlock = applyTransform(priorBlock, transform);
 
+		// INVARIANT: `latest` must never advance past a revision this node can materialize.
+		// `applyTransform` silently drops `updates` when there is no block to apply them to, so a
+		// member that missed the block's CREATING revision would otherwise record rev N while storing
+		// nothing to serve it from. `latest === undefined` is precisely the "nothing below to fall
+		// back to" case: materializeBlock's descending walk needs some materialization at or below the
+		// target, and with no prior revision there is none. With a prior `latest` an absent newBlock is
+		// a legitimate tombstone (the walk resolves to an earlier materialization), so it stays allowed.
+		if (!newBlock && latest === undefined) {
+			return await this.refuseMissingBase(blockId, actionId, rev, storage,
+				'no committed revision to apply the transform to');
+		}
+
 		if (newBlock) {
 			await storage.saveMaterializedBlock(actionId, newBlock);
 		}
@@ -682,6 +778,67 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 		// undefined so the caller skips it rather than emitting a bogus event.
 		return newBlock?.header.collectionId ?? priorBlock?.header.collectionId;
 	}
+
+	/**
+	 * The materialization this commit builds on: the block at `latest`, or `undefined` when the block
+	 * holds no committed revision yet (the normal insert case).
+	 *
+	 * `getBlock` THROWS when this node holds a `latest` it cannot materialize — a block already wedged
+	 * by a pre-fix commit, or by truncated history. That is the same divergence as having no base at
+	 * all, so it is translated into {@link MissingBaseRevisionError} rather than surfacing as an opaque
+	 * storage fault: the healing path can then repair the block instead of the fault resetting the
+	 * cluster stream, and a wedged node recovers on the next write touching the block.
+	 *
+	 * The catch is deliberately UNNARROWED — it also absorbs a transient fault (a raw-storage read
+	 * error, a `restoreCallback` timeout on a block whose `ranges` do not cover its own `latest`).
+	 * BlockStorage reports every one of these as a bare `Error`, so they cannot be told apart here,
+	 * and treating them as divergence is the safe default: this node genuinely cannot materialize the
+	 * base right now, and the cluster's policy is to heal rather than throw out of consensus. The
+	 * price is that a transient fault ALSO drops the pending (see {@link refuseMissingBase}), so the
+	 * block converges by replication instead of by a replay the retry could have done. Narrowing this
+	 * would require typed faults out of BlockStorage; until then, prefer the tolerant reading.
+	 */
+	private async readCommitBase(
+		blockId: BlockId,
+		actionId: ActionId,
+		rev: number,
+		storage: IBlockStorage,
+		latest: ActionRev | undefined
+	): Promise<IBlock | undefined> {
+		if (!latest) {
+			return undefined;
+		}
+		try {
+			return (await storage.getBlock(latest.rev))?.block;
+		} catch (err) {
+			log('commit:unmaterializable-base blockId=%s baseRev=%d error=%s', blockId, latest.rev,
+				err instanceof Error ? err.message : String(err));
+			return await this.refuseMissingBase(blockId, actionId, rev, storage,
+				`local rev ${latest.rev} is not materializable here`);
+		}
+	}
+
+	/**
+	 * Refuse a commit this node cannot materialize. Always throws {@link MissingBaseRevisionError};
+	 * nothing durable has been written at this point, so the block is left exactly as it was minus the
+	 * pending record.
+	 *
+	 * The pending is dropped because it can never be promoted here: promotion needs a base this node
+	 * must obtain out-of-band, and once the healing path lands that revision `latest` is already >= rev,
+	 * so a commit retry partitions the block as already-done/stale and never revisits the pending.
+	 * Leaving it would also report a phantom conflicting action from {@link pend} for every later write.
+	 */
+	private async refuseMissingBase(
+		blockId: BlockId,
+		actionId: ActionId,
+		rev: number,
+		storage: IBlockStorage,
+		detail: string
+	): Promise<never> {
+		await storage.deletePendingTransaction(actionId);
+		log('commit:missing-base blockId=%s rev=%d actionId=%s detail=%s', blockId, rev, actionId, detail);
+		throw new MissingBaseRevisionError(blockId, rev, detail);
+	}
 }
 
 /**

package/src/sync/service.ts

@@ -1,4 +1,4 @@
-import type { ComponentLogger, Startable } from '@libp2p/interface';
+import type { ComponentLogger, Connection, Startable } from '@libp2p/interface';
 import type { IRepo } from '@optimystic/db-core';
 import { buildSyncProtocol, type SyncRequest, type SyncResponse } from './protocol.js';
 import { pipe } from 'it-pipe';
@@ -6,8 +6,9 @@ import { toString as u8ToString } from 'uint8arrays/to-string';
 import * as lp from 'it-length-prefixed';
 import { MAX_CONTROL_MESSAGE_BYTES } from '../protocol-limits.js';
 import type { Uint8ArrayList } from 'uint8arraylist';
+import { createInboundStreamAuthorization, type InboundStreamAuthorization, type InboundStreamAuthorizationInit } from '../inbound-authorization.js';
 
-export interface SyncServiceInit {
+export interface SyncServiceInit extends InboundStreamAuthorizationInit {
 	protocolPrefix?: string;
 }
 
@@ -35,6 +36,8 @@ export class SyncService implements Startable {
 	private readonly protocol: string;
 	private readonly repo: IRepo;
 	private readonly registrar: { handle: (...args: any[]) => Promise<void>, unhandle: (...args: any[]) => Promise<void> };
+	/** Optional embedder authorization gate; `undefined` (the default) means no check runs. */
+	private readonly authorization: InboundStreamAuthorization | undefined;
 
 	constructor(
 		components: SyncServiceComponents,
@@ -44,6 +47,7 @@ export class SyncService implements Startable {
 		this.protocol = buildSyncProtocol(init.protocolPrefix ?? '');
 		this.repo = components.repo;
 		this.registrar = components.registrar;
+		this.authorization = createInboundStreamAuthorization(init, this.protocol, (msg, ...args) => this.log.error(msg, ...args));
 	}
 
 	async start(): Promise<void> {
@@ -62,21 +66,22 @@ export class SyncService implements Startable {
 		this.log('Sync service stopped');
 	}
 
-	/**
-	 * Handle an incoming sync request stream.
-	 * Uses a streaming pipeline (like the repo service) to process the
-	 * first request and yield a response without waiting for the client
-	 * to close its write side — avoids a read/write deadlock.
-	 */
 	/**
 	 * Handle an incoming sync request stream.
 	 * Uses a streaming pipeline (like the repo service) to process the
 	 * request and yield a response immediately — avoids a read/write deadlock.
 	 */
-	private handleSyncRequest(stream: any): void {
+	private handleSyncRequest(stream: any, connection?: Connection): void {
 		const self = this;
+		// The registrar handler receives the stream (or, in an older shape, a `{ stream }`
+		// wrapper) directly; resolve it once so the authorization abort, the pipeline and the
+		// error path all act on the same object.
+		const actualStream = stream.stream ?? stream;
 		void (async () => {
 			try {
+				// Authorization runs before ANY decoding or execution. Guarded on the field so a
+				// node without a predicate keeps the original path untouched.
+				if (self.authorization && await self.authorization.deny(actualStream, connection?.remotePeer?.toString())) return;
 				const processStream = async function* (source: AsyncIterable<Uint8ArrayList> | Iterable<Uint8ArrayList>) {
 					for await (const msg of source) {
 						const json = u8ToString(msg.subarray(), 'utf8');
@@ -111,8 +116,6 @@ export class SyncService implements Startable {
 				};
 
 				// Use the same streaming pipeline pattern as the repo service.
-				// The registrar handler receives the stream (or data object) directly.
-				const actualStream = stream.stream ?? stream;
 				const responses = pipe(
 					actualStream,
 					(source: any) => lp.decode(source, { maxDataLength: MAX_CONTROL_MESSAGE_BYTES }),
@@ -125,7 +128,6 @@ export class SyncService implements Startable {
 				await actualStream.close();
 			} catch (error) {
 				self.log.error('Error handling sync request:', error);
-				const actualStream = stream.stream ?? stream;
 				try { actualStream.abort(error instanceof Error ? error : new Error(String(error))); } catch { /* ignore */ }
 			}
 		})();

package/src/testing/mesh-harness.ts

@@ -107,6 +107,32 @@ export interface Mesh {
 	keyNetwork: IKeyNetwork;
 }
 
+/**
+ * Pull a block's committed content from a sibling cohort node into `storageRepo` — the mesh analogue
+ * of `libp2p-node-base`'s `createReconcileBlock` (SyncClient archive fetch + `saveReplicatedBlock`).
+ *
+ * Deliberately shared by BOTH callers, exactly as the live node shares one callback between them: the
+ * commit path (`clusterMember.reconcileBlock`, for a block committed without a materializable base)
+ * and the read path (`CoordinatorRepo.acquireBlockFromCohort`, for a corroborated revision the reader
+ * cannot promote locally). Stateless, so it is simply rebuilt per caller.
+ *
+ * `nodes` is captured by reference and is fully populated by the time either caller invokes it.
+ */
+const makeReconcileBlock = (nodes: MeshNode[], selfPeerId: string, storageRepo: StorageRepo): ReconcileBlockCallback =>
+	async (blockId, committed, cohortPeerIds) => {
+		for (const peerIdStr of cohortPeerIds) {
+			if (peerIdStr === selfPeerId) continue;
+			const target = nodes.find(n => n.peerId.toString() === peerIdStr);
+			if (!target) continue;
+			const result = await target.storageRepo.get({ blockIds: [blockId] }, { skipClusterFetch: true } as any);
+			const entry = result[blockId];
+			const latest = entry?.state?.latest;
+			if (!latest || !entry?.block || latest.rev < committed.rev) continue;
+			await storageRepo.saveReplicatedBlock(blockId, entry.block, latest);
+			return;
+		}
+	};
+
 /**
  * Creates N interconnected mesh nodes with real components and mock transport.
  * ClusterClient calls route directly to target ClusterMember instances.
@@ -125,8 +151,6 @@ export async function createMesh(nodeCount: number, options: MeshOptions): Promi
 	// Build nodes array (partially — coordinatorRepo added after keyNetwork is ready)
 	const nodes: MeshNode[] = [];
 	const peerNetwork = new MockPeerNetwork();
-	// Map peerId → rawStorage for data sync simulation in clusterLatestCallback
-	const rawStorages = new Map<string, IRawStorage>();
 
 	// Phase 1: create storage + cluster members
 	let nodeIndex = 0;
@@ -134,7 +158,6 @@ export async function createMesh(nodeCount: number, options: MeshOptions): Promi
 		const rawStorage = options.rawStorageFactory
 			? options.rawStorageFactory(nodeIndex)
 			: new MemoryRawStorage();
-		rawStorages.set(peerId.toString(), rawStorage);
 		nodeIndex++;
 		const storageRepo = new StorageRepo(
 			(blockId: BlockId) => new BlockStorage(blockId, rawStorage)
@@ -152,24 +175,9 @@ export async function createMesh(nodeCount: number, options: MeshOptions): Promi
 			allowUnvalidatedSmallCluster: true
 		};
 
-		// Active reconciliation simulation: when a member commits a block it never
-		// pended (cohort drift), pull the committed revision from a sibling cohort node
-		// that holds it and persist it locally — the mesh analogue of the SyncClient
-		// fetch + saveReplicatedBlock path wired in `libp2p-node-base`. `nodes` is captured
-		// by reference and fully populated by the time the callback is actually invoked.
-		const reconcileBlock: ReconcileBlockCallback = async (blockId, committed, cohortPeerIds) => {
-			for (const peerIdStr of cohortPeerIds) {
-				if (peerIdStr === peerId.toString()) continue;
-				const target = nodes.find(n => n.peerId.toString() === peerIdStr);
-				if (!target) continue;
-				const result = await target.storageRepo.get({ blockIds: [blockId] }, { skipClusterFetch: true } as any);
-				const entry = result[blockId];
-				const latest = entry?.state?.latest;
-				if (!latest || !entry?.block || latest.rev < committed.rev) continue;
-				await storageRepo.saveReplicatedBlock(blockId, entry.block, latest);
-				return;
-			}
-		};
+		// Active reconciliation: when a member commits a block it never pended (cohort drift),
+		// pull the committed revision from a sibling cohort node that holds it.
+		const reconcileBlock = makeReconcileBlock(nodes, peerId.toString(), storageRepo);
 
 		const member = clusterMember({
 			storageRepo,
@@ -208,10 +216,12 @@ export async function createMesh(nodeCount: number, options: MeshOptions): Promi
 	};
 
 	for (const node of nodes) {
-		const localRawStorage = rawStorages.get(node.peerId.toString())!;
-
-		// Per-node callback: queries remote peer and replicates committed data locally
-		// (simulates what SyncClient does in production)
+		// Per-node callback: reports the queried peer's latest revision, and NOTHING else. It used to
+		// also write the peer's block into local storage ("simulate data sync"), which made every
+		// read-repair assertion on this harness observe a convergence the production callback does not
+		// provide — masking exactly the defect that ticket `read-repair-cannot-transfer-block-content`
+		// existed to expose. Transfer now happens where it does in production: through
+		// `acquireBlockFromCohort` below, gated on a corroborated revision.
 		const clusterLatestCallback: ClusterLatestCallback = async (peerId: PeerId, blockId: BlockId, context?): Promise<ActionRev | undefined> => {
 			const target = nodes.find(n => n.peerId.equals(peerId));
 			if (!target) return undefined;
@@ -219,28 +229,7 @@ export async function createMesh(nodeCount: number, options: MeshOptions): Promi
 				{ blockIds: [blockId], context },
 				{ skipClusterFetch: true } as any
 			);
-			const entry = result[blockId];
-			const latest = entry?.state?.latest;
-
-			// Simulate data sync: replicate committed block data to local storage
-			if (latest && entry?.block) {
-				const localBlockStorage = new BlockStorage(blockId as BlockId, localRawStorage);
-				const localLatest = await localBlockStorage.getLatest();
-				if (!localLatest || localLatest.rev < latest.rev) {
-					// Ensure metadata exists
-					const meta = await localRawStorage.getMetadata(blockId as BlockId);
-					if (!meta) {
-						// Seed empty ranges (honest: nothing reconstructible yet); the setLatest
-						// below merges [latest.rev, latest.rev+1] once the revision is persisted.
-						await localRawStorage.saveMetadata(blockId as BlockId, { latest: undefined, ranges: [] });
-					}
-					await localBlockStorage.saveMaterializedBlock(latest.actionId, entry.block);
-					await localBlockStorage.saveRevision(latest.rev, latest.actionId);
-					await localBlockStorage.setLatest(latest);
-				}
-			}
-
-			return latest;
+			return result[blockId]?.state?.latest;
 		};
 		// Wrap key network to include self in findCluster (matches real Libp2pKeyPeerNetwork behavior)
 		const nodeKeyNetwork: IKeyNetwork = {
@@ -259,7 +248,7 @@ export async function createMesh(nodeCount: number, options: MeshOptions): Promi
 		};
 		const factory = coordinatorRepo(
 			nodeKeyNetwork,
-			(peerId: PeerId) => createClusterClient(peerId) as any,
+			createClusterClient,
 			{
 				clusterSize: options.clusterSize ?? nodeCount,
 				superMajorityThreshold: options.superMajorityThreshold ?? DEFAULT_SUPER_MAJORITY_THRESHOLD,
@@ -273,13 +262,30 @@ export async function createMesh(nodeCount: number, options: MeshOptions): Promi
 			storageRepo: node.storageRepo,
 			localCluster: node.clusterMember,
 			localPeerId: node.peerId,
-			clusterLatestCallback
+			clusterLatestCallback,
+			// The read path's transfer mechanism — the same callback the member uses on the commit path,
+			// mirroring how `libp2p-node-base` shares one `reconcileBlock` between both.
+			acquireBlockFromCohort: makeReconcileBlock(nodes, node.peerId.toString(), node.storageRepo)
 		});
 	}
 
 	return { nodes, failures, keyNetwork };
 }
 
+/**
+ * The nodes the key network keeps OUT of `blockId`'s cohort — peers that receive none of the
+ * block's cluster traffic, and so hold none of its content until something repairs them.
+ *
+ * Peer ids are generated fresh per mesh, so which node is responsible for a given block is random
+ * from run to run: in a 3-node `responsibilityK: 1` mesh, `nodes[1]` is the block's sole responsible
+ * peer about a third of the time, and then it receives the writer's commit directly. A test that
+ * needs a genuinely non-responsible node has to ask the routing layer rather than assume an index.
+ */
+export async function nonResponsibleNodes(mesh: Mesh, blockId: string): Promise<MeshNode[]> {
+	const cohort = await mesh.keyNetwork.findCluster(new TextEncoder().encode(blockId));
+	return mesh.nodes.filter(node => !(node.peerId.toString() in cohort));
+}
+
 export interface BuildTransactorOptions {
 	timeoutMs?: number;
 	abortOrCancelTimeoutMs?: number;