@optimystic/db-p2p 0.17.0 → 0.19.0

package/src/storage/storage-repo.ts

@@ -1,13 +1,14 @@
 import type {
 	IRepo, MessageOptions, BlockId, CommitRequest, CommitResult, GetBlockResults, PendRequest, PendResult, ActionBlocks,
 	ActionId, BlockGets, ActionPending, PendSuccess, ActionTransform, ActionTransforms,
-	GetBlockResult, IBlock, ActionRev,
+	GetBlockResult, IBlock, ActionRev, BlockUnavailableReason,
 	PendValidationHook,
-	CollectionId, IBlockChangeNotifier, CollectionChangeListener, CollectionChangeEvent
+	CollectionId, IBlockChangeNotifier, CollectionChangeListener, CollectionChangeEvent,
+	StaleFailure
 } from "@optimystic/db-core";
 import {
 	Latches, transformForBlockId, applyTransform, groupBy, concatTransform, emptyTransforms,
-	blockIdsForTransforms, transformsFromTransform
+	blockIdsForTransforms, transformsFromTransform, highestStaleAt
 } from "@optimystic/db-core";
 import { asyncIteratorToArray } from "../it-utility.js";
 import type { IBlockStorage } from "./i-block-storage.js";
@@ -186,6 +187,11 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 		const promotions: { collectionId: CollectionId, blockId: BlockId, actionId: ActionId, rev: number }[] = [];
 		const results = await Promise.all(distinctBlockIds.map(async (blockId) => {
 			const blockStorage = this.createBlockStorage(blockId);
+			// Set when this node KNOWS its answer for the block is a guess: the promotion
+			// below refused for a missing base, or getBlock() threw (truncated history /
+			// failed restore). An absent-reading block then reports `unavailable` instead of
+			// posing as an authoritative "never existed" — see BlockUnavailableReason.
+			let unavailable: BlockUnavailableReason | undefined;
 
 			// Ensure that all outstanding transactions in the context are committed.
 			// This promotes a landed-elsewhere pending via internalCommit, which mutates
@@ -235,6 +241,10 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 							if (!(err instanceof MissingBaseRevisionError)) {
 								throw err;
 							}
+							// This node holds records PROVING the block exists (a pending it could not
+							// promote); if the block then reads as absent below, the answer is a guess,
+							// not an authoritative "never existed".
+							unavailable = 'unmaterializable';
 							log('get:promote-skipped-missing-base blockId=%s rev=%d reason=%s', blockId, err.rev, err.message);
 						}
 					});
@@ -248,7 +258,26 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 			// (actionId, rev) self-heals it via storage.recover() in commit(). Not repaired lazily on
 			// the read path because get() holds no commit latch; if stale reads on unwritten blocks
 			// ever become a problem, add a latched lazy recover() here.
-			const blockRev = await blockStorage.getBlock(context?.rev);
+			//
+			// getBlock() THROWS when this node holds a `latest` it cannot materialize (truncated
+			// history: "Failed to find materialized block", or a failed restore). Caught PER BLOCK so
+			// one broken block cannot fail the whole batch's Promise.all and take healthy siblings
+			// down with it. The read still fails for THIS block — TransactorSource throws
+			// BlockUnavailableError on the flagged entry — so nothing is swallowed.
+			let blockRev: Awaited<ReturnType<IBlockStorage['getBlock']>>;
+			try {
+				blockRev = await blockStorage.getBlock(context?.rev);
+			} catch (err) {
+				// NOTE: the entry drops `state.latest`, which this node does know (getLatest() does not
+				// materialize, so it does not throw). Empty state is what makes CoordinatorRepo treat the
+				// block as missing and consult the cohort — exactly the repair this block needs. If a
+				// consumer ever needs the revision behind an unavailable answer (e.g. to ask the cohort
+				// for a specific rev instead of the whole block), carry `latest` here and widen the
+				// coordinator's consult trigger to `isMissing || unavailable` so repair still fires.
+				log('get:unmaterializable blockId=%s error=%s', blockId,
+					err instanceof Error ? err.message : String(err));
+				return [blockId, { state: {}, unavailable: 'unmaterializable' } as GetBlockResult];
+			}
 
 			// Include pending action if requested — handled first so a pending-only
 			// insert (no committed revision yet) can still be served by applying the
@@ -256,6 +285,8 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 			if (context?.actionId !== undefined) {
 				const pendingTransform = await blockStorage.getPendingTransaction(context.actionId);
 				if (!pendingTransform) {
+					// Caller-contract violation (the caller asserted a pending this repo never had, or
+					// cancelled) — an error, not an availability question. Deliberately NOT `unavailable`.
 					throw new Error(`Pending action ${context.actionId} not found`);
 				}
 				const block = applyTransform(blockRev?.block, pendingTransform);
@@ -264,12 +295,21 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 					state: {
 						latest: await blockStorage.getLatest(),
 						pendings: [context.actionId]
-					}
-				}];
+					},
+					// A pending applied to a missing base can materialize nothing (applyTransform drops
+					// updates with no block to apply them to) — that absence is a guess, and is flagged.
+					// A materialized block is a real answer regardless of the earlier refusal.
+					...(unavailable !== undefined && block === undefined ? { unavailable } : {})
+				} as GetBlockResult];
 			}
 
 			if (!blockRev) {
-				return [blockId, { state: {} } as GetBlockResult];
+				// `unavailable` distinguishes "never existed" (the common insert-probe case, no flag)
+				// from "this node cannot reconstruct it" (the promotion above refused for a missing
+				// base). A tombstoned block also lands here with meta.latest set, but it never enters
+				// the missing-base catch, so it stays an authoritative absent — keyed off the explicit
+				// flag, not off "no block".
+				return [blockId, { state: {}, ...(unavailable !== undefined ? { unavailable } : {}) } as GetBlockResult];
 			}
 
 			const pendings = await asyncIteratorToArray(blockStorage.listPendingTransactions());
@@ -322,6 +362,8 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 		if (this.validatePend && request.transaction && request.operationsHash) {
 			const validationResult = await this.validatePend(request.transaction, request.operationsHash);
 			if (!validationResult.valid) {
+				// Hard rejection: the transaction itself is invalid, so no `conflict` flag — a
+				// re-read and re-pend would fail the same way and only burn the retry budget.
 				return {
 					success: false,
 					reason: validationResult.reason ?? 'Transaction validation failed'
@@ -333,6 +375,10 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 		log('pend actionId=%s blockIds=%d rev=%s', request.actionId, blockIds.length, request.rev);
 		const pendings: ActionPending[] = [];
 		const missing: ActionTransforms[] = [];
+		// Highest revision this node confirms holding among the blocks that are at or past the
+		// requested one — reported as StaleFailure.staleAt so a losing writer learns the number
+		// instead of parsing prose. Confirmed-local only: we read it from our own storage below.
+		let staleAt: StaleFailure['staleAt'];
 
 		// Potential race condition: A concurrent commit operation could complete
 		// between the conflict checks (latest.rev, listPendingTransactions) and the
@@ -353,6 +399,13 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 			if (request.rev !== undefined || transforms.insert) {
 				const latest = await blockStorage.getLatest();
 				if (latest && latest.rev >= (request.rev ?? 0)) {
+					// Only a real revision race yields a meaningful `staleAt`. When `request.rev` is
+					// undefined this same branch fires for an insert collision (the comparison degrades
+					// to `latest.rev >= 0`, true for any existing block), and reporting that block's
+					// revision would be a number that answers a question nobody asked.
+					if (request.rev !== undefined) {
+						staleAt = highestStaleAt([staleAt, { blockId, rev: latest.rev }]);
+					}
 					const transforms = await asyncIteratorToArray(blockStorage.listRevisions(request.rev ?? 0, latest.rev));
 					for (const actionRev of transforms) {
 						const transform = await blockStorage.getTransaction(actionRev.actionId);
@@ -373,16 +426,19 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 			log('pend:stale actionId=%s missing=%d', request.actionId, missing.length);
 			return {
 				success: false,
-				missing
+				conflict: true,
+				missing,
+				...(staleAt === undefined ? {} : { staleAt })
 			};
 		}
 
 		if (pendings.length > 0) {
 			if (request.policy === 'f') {	// Fail on pending actions
-				return { success: false, pending: pendings };
+				return { success: false, conflict: true, pending: pendings };
 			} else if (request.policy === 'r') {	// Return populated pending actions
 				return {
 					success: false,
+					conflict: true,
 					pending: await Promise.all(pendings.map(async action => {
 						const blockStorage = this.createBlockStorage(action.blockId);
 						return {
@@ -459,6 +515,10 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 			//   - toCommit: latest.rev < request.rev or no latest yet → run internalCommit.
 			const toCommit: { blockId: BlockId, storage: IBlockStorage }[] = [];
 			const missedCommits: { blockId: BlockId, transforms: ActionTransform[] }[] = [];
+			// Highest revision among the blocks confirmed lost to a newer one — reported as
+			// StaleFailure.staleAt. The idempotent-retry `continue` below is a no-op, not a loss,
+			// so it never seeds this.
+			let staleAt: StaleFailure['staleAt'];
 			for (const entry of blockStorages) {
 				const { blockId, storage } = entry;
 				const latest = await storage.getLatest();
@@ -467,6 +527,7 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 						// Idempotent no-op for this block — already committed with this exact (actionId, rev).
 						continue;
 					}
+					staleAt = highestStaleAt([staleAt, { blockId, rev: latest.rev }]);
 					const transforms: ActionTransform[] = [];
 					for await (const actionRev of storage.listRevisions(request.rev, latest.rev)) {
 						const transform = await storage.getTransaction(actionRev.actionId);
@@ -489,7 +550,8 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 				log('commit:stale actionId=%s missed=%d', request.actionId, missedCommits.length);
 				return { // Return directly, locks will be released in finally
 					success: false,
-					missing: perBlockActionTransformsToPerAction(missedCommits)
+					missing: perBlockActionTransformsToPerAction(missedCommits),
+					...(staleAt === undefined ? {} : { staleAt })
 				};
 			}
 
@@ -745,6 +807,15 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 	 * all, so it is translated into {@link MissingBaseRevisionError} rather than surfacing as an opaque
 	 * storage fault: the healing path can then repair the block instead of the fault resetting the
 	 * cluster stream, and a wedged node recovers on the next write touching the block.
+	 *
+	 * The catch is deliberately UNNARROWED — it also absorbs a transient fault (a raw-storage read
+	 * error, a `restoreCallback` timeout on a block whose `ranges` do not cover its own `latest`).
+	 * BlockStorage reports every one of these as a bare `Error`, so they cannot be told apart here,
+	 * and treating them as divergence is the safe default: this node genuinely cannot materialize the
+	 * base right now, and the cluster's policy is to heal rather than throw out of consensus. The
+	 * price is that a transient fault ALSO drops the pending (see {@link refuseMissingBase}), so the
+	 * block converges by replication instead of by a replay the retry could have done. Narrowing this
+	 * would require typed faults out of BlockStorage; until then, prefer the tolerant reading.
 	 */
 	private async readCommitBase(
 		blockId: BlockId,

package/src/testing/mesh-harness.ts

@@ -42,6 +42,13 @@ export interface MeshFailureConfig {
 	failingPeers?: Set<string>;
 	/** Make findCluster return empty (simulate DHT failure) */
 	findClusterFails?: boolean;
+	/**
+	 * Peers that are unreachable on the READ path: their latest-revision consult
+	 * (`ClusterLatestCallback`) REJECTS — silence the coordinator must count as "did not
+	 * answer", never as the peer claiming absence — and the reconcile/acquire transfer skips
+	 * them as a source. Distinct from `failingPeers`, which fails cluster (write) updates.
+	 */
+	silentPeers?: Set<string>;
 }
 
 class MockPeerNetwork implements IPeerNetwork {
@@ -118,10 +125,13 @@ export interface Mesh {
  *
  * `nodes` is captured by reference and is fully populated by the time either caller invokes it.
  */
-const makeReconcileBlock = (nodes: MeshNode[], selfPeerId: string, storageRepo: StorageRepo): ReconcileBlockCallback =>
+const makeReconcileBlock = (nodes: MeshNode[], selfPeerId: string, storageRepo: StorageRepo, failures: MeshFailureConfig): ReconcileBlockCallback =>
 	async (blockId, committed, cohortPeerIds) => {
 		for (const peerIdStr of cohortPeerIds) {
 			if (peerIdStr === selfPeerId) continue;
+			// A silent peer cannot serve bytes either — without this, a test that silenced a
+			// peer's consult could still accidentally converge THROUGH that peer.
+			if (failures.silentPeers?.has(peerIdStr)) continue;
 			const target = nodes.find(n => n.peerId.toString() === peerIdStr);
 			if (!target) continue;
 			const result = await target.storageRepo.get({ blockIds: [blockId] }, { skipClusterFetch: true } as any);
@@ -177,7 +187,7 @@ export async function createMesh(nodeCount: number, options: MeshOptions): Promi
 
 		// Active reconciliation: when a member commits a block it never pended (cohort drift),
 		// pull the committed revision from a sibling cohort node that holds it.
-		const reconcileBlock = makeReconcileBlock(nodes, peerId.toString(), storageRepo);
+		const reconcileBlock = makeReconcileBlock(nodes, peerId.toString(), storageRepo, failures);
 
 		const member = clusterMember({
 			storageRepo,
@@ -223,6 +233,12 @@ export async function createMesh(nodeCount: number, options: MeshOptions): Promi
 		// existed to expose. Transfer now happens where it does in production: through
 		// `acquireBlockFromCohort` below, gated on a corroborated revision.
 		const clusterLatestCallback: ClusterLatestCallback = async (peerId: PeerId, blockId: BlockId, context?): Promise<ActionRev | undefined> => {
+			// Silence: the peer never answers. REJECTS, mirroring what a dial failure does to the
+			// production callback — the coordinator must count this as "did not answer", never as
+			// an absent claim (a resolved `undefined` remains the peer answering "I hold nothing").
+			if (failures.silentPeers?.has(peerId.toString())) {
+				throw new Error(`Peer ${peerId.toString()} is silent`);
+			}
 			const target = nodes.find(n => n.peerId.equals(peerId));
 			if (!target) return undefined;
 			const result = await target.storageRepo.get(
@@ -248,7 +264,7 @@ export async function createMesh(nodeCount: number, options: MeshOptions): Promi
 		};
 		const factory = coordinatorRepo(
 			nodeKeyNetwork,
-			(peerId: PeerId) => createClusterClient(peerId) as any,
+			createClusterClient,
 			{
 				clusterSize: options.clusterSize ?? nodeCount,
 				superMajorityThreshold: options.superMajorityThreshold ?? DEFAULT_SUPER_MAJORITY_THRESHOLD,
@@ -265,13 +281,27 @@ export async function createMesh(nodeCount: number, options: MeshOptions): Promi
 			clusterLatestCallback,
 			// The read path's transfer mechanism — the same callback the member uses on the commit path,
 			// mirroring how `libp2p-node-base` shares one `reconcileBlock` between both.
-			acquireBlockFromCohort: makeReconcileBlock(nodes, node.peerId.toString(), node.storageRepo)
+			acquireBlockFromCohort: makeReconcileBlock(nodes, node.peerId.toString(), node.storageRepo, failures)
 		});
 	}
 
 	return { nodes, failures, keyNetwork };
 }
 
+/**
+ * The nodes the key network keeps OUT of `blockId`'s cohort — peers that receive none of the
+ * block's cluster traffic, and so hold none of its content until something repairs them.
+ *
+ * Peer ids are generated fresh per mesh, so which node is responsible for a given block is random
+ * from run to run: in a 3-node `responsibilityK: 1` mesh, `nodes[1]` is the block's sole responsible
+ * peer about a third of the time, and then it receives the writer's commit directly. A test that
+ * needs a genuinely non-responsible node has to ask the routing layer rather than assume an index.
+ */
+export async function nonResponsibleNodes(mesh: Mesh, blockId: string): Promise<MeshNode[]> {
+	const cohort = await mesh.keyNetwork.findCluster(new TextEncoder().encode(blockId));
+	return mesh.nodes.filter(node => !(node.peerId.toString() in cohort));
+}
+
 export interface BuildTransactorOptions {
 	timeoutMs?: number;
 	abortOrCancelTimeoutMs?: number;