@optimystic/db-p2p 0.17.0 → 0.18.0

package/src/storage/storage-repo.ts

@@ -1,7 +1,7 @@
 import type {
 	IRepo, MessageOptions, BlockId, CommitRequest, CommitResult, GetBlockResults, PendRequest, PendResult, ActionBlocks,
 	ActionId, BlockGets, ActionPending, PendSuccess, ActionTransform, ActionTransforms,
-	GetBlockResult, IBlock, ActionRev,
+	GetBlockResult, IBlock, ActionRev, BlockUnavailableReason,
 	PendValidationHook,
 	CollectionId, IBlockChangeNotifier, CollectionChangeListener, CollectionChangeEvent
 } from "@optimystic/db-core";
@@ -186,6 +186,11 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 		const promotions: { collectionId: CollectionId, blockId: BlockId, actionId: ActionId, rev: number }[] = [];
 		const results = await Promise.all(distinctBlockIds.map(async (blockId) => {
 			const blockStorage = this.createBlockStorage(blockId);
+			// Set when this node KNOWS its answer for the block is a guess: the promotion
+			// below refused for a missing base, or getBlock() threw (truncated history /
+			// failed restore). An absent-reading block then reports `unavailable` instead of
+			// posing as an authoritative "never existed" — see BlockUnavailableReason.
+			let unavailable: BlockUnavailableReason | undefined;
 
 			// Ensure that all outstanding transactions in the context are committed.
 			// This promotes a landed-elsewhere pending via internalCommit, which mutates
@@ -235,6 +240,10 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 							if (!(err instanceof MissingBaseRevisionError)) {
 								throw err;
 							}
+							// This node holds records PROVING the block exists (a pending it could not
+							// promote); if the block then reads as absent below, the answer is a guess,
+							// not an authoritative "never existed".
+							unavailable = 'unmaterializable';
 							log('get:promote-skipped-missing-base blockId=%s rev=%d reason=%s', blockId, err.rev, err.message);
 						}
 					});
@@ -248,7 +257,26 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 			// (actionId, rev) self-heals it via storage.recover() in commit(). Not repaired lazily on
 			// the read path because get() holds no commit latch; if stale reads on unwritten blocks
 			// ever become a problem, add a latched lazy recover() here.
-			const blockRev = await blockStorage.getBlock(context?.rev);
+			//
+			// getBlock() THROWS when this node holds a `latest` it cannot materialize (truncated
+			// history: "Failed to find materialized block", or a failed restore). Caught PER BLOCK so
+			// one broken block cannot fail the whole batch's Promise.all and take healthy siblings
+			// down with it. The read still fails for THIS block — TransactorSource throws
+			// BlockUnavailableError on the flagged entry — so nothing is swallowed.
+			let blockRev: Awaited<ReturnType<IBlockStorage['getBlock']>>;
+			try {
+				blockRev = await blockStorage.getBlock(context?.rev);
+			} catch (err) {
+				// NOTE: the entry drops `state.latest`, which this node does know (getLatest() does not
+				// materialize, so it does not throw). Empty state is what makes CoordinatorRepo treat the
+				// block as missing and consult the cohort — exactly the repair this block needs. If a
+				// consumer ever needs the revision behind an unavailable answer (e.g. to ask the cohort
+				// for a specific rev instead of the whole block), carry `latest` here and widen the
+				// coordinator's consult trigger to `isMissing || unavailable` so repair still fires.
+				log('get:unmaterializable blockId=%s error=%s', blockId,
+					err instanceof Error ? err.message : String(err));
+				return [blockId, { state: {}, unavailable: 'unmaterializable' } as GetBlockResult];
+			}
 
 			// Include pending action if requested — handled first so a pending-only
 			// insert (no committed revision yet) can still be served by applying the
@@ -256,6 +284,8 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 			if (context?.actionId !== undefined) {
 				const pendingTransform = await blockStorage.getPendingTransaction(context.actionId);
 				if (!pendingTransform) {
+					// Caller-contract violation (the caller asserted a pending this repo never had, or
+					// cancelled) — an error, not an availability question. Deliberately NOT `unavailable`.
 					throw new Error(`Pending action ${context.actionId} not found`);
 				}
 				const block = applyTransform(blockRev?.block, pendingTransform);
@@ -264,12 +294,21 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 					state: {
 						latest: await blockStorage.getLatest(),
 						pendings: [context.actionId]
-					}
-				}];
+					},
+					// A pending applied to a missing base can materialize nothing (applyTransform drops
+					// updates with no block to apply them to) — that absence is a guess, and is flagged.
+					// A materialized block is a real answer regardless of the earlier refusal.
+					...(unavailable !== undefined && block === undefined ? { unavailable } : {})
+				} as GetBlockResult];
 			}
 
 			if (!blockRev) {
-				return [blockId, { state: {} } as GetBlockResult];
+				// `unavailable` distinguishes "never existed" (the common insert-probe case, no flag)
+				// from "this node cannot reconstruct it" (the promotion above refused for a missing
+				// base). A tombstoned block also lands here with meta.latest set, but it never enters
+				// the missing-base catch, so it stays an authoritative absent — keyed off the explicit
+				// flag, not off "no block".
+				return [blockId, { state: {}, ...(unavailable !== undefined ? { unavailable } : {}) } as GetBlockResult];
 			}
 
 			const pendings = await asyncIteratorToArray(blockStorage.listPendingTransactions());
@@ -322,6 +361,8 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 		if (this.validatePend && request.transaction && request.operationsHash) {
 			const validationResult = await this.validatePend(request.transaction, request.operationsHash);
 			if (!validationResult.valid) {
+				// Hard rejection: the transaction itself is invalid, so no `conflict` flag — a
+				// re-read and re-pend would fail the same way and only burn the retry budget.
 				return {
 					success: false,
 					reason: validationResult.reason ?? 'Transaction validation failed'
@@ -373,16 +414,18 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 			log('pend:stale actionId=%s missing=%d', request.actionId, missing.length);
 			return {
 				success: false,
+				conflict: true,
 				missing
 			};
 		}
 
 		if (pendings.length > 0) {
 			if (request.policy === 'f') {	// Fail on pending actions
-				return { success: false, pending: pendings };
+				return { success: false, conflict: true, pending: pendings };
 			} else if (request.policy === 'r') {	// Return populated pending actions
 				return {
 					success: false,
+					conflict: true,
 					pending: await Promise.all(pendings.map(async action => {
 						const blockStorage = this.createBlockStorage(action.blockId);
 						return {
@@ -745,6 +788,15 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
 	 * all, so it is translated into {@link MissingBaseRevisionError} rather than surfacing as an opaque
 	 * storage fault: the healing path can then repair the block instead of the fault resetting the
 	 * cluster stream, and a wedged node recovers on the next write touching the block.
+	 *
+	 * The catch is deliberately UNNARROWED — it also absorbs a transient fault (a raw-storage read
+	 * error, a `restoreCallback` timeout on a block whose `ranges` do not cover its own `latest`).
+	 * BlockStorage reports every one of these as a bare `Error`, so they cannot be told apart here,
+	 * and treating them as divergence is the safe default: this node genuinely cannot materialize the
+	 * base right now, and the cluster's policy is to heal rather than throw out of consensus. The
+	 * price is that a transient fault ALSO drops the pending (see {@link refuseMissingBase}), so the
+	 * block converges by replication instead of by a replay the retry could have done. Narrowing this
+	 * would require typed faults out of BlockStorage; until then, prefer the tolerant reading.
 	 */
 	private async readCommitBase(
 		blockId: BlockId,

package/src/testing/mesh-harness.ts

@@ -248,7 +248,7 @@ export async function createMesh(nodeCount: number, options: MeshOptions): Promi
 		};
 		const factory = coordinatorRepo(
 			nodeKeyNetwork,
-			(peerId: PeerId) => createClusterClient(peerId) as any,
+			createClusterClient,
 			{
 				clusterSize: options.clusterSize ?? nodeCount,
 				superMajorityThreshold: options.superMajorityThreshold ?? DEFAULT_SUPER_MAJORITY_THRESHOLD,
@@ -272,6 +272,20 @@ export async function createMesh(nodeCount: number, options: MeshOptions): Promi
 	return { nodes, failures, keyNetwork };
 }
 
+/**
+ * The nodes the key network keeps OUT of `blockId`'s cohort — peers that receive none of the
+ * block's cluster traffic, and so hold none of its content until something repairs them.
+ *
+ * Peer ids are generated fresh per mesh, so which node is responsible for a given block is random
+ * from run to run: in a 3-node `responsibilityK: 1` mesh, `nodes[1]` is the block's sole responsible
+ * peer about a third of the time, and then it receives the writer's commit directly. A test that
+ * needs a genuinely non-responsible node has to ask the routing layer rather than assume an index.
+ */
+export async function nonResponsibleNodes(mesh: Mesh, blockId: string): Promise<MeshNode[]> {
+	const cohort = await mesh.keyNetwork.findCluster(new TextEncoder().encode(blockId));
+	return mesh.nodes.filter(node => !(node.peerId.toString() in cohort));
+}
+
 export interface BuildTransactorOptions {
 	timeoutMs?: number;
 	abortOrCancelTimeoutMs?: number;