@optimystic/db-p2p 0.24.1 → 0.25.0

package/src/repo/cluster-coordinator.ts

@@ -1,5 +1,5 @@
 import { peerIdFromString } from "@libp2p/peer-id";
-import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, ClusterConsensusConfig, ICluster } from "@optimystic/db-core";
+import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, ClusterConsensusConfig, ICluster, PendResult, CommitResult } from "@optimystic/db-core";
 import { CURRENT_MEMBERSHIP_VERSION, computeClusterMessageHash, membershipDigest } from "@optimystic/db-core";
 import { Pending } from "@optimystic/db-core";
 import type { PeerId } from "@libp2p/interface";
@@ -37,10 +37,17 @@ export class ValidatorRejectionError extends Error {
  * vote (they hold a rival transaction that won the deterministic race on the same blocks) and
  * approvals fell short of super-majority. Distinct from {@link ValidatorRejectionError} — nobody
  * judged this write invalid; it lost an optimistic-concurrency race and a fresh retry can win.
- * `CoordinatorRepo.pend` converts this into a `StaleFailure` with `conflict: true` so the normal
- * retry machinery (`isConflictFailure`) absorbs it; it should escape as a thrown error only from
- * paths other than pend. The conflicting peers and the winning hashes ride as structured data
- * (from the signed `conflictWith` fields), never parsed out of prose.
+ * `CoordinatorRepo.pend` AND `CoordinatorRepo.commit` both convert this into a `StaleFailure` with
+ * `conflict: true` so the normal retry machinery (`isConflictFailure`) absorbs it; it should escape
+ * as a thrown error only from other paths. The commit conversion matters as much as the pend one:
+ * at the moment this is thrown zero members approved and the members hold the winner — nothing of
+ * the loser landed — yet a THROWN commit error is retried verbatim by db-core's `commitCollection`
+ * (it treats throws as transport faults), and that re-driven commit races into the window after
+ * members apply the winner and clear its reservation, where it can assemble a consensus no member
+ * will durably store. A returned conflict is instead surfaced immediately as a stale loss, and the
+ * writer re-reads and re-drives the whole pend+commit at a fresh revision. The conflicting peers
+ * and the winning hashes ride as structured data (from the signed `conflictWith` fields), never
+ * parsed out of prose.
  */
 export class ConflictRaceLostError extends Error {
 	constructor(
@@ -123,6 +130,10 @@ export class ClusterCoordinator {
 			update: (record: ClusterRecord) => Promise<ClusterRecord>;
 			peerId: PeerId;
 			wasTransactionExecuted?: (messageHash: string) => boolean;
+			/** Local storage's verdict for a pend applied during consensus; see ClusterMember.getExecutedPendResult. */
+			getExecutedPendResult?: (messageHash: string) => PendResult | undefined;
+			/** Local storage's verdict for a commit applied during consensus; see ClusterMember.getExecutedCommitResult. */
+			getExecutedCommitResult?: (messageHash: string) => CommitResult | undefined;
 		},
 		private readonly fretService?: FretService,
 		private readonly reputation?: IPeerReputation,
@@ -183,7 +194,7 @@ export class ClusterCoordinator {
 	 * reproduces the legacy v1 hash byte-for-byte.
 	 *
 	 * NOTE: the whole `message` is hashed (canonicalJson), so a transaction's advisory aged priority —
-	 * which rides inside the pend operation as `pend.transaction.priority` (multi-collection) or
+	 * which rides inside the pend operation as `pend.validation.transaction.priority` (multi-collection) or
 	 * `pend.priority` (single-collection) — is automatically covered here and by the derived
 	 * promise/commit hashes. That is what makes priority integrity-protected in transit: a relaying peer
 	 * cannot strip or inflate it without invalidating the message hash the members verify. No separate
@@ -219,7 +230,6 @@ export class ClusterCoordinator {
 			membershipVersion: CURRENT_MEMBERSHIP_VERSION,
 			membershipDigest: membershipDigestValue,
 			message,
-			coordinatingBlockIds: message.coordinatingBlockIds,
 			promises: {},
 			commits: {},
 			suggestedClusterSize: peerCount || undefined,
@@ -249,7 +259,41 @@ export class ClusterCoordinator {
 	async executeClusterTransaction(blockId: BlockId, message: RepoMessage, _options?: MessageOptions): Promise<{
 		record: ClusterRecord;
 		localExecuted: boolean;
+		/**
+		 * Local storage's verdict for a pend operation this node's own cluster member applied during
+		 * consensus, when the member retained one. Meaningful only when `localExecuted` is true;
+		 * absent for non-pend messages, for a member that predates the retention, or after the
+		 * retention TTL. `CoordinatorRepo.pend` returns this instead of fabricating a success.
+		 */
+		localPendResult?: PendResult;
+		/**
+		 * Local storage's verdict for a commit operation this node's own cluster member applied
+		 * during consensus, when the member retained one. Same availability contract as
+		 * `localPendResult`. `CoordinatorRepo.commit` uses a retained refusal to detect a rival's
+		 * win swallowed by the member-side ahead-divergence tolerance, instead of fabricating a
+		 * success no member durably stored.
+		 */
+		localCommitResult?: CommitResult;
 	}> {
+		// The coordinating block id is derived HERE, from the key this method is already handed, rather
+		// than being set by each caller's message builder: a member's membership admission gate derives
+		// its own cohort view from this field, and a builder that forgets it silently downgrades the gate
+		// to its fallback floor on that path (which is how `commit` and `cancel` used to strand writes —
+		// admitted at pend, refused at commit). Doing it at the single choke point means a future message
+		// builder cannot reintroduce the gap.
+		//
+		// Two constraints this shape exists to satisfy:
+		//  - COPY, never mutate: `CoordinatorRepo.cancel` builds ONE message and hands the same object to
+		//    N concurrent calls, one per block. In-place mutation would leak one block's id into another
+		//    block's transaction.
+		//  - Preserve an already-present list: `pend` deliberately declares the whole consolidated batch,
+		//    not just its first block, so this must not overwrite it. Tested on `length`, not on the
+		//    field: an empty list carries no id for a member to derive from, so preserving one would be
+		//    the same silent downgrade to the fallback floor this choke point exists to prevent.
+		const coordinated: RepoMessage = message.coordinatingBlockIds?.length
+			? message
+			: { ...message, coordinatingBlockIds: [blockId] };
+
 		// Get the cluster peers for this block
 		const peers = await this.getClusterForBlock(blockId);
 
@@ -258,11 +302,14 @@ export class ClusterCoordinator {
 		// than one hash with a silent internal disagreement about who is responsible.
 		const membershipDigestValue = await membershipDigest(peers);
 
-		// Create a unique hash for this transaction (over message + membership digest)
-		const messageHash = await this.createMessageHash(message, membershipDigestValue);
+		// Create a unique hash for this transaction (over message + membership digest). Hashing the
+		// coordinating-block-bearing copy is what makes the field tamper-evident in transit — and it also
+		// makes a multi-block `cancel` produce a distinct hash per block, where before two blocks with
+		// identical cohorts collided on one `messageHash` in `this.transactions` / `wasTransactionExecuted`.
+		const messageHash = await this.createMessageHash(coordinated, membershipDigestValue);
 
 		// Create a cluster record for this transaction
-		const record = this.makeRecord(peers, messageHash, message, membershipDigestValue);
+		const record = this.makeRecord(peers, messageHash, coordinated, membershipDigestValue);
 		log('cluster-tx:start', {
 			messageHash,
 			blockId,
@@ -296,7 +343,14 @@ export class ClusterCoordinator {
 			const result = await pending.result();
 			// Check if the local cluster already executed the operations during consensus
 			const localExecuted = this.localCluster?.wasTransactionExecuted?.(messageHash) ?? false;
-			return { record: result, localExecuted };
+			const localPendResult = localExecuted ? this.localCluster?.getExecutedPendResult?.(messageHash) : undefined;
+			const localCommitResult = localExecuted ? this.localCluster?.getExecutedCommitResult?.(messageHash) : undefined;
+			return {
+				record: result,
+				localExecuted,
+				...(localPendResult === undefined ? {} : { localPendResult }),
+				...(localCommitResult === undefined ? {} : { localCommitResult })
+			};
 		} finally {
 			const stored = this.transactions.get(messageHash);
 			const retrySnapshot = stored?.retry ? {