@optimystic/db-p2p 1.0.0-beta.3 → 1.1.0

package/src/repo/cluster-coordinator.ts

@@ -1,6 +1,6 @@
 import { peerIdFromString } from "@libp2p/peer-id";
 import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, ClusterConsensusConfig, ICluster, PendResult, CommitResult, StaleFailure } from "@optimystic/db-core";
-import { CURRENT_MEMBERSHIP_VERSION, computeClusterMessageHash, isConflictFailure, membershipDigest } from "@optimystic/db-core";
+import { CURRENT_MEMBERSHIP_VERSION, computeClusterMessageHash, isConflictFailure, membershipDigest, routingKeyForBlock } from "@optimystic/db-core";
 import { Pending } from "@optimystic/db-core";
 import type { PeerId } from "@libp2p/interface";
 import { createLogger, verbose } from '../logger.js'
@@ -9,6 +9,7 @@ import type { FretService } from "p2p-fret";
 import type { IPeerReputation } from "../reputation/types.js";
 import { PenaltyReason } from "../reputation/types.js";
 import type { ITransactionStateStore } from "../cluster/i-transaction-state-store.js";
+import { ResponsibilityRefusalError } from "./responsibility.js";
 
 const log = createLogger('cluster')
 
@@ -91,6 +92,33 @@ export class ConflictRaceLostError extends Error {
 	}
 }
 
+/**
+ * The transaction's pend could not proceed because one or more members answered with a signed `held`
+ * vote: the requested blocks are reserved by a different unresolved action in that member's durable
+ * storage. Sibling of {@link ConflictRaceLostError} and retryable for the same reason — nobody judged
+ * this write invalid; it queued behind a reservation that disappears when the holder commits or
+ * cancels.
+ *
+ * The two are separate because they name different things. A conflict vote names the winning rival's
+ * `messageHash`, which the member holds whole; a held vote can only name the rival's **action id**,
+ * because it fires in the window where the rival has left the member's in-memory table but not yet its
+ * storage. `CoordinatorRepo.pend` converts this into a `StaleFailure` with `conflict: true` so the
+ * normal retry machinery (`isConflictFailure`) absorbs it, exactly as it does a lost race.
+ *
+ * Only a PEND record can produce it: `held` votes come from `ClusterMember.validatePendOperations`,
+ * which inspects pend operations only, so `CoordinatorRepo.commit` never meets one.
+ */
+export class BlocksHeldError extends Error {
+	constructor(
+		message: string,
+		/** peerId → actionId of the unresolved action that member's storage says holds the blocks. */
+		readonly heldBy: Record<string, string>
+	) {
+		super(message);
+		this.name = 'BlocksHeldError';
+	}
+}
+
 /** Cancel handle for an injected timer; cancels a not-yet-fired timer (safe no-op after fire/cancel). */
 export type TimerCancel = () => void;
 
@@ -140,6 +168,16 @@ interface ClusterTransactionState {
 }
 
 /** Manages distributed transactions across clusters */
+/**
+ * What a cohort lookup established about a block's cohort. `resolved: false` covers BOTH a lookup
+ * that threw and one that answered with nobody: neither names a destination for a write, and the
+ * durability class both produce is the same (`unrouted`). `reason` is for logs only — never branch
+ * on it.
+ */
+export type CohortResolution =
+	| { readonly resolved: true; readonly peerIds: readonly string[] }
+	| { readonly resolved: false; readonly reason: string };
+
 export class ClusterCoordinator {
 	private transactions: Map<string, ClusterTransactionState> = new Map();
 	private readonly retryInitialIntervalMs: number;
@@ -165,6 +203,8 @@ export class ClusterCoordinator {
 			getExecutedPendResult?: (messageHash: string) => PendResult | undefined;
 			/** Local storage's verdict for a commit applied during consensus; see ClusterMember.getExecutedCommitResult. */
 			getExecutedCommitResult?: (messageHash: string) => CommitResult | undefined;
+			/** One more reconcile for a behind-refused commit, once remote members hold it; see ClusterMember.reconcileRefusedCommit. */
+			reconcileRefusedCommit?: (record: ClusterRecord) => Promise<void>;
 		},
 		private readonly fretService?: FretService,
 		private readonly reputation?: IPeerReputation,
@@ -236,21 +276,76 @@ export class ClusterCoordinator {
 	}
 
 	/**
-	 * Gets all peers in the cluster for a specific block ID
+	 * The ONE cohort lookup every accessor on this class derives from: the raw peer map when the key
+	 * network answered, otherwise the reason it did not. A thrown `findCluster` is logged here and
+	 * nowhere else. Callers that need the map (`executeClusterTransaction`, which builds the record's
+	 * `peers`) go through {@link getClusterForBlock}; callers that need to know whether the cohort
+	 * RESOLVED go through {@link resolveCohort}.
 	 */
-	private async getClusterForBlock(blockId: BlockId): Promise<ClusterPeers> {
-		const blockIdBytes = new TextEncoder().encode(blockId);
+	private async lookupCluster(blockId: BlockId): Promise<{ peers: ClusterPeers } | { reason: string }> {
 		try {
-			const peers = await this.keyNetwork.findCluster(blockIdBytes);
+			const peers = await this.keyNetwork.findCluster(routingKeyForBlock(blockId));
 			const peerIds = Object.keys(peers ?? {});
 			log('cluster-tx:cluster-members', { blockId, peerIds });
-			return peers;
+			return { peers: peers ?? {} };
 		} catch (e) {
 			log('WARN findCluster failed for %s: %o', blockId, e)
-			return {} as ClusterPeers
+			return { reason: `findCluster threw: ${(e as Error)?.message ?? String(e)}` };
 		}
 	}
 
+	/**
+	 * Gets all peers in the cluster for a specific block ID. Empty when the lookup failed — the
+	 * consensus path treats "no cohort" and "lookup failed" alike (there is nobody to run consensus
+	 * with either way); a caller that must tell them apart uses {@link resolveCohort}.
+	 */
+	private async getClusterForBlock(blockId: BlockId): Promise<ClusterPeers> {
+		const outcome = await this.lookupCluster(blockId);
+		return 'peers' in outcome ? outcome.peers : {};
+	}
+
+	/**
+	 * Whether the block's cohort could be established, and who it is. The primitive behind
+	 * {@link getClusterPeerIds} and {@link getClusterSize}: a lookup that threw and a lookup that named
+	 * nobody used to reach every caller as the same empty list, and `CoordinatorRepo`'s solo
+	 * short-circuit then acknowledged a write it had no idea where to send exactly as it acknowledged a
+	 * write to a genuine cohort of one (GitHub #19). Both shapes are still `resolved: false` here —
+	 * neither names a destination — but they are distinguishable from a resolved cohort, which is what
+	 * the write's durability class needs (`unrouted` vs `local`).
+	 */
+	async resolveCohort(blockId: BlockId): Promise<CohortResolution> {
+		const outcome = await this.lookupCluster(blockId);
+		if ('reason' in outcome) return { resolved: false, reason: outcome.reason };
+		const peerIds = Object.keys(outcome.peers);
+		if (peerIds.length === 0) return { resolved: false, reason: 'findCluster named nobody' };
+		return { resolved: true, peerIds };
+	}
+
+	/**
+	 * A node never runs a cluster transaction for a cohort it is not in. Behind members reconcile from the
+	 * coordinator's own proof-carrying copy (its member applies before the merged record fans out), and a
+	 * coordinator outside `record.peers` is not a reconcile target — so a cohort with no holder would stay
+	 * behind and the commit durability gate would refuse, having first put this node's vote and storage
+	 * where the cohort does not look. The invariant is held here, at the one place a record's `peers` is
+	 * chosen, rather than left to the routing convention.
+	 *
+	 * Fires only on a RESOLVED cohort (at least one peer) that excludes the wired local member. An empty
+	 * cohort is a failed lookup, not a cohort this node is outside of, so it is left to `executeTransaction`'s
+	 * size checks; `CoordinatorRepo`'s solo short-circuit keeps unresolved and single-peer cohorts away from
+	 * this method altogether in any case. After its responsibility check, what remains is a multi-member
+	 * cohort that changed inside the responsibility cache's staleness window. With no local member wired the guard does not apply: that
+	 * bypass exists for wiring without an identity (direct constructors, some tests), never for production.
+	 */
+	private assertLocalMemberInCohort(blockId: BlockId, peers: ClusterPeers): void {
+		if (!this.localCluster) return;
+		const peerIds = Object.keys(peers);
+		const selfId = this.localCluster.peerId.toString();
+		if (peerIds.length === 0 || peerIds.includes(selfId)) return;
+		log('cluster-tx:not-in-cohort', { blockId, selfId, peerIds });
+		throw new ResponsibilityRefusalError('not-responsible', [blockId],
+			`refusing to coordinate a cluster transaction for a cohort this node is not in: ${peerIds.join(', ')}`);
+	}
+
 	private makeRecord(peers: ClusterPeers, messageHash: string, message: RepoMessage, membershipDigestValue: string): ClusterRecord {
 		const peerCount = Object.keys(peers ?? {}).length;
 		const record: ClusterRecord = {
@@ -302,7 +397,9 @@ export class ClusterCoordinator {
 		 * during consensus, when the member retained one. Same availability contract as
 		 * `localPendResult`. `CoordinatorRepo.commit` uses a retained refusal to detect a rival's
 		 * win swallowed by the member-side ahead-divergence tolerance, instead of fabricating a
-		 * success no member durably stored.
+		 * success no member durably stored. Read after the commit broadcast, so a behind member's
+		 * verdict already reflects the second reconcile `broadcastMergedRecord` gives it once a
+		 * remote member holds the revision.
 		 */
 		localCommitResult?: CommitResult;
 		/**
@@ -356,6 +453,7 @@ export class ClusterCoordinator {
 
 		// Get the cluster peers for this block
 		const peers = await this.getClusterForBlock(blockId);
+		this.assertLocalMemberInCohort(blockId, peers);
 
 		// Bind the responsible membership into the transaction identity (v2): the digest is folded into
 		// the messageHash below, so two different peer sets produce two different messageHashes rather
@@ -494,18 +592,24 @@ export class ClusterCoordinator {
 		const promised = await this.collectPromises(peers, record);
 		const superMajority = Math.ceil(peerCount * this.cfg.superMajorityThreshold);
 
-		// Count approvals, rejections and conflict votes separately. A `conflict` vote is a member
-		// saying "not now — I hold the race winner": it must count toward NEITHER approvals NOR
-		// rejections, or a lost race would masquerade as a validator rejection (permanent) or as
-		// silence (indistinguishable from an unreachable cohort) — both wrong.
+		// Count approvals, rejections and the two RETRYABLE refusals separately. A `conflict` vote is a
+		// member saying "not now — I hold the race winner"; a `held` vote is a member saying "not now —
+		// a different unresolved action holds these blocks in my storage". Neither may count toward
+		// approvals OR rejections, or a transient refusal would masquerade as a validator rejection
+		// (permanent) or as silence (indistinguishable from an unreachable cohort) — both wrong.
 		const promises = promised.record.promises;
 		const approvalCount = Object.values(promises).filter(sig => sig.type === 'approve').length;
 		const rejectionCount = Object.values(promises).filter(sig => sig.type === 'reject').length;
 		const conflictCount = Object.values(promises).filter(sig => sig.type === 'conflict').length;
+		const heldCount = Object.values(promises).filter(sig => sig.type === 'held').length;
 
 		// Check if rejections make super-majority impossible
 		// If more than (peerCount - superMajority) nodes reject, we can never reach super-majority
 		const maxAllowedRejections = peerCount - superMajority;
+		// Whether the merged record itself PROVES super-majority unreachable — the same sum a member
+		// re-derives as `ConflictSuperseded`/`Rejected` from the signed votes, which is what makes an
+		// abandonment broadcast proof-carrying rather than an unauthenticated "forget this".
+		const refusalsProveUnreachable = rejectionCount + conflictCount + heldCount > maxAllowedRejections;
 		if (rejectionCount > maxAllowedRejections) {
 			const rejectReasonsByPeer = Object.fromEntries(Object.entries(promises)
 				.flatMap(([peerId, sig]) => sig.type === 'reject' ? [[peerId, sig.rejectReason ?? 'unknown'] as const] : []));
@@ -552,7 +656,7 @@ export class ClusterCoordinator {
 			// super-majority (members re-derive ConflictSuperseded/Rejected from the signed votes and
 			// clear their reservations immediately). Below that bar the record proves nothing and a
 			// broadcast would be the unauthenticated "forget this" the shortfall NOTE below refuses.
-			if (rejectionCount + conflictCount > maxAllowedRejections) {
+			if (refusalsProveUnreachable) {
 				this.broadcastAbandonment(promised.record, 'conflict-race-lost');
 			}
 			throw new ConflictRaceLostError(
@@ -560,6 +664,31 @@ export class ClusterCoordinator {
 				conflicts);
 		}
 
+		// A `held`-answered shortfall is the OTHER retryable refusal: the pend queued behind a rival's
+		// unresolved reservation. Checked after the conflict branch so a lost race still wins when both
+		// answer — a conflict vote names the winning transaction's messageHash, which is strictly more
+		// actionable than an action id — and, like it, before the generic shortfall, which must stay
+		// reserved for the genuinely-silent cohort.
+		if (heldCount > 0 && approvalCount < superMajority) {
+			const heldBy = Object.fromEntries(Object.entries(promises)
+				.flatMap(([peerId, sig]) => sig.type === 'held' ? [[peerId, sig.heldBy] as const] : []));
+			log('cluster-tx:pend-blocks-held', {
+				messageHash: record.messageHash,
+				peerCount,
+				approvals: approvalCount,
+				rejections: rejectionCount,
+				heldBy,
+				superMajority
+			});
+			this.updateTransactionRecord(promised.record, 'pend-blocks-held');
+			if (refusalsProveUnreachable) {
+				this.broadcastAbandonment(promised.record, 'pend-blocks-held');
+			}
+			throw new BlocksHeldError(
+				`Pend blocks held: ${heldCount}/${peerCount} member(s) hold an unresolved rival action (${approvalCount}/${superMajority} approvals)`,
+				heldBy);
+		}
+
 		if (peerCount > 1 && approvalCount < superMajority) {
 			log('cluster-tx:supermajority-failed', {
 				messageHash: record.messageHash,
@@ -578,7 +707,8 @@ export class ClusterCoordinator {
 			// their own staleness sweep instead.
 			// NOTE: the message below is load-bearing wire text — the consuming repo
 			// (sereus cadre-core control-write-retry) matches it verbatim to retry a genuinely-silent
-			// cohort. Keep it byte-identical, and never fold conflict votes into its rejection count.
+			// cohort. Keep it byte-identical, and never fold `conflict` or `held` votes into its
+			// rejection count.
 			throw new Error(`Failed to get super-majority: ${approvalCount}/${peerCount} approvals (needed ${superMajority}, ${rejectionCount} rejections)`);
 		}
 
@@ -614,14 +744,14 @@ export class ClusterCoordinator {
 	}
 
 	/**
-	 * The block's cohort peer ids as currently derivable. Empty when `findCluster` fails
-	 * (getClusterForBlock swallows the throw), so a caller branching on `length <= 1` is also taking
-	 * the degraded-routing branch; `CoordinatorRepo.commit` uses the ids to log whether a solo cohort
-	 * is genuinely just self or a routing failure.
+	 * The block's cohort peer ids as currently derivable. Empty when the cohort did not resolve
+	 * ({@link resolveCohort}: `findCluster` threw, or named nobody), so a caller branching on
+	 * `length <= 1` is also taking the degraded-routing branch. Derived from `resolveCohort` rather
+	 * than re-deriving the cohort, so there is exactly one lookup rule.
 	 */
 	async getClusterPeerIds(blockId: BlockId): Promise<string[]> {
-		const peers = await this.getClusterForBlock(blockId);
-		return Object.keys(peers ?? {});
+		const cohort = await this.resolveCohort(blockId);
+		return cohort.resolved ? [...cohort.peerIds] : [];
 	}
 
 	/** {@link getClusterPeerIds}, counted. Derived from it rather than re-deriving the cohort, so the
@@ -922,11 +1052,14 @@ export class ClusterCoordinator {
 	 * reconcile target and gains nothing from this ordering; the durability gate in
 	 * `CoordinatorRepo.commit` is what makes that shape refuse rather than acknowledge.
 	 *
-	 * NOTE: when the coordinating member is ITSELF behind (it never saw the pend), its reconcile
-	 * runs here before any remote member has applied, finds no holder, and reports not-durable; the
-	 * remote members then apply and may carry the majority on their own. Fine while the coordinator
-	 * ordinarily saw the pend; if coordinators are routinely picked after the pend phase, deliver
-	 * local-first only when the local member holds the pend, or reconcile it once more afterwards.
+	 * The mirror case — the coordinating member is ITSELF behind (no pend, or no base for the block)
+	 * — is the price of that order: its reconcile runs before any remote member has applied, finds
+	 * no holder, and retains a refusal. So once the remote members have answered, and at least one
+	 * reported holding the revision, this node's own member gets one more reconcile
+	 * (`reconcileRefusedCommit`). The member skips it unless its retained refusal has the behind
+	 * shape, so only a behind coordinator pays the extra fetch. It finishes before this method
+	 * returns, so `executeClusterTransaction` reads the refreshed verdict, and a two-member cohort
+	 * whose members both end up holding the commit is no longer refused as not durable.
 	 */
 	private async broadcastMergedRecord(record: ClusterRecord, peerIds: string[]): Promise<{ failures: string[]; applyOutcomes?: ClusterRecord['applyOutcomes'] }> {
 		const deliver = async (peerIdStr: string) => {
@@ -958,9 +1091,34 @@ export class ClusterCoordinator {
 		// asked, so a member cannot report an outcome on another member's behalf by echoing a record
 		// full of entries. Unsigned and advisory either way — see ClusterRecord.applyOutcomes.
 		const applyOutcomes = collectApplyOutcomes(results);
+		// NOTE: after a healing second reconcile, `applyOutcomes[selfId].commit` still carries the
+		// pre-reconcile refusal. Nothing reads the self entry today (the gate reads
+		// `localCommitResult`); if anything starts to, re-stamp it from `getExecutedCommitResult` here.
+		// NOTE: in a 3+ cohort this also runs when the remote holders already form a majority without
+		// this member — one extra fetch that heals its copy; gate on the remote count if it ever shows up.
+		const remoteHolds = remote.some(id => applyOutcomes?.[id]?.commit?.success === true);
+		if (remoteHolds && localResults.some(r => r.success)) {
+			await this.reconcileLocalMemberAgain(record);
+		}
 		return { failures, ...(applyOutcomes === undefined ? {} : { applyOutcomes }) };
 	}
 
+	/**
+	 * Give this node's own member its second reconcile (see {@link broadcastMergedRecord}). The
+	 * member contract is never to throw; the catch keeps a broken seam from failing a transaction
+	 * the remote members already applied.
+	 */
+	private async reconcileLocalMemberAgain(record: ClusterRecord): Promise<void> {
+		try {
+			await this.localCluster?.reconcileRefusedCommit?.(record);
+		} catch (err) {
+			log('cluster-tx:local-reconcile-again-error', {
+				messageHash: record.messageHash,
+				error: err instanceof Error ? err.message : String(err)
+			});
+		}
+	}
+
 	/**
 	 * Fire-and-forget replay of an abandoned transaction's record to every peer in its cohort.
 	 *