@optimystic/db-p2p 0.13.4 → 0.14.0

package/src/cluster/cluster-repo.ts

@@ -1,7 +1,9 @@
-import type { IRepo, ClusterRecord, Signature, RepoMessage, ITransactionValidator, ClusterConsensusConfig } from "@optimystic/db-core";
+import type { IRepo, ClusterRecord, Signature, RepoMessage, ITransactionValidator, ClusterConsensusConfig, CommitResult, BlockId, ActionId, ActionRev, CommitRequest, CommitCert, InvalidateRequest } from "@optimystic/db-core";
 import type { ICluster } from "@optimystic/db-core";
 import type { IPeerNetwork } from "@optimystic/db-core";
 import { blockIdsForTransforms } from "@optimystic/db-core";
+import { verifyInvalidationCertificate, type ArbitratorSetRecompute } from "../dispute/invalidation.js";
+import { buildCommitCert, invalidationActionId } from "./commit-cert.js";
 import { ClusterClient } from "./client.js";
 import type { PeerId, PrivateKey } from "@libp2p/interface";
 import { peerIdFromString } from "@libp2p/peer-id";
@@ -35,6 +37,54 @@ interface TransactionState {
 	lastUpdate: number;
 }
 
+/**
+ * Actively reconciles a block this member committed without having seen the matching
+ * pend (cohort drift between the independent pend and commit cluster-transactions).
+ * Pulls the committed revision from a cohort peer that holds it and restores it into
+ * local storage. Injected so {@link ClusterMember} stays transport-agnostic — mirrors
+ * how `CoordinatorRepo` receives its `clusterLatestCallback`.
+ *
+ * @param blockId       the under-replicated block to restore
+ * @param committed     the committed `(actionId, rev)` agreed by consensus
+ * @param cohortPeerIds cohort members to pull from (self already excluded)
+ */
+export type ReconcileBlockCallback = (blockId: BlockId, committed: ActionRev, cohortPeerIds: string[]) => Promise<void>;
+
+/**
+ * Sink for the {@link CommitCert} this member assembled at consensus, fired once per committed action
+ * **before** the commit is applied to local storage (so it is already retained when
+ * {@link StorageRepo.commit} emits the matching `CollectionChangeEvent` the reactivity bridge reads).
+ * The bytes are the cluster's own `approve` commit signatures, forwarded UNCHANGED — never re-signed.
+ * Optional; absent on nodes that do not originate reactivity notifications. A throwing sink is
+ * isolated + logged (it must never break consensus).
+ */
+export type CommitCertificateSink = (actionId: ActionId, cert: CommitCert) => void;
+
+/**
+ * Applies a consensus-ordered {@link InvalidateRequest} to local storage — the deterministic
+ * reversal every member runs once consensus on the invalidation is reached. The implementation
+ * recomputes the per-block as-if-`T_inv`-absent content, writes the compensating revisions, and
+ * appends the durable invalidation log entry (see `applyInvalidation` in the dispute module). It
+ * is injected so {@link ClusterMember} stays storage/log-agnostic (mirrors {@link ReconcileBlockCallback}).
+ *
+ * The certificate is verified by {@link ClusterMember} *before* the sink is invoked, so a sink
+ * implementation may assume `request.resolution` is already a valid challenger-wins certificate.
+ * A throwing sink is logged and tolerated (never resets the cluster stream); the in-memory dedup
+ * marker is rolled back so a re-broadcast can retry.
+ */
+export type InvalidationApplySink = (request: InvalidateRequest) => Promise<void>;
+
+/**
+ * Optional **layer-2** capability for invalidation-certificate verification: re-derives the
+ * legitimately-selected arbitrator set from this member's topology view and judges the carried set
+ * (see {@link ArbitratorSetRecompute}). Injected so {@link ClusterMember} stays network-agnostic; the
+ * composition root supplies it from FRET when available. When absent (or when the member cannot
+ * reconstruct the historical topology), {@link applyConsensusInvalidation} accepts a layer-1-valid
+ * certificate and logs that it applied an invalidation it could not fully anchor — the documented
+ * interim posture (see `tickets/plan/cohort-topic-membership-cert-trust-anchoring.md`).
+ */
+export type RecomputeArbitratorSetCapability = ArbitratorSetRecompute;
+
 interface ClusterMemberComponents {
 	storageRepo: IRepo;
 	peerNetwork: IPeerNetwork;
@@ -48,6 +98,14 @@ interface ClusterMemberComponents {
 	reputation?: IPeerReputation;
 	consensusConfig?: ClusterConsensusConfig;
 	stateStore?: ITransactionStateStore;
+	/** Restores a block under-replicated by cohort drift; see {@link ReconcileBlockCallback}. */
+	reconcileBlock?: ReconcileBlockCallback;
+	/** Receives the consensus commit cert per committed action; see {@link CommitCertificateSink}. */
+	onCommitCertificate?: CommitCertificateSink;
+	/** Applies a consensus-ordered invalidation to local storage; see {@link InvalidationApplySink}. */
+	onInvalidate?: InvalidationApplySink;
+	/** Layer-2 arbitrator-set recompute for invalidation verification; see {@link RecomputeArbitratorSetCapability}. */
+	recomputeArbitratorSet?: RecomputeArbitratorSetCapability;
 }
 
 export function clusterMember(components: ClusterMemberComponents): ClusterMember {
@@ -62,13 +120,32 @@ export function clusterMember(components: ClusterMemberComponents): ClusterMembe
 		components.validator,
 		components.reputation,
 		components.consensusConfig,
-		components.stateStore
+		components.stateStore,
+		components.reconcileBlock,
+		components.onCommitCertificate,
+		components.onInvalidate,
+		components.recomputeArbitratorSet
 	);
 }
 
 // How long to keep executed transaction records (10 minutes)
 const ExecutedTransactionTtlMs = 10 * 60 * 1000;
 
+// Upper bound on an awaited active reconciliation of a divergent commit. Bounds the
+// consensus path so a slow/unreachable cohort peer can't stall the cluster stream;
+// a timeout is logged and tolerated (never thrown — that would reset the stream).
+const ReconcileTimeoutMs = 5000;
+
+/**
+ * True when a thrown storage error reports a missing pending action — i.e. this
+ * member reached commit-consensus without having seen the matching pend phase.
+ * That is recoverable local divergence (reconciled via sync / read-repair), not
+ * a transaction fault, so it must not reset the cluster stream.
+ */
+function isMissingPendingActionError(err: unknown): boolean {
+	return err instanceof Error && /pending action .+ not found/i.test(err.message);
+}
+
 /**
  * Handles cluster-side operations, managing promises and commits for cluster updates
  * and coordinating with the local storage repo.
@@ -78,12 +155,15 @@ export class ClusterMember implements ICluster {
 	private activeTransactions: Map<string, TransactionState> = new Map();
 	// Track executed consensus transactions to prevent duplicate execution (messageHash -> executedAt timestamp)
 	private executedTransactions: Map<string, number> = new Map();
+	// Fast in-memory dedup for applied invalidations, keyed `${invalidatedActionId}:${disputeId}`.
+	// The durable source of truth is the invalidation log entry (Log.findInvalidation, re-checked
+	// inside the sink); this map only spares redundant work when the same invalidation reaches
+	// consensus twice (rebroadcast / sync) under different message hashes. (-> appliedAt timestamp)
+	private appliedInvalidations: Map<string, number> = new Map();
 	// Queue of transactions to clean up
 	private cleanupQueue: string[] = [];
 	// Serialize concurrent updates for the same transaction
 	private pendingUpdates: Map<string, Promise<ClusterRecord>> = new Map();
-	// Temporarily set during validateSignatures so verifySignature can access the record
-	private currentValidationRecord?: ClusterRecord;
 	// Interval handles for periodic cleanup (stored so dispose() can clear them)
 	private readonly expirationInterval: NodeJS.Timeout;
 	private readonly cleanupInterval: NodeJS.Timeout;
@@ -103,7 +183,11 @@ export class ClusterMember implements ICluster {
 		private readonly validator?: ITransactionValidator,
 		private readonly reputation?: IPeerReputation,
 		consensusConfig?: ClusterConsensusConfig,
-		private readonly stateStore?: ITransactionStateStore
+		private readonly stateStore?: ITransactionStateStore,
+		private readonly reconcileBlock?: ReconcileBlockCallback,
+		private readonly onCommitCertificate?: CommitCertificateSink,
+		private readonly onInvalidate?: InvalidationApplySink,
+		private readonly recomputeArbitratorSet?: RecomputeArbitratorSetCapability
 	) {
 		this.superMajorityThreshold = consensusConfig?.superMajorityThreshold ?? 1.0;
 		// Periodically clean up expired transactions (.unref() so tests/short-lived processes can exit)
@@ -440,27 +524,22 @@ export class ClusterMember implements ICluster {
 	}
 
 	private async validateSignatures(record: ClusterRecord): Promise<void> {
-		this.currentValidationRecord = record;
-		try {
-			// Validate promise signatures
-			const promiseHash = await this.computePromiseHash(record);
-			for (const [peerId, signature] of Object.entries(record.promises)) {
-				if (!await this.verifySignature(peerId, promiseHash, signature)) {
-					this.reputation?.reportPeer(peerId, PenaltyReason.InvalidSignature, `promise:${record.messageHash}`);
-					throw new Error(`Invalid promise signature from ${peerId}`);
-				}
+		// Validate promise signatures
+		const promiseHash = await this.computePromiseHash(record);
+		for (const [peerId, signature] of Object.entries(record.promises)) {
+			if (!await this.verifySignature(record, peerId, promiseHash, signature)) {
+				this.reputation?.reportPeer(peerId, PenaltyReason.InvalidSignature, `promise:${record.messageHash}`);
+				throw new Error(`Invalid promise signature from ${peerId}`);
 			}
+		}
 
-			// Validate commit signatures
-			const commitHash = await this.computeCommitHash(record);
-			for (const [peerId, signature] of Object.entries(record.commits)) {
-				if (!await this.verifySignature(peerId, commitHash, signature)) {
-					this.reputation?.reportPeer(peerId, PenaltyReason.InvalidSignature, `commit:${record.messageHash}`);
-					throw new Error(`Invalid commit signature from ${peerId}`);
-				}
+		// Validate commit signatures
+		const commitHash = await this.computeCommitHash(record);
+		for (const [peerId, signature] of Object.entries(record.commits)) {
+			if (!await this.verifySignature(record, peerId, commitHash, signature)) {
+				this.reputation?.reportPeer(peerId, PenaltyReason.InvalidSignature, `commit:${record.messageHash}`);
+				throw new Error(`Invalid commit signature from ${peerId}`);
 			}
-		} finally {
-			this.currentValidationRecord = undefined;
 		}
 	}
 
@@ -496,8 +575,8 @@ export class ClusterMember implements ICluster {
 		return uint8ArrayToString(sigBytes, 'base64url');
 	}
 
-	private async verifySignature(peerId: string, hash: string, signature: Signature): Promise<boolean> {
-		const peerInfo = this.currentValidationRecord?.peers[peerId];
+	private async verifySignature(record: ClusterRecord, peerId: string, hash: string, signature: Signature): Promise<boolean> {
+		const peerInfo = record.peers[peerId];
 		if (!peerInfo?.publicKey?.length) {
 			throw new Error(`No public key for peer ${peerId}`);
 		}
@@ -677,44 +756,305 @@ export class ClusterMember implements ICluster {
 			.catch(err => log('cluster-member:persist-executed-error', { messageHash: record.messageHash, error: (err as Error).message }));
 
 		try {
-			// Execute the operations - check return values for failures
 			for (const operation of record.message.operations) {
-				if ('get' in operation) {
-					await this.storageRepo.get(operation.get);
-				} else if ('pend' in operation) {
-					const result = await this.storageRepo.pend(operation.pend);
-					if (!result.success) {
-						log('cluster-member:consensus-pend-failed', {
-							messageHash: record.messageHash,
-							actionId: operation.pend.actionId,
-							reason: result.reason,
-							hasMissing: !!result.missing?.length,
-							hasPending: !!result.pending?.length
-						});
-						throw new Error(`Consensus pend failed for action ${operation.pend.actionId}: ${result.reason ?? 'stale revision'}`);
-					}
-				} else if ('commit' in operation) {
-					const result = await this.storageRepo.commit(operation.commit);
-					if (!result.success) {
-						log('cluster-member:consensus-commit-failed', {
-							messageHash: record.messageHash,
-							actionId: operation.commit.actionId,
-							reason: result.reason,
-							hasMissing: !!result.missing?.length
-						});
-						throw new Error(`Consensus commit failed for action ${operation.commit.actionId}: ${result.reason ?? 'stale revision'}`);
-					}
-				} else if ('cancel' in operation) {
-					await this.storageRepo.cancel(operation.cancel.actionRef);
-				}
+				await this.applyConsensusOperation(record, operation);
 			}
 		} catch (err) {
-			// On failure, remove from executedTransactions so it can be retried
+			// A genuinely unexpected fault (e.g. storage I/O) — roll back the executed
+			// marker so a corrected retry can re-run, and propagate so the caller learns
+			// the real cause. Recoverable local divergence is absorbed inside
+			// applyConsensusOperation and never reaches here.
 			this.executedTransactions.delete(record.messageHash);
 			throw err;
 		}
 	}
 
+	/**
+	 * Applies one consensus-approved operation to local storage.
+	 *
+	 * By the time we reach here the cluster has *already* reached consensus, so the
+	 * operation is authoritative cluster-wide. A failure applying it to THIS member's
+	 * local store therefore does not mean the operation is invalid — it means our
+	 * local state has diverged from the agreed history:
+	 *
+	 *   - **ahead**: we already hold a newer revision, so a stale pend/commit is a
+	 *     no-op for us (`StorageRepo.commit` returns `success:false` with `missing`);
+	 *   - **behind**: we missed the prior `pend` cluster-transaction (cohort drift
+	 *     between the independent pend and commit phases, or transient unreachability),
+	 *     so we lack the pending action — `StorageRepo.commit` *throws* "Pending
+	 *     action … not found".
+	 *
+	 * For the **behind** case we hold no revision of the committed blocks at all, so we
+	 * actively reconcile: pull the committed revision from a cohort peer that holds it
+	 * (`reconcileBlock`) and restore it locally. Lazy read-repair on a later read cannot
+	 * recover it on its own when cohort drift has left the block under-replicated (no
+	 * reachable peer the reader sees holds the newer rev), so reconciling here is what
+	 * keeps cross-cohort transactions converging. For the **ahead** case we already hold
+	 * ≥ the committed rev, so we tolerate the no-op without reconciling downward.
+	 *
+	 * Whatever happens, we must NOT reset the stream: throwing here would reset the
+	 * cluster stream the coordinator is awaiting and surface as a spurious
+	 * `StreamResetError`, sinking an otherwise-successful transaction. So divergence —
+	 * and any reconciliation failure — is tolerated (logged for observability). A
+	 * genuinely *invalid* pend can never get this far: it is rejected during the promise
+	 * phase (`validatePendOperations`, which validates pend ops only — commits carry no
+	 * promise-phase validation).
+	 *
+	 * The propagate-vs-tolerate split keys off the failure's *nature* via `CommitResult`,
+	 * not throw-vs-return: a missing pend (thrown "not found") or a stale/ahead commit
+	 * (`success:false` with `missing`) is divergence and tolerated, whereas a genuine
+	 * mid-commit `internalCommit` fault (`success:false` with a bare `reason`, no
+	 * `missing`) is propagated so {@link handleConsensus} rolls back the executed marker
+	 * and rethrows — exactly like an unexpected thrown fault.
+	 */
+	private async applyConsensusOperation(record: ClusterRecord, operation: RepoMessage['operations'][number]): Promise<void> {
+		const messageHash = record.messageHash;
+		if ('get' in operation) {
+			await this.storageRepo.get(operation.get);
+			return;
+		}
+		if ('cancel' in operation) {
+			await this.storageRepo.cancel(operation.cancel.actionRef);
+			return;
+		}
+		if ('pend' in operation) {
+			const result = await this.storageRepo.pend(operation.pend);
+			if (!result.success) {
+				log('cluster-member:consensus-pend-diverged', {
+					messageHash,
+					actionId: operation.pend.actionId,
+					reason: result.reason,
+					hasMissing: !!result.missing?.length,
+					hasPending: !!result.pending?.length
+				});
+			}
+			return;
+		}
+		if ('commit' in operation) {
+			const commit = operation.commit;
+			// Capture the consensus commit cert BEFORE applying to storage: StorageRepo.commit emits the
+			// CollectionChangeEvent synchronously at the end of the call below, and the reactivity bridge
+			// resolves the cert from that event — so it must already be retained when commit() returns.
+			//
+			// The commit-vote signed preimage is computed HERE, before the synchronous capture+commit
+			// sequence below: it is the exact bytes each approving signer endorsed
+			// (`utf8(commitHash + ":approve")`), reproduced from this already-validated `record` (its
+			// commit signatures were verified against the same `computeCommitHash`). Reactivity sets a
+			// notification's `digest` from it so a subscriber's threshold-verify over `digest` succeeds.
+			// The one `await` happens before `captureCommitCert` runs, so the cert is still retained
+			// synchronously before `commit()` emits its change event (do not move this past the commit).
+			// Gated on the sink: with no reactivity wired the preimage has no consumer, so a sink-less
+			// node pays neither the extra `sha256` nor the extra microtask — the true zero-cost default.
+			if (this.onCommitCertificate) {
+				const commitSignedPayload = this.computeSigningPayload(await this.computeCommitHash(record), 'approve');
+				this.captureCommitCert(record, commit.actionId, commitSignedPayload);
+			}
+			let result: CommitResult;
+			try {
+				result = await this.storageRepo.commit(commit);
+			} catch (err) {
+				// `StorageRepo.commit` throws (rather than returning success:false) when
+				// the pending action is missing — the canonical "behind" signal: this
+				// member reached commit-consensus without the matching pend (cohort drift).
+				if (isMissingPendingActionError(err)) {
+					log('cluster-member:consensus-commit-diverged', {
+						messageHash,
+						actionId: commit.actionId,
+						divergence: 'behind',
+						reason: (err as Error).message
+					});
+					// We hold no revision of these blocks; pull the committed revision from a
+					// cohort peer so the block is no longer under-replicated. Best-effort:
+					// failures are logged inside, never thrown (a throw would reset the stream).
+					await this.reconcileDivergentCommit(record, commit);
+					return;
+				}
+				throw err;
+			}
+			if (!result.success) {
+				// success:false is a StaleFailure. `missing` ⇒ ahead/stale divergence
+				// (we already hold ≥ this rev): tolerate, do NOT reconcile downward. A bare
+				// `reason` with no `missing` ⇒ a genuine internalCommit fault: propagate so
+				// handleConsensus rolls back the executed marker and rethrows.
+				if (result.missing?.length) {
+					log('cluster-member:consensus-commit-diverged', {
+						messageHash,
+						actionId: commit.actionId,
+						divergence: 'ahead',
+						reason: result.reason,
+						hasMissing: true
+					});
+					return;
+				}
+				throw new Error(`Consensus commit for action ${commit.actionId} failed: ${result.reason ?? 'unknown reason'}`);
+			}
+			return;
+		}
+		if ('invalidate' in operation) {
+			await this.applyConsensusInvalidation(record, operation.invalidate);
+			return;
+		}
+	}
+
+	/**
+	 * Applies a consensus-ordered invalidation on this member: dedup → certificate verification →
+	 * capture the invalidation's commit cert (for reactivity reuse) → delegate the compensating write +
+	 * durable log append to the injected {@link InvalidationApplySink}.
+	 *
+	 * Like every other branch of {@link applyConsensusOperation}, a failure here is tolerated rather
+	 * than thrown — a throw would reset the cluster stream. A forged/sub-threshold certificate is
+	 * rejected (and never reaches the sink); a sink fault is logged and its dedup marker rolled back
+	 * so a re-broadcast can retry. The durable, authoritative dedup is the invalidation log entry the
+	 * sink consults; the in-memory map is only a fast path.
+	 */
+	private async applyConsensusInvalidation(record: ClusterRecord, request: InvalidateRequest): Promise<void> {
+		const messageHash = record.messageHash;
+		const dedupKey = `${request.invalidatedActionId}:${request.resolution.disputeId}`;
+		if (this.appliedInvalidations.has(dedupKey)) {
+			log('cluster-member:consensus-invalidate-duplicate', { messageHash, dedupKey });
+			return;
+		}
+
+		// Certificate verification BEFORE apply — never trust the originator's say-so. Verify the proof
+		// against THIS request's own target: the votes are bound to the transaction the dispute resolved,
+		// so a genuine proof carried in a request that points at a different (innocent) action/blocks fails
+		// here (the network-facing replay boundary, #2). The proof is also bound to the legitimately-selected
+		// arbitrator set (#1): membership + the challenger's set signature gate it (layer 1), and when a
+		// recompute capability is wired we re-derive the eligible set from our topology and reject a forged
+		// one (layer 2). When we cannot reconstruct the historical topology, we accept on layer 1 and LOG
+		// that the invalidation was applied without full anchoring — the documented interim posture.
+		const certified = await verifyInvalidationCertificate(
+			request.resolution,
+			{ invalidatedActionId: request.invalidatedActionId, blockIds: request.blockIds },
+			{
+				recomputeArbitratorSet: this.recomputeArbitratorSet,
+				onUnanchored: (info) => log('cluster-member:consensus-invalidate-unanchored', {
+					messageHash,
+					invalidatedActionId: request.invalidatedActionId,
+					disputeId: info.disputeId,
+					reason: info.reason,
+					arbitratorSetSize: info.arbitratorSet.length
+				})
+			}
+		);
+		if (!certified) {
+			log('cluster-member:consensus-invalidate-reject-certificate', {
+				messageHash,
+				invalidatedActionId: request.invalidatedActionId,
+				disputeId: request.resolution.disputeId,
+				outcome: request.resolution.outcome
+			});
+			return;
+		}
+
+		// Capture the invalidation's own commit cert — the threshold signature the cohort produced for
+		// THIS consensus-ordered `invalidate` op (over `computeCommitHash(record)`), the reversal analogue
+		// of the commit-cert capture in {@link applyConsensusOperation}. Reactivity reuses it bit-for-bit
+		// as the invalidation notification's `sig` (never re-signed). Keyed on the deterministic
+		// {@link invalidationActionId} the invalidation's change event also carries, so the bridge's
+		// cert extractor resolves it. Gated on the sink — a node with no reactivity wired pays nothing.
+		if (this.onCommitCertificate) {
+			const invSignedPayload = this.computeSigningPayload(await this.computeCommitHash(record), 'approve');
+			this.captureCommitCert(record, invalidationActionId(request.invalidatedActionId, request.resolution.disputeId), invSignedPayload);
+		}
+
+		if (!this.onInvalidate) {
+			log('cluster-member:consensus-invalidate-no-sink', { messageHash, dedupKey });
+			return;
+		}
+
+		this.appliedInvalidations.set(dedupKey, Date.now());
+		try {
+			await this.onInvalidate(request);
+			log('cluster-member:consensus-invalidate-applied', {
+				messageHash,
+				invalidatedActionId: request.invalidatedActionId,
+				disputeId: request.resolution.disputeId,
+				blockCount: request.blockIds.length
+			});
+		} catch (err) {
+			// Tolerate (don't reset the stream); roll back the marker so a re-broadcast retries.
+			this.appliedInvalidations.delete(dedupKey);
+			log('cluster-member:consensus-invalidate-sink-error', { messageHash, dedupKey, error: (err as Error).message });
+		}
+	}
+
+	/**
+	 * Record the consensus commit cert for `actionId` into the injected {@link CommitCertificateSink},
+	 * if one is wired. The cert is built from the agreed `record.commits` (the per-member `approve`
+	 * commit signatures), forwarded UNCHANGED for reactivity to reuse. `signedPayload` is the exact
+	 * commit-vote preimage those signatures were produced over (`utf8(commitHash + ":approve")`),
+	 * computed by the caller from the same `record` *before* the synchronous commit — reactivity sets a
+	 * notification's `digest` from it so a real threshold-verify over `digest` succeeds. Stays
+	 * **synchronous** (no `await` here) so the cert is retained before `StorageRepo.commit` emits its
+	 * change event. No-op (zero cost) when no sink is configured; a throwing sink is isolated + logged so
+	 * it can never break consensus.
+	 */
+	private captureCommitCert(record: ClusterRecord, actionId: ActionId, signedPayload: Uint8Array): void {
+		if (!this.onCommitCertificate) {
+			return;
+		}
+		const minSigs = Math.ceil(Object.keys(record.peers).length * this.superMajorityThreshold);
+		try {
+			this.onCommitCertificate(actionId, buildCommitCert(record, minSigs, signedPayload));
+		} catch (err) {
+			log('cluster-member:commit-cert-sink-error', { actionId, error: (err as Error).message });
+		}
+	}
+
+	/**
+	 * After tolerating a "behind" commit divergence, pull the committed revision of
+	 * each block from a cohort peer that holds it and restore it locally. Best-effort:
+	 * a missing callback, an empty cohort, or a per-block failure/timeout is logged and
+	 * tolerated — never thrown, since a throw out of consensus execution resets the
+	 * cluster stream.
+	 */
+	private async reconcileDivergentCommit(record: ClusterRecord, commit: CommitRequest): Promise<void> {
+		if (!this.reconcileBlock) {
+			log('cluster-member:consensus-commit-reconcile-skip', { messageHash: record.messageHash, reason: 'no-callback' });
+			return;
+		}
+		const cohortPeerIds = Object.keys(record.peers).filter(id => id !== this.peerId.toString());
+		if (cohortPeerIds.length === 0) {
+			log('cluster-member:consensus-commit-reconcile-skip', { messageHash: record.messageHash, reason: 'no-cohort-peers' });
+			return;
+		}
+		const committed: ActionRev = { actionId: commit.actionId, rev: commit.rev };
+		await Promise.all(
+			commit.blockIds.map(blockId => this.reconcileOneBlock(record.messageHash, blockId, committed, cohortPeerIds))
+		);
+	}
+
+	/** Reconcile a single block, bounding the awaited callback and swallowing failures. */
+	private async reconcileOneBlock(messageHash: string, blockId: BlockId, committed: ActionRev, cohortPeerIds: string[]): Promise<void> {
+		try {
+			await this.withReconcileTimeout(this.reconcileBlock!(blockId, committed, cohortPeerIds), blockId);
+			log('cluster-member:consensus-commit-reconciled', { messageHash, blockId, rev: committed.rev });
+		} catch (err) {
+			log('cluster-member:consensus-commit-reconcile-failed', {
+				messageHash,
+				blockId,
+				rev: committed.rev,
+				error: (err as Error).message
+			});
+		}
+	}
+
+	/** Bound an awaited reconcile so a slow/unreachable cohort peer can't stall consensus. */
+	private withReconcileTimeout<T>(promise: Promise<T>, blockId: BlockId): Promise<T> {
+		let timer: NodeJS.Timeout | undefined;
+		const timeout = new Promise<never>((_, reject) => {
+			timer = setTimeout(
+				() => reject(new Error(`reconcile for block ${blockId} timed out after ${ReconcileTimeoutMs}ms`)),
+				ReconcileTimeoutMs
+			);
+			timer.unref();
+		});
+		return Promise.race([promise, timeout]).finally(() => {
+			if (timer) clearTimeout(timer);
+		});
+	}
+
 	private async handleRejection(_record: ClusterRecord): Promise<void> {
 		// Clean up any resources - will be cleared by shouldPersist = false in the main flow
 	}
@@ -867,6 +1207,10 @@ export class ClusterMember implements ICluster {
 				operation.commit.blockIds.forEach(id => blockIds.add(id));
 			} else if ('cancel' in operation) {
 				operation.cancel.actionRef.blockIds.forEach(id => blockIds.add(id));
+			} else if ('invalidate' in operation) {
+				// The invalidation writes compensating revisions to these blocks; surfacing them lets
+				// conflict detection serialize a concurrent commit racing the invalidation on a block.
+				operation.invalidate.blockIds.forEach(id => blockIds.add(id));
 			}
 		}
 
@@ -939,6 +1283,12 @@ export class ClusterMember implements ICluster {
 				this.executedTransactions.delete(messageHash);
 			}
 		}
+		// Prune old applied-invalidation dedup markers on the same TTL.
+		for (const [dedupKey, appliedAt] of Array.from(this.appliedInvalidations.entries())) {
+			if (appliedAt < expirationThreshold) {
+				this.appliedInvalidations.delete(dedupKey);
+			}
+		}
 		this.stateStore?.pruneExecuted(expirationThreshold)
 			.catch(err => log('cluster-member:prune-executed-error', { error: (err as Error).message }));
 	}