@optimystic/db-core 0.14.0 → 0.16.2

package/src/transaction/coordinator.ts

@@ -1,22 +1,38 @@
-import type { ITransactor, BlockId, CollectionId, Transforms, PendRequest, CommitRequest, ActionId, IBlock, BlockOperations } from "../index.js";
+import type { ITransactor, BlockId, CollectionId, Transforms, PendRequest, CommitRequest, ActionId } from "../index.js";
 import type { Transaction, ExecutionResult, ITransactionEngine, CollectionActions, ReadDependency } from "./transaction.js";
 import type { PeerId } from "../network/types.js";
 import type { Collection } from "../collection/collection.js";
-import { TransactionContext } from "./context.js";
-import { ActionsEngine } from "./actions-engine.js";
-import { createActionsStatements, createTransactionStamp, createTransactionId, isTransactionExpired } from "./transaction.js";
-import { Log, blockIdsForTransforms, hashString } from "../index.js";
+import type { SyncOptions } from "../collection/index.js";
+import { isTransactionExpired, clampPriority } from "./transaction.js";
+import { Log, blockIdsForTransforms } from "../index.js";
+import { collectOperations, hashOperations } from "./operations-hash.js";
+import { CoordinatorPartialCommitError, CoordinatorStaleLossError } from "./errors.js";
+import { jitteredBackoffMs, abortableDelay, makeAbortError } from "../utility/backoff.js";
 import { createLogger } from "../logger.js";
 
 const log = createLogger('trx:coordinator');
 
+/** Default max consecutive clean-stale-loss retries before {@link TransactionCoordinator.commit}
+ * gives up. Mirrors the single-collection sync default so the two retry loops share one policy. */
+const DefaultMaxAttempts = 10;
+/** Default base backoff (ms) before the first commit retry. */
+const DefaultBaseBackoffMs = 100;
+/** Default ceiling (ms) on a single commit-retry backoff sleep. */
+const DefaultMaxBackoffMs = 5000;
+
 /**
- * Represents an operation on a block within a collection.
+ * A pend that failed. `conflict` marks the retryable class — an optimistic-concurrency collision (a
+ * committed `missing` action newer than our rev, or a `pending` action on a touched block) that a
+ * re-read + re-pend can clear. A bare rejection reason (storage full, policy) is NOT a conflict and
+ * is not worth re-driving. Thrown by {@link TransactionCoordinator.pendCollection} so the fan-out in
+ * pendPhase can settle it and read the flag off the rejection.
  */
-type Operation =
-	| { readonly type: 'insert'; readonly collectionId: CollectionId; readonly blockId: BlockId; readonly block: IBlock }
-	| { readonly type: 'update'; readonly collectionId: CollectionId; readonly blockId: BlockId; readonly operations: BlockOperations }
-	| { readonly type: 'delete'; readonly collectionId: CollectionId; readonly blockId: BlockId };
+class PendRejectedError extends Error {
+	constructor(collectionId: CollectionId, readonly conflict: boolean, reason?: string) {
+		super(`Pend failed for collection ${collectionId}: ${reason ?? (conflict ? 'stale conflict' : 'rejected')}`);
+		this.name = 'PendRejectedError';
+	}
+}
 
 /**
  * Coordinates multi-collection transactions.
@@ -94,19 +110,105 @@ export class TransactionCoordinator {
 	}
 
 	/**
-	 * Commit a transaction: materialise a log entry from each collection's staged
+	 * Commit a transaction with a bounded, jittered backoff retry around a CLEAN stale loss.
+	 *
+	 * The single-attempt work lives in {@link commitOnce}; this wrapper re-drives it when the attempt
+	 * fails as a clean optimistic-concurrency loss ({@link CoordinatorStaleLossError} — nothing
+	 * durably committed, every tracker restored to its pre-append state). Before each re-attempt it
+	 * re-reads each collection to fresh revisions (so the retry pends against current state rather
+	 * than immediately re-failing stale), then backs off with the same jitter policy as
+	 * {@link Collection.sync}. Retry is bounded by `maxAttempts` and an optional wall-clock
+	 * `deadlineMs`, and honours an abort `signal`.
+	 *
+	 * A {@link CoordinatorPartialCommitError} (a partial landing — some collection durably committed)
+	 * is NOT retryable and escapes immediately: blindly retrying would re-log already-durable actions.
+	 * Any other failure (expired transaction, unavailable transactor, unreachable cluster) also
+	 * propagates without retry — only genuine clean stale losses are re-driven.
+	 *
+	 * Defaults are safe out of the box: a caller that passes no options gets bounded, jittered retry.
+	 *
+	 * @param transaction - The transaction to commit
+	 * @param options - Retry knobs; shares the {@link SyncOptions} vocabulary with `Collection.sync`.
+	 */
+	async commit(transaction: Transaction, options?: SyncOptions): Promise<void> {
+		const maxAttempts = options?.maxAttempts ?? DefaultMaxAttempts;
+		const baseBackoffMs = options?.baseBackoffMs ?? DefaultBaseBackoffMs;
+		const maxBackoffMs = options?.maxBackoffMs ?? DefaultMaxBackoffMs;
+		const deadlineMs = options?.deadlineMs;
+		const signal = options?.signal;
+		const startedAt = Date.now();
+
+		// Count of consecutive clean stale losses. There is no forward-progress notion here (a
+		// single commit either lands or it does not), so this simply bounds how many times we
+		// re-drive a losing transaction before surfacing a terminal error.
+		let staleLosses = 0;
+		let lastLoss: CoordinatorStaleLossError | undefined;
+		for (;;) {
+			if (signal?.aborted) {
+				throw makeAbortError(signal);
+			}
+			// Progress-agnostic ceiling: once we've taken at least one loss, give up if the
+			// wall-clock deadline passed (independent of the attempt cap).
+			if (deadlineMs !== undefined && lastLoss && Date.now() - startedAt >= deadlineMs) {
+				throw lastLoss;
+			}
+
+			// Age the transaction's advisory priority by the number of losses taken so far, so a
+			// repeatedly-losing transaction out-ranks fresh (priority-0) rivals in the cluster's
+			// resolveRace. Fairness-only and capped at MaxPriority; excluded from the tx id / client
+			// signature, so bumping it here does not churn identity. Left untouched on the first
+			// attempt (staleLosses == 0) so the initial pend serializes exactly as before.
+			if (staleLosses > 0) {
+				transaction.priority = clampPriority(staleLosses);
+			}
+
+			try {
+				await this.commitOnce(transaction);
+				return;
+			} catch (err) {
+				// Only a CLEAN stale loss is retryable. A partial landing, an expired transaction, an
+				// unavailable transactor, etc. all propagate unchanged.
+				if (!(err instanceof CoordinatorStaleLossError)) {
+					throw err;
+				}
+				lastLoss = err;
+				staleLosses++;
+				if (staleLosses >= maxAttempts) {
+					throw err;
+				}
+				const delay = jitteredBackoffMs(staleLosses - 1, { baseMs: baseBackoffMs, capMs: maxBackoffMs }, options?.rand);
+				await abortableDelay(delay, signal);
+				// Re-read fresh state before re-attempting so the next commit pends against current
+				// revisions (mirrors how Collection.sync calls updateInternal() before retrying).
+				// NOTE: refreshes EVERY registered collection, not only the participants of this
+				// transaction. Harmless (a non-participant's update() just fetches latest) and the
+				// registered set is small today; if a coordinator ever holds many collections and this
+				// shows up as retry latency, narrow it to the transaction's participating collections.
+				for (const collection of this.collections.values()) {
+					await collection.update();
+				}
+			}
+		}
+	}
+
+	/**
+	 * Commit a transaction (single attempt): materialise a log entry from each collection's staged
 	 * pending actions, then orchestrate the distributed consensus (GATHER/PEND/COMMIT).
 	 *
-	 * Called by TransactionSession.commit() after all statements have executed. The
+	 * Called by {@link commit} (which wraps it in the backoff+jitter retry loop). The
 	 * staged mutations already live in each collection's tracker — applied either via
 	 * applyActions() (engine-driven path) or directly via Collection.act()/Tree.stage
 	 * (the vtab's deferred-DML path) — but in BOTH cases without a log entry yet, so
 	 * this method appends that entry here (see the inline note below) before pending,
 	 * and folds the committed transforms back into each collection's read cache.
 	 *
+	 * On a clean stale loss (nothing durable, every tracker restored) it throws
+	 * {@link CoordinatorStaleLossError} so the caller can retry; on a partial landing it throws
+	 * {@link CoordinatorPartialCommitError} (not retryable).
+	 *
 	 * @param transaction - The transaction to commit
 	 */
-	async commit(transaction: Transaction): Promise<void> {
+	private async commitOnce(transaction: Transaction): Promise<void> {
 		if (isTransactionExpired(transaction.stamp)) {
 			throw new Error(`Transaction expired at ${transaction.stamp.expiration}`);
 		}
@@ -144,44 +246,117 @@ export class TransactionCoordinator {
 		const collectionTransforms = new Map<CollectionId, Transforms>();
 		const criticalBlocks = new Map<CollectionId, BlockId>();
 
+		// Snapshot EVERY participating collection's staged state (transforms + pending
+		// queue) BEFORE the append loop mutates any tracker. The loop appends log
+		// entries sequentially, so a failure on the Nth collection must also undo the
+		// 0..N-1 collections that already appended — and coordinateTransaction can fail
+		// after ALL of them appended. On any throw below we restore every snapshot, so a
+		// failed commit leaves each tracker exactly as it was: a retry re-appends cleanly
+		// (no duplicate log entry) and a directly-staged tree's rollback (which no-ops
+		// when the stamp was never tracked via applyActions) has nothing poisoned to undo.
+		const preCommitSnapshots = new Map<CollectionId, ReturnType<Collection<any>['snapshotPending']>>();
 		for (const { collectionId, collection } of collectionData) {
-			const applyResult = await this.applyActionsToCollection(
-				{ collectionId, actions: collection.getPendingActions() },
-				transaction,
-				allCollectionIds
-			);
-			if (!applyResult.success) {
-				throw new Error(`Transaction commit failed: ${applyResult.error}`);
+			preCommitSnapshots.set(collectionId, collection.snapshotPending());
+		}
+
+		let coordResult: {
+			success: boolean;
+			error?: string;
+			committedCollections?: Set<CollectionId>;
+			failedCollections?: Set<CollectionId>;
+			staleLoss?: boolean;
+		};
+		try {
+			for (const { collectionId, collection } of collectionData) {
+				const applyResult = await this.applyActionsToCollection(
+					{ collectionId, actions: collection.getPendingActions() },
+					transaction,
+					allCollectionIds
+				);
+				if (!applyResult.success) {
+					throw new Error(`Transaction commit failed: ${applyResult.error}`);
+				}
+				collectionTransforms.set(collectionId, applyResult.transforms!);
+				criticalBlocks.set(collectionId, applyResult.logTailBlockId!);
 			}
-			collectionTransforms.set(collectionId, applyResult.transforms!);
-			criticalBlocks.set(collectionId, applyResult.logTailBlockId!);
-		}
-
-		// Compute hash of ALL operations across ALL collections (post-log-append).
-		// Validators re-execute the transaction and compare their computed hash.
-		const allOperations = Array.from(collectionTransforms.entries()).flatMap(([collectionId, transforms]) => [
-			...Object.entries(transforms.inserts ?? {}).map(([blockId, block]) =>
-				({ type: 'insert' as const, collectionId, blockId, block })
-			),
-			...Object.entries(transforms.updates ?? {}).map(([blockId, operations]) =>
-				({ type: 'update' as const, collectionId, blockId, operations })
-			),
-			...(transforms.deletes ?? []).map(blockId =>
-				({ type: 'delete' as const, collectionId, blockId })
-			)
-		]);
 
-		const operationsHash = await this.hashOperations(allOperations);
+			// Compute hash of ALL operations across ALL collections (post-log-append).
+			// Validators re-execute the transaction and compare their computed hash.
+			// The shared operations-hash module canonicalises (sort + canonical JSON) so
+			// this order-independent fingerprint matches what a validator recomputes.
+			const operationsHash = await hashOperations(collectOperations(collectionTransforms));
 
-		// Execute consensus phases (GATHER, PEND, COMMIT)
-		const coordResult = await this.coordinateTransaction(
-			transaction,
-			operationsHash,
-			collectionTransforms,
-			criticalBlocks
-		);
+			// Execute consensus phases (GATHER, PEND, COMMIT)
+			coordResult = await this.coordinateTransaction(
+				transaction,
+				operationsHash,
+				collectionTransforms,
+				criticalBlocks
+			);
+		} catch (err) {
+			// A throw here means the failure happened BEFORE any collection could
+			// durably commit (a log-append failure, or coordinateTransaction rejecting
+			// unexpectedly). Nothing landed on the cluster, so roll every tracker back
+			// to its pre-append snapshot — a genuinely clean rollback that leaves each
+			// tracker pristine for retry (see txn-failed-commit-leaves-staged-log-entry).
+			for (const { collectionId, collection } of collectionData) {
+				collection.restorePending(preCommitSnapshots.get(collectionId)!);
+			}
+			throw err;
+		}
 
 		if (!coordResult.success) {
+			const committed = coordResult.committedCollections ?? new Set<CollectionId>();
+			if (committed.size > 0) {
+				// PARTIAL COMMIT: at least one collection durably committed via consensus
+				// while another failed permanently. A uniform pre-append restore would
+				// corrupt the committed half — re-staging its already-durable actions as
+				// still-pending, so tracker memory would disagree with cluster storage.
+				// Split the local handling instead:
+				for (const { collectionId, collection } of collectionData) {
+					if (committed.has(collectionId)) {
+						// Committed → the success-path local treatment (see below): fold the
+						// committed transforms into the read cache BEFORE resetting the tracker,
+						// then drop the now-durable pending actions so a retry cannot re-log them.
+						// NOTE: no-double-apply on retry depends on clearPendingActions() running for
+						// EVERY committed collection here before any re-drive of commit(). If a committed
+						// collection kept its pending queue, a subsequent commit() would re-append and
+						// re-log its already-durable actions — a duplicate log entry on the winner. The
+						// no-double-apply-on-retry test in transaction.spec.ts locks this.
+						const rev = collection.recordCommitted(transaction.id);
+						collection.applyCommittedToCache(collectionTransforms.get(collectionId)!, rev);
+						collection.tracker.reset();
+						collection.clearPendingActions();
+					} else {
+						// Failed / never-committed → restore the pre-append snapshot so a retry
+						// re-appends cleanly (no duplicate log entry).
+						collection.restorePending(preCommitSnapshots.get(collectionId)!);
+					}
+				}
+				// The transaction half-landed, so it is neither cleanly retryable nor
+				// cleanly abortable: drop its stamp tracking (the success path does the
+				// same at the end) and surface the structured signal for reconciliation.
+				this.stampData.delete(transaction.stamp.id);
+				throw new CoordinatorPartialCommitError(
+					[...committed],
+					[...(coordResult.failedCollections ?? new Set<CollectionId>())],
+					coordResult.error
+				);
+			}
+
+			// EMPTY committed set: PEND failed, or the whole commit failed cleanly with
+			// nothing durable. Restore every tracker so each is pristine for retry.
+			for (const { collectionId, collection } of collectionData) {
+				collection.restorePending(preCommitSnapshots.get(collectionId)!);
+			}
+			// Distinguish a genuine optimistic-concurrency conflict (a stale loss / pending
+			// contention — retryable after a re-read) from a hard failure (unavailable transactor,
+			// storage rejection, expired). Only the former is worth re-driving; the retry wrapper in
+			// commit() catches CoordinatorStaleLossError and re-attempts, while a plain Error escapes
+			// immediately (preserving the historical fail-fast behaviour for hard failures).
+			if (coordResult.staleLoss) {
+				throw new CoordinatorStaleLossError([...(coordResult.failedCollections ?? new Set(allCollectionIds))], coordResult.error);
+			}
 			throw new Error(`Transaction commit failed: ${coordResult.error}`);
 		}
 
@@ -193,15 +368,8 @@ export class TransactionCoordinator {
 		// serves the new revision instead of the stale cached one. Clearing
 		// pending keeps a subsequent commit from re-logging these actions.
 		for (const { collectionId, collection } of collectionData) {
-			const newRev = (collection['source'].actionContext?.rev ?? 0) + 1;
-			collection['source'].actionContext = {
-				committed: [
-					...(collection['source'].actionContext?.committed ?? []),
-					{ actionId: transaction.id, rev: newRev }
-				],
-				rev: newRev,
-			};
-			collection.applyCommittedToCache(collectionTransforms.get(collectionId)!);
+			const rev = collection.recordCommitted(transaction.id);
+			collection.applyCommittedToCache(collectionTransforms.get(collectionId)!, rev);
 			collection.tracker.reset();
 			collection.clearPendingActions();
 		}
@@ -318,64 +486,10 @@ export class TransactionCoordinator {
 		}
 	}
 
-	/**
-	 * Compute hash of all operations in a transaction.
-	 * This hash is used for validation - validators re-execute the transaction
-	 * and compare their computed operations hash with this one.
-	 */
-	private async hashOperations(operations: readonly Operation[]): Promise<string> {
-		const operationsData = JSON.stringify(operations);
-		return `ops:${await hashString(operationsData)}`;
-	}
-
-	/**
-	 * Commit a transaction context.
-	 *
-	 * @deprecated Use TransactionSession instead of TransactionContext
-	 * This is called by TransactionContext.commit().
-	 *
-	 * @param context - The transaction context to commit
-	 * @returns Execution result with actions and results
-	 */
-	async commitTransaction(context: TransactionContext): Promise<ExecutionResult> {
-		const collectionActions = Array.from(context.getCollectionActions().entries()).map(
-			([collectionId, actions]) => ({ collectionId, actions })
-		);
-
-		if (collectionActions.length === 0) {
-			return { success: true }; // Nothing to commit
-		}
-
-		// Create transaction statements
-		const statements = createActionsStatements(collectionActions);
-		const reads = context.getReads();
-
-		// Create stamp from context
-		const stamp = await createTransactionStamp(
-			'local', // TODO: Get from context or coordinator
-			Date.now(),
-			'', // TODO: Get from engine
-			context.engine
-		);
-
-		const transaction: Transaction = {
-			stamp,
-			statements,
-			reads,
-			id: await createTransactionId(stamp.id, statements, reads)
-		};
-
-		const engine = new ActionsEngine(this);
-
-		// Execute through standard path
-		return await this.execute(transaction, engine);
-	}
-
 	/**
 	 * Execute a fully-formed transaction.
 	 *
-	 * This can be called directly with a complete transaction (e.g., from Quereus),
-	 * or indirectly via commitTransaction().
+	 * This is called with a complete transaction (e.g., from Quereus).
 	 *
 	 * @param transaction - The transaction to execute
 	 * @param engine - The engine to use for executing the transaction
@@ -405,7 +519,41 @@ export class TransactionCoordinator {
 			return { success: true }; // Nothing to do
 		}
 
-		// 2. Apply actions to collections and collect transforms
+		// 1b. Stage the returned actions into the collection trackers.
+		//
+		// Reaching here means the engine RETURNED non-empty actions — i.e. the pure-
+		// translator model (see the ITransactionEngine contract): it translated the
+		// statements but did NOT apply them. So THIS path owns application — we stage the
+		// actions here via applyActions() (which also snapshots/tracks the stamp for
+		// rollback) BEFORE the loop below reads each tracker's transforms to materialise
+		// the log entry. (Previously ActionsEngine applied as a side effect and this
+		// method merely re-read the already-staged trackers; that side effect is gone, so
+		// the application must happen explicitly here. A side-effecting engine that
+		// applied internally would instead return EMPTY actions and short-circuit at the
+		// guard above.)
+		//
+		// applyActions() throws if a referenced collection is not registered — the same
+		// "Collection not found" the engine's side-effecting apply used to surface. Convert
+		// it back into a failure result so execute() keeps its return contract.
+		try {
+			await this.applyActions(result.actions, transaction.stamp.id);
+		} catch (error) {
+			const engineMs = Date.now() - tEngine;
+			log('execute:done trxId=%s engine=%dms apply-failed=true total=%dms', trxId, engineMs, Date.now() - t0);
+			return { success: false, error: error instanceof Error ? error.message : String(error) };
+		}
+
+		// 2. Build a log entry per collection from the now-staged tracker transforms.
+		//
+		// NOTE: like commit(), this loop appends a log entry into each collection's
+		// tracker and these failure returns do NOT restore that state — so a partially
+		// applied engine transaction leaves appended-but-uncommitted entries in the
+		// trackers. This is deliberately NOT snapshot/restore-wrapped the way commit()
+		// is, because execute()'s asymmetry makes it lower risk: it is not the retryable
+		// session.commit() entry point (a failed execute() is not re-driven through the
+		// same loop), and its actions were tracked via applyActions() so rollback(stampId)
+		// CAN unwind them (unlike commit()'s directly-staged path). If execute() ever
+		// becomes retryable, mirror the commit() snapshot/restore fix here.
 		const tApply = Date.now();
 		const collectionTransforms = new Map<CollectionId, Transforms>();
 		const criticalBlocks = new Map<CollectionId, BlockId>();
@@ -428,19 +576,8 @@ export class TransactionCoordinator {
 			actionResults.set(collectionActions.collectionId, applyResult.results!);
 		}
 
-		// 3. Compute operations hash for validation
-		const allOperations = Array.from(collectionTransforms.entries()).flatMap(([collectionId, transforms]) => [
-			...Object.entries(transforms.inserts ?? {}).map(([blockId, block]) =>
-				({ type: 'insert' as const, collectionId, blockId, block })
-			),
-			...Object.entries(transforms.updates ?? {}).map(([blockId, operations]) =>
-				({ type: 'update' as const, collectionId, blockId, operations })
-			),
-			...(transforms.deletes ?? []).map(blockId =>
-				({ type: 'delete' as const, collectionId, blockId })
-			)
-		]);
-		const operationsHash = await this.hashOperations(allOperations);
+		// 3. Compute operations hash for validation (order-independent; see commit()).
+		const operationsHash = await hashOperations(collectOperations(collectionTransforms));
 
 		const applyMs = Date.now() - tApply;
 
@@ -456,22 +593,34 @@ export class TransactionCoordinator {
 		const coordMs = Date.now() - tCoord;
 		if (!coordResult.success) {
 			log('execute:done trxId=%s engine=%dms apply=%dms coordinate=%dms success=false total=%dms', trxId, engineMs, applyMs, coordMs, Date.now() - t0);
-			return coordResult;
+			// Stop lying to the caller about a partial commit: if some collections durably
+			// committed, surface that set. execute() is not snapshot/restore-wrapped (see the
+			// note above), but the committed subset must still get the success-path local
+			// treatment (recordCommitted + tracker.reset, as on the success path below) so its
+			// trackers aren't left mis-tracking already-durable state.
+			const committed = coordResult.committedCollections ?? new Set<CollectionId>();
+			if (committed.size > 0) {
+				for (const collectionActions of result.actions) {
+					const collection = this.collections.get(collectionActions.collectionId);
+					if (collection && committed.has(collectionActions.collectionId)) {
+						collection.recordCommitted(transaction.id);
+						collection.tracker.reset();
+					}
+				}
+			}
+			return {
+				success: false,
+				error: coordResult.error,
+				committedCollections: committed.size > 0 ? [...committed] : undefined,
+				failedCollections: coordResult.failedCollections ? [...coordResult.failedCollections] : undefined,
+			};
 		}
 
 		// 5. Update actionContext and reset trackers after successful commit
 		for (const collectionActions of result.actions) {
 			const collection = this.collections.get(collectionActions.collectionId);
 			if (collection) {
-				const newRev = (collection['source'].actionContext?.rev ?? 0) + 1;
-				const actionId = transaction.id;
-				collection['source'].actionContext = {
-					committed: [
-						...(collection['source'].actionContext?.committed ?? []),
-						{ actionId, rev: newRev }
-					],
-					rev: newRev,
-				};
+				collection.recordCommitted(transaction.id);
 				collection.tracker.reset();
 			}
 		}
@@ -513,7 +662,7 @@ export class TransactionCoordinator {
 		}
 
 		// At this point, actions have already been executed through collection.act()
-		// when they were added to the TransactionContext. The collection's tracker
+		// (via the engine or the vtab's staging path). The collection's tracker
 		// already has the transforms, and the actions are in the pending buffer.
 
 		// Get transforms from the collection's tracker
@@ -530,7 +679,7 @@ export class TransactionCoordinator {
 
 		// Generate action ID from transaction ID
 		const actionId = transaction.id;
-		const newRev = (collection['source'].actionContext?.rev ?? 0) + 1;
+		const newRev = collection.getNextRev();
 
 		// Add actions to log (this updates the tracker with log block changes).
 		// Persist the transaction's read set on the entry so a later invalidation cascade can
@@ -569,7 +718,15 @@ export class TransactionCoordinator {
 		operationsHash: string,
 		collectionTransforms: Map<CollectionId, Transforms>,
 		criticalBlocks: Map<CollectionId, BlockId>
-	): Promise<{ success: boolean; error?: string }> {
+	): Promise<{
+		success: boolean;
+		error?: string;
+		committedCollections?: Set<CollectionId>;
+		failedCollections?: Set<CollectionId>;
+		/** True when the failure was a clean optimistic-concurrency conflict (stale loss / pending
+		 * contention) with nothing durable — i.e. safe to re-drive after a re-read. */
+		staleLoss?: boolean;
+	}> {
 		const trxId = transaction.id;
 		const t0 = Date.now();
 
@@ -609,7 +766,16 @@ export class TransactionCoordinator {
 				commitResult.committedCollections
 			);
 			log('trx:phases trxId=%s gather=%dms pend=%dms commit=%dms (failed) total=%dms', trxId, gatherMs, pendMs, commitMs, Date.now() - t0);
-			return { success: false, error: commitResult.error };
+			// Surface the committed/failed partition so commit()/execute() can report which
+			// collections durably landed. A non-empty committedCollections is a PARTIAL commit:
+			// those collections cannot be rolled back and the caller must reconcile.
+			return {
+				success: false,
+				error: commitResult.error,
+				committedCollections: commitResult.committedCollections,
+				failedCollections: commitResult.failedCollections,
+				staleLoss: commitResult.staleLoss,
+			};
 		}
 
 		// 4. PROPAGATE and CHECKPOINT phases are handled by clusters automatically
@@ -647,16 +813,19 @@ export class TransactionCoordinator {
 		);
 		const results = await Promise.all(nomineePromises);
 
-		// Merge all nominees into a single set
-		const supercluster = results.reduce(
+		// Merge all nominees into a single set, deduped by peer identity. Each
+		// queryClusterNominees builds a fresh PeerId object per call (peerIdFromString),
+		// so a Set keyed by object reference would keep the same physical peer twice when
+		// it nominates for two critical clusters. Key by toString() to collapse duplicates.
+		const byId = results.reduce(
 			(acc, result) => {
-				result.nominees.forEach(nominee => acc.add(nominee));
+				result.nominees.forEach(nominee => acc.set(nominee.toString(), nominee));
 				return acc;
 			},
-			new Set<PeerId>()
+			new Map<string, PeerId>()
 		);
 
-		return supercluster;
+		return new Set(byId.values());
 	}
 
 	/**
@@ -672,56 +841,104 @@ export class TransactionCoordinator {
 		operationsHash: string,
 		collectionTransforms: ReadonlyMap<CollectionId, Transforms>,
 		superclusterNominees: ReadonlySet<PeerId> | null
-	): Promise<{ success: boolean; error?: string; pendedBlockIds?: Map<CollectionId, BlockId[]> }> {
+	): Promise<{ success: boolean; error?: string; pendedBlockIds?: Map<CollectionId, BlockId[]>; staleLoss?: boolean }> {
 		if (collectionTransforms.size === 0) {
 			return { success: false, error: 'No transforms to pend' };
 		}
 
-		const pendedBlockIds = new Map<CollectionId, BlockId[]>();
 		const actionId = transaction.id as ActionId;
 		const nominees = superclusterNominees ? Array.from(superclusterNominees) : undefined;
 
-		// Pend each collection's transforms
-		for (const [collectionId, transforms] of collectionTransforms.entries()) {
-			const collection = this.collections.get(collectionId);
-			if (!collection) {
-				return { success: false, error: `Collection not found: ${collectionId}` };
-			}
-
-			// Get revision from the collection's source
-			const rev = (collection['source'].actionContext?.rev ?? 0) + 1;
-
-			// Create pend request with transaction and operations hash for validation
-			const pendRequest: PendRequest = {
-				actionId,
-				rev,
-				transforms,
-				policy: 'r', // Return policy: fail but return pending actions
-				transaction,
-				operationsHash,
-				superclusterNominees: nominees
-			};
+		// Fan out the independent per-collection pends concurrently. Each settles to a
+		// { collectionId, blockIds } on success, or rejects with the per-collection reason.
+		// NOTE: unbounded fan-out — one concurrent coordinator round-trip per collection.
+		// Transactions touch few collections today; if one ever spans very many, bound this
+		// with a concurrency limiter so peak in-flight round-trips stays sane. Same for commitPhase.
+		const outcomes = await Promise.allSettled(
+			Array.from(collectionTransforms.entries()).map(([collectionId, transforms]) =>
+				this.pendCollection(transaction, operationsHash, collectionId, transforms, actionId, nominees)
+			)
+		);
 
-			// Pend the transaction
-			const pendResult = await this.transactor.pend(pendRequest);
-			if (!pendResult.success) {
-				// Cancel any already-pended collections before returning
-				for (const [, pendedBlockIdList] of pendedBlockIds.entries()) {
-					await this.transactor.cancel({ actionId, blockIds: pendedBlockIdList });
+		// Partition settled results: every collection that DID pend (keyed with its block
+		// ids), plus the first failure reason if any collection failed.
+		const pendedBlockIds = new Map<CollectionId, BlockId[]>();
+		let failure: string | undefined;
+		// Classify across ALL failures (mirroring commitPhase, and independent of iteration order):
+		// the pend is a retryable clean stale loss only if at least one failure was a conflicting pend
+		// (PendRejectedError.conflict) AND none was a hard failure. A single hard failure (storage/
+		// policy rejection, or a thrown/unavailable transactor) will not clear on a re-read, so
+		// re-driving it would just burn the retry budget — fail fast instead.
+		let anyConflict = false;
+		let anyHard = false;
+		for (const outcome of outcomes) {
+			if (outcome.status === 'fulfilled') {
+				pendedBlockIds.set(outcome.value.collectionId, outcome.value.blockIds);
+			} else {
+				if (failure === undefined) {
+					failure = outcome.reason instanceof Error ? outcome.reason.message : String(outcome.reason);
 				}
-				return {
-					success: false,
-					error: `Pend failed for collection ${collectionId}: ${pendResult.reason}`
-				};
+				if (outcome.reason instanceof PendRejectedError && outcome.reason.conflict) anyConflict = true;
+				else anyHard = true;
 			}
+		}
 
-			// Store the pended block IDs for commit phase
-			pendedBlockIds.set(collectionId, pendResult.blockIds);
+		if (failure !== undefined) {
+			// Any failure aborts the whole pend. With concurrency several collections may
+			// have pended in parallel, so cancel EVERY successfully-pended collection — not
+			// only those started before the failure. Cancels are best-effort (cancelPhase
+			// swallows their errors) so they cannot mask the original pend failure.
+			await this.cancelPhase(actionId, pendedBlockIds);
+			return { success: false, error: failure, staleLoss: anyConflict && !anyHard };
 		}
 
 		return { success: true, pendedBlockIds };
 	}
 
+	/**
+	 * Pend a single collection's transforms. Resolves with the collection id and its
+	 * pended block ids on success; throws with a per-collection reason on failure so the
+	 * fan-out in {@link pendPhase} can settle it as a rejection.
+	 */
+	private async pendCollection(
+		transaction: Transaction,
+		operationsHash: string,
+		collectionId: CollectionId,
+		transforms: Transforms,
+		actionId: ActionId,
+		nominees: PeerId[] | undefined
+	): Promise<{ collectionId: CollectionId; blockIds: BlockId[] }> {
+		const collection = this.collections.get(collectionId);
+		if (!collection) {
+			throw new Error(`Collection not found: ${collectionId}`);
+		}
+
+		// Get revision from the collection's source
+		const rev = collection.getNextRev();
+
+		// Create pend request with transaction and operations hash for validation
+		const pendRequest: PendRequest = {
+			actionId,
+			rev,
+			transforms,
+			policy: 'r', // Return policy: fail but return pending actions
+			transaction,
+			operationsHash,
+			superclusterNominees: nominees
+		};
+
+		const pendResult = await this.transactor.pend(pendRequest);
+		if (!pendResult.success) {
+			// A committed `missing` action or a `pending` action on a touched block is an
+			// optimistic-concurrency conflict — retryable after a re-read. A bare `reason` is a hard
+			// rejection (storage/policy) that re-driving won't fix.
+			const conflict = Boolean(pendResult.missing?.length || pendResult.pending?.length);
+			throw new PendRejectedError(collectionId, conflict, pendResult.reason);
+		}
+
+		return { collectionId, blockIds: pendResult.blockIds };
+	}
+
 	/**
 	 * COMMIT phase: Commit to all critical blocks with retry for transient failures.
 	 *
@@ -738,69 +955,114 @@ export class TransactionCoordinator {
 		error?: string;
 		committedCollections: Set<CollectionId>;
 		failedCollections: Set<CollectionId>;
+		staleLoss?: boolean;
 	}> {
+		// Fan out the independent per-collection commit-with-retry concurrently, then
+		// aggregate the committed/failed partition from the settled results.
+		const outcomes = await Promise.allSettled(
+			Array.from(pendedBlockIds.entries()).map(([collectionId, blockIds]) =>
+				this.commitCollection(actionId, criticalBlockIds, collectionId, blockIds)
+			)
+		);
+
 		const committedCollections = new Set<CollectionId>();
 		const failedCollections = new Set<CollectionId>();
-
-		// Commit each collection's transaction with retry
-		for (const [collectionId, blockIds] of pendedBlockIds.entries()) {
-			const collection = this.collections.get(collectionId);
-			if (!collection) {
-				failedCollections.add(collectionId);
-				return {
-					success: false,
-					error: `Collection not found: ${collectionId}`,
-					committedCollections,
-					failedCollections
-				};
+		const errors: string[] = [];
+		// Classify failures: a returned stale loss (someone committed a newer rev) is retryable after
+		// a re-read; a thrown/transient-exhausted or structural failure is not. staleLoss holds only
+		// if EVERY failure was a stale loss — a single hard failure makes the whole attempt not worth
+		// re-driving.
+		let anyStale = false;
+		let anyHard = false;
+		for (const outcome of outcomes) {
+			if (outcome.status === 'fulfilled') {
+				const { collectionId, committed, error, stale } = outcome.value;
+				if (committed) {
+					committedCollections.add(collectionId);
+				} else {
+					failedCollections.add(collectionId);
+					if (error) errors.push(error);
+					if (stale) anyStale = true; else anyHard = true;
+				}
+			} else {
+				// commitCollection resolves rather than rejects, but treat any unexpected
+				// rejection as a (hard) failure so the partitioned sets stay honest.
+				errors.push(outcome.reason instanceof Error ? outcome.reason.message : String(outcome.reason));
+				anyHard = true;
 			}
+		}
 
-			// Get revision
-			const rev = (collection['source'].actionContext?.rev ?? 0) + 1;
+		if (failedCollections.size > 0 || errors.length > 0) {
+			return {
+				success: false,
+				error: errors.join('; ') || 'Commit failed',
+				committedCollections,
+				failedCollections,
+				staleLoss: anyStale && !anyHard,
+			};
+		}
 
-			// Find the critical block (log tail) for this collection
-			const logTailBlockId = criticalBlockIds.find(blockId =>
-				blockIds.includes(blockId)
-			);
+		return { success: true, committedCollections, failedCollections };
+	}
 
-			if (!logTailBlockId) {
-				failedCollections.add(collectionId);
-				return {
-					success: false,
-					error: `Log tail block not found for collection ${collectionId}`,
-					committedCollections,
-					failedCollections
-				};
-			}
+	/**
+	 * Commit a single collection's pended blocks, retrying transient failures up to three
+	 * times (forward recovery). Always resolves — success is carried in the returned
+	 * `committed` flag — so the fan-out in {@link commitPhase} can aggregate every result.
+	 */
+	private async commitCollection(
+		actionId: ActionId,
+		criticalBlockIds: BlockId[],
+		collectionId: CollectionId,
+		blockIds: BlockId[]
+	): Promise<{ collectionId: CollectionId; committed: boolean; error?: string; stale?: boolean }> {
+		const collection = this.collections.get(collectionId);
+		if (!collection) {
+			return { collectionId, committed: false, error: `Collection not found: ${collectionId}` };
+		}
 
-			// Create commit request
-			const commitRequest: CommitRequest = {
-				actionId,
-				blockIds,
-				tailId: logTailBlockId,
-				rev
-			};
+		// Get revision
+		const rev = collection.getNextRev();
+
+		// Find the critical block (log tail) for this collection
+		const logTailBlockId = criticalBlockIds.find(blockId => blockIds.includes(blockId));
+		if (!logTailBlockId) {
+			return { collectionId, committed: false, error: `Log tail block not found for collection ${collectionId}` };
+		}
+
+		// Create commit request
+		const commitRequest: CommitRequest = {
+			actionId,
+			blockIds,
+			tailId: logTailBlockId,
+			rev
+		};
 
-			// Retry up to 3 attempts for transient failures
-			let committed = false;
-			for (let attempt = 0; attempt < 3 && !committed; attempt++) {
+		// Retry ONLY transient/thrown failures (unreachable peers, timeout) — forward recovery.
+		// A returned { success:false } is a permanent stale loss (someone committed a newer rev);
+		// the identical request can never win, so return immediately without retrying. Either way
+		// cancelPhase (run by coordinateTransaction on commitPhase failure) releases the pend
+		// exactly once — commit itself no longer self-cancels.
+		let lastTransientError: string | undefined;
+		for (let attempt = 0; attempt < 3; attempt++) {
+			try {
 				const commitResult = await this.transactor.commit(commitRequest);
 				if (commitResult.success) {
-					committed = true;
-					committedCollections.add(collectionId);
-				} else if (attempt === 2) {
-					failedCollections.add(collectionId);
-					return {
-						success: false,
-						error: `Commit failed for collection ${collectionId} after 3 attempts`,
-						committedCollections,
-						failedCollections
-					};
+					return { collectionId, committed: true };
 				}
+				// Permanent stale failure: do not retry here. It IS a clean stale loss, though, so
+				// mark it retryable at the coordinator level (after a re-read advances the rev).
+				return {
+					collectionId,
+					committed: false,
+					stale: true,
+					error: commitResult.reason ?? `Stale commit for collection ${collectionId}`
+				};
+			} catch (e) {
+				lastTransientError = e instanceof Error ? e.message : String(e);
 			}
 		}
-
-		return { success: true, committedCollections, failedCollections };
+		return { collectionId, committed: false, error: `Commit failed for collection ${collectionId} after 3 attempts: ${lastTransientError}` };
 	}
 
 	/**
@@ -814,10 +1076,17 @@ export class TransactionCoordinator {
 		pendedBlockIds: Map<CollectionId, BlockId[]>,
 		excludeCollections?: Set<CollectionId>
 	): Promise<void> {
-		for (const [collectionId, blockIds] of pendedBlockIds.entries()) {
-			if (excludeCollections?.has(collectionId)) continue;
-			await this.transactor.cancel({ actionId, blockIds });
-		}
+		// Fan out the per-collection cancels concurrently. Each is best-effort: a cancel
+		// fault is logged and swallowed so it cannot mask the pend/commit failure that
+		// triggered this sweep, and so one failed cancel does not abort the others.
+		const cancels = Array.from(pendedBlockIds.entries())
+			.filter(([collectionId]) => !excludeCollections?.has(collectionId))
+			.map(([collectionId, blockIds]) =>
+				this.transactor.cancel({ actionId, blockIds }).catch(err => {
+					log('cancelPhase: best-effort cancel failed collection=%s: %o', collectionId, err);
+				})
+			);
+		await Promise.all(cancels);
 	}
 
 }