@optimystic/db-core 1.0.0-beta.2 → 1.0.0

package/src/transaction/coordinator.ts

@@ -2,12 +2,13 @@ import type { ITransactor, BlockId, CollectionId, Transforms, PendRequest, Commi
 import type { Transaction, ExecutionResult, ITransactionEngine, CollectionActions, ReadDependency } from "./transaction.js";
 import type { PeerId } from "../network/types.js";
 import { isConflictFailure } from "../network/stale-failure.js";
-import type { Collection, CollectionSnapshot } from "../collection/collection.js";
+import type { Collection, CollectionSnapshot, RefreshReport } from "../collection/collection.js";
 import type { SyncOptions } from "../collection/index.js";
+import { TornActionError } from "../collection/struct.js";
 import { isTransactionExpired, clampPriority } from "./transaction.js";
 import { Log } from "../log/log.js";
-import { blockIdsForTransforms } from "../transform/helpers.js";
-import { computeBlockContentDigests, blockDigestsField } from "../transform/digest.js";
+import { blockIdsForTransforms, copyTransforms } from "../transform/helpers.js";
+import { computeBlockContentDigests, blockDigestsField, baseRevsField } from "../transform/digest.js";
 import { collectOperations, hashOperations } from "./operations-hash.js";
 import { CoordinatorConcurrentStampError, CoordinatorPartialCommitError, CoordinatorStaleLossError } from "./errors.js";
 import { jitteredBackoffMs, abortableDelay, makeAbortError } from "../utility/backoff.js";
@@ -46,6 +47,55 @@ class PendRejectedError extends Error {
 	}
 }
 
+/**
+ * What one {@link TransactionCoordinator.commit} call keeps across all of its attempts. Created
+ * once per commit, filled in by each attempt and by the refresh between attempts, and read when
+ * the commit fails, so the failure can say which participants are already saved.
+ */
+type CommitCycle = {
+	/** Disposers for the in-flight marks each attempt sets on the participants it latches. They
+	 * must outlive the individual attempt: the inter-attempt refresh is the ONLY reader of the mark,
+	 * and it deliberately runs after the commit span released its latches (`Latches` is
+	 * non-reentrant), so a clear tied to the latch would already have run. Each disposer is
+	 * id-guarded, so re-marking on a later attempt is harmless and a stale disposer cannot wipe a
+	 * newer mark. */
+	inFlightDisposers: (() => void)[];
+	/** Every collection any attempt included — one with staged changes when that attempt began. A
+	 * registered collection no attempt included is not part of this transaction and appears in no
+	 * report. */
+	participants: Set<CollectionId>;
+	/** Participants that are durably committed while the commit is still being driven: a refresh
+	 * between attempts found this transaction's own log entry and finished it (see
+	 * {@link RefreshReport}), or an attempt committed them and was refused on a sibling that can
+	 * still be finished (the partial-retry branch of {@link commitOnceLatched}). Once non-empty,
+	 * the commit can no longer fail cleanly: every later failure is reported through
+	 * {@link reportSaved} as a partial landing naming these as committed. A successful attempt
+	 * ends the commit and records nothing here; a partial landing on a hard failure reports its
+	 * own committed set at once. */
+	saved: Set<CollectionId>;
+};
+
+/**
+ * What a pend this coordinator sends carries for the receiving cluster members to re-check.
+ *
+ * - `'transaction'` (default): the transaction and the hash of every operation it produced — the
+ *   session-mode shape. A member with a transaction validator re-executes the statements and
+ *   compares hashes; the aged retry priority rides inside the transaction.
+ * - `'none'`: bare transforms, exactly the shape a single-collection `Collection.sync` pend has.
+ *   For a commit that has no statements to re-execute — the Quereus adapter's legacy multi-tree
+ *   commit, whose rows were staged directly into the trees — a validating member would otherwise
+ *   refuse the pend as a validator fault. The aged priority then rides on the pend itself
+ *   (`PendRequest.priority`), which is the other carrier a member's race resolution reads, so a
+ *   repeatedly-losing commit still ages. Members configured `unvalidatablePendPolicy: 'reject'`
+ *   refuse these pends, as they refuse every unvalidatable pend.
+ */
+export type PendValidationMode = 'transaction' | 'none';
+
+export interface CoordinatorOptions {
+	/** See {@link PendValidationMode}. Default `'transaction'`. */
+	pendValidation?: PendValidationMode;
+}
+
 /**
  * Coordinates multi-collection transactions.
  *
@@ -95,10 +145,35 @@ export class TransactionCoordinator {
 		preSnapshot: Map<Collection<any>, CollectionSnapshot<any>>;
 	}>();
 
+	/** See {@link PendValidationMode}; fixed for the coordinator's lifetime, so every pend it
+	 * sends has one shape. */
+	private readonly pendValidation: PendValidationMode;
+
 	constructor(
 		private readonly transactor: ITransactor,
-		private readonly collections: Map<CollectionId, Collection<any>>
-	) {}
+		private readonly collections: Map<CollectionId, Collection<any>>,
+		options: CoordinatorOptions = {}
+	) {
+		this.pendValidation = options.pendValidation ?? 'transaction';
+	}
+
+	/**
+	 * The validation pair every pend of this attempt carries, or `undefined` when this coordinator
+	 * sends unvalidated pends (see {@link PendValidationMode}). Built once per attempt, AFTER the
+	 * log append: the hash covers ALL operations across ALL collections, which a validator
+	 * recomputes from its own re-execution and compares. The shared operations-hash module
+	 * canonicalises (sort + canonical JSON), so the fingerprint is order-independent. Skipped
+	 * entirely — not computed and discarded — when no pend will carry it.
+	 */
+	private async pendValidationFor(
+		transaction: Transaction,
+		collectionTransforms: Map<CollectionId, Transforms>
+	): Promise<PendRequest['validation']> {
+		if (this.pendValidation === 'none') {
+			return undefined;
+		}
+		return { transaction, operationsHash: await hashOperations(collectOperations(collectionTransforms)) };
+	}
 
 	/**
 	 * Apply actions to collections (called by engines during statement execution).
@@ -213,21 +288,43 @@ export class TransactionCoordinator {
 	/**
 	 * Commit a transaction with a bounded, jittered backoff retry around a CLEAN stale loss.
 	 *
-	 * The single-attempt work lives in {@link commitOnce}; this wrapper re-drives it when the attempt
-	 * fails as a clean optimistic-concurrency loss ({@link CoordinatorStaleLossError} — nothing
-	 * durably committed, every tracker restored to its pre-append state). Before each re-attempt it
-	 * re-reads each collection to fresh revisions (so the retry pends against current state rather
-	 * than immediately re-failing stale), then backs off with the same jitter policy as
+	 * The single-attempt work lives in {@link commitOnce}; {@link commitAttempts} re-drives it when
+	 * the attempt fails as a clean optimistic-concurrency loss ({@link CoordinatorStaleLossError} —
+	 * nothing durably committed, every tracker restored to its pre-append state). Before each
+	 * re-attempt it re-reads each collection to fresh revisions (so the retry pends against current
+	 * state rather than immediately re-failing stale), then backs off with the same jitter policy as
 	 * {@link Collection.sync}. Retry is bounded by `maxAttempts` and an optional wall-clock
 	 * `deadlineMs`, and honours an abort `signal`.
 	 *
-	 * A {@link CoordinatorPartialCommitError} (a partial landing — some collection durably committed)
-	 * is NOT retryable and escapes immediately: blindly retrying would re-log already-durable actions.
-	 * Any other failure (expired transaction, unavailable transactor, unreachable cluster) also
-	 * propagates without retry — only genuine clean stale losses are re-driven.
+	 * A PARTIAL landing — some collection durably committed while a sibling's commit was refused —
+	 * is re-driven too when every refusal was a returned one (a member behind on the block's base,
+	 * a rival that took the revision slot): the committed collections are recorded as saved and the
+	 * next attempt commits only what is still staged, so the durable half is never re-logged and the
+	 * refused collection gets the same refresh-and-finish recovery a clean loss gets. A caller must
+	 * still never re-drive a partial landing itself. A partial landing on a HARD failure (the
+	 * transport budget spent, a structural rejection) escapes at once as
+	 * {@link CoordinatorPartialCommitError}, and any other failure (expired transaction,
+	 * unavailable transactor, unreachable cluster) also propagates without retry.
+	 *
+	 * A participant can also be SAVED between attempts: an attempt that reported a loss may still
+	 * have stored that participant's log tail, and the refresh before the next attempt then finishes
+	 * and consumes the participant's own entry (see {@link refreshBetweenAttempts}). From that moment
+	 * the commit can no longer fail cleanly, so EVERY failure that escapes afterwards — a torn sibling,
+	 * a sibling that keeps losing until the budget runs out, an abort, an expiry, a hard error — is
+	 * rethrown as {@link CoordinatorPartialCommitError}, counting the refresh-saved participants as
+	 * committed and carrying the original error as its `reason` (see {@link reportSaved}). A commit in
+	 * which nothing was saved fails with exactly the error it failed with.
 	 *
 	 * Defaults are safe out of the box: a caller that passes no options gets bounded, jittered retry.
 	 *
+	 * NOTE: this returns nothing, so a multi-collection commit reports NO durability — unlike
+	 * `Collection.sync`, which now answers with the {@link WriteDurability} of what it committed. The
+	 * gap is deliberate and tracked in `tickets/backlog/feat-multi-collection-commit-reports-durability.md`:
+	 * folding a class across collections that may individually have committed or been dropped (see
+	 * {@link CoordinatorPartialCommitError}) is its own design problem, not a pass-through. Until that
+	 * lands, a caller that needs to know whether a write is fully held has to use the single-collection
+	 * path; do not infer one here from a plain successful return.
+	 *
 	 * @param transaction - The transaction to commit
 	 * @param options - Retry knobs; shares the {@link SyncOptions} vocabulary with `Collection.sync`.
 	 */
@@ -241,6 +338,31 @@ export class TransactionCoordinator {
 		if (open !== undefined) {
 			throw new CoordinatorConcurrentStampError(open, transaction.stamp.id);
 		}
+
+		const cycle: CommitCycle = {
+			inFlightDisposers: [],
+			participants: new Set(),
+			saved: new Set(),
+		};
+		try {
+			await this.commitAttempts(transaction, cycle, options);
+		} catch (err) {
+			// The ONE place a failure leaves this commit, so no exit — present or future — can skip
+			// reporting a participant that is already saved.
+			throw this.reportSaved(transaction, cycle, err);
+		} finally {
+			// The in-flight marks must outlive each individual attempt (see CommitCycle), so they are
+			// cleared here, on every exit: return, stale-loss exhaustion, a partial landing, a hard
+			// error, an abort.
+			for (const dispose of cycle.inFlightDisposers) {
+				dispose();
+			}
+		}
+	}
+
+	/** The retry loop behind {@link commit}: every attempt, and the refresh between them. Records
+	 * what it learns in `cycle`; {@link commit} owns turning a failure into the right report. */
+	private async commitAttempts(transaction: Transaction, cycle: CommitCycle, options?: SyncOptions): Promise<void> {
 		const maxAttempts = options?.maxAttempts ?? DefaultMaxAttempts;
 		const baseBackoffMs = options?.baseBackoffMs ?? DefaultBaseBackoffMs;
 		const maxBackoffMs = options?.maxBackoffMs ?? DefaultMaxBackoffMs;
@@ -254,74 +376,198 @@ export class TransactionCoordinator {
 		let staleLosses = 0;
 		let lastLoss: CoordinatorStaleLossError | undefined;
 
-		// Disposers for the in-flight marks {@link commitOnce} sets on each participant it latches.
-		// They must outlive the individual attempt: the inter-attempt refresh below is the ONLY
-		// reader of the mark, and it deliberately runs after the commit span released its latches
-		// (`Latches` is non-reentrant), so a clear tied to the latch would already have run. Hence
-		// the finally spans the WHOLE retry loop — every exit clears: return, stale-loss exhaustion,
-		// a partial landing, a hard error, an abort. Each disposer is id-guarded, so re-marking on a
-		// later attempt is harmless and a stale disposer cannot wipe a newer mark.
-		const inFlightDisposers: (() => void)[] = [];
-		try {
-			for (;;) {
-				if (signal?.aborted) {
-					throw makeAbortError(signal);
-				}
-				// Progress-agnostic ceiling: once we've taken at least one loss, give up if the
-				// wall-clock deadline passed (independent of the attempt cap).
-				if (deadlineMs !== undefined && lastLoss && Date.now() - startedAt >= deadlineMs) {
-					throw lastLoss;
-				}
+		for (;;) {
+			if (signal?.aborted) {
+				throw makeAbortError(signal);
+			}
+			// Progress-agnostic ceiling: once we've taken at least one loss, give up if the
+			// wall-clock deadline passed (independent of the attempt cap).
+			if (deadlineMs !== undefined && lastLoss && Date.now() - startedAt >= deadlineMs) {
+				throw lastLoss;
+			}
 
-				// Age the transaction's advisory priority by the number of losses taken so far, so a
-				// repeatedly-losing transaction out-ranks fresh (priority-0) rivals in the cluster's
-				// resolveRace. Fairness-only and capped at MaxPriority; excluded from the tx id / client
-				// signature, so bumping it here does not churn identity. Left untouched on the first
-				// attempt (staleLosses == 0) so the initial pend serializes exactly as before.
-				if (staleLosses > 0) {
-					transaction.priority = clampPriority(staleLosses);
-				}
+			// Age the transaction's advisory priority by the number of losses taken so far, so a
+			// repeatedly-losing transaction out-ranks fresh (priority-0) rivals in the cluster's
+			// resolveRace. Fairness-only and capped at MaxPriority; excluded from the tx id / client
+			// signature, so bumping it here does not churn identity. Left untouched on the first
+			// attempt (staleLosses == 0) so the initial pend serializes exactly as before.
+			if (staleLosses > 0) {
+				transaction.priority = clampPriority(staleLosses);
+			}
 
-				try {
-					await this.commitOnce(transaction, inFlightDisposers);
-					return;
-				} catch (err) {
-					// Only a CLEAN stale loss is retryable. A partial landing, an expired transaction, an
-					// unavailable transactor, etc. all propagate unchanged.
-					if (!(err instanceof CoordinatorStaleLossError)) {
-						throw err;
-					}
-					lastLoss = err;
-					staleLosses++;
-					if (staleLosses >= maxAttempts) {
-						throw err;
-					}
+			try {
+				await this.commitOnce(transaction, cycle);
+				return;
+			} catch (err) {
+				// Only a CLEAN stale loss is retryable. A partial landing, an expired transaction, an
+				// unavailable transactor, etc. all propagate unchanged.
+				if (!(err instanceof CoordinatorStaleLossError)) {
+					throw err;
+				}
+				lastLoss = err;
+				staleLosses++;
+				if (staleLosses >= maxAttempts) {
+					throw err;
+				}
+				// Re-read fresh state before re-attempting so the next commit pends against current
+				// revisions (mirrors how Collection.sync calls updateInternal() before retrying).
+				//
+				// A participant whose log tail landed despite the reported loss finds its own entry
+				// here and FINISHES that action before consuming it (Collection.completeOwnEntry,
+				// from the attempt commitOnceLatched retained). When finishing is refused for a cause
+				// that can clear, the refresh — not the commit — is what gets retried: commitOnce
+				// would rebuild the participant's log entry with a fresh timestamp and send a second
+				// version of a log tail already stored under this transaction's id and revision.
+				// Each such round counts against the same budget as a stale loss; a permanent
+				// refusal (a rival holds the revision) escapes as the named TornActionError at once.
+				// Re-refreshing collections an earlier round already refreshed is a no-op.
+				// The round that would spend the last of the budget is told so, which makes each
+				// participant SETTLE a half-landed write rather than ask for another round: the
+				// error thrown below then says whether the write can still land (`final`). A
+				// deadline cannot be foreseen the same way; a write given up on it stays unsettled.
+				for (;;) {
 					const delay = jitteredBackoffMs(staleLosses - 1, { baseMs: baseBackoffMs, capMs: maxBackoffMs }, options?.rand);
 					await abortableDelay(delay, signal);
-					// Re-read fresh state before re-attempting so the next commit pends against current
-					// revisions (mirrors how Collection.sync calls updateInternal() before retrying).
-					// NOTE: refreshes EVERY registered collection, not only the participants of this
-					// transaction. Not free: a non-participant's update() throws CollectionHeaderVanishedError
-					// if its header momentarily reads absent while it holds a committed revision, aborting
-					// this retry. The registered set is small today; if that (or retry latency) ever bites,
-					// narrow this to the transaction's participating collections.
-					for (const collection of this.collections.values()) {
-						await collection.update();
+					const refused = await this.refreshBetweenAttempts(cycle, staleLosses + 1 >= maxAttempts);
+					if (refused === undefined) {
+						break;
+					}
+					staleLosses++;
+					if (staleLosses >= maxAttempts
+						|| (deadlineMs !== undefined && Date.now() - startedAt >= deadlineMs)) {
+						throw refused;
 					}
 				}
+				if (cycle.saved.size > 0 && this.stagedCollections().length === 0) {
+					// The refresh saved what was left: nothing is staged any more, so this transaction
+					// is committed. Return now rather than go round again, where an abort, the deadline
+					// or an expiry check could still report failure for a transaction that is entirely
+					// saved. The refresh's consume and replay already gave each saved participant the
+					// success path's local treatment, so only the stamp is left to release.
+					// Tested on what is still STAGED, not on "every participant saved": a participant
+					// can be saved and still hold actions staged after the attempt began, and those
+					// the next attempt must commit.
+					this.stampData.delete(transaction.stamp.id);
+					return;
+				}
 			}
-		} finally {
-			for (const dispose of inFlightDisposers) {
-				dispose();
+		}
+	}
+
+	/**
+	 * Refresh every registered collection between attempts, recording in `cycle.saved` each one
+	 * whose refresh finished this transaction's own log entry (see {@link Collection.refreshInFlight}).
+	 *
+	 * Visits EVERY collection even after one throws. Stopping at the first throw would leave a
+	 * participant later in the map unfinished whenever an earlier one is torn — its log tail stored
+	 * and its other blocks never landed, the silent loss `Collection.completeOwnEntry` exists to
+	 * prevent — and unreported. Each refresh is independent (its own latch, its own instance state),
+	 * so one failing says nothing about the next.
+	 *
+	 * NOTE: refreshes EVERY registered collection, not only the participants of this transaction.
+	 * Not free: a non-participant's refresh throws CollectionHeaderVanishedError if its header
+	 * momentarily reads absent while it holds a committed revision, aborting this retry. And because
+	 * a throw no longer stops the round, an unreachable cluster costs one failed read per registered
+	 * collection rather than one. The registered set is small today; if that (or retry latency) ever
+	 * bites, narrow this to the transaction's participants.
+	 *
+	 * @param lastChance - this is the round that spends the last of the retry budget, so a
+	 * participant whose half-landed write is refused again settles it (`Collection.completeOwnEntry`)
+	 * rather than asking for a round it will not get; the error returned then carries `final`.
+	 * @returns the first `completion-refused` {@link TornActionError} when that is the only way any
+	 * refresh failed — the round is worth retrying; `undefined` when every refresh succeeded.
+	 * @throws the first other error, once every collection has been visited.
+	 */
+	private async refreshBetweenAttempts(cycle: CommitCycle, lastChance: boolean): Promise<TornActionError | undefined> {
+		let terminal: { error: unknown } | undefined;
+		let refused: TornActionError | undefined;
+		for (const [collectionId, collection] of this.collections) {
+			const report: RefreshReport = {};
+			try {
+				await collection.refreshInFlight(report, lastChance);
+			} catch (err) {
+				if (err instanceof TornActionError && err.reason === 'completion-refused') {
+					refused ??= err;
+				} else {
+					terminal ??= { error: err };
+				}
+			}
+			// Read after the try, not only on success: a refresh can throw AFTER it finished the
+			// entry, and that participant is saved either way.
+			if (report.ownEntryFinished !== undefined) {
+				cycle.saved.add(collectionId);
 			}
 		}
+		if (terminal !== undefined) {
+			throw terminal.error;
+		}
+		return refused;
+	}
+
+	/** Every registered collection with something to push — the participants an attempt starting
+	 * now would commit. The predicate is {@link Collection.hasUnsyncedChanges}, the one
+	 * `Collection.sync` loops on: staged actions OR uncommitted tracker transforms. Not the
+	 * transforms alone: an action can be staged and have changed no block — a delete of an index
+	 * entry this instance's stale view never held (the shape `blind-index-delete-is-logged-but-
+	 * never-applied` fixed for the single-collection path) — and such a collection MUST still take
+	 * part. Its stale pend is refused, and the refresh between attempts replays the action against
+	 * the adopted revision, where it finds the entry and produces the delete the retry commits.
+	 * Filtering on transforms dropped it from the batch entirely, so the row went and its index
+	 * entry stayed behind, on every node. */
+	private stagedCollections(): { collectionId: CollectionId; collection: Collection<any>; transforms: Transforms }[] {
+		return Array.from(this.collections.entries())
+			.filter(([, collection]) => collection.hasUnsyncedChanges())
+			.map(([collectionId, collection]) => ({
+				collectionId,
+				collection,
+				transforms: collection.tracker.transforms
+			}));
+	}
+
+	/**
+	 * What a failure escaping {@link commit} must be reported as. When nothing is saved, the error
+	 * itself, untouched. When a refresh between attempts already saved a participant, the commit
+	 * half-landed whatever the error says — a {@link CoordinatorStaleLossError}'s "nothing durably
+	 * committed, safe to re-drive" would be false, and a re-drive would apply the saved half twice —
+	 * so it becomes a {@link CoordinatorPartialCommitError}:
+	 *
+	 * - `committedCollections`: every refresh-saved participant, plus the committed set of `err`
+	 *   when it is itself a partial landing (merged, never wrapped twice — its `reason` is kept);
+	 * - `failedCollections`: every other participant of any attempt. Empty when a failure came after
+	 *   every participant was saved (a refresh that finished its entry and then threw, say);
+	 * - `reason`: the original error.
+	 *
+	 * Local state already matches that report: a saved participant was consumed by its refresh, and
+	 * an unsaved one kept its staged actions (a failed finish throws before touching anything, and
+	 * commitOnceLatched restores every participant it did not commit). The stamp is released, as on
+	 * the existing partial path, because the transaction can no longer be cleanly rolled back.
+	 */
+	private reportSaved(transaction: Transaction, cycle: CommitCycle, err: unknown): unknown {
+		if (cycle.saved.size === 0) {
+			return err;
+		}
+		const committed = new Set(cycle.saved);
+		let reason = err;
+		if (err instanceof CoordinatorPartialCommitError) {
+			for (const id of err.committedCollections) committed.add(id);
+			reason = err.reason;
+		}
+		// NOTE: a participant is in exactly one list, so one saved by a refresh that ALSO held actions
+		// staged after its attempt began (and a later attempt failed to commit them) is listed only as
+		// committed. Unreachable while a session stages and commits on one call path; if staging ever
+		// races a commit, report such a participant in both lists.
+		const failed = [...cycle.participants].filter(id => !committed.has(id));
+		log('commit:partial-after-refresh tx=%s committed=%o failed=%o reason=%s', transaction.id,
+			[...committed], failed, reason instanceof Error ? reason.name : String(reason));
+		this.stampData.delete(transaction.stamp.id);
+		return new CoordinatorPartialCommitError([...committed], failed, reason);
 	}
 
 	/**
 	 * Commit a transaction (single attempt): materialise a log entry from each collection's staged
 	 * pending actions, then orchestrate the distributed consensus (GATHER/PEND/COMMIT).
 	 *
-	 * Called by {@link commit} (which wraps it in the backoff+jitter retry loop). The
+	 * Called by {@link commitAttempts} (the backoff+jitter retry loop behind {@link commit}). The
 	 * staged mutations already live in each collection's tracker — applied either via
 	 * applyActions() (engine-driven path) or directly via Collection.act()/Tree.stage
 	 * (the vtab's deferred-DML path) — but in BOTH cases without a log entry yet, so
@@ -329,37 +575,36 @@ export class TransactionCoordinator {
 	 * and folds the committed transforms back into each collection's read cache.
 	 *
 	 * On a clean stale loss (nothing durable, every tracker restored) it throws
-	 * {@link CoordinatorStaleLossError} so the caller can retry; on a partial landing it throws
+	 * {@link CoordinatorStaleLossError} so the caller can retry. On a partial landing it throws the
+	 * same, after recording the committed collections in `cycle.saved`, when every refusal was a
+	 * returned one (the retry finishes the refused collection); on a hard failure it throws
 	 * {@link CoordinatorPartialCommitError} (not retryable).
 	 *
 	 * @param transaction - The transaction to commit
-	 * @param inFlightDisposers - Collects one disposer per participant marked in flight under this
-	 * transaction's id (see {@link Collection.beginInFlightAction}). REQUIRED, so a future caller
-	 * cannot silently reintroduce the unmarked refresh this parameter exists to prevent: a caller
-	 * that never refreshes between attempts passes a throwaway array and simply ignores it. The
-	 * caller owns clearing them, because the mark has to survive past this attempt — see the
-	 * array's declaration in {@link commit}.
+	 * @param cycle - The state this commit keeps across attempts (see {@link CommitCycle}). This
+	 * attempt adds its participants to `cycle.participants` and one disposer per participant it marks
+	 * in flight under this transaction's id (see {@link Collection.beginInFlightAction}) to
+	 * `cycle.inFlightDisposers`. REQUIRED, so a future caller cannot silently reintroduce the
+	 * unmarked refresh the disposers exist to prevent, nor lose track of who took part. The caller
+	 * owns clearing the marks, because they have to survive past this attempt.
 	 */
-	private async commitOnce(transaction: Transaction, inFlightDisposers: (() => void)[]): Promise<void> {
+	private async commitOnce(transaction: Transaction, cycle: CommitCycle): Promise<void> {
 		if (isTransactionExpired(transaction.stamp)) {
 			throw new Error(`Transaction expired at ${transaction.stamp.expiration}`);
 		}
 
 		// Collect collections with staged (un-synced) changes.
-		const collectionData = Array.from(this.collections.entries())
-			.map(([collectionId, collection]) => ({
-				collectionId,
-				collection,
-				transforms: collection.tracker.transforms
-			}))
-			.filter(({ transforms }) =>
-				Object.keys(transforms.inserts ?? {}).length +
-				Object.keys(transforms.updates ?? {}).length +
-				(transforms.deletes?.length ?? 0) > 0
-			);
+		const collectionData = this.stagedCollections();
 
 		if (collectionData.length === 0) {
-			return; // Nothing to commit
+			// Nothing to commit is a successful commit: release the stamp as the success path does,
+			// or a read-only transaction that opened one (the empty pre-stage barrier) wedges every
+			// later stamp on this coordinator.
+			this.stampData.delete(transaction.stamp.id);
+			return;
+		}
+		for (const { collectionId } of collectionData) {
+			cycle.participants.add(collectionId);
 		}
 		// NOTE: this selection reads each tracker BEFORE the latches below are held, so a stage
 		// that lands between the filter and the acquisition is simply not part of this commit.
@@ -376,7 +621,7 @@ export class TransactionCoordinator {
 		// (db-p2p/src/storage/block-latch.ts), so two concurrent commits over overlapping
 		// participant sets cannot deadlock. `Latches` is non-reentrant, so nothing inside the
 		// held span may call a latched Collection method (act/update/sync/updateAndSync) on a
-		// participant — the retry loop's blanket collection.update() in commit() runs OUTSIDE
+		// participant — the retry loop's blanket refresh in commitAttempts runs OUTSIDE
 		// this span, after release.
 		// NOTE: the span covers the pend/commit consensus round trips, so every latched method on
 		// a participant instance (act/update/sync) queues for as long as the transactor takes.
@@ -390,15 +635,16 @@ export class TransactionCoordinator {
 			for (const { collection } of latchOrder) {
 				latchReleases.push(await collection.acquireLatch());
 				// Mark THIS attempt's action id on each participant while its latch is held, so the
-				// inter-attempt refresh in commit() recognises a log entry this transaction itself
-				// made durable (a torn commit: header and log tail committed, a later sweep block
-				// reported the conflict) and consumes it instead of replaying it into a second entry
-				// under the same id. `transaction.id` is stable across retries, so re-marking on a
+				// inter-attempt refresh in commitAttempts recognises a log entry this transaction itself
+				// stored (a torn commit: the log tail landed, and either its own answer or a later
+				// sweep block's reported failure), finishes that action's remaining blocks, and then
+				// consumes the entry instead of replaying it into a second entry under the same id
+				// (Collection.completeOwnEntry). `transaction.id` is stable across retries, so re-marking on a
 				// later attempt re-states the same fact. Only participants are marked; a registered
 				// non-participant is left unmarked and its refresh behaves exactly as a reader's.
-				inFlightDisposers.push(collection.beginInFlightAction(transaction.id));
+				cycle.inFlightDisposers.push(collection.beginInFlightAction(transaction.id));
 			}
-			await this.commitOnceLatched(transaction, collectionData);
+			await this.commitOnceLatched(transaction, collectionData, cycle);
 		} finally {
 			for (const release of latchReleases.reverse()) {
 				release();
@@ -412,10 +658,15 @@ export class TransactionCoordinator {
 	 * participant's latch — every Collection member it touches (snapshotPending, getPendingActions,
 	 * recordCommitted, applyCommittedToCache, restorePending, clearPendingActions, tracker.reset)
 	 * is latch-free by contract.
+	 *
+	 * `cycle` is written only on a partial landing that is retried forward: the participants this
+	 * attempt committed go into `cycle.saved`, so the retry loop reports every later failure as the
+	 * partial landing it is (see {@link reportSaved}).
 	 */
 	private async commitOnceLatched(
 		transaction: Transaction,
-		collectionData: { collectionId: CollectionId; collection: Collection<any> }[]
+		collectionData: { collectionId: CollectionId; collection: Collection<any> }[],
+		cycle: CommitCycle
 	): Promise<void> {
 		// Append each collection's staged actions to its log, then collect the
 		// resulting transforms + critical (log-tail) block for consensus.
@@ -437,6 +688,15 @@ export class TransactionCoordinator {
 		// through pend, commit, and the local recordCommitted, so all four name the same number.
 		const pendedRevs = new Map<CollectionId, number>();
 
+		// A participant is never pended over a base that moved under its staged operations
+		// (Collection.restageIfBasesMoved). Each re-stages its own queue only. BEFORE the snapshots
+		// below, so a failed attempt restores transforms that agree with the tracker's pins; and
+		// before the append loop, since a replay resets the tracker and would drop the appended
+		// log entry.
+		for (const { collection } of collectionData) {
+			await collection.restageIfBasesMoved();
+		}
+
 		// Snapshot EVERY participating collection's staged state (transforms + pending
 		// queue) BEFORE the append loop mutates any tracker. The loop appends log
 		// entries sequentially, so a failure on the Nth collection must also undo the
@@ -472,16 +732,13 @@ export class TransactionCoordinator {
 				pendedRevs.set(collectionId, applyResult.rev!);
 			}
 
-			// Compute hash of ALL operations across ALL collections (post-log-append).
-			// Validators re-execute the transaction and compare their computed hash.
-			// The shared operations-hash module canonicalises (sort + canonical JSON) so
-			// this order-independent fingerprint matches what a validator recomputes.
-			const operationsHash = await hashOperations(collectOperations(collectionTransforms));
+			// What each pend carries for members to re-check (post-log-append) — see pendValidationFor.
+			const validation = await this.pendValidationFor(transaction, collectionTransforms);
 
 			// Execute consensus phases (GATHER, PEND, COMMIT)
 			coordResult = await this.coordinateTransaction(
 				transaction,
-				operationsHash,
+				validation,
 				collectionTransforms,
 				criticalBlocks,
 				pendedRevs
@@ -500,6 +757,24 @@ export class TransactionCoordinator {
 
 		if (!coordResult.success) {
 			const committed = coordResult.committedCollections ?? new Set<CollectionId>();
+			// Keep exactly what each participant that did NOT commit sent, BEFORE the restores below
+			// swap its tracker out. A refused commit is not proof nothing landed: a participant's log
+			// tail is committed first and can be stored while the answer is still a failure, and
+			// cancelPhase has just dropped the pending records of every block that did not land. If
+			// the next refresh of that participant finds this transaction's own log entry, these are
+			// the transforms that finish it (Collection.completeOwnEntry) — without them the refresh
+			// can only refuse the write as torn, never save it. Copied, because until the restore
+			// this is the participant's LIVE tracker state. No block digests are retained: they are
+			// computed per commit from the live tracker (commitCollection), which is gone by the time
+			// a refresh re-sends, and an undeclared block falls back to member-side corroboration.
+			for (const { collectionId, collection } of collectionData) {
+				if (committed.has(collectionId)) continue;
+				collection.retainInFlightAttempt(transaction.id, {
+					rev: pendedRevs.get(collectionId)!,
+					transforms: copyTransforms(collectionTransforms.get(collectionId)!),
+					tailId: criticalBlocks.get(collectionId)!,
+				});
+			}
 			if (committed.size > 0) {
 				// PARTIAL COMMIT: at least one collection durably committed via consensus
 				// while another failed permanently. A uniform pre-append restore would
@@ -526,15 +801,38 @@ export class TransactionCoordinator {
 						collection.restorePending(preCommitSnapshots.get(collectionId)!);
 					}
 				}
-				// The transaction half-landed, so it is neither cleanly retryable nor
-				// cleanly abortable: drop its stamp tracking (the success path does the
-				// same at the end) and surface the structured signal for reconciliation.
+				const failed = [...(coordResult.failedCollections ?? new Set<CollectionId>())];
+				if (coordResult.staleLoss) {
+					// Every failure was a RETURNED refusal: a member behind on the block's base (the
+					// durability gate's `commit-not-durable`, from a replica that has not caught up —
+					// the writer's own, on a node that has just joined), or a rival that took the
+					// revision slot. The siblings above have landed and cannot be rolled back, so
+					// FORWARD recovery — finishing this collection — is the only way back to
+					// all-or-nothing, and the retry loop already does it: the refresh between attempts
+					// finds this transaction's own log entry where the tail landed and re-sends the
+					// retained attempt at the same revision (Collection.completeOwnEntry), or the next
+					// attempt re-pends afresh when nothing of it landed. The committed siblings are
+					// recorded as saved, so every failure that still escapes is reported as the partial
+					// landing it is (reportSaved), and the next attempt commits only what is still
+					// staged (stagedCollections) — the durable half is never re-logged. The stamp stays
+					// open for the retry; reportSaved or the success path releases it.
+					// The committed siblings keep their in-flight mark until the cycle ends, so the
+					// blanket refresh between attempts WALKS their log (the mark disables the tail
+					// shortcut) but cannot find their own entry: recordCommitted above advanced each
+					// one's held revision to the entry's own, and Log.getFrom returns only entries
+					// ABOVE the held revision. Collection.consumeOwnEntry is therefore unreachable for
+					// a sibling committed here, and no attempt was retained for it (the loop above
+					// skips committed participants) — the refresh is a reader's, at a cost of one walk.
+					for (const collectionId of committed) cycle.saved.add(collectionId);
+					log('commit:partial-retry tx=%s committed=%o failed=%o reason=%s', transaction.id, [...committed], failed, coordResult.error);
+					throw new CoordinatorStaleLossError(failed, coordResult.error);
+				}
+				// Half-landed on a HARD failure (the transport budget spent on a thrown fault, a
+				// structural rejection): neither cleanly retryable nor cleanly abortable, so drop the
+				// stamp tracking (the success path does the same at the end) and surface the
+				// structured signal for reconciliation at once.
 				this.stampData.delete(transaction.stamp.id);
-				throw new CoordinatorPartialCommitError(
-					[...committed],
-					[...(coordResult.failedCollections ?? new Set<CollectionId>())],
-					coordResult.error
-				);
+				throw new CoordinatorPartialCommitError([...committed], failed, coordResult.error);
 			}
 
 			// EMPTY committed set: PEND failed, or the whole commit failed cleanly with
@@ -855,6 +1153,12 @@ export class TransactionCoordinator {
 				}
 			}
 
+			// Same pre-pend rule as commitOnceLatched: no participant is pended over a moved base.
+			// Under the latches and before the append loop, which a replay's tracker reset would undo.
+			for (const collectionId of allCollectionIds) {
+				await this.collections.get(collectionId)?.restageIfBasesMoved();
+			}
+
 			for (const [collectionId, actions] of batches) {
 				const applyResult = await this.applyActionsToCollection(
 					{ collectionId, actions },
@@ -872,8 +1176,8 @@ export class TransactionCoordinator {
 				pendedRevs.set(collectionId, applyResult.rev!);
 			}
 
-			// 3. Compute operations hash for validation (order-independent; see commit()).
-			const operationsHash = await hashOperations(collectOperations(collectionTransforms));
+			// 3. What each pend carries for members to re-check (see pendValidationFor).
+			const validation = await this.pendValidationFor(transaction, collectionTransforms);
 
 			const applyMs = Date.now() - tApply;
 
@@ -881,7 +1185,7 @@ export class TransactionCoordinator {
 			const tCoord = Date.now();
 			const coordResult = await this.coordinateTransaction(
 				transaction,
-				operationsHash,
+				validation,
 				collectionTransforms,
 				criticalBlocks,
 				pendedRevs
@@ -899,6 +1203,14 @@ export class TransactionCoordinator {
 				// unsafe all-or-nothing undo handle survives (rollback() would rewind the winner).
 				// Kept await-free for the same reason the success fold is — restorePending is
 				// synchronous and latch-free by contract, so it is safe inside this latched span.
+				// NOTE: execute() has no retry loop at all, so a returned refusal after a sibling
+				// committed is reported here at once, where commit() would re-drive the refused
+				// collection forward (the partial-retry branch of commitOnceLatched). No production
+				// caller reaches this today — the plugin's session mode stages through applyActions
+				// and commits through commit() — so the split window is documented, not closed
+				// (docs/transactions.md). If a host ever commits multi-collection transactions
+				// through execute(), route its coordination through commit()'s retry rather than
+				// duplicating that recovery here.
 				const committed = coordResult.committedCollections ?? new Set<CollectionId>();
 				if (committed.size > 0) {
 					for (const collectionId of batches.keys()) {
@@ -1032,9 +1344,10 @@ export class TransactionCoordinator {
 		// which collection's log the dependent landed in.
 		// NOTE: `allCollectionIds` names the participants of THIS attempt, and a retry's participant
 		// set can be SMALLER than the first attempt's. After a torn commit, the participant whose
-		// entry landed durably consumes that entry on the inter-attempt refresh
-		// (Collection.inFlightActionId), empties its pending queue and resets its tracker, so
-		// commitOnce's non-empty-transforms filter drops it from the next attempt. The retry's
+		// log entry landed finishes that action on the inter-attempt refresh — landing the blocks
+		// the refused commit left behind, at the same id and revision — and only then consumes the
+		// entry (Collection.completeOwnEntry), emptying its pending queue and resetting its tracker,
+		// so commitOnce's non-empty-transforms filter drops it from the next attempt. The retry's
 		// entries therefore list only the REMAINING participants, while the torn participant's
 		// already-durable entry lists them all — one transaction id, two different
 		// `allCollectionIds` values across its entries. Nothing today keys off that list for
@@ -1064,7 +1377,8 @@ export class TransactionCoordinator {
 	 * Coordinate a transaction across multiple collections.
 	 *
 	 * @param transaction - The transaction to coordinate
-	 * @param operationsHash - Hash of all operations for validation
+	 * @param validation - What every pend carries for members to re-check, or undefined for an
+	 * unvalidated pend (see {@link pendValidationFor})
 	 * @param collectionTransforms - Map of collectionId to its transforms
 	 * @param criticalBlocks - Map of collectionId to its log tail blockId
 	 * @param pendedRevs - Per collection, the revision its log entry was stamped with (from
@@ -1073,7 +1387,7 @@ export class TransactionCoordinator {
 	 */
 	private async coordinateTransaction(
 		transaction: Transaction,
-		operationsHash: string,
+		validation: PendRequest['validation'],
 		collectionTransforms: Map<CollectionId, Transforms>,
 		criticalBlocks: Map<CollectionId, BlockId>,
 		pendedRevs: ReadonlyMap<CollectionId, number>
@@ -1099,7 +1413,7 @@ export class TransactionCoordinator {
 		const tPend = Date.now();
 		const pendResult = await this.pendPhase(
 			transaction,
-			operationsHash,
+			validation,
 			collectionTransforms,
 			pendedRevs,
 			superclusterNominees
@@ -1192,8 +1506,10 @@ export class TransactionCoordinator {
 	/**
 	 * PEND phase: Distribute transaction to all affected block clusters.
 	 *
-	 * @param transaction - The full transaction for replay/validation
-	 * @param operationsHash - Hash of all operations for validation
+	 * @param transaction - The transaction being pended (its id names the action; its aged
+	 * priority rides on an unvalidated pend)
+	 * @param validation - What every pend carries for members to re-check, or undefined for an
+	 * unvalidated pend (see {@link pendValidationFor})
 	 * @param collectionTransforms - Map of collectionId to its transforms
 	 * @param pendedRevs - Per collection, the revision its log entry was stamped with — the pend
 	 * request repeats it verbatim rather than recomputing from the collection (a recompute after
@@ -1202,7 +1518,7 @@ export class TransactionCoordinator {
 	 */
 	private async pendPhase(
 		transaction: Transaction,
-		operationsHash: string,
+		validation: PendRequest['validation'],
 		collectionTransforms: ReadonlyMap<CollectionId, Transforms>,
 		pendedRevs: ReadonlyMap<CollectionId, number>,
 		superclusterNominees: ReadonlySet<PeerId> | null
@@ -1221,7 +1537,7 @@ export class TransactionCoordinator {
 		// with a concurrency limiter so peak in-flight round-trips stays sane. Same for commitPhase.
 		const outcomes = await Promise.allSettled(
 			Array.from(collectionTransforms.entries()).map(([collectionId, transforms]) =>
-				this.pendCollection(transaction, operationsHash, collectionId, transforms, pendedRevs.get(collectionId)!, actionId, nominees)
+				this.pendCollection(transaction, validation, collectionId, transforms, pendedRevs.get(collectionId)!, actionId, nominees)
 			)
 		);
 
@@ -1267,7 +1583,7 @@ export class TransactionCoordinator {
 	 */
 	private async pendCollection(
 		transaction: Transaction,
-		operationsHash: string,
+		validation: PendRequest['validation'],
 		collectionId: CollectionId,
 		transforms: Transforms,
 		/** The revision the log entry was stamped with (threaded from applyActionsToCollection),
@@ -1282,14 +1598,29 @@ export class TransactionCoordinator {
 			throw new Error(`Collection not found: ${collectionId}`);
 		}
 
-		// Create pend request with the validation payload (transaction + operations hash) —
-		// always BOTH, as one pair: this is the only producer of PendRequest.validation.
+		// Create the pend request. This is the only producer of PendRequest.validation, and it
+		// always sends the pair whole (transaction + operations hash) or not at all. The aged
+		// priority has one carrier per shape — inside the transaction when the pend carries it,
+		// on the pend itself when it does not — because a member's race resolution reads the
+		// transaction's priority first and falls back to the pend's (recordPriority in db-p2p's
+		// race-resolution.ts); an unvalidated pend that left both empty would never age. Omitted
+		// at 0, exactly as the single-collection path sends it (TransactorSource.transact), so
+		// the common first-attempt pend is indistinguishable from a Collection.sync pend.
+		const priority = clampPriority(transaction.priority);
+		// The base each update-only block's operations were computed against, read from the same
+		// tracker the commit side digests (commitCollection); it still holds this transaction's staged
+		// transforms and pins. Storage keeps it with the pending record so the record is never applied
+		// to a different version of the block (see PendRequest.baseRevs). Omitted when empty.
+		const baseRevs = collection.tracker.stagedBaseRevs(blockIdsForTransforms(transforms));
 		const pendRequest: PendRequest = {
 			actionId,
 			rev,
 			transforms,
 			policy: 'r', // Return policy: fail but return pending actions
-			validation: { transaction, operationsHash },
+			...(validation === undefined
+				? (priority > 0 ? { priority } : {})
+				: { validation }),
+			...baseRevsField(baseRevs),
 			superclusterNominees: nominees
 		};
 
@@ -1415,10 +1746,14 @@ export class TransactionCoordinator {
 		};
 
 		// Retry ONLY transient/thrown failures (unreachable peers, timeout) — forward recovery.
-		// A returned { success:false } is a permanent stale loss (someone committed a newer rev);
-		// the identical request can never win, so return immediately without retrying. Either way
-		// cancelPhase (run by coordinateTransaction on commitPhase failure) releases the pend
-		// exactly once — commit itself no longer self-cancels.
+		// A returned { success:false } is a refusal the IDENTICAL request can never win (a rival
+		// committed a newer rev, or a member behind on the base refused and dropped its pending
+		// record), so return immediately without retrying it here. The refusal is retried at the
+		// coordinator level instead, as a DIFFERENT request: the retry loop refreshes, finishes the
+		// collection's own log entry where the tail landed, or re-pends afresh — and it does so even
+		// when a sibling collection has already committed (the partial-retry branch of
+		// commitOnceLatched). Either way cancelPhase (run by coordinateTransaction on commitPhase
+		// failure) releases the pend exactly once — commit itself no longer self-cancels.
 		let lastTransientError: string | undefined;
 		for (let attempt = 0; attempt < 3; attempt++) {
 			try {
@@ -1430,7 +1765,11 @@ export class TransactionCoordinator {
 				// mark it retryable at the coordinator level (after a re-read advances the rev).
 				// NOTE: deliberately does NOT consult `isConflictFailure` / `StaleFailure.conflict`
 				// like the pend path does. Once the pend succeeded, a returned commit failure means
-				// the revision slot moved. One commit producer DOES set `conflict` now —
+				// the revision slot moved, or a member behind on the base refused — the durability
+				// gate's `commit-not-durable` (which does set `conflict`) or a solo StorageRepo's
+				// `missing-base-revision` (a bare `reason`, no `conflict`). All of these clear on a
+				// re-drive, so gating on `conflict` here would fail the last of them fast. One commit
+				// producer DOES set `conflict` now —
 				// db-p2p's CoordinatorRepo.commit returns lost commit-consensus races and classified
 				// stale-commit rejections as `{ success:false, conflict:true }` (returning, not
 				// throwing, is what keeps them out of the verbatim retry above) — but every returned