@optimystic/db-p2p 0.24.1 → 0.25.0

package/dist/src/storage/storage-repo.js

@@ -1,34 +1,11 @@
-import { Latches, transformForBlockId, applyTransform, groupBy, concatTransform, emptyTransforms, blockIdsForTransforms, transformsFromTransform, highestStaleAt } from "@optimystic/db-core";
+import { transformForBlockId, applyTransform, groupBy, concatTransform, emptyTransforms, blockIdsForTransforms, transformsFromTransform, highestStaleAt, isOwnRevision, canonicalBlockHash } from "@optimystic/db-core";
 import { asyncIteratorToArray } from "../it-utility.js";
+import { proofDeclaredDigest } from "../cluster/commit-proof.js";
+import { RevisionNotCoveredError } from "./i-block-storage.js";
+import { acquireBlockWriteLatches, withBlockWriteLatch } from "./block-latch.js";
 import { createLogger } from "../logger.js";
+import { checkPendValidation } from "../pend-validation.js";
 const log = createLogger('storage-repo');
-/**
- * Single source of truth for the per-block commit latch key. Held by {@link StorageRepo.commit} and
- * {@link StorageRepo.saveReplicatedBlock}, and — through an injected runner ({@link withBlockCommitLatch})
- * — by the invalidation-apply path. Every out-of-band writer of a block's `meta.latest` must serialize
- * on this key against a concurrent local commit on the same block; keeping all call sites on this helper
- * is what prevents the key from drifting between them.
- */
-export const commitLatchKey = (blockId) => `StorageRepo.commit:${blockId}`;
-/**
- * Runs `fn` while holding the per-block commit latch {@link commitLatchKey}. This is the capability the
- * dispute module's `applyInvalidation` is handed (through its context) so its compensating
- * `saveReplica`/`saveDeletion` read-modify-write of `meta.latest` is mutually exclusive with a concurrent
- * {@link StorageRepo.commit} on the same block — otherwise an invalidation advancing `latest` outside
- * that latch is invisible to commit's staleness guard and can be clobbered (a non-monotonic regression).
- *
- * Acquire/release is per call, so a caller holds at most one block latch at any instant and cannot
- * deadlock against commit's sorted, up-front multi-latch acquisition.
- */
-export async function withBlockCommitLatch(blockId, fn) {
-    const release = await Latches.acquire(commitLatchKey(blockId));
-    try {
-        return await fn();
-    }
-    finally {
-        release();
-    }
-}
 /**
  * Stable, greppable prefix on the failure reason a commit carries when this node cannot materialize
  * the revision it was asked to record. It is a STRING marker rather than only an error class because
@@ -36,6 +13,13 @@ export async function withBlockCommitLatch(blockId, fn) {
  * also crosses the wire), so the class identity is lost by the time a caller inspects the result.
  */
 export const MISSING_BASE_REVISION_REASON = 'missing-base-revision';
+/**
+ * The two stable reject-reason prefixes a validating receiver emits, re-exported here (and from
+ * `cluster/cluster-repo.ts`) next to their siblings so a caller inspecting a `PendResult` reason
+ * need not know which module defines them. Both tiers refuse with the same prefixes because both
+ * run the same {@link checkPendValidation}.
+ */
+export { PEND_NOT_VALIDATABLE, VALIDATOR_FAULT } from "../pend-validation.js";
 /**
  * This node was asked to commit revision N of a block it holds no materializable base for, so
  * applying the transform would materialize nothing while `latest` advanced to N — a block that is
@@ -64,6 +48,7 @@ export function isMissingBaseRevisionFailure(result) {
 export class StorageRepo {
     createBlockStorage;
     validatePend;
+    unvalidatablePendPolicy;
     /** Per-collection change listeners; empty sets are pruned on unsubscribe. */
     changeListeners = new Map();
     /** Catch-all change listeners — fire for EVERY collection's commit on this node. */
@@ -71,6 +56,7 @@ export class StorageRepo {
     constructor(createBlockStorage, options) {
         this.createBlockStorage = createBlockStorage;
         this.validatePend = options?.validatePend;
+        this.unvalidatablePendPolicy = options?.unvalidatablePendPolicy ?? 'accept';
     }
     /**
      * Subscribe to commits that mutate `collectionId`'s blocks on this node.
@@ -169,9 +155,9 @@ export class StorageRepo {
             // posing as an authoritative "never existed" — see BlockUnavailableReason.
             let unavailable;
             // Ensure that all outstanding transactions in the context are committed.
-            // This promotes a landed-elsewhere pending via internalCommit, which mutates
-            // meta.latest — the same read-modify-write commit()/saveReplicatedBlock guard
-            // with the per-block commit latch. It MUST hold that latch too, or a promotion
+            // This promotes a landed-elsewhere pending via internalCommit, which writes the
+            // block's metadata — the same read-modify-write commit()/saveReplicatedBlock guard
+            // with the per-block write latch. It MUST hold that latch too, or a promotion
             // racing a concurrent commit on the block regresses latest non-monotonically /
             // cross-writes a revision. Cheap unlatched pre-scan first so the common
             // contextless read and no-pending read never pay for latch acquisition; the
@@ -182,7 +168,7 @@ export class StorageRepo {
                     ? context.committed.filter(c => c.rev > preLatest.rev)
                     : context.committed;
                 if (preMissing.length > 0) {
-                    await withBlockCommitLatch(blockId, async () => {
+                    await withBlockWriteLatch(blockId, async (latch) => {
                         // Re-read authoritative state under the latch: a concurrent commit may have
                         // promoted or superseded a pending between the unlatched pre-scan and here.
                         // Recompute which committed entries are still ahead of `latest` (drops the
@@ -201,7 +187,7 @@ export class StorageRepo {
                             for (const { actionId, rev } of [...missing].sort((a, b) => a.rev - b.rev)) {
                                 const pending = await blockStorage.getPendingTransaction(actionId);
                                 if (pending) {
-                                    const collectionId = await this.internalCommit(blockId, actionId, rev, blockStorage);
+                                    const collectionId = await this.internalCommit(blockId, actionId, rev, blockStorage, latch);
                                     if (collectionId !== undefined) {
                                         promotions.push({ collectionId, blockId, actionId, rev });
                                     }
@@ -231,17 +217,17 @@ export class StorageRepo {
             // context-driven get skips promotion (pending gone) and a default getBlock() sees the
             // stale latest. It is soft-wedged (stale), not hard-wedged: the next commit-retry for
             // (actionId, rev) self-heals it via storage.recover() in commit(). Not repaired lazily on
-            // the read path because get() holds no commit latch; if stale reads on unwritten blocks
-            // ever become a problem, add a latched lazy recover() here.
+            // the read path because the plain read below holds no write latch; if stale reads on
+            // unwritten blocks ever become a problem, add a latched lazy recover() here.
             //
-            // getBlock() THROWS when this node holds a `latest` it cannot materialize (truncated
-            // history: "Failed to find materialized block", or a failed restore). Caught PER BLOCK so
-            // one broken block cannot fail the whole batch's Promise.all and take healthy siblings
-            // down with it. The read still fails for THIS block — TransactorSource throws
+            // readBlockHealing() THROWS when this node holds a `latest` it cannot materialize
+            // (truncated history: "Failed to find materialized block", or a failed restore). Caught
+            // PER BLOCK so one broken block cannot fail the whole batch's Promise.all and take healthy
+            // siblings down with it. The read still fails for THIS block — TransactorSource throws
             // BlockUnavailableError on the flagged entry — so nothing is swallowed.
             let blockRev;
             try {
-                blockRev = await blockStorage.getBlock(context?.rev);
+                blockRev = await this.readBlockHealing(blockId, blockStorage, context?.rev);
             }
             catch (err) {
                 // NOTE: the entry drops `state.latest`, which this node does know (getLatest() does not
@@ -291,7 +277,7 @@ export class StorageRepo {
                         // of its own, so the honest answer is the base it was applied to. Absent when there
                         // was no base at all — a pending-only insert served over an absent committed base,
                         // where fabricating a revision would claim content this node never committed.
-                        ...(blockRev ? { materializedRev: blockRev.actionRev.rev } : {}),
+                        ...(blockRev ? { materialized: blockRev.actionRev } : {}),
                         // A pending applied to a missing base can materialize nothing (applyTransform drops
                         // updates with no block to apply them to) — that absence is a guess, and is flagged.
                         // A materialized block is a real answer regardless of the earlier refusal. TWO ways
@@ -322,7 +308,7 @@ export class StorageRepo {
                     // content. `state.latest` deliberately stays the node's newest revision for the block
                     // (StorageRepo.get's own promotion pre-scan and CoordinatorRepo's read-repair compare
                     // against it), so the two disagree exactly when a pinned read is serving older content.
-                    materializedRev: blockRev.actionRev.rev,
+                    materialized: blockRev.actionRev,
                     state: {
                         latest: await blockStorage.getLatest(),
                         pendings
@@ -335,6 +321,48 @@ export class StorageRepo {
         this.emitPromotions(promotions);
         return Object.fromEntries(results);
     }
+    /**
+     * The one place a local coverage gap is healed from a peer. `getBlock` is local-only; when it
+     * reports the target revision as not covered ({@link RevisionNotCoveredError}) this fetches it
+     * through `restoreRevision` under the block's write latch — the restore writes revision records
+     * and merges coverage into the metadata blob, so it must serialize against every other writer of
+     * the block — and re-reads. Only the restore is latched; the reads on either side are not, and
+     * the latch is never held across the two.
+     *
+     * A restore that fails on a **pending-only** block (metadata seeded by a pend, no committed
+     * revision) reads as ABSENT, not as a fault: the named revision was a guess about content this
+     * node never held, and the caller's insert-probe / pending-overlay logic already treats an absent
+     * base as "nothing committed here". A failed restore on a block that DOES hold a `latest` is a
+     * real fault (a `latest` this node cannot serve) and propagates, so the caller reports the block
+     * as unavailable. Any throw from the second read (records restored but nothing materializable
+     * under them) propagates the same way.
+     */
+    async readBlockHealing(blockId, storage, rev) {
+        try {
+            return await storage.getBlock(rev);
+        }
+        catch (err) {
+            if (!(err instanceof RevisionNotCoveredError)) {
+                throw err;
+            }
+            try {
+                // NOTE: the peer fetch inside restoreRevision runs UNDER the block's write latch, so a
+                // slow restore queues every commit/pend/replica on this block behind one network
+                // round-trip. Fine at today's restore rates (a gap is healed once, then served
+                // locally); if restore latency ever shows up delaying commits, fetch + vet OUTSIDE the
+                // latch and take it only to write, re-checking coverage inside.
+                await withBlockWriteLatch(blockId, latch => storage.restoreRevision(err.rev, latch));
+            }
+            catch (restoreErr) {
+                if (await storage.getLatest() === undefined) {
+                    log('get:restore-failed-pending-only blockId=%s rev=%d error=%s', blockId, err.rev, restoreErr instanceof Error ? restoreErr.message : String(restoreErr));
+                    return undefined;
+                }
+                throw restoreErr;
+            }
+            return await storage.getBlock(rev);
+        }
+    }
     /**
      * Emit a {@link CollectionChangeEvent} for each read-driven promotion that landed
      * during a {@link get}. A single get() can promote multiple distinct actions, each
@@ -362,17 +390,20 @@ export class StorageRepo {
         }
     }
     async pend(request, _options) {
-        // Validate transaction if present and validation hook is configured
-        if (this.validatePend && request.transaction && request.operationsHash) {
-            const validationResult = await this.validatePend(request.transaction, request.operationsHash);
-            if (!validationResult.valid) {
-                // Hard rejection: the transaction itself is invalid, so no `conflict` flag — a
-                // re-read and re-pend would fail the same way and only burn the retry budget.
-                return {
-                    success: false,
-                    reason: validationResult.reason ?? 'Transaction validation failed'
-                };
-            }
+        // Re-check the transaction when a validation hook is configured — the unvalidatable-pend
+        // policy and the throwing-hook catch both live in the shared `checkPendValidation`, so this
+        // tier and the cluster tier cannot drift apart on what they refuse.
+        const hook = this.validatePend;
+        const validation = await checkPendValidation(request, hook && (({ transaction, operationsHash }) => hook(transaction, operationsHash)), this.unvalidatablePendPolicy, event => event.kind === 'unvalidatable'
+            ? log('pend-unvalidatable actionId=%s policy=%s', request.actionId, event.policy)
+            : log('pend validator-fault actionId=%s error=%s', request.actionId, event.error));
+        if (!validation.valid) {
+            // Hard rejection: no `conflict` flag, because re-driving the same request fails the same
+            // way and would only burn the writer's retry budget.
+            return {
+                success: false,
+                reason: validation.reason ?? 'Transaction validation failed'
+            };
         }
         const blockIds = blockIdsForTransforms(request.transforms);
         log('pend actionId=%s blockIds=%d rev=%s', request.actionId, blockIds.length, request.rev);
@@ -382,6 +413,11 @@ export class StorageRepo {
         // requested one — reported as StaleFailure.staleAt so a losing writer learns the number
         // instead of parsing prose. Confirmed-local only: we read it from our own storage below.
         let staleAt;
+        // Blocks this action ALREADY committed at exactly the requested revision — the durable half
+        // of a torn action whose retry reuses the same actionId. Sibling of the `alreadyDone`
+        // partition in `commit` below: satisfied, not merely non-stale, so no pending is recorded
+        // for them (see the fan-out at the end of this method).
+        const satisfied = new Set();
         // Potential race condition: A concurrent commit operation could complete
         // between the conflict checks (latest.rev, listPendingTransactions) and the
         // savePendingTransaction call below. This pend operation might succeed based on
@@ -392,12 +428,22 @@ export class StorageRepo {
         for (const blockId of blockIds) {
             const blockStorage = this.createBlockStorage(blockId);
             const transforms = transformForBlockId(request.transforms, blockId);
-            // First handle any pending actions
-            const pending = await asyncIteratorToArray(blockStorage.listPendingTransactions());
-            pendings.push(...pending.map(actionId => ({ blockId, actionId })));
-            // Handle any conflicting revisions
+            // Handle any conflicting revisions FIRST: a block this same action already committed at
+            // exactly the requested revision is satisfied, and skips both this check and the
+            // pending-action listing below.
             if (request.rev !== undefined || transforms.insert) {
                 const latest = await blockStorage.getLatest();
+                // Our own already-durable work, met again by a retry (see {@link isOwnRevision}):
+                // treating it as a stale rival would refuse the writer with its own commit.
+                // NOTE: a rev-less pend (`request.rev === undefined`, an insert-only claim) can
+                // never match, so a torn action retried WITHOUT a revision is still refused by its
+                // own insert. No production caller sends one — `TransactorSource.transact` and the
+                // multi-collection coordinator both require a rev — so this is unreachable today;
+                // if a rev-less write path ever appears, match on `latest.actionId` alone here.
+                if (isOwnRevision(latest, request.rev, request.actionId)) {
+                    satisfied.add(blockId);
+                    continue;
+                }
                 if (latest && latest.rev >= (request.rev ?? 0)) {
                     // Only a real revision race yields a meaningful `staleAt`. When `request.rev` is
                     // undefined this same branch fires for an insert collision (the comparison degrades
@@ -420,6 +466,9 @@ export class StorageRepo {
                     }
                 }
             }
+            // Then handle any pending actions
+            const pending = await asyncIteratorToArray(blockStorage.listPendingTransactions());
+            pendings.push(...pending.map(actionId => ({ blockId, actionId })));
         }
         if (missing.length) {
             log('pend:stale actionId=%s missing=%d', request.actionId, missing.length);
@@ -454,10 +503,24 @@ export class StorageRepo {
         // Note: that this is not atomic, after we checked for conflicts and pending actions
         // new pending or committed actions may have been added.  This is okay, because
         // this check during pend is conservative.
-        await Promise.all(blockIds.map(blockId => {
+        //
+        // Each block's pending write runs under THAT block's write latch, one latch per branch and
+        // never nested: savePendingTransaction seeds the block's metadata blob when it has none, and
+        // an unlatched seed racing a concurrent commit/replica on a fresh block erases the `latest`
+        // the other writer just landed. Never more than one block latch is held by a branch, so this
+        // cannot deadlock against commit's sorted multi-latch acquisition.
+        //
+        // `satisfied` blocks are skipped: `commit`'s `alreadyDone` arm skips `internalCommit`, the
+        // only thing that promotes (and thereby removes) a pending record, so a pending saved here
+        // would never clear — a permanent durable reservation that the rival-pending checks (this
+        // method's listPendingTransactions scan, and `ClusterMember.validatePendOperations`) refuse
+        // every future writer against, a worse wedge than the one this carve-out fixes. They still
+        // ride in the returned `blockIds` so `cancel` covers them (deleting an absent pending is a
+        // no-op that writes no metadata).
+        await Promise.all(blockIds.filter(blockId => !satisfied.has(blockId)).map(blockId => {
             const blockStorage = this.createBlockStorage(blockId);
             const blockTransform = transformForBlockId(request.transforms, blockId);
-            return blockStorage.savePendingTransaction(request.actionId, blockTransform);
+            return withBlockWriteLatch(blockId, latch => blockStorage.savePendingTransaction(request.actionId, blockTransform, latch));
         }));
         return {
             success: true,
@@ -469,11 +532,11 @@ export class StorageRepo {
         log('cancel actionId=%s blockIds=%d', actionRef.actionId, actionRef.blockIds.length);
         await Promise.all(actionRef.blockIds.map(blockId => {
             const blockStorage = this.createBlockStorage(blockId);
-            return blockStorage.deletePendingTransaction(actionRef.actionId);
+            return withBlockWriteLatch(blockId, latch => blockStorage.deletePendingTransaction(actionRef.actionId, latch));
         }));
     }
     /**
-     * Commit a previously-pended action across its blocks, under the per-block commit latches.
+     * Commit a previously-pended action across its blocks, under the block write latches.
      *
      * **Divergence vs genuine fault.** When the batch cannot be completed, the reason decides what
      * happens to the pending records the pend left behind. `ClusterMember.applyConsensusOperation`
@@ -500,10 +563,12 @@ export class StorageRepo {
      * its `cancel` therefore still strands the record; that is pre-existing and orthogonal to the
      * divergence split above.
      */
-    async commit(request, _options) {
+    async commit(request, _options, proof) {
         log('commit actionId=%s rev=%d blockIds=%d', request.actionId, request.rev, request.blockIds.length);
-        const uniqueBlockIds = Array.from(new Set(request.blockIds)).sort();
-        const releases = [];
+        // Deduped ONCE, in request order — the order blocks are committed and reported in. The latches
+        // are acquired in sorted order by `acquireBlockWriteLatches` over this same set, so every
+        // `latches.get(blockId)!` below resolves.
+        const blockIds = Array.from(new Set(request.blockIds));
         // Collects the blocks newly committed in this call, grouped by collection,
         // so we can emit change events once locks are released. Blocks that land before
         // a mid-loop failure stay here and are still emitted (Option A — emit eagerly):
@@ -512,17 +577,16 @@ export class StorageRepo {
         // Captured when internalCommit throws mid-loop; we break (rather than return)
         // so locks release and accumulated landings still emit before we report failure.
         let failure;
+        // Every block's token is kept so each write below can prove it runs inside that block's latch.
+        const { latches, release } = await acquireBlockWriteLatches(blockIds);
         try {
-            // Acquire locks sequentially based on sorted IDs to prevent deadlocks
-            for (const id of uniqueBlockIds) {
-                const lockId = commitLatchKey(id);
-                const release = await Latches.acquire(lockId);
-                releases.push(release);
-            }
             // --- Start of Critical Section ---
-            const blockStorages = request.blockIds.map(blockId => ({
+            // Request order, deduped (NOT the sorted acquisition order): the order here is the order
+            // blocks are committed and reported in change events, which callers may observe.
+            const blockStorages = blockIds.map(blockId => ({
                 blockId,
-                storage: this.createBlockStorage(blockId)
+                storage: this.createBlockStorage(blockId),
+                latch: latches.get(blockId)
             }));
             // Partition blocks into:
             //   - alreadyDone: latest.rev === request.rev && latest.actionId === request.actionId
@@ -538,11 +602,15 @@ export class StorageRepo {
             // so it never seeds this.
             let staleAt;
             for (const entry of blockStorages) {
-                const { blockId, storage } = entry;
+                const { blockId, storage, latch } = entry;
                 const latest = await storage.getLatest();
                 if (latest && latest.rev >= request.rev) {
-                    if (latest.rev === request.rev && latest.actionId === request.actionId) {
+                    if (isOwnRevision(latest, request.rev, request.actionId)) {
                         // Idempotent no-op for this block — already committed with this exact (actionId, rev).
+                        // A retry can carry a proof the original commit lacked (or crashed before writing):
+                        // back-fill it, strictly additively, under the same digest-match retention rule the
+                        // original commit applies. Runs inside the latched critical section.
+                        await this.backFillProof(blockId, storage, request.rev, request.actionId, proof, latch);
                         continue;
                     }
                     staleAt = highestStaleAt([staleAt, { blockId, rev: latest.rev }]);
@@ -581,7 +649,7 @@ export class StorageRepo {
             //     is gone. getTransaction(actionId) returns the promoted transform. Self-heal here
             //     via storage.recover() (redoes the lost setLatest, advancing latest to the highest
             //     contiguous promoted rev, >= request.rev). recover() is idempotent + monotonic, so
-            //     calling it under the already-held commit latch is safe. Recovered blocks are then
+            //     calling it under the already-held block write latch is safe. Recovered blocks are then
             //     excluded from the internalCommit loop below — their pending is gone, so
             //     internalCommit would throw.
             //   - Genuine missing pend: the action was never promoted (getTransaction → undefined),
@@ -589,7 +657,7 @@ export class StorageRepo {
             // Crash-D2 never reaches this branch: its pending record is still present.
             const missingPends = [];
             const recovered = new Set();
-            for (const { blockId, storage } of toCommit) {
+            for (const { blockId, storage, latch } of toCommit) {
                 const pendingAction = await storage.getPendingTransaction(request.actionId);
                 if (pendingAction) {
                     continue;
@@ -600,7 +668,7 @@ export class StorageRepo {
                     continue;
                 }
                 // Crash-D3 signature (pending absent + action durably promoted). Redo the lost setLatest.
-                const result = await storage.recover();
+                const result = await storage.recover(latch);
                 if (result.latest !== undefined && result.latest.rev >= request.rev) {
                     recovered.add(blockId);
                 }
@@ -635,7 +703,7 @@ export class StorageRepo {
             // internalCommit does — otherwise a recovered delete would silently fail to wake watchers.
             // Only when neither resolves (a delete-only block with no prior materialization) is the
             // emit skipped, the same terminal fallback internalCommit uses.
-            for (const { blockId, storage } of toCommit) {
+            for (const { blockId, storage, latch } of toCommit) {
                 if (!recovered.has(blockId)) {
                     continue;
                 }
@@ -646,6 +714,10 @@ export class StorageRepo {
                     list.push(blockId);
                     collectionBlocks.set(collectionId, list);
                 }
+                // The recovered block IS committed at request.rev, but it is excluded from the
+                // internalCommit loop below — so without this it would be the one landing path that
+                // never retains the cohort's proof, even though this very call is carrying it.
+                await this.backFillProof(blockId, storage, request.rev, request.actionId, proof, latch);
             }
             // Commit the action for each block that still needs it.
             // This loop will execute atomically for all blocks due to the acquired locks.
@@ -655,13 +727,13 @@ export class StorageRepo {
             // Set when the mid-loop failure was a divergence rather than a genuine fault — the split
             // documented on commit() above, which decides the fate of the batch's pending records.
             let divergentFailure = false;
-            for (const { blockId, storage } of toCommit) {
+            for (const { blockId, storage, latch } of toCommit) {
                 if (recovered.has(blockId)) {
                     continue;
                 }
                 try {
                     // internalCommit will throw if it encounters an issue
-                    const collectionId = await this.internalCommit(blockId, request.actionId, request.rev, storage);
+                    const collectionId = await this.internalCommit(blockId, request.actionId, request.rev, storage, latch, proof);
                     if (collectionId !== undefined) {
                         const list = collectionBlocks.get(collectionId) ?? [];
                         list.push(blockId);
@@ -689,8 +761,8 @@ export class StorageRepo {
             }
         }
         finally {
-            // Release locks in reverse order of acquisition
-            releases.reverse().forEach(release => release());
+            // Releases every block latch, in reverse acquisition order.
+            release();
         }
         // Notify after the critical section, for every block newly committed here —
         // including those that landed before a mid-loop failure (alreadyDone / stale
@@ -723,9 +795,9 @@ export class StorageRepo {
             return;
         }
         log('commit:drop-unpromotable-pendings actionId=%s blockIds=%d', actionId, blocks.length);
-        await Promise.all(blocks.map(async ({ blockId, storage }) => {
+        await Promise.all(blocks.map(async ({ blockId, storage, latch }) => {
             try {
-                await storage.deletePendingTransaction(actionId);
+                await storage.deletePendingTransaction(actionId, latch);
             }
             catch (err) {
                 log('commit:drop-unpromotable-pending-failed blockId=%s actionId=%s error=%s', blockId, actionId, err instanceof Error ? err.message : String(err));
@@ -742,13 +814,13 @@ export class StorageRepo {
     async recoverBlock(blockId) {
         log('recoverBlock blockId=%s', blockId);
         const storage = this.createBlockStorage(blockId);
-        // Hold the per-block commit latch: recover() is a read-modify-write of meta.latest that
-        // blindly writes back the metadata object it read, so its "advance only" guard is TOCTOU —
-        // racing a concurrent commit()/saveReplicatedBlock that advanced latest in between would
-        // clobber it (a non-monotonic regression). Same latching invariant as every other
-        // latest-mutating site. commit() calls storage.recover() directly under its own held latch,
-        // so it never routes through here — no double-acquire / deadlock.
-        await withBlockCommitLatch(blockId, () => storage.recover());
+        // Hold the block write latch: recover() is a read-modify-write of the metadata blob that
+        // blindly writes back the object it read, so its "advance only" guard is TOCTOU — racing a
+        // concurrent commit()/saveReplicatedBlock that advanced latest in between would clobber it
+        // (a non-monotonic regression). Same latching invariant as every other metadata writer.
+        // commit() calls storage.recover(latch) directly under its own held latch, so it never
+        // routes through here — no double-acquire / deadlock.
+        await withBlockWriteLatch(blockId, latch => storage.recover(latch));
     }
     /**
      * Persist a replica of a block received out-of-band (churn re-replication) into
@@ -756,20 +828,34 @@ export class StorageRepo {
      * already materialized from a departing owner, not as a pend/commit. See
      * {@link IBlockStorage.saveReplica} for the durability/monotonicity contract.
      *
-     * Held under the same `StorageRepo.commit:<id>` latch as {@link commit} so the
-     * replica's read-modify-write of `latest` is mutually exclusive with a concurrent
+     * Held under the same block write latch as {@link commit} so the replica's
+     * read-modify-write of the metadata blob is mutually exclusive with a concurrent
      * local commit on the same block — otherwise `saveReplica`'s monotonic guard could
      * read a stale `latest` and clobber a commit that advanced it in between.
+     *
+     * `verifiedProof` is retained when supplied: both the reconcile path
+     * (`cluster/reconcile-block.ts`) and the certified push path (`BlockTransferService.handlePush`)
+     * pass the {@link BlockCommitProof} they verified against these exact bytes (`certifyContent`'s
+     * digest check), so a repaired replica serves the proof onward and certification no longer decays
+     * across repair hops.
+     *
+     * When the push does NOT advance `latest` (this node already holds that revision), `saveReplica`
+     * is a no-op and persists nothing — so the proof is back-filled here instead, through
+     * {@link backFillProof}'s digest-match rule. It is deliberately NOT persisted inside
+     * `saveReplica`: the proof was verified against the PUSHED bytes, while a back-fill attaches it to
+     * this node's HELD materialization, and a diverged holder's bytes at the same `(rev, actionId)`
+     * may differ. Storing a proof whose declared digest contradicts local content would make this node
+     * serve content that fails its own proof — `digest-mismatch` is an ATTRIBUTABLE fault in
+     * `certified-claims.ts`, so every receiver would penalize it.
      */
-    async saveReplicatedBlock(blockId, block, source) {
+    async saveReplicatedBlock(blockId, block, source, verifiedProof) {
         log('saveReplicatedBlock blockId=%s rev=%s', blockId, source?.rev);
         const storage = this.createBlockStorage(blockId);
-        const release = await Latches.acquire(commitLatchKey(blockId));
         // Captured under the latch; emitted after release to match commit's ordering.
         let landed;
-        try {
+        await withBlockWriteLatch(blockId, async (latch) => {
             const priorLatest = await storage.getLatest();
-            const effective = await storage.saveReplica(block, source);
+            const effective = await storage.saveReplica(block, source, verifiedProof, latch);
             // Advanced iff there was no prior revision or the effective rev moved past it. On the
             // monotonic no-op, saveReplica returns the held latest unchanged → effective.rev === priorLatest.rev.
             const advanced = priorLatest === undefined || effective.rev > priorLatest.rev;
@@ -777,10 +863,31 @@ export class StorageRepo {
             if (advanced && collectionId !== undefined) {
                 landed = { collectionId, actionId: effective.actionId, rev: effective.rev };
             }
-        }
-        finally {
-            release();
-        }
+            if (!advanced && verifiedProof !== undefined && source !== undefined
+                && effective.rev === source.rev && effective.actionId === source.actionId) {
+                // The push named exactly the revision this node already holds, and carried a verified
+                // proof for it. Back-fill so a proof-lessly-landed revision stops being corroboration-only
+                // the moment valid evidence for it arrives. Requires agreement on BOTH rev and actionId:
+                // same rev under a different action is a divergence, not the same revision.
+                //
+                // A held revision NEWER than the pushed one is deliberately not back-filled: `servableProof`
+                // only ever serves the proof for `latest.rev`, so the proof would be keyed to a revision
+                // this node will never serve, for content it may not even materialize.
+                //
+                // Runs under the block write latch already held here — the same latch the commit-path
+                // back-fill sites hold, so no new latch interaction. `backFillProof` never throws: the
+                // revision is already durable, and a proof-persist fault must not turn a no-op into a
+                // failure.
+                //
+                // NOTE: once a proof IS retained this costs one key lookup per duplicate push
+                // (`backFillProof` returns before materializing). A holder whose bytes diverge from the
+                // cohort's never retains one, so it re-materializes and re-hashes the block on EVERY
+                // certified push of that revision. Bounded by push frequency and fine at spread-on-churn
+                // rates; if a diverged holder under repeated push ever shows up in a profile, remember the
+                // withheld `(rev, actionId)` and skip the re-check.
+                await this.backFillProof(blockId, storage, effective.rev, effective.actionId, verifiedProof, latch);
+            }
+        });
         // Replica-persist has no CommitRequest, hence no tailId — like a read-driven promotion,
         // this wakes local onCollectionChange watchers but is cert-gated out of cohort-topic
         // re-origination downstream (change-bridge selfIsCohortMember treats a tail-less event as
@@ -789,12 +896,104 @@ export class StorageRepo {
             this.emitCollectionChanges(new Map([[landed.collectionId, [blockId]]]), landed.actionId, landed.rev);
         }
     }
-    async internalCommit(blockId, actionId, rev, storage) {
-        // Note: This method is called under the per-block commit latch — by commit() (within its
-        // locked critical section) and by the read-driven promotion in get() (which now takes the
-        // same latch). So, operations like getPendingTransaction, getLatest, getBlock,
-        // saveMaterializedBlock, saveRevision, promotePendingTransaction, setLatest are protected
-        // against concurrent commits for the *same blockId*.
+    /**
+     * The digest the block WOULD materialize to if `actionId`'s pending transform committed at `rev`,
+     * plus the base revision it was computed from. Read-only: touches no durable state and takes no
+     * block write latch.
+     *
+     * Mirrors {@link internalCommit}'s reads (pending transform → latest → base → applyTransform) so
+     * the prediction and the eventual commit cannot drift. Consumed by the cluster member's
+     * promise-round content-digest check (`ClusterMember.validateCommitOperations`), which compares it
+     * against the digest the transaction author declared on the commit request.
+     *
+     * Deliberately does NOT take the block write latch: this runs on the vote path, ahead of the
+     * commit that will take it, so taking it here would serialize voting behind commits and risks
+     * deadlocking against commit's sorted up-front multi-block latch acquisition. The price is that a
+     * concurrent commit can move `latest` mid-preview; the caller's checkable rule (base-independent,
+     * or `baseRev` agreement) makes a torn read at worst an abstain, never a false reject of honest
+     * content.
+     *
+     * `rev` is accepted for parity/logging with the commit that would follow; materialization does not
+     * depend on it (internalCommit only records it).
+     *
+     * Returns `undefined` when this node holds no pending transform for the action (it never saw the
+     * pend) — distinct from a defined preview with `digest: undefined` (see {@link CommitDigestPreview}).
+     */
+    async previewCommitDigest(blockId, actionId, rev) {
+        const storage = this.createBlockStorage(blockId);
+        const transform = await storage.getPendingTransaction(actionId);
+        if (!transform) {
+            return undefined;
+        }
+        // An insert replaces the block wholesale before updates apply, so the result is the same on
+        // every member no matter what base it holds — do not read a base at all (the block may even be
+        // locally wedged/unmaterializable, which must not degrade a base-independent preview).
+        const baseIndependent = transform.insert !== undefined;
+        let base;
+        let baseRev;
+        if (!baseIndependent) {
+            const latest = await storage.getLatest();
+            if (latest) {
+                baseRev = latest.rev;
+                try {
+                    base = (await storage.getBlock(latest.rev))?.block;
+                }
+                catch (err) {
+                    // This node holds a `latest` it cannot materialize (see readCommitBase). That is a
+                    // local deficiency, not a content mismatch — report "cannot check" so the caller
+                    // abstains. Unlike the commit path's refuseMissingBase, this must NOT delete the
+                    // pending record or throw: preview is read-only and runs before any commit exists.
+                    log('previewCommitDigest:unmaterializable-base blockId=%s baseRev=%d rev=%d error=%s', blockId, latest.rev, rev, err instanceof Error ? err.message : String(err));
+                    return { baseIndependent: false, baseRev, digest: undefined };
+                }
+            }
+        }
+        // Clone both: applyTransform assigns `transform.insert` into the result by reference and
+        // applyOperations mutates the block in place, so materializing on live storage/pending objects
+        // would corrupt them for the real commit that follows.
+        const newBlock = applyTransform(structuredClone(base), structuredClone(transform));
+        // `undefined` covers the tombstone (delete transform) and updates-with-no-base (applyTransform
+        // drops updates when there is no block to apply them to) — both materialize nothing.
+        const digest = newBlock ? await canonicalBlockHash(newBlock) : undefined;
+        return { digest, baseRev, baseIndependent };
+    }
+    /**
+     * See {@link IRevisionActionReader}. Reads the block's revision index directly
+     * (`listRevisions(rev, rev)` — both bounds inclusive per the `IBlockStorage` contract); an empty
+     * range means this node holds no record for that revision.
+     */
+    async getRevisionAction(blockId, rev) {
+        const storage = this.createBlockStorage(blockId);
+        for await (const actionRev of storage.listRevisions(rev, rev)) {
+            return actionRev.actionId;
+        }
+        return undefined;
+    }
+    /**
+     * The {@link BlockCommitProof} this node retained for `blockId` at `rev`, or `undefined` when it
+     * kept none — a revision committed before proofs were persisted, a member whose materialization
+     * diverged from the declared digest (see {@link persistProofIfContentMatches}), or simply a
+     * revision this node never landed.
+     *
+     * Public because a peer answering a block-repair fetch serves the proof alongside the revision
+     * (`serveBlockArchive`), which is the only way a requester can check a lone holder's claim
+     * without a second holder to corroborate it. Read-only and unlatched: a proof is written once
+     * and never mutated, so a concurrent commit can only make this return a proof for a revision
+     * that just became stale — which the caller pairs with the revision it actually read.
+     */
+    async getBlockProof(blockId, rev) {
+        return await this.createBlockStorage(blockId).getBlockProof(rev);
+    }
+    async internalCommit(blockId, actionId, rev, storage, latch, proof) {
+        // Note: This method is called under the block write latch — by commit() (within its locked
+        // critical section) and by the read-driven promotion in get() (which takes the same latch);
+        // `latch` is the proof of that. So, operations like getPendingTransaction, getLatest,
+        // getBlock, saveMaterializedBlock, saveRevision, promotePendingTransaction, setLatest are
+        // protected against concurrent writers for the *same blockId*.
+        //
+        // `getBlock` here (via readCommitBase) is LOCAL-ONLY: the commit path never fetches from a
+        // peer while holding N block latches. A coverage gap reads as a missing base, which the
+        // healing path repairs by replication instead.
         const transform = await storage.getPendingTransaction(actionId);
         // No need to check if !transform here, as the caller (commit) already verified this.
         // If it's null here, it indicates a logic error or race condition bypassed the lock (unlikely).
@@ -803,7 +1002,7 @@ export class StorageRepo {
         }
         // Get prior materialized block if it exists
         const latest = await storage.getLatest();
-        const priorBlock = await this.readCommitBase(blockId, actionId, rev, storage, latest);
+        const priorBlock = await this.readCommitBase(blockId, actionId, rev, storage, latest, latch);
         // Apply transform and save materialized block
         // applyTransform handles undefined priorBlock correctly for inserts
         const newBlock = applyTransform(priorBlock, transform);
@@ -815,24 +1014,32 @@ export class StorageRepo {
         // target, and with no prior revision there is none. With a prior `latest` an absent newBlock is
         // a legitimate tombstone (the walk resolves to an earlier materialization), so it stays allowed.
         if (!newBlock && latest === undefined) {
-            return await this.refuseMissingBase(blockId, actionId, rev, storage, 'no committed revision to apply the transform to');
+            return await this.refuseMissingBase(blockId, actionId, rev, storage, latch, 'no committed revision to apply the transform to');
         }
         if (newBlock) {
-            await storage.saveMaterializedBlock(actionId, newBlock);
+            await storage.saveMaterializedBlock(actionId, newBlock, latch);
         }
         // Save revision and promote action *before* updating latest
         // This ensures that if the process crashes between these steps,
         // the 'latest' pointer doesn't point to a revision that hasn't been fully recorded.
-        await storage.saveRevision(rev, actionId);
-        await storage.promotePendingTransaction(actionId);
+        await storage.saveRevision(rev, actionId, latch);
+        await storage.promotePendingTransaction(actionId, latch);
         // Update latest revision *last*
-        await storage.setLatest({ actionId, rev });
+        await storage.setLatest({ actionId, rev }, latch);
+        // Persist the cohort's commit proof AFTER the commit is durably latest — the proof is
+        // evidence about a landed revision, never a precondition of landing it. The retention rule
+        // (persist only when the LOCAL materialization matches the digest the commit op declared)
+        // and its failure logging live in the shared helper; a proof-persist fault must not fail a
+        // commit that already landed, so the helper never throws.
+        if (proof !== undefined) {
+            await this.persistProofIfContentMatches(blockId, actionId, rev, storage, proof, newBlock, latch);
+        }
         // Prune the now-superseded prior materialization (checkpoint retention). Runs LAST — after the
         // new rev's materialization + revision + transform + setLatest are all durable — so no crash
         // point can leave a rev unrecoverable: a crash BEFORE this leaves a redundant (harmless)
         // materialization the next commit's prune reclaims; a crash AFTER is fully consistent. The prune
         // only ever deletes a materialization reconstructible from the retained floor + transforms. Runs
-        // under the per-block commit latch already held here, so it serializes against concurrent commits.
+        // under the block write latch already held here, so it serializes against concurrent commits.
         // NOTE: prune targets ONLY the immediate prior. A crash between setLatest and this call leaves that
         // one prior materialization un-pruned; since a later commit prunes ITS OWN prior (never the earlier
         // leaked rev), that copy is NOT auto-reclaimed — a bounded (≤1 block-copy per crash), harmless leak
@@ -840,7 +1047,7 @@ export class StorageRepo {
         // add a bounded look-back (prune non-retained mats in [rev-checkpointInterval, rev)) here, or a
         // periodic reconciliation sweep — do NOT reintroduce a per-read re-cache.
         if (latest !== undefined) {
-            await storage.pruneSupersededMaterialization(latest);
+            await storage.pruneSupersededMaterialization(latest, latch);
         }
         // Report the affected collection for change-event routing. For a delete the
         // materialized block is undefined, so fall back to the prior block's header.
@@ -848,6 +1055,81 @@ export class StorageRepo {
         // undefined so the caller skips it rather than emitting a bogus event.
         return newBlock?.header.collectionId ?? priorBlock?.header.collectionId;
     }
+    /**
+     * Retain `proof` for a block this call found ALREADY committed at `(rev, actionId)` — the paths
+     * that land (or find already landed) a revision without running {@link internalCommit}, and would
+     * otherwise never retain a proof: the idempotent re-commit partition, the Crash-D3 `recover()`
+     * partition, and {@link saveReplicatedBlock}'s monotonic no-op on a certified push. Strictly
+     * additive: an existing proof is left alone, and the same digest-match rule as the fresh-commit
+     * site decides retention.
+     *
+     * `rev`/`actionId` are passed separately rather than as a `CommitRequest` because the replica
+     * caller has no commit request — it has the `(rev, actionId)` the push and the held revision
+     * agree on.
+     *
+     * Callers must hold the block's write latch (`latch`). `getBlock` is local-only and can throw on
+     * an unmaterializable or uncovered base — treated as "no local content", i.e. the proof is withheld.
+     */
+    async backFillProof(blockId, storage, rev, actionId, proof, latch) {
+        if (proof === undefined || await storage.getBlockProof(rev) !== undefined) {
+            return;
+        }
+        let committedBlock;
+        try {
+            committedBlock = (await storage.getBlock(rev))?.block;
+        }
+        catch {
+            committedBlock = undefined;
+        }
+        await this.persistProofIfContentMatches(blockId, actionId, rev, storage, proof, committedBlock, latch);
+    }
+    /**
+     * The single retention rule for {@link BlockCommitProof}s, shared by the fresh-commit site
+     * ({@link internalCommit}, after `setLatest`) and the already-landed back-fill
+     * ({@link backFillProof}):
+     *
+     * > **A member persists the proof only when its own materialization matches the digest the
+     * > commit operation declared for this block.**
+     *
+     * One rule covers every awkward case without a second flag: a DIVERGED member (committed onto a
+     * lagging base) computes a different hash, stores no proof, and falls back to corroboration
+     * exactly as today — the `commit:proof-digest-mismatch` log line is also the first signal this
+     * system has ever had that a member diverged. A member that abstained at vote time still checks
+     * here (by commit time it HAS materialized) and legitimately keeps the proof on agreement. A
+     * tombstone (no `block`) and a commit with no `blockDigests` (pre-upgrade client) declare no
+     * digest and store no proof (`commit:proof-undeclared`).
+     *
+     * Never throws: the commit this proof describes already durably landed, so a proof-persist
+     * fault must not turn `commit()` into `success:false` for a landed commit — it is logged and
+     * the proof simply is not retained (repair falls back to corroboration).
+     *
+     * NOTE: one commit of N blocks stores the SAME proof under each block's `(blockId, rev)` proofs-store key, and
+     * the proof itself carries the commit op's N `blockIds`/`blockDigests` — so bytes retained per
+     * commit grow with N². Measured base cost is ~4.6 KB for a 10-peer 2-block commit
+     * (`test/commit-proof.spec.ts` "size"), and nothing today bounds `CommitRequest.blockIds`. Fine
+     * at the handful-of-blocks batches the transactor produces now; if per-coordinator batches ever
+     * grow large, store the proof once under its `messageHash` and key each revision to a pointer.
+     */
+    async persistProofIfContentMatches(blockId, actionId, rev, storage, proof, block, latch) {
+        try {
+            const declaredDigest = proofDeclaredDigest(proof, { blockId, rev, actionId });
+            if (declaredDigest === undefined) {
+                log('commit:proof-undeclared blockId=%s rev=%d actionId=%s', blockId, rev, actionId);
+                return;
+            }
+            // A digest was declared but this node materialized nothing (tombstone / unmaterializable
+            // read on the back-fill path): the local content provably is not the declared content.
+            const localDigest = block === undefined ? undefined : await canonicalBlockHash(block);
+            if (localDigest !== declaredDigest) {
+                log('commit:proof-digest-mismatch blockId=%s rev=%d actionId=%s declared=%s local=%s', blockId, rev, actionId, declaredDigest, localDigest);
+                return;
+            }
+            await storage.saveBlockProof(rev, proof, latch);
+        }
+        catch (err) {
+            log('commit:proof-persist-failed blockId=%s rev=%d actionId=%s error=%s', blockId, rev, actionId, err instanceof Error ? err.message : String(err));
+        }
+    }
     /**
      * The materialization this commit builds on: the block at `latest`, or `undefined` when the block
      * holds no committed revision yet (the normal insert case).
@@ -871,16 +1153,37 @@ export class StorageRepo {
      * Narrowing this would require typed faults out of BlockStorage; until then, prefer the tolerant
      * reading.
      */
-    async readCommitBase(blockId, actionId, rev, storage, latest) {
+    async readCommitBase(blockId, actionId, rev, storage, latest, latch) {
         if (!latest) {
             return undefined;
         }
+        // NOTE: this read is deliberately LOCAL-ONLY and does not heal. `getBlock` no longer restores
+        // from a peer (that moved to the explicit `restoreRevision`, which `StorageRepo.get` calls), so a
+        // base this node cannot materialize locally raises {@link MissingBaseRevisionError} here instead
+        // of being fetched in line. The reason is the calling context, not the cost of a fetch: `commit`
+        // holds the write latch of EVERY block in the batch across this call, and network I/O inside that
+        // critical section makes one unreachable peer stall every writer of every block in the batch for
+        // the length of a round trip. Healing is out-of-band instead — cohort reconcile supplies the
+        // revision (`ClusterMember` → `saveReplicatedBlock`) and the action is retried, by which point
+        // this read succeeds locally. Pinned by `test/storage-repo.spec.ts` "commit reads its base
+        // locally", which wires a restore callback that would have answered and asserts it is never
+        // called. Do not reintroduce a restore on this path; if a commit ever genuinely needs one, fetch
+        // BEFORE taking the latches, not underneath them.
+        //
+        // NOTE: `latest.rev` is always inside `meta.ranges` today — every writer of `latest`
+        // (`setLatest`, `saveForwardRevision`, `recover`) merges an open-ended range anchored at or
+        // below the new latest in the same `saveMetadata` — so the RevisionNotCoveredError arm below
+        // is unreachable from here and only truncated-history corruption lands in the catch. If a
+        // future change can leave `latest` uncovered, the ordering in `get` becomes load-bearing: the
+        // read-driven promotion runs BEFORE `readBlockHealing`, so a coverage gap under `latest` would
+        // make `refuseMissingBase` delete the pending record moments before the healing read would
+        // have restored it. Heal before refusing if that day comes.
         try {
             return (await storage.getBlock(latest.rev))?.block;
         }
         catch (err) {
             log('commit:unmaterializable-base blockId=%s baseRev=%d error=%s', blockId, latest.rev, err instanceof Error ? err.message : String(err));
-            return await this.refuseMissingBase(blockId, actionId, rev, storage, `local rev ${latest.rev} is not materializable here`);
+            return await this.refuseMissingBase(blockId, actionId, rev, storage, latch, `local rev ${latest.rev} is not materializable here`);
         }
     }
     /**
@@ -893,8 +1196,8 @@ export class StorageRepo {
      * so a commit retry partitions the block as already-done/stale and never revisits the pending.
      * Leaving it would also report a phantom conflicting action from {@link pend} for every later write.
      */
-    async refuseMissingBase(blockId, actionId, rev, storage, detail) {
-        await storage.deletePendingTransaction(actionId);
+    async refuseMissingBase(blockId, actionId, rev, storage, latch, detail) {
+        await storage.deletePendingTransaction(actionId, latch);
         log('commit:missing-base blockId=%s rev=%d actionId=%s detail=%s', blockId, rev, actionId, detail);
         throw new MissingBaseRevisionError(blockId, rev, detail);
     }