@optimystic/db-p2p 0.17.0 → 0.19.0

package/dist/src/repo/coordinator-repo.js

@@ -1,14 +1,17 @@
-import { LruMap, blockIdsForTransforms, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
-import { ClusterCoordinator } from "./cluster-coordinator.js";
+import { LruMap, blockIdsForTransforms, highestStaleAt, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
+import { ClusterCoordinator, ValidatorRejectionError } from "./cluster-coordinator.js";
 import { peerIdFromString } from "@libp2p/peer-id";
 import { createLogger } from '../logger.js';
 import { PenaltyReason } from "../reputation/types.js";
-import { quorumSize, selectQuorumRev } from "../cluster/quorum-restore.js";
+import { quorumSize, corroboratorCapacity, selectQuorumRev } from "../cluster/quorum-restore.js";
 import { RECONCILE_TIMEOUT_MS } from "../cluster/reconcile-block.js";
+import { isMissingBaseRevisionFailure, MISSING_BASE_REVISION_REASON } from "../storage/storage-repo.js";
 const log = createLogger('coordinator-repo');
+/** How long one cohort peer gets to answer the latest-revision consult before it counts as silent. */
+const LATEST_QUERY_TIMEOUT_MS = 1000;
 /** True when a freshly-read local revision is strictly ahead of the baseline the repair started from. */
-function isAdvanceOver(rev, baseline) {
-    return typeof rev === 'number' && (baseline === undefined || rev > baseline.rev);
+function isAdvanceOver(rev, baselineRev) {
+    return typeof rev === 'number' && (baselineRev === undefined || rev > baselineRev);
 }
 /**
  * Reject if `promise` has not settled within `ms`. The timer is cleared on either outcome, so no
@@ -45,8 +48,13 @@ export class CoordinatorRepo {
     readRepairSampleRate;
     /** Simple-majority threshold from the consensus policy; drives the read-repair corroboration quorum. */
     simpleMajorityThreshold;
-    /** Configured full cluster size; the operator's declaration of how many corroborators should exist. */
-    clusterSize;
+    /**
+     * Yardstick the read-repair corroboration floor is measured against; the floor for
+     * {@link corroboratorCapacity}. Resolved by `resolveClusterPolicy` for a real node; falls back to
+     * `assumedClusterSize` and then `clusterSize` for direct constructors (see the constructor), so a
+     * caller that has adopted neither field keeps today's behavior exactly.
+     */
+    repairCorroborationClusterSize;
     /** Resolved super-majority threshold the coordinator commits on (mirrors the value handed to ClusterCoordinator). */
     superMajorityThreshold;
     reputation;
@@ -63,6 +71,7 @@ export class CoordinatorRepo {
         this.localPeerId = localPeerId;
         const policy = {
             clusterSize: cfg?.clusterSize ?? 10,
+            assumedClusterSize: cfg?.assumedClusterSize,
             superMajorityThreshold: cfg?.superMajorityThreshold ?? DEFAULT_SUPER_MAJORITY_THRESHOLD,
             simpleMajorityThreshold: cfg?.simpleMajorityThreshold ?? 0.51,
             minAbsoluteClusterSize: cfg?.minAbsoluteClusterSize ?? 3,
@@ -87,7 +96,15 @@ export class CoordinatorRepo {
         this.readRepairSampleRate = policy.readRepairSampleRate;
         this.simpleMajorityThreshold = policy.simpleMajorityThreshold;
         this.superMajorityThreshold = policy.superMajorityThreshold;
-        this.clusterSize = policy.clusterSize;
+        // Unlike the membership admission gate (which treats an absent assumedClusterSize as "unknown"
+        // and admits — refusing writes outright is unacceptable), this falls back to the replication
+        // factor and stays strict: the failure mode of getting this wrong is a block that goes
+        // unrepaired, degraded rather than dead, so there is no reason to relax it for a caller that
+        // has not adopted the new field. A real node is handed an explicit
+        // `repairCorroborationClusterSize` by `resolveClusterPolicy`; the `assumedClusterSize` middle
+        // term keeps direct constructors (embedders, existing tests) behaving as before.
+        this.repairCorroborationClusterSize =
+            cfg?.repairCorroborationClusterSize ?? policy.assumedClusterSize ?? policy.clusterSize;
         this.reputation = reputation;
         const localClusterRef = localCluster && localPeerId ? {
             update: localCluster.update.bind(localCluster),
@@ -154,6 +171,13 @@ export class CoordinatorRepo {
     }
     async get(blockGets, options) {
         // Soft proximity check — warn but still serve reads for graceful degradation
+        // NOTE: a soft-served read now also *acquires* the block durably (see restoreCorroborated), where
+        // before it could at most promote a pending this node already held. So a soft serve leaves behind
+        // a replica of a block this node is not responsible for, and nothing sweeps those: ring-shift
+        // sheds a keyspace RANGE, not "blocks outside my cohort". Fine while soft serves are what they
+        // are meant to be — a rare degradation during routing churn — since routing already placed this
+        // node near the block. If they ever become routine, gate acquisition (not the serve itself) on
+        // isResponsibleForBlock.
         for (const blockId of blockGets.blockIds) {
             if (!await this.isResponsibleForBlock(blockId)) {
                 log('proximity:get-warning', { blockId, msg: 'serving read for non-responsible block' });
@@ -165,14 +189,22 @@ export class CoordinatorRepo {
         //   (a) Missing — block isn't present locally at all (legacy behavior).
         //   (b) Stale-by-policy — block is present but read-repair policy says verify.
         // Skip cluster fetch if this is already a sync request (to prevent recursive queries).
+        // A sync read is also never marked `unavailable` here — the consult it skips is the
+        // one whose failure the flag reports, and flagging would feed the recursion this
+        // bypass exists to prevent. (Storage-level 'unmaterializable' flags still pass
+        // through untouched; they report local state, not the consult.)
         const skipClusterFetch = options?.skipClusterFetch;
         // NOTE: NetworkTransactor.get treats an authoritative "absent" ({ state: {} })
         // as final and no longer retries it (ticket txn-perf-authoritative-notfound),
-        // relying on this cluster reconciliation to have already run. If a coordinator
-        // is ever configured WITHOUT clusterLatestCallback, a missing block is answered
-        // from local state alone with no transactor-level retry to compensate. That is
-        // fine today (such a coordinator has no cluster to reconcile against), but keep
-        // this coupling in mind if a partial-cluster read path is added.
+        // relying on this cluster reconciliation to have already run. When the consult
+        // FAILS outright — or runs while part of the cohort stays SILENT and the block
+        // stays missing — the entry is flagged `unavailable: 'peers-unreachable'` below,
+        // which re-enables the transactor-level retry against a different peer. If a
+        // coordinator is configured WITHOUT clusterLatestCallback, there is no cohort to
+        // consult and the local answer IS the whole truth — it stays authoritative, with
+        // no flag and no transactor-level retry to compensate. That is fine (such a
+        // coordinator has no cluster to reconcile against), but keep this coupling in
+        // mind if a partial-cluster read path is added.
         if (this.clusterLatestCallback && !skipClusterFetch) {
             for (const blockId of blockGets.blockIds) {
                 const localEntry = localResult[blockId];
@@ -190,7 +222,7 @@ export class CoordinatorRepo {
                     });
                 }
                 try {
-                    await this.fetchBlockFromCluster(blockId, blockGets.context);
+                    const { inconclusive } = await this.fetchBlockFromCluster(blockId, blockGets.context, localRev);
                     const refreshed = await this.storageRepo.get({ blockIds: [blockId], context: blockGets.context }, options);
                     const newRev = refreshed[blockId]?.state?.latest?.rev;
                     if (refreshed[blockId]) {
@@ -204,14 +236,43 @@ export class CoordinatorRepo {
                             log('cluster-tx:read-repair-noop', { blockId });
                         }
                     }
+                    // The consult ran but came back INCONCLUSIVE (a silent cohort peer, or a
+                    // corroborated revision this node could not acquire — see
+                    // fetchBlockFromCluster). Either way the reader cannot rule the block out,
+                    // so a still-missing block must not pose as an authoritative absent. When
+                    // the whole cohort answers "holds nothing" the absent stays authoritative —
+                    // the new-collection probe against a healthy cohort stays one round-trip.
+                    if (isMissing && inconclusive) {
+                        this.flagUnconfirmedAbsence(localResult, blockId);
+                    }
                 }
                 catch (err) {
                     log('cluster-fetch:error', { blockId, error: err.message });
+                    // The consult that was supposed to make this answer trustworthy did not run.
+                    if (isMissing) {
+                        this.flagUnconfirmedAbsence(localResult, blockId);
+                    }
                 }
             }
         }
         return localResult;
     }
+    /**
+     * Downgrade an absence the coordinator could not confirm to `unavailable: 'peers-unreachable'` —
+     * the flag `NetworkTransactor.get` retries against another peer instead of taking as final.
+     *
+     * No-op once the entry carries a real answer (the consult restored the block) or a sharper flag
+     * (storage's `'unmaterializable'`), so callers only need to establish that the answer is a guess.
+     */
+    flagUnconfirmedAbsence(results, blockId) {
+        const entry = results[blockId];
+        if (!entry) {
+            results[blockId] = { state: {}, unavailable: 'peers-unreachable' };
+        }
+        else if (!entry.state?.latest && entry.unavailable === undefined) {
+            entry.unavailable = 'peers-unreachable';
+        }
+    }
     /** Decide whether the read-repair policy wants us to consult the cluster for a present-but-possibly-stale block. */
     shouldReadRepair(blockId) {
         switch (this.readRepairMode) {
@@ -249,14 +310,27 @@ export class CoordinatorRepo {
     setLastSeenForTest(blockId, ts) {
         this.lastSeenCommitMs.set(blockId, ts);
     }
-    async fetchBlockFromCluster(blockId, context) {
+    /**
+     * One repair pass for a block: ask the cohort what it holds, and converge onto that if it is
+     * ahead of `localRev` — the revision the caller's read already loaded, and the baseline every
+     * decision below is measured against.
+     *
+     * Returns the one thing `get` needs beyond the storage side effects: whether the pass was
+     * INCONCLUSIVE — it neither confirmed the cohort holds nothing nor left this node holding the
+     * block. Two ways that happens: a cohort peer other than this node stayed SILENT (rejected
+     * callback or per-peer deadline), or a revision WAS corroborated and the convergence onto it
+     * failed. In both, `get` has learned that its local absence may be wrong, so it must not report
+     * a still-missing block as an authoritative absent. Paths that consult nobody (no cohort,
+     * solo-self) are conclusive: there, the local answer genuinely is the whole truth.
+     */
+    async fetchBlockFromCluster(blockId, context, localRev) {
         if (!this.clusterLatestCallback)
-            return;
+            return { inconclusive: false };
         const blockIdBytes = new TextEncoder().encode(blockId);
         const peers = await this.keyNetwork.findCluster(blockIdBytes);
         const peerIds = peers ? Object.keys(peers) : [];
         if (peerIds.length === 0)
-            return;
+            return { inconclusive: false };
         // Solo-cluster short-circuit: the only responsible peer is us. There is no
         // remote to sync from, so skip the callback entirely. Querying ourselves
         // would dial self via SyncClient — pointless at best, and on nodes without
@@ -265,24 +339,41 @@ export class CoordinatorRepo {
             && this.localPeerId
             && peerIds[0] === this.localPeerId.toString()) {
             log('cluster-fetch:solo-self-skip', { blockId });
-            return;
+            return { inconclusive: false };
         }
-        const { corroborated, local } = await this.queryClusterForLatest(peerIds, blockId, context);
+        const { corroborated, local, silent } = await this.queryClusterForLatest(peerIds, blockId, context);
+        // Any silence flags the WHOLE consult, not a fraction of it (fail-closed): one silent
+        // peer could be the sole holder, and the cost — an extra transactor-level retry against
+        // another coordinator — is paid only while a peer is actually unreachable.
+        const cohortSilent = silent.length > 0;
         // Nothing corroborated: keep local data AND stay eligible for repair — marking the
         // block seen here would suppress the next attempt for the whole read-repair window.
         if (!corroborated)
-            return;
+            return { inconclusive: cohortSilent };
+        // The self answer is the sharper baseline (same storage, same context, read alongside the
+        // cohort's), but it exists only when `findCluster` returned this node. A soft serve for a
+        // block this node is no longer responsible for is absent from its own cohort view, so fall
+        // back to the revision the caller's read already loaded. Without the fallback both decisions
+        // below degrade to "any local revision is an advance", which restores backwards and reports
+        // a sync at the revision the pass started from.
+        const baselineRev = local?.rev ?? localRev;
         // Never restore backwards. With this node's own claim excluded from the quorum, a
         // cohort that lags behind the reader corroborates an OLDER revision; adopting it
         // would be a regression, and logging it as a sync would be a lie. The cohort did
         // answer, so the block is verified fresh — mark it seen.
-        if (local && corroborated.rev <= local.rev) {
-            log('cluster-fetch:local-current', { blockId, localRev: local.rev, clusterRev: corroborated.rev });
+        // NOTE: in a cohort of two, that sole peer is the only corroborator, so a lying one can park
+        // the reader here — corroborating the revision it already holds — and re-arm the lazy window
+        // on every pass, hiding a real divergence. Bounded by `readRepairWindowMs` (10s default) and
+        // no worse than the peer simply staying silent. If two-member cohorts become a supported
+        // production topology rather than a dev convenience, stop re-arming the window on a
+        // corroboration that came from a single voter.
+        if (baselineRev !== undefined && corroborated.rev <= baselineRev) {
+            log('cluster-fetch:local-current', { blockId, localRev: baselineRev, clusterRev: corroborated.rev });
             this.markBlocksSeen([blockId]);
-            return;
+            return { inconclusive: cohortSilent };
         }
         // Corroborated revision is ahead of ours — converge onto it.
-        const rev = await this.restoreCorroborated(blockId, corroborated, local, peerIds);
+        const rev = await this.restoreCorroborated(blockId, corroborated, baselineRev, peerIds);
         // Log the OUTCOME, not the attempt. Logging `synced` unconditionally reported hundreds of
         // phantom convergences per run and made a real replication defect invisible for two debugging
         // sessions.
@@ -290,23 +381,28 @@ export class CoordinatorRepo {
             log('cluster-fetch:synced', { blockId, rev });
         }
         else {
-            log('cluster-fetch:not-restored', { blockId, localRev: local?.rev, clusterRev: corroborated.rev });
+            log('cluster-fetch:not-restored', { blockId, localRev: baselineRev, clusterRev: corroborated.rev });
         }
+        // A corroborated revision this node failed to converge onto is inconclusive in its own right,
+        // even with the whole cohort answering: the reader has just been TOLD the block exists, so
+        // reporting it absent would be a lie of the same kind a silent peer causes (see `get`).
+        const inconclusive = cohortSilent || rev === undefined;
         // The block is marked seen either way — the cohort DID answer, so its freshness was checked,
         // which is what the read-repair window tracks. A failed convergence therefore waits out the
         // window before retrying.
         // NOTE: that damping covers only a block this node holds at an OLDER revision. A block entirely
         // missing locally never consults the window (`get` triggers on `isMissing` before
-        // `shouldReadRepair`), so a persistently failing acquisition — e.g. a two-node deployment left
-        // at the default `clusterSize: 10`, where the content quorum can never be met — re-fetches an
+        // `shouldReadRepair`), so a persistently failing acquisition — e.g. a two-node deployment that
+        // never set `assumedClusterSize`, where the content quorum can never be met — re-fetches an
         // archive on every read of that block. Correct, and self-limiting once the cohort can agree; if
         // it ever shows as read amplification, gate the acquisition step (not the latest-query) on the
         // same window rather than widening `isMissing`.
         this.markBlocksSeen([blockId]);
+        return { inconclusive };
     }
     /**
      * Bring this node up to the cohort-corroborated `corroborated`, returning the revision it holds
-     * afterwards when that is an advance over `local`, else `undefined`.
+     * afterwards when that is an advance over `baselineRev`, else `undefined`.
      *
      * Two mechanisms, cheapest first:
      *  1. **Promote a local pending** — free, no network, and the only mechanism that existed before
@@ -326,9 +422,9 @@ export class CoordinatorRepo {
      * Cohort peer ids are passed straight through: the callback filters self out and caps its own
      * corroboration quorum by how many peers could answer at all.
      */
-    async restoreCorroborated(blockId, corroborated, local, cohortPeerIds) {
+    async restoreCorroborated(blockId, corroborated, baselineRev, cohortPeerIds) {
         const promoted = await this.promoteCorroborated(blockId, corroborated);
-        if (isAdvanceOver(promoted, local)) {
+        if (isAdvanceOver(promoted, baselineRev)) {
             return promoted;
         }
         if (!this.acquireBlockFromCohort) {
@@ -339,6 +435,12 @@ export class CoordinatorRepo {
             // inside the callback via `saveReplicatedBlock`, which takes the per-block commit latch —
             // safe to call from here because the read path holds no latch of its own (`StorageRepo.get`
             // acquires and releases it around the promotion above, and nothing wraps this method).
+            // NOTE: `get` walks its block ids sequentially, so the bound is per block, not per call — a
+            // multi-block read that is missing N blocks against a wholly stalled cohort waits N × this.
+            // Acceptable today (the underlying per-peer archive fetch is itself 1s-bounded and runs the
+            // cohort in parallel, so the 5s is a stall ceiling, not a typical cost). If a cold reader
+            // batching a wide read ever times out above this layer, repair the block ids concurrently
+            // rather than shortening the bound.
             await withDeadline(this.acquireBlockFromCohort(blockId, corroborated, cohortPeerIds), RECONCILE_TIMEOUT_MS, `block acquisition for ${blockId}`);
         }
         catch (err) {
@@ -347,47 +449,42 @@ export class CoordinatorRepo {
             return undefined;
         }
         const acquired = await this.readLocalRev(blockId);
-        return isAdvanceOver(acquired, local) ? acquired : undefined;
+        return isAdvanceOver(acquired, baselineRev) ? acquired : undefined;
     }
     /**
      * Promote a corroborated action this node already holds as a local pending — the no-network half of
      * the repair. Returns the local revision afterwards.
      *
      * A pending-only block (metadata seeded by `savePendingTransaction`, no committed revision) asked
-     * for a forward revision throws out of `BlockStorage.ensureRevision` when no restore can supply it.
-     * On THIS path that is an absence, not a read failure — acquisition is precisely the mechanism that
-     * can supply it — so the throw is logged and swallowed rather than short-circuiting the caller.
+     * for a forward revision no promotion can reach used to throw out of `BlockStorage.ensureRevision`;
+     * `StorageRepo.get` now reports it as an entry flagged `unavailable` instead (ticket
+     * repo-reports-unavailable-vs-absent). On THIS path either shape is an absence, not a read failure —
+     * acquisition is precisely the mechanism that can supply the revision — so both are logged as
+     * `promote-unavailable` and stepped over rather than short-circuiting the caller.
      */
     async promoteCorroborated(blockId, corroborated) {
         try {
-            return await this.readLocalRev(blockId, { committed: [corroborated], rev: corroborated.rev });
+            const entry = await this.readLocalEntry(blockId, { committed: [corroborated], rev: corroborated.rev });
+            if (entry?.unavailable !== undefined) {
+                log('cluster-fetch:promote-unavailable', { blockId, rev: corroborated.rev, error: entry.unavailable });
+                return undefined;
+            }
+            return entry?.state?.latest?.rev;
         }
         catch (err) {
             log('cluster-fetch:promote-unavailable', { blockId, rev: corroborated.rev, error: err.message });
             return undefined;
         }
     }
-    /** This node's own `latest.rev` for a block, optionally driving a promotion context through the read. */
-    async readLocalRev(blockId, context) {
+    /** This node's own answer for a block, optionally driving a promotion context through the read.
+     *  Callers that care whether the answer is authoritative inspect `entry.unavailable`. */
+    async readLocalEntry(blockId, context) {
         const result = await this.storageRepo.get({ blockIds: [blockId], context });
-        return result[blockId]?.state?.latest?.rev;
+        return result[blockId];
     }
-    /**
-     * How many peers other than this node could corroborate a claim about a block, given a
-     * cohort view of `peerIds`. Deliberately the MAX of what we observe and what the
-     * configured cluster size implies: the corroboration floor may only be relaxed for a
-     * cohort that is genuinely small, never for one that merely *looks* small. `findCluster`
-     * results are unauthenticated, so a partition — or an attacker with routing influence —
-     * can shrink this node's view to itself plus one peer; measuring against the configured
-     * size keeps that shrunken view from talking the requirement down to a single voter.
-     * The escape hatch for a real two-node deployment is therefore to configure
-     * `clusterSize: 2`, an explicit operator declaration, mirroring how
-     * `allowUnvalidatedSmallCluster` gates the membership admission floor.
-     */
-    corroboratorCapacity(peerIds) {
-        const selfId = this.localPeerId?.toString();
-        const observed = peerIds.filter(id => id !== selfId).length;
-        return Math.max(observed, this.clusterSize - 1);
+    /** This node's own `latest.rev` for a block, optionally driving a promotion context through the read. */
+    async readLocalRev(blockId, context) {
+        return (await this.readLocalEntry(blockId, context))?.state?.latest?.rev;
     }
     /**
      * Query cluster peers for their latest revision and return the highest revision
@@ -410,32 +507,54 @@ export class CoordinatorRepo {
      * `debt-read-repair-commit-cert-verification`.
      */
     async queryClusterForLatest(peerIds, blockId, context) {
-        // Add timeout wrapper to prevent hanging on unresponsive peers
-        const withTimeout = (promise, timeoutMs) => Promise.race([
-            promise,
-            new Promise(resolve => setTimeout(() => resolve(undefined), timeoutMs))
-        ]);
-        // Query peers in parallel for their latest revision (with 1s timeout per peer),
-        // tagging each response with the peer that made it so votes stay distinct.
+        // Query peers in parallel for their latest revision. Each query is DEADLINED (rejects), not
+        // raced-to-undefined: a peer that blows the deadline lands in the silent set below exactly
+        // like a dial failure, because a slow peer and a peer claiming "I hold nothing" must produce
+        // different answers (ticket cluster-read-consult-cannot-report-unreachable).
+        // NOTE: LATEST_QUERY_TIMEOUT_MS is a LAN-shaped budget. A cohort whose round trip honestly
+        // exceeds it now reads as permanently silent, which is safe (the read is flagged, not
+        // mis-reported) but makes every miss cost a transactor-level retry. If a WAN deployment shows
+        // steady `cluster-fetch:peers-silent` against healthy peers, raise this rather than softening
+        // the deadline back into an absent claim.
         const latestResults = await Promise.allSettled(peerIds.map(async (peerIdStr) => {
             const peerId = peerIdFromString(peerIdStr);
-            const value = await withTimeout(this.clusterLatestCallback(peerId, blockId, context), 1000);
-            return { peerIdStr, value };
+            return await withDeadline(this.clusterLatestCallback(peerId, blockId, context), LATEST_QUERY_TIMEOUT_MS, `latest query to ${peerIdStr}`);
         }));
+        // NOTE: self-exclusion is keyed on `localPeerId`, which is optional for the single-node/test
+        // construction this class has always tolerated. Left unset, this node's own answer is counted
+        // as a peer claim again. Harmless today — the self answer can only ever corroborate the
+        // revision already held, so the pass declines as `local-current` — but if a future caller can
+        // make self report something the reader does not hold, make `localPeerId` required instead.
         const selfId = this.localPeerId?.toString();
         let local;
         const claims = [];
-        for (const result of latestResults) {
-            if (result.status !== 'fulfilled' || !result.value.value)
+        const silent = [];
+        // `allSettled` preserves input order, so results correlate to `peerIds` by index — a
+        // rejected entry carries no payload of its own, and its peer id is what `silent` records.
+        for (let i = 0; i < latestResults.length; i++) {
+            const result = latestResults[i];
+            const peerIdStr = peerIds[i];
+            if (result.status !== 'fulfilled') {
+                // Silence: the callback rejected or the deadline expired. Never a claim. Self is
+                // excluded — its short-circuit reads local storage, and a local read error is not a
+                // cohort peer being unreachable.
+                if (peerIdStr !== selfId)
+                    silent.push(peerIdStr);
                 continue;
-            const { peerIdStr, value } = result.value;
+            }
+            const value = result.value;
             if (peerIdStr === selfId) {
                 local = value;
                 continue;
             }
+            if (!value)
+                continue; // responded, holds nothing — an absent claim, not silence
             claims.push({ peerId: peerIdStr, rev: value.rev, actionId: value.actionId });
         }
-        const capacity = this.corroboratorCapacity(peerIds);
+        if (silent.length > 0) {
+            log('cluster-fetch:peers-silent', { blockId, silent: silent.length, consulted: peerIds.length });
+        }
+        const capacity = corroboratorCapacity(peerIds.filter(id => id !== selfId).length, this.repairCorroborationClusterSize);
         const selected = selectQuorumRev(claims, this.simpleMajorityThreshold, capacity);
         if (!selected) {
             log('cluster-fetch:no-quorum', {
@@ -443,13 +562,13 @@ export class CoordinatorRepo {
                 responders: claims.length,
                 required: quorumSize(claims.length, this.simpleMajorityThreshold, capacity)
             });
-            return { local };
+            return { local, silent };
         }
         // Best-effort: penalize peers whose claim contradicts the corroborated pair
         // (an inflated rev the quorum outvoted, or conflicting content at the agreed
         // rev). A lower rev is just lag, never penalized. Never let this throw.
         this.penalizeContradictingRevClaims(claims, selected, blockId);
-        return { corroborated: { actionId: selected.actionId, rev: selected.rev }, local };
+        return { corroborated: { actionId: selected.actionId, rev: selected.rev }, local, silent };
     }
     /**
      * Report peers whose reported latest contradicts the quorum-corroborated pair. Best-effort.
@@ -517,9 +636,78 @@ export class CoordinatorRepo {
         }
         catch (error) {
             log('coordinator-repo:pend-error', { actionId: request.actionId, error: error.message });
+            const stale = await this.classifyStaleRejection(error, request, allBlockIds);
+            if (stale)
+                return stale;
             throw error;
         }
     }
+    /**
+     * Decide whether a cluster validator rejection was an optimistic-concurrency loss — the block
+     * already advanced past the requested revision — rather than a genuine validation fault.
+     * A confirmed loss returns a {@link StaleFailure} carrying `conflict: true` so the caller
+     * receives a non-success *response* that says plainly it is a lost race: network-transactor's
+     * pend then takes its stale branch and both writers (`Collection.sync`, and the coordinator's
+     * multi-collection pendPhase via `isConflictFailure`) retry, instead of a thrown error escaping
+     * mid-batch (which splits multi-tree commits — see PartialCommitError).
+     *
+     * The failure carries no `missing` list: confirmation is a local re-read that reveals the
+     * revision is taken but not which actions took it, and no consumer rebases from `missing`
+     * anyway (it is only counted or logged). `conflict` conveys retryability directly instead.
+     *
+     * Confirmation is purely local: re-read the affected blocks from our own storage and require
+     * `latest.rev >= request.rev`. The signed reject-reason text is never consulted — it is
+     * free-form wire-visible prose and must not become control flow. Anything unconfirmed
+     * (including read errors during confirmation) stays a throw, preserving fail-fast for
+     * genuine validation faults.
+     */
+    async classifyStaleRejection(error, request, blockIds) {
+        const requestedRev = request.rev;
+        if (!(error instanceof ValidatorRejectionError) || requestedRev === undefined)
+            return undefined;
+        let results;
+        try {
+            results = await this.storageRepo.get({ blockIds });
+        }
+        catch (readError) {
+            log('coordinator-repo:pend-stale-classify-read-error', {
+                actionId: request.actionId,
+                error: readError.message
+            });
+            return undefined;
+        }
+        // Scan EVERY block rather than stopping at the first confirmation: several of the request's
+        // blocks can be past the requested revision at different revisions, and it is the highest
+        // that the loser's next request has to clear (see `highestStaleAt`). Both the reported
+        // number and the reason prose name that block, so they never disagree.
+        const staleAt = highestStaleAt(blockIds.map(blockId => {
+            const latest = results[blockId]?.state.latest;
+            return latest && latest.rev >= requestedRev ? { blockId, rev: latest.rev } : undefined;
+        }));
+        if (staleAt) {
+            log('coordinator-repo:pend-stale-classified', {
+                actionId: request.actionId,
+                blockId: staleAt.blockId,
+                latestRev: staleAt.rev,
+                requestedRev
+            });
+            return {
+                success: false,
+                conflict: true,
+                reason: `stale revision: block ${staleAt.blockId} at rev ${staleAt.rev}, requested rev ${requestedRev}`,
+                // The same fact as the reason prose, but as data. This is the ONLY place a losing
+                // writer can learn the revision it lost to, since this failure deliberately carries
+                // no `missing`. Confirmed-local: read out of our own storage just above.
+                staleAt
+            };
+        }
+        // NOTE: conservative — when only remote members saw the newer revision (local storage still
+        // behind), staleness can't be confirmed locally and the rejection stays a throw. If that
+        // shows up in practice, extend confirmation with a quorum read; never trust the reject text.
+        // `staleAt` is absent on this path for the same reason, and deliberately so — there is no
+        // confirmed number to report, and the field's contract forbids inferring one from that text.
+        return undefined;
+    }
     async cancel(actionRef, options) {
         const blockIds = actionRef.blockIds;
         await this.verifyResponsibility(blockIds);
@@ -564,25 +752,33 @@ export class CoordinatorRepo {
                 this.markBlocksSeen(blockIds);
                 return { success: true };
             }
-            // Local cluster didn't execute during consensus. Attempt a local commit,
-            // but tolerate failure (e.g., "pending action not found") when the cluster
-            // already reached consensus — this coordinator was likely picked for commit
-            // after missing the pend phase (unreachable during pend, fresh join, etc.).
-            // The cluster's majority is authoritative; this peer will catch up via sync.
+            // Local cluster didn't execute during consensus. Attempt a local commit, but tolerate
+            // local divergence when the cluster already reached consensus — this coordinator was
+            // likely picked for commit after missing the pend phase (unreachable during pend, fresh
+            // join, etc.). The cluster's majority is authoritative; this peer catches up via sync.
+            //
+            // Divergence reaches us in BOTH shapes and both must be tolerated identically:
+            //   - a THROW ("Pending action … not found"), when we never saw the pend;
+            //   - a RETURNED `success:false` carrying `missing-base-revision`, when we saw the pend
+            //     but not the revision that created the block (see StorageRepo.internalCommit).
+            // Only the throw was tolerated before the refusal existed. Reporting the refusal to the
+            // caller instead would surface a committed transaction as a stale loss: db-core's
+            // commitPhase treats any returned `success:false` as a permanent stale failure, so the
+            // client would retry an action the cluster already landed until it exhausted its budget.
             try {
                 const result = await this.storageRepo.commit(request, options);
-                if (result.success)
+                if (result.success) {
                     this.markBlocksSeen(blockIds);
+                    return result;
+                }
+                if (isMissingBaseRevisionFailure(result) && clusterReachedCommitConsensus(record)) {
+                    return this.tolerateLocalCommitDivergence(request, blockIds, result.reason ?? MISSING_BASE_REVISION_REASON);
+                }
                 return result;
             }
             catch (err) {
                 if (clusterReachedCommitConsensus(record)) {
-                    log('coordinator-repo:commit-local-failed-cluster-succeeded', {
-                        actionId: request.actionId,
-                        error: err.message
-                    });
-                    this.markBlocksSeen(blockIds);
-                    return { success: true };
+                    return this.tolerateLocalCommitDivergence(request, blockIds, err.message);
                 }
                 throw err;
             }
@@ -592,6 +788,16 @@ export class CoordinatorRepo {
             throw error;
         }
     }
+    /**
+     * Report success for a commit the cluster carried but this peer could not apply locally. The
+     * blocks are marked seen so the read path treats them as freshness-checked; convergence comes
+     * from replication (cohort reconcile, or read-driven acquisition), not from replay here.
+     */
+    tolerateLocalCommitDivergence(request, blockIds, detail) {
+        log('coordinator-repo:commit-local-failed-cluster-succeeded', { actionId: request.actionId, error: detail });
+        this.markBlocksSeen(blockIds);
+        return { success: true };
+    }
 }
 /** True if a simple majority of cluster peers signed an approving commit. */
 function clusterReachedCommitConsensus(record) {