@optimystic/db-p2p 0.26.0 → 0.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/dist/src/cluster/certified-claims.d.ts +17 -3
  2. package/dist/src/cluster/certified-claims.d.ts.map +1 -1
  3. package/dist/src/cluster/certified-claims.js +5 -3
  4. package/dist/src/cluster/certified-claims.js.map +1 -1
  5. package/dist/src/cluster/cluster-repo.d.ts +10 -1
  6. package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
  7. package/dist/src/cluster/cluster-repo.js +11 -1
  8. package/dist/src/cluster/cluster-repo.js.map +1 -1
  9. package/dist/src/cluster/commit-proof.d.ts +16 -0
  10. package/dist/src/cluster/commit-proof.d.ts.map +1 -1
  11. package/dist/src/cluster/commit-proof.js +32 -1
  12. package/dist/src/cluster/commit-proof.js.map +1 -1
  13. package/dist/src/cluster/quorum-restore.d.ts +81 -28
  14. package/dist/src/cluster/quorum-restore.d.ts.map +1 -1
  15. package/dist/src/cluster/quorum-restore.js +148 -51
  16. package/dist/src/cluster/quorum-restore.js.map +1 -1
  17. package/dist/src/cluster/reconcile-block.d.ts +9 -4
  18. package/dist/src/cluster/reconcile-block.d.ts.map +1 -1
  19. package/dist/src/cluster/reconcile-block.js +28 -11
  20. package/dist/src/cluster/reconcile-block.js.map +1 -1
  21. package/dist/src/libp2p-node-base.d.ts +6 -0
  22. package/dist/src/libp2p-node-base.d.ts.map +1 -1
  23. package/dist/src/libp2p-node-base.js.map +1 -1
  24. package/dist/src/repo/cluster-coordinator.d.ts +9 -0
  25. package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
  26. package/dist/src/repo/cluster-coordinator.js +13 -2
  27. package/dist/src/repo/cluster-coordinator.js.map +1 -1
  28. package/dist/src/repo/coordinator-repo.d.ts +34 -2
  29. package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
  30. package/dist/src/repo/coordinator-repo.js +57 -3
  31. package/dist/src/repo/coordinator-repo.js.map +1 -1
  32. package/package.json +2 -2
  33. package/src/cluster/certified-claims.ts +22 -9
  34. package/src/cluster/cluster-repo.ts +12 -1
  35. package/src/cluster/commit-proof.ts +38 -2
  36. package/src/cluster/quorum-restore.ts +183 -56
  37. package/src/cluster/reconcile-block.ts +34 -11
  38. package/src/libp2p-node-base.ts +6 -0
  39. package/src/repo/cluster-coordinator.ts +1039 -1027
  40. package/src/repo/coordinator-repo.ts +1937 -1855
@@ -1,1855 +1,1937 @@
1
- import type { PendRequest, ActionBlocks, IRepo, MessageOptions, CommitResult, GetBlockResults, PendResult, StaleFailure, BlockGets, CommitRequest, RepoMessage, IKeyNetwork, ICluster, ClusterConsensusConfig, BlockId, ActionId, ActionRev, ActionContext, ClusterRecord, BlockUnavailableReason, ActionPending } from "@optimystic/db-core";
2
- import { LruMap, blockIdsForTransforms, highestStaleAt, isConflictFailure, isOwnRevision, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
3
- import { ClusterCoordinator, ConflictRaceLostError, ValidatorRejectionError } from "./cluster-coordinator.js";
4
- import type { PeerId } from "@libp2p/interface";
5
- import { peerIdFromString } from "@libp2p/peer-id";
6
- import type { FretService } from "p2p-fret";
7
- import { createLogger } from '../logger.js';
8
- import type { IPeerReputation } from "../reputation/types.js";
9
- import { PenaltyReason } from "../reputation/types.js";
10
- import type { ITransactionStateStore } from "../cluster/i-transaction-state-store.js";
11
- import { quorumSize, corroboratorCapacity, selectQuorumRev, certifiedEquivocation, CORROBORATION_FLOOR, type RevClaim, type QuorumRev } from "../cluster/quorum-restore.js";
12
- import { certifyClaim, isAttributableProofFailure, proofThresholds, type ProofAnchoring } from "../cluster/certified-claims.js";
13
- import { DEFAULT_CLUSTER_SIZE } from "../cluster/cluster-policy.js";
14
- import { RECONCILE_TIMEOUT_MS } from "../cluster/reconcile-block.js";
15
- import { isMissingBaseRevisionFailure, MISSING_BASE_REVISION_REASON, type IRevisionActionReader } from "../storage/storage-repo.js";
16
- import type { ReconcileBlockCallback } from "../cluster/cluster-repo.js";
17
- import type { CertifiedActionRev } from "../storage/block-archive.js";
18
-
19
- /**
20
- * Acquire a block's content for a cohort-corroborated revision, from the cohort, and persist it.
21
- *
22
- * Deliberately the SAME shape as the commit path's {@link ReconcileBlockCallback}, and in the live
23
- * node the very same instance (`libp2p-node-base` passes its `reconcileBlock` to both): read-driven
24
- * acquisition needs exactly what reconcile already provides a per-peer-bounded archive fetch, a
25
- * quorum vote on the target `(rev, actionId)`, a quorum vote on the *content* at that revision, and a
26
- * persist through the monotonic, commit-latched `StorageRepo.saveReplicatedBlock` funnel. Reusing it
27
- * is what keeps read-repair from being a weaker trust path than reconcile.
28
- */
29
- export type AcquireBlockCallback = ReconcileBlockCallback;
30
-
31
- /** How long one cohort peer gets to answer the latest-revision consult before it counts as silent. */
32
- const LATEST_QUERY_TIMEOUT_MS = 1000;
33
-
34
- /** True when a freshly-read local revision is strictly ahead of the baseline the repair started from. */
35
- function isAdvanceOver(rev: number | undefined, baselineRev: number | undefined): boolean {
36
- return typeof rev === 'number' && (baselineRev === undefined || rev > baselineRev);
37
- }
38
-
39
- /**
40
- * Reject if `promise` has not settled within `ms`. The timer is cleared on either outcome, so no
41
- * handle outlives the race (hence no `unref`, which does not exist off Node).
42
- */
43
- function withDeadline<T>(promise: Promise<T>, ms: number, label: string): Promise<T> {
44
- let timer: ReturnType<typeof setTimeout> | undefined;
45
- const deadline = new Promise<never>((_, reject) => {
46
- timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms);
47
- });
48
- return Promise.race([promise, deadline]).finally(() => {
49
- if (timer !== undefined) clearTimeout(timer);
50
- });
51
- }
52
-
53
- /**
54
- * What one round of polling the cohort learned about a block: the revision the OTHER
55
- * cohort members corroborated, and what this node itself already holds. The two are kept
56
- * apart on purpose the local revision is the baseline being repaired, never evidence
57
- * about the cluster (see {@link CoordinatorRepo.queryClusterForLatest}) but the caller
58
- * still needs it to tell whether the corroborated revision is actually an advance.
59
- */
60
- interface ClusterLatestQuery {
61
- /** Highest `(rev, actionId)` corroborated by peers other than this node, if any. */
62
- corroborated?: ActionRev;
63
- /**
64
- * This node's own latest for the block, as answered by the callback's self short-circuit.
65
- * Typed as a {@link CertifiedActionRev} because that short-circuit reads the local proof too —
66
- * of no use to this node (it trusts its own storage), but the type stays honest about what the
67
- * value carries rather than silently erasing it.
68
- */
69
- local?: CertifiedActionRev;
70
- /**
71
- * Cohort peers (self excluded) that never answered the consult — the callback rejected
72
- * (dial failure, protocol error) or blew the per-peer deadline. Silence, not evidence:
73
- * these are never counted as claims, but while this is non-empty a caller must not treat
74
- * "nothing corroborated" as an authoritative absence, because a silent peer could be the
75
- * sole holder.
76
- */
77
- silent: string[];
78
- /**
79
- * Highest revision any cohort peer CLAIMED when no claim met the corroboration quorum
80
- * (set only alongside an absent `corroborated`). The claim failed quorum, so it must
81
- * never drive restoration it exists so `get` can report content it serves below this
82
- * revision as possibly behind ({@link GetBlockResult.unconfirmedAheadRev}) instead of
83
- * confirmed. When a quorum DOES corroborate, higher uncorroborated claims are dropped
84
- * as before: the quorum's affirmative answer outweighs a lone voter (which may simply
85
- * be ahead on an in-flight commit), and stamping doubt there would mark every read that
86
- * races a commit broadcast.
87
- */
88
- uncorroboratedRev?: number;
89
- /**
90
- * How many cohort peers OTHER than this node answered the consult at all — with a claim
91
- * or with "I hold nothing". `silent` says who could not be asked; this says how many
92
- * could. Zero with a non-empty `silent` means this node reached NOBODY, which is a
93
- * different fact from partial silence: there is no better-informed answer to be had from
94
- * this node's position (see {@link AbsenceVerdict}).
95
- */
96
- answered: number;
97
- }
98
-
99
- /**
100
- * What earlier repair passes left unresolved for one block. Two independent facts share one entry
101
- * and one map on purpose: both are "what the last repair pass could not finish", both are
102
- * cleared by the same event (the block converging), and `CoordinatorRepo` already keeps more
103
- * per-block maps than anyone can hold in their head (backlog
104
- * `debt-freshness-state-scattered-across-coordinator-repo`).
105
- *
106
- * An entry exists exactly while at least one of the two is set; both clear together in
107
- * {@link CoordinatorRepo.flagUnconfirmedCurrency} once this node reaches the claimed revision.
108
- */
109
- interface AheadClaimState {
110
- /**
111
- * The cohort-claimed revision the last freshness consult could not settle — the doubt
112
- * {@link CoordinatorRepo.flagUnconfirmedCurrency} stamps onto reads served below it. Absent once a
113
- * consult finds nothing ahead of what this node holds.
114
- */
115
- rev?: number;
116
- /**
117
- * Which `cluster-fetch:repair-deadlock` reasons have already been said for this block (see
118
- * {@link CoordinatorRepo.reportRepairDeadlock}). Neither reason is about any one revision one is
119
- * about the cohort's size, the other about how many of its peers hold the block so both survive
120
- * {@link CoordinatorRepo.recordAheadClaim} clearing `rev`: without that, a block whose cohort
121
- * claims nothing *ahead* of the reader would re-announce the same permanent condition on every
122
- * single pass, the noise this line exists to replace.
123
- *
124
- * Tracked per REASON rather than as one flag: the two diagnose different faults and send the
125
- * operator to different places, so an episode that starts as `cohort-too-small` and becomes
126
- * `sole-holder` (the operator added machines, which is what that reason told them to do) has to be
127
- * able to say the second thing. Bounded at two entries by the reason union itself.
128
- */
129
- deadlocksReported?: readonly DeadlockReason[];
130
- }
131
-
132
- /**
133
- * Why a corroboration decline is provably permanent — see {@link CoordinatorRepo.reportRepairDeadlock}
134
- * for what makes each provable and which remedy each sends the operator to.
135
- */
136
- type DeadlockReason = 'cohort-too-small' | 'sole-holder';
137
-
138
- /** The `cohort-too-small` wording: the cohort cannot field the quorum however healthy its peers are. */
139
- function cohortTooSmallMessage(
140
- cohortPeers: number,
141
- claimants: number,
142
- requiredEvenIfAllAnswered: number,
143
- repairCorroborationClusterSize: number
144
- ): string {
145
- return `Block repair cannot converge for this block and the condition is PERMANENT, not transient: ` +
146
- `this node's cohort has ${cohortPeers} peer(s) besides itself, all of them answered ` +
147
- `(${claimants} hold the block), but accepting a revision would need ${requiredEvenIfAllAnswered} ` +
148
- `agreeing peers even if every one of those ${cohortPeers} answered and agreed. No later pass can reach ` +
149
- `that, however healthy every peer is, so this node's copy of the block stays as it is. Repair needs ` +
150
- `${CORROBORATION_FLOOR} cohort peers BESIDES the reader to answer and agree, relaxed to 1 only for a ` +
151
- `cohort that DECLARES it is smaller; repairCorroborationClusterSize currently resolves to ` +
152
- `${repairCorroborationClusterSize}. Two things produce this, and this node cannot tell them ` +
153
- `apart: (1) the deployment really does run this few machines set clusterPolicy.assumedClusterSize ` +
154
- `to the number you actually run (it does not lower clusterSize / the replication factor), or set an ` +
155
- `honest clusterSize, and run at least ${CORROBORATION_FLOOR + 2} machines for any tolerance of one ` +
156
- `unreachable peer; or (2) this node's view of the cohort has shrunk below the real deployment — a ` +
157
- `partition or a routing problem, which configuration will not fix. Check the peer count above ` +
158
- `against the machines you run before changing anything.`;
159
- }
160
-
161
- /**
162
- * The `sole-holder` wording: the cohort is big enough, but only one of its peers holds the block.
163
- *
164
- * Every claim here is scoped to THIS NODE'S COHORT PEERS, which is the whole of what the pass
165
- * observed. It deliberately does not say "only one machine in the deployment holds this block": this
166
- * node's own copy is excluded from the claim set (it cannot corroborate the revision it is trying to
167
- * repair), so a reader that holds the block itself would make that reading false and a scary
168
- * all-caps line an operator can disprove by looking at their own disks is worth less than no line.
169
- * For the same reason the remedy is "another COHORT PEER holding it" rather than "a second copy":
170
- * with the reader holding one, a second copy already exists and is still not enough.
171
- */
172
- function soleHolderMessage(cohortPeers: number): string {
173
- return `Block repair cannot converge for this block and the condition is PERMANENT, not transient: ` +
174
- `ONLY ONE COHORT PEER HOLDS THIS BLOCK. Of this node's ${cohortPeers} cohort peers, 1 reports holding ` +
175
- `it and the other ${cohortPeers - 1} answered that they hold NOTHING — an answer, not silence, so this ` +
176
- `is the whole picture and not a slow pass. Repair adopts a revision only when ${CORROBORATION_FLOOR} ` +
177
- `peers BESIDES this node agree on it, and a lone holder cannot second itself, so every later pass ` +
178
- `declines identically. This node's own copy, if it has one, is the copy being repaired and does not ` +
179
- `count toward that number. MORE MACHINES DO NOT FIX THIS, and neither does any cluster-size setting ` +
180
- `what is missing is ANOTHER COHORT PEER HOLDING THE BLOCK. The usual cause is data written while the ` +
181
- `deployment (or this block's cohort) was smaller: a block that had one holder then still has one holder ` +
182
- `now, because the two paths that would replicate it read-repair and reconcile both decline on this ` +
183
- `same rule. Committing any new revision of the block writes it to the current cohort and clears this. ` +
184
- `(A lone holder whose answer carries a valid cohort commit proof for its revision IS adopted without a ` +
185
- `second voter reaching this message means the one holder attached no such proof, or one that did not ` +
186
- `verify.)`;
187
- }
188
-
189
- /**
190
- * What one repair pass established about a block that is still MISSING locally after it.
191
- * Ordered by how firmly the block is ruled out; `get` consults it only on the missing path.
192
- */
193
- type AbsenceVerdict =
194
- /** Nobody to ask (empty cohort, or solo-self), or every non-self cohort member answered
195
- * "I hold nothing". As confirmed as an absence gets stays authoritative, which is what
196
- * keeps the routine new-collection probe at one round trip. */
197
- | 'confirmed'
198
- /** Some of the cohort answered and some could not be asked. (A consult that THROWS produces
199
- * no verdict at all `get`'s catch arm reports it directly.) */
200
- | 'unconfirmed'
201
- /** No cohort member outside this node could be asked at all. Mutually exclusive with
202
- * `claimed` in practice: a claim requires a non-self peer to have answered, which is
203
- * exactly what this verdict rules out. The precedence below still orders the pair, so
204
- * the mapping stays total, but there is no reachable case to test. */
205
- | 'isolated'
206
- /** A peer claimed a revision this pass did not converge onto — quorum declined it, or a
207
- * quorum corroborated it and acquisition failed. */
208
- | 'claimed';
209
-
210
- /**
211
- * Extended cluster interface that includes the ability to check if a transaction was executed.
212
- * This is used by CoordinatorRepo to avoid duplicate execution.
213
- */
214
- interface LocalClusterWithExecutionTracking extends ICluster {
215
- wasTransactionExecuted?(messageHash: string): boolean;
216
- /** Local storage's verdict for a pend applied during consensus; see ClusterMember.getExecutedPendResult. */
217
- getExecutedPendResult?(messageHash: string): PendResult | undefined;
218
- /** Local storage's verdict for a commit applied during consensus; see ClusterMember.getExecutedCommitResult. */
219
- getExecutedCommitResult?(messageHash: string): CommitResult | undefined;
220
- }
221
-
222
- /**
223
- * A cohort peer's answer to the latest-revision consult. Defined with the archive shape it is
224
- * projected from (`storage/block-archive.ts`) and re-exported here so it reads next to the callback
225
- * that returns it; see {@link CertifiedActionRev} there for what the optional proof does and does
226
- * not mean.
227
- */
228
- export type { CertifiedActionRev } from "../storage/block-archive.js";
229
-
230
- /**
231
- * Callback to query a cluster peer for their latest revision of a block. Three-way contract:
232
- * - resolves a `CertifiedActionRev` — the peer answered and holds the block at that revision;
233
- * - resolves `undefined` the peer answered and holds NOTHING (an absent claim);
234
- * - REJECTS — the peer could not be asked at all (dial failure, protocol error).
235
- *
236
- * The distinction between the last two is load-bearing: `queryClusterForLatest` counts a
237
- * rejection as a SILENT peer, which stops `CoordinatorRepo.get` from reporting a locally-missing
238
- * block as an authoritative absent, while a resolved `undefined` is a real answer that keeps the
239
- * absent authoritative. Implementations must therefore let transport errors propagate rather than
240
- * swallowing them into `undefined` (see the implementations in `libp2p-node-base` and the mesh
241
- * harness's `silentPeers` failure knob). Slowness needs no handling here the caller deadlines
242
- * each query and treats expiry as silence.
243
- */
244
- export type ClusterLatestCallback = (peerId: PeerId, blockId: BlockId, context?: ActionContext) => Promise<CertifiedActionRev | undefined>;
245
-
246
- interface CoordinatorRepoComponents {
247
- storageRepo: IRepo;
248
- localCluster?: LocalClusterWithExecutionTracking;
249
- localPeerId?: PeerId;
250
- /**
251
- * Optional callback to query cluster peers for their latest block revision.
252
- * Used for read-path cluster verification to discover unknown revisions.
253
- */
254
- clusterLatestCallback?: ClusterLatestCallback;
255
- /**
256
- * Optional callback that actually moves a block's bytes from the cohort into local storage once
257
- * {@link clusterLatestCallback} has established a corroborated revision this node lacks. Absent →
258
- * the read path can still *select* the right revision but converges only when the node already
259
- * holds the corroborated action as a promotable pending. See {@link AcquireBlockCallback}.
260
- */
261
- acquireBlockFromCohort?: AcquireBlockCallback;
262
- /**
263
- * Optional layer-2 anchoring for the cohort commit proofs the latest-revision consult verifies
264
- * (`cluster/certified-claims.ts`): re-derive the block's cohort and LOG the overlap with the
265
- * proof's signers, plus surface proofs accepted without that comparison. Purely observational —
266
- * never a gate — and absent in production wiring today; `certifyClaim` logs unanchored
267
- * acceptance internally regardless.
268
- */
269
- proofAnchoring?: ProofAnchoring;
270
- }
271
-
272
- /**
273
- * Consensus config for the coordinator side, plus the repair yardstick the read-repair path measures
274
- * a (possibly shrunken) cohort view against. `repairCorroborationClusterSize` is deliberately its own
275
- * field rather than an overload of {@link ClusterConsensusConfig.assumedClusterSize}: this same object
276
- * also builds the `ClusterCoordinator`, and a field whose value silently differed from the cluster
277
- * member's copy of it would be a trap. See `cluster/cluster-policy.ts` for why the two differ.
278
- */
279
- export type CoordinatorRepoConfig = Partial<ClusterConsensusConfig> & {
280
- clusterSize?: number;
281
- repairCorroborationClusterSize?: number;
282
- };
283
-
284
- export function coordinatorRepo(
285
- keyNetwork: IKeyNetwork,
286
- createClusterClient: (peerId: PeerId) => ICluster,
287
- cfg?: CoordinatorRepoConfig,
288
- fretService?: FretService,
289
- reputation?: IPeerReputation,
290
- stateStore?: ITransactionStateStore
291
- ): (components: CoordinatorRepoComponents) => CoordinatorRepo {
292
- return (components: CoordinatorRepoComponents) => new CoordinatorRepo(
293
- keyNetwork,
294
- createClusterClient,
295
- components.storageRepo,
296
- cfg,
297
- components.localCluster,
298
- components.localPeerId,
299
- fretService,
300
- components.clusterLatestCallback,
301
- reputation,
302
- stateStore,
303
- components.acquireBlockFromCohort,
304
- components.proofAnchoring
305
- );
306
- }
307
-
308
- /** Cluster coordination repo - uses local store, as well as distributes changes to other nodes using cluster consensus. */
309
- export class CoordinatorRepo implements IRepo {
310
- private coordinator: ClusterCoordinator;
311
- private readonly DEFAULT_TIMEOUT = 30000; // 30 seconds default timeout
312
- private readonly localPeerId?: PeerId;
313
- private readonly responsibilityCache = new LruMap<string, { inCluster: boolean, expires: number }>(1000);
314
- private static readonly RESPONSIBILITY_TTL_MS = 60_000;
315
- private readonly lastSeenCommitMs = new LruMap<string, number>(1000);
316
- /** Per block, what earlier repair passes left unresolved see {@link AheadClaimState}.
317
- * Outlives the consult on purpose: the read-repair window skips consults for blocks checked
318
- * recently, and a doubt dropped there is a stale answer served as confirmed again.
319
- * NOTE: LRU-bounded like `lastSeenCommitMs`; an eviction under >1000 doubted blocks loses the
320
- * doubt until the next consult re-derives it (one read-repair window later, at worst) and lets
321
- * {@link reportRepairDeadlock} say its piece a second time. */
322
- private readonly unsettledAheadClaims = new LruMap<string, AheadClaimState>(1000);
323
- private readonly readRepairMode: 'off' | 'lazy' | 'paranoid';
324
- private readonly readRepairWindowMs: number;
325
- private readonly readRepairSampleRate: number;
326
- /** Simple-majority threshold from the consensus policy; drives the read-repair corroboration quorum. */
327
- private readonly simpleMajorityThreshold: number;
328
- /**
329
- * Yardstick the read-repair corroboration floor is measured against; the floor for
330
- * {@link corroboratorCapacity}. Resolved by `resolveClusterPolicy` for a real node; falls back to
331
- * `assumedClusterSize` and then `clusterSize` for direct constructors (see the constructor), so a
332
- * caller that has adopted neither field keeps today's behavior exactly.
333
- */
334
- private readonly repairCorroborationClusterSize: number;
335
- /** Resolved super-majority threshold the coordinator commits on (mirrors the value handed to ClusterCoordinator). */
336
- private readonly superMajorityThreshold: number;
337
- private readonly reputation?: IPeerReputation;
338
- /** Per-instance logger, namespaced by peer id when `localPeerId` is known (degrades to the un-suffixed namespace when not — the single-node/test construction has always tolerated its absence). */
339
- private readonly log: ReturnType<typeof createLogger>;
340
- /** Test seam: overridable clock for window-based read-repair gating. */
341
- now: () => number = () => Date.now();
342
- /** Test seam: overridable RNG (0..1) for sample-rate gating. */
343
- rand: () => number = () => Math.random();
344
-
345
- constructor(
346
- readonly keyNetwork: IKeyNetwork,
347
- readonly createClusterClient: (peerId: PeerId) => ICluster,
348
- private readonly storageRepo: IRepo,
349
- cfg?: CoordinatorRepoConfig,
350
- localCluster?: LocalClusterWithExecutionTracking,
351
- localPeerId?: PeerId,
352
- fretService?: FretService,
353
- private readonly clusterLatestCallback?: ClusterLatestCallback,
354
- reputation?: IPeerReputation,
355
- stateStore?: ITransactionStateStore,
356
- private readonly acquireBlockFromCohort?: AcquireBlockCallback,
357
- private readonly proofAnchoring?: ProofAnchoring
358
- ) {
359
- this.localPeerId = localPeerId;
360
- this.log = createLogger('coordinator-repo', localPeerId?.toString());
361
- const policy: ClusterConsensusConfig & { clusterSize: number } = {
362
- // Same constant `resolveClusterPolicy` gives a node that declares no clusterSize, not a
363
- // second literal: a direct constructor (the readme's manual-wiring path) and the node
364
- // assembly must land on the same width or the two disagree about the same key's cohort.
365
- clusterSize: cfg?.clusterSize ?? DEFAULT_CLUSTER_SIZE,
366
- assumedClusterSize: cfg?.assumedClusterSize,
367
- superMajorityThreshold: cfg?.superMajorityThreshold ?? DEFAULT_SUPER_MAJORITY_THRESHOLD,
368
- simpleMajorityThreshold: cfg?.simpleMajorityThreshold ?? 0.51,
369
- minAbsoluteClusterSize: cfg?.minAbsoluteClusterSize ?? 3,
370
- allowClusterDownsize: cfg?.allowClusterDownsize ?? true,
371
- clusterSizeTolerance: cfg?.clusterSizeTolerance ?? 0.5,
372
- partitionDetectionWindow: cfg?.partitionDetectionWindow ?? 60000,
373
- commitBroadcastRetryInitialMs: cfg?.commitBroadcastRetryInitialMs ?? 250,
374
- commitBroadcastRetryBackoffFactor: cfg?.commitBroadcastRetryBackoffFactor ?? 2,
375
- commitBroadcastRetryMaxIntervalMs: cfg?.commitBroadcastRetryMaxIntervalMs ?? 8000,
376
- commitBroadcastRetryMaxAttempts: cfg?.commitBroadcastRetryMaxAttempts ?? 5,
377
- commitBroadcastImmediateRetries: cfg?.commitBroadcastImmediateRetries ?? 1,
378
- promiseImmediateRetries: cfg?.promiseImmediateRetries ?? 1,
379
- readRepairMode: cfg?.readRepairMode ?? 'lazy',
380
- readRepairWindowMs: cfg?.readRepairWindowMs ?? 10000,
381
- readRepairSampleRate: cfg?.readRepairSampleRate ?? 0,
382
- // Default false: an undersized cluster with no confident network-size estimate
383
- // is REJECTED (fail closed). Callers only opt in for single-node/local/test meshes.
384
- allowUnvalidatedSmallCluster: cfg?.allowUnvalidatedSmallCluster ?? false
385
- };
386
- this.readRepairMode = policy.readRepairMode!;
387
- this.readRepairWindowMs = policy.readRepairWindowMs!;
388
- this.readRepairSampleRate = policy.readRepairSampleRate!;
389
- this.simpleMajorityThreshold = policy.simpleMajorityThreshold;
390
- this.superMajorityThreshold = policy.superMajorityThreshold;
391
- // Unlike the membership admission gate (which treats an absent assumedClusterSize as "unknown"
392
- // and admits — refusing writes outright is unacceptable), this falls back to the replication
393
- // factor and stays strict: the failure mode of getting this wrong is a block that goes
394
- // unrepaired, degraded rather than dead, so there is no reason to relax it for a caller that
395
- // has not adopted the new field. A real node is handed an explicit
396
- // `repairCorroborationClusterSize` by `resolveClusterPolicy`; the `assumedClusterSize` middle
397
- // term keeps direct constructors (embedders, existing tests) behaving as before.
398
- this.repairCorroborationClusterSize =
399
- cfg?.repairCorroborationClusterSize ?? policy.assumedClusterSize ?? policy.clusterSize;
400
- this.reputation = reputation;
401
- const localClusterRef = localCluster && localPeerId ? {
402
- update: localCluster.update.bind(localCluster),
403
- peerId: localPeerId,
404
- wasTransactionExecuted: localCluster.wasTransactionExecuted?.bind(localCluster),
405
- getExecutedPendResult: localCluster.getExecutedPendResult?.bind(localCluster),
406
- getExecutedCommitResult: localCluster.getExecutedCommitResult?.bind(localCluster)
407
- } : undefined;
408
- this.coordinator = new ClusterCoordinator(keyNetwork, createClusterClient, policy, localClusterRef, fretService, reputation, stateStore);
409
- }
410
-
411
- /**
412
- * The resolved super-majority threshold this coordinator commits on. Exposed so the composition root
413
- * can fail-fast if the coordinator and the cluster member would run different thresholds (see the
414
- * coupling assertion in `libp2p-node-base.ts`).
415
- */
416
- get effectiveSuperMajorityThreshold(): number {
417
- return this.superMajorityThreshold;
418
- }
419
-
420
- /** Recover coordinator transactions from persistent store after a restart. */
421
- async recoverTransactions(): Promise<void> {
422
- await this.coordinator.recoverTransactions();
423
- }
424
-
425
- /**
426
- * Check if this node is in the cluster for a given block.
427
- * Uses findCluster membership in the real network layer, self is always
428
- * included in the cohort when this node is responsible. This serves as a
429
- * defense-in-depth guard for requests that arrive at the wrong node.
430
- * Returns true if localPeerId is not set (backward compat for single-node/test setups).
431
- */
432
- private async isResponsibleForBlock(blockId: BlockId): Promise<boolean> {
433
- if (!this.localPeerId) return true;
434
-
435
- const cached = this.responsibilityCache.get(blockId);
436
- if (cached && cached.expires > Date.now()) {
437
- return cached.inCluster;
438
- }
439
-
440
- const blockIdBytes = new TextEncoder().encode(blockId);
441
- let inCluster: boolean;
442
- try {
443
- const peers = await this.keyNetwork.findCluster(blockIdBytes);
444
- inCluster = this.localPeerId.toString() in peers;
445
- } catch (err) {
446
- this.log('proximity:check-error', { blockId, error: (err as Error).message });
447
- // On failure, assume responsible to avoid false rejections
448
- return true;
449
- }
450
-
451
- this.responsibilityCache.set(blockId, { inCluster, expires: Date.now() + CoordinatorRepo.RESPONSIBILITY_TTL_MS });
452
- this.log('proximity:checked', { blockId, inCluster });
453
- return inCluster;
454
- }
455
-
456
- /**
457
- * Verify this node is responsible for all given block IDs. Throws if not.
458
- */
459
- private async verifyResponsibility(blockIds: BlockId[]): Promise<void> {
460
- const notResponsible: BlockId[] = [];
461
- for (const blockId of blockIds) {
462
- if (!await this.isResponsibleForBlock(blockId)) {
463
- notResponsible.push(blockId);
464
- }
465
- }
466
- if (notResponsible.length > 0) {
467
- this.log('proximity:rejected', { blockIds: notResponsible });
468
- throw new Error(`Not responsible for block(s): ${notResponsible.join(', ')}`);
469
- }
470
- }
471
-
472
- async get(blockGets: BlockGets, options?: MessageOptions): Promise<GetBlockResults> {
473
- // Soft proximity check — warn but still serve reads for graceful degradation
474
- // NOTE: a soft-served read now also *acquires* the block durably (see restoreCorroborated), where
475
- // before it could at most promote a pending this node already held. So a soft serve leaves behind
476
- // a replica of a block this node is not responsible for, and nothing sweeps those: ring-shift
477
- // sheds a keyspace RANGE, not "blocks outside my cohort". Fine while soft serves are what they
478
- // are meant to be a rare degradation during routing churn — since routing already placed this
479
- // node near the block. If they ever become routine, gate acquisition (not the serve itself) on
480
- // isResponsibleForBlock.
481
- for (const blockId of blockGets.blockIds) {
482
- if (!await this.isResponsibleForBlock(blockId)) {
483
- this.log('proximity:get-warning', { blockId, msg: 'serving read for non-responsible block' });
484
- }
485
- }
486
-
487
- // First try local storage
488
- const localResult = await this.storageRepo.get(blockGets, options);
489
-
490
- // Decide per-block whether to consult cluster peers. Two triggers:
491
- // (a) Missing block isn't present locally at all (legacy behavior).
492
- // (b) Stale-by-policy block is present but read-repair policy says verify.
493
- // Skip cluster fetch if this is already a sync request (to prevent recursive queries).
494
- // A sync read is also never marked `unavailable` here — the consult it skips is the
495
- // one whose failure the flag reports, and flagging would feed the recursion this
496
- // bypass exists to prevent. (Storage-level 'unmaterializable' flags still pass
497
- // through untouched; they report local state, not the consult.)
498
- const skipClusterFetch = (options as any)?.skipClusterFetch;
499
- // NOTE: NetworkTransactor.get treats an authoritative "absent" ({ state: {} })
500
- // as final and no longer retries it (ticket txn-perf-authoritative-notfound),
501
- // relying on this cluster reconciliation to have already run. When the consult
502
- // FAILS outright — or runs without ruling the block out and the block stays
503
- // missing — the entry is flagged `unavailable` below with a reason naming what
504
- // the consult established (see AbsenceVerdict and the mapping in the loop body),
505
- // which re-enables the transactor-level retry against a different peer. If a
506
- // coordinator is configured WITHOUT clusterLatestCallback, there is no cohort to
507
- // consult and the local answer IS the whole truth it stays authoritative, with
508
- // no flag and no transactor-level retry to compensate. That is fine (such a
509
- // coordinator has no cluster to reconcile against), but keep this coupling in
510
- // mind if a partial-cluster read path is added.
511
- if (this.clusterLatestCallback && !skipClusterFetch) {
512
- for (const blockId of blockGets.blockIds) {
513
- const localEntry = localResult[blockId];
514
- const localRev = localEntry?.state?.latest?.rev;
515
- const isMissing = !localEntry?.state?.latest;
516
- const isStale = !isMissing && this.shouldReadRepair(blockId);
517
- if (!isMissing && !isStale) {
518
- // No consult this pass — the read-repair window says this block was checked
519
- // recently. An unsettled claim an earlier pass recorded still applies: the doubt
520
- // is a property of what this node HOLDS, not of whether a consult just ran.
521
- // Without this, every read inside the window after a failed convergence would
522
- // serve the same content as confirmed the exact silent lie this marker exists
523
- // to end, re-opened for `readRepairWindowMs` at a time.
524
- this.flagUnconfirmedCurrency(localResult, blockId, blockGets.context);
525
- continue;
526
- }
527
-
528
- if (isStale) {
529
- this.log('cluster-tx:read-repair-triggered', {
530
- blockId,
531
- mode: this.readRepairMode,
532
- ageMs: this.ageMs(blockId),
533
- localRev
534
- });
535
- }
536
-
537
- try {
538
- const { absence, claimedAheadRev } = await this.fetchBlockFromCluster(blockId, blockGets.context, localRev);
539
- const refreshed = await this.storageRepo.get({ blockIds: [blockId], context: blockGets.context }, options);
540
- const newRev = refreshed[blockId]?.state?.latest?.rev;
541
- if (refreshed[blockId]) {
542
- localResult[blockId] = refreshed[blockId];
543
- }
544
- if (isStale) {
545
- if (typeof newRev === 'number' && typeof localRev === 'number' && newRev > localRev) {
546
- this.log('cluster-tx:read-repair-applied', { blockId, oldRev: localRev, newRev });
547
- } else {
548
- this.log('cluster-tx:read-repair-noop', { blockId });
549
- }
550
- }
551
- // The consult ran but could not rule the block out, and the verdict names the
552
- // evidence (see AbsenceVerdict): part of the cohort was silent (`unconfirmed`
553
- // 'peers-unreachable' another coordinator may know better), no cohort
554
- // member outside this node could be asked at all (`isolated`
555
- // 'cohort-unreachable' there is no better-connected coordinator to re-ask),
556
- // or a peer positively claimed a revision this pass could neither corroborate
557
- // nor acquire (`claimed` → 'claimed-elsewhere' — the block is known to exist
558
- // somewhere). Either way a still-missing block must not pose as an
559
- // authoritative absent. When the whole cohort answers "holds nothing" the
560
- // absent stays authoritative (`confirmed`) — the new-collection probe against
561
- // a healthy cohort stays one round-trip.
562
- if (isMissing && absence !== 'confirmed') {
563
- this.flagUnconfirmedAbsence(localResult, blockId,
564
- absence === 'claimed' ? 'claimed-elsewhere'
565
- : absence === 'isolated' ? 'cohort-unreachable'
566
- : 'peers-unreachable');
567
- }
568
- // A PRESENT block served below a cohort claim the repair could not settle is
569
- // the mirror lie: real content posing as confirmed-current. This consult is the
570
- // authority on that claim, so it replaces whatever an earlier one recorded —
571
- // including clearing it when nobody claims anything any more. The missing case
572
- // is excluded — it is the absence path above, and a bare absent below a claim
573
- // already reads as either authoritative (cohort answered, nothing corroborated)
574
- // or flagged.
575
- if (!isMissing) {
576
- this.recordAheadClaim(blockId, claimedAheadRev);
577
- this.flagUnconfirmedCurrency(localResult, blockId, blockGets.context);
578
- }
579
- } catch (err) {
580
- this.log('cluster-fetch:error', { blockId, error: (err as Error).message });
581
- // The consult that was supposed to make this answer trustworthy did not run.
582
- // NOTE: a consult that THROWS (e.g. `findCluster` itself rejected) is reported
583
- // 'peers-unreachable' even on an isolated node: a failed cohort lookup is a
584
- // routing failure and says nothing about how many cohort members were
585
- // reachable. If `findCluster` on an isolated node turns out to throw routinely
586
- // rather than return a stale cohort view, revisit that would put the
587
- // isolated case back under this vaguer reason.
588
- if (isMissing) {
589
- this.flagUnconfirmedAbsence(localResult, blockId, 'peers-unreachable');
590
- } else {
591
- // It told us nothing, so it refutes nothing: an earlier pass's unsettled
592
- // claim stands.
593
- this.flagUnconfirmedCurrency(localResult, blockId, blockGets.context);
594
- }
595
- }
596
- }
597
- }
598
-
599
- return localResult;
600
- }
601
-
602
- /**
603
- * Downgrade an absence the coordinator could not confirm to the given `unavailable` reason —
604
- * a flag `NetworkTransactor.get` retries against another peer instead of taking as final.
605
- * The reason names the evidence (see {@link AbsenceVerdict} for the mapping in `get`); this
606
- * method only decides WHETHER the entry may carry a flag at all.
607
- *
608
- * No-op once the entry carries a real answer (the consult restored the block) or a sharper flag
609
- * (storage's `'unmaterializable'`), so callers only need to establish that the answer is a guess.
610
- *
611
- * "Carries a real answer" is tested as `entry.block !== undefined`, NOT as `state.latest` being
612
- * set. The two used to move together, so `state.latest` read as a serviceable proxy — but a
613
- * pending-only insert (pended, not yet committed) served through the pending overlay has real
614
- * CONTENT and no committed revision at all, so its `state.latest` is undefined. Flagging that
615
- * entry would mark a block this node is positively holding as an unconfirmed absence, and
616
- * `NetworkTransactor`'s `isAuthoritative` keys off the flag alone the read would burn its
617
- * retry budget re-asking other peers for content it already has. `state.latest` stays in the
618
- * test as well so a stale-but-real committed answer is likewise never downgraded.
619
- */
620
- private flagUnconfirmedAbsence(results: GetBlockResults, blockId: BlockId, reason: BlockUnavailableReason): void {
621
- const entry = results[blockId];
622
- if (!entry) {
623
- results[blockId] = { state: {}, unavailable: reason };
624
- } else if (entry.block === undefined && !entry.state?.latest && entry.unavailable === undefined) {
625
- entry.unavailable = reason;
626
- }
627
- }
628
-
629
- /**
630
- * Remember (or forget) the cohort claim a freshness consult could not settle for a block.
631
- * Only a consult that actually RAN may call this: it is the authority, so `undefined` clears
632
- * a claim an earlier pass recorded. Entries are also dropped once this node reaches the
633
- * claimed revision (see {@link flagUnconfirmedCurrency}), which is what bounds the map.
634
- */
635
- private recordAheadClaim(blockId: BlockId, claimedRev: number | undefined): void {
636
- const prior = this.unsettledAheadClaims.get(blockId);
637
- if (claimedRev === undefined) {
638
- // The consult is the authority on the CLAIM, and only on the claim. A recorded deadlock is
639
- // not about any revision — it is about how many machines this deployment can field, or how
640
- // many of them hold the block so it outlives the claim that first exposed it and is
641
- // dropped only when the block converges (see {@link flagUnconfirmedCurrency}).
642
- if (prior?.deadlocksReported) this.unsettledAheadClaims.set(blockId, { deadlocksReported: prior.deadlocksReported });
643
- else this.unsettledAheadClaims.delete(blockId);
644
- return;
645
- }
646
- this.unsettledAheadClaims.set(blockId, {
647
- rev: claimedRev,
648
- ...(prior?.deadlocksReported ? { deadlocksReported: prior.deadlocksReported } : {})
649
- });
650
- }
651
-
652
- /**
653
- * Stamp {@link GetBlockResult.unconfirmedAheadRev} on an entry sitting behind an unsettled
654
- * cohort claim — served committed content the coordinator cannot confirm is current.
655
- * Deliberately narrow; ALL of these must hold:
656
- * - a consult (this read's or an earlier one's, see {@link recordAheadClaim}) left a claim
657
- * unsettled for this block;
658
- * - the entry carries a committed revision (a present block, or a committed tombstone) —
659
- * never a plain absent, which is the absence path's business;
660
- * - that served revision is still strictly BELOW the claim: the repair did not converge, and
661
- * nothing committed past the claim in the meantime (if it did, the claim is settled and the
662
- * memo is dropped here);
663
- * - the caller asked for a view that should contain the claim: an unpinned "latest" read, or
664
- * a pin at/above the claimed revision. A read pinned BELOW the claim is being served
665
- * correctly and stays unstamped this keeps a collection's context-pinned data reads
666
- * quiet while its unpinned tail read (the one seam where fresher truth could arrive —
667
- * Collection.bootstrapContext) speaks up.
668
- * NOT covered, on purpose: a cohort that is merely silent and claims nothing (pinned as
669
- * authoritative by the merely-STALE spec in coordinator-repo-unavailable.spec.ts) — silence
670
- * carries no revision to be behind of.
671
- *
672
- * Pin comparability: `ActionContext.rev` and a block's `state.latest.rev` count the same
673
- * per-collection revision sequence `Collection.bootstrapContext` seeds the context straight
674
- * from the tail block's `latest.rev`, and `syncInternal` commits every block of an action at
675
- * `context.rev + 1` — so `context.rev >= claimedRev` is a well-defined comparison. `state.latest`
676
- * is this node's newest revision for the block even on a pinned read (StorageRepo reports the
677
- * content's own revision separately as `materialized`), which is exactly the number "is this
678
- * node behind the claim?" asks about.
679
- */
680
- private flagUnconfirmedCurrency(results: GetBlockResults, blockId: BlockId, context?: ActionContext): void {
681
- const claimedRev = this.unsettledAheadClaims.get(blockId)?.rev;
682
- if (claimedRev === undefined) return;
683
- const entry = results[blockId];
684
- if (!entry || entry.unavailable !== undefined) return;
685
- const servedRev = entry.state?.latest?.rev;
686
- if (typeof servedRev !== 'number') return;
687
- if (servedRev >= claimedRev) {
688
- // Caught up — by this pass's repair or by a commit that landed since. Nothing to doubt, and
689
- // nothing deadlocked either: repair demonstrably converged for this block, so a later
690
- // non-convergence is a new episode and gets to say so again.
691
- this.unsettledAheadClaims.delete(blockId);
692
- return;
693
- }
694
- if (context !== undefined && context.rev < claimedRev) return;
695
- entry.unconfirmedAheadRev = claimedRev;
696
- this.log('cluster-tx:read-unconfirmed', { blockId, servedRev, claimedAheadRev: claimedRev });
697
- }
698
-
699
- /** Decide whether the read-repair policy wants us to consult the cluster for a present-but-possibly-stale block. */
700
- private shouldReadRepair(blockId: BlockId): boolean {
701
- switch (this.readRepairMode) {
702
- case 'off': return false;
703
- case 'paranoid': return true;
704
- case 'lazy': {
705
- const lastSeen = this.lastSeenCommitMs.get(blockId);
706
- if (lastSeen == null) return true;
707
- if (this.now() - lastSeen > this.readRepairWindowMs) return true;
708
- if (this.readRepairSampleRate > 0 && this.rand() < this.readRepairSampleRate) return true;
709
- return false;
710
- }
711
- }
712
- }
713
-
714
- /** Milliseconds since we last marked this block fresh, or undefined if never. */
715
- private ageMs(blockId: BlockId): number | undefined {
716
- const lastSeen = this.lastSeenCommitMs.get(blockId);
717
- return lastSeen == null ? undefined : this.now() - lastSeen;
718
- }
719
-
720
- /** Mark blocks as freshly observed from cluster authority (post-commit or post-fetch). */
721
- private markBlocksSeen(blockIds: BlockId[]): void {
722
- const now = this.now();
723
- for (const id of blockIds) {
724
- this.lastSeenCommitMs.set(id, now);
725
- }
726
- }
727
-
728
- /**
729
- * Test seam: directly set the last-seen timestamp for a block. Used by read-repair
730
- * specs to simulate "the local commit happened at time T" without needing to drive
731
- * a full pend/commit cycle through the cluster coordinator.
732
- */
733
- setLastSeenForTest(blockId: BlockId, ts: number): void {
734
- this.lastSeenCommitMs.set(blockId, ts);
735
- }
736
-
737
- /**
738
- * One repair pass for a block: ask the cohort what it holds, and converge onto that if it is
739
- * ahead of `localRev` the revision the caller's read already loaded, and the baseline every
740
- * decision below is measured against.
741
- *
742
- * Returns the two things `get` needs beyond the storage side effects:
743
- * - `absence` — the verdict on this node's local absence of the block (see
744
- * {@link AbsenceVerdict}): whether the pass may rule the block out, and on what evidence.
745
- * Only `'confirmed'` lets a still-missing block be reported as an authoritative absent.
746
- * Paths that consult nobody (no cohort, solo-self) are `'confirmed'`: there, the local
747
- * answer genuinely is the whole truth. When several verdicts apply at once the sharpest
748
- * evidence wins: `claimed` > `isolated` > `unconfirmed` > `confirmed` — a peer positively
749
- * saying "it exists" outranks any amount of silence.
750
- * - `claimedAheadRev` — a cohort peer claimed a revision strictly ahead of what this node
751
- * holds and the pass did NOT converge onto it: the claim failed the corroboration quorum,
752
- * or was corroborated but could not be acquired. Content `get` serves below this revision
753
- * cannot be confirmed current (see {@link GetBlockResult.unconfirmedAheadRev}); the claim
754
- * itself must never drive restoration.
755
- */
756
- private async fetchBlockFromCluster(blockId: BlockId, context?: ActionContext, localRev?: number): Promise<{ absence: AbsenceVerdict; claimedAheadRev?: number }> {
757
- if (!this.clusterLatestCallback) return { absence: 'confirmed' };
758
-
759
- const blockIdBytes = new TextEncoder().encode(blockId);
760
- const peers = await this.keyNetwork.findCluster(blockIdBytes);
761
- const peerIds = peers ? Object.keys(peers) : [];
762
- if (peerIds.length === 0) return { absence: 'confirmed' };
763
-
764
- // Solo-cluster short-circuit: the only responsible peer is us. There is no
765
- // remote to sync from, so skip the callback entirely. Querying ourselves
766
- // would dial self via SyncClient — pointless at best, and on nodes without
767
- // listen addresses (e.g. solo WebSocket-only) the dial can hang.
768
- if (
769
- peerIds.length === 1
770
- && this.localPeerId
771
- && peerIds[0] === this.localPeerId.toString()
772
- ) {
773
- this.log('cluster-fetch:solo-self-skip', { blockId });
774
- return { absence: 'confirmed' };
775
- }
776
-
777
- const { corroborated, local, silent, answered, uncorroboratedRev } = await this.queryClusterForLatest(peerIds, blockId, context);
778
- // Any silence taints the WHOLE consult, not a fraction of it (fail-closed): one silent
779
- // peer could be the sole holder, and the cost an extra transactor-level retry against
780
- // another coordinator is paid only while a peer is actually unreachable. Silence with
781
- // NOBODY else reached at all is its own verdict: partial silence says "ask a better-
782
- // connected coordinator", total silence says there is no better-informed answer to be
783
- // had from this node.
784
- const silenceVerdict: AbsenceVerdict =
785
- silent.length > 0 ? (answered === 0 ? 'isolated' : 'unconfirmed') : 'confirmed';
786
- // Nothing corroborated: keep local data AND stay eligible for repair — marking the
787
- // block seen here would suppress the next attempt for the whole read-repair window.
788
- // An uncorroborated claim strictly ahead of what this node holds still travels up as
789
- // doubt: the answer about to be served may be behind it, and only the caller knows
790
- // whether that matters for the view it was asked for.
791
- if (!corroborated) {
792
- const uncorroboratedBaseline = local?.rev ?? localRev;
793
- const claimIsAhead = uncorroboratedRev !== undefined
794
- && (uncorroboratedBaseline === undefined || uncorroboratedRev > uncorroboratedBaseline);
795
- // A claim — even one the quorum declined — is a peer positively attesting the block
796
- // exists, the sharpest fact this pass can surface. It outranks silence.
797
- const absence: AbsenceVerdict = uncorroboratedRev !== undefined ? 'claimed' : silenceVerdict;
798
- return { absence, ...(claimIsAhead ? { claimedAheadRev: uncorroboratedRev } : {}) };
799
- }
800
-
801
- // The self answer is the sharper baseline (same storage, same context, read alongside the
802
- // cohort's), but it exists only when `findCluster` returned this node. A soft serve for a
803
- // block this node is no longer responsible for is absent from its own cohort view, so fall
804
- // back to the revision the caller's read already loaded. Without the fallback both decisions
805
- // below degrade to "any local revision is an advance", which restores backwards and reports
806
- // a sync at the revision the pass started from.
807
- const baselineRev = local?.rev ?? localRev;
808
-
809
- // Never restore backwards. With this node's own claim excluded from the quorum, a
810
- // cohort that lags behind the reader corroborates an OLDER revision; adopting it
811
- // would be a regression, and logging it as a sync would be a lie. The cohort did
812
- // answer, so the block is verified fresh mark it seen.
813
- // NOTE: in a cohort of two, that sole peer is the only corroborator, so a lying one can park
814
- // the reader here corroborating the revision it already holds — and re-arm the lazy window
815
- // on every pass, hiding a real divergence. Bounded by `readRepairWindowMs` (10s default) and
816
- // no worse than the peer simply staying silent. If two-member cohorts become a supported
817
- // production topology rather than a dev convenience, stop re-arming the window on a
818
- // corroboration that came from a single voter.
819
- if (baselineRev !== undefined && corroborated.rev <= baselineRev) {
820
- this.log('cluster-fetch:local-current', { blockId, localRev: baselineRev, clusterRev: corroborated.rev });
821
- this.markBlocksSeen([blockId]);
822
- // Only reachable when this node HOLDS a revision (the baseline), so `get` never
823
- // consults this verdict — computed consistently rather than hard-coded.
824
- return { absence: silenceVerdict };
825
- }
826
-
827
- // Corroborated revision is ahead of oursconverge onto it.
828
- const rev = await this.restoreCorroborated(blockId, corroborated, baselineRev, peerIds);
829
-
830
- // Log the OUTCOME, not the attempt. Logging `synced` unconditionally reported hundreds of
831
- // phantom convergences per run and made a real replication defect invisible for two debugging
832
- // sessions.
833
- if (rev !== undefined) {
834
- this.log('cluster-fetch:synced', { blockId, rev });
835
- } else {
836
- this.log('cluster-fetch:not-restored', { blockId, localRev: baselineRev, clusterRev: corroborated.rev });
837
- }
838
- // A corroborated revision this node failed to converge onto rules nothing out, even with
839
- // the whole cohort answering: the reader has just been TOLD the block exists, so reporting
840
- // it absent would be a lie regardless of silence — that is the `claimed` verdict, and it
841
- // outranks whatever the silence-based mapping would have said.
842
- const absence: AbsenceVerdict = rev === undefined ? 'claimed' : silenceVerdict;
843
- // Converged means REACHED the corroborated revision, not merely advanced: a promotion that
844
- // landed short of it (possible in principlerestoreCorroborated only requires an advance
845
- // over the baseline) still leaves the served answer behind a revision the cohort attested.
846
- const converged = rev !== undefined && rev >= corroborated.rev;
847
- // The block is marked seen either way the cohort DID answer, so its freshness was checked,
848
- // which is what the read-repair window tracks. A failed convergence therefore waits out the
849
- // window before retrying. The DOUBT it produced does not wait: `get` remembers the
850
- // unsettled claim (`recordAheadClaim`) and keeps stamping reads served below it while the
851
- // window suppresses the retry the window damps repair effort, not honesty.
852
- // NOTE: that damping covers only a block this node holds at an OLDER revision. A block entirely
853
- // missing locally never consults the window (`get` triggers on `isMissing` before
854
- // `shouldReadRepair`), so a persistently failing acquisition e.g. a two-node deployment that
855
- // never set `assumedClusterSize`, where the content quorum can never be met — re-fetches an
856
- // archive on every read of that block. Correct, and self-limiting once the cohort can agree; if
857
- // it ever shows as read amplification, gate the acquisition step (not the latest-query) on the
858
- // same window rather than widening `isMissing`.
859
- this.markBlocksSeen([blockId]);
860
- return { absence, ...(converged ? {} : { claimedAheadRev: corroborated.rev }) };
861
- }
862
-
863
- /**
864
- * Bring this node up to the cohort-corroborated `corroborated`, returning the revision it holds
865
- * afterwards when that is an advance over `baselineRev`, else `undefined`.
866
- *
867
- * Two mechanisms, cheapest first:
868
- * 1. **Promote a local pending** — free, no network, and the only mechanism that existed before
869
- * block acquisition. Covers the node that saw the pend and missed the commit broadcast.
870
- * 2. **Acquire the bytes from the cohort** ({@link AcquireBlockCallback}) covers everything else,
871
- * including a block this node has never seen at all.
872
- *
873
- * **Why acquisition is gated here and not on a plain local miss.** `BlockStorage.getBlock` returns
874
- * `undefined` for a block with no local metadata *without* consulting its restore callback, so that
875
- * an insert probing a fresh random block id for a collision does not cost a network fetch. That
876
- * remains true: this method runs only after {@link queryClusterForLatest} produced a quorum-
877
- * corroborated `(rev, actionId)`, which a genuinely non-existent block can never produce (no peer
878
- * claims it, so `selectQuorumRev` declines and `fetchBlockFromCluster` returns before reaching
879
- * here). The cost of a genuine absence is unchanged — the latest-query round trip that already
880
- * happened while a block the cohort demonstrably holds is no longer thrown away.
881
- *
882
- * Cohort peer ids are passed straight through: the callback filters self out and caps its own
883
- * corroboration quorum by how many peers could answer at all.
884
- */
885
- private async restoreCorroborated(
886
- blockId: BlockId,
887
- corroborated: ActionRev,
888
- baselineRev: number | undefined,
889
- cohortPeerIds: string[]
890
- ): Promise<number | undefined> {
891
- const promoted = await this.promoteCorroborated(blockId, corroborated);
892
- if (isAdvanceOver(promoted, baselineRev)) {
893
- return promoted;
894
- }
895
-
896
- if (!this.acquireBlockFromCohort) {
897
- return undefined;
898
- }
899
- try {
900
- // Bounded: a stalled cohort peer must not hold up the caller's read. Persisting happens
901
- // inside the callback via `saveReplicatedBlock`, which takes the block write latch
902
- // safe to call from here because the read path holds no latch of its own (`StorageRepo.get`
903
- // acquires and releases it around the promotion above, and nothing wraps this method).
904
- // NOTE: `get` walks its block ids sequentially, so the bound is per block, not per call — a
905
- // multi-block read that is missing N blocks against a wholly stalled cohort waits N × this.
906
- // Acceptable today (the underlying per-peer archive fetch is itself 1s-bounded and runs the
907
- // cohort in parallel, so the 5s is a stall ceiling, not a typical cost). If a cold reader
908
- // batching a wide read ever times out above this layer, repair the block ids concurrently
909
- // rather than shortening the bound.
910
- await withDeadline(
911
- this.acquireBlockFromCohort(blockId, corroborated, cohortPeerIds),
912
- RECONCILE_TIMEOUT_MS,
913
- `block acquisition for ${blockId}`
914
- );
915
- } catch (err) {
916
- // Declines are cheap and retryable — nothing was persisted. Report and leave the block behind.
917
- this.log('cluster-fetch:acquire-error', { blockId, rev: corroborated.rev, error: (err as Error).message });
918
- return undefined;
919
- }
920
- const acquired = await this.readLocalRev(blockId);
921
- return isAdvanceOver(acquired, baselineRev) ? acquired : undefined;
922
- }
923
-
924
- /**
925
- * Promote a corroborated action this node already holds as a local pending — the no-network half of
926
- * the repair. Returns the local revision afterwards.
927
- *
928
- * A pending-only block (metadata seeded by `savePendingTransaction`, no committed revision) asked
929
- * for a forward revision no promotion can reach used to throw out of the restore step (now
930
- * `BlockStorage.restoreRevision`, driven by `StorageRepo.get`'s healing helper).
931
- * It no longer does: "no committed base here" is an absence, so that read comes back as a plain
932
- * unflagged `{ state: {} }` and this method simply returns `undefined` acquisition then supplies
933
- * the revision. The `unavailable` arm below still fires for the shapes that ARE a guess (a `latest`
934
- * this node cannot materialize, a missing-base promotion refusal); on THIS path those are an
935
- * absence too rather than a read failure, so they are logged as `promote-unavailable` and stepped
936
- * over rather than short-circuiting the caller. The catch stays for any other fault, same reason.
937
- */
938
- private async promoteCorroborated(blockId: BlockId, corroborated: ActionRev): Promise<number | undefined> {
939
- try {
940
- const entry = await this.readLocalEntry(blockId, { committed: [corroborated], rev: corroborated.rev });
941
- if (entry?.unavailable !== undefined) {
942
- this.log('cluster-fetch:promote-unavailable', { blockId, rev: corroborated.rev, error: entry.unavailable });
943
- return undefined;
944
- }
945
- return entry?.state?.latest?.rev;
946
- } catch (err) {
947
- this.log('cluster-fetch:promote-unavailable', { blockId, rev: corroborated.rev, error: (err as Error).message });
948
- return undefined;
949
- }
950
- }
951
-
952
- /** This node's own answer for a block, optionally driving a promotion context through the read.
953
- * Callers that care whether the answer is authoritative inspect `entry.unavailable`. */
954
- private async readLocalEntry(blockId: BlockId, context?: ActionContext) {
955
- const result = await this.storageRepo.get({ blockIds: [blockId], context });
956
- return result[blockId];
957
- }
958
-
959
- /** This node's own `latest.rev` for a block, optionally driving a promotion context through the read. */
960
- private async readLocalRev(blockId: BlockId, context?: ActionContext): Promise<number | undefined> {
961
- return (await this.readLocalEntry(blockId, context))?.state?.latest?.rev;
962
- }
963
-
964
- /**
965
- * Query cluster peers for their latest revision and return the highest revision
966
- * corroborated by a quorum of distinct peers, alongside this node's own latest.
967
- *
968
- * Replaces the old "max rev any single peer reports" which let one lying
969
- * peer over-reporting its revision steer restoration — with quorum
970
- * corroboration on the exact `(rev, actionId)` pair (see {@link selectQuorumRev}).
971
- *
972
- * This node's own answer is split out of the claim set rather than counted in it:
973
- * `clusterLatestCallback` short-circuits self to local storage, so including it let a
974
- * reader whose only peer timed out "corroborate" the very revision it was trying to
975
- * repair. It is returned separately so the caller can compare, not vote.
976
- *
977
- * NOTE: the quorum is corroboration-of-a-claim, NOT Sybil-resistant cohort
978
- * membership a peer minting fresh keypairs still casts a vote. A claim that arrives
979
- * with a cohort commit proof is additionally VERIFIED here (`certifyClaim`,
980
- * `cluster/certified-claims.ts`); when the proof holds, the claim is certified and
981
- * {@link selectQuorumRev} accepts it without a second voter — the cohort's signature set
982
- * is its corroboration. What a passing proof does NOT prove is that its signers are the
983
- * block's responsible cohort (anyone controlling N keys can sign their own N-peer
984
- * proof); anchoring the signer set to topology is the optional, observational-only
985
- * {@link ProofAnchoring} layer, unwired in production today.
986
- */
987
- private async queryClusterForLatest(peerIds: string[], blockId: BlockId, context?: ActionContext): Promise<ClusterLatestQuery> {
988
- // Query peers in parallel for their latest revision. Each query is DEADLINED (rejects), not
989
- // raced-to-undefined: a peer that blows the deadline lands in the silent set below exactly
990
- // like a dial failure, because a slow peer and a peer claiming "I hold nothing" must produce
991
- // different answers (ticket cluster-read-consult-cannot-report-unreachable).
992
- // NOTE: LATEST_QUERY_TIMEOUT_MS is a LAN-shaped budget. A cohort whose round trip honestly
993
- // exceeds it now reads as permanently silent, which is safe (the read is flagged, not
994
- // mis-reported) but makes every miss cost a transactor-level retry. If a WAN deployment shows
995
- // steady `cluster-fetch:peers-silent` against healthy peers, raise this rather than softening
996
- // the deadline back into an absent claim.
997
- const latestResults = await Promise.allSettled(
998
- peerIds.map(async peerIdStr => {
999
- const peerId = peerIdFromString(peerIdStr);
1000
- return await withDeadline(
1001
- this.clusterLatestCallback!(peerId, blockId, context),
1002
- LATEST_QUERY_TIMEOUT_MS,
1003
- `latest query to ${peerIdStr}`
1004
- );
1005
- })
1006
- );
1007
-
1008
- // NOTE: self-exclusion is keyed on `localPeerId`, which is optional for the single-node/test
1009
- // construction this class has always tolerated. Left unset, this node's own answer is counted
1010
- // as a peer claim again. Harmless today the self answer can only ever corroborate the
1011
- // revision already held, so the pass declines as `local-current` — but if a future caller can
1012
- // make self report something the reader does not hold, make `localPeerId` required instead.
1013
- // The same unset-`localPeerId` tolerance also lets self count toward `answered` below, and
1014
- // lets a self read that REJECTS land in `silent`: a solo repo whose own storage throws then
1015
- // reads as `answered === 0` and reports isolation ('cohort-unreachable') rather than a local
1016
- // fault. Same fix if it ever matters require `localPeerId`.
1017
- const selfId = this.localPeerId?.toString();
1018
- let local: CertifiedActionRev | undefined;
1019
- const claims: RevClaim[] = [];
1020
- const silent: string[] = [];
1021
- // `allSettled` preserves input order, so results correlate to `peerIds` by index a
1022
- // rejected entry carries no payload of its own, and its peer id is what `silent` records.
1023
- for (let i = 0; i < latestResults.length; i++) {
1024
- const result = latestResults[i]!;
1025
- const peerIdStr = peerIds[i]!;
1026
- if (result.status !== 'fulfilled') {
1027
- // Silence: the callback rejected or the deadline expired. Never a claim. Self is
1028
- // excluded its short-circuit reads local storage, and a local read error is not a
1029
- // cohort peer being unreachable.
1030
- if (peerIdStr !== selfId) silent.push(peerIdStr);
1031
- continue;
1032
- }
1033
- const value = result.value;
1034
- if (peerIdStr === selfId) {
1035
- local = value;
1036
- continue;
1037
- }
1038
- if (!value) continue; // responded, holds nothing — an absent claim, not silence
1039
- // The proof rides along here and is verified BELOW (certifyClaim) before selection reads
1040
- // the claim set: presence proves nothing the peer chose what to attach — but a proof
1041
- // that verifies certifies the claim, and a certified claim needs no second voter.
1042
- claims.push({
1043
- peerId: peerIdStr, rev: value.rev, actionId: value.actionId,
1044
- ...(value.proof ? { proof: value.proof } : {})
1045
- });
1046
- }
1047
- if (silent.length > 0) {
1048
- this.log('cluster-fetch:peers-silent', { blockId, silent: silent.length, consulted: peerIds.length });
1049
- }
1050
-
1051
- // Verify every attached proof, in parallel, BEFORE selection — and penalize provable proof
1052
- // misbehavior HERE, at verification time, independent of what selection later does with the
1053
- // claim. Only attributable failures (isAttributableProofFailure) are penalized: a failure
1054
- // whose signer identities were never proven unknown/non-ed25519 signer, malformed
1055
- // signature or proof, a legacy record, the oversized-cohort cap — could have been authored
1056
- // by anyone in the chain, and penalizing on it would let an attacker frame a peer (the same
1057
- // discipline as VerifyOutcome.penalize in cluster-repo.ts). A claim whose proof fails stays
1058
- // in the claim set UNCERTIFIED: it still corroborates by distinct-peer count exactly as a
1059
- // proof-less claim does a peer that could fabricate a bad proof could equally have sent no
1060
- // proof, so dropping the vote would buy nothing.
1061
- // NOTE: cost is one verification pass per proof-carrying answer per consult, each bounded by
1062
- // MAX_PROOF_SIGNERS (256) signature checks. In `lazy` mode consults are rate-limited by the
1063
- // read-repair window; in `paranoid` mode every read of every block pays cohort-width
1064
- // verifications. Fine at deployment cohort sizes (~10) — if paranoid readers ever show CPU
1065
- // time in `certifyClaim`, cache verdicts per (blockId, rev, actionId, proof hash) rather than
1066
- // skipping verification.
1067
- await Promise.all(claims.map(async claim => {
1068
- if (!claim.proof) return;
1069
- const verdict = await certifyClaim(
1070
- claim.proof,
1071
- { blockId, rev: claim.rev, actionId: claim.actionId },
1072
- // Shared with the reconcile path, so the two cannot drift on what the members actually
1073
- // enforced see `proofThresholds` for why the simple-majority term is not
1074
- // this.simpleMajorityThreshold.
1075
- proofThresholds(this.superMajorityThreshold),
1076
- this.proofAnchoring
1077
- );
1078
- if (verdict.certified) {
1079
- claim.certified = true;
1080
- return;
1081
- }
1082
- this.log('cluster-fetch:proof-uncertified', {
1083
- blockId, peerId: claim.peerId, rev: claim.rev, failure: verdict.failure
1084
- });
1085
- if (isAttributableProofFailure(verdict.failure)) {
1086
- this.penalizeProofService(claim.peerId, blockId);
1087
- }
1088
- }));
1089
-
1090
- const nonSelfCount = peerIds.filter(id => id !== selfId).length;
1091
- const answered = nonSelfCount - silent.length;
1092
- const capacity = corroboratorCapacity(nonSelfCount, this.repairCorroborationClusterSize);
1093
- const required = quorumSize(claims.length, this.simpleMajorityThreshold, capacity);
1094
- const selected = selectQuorumRev(claims, this.simpleMajorityThreshold, capacity);
1095
- if (!selected) {
1096
- // A decline can be the certified path REFUSING to pick a side: two distinct actions each
1097
- // carrying a verified cohort proof for the same top revision. Name that apart from the
1098
- // routine no-quorum — the cohort (or whoever holds its keys) provably signed both sides,
1099
- // an incident rather than a shortage of answers. Neither claimant is penalized: both
1100
- // proofs verified, so which side is "wrong" is exactly what this node cannot know.
1101
- const equivocation = certifiedEquivocation(claims);
1102
- if (equivocation) {
1103
- this.log('cluster-fetch:certified-equivocation', {
1104
- blockId, rev: equivocation.rev, actionIds: equivocation.actionIds
1105
- });
1106
- }
1107
- // The three populations are reported SEPARATELY, never rolled into one "responders" count:
1108
- // "1 of 2 responded" and "1 holder, 1 confirmed non-holder, 0 silent" call for completely
1109
- // different operator actions — the first says wait or fix reachability, the second says the
1110
- // block has only one copy and no amount of waiting produces a second.
1111
- this.log('cluster-fetch:no-quorum', {
1112
- blockId,
1113
- cohortPeers: nonSelfCount,
1114
- holders: claims.length,
1115
- absent: answered - claims.length,
1116
- silent: silent.length,
1117
- required,
1118
- repairCorroborationClusterSize: this.repairCorroborationClusterSize
1119
- });
1120
- // ...and, when this decline is provably permanent rather than transient, say THAT once,
1121
- // in words. The `no-quorum` line above fires on every pass and cannot tell the two apart.
1122
- this.reportRepairDeadlock({
1123
- blockId, claims, silentCount: silent.length, cohortPeers: nonSelfCount, answered, required, capacity
1124
- });
1125
- // The claims themselves must not drive restoration — but their existence is
1126
- // evidence the caller needs: an answer served below the highest claim cannot be
1127
- // confirmed current (see ClusterLatestQuery.uncorroboratedRev).
1128
- // NOTE: ONE claim is enough to raise that doubt, and the claims reaching this branch
1129
- // are unverified assertions a certified claim converges above instead of declining
1130
- // (the only certified shape that lands here is the equivocation decline). So a single
1131
- // lying cohort peer can deny unpinned reads of a block by claiming a revision nobody
1132
- // else holds an availability lever it did not have while uncorroborated claims were
1133
- // discarded. Deliberate for now: the alternative is the silent stale serve this marker
1134
- // exists to end, and the same liar can already force a silent-treated absence by
1135
- // staying quiet. If the lever is ever exercised, gate the stamp on a certified claim
1136
- // (the verification machinery now exists) rather than on the bare assertion — at the
1137
- // cost of re-opening the stale-serve window for the proof-less honest majority.
1138
- const uncorroboratedRev = claims.length > 0 ? Math.max(...claims.map(c => c.rev)) : undefined;
1139
- return { local, silent, answered, ...(uncorroboratedRev !== undefined ? { uncorroboratedRev } : {}) };
1140
- }
1141
-
1142
- if (selected.certified) {
1143
- // Which rule won matters when reading a repair log: a certified selection may rest on a
1144
- // SINGLE claimant whose corroboration is the cohort's signature set, not other voters.
1145
- this.log('cluster-fetch:certified-selected', {
1146
- blockId, rev: selected.rev, claimants: selected.supporters.length
1147
- });
1148
- }
1149
-
1150
- // Best-effort: penalize peers whose claim contradicts a CORROBORATED selection — a different
1151
- // action at the very same revision. A higher rev may be honest leadership and a lower rev is
1152
- // just lag; neither is penalized, nor is anything contradicting a certified-only selection
1153
- // (an unanchored proof must not be able to convict the honest cohort). Never let this throw.
1154
- this.penalizeContradictingRevClaims(claims, selected, blockId);
1155
-
1156
- return { corroborated: { actionId: selected.actionId, rev: selected.rev }, local, silent, answered };
1157
- }
1158
-
1159
- /**
1160
- * Say ONCE per block, in words, when a corroboration decline is provably PERMANENT rather than a
1161
- * transient shortage of answers. There are exactly TWO permanent shapes, and they send the operator
1162
- * to different places, so each gets its own `reason` and its own wording:
1163
- *
1164
- * - `cohort-too-small` this node's cohort has fewer peers than the quorum would demand even if
1165
- * every one of them answered and agreed. The remedy is machines or an honest declared size.
1166
- * - `sole-holder` — the cohort is big enough, but exactly ONE of its peers holds the block at all
1167
- * and every other peer answered that it holds nothing. The remedy is another cohort peer
1168
- * holding the block; machines and configuration are both irrelevant. Note the scope: this node's
1169
- * own copy is excluded from the claim set, so a reader that holds the block itself still sees
1170
- * `sole-holder` the message says "cohort peer", never "machine in the deployment".
1171
- *
1172
- * **What makes `cohort-too-small` provable.** Not "this pass fell short" a pass falls short
1173
- * whenever some peer simply does not hold the block *yet*. The decisive question is whether the
1174
- * cohort could supply the quorum AT ALL: ask what would be required if every cohort peer answered
1175
- * and agreed — the best case any later pass can reach without new machines — and compare it to how
1176
- * many peers the cohort has. Short of that best case the shortfall is not the machine count, and
1177
- * saying PERMANENT would send the operator to change a number that was never the problem. Twelve
1178
- * days of log archaeology went into re-deriving the real condition from a thousand identical
1179
- * `cluster-fetch:no-quorum` lines; the node knows it at the moment of each decline.
1180
- *
1181
- * **What makes `sole-holder` provable.** Note it is only reachable for a lone UNCERTIFIED holder:
1182
- * a lone holder whose cohort commit proof verified is selected by the certified path and converges
1183
- * before any decline — so the wording's "a lone holder cannot second itself" stays accurate for
1184
- * every claim that gets here. "That peer will hold it later" is an assumption, and for a
1185
- * peer that ANSWERED "I hold nothing" it is false: the only two mechanisms that would turn a
1186
- * non-holder into a holder `queryClusterForLatest` (read-repair) and `createReconcileBlock`
1187
- * (reconcile) consume this very decision, so they decline for exactly the same reason on that
1188
- * peer. Every peer answered, one holds the block, the rest hold nothing, and no later pass changes
1189
- * any of that. What DOES change it is a new copy: a commit that writes the block again pushes it to
1190
- * the current cohort. (Sibling work `replicate-owned-blocks-when-the-cohort-grows` makes that
1191
- * automatic; until it lands the operator has to cause the write.)
1192
- *
1193
- * **What is deliberately NOT reported.** A cohort that answers unanimously "I hold nothing" — an
1194
- * agreed absence is an answer, not a failed repair. A pass with any silent peer: silence cannot
1195
- * change the arithmetic (`cohortPeers` counts silent peers too), but it does mean this node saw less
1196
- * than the whole picture, and the next clean pass says the same thing at no cost. Note there is
1197
- * deliberately NO "the claims disagreed" exemption for `cohort-too-small`: a cohort too small to
1198
- * reach quorum stays too small whether its peers agree or not, so disagreement would suppress a line
1199
- * that is still true. Two or more disagreeing holders DO suppress `sole-holder`, because that is a
1200
- * cohort with two copies whose peers have not settled yet a later pass can settle it.
1201
- *
1202
- * **Never a lever.** This only classifies and logs; it never relaxes a floor. Which is also why the
1203
- * `cohort-too-small` message names *two* readings of the same numbers a deployment that genuinely
1204
- * runs this few machines, or a cohort view shrunk below the real deployment by a partition or by an
1205
- * attacker with routing influence. `corroboratorCapacity` keeps the shrunken view out of the relaxed
1206
- * branch, but this node cannot tell the two apart from the inside, and an operator sent to fix the
1207
- * wrong one is the failure this line exists to end.
1208
- *
1209
- * NOTE: the reader is still told only "this may be stale" `BlockPossiblyStaleError` implies a
1210
- * retry might help, which is wrong advice for a block whose repair is deadlocked as configured.
1211
- * Carrying this condition into the error needs a new field on `GetBlockResult` plus a change to
1212
- * that error's documented contract; deliberately out of scope here (see the ticket
1213
- * `repair-deadlock-is-never-named`, *Not this ticket*).
1214
- */
1215
- private reportRepairDeadlock(pass: {
1216
- blockId: BlockId;
1217
- claims: RevClaim[];
1218
- silentCount: number;
1219
- /** Cohort peers besides this node, from the cohort view whether they answered or not. */
1220
- cohortPeers: number;
1221
- answered: number;
1222
- /** The quorum THIS pass demanded, computed from the peers that actually claimed. */
1223
- required: number;
1224
- /** `corroboratorCapacity` for this pass a function of the view and the resolved size, not of who answered. */
1225
- capacity: number;
1226
- }): void {
1227
- const { blockId, claims, silentCount, cohortPeers, answered, required, capacity } = pass;
1228
- // An incomplete picture proves nothing about the deployment; the next clean pass says it.
1229
- if (silentCount > 0) return;
1230
- // Nobody claimed anything: the cohort agrees the block is absent, which is an answer, not a
1231
- // deadlock.
1232
- if (claims.length === 0) return;
1233
- // The decisive test for the first shape. `requiredEvenIfAllAnswered` is the quorum this cohort
1234
- // would face with every one of its peers answering and agreeing the best case reachable
1235
- // without adding machines. A cohort that can meet it is not too small.
1236
- const requiredEvenIfAllAnswered = quorumSize(cohortPeers, this.simpleMajorityThreshold, capacity);
1237
- const cohortTooSmall = cohortPeers < requiredEvenIfAllAnswered;
1238
- // The second shape: exactly one cohort peer holds the block AT ALL, and since a claim is one
1239
- // peer's latest, so a single claim is a single distinct (rev, actionId) group with a single
1240
- // supporter every other cohort peer answered that it holds nothing. `answered === cohortPeers`
1241
- // is already implied by the silence guard above; it is stated because the two counts arrive as
1242
- // independent parameters and "everybody answered" is half of what makes this provable.
1243
- //
1244
- // NOTE: there is a narrow window where `sole-holder` is true of the instant but not of the
1245
- // deployment a commit that has landed on one cohort member and has not yet been pushed to the
1246
- // rest presents exactly this shape. Calling it PERMANENT is defensible even there (repair
1247
- // genuinely cannot converge until the push lands, and the once-per-episode flag clears the
1248
- // moment the block converges, so the line does not repeat), and widening the window is what the
1249
- // push path's own threat model decides — see
1250
- // `tickets/blocked/repair-floor-defends-a-door-the-push-path-leaves-open`. If commit-to-push
1251
- // latency ever grows enough that operators see `sole-holder` on blocks that heal moments later,
1252
- // gate the line on the block having been quiet for longer than that latency rather than
1253
- // softening the wording.
1254
- const soleHolder = claims.length === 1 && answered === cohortPeers;
1255
- if (!cohortTooSmall && !soleHolder) return;
1256
-
1257
- // Both shapes can hold at once (an undeclared two-machine deployment whose single peer holds the
1258
- // block is both). `cohort-too-small` is reported in preference because its remedy is the one
1259
- // that actually works there: declaring the real size makes the floor reachable, after which the
1260
- // lone peer's claim IS adopted — so calling it a sole-holder problem would send the operator
1261
- // looking for a copy they do not need.
1262
- const reason: DeadlockReason = cohortTooSmall ? 'cohort-too-small' : 'sole-holder';
1263
- const state = this.unsettledAheadClaims.get(blockId);
1264
- const alreadySaid = state?.deadlocksReported ?? [];
1265
- // Suppressed per REASON, not once outright: an episode that starts as `cohort-too-small` and
1266
- // becomes `sole-holder` — the operator added the machines that reason asked for, and the block
1267
- // is still stuck — has a second thing to say, and a silent log there is the failure this line
1268
- // exists to end. Neither reason repeats within an episode.
1269
- if (alreadySaid.includes(reason)) return;
1270
-
1271
- this.log('cluster-fetch:repair-deadlock', {
1272
- blockId,
1273
- reason,
1274
- cohortPeers,
1275
- answered,
1276
- claimants: claims.length,
1277
- required,
1278
- requiredEvenIfAllAnswered,
1279
- repairCorroborationClusterSize: this.repairCorroborationClusterSize,
1280
- message: cohortTooSmall
1281
- ? cohortTooSmallMessage(cohortPeers, claims.length, requiredEvenIfAllAnswered, this.repairCorroborationClusterSize)
1282
- : soleHolderMessage(cohortPeers)
1283
- });
1284
- // Hung off the existing per-block freshness entry rather than a fourth per-block map. The entry
1285
- // survives `recordAheadClaim` clearing its `rev`, and is dropped wholesale once the block
1286
- // converges so each reason is said once per non-convergence episode, not once per pass.
1287
- // NOTE: per BLOCK, though the condition is a property of the cohort, not of any block — so a node
1288
- // in this state that reads N distinct blocks emits N lines. Deliberate: the operator wants to
1289
- // know which blocks are stuck, and N is bounded by blocks actually read (1821 lines for a single
1290
- // block was the defect). If a deployment in this state ever makes this the noisy line again, add
1291
- // a node-level once-flag keyed on (cohortPeers, requiredEvenIfAllAnswered) and let the per-block
1292
- // entry only suppress repeats.
1293
- this.unsettledAheadClaims.set(blockId, { ...(state ?? {}), deadlocksReported: [...alreadySaid, reason] });
1294
- }
1295
-
1296
- /**
1297
- * Report peers whose reported latest PROVABLY contradicts a CORROBORATED selection: the same
1298
- * revision under a different actionId. Two actions cannot both be the commit at one revision,
1299
- * and the pair a quorum of distinct peers agreed on is the one this node can stand behind, so
1300
- * the disagreeing claimant is wrong. Best-effort.
1301
- *
1302
- * A CERTIFIED selection is deliberately excluded no claim is penalized against it. A passing
1303
- * proof shows the cohort it names signed the commit, never that those signers are the block's
1304
- * responsible cohort: anyone holding N keys can mint a proof that verifies (see caller
1305
- * obligation #1 in `cluster/commit-proof.ts`, and the unwired {@link ProofAnchoring} layer).
1306
- * Penalizing here would therefore hand one forged proof a lever it must not have — every honest
1307
- * peer holding the real action at that revision reported for InvalidRestoration (weight 30,
1308
- * above the deprioritize threshold of 20), on every consult. Losing the selection to the proof
1309
- * is already the accepted cost of the certified path; deprioritizing the honest cohort on top of
1310
- * it is not. Revisit when certification is anchored to the block's derived cohort
1311
- * (`feat-cluster-membership-threshold-cert-anchoring`): a gated proof makes the contradiction
1312
- * provable again.
1313
- *
1314
- * A claim at a HIGHER rev than the selection is deliberately NOT penalized: a peer can honestly
1315
- * be ahead of the sampled quorum — an in-flight commit it durably stored before the rest of the
1316
- * cohort, or other honest holders dropped from the sample by the 1s per-peer consult deadline —
1317
- * and the InvalidRestoration weight (30) sits above the deprioritize threshold (20), so a single
1318
- * false hit used to deprioritize an honest, up-to-date peer. Declining to RESTORE from the
1319
- * uncorroborated higher claim already happens in selection; the affirmative penalty on that
1320
- * ambiguous evidence is what this method no longer applies. Provably-bad proof SERVICE is
1321
- * penalized at verification time instead (the certifyClaim pass in
1322
- * {@link queryClusterForLatest}).
1323
- */
1324
- private penalizeContradictingRevClaims(claims: RevClaim[], selected: QuorumRev, blockId: BlockId): void {
1325
- if (!this.reputation || selected.certified) return;
1326
- try {
1327
- for (const c of claims) {
1328
- if (c.rev === selected.rev && c.actionId !== selected.actionId) {
1329
- this.reputation.reportPeer(c.peerId, PenaltyReason.InvalidRestoration, `read-repair:${blockId}`);
1330
- }
1331
- }
1332
- } catch (err) {
1333
- this.log('cluster-fetch:penalize-error', { blockId, error: (err as Error).message });
1334
- }
1335
- }
1336
-
1337
- /**
1338
- * Best-effort penalty for a peer whose SERVED PROOF provably lies or provably does not cover the
1339
- * claim it was attached to (see the attributability classification in
1340
- * `cluster/certified-claims.ts`). Never throws mirrors
1341
- * {@link penalizeContradictingRevClaims}.
1342
- */
1343
- private penalizeProofService(peerId: string, blockId: BlockId): void {
1344
- if (!this.reputation) return;
1345
- try {
1346
- this.reputation.reportPeer(peerId, PenaltyReason.InvalidRestoration, `read-repair:${blockId}`);
1347
- } catch (err) {
1348
- this.log('cluster-fetch:penalize-error', { blockId, error: (err as Error).message });
1349
- }
1350
- }
1351
-
1352
- async pend(request: PendRequest, options?: MessageOptions): Promise<PendResult> {
1353
- const allBlockIds = blockIdsForTransforms(request.transforms);
1354
- await this.verifyResponsibility(allBlockIds);
1355
- const coordinatingBlockIds = options?.coordinatingBlockIds ?? allBlockIds;
1356
-
1357
- const peerCount = await this.coordinator.getClusterSize(coordinatingBlockIds[0]!);
1358
- if (peerCount <= 1) {
1359
- return await this.storageRepo.pend(request, options);
1360
- }
1361
-
1362
- const message: RepoMessage = {
1363
- operations: [{ pend: request }],
1364
- expiration: options?.expiration ?? Date.now() + this.DEFAULT_TIMEOUT,
1365
- coordinatingBlockIds
1366
- };
1367
-
1368
- try {
1369
- const { localExecuted, localPendResult } = await this.coordinator.executeClusterTransaction(coordinatingBlockIds[0]!, message, options);
1370
- this.log('coordinator-repo:pend-cluster-complete', {
1371
- actionId: request.actionId,
1372
- localExecuted,
1373
- localVerdict: localPendResult === undefined ? 'none'
1374
- : localPendResult.success ? 'success'
1375
- : isConflictFailure(localPendResult) ? 'conflict' : 'fault'
1376
- });
1377
- // Only call storageRepo if local cluster didn't already execute during consensus
1378
- if (!localExecuted) {
1379
- const result = await this.storageRepo.pend(request, options);
1380
- this.log('coordinator-repo:pend-fallback-result', {
1381
- actionId: request.actionId,
1382
- success: result.success,
1383
- hasMissing: !!(result as any).missing?.length,
1384
- hasPending: !!(result as any).pending?.length
1385
- });
1386
- return result;
1387
- }
1388
- // Local cluster already executed during consensus — return storage's own verdict rather
1389
- // than fabricating a success (the peerCount <= 1 path above returns storage's real result
1390
- // verbatim; the cluster path must never answer differently). Pend-consensus confers no
1391
- // durability: a refusal carrying `pending` (a rival's unresolved action holds the blocks)
1392
- // or `missing` (the requested revision is already committed) is the optimistic-concurrency
1393
- // verdict — the same scan every member runs, not a local fault — and must reach the writer
1394
- // as a retryable conflict so NetworkTransactor.pendPhase cancels the partial pend and the
1395
- // writer rebases. This deliberately differs from `commit`'s divergence split below: a
1396
- // commit that reached commit-consensus IS the authoritative commit (Theorem 9), whereas a
1397
- // pend that reached pend-consensus may still have been stored by nobody.
1398
- if (localPendResult !== undefined) {
1399
- if (localPendResult.success || isConflictFailure(localPendResult)) {
1400
- return localPendResult;
1401
- }
1402
- // A bare-reason refusal (no pending/missing — e.g. a local validation-hook fault)
1403
- // stays tolerated local divergence: consensus is authoritative and the pend may well
1404
- // have landed on the rest of the cohort.
1405
- this.log('coordinator-repo:pend-local-fault-tolerated', {
1406
- actionId: request.actionId,
1407
- reason: localPendResult.reason
1408
- });
1409
- }
1410
- // No verdict retained (member predates retention, restart, or TTL): the prior shape.
1411
- return {
1412
- success: true,
1413
- pending: [],
1414
- blockIds: allBlockIds
1415
- };
1416
- } catch (error) {
1417
- this.log('coordinator-repo:pend-error', { actionId: request.actionId, error: (error as Error).message });
1418
- // A lost conflict race is an optimistic-concurrency loss, not a fault: surface it as the
1419
- // StaleFailure shape the retry machinery already understands (`Collection.sync` and the
1420
- // multi-collection pendPhase retry it via `isConflictFailure`), exactly as a confirmed
1421
- // stale revision is. `staleAt` stays absent deliberately — it is confirmed-only, and a
1422
- // lost race is a rival *pend* holding the blocks, not a revision claim.
1423
- //
1424
- // NOTE: `error.conflicts` (peerId winning messageHash) is dropped here — `StaleFailure`
1425
- // has no field for it and the retry loop only needs "retryable". If a caller ever needs to
1426
- // know WHICH transaction won (e.g. to wait on it rather than re-race it), add a typed field
1427
- // for it; never recover it by parsing `reason`.
1428
- //
1429
- // NOTE: with three or more contenders the members can split so that EVERY contender is
1430
- // told it lost the race an all-lose round where nobody wins and each writer retries.
1431
- // The cause is `ClusterMember.resolveRace`'s approvals-first rule, not its tie-break:
1432
- // each member compares the rivals as IT holds them, so a member that already approved X
1433
- // keeps X while a member that approved Y first keeps Y, and no rival reaches a promise
1434
- // supermajority. (The hash tie-break is already symmetricit cannot be the fix.)
1435
- // Fine as it stands: since the torn-action fixes landed, an all-lose round costs one
1436
- // retry cycle rather than wedging, and the contenders are separated next round by the
1437
- // jittered backoff plus the aged retry priority carried on the re-pend
1438
- // (`clampPriority(consecutiveFailures)` in `Collection.syncInternal`), which out-ranks
1439
- // fresh priority-0 rivals at EQUAL approval counts — priority sits below the approval
1440
- // count in `resolveRace`, so it does not displace a more-progressed rival. If a
1441
- // high-contention workload ever shows syncs exhausting `maxAttempts` on repeated
1442
- // all-lose rounds, the fix is reserve/defer at pend time (backlog
1443
- // `feat-occ-priority-reservation`, which `resolveRace`'s own residual-fairness NOTE
1444
- // already points at) rather than raising maxAttempts.
1445
- if (error instanceof ConflictRaceLostError) {
1446
- return { success: false, conflict: true, reason: error.message };
1447
- }
1448
- const stale = await this.classifyStaleRejection(error, request, allBlockIds)
1449
- ?? await this.classifyPendingConflictRejection(error, request, allBlockIds);
1450
- if (stale) return stale;
1451
- throw error;
1452
- }
1453
- }
1454
-
1455
- /**
1456
- * Decide whether a cluster validator rejection was an optimistic-concurrency loss — the block
1457
- * already advanced past the requested revision — rather than a genuine validation fault.
1458
- * A confirmed loss returns a {@link StaleFailure} carrying `conflict: true` so the caller
1459
- * receives a non-success *response* that says plainly it is a lost race: network-transactor's
1460
- * pend then takes its stale branch and both writers (`Collection.sync`, and the coordinator's
1461
- * multi-collection pendPhase via `isConflictFailure`) retry, instead of a thrown error escaping
1462
- * mid-batch (which splits multi-tree commitssee PartialCommitError).
1463
- *
1464
- * The failure carries no `missing` list: confirmation is a local re-read that reveals the
1465
- * revision is taken but not which actions took it, and no consumer rebases from `missing`
1466
- * anyway (it is only counted or logged). `conflict` conveys retryability directly instead.
1467
- *
1468
- * Confirmation is purely local: re-read the affected blocks from our own storage and require
1469
- * `latest.rev >= request.rev`. The signed reject-reason text is never consulted — it is
1470
- * free-form wire-visible prose and must not become control flow. Anything unconfirmed
1471
- * (including read errors during confirmation) stays a throw, preserving fail-fast for
1472
- * genuine validation faults.
1473
- */
1474
- private async classifyStaleRejection(error: unknown, request: PendRequest, blockIds: BlockId[]): Promise<StaleFailure | undefined> {
1475
- const requestedRev = request.rev;
1476
- if (!(error instanceof ValidatorRejectionError) || requestedRev === undefined) return undefined;
1477
- let results: GetBlockResults;
1478
- try {
1479
- results = await this.storageRepo.get({ blockIds });
1480
- } catch (readError) {
1481
- this.log('coordinator-repo:pend-stale-classify-read-error', {
1482
- actionId: request.actionId,
1483
- error: (readError as Error).message
1484
- });
1485
- return undefined;
1486
- }
1487
- // Scan EVERY block rather than stopping at the first confirmation: several of the request's
1488
- // blocks can be past the requested revision at different revisions, and it is the highest
1489
- // that the loser's next request has to clear (see `highestStaleAt`). Both the reported
1490
- // number and the reason prose name that block, so they never disagree.
1491
- const staleAt = highestStaleAt(blockIds.map(blockId => {
1492
- const latest = results[blockId]?.state.latest;
1493
- if (!latest || latest.rev < requestedRev) return undefined;
1494
- // Per-block self-exclusion (see {@link isOwnRevision}): our own durable half of a torn
1495
- // action is not a confirmed loss. Deliberately per-block, NOT the bail-entirely
1496
- // 'own-durable' shape of confirmCommitRivalAgainstLocal — a confirmed rival on ANOTHER
1497
- // block still confirms, and when none is confirmed anywhere the rejection stays a throw
1498
- // exactly as before. With the two pend-tier sites upstream fixed (StorageRepo.pend and
1499
- // ClusterMember.validatePendOperations) this shape should not reach here; mirrored so
1500
- // all three pend-tier checks agree.
1501
- if (isOwnRevision(latest, requestedRev, request.actionId)) return undefined;
1502
- return { blockId, rev: latest.rev };
1503
- }));
1504
- if (staleAt) {
1505
- this.log('coordinator-repo:pend-stale-classified', {
1506
- actionId: request.actionId,
1507
- blockId: staleAt.blockId,
1508
- latestRev: staleAt.rev,
1509
- requestedRev
1510
- });
1511
- return {
1512
- success: false,
1513
- conflict: true,
1514
- reason: `stale revision: block ${staleAt.blockId} at rev ${staleAt.rev}, requested rev ${requestedRev}`,
1515
- // The same fact as the reason prose, but as data. This is the ONLY place a losing
1516
- // writer can learn the revision it lost to, since this failure deliberately carries
1517
- // no `missing`. Confirmed-local: read out of our own storage just above.
1518
- staleAt
1519
- };
1520
- }
1521
- // NOTE: conservative — when only remote members saw the newer revision (local storage still
1522
- // behind), staleness can't be confirmed locally and the rejection stays a throw. If that
1523
- // shows up in practice, extend confirmation with a quorum read; never trust the reject text.
1524
- // `staleAt` is absent on this path for the same reason, and deliberately so — there is no
1525
- // confirmed number to report, and the field's contract forbids inferring one from that text.
1526
- return undefined;
1527
- }
1528
-
1529
- /**
1530
- * Sibling of {@link classifyStaleRejection} for the OTHER optimistic-concurrency refusal shape:
1531
- * the promise-phase pending-conflict vote (`validatePendOperations` rejecting a pend whose
1532
- * blocks are held by a different unresolved pending action). That vote surfaces here as a
1533
- * {@link ValidatorRejectionError}, and without classification it would escape as a throw —
1534
- * splitting multi-tree pends mid-batch instead of taking the retry path a lost race deserves.
1535
- *
1536
- * Same confirmation discipline as the stale classifier: purely local. Re-read the affected
1537
- * blocks from our own storage and require some block's `state.pendings` to carry a rival
1538
- * actionId; the signed reject text is never consulted. A confirmed rival returns a
1539
- * {@link StaleFailure} with `conflict: true` and the rivals as `pending` (`ActionPending`
1540
- * without `transform` the type allows it, and no consumer rebases from it). Unconfirmed —
1541
- * including read errors during confirmation — stays a throw, preserving fail-fast for genuine
1542
- * validation faults. Checked after `classifyStaleRejection` so a confirmed committed loss
1543
- * (which carries the sharper `staleAt`) wins when both hold.
1544
- */
1545
- private async classifyPendingConflictRejection(error: unknown, request: PendRequest, blockIds: BlockId[]): Promise<StaleFailure | undefined> {
1546
- if (!(error instanceof ValidatorRejectionError)) return undefined;
1547
- let results: GetBlockResults;
1548
- try {
1549
- results = await this.storageRepo.get({ blockIds });
1550
- } catch (readError) {
1551
- this.log('coordinator-repo:pend-conflict-classify-read-error', {
1552
- actionId: request.actionId,
1553
- error: (readError as Error).message
1554
- });
1555
- return undefined;
1556
- }
1557
- const pending: ActionPending[] = [];
1558
- for (const blockId of blockIds) {
1559
- for (const actionId of results[blockId]?.state?.pendings ?? []) {
1560
- if (actionId !== request.actionId) pending.push({ blockId, actionId });
1561
- }
1562
- }
1563
- if (pending.length === 0) return undefined;
1564
- this.log('coordinator-repo:pend-conflict-classified', {
1565
- actionId: request.actionId,
1566
- rivals: pending.map(p => `${p.blockId}:${p.actionId}`)
1567
- });
1568
- return {
1569
- success: false,
1570
- conflict: true,
1571
- pending,
1572
- reason: `pending conflict: block(s) held by unresolved rival action(s) ${[...new Set(pending.map(p => p.actionId))].join(', ')}`
1573
- };
1574
- }
1575
-
1576
- async cancel(actionRef: ActionBlocks, options?: MessageOptions): Promise<void> {
1577
- const blockIds = actionRef.blockIds;
1578
- await this.verifyResponsibility(blockIds);
1579
-
1580
- // Create a message for this cancel operation with timeout
1581
- const message: RepoMessage = {
1582
- operations: [{ cancel: { actionRef } }],
1583
- expiration: options?.expiration ?? Date.now() + this.DEFAULT_TIMEOUT
1584
- };
1585
-
1586
- try {
1587
- // One cluster transaction per block ID — but a block whose cohort is just this node
1588
- // short-circuits to local storage, exactly as `pend` and `commit` do above. Without the
1589
- // short-circuit a solo cohort enters `executeTransaction`, fails `minAbsoluteClusterSize`
1590
- // (2), and throws `Cluster size 1 below minimum 2 and not validated` — so a single-peer
1591
- // deployment could pend and commit but never cancel, unless the operator had opened the
1592
- // `allowUnvalidatedSmallCluster` hatch. Decided per block rather than once for
1593
- // `blockIds[0]`, because a multi-block cancel can span cohorts of different sizes.
1594
- //
1595
- // NOTE: `getClusterSize` is a second `findCluster` for the same key that
1596
- // `executeClusterTransaction` is about to look up again, so a cancel over N blocks now
1597
- // costs 2N cohort lookups instead of N. Same shape `pend` and `commit` already pay, but
1598
- // they pay it once (they only ever consult `blockIds[0]`) where this scales with N. Fine
1599
- // while cancels span a handful of blocks; if wide multi-block cancels ever show up hot,
1600
- // have `executeClusterTransaction` return the cohort it already fetched (or own the
1601
- // short-circuit itself) rather than adding a cache here.
1602
- const results = await Promise.all(blockIds.map(async blockId => {
1603
- const peerCount = await this.coordinator.getClusterSize(blockId);
1604
- if (peerCount <= 1) return false;
1605
- const { localExecuted } = await this.coordinator.executeClusterTransaction(blockId, message, options);
1606
- return localExecuted;
1607
- }));
1608
-
1609
- // Only call storageRepo if local cluster didn't already execute during consensus
1610
- const anyLocalExecuted = results.some(Boolean);
1611
- if (!anyLocalExecuted) {
1612
- await this.storageRepo.cancel(actionRef, options);
1613
- }
1614
- } catch (error) {
1615
- this.log('coordinator-repo:cancel-error', { actionId: actionRef.actionId, error: (error as Error).message });
1616
- throw error;
1617
- }
1618
- }
1619
-
1620
- async commit(request: CommitRequest, options?: MessageOptions): Promise<CommitResult> {
1621
- const blockIds = request.blockIds;
1622
- await this.verifyResponsibility(blockIds);
1623
-
1624
- const peerCount = await this.coordinator.getClusterSize(blockIds[0]!);
1625
- if (peerCount <= 1) {
1626
- const result = await this.storageRepo.commit(request, options);
1627
- if (result.success) this.markBlocksSeen(blockIds);
1628
- return result;
1629
- }
1630
-
1631
- const message: RepoMessage = {
1632
- operations: [{ commit: request }],
1633
- expiration: options?.expiration ?? Date.now() + this.DEFAULT_TIMEOUT
1634
- };
1635
-
1636
- try {
1637
- const { record, localExecuted, localCommitResult } = await this.coordinator.executeClusterTransaction(blockIds[0]!, message, options);
1638
- if (localExecuted) {
1639
- // Our own member applied this commit during consensus. Its retained storage verdict is
1640
- // the one honest signal we have about durability: the member-side apply tolerates an
1641
- // "ahead" refusal as divergence (see the NOTE in ClusterMember.applyConsensusOperation),
1642
- // which is correct for a redelivered or lagging commit — but when the refusal's real
1643
- // cause is a RIVAL action holding the requested revision, that tolerance turns a commit
1644
- // no member durably stored into a fabricated success. This is the
1645
- // signed-but-not-yet-applied window: two commits for one revision can BOTH assemble
1646
- // consensus when every member signs the second after signing (but before applying) the
1647
- // first, because signing drops the member's reservation. Confirm the rival against local
1648
- // storage (never the verdict's prose) and answer the writer with a retryable conflict so
1649
- // it re-drives at a fresh revision. Own-action or unconfirmed refusals keep the
1650
- // prior fabricated-success shape: consensus is authoritative and this member converges
1651
- // via replication.
1652
- //
1653
- // NOTE: a CONFIRMED rival is trusted over the consensus outcome here. That is right in
1654
- // the window this closes (the cohort refused the loser too), but it inverts if the two
1655
- // ever disagree — a local rival at the requested revision while a super-majority
1656
- // approved OUR commit means this node is on a forked lineage, and refusing then tells a
1657
- // writer whose write did land to re-drive it (a duplicate entry). Members holding the
1658
- // rival reject at the promise round, so consensus and a local rival can only disagree
1659
- // after a fork; that is partition-healing scope (docs/partition-healing.md). If forks
1660
- // are ever observed here, weigh the retained verdict against the cohort's votes instead
1661
- // of trusting the local re-read alone.
1662
- if (localCommitResult !== undefined && !localCommitResult.success) {
1663
- const rival = await this.confirmCommitRivalAgainstLocal(request);
1664
- if (typeof rival === 'object') return rival;
1665
- this.log('coordinator-repo:commit-local-refusal-tolerated', {
1666
- actionId: request.actionId,
1667
- confirmation: rival ?? 'unconfirmed',
1668
- reason: localCommitResult.reason
1669
- });
1670
- }
1671
- this.markBlocksSeen(blockIds);
1672
- return { success: true };
1673
- }
1674
- // Local cluster didn't execute during consensus. Attempt a local commit, but tolerate
1675
- // local divergence when the cluster already reached consensus this coordinator was
1676
- // likely picked for commit after missing the pend phase (unreachable during pend, fresh
1677
- // join, etc.). The cluster's majority is authoritative; this peer catches up via sync.
1678
- //
1679
- // Divergence reaches us in BOTH shapes and both must be tolerated identically:
1680
- // - a THROW ("Pending action … not found"), when we never saw the pend;
1681
- // - a RETURNED `success:false` carrying `missing-base-revision`, when we saw the pend
1682
- // but not the revision that created the block (see StorageRepo.internalCommit).
1683
- // Only the throw was tolerated before the refusal existed. Reporting the refusal to the
1684
- // caller instead would surface a committed transaction as a stale loss: db-core's
1685
- // commitPhase treats any returned `success:false` as a permanent stale failure, so the
1686
- // client would retry an action the cluster already landed until it exhausted its budget.
1687
- try {
1688
- const result = await this.storageRepo.commit(request, options);
1689
- if (result.success) {
1690
- this.markBlocksSeen(blockIds);
1691
- return result;
1692
- }
1693
- if (isMissingBaseRevisionFailure(result) && clusterReachedCommitConsensus(record)) {
1694
- return this.tolerateLocalCommitDivergence(request, blockIds, result.reason ?? MISSING_BASE_REVISION_REASON);
1695
- }
1696
- return result;
1697
- } catch (err) {
1698
- if (clusterReachedCommitConsensus(record)) {
1699
- return this.tolerateLocalCommitDivergence(request, blockIds, (err as Error).message);
1700
- }
1701
- throw err;
1702
- }
1703
- } catch (error) {
1704
- this.log('coordinator-repo:commit-error', { actionId: request.actionId, error: (error as Error).message });
1705
- // A lost commit-consensus race is an optimistic-concurrency loss, not a fault — mirror
1706
- // `pend`'s conversion above. At the moment this is thrown, zero members approved and the
1707
- // members hold the winner: nothing of the loser landed, so a retryable-conflict answer is
1708
- // truthful. Returning it (rather than rethrowing) matters more here than on the pend path:
1709
- // db-core's `commitCollection` retries a THROWN commit error verbatim up to 3 times, and by
1710
- // the retry the members have applied the winner and cleared its reservation — the re-driven
1711
- // commit can then assemble a consensus no member will durably store (the writer's append
1712
- // fulfills, the entry exists on no node). A RETURNED `success:false` is instead surfaced
1713
- // immediately as a stale loss; the writer cancels the pend, re-reads, and re-drives the
1714
- // whole pend+commit at a fresh revision. `staleAt` stays absent for the same reason as
1715
- // pend's: it is confirmed-only, and a lost race is a rival commit racing the same revision,
1716
- // not a locally-confirmed revision claim.
1717
- if (error instanceof ConflictRaceLostError) {
1718
- return { success: false, conflict: true, reason: error.message };
1719
- }
1720
- // A promise-phase stale-commit reject (`ClusterMember.validateCommitRevisions` a member
1721
- // holds the requested revision under a different action) surfaces here as a
1722
- // ValidatorRejectionError; classify it against local storage the way `pend` does, so the
1723
- // writer gets a clean retryable conflict instead of three verbatim re-drives and a hard
1724
- // failure.
1725
- const stale = await this.classifyCommitStaleRejection(error, request);
1726
- if (stale) return stale;
1727
- throw error;
1728
- }
1729
- }
1730
-
1731
- /**
1732
- * Commit-shaped sibling of {@link classifyStaleRejection}: decide whether a cluster validator
1733
- * rejection of a COMMIT was an optimistic-concurrency loss — the requested revision is already
1734
- * committed under a different action — rather than a genuine validation fault. A confirmed loss
1735
- * returns a {@link StaleFailure} with `conflict: true` so db-core's `commitCollection` surfaces
1736
- * it immediately as a stale loss (no verbatim retry) and the writer re-drives at a fresh
1737
- * revision.
1738
- *
1739
- * Same confirmation discipline as the pend classifiers: purely local re-read; the signed reject
1740
- * text is never consulted. One commit-specific delta — confirmation must EXCLUDE the
1741
- * own-action-at-rev case: a block whose requested revision is held by THIS action is already
1742
- * durable, and answering `conflict` for it would make the writer rebase and re-append an
1743
- * already-committed action at a new revision — a duplicate entry. So:
1744
- * - `latest.rev === request.rev` → compare `latest.actionId`: ours ⇒ bail (stays a throw),
1745
- * a rival's ⇒ confirmed loss;
1746
- * - `latest.rev > request.rev` ask the {@link IRevisionActionReader} capability who holds
1747
- * `request.rev`: ours bail, a rival's confirmed loss, unknown/absent/fault unconfirmed;
1748
- * - anything unconfirmed (including read errors) stays a throw fail-fast for genuine faults.
1749
- */
1750
- private async classifyCommitStaleRejection(error: unknown, request: CommitRequest): Promise<StaleFailure | undefined> {
1751
- if (!(error instanceof ValidatorRejectionError)) return undefined;
1752
- const rival = await this.confirmCommitRivalAgainstLocal(request);
1753
- // 'own-durable' and unconfirmed both stay a throw here: fail-fast for genuine faults, and a
1754
- // commit already durable under this action must never be answered `conflict` (the writer
1755
- // would rebase and re-append it a duplicate entry).
1756
- return typeof rival === 'object' ? rival : undefined;
1757
- }
1758
-
1759
- /**
1760
- * Shared confirmation core for the two commit-tier conversion sites ({@link classifyCommitStaleRejection}
1761
- * and the locally-executed refusal check in {@link commit}): decide, from LOCAL storage only, who
1762
- * holds the requested revision.
1763
- * - a confirmed RIVAL the {@link StaleFailure} conflict answer (with `staleAt` = highest
1764
- * confirmed holder);
1765
- * - our OWN action durable at the requested revision `'own-durable'` (callers must not answer
1766
- * `conflict` the writer would rebase an already-landed action into a duplicate entry);
1767
- * - anything else (behind, truncated history, read faults, capability absent) → `undefined`,
1768
- * unconfirmed.
1769
- * The signed reject text / retained verdict prose is never consulted.
1770
- */
1771
- private async confirmCommitRivalAgainstLocal(request: CommitRequest): Promise<StaleFailure | 'own-durable' | undefined> {
1772
- const blockIds = request.blockIds;
1773
- let results: GetBlockResults;
1774
- try {
1775
- results = await this.storageRepo.get({ blockIds });
1776
- } catch (readError) {
1777
- this.log('coordinator-repo:commit-stale-classify-read-error', {
1778
- actionId: request.actionId,
1779
- error: (readError as Error).message
1780
- });
1781
- return undefined;
1782
- }
1783
- const reader = this.storageRepo as IRepo & Partial<IRevisionActionReader>;
1784
- // Scan EVERY block (same rule as the pend classifier): report the highest confirmed rival
1785
- // revision, but bail the moment any block shows OUR action durable at the requested revision.
1786
- const rivalStales: ({ blockId: BlockId; rev: number } | undefined)[] = [];
1787
- for (const blockId of blockIds) {
1788
- const latest = results[blockId]?.state?.latest;
1789
- if (!latest || latest.rev < request.rev) continue;
1790
- if (latest.rev === request.rev) {
1791
- if (latest.actionId === request.actionId) {
1792
- this.log('coordinator-repo:commit-stale-classify-own-action', {
1793
- actionId: request.actionId, blockId, rev: request.rev
1794
- });
1795
- return 'own-durable';
1796
- }
1797
- rivalStales.push({ blockId, rev: latest.rev });
1798
- continue;
1799
- }
1800
- // latest.rev > request.rev latest can no longer name who took request.rev.
1801
- if (typeof reader.getRevisionAction !== 'function') continue;
1802
- let takenBy: ActionId | undefined;
1803
- try {
1804
- takenBy = await reader.getRevisionAction(blockId, request.rev);
1805
- } catch (readError) {
1806
- this.log('coordinator-repo:commit-stale-classify-revision-read-error', {
1807
- actionId: request.actionId, blockId, rev: request.rev,
1808
- error: (readError as Error).message
1809
- });
1810
- continue;
1811
- }
1812
- if (takenBy === request.actionId) {
1813
- this.log('coordinator-repo:commit-stale-classify-own-action', {
1814
- actionId: request.actionId, blockId, rev: request.rev, latestRev: latest.rev
1815
- });
1816
- return 'own-durable';
1817
- }
1818
- if (takenBy !== undefined) rivalStales.push({ blockId, rev: latest.rev });
1819
- // takenBy undefined (truncated history): unconfirmed for this block.
1820
- }
1821
- const staleAt = highestStaleAt(rivalStales);
1822
- if (!staleAt) return undefined;
1823
- this.log('coordinator-repo:commit-stale-classified', {
1824
- actionId: request.actionId,
1825
- blockId: staleAt.blockId,
1826
- latestRev: staleAt.rev,
1827
- requestedRev: request.rev
1828
- });
1829
- return {
1830
- success: false,
1831
- conflict: true,
1832
- reason: `stale commit: block ${staleAt.blockId} at rev ${staleAt.rev}, requested rev ${request.rev}`,
1833
- staleAt
1834
- };
1835
- }
1836
-
1837
- /**
1838
- * Report success for a commit the cluster carried but this peer could not apply locally. The
1839
- * blocks are marked seen so the read path treats them as freshness-checked; convergence comes
1840
- * from replication (cohort reconcile, or read-driven acquisition), not from replay here.
1841
- */
1842
- private tolerateLocalCommitDivergence(request: CommitRequest, blockIds: BlockId[], detail: string): CommitResult {
1843
- this.log('coordinator-repo:commit-local-failed-cluster-succeeded', { actionId: request.actionId, error: detail });
1844
- this.markBlocksSeen(blockIds);
1845
- return { success: true };
1846
- }
1847
- }
1848
-
1849
- /** True if a simple majority of cluster peers signed an approving commit. */
1850
- function clusterReachedCommitConsensus(record: ClusterRecord): boolean {
1851
- const peerCount = Object.keys(record.peers).length;
1852
- if (peerCount === 0) return false;
1853
- const approvedCommits = Object.values(record.commits).filter(s => s.type === 'approve').length;
1854
- return approvedCommits > peerCount / 2;
1855
- }
1
+ import type { PendRequest, ActionBlocks, IRepo, MessageOptions, CommitResult, GetBlockResults, PendResult, StaleFailure, BlockGets, CommitRequest, RepoMessage, IKeyNetwork, ICluster, ClusterConsensusConfig, BlockId, ActionId, ActionRev, ActionContext, ClusterRecord, BlockUnavailableReason, ActionPending } from "@optimystic/db-core";
2
+ import { LruMap, blockIdsForTransforms, highestStaleAt, isConflictFailure, isOwnRevision, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
3
+ import { ClusterCoordinator, ConflictRaceLostError, ValidatorRejectionError } from "./cluster-coordinator.js";
4
+ import type { PeerId } from "@libp2p/interface";
5
+ import { peerIdFromString } from "@libp2p/peer-id";
6
+ import type { FretService } from "p2p-fret";
7
+ import { createLogger } from '../logger.js';
8
+ import type { IPeerReputation } from "../reputation/types.js";
9
+ import { PenaltyReason } from "../reputation/types.js";
10
+ import type { ITransactionStateStore } from "../cluster/i-transaction-state-store.js";
11
+ import { quorumSize, corroboratorCapacity, selectQuorumRev, certifiedEquivocation, CORROBORATION_FLOOR, type RevClaim, type QuorumRev } from "../cluster/quorum-restore.js";
12
+ import { certifyClaim, isAttributableProofFailure, proofThresholds, type ProofAnchoring } from "../cluster/certified-claims.js";
13
+ import { DEFAULT_CLUSTER_SIZE } from "../cluster/cluster-policy.js";
14
+ import { RECONCILE_TIMEOUT_MS } from "../cluster/reconcile-block.js";
15
+ import { isMissingBaseRevisionFailure, MISSING_BASE_REVISION_REASON, type ICommitProofPersister, type IRevisionActionReader } from "../storage/storage-repo.js";
16
+ import { buildBlockCommitProof, type BlockCommitProof } from "../cluster/commit-proof.js";
17
+ import type { ReconcileBlockCallback } from "../cluster/cluster-repo.js";
18
+ import type { CertifiedActionRev } from "../storage/block-archive.js";
19
+
20
+ /**
21
+ * Acquire a block's content for a cohort-corroborated revision, from the cohort, and persist it.
22
+ *
23
+ * Deliberately the SAME shape as the commit path's {@link ReconcileBlockCallback}, and in the live
24
+ * node the very same instance (`libp2p-node-base` passes its `reconcileBlock` to both): read-driven
25
+ * acquisition needs exactly what reconcile already provides a per-peer-bounded archive fetch, a
26
+ * quorum vote on the target `(rev, actionId)`, a quorum vote on the *content* at that revision, and a
27
+ * persist through the monotonic, commit-latched `StorageRepo.saveReplicatedBlock` funnel. Reusing it
28
+ * is what keeps read-repair from being a weaker trust path than reconcile.
29
+ */
30
+ export type AcquireBlockCallback = ReconcileBlockCallback;
31
+
32
+ /** How long one cohort peer gets to answer the latest-revision consult before it counts as silent. */
33
+ const LATEST_QUERY_TIMEOUT_MS = 1000;
34
+
35
+ /** True when a freshly-read local revision is strictly ahead of the baseline the repair started from. */
36
+ function isAdvanceOver(rev: number | undefined, baselineRev: number | undefined): boolean {
37
+ return typeof rev === 'number' && (baselineRev === undefined || rev > baselineRev);
38
+ }
39
+
40
+ /**
41
+ * Reject if `promise` has not settled within `ms`. The timer is cleared on either outcome, so no
42
+ * handle outlives the race (hence no `unref`, which does not exist off Node).
43
+ */
44
+ function withDeadline<T>(promise: Promise<T>, ms: number, label: string): Promise<T> {
45
+ let timer: ReturnType<typeof setTimeout> | undefined;
46
+ const deadline = new Promise<never>((_, reject) => {
47
+ timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms);
48
+ });
49
+ return Promise.race([promise, deadline]).finally(() => {
50
+ if (timer !== undefined) clearTimeout(timer);
51
+ });
52
+ }
53
+
54
+ /**
55
+ * What one round of polling the cohort learned about a block: the revision the OTHER
56
+ * cohort members corroborated, and what this node itself already holds. The two are kept
57
+ * apart on purpose the local revision is the baseline being repaired, never evidence
58
+ * about the cluster (see {@link CoordinatorRepo.queryClusterForLatest}) but the caller
59
+ * still needs it to tell whether the corroborated revision is actually an advance.
60
+ */
61
+ interface ClusterLatestQuery {
62
+ /** Highest `(rev, actionId)` corroborated by peers other than this node, if any. */
63
+ corroborated?: ActionRev;
64
+ /**
65
+ * This node's own latest for the block, as answered by the callback's self short-circuit.
66
+ * Typed as a {@link CertifiedActionRev} because that short-circuit reads the local proof too
67
+ * of no use to this node (it trusts its own storage), but the type stays honest about what the
68
+ * value carries rather than silently erasing it.
69
+ */
70
+ local?: CertifiedActionRev;
71
+ /**
72
+ * Cohort peers (self excluded) that never answered the consult the callback rejected
73
+ * (dial failure, protocol error) or blew the per-peer deadline. Silence, not evidence:
74
+ * these are never counted as claims, but while this is non-empty a caller must not treat
75
+ * "nothing corroborated" as an authoritative absence, because a silent peer could be the
76
+ * sole holder.
77
+ */
78
+ silent: string[];
79
+ /**
80
+ * Highest revision any cohort peer CLAIMED when no claim met the corroboration quorum
81
+ * (set only alongside an absent `corroborated`). The claim failed quorum, so it must
82
+ * never drive restoration it exists so `get` can report content it serves below this
83
+ * revision as possibly behind ({@link GetBlockResult.unconfirmedAheadRev}) instead of
84
+ * confirmed. When a quorum DOES corroborate, higher uncorroborated claims are dropped
85
+ * as before: the quorum's affirmative answer outweighs a lone voter (which may simply
86
+ * be ahead on an in-flight commit), and stamping doubt there would mark every read that
87
+ * races a commit broadcast.
88
+ */
89
+ uncorroboratedRev?: number;
90
+ /**
91
+ * How many cohort peers OTHER than this node answered the consult at all with a claim
92
+ * or with "I hold nothing". `silent` says who could not be asked; this says how many
93
+ * could. Zero with a non-empty `silent` means this node reached NOBODY, which is a
94
+ * different fact from partial silence: there is no better-informed answer to be had from
95
+ * this node's position (see {@link AbsenceVerdict}).
96
+ */
97
+ answered: number;
98
+ }
99
+
100
+ /**
101
+ * What earlier repair passes left unresolved for one block. Two independent facts share one entry
102
+ * and one map on purpose: both are "what the last repair pass could not finish", both are
103
+ * cleared by the same event (the block converging), and `CoordinatorRepo` already keeps more
104
+ * per-block maps than anyone can hold in their head (backlog
105
+ * `debt-freshness-state-scattered-across-coordinator-repo`).
106
+ *
107
+ * An entry exists exactly while at least one of the two is set; both clear together in
108
+ * {@link CoordinatorRepo.flagUnconfirmedCurrency} once this node reaches the claimed revision.
109
+ */
110
+ interface AheadClaimState {
111
+ /**
112
+ * The cohort-claimed revision the last freshness consult could not settle — the doubt
113
+ * {@link CoordinatorRepo.flagUnconfirmedCurrency} stamps onto reads served below it. Absent once a
114
+ * consult finds nothing ahead of what this node holds.
115
+ */
116
+ rev?: number;
117
+ /**
118
+ * Which `cluster-fetch:repair-deadlock` reasons have already been said for this block (see
119
+ * {@link CoordinatorRepo.reportRepairDeadlock}). Neither reason is about any one revisionone is
120
+ * about the cohort's size, the other about how many of its peers hold the block so both survive
121
+ * {@link CoordinatorRepo.recordAheadClaim} clearing `rev`: without that, a block whose cohort
122
+ * claims nothing *ahead* of the reader would re-announce the same permanent condition on every
123
+ * single pass, the noise this line exists to replace.
124
+ *
125
+ * Tracked per REASON rather than as one flag: the two diagnose different faults and send the
126
+ * operator to different places, so an episode that starts as `cohort-too-small` and becomes
127
+ * `sole-holder` (the operator added machines, which is what that reason told them to do) has to be
128
+ * able to say the second thing. Bounded at two entries by the reason union itself.
129
+ */
130
+ deadlocksReported?: readonly DeadlockReason[];
131
+ }
132
+
133
+ /**
134
+ * Why a corroboration decline is provably permanent see {@link CoordinatorRepo.reportRepairDeadlock}
135
+ * for what makes each provable and which remedy each sends the operator to.
136
+ */
137
+ type DeadlockReason = 'cohort-too-small' | 'sole-holder';
138
+
139
+ /** The `cohort-too-small` wording: the cohort cannot field the quorum however healthy its peers are. */
140
+ function cohortTooSmallMessage(
141
+ cohortPeers: number,
142
+ claimants: number,
143
+ requiredEvenIfAllAnswered: number,
144
+ repairCorroborationClusterSize: number
145
+ ): string {
146
+ return `Block repair cannot converge for this block and the condition is PERMANENT, not transient: ` +
147
+ `this node's cohort has ${cohortPeers} peer(s) besides itself, all of them answered ` +
148
+ `(${claimants} hold the block), but accepting a revision would need ${requiredEvenIfAllAnswered} ` +
149
+ `agreeing peers even if every one of those ${cohortPeers} answered and agreed. No later pass can reach ` +
150
+ `that, however healthy every peer is, so this node's copy of the block stays as it is. Repair needs ` +
151
+ `${CORROBORATION_FLOOR} cohort peers BESIDES the reader to answer and agree, relaxed to 1 only for a ` +
152
+ `cohort that DECLARES it is smaller; repairCorroborationClusterSize currently resolves to ` +
153
+ `${repairCorroborationClusterSize}. Two things produce this, and this node cannot tell them ` +
154
+ `apart: (1) the deployment really does run this few machines set clusterPolicy.assumedClusterSize ` +
155
+ `to the number you actually run (it does not lower clusterSize / the replication factor), or set an ` +
156
+ `honest clusterSize, and run at least ${CORROBORATION_FLOOR + 2} machines for any tolerance of one ` +
157
+ `unreachable peer; or (2) this node's view of the cohort has shrunk below the real deployment a ` +
158
+ `partition or a routing problem, which configuration will not fix. Check the peer count above ` +
159
+ `against the machines you run before changing anything.`;
160
+ }
161
+
162
+ /**
163
+ * The `sole-holder` wording: the cohort is big enough, but only one of its peers holds the block.
164
+ *
165
+ * Every claim here is scoped to THIS NODE'S COHORT PEERS, which is the whole of what the pass
166
+ * observed. It deliberately does not say "only one machine in the deployment holds this block": this
167
+ * node's own copy is excluded from the claim set (it cannot corroborate the revision it is trying to
168
+ * repair), so a reader that holds the block itself would make that reading false and a scary
169
+ * all-caps line an operator can disprove by looking at their own disks is worth less than no line.
170
+ * For the same reason the remedy is "another COHORT PEER holding it" rather than "a second copy":
171
+ * with the reader holding one, a second copy already exists and is still not enough.
172
+ */
173
+ function soleHolderMessage(cohortPeers: number): string {
174
+ return `Block repair cannot converge for this block and the condition is PERMANENT, not transient: ` +
175
+ `ONLY ONE COHORT PEER HOLDS THIS BLOCK. Of this node's ${cohortPeers} cohort peers, 1 reports holding ` +
176
+ `it and the other ${cohortPeers - 1} answered that they hold NOTHING an answer, not silence, so this ` +
177
+ `is the whole picture and not a slow pass. Repair adopts a revision only when ${CORROBORATION_FLOOR} ` +
178
+ `peers BESIDES this node agree on it, and a lone holder cannot second itself, so every later pass ` +
179
+ `declines identically. This node's own copy, if it has one, is the copy being repaired and does not ` +
180
+ `count toward that number. MORE MACHINES DO NOT FIX THIS, and neither does any cluster-size setting ` +
181
+ `what is missing is ANOTHER COHORT PEER HOLDING THE BLOCK. The usual cause is data written while the ` +
182
+ `deployment (or this block's cohort) was smaller: a block that had one holder then still has one holder ` +
183
+ `now, because the two paths that would replicate it read-repair and reconcile both decline on this ` +
184
+ `same rule. Committing any new revision of the block writes it to the current cohort and clears this. ` +
185
+ `(A lone holder whose answer carries a valid cohort commit proof for its revision IS adopted without a ` +
186
+ `second voter — reaching this message means the one holder attached no such proof, or one that did not ` +
187
+ `verify.)`;
188
+ }
189
+
190
+ /**
191
+ * What one repair pass established about a block that is still MISSING locally after it.
192
+ * Ordered by how firmly the block is ruled out; `get` consults it only on the missing path.
193
+ */
194
+ type AbsenceVerdict =
195
+ /** Nobody to ask (empty cohort, or solo-self), or every non-self cohort member answered
196
+ * "I hold nothing". As confirmed as an absence gets — stays authoritative, which is what
197
+ * keeps the routine new-collection probe at one round trip. */
198
+ | 'confirmed'
199
+ /** Some of the cohort answered and some could not be asked. (A consult that THROWS produces
200
+ * no verdict at all — `get`'s catch arm reports it directly.) */
201
+ | 'unconfirmed'
202
+ /** No cohort member outside this node could be asked at all. Mutually exclusive with
203
+ * `claimed` in practice: a claim requires a non-self peer to have answered, which is
204
+ * exactly what this verdict rules out. The precedence below still orders the pair, so
205
+ * the mapping stays total, but there is no reachable case to test. */
206
+ | 'isolated'
207
+ /** A peer claimed a revision this pass did not converge onto — quorum declined it, or a
208
+ * quorum corroborated it and acquisition failed. */
209
+ | 'claimed';
210
+
211
+ /**
212
+ * Extended cluster interface that includes the ability to check if a transaction was executed.
213
+ * This is used by CoordinatorRepo to avoid duplicate execution.
214
+ */
215
+ interface LocalClusterWithExecutionTracking extends ICluster {
216
+ wasTransactionExecuted?(messageHash: string): boolean;
217
+ /** Local storage's verdict for a pend applied during consensus; see ClusterMember.getExecutedPendResult. */
218
+ getExecutedPendResult?(messageHash: string): PendResult | undefined;
219
+ /** Local storage's verdict for a commit applied during consensus; see ClusterMember.getExecutedCommitResult. */
220
+ getExecutedCommitResult?(messageHash: string): CommitResult | undefined;
221
+ /** Self-sign a one-peer commit proof for the solo-cohort commit path; see ClusterMember.mintSoloCommitProof.
222
+ * Optional like its siblings: absent on a bare ICluster double, and then the solo path simply
223
+ * commits proof-less exactly the pre-mint behavior. */
224
+ mintSoloCommitProof?(message: RepoMessage): Promise<BlockCommitProof>;
225
+ }
226
+
227
+ /**
228
+ * A cohort peer's answer to the latest-revision consult. Defined with the archive shape it is
229
+ * projected from (`storage/block-archive.ts`) and re-exported here so it reads next to the callback
230
+ * that returns it; see {@link CertifiedActionRev} there for what the optional proof does and does
231
+ * not mean.
232
+ */
233
+ export type { CertifiedActionRev } from "../storage/block-archive.js";
234
+
235
+ /**
236
+ * Callback to query a cluster peer for their latest revision of a block. Three-way contract:
237
+ * - resolves a `CertifiedActionRev` — the peer answered and holds the block at that revision;
238
+ * - resolves `undefined` the peer answered and holds NOTHING (an absent claim);
239
+ * - REJECTS the peer could not be asked at all (dial failure, protocol error).
240
+ *
241
+ * The distinction between the last two is load-bearing: `queryClusterForLatest` counts a
242
+ * rejection as a SILENT peer, which stops `CoordinatorRepo.get` from reporting a locally-missing
243
+ * block as an authoritative absent, while a resolved `undefined` is a real answer that keeps the
244
+ * absent authoritative. Implementations must therefore let transport errors propagate rather than
245
+ * swallowing them into `undefined` (see the implementations in `libp2p-node-base` and the mesh
246
+ * harness's `silentPeers` failure knob). Slowness needs no handling here — the caller deadlines
247
+ * each query and treats expiry as silence.
248
+ */
249
+ export type ClusterLatestCallback = (peerId: PeerId, blockId: BlockId, context?: ActionContext) => Promise<CertifiedActionRev | undefined>;
250
+
251
+ interface CoordinatorRepoComponents {
252
+ storageRepo: IRepo;
253
+ localCluster?: LocalClusterWithExecutionTracking;
254
+ localPeerId?: PeerId;
255
+ /**
256
+ * Optional callback to query cluster peers for their latest block revision.
257
+ * Used for read-path cluster verification to discover unknown revisions.
258
+ */
259
+ clusterLatestCallback?: ClusterLatestCallback;
260
+ /**
261
+ * Optional callback that actually moves a block's bytes from the cohort into local storage once
262
+ * {@link clusterLatestCallback} has established a corroborated revision this node lacks. Absent →
263
+ * the read path can still *select* the right revision but converges only when the node already
264
+ * holds the corroborated action as a promotable pending. See {@link AcquireBlockCallback}.
265
+ */
266
+ acquireBlockFromCohort?: AcquireBlockCallback;
267
+ /**
268
+ * Optional layer-2 anchoring for the cohort commit proofs the latest-revision consult verifies
269
+ * (`cluster/certified-claims.ts`): re-derive the block's cohort and LOG the overlap with the
270
+ * proof's signers, plus surface proofs accepted without that comparison. Purely observational —
271
+ * never a gate — and absent in production wiring today; `certifyClaim` logs unanchored
272
+ * acceptance internally regardless.
273
+ */
274
+ proofAnchoring?: ProofAnchoring;
275
+ }
276
+
277
+ /**
278
+ * Consensus config for the coordinator side, plus the repair yardstick the read-repair path measures
279
+ * a (possibly shrunken) cohort view against. `repairCorroborationClusterSize` is deliberately its own
280
+ * field rather than an overload of {@link ClusterConsensusConfig.assumedClusterSize}: this same object
281
+ * also builds the `ClusterCoordinator`, and a field whose value silently differed from the cluster
282
+ * member's copy of it would be a trap. See `cluster/cluster-policy.ts` for why the two differ.
283
+ */
284
+ export type CoordinatorRepoConfig = Partial<ClusterConsensusConfig> & {
285
+ clusterSize?: number;
286
+ repairCorroborationClusterSize?: number;
287
+ };
288
+
289
+ export function coordinatorRepo(
290
+ keyNetwork: IKeyNetwork,
291
+ createClusterClient: (peerId: PeerId) => ICluster,
292
+ cfg?: CoordinatorRepoConfig,
293
+ fretService?: FretService,
294
+ reputation?: IPeerReputation,
295
+ stateStore?: ITransactionStateStore
296
+ ): (components: CoordinatorRepoComponents) => CoordinatorRepo {
297
+ return (components: CoordinatorRepoComponents) => new CoordinatorRepo(
298
+ keyNetwork,
299
+ createClusterClient,
300
+ components.storageRepo,
301
+ cfg,
302
+ components.localCluster,
303
+ components.localPeerId,
304
+ fretService,
305
+ components.clusterLatestCallback,
306
+ reputation,
307
+ stateStore,
308
+ components.acquireBlockFromCohort,
309
+ components.proofAnchoring
310
+ );
311
+ }
312
+
313
+ /**
314
+ * The slice of {@link ClusterCoordinator} that {@link CoordinatorRepo} actually consumes.
315
+ *
316
+ * It exists so a test double has something COMPLETE to satisfy. The doubles in
317
+ * `test/coordinator-repo-*.spec.ts` replace the private `coordinator` field wholesale, and while
318
+ * that field was typed as the whole class and the doubles were assigned through
319
+ * `as unknown as { coordinator: unknown }`, every method this class newly called on the coordinator
320
+ * type-checked fine and then threw `... is not a function` at runtime in every spec holding a
321
+ * double `getClusterPeerIds` cost 21 specs exactly that way. Widening this interface now breaks
322
+ * the doubles at COMPILE time, at the point of widening, which is where the cost belongs.
323
+ *
324
+ * NOTE: the doubles still name the private field by string (`{ coordinator: ... }`), so renaming
325
+ * `CoordinatorRepo.coordinator` would make those casts silently stop applying and every double
326
+ * revert to being ignored. If that field is ever renamed, grep the specs for `coordinator:`.
327
+ */
328
+ export interface ICoordinatorClusterSeam {
329
+ getClusterSize(blockId: BlockId): Promise<number>;
330
+ getClusterPeerIds(blockId: BlockId): Promise<string[]>;
331
+ executeClusterTransaction(blockId: BlockId, message: RepoMessage, options?: MessageOptions): Promise<{
332
+ record: ClusterRecord;
333
+ localExecuted: boolean;
334
+ localPendResult?: PendResult;
335
+ localCommitResult?: CommitResult;
336
+ }>;
337
+ recoverTransactions(): Promise<void>;
338
+ }
339
+
340
+ /** Cluster coordination repo - uses local store, as well as distributes changes to other nodes using cluster consensus. */
341
+ export class CoordinatorRepo implements IRepo {
342
+ private coordinator: ICoordinatorClusterSeam;
343
+ private readonly DEFAULT_TIMEOUT = 30000; // 30 seconds default timeout
344
+ private readonly localPeerId?: PeerId;
345
+ private readonly responsibilityCache = new LruMap<string, { inCluster: boolean, expires: number }>(1000);
346
+ private static readonly RESPONSIBILITY_TTL_MS = 60_000;
347
+ private readonly lastSeenCommitMs = new LruMap<string, number>(1000);
348
+ /** Per block, what earlier repair passes left unresolved — see {@link AheadClaimState}.
349
+ * Outlives the consult on purpose: the read-repair window skips consults for blocks checked
350
+ * recently, and a doubt dropped there is a stale answer served as confirmed again.
351
+ * NOTE: LRU-bounded like `lastSeenCommitMs`; an eviction under >1000 doubted blocks loses the
352
+ * doubt until the next consult re-derives it (one read-repair window later, at worst) and lets
353
+ * {@link reportRepairDeadlock} say its piece a second time. */
354
+ private readonly unsettledAheadClaims = new LruMap<string, AheadClaimState>(1000);
355
+ private readonly readRepairMode: 'off' | 'lazy' | 'paranoid';
356
+ private readonly readRepairWindowMs: number;
357
+ private readonly readRepairSampleRate: number;
358
+ /** Simple-majority threshold from the consensus policy; drives the read-repair corroboration quorum. */
359
+ private readonly simpleMajorityThreshold: number;
360
+ /**
361
+ * Yardstick the read-repair corroboration floor is measured against; the floor for
362
+ * {@link corroboratorCapacity}. Resolved by `resolveClusterPolicy` for a real node; falls back to
363
+ * `assumedClusterSize` and then `clusterSize` for direct constructors (see the constructor), so a
364
+ * caller that has adopted neither field keeps today's behavior exactly.
365
+ */
366
+ private readonly repairCorroborationClusterSize: number;
367
+ /** Resolved super-majority threshold the coordinator commits on (mirrors the value handed to ClusterCoordinator). */
368
+ private readonly superMajorityThreshold: number;
369
+ private readonly reputation?: IPeerReputation;
370
+ /** Per-instance logger, namespaced by peer id when `localPeerId` is known (degrades to the un-suffixed namespace when not — the single-node/test construction has always tolerated its absence). */
371
+ private readonly log: ReturnType<typeof createLogger>;
372
+ /** Test seam: overridable clock for window-based read-repair gating. */
373
+ now: () => number = () => Date.now();
374
+ /** Test seam: overridable RNG (0..1) for sample-rate gating. */
375
+ rand: () => number = () => Math.random();
376
+
377
+ constructor(
378
+ readonly keyNetwork: IKeyNetwork,
379
+ readonly createClusterClient: (peerId: PeerId) => ICluster,
380
+ private readonly storageRepo: IRepo,
381
+ cfg?: CoordinatorRepoConfig,
382
+ private readonly localCluster?: LocalClusterWithExecutionTracking,
383
+ localPeerId?: PeerId,
384
+ fretService?: FretService,
385
+ private readonly clusterLatestCallback?: ClusterLatestCallback,
386
+ reputation?: IPeerReputation,
387
+ stateStore?: ITransactionStateStore,
388
+ private readonly acquireBlockFromCohort?: AcquireBlockCallback,
389
+ private readonly proofAnchoring?: ProofAnchoring
390
+ ) {
391
+ this.localPeerId = localPeerId;
392
+ this.log = createLogger('coordinator-repo', localPeerId?.toString());
393
+ const policy: ClusterConsensusConfig & { clusterSize: number } = {
394
+ // Same constant `resolveClusterPolicy` gives a node that declares no clusterSize, not a
395
+ // second literal: a direct constructor (the readme's manual-wiring path) and the node
396
+ // assembly must land on the same width or the two disagree about the same key's cohort.
397
+ clusterSize: cfg?.clusterSize ?? DEFAULT_CLUSTER_SIZE,
398
+ assumedClusterSize: cfg?.assumedClusterSize,
399
+ superMajorityThreshold: cfg?.superMajorityThreshold ?? DEFAULT_SUPER_MAJORITY_THRESHOLD,
400
+ simpleMajorityThreshold: cfg?.simpleMajorityThreshold ?? 0.51,
401
+ minAbsoluteClusterSize: cfg?.minAbsoluteClusterSize ?? 3,
402
+ allowClusterDownsize: cfg?.allowClusterDownsize ?? true,
403
+ clusterSizeTolerance: cfg?.clusterSizeTolerance ?? 0.5,
404
+ partitionDetectionWindow: cfg?.partitionDetectionWindow ?? 60000,
405
+ commitBroadcastRetryInitialMs: cfg?.commitBroadcastRetryInitialMs ?? 250,
406
+ commitBroadcastRetryBackoffFactor: cfg?.commitBroadcastRetryBackoffFactor ?? 2,
407
+ commitBroadcastRetryMaxIntervalMs: cfg?.commitBroadcastRetryMaxIntervalMs ?? 8000,
408
+ commitBroadcastRetryMaxAttempts: cfg?.commitBroadcastRetryMaxAttempts ?? 5,
409
+ commitBroadcastImmediateRetries: cfg?.commitBroadcastImmediateRetries ?? 1,
410
+ promiseImmediateRetries: cfg?.promiseImmediateRetries ?? 1,
411
+ readRepairMode: cfg?.readRepairMode ?? 'lazy',
412
+ readRepairWindowMs: cfg?.readRepairWindowMs ?? 10000,
413
+ readRepairSampleRate: cfg?.readRepairSampleRate ?? 0,
414
+ // Default false: an undersized cluster with no confident network-size estimate
415
+ // is REJECTED (fail closed). Callers only opt in for single-node/local/test meshes.
416
+ allowUnvalidatedSmallCluster: cfg?.allowUnvalidatedSmallCluster ?? false
417
+ };
418
+ this.readRepairMode = policy.readRepairMode!;
419
+ this.readRepairWindowMs = policy.readRepairWindowMs!;
420
+ this.readRepairSampleRate = policy.readRepairSampleRate!;
421
+ this.simpleMajorityThreshold = policy.simpleMajorityThreshold;
422
+ this.superMajorityThreshold = policy.superMajorityThreshold;
423
+ // Unlike the membership admission gate (which treats an absent assumedClusterSize as "unknown"
424
+ // and admits — refusing writes outright is unacceptable), this falls back to the replication
425
+ // factor and stays strict: the failure mode of getting this wrong is a block that goes
426
+ // unrepaired, degraded rather than dead, so there is no reason to relax it for a caller that
427
+ // has not adopted the new field. A real node is handed an explicit
428
+ // `repairCorroborationClusterSize` by `resolveClusterPolicy`; the `assumedClusterSize` middle
429
+ // term keeps direct constructors (embedders, existing tests) behaving as before.
430
+ this.repairCorroborationClusterSize =
431
+ cfg?.repairCorroborationClusterSize ?? policy.assumedClusterSize ?? policy.clusterSize;
432
+ this.reputation = reputation;
433
+ const localClusterRef = localCluster && localPeerId ? {
434
+ update: localCluster.update.bind(localCluster),
435
+ peerId: localPeerId,
436
+ wasTransactionExecuted: localCluster.wasTransactionExecuted?.bind(localCluster),
437
+ getExecutedPendResult: localCluster.getExecutedPendResult?.bind(localCluster),
438
+ getExecutedCommitResult: localCluster.getExecutedCommitResult?.bind(localCluster)
439
+ } : undefined;
440
+ this.coordinator = new ClusterCoordinator(keyNetwork, createClusterClient, policy, localClusterRef, fretService, reputation, stateStore);
441
+ }
442
+
443
+ /**
444
+ * The resolved super-majority threshold this coordinator commits on. Exposed so the composition root
445
+ * can fail-fast if the coordinator and the cluster member would run different thresholds (see the
446
+ * coupling assertion in `libp2p-node-base.ts`).
447
+ */
448
+ get effectiveSuperMajorityThreshold(): number {
449
+ return this.superMajorityThreshold;
450
+ }
451
+
452
+ /** Recover coordinator transactions from persistent store after a restart. */
453
+ async recoverTransactions(): Promise<void> {
454
+ await this.coordinator.recoverTransactions();
455
+ }
456
+
457
+ /**
458
+ * Check if this node is in the cluster for a given block.
459
+ * Uses findCluster membership in the real network layer, self is always
460
+ * included in the cohort when this node is responsible. This serves as a
461
+ * defense-in-depth guard for requests that arrive at the wrong node.
462
+ * Returns true if localPeerId is not set (backward compat for single-node/test setups).
463
+ */
464
+ private async isResponsibleForBlock(blockId: BlockId): Promise<boolean> {
465
+ if (!this.localPeerId) return true;
466
+
467
+ const cached = this.responsibilityCache.get(blockId);
468
+ if (cached && cached.expires > Date.now()) {
469
+ return cached.inCluster;
470
+ }
471
+
472
+ const blockIdBytes = new TextEncoder().encode(blockId);
473
+ let inCluster: boolean;
474
+ try {
475
+ const peers = await this.keyNetwork.findCluster(blockIdBytes);
476
+ inCluster = this.localPeerId.toString() in peers;
477
+ } catch (err) {
478
+ this.log('proximity:check-error', { blockId, error: (err as Error).message });
479
+ // On failure, assume responsible to avoid false rejections
480
+ return true;
481
+ }
482
+
483
+ this.responsibilityCache.set(blockId, { inCluster, expires: Date.now() + CoordinatorRepo.RESPONSIBILITY_TTL_MS });
484
+ this.log('proximity:checked', { blockId, inCluster });
485
+ return inCluster;
486
+ }
487
+
488
+ /**
489
+ * Verify this node is responsible for all given block IDs. Throws if not.
490
+ */
491
+ private async verifyResponsibility(blockIds: BlockId[]): Promise<void> {
492
+ const notResponsible: BlockId[] = [];
493
+ for (const blockId of blockIds) {
494
+ if (!await this.isResponsibleForBlock(blockId)) {
495
+ notResponsible.push(blockId);
496
+ }
497
+ }
498
+ if (notResponsible.length > 0) {
499
+ this.log('proximity:rejected', { blockIds: notResponsible });
500
+ throw new Error(`Not responsible for block(s): ${notResponsible.join(', ')}`);
501
+ }
502
+ }
503
+
504
+ async get(blockGets: BlockGets, options?: MessageOptions): Promise<GetBlockResults> {
505
+ // Soft proximity check warn but still serve reads for graceful degradation
506
+ // NOTE: a soft-served read now also *acquires* the block durably (see restoreCorroborated), where
507
+ // before it could at most promote a pending this node already held. So a soft serve leaves behind
508
+ // a replica of a block this node is not responsible for, and nothing sweeps those: ring-shift
509
+ // sheds a keyspace RANGE, not "blocks outside my cohort". Fine while soft serves are what they
510
+ // are meant to be — a rare degradation during routing churn — since routing already placed this
511
+ // node near the block. If they ever become routine, gate acquisition (not the serve itself) on
512
+ // isResponsibleForBlock.
513
+ for (const blockId of blockGets.blockIds) {
514
+ if (!await this.isResponsibleForBlock(blockId)) {
515
+ this.log('proximity:get-warning', { blockId, msg: 'serving read for non-responsible block' });
516
+ }
517
+ }
518
+
519
+ // First try local storage
520
+ const localResult = await this.storageRepo.get(blockGets, options);
521
+
522
+ // Decide per-block whether to consult cluster peers. Two triggers:
523
+ // (a) Missing block isn't present locally at all (legacy behavior).
524
+ // (b) Stale-by-policy — block is present but read-repair policy says verify.
525
+ // Skip cluster fetch if this is already a sync request (to prevent recursive queries).
526
+ // A sync read is also never marked `unavailable` here — the consult it skips is the
527
+ // one whose failure the flag reports, and flagging would feed the recursion this
528
+ // bypass exists to prevent. (Storage-level 'unmaterializable' flags still pass
529
+ // through untouched; they report local state, not the consult.)
530
+ const skipClusterFetch = (options as any)?.skipClusterFetch;
531
+ // NOTE: NetworkTransactor.get treats an authoritative "absent" ({ state: {} })
532
+ // as final and no longer retries it (ticket txn-perf-authoritative-notfound),
533
+ // relying on this cluster reconciliation to have already run. When the consult
534
+ // FAILS outright — or runs without ruling the block out and the block stays
535
+ // missing — the entry is flagged `unavailable` below with a reason naming what
536
+ // the consult established (see AbsenceVerdict and the mapping in the loop body),
537
+ // which re-enables the transactor-level retry against a different peer. If a
538
+ // coordinator is configured WITHOUT clusterLatestCallback, there is no cohort to
539
+ // consult and the local answer IS the whole truth — it stays authoritative, with
540
+ // no flag and no transactor-level retry to compensate. That is fine (such a
541
+ // coordinator has no cluster to reconcile against), but keep this coupling in
542
+ // mind if a partial-cluster read path is added.
543
+ if (this.clusterLatestCallback && !skipClusterFetch) {
544
+ for (const blockId of blockGets.blockIds) {
545
+ const localEntry = localResult[blockId];
546
+ const localRev = localEntry?.state?.latest?.rev;
547
+ const isMissing = !localEntry?.state?.latest;
548
+ const isStale = !isMissing && this.shouldReadRepair(blockId);
549
+ if (!isMissing && !isStale) {
550
+ // No consult this pass — the read-repair window says this block was checked
551
+ // recently. An unsettled claim an earlier pass recorded still applies: the doubt
552
+ // is a property of what this node HOLDS, not of whether a consult just ran.
553
+ // Without this, every read inside the window after a failed convergence would
554
+ // serve the same content as confirmed the exact silent lie this marker exists
555
+ // to end, re-opened for `readRepairWindowMs` at a time.
556
+ this.flagUnconfirmedCurrency(localResult, blockId, blockGets.context);
557
+ continue;
558
+ }
559
+
560
+ if (isStale) {
561
+ this.log('cluster-tx:read-repair-triggered', {
562
+ blockId,
563
+ mode: this.readRepairMode,
564
+ ageMs: this.ageMs(blockId),
565
+ localRev
566
+ });
567
+ }
568
+
569
+ try {
570
+ const { absence, claimedAheadRev } = await this.fetchBlockFromCluster(blockId, blockGets.context, localRev);
571
+ const refreshed = await this.storageRepo.get({ blockIds: [blockId], context: blockGets.context }, options);
572
+ const newRev = refreshed[blockId]?.state?.latest?.rev;
573
+ if (refreshed[blockId]) {
574
+ localResult[blockId] = refreshed[blockId];
575
+ }
576
+ if (isStale) {
577
+ if (typeof newRev === 'number' && typeof localRev === 'number' && newRev > localRev) {
578
+ this.log('cluster-tx:read-repair-applied', { blockId, oldRev: localRev, newRev });
579
+ } else {
580
+ this.log('cluster-tx:read-repair-noop', { blockId });
581
+ }
582
+ }
583
+ // The consult ran but could not rule the block out, and the verdict names the
584
+ // evidence (see AbsenceVerdict): part of the cohort was silent (`unconfirmed`
585
+ // 'peers-unreachable' another coordinator may know better), no cohort
586
+ // member outside this node could be asked at all (`isolated`
587
+ // 'cohort-unreachable' there is no better-connected coordinator to re-ask),
588
+ // or a peer positively claimed a revision this pass could neither corroborate
589
+ // nor acquire (`claimed` 'claimed-elsewhere' — the block is known to exist
590
+ // somewhere). Either way a still-missing block must not pose as an
591
+ // authoritative absent. When the whole cohort answers "holds nothing" the
592
+ // absent stays authoritative (`confirmed`) — the new-collection probe against
593
+ // a healthy cohort stays one round-trip.
594
+ if (isMissing && absence !== 'confirmed') {
595
+ this.flagUnconfirmedAbsence(localResult, blockId,
596
+ absence === 'claimed' ? 'claimed-elsewhere'
597
+ : absence === 'isolated' ? 'cohort-unreachable'
598
+ : 'peers-unreachable');
599
+ }
600
+ // A PRESENT block served below a cohort claim the repair could not settle is
601
+ // the mirror lie: real content posing as confirmed-current. This consult is the
602
+ // authority on that claim, so it replaces whatever an earlier one recorded —
603
+ // including clearing it when nobody claims anything any more. The missing case
604
+ // is excluded it is the absence path above, and a bare absent below a claim
605
+ // already reads as either authoritative (cohort answered, nothing corroborated)
606
+ // or flagged.
607
+ if (!isMissing) {
608
+ this.recordAheadClaim(blockId, claimedAheadRev);
609
+ this.flagUnconfirmedCurrency(localResult, blockId, blockGets.context);
610
+ }
611
+ } catch (err) {
612
+ this.log('cluster-fetch:error', { blockId, error: (err as Error).message });
613
+ // The consult that was supposed to make this answer trustworthy did not run.
614
+ // NOTE: a consult that THROWS (e.g. `findCluster` itself rejected) is reported
615
+ // 'peers-unreachable' even on an isolated node: a failed cohort lookup is a
616
+ // routing failure and says nothing about how many cohort members were
617
+ // reachable. If `findCluster` on an isolated node turns out to throw routinely
618
+ // rather than return a stale cohort view, revisit that would put the
619
+ // isolated case back under this vaguer reason.
620
+ if (isMissing) {
621
+ this.flagUnconfirmedAbsence(localResult, blockId, 'peers-unreachable');
622
+ } else {
623
+ // It told us nothing, so it refutes nothing: an earlier pass's unsettled
624
+ // claim stands.
625
+ this.flagUnconfirmedCurrency(localResult, blockId, blockGets.context);
626
+ }
627
+ }
628
+ }
629
+ }
630
+
631
+ return localResult;
632
+ }
633
+
634
+ /**
635
+ * Downgrade an absence the coordinator could not confirm to the given `unavailable` reason —
636
+ * a flag `NetworkTransactor.get` retries against another peer instead of taking as final.
637
+ * The reason names the evidence (see {@link AbsenceVerdict} for the mapping in `get`); this
638
+ * method only decides WHETHER the entry may carry a flag at all.
639
+ *
640
+ * No-op once the entry carries a real answer (the consult restored the block) or a sharper flag
641
+ * (storage's `'unmaterializable'`), so callers only need to establish that the answer is a guess.
642
+ *
643
+ * "Carries a real answer" is tested as `entry.block !== undefined`, NOT as `state.latest` being
644
+ * set. The two used to move together, so `state.latest` read as a serviceable proxy — but a
645
+ * pending-only insert (pended, not yet committed) served through the pending overlay has real
646
+ * CONTENT and no committed revision at all, so its `state.latest` is undefined. Flagging that
647
+ * entry would mark a block this node is positively holding as an unconfirmed absence, and
648
+ * `NetworkTransactor`'s `isAuthoritative` keys off the flag alone — the read would burn its
649
+ * retry budget re-asking other peers for content it already has. `state.latest` stays in the
650
+ * test as well so a stale-but-real committed answer is likewise never downgraded.
651
+ */
652
+ private flagUnconfirmedAbsence(results: GetBlockResults, blockId: BlockId, reason: BlockUnavailableReason): void {
653
+ const entry = results[blockId];
654
+ if (!entry) {
655
+ results[blockId] = { state: {}, unavailable: reason };
656
+ } else if (entry.block === undefined && !entry.state?.latest && entry.unavailable === undefined) {
657
+ entry.unavailable = reason;
658
+ }
659
+ }
660
+
661
+ /**
662
+ * Remember (or forget) the cohort claim a freshness consult could not settle for a block.
663
+ * Only a consult that actually RAN may call this: it is the authority, so `undefined` clears
664
+ * a claim an earlier pass recorded. Entries are also dropped once this node reaches the
665
+ * claimed revision (see {@link flagUnconfirmedCurrency}), which is what bounds the map.
666
+ */
667
+ private recordAheadClaim(blockId: BlockId, claimedRev: number | undefined): void {
668
+ const prior = this.unsettledAheadClaims.get(blockId);
669
+ if (claimedRev === undefined) {
670
+ // The consult is the authority on the CLAIM, and only on the claim. A recorded deadlock is
671
+ // not about any revision — it is about how many machines this deployment can field, or how
672
+ // many of them hold the block so it outlives the claim that first exposed it and is
673
+ // dropped only when the block converges (see {@link flagUnconfirmedCurrency}).
674
+ if (prior?.deadlocksReported) this.unsettledAheadClaims.set(blockId, { deadlocksReported: prior.deadlocksReported });
675
+ else this.unsettledAheadClaims.delete(blockId);
676
+ return;
677
+ }
678
+ this.unsettledAheadClaims.set(blockId, {
679
+ rev: claimedRev,
680
+ ...(prior?.deadlocksReported ? { deadlocksReported: prior.deadlocksReported } : {})
681
+ });
682
+ }
683
+
684
+ /**
685
+ * Stamp {@link GetBlockResult.unconfirmedAheadRev} on an entry sitting behind an unsettled
686
+ * cohort claim served committed content the coordinator cannot confirm is current.
687
+ * Deliberately narrow; ALL of these must hold:
688
+ * - a consult (this read's or an earlier one's, see {@link recordAheadClaim}) left a claim
689
+ * unsettled for this block;
690
+ * - the entry carries a committed revision (a present block, or a committed tombstone) —
691
+ * never a plain absent, which is the absence path's business;
692
+ * - that served revision is still strictly BELOW the claim: the repair did not converge, and
693
+ * nothing committed past the claim in the meantime (if it did, the claim is settled and the
694
+ * memo is dropped here);
695
+ * - the caller asked for a view that should contain the claim: an unpinned "latest" read, or
696
+ * a pin at/above the claimed revision. A read pinned BELOW the claim is being served
697
+ * correctly and stays unstamped — this keeps a collection's context-pinned data reads
698
+ * quiet while its unpinned tail read (the one seam where fresher truth could arrive —
699
+ * Collection.bootstrapContext) speaks up.
700
+ * NOT covered, on purpose: a cohort that is merely silent and claims nothing (pinned as
701
+ * authoritative by the merely-STALE spec in coordinator-repo-unavailable.spec.ts) — silence
702
+ * carries no revision to be behind of.
703
+ *
704
+ * Pin comparability: `ActionContext.rev` and a block's `state.latest.rev` count the same
705
+ * per-collection revision sequence — `Collection.bootstrapContext` seeds the context straight
706
+ * from the tail block's `latest.rev`, and `syncInternal` commits every block of an action at
707
+ * `context.rev + 1` so `context.rev >= claimedRev` is a well-defined comparison. `state.latest`
708
+ * is this node's newest revision for the block even on a pinned read (StorageRepo reports the
709
+ * content's own revision separately as `materialized`), which is exactly the number "is this
710
+ * node behind the claim?" asks about.
711
+ */
712
+ private flagUnconfirmedCurrency(results: GetBlockResults, blockId: BlockId, context?: ActionContext): void {
713
+ const claimedRev = this.unsettledAheadClaims.get(blockId)?.rev;
714
+ if (claimedRev === undefined) return;
715
+ const entry = results[blockId];
716
+ if (!entry || entry.unavailable !== undefined) return;
717
+ const servedRev = entry.state?.latest?.rev;
718
+ if (typeof servedRev !== 'number') return;
719
+ if (servedRev >= claimedRev) {
720
+ // Caught up by this pass's repair or by a commit that landed since. Nothing to doubt, and
721
+ // nothing deadlocked either: repair demonstrably converged for this block, so a later
722
+ // non-convergence is a new episode and gets to say so again.
723
+ this.unsettledAheadClaims.delete(blockId);
724
+ return;
725
+ }
726
+ if (context !== undefined && context.rev < claimedRev) return;
727
+ entry.unconfirmedAheadRev = claimedRev;
728
+ this.log('cluster-tx:read-unconfirmed', { blockId, servedRev, claimedAheadRev: claimedRev });
729
+ }
730
+
731
+ /** Decide whether the read-repair policy wants us to consult the cluster for a present-but-possibly-stale block. */
732
+ private shouldReadRepair(blockId: BlockId): boolean {
733
+ switch (this.readRepairMode) {
734
+ case 'off': return false;
735
+ case 'paranoid': return true;
736
+ case 'lazy': {
737
+ const lastSeen = this.lastSeenCommitMs.get(blockId);
738
+ if (lastSeen == null) return true;
739
+ if (this.now() - lastSeen > this.readRepairWindowMs) return true;
740
+ if (this.readRepairSampleRate > 0 && this.rand() < this.readRepairSampleRate) return true;
741
+ return false;
742
+ }
743
+ }
744
+ }
745
+
746
+ /** Milliseconds since we last marked this block fresh, or undefined if never. */
747
+ private ageMs(blockId: BlockId): number | undefined {
748
+ const lastSeen = this.lastSeenCommitMs.get(blockId);
749
+ return lastSeen == null ? undefined : this.now() - lastSeen;
750
+ }
751
+
752
+ /** Mark blocks as freshly observed from cluster authority (post-commit or post-fetch). */
753
+ private markBlocksSeen(blockIds: BlockId[]): void {
754
+ const now = this.now();
755
+ for (const id of blockIds) {
756
+ this.lastSeenCommitMs.set(id, now);
757
+ }
758
+ }
759
+
760
+ /**
761
+ * Test seam: directly set the last-seen timestamp for a block. Used by read-repair
762
+ * specs to simulate "the local commit happened at time T" without needing to drive
763
+ * a full pend/commit cycle through the cluster coordinator.
764
+ */
765
+ setLastSeenForTest(blockId: BlockId, ts: number): void {
766
+ this.lastSeenCommitMs.set(blockId, ts);
767
+ }
768
+
769
+ /**
770
+ * One repair pass for a block: ask the cohort what it holds, and converge onto that if it is
771
+ * ahead of `localRev` — the revision the caller's read already loaded, and the baseline every
772
+ * decision below is measured against.
773
+ *
774
+ * Returns the two things `get` needs beyond the storage side effects:
775
+ * - `absence` — the verdict on this node's local absence of the block (see
776
+ * {@link AbsenceVerdict}): whether the pass may rule the block out, and on what evidence.
777
+ * Only `'confirmed'` lets a still-missing block be reported as an authoritative absent.
778
+ * Paths that consult nobody (no cohort, solo-self) are `'confirmed'`: there, the local
779
+ * answer genuinely is the whole truth. When several verdicts apply at once the sharpest
780
+ * evidence wins: `claimed` > `isolated` > `unconfirmed` > `confirmed` — a peer positively
781
+ * saying "it exists" outranks any amount of silence.
782
+ * - `claimedAheadRev` a cohort peer claimed a revision strictly ahead of what this node
783
+ * holds and the pass did NOT converge onto it: the claim failed the corroboration quorum,
784
+ * or was corroborated but could not be acquired. Content `get` serves below this revision
785
+ * cannot be confirmed current (see {@link GetBlockResult.unconfirmedAheadRev}); the claim
786
+ * itself must never drive restoration.
787
+ */
788
+ private async fetchBlockFromCluster(blockId: BlockId, context?: ActionContext, localRev?: number): Promise<{ absence: AbsenceVerdict; claimedAheadRev?: number }> {
789
+ if (!this.clusterLatestCallback) return { absence: 'confirmed' };
790
+
791
+ const blockIdBytes = new TextEncoder().encode(blockId);
792
+ const peers = await this.keyNetwork.findCluster(blockIdBytes);
793
+ const peerIds = peers ? Object.keys(peers) : [];
794
+ if (peerIds.length === 0) return { absence: 'confirmed' };
795
+
796
+ // Solo-cluster short-circuit: the only responsible peer is us. There is no
797
+ // remote to sync from, so skip the callback entirely. Querying ourselves
798
+ // would dial self via SyncClient pointless at best, and on nodes without
799
+ // listen addresses (e.g. solo WebSocket-only) the dial can hang.
800
+ if (
801
+ peerIds.length === 1
802
+ && this.localPeerId
803
+ && peerIds[0] === this.localPeerId.toString()
804
+ ) {
805
+ this.log('cluster-fetch:solo-self-skip', { blockId });
806
+ return { absence: 'confirmed' };
807
+ }
808
+
809
+ const { corroborated, local, silent, answered, uncorroboratedRev } = await this.queryClusterForLatest(peerIds, blockId, context);
810
+ // Any silence taints the WHOLE consult, not a fraction of it (fail-closed): one silent
811
+ // peer could be the sole holder, and the cost an extra transactor-level retry against
812
+ // another coordinator is paid only while a peer is actually unreachable. Silence with
813
+ // NOBODY else reached at all is its own verdict: partial silence says "ask a better-
814
+ // connected coordinator", total silence says there is no better-informed answer to be
815
+ // had from this node.
816
+ const silenceVerdict: AbsenceVerdict =
817
+ silent.length > 0 ? (answered === 0 ? 'isolated' : 'unconfirmed') : 'confirmed';
818
+ // Nothing corroborated: keep local data AND stay eligible for repair — marking the
819
+ // block seen here would suppress the next attempt for the whole read-repair window.
820
+ // An uncorroborated claim strictly ahead of what this node holds still travels up as
821
+ // doubt: the answer about to be served may be behind it, and only the caller knows
822
+ // whether that matters for the view it was asked for.
823
+ if (!corroborated) {
824
+ const uncorroboratedBaseline = local?.rev ?? localRev;
825
+ const claimIsAhead = uncorroboratedRev !== undefined
826
+ && (uncorroboratedBaseline === undefined || uncorroboratedRev > uncorroboratedBaseline);
827
+ // A claim even one the quorum declined is a peer positively attesting the block
828
+ // exists, the sharpest fact this pass can surface. It outranks silence.
829
+ const absence: AbsenceVerdict = uncorroboratedRev !== undefined ? 'claimed' : silenceVerdict;
830
+ return { absence, ...(claimIsAhead ? { claimedAheadRev: uncorroboratedRev } : {}) };
831
+ }
832
+
833
+ // The self answer is the sharper baseline (same storage, same context, read alongside the
834
+ // cohort's), but it exists only when `findCluster` returned this node. A soft serve for a
835
+ // block this node is no longer responsible for is absent from its own cohort view, so fall
836
+ // back to the revision the caller's read already loaded. Without the fallback both decisions
837
+ // below degrade to "any local revision is an advance", which restores backwards and reports
838
+ // a sync at the revision the pass started from.
839
+ const baselineRev = local?.rev ?? localRev;
840
+
841
+ // Never restore backwards. With this node's own claim excluded from the quorum, a
842
+ // cohort that lags behind the reader corroborates an OLDER revision; adopting it
843
+ // would be a regression, and logging it as a sync would be a lie. The cohort did
844
+ // answer, so the block is verified freshmark it seen.
845
+ // NOTE: in a cohort of two, that sole peer is the only corroborator, so a lying one can park
846
+ // the reader here corroborating the revision it already holds — and re-arm the lazy window
847
+ // on every pass, hiding a real divergence. Bounded by `readRepairWindowMs` (10s default) and
848
+ // no worse than the peer simply staying silent. If two-member cohorts become a supported
849
+ // production topology rather than a dev convenience, stop re-arming the window on a
850
+ // corroboration that came from a single voter.
851
+ if (baselineRev !== undefined && corroborated.rev <= baselineRev) {
852
+ this.log('cluster-fetch:local-current', { blockId, localRev: baselineRev, clusterRev: corroborated.rev });
853
+ this.markBlocksSeen([blockId]);
854
+ // Only reachable when this node HOLDS a revision (the baseline), so `get` never
855
+ // consults this verdict computed consistently rather than hard-coded.
856
+ return { absence: silenceVerdict };
857
+ }
858
+
859
+ // Corroborated revision is ahead of ours — converge onto it.
860
+ const rev = await this.restoreCorroborated(blockId, corroborated, baselineRev, peerIds);
861
+
862
+ // Log the OUTCOME, not the attempt. Logging `synced` unconditionally reported hundreds of
863
+ // phantom convergences per run and made a real replication defect invisible for two debugging
864
+ // sessions.
865
+ if (rev !== undefined) {
866
+ this.log('cluster-fetch:synced', { blockId, rev });
867
+ } else {
868
+ this.log('cluster-fetch:not-restored', { blockId, localRev: baselineRev, clusterRev: corroborated.rev });
869
+ }
870
+ // A corroborated revision this node failed to converge onto rules nothing out, even with
871
+ // the whole cohort answering: the reader has just been TOLD the block exists, so reporting
872
+ // it absent would be a lie regardless of silence — that is the `claimed` verdict, and it
873
+ // outranks whatever the silence-based mapping would have said.
874
+ const absence: AbsenceVerdict = rev === undefined ? 'claimed' : silenceVerdict;
875
+ // Converged means REACHED the corroborated revision, not merely advanced: a promotion that
876
+ // landed short of it (possible in principle restoreCorroborated only requires an advance
877
+ // over the baseline) still leaves the served answer behind a revision the cohort attested.
878
+ const converged = rev !== undefined && rev >= corroborated.rev;
879
+ // The block is marked seen either way — the cohort DID answer, so its freshness was checked,
880
+ // which is what the read-repair window tracks. A failed convergence therefore waits out the
881
+ // window before retrying. The DOUBT it produced does not wait: `get` remembers the
882
+ // unsettled claim (`recordAheadClaim`) and keeps stamping reads served below it while the
883
+ // window suppresses the retry the window damps repair effort, not honesty.
884
+ // NOTE: that damping covers only a block this node holds at an OLDER revision. A block entirely
885
+ // missing locally never consults the window (`get` triggers on `isMissing` before
886
+ // `shouldReadRepair`), so a persistently failing acquisition — e.g. a two-node deployment that
887
+ // never set `assumedClusterSize`, where the content quorum can never be met — re-fetches an
888
+ // archive on every read of that block. Correct, and self-limiting once the cohort can agree; if
889
+ // it ever shows as read amplification, gate the acquisition step (not the latest-query) on the
890
+ // same window rather than widening `isMissing`.
891
+ this.markBlocksSeen([blockId]);
892
+ return { absence, ...(converged ? {} : { claimedAheadRev: corroborated.rev }) };
893
+ }
894
+
895
+ /**
896
+ * Bring this node up to the cohort-corroborated `corroborated`, returning the revision it holds
897
+ * afterwards when that is an advance over `baselineRev`, else `undefined`.
898
+ *
899
+ * Two mechanisms, cheapest first:
900
+ * 1. **Promote a local pending** free, no network, and the only mechanism that existed before
901
+ * block acquisition. Covers the node that saw the pend and missed the commit broadcast.
902
+ * 2. **Acquire the bytes from the cohort** ({@link AcquireBlockCallback}) covers everything else,
903
+ * including a block this node has never seen at all.
904
+ *
905
+ * **Why acquisition is gated here and not on a plain local miss.** `BlockStorage.getBlock` returns
906
+ * `undefined` for a block with no local metadata *without* consulting its restore callback, so that
907
+ * an insert probing a fresh random block id for a collision does not cost a network fetch. That
908
+ * remains true: this method runs only after {@link queryClusterForLatest} produced a quorum-
909
+ * corroborated `(rev, actionId)`, which a genuinely non-existent block can never produce (no peer
910
+ * claims it, so `selectQuorumRev` declines and `fetchBlockFromCluster` returns before reaching
911
+ * here). The cost of a genuine absence is unchanged — the latest-query round trip that already
912
+ * happened — while a block the cohort demonstrably holds is no longer thrown away.
913
+ *
914
+ * Cohort peer ids are passed straight through: the callback filters self out and caps its own
915
+ * corroboration quorum by how many peers could answer at all.
916
+ */
917
+ private async restoreCorroborated(
918
+ blockId: BlockId,
919
+ corroborated: ActionRev,
920
+ baselineRev: number | undefined,
921
+ cohortPeerIds: string[]
922
+ ): Promise<number | undefined> {
923
+ const promoted = await this.promoteCorroborated(blockId, corroborated);
924
+ if (isAdvanceOver(promoted, baselineRev)) {
925
+ return promoted;
926
+ }
927
+
928
+ if (!this.acquireBlockFromCohort) {
929
+ return undefined;
930
+ }
931
+ try {
932
+ // Bounded: a stalled cohort peer must not hold up the caller's read. Persisting happens
933
+ // inside the callback via `saveReplicatedBlock`, which takes the block write latch
934
+ // safe to call from here because the read path holds no latch of its own (`StorageRepo.get`
935
+ // acquires and releases it around the promotion above, and nothing wraps this method).
936
+ // NOTE: `get` walks its block ids sequentially, so the bound is per block, not per call — a
937
+ // multi-block read that is missing N blocks against a wholly stalled cohort waits N × this.
938
+ // Acceptable today (the underlying per-peer archive fetch is itself 1s-bounded and runs the
939
+ // cohort in parallel, so the 5s is a stall ceiling, not a typical cost). If a cold reader
940
+ // batching a wide read ever times out above this layer, repair the block ids concurrently
941
+ // rather than shortening the bound.
942
+ await withDeadline(
943
+ this.acquireBlockFromCohort(blockId, corroborated, cohortPeerIds),
944
+ RECONCILE_TIMEOUT_MS,
945
+ `block acquisition for ${blockId}`
946
+ );
947
+ } catch (err) {
948
+ // Declines are cheap and retryable — nothing was persisted. Report and leave the block behind.
949
+ this.log('cluster-fetch:acquire-error', { blockId, rev: corroborated.rev, error: (err as Error).message });
950
+ return undefined;
951
+ }
952
+ const acquired = await this.readLocalRev(blockId);
953
+ return isAdvanceOver(acquired, baselineRev) ? acquired : undefined;
954
+ }
955
+
956
+ /**
957
+ * Promote a corroborated action this node already holds as a local pending — the no-network half of
958
+ * the repair. Returns the local revision afterwards.
959
+ *
960
+ * A pending-only block (metadata seeded by `savePendingTransaction`, no committed revision) asked
961
+ * for a forward revision no promotion can reach used to throw out of the restore step (now
962
+ * `BlockStorage.restoreRevision`, driven by `StorageRepo.get`'s healing helper).
963
+ * It no longer does: "no committed base here" is an absence, so that read comes back as a plain
964
+ * unflagged `{ state: {} }` and this method simply returns `undefined` — acquisition then supplies
965
+ * the revision. The `unavailable` arm below still fires for the shapes that ARE a guess (a `latest`
966
+ * this node cannot materialize, a missing-base promotion refusal); on THIS path those are an
967
+ * absence too rather than a read failure, so they are logged as `promote-unavailable` and stepped
968
+ * over rather than short-circuiting the caller. The catch stays for any other fault, same reason.
969
+ */
970
+ private async promoteCorroborated(blockId: BlockId, corroborated: ActionRev): Promise<number | undefined> {
971
+ try {
972
+ const entry = await this.readLocalEntry(blockId, { committed: [corroborated], rev: corroborated.rev });
973
+ if (entry?.unavailable !== undefined) {
974
+ this.log('cluster-fetch:promote-unavailable', { blockId, rev: corroborated.rev, error: entry.unavailable });
975
+ return undefined;
976
+ }
977
+ return entry?.state?.latest?.rev;
978
+ } catch (err) {
979
+ this.log('cluster-fetch:promote-unavailable', { blockId, rev: corroborated.rev, error: (err as Error).message });
980
+ return undefined;
981
+ }
982
+ }
983
+
984
+ /** This node's own answer for a block, optionally driving a promotion context through the read.
985
+ * Callers that care whether the answer is authoritative inspect `entry.unavailable`. */
986
+ private async readLocalEntry(blockId: BlockId, context?: ActionContext) {
987
+ const result = await this.storageRepo.get({ blockIds: [blockId], context });
988
+ return result[blockId];
989
+ }
990
+
991
+ /** This node's own `latest.rev` for a block, optionally driving a promotion context through the read. */
992
+ private async readLocalRev(blockId: BlockId, context?: ActionContext): Promise<number | undefined> {
993
+ return (await this.readLocalEntry(blockId, context))?.state?.latest?.rev;
994
+ }
995
+
996
+ /**
997
+ * Query cluster peers for their latest revision and return the highest revision
998
+ * corroborated by a quorum of distinct peers, alongside this node's own latest.
999
+ *
1000
+ * Replaces the old "max rev any single peer reports" — which let one lying
1001
+ * peer over-reporting its revision steer restoration — with quorum
1002
+ * corroboration on the exact `(rev, actionId)` pair (see {@link selectQuorumRev}).
1003
+ *
1004
+ * This node's own answer is split out of the claim set rather than counted in it:
1005
+ * `clusterLatestCallback` short-circuits self to local storage, so including it let a
1006
+ * reader whose only peer timed out "corroborate" the very revision it was trying to
1007
+ * repair. It is returned separately so the caller can compare, not vote.
1008
+ *
1009
+ * NOTE: the quorum is corroboration-of-a-claim, NOT Sybil-resistant cohort
1010
+ * membership a peer minting fresh keypairs still casts a vote. A claim that arrives
1011
+ * with a cohort commit proof is additionally VERIFIED here (`certifyClaim`,
1012
+ * `cluster/certified-claims.ts`); when the proof holds, the claim is certified and
1013
+ * {@link selectQuorumRev} accepts it without a second voter the cohort's signature set
1014
+ * is its corroboration. What a passing proof does NOT prove is that its signers are the
1015
+ * block's responsible cohort (anyone controlling N keys can sign their own N-peer
1016
+ * proof); anchoring the signer set to topology is the optional, observational-only
1017
+ * {@link ProofAnchoring} layer, unwired in production today.
1018
+ */
1019
+ private async queryClusterForLatest(peerIds: string[], blockId: BlockId, context?: ActionContext): Promise<ClusterLatestQuery> {
1020
+ // Query peers in parallel for their latest revision. Each query is DEADLINED (rejects), not
1021
+ // raced-to-undefined: a peer that blows the deadline lands in the silent set below exactly
1022
+ // like a dial failure, because a slow peer and a peer claiming "I hold nothing" must produce
1023
+ // different answers (ticket cluster-read-consult-cannot-report-unreachable).
1024
+ // NOTE: LATEST_QUERY_TIMEOUT_MS is a LAN-shaped budget. A cohort whose round trip honestly
1025
+ // exceeds it now reads as permanently silent, which is safe (the read is flagged, not
1026
+ // mis-reported) but makes every miss cost a transactor-level retry. If a WAN deployment shows
1027
+ // steady `cluster-fetch:peers-silent` against healthy peers, raise this rather than softening
1028
+ // the deadline back into an absent claim.
1029
+ const latestResults = await Promise.allSettled(
1030
+ peerIds.map(async peerIdStr => {
1031
+ const peerId = peerIdFromString(peerIdStr);
1032
+ return await withDeadline(
1033
+ this.clusterLatestCallback!(peerId, blockId, context),
1034
+ LATEST_QUERY_TIMEOUT_MS,
1035
+ `latest query to ${peerIdStr}`
1036
+ );
1037
+ })
1038
+ );
1039
+
1040
+ // NOTE: self-exclusion is keyed on `localPeerId`, which is optional for the single-node/test
1041
+ // construction this class has always tolerated. Left unset, this node's own answer is counted
1042
+ // as a peer claim again. Harmless today — the self answer can only ever corroborate the
1043
+ // revision already held, so the pass declines as `local-current` — but if a future caller can
1044
+ // make self report something the reader does not hold, make `localPeerId` required instead.
1045
+ // The same unset-`localPeerId` tolerance also lets self count toward `answered` below, and
1046
+ // lets a self read that REJECTS land in `silent`: a solo repo whose own storage throws then
1047
+ // reads as `answered === 0` and reports isolation ('cohort-unreachable') rather than a local
1048
+ // fault. Same fix if it ever matters — require `localPeerId`.
1049
+ const selfId = this.localPeerId?.toString();
1050
+ let local: CertifiedActionRev | undefined;
1051
+ const claims: RevClaim[] = [];
1052
+ const silent: string[] = [];
1053
+ // `allSettled` preserves input order, so results correlate to `peerIds` by index — a
1054
+ // rejected entry carries no payload of its own, and its peer id is what `silent` records.
1055
+ for (let i = 0; i < latestResults.length; i++) {
1056
+ const result = latestResults[i]!;
1057
+ const peerIdStr = peerIds[i]!;
1058
+ if (result.status !== 'fulfilled') {
1059
+ // Silence: the callback rejected or the deadline expired. Never a claim. Self is
1060
+ // excluded — its short-circuit reads local storage, and a local read error is not a
1061
+ // cohort peer being unreachable.
1062
+ if (peerIdStr !== selfId) silent.push(peerIdStr);
1063
+ continue;
1064
+ }
1065
+ const value = result.value;
1066
+ if (peerIdStr === selfId) {
1067
+ local = value;
1068
+ continue;
1069
+ }
1070
+ if (!value) continue; // responded, holds nothing — an absent claim, not silence
1071
+ // The proof rides along here and is verified BELOW (certifyClaim) before selection reads
1072
+ // the claim set: presence proves nothing the peer chose what to attach but a proof
1073
+ // that verifies certifies the claim, and a certified claim needs no second voter.
1074
+ claims.push({
1075
+ peerId: peerIdStr, rev: value.rev, actionId: value.actionId,
1076
+ ...(value.proof ? { proof: value.proof } : {})
1077
+ });
1078
+ }
1079
+ if (silent.length > 0) {
1080
+ this.log('cluster-fetch:peers-silent', { blockId, silent: silent.length, consulted: peerIds.length });
1081
+ }
1082
+
1083
+ // Verify every attached proof, in parallel, BEFORE selection — and penalize provable proof
1084
+ // misbehavior HERE, at verification time, independent of what selection later does with the
1085
+ // claim. Only attributable failures (isAttributableProofFailure) are penalized: a failure
1086
+ // whose signer identities were never proven — unknown/non-ed25519 signer, malformed
1087
+ // signature or proof, a legacy record, the oversized-cohort cap — could have been authored
1088
+ // by anyone in the chain, and penalizing on it would let an attacker frame a peer (the same
1089
+ // discipline as VerifyOutcome.penalize in cluster-repo.ts). A claim whose proof fails stays
1090
+ // in the claim set UNCERTIFIED: it still corroborates by distinct-peer count exactly as a
1091
+ // proof-less claim does a peer that could fabricate a bad proof could equally have sent no
1092
+ // proof, so dropping the vote would buy nothing.
1093
+ // NOTE: cost is one verification pass per proof-carrying answer per consult, each bounded by
1094
+ // MAX_PROOF_SIGNERS (256) signature checks. In `lazy` mode consults are rate-limited by the
1095
+ // read-repair window; in `paranoid` mode every read of every block pays cohort-width
1096
+ // verifications. Fine at deployment cohort sizes (~10) if paranoid readers ever show CPU
1097
+ // time in `certifyClaim`, cache verdicts per (blockId, rev, actionId, proof hash) rather than
1098
+ // skipping verification.
1099
+ await Promise.all(claims.map(async claim => {
1100
+ if (!claim.proof) return;
1101
+ const verdict = await certifyClaim(
1102
+ claim.proof,
1103
+ { blockId, rev: claim.rev, actionId: claim.actionId },
1104
+ // Shared with the reconcile path, so the two cannot drift on what the members actually
1105
+ // enforced — see `proofThresholds` for why the simple-majority term is not
1106
+ // this.simpleMajorityThreshold.
1107
+ proofThresholds(this.superMajorityThreshold),
1108
+ this.proofAnchoring
1109
+ );
1110
+ if (verdict.certified) {
1111
+ claim.certified = true;
1112
+ // The verdict's signer count, never proof.peerIds read here — selection weighs a
1113
+ // single-signer certification below multi-peer corroboration (quorum-restore.ts).
1114
+ claim.certifiedSignerCount = verdict.signerCount;
1115
+ return;
1116
+ }
1117
+ this.log('cluster-fetch:proof-uncertified', {
1118
+ blockId, peerId: claim.peerId, rev: claim.rev, failure: verdict.failure
1119
+ });
1120
+ if (isAttributableProofFailure(verdict.failure)) {
1121
+ this.penalizeProofService(claim.peerId, blockId);
1122
+ }
1123
+ }));
1124
+
1125
+ const nonSelfCount = peerIds.filter(id => id !== selfId).length;
1126
+ const answered = nonSelfCount - silent.length;
1127
+ const capacity = corroboratorCapacity(nonSelfCount, this.repairCorroborationClusterSize);
1128
+ const required = quorumSize(claims.length, this.simpleMajorityThreshold, capacity);
1129
+ const selected = selectQuorumRev(claims, this.simpleMajorityThreshold, capacity);
1130
+ if (!selected) {
1131
+ // A decline can be the certified path REFUSING to pick a side: two distinct actions each
1132
+ // carrying a verified cohort proof for the same top revision. Name that apart from the
1133
+ // routine no-quorum the cohort (or whoever holds its keys) provably signed both sides,
1134
+ // an incident rather than a shortage of answers. Neither claimant is penalized: both
1135
+ // proofs verified, so which side is "wrong" is exactly what this node cannot know.
1136
+ const equivocation = certifiedEquivocation(claims);
1137
+ if (equivocation) {
1138
+ this.log('cluster-fetch:certified-equivocation', {
1139
+ blockId, rev: equivocation.rev, actionIds: equivocation.actionIds
1140
+ });
1141
+ }
1142
+ // The three populations are reported SEPARATELY, never rolled into one "responders" count:
1143
+ // "1 of 2 responded" and "1 holder, 1 confirmed non-holder, 0 silent" call for completely
1144
+ // different operator actions the first says wait or fix reachability, the second says the
1145
+ // block has only one copy and no amount of waiting produces a second.
1146
+ this.log('cluster-fetch:no-quorum', {
1147
+ blockId,
1148
+ cohortPeers: nonSelfCount,
1149
+ holders: claims.length,
1150
+ absent: answered - claims.length,
1151
+ silent: silent.length,
1152
+ required,
1153
+ repairCorroborationClusterSize: this.repairCorroborationClusterSize
1154
+ });
1155
+ // ...and, when this decline is provably permanent rather than transient, say THAT once,
1156
+ // in words. The `no-quorum` line above fires on every pass and cannot tell the two apart.
1157
+ this.reportRepairDeadlock({
1158
+ blockId, claims, silentCount: silent.length, cohortPeers: nonSelfCount, answered, required, capacity
1159
+ });
1160
+ // The claims themselves must not drive restoration but their existence is
1161
+ // evidence the caller needs: an answer served below the highest claim cannot be
1162
+ // confirmed current (see ClusterLatestQuery.uncorroboratedRev).
1163
+ // NOTE: ONE claim is enough to raise that doubt, and the claims reaching this branch
1164
+ // are unverified assertions a certified claim converges above instead of declining
1165
+ // (the only certified shape that lands here is the equivocation decline). So a single
1166
+ // lying cohort peer can deny unpinned reads of a block by claiming a revision nobody
1167
+ // else holds an availability lever it did not have while uncorroborated claims were
1168
+ // discarded. Deliberate for now: the alternative is the silent stale serve this marker
1169
+ // exists to end, and the same liar can already force a silent-treated absence by
1170
+ // staying quiet. If the lever is ever exercised, gate the stamp on a certified claim
1171
+ // (the verification machinery now exists) rather than on the bare assertion — at the
1172
+ // cost of re-opening the stale-serve window for the proof-less honest majority.
1173
+ const uncorroboratedRev = claims.length > 0 ? Math.max(...claims.map(c => c.rev)) : undefined;
1174
+ return { local, silent, answered, ...(uncorroboratedRev !== undefined ? { uncorroboratedRev } : {}) };
1175
+ }
1176
+
1177
+ if (selected.certified) {
1178
+ // Which rule won matters when reading a repair log: a certified selection may rest on a
1179
+ // SINGLE claimant whose corroboration is the cohort's signature set, not other voters.
1180
+ this.log('cluster-fetch:certified-selected', {
1181
+ blockId, rev: selected.rev, claimants: selected.supporters.length
1182
+ });
1183
+ }
1184
+
1185
+ // Best-effort: penalize peers whose claim contradicts a CORROBORATED selection a different
1186
+ // action at the very same revision. A higher rev may be honest leadership and a lower rev is
1187
+ // just lag; neither is penalized, nor is anything contradicting a certified-only selection
1188
+ // (an unanchored proof must not be able to convict the honest cohort). Never let this throw.
1189
+ this.penalizeContradictingRevClaims(claims, selected, blockId);
1190
+
1191
+ return { corroborated: { actionId: selected.actionId, rev: selected.rev }, local, silent, answered };
1192
+ }
1193
+
1194
+ /**
1195
+ * Say ONCE per block, in words, when a corroboration decline is provably PERMANENT rather than a
1196
+ * transient shortage of answers. There are exactly TWO permanent shapes, and they send the operator
1197
+ * to different places, so each gets its own `reason` and its own wording:
1198
+ *
1199
+ * - `cohort-too-small` this node's cohort has fewer peers than the quorum would demand even if
1200
+ * every one of them answered and agreed. The remedy is machines or an honest declared size.
1201
+ * - `sole-holder` — the cohort is big enough, but exactly ONE of its peers holds the block at all
1202
+ * and every other peer answered that it holds nothing. The remedy is another cohort peer
1203
+ * holding the block; machines and configuration are both irrelevant. Note the scope: this node's
1204
+ * own copy is excluded from the claim set, so a reader that holds the block itself still sees
1205
+ * `sole-holder` the message says "cohort peer", never "machine in the deployment".
1206
+ *
1207
+ * **What makes `cohort-too-small` provable.** Not "this pass fell short" — a pass falls short
1208
+ * whenever some peer simply does not hold the block *yet*. The decisive question is whether the
1209
+ * cohort could supply the quorum AT ALL: ask what would be required if every cohort peer answered
1210
+ * and agreed the best case any later pass can reach without new machines and compare it to how
1211
+ * many peers the cohort has. Short of that best case the shortfall is not the machine count, and
1212
+ * saying PERMANENT would send the operator to change a number that was never the problem. Twelve
1213
+ * days of log archaeology went into re-deriving the real condition from a thousand identical
1214
+ * `cluster-fetch:no-quorum` lines; the node knows it at the moment of each decline.
1215
+ *
1216
+ * **What makes `sole-holder` provable.** Note it is only reachable for a lone UNCERTIFIED holder:
1217
+ * a lone holder whose cohort commit proof verified is selected by the certified path and converges
1218
+ * before any decline — so the wording's "a lone holder cannot second itself" stays accurate for
1219
+ * every claim that gets here. "That peer will hold it later" is an assumption, and for a
1220
+ * peer that ANSWERED "I hold nothing" it is false: the only two mechanisms that would turn a
1221
+ * non-holder into a holder — `queryClusterForLatest` (read-repair) and `createReconcileBlock`
1222
+ * (reconcile) consume this very decision, so they decline for exactly the same reason on that
1223
+ * peer. Every peer answered, one holds the block, the rest hold nothing, and no later pass changes
1224
+ * any of that. What DOES change it is a new copy: a commit that writes the block again pushes it to
1225
+ * the current cohort. (Sibling work `replicate-owned-blocks-when-the-cohort-grows` makes that
1226
+ * automatic; until it lands the operator has to cause the write.)
1227
+ *
1228
+ * **What is deliberately NOT reported.** A cohort that answers unanimously "I hold nothing" — an
1229
+ * agreed absence is an answer, not a failed repair. A pass with any silent peer: silence cannot
1230
+ * change the arithmetic (`cohortPeers` counts silent peers too), but it does mean this node saw less
1231
+ * than the whole picture, and the next clean pass says the same thing at no cost. Note there is
1232
+ * deliberately NO "the claims disagreed" exemption for `cohort-too-small`: a cohort too small to
1233
+ * reach quorum stays too small whether its peers agree or not, so disagreement would suppress a line
1234
+ * that is still true. Two or more disagreeing holders DO suppress `sole-holder`, because that is a
1235
+ * cohort with two copies whose peers have not settled yet a later pass can settle it.
1236
+ *
1237
+ * **Never a lever.** This only classifies and logs; it never relaxes a floor. Which is also why the
1238
+ * `cohort-too-small` message names *two* readings of the same numbers — a deployment that genuinely
1239
+ * runs this few machines, or a cohort view shrunk below the real deployment by a partition or by an
1240
+ * attacker with routing influence. `corroboratorCapacity` keeps the shrunken view out of the relaxed
1241
+ * branch, but this node cannot tell the two apart from the inside, and an operator sent to fix the
1242
+ * wrong one is the failure this line exists to end.
1243
+ *
1244
+ * NOTE: the reader is still told only "this may be stale" `BlockPossiblyStaleError` implies a
1245
+ * retry might help, which is wrong advice for a block whose repair is deadlocked as configured.
1246
+ * Carrying this condition into the error needs a new field on `GetBlockResult` plus a change to
1247
+ * that error's documented contract; deliberately out of scope here (see the ticket
1248
+ * `repair-deadlock-is-never-named`, *Not this ticket*).
1249
+ */
1250
+ private reportRepairDeadlock(pass: {
1251
+ blockId: BlockId;
1252
+ claims: RevClaim[];
1253
+ silentCount: number;
1254
+ /** Cohort peers besides this node, from the cohort view — whether they answered or not. */
1255
+ cohortPeers: number;
1256
+ answered: number;
1257
+ /** The quorum THIS pass demanded, computed from the peers that actually claimed. */
1258
+ required: number;
1259
+ /** `corroboratorCapacity` for this pass a function of the view and the resolved size, not of who answered. */
1260
+ capacity: number;
1261
+ }): void {
1262
+ const { blockId, claims, silentCount, cohortPeers, answered, required, capacity } = pass;
1263
+ // An incomplete picture proves nothing about the deployment; the next clean pass says it.
1264
+ if (silentCount > 0) return;
1265
+ // Nobody claimed anything: the cohort agrees the block is absent, which is an answer, not a
1266
+ // deadlock.
1267
+ if (claims.length === 0) return;
1268
+ // The decisive test for the first shape. `requiredEvenIfAllAnswered` is the quorum this cohort
1269
+ // would face with every one of its peers answering and agreeing — the best case reachable
1270
+ // without adding machines. A cohort that can meet it is not too small.
1271
+ const requiredEvenIfAllAnswered = quorumSize(cohortPeers, this.simpleMajorityThreshold, capacity);
1272
+ const cohortTooSmall = cohortPeers < requiredEvenIfAllAnswered;
1273
+ // The second shape: exactly one cohort peer holds the block AT ALL, and — since a claim is one
1274
+ // peer's latest, so a single claim is a single distinct (rev, actionId) group with a single
1275
+ // supporter — every other cohort peer answered that it holds nothing. `answered === cohortPeers`
1276
+ // is already implied by the silence guard above; it is stated because the two counts arrive as
1277
+ // independent parameters and "everybody answered" is half of what makes this provable.
1278
+ //
1279
+ // NOTE: there is a narrow window where `sole-holder` is true of the instant but not of the
1280
+ // deployment — a commit that has landed on one cohort member and has not yet been pushed to the
1281
+ // rest presents exactly this shape. Calling it PERMANENT is defensible even there (repair
1282
+ // genuinely cannot converge until the push lands, and the once-per-episode flag clears the
1283
+ // moment the block converges, so the line does not repeat), and widening the window is what the
1284
+ // push path's own threat model decides see
1285
+ // `tickets/blocked/repair-floor-defends-a-door-the-push-path-leaves-open`. If commit-to-push
1286
+ // latency ever grows enough that operators see `sole-holder` on blocks that heal moments later,
1287
+ // gate the line on the block having been quiet for longer than that latency rather than
1288
+ // softening the wording.
1289
+ const soleHolder = claims.length === 1 && answered === cohortPeers;
1290
+ if (!cohortTooSmall && !soleHolder) return;
1291
+
1292
+ // Both shapes can hold at once (an undeclared two-machine deployment whose single peer holds the
1293
+ // block is both). `cohort-too-small` is reported in preference because its remedy is the one
1294
+ // that actually works there: declaring the real size makes the floor reachable, after which the
1295
+ // lone peer's claim IS adopted — so calling it a sole-holder problem would send the operator
1296
+ // looking for a copy they do not need.
1297
+ const reason: DeadlockReason = cohortTooSmall ? 'cohort-too-small' : 'sole-holder';
1298
+ const state = this.unsettledAheadClaims.get(blockId);
1299
+ const alreadySaid = state?.deadlocksReported ?? [];
1300
+ // Suppressed per REASON, not once outright: an episode that starts as `cohort-too-small` and
1301
+ // becomes `sole-holder` — the operator added the machines that reason asked for, and the block
1302
+ // is still stuck has a second thing to say, and a silent log there is the failure this line
1303
+ // exists to end. Neither reason repeats within an episode.
1304
+ if (alreadySaid.includes(reason)) return;
1305
+
1306
+ this.log('cluster-fetch:repair-deadlock', {
1307
+ blockId,
1308
+ reason,
1309
+ cohortPeers,
1310
+ answered,
1311
+ claimants: claims.length,
1312
+ required,
1313
+ requiredEvenIfAllAnswered,
1314
+ repairCorroborationClusterSize: this.repairCorroborationClusterSize,
1315
+ message: cohortTooSmall
1316
+ ? cohortTooSmallMessage(cohortPeers, claims.length, requiredEvenIfAllAnswered, this.repairCorroborationClusterSize)
1317
+ : soleHolderMessage(cohortPeers)
1318
+ });
1319
+ // Hung off the existing per-block freshness entry rather than a fourth per-block map. The entry
1320
+ // survives `recordAheadClaim` clearing its `rev`, and is dropped wholesale once the block
1321
+ // converges so each reason is said once per non-convergence episode, not once per pass.
1322
+ // NOTE: per BLOCK, though the condition is a property of the cohort, not of any block — so a node
1323
+ // in this state that reads N distinct blocks emits N lines. Deliberate: the operator wants to
1324
+ // know which blocks are stuck, and N is bounded by blocks actually read (1821 lines for a single
1325
+ // block was the defect). If a deployment in this state ever makes this the noisy line again, add
1326
+ // a node-level once-flag keyed on (cohortPeers, requiredEvenIfAllAnswered) and let the per-block
1327
+ // entry only suppress repeats.
1328
+ this.unsettledAheadClaims.set(blockId, { ...(state ?? {}), deadlocksReported: [...alreadySaid, reason] });
1329
+ }
1330
+
1331
+ /**
1332
+ * Report peers whose reported latest PROVABLY contradicts a CORROBORATED selection: the same
1333
+ * revision under a different actionId. Two actions cannot both be the commit at one revision,
1334
+ * and the pair a quorum of distinct peers agreed on is the one this node can stand behind, so
1335
+ * the disagreeing claimant is wrong. Best-effort.
1336
+ *
1337
+ * A CERTIFIED selection is deliberately excluded — no claim is penalized against it. A passing
1338
+ * proof shows the cohort it names signed the commit, never that those signers are the block's
1339
+ * responsible cohort: anyone holding N keys can mint a proof that verifies (see caller
1340
+ * obligation #1 in `cluster/commit-proof.ts`, and the unwired {@link ProofAnchoring} layer).
1341
+ * Penalizing here would therefore hand one forged proof a lever it must not have — every honest
1342
+ * peer holding the real action at that revision reported for InvalidRestoration (weight 30,
1343
+ * above the deprioritize threshold of 20), on every consult. Losing the selection to the proof
1344
+ * is already the accepted cost of the certified path; deprioritizing the honest cohort on top of
1345
+ * it is not. Revisit when certification is anchored to the block's derived cohort
1346
+ * (`feat-cluster-membership-threshold-cert-anchoring`): a gated proof makes the contradiction
1347
+ * provable again.
1348
+ *
1349
+ * A claim at a HIGHER rev than the selection is deliberately NOT penalized: a peer can honestly
1350
+ * be ahead of the sampled quorum — an in-flight commit it durably stored before the rest of the
1351
+ * cohort, or other honest holders dropped from the sample by the 1s per-peer consult deadline —
1352
+ * and the InvalidRestoration weight (30) sits above the deprioritize threshold (20), so a single
1353
+ * false hit used to deprioritize an honest, up-to-date peer. Declining to RESTORE from the
1354
+ * uncorroborated higher claim already happens in selection; the affirmative penalty on that
1355
+ * ambiguous evidence is what this method no longer applies. Provably-bad proof SERVICE is
1356
+ * penalized at verification time instead (the certifyClaim pass in
1357
+ * {@link queryClusterForLatest}).
1358
+ */
1359
+ private penalizeContradictingRevClaims(claims: RevClaim[], selected: QuorumRev, blockId: BlockId): void {
1360
+ if (!this.reputation || selected.certified) return;
1361
+ try {
1362
+ for (const c of claims) {
1363
+ // A CERTIFIED disagreeing claim is exempt: its proof verified, so the peer honestly
1364
+ // served a commit that really happened — when multi-peer corroboration outweighs its
1365
+ // single-signer proof at the same rev (quorum-restore.ts), that peer is a partition
1366
+ // casualty on the losing side of a fork, not a liar. Provably-bad proof SERVICE was
1367
+ // already penalized at verification time above.
1368
+ if (c.certified === true) continue;
1369
+ if (c.rev === selected.rev && c.actionId !== selected.actionId) {
1370
+ this.reputation.reportPeer(c.peerId, PenaltyReason.InvalidRestoration, `read-repair:${blockId}`);
1371
+ }
1372
+ }
1373
+ } catch (err) {
1374
+ this.log('cluster-fetch:penalize-error', { blockId, error: (err as Error).message });
1375
+ }
1376
+ }
1377
+
1378
+ /**
1379
+ * Best-effort penalty for a peer whose SERVED PROOF provably lies or provably does not cover the
1380
+ * claim it was attached to (see the attributability classification in
1381
+ * `cluster/certified-claims.ts`). Never throws — mirrors
1382
+ * {@link penalizeContradictingRevClaims}.
1383
+ */
1384
+ private penalizeProofService(peerId: string, blockId: BlockId): void {
1385
+ if (!this.reputation) return;
1386
+ try {
1387
+ this.reputation.reportPeer(peerId, PenaltyReason.InvalidRestoration, `read-repair:${blockId}`);
1388
+ } catch (err) {
1389
+ this.log('cluster-fetch:penalize-error', { blockId, error: (err as Error).message });
1390
+ }
1391
+ }
1392
+
1393
+ async pend(request: PendRequest, options?: MessageOptions): Promise<PendResult> {
1394
+ const allBlockIds = blockIdsForTransforms(request.transforms);
1395
+ await this.verifyResponsibility(allBlockIds);
1396
+ const coordinatingBlockIds = options?.coordinatingBlockIds ?? allBlockIds;
1397
+
1398
+ const peerCount = await this.coordinator.getClusterSize(coordinatingBlockIds[0]!);
1399
+ if (peerCount <= 1) {
1400
+ return await this.storageRepo.pend(request, options);
1401
+ }
1402
+
1403
+ const message: RepoMessage = {
1404
+ operations: [{ pend: request }],
1405
+ expiration: options?.expiration ?? Date.now() + this.DEFAULT_TIMEOUT,
1406
+ coordinatingBlockIds
1407
+ };
1408
+
1409
+ try {
1410
+ const { localExecuted, localPendResult } = await this.coordinator.executeClusterTransaction(coordinatingBlockIds[0]!, message, options);
1411
+ this.log('coordinator-repo:pend-cluster-complete', {
1412
+ actionId: request.actionId,
1413
+ localExecuted,
1414
+ localVerdict: localPendResult === undefined ? 'none'
1415
+ : localPendResult.success ? 'success'
1416
+ : isConflictFailure(localPendResult) ? 'conflict' : 'fault'
1417
+ });
1418
+ // Only call storageRepo if local cluster didn't already execute during consensus
1419
+ if (!localExecuted) {
1420
+ const result = await this.storageRepo.pend(request, options);
1421
+ this.log('coordinator-repo:pend-fallback-result', {
1422
+ actionId: request.actionId,
1423
+ success: result.success,
1424
+ hasMissing: !!(result as any).missing?.length,
1425
+ hasPending: !!(result as any).pending?.length
1426
+ });
1427
+ return result;
1428
+ }
1429
+ // Local cluster already executed during consensus return storage's own verdict rather
1430
+ // than fabricating a success (the peerCount <= 1 path above returns storage's real result
1431
+ // verbatim; the cluster path must never answer differently). Pend-consensus confers no
1432
+ // durability: a refusal carrying `pending` (a rival's unresolved action holds the blocks)
1433
+ // or `missing` (the requested revision is already committed) is the optimistic-concurrency
1434
+ // verdict the same scan every member runs, not a local fault and must reach the writer
1435
+ // as a retryable conflict so NetworkTransactor.pendPhase cancels the partial pend and the
1436
+ // writer rebases. This deliberately differs from `commit`'s divergence split below: a
1437
+ // commit that reached commit-consensus IS the authoritative commit (Theorem 9), whereas a
1438
+ // pend that reached pend-consensus may still have been stored by nobody.
1439
+ if (localPendResult !== undefined) {
1440
+ if (localPendResult.success || isConflictFailure(localPendResult)) {
1441
+ return localPendResult;
1442
+ }
1443
+ // A bare-reason refusal (no pending/missing — e.g. a local validation-hook fault)
1444
+ // stays tolerated local divergence: consensus is authoritative and the pend may well
1445
+ // have landed on the rest of the cohort.
1446
+ this.log('coordinator-repo:pend-local-fault-tolerated', {
1447
+ actionId: request.actionId,
1448
+ reason: localPendResult.reason
1449
+ });
1450
+ }
1451
+ // No verdict retained (member predates retention, restart, or TTL): the prior shape.
1452
+ return {
1453
+ success: true,
1454
+ pending: [],
1455
+ blockIds: allBlockIds
1456
+ };
1457
+ } catch (error) {
1458
+ this.log('coordinator-repo:pend-error', { actionId: request.actionId, error: (error as Error).message });
1459
+ // A lost conflict race is an optimistic-concurrency loss, not a fault: surface it as the
1460
+ // StaleFailure shape the retry machinery already understands (`Collection.sync` and the
1461
+ // multi-collection pendPhase retry it via `isConflictFailure`), exactly as a confirmed
1462
+ // stale revision is. `staleAt` stays absent deliberately it is confirmed-only, and a
1463
+ // lost race is a rival *pend* holding the blocks, not a revision claim.
1464
+ //
1465
+ // NOTE: `error.conflicts` (peerId winning messageHash) is dropped here `StaleFailure`
1466
+ // has no field for it and the retry loop only needs "retryable". If a caller ever needs to
1467
+ // know WHICH transaction won (e.g. to wait on it rather than re-race it), add a typed field
1468
+ // for it; never recover it by parsing `reason`.
1469
+ //
1470
+ // NOTE: with three or more contenders the members can split so that EVERY contender is
1471
+ // told it lost the race an all-lose round where nobody wins and each writer retries.
1472
+ // The cause is `ClusterMember.resolveRace`'s approvals-first rule, not its tie-break:
1473
+ // each member compares the rivals as IT holds them, so a member that already approved X
1474
+ // keeps X while a member that approved Y first keeps Y, and no rival reaches a promise
1475
+ // supermajority. (The hash tie-break is already symmetric — it cannot be the fix.)
1476
+ // Fine as it stands: since the torn-action fixes landed, an all-lose round costs one
1477
+ // retry cycle rather than wedging, and the contenders are separated next round by the
1478
+ // jittered backoff plus the aged retry priority carried on the re-pend
1479
+ // (`clampPriority(consecutiveFailures)` in `Collection.syncInternal`), which out-ranks
1480
+ // fresh priority-0 rivals at EQUAL approval counts — priority sits below the approval
1481
+ // count in `resolveRace`, so it does not displace a more-progressed rival. If a
1482
+ // high-contention workload ever shows syncs exhausting `maxAttempts` on repeated
1483
+ // all-lose rounds, the fix is reserve/defer at pend time (backlog
1484
+ // `feat-occ-priority-reservation`, which `resolveRace`'s own residual-fairness NOTE
1485
+ // already points at) rather than raising maxAttempts.
1486
+ if (error instanceof ConflictRaceLostError) {
1487
+ return { success: false, conflict: true, reason: error.message };
1488
+ }
1489
+ const stale = await this.classifyStaleRejection(error, request, allBlockIds)
1490
+ ?? await this.classifyPendingConflictRejection(error, request, allBlockIds);
1491
+ if (stale) return stale;
1492
+ throw error;
1493
+ }
1494
+ }
1495
+
1496
+ /**
1497
+ * Decide whether a cluster validator rejection was an optimistic-concurrency loss the block
1498
+ * already advanced past the requested revision rather than a genuine validation fault.
1499
+ * A confirmed loss returns a {@link StaleFailure} carrying `conflict: true` so the caller
1500
+ * receives a non-success *response* that says plainly it is a lost race: network-transactor's
1501
+ * pend then takes its stale branch and both writers (`Collection.sync`, and the coordinator's
1502
+ * multi-collection pendPhase via `isConflictFailure`) retry, instead of a thrown error escaping
1503
+ * mid-batch (which splits multi-tree commits — see PartialCommitError).
1504
+ *
1505
+ * The failure carries no `missing` list: confirmation is a local re-read that reveals the
1506
+ * revision is taken but not which actions took it, and no consumer rebases from `missing`
1507
+ * anyway (it is only counted or logged). `conflict` conveys retryability directly instead.
1508
+ *
1509
+ * Confirmation is purely local: re-read the affected blocks from our own storage and require
1510
+ * `latest.rev >= request.rev`. The signed reject-reason text is never consulted — it is
1511
+ * free-form wire-visible prose and must not become control flow. Anything unconfirmed
1512
+ * (including read errors during confirmation) stays a throw, preserving fail-fast for
1513
+ * genuine validation faults.
1514
+ */
1515
+ private async classifyStaleRejection(error: unknown, request: PendRequest, blockIds: BlockId[]): Promise<StaleFailure | undefined> {
1516
+ const requestedRev = request.rev;
1517
+ if (!(error instanceof ValidatorRejectionError) || requestedRev === undefined) return undefined;
1518
+ let results: GetBlockResults;
1519
+ try {
1520
+ results = await this.storageRepo.get({ blockIds });
1521
+ } catch (readError) {
1522
+ this.log('coordinator-repo:pend-stale-classify-read-error', {
1523
+ actionId: request.actionId,
1524
+ error: (readError as Error).message
1525
+ });
1526
+ return undefined;
1527
+ }
1528
+ // Scan EVERY block rather than stopping at the first confirmation: several of the request's
1529
+ // blocks can be past the requested revision at different revisions, and it is the highest
1530
+ // that the loser's next request has to clear (see `highestStaleAt`). Both the reported
1531
+ // number and the reason prose name that block, so they never disagree.
1532
+ const staleAt = highestStaleAt(blockIds.map(blockId => {
1533
+ const latest = results[blockId]?.state.latest;
1534
+ if (!latest || latest.rev < requestedRev) return undefined;
1535
+ // Per-block self-exclusion (see {@link isOwnRevision}): our own durable half of a torn
1536
+ // action is not a confirmed loss. Deliberately per-block, NOT the bail-entirely
1537
+ // 'own-durable' shape of confirmCommitRivalAgainstLocal a confirmed rival on ANOTHER
1538
+ // block still confirms, and when none is confirmed anywhere the rejection stays a throw
1539
+ // exactly as before. With the two pend-tier sites upstream fixed (StorageRepo.pend and
1540
+ // ClusterMember.validatePendOperations) this shape should not reach here; mirrored so
1541
+ // all three pend-tier checks agree.
1542
+ if (isOwnRevision(latest, requestedRev, request.actionId)) return undefined;
1543
+ return { blockId, rev: latest.rev };
1544
+ }));
1545
+ if (staleAt) {
1546
+ this.log('coordinator-repo:pend-stale-classified', {
1547
+ actionId: request.actionId,
1548
+ blockId: staleAt.blockId,
1549
+ latestRev: staleAt.rev,
1550
+ requestedRev
1551
+ });
1552
+ return {
1553
+ success: false,
1554
+ conflict: true,
1555
+ reason: `stale revision: block ${staleAt.blockId} at rev ${staleAt.rev}, requested rev ${requestedRev}`,
1556
+ // The same fact as the reason prose, but as data. This is the ONLY place a losing
1557
+ // writer can learn the revision it lost to, since this failure deliberately carries
1558
+ // no `missing`. Confirmed-local: read out of our own storage just above.
1559
+ staleAt
1560
+ };
1561
+ }
1562
+ // NOTE: conservative — when only remote members saw the newer revision (local storage still
1563
+ // behind), staleness can't be confirmed locally and the rejection stays a throw. If that
1564
+ // shows up in practice, extend confirmation with a quorum read; never trust the reject text.
1565
+ // `staleAt` is absent on this path for the same reason, and deliberately so — there is no
1566
+ // confirmed number to report, and the field's contract forbids inferring one from that text.
1567
+ return undefined;
1568
+ }
1569
+
1570
+ /**
1571
+ * Sibling of {@link classifyStaleRejection} for the OTHER optimistic-concurrency refusal shape:
1572
+ * the promise-phase pending-conflict vote (`validatePendOperations` rejecting a pend whose
1573
+ * blocks are held by a different unresolved pending action). That vote surfaces here as a
1574
+ * {@link ValidatorRejectionError}, and without classification it would escape as a throw —
1575
+ * splitting multi-tree pends mid-batch instead of taking the retry path a lost race deserves.
1576
+ *
1577
+ * Same confirmation discipline as the stale classifier: purely local. Re-read the affected
1578
+ * blocks from our own storage and require some block's `state.pendings` to carry a rival
1579
+ * actionId; the signed reject text is never consulted. A confirmed rival returns a
1580
+ * {@link StaleFailure} with `conflict: true` and the rivals as `pending` (`ActionPending`
1581
+ * without `transform` the type allows it, and no consumer rebases from it). Unconfirmed —
1582
+ * including read errors during confirmation — stays a throw, preserving fail-fast for genuine
1583
+ * validation faults. Checked after `classifyStaleRejection` so a confirmed committed loss
1584
+ * (which carries the sharper `staleAt`) wins when both hold.
1585
+ */
1586
+ private async classifyPendingConflictRejection(error: unknown, request: PendRequest, blockIds: BlockId[]): Promise<StaleFailure | undefined> {
1587
+ if (!(error instanceof ValidatorRejectionError)) return undefined;
1588
+ let results: GetBlockResults;
1589
+ try {
1590
+ results = await this.storageRepo.get({ blockIds });
1591
+ } catch (readError) {
1592
+ this.log('coordinator-repo:pend-conflict-classify-read-error', {
1593
+ actionId: request.actionId,
1594
+ error: (readError as Error).message
1595
+ });
1596
+ return undefined;
1597
+ }
1598
+ const pending: ActionPending[] = [];
1599
+ for (const blockId of blockIds) {
1600
+ for (const actionId of results[blockId]?.state?.pendings ?? []) {
1601
+ if (actionId !== request.actionId) pending.push({ blockId, actionId });
1602
+ }
1603
+ }
1604
+ if (pending.length === 0) return undefined;
1605
+ this.log('coordinator-repo:pend-conflict-classified', {
1606
+ actionId: request.actionId,
1607
+ rivals: pending.map(p => `${p.blockId}:${p.actionId}`)
1608
+ });
1609
+ return {
1610
+ success: false,
1611
+ conflict: true,
1612
+ pending,
1613
+ reason: `pending conflict: block(s) held by unresolved rival action(s) ${[...new Set(pending.map(p => p.actionId))].join(', ')}`
1614
+ };
1615
+ }
1616
+
1617
+ async cancel(actionRef: ActionBlocks, options?: MessageOptions): Promise<void> {
1618
+ const blockIds = actionRef.blockIds;
1619
+ await this.verifyResponsibility(blockIds);
1620
+
1621
+ // Create a message for this cancel operation with timeout
1622
+ const message: RepoMessage = {
1623
+ operations: [{ cancel: { actionRef } }],
1624
+ expiration: options?.expiration ?? Date.now() + this.DEFAULT_TIMEOUT
1625
+ };
1626
+
1627
+ try {
1628
+ // One cluster transaction per block ID — but a block whose cohort is just this node
1629
+ // short-circuits to local storage, exactly as `pend` and `commit` do above. Without the
1630
+ // short-circuit a solo cohort enters `executeTransaction`, fails `minAbsoluteClusterSize`
1631
+ // (2), and throws `Cluster size 1 below minimum 2 and not validated` — so a single-peer
1632
+ // deployment could pend and commit but never cancel, unless the operator had opened the
1633
+ // `allowUnvalidatedSmallCluster` hatch. Decided per block rather than once for
1634
+ // `blockIds[0]`, because a multi-block cancel can span cohorts of different sizes.
1635
+ //
1636
+ // NOTE: `getClusterSize` is a second `findCluster` for the same key that
1637
+ // `executeClusterTransaction` is about to look up again, so a cancel over N blocks now
1638
+ // costs 2N cohort lookups instead of N. Same shape `pend` and `commit` already pay, but
1639
+ // they pay it once (they only ever consult `blockIds[0]`) where this scales with N. Fine
1640
+ // while cancels span a handful of blocks; if wide multi-block cancels ever show up hot,
1641
+ // have `executeClusterTransaction` return the cohort it already fetched (or own the
1642
+ // short-circuit itself) rather than adding a cache here.
1643
+ const results = await Promise.all(blockIds.map(async blockId => {
1644
+ const peerCount = await this.coordinator.getClusterSize(blockId);
1645
+ if (peerCount <= 1) return false;
1646
+ const { localExecuted } = await this.coordinator.executeClusterTransaction(blockId, message, options);
1647
+ return localExecuted;
1648
+ }));
1649
+
1650
+ // Only call storageRepo if local cluster didn't already execute during consensus
1651
+ const anyLocalExecuted = results.some(Boolean);
1652
+ if (!anyLocalExecuted) {
1653
+ await this.storageRepo.cancel(actionRef, options);
1654
+ }
1655
+ } catch (error) {
1656
+ this.log('coordinator-repo:cancel-error', { actionId: actionRef.actionId, error: (error as Error).message });
1657
+ throw error;
1658
+ }
1659
+ }
1660
+
1661
+ async commit(request: CommitRequest, options?: MessageOptions): Promise<CommitResult> {
1662
+ const blockIds = request.blockIds;
1663
+ await this.verifyResponsibility(blockIds);
1664
+
1665
+ const cohortPeerIds = await this.coordinator.getClusterPeerIds(blockIds[0]!);
1666
+ const peerCount = cohortPeerIds.length;
1667
+ if (peerCount <= 1) {
1668
+ // Solo cohort: consensus never runs, so no ClusterRecord exists to project a proof from —
1669
+ // the lone member self-signs a one-peer proof instead (mintSoloCommitProof), which is what
1670
+ // lets a block born on a cohort of one ever gain a second holder under the certified-push
1671
+ // default (handlePush refuses a proof-less block). Minted even when peerCount is 0 or the
1672
+ // sole peer is not self — findCluster failing (getClusterPeerIds returns []) puts the
1673
+ // DEGRADED-ROUTING case in this same branch, and self genuinely committed these bytes
1674
+ // either way; a proof's peer list is already not evidence of cohort membership by design
1675
+ // (caller obligation #1 on verifyBlockCommitProofClaim), so gating the mint on cohort
1676
+ // composition would buy no safety while opening a silent no-proof hole exactly when
1677
+ // routing is degraded. The log line is how an operator tells a real cohort of one
1678
+ // (cohortSize 1, soleIsSelf true) from a routing failure (cohortSize 0, or a sole peer
1679
+ // that is not this node).
1680
+ this.log('commit:solo-cohort', {
1681
+ blockId: blockIds[0],
1682
+ cohortSize: peerCount,
1683
+ soleIsSelf: peerCount === 1 && this.localPeerId !== undefined
1684
+ && cohortPeerIds[0] === this.localPeerId.toString()
1685
+ });
1686
+ // Same message shape the multi-peer path produces executeClusterTransaction stamps
1687
+ // coordinatingBlockIds at its choke point, so the solo artifact must carry it too or a
1688
+ // solo proof's message is distinguishable from every other proof's.
1689
+ const message: RepoMessage = {
1690
+ operations: [{ commit: request }],
1691
+ coordinatingBlockIds: [blockIds[0]!],
1692
+ expiration: options?.expiration ?? Date.now() + this.DEFAULT_TIMEOUT
1693
+ };
1694
+ // `undefined` when no local cluster is wired (direct constructors, unit-test doubles)
1695
+ // then the commit lands proof-less, exactly the pre-mint behavior. The cast is the named
1696
+ // ICommitProofPersister contract; a plain IRepo double ignores the extra argument.
1697
+ const proof = await this.localCluster?.mintSoloCommitProof?.(message);
1698
+ const result = await (this.storageRepo as IRepo & ICommitProofPersister).commit(request, options, proof);
1699
+ if (result.success) this.markBlocksSeen(blockIds);
1700
+ return result;
1701
+ }
1702
+
1703
+ const message: RepoMessage = {
1704
+ operations: [{ commit: request }],
1705
+ expiration: options?.expiration ?? Date.now() + this.DEFAULT_TIMEOUT
1706
+ };
1707
+
1708
+ try {
1709
+ const { record, localExecuted, localCommitResult } = await this.coordinator.executeClusterTransaction(blockIds[0]!, message, options);
1710
+ if (localExecuted) {
1711
+ // Our own member applied this commit during consensus. Its retained storage verdict is
1712
+ // the one honest signal we have about durability: the member-side apply tolerates an
1713
+ // "ahead" refusal as divergence (see the NOTE in ClusterMember.applyConsensusOperation),
1714
+ // which is correct for a redelivered or lagging commit but when the refusal's real
1715
+ // cause is a RIVAL action holding the requested revision, that tolerance turns a commit
1716
+ // no member durably stored into a fabricated success. This is the
1717
+ // signed-but-not-yet-applied window: two commits for one revision can BOTH assemble
1718
+ // consensus when every member signs the second after signing (but before applying) the
1719
+ // first, because signing drops the member's reservation. Confirm the rival against local
1720
+ // storage (never the verdict's prose) and answer the writer with a retryable conflict so
1721
+ // it re-drives at a fresh revision. Own-action or unconfirmed refusals keep the
1722
+ // prior fabricated-success shape: consensus is authoritative and this member converges
1723
+ // via replication.
1724
+ //
1725
+ // NOTE: a CONFIRMED rival is trusted over the consensus outcome here. That is right in
1726
+ // the window this closes (the cohort refused the loser too), but it inverts if the two
1727
+ // ever disagree — a local rival at the requested revision while a super-majority
1728
+ // approved OUR commit means this node is on a forked lineage, and refusing then tells a
1729
+ // writer whose write did land to re-drive it (a duplicate entry). Members holding the
1730
+ // rival reject at the promise round, so consensus and a local rival can only disagree
1731
+ // after a fork; that is partition-healing scope (docs/partition-healing.md). If forks
1732
+ // are ever observed here, weigh the retained verdict against the cohort's votes instead
1733
+ // of trusting the local re-read alone.
1734
+ if (localCommitResult !== undefined && !localCommitResult.success) {
1735
+ const rival = await this.confirmCommitRivalAgainstLocal(request);
1736
+ if (typeof rival === 'object') return rival;
1737
+ this.log('coordinator-repo:commit-local-refusal-tolerated', {
1738
+ actionId: request.actionId,
1739
+ confirmation: rival ?? 'unconfirmed',
1740
+ reason: localCommitResult.reason
1741
+ });
1742
+ }
1743
+ this.markBlocksSeen(blockIds);
1744
+ return { success: true };
1745
+ }
1746
+ // Local cluster didn't execute during consensus. Attempt a local commit, but tolerate
1747
+ // local divergence when the cluster already reached consensus this coordinator was
1748
+ // likely picked for commit after missing the pend phase (unreachable during pend, fresh
1749
+ // join, etc.). The cluster's majority is authoritative; this peer catches up via sync.
1750
+ //
1751
+ // Divergence reaches us in BOTH shapes and both must be tolerated identically:
1752
+ // - a THROW ("Pending action … not found"), when we never saw the pend;
1753
+ // - a RETURNED `success:false` carrying `missing-base-revision`, when we saw the pend
1754
+ // but not the revision that created the block (see StorageRepo.internalCommit).
1755
+ // Only the throw was tolerated before the refusal existed. Reporting the refusal to the
1756
+ // caller instead would surface a committed transaction as a stale loss: db-core's
1757
+ // commitPhase treats any returned `success:false` as a permanent stale failure, so the
1758
+ // client would retry an action the cluster already landed until it exhausted its budget.
1759
+ //
1760
+ // Deliberately NOT self-signed here (unlike the solo short-circuit above): consensus for
1761
+ // this commit ran on the cohort, so a one-peer minted proof would be a FALSE statement
1762
+ // about the committing cohort. Thread the record's REAL proof instead —
1763
+ // executeClusterTransaction resolves only after the record's promise/commit votes are
1764
+ // populated, so the projection is genuine. Passed unconditionally, threshold or not:
1765
+ // persistProofIfContentMatches retains it under the digest-match rule, and a record that
1766
+ // never reached threshold simply yields a proof no verifier accepts the same posture
1767
+ // ClusterMember.applyConsensusOperation takes with its own projection.
1768
+ const consensusProof = buildBlockCommitProof(record);
1769
+ try {
1770
+ const result = await (this.storageRepo as IRepo & ICommitProofPersister).commit(request, options, consensusProof);
1771
+ if (result.success) {
1772
+ this.markBlocksSeen(blockIds);
1773
+ return result;
1774
+ }
1775
+ if (isMissingBaseRevisionFailure(result) && clusterReachedCommitConsensus(record)) {
1776
+ return this.tolerateLocalCommitDivergence(request, blockIds, result.reason ?? MISSING_BASE_REVISION_REASON);
1777
+ }
1778
+ return result;
1779
+ } catch (err) {
1780
+ if (clusterReachedCommitConsensus(record)) {
1781
+ return this.tolerateLocalCommitDivergence(request, blockIds, (err as Error).message);
1782
+ }
1783
+ throw err;
1784
+ }
1785
+ } catch (error) {
1786
+ this.log('coordinator-repo:commit-error', { actionId: request.actionId, error: (error as Error).message });
1787
+ // A lost commit-consensus race is an optimistic-concurrency loss, not a fault — mirror
1788
+ // `pend`'s conversion above. At the moment this is thrown, zero members approved and the
1789
+ // members hold the winner: nothing of the loser landed, so a retryable-conflict answer is
1790
+ // truthful. Returning it (rather than rethrowing) matters more here than on the pend path:
1791
+ // db-core's `commitCollection` retries a THROWN commit error verbatim up to 3 times, and by
1792
+ // the retry the members have applied the winner and cleared its reservation — the re-driven
1793
+ // commit can then assemble a consensus no member will durably store (the writer's append
1794
+ // fulfills, the entry exists on no node). A RETURNED `success:false` is instead surfaced
1795
+ // immediately as a stale loss; the writer cancels the pend, re-reads, and re-drives the
1796
+ // whole pend+commit at a fresh revision. `staleAt` stays absent for the same reason as
1797
+ // pend's: it is confirmed-only, and a lost race is a rival commit racing the same revision,
1798
+ // not a locally-confirmed revision claim.
1799
+ if (error instanceof ConflictRaceLostError) {
1800
+ return { success: false, conflict: true, reason: error.message };
1801
+ }
1802
+ // A promise-phase stale-commit reject (`ClusterMember.validateCommitRevisions` — a member
1803
+ // holds the requested revision under a different action) surfaces here as a
1804
+ // ValidatorRejectionError; classify it against local storage the way `pend` does, so the
1805
+ // writer gets a clean retryable conflict instead of three verbatim re-drives and a hard
1806
+ // failure.
1807
+ const stale = await this.classifyCommitStaleRejection(error, request);
1808
+ if (stale) return stale;
1809
+ throw error;
1810
+ }
1811
+ }
1812
+
1813
+ /**
1814
+ * Commit-shaped sibling of {@link classifyStaleRejection}: decide whether a cluster validator
1815
+ * rejection of a COMMIT was an optimistic-concurrency loss — the requested revision is already
1816
+ * committed under a different action — rather than a genuine validation fault. A confirmed loss
1817
+ * returns a {@link StaleFailure} with `conflict: true` so db-core's `commitCollection` surfaces
1818
+ * it immediately as a stale loss (no verbatim retry) and the writer re-drives at a fresh
1819
+ * revision.
1820
+ *
1821
+ * Same confirmation discipline as the pend classifiers: purely local re-read; the signed reject
1822
+ * text is never consulted. One commit-specific delta — confirmation must EXCLUDE the
1823
+ * own-action-at-rev case: a block whose requested revision is held by THIS action is already
1824
+ * durable, and answering `conflict` for it would make the writer rebase and re-append an
1825
+ * already-committed action at a new revision — a duplicate entry. So:
1826
+ * - `latest.rev === request.rev` → compare `latest.actionId`: ours ⇒ bail (stays a throw),
1827
+ * a rival's ⇒ confirmed loss;
1828
+ * - `latest.rev > request.rev` → ask the {@link IRevisionActionReader} capability who holds
1829
+ * `request.rev`: ours ⇒ bail, a rival's ⇒ confirmed loss, unknown/absent/fault ⇒ unconfirmed;
1830
+ * - anything unconfirmed (including read errors) stays a throw — fail-fast for genuine faults.
1831
+ */
1832
+ private async classifyCommitStaleRejection(error: unknown, request: CommitRequest): Promise<StaleFailure | undefined> {
1833
+ if (!(error instanceof ValidatorRejectionError)) return undefined;
1834
+ const rival = await this.confirmCommitRivalAgainstLocal(request);
1835
+ // 'own-durable' and unconfirmed both stay a throw here: fail-fast for genuine faults, and a
1836
+ // commit already durable under this action must never be answered `conflict` (the writer
1837
+ // would rebase and re-append it — a duplicate entry).
1838
+ return typeof rival === 'object' ? rival : undefined;
1839
+ }
1840
+
1841
+ /**
1842
+ * Shared confirmation core for the two commit-tier conversion sites ({@link classifyCommitStaleRejection}
1843
+ * and the locally-executed refusal check in {@link commit}): decide, from LOCAL storage only, who
1844
+ * holds the requested revision.
1845
+ * - a confirmed RIVAL → the {@link StaleFailure} conflict answer (with `staleAt` = highest
1846
+ * confirmed holder);
1847
+ * - our OWN action durable at the requested revision → `'own-durable'` (callers must not answer
1848
+ * `conflict` — the writer would rebase an already-landed action into a duplicate entry);
1849
+ * - anything else (behind, truncated history, read faults, capability absent) `undefined`,
1850
+ * unconfirmed.
1851
+ * The signed reject text / retained verdict prose is never consulted.
1852
+ */
1853
+ private async confirmCommitRivalAgainstLocal(request: CommitRequest): Promise<StaleFailure | 'own-durable' | undefined> {
1854
+ const blockIds = request.blockIds;
1855
+ let results: GetBlockResults;
1856
+ try {
1857
+ results = await this.storageRepo.get({ blockIds });
1858
+ } catch (readError) {
1859
+ this.log('coordinator-repo:commit-stale-classify-read-error', {
1860
+ actionId: request.actionId,
1861
+ error: (readError as Error).message
1862
+ });
1863
+ return undefined;
1864
+ }
1865
+ const reader = this.storageRepo as IRepo & Partial<IRevisionActionReader>;
1866
+ // Scan EVERY block (same rule as the pend classifier): report the highest confirmed rival
1867
+ // revision, but bail the moment any block shows OUR action durable at the requested revision.
1868
+ const rivalStales: ({ blockId: BlockId; rev: number } | undefined)[] = [];
1869
+ for (const blockId of blockIds) {
1870
+ const latest = results[blockId]?.state?.latest;
1871
+ if (!latest || latest.rev < request.rev) continue;
1872
+ if (latest.rev === request.rev) {
1873
+ if (latest.actionId === request.actionId) {
1874
+ this.log('coordinator-repo:commit-stale-classify-own-action', {
1875
+ actionId: request.actionId, blockId, rev: request.rev
1876
+ });
1877
+ return 'own-durable';
1878
+ }
1879
+ rivalStales.push({ blockId, rev: latest.rev });
1880
+ continue;
1881
+ }
1882
+ // latest.rev > request.rev — latest can no longer name who took request.rev.
1883
+ if (typeof reader.getRevisionAction !== 'function') continue;
1884
+ let takenBy: ActionId | undefined;
1885
+ try {
1886
+ takenBy = await reader.getRevisionAction(blockId, request.rev);
1887
+ } catch (readError) {
1888
+ this.log('coordinator-repo:commit-stale-classify-revision-read-error', {
1889
+ actionId: request.actionId, blockId, rev: request.rev,
1890
+ error: (readError as Error).message
1891
+ });
1892
+ continue;
1893
+ }
1894
+ if (takenBy === request.actionId) {
1895
+ this.log('coordinator-repo:commit-stale-classify-own-action', {
1896
+ actionId: request.actionId, blockId, rev: request.rev, latestRev: latest.rev
1897
+ });
1898
+ return 'own-durable';
1899
+ }
1900
+ if (takenBy !== undefined) rivalStales.push({ blockId, rev: latest.rev });
1901
+ // takenBy undefined (truncated history): unconfirmed for this block.
1902
+ }
1903
+ const staleAt = highestStaleAt(rivalStales);
1904
+ if (!staleAt) return undefined;
1905
+ this.log('coordinator-repo:commit-stale-classified', {
1906
+ actionId: request.actionId,
1907
+ blockId: staleAt.blockId,
1908
+ latestRev: staleAt.rev,
1909
+ requestedRev: request.rev
1910
+ });
1911
+ return {
1912
+ success: false,
1913
+ conflict: true,
1914
+ reason: `stale commit: block ${staleAt.blockId} at rev ${staleAt.rev}, requested rev ${request.rev}`,
1915
+ staleAt
1916
+ };
1917
+ }
1918
+
1919
+ /**
1920
+ * Report success for a commit the cluster carried but this peer could not apply locally. The
1921
+ * blocks are marked seen so the read path treats them as freshness-checked; convergence comes
1922
+ * from replication (cohort reconcile, or read-driven acquisition), not from replay here.
1923
+ */
1924
+ private tolerateLocalCommitDivergence(request: CommitRequest, blockIds: BlockId[], detail: string): CommitResult {
1925
+ this.log('coordinator-repo:commit-local-failed-cluster-succeeded', { actionId: request.actionId, error: detail });
1926
+ this.markBlocksSeen(blockIds);
1927
+ return { success: true };
1928
+ }
1929
+ }
1930
+
1931
+ /** True if a simple majority of cluster peers signed an approving commit. */
1932
+ function clusterReachedCommitConsensus(record: ClusterRecord): boolean {
1933
+ const peerCount = Object.keys(record.peers).length;
1934
+ if (peerCount === 0) return false;
1935
+ const approvedCommits = Object.values(record.commits).filter(s => s.type === 'approve').length;
1936
+ return approvedCommits > peerCount / 2;
1937
+ }