@optimystic/db-p2p 0.27.0 → 0.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/dist/src/cluster/block-transfer-service.d.ts +0 -10
  2. package/dist/src/cluster/block-transfer-service.d.ts.map +1 -1
  3. package/dist/src/cluster/block-transfer-service.js +4 -2
  4. package/dist/src/cluster/block-transfer-service.js.map +1 -1
  5. package/dist/src/cluster/cluster-policy.d.ts +101 -18
  6. package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
  7. package/dist/src/cluster/cluster-policy.js +153 -30
  8. package/dist/src/cluster/cluster-policy.js.map +1 -1
  9. package/dist/src/cluster/quorum-restore.d.ts +4 -2
  10. package/dist/src/cluster/quorum-restore.d.ts.map +1 -1
  11. package/dist/src/cluster/quorum-restore.js +4 -2
  12. package/dist/src/cluster/quorum-restore.js.map +1 -1
  13. package/dist/src/cluster/service.d.ts +12 -9
  14. package/dist/src/cluster/service.d.ts.map +1 -1
  15. package/dist/src/cluster/service.js +6 -6
  16. package/dist/src/cluster/service.js.map +1 -1
  17. package/dist/src/dispute/service.d.ts +1 -4
  18. package/dist/src/dispute/service.d.ts.map +1 -1
  19. package/dist/src/dispute/service.js +2 -1
  20. package/dist/src/dispute/service.js.map +1 -1
  21. package/dist/src/libp2p-key-network.d.ts +38 -0
  22. package/dist/src/libp2p-key-network.d.ts.map +1 -1
  23. package/dist/src/libp2p-key-network.js +47 -1
  24. package/dist/src/libp2p-key-network.js.map +1 -1
  25. package/dist/src/libp2p-node-base.d.ts.map +1 -1
  26. package/dist/src/libp2p-node-base.js +22 -19
  27. package/dist/src/libp2p-node-base.js.map +1 -1
  28. package/dist/src/logger.d.ts +28 -1
  29. package/dist/src/logger.d.ts.map +1 -1
  30. package/dist/src/logger.js +143 -1
  31. package/dist/src/logger.js.map +1 -1
  32. package/dist/src/network/network-manager-service.d.ts +1 -4
  33. package/dist/src/network/network-manager-service.d.ts.map +1 -1
  34. package/dist/src/network/network-manager-service.js +2 -1
  35. package/dist/src/network/network-manager-service.js.map +1 -1
  36. package/dist/src/repo/coordinator-repo.d.ts +183 -13
  37. package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
  38. package/dist/src/repo/coordinator-repo.js +686 -107
  39. package/dist/src/repo/coordinator-repo.js.map +1 -1
  40. package/dist/src/repo/service.d.ts +9 -6
  41. package/dist/src/repo/service.d.ts.map +1 -1
  42. package/dist/src/repo/service.js +4 -5
  43. package/dist/src/repo/service.js.map +1 -1
  44. package/dist/src/sync/service.d.ts +1 -2
  45. package/dist/src/sync/service.d.ts.map +1 -1
  46. package/dist/src/sync/service.js +2 -1
  47. package/dist/src/sync/service.js.map +1 -1
  48. package/dist/src/testing/mesh-harness.d.ts +7 -1
  49. package/dist/src/testing/mesh-harness.d.ts.map +1 -1
  50. package/dist/src/testing/mesh-harness.js +2 -1
  51. package/dist/src/testing/mesh-harness.js.map +1 -1
  52. package/package.json +2 -2
  53. package/readme.md +19 -0
  54. package/src/cluster/block-transfer-service.ts +4 -8
  55. package/src/cluster/cluster-policy.ts +196 -36
  56. package/src/cluster/quorum-restore.ts +4 -2
  57. package/src/cluster/service.ts +14 -9
  58. package/src/dispute/service.ts +3 -3
  59. package/src/libp2p-key-network.ts +50 -1
  60. package/src/libp2p-node-base.ts +22 -19
  61. package/src/logger.ts +196 -2
  62. package/src/network/network-manager-service.ts +414 -414
  63. package/src/protocol-client.ts +196 -196
  64. package/src/repo/coordinator-repo.ts +833 -122
  65. package/src/repo/service.ts +12 -9
  66. package/src/sync/service.ts +3 -5
  67. package/src/testing/mesh-harness.ts +8 -1
@@ -10,7 +10,7 @@ import { PenaltyReason } from "../reputation/types.js";
10
10
  import type { ITransactionStateStore } from "../cluster/i-transaction-state-store.js";
11
11
  import { quorumSize, corroboratorCapacity, selectQuorumRev, certifiedEquivocation, CORROBORATION_FLOOR, type RevClaim, type QuorumRev } from "../cluster/quorum-restore.js";
12
12
  import { certifyClaim, isAttributableProofFailure, proofThresholds, type ProofAnchoring } from "../cluster/certified-claims.js";
13
- import { DEFAULT_CLUSTER_SIZE } from "../cluster/cluster-policy.js";
13
+ import { DEFAULT_CLUSTER_SIZE, resolveRepairCorroborationClusterSize } from "../cluster/cluster-policy.js";
14
14
  import { RECONCILE_TIMEOUT_MS } from "../cluster/reconcile-block.js";
15
15
  import { isMissingBaseRevisionFailure, MISSING_BASE_REVISION_REASON, type ICommitProofPersister, type IRevisionActionReader } from "../storage/storage-repo.js";
16
16
  import { buildBlockCommitProof, type BlockCommitProof } from "../cluster/commit-proof.js";
@@ -88,13 +88,57 @@ interface ClusterLatestQuery {
88
88
  */
89
89
  uncorroboratedRev?: number;
90
90
  /**
91
- * How many cohort peers OTHER than this node answered the consult at all — with a claim
92
- * or with "I hold nothing". `silent` says who could not be asked; this says how many
93
- * could. Zero with a non-empty `silent` means this node reached NOBODY, which is a
91
+ * Cohort peers OTHER than this node that answered the consult at all — with a claim or with
92
+ * "I hold nothing". `silent` says who could not be asked; this says who could, and the two
93
+ * together are exactly the non-self cohort view this pass saw (`answered` `silent`, no
94
+ * overlap). Empty with a non-empty `silent` means this node reached NOBODY, which is a
94
95
  * different fact from partial silence: there is no better-informed answer to be had from
95
96
  * this node's position (see {@link AbsenceVerdict}).
97
+ *
98
+ * Was a bare count, and `answered.length` is still that number wherever only the count is
99
+ * wanted. The IDENTITIES are what let a claim recorded by an EARLIER pass be matched against
100
+ * the peers that actually made it: an answer refutes only the claim it bears on (see
101
+ * {@link CoordinatorRepo.recordAheadClaim}).
96
102
  */
97
- answered: number;
103
+ answered: string[];
104
+ /**
105
+ * Every claim this consult collected, whatever the quorum then did with it — the ones a
106
+ * corroborated selection was built from included. The caller derives a claim's CLAIMANTS
107
+ * from these (the peers claiming at or above the revision in doubt), which is what
108
+ * {@link CoordinatorRepo.recordAheadClaim} later matches against `silent` to decide whether
109
+ * a recorded memo may be weakened. Read-only: selection has already run and nothing
110
+ * downstream may re-vote off these.
111
+ */
112
+ claims: readonly RevClaim[];
113
+ /**
114
+ * Why this pass's corroboration decline is provably PERMANENT, when it is (see
115
+ * {@link DeadlockReason}). Computed fresh on EVERY declining pass — the once-per-episode
116
+ * suppression in {@link CoordinatorRepo.reportRepairDeadlock} applies to the LOG LINE, never to
117
+ * this verdict — because the caller's arming decision has to fire on every qualifying pass (the
118
+ * read-repair window expires and the next pass must re-arm), not only on the first. Set only
119
+ * alongside an absent {@link corroborated}; absent here means the decline is, or at least may
120
+ * be, transient — a silent peer, a plain shortfall, an agreed absence — and re-asking can learn.
121
+ */
122
+ deadlock?: DeadlockReason;
123
+ /**
124
+ * How the corroborated selection was supported, present exactly when {@link corroborated} is:
125
+ * `voters` counts the distinct corroborating supporters (the certified claimants, when the
126
+ * certified rule selected), and `certified` marks a selection whose corroboration is a verified
127
+ * cohort commit proof's signature set rather than other voters. Observability only — selection
128
+ * has already run and nothing downstream re-votes off this; it exists so "this node's currency
129
+ * rests on one peer's word" is visible in the `cluster-fetch:local-current` line without
130
+ * re-deriving it from the claim set. Read the two together: `voters: 1` WITHOUT `certified` is one
131
+ * peer's uncorroborated word; with it, the corroboration is a proof and the voter count is beside
132
+ * the point.
133
+ *
134
+ * NOTE: `certified: true` with `voters: 1` does not distinguish a multi-signer cohort proof from a
135
+ * solo cohort's self-signed receipt — a distinction selection itself makes
136
+ * (`RevClaim.certifiedSignerCount`, weighed in `selectQuorumRev`) but {@link QuorumRev} does not
137
+ * carry out. Fine while the flag is read as "proof-backed, not peer-backed"; if operators ever
138
+ * need to tell the two apart from a log, add the signer count to `QuorumRev` and emit it here
139
+ * rather than re-deriving it from the claim set.
140
+ */
141
+ corroboration?: { voters: number; certified?: true };
98
142
  }
99
143
 
100
144
  /**
@@ -114,6 +158,19 @@ interface AheadClaimState {
114
158
  * consult finds nothing ahead of what this node holds.
115
159
  */
116
160
  rev?: number;
161
+ /**
162
+ * Which cohort peers reported {@link rev} when the claim was recorded — the peers whose later
163
+ * answer is evidence ABOUT this claim, and the only peers whose silence may keep it standing
164
+ * against a consult that would retire it or revise it downward. Nobody else can speak for them:
165
+ * a peer that never knew the claimed revision saying "I hold nothing" is not a refutation of it
166
+ * (see {@link CoordinatorRepo.recordAheadClaim}).
167
+ *
168
+ * One pass contributes at most a cohort's width; passes at an unchanged `rev` union rather than
169
+ * replace, so under churn the set is bounded by the distinct peers that ever claimed that exact
170
+ * revision (see the NOTE at the union). Meaningless without `rev`, and written and cleared with
171
+ * it — the pair is one fact. `deadlocksReported` keeps its own independent lifetime.
172
+ */
173
+ claimants?: readonly string[];
117
174
  /**
118
175
  * Which `cluster-fetch:repair-deadlock` reasons have already been said for this block (see
119
176
  * {@link CoordinatorRepo.reportRepairDeadlock}). Neither reason is about any one revision — one is
@@ -136,6 +193,25 @@ interface AheadClaimState {
136
193
  */
137
194
  type DeadlockReason = 'cohort-too-small' | 'sole-holder';
138
195
 
196
+ /**
197
+ * One corroboration-declining pass, as {@link CoordinatorRepo.classifyRepairDeadlock} weighs it and
198
+ * {@link CoordinatorRepo.reportRepairDeadlock} reports it. Named rather than written inline at both,
199
+ * so the field meanings — several of which are easy to confuse with a neighbour — are stated once.
200
+ */
201
+ interface RepairDeclinePass {
202
+ blockId: BlockId;
203
+ claims: RevClaim[];
204
+ silentCount: number;
205
+ /** Cohort peers besides this node, from the cohort view — whether they answered or not. */
206
+ cohortPeers: number;
207
+ /** How many of those peers answered at all, holders and confirmed non-holders alike. */
208
+ answered: number;
209
+ /** The quorum THIS pass demanded, computed from the peers that actually claimed. */
210
+ required: number;
211
+ /** `corroboratorCapacity` for this pass — a function of the view and the resolved size, not of who answered. */
212
+ capacity: number;
213
+ }
214
+
139
215
  /** The `cohort-too-small` wording: the cohort cannot field the quorum however healthy its peers are. */
140
216
  function cohortTooSmallMessage(
141
217
  cohortPeers: number,
@@ -151,8 +227,11 @@ function cohortTooSmallMessage(
151
227
  `${CORROBORATION_FLOOR} cohort peers BESIDES the reader to answer and agree, relaxed to 1 only for a ` +
152
228
  `cohort that DECLARES it is smaller; repairCorroborationClusterSize currently resolves to ` +
153
229
  `${repairCorroborationClusterSize}. Two things produce this, and this node cannot tell them ` +
154
- `apart: (1) the deployment really does run this few machines — set clusterPolicy.assumedClusterSize ` +
155
- `to the number you actually run (it does not lower clusterSize / the replication factor), or set an ` +
230
+ `apart: (1) the deployment really does run this few machines — set ` +
231
+ `clusterPolicy.repairCorroborationClusterSize to the number you actually run, which moves ONLY this ` +
232
+ `repair yardstick, or set clusterPolicy.assumedClusterSize, which moves it AND the membership ` +
233
+ `admission gate's low-confidence write floor (neither lowers clusterSize / the replication factor), ` +
234
+ `or set an ` +
156
235
  `honest clusterSize, and run at least ${CORROBORATION_FLOOR + 2} machines for any tolerance of one ` +
157
236
  `unreachable peer; or (2) this node's view of the cohort has shrunk below the real deployment — a ` +
158
237
  `partition or a routing problem, which configuration will not fix. Check the peer count above ` +
@@ -187,6 +266,111 @@ function soleHolderMessage(cohortPeers: number): string {
187
266
  `verify.)`;
188
267
  }
189
268
 
269
+ /**
270
+ * What one block's pending-conflict refusals have added up to, for ONE unchanged set of holders.
271
+ *
272
+ * A block is reserved by an unresolved pending action for the span between that action's pend and
273
+ * its commit or cancel, and while the reservation stands every OTHER writer's pend for the block is
274
+ * refused. That is the healthy optimistic-concurrency loss. The unhealthy case has the identical
275
+ * per-refusal shape and differs only in repetition: the same holder refusing DISTINCT later actions
276
+ * without end, because the holder is never going to commit or cancel (see
277
+ * {@link CoordinatorRepo.noteStuckReservation}).
278
+ *
279
+ * Counted per (block, holders) rather than per block: a holder that changes is the healthy cycle —
280
+ * the previous reservation resolved and another writer took the block — so a new holder starts a new
281
+ * count and gets its own chance to speak.
282
+ */
283
+ interface StuckReservationWatch {
284
+ /** The rival action ids the refusals in this episode named, sorted so the comparison is stable. */
285
+ holders: readonly ActionId[];
286
+ /**
287
+ * Distinct action ids these holders have refused. Distinct ACTIONS, not refusals: one writer
288
+ * retrying is one writer, because a sync reuses a single action id across all of its retry
289
+ * attempts (`Collection.syncInternal` mints the id once and `syncAttempts` reuses it for every
290
+ * attempt of that cycle). Emptied at the
291
+ * moment the episode is reported — the count is in the line, and nothing reads the ids again —
292
+ * so the set is bounded by {@link STUCK_RESERVATION_DISTINCT_ACTIONS}.
293
+ */
294
+ refused: Set<ActionId>;
295
+ /** True once this episode has been named; suppresses every later refusal against these holders. */
296
+ reported: boolean;
297
+ }
298
+
299
+ /**
300
+ * How many DISTINCT later actions one unchanged holder must refuse on a block before the refusals are
301
+ * named as a stuck reservation rather than as an ordinary lost race.
302
+ *
303
+ * **Why a count of distinct actions and not something else.** Elapsed time answers the wrong question
304
+ * — a slow writer is not a stuck one, and a holder legitimately keeps its reservation for as long as
305
+ * its own commit takes. A raw refusal count answers the wrong question too: a single writer retrying
306
+ * a lost race produces a run of refusals under ONE action id (see {@link StuckReservationWatch.refused}).
307
+ * What no healthy holder can produce is an unbounded stream of *different* writers all losing to it,
308
+ * because a healthy holder's reservation lasts one pend-to-commit window.
309
+ *
310
+ * **Why 8.** The bound to clear is how many distinct actions can honestly be refused inside one such
311
+ * window. Measured on the in-process mesh, in the healthy-contention arm of
312
+ * `test/stuck-reservation-named.spec.ts`: a holder that pends, is raced by other writers, and then
313
+ * commits refuses **2** distinct actions per episode — the two rivals — and the count resets on every
314
+ * holder change. `concurrent-diary-append-acknowledgement.spec.ts` races three writers at one diary
315
+ * and cannot exceed that either, for the same reason: at most (writers - 1) rivals can lose to one
316
+ * winner. 8 is four times the measured healthy figure, and it is a floor a genuinely stuck block
317
+ * clears trivially (the field instance refused hundreds).
318
+ *
319
+ * **The bound stated exactly.** It is distinct SYNC CYCLES, not distinct writers: one writer that
320
+ * exhausts a sync's retry budget and is re-driven by its caller mints a fresh id for the next cycle,
321
+ * so it can contribute more than one. That does not widen the window much — a cycle only ends in
322
+ * exhaustion after `DefaultMaxAttempts` (10) attempts of backoff, roughly 21s (see the exhaustion
323
+ * NOTE in `Collection.syncAttempts`), so a lone writer needs a holder to keep the block for upwards
324
+ * of two and a half minutes before it reaches 8 by itself, which is not a healthy holder.
325
+ *
326
+ * **What the margin does NOT cover, stated honestly.** A block with more than 8 distinct writers
327
+ * racing it inside a single pend-to-commit round trip could reach 8 with a perfectly healthy holder.
328
+ * That is a diagnostic false positive on a log line and nothing else — this counter never refuses,
329
+ * expires, or deletes anything (see {@link CoordinatorRepo.noteStuckReservation}) — and the remedy if
330
+ * a deployment ever hits it is to raise this number, not to add a control path. Raising it costs
331
+ * detection latency on low-traffic blocks, which need this many distinct write ATTEMPTS before the
332
+ * condition can be named at all.
333
+ */
334
+ const STUCK_RESERVATION_DISTINCT_ACTIONS = 8;
335
+
336
+ /** Whether two sorted holder lists name the same reservation — i.e. whether a refusal continues an
337
+ * existing episode or starts a new one. Both sides come from the same sort, so this is a plain
338
+ * element-wise comparison; a block normally has exactly one holder, since a member's own pend refuses
339
+ * a second one (`ClusterMember.validatePendOperations`). */
340
+ function sameHolders(a: readonly ActionId[], b: readonly ActionId[]): boolean {
341
+ return a.length === b.length && a.every((id, i) => id === b[i]);
342
+ }
343
+
344
+ /**
345
+ * The stuck-reservation wording: written for an operator reading logs, in the same register as
346
+ * {@link cohortTooSmallMessage} and {@link soleHolderMessage} — what is stuck, what will and will not
347
+ * clear it, and what to do next.
348
+ *
349
+ * The claim is deliberately about the RESERVATION, not about the writer's intent: this node cannot
350
+ * see whether the holding process is alive, only that it has held the block across enough unrelated
351
+ * later actions that no retry is going to win. So the line says what is provable (the block accepts
352
+ * no writes while this record stands, and nothing on the node removes it) and points at the one check
353
+ * that settles the rest.
354
+ */
355
+ function stuckReservationMessage(holders: readonly ActionId[], refusedActions: number): string {
356
+ const held = holders.join(', ');
357
+ return `This block is WEDGED BEHIND A PENDING WRITE THAT IS NOT COMPLETING, and retrying will never ` +
358
+ `clear it: action(s) ${held} reserved the block and have now refused ${refusedActions} DISTINCT, ` +
359
+ `unrelated later actions. Each of those refusals on its own looks exactly like an ordinary ` +
360
+ `optimistic-concurrency loss, which is normal and healthy — the repetition is what is not. A ` +
361
+ `healthy rival holds a block only for its own pend-to-commit window and then releases it by ` +
362
+ `committing or cancelling; a reservation that keeps refusing NEW writers is holding the block ` +
363
+ `against every writer on every machine, and each of them loses again identically. EXACTLY TWO ` +
364
+ `THINGS CLEAR IT: a cancel for action(s) ${held} on this block (route it through the cohort so ` +
365
+ `every member drops the record), or that same action's own commit landing. Nothing on the node ` +
366
+ `expires it — there is no sweep for abandoned pending records — so until one of those two happens ` +
367
+ `the block takes NO writes while continuing to serve reads and to look healthy in every other ` +
368
+ `respect. The usual cause is a writer that went away between a failed or half-applied commit and ` +
369
+ `the cancel it owed, so check whether whatever ran ${held} still exists before cancelling on its ` +
370
+ `behalf. This line is a diagnosis and nothing more: this node does not expire, refuse, or delete ` +
371
+ `the record on the strength of it.`;
372
+ }
373
+
190
374
  /**
191
375
  * What one repair pass established about a block that is still MISSING locally after it.
192
376
  * Ordered by how firmly the block is ruled out; `get` consults it only on the missing path.
@@ -208,6 +392,42 @@ type AbsenceVerdict =
208
392
  * quorum corroborated it and acquisition failed. */
209
393
  | 'claimed';
210
394
 
395
+ /**
396
+ * What one consult established about whether this node's copy is CURRENT — the currency
397
+ * counterpart to {@link AbsenceVerdict}. Three cases, and two rules the difference between them
398
+ * exists to state:
399
+ *
400
+ * - A CONSULT THAT REACHED NOBODY REFUTES NOTHING. An earlier pass's recorded doubt
401
+ * ({@link CoordinatorRepo.unsettledAheadClaims}) may only be cleared by evidence, and "nobody
402
+ * answered" is not evidence.
403
+ * - AN ANSWER REFUTES ONLY THE CLAIM IT BEARS ON. A peer that never reported the claimed
404
+ * revision saying "I hold nothing" says nothing about a claim another peer made; only the
405
+ * claimants can retire their own word. So BOTH cases that reached somebody carry `silent` —
406
+ * who could not be asked — rather than a pre-baked "refuted", and
407
+ * {@link CoordinatorRepo.recordAheadClaim} weighs it against the claim it actually holds.
408
+ * That applies to `unsettled-claim` too, which weakens a recorded claim whenever its own
409
+ * revision is LOWER: same erasure, one step less obvious.
410
+ *
411
+ * Kept as a named union rather than an optional number so a `return` added later cannot mean
412
+ * "nothing is ahead" by leaving a field off — the compiler asks.
413
+ */
414
+ type CurrencyVerdict =
415
+ /** The consult reached the cohort and nothing it heard is ahead of this node. NOT by itself a
416
+ * refutation of a recorded memo: it carries the evidence — `answered` and `silent`, the two
417
+ * halves of the non-self cohort view — so `recordAheadClaim` can ask whether the peers that
418
+ * MADE the recorded claim are among the ones that spoke. */
419
+ | { kind: 'nothing-ahead'; answered: readonly string[]; silent: readonly string[] }
420
+ /** No cohort member outside this node was asked (no cohort, solo-self, no callback) or none
421
+ * answered. No evidence either way: a recorded memo stands exactly as it was. */
422
+ | { kind: 'no-evidence' }
423
+ /** Cohort peers `claimants` claim `rev`, strictly ahead of what this node holds, and this pass
424
+ * did not converge onto it — quorum declined the claim, or corroborated it and acquisition
425
+ * failed. `claimants` is recorded with the claim: it is who must later answer for it.
426
+ * `silent` is the same evidence `nothing-ahead` carries, needed for the same reason: this
427
+ * verdict REPLACES a recorded claim, so when its `rev` is lower it must not do so behind the
428
+ * back of a claimant that could not be asked. */
429
+ | { kind: 'unsettled-claim'; rev: number; claimants: readonly string[]; silent: readonly string[] };
430
+
211
431
  /**
212
432
  * Extended cluster interface that includes the ability to check if a transaction was executed.
213
433
  * This is used by CoordinatorRepo to avoid duplicate execution.
@@ -352,6 +572,29 @@ export class CoordinatorRepo implements IRepo {
352
572
  * doubt until the next consult re-derives it (one read-repair window later, at worst) and lets
353
573
  * {@link reportRepairDeadlock} say its piece a second time. */
354
574
  private readonly unsettledAheadClaims = new LruMap<string, AheadClaimState>(1000);
575
+ /**
576
+ * Per block, what its pending-conflict refusals have added up to — see {@link StuckReservationWatch}
577
+ * and {@link noteStuckReservation}. Deliberately its OWN map rather than a third fact hung off
578
+ * {@link unsettledAheadClaims}: that entry belongs to the read-repair path and is cleared by a block
579
+ * converging on a revision, whereas this one belongs to the write path and is cleared by the block
580
+ * accepting a write. Sharing the entry would mean teaching both of those lifetimes about a fact
581
+ * neither owns (backlog `debt-freshness-state-scattered-across-coordinator-repo` is the standing
582
+ * argument for collapsing all of this per-block state behind one collaborator; adding a fourth
583
+ * carve-out to the freshness entry would have made that harder, not easier).
584
+ *
585
+ * NOTE: LRU-bounded like its siblings. An eviction under >1000 conflicted blocks loses an episode's
586
+ * say-once flag, so the line can repeat once for that block — the same bounded duplication
587
+ * {@link reportRepairDeadlock} accepts, and far cheaper than the unbounded silence it replaces.
588
+ *
589
+ * NOTE: per COORDINATOR, and a block's coordinator is whichever peer the writer's key lookup
590
+ * resolved — normally stable, but cohort churn or a routing change moves it. When it moves, the
591
+ * count restarts on the new coordinator (the condition is named later) and the old one may name the
592
+ * same episode again (the condition is named twice). Fine while a wedged block is diagnosed by
593
+ * searching for its id; if a churning deployment ever makes duplicate lines the noisy failure this
594
+ * one exists to replace, the say-once state has to move to where the record lives (the member's own
595
+ * storage) rather than to where the refusal was classified.
596
+ */
597
+ private readonly stuckReservations = new LruMap<string, StuckReservationWatch>(1000);
355
598
  private readonly readRepairMode: 'off' | 'lazy' | 'paranoid';
356
599
  private readonly readRepairWindowMs: number;
357
600
  private readonly readRepairSampleRate: number;
@@ -426,9 +669,12 @@ export class CoordinatorRepo implements IRepo {
426
669
  // unrepaired, degraded rather than dead, so there is no reason to relax it for a caller that
427
670
  // has not adopted the new field. A real node is handed an explicit
428
671
  // `repairCorroborationClusterSize` by `resolveClusterPolicy`; the `assumedClusterSize` middle
429
- // term keeps direct constructors (embedders, existing tests) behaving as before.
430
- this.repairCorroborationClusterSize =
431
- cfg?.repairCorroborationClusterSize ?? policy.assumedClusterSize ?? policy.clusterSize;
672
+ // term keeps direct constructors (embedders, existing tests) behaving as before. The chain
673
+ // itself is `resolveClusterPolicy`'s own, called rather than restated so this manual-wiring
674
+ // path and the node assembly cannot drift on how much trust a lone peer gets — including on
675
+ // degenerate declarations, which fall through here exactly as they do there.
676
+ this.repairCorroborationClusterSize = resolveRepairCorroborationClusterSize(
677
+ cfg?.repairCorroborationClusterSize, policy.assumedClusterSize, policy.clusterSize);
432
678
  this.reputation = reputation;
433
679
  const localClusterRef = localCluster && localPeerId ? {
434
680
  update: localCluster.update.bind(localCluster),
@@ -567,7 +813,7 @@ export class CoordinatorRepo implements IRepo {
567
813
  }
568
814
 
569
815
  try {
570
- const { absence, claimedAheadRev } = await this.fetchBlockFromCluster(blockId, blockGets.context, localRev);
816
+ const { absence, currency } = await this.fetchBlockFromCluster(blockId, blockGets.context, localRev);
571
817
  const refreshed = await this.storageRepo.get({ blockIds: [blockId], context: blockGets.context }, options);
572
818
  const newRev = refreshed[blockId]?.state?.latest?.rev;
573
819
  if (refreshed[blockId]) {
@@ -598,14 +844,19 @@ export class CoordinatorRepo implements IRepo {
598
844
  : 'peers-unreachable');
599
845
  }
600
846
  // A PRESENT block served below a cohort claim the repair could not settle is
601
- // the mirror lie: real content posing as confirmed-current. This consult is the
602
- // authority on that claim, so it replaces whatever an earlier one recorded —
603
- // including clearing it when nobody claims anything any more. The missing case
604
- // is excluded it is the absence path above, and a bare absent below a claim
605
- // already reads as either authoritative (cohort answered, nothing corroborated)
606
- // or flagged.
847
+ // the mirror lie: real content posing as confirmed-current. A consult that
848
+ // REACHED THE CLAIMANTS is the authority on that claim, so it replaces whatever
849
+ // an earlier one recorded — including clearing it when those peers claim nothing
850
+ // any more. One that reached nobody (`currency.kind === 'no-evidence'` no
851
+ // cohort, solo-self, or total silence) refutes nothing, and neither does one
852
+ // whose answers came from peers that never made the claim; `recordAheadClaim`
853
+ // weighs the verdict's evidence against the recorded claimants, and only in the
854
+ // direction that could hide a stale serve (a higher claim always lands). The missing
855
+ // case is excluded — it is the absence path above, and a bare absent below a
856
+ // claim already reads as either authoritative (cohort answered, nothing
857
+ // corroborated) or flagged.
607
858
  if (!isMissing) {
608
- this.recordAheadClaim(blockId, claimedAheadRev);
859
+ this.recordAheadClaim(blockId, currency);
609
860
  this.flagUnconfirmedCurrency(localResult, blockId, blockGets.context);
610
861
  }
611
862
  } catch (err) {
@@ -659,14 +910,76 @@ export class CoordinatorRepo implements IRepo {
659
910
  }
660
911
 
661
912
  /**
662
- * Remember (or forget) the cohort claim a freshness consult could not settle for a block.
663
- * Only a consult that actually RAN may call this: it is the authority, so `undefined` clears
664
- * a claim an earlier pass recorded. Entries are also dropped once this node reaches the
913
+ * Remember (or forget) the cohort claim a freshness consult could not settle for a block, per
914
+ * that consult's {@link CurrencyVerdict}. Entries are also dropped once this node reaches the
665
915
  * claimed revision (see {@link flagUnconfirmedCurrency}), which is what bounds the map.
916
+ *
917
+ * **The weakening rule, in one sentence.** A recorded claim may be WEAKENED — retired, or
918
+ * revised down to a lower revision — when at least one non-self cohort member answered this
919
+ * consult AND no peer that MADE the claim was silent in it. Everything else about the rule
920
+ * follows from that, and each of the three cases a claimant can be in has its own reason:
921
+ *
922
+ * - A claimant that ANSWERED, on a consult that found nothing ahead of it, has retired its
923
+ * own word. Its answer is evidence about its own claim — the only kind that counts.
924
+ * - A claimant that was SILENT blocks weakening. Nobody else can speak for it; a peer that
925
+ * never knew the claimed revision answering "I hold nothing" is not a refutation.
926
+ * - A claimant that is NEITHER answered nor silent has left this node's cohort view:
927
+ * `findCluster` no longer holds it responsible for the block, so its old word no longer
928
+ * binds the current cohort. This is what BOUNDS the doubt — see the membership discussion
929
+ * at `nothingAheadVerdict` in {@link fetchBlockFromCluster}.
930
+ *
931
+ * Note the arithmetic that makes it a one-liner: the non-self cohort IS `answered` ∪ `silent`,
932
+ * so "answered, or gone from the cohort" is exactly "not silent". No membership set has to be
933
+ * carried or diffed.
934
+ *
935
+ * RAISING a claim needs no licence — a higher claimed revision subsumes the one it replaces,
936
+ * so `flagUnconfirmedCurrency` stamps everything it used to and more. Only the weakening
937
+ * direction can hide a stale serve, which is why the gate below is one test applied to both
938
+ * shapes rather than a rule attached to retirement alone.
939
+ *
940
+ * Forgetting requires EVIDENCE. A consult that reached nobody (`no-evidence`) refutes nothing
941
+ * and leaves the memo exactly as it was. That used to be a rule the caller had to obey and
942
+ * could not: the old `claimedAheadRev: number | undefined` said the same thing for "peers
943
+ * refuted the claim" and for "nobody was asked", so every non-consulting exit silently erased
944
+ * the doubt and served stale content as confirmed-current. The verdict type states the
945
+ * difference instead, so a new exit cannot mean the wrong one by omission — and `nothing-ahead`
946
+ * now carries the evidence rather than a pre-baked refutation for the same reason, so a caller
947
+ * cannot assert a refutation it did not observe.
666
948
  */
667
- private recordAheadClaim(blockId: BlockId, claimedRev: number | undefined): void {
949
+ private recordAheadClaim(blockId: BlockId, currency: CurrencyVerdict): void {
950
+ if (currency.kind === 'no-evidence') return;
668
951
  const prior = this.unsettledAheadClaims.get(blockId);
669
- if (claimedRev === undefined) {
952
+ const priorRev = prior?.rev;
953
+ // The one gate, applied to both shapes that can weaken a recorded claim: retiring it
954
+ // (`nothing-ahead`) and revising it down to a lower revision (`unsettled-claim` below
955
+ // `priorRev`) are the same act, and a lower claim replacing a higher one used to slip past
956
+ // the retirement rule and erase the higher claimant's word exactly as an unrelated peer's
957
+ // "I hold nothing" once did. Only the claimants' own silence protects their claim, though:
958
+ // everyone else's silence is irrelevant to it, and degrading this into "any silence blocks
959
+ // weakening" would flag a block forever behind one permanently unreachable cohort peer.
960
+ const weakensPrior = priorRev !== undefined
961
+ && (currency.kind === 'nothing-ahead' || currency.rev < priorRev);
962
+ if (weakensPrior) {
963
+ const silentClaimants = (prior?.claimants ?? []).filter(id => currency.silent.includes(id));
964
+ if (silentClaimants.length > 0) {
965
+ // The operator's only signal that a block's doubt CANNOT settle from this node's
966
+ // position: the peer that claimed the revision is unreachable, so every read of the
967
+ // block stays marked possibly-stale until it answers or leaves the cohort view.
968
+ // NOTE: emitted per pass, not say-once. In `lazy` mode `readRepairWindowMs` (10s
969
+ // default) rate-limits consults, so this is at most one line per window; in
970
+ // `paranoid` mode a consult runs on EVERY read and so this logs on every read. If
971
+ // that ever shows as noise, move it behind a say-once flag on the entry (beside
972
+ // `deadlocksReported`) rather than dropping the line — it is the only place the
973
+ // condition is named.
974
+ this.log('cluster-fetch:claim-unrefutable', { blockId, rev: priorRev, silentClaimants });
975
+ return; // the memo stands exactly as recorded
976
+ }
977
+ }
978
+ if (currency.kind === 'nothing-ahead') {
979
+ // Nothing recorded to retire, or a consult nobody outside this node answered — which
980
+ // `fetchBlockFromCluster` reports as `no-evidence` rather than this verdict, so the
981
+ // second test is the type's guarantee restated, not a reachable path.
982
+ if (priorRev === undefined || currency.answered.length === 0) return;
670
983
  // The consult is the authority on the CLAIM, and only on the claim. A recorded deadlock is
671
984
  // not about any revision — it is about how many machines this deployment can field, or how
672
985
  // many of them hold the block — so it outlives the claim that first exposed it and is
@@ -675,8 +988,23 @@ export class CoordinatorRepo implements IRepo {
675
988
  else this.unsettledAheadClaims.delete(blockId);
676
989
  return;
677
990
  }
991
+ // Repeated claims at the SAME revision accumulate claimants: pass 1 hears it from A, pass 2
992
+ // hears it from B while A is silent, and retiring later on B's answer alone would ignore A's
993
+ // still-unanswered word. A claim at a DIFFERENT revision replaces outright — a higher one
994
+ // subsumes the old, and a lower one has just passed the weakening gate above, so every peer
995
+ // that made the older claim has either answered this pass or left the cohort view.
996
+ // NOTE: the same-revision union never prunes, so a claimant that departed the cohort while
997
+ // the revision stayed stuck lingers in the list. Inert (a departed peer is never `silent`,
998
+ // so it never blocks weakening) and bounded per pass by cohort width, but across many passes
999
+ // with churn the list is bounded only by how many distinct peers ever claimed that exact
1000
+ // revision. Prune against the pass's `answered` ∪ `silent` if a long-lived stuck block ever
1001
+ // shows this entry growing.
1002
+ const claimants = priorRev === currency.rev
1003
+ ? [...new Set([...(prior?.claimants ?? []), ...currency.claimants])]
1004
+ : [...currency.claimants];
678
1005
  this.unsettledAheadClaims.set(blockId, {
679
- rev: claimedRev,
1006
+ rev: currency.rev,
1007
+ claimants,
680
1008
  ...(prior?.deadlocksReported ? { deadlocksReported: prior.deadlocksReported } : {})
681
1009
  });
682
1010
  }
@@ -757,6 +1085,34 @@ export class CoordinatorRepo implements IRepo {
757
1085
  }
758
1086
  }
759
1087
 
1088
+ /**
1089
+ * True when a commit's approve votes form a strict majority of the FULL cohort — the only case
1090
+ * where "this node committed" is evidence that no rival commit moved past it. Two strict
1091
+ * majorities of one cohort must share a voter, so a rival that also reached a full-cohort
1092
+ * majority would have surfaced here as a conflict vote instead of approving both. A commit below
1093
+ * that bar — the solo short-circuit under a larger declared cohort, or a record that enrolled
1094
+ * fewer than a full-cohort majority — rules nothing out; it still succeeds, it just must not arm
1095
+ * the lazy read-repair window ({@link markBlocksSeen}), so the next read consults the cohort as
1096
+ * if the commit had not happened.
1097
+ *
1098
+ * What it does NOT rule out, at any vote count: a rival that assembled its own quorum on a
1099
+ * SHRUNKEN cohort view, which needs no full-cohort majority to complete
1100
+ * ({@link clusterReachedCommitConsensus} accepts an enrolled-subset majority). Two commits that
1101
+ * land at one revision on disjoint quorums are a fork, and healing one is
1102
+ * `docs/partition-healing.md`'s business, not a freshness window's.
1103
+ *
1104
+ * Denominator: the full cohort, never the enrolled/reachable subset — `record.peers` is exactly
1105
+ * the thing a downsize shrinks (contrast `clusterReachedCommitConsensus`, whose enrolled-subset
1106
+ * majority answers "did consensus complete", a different question — leave it be).
1107
+ * {@link repairCorroborationClusterSize} is the declared yardstick resolved for this same
1108
+ * shrunken-view trap on the repair side, maxed with the observed cohort for the case where
1109
+ * routing sees more peers than were declared.
1110
+ */
1111
+ private commitQuorumRulesOutRivals(approvals: number, observedCohortSize: number): boolean {
1112
+ const fullCohortSize = Math.max(observedCohortSize, this.repairCorroborationClusterSize);
1113
+ return approvals > fullCohortSize / 2;
1114
+ }
1115
+
760
1116
  /**
761
1117
  * Test seam: directly set the last-seen timestamp for a block. Used by read-repair
762
1118
  * specs to simulate "the local commit happened at time T" without needing to drive
@@ -779,19 +1135,33 @@ export class CoordinatorRepo implements IRepo {
779
1135
  * answer genuinely is the whole truth. When several verdicts apply at once the sharpest
780
1136
  * evidence wins: `claimed` > `isolated` > `unconfirmed` > `confirmed` — a peer positively
781
1137
  * saying "it exists" outranks any amount of silence.
782
- * - `claimedAheadRev` — a cohort peer claimed a revision strictly ahead of what this node
783
- * holds and the pass did NOT converge onto it: the claim failed the corroboration quorum,
784
- * or was corroborated but could not be acquired. Content `get` serves below this revision
785
- * cannot be confirmed current (see {@link GetBlockResult.unconfirmedAheadRev}); the claim
786
- * itself must never drive restoration.
1138
+ * - `currency` — the verdict on whether what this node holds is CURRENT (see
1139
+ * {@link CurrencyVerdict}). `unsettled-claim` means a cohort peer claimed a revision
1140
+ * strictly ahead of what this node holds and the pass did NOT converge onto it: the claim
1141
+ * failed the corroboration quorum, or was corroborated but could not be acquired. Content
1142
+ * `get` serves below that revision cannot be confirmed current (see
1143
+ * {@link GetBlockResult.unconfirmedAheadRev}); the claim itself must never drive
1144
+ * restoration. `nothing-ahead` and `no-evidence` differ in exactly one way that matters:
1145
+ * `nothing-ahead` carries the evidence that can retire an earlier pass's memo (and does
1146
+ * retire it when the memo's claimants are among the peers that answered), `no-evidence`
1147
+ * leaves it standing untouched. Required, not optional, so an exit added later has to say
1148
+ * which it means.
787
1149
  */
788
- private async fetchBlockFromCluster(blockId: BlockId, context?: ActionContext, localRev?: number): Promise<{ absence: AbsenceVerdict; claimedAheadRev?: number }> {
789
- if (!this.clusterLatestCallback) return { absence: 'confirmed' };
1150
+ private async fetchBlockFromCluster(blockId: BlockId, context?: ActionContext, localRev?: number): Promise<{ absence: AbsenceVerdict; currency: CurrencyVerdict }> {
1151
+ if (!this.clusterLatestCallback) return { absence: 'confirmed', currency: { kind: 'no-evidence' } };
790
1152
 
791
1153
  const blockIdBytes = new TextEncoder().encode(blockId);
792
1154
  const peers = await this.keyNetwork.findCluster(blockIdBytes);
793
1155
  const peerIds = peers ? Object.keys(peers) : [];
794
- if (peerIds.length === 0) return { absence: 'confirmed' };
1156
+ // NOTE: deliberately does NOT call `markBlocksSeen` unlike the solo-self exit below.
1157
+ // An empty cohort is a ROUTING FAILURE, not a settled answer, and `Libp2pKeyPeerNetwork`
1158
+ // cannot even produce one (its `findCluster` always includes self); the only producer
1159
+ // today is the mesh harness's injected `findClusterFails`. Arming the read-repair window
1160
+ // here would suppress a genuine repair for a whole `readRepairWindowMs` after a transient
1161
+ // blip, and re-entering costs no network work beyond the `findCluster` the read already
1162
+ // makes. Do not "fix" this by symmetry with the solo-self exit.
1163
+ // Currency: nobody was asked, so nothing was refuted — an earlier pass's unsettled claim stands.
1164
+ if (peerIds.length === 0) return { absence: 'confirmed', currency: { kind: 'no-evidence' } };
795
1165
 
796
1166
  // Solo-cluster short-circuit: the only responsible peer is us. There is no
797
1167
  // remote to sync from, so skip the callback entirely. Querying ourselves
@@ -803,10 +1173,34 @@ export class CoordinatorRepo implements IRepo {
803
1173
  && peerIds[0] === this.localPeerId.toString()
804
1174
  ) {
805
1175
  this.log('cluster-fetch:solo-self-skip', { blockId });
806
- return { absence: 'confirmed' };
1176
+ // Arm the lazy read-repair window. Without this the window is never stamped,
1177
+ // `shouldReadRepair` reads `lastSeen == null` forever, and every read re-enters this
1178
+ // exit — read -> stale -> consult -> solo skip -> no-op -> still stale, unbounded
1179
+ // (GitHub issue #8: a solo node spent 47 minutes on a cold schema apply, logging 3,880
1180
+ // triggers against 3,879 no-ops).
1181
+ // What arming claims, precisely: NOT that this node has no rivals. A self-only cohort
1182
+ // is also what `Libp2pKeyPeerNetwork.findCluster` returns while genuine same-network
1183
+ // peers are still 'unknown' mid-identify (see its membership-scoping comment). It
1184
+ // claims only that re-asking sooner than one window cannot learn anything the next
1185
+ // `findCluster` would not, since this exit runs no other query — so a cohort that
1186
+ // appears, grows, or finishes identifying is consulted within one `readRepairWindowMs`.
1187
+ // NOTE: that self-heal delay IS `readRepairWindowMs` — 10s by default, well under
1188
+ // bootstrap. If it is ever configured into the minutes, a cold node would serve
1189
+ // unverified reads for that whole period; gate arming on cohort provenance if so.
1190
+ // Deliberately opposite to the commit-side rule that withholds arming from a commit
1191
+ // whose quorum proves nothing about rivals: that damps nothing, this bounds an
1192
+ // otherwise unbounded loop. Landing both, keep both — see the specs for each.
1193
+ this.markBlocksSeen([blockId]);
1194
+ // Currency: this exit queried NOBODY, so it refutes nothing — an earlier pass's unsettled
1195
+ // claim survives it. Note the coupling with the arming just above: retained doubt now
1196
+ // persists for up to `readRepairWindowMs` before a consult can refute it. That is correct
1197
+ // and deliberate — the window damps repair EFFORT, not honesty — and it is the same
1198
+ // coupling the comment at the final exit below describes. Arming the window and keeping
1199
+ // the memo are answers to different questions; do not collapse them.
1200
+ return { absence: 'confirmed', currency: { kind: 'no-evidence' } };
807
1201
  }
808
1202
 
809
- const { corroborated, local, silent, answered, uncorroboratedRev } = await this.queryClusterForLatest(peerIds, blockId, context);
1203
+ const { corroborated, corroboration, local, silent, answered, claims, uncorroboratedRev, deadlock } = await this.queryClusterForLatest(peerIds, blockId, context);
810
1204
  // Any silence taints the WHOLE consult, not a fraction of it (fail-closed): one silent
811
1205
  // peer could be the sole holder, and the cost — an extra transactor-level retry against
812
1206
  // another coordinator — is paid only while a peer is actually unreachable. Silence with
@@ -814,20 +1208,89 @@ export class CoordinatorRepo implements IRepo {
814
1208
  // connected coordinator", total silence says there is no better-informed answer to be
815
1209
  // had from this node.
816
1210
  const silenceVerdict: AbsenceVerdict =
817
- silent.length > 0 ? (answered === 0 ? 'isolated' : 'unconfirmed') : 'confirmed';
818
- // Nothing corroborated: keep local data AND stay eligible for repair marking the
819
- // block seen here would suppress the next attempt for the whole read-repair window.
1211
+ silent.length > 0 ? (answered.length === 0 ? 'isolated' : 'unconfirmed') : 'confirmed';
1212
+ // The CURRENCY half's counterpart: the verdict to use at every exit that found nothing ahead.
1213
+ // A consult that reached NOBODY outside this node refutes nothing, so a memo an earlier pass
1214
+ // recorded must survive it. Keyed on `answered` (cohort members other than this node that
1215
+ // answered at all) rather than on `silenceVerdict === 'isolated'`: the two agree in every
1216
+ // reachable state — an empty `answered` with an EMPTY silent set would need a cohort of
1217
+ // nothing but self, which the solo short-circuit above already took — but stating the rule
1218
+ // in terms of "did anyone answer" says what it means without depending on that argument.
1219
+ // Computed once here, like `silenceVerdict`, so it is stated in one place instead of
1220
+ // re-derived at four `return`s.
1221
+ //
1222
+ // Deliberately NOT the same shape as `silenceVerdict` above, and no longer a defect that it
1223
+ // is not: the two answer DIFFERENT QUESTIONS. `silenceVerdict` asks whether this pass saw
1224
+ // the whole cohort, which partial silence answers "no" (fail-closed: one silent peer could
1225
+ // be the sole holder of a block this node lacks entirely). This one carries EVIDENCE rather
1226
+ // than a verdict, because whether an answer refutes anything depends on the claim being
1227
+ // weighed: `recordAheadClaim` retires a memo only when a claimant answered and no claimant
1228
+ // was silent. Do not "harmonize" them into one three-level verdict; and do not collapse this
1229
+ // back to a bare `refuted`, which is what let a peer that never knew the claim retire it.
1230
+ //
1231
+ // **How doubt settles when a claimant never comes back — not a timer.** A permanently silent
1232
+ // claimant would keep a block marked forever, so what bounds the doubt is COHORT MEMBERSHIP:
1233
+ // `Libp2pKeyPeerNetwork.findCluster` builds `peerIds` from the live routing table, so a peer
1234
+ // that is genuinely gone leaves the cohort view, stops appearing in `silent`, and stops
1235
+ // blocking retirement. Two consequences, both accepted:
1236
+ // - If a dead peer never leaves the cohort view, the memo stands forever. That is a
1237
+ // membership defect to fix in the membership layer, not a licence to lie here — the block
1238
+ // genuinely IS possibly stale while a responsible cohort member holds a revision nobody
1239
+ // can obtain. `cluster-fetch:claim-unrefutable` (see `recordAheadClaim`) gives the
1240
+ // operator a name for it rather than erasing the doubt silently.
1241
+ // - A transiently-shrunken cohort view can retire a live claim early: `findCluster` never
1242
+ // admits a not-yet-identified peer, so one mid-identify is briefly absent and briefly
1243
+ // looks departed. Self-correcting — when it rejoins still holding the higher revision, the
1244
+ // next consult re-records the claim from its answer. One window of clean reads, versus the
1245
+ // permanent denial the alternative rule ("only a fully-answered cohort refutes") produces
1246
+ // from a single unreachable peer.
1247
+ const nothingAheadVerdict: CurrencyVerdict =
1248
+ answered.length === 0 ? { kind: 'no-evidence' } : { kind: 'nothing-ahead', answered, silent };
1249
+ /** The peers whose word this pass would be recording: those claiming at or above `rev`. A
1250
+ * claimant is who must later answer for the claim, so a peer claiming something LOWER is
1251
+ * not one — its answer says nothing about the revision in doubt. */
1252
+ const claimantsAtOrAbove = (rev: number): string[] =>
1253
+ claims.filter(c => c.rev >= rev).map(c => c.peerId);
1254
+ // Nothing corroborated: keep local data AND (usually) stay eligible for repair — marking the
1255
+ // block seen suppresses the next attempt for the whole read-repair window, which is only
1256
+ // right when re-asking sooner could not teach this node anything (see the `deadlock` arming
1257
+ // below for the one decline where that is provable).
820
1258
  // An uncorroborated claim strictly ahead of what this node holds still travels up as
821
1259
  // doubt: the answer about to be served may be behind it, and only the caller knows
822
1260
  // whether that matters for the view it was asked for.
823
1261
  if (!corroborated) {
824
1262
  const uncorroboratedBaseline = local?.rev ?? localRev;
825
- const claimIsAhead = uncorroboratedRev !== undefined
826
- && (uncorroboratedBaseline === undefined || uncorroboratedRev > uncorroboratedBaseline);
827
1263
  // A claim — even one the quorum declined — is a peer positively attesting the block
828
1264
  // exists, the sharpest fact this pass can surface. It outranks silence.
829
1265
  const absence: AbsenceVerdict = uncorroboratedRev !== undefined ? 'claimed' : silenceVerdict;
830
- return { absence, ...(claimIsAhead ? { claimedAheadRev: uncorroboratedRev } : {}) };
1266
+ // A claim present but NOT ahead of the baseline falls to the shared verdict — which
1267
+ // resolves to `nothing-ahead` in practice here, since `uncorroboratedRev` can only exist
1268
+ // when a peer answered. Using the shared value anyway keeps the rule stated once.
1269
+ const currency: CurrencyVerdict =
1270
+ uncorroboratedRev !== undefined
1271
+ && (uncorroboratedBaseline === undefined || uncorroboratedRev > uncorroboratedBaseline)
1272
+ ? { kind: 'unsettled-claim', rev: uncorroboratedRev, claimants: claimantsAtOrAbove(uncorroboratedRev), silent }
1273
+ : nothingAheadVerdict;
1274
+ // Arm the lazy read-repair window when — and only when — the decline is provably
1275
+ // PERMANENT for the cohort-size reason: `cohort-too-small` means the cohort cannot field
1276
+ // the quorum even if every member answered and agreed (an undeclared two-machine cohort
1277
+ // against the default floor is the ordinary producer), so repeating the identical
1278
+ // hopeless consult sooner than one window teaches nothing — the same justification the
1279
+ // solo-self exit arms under. Every other decline stays unarmed, because there re-asking
1280
+ // CAN genuinely learn: a silent peer can recover (and the verdict is never computed off a
1281
+ // pass with silence — the guard in classifyRepairDeadlock is load-bearing here, not just
1282
+ // for the log), a `sole-holder`'s missing copy can arrive (the cohort-growth push, or a
1283
+ // commit), and a plain shortfall can fill. The doubt memo above is untouched either way:
1284
+ // the window damps repair EFFORT, never honesty — reads inside it still carry
1285
+ // `unconfirmedAheadRev` via flagUnconfirmedCurrency (pinned in
1286
+ // coordinator-repo-small-cohort-arming.spec.ts). Note the verdict arrives on EVERY
1287
+ // qualifying pass while the deadlock LOG stays once-per-episode — the split in
1288
+ // reportRepairDeadlock is what lets the window re-arm after it expires without the log
1289
+ // repeating.
1290
+ if (deadlock === 'cohort-too-small') {
1291
+ this.markBlocksSeen([blockId]);
1292
+ }
1293
+ return { absence, currency };
831
1294
  }
832
1295
 
833
1296
  // The self answer is the sharper baseline (same storage, same context, read alongside the
@@ -842,18 +1305,45 @@ export class CoordinatorRepo implements IRepo {
842
1305
  // cohort that lags behind the reader corroborates an OLDER revision; adopting it
843
1306
  // would be a regression, and logging it as a sync would be a lie. The cohort did
844
1307
  // answer, so the block is verified fresh — mark it seen.
845
- // NOTE: in a cohort of two, that sole peer is the only corroborator, so a lying one can park
846
- // the reader here corroborating the revision it already holds and re-arm the lazy window
847
- // on every pass, hiding a real divergence. Bounded by `readRepairWindowMs` (10s default) and
848
- // no worse than the peer simply staying silent. If two-member cohorts become a supported
849
- // production topology rather than a dev convenience, stop re-arming the window on a
850
- // corroboration that came from a single voter.
1308
+ // NOTE: accepted tradeoff this arm arms the lazy window even when the corroboration came
1309
+ // from a SINGLE voter (a two-member cohort's sole partner, or a single-signer certified
1310
+ // claim). The once-proposed remedy stop re-arming on a single-voter corroboration was
1311
+ // weighed and REJECTED when cohorts of one and two became supported production topology
1312
+ // (ticket `small-cohort-arming-rule`), for three reasons:
1313
+ // 1. Re-asking the sole partner sooner learns nothing: the repeat consult reaches the same
1314
+ // one peer, and a lying peer repeats the lie. Same justification as the solo-self exit —
1315
+ // not "the partner is trusted", but "a faster cadence cannot produce new evidence".
1316
+ // 2. Not arming punishes only the honest: an honest partner corroborating "you are current"
1317
+ // is also a single voter, so refusing to arm makes every healthy two-machine cohort pay
1318
+ // one network round trip per read, forever, while buying zero protection against a
1319
+ // dishonest partner.
1320
+ // 3. The residual threat is a WITHHOLDING attack, and no consult cadence or proof rule
1321
+ // touches it: a commit proof certifies "revision R was committed", nothing can certify
1322
+ // "no revision after R exists", so a sole partner withholding a newer revision is
1323
+ // indistinguishable from that revision never existing. Kept narrow by the commit rule —
1324
+ // a two-member commit needs BOTH members' signatures (super-majority and majority both
1325
+ // resolve to 2 of 2; ClusterMember.hasMajority) — so a revision the reader never
1326
+ // co-signed can exist only across the reader's own storage loss, or from a commit under
1327
+ // a different cohort shape (a partition-era solo commit, whose single-signer proof the
1328
+ // partner can present or withhold).
1329
+ // What was bought instead is observability: `voters` (and `certified`) on the line below say
1330
+ // when this node's currency rests on one peer's word, without re-deriving it from logs.
1331
+ // Revisit only if a cadence-independent freshness signal (e.g. cross-cohort anchoring) ever
1332
+ // exists to arm against — a shorter cadence alone can never be the fix, per reason 1.
851
1333
  if (baselineRev !== undefined && corroborated.rev <= baselineRev) {
852
- this.log('cluster-fetch:local-current', { blockId, localRev: baselineRev, clusterRev: corroborated.rev });
1334
+ this.log('cluster-fetch:local-current', {
1335
+ blockId, localRev: baselineRev, clusterRev: corroborated.rev,
1336
+ voters: corroboration?.voters,
1337
+ ...(corroboration?.certified ? { certified: true } : {})
1338
+ });
853
1339
  this.markBlocksSeen([blockId]);
854
1340
  // Only reachable when this node HOLDS a revision (the baseline), so `get` never
855
1341
  // consults this verdict — computed consistently rather than hard-coded.
856
- return { absence: silenceVerdict };
1342
+ // Currency: the cohort corroborated at or below what this node holds, so nothing is
1343
+ // ahead. Only reachable when a peer answered (a corroboration requires claims), so the
1344
+ // shared verdict resolves to `nothing-ahead` and a memo whose claimants are among those
1345
+ // answers is cleared.
1346
+ return { absence: silenceVerdict, currency: nothingAheadVerdict };
857
1347
  }
858
1348
 
859
1349
  // Corroborated revision is ahead of ours — converge onto it.
@@ -889,7 +1379,13 @@ export class CoordinatorRepo implements IRepo {
889
1379
  // it ever shows as read amplification, gate the acquisition step (not the latest-query) on the
890
1380
  // same window rather than widening `isMissing`.
891
1381
  this.markBlocksSeen([blockId]);
892
- return { absence, ...(converged ? {} : { claimedAheadRev: corroborated.rev }) };
1382
+ // Converged: the corroboration is itself the evidence that nothing is ahead, and it came from
1383
+ // peers that answered — the shared verdict resolves to `nothing-ahead`, and the memo retires
1384
+ // if its claimants are among those peers.
1385
+ const currency: CurrencyVerdict = converged
1386
+ ? nothingAheadVerdict
1387
+ : { kind: 'unsettled-claim', rev: corroborated.rev, claimants: claimantsAtOrAbove(corroborated.rev), silent };
1388
+ return { absence, currency };
893
1389
  }
894
1390
 
895
1391
  /**
@@ -1044,8 +1540,16 @@ export class CoordinatorRepo implements IRepo {
1044
1540
  // make self report something the reader does not hold, make `localPeerId` required instead.
1045
1541
  // The same unset-`localPeerId` tolerance also lets self count toward `answered` below, and
1046
1542
  // lets a self read that REJECTS land in `silent`: a solo repo whose own storage throws then
1047
- // reads as `answered === 0` and reports isolation ('cohort-unreachable') rather than a local
1048
- // fault. Same fix if it ever matters — require `localPeerId`.
1543
+ // reads as an EMPTY `answered` and reports isolation ('cohort-unreachable') rather than a
1544
+ // local fault. Same fix if it ever matters — require `localPeerId`.
1545
+ // What this tolerance no longer does is let this node erase its own doubt. `answered` used
1546
+ // to be a bare count, so an unset `localPeerId` on a self-only-answering pass reached
1547
+ // "somebody answered, nothing is ahead" off this node's own answer and dropped the memo —
1548
+ // the exact erasure the marker exists to prevent. Claim PROVENANCE closes it without
1549
+ // touching the constructor: the claimant recorded is the remote peer that reported the
1550
+ // revision (never self, whose answer reads the storage being repaired, per the argument
1551
+ // above), so when that peer goes silent it is a silent claimant and the memo stands. This
1552
+ // node agreeing with itself retires nothing.
1049
1553
  const selfId = this.localPeerId?.toString();
1050
1554
  let local: CertifiedActionRev | undefined;
1051
1555
  const claims: RevClaim[] = [];
@@ -1123,7 +1627,10 @@ export class CoordinatorRepo implements IRepo {
1123
1627
  }));
1124
1628
 
1125
1629
  const nonSelfCount = peerIds.filter(id => id !== selfId).length;
1126
- const answered = nonSelfCount - silent.length;
1630
+ // Identities, not a count: the caller matches a memo's recorded claimants against who spoke
1631
+ // this pass. `answered` and `silent` partition the non-self cohort view, so the old
1632
+ // `nonSelfCount - silent.length` is exactly `answered.length`.
1633
+ const answered = peerIds.filter(id => id !== selfId && !silent.includes(id));
1127
1634
  const capacity = corroboratorCapacity(nonSelfCount, this.repairCorroborationClusterSize);
1128
1635
  const required = quorumSize(claims.length, this.simpleMajorityThreshold, capacity);
1129
1636
  const selected = selectQuorumRev(claims, this.simpleMajorityThreshold, capacity);
@@ -1147,15 +1654,17 @@ export class CoordinatorRepo implements IRepo {
1147
1654
  blockId,
1148
1655
  cohortPeers: nonSelfCount,
1149
1656
  holders: claims.length,
1150
- absent: answered - claims.length,
1657
+ absent: answered.length - claims.length,
1151
1658
  silent: silent.length,
1152
1659
  required,
1153
1660
  repairCorroborationClusterSize: this.repairCorroborationClusterSize
1154
1661
  });
1155
1662
  // ...and, when this decline is provably permanent rather than transient, say THAT once,
1156
- // in words. The `no-quorum` line above fires on every pass and cannot tell the two apart.
1157
- this.reportRepairDeadlock({
1158
- blockId, claims, silentCount: silent.length, cohortPeers: nonSelfCount, answered, required, capacity
1663
+ // in words and return the verdict, which the caller acts on EVERY pass (arming the
1664
+ // read-repair window on `cohort-too-small`; see fetchBlockFromCluster). The `no-quorum`
1665
+ // line above fires on every pass and cannot tell the two apart.
1666
+ const deadlock = this.reportRepairDeadlock({
1667
+ blockId, claims, silentCount: silent.length, cohortPeers: nonSelfCount, answered: answered.length, required, capacity
1159
1668
  });
1160
1669
  // The claims themselves must not drive restoration — but their existence is
1161
1670
  // evidence the caller needs: an answer served below the highest claim cannot be
@@ -1171,7 +1680,11 @@ export class CoordinatorRepo implements IRepo {
1171
1680
  // (the verification machinery now exists) rather than on the bare assertion — at the
1172
1681
  // cost of re-opening the stale-serve window for the proof-less honest majority.
1173
1682
  const uncorroboratedRev = claims.length > 0 ? Math.max(...claims.map(c => c.rev)) : undefined;
1174
- return { local, silent, answered, ...(uncorroboratedRev !== undefined ? { uncorroboratedRev } : {}) };
1683
+ return {
1684
+ local, silent, answered, claims,
1685
+ ...(uncorroboratedRev !== undefined ? { uncorroboratedRev } : {}),
1686
+ ...(deadlock !== undefined ? { deadlock } : {})
1687
+ };
1175
1688
  }
1176
1689
 
1177
1690
  if (selected.certified) {
@@ -1188,13 +1701,27 @@ export class CoordinatorRepo implements IRepo {
1188
1701
  // (an unanchored proof must not be able to convict the honest cohort). Never let this throw.
1189
1702
  this.penalizeContradictingRevClaims(claims, selected, blockId);
1190
1703
 
1191
- return { corroborated: { actionId: selected.actionId, rev: selected.rev }, local, silent, answered };
1704
+ return {
1705
+ corroborated: { actionId: selected.actionId, rev: selected.rev },
1706
+ // The supporters used to be dropped on the floor here, leaving "how many peers this
1707
+ // currency judgment rests on" underivable downstream — see ClusterLatestQuery.corroboration.
1708
+ corroboration: {
1709
+ voters: selected.supporters.length,
1710
+ ...(selected.certified ? { certified: true as const } : {})
1711
+ },
1712
+ local, silent, answered, claims
1713
+ };
1192
1714
  }
1193
1715
 
1194
1716
  /**
1195
1717
  * Say ONCE per block, in words, when a corroboration decline is provably PERMANENT rather than a
1196
- * transient shortage of answers. There are exactly TWO permanent shapes, and they send the operator
1197
- * to different places, so each gets its own `reason` and its own wording:
1718
+ * transient shortage of answers and RETURN the verdict, computed fresh on every pass by
1719
+ * {@link classifyRepairDeadlock} (the pure half; the say-once suppression below applies only to
1720
+ * the log line). The caller acts on the returned reason every pass: `fetchBlockFromCluster` arms
1721
+ * the lazy read-repair window on `cohort-too-small`, because a decline the cohort provably cannot
1722
+ * escape teaches nothing when repeated sooner than one window. There are exactly TWO permanent
1723
+ * shapes, and they send the operator to different places, so each gets its own `reason` and its
1724
+ * own wording:
1198
1725
  *
1199
1726
  * - `cohort-too-small` — this node's cohort has fewer peers than the quorum would demand even if
1200
1727
  * every one of them answered and agreed. The remedy is machines or an honest declared size.
@@ -1247,61 +1774,22 @@ export class CoordinatorRepo implements IRepo {
1247
1774
  * that error's documented contract; deliberately out of scope here (see the ticket
1248
1775
  * `repair-deadlock-is-never-named`, *Not this ticket*).
1249
1776
  */
1250
- private reportRepairDeadlock(pass: {
1251
- blockId: BlockId;
1252
- claims: RevClaim[];
1253
- silentCount: number;
1254
- /** Cohort peers besides this node, from the cohort view whether they answered or not. */
1255
- cohortPeers: number;
1256
- answered: number;
1257
- /** The quorum THIS pass demanded, computed from the peers that actually claimed. */
1258
- required: number;
1259
- /** `corroboratorCapacity` for this pass — a function of the view and the resolved size, not of who answered. */
1260
- capacity: number;
1261
- }): void {
1262
- const { blockId, claims, silentCount, cohortPeers, answered, required, capacity } = pass;
1263
- // An incomplete picture proves nothing about the deployment; the next clean pass says it.
1264
- if (silentCount > 0) return;
1265
- // Nobody claimed anything: the cohort agrees the block is absent, which is an answer, not a
1266
- // deadlock.
1267
- if (claims.length === 0) return;
1268
- // The decisive test for the first shape. `requiredEvenIfAllAnswered` is the quorum this cohort
1269
- // would face with every one of its peers answering and agreeing — the best case reachable
1270
- // without adding machines. A cohort that can meet it is not too small.
1777
+ private reportRepairDeadlock(pass: RepairDeclinePass): DeadlockReason | undefined {
1778
+ const { blockId, claims, cohortPeers, answered, required, capacity } = pass;
1779
+ const reason = this.classifyRepairDeadlock(pass);
1780
+ if (reason === undefined) return undefined;
1781
+ // From here down is LOGGING only, under the once-per-episode suppression. The verdict above
1782
+ // is returned regardless: the caller arms the read-repair window off it on every qualifying
1783
+ // pass (a window that expired must re-arm), and letting the say-once flag swallow the verdict
1784
+ // would arm exactly once per episode — one quiet window, then the re-ask-forever loop back.
1271
1785
  const requiredEvenIfAllAnswered = quorumSize(cohortPeers, this.simpleMajorityThreshold, capacity);
1272
- const cohortTooSmall = cohortPeers < requiredEvenIfAllAnswered;
1273
- // The second shape: exactly one cohort peer holds the block AT ALL, and — since a claim is one
1274
- // peer's latest, so a single claim is a single distinct (rev, actionId) group with a single
1275
- // supporter — every other cohort peer answered that it holds nothing. `answered === cohortPeers`
1276
- // is already implied by the silence guard above; it is stated because the two counts arrive as
1277
- // independent parameters and "everybody answered" is half of what makes this provable.
1278
- //
1279
- // NOTE: there is a narrow window where `sole-holder` is true of the instant but not of the
1280
- // deployment — a commit that has landed on one cohort member and has not yet been pushed to the
1281
- // rest presents exactly this shape. Calling it PERMANENT is defensible even there (repair
1282
- // genuinely cannot converge until the push lands, and the once-per-episode flag clears the
1283
- // moment the block converges, so the line does not repeat), and widening the window is what the
1284
- // push path's own threat model decides — see
1285
- // `tickets/blocked/repair-floor-defends-a-door-the-push-path-leaves-open`. If commit-to-push
1286
- // latency ever grows enough that operators see `sole-holder` on blocks that heal moments later,
1287
- // gate the line on the block having been quiet for longer than that latency rather than
1288
- // softening the wording.
1289
- const soleHolder = claims.length === 1 && answered === cohortPeers;
1290
- if (!cohortTooSmall && !soleHolder) return;
1291
-
1292
- // Both shapes can hold at once (an undeclared two-machine deployment whose single peer holds the
1293
- // block is both). `cohort-too-small` is reported in preference because its remedy is the one
1294
- // that actually works there: declaring the real size makes the floor reachable, after which the
1295
- // lone peer's claim IS adopted — so calling it a sole-holder problem would send the operator
1296
- // looking for a copy they do not need.
1297
- const reason: DeadlockReason = cohortTooSmall ? 'cohort-too-small' : 'sole-holder';
1298
1786
  const state = this.unsettledAheadClaims.get(blockId);
1299
1787
  const alreadySaid = state?.deadlocksReported ?? [];
1300
1788
  // Suppressed per REASON, not once outright: an episode that starts as `cohort-too-small` and
1301
1789
  // becomes `sole-holder` — the operator added the machines that reason asked for, and the block
1302
1790
  // is still stuck — has a second thing to say, and a silent log there is the failure this line
1303
1791
  // exists to end. Neither reason repeats within an episode.
1304
- if (alreadySaid.includes(reason)) return;
1792
+ if (alreadySaid.includes(reason)) return reason;
1305
1793
 
1306
1794
  this.log('cluster-fetch:repair-deadlock', {
1307
1795
  blockId,
@@ -1312,7 +1800,7 @@ export class CoordinatorRepo implements IRepo {
1312
1800
  required,
1313
1801
  requiredEvenIfAllAnswered,
1314
1802
  repairCorroborationClusterSize: this.repairCorroborationClusterSize,
1315
- message: cohortTooSmall
1803
+ message: reason === 'cohort-too-small'
1316
1804
  ? cohortTooSmallMessage(cohortPeers, claims.length, requiredEvenIfAllAnswered, this.repairCorroborationClusterSize)
1317
1805
  : soleHolderMessage(cohortPeers)
1318
1806
  });
@@ -1326,6 +1814,72 @@ export class CoordinatorRepo implements IRepo {
1326
1814
  // a node-level once-flag keyed on (cohortPeers, requiredEvenIfAllAnswered) and let the per-block
1327
1815
  // entry only suppress repeats.
1328
1816
  this.unsettledAheadClaims.set(blockId, { ...(state ?? {}), deadlocksReported: [...alreadySaid, reason] });
1817
+ return reason;
1818
+ }
1819
+
1820
+ /**
1821
+ * The PURE half of {@link reportRepairDeadlock}: classify one declining pass as provably
1822
+ * permanent (`cohort-too-small` / `sole-holder`) or not (`undefined`), with no logging and no
1823
+ * state. Split out so the verdict can run on EVERY pass — the caller in
1824
+ * `fetchBlockFromCluster` arms the lazy read-repair window off `cohort-too-small` each time,
1825
+ * and the window has to re-arm after it expires — while the log line keeps its once-per-episode
1826
+ * suppression. Before the split the verdict was computed for the log line and thrown away,
1827
+ * which is what left an undeclared two-machine cohort re-running the provably hopeless consult
1828
+ * on every read (measured: 6 peer queries across three reads inside one 10 s window, versus 0
1829
+ * with the size declared).
1830
+ *
1831
+ * The guards are part of the VERDICT, not merely of the logging — the arming consumer depends
1832
+ * on both:
1833
+ * - a pass with ANY silent peer proves nothing (an incomplete picture — the silent peer could
1834
+ * recover, so re-asking can learn; and permanent claims are not made off partial views);
1835
+ * - a pass with ZERO claims is an agreed absence — an answer, not a deadlock.
1836
+ */
1837
+ private classifyRepairDeadlock(pass: RepairDeclinePass): DeadlockReason | undefined {
1838
+ const { claims, silentCount, cohortPeers, answered, capacity } = pass;
1839
+ // An incomplete picture proves nothing about the deployment; the next clean pass says it.
1840
+ if (silentCount > 0) return undefined;
1841
+ // Nobody claimed anything: the cohort agrees the block is absent, which is an answer, not a
1842
+ // deadlock.
1843
+ if (claims.length === 0) return undefined;
1844
+ // The decisive test for the first shape. `requiredEvenIfAllAnswered` is the quorum this cohort
1845
+ // would face with every one of its peers answering and agreeing — the best case reachable
1846
+ // without adding machines. A cohort that can meet it is not too small.
1847
+ const requiredEvenIfAllAnswered = quorumSize(cohortPeers, this.simpleMajorityThreshold, capacity);
1848
+ const cohortTooSmall = cohortPeers < requiredEvenIfAllAnswered;
1849
+ // The second shape: exactly one cohort peer holds the block AT ALL, and — since a claim is one
1850
+ // peer's latest, so a single claim is a single distinct (rev, actionId) group with a single
1851
+ // supporter — every other cohort peer answered that it holds nothing. `answered === cohortPeers`
1852
+ // is already implied by the silence guard above; it is stated because the two counts arrive as
1853
+ // independent parameters and "everybody answered" is half of what makes this provable.
1854
+ //
1855
+ // NOTE: there is a narrow window where `sole-holder` is true of the instant but not of the
1856
+ // deployment — a commit that has landed on one cohort member and has not yet been pushed to the
1857
+ // rest presents exactly this shape. Calling it PERMANENT is defensible even there (repair
1858
+ // genuinely cannot converge until the push lands, and the once-per-episode flag clears the
1859
+ // moment the block converges, so the line does not repeat), and widening the window is what the
1860
+ // push path's own threat model decides — see
1861
+ // `tickets/blocked/repair-floor-defends-a-door-the-push-path-leaves-open`. If commit-to-push
1862
+ // latency ever grows enough that operators see `sole-holder` on blocks that heal moments later,
1863
+ // gate the line on the block having been quiet for longer than that latency rather than
1864
+ // softening the wording. (`sole-holder` never arms the read-repair window — the missing thing
1865
+ // is a COPY, which the cohort-growth push or the next commit can deliver at any moment, so
1866
+ // re-asking can genuinely learn.)
1867
+ const soleHolder = claims.length === 1 && answered === cohortPeers;
1868
+ if (!cohortTooSmall && !soleHolder) return undefined;
1869
+
1870
+ // Both shapes hold at once whenever `cohortTooSmall` does — not merely "can". At the fixed
1871
+ // simple-majority term (0.51) the proportional quorum never exceeds the peer count, so
1872
+ // `cohortTooSmall` reduces to exactly ONE non-self cohort peer with the resolved size at three
1873
+ // or more; combined with the guards above (nobody silent, somebody claimed) that peer is also
1874
+ // the only claimant and everybody answered, which is `soleHolder`. So this precedence is
1875
+ // load-bearing on every cohort-too-small pass, not a rare tie-break — and it is what keeps the
1876
+ // ARMING consumer keyed on the reason that means "no cadence can help" rather than on the one
1877
+ // that means "a copy has not arrived yet".
1878
+ // `cohort-too-small` is reported in preference because its remedy is the one that actually
1879
+ // works there: declaring the real size makes the floor reachable, after which the lone peer's
1880
+ // claim IS adopted — so calling it a sole-holder problem would send the operator looking for a
1881
+ // copy they do not need.
1882
+ return cohortTooSmall ? 'cohort-too-small' : 'sole-holder';
1329
1883
  }
1330
1884
 
1331
1885
  /**
@@ -1393,6 +1947,21 @@ export class CoordinatorRepo implements IRepo {
1393
1947
  async pend(request: PendRequest, options?: MessageOptions): Promise<PendResult> {
1394
1948
  const allBlockIds = blockIdsForTransforms(request.transforms);
1395
1949
  await this.verifyResponsibility(allBlockIds);
1950
+ const result = await this.pendThroughCluster(request, allBlockIds, options);
1951
+ // A pend the blocks ACCEPTED is the proof that no reservation is holding them any more — the
1952
+ // only such proof this node gets without asking a question it has no reason to ask.
1953
+ // Scope, honestly: this is NOT what re-arms the line for a later wedge. The holder comparison
1954
+ // in `noteStuckReservation` already does that on its own, and every real holder set is new,
1955
+ // since an action id is 16 random bytes minted per sync cycle (`Collection.syncInternal`). What
1956
+ // forgetting buys is that a settled episode stops occupying an LRU slot it can only use to
1957
+ // evict a live one — plus defence in depth if an action id ever does repeat.
1958
+ if (result.success) this.clearStuckReservations(allBlockIds);
1959
+ return result;
1960
+ }
1961
+
1962
+ /** The cluster half of {@link pend}, after responsibility is verified: consensus, the local-verdict
1963
+ * arms, and the two optimistic-concurrency classifiers a rejection is run through. */
1964
+ private async pendThroughCluster(request: PendRequest, allBlockIds: BlockId[], options?: MessageOptions): Promise<PendResult> {
1396
1965
  const coordinatingBlockIds = options?.coordinatingBlockIds ?? allBlockIds;
1397
1966
 
1398
1967
  const peerCount = await this.coordinator.getClusterSize(coordinatingBlockIds[0]!);
@@ -1602,9 +2171,21 @@ export class CoordinatorRepo implements IRepo {
1602
2171
  }
1603
2172
  }
1604
2173
  if (pending.length === 0) return undefined;
2174
+ // Counted as its own statement, never inside the log payload below: this call is the detection
2175
+ // mechanism, not a formatting step, and payload expressions in this repo are fair game to wrap
2176
+ // in an `enabled` gate (`Collection.advanceContext` does exactly that). A gate added there
2177
+ // later would silently stop the counter and with it `coordinator-repo:stuck-reservation`.
2178
+ // {@link reportRepairDeadlock} keeps its say-once bookkeeping outside its own log call for the
2179
+ // same reason.
2180
+ const distinctRefusedActions = this.noteStuckReservation(pending, request.actionId);
1605
2181
  this.log('coordinator-repo:pend-conflict-classified', {
1606
2182
  actionId: request.actionId,
1607
- rivals: pending.map(p => `${p.blockId}:${p.actionId}`)
2183
+ rivals: pending.map(p => `${p.blockId}:${p.actionId}`),
2184
+ // How many distinct actions the most-refusing of this request's holders has now turned away
2185
+ // (see {@link noteStuckReservation}). Carried on every classification, not just the stuck
2186
+ // ones, so a healthy contended deployment's real figure is readable from its own logs rather
2187
+ // than assumed — which is what calibrates STUCK_RESERVATION_DISTINCT_ACTIONS.
2188
+ distinctRefusedActions
1608
2189
  });
1609
2190
  return {
1610
2191
  success: false,
@@ -1614,6 +2195,108 @@ export class CoordinatorRepo implements IRepo {
1614
2195
  };
1615
2196
  }
1616
2197
 
2198
+ /**
2199
+ * Count one confirmed pending-conflict refusal against the holder(s) of each block it names, and say
2200
+ * ONCE — in words, at the moment it becomes provable — when a block is wedged behind a reservation
2201
+ * that is not going to clear.
2202
+ *
2203
+ * **Why this needs saying at all.** Every individual refusal here is indistinguishable from an
2204
+ * ordinary lost race, which is a normal and healthy event, so the logs of a permanently wedged block
2205
+ * read exactly like the logs of a busy one. Finding the difference today means noticing that the
2206
+ * SAME rival action id keeps appearing across unrelated writers for as long as the process lives —
2207
+ * a pattern nothing points at, and one that cost a downstream project several tickets and weeks to
2208
+ * re-derive from raw traces. The node has the fact in hand at every refusal; this makes it sayable.
2209
+ *
2210
+ * **The signal, and the two things that are NOT the signal.** The discriminator is repetition
2211
+ * against an unchanged holder — see {@link STUCK_RESERVATION_DISTINCT_ACTIONS} for why distinct
2212
+ * refused actions is the right counter and for the measured threshold. Two cheaper-looking tests
2213
+ * were tried and do not work: the members' in-memory reservation table
2214
+ * (`ClusterMember.activeTransactions`) clears the moment a rival's pend reaches consensus, so a
2215
+ * perfectly healthy rival inside its pend-to-commit window is absent from it too and absence there
2216
+ * says nothing; and "the block already passed this pending record's revision" catches a different
2217
+ * orphan class entirely — in the verified instance the wedged block sat at revision 1 while the
2218
+ * orphaned record was for revision 2, still nominally promotable.
2219
+ *
2220
+ * **Never a control path.** This classifies and logs; it never refuses, expires, or deletes
2221
+ * anything. Deciding when a durable pending record may be removed is precisely the hard problem
2222
+ * backlog `debt-unpromotable-pending-records-need-a-sweep` exists for — deleting a live reservation
2223
+ * is worse than the leak — and a counter accurate enough for a log line is not evidence enough to
2224
+ * destroy state.
2225
+ *
2226
+ * Returns the highest distinct-refusal count any of this refusal's blocks has now reached, for the
2227
+ * classification line to carry; it saturates at the threshold once an episode has been reported,
2228
+ * since the ids are dropped at that point.
2229
+ *
2230
+ * NOTE: fed only by {@link classifyPendingConflictRejection}, i.e. by refusals that arrive as a
2231
+ * cohort-wide validator rejection. A block only PART of whose cohort holds the stranded record can
2232
+ * still reach approval super-majority, and its refusal then comes back through the retained local
2233
+ * apply verdict (`getExecutedPendResult`) instead, which this never sees — so a partially wedged
2234
+ * block goes unnamed. That is the weaker condition (the write does land on the healthy members),
2235
+ * and instrumenting the second path would count a refusal that the cohort as a whole did not make.
2236
+ * If partial strands ever turn out to be the common shape in the field, the counter belongs on the
2237
+ * member side (`ClusterMember.validatePendOperations`), where each member sees its own votes.
2238
+ */
2239
+ private noteStuckReservation(pending: ActionPending[], refusedActionId: ActionId): number {
2240
+ const rivalsByBlock = new Map<BlockId, ActionId[]>();
2241
+ for (const { blockId, actionId } of pending) {
2242
+ const rivals = rivalsByBlock.get(blockId);
2243
+ if (rivals) rivals.push(actionId);
2244
+ else rivalsByBlock.set(blockId, [actionId]);
2245
+ }
2246
+ let highest = 0;
2247
+ for (const [blockId, rivals] of rivalsByBlock) {
2248
+ const holders = [...new Set(rivals)].sort();
2249
+ const prior = this.stuckReservations.get(blockId);
2250
+ // A different holder set is a DIFFERENT episode — the block changed hands, which is the
2251
+ // healthy cycle — so the count starts over and the new holder gets its own chance to speak.
2252
+ const watch: StuckReservationWatch = prior !== undefined && sameHolders(prior.holders, holders)
2253
+ ? prior
2254
+ : { holders, refused: new Set<ActionId>(), reported: false };
2255
+ if (watch !== prior) this.stuckReservations.set(blockId, watch);
2256
+ if (watch.reported) {
2257
+ highest = Math.max(highest, STUCK_RESERVATION_DISTINCT_ACTIONS);
2258
+ continue;
2259
+ }
2260
+ watch.refused.add(refusedActionId);
2261
+ highest = Math.max(highest, watch.refused.size);
2262
+ if (watch.refused.size < STUCK_RESERVATION_DISTINCT_ACTIONS) continue;
2263
+ this.log('coordinator-repo:stuck-reservation', {
2264
+ blockId,
2265
+ // The ids an operator needs to grep for and to cancel, kept as data beside the prose so a
2266
+ // log search finds the block and the action without parsing English.
2267
+ holdingActionIds: holders,
2268
+ distinctRefusedActions: watch.refused.size,
2269
+ message: stuckReservationMessage(holders, watch.refused.size)
2270
+ });
2271
+ watch.reported = true;
2272
+ // Said once per episode: from here the flag alone suppresses, and the ids have done their
2273
+ // work (their count is in the line above), so drop them rather than growing a set for the
2274
+ // unbounded remainder of a permanent condition.
2275
+ watch.refused.clear();
2276
+ }
2277
+ return highest;
2278
+ }
2279
+
2280
+ /**
2281
+ * Forget any stuck-reservation episode recorded for these blocks, optionally only when
2282
+ * `holderActionId` is one of the actions that episode named.
2283
+ *
2284
+ * Called from the two events the message itself names as the only cures: a write the block accepted
2285
+ * ({@link pend}), and a cancel for the holding action ({@link cancel}). Forgetting is the whole
2286
+ * effect, and it is LRU hygiene rather than behaviour: a later wedge is named by a DIFFERENT holder,
2287
+ * which `noteStuckReservation`'s holder comparison already treats as a new episode whether the old
2288
+ * entry is still there or not. Both call sites therefore have no observable effect through the
2289
+ * public surface (no spec can distinguish them — reproducing a repeated holder set would mean
2290
+ * reusing an action id, which nothing that mints them does), and both are kept because a settled
2291
+ * episode holding an LRU slot can only evict a live one.
2292
+ */
2293
+ private clearStuckReservations(blockIds: BlockId[], holderActionId?: ActionId): void {
2294
+ for (const blockId of blockIds) {
2295
+ if (holderActionId !== undefined && !this.stuckReservations.peek(blockId)?.holders.includes(holderActionId)) continue;
2296
+ this.stuckReservations.delete(blockId);
2297
+ }
2298
+ }
2299
+
1617
2300
  async cancel(actionRef: ActionBlocks, options?: MessageOptions): Promise<void> {
1618
2301
  const blockIds = actionRef.blockIds;
1619
2302
  await this.verifyResponsibility(blockIds);
@@ -1652,6 +2335,11 @@ export class CoordinatorRepo implements IRepo {
1652
2335
  if (!anyLocalExecuted) {
1653
2336
  await this.storageRepo.cancel(actionRef, options);
1654
2337
  }
2338
+ // The remedy a stuck-reservation line names, actually taken: the record this node was
2339
+ // complaining about is gone, so forget the episode and let a LATER wedge on the same block
2340
+ // speak. Scoped to the cancelled action — a cancel for some OTHER action says nothing about
2341
+ // the reservation that is holding these blocks.
2342
+ this.clearStuckReservations(blockIds, actionRef.actionId);
1655
2343
  } catch (error) {
1656
2344
  this.log('coordinator-repo:cancel-error', { actionId: actionRef.actionId, error: (error as Error).message });
1657
2345
  throw error;
@@ -1696,7 +2384,14 @@ export class CoordinatorRepo implements IRepo {
1696
2384
  // ICommitProofPersister contract; a plain IRepo double ignores the extra argument.
1697
2385
  const proof = await this.localCluster?.mintSoloCommitProof?.(message);
1698
2386
  const result = await (this.storageRepo as IRepo & ICommitProofPersister).commit(request, options, proof);
1699
- if (result.success) this.markBlocksSeen(blockIds);
2387
+ // One self-approval arms the read-repair window only where the DECLARED cohort is also one
2388
+ // — then no rival quorum can exist to be missed. At any larger declared size (including
2389
+ // an undeclared one, which resolves to the replication factor, and including degraded
2390
+ // routing where peerCount is 0) this commit proves nothing about rival quorums — see
2391
+ // commitQuorumRulesOutRivals — so the window stays unarmed and the read path's
2392
+ // solo-self-skip exit re-arms it once per consult instead (which keeps GitHub issue #8's
2393
+ // consult storm bounded at one per window).
2394
+ if (result.success && this.commitQuorumRulesOutRivals(1, peerCount)) this.markBlocksSeen(blockIds);
1700
2395
  return result;
1701
2396
  }
1702
2397
 
@@ -1707,6 +2402,14 @@ export class CoordinatorRepo implements IRepo {
1707
2402
 
1708
2403
  try {
1709
2404
  const { record, localExecuted, localCommitResult } = await this.coordinator.executeClusterTransaction(blockIds[0]!, message, options);
2405
+ // Decided once for every success shape below (local-executed, local fallback, tolerated
2406
+ // divergence): whether this commit's quorum is freshness evidence or merely a commit.
2407
+ // NOTE: one verdict covers every block in `blockIds`, though it is measured against
2408
+ // `blockIds[0]`'s cohort alone. Consistent with the rest of this path — consensus for the
2409
+ // whole commit runs on that one cohort — so a per-block verdict would be measuring a
2410
+ // quorum that never voted. If commits ever coordinate per-block cohorts separately (see
2411
+ // `debt-sender-side-coordinating-block-binding-is-unchecked`), this must follow them.
2412
+ const armFreshness = this.commitQuorumRulesOutRivals(countApprovingCommitVotes(record), peerCount);
1710
2413
  if (localExecuted) {
1711
2414
  // Our own member applied this commit during consensus. Its retained storage verdict is
1712
2415
  // the one honest signal we have about durability: the member-side apply tolerates an
@@ -1740,7 +2443,7 @@ export class CoordinatorRepo implements IRepo {
1740
2443
  reason: localCommitResult.reason
1741
2444
  });
1742
2445
  }
1743
- this.markBlocksSeen(blockIds);
2446
+ if (armFreshness) this.markBlocksSeen(blockIds);
1744
2447
  return { success: true };
1745
2448
  }
1746
2449
  // Local cluster didn't execute during consensus. Attempt a local commit, but tolerate
@@ -1769,16 +2472,16 @@ export class CoordinatorRepo implements IRepo {
1769
2472
  try {
1770
2473
  const result = await (this.storageRepo as IRepo & ICommitProofPersister).commit(request, options, consensusProof);
1771
2474
  if (result.success) {
1772
- this.markBlocksSeen(blockIds);
2475
+ if (armFreshness) this.markBlocksSeen(blockIds);
1773
2476
  return result;
1774
2477
  }
1775
2478
  if (isMissingBaseRevisionFailure(result) && clusterReachedCommitConsensus(record)) {
1776
- return this.tolerateLocalCommitDivergence(request, blockIds, result.reason ?? MISSING_BASE_REVISION_REASON);
2479
+ return this.tolerateLocalCommitDivergence(request, blockIds, result.reason ?? MISSING_BASE_REVISION_REASON, armFreshness);
1777
2480
  }
1778
2481
  return result;
1779
2482
  } catch (err) {
1780
2483
  if (clusterReachedCommitConsensus(record)) {
1781
- return this.tolerateLocalCommitDivergence(request, blockIds, (err as Error).message);
2484
+ return this.tolerateLocalCommitDivergence(request, blockIds, (err as Error).message, armFreshness);
1782
2485
  }
1783
2486
  throw err;
1784
2487
  }
@@ -1917,13 +2620,17 @@ export class CoordinatorRepo implements IRepo {
1917
2620
  }
1918
2621
 
1919
2622
  /**
1920
- * Report success for a commit the cluster carried but this peer could not apply locally. The
1921
- * blocks are marked seen so the read path treats them as freshness-checked; convergence comes
1922
- * from replication (cohort reconcile, or read-driven acquisition), not from replay here.
2623
+ * Report success for a commit the cluster carried but this peer could not apply locally.
2624
+ * Convergence comes from replication (cohort reconcile, or read-driven acquisition), not from
2625
+ * replay here. `armFreshness` says whether the commit's quorum was strong enough
2626
+ * ({@link commitQuorumRulesOutRivals}) for the read path to treat the blocks as
2627
+ * freshness-checked; a divergence tolerated on a downsized quorum leaves the window unarmed —
2628
+ * this peer is known to be behind here, the last place a self-referential freshness stamp
2629
+ * belongs.
1923
2630
  */
1924
- private tolerateLocalCommitDivergence(request: CommitRequest, blockIds: BlockId[], detail: string): CommitResult {
2631
+ private tolerateLocalCommitDivergence(request: CommitRequest, blockIds: BlockId[], detail: string, armFreshness: boolean): CommitResult {
1925
2632
  this.log('coordinator-repo:commit-local-failed-cluster-succeeded', { actionId: request.actionId, error: detail });
1926
- this.markBlocksSeen(blockIds);
2633
+ if (armFreshness) this.markBlocksSeen(blockIds);
1927
2634
  return { success: true };
1928
2635
  }
1929
2636
  }
@@ -1932,6 +2639,10 @@ export class CoordinatorRepo implements IRepo {
1932
2639
  function clusterReachedCommitConsensus(record: ClusterRecord): boolean {
1933
2640
  const peerCount = Object.keys(record.peers).length;
1934
2641
  if (peerCount === 0) return false;
1935
- const approvedCommits = Object.values(record.commits).filter(s => s.type === 'approve').length;
1936
- return approvedCommits > peerCount / 2;
2642
+ return countApprovingCommitVotes(record) > peerCount / 2;
2643
+ }
2644
+
2645
+ /** Approve-typed commit votes on a consensus record — the numerator `commitQuorumRulesOutRivals` measures against the full cohort. */
2646
+ function countApprovingCommitVotes(record: ClusterRecord): number {
2647
+ return Object.values(record.commits).filter(s => s.type === 'approve').length;
1937
2648
  }