@optimystic/db-p2p 1.0.0-beta.2 → 1.0.0-beta.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/dist/src/cluster/cluster-repo.d.ts +46 -11
  2. package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
  3. package/dist/src/cluster/cluster-repo.js +177 -101
  4. package/dist/src/cluster/cluster-repo.js.map +1 -1
  5. package/dist/src/libp2p-key-network.d.ts.map +1 -1
  6. package/dist/src/libp2p-key-network.js +7 -0
  7. package/dist/src/libp2p-key-network.js.map +1 -1
  8. package/dist/src/logger.d.ts.map +1 -1
  9. package/dist/src/logger.js +13 -6
  10. package/dist/src/logger.js.map +1 -1
  11. package/dist/src/repo/cluster-coordinator.d.ts +42 -0
  12. package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
  13. package/dist/src/repo/cluster-coordinator.js +50 -9
  14. package/dist/src/repo/cluster-coordinator.js.map +1 -1
  15. package/dist/src/repo/coordinator-repo.d.ts +74 -7
  16. package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
  17. package/dist/src/repo/coordinator-repo.js +290 -55
  18. package/dist/src/repo/coordinator-repo.js.map +1 -1
  19. package/dist/src/storage/storage-repo.d.ts +15 -0
  20. package/dist/src/storage/storage-repo.d.ts.map +1 -1
  21. package/dist/src/storage/storage-repo.js +28 -0
  22. package/dist/src/storage/storage-repo.js.map +1 -1
  23. package/dist/src/testing/mesh-harness.d.ts +15 -0
  24. package/dist/src/testing/mesh-harness.d.ts.map +1 -1
  25. package/dist/src/testing/mesh-harness.js +21 -4
  26. package/dist/src/testing/mesh-harness.js.map +1 -1
  27. package/package.json +2 -2
  28. package/src/cluster/cluster-repo.ts +2749 -2671
  29. package/src/libp2p-key-network.ts +7 -0
  30. package/src/logger.ts +14 -6
  31. package/src/repo/cluster-coordinator.ts +1170 -1113
  32. package/src/repo/coordinator-repo.ts +2940 -2687
  33. package/src/storage/storage-repo.ts +30 -0
  34. package/src/testing/mesh-harness.ts +37 -4
@@ -7,7 +7,7 @@ import { quorumSize, corroboratorCapacity, selectQuorumRev, certifiedEquivocatio
7
7
  import { certifyClaim, isAttributableProofFailure, proofThresholds } from "../cluster/certified-claims.js";
8
8
  import { DEFAULT_CLUSTER_SIZE, resolveRepairCorroborationClusterSize } from "../cluster/cluster-policy.js";
9
9
  import { RECONCILE_TIMEOUT_MS } from "../cluster/reconcile-block.js";
10
- import { isMissingBaseRevisionFailure, MISSING_BASE_REVISION_REASON } from "../storage/storage-repo.js";
10
+ import { isMissingBaseRevisionFailure, COMMIT_NOT_DURABLE_REASON, MISSING_BASE_REVISION_REASON } from "../storage/storage-repo.js";
11
11
  import { buildBlockCommitProof } from "../cluster/commit-proof.js";
12
12
  /** How long one cohort peer gets to answer the latest-revision consult before it counts as silent. */
13
13
  const LATEST_QUERY_TIMEOUT_MS = 1000;
@@ -166,6 +166,31 @@ export class CoordinatorRepo {
166
166
  responsibilityCache = new LruMap(1000);
167
167
  static RESPONSIBILITY_TTL_MS = 60_000;
168
168
  lastSeenCommitMs = new LruMap(1000);
169
+ /**
170
+ * Per block, when (`this.now()` at the consult's start) a consult last SETTLED that this node's
171
+ * absence of it is the cohort's answer too — see `absenceSettled` on {@link fetchBlockFromCluster}'s
172
+ * result. Only the solo-self exit settles today: when this node is the block's whole cohort, every
173
+ * acknowledged commit of it lands in this node's storage before the writer hears success, so the
174
+ * block reads present and the memo dies. Read only for a block still missing locally
175
+ * ({@link absenceIsSettled}); cleared by any pend or commit this node coordinates for the block, by
176
+ * seeing it present, and by any later consult of it that does not settle.
177
+ *
178
+ * Deliberately its OWN map rather than `lastSeenCommitMs`: that one is also stamped for missing
179
+ * blocks at exits whose verdict is NOT a confirmed absence (the `cohort-too-small` arm when a peer
180
+ * claimed a revision, the post-restore arm when acquisition failed), so reading it here would turn
181
+ * a `claimed-elsewhere` absence into an authoritative one for a whole window.
182
+ *
183
+ * NOTE: accepted tradeoff — a cohort-of-one view can be transient (peers still mid-identify; see
184
+ * the solo exit), and a block another node creates through the cohort it sees is then reported
185
+ * absent here for up to one `readRepairWindowMs` after this node's view grows — the same one-window
186
+ * cost the held-block window pays for a growing cohort. Revisit if a caller ever needs
187
+ * create-visibility across coordinators tighter than one window — such a caller needs a revision
188
+ * floor (backlog `feat-refresh-can-demand-a-revision-floor`), not a shorter window.
189
+ *
190
+ * NOTE: LRU-bounded like its siblings; an eviction under >1000 settled blocks loses a memo and
191
+ * costs one extra consult — the safe direction.
192
+ */
193
+ settledAbsences = new LruMap(1000);
169
194
  /** Per block, what earlier repair passes left unresolved — see {@link AheadClaimState}.
170
195
  * Outlives the consult on purpose: the read-repair window skips consults for blocks checked
171
196
  * recently, and a doubt dropped there is a stale answer served as confirmed again.
@@ -343,6 +368,12 @@ export class CoordinatorRepo {
343
368
  // are meant to be — a rare degradation during routing churn — since routing already placed this
344
369
  // node near the block. If they ever become routine, gate acquisition (not the serve itself) on
345
370
  // isResponsibleForBlock.
371
+ //
372
+ // NOTE: accepted tradeoff — this check and `fetchBlockFromCluster` below each run their own
373
+ // `findCluster` for the same block, so a cold read costs two cohort lookups where one shared
374
+ // lookup would do. Declined 2026-09-11: at 0.009 ms per solo lookup there is nothing to win,
375
+ // and this check's future is open (blocked `writer-and-servers-disagree-on-where-a-block-lives`,
376
+ // option D2(b)). Revisit if a device profile shows `findCluster` as material.
346
377
  for (const blockId of blockGets.blockIds) {
347
378
  if (!await this.isResponsibleForBlock(blockId)) {
348
379
  this.log('proximity:get-warning', { blockId, msg: 'serving read for non-responsible block' });
@@ -351,7 +382,9 @@ export class CoordinatorRepo {
351
382
  // First try local storage
352
383
  const localResult = await this.storageRepo.get(blockGets, options);
353
384
  // Decide per-block whether to consult cluster peers. Two triggers:
354
- // (a) Missing — block isn't present locally at all (legacy behavior).
385
+ // (a) Missing — block isn't present locally at all. Consults unless an earlier consult
386
+ // SETTLED its absence within the last `readRepairWindowMs` (see `settledAbsences` and
387
+ // `absenceIsSettled`) — today only when this node is the block's whole cohort.
355
388
  // (b) Stale-by-policy — block is present but read-repair policy says verify.
356
389
  // Skip cluster fetch if this is already a sync request (to prevent recursive queries).
357
390
  // A sync read is also never marked `unavailable` here — the consult it skips is the
@@ -361,21 +394,38 @@ export class CoordinatorRepo {
361
394
  const skipClusterFetch = options?.skipClusterFetch;
362
395
  // NOTE: NetworkTransactor.get treats an authoritative "absent" ({ state: {} })
363
396
  // as final and no longer retries it (ticket txn-perf-authoritative-notfound),
364
- // relying on this cluster reconciliation to have already run. When the consult
365
- // FAILS outright or runs without ruling the block out and the block stays
366
- // missing the entry is flagged `unavailable` below with a reason naming what
367
- // the consult established (see AbsenceVerdict and the mapping in the loop body),
368
- // which re-enables the transactor-level retry against a different peer. If a
369
- // coordinator is configured WITHOUT clusterLatestCallback, there is no cohort to
370
- // consult and the local answer IS the whole truth it stays authoritative, with
371
- // no flag and no transactor-level retry to compensate. That is fine (such a
372
- // coordinator has no cluster to reconcile against), but keep this coupling in
373
- // mind if a partial-cluster read path is added.
397
+ // relying on this cluster reconciliation to have already run within the last
398
+ // `readRepairWindowMs`: an unflagged absent means the cohort confirmed this absence
399
+ // at most one window ago (a read inside the window serves the memo of that
400
+ // confirmation, see `settledAbsences`), the same currency guarantee a held block's
401
+ // content already carries. When the consult FAILS outright or runs without ruling
402
+ // the block out and the block stays missing — the entry is flagged `unavailable`
403
+ // below with a reason naming what the consult established (see AbsenceVerdict and
404
+ // the mapping in the loop body), which re-enables the transactor-level retry against
405
+ // a different peer; none of those outcomes is ever remembered. If a coordinator is
406
+ // configured WITHOUT clusterLatestCallback, there is no cohort to consult and the
407
+ // local answer IS the whole truth — it stays authoritative, with no flag and no
408
+ // transactor-level retry to compensate. That is fine (such a coordinator has no
409
+ // cluster to reconcile against), but keep this coupling in mind if a partial-cluster
410
+ // read path is added.
374
411
  if (this.clusterLatestCallback && !skipClusterFetch) {
375
412
  for (const blockId of blockGets.blockIds) {
376
413
  const localEntry = localResult[blockId];
377
414
  const localRev = localEntry?.state?.latest?.rev;
378
415
  const isMissing = !localEntry?.state?.latest;
416
+ if (!isMissing) {
417
+ // Present: any remembered absence is dead. Dropped now rather than left to age out,
418
+ // so a block that vanishes again inside the same window consults instead of being
419
+ // served the old memo.
420
+ this.settledAbsences.delete(blockId);
421
+ }
422
+ else if (this.absenceIsSettled(blockId)) {
423
+ // An earlier consult settled this absence within one window: serve the local entry
424
+ // as it stands — an authoritative absent, or pending-only content — with no flag and
425
+ // no log line. The held-block skip below logs nothing either, and a per-read line
426
+ // here would recreate the volume this skip exists to remove.
427
+ continue;
428
+ }
379
429
  const isStale = !isMissing && this.shouldReadRepair(blockId);
380
430
  if (!isMissing && !isStale) {
381
431
  // No consult this pass — the read-repair window says this block was checked
@@ -395,13 +445,33 @@ export class CoordinatorRepo {
395
445
  localRev
396
446
  });
397
447
  }
448
+ // Evidence gathered during the consult is at least this old, so a memo stamped with it never
449
+ // outlives "confirmed within one window" by the consult's own duration.
450
+ const consultStartedAt = this.now();
398
451
  try {
399
- const { absence, currency } = await this.fetchBlockFromCluster(blockId, blockGets.context, localRev);
452
+ const { absence, currency, absenceSettled } = await this.fetchBlockFromCluster(blockId, blockGets.context, localRev);
400
453
  const refreshed = await this.storageRepo.get({ blockIds: [blockId], context: blockGets.context }, options);
401
454
  const newRev = refreshed[blockId]?.state?.latest?.rev;
402
455
  if (refreshed[blockId]) {
403
456
  localResult[blockId] = refreshed[blockId];
404
457
  }
458
+ // Remember a settled absence for one window; forget it after every other outcome. A
459
+ // consult that ended `unconfirmed`, `isolated` or `claimed`, that asked a multi-peer
460
+ // cohort, or that rested on no real cohort view, leaves no memo, so the next read consults (and flags) exactly as before —
461
+ // including after a sampled or paranoid re-consult of a block whose memo was still fresh.
462
+ // NOTE: a local pend/commit that clears the memo WHILE this consult is in flight is undone
463
+ // by the stamp below — one read inside the window can then serve an absent the refused
464
+ // writer knows is wrong. Bounded: that writer's next pend clears the memo again. If it ever
465
+ // shows as a stuck writer, skip this stamp when a clear of this block landed after
466
+ // `consultStartedAt`.
467
+ if (isMissing) {
468
+ if (!refreshed[blockId]?.state?.latest && absence === 'confirmed' && absenceSettled) {
469
+ this.settledAbsences.set(blockId, consultStartedAt);
470
+ }
471
+ else {
472
+ this.settledAbsences.delete(blockId);
473
+ }
474
+ }
405
475
  if (isStale) {
406
476
  if (typeof newRev === 'number' && typeof localRev === 'number' && newRev > localRev) {
407
477
  this.log('cluster-tx:read-repair-applied', { blockId, oldRev: localRev, newRev });
@@ -453,6 +523,9 @@ export class CoordinatorRepo {
453
523
  // rather than return a stale cohort view, revisit — that would put the
454
524
  // isolated case back under this vaguer reason.
455
525
  if (isMissing) {
526
+ // A consult that threw settled nothing — not even a memo that was still fresh
527
+ // before a sampled or paranoid re-consult.
528
+ this.settledAbsences.delete(blockId);
456
529
  this.flagUnconfirmedAbsence(localResult, blockId, 'peers-unreachable');
457
530
  }
458
531
  else {
@@ -662,6 +735,28 @@ export class CoordinatorRepo {
662
735
  }
663
736
  }
664
737
  }
738
+ /**
739
+ * Decide whether a block MISSING locally may skip its consult: an earlier consult settled its
740
+ * absence within one read-repair window (see {@link settledAbsences}). The absence counterpart of
741
+ * {@link shouldReadRepair}, giving absence the guarantee the window already gives content —
742
+ * "checked with the cohort within one window" — and nothing weaker.
743
+ *
744
+ * Modes: `paranoid` never skips ("verify every read" means every read). `lazy` uses the window and
745
+ * the sample rate, exactly as {@link shouldReadRepair} does. `off` uses the window but not the
746
+ * sample rate (a `lazy`-only knob, as it is for content): `off` disables STALE-CONTENT repair, and
747
+ * the absence consult is not that — it is what makes an absent answer authoritative at all — so
748
+ * leaving it unbounded would make the mode meant to do less network work do more.
749
+ */
750
+ absenceIsSettled(blockId) {
751
+ if (this.readRepairMode === 'paranoid')
752
+ return false;
753
+ const at = this.settledAbsences.get(blockId);
754
+ if (at == null || this.now() - at > this.readRepairWindowMs)
755
+ return false;
756
+ if (this.readRepairMode === 'lazy' && this.readRepairSampleRate > 0 && this.rand() < this.readRepairSampleRate)
757
+ return false;
758
+ return true;
759
+ }
665
760
  /** Milliseconds since we last marked this block fresh, or undefined if never. */
666
761
  ageMs(blockId) {
667
762
  const lastSeen = this.lastSeenCommitMs.get(blockId);
@@ -674,6 +769,16 @@ export class CoordinatorRepo {
674
769
  this.lastSeenCommitMs.set(id, now);
675
770
  }
676
771
  }
772
+ /**
773
+ * Drop any settled-absence memo for `blockIds` ({@link settledAbsences}) — called at the top of a
774
+ * local pend or commit, before routing and whatever the outcome. Clearing is always the safe
775
+ * direction: the most it can cost is one extra consult.
776
+ */
777
+ forgetSettledAbsences(blockIds) {
778
+ for (const id of blockIds) {
779
+ this.settledAbsences.delete(id);
780
+ }
781
+ }
677
782
  /**
678
783
  * True when a commit's approve votes form a strict majority of the FULL cohort — the only case
679
784
  * where "this node committed" is evidence that no rival commit moved past it. Two strict
@@ -733,10 +838,20 @@ export class CoordinatorRepo {
733
838
  * retire it when the memo's claimants are among the peers that answered), `no-evidence`
734
839
  * leaves it standing untouched. Required, not optional, so an exit added later has to say
735
840
  * which it means.
841
+ * - `absenceSettled` — whether this pass SETTLED the block's absence for one read-repair window:
842
+ * no write of it this node has not seen can exist. Only the solo-self exit can say that today
843
+ * (this node is the whole cohort, so every acknowledged commit lands here). `get` remembers a
844
+ * settled absence ({@link settledAbsences}) and skips the consult while the block stays missing.
845
+ * `absence === 'confirmed'` alone is not enough — the empty-cohort exit reports it without
846
+ * asking anyone, and a multi-peer cohort's unanimous "nothing" can be overtaken by a commit
847
+ * still on its way to this node. Required, like
848
+ * `currency`, so an exit added later has to say which side it falls on rather than
849
+ * remembering (or forgetting) to arm something.
736
850
  */
737
851
  async fetchBlockFromCluster(blockId, context, localRev) {
852
+ // Unreachable from `get` (it guards on the callback); nothing to settle against.
738
853
  if (!this.clusterLatestCallback)
739
- return { absence: 'confirmed', currency: { kind: 'no-evidence' } };
854
+ return { absence: 'confirmed', currency: { kind: 'no-evidence' }, absenceSettled: false };
740
855
  const blockIdBytes = new TextEncoder().encode(blockId);
741
856
  const peers = await this.keyNetwork.findCluster(blockIdBytes);
742
857
  const peerIds = peers ? Object.keys(peers) : [];
@@ -747,9 +862,11 @@ export class CoordinatorRepo {
747
862
  // here would suppress a genuine repair for a whole `readRepairWindowMs` after a transient
748
863
  // blip, and re-entering costs no network work beyond the `findCluster` the read already
749
864
  // makes. Do not "fix" this by symmetry with the solo-self exit.
865
+ // For the same reason it returns `absenceSettled: false`: remembering the absence would serve an
866
+ // authoritative absent for a whole window on the strength of a lookup that returned nobody.
750
867
  // Currency: nobody was asked, so nothing was refuted — an earlier pass's unsettled claim stands.
751
868
  if (peerIds.length === 0)
752
- return { absence: 'confirmed', currency: { kind: 'no-evidence' } };
869
+ return { absence: 'confirmed', currency: { kind: 'no-evidence' }, absenceSettled: false };
753
870
  // Solo-cluster short-circuit: the only responsible peer is us. There is no
754
871
  // remote to sync from, so skip the callback entirely. Querying ourselves
755
872
  // would dial self via SyncClient — pointless at best, and on nodes without
@@ -782,7 +899,11 @@ export class CoordinatorRepo {
782
899
  // and deliberate — the window damps repair EFFORT, not honesty — and it is the same
783
900
  // coupling the comment at the final exit below describes. Arming the window and keeping
784
901
  // the memo are answers to different questions; do not collapse them.
785
- return { absence: 'confirmed', currency: { kind: 'no-evidence' } };
902
+ // Absence: settled. Nobody else could hold the block, and re-asking inside one window learns
903
+ // nothing the next `findCluster` would not — the same argument, with the same one-window
904
+ // self-heal when a cohort appears, as the arming above. (That arming's stamp is NOT what
905
+ // suppresses the next read of a missing block: `get` reads `settledAbsences` for those.)
906
+ return { absence: 'confirmed', currency: { kind: 'no-evidence' }, absenceSettled: true };
786
907
  }
787
908
  const { corroborated, corroboration, local, silent, answered, claims, uncorroboratedRev, deadlock } = await this.queryClusterForLatest(peerIds, blockId, context);
788
909
  // Any silence taints the WHOLE consult, not a fraction of it (fail-closed): one silent
@@ -870,7 +991,14 @@ export class CoordinatorRepo {
870
991
  if (deadlock === 'cohort-too-small') {
871
992
  this.markBlocksSeen([blockId]);
872
993
  }
873
- return { absence, currency };
994
+ // Absence NOT settled, even when every cohort member answered "I hold nothing". This node
995
+ // takes part in other coordinators' writes as a cohort member through ClusterRepo, straight
996
+ // into storage and never through this class's pend/commit, so nothing here would clear the
997
+ // memo; and a commit acknowledged at super-majority reaches the remaining members in the
998
+ // background. Remembering this answer served a writer's own create as absent through a
999
+ // lagging member (fresh-node-ddl-multi Scenario B, 5 of 20 runs). Only the solo exit
1000
+ // settles — see backlog `feat-a-cohort-member-remembers-a-settled-absence`.
1001
+ return { absence, currency, absenceSettled: false };
874
1002
  }
875
1003
  // The self answer is the sharper baseline (same storage, same context, read alongside the
876
1004
  // cohort's), but it exists only when `findCluster` returned this node. A soft serve for a
@@ -921,7 +1049,8 @@ export class CoordinatorRepo {
921
1049
  // ahead. Only reachable when a peer answered (a corroboration requires claims), so the
922
1050
  // shared verdict resolves to `nothing-ahead` and a memo whose claimants are among those
923
1051
  // answers is cleared.
924
- return { absence: silenceVerdict, currency: nothingAheadVerdict };
1052
+ // `absenceSettled`: never read for a held block; false for the same reason as the exit above.
1053
+ return { absence: silenceVerdict, currency: nothingAheadVerdict, absenceSettled: false };
925
1054
  }
926
1055
  // Corroborated revision is ahead of ours — converge onto it.
927
1056
  const rev = await this.restoreCorroborated(blockId, corroborated, baselineRev, peerIds);
@@ -949,12 +1078,13 @@ export class CoordinatorRepo {
949
1078
  // unsettled claim (`recordAheadClaim`) and keeps stamping reads served below it while the
950
1079
  // window suppresses the retry — the window damps repair effort, not honesty.
951
1080
  // NOTE: that damping covers only a block this node holds at an OLDER revision. A block entirely
952
- // missing locally never consults the window (`get` triggers on `isMissing` before
953
- // `shouldReadRepair`), so a persistently failing acquisition e.g. a two-node deployment that
954
- // never set `assumedClusterSize`, where the content quorum can never be met — re-fetches an
955
- // archive on every read of that block. Correct, and self-limiting once the cohort can agree; if
956
- // it ever shows as read amplification, gate the acquisition step (not the latest-query) on the
957
- // same window rather than widening `isMissing`.
1081
+ // missing locally is suppressed only by its own absence memo (`settledAbsences`), and this exit
1082
+ // never arms it (`absenceSettled: false` below a failed acquisition is `claimed`). So a
1083
+ // persistently failing acquisition e.g. a two-node deployment that never set
1084
+ // `assumedClusterSize`, where the content quorum can never be met still re-fetches an archive
1085
+ // on every read of that block. Correct, and self-limiting once the cohort can agree; if it ever
1086
+ // shows as read amplification, gate the acquisition step (not the latest-query) on the same
1087
+ // window rather than letting a `claimed` absence settle.
958
1088
  this.markBlocksSeen([blockId]);
959
1089
  // Converged: the corroboration is itself the evidence that nothing is ahead, and it came from
960
1090
  // peers that answered — the shared verdict resolves to `nothing-ahead`, and the memo retires
@@ -962,7 +1092,9 @@ export class CoordinatorRepo {
962
1092
  const currency = converged
963
1093
  ? nothingAheadVerdict
964
1094
  : { kind: 'unsettled-claim', rev: corroborated.rev, claimants: claimantsAtOrAbove(corroborated.rev), silent };
965
- return { absence, currency };
1095
+ // Never settles an absence: restored → the block is now present (nothing to remember); not
1096
+ // restored → `claimed`.
1097
+ return { absence, currency, absenceSettled: false };
966
1098
  }
967
1099
  /**
968
1100
  * Bring this node up to the cohort-corroborated `corroborated`, returning the revision it holds
@@ -1500,6 +1632,12 @@ export class CoordinatorRepo {
1500
1632
  }
1501
1633
  async pend(request, options) {
1502
1634
  const allBlockIds = blockIdsForTransforms(request.transforms);
1635
+ // Forget any settled absence of these blocks FIRST — before responsibility, routing, or the
1636
+ // outcome is known. A pend refused because a block already exists somewhere is the strongest
1637
+ // evidence there is that the memo was wrong, and the writer's retry re-reads: that read must
1638
+ // consult (and restore) rather than serve the memo's authoritative absent for the rest of the
1639
+ // window. After a success the block is present and the memo is dead anyway.
1640
+ this.forgetSettledAbsences(allBlockIds);
1503
1641
  await this.verifyResponsibility(allBlockIds);
1504
1642
  const result = await this.pendThroughCluster(request, allBlockIds, options);
1505
1643
  // A pend the blocks ACCEPTED is the proof that no reservation is holding them any more — the
@@ -1914,11 +2052,12 @@ export class CoordinatorRepo {
1914
2052
  //
1915
2053
  // NOTE: `getClusterSize` is a second `findCluster` for the same key that
1916
2054
  // `executeClusterTransaction` is about to look up again, so a cancel over N blocks now
1917
- // costs 2N cohort lookups instead of N. Same shape `pend` and `commit` already pay, but
1918
- // they pay it once (they only ever consult `blockIds[0]`) where this scales with N. Fine
1919
- // while cancels span a handful of blocks; if wide multi-block cancels ever show up hot,
1920
- // have `executeClusterTransaction` return the cohort it already fetched (or own the
1921
- // short-circuit itself) rather than adding a cache here.
2055
+ // costs 2N cohort lookups instead of N. `pend` and `commit` pay the same doubled lookup
2056
+ // for a multi-peer cohort, but theirs is CONSTANT — they only ever consult `blockIds[0]`
2057
+ // — where this scales with N. Fine while cancels span a handful of blocks; if wide
2058
+ // multi-block cancels ever show up hot, have `executeClusterTransaction` return the
2059
+ // cohort it already fetched (or own the short-circuit itself) rather than adding a cache
2060
+ // here.
1922
2061
  const results = await Promise.all(blockIds.map(async (blockId) => {
1923
2062
  const peerCount = await this.coordinator.getClusterSize(blockId);
1924
2063
  if (peerCount <= 1)
@@ -1944,6 +2083,8 @@ export class CoordinatorRepo {
1944
2083
  }
1945
2084
  async commit(request, options) {
1946
2085
  const blockIds = request.blockIds;
2086
+ // Same as `pend`: forget any settled absence of these blocks before routing, whatever the outcome.
2087
+ this.forgetSettledAbsences(blockIds);
1947
2088
  await this.verifyResponsibility(blockIds);
1948
2089
  const cohortPeerIds = await this.coordinator.getClusterPeerIds(blockIds[0]);
1949
2090
  const peerCount = cohortPeerIds.length;
@@ -1995,7 +2136,7 @@ export class CoordinatorRepo {
1995
2136
  expiration: options?.expiration ?? Date.now() + this.DEFAULT_TIMEOUT
1996
2137
  };
1997
2138
  try {
1998
- const { record, localExecuted, localCommitResult } = await this.coordinator.executeClusterTransaction(blockIds[0], message, options);
2139
+ const { record, localExecuted, localCommitResult, cohortCommitOutcomes } = await this.coordinator.executeClusterTransaction(blockIds[0], message, options);
1999
2140
  // Decided once for every success shape below (local-executed, local fallback, tolerated
2000
2141
  // divergence): whether this commit's quorum is freshness evidence or merely a commit.
2001
2142
  // NOTE: one verdict covers every block in `blockIds`, though it is measured against
@@ -2004,20 +2145,48 @@ export class CoordinatorRepo {
2004
2145
  // quorum that never voted. If commits ever coordinate per-block cohorts separately (see
2005
2146
  // `debt-sender-side-coordinating-block-binding-is-unchecked`), this must follow them.
2006
2147
  const armFreshness = this.commitQuorumRulesOutRivals(countApprovingCommitVotes(record), peerCount);
2148
+ // THE DURABILITY GATE. A commit is acknowledged to the writer only when more than half of
2149
+ // the cohort it ran on (`record.peers`) reports that its storage durably holds the
2150
+ // committed revision under this action, each member measured after its own reconcile.
2151
+ // Consensus votes are necessary for the commit to be authoritative; they were never
2152
+ // evidence that it was stored — every member can sign, then refuse at apply (no base for
2153
+ // the block, a missed pend), and the coordinator used to count the votes as success. The
2154
+ // same strict majority `clusterReachedCommitConsensus` measures votes against; the
2155
+ // freshness verdict above stays tied to the vote count, the two quorums are not
2156
+ // conflated. Other members' reports arrive as `cohortCommitOutcomes`; this node's own
2157
+ // member is added per arm below — its retained verdict when it executed, its fallback
2158
+ // commit when it did not.
2159
+ //
2160
+ // NOTE: a refusal here means "not confirmed durable at a quorum", never "guaranteed
2161
+ // absent" — the two-phase ambiguity. Members that refused with `missing-base-revision`
2162
+ // have already dropped their pending records, so the writer must cancel and re-drive at a
2163
+ // fresh revision, which is what the `conflict: true` shape makes `Collection.syncAttempts`
2164
+ // and the multi-collection `pendPhase` do. But a member reached only by
2165
+ // `scheduleCommitRetry` can still land the refused revision later, and this node's own
2166
+ // member (or its fallback commit) may hold it already. The writer's retry with the SAME
2167
+ // action id converges either way: `isOwnRevision` in `StorageRepo.pend` and `commit`
2168
+ // treats an already-landed own revision as satisfied, and `inFlightActionId` in
2169
+ // `Collection.updateInternal` keeps the retry on the same action.
2170
+ const durability = cohortDurability(record, cohortCommitOutcomes);
2007
2171
  if (localExecuted) {
2008
2172
  // Our own member applied this commit during consensus. Its retained storage verdict is
2009
2173
  // the one honest signal we have about durability: the member-side apply tolerates an
2010
- // "ahead" refusal as divergence (see the NOTE in ClusterMember.applyConsensusOperation),
2174
+ // "ahead" refusal as divergence (see the NOTE in ClusterMember.applyCommitToStorage),
2011
2175
  // which is correct for a redelivered or lagging commit — but when the refusal's real
2012
- // cause is a RIVAL action holding the requested revision, that tolerance turns a commit
2013
- // no member durably stored into a fabricated success. This is the
2176
+ // cause is a RIVAL action holding the requested revision, that tolerance would turn a
2177
+ // commit no member durably stored into a fabricated success. This is the
2014
2178
  // signed-but-not-yet-applied window: two commits for one revision can BOTH assemble
2015
2179
  // consensus when every member signs the second after signing (but before applying) the
2016
2180
  // first, because signing drops the member's reservation. Confirm the rival against local
2017
2181
  // storage (never the verdict's prose) and answer the writer with a retryable conflict so
2018
- // it re-drives at a fresh revision. Own-action or unconfirmed refusals keep the
2019
- // prior fabricated-success shape: consensus is authoritative and this member converges
2020
- // via replication.
2182
+ // it re-drives at a fresh revision. An own-action confirmation only clears the conflict
2183
+ // answer; it does NOT make this node a holder for the gate below. The confirmation fires
2184
+ // on the FIRST block found held under this action, and a multi-block commit torn locally
2185
+ // (`StorageRepo.commit` lands blocks in order and stops at the first refusal) holds some
2186
+ // blocks and not others — whereas the member's retained verdict is already a success
2187
+ // whenever EVERY block is held (`ClusterMember.durableCommitVerdict`, measured after its
2188
+ // reconcile). So the retained verdict is the only local contribution the gate counts;
2189
+ // a refused one, confirmed or not, leaves it to the rest of the cohort.
2021
2190
  //
2022
2191
  // NOTE: a CONFIRMED rival is trusted over the consensus outcome here. That is right in
2023
2192
  // the window this closes (the cohort refused the loser too), but it inverts if the two
@@ -2028,6 +2197,7 @@ export class CoordinatorRepo {
2028
2197
  // after a fork; that is partition-healing scope (docs/partition-healing.md). If forks
2029
2198
  // are ever observed here, weigh the retained verdict against the cohort's votes instead
2030
2199
  // of trusting the local re-read alone.
2200
+ const localDurable = localCommitResult?.success === true;
2031
2201
  if (localCommitResult !== undefined && !localCommitResult.success) {
2032
2202
  const rival = await this.confirmCommitRivalAgainstLocal(request);
2033
2203
  if (typeof rival === 'object')
@@ -2038,23 +2208,42 @@ export class CoordinatorRepo {
2038
2208
  reason: localCommitResult.reason
2039
2209
  });
2040
2210
  }
2211
+ // An absent verdict (a member that predates retention, a restart, the TTL) is not
2212
+ // evidence of holding anything: it is simply not counted.
2213
+ const durableHolders = durability.remoteHolders + (localDurable ? 1 : 0);
2214
+ if (!isDurableMajority(durableHolders, durability.cohortSize)) {
2215
+ return this.refuseCommitNotDurable(request, durableHolders, durability, 'local-executed');
2216
+ }
2041
2217
  if (armFreshness)
2042
2218
  this.markBlocksSeen(blockIds);
2043
2219
  return { success: true };
2044
2220
  }
2045
2221
  // Local cluster didn't execute during consensus. Attempt a local commit, but tolerate
2046
- // local divergence when the cluster already reached consensus this coordinator was
2047
- // likely picked for commit after missing the pend phase (unreachable during pend, fresh
2048
- // join, etc.). The cluster's majority is authoritative; this peer catches up via sync.
2222
+ // local divergence when the cluster already reached consensus AND a durable majority of
2223
+ // the cohort reports holding the revision this coordinator was likely picked for commit
2224
+ // after missing the pend phase (unreachable during pend, fresh join, etc.), and it catches
2225
+ // up via replication.
2049
2226
  //
2050
- // Divergence reaches us in BOTH shapes and both must be tolerated identically:
2227
+ // Divergence reaches us in BOTH shapes and both are handled identically:
2051
2228
  // - a THROW ("Pending action … not found"), when we never saw the pend;
2052
2229
  // - a RETURNED `success:false` carrying `missing-base-revision`, when we saw the pend
2053
2230
  // but not the revision that created the block (see StorageRepo.internalCommit).
2054
- // Only the throw was tolerated before the refusal existed. Reporting the refusal to the
2055
- // caller instead would surface a committed transaction as a stale loss: db-core's
2056
- // commitPhase treats any returned `success:false` as a permanent stale failure, so the
2057
- // client would retry an action the cluster already landed until it exhausted its budget.
2231
+ // Reporting a tolerated divergence to the caller as the raw refusal would surface a
2232
+ // committed transaction as a stale loss: db-core's commitPhase treats any returned
2233
+ // `success:false` as a permanent stale failure. So a divergence on a durable majority is
2234
+ // reported as success, and one WITHOUT a durable majority as the durability gate's
2235
+ // retryable refusal (`tolerateLocalCommitDivergence`).
2236
+ //
2237
+ // The gate is evaluated BEFORE the local fallback commit, and a failing gate skips it.
2238
+ // This node counts toward the majority only when it is in the cohort the commit ran on;
2239
+ // an off-cohort coordinator's copy is a lone holder no cohort member will ever reconcile
2240
+ // from (members reconcile from `record.peers`), and a refused commit must not create one —
2241
+ // that lone off-cohort copy is exactly the seed of the "revision exists on one node that
2242
+ // is not responsible for it" placement the durability gate exists to prevent.
2243
+ const selfInCohort = this.localPeerId !== undefined && this.localPeerId.toString() in record.peers;
2244
+ if (!isDurableMajority(durability.remoteHolders + (selfInCohort ? 1 : 0), durability.cohortSize)) {
2245
+ return this.refuseCommitNotDurable(request, durability.remoteHolders, durability, 'fallback-skipped');
2246
+ }
2058
2247
  //
2059
2248
  // Deliberately NOT self-signed here (unlike the solo short-circuit above): consensus for
2060
2249
  // this commit ran on the cohort, so a one-peer minted proof would be a FALSE statement
@@ -2068,18 +2257,20 @@ export class CoordinatorRepo {
2068
2257
  try {
2069
2258
  const result = await this.storageRepo.commit(request, options, consensusProof);
2070
2259
  if (result.success) {
2260
+ // The gate above already admitted this shape: the remote holders plus this node
2261
+ // (when it is in the cohort) form the majority, and this node now holds it.
2071
2262
  if (armFreshness)
2072
2263
  this.markBlocksSeen(blockIds);
2073
2264
  return result;
2074
2265
  }
2075
2266
  if (isMissingBaseRevisionFailure(result) && clusterReachedCommitConsensus(record)) {
2076
- return this.tolerateLocalCommitDivergence(request, blockIds, result.reason ?? MISSING_BASE_REVISION_REASON, armFreshness);
2267
+ return this.tolerateLocalCommitDivergence(request, blockIds, result.reason ?? MISSING_BASE_REVISION_REASON, armFreshness, durability);
2077
2268
  }
2078
2269
  return result;
2079
2270
  }
2080
2271
  catch (err) {
2081
2272
  if (clusterReachedCommitConsensus(record)) {
2082
- return this.tolerateLocalCommitDivergence(request, blockIds, err.message, armFreshness);
2273
+ return this.tolerateLocalCommitDivergence(request, blockIds, err.message, armFreshness, durability);
2083
2274
  }
2084
2275
  throw err;
2085
2276
  }
@@ -2224,20 +2415,64 @@ export class CoordinatorRepo {
2224
2415
  };
2225
2416
  }
2226
2417
  /**
2227
- * Report success for a commit the cluster carried but this peer could not apply locally.
2228
- * Convergence comes from replication (cohort reconcile, or read-driven acquisition), not from
2229
- * replay here. `armFreshness` says whether the commit's quorum was strong enough
2230
- * ({@link commitQuorumRulesOutRivals}) for the read path to treat the blocks as
2231
- * freshness-checked; a divergence tolerated on a downsized quorum leaves the window unarmed
2232
- * this peer is known to be behind here, the last place a self-referential freshness stamp
2233
- * belongs.
2418
+ * A commit the cluster carried but this peer could not apply locally. Reported as success only
2419
+ * when a durable majority of the OTHER cohort members holds it (this node, having diverged, is
2420
+ * not a holder) convergence then comes from replication (cohort reconcile, or read-driven
2421
+ * acquisition), not from replay here. Without that majority it is the durability gate's
2422
+ * retryable refusal: nothing is known to hold the revision. `armFreshness` says whether the
2423
+ * commit's quorum was strong enough ({@link commitQuorumRulesOutRivals}) for the read path to
2424
+ * treat the blocks as freshness-checked; a divergence tolerated on a downsized quorum leaves the
2425
+ * window unarmed — this peer is known to be behind here, the last place a self-referential
2426
+ * freshness stamp belongs.
2234
2427
  */
2235
- tolerateLocalCommitDivergence(request, blockIds, detail, armFreshness) {
2428
+ tolerateLocalCommitDivergence(request, blockIds, detail, armFreshness, durability) {
2429
+ if (!isDurableMajority(durability.remoteHolders, durability.cohortSize)) {
2430
+ return this.refuseCommitNotDurable(request, durability.remoteHolders, durability, `fallback-diverged: ${detail}`);
2431
+ }
2236
2432
  this.log('coordinator-repo:commit-local-failed-cluster-succeeded', { actionId: request.actionId, error: detail });
2237
2433
  if (armFreshness)
2238
2434
  this.markBlocksSeen(blockIds);
2239
2435
  return { success: true };
2240
2436
  }
2437
+ /**
2438
+ * The durability gate's answer: a retryable refusal ({@link COMMIT_NOT_DURABLE_REASON}) naming how
2439
+ * many of the cohort reported holding the revision. `conflict: true` because the members that
2440
+ * refused have dropped their pending records, so the writer's only way forward is to cancel and
2441
+ * re-drive at a fresh revision — the path a conflict-shaped answer already puts it on. The
2442
+ * read-repair window is deliberately NOT armed: nothing about this commit is freshness evidence.
2443
+ */
2444
+ refuseCommitNotDurable(request, durableHolders, durability, arm) {
2445
+ this.log('coordinator-repo:commit-not-durable', {
2446
+ actionId: request.actionId,
2447
+ rev: request.rev,
2448
+ durableHolders,
2449
+ cohortSize: durability.cohortSize,
2450
+ remoteHolders: durability.remoteHolders,
2451
+ remoteRefusals: durability.remoteRefusals,
2452
+ arm
2453
+ });
2454
+ return {
2455
+ success: false,
2456
+ conflict: true,
2457
+ reason: `${COMMIT_NOT_DURABLE_REASON}: ${durableHolders} of ${durability.cohortSize} cohort member(s) report holding rev ${request.rev} of action ${request.actionId} (${arm})`
2458
+ };
2459
+ }
2460
+ }
2461
+ function cohortDurability(record, outcomes) {
2462
+ const reports = Object.values(outcomes ?? {});
2463
+ return {
2464
+ cohortSize: Object.keys(record.peers).length,
2465
+ remoteHolders: reports.filter(report => report.success).length,
2466
+ remoteRefusals: reports.filter(report => !report.success).length
2467
+ };
2468
+ }
2469
+ /**
2470
+ * The durability gate's rule: a STRICT majority of the cohort the commit ran on — the same majority
2471
+ * {@link clusterReachedCommitConsensus} measures votes against, applied to durable holders instead
2472
+ * of approving votes.
2473
+ */
2474
+ function isDurableMajority(durableHolders, cohortSize) {
2475
+ return cohortSize > 0 && durableHolders > cohortSize / 2;
2241
2476
  }
2242
2477
  /** True if a simple majority of cluster peers signed an approving commit. */
2243
2478
  function clusterReachedCommitConsensus(record) {