@optimystic/db-p2p 1.0.0-beta.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (234) hide show
  1. package/dist/src/cluster/block-transfer-service.d.ts +73 -1
  2. package/dist/src/cluster/block-transfer-service.d.ts.map +1 -1
  3. package/dist/src/cluster/block-transfer-service.js +70 -3
  4. package/dist/src/cluster/block-transfer-service.js.map +1 -1
  5. package/dist/src/cluster/block-transfer.d.ts +16 -1
  6. package/dist/src/cluster/block-transfer.d.ts.map +1 -1
  7. package/dist/src/cluster/block-transfer.js +73 -72
  8. package/dist/src/cluster/block-transfer.js.map +1 -1
  9. package/dist/src/cluster/client.d.ts +4 -6
  10. package/dist/src/cluster/client.d.ts.map +1 -1
  11. package/dist/src/cluster/client.js +8 -13
  12. package/dist/src/cluster/client.js.map +1 -1
  13. package/dist/src/cluster/cluster-policy.d.ts +2 -1
  14. package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
  15. package/dist/src/cluster/cluster-policy.js.map +1 -1
  16. package/dist/src/cluster/cluster-repo.d.ts +172 -17
  17. package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
  18. package/dist/src/cluster/cluster-repo.js +553 -149
  19. package/dist/src/cluster/cluster-repo.js.map +1 -1
  20. package/dist/src/cluster/commit-proof.js +2 -2
  21. package/dist/src/cluster/race-resolution.d.ts +51 -7
  22. package/dist/src/cluster/race-resolution.d.ts.map +1 -1
  23. package/dist/src/cluster/race-resolution.js +71 -7
  24. package/dist/src/cluster/race-resolution.js.map +1 -1
  25. package/dist/src/cluster/rebalance-monitor.d.ts +67 -5
  26. package/dist/src/cluster/rebalance-monitor.d.ts.map +1 -1
  27. package/dist/src/cluster/rebalance-monitor.js +120 -26
  28. package/dist/src/cluster/rebalance-monitor.js.map +1 -1
  29. package/dist/src/cluster/spread-on-churn.d.ts.map +1 -1
  30. package/dist/src/cluster/spread-on-churn.js +34 -47
  31. package/dist/src/cluster/spread-on-churn.js.map +1 -1
  32. package/dist/src/cohort-topic/host.d.ts.map +1 -1
  33. package/dist/src/cohort-topic/host.js +8 -6
  34. package/dist/src/cohort-topic/host.js.map +1 -1
  35. package/dist/src/cohort-topic/membership-source.js +1 -1
  36. package/dist/src/cohort-topic/membership-source.js.map +1 -1
  37. package/dist/src/cohort-topic/peer-codec.d.ts.map +1 -1
  38. package/dist/src/cohort-topic/peer-codec.js +5 -2
  39. package/dist/src/cohort-topic/peer-codec.js.map +1 -1
  40. package/dist/src/cohort-topic/stream-util.d.ts +24 -3
  41. package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
  42. package/dist/src/cohort-topic/stream-util.js +36 -3
  43. package/dist/src/cohort-topic/stream-util.js.map +1 -1
  44. package/dist/src/cohort-topic/topic-router.d.ts +9 -0
  45. package/dist/src/cohort-topic/topic-router.d.ts.map +1 -1
  46. package/dist/src/cohort-topic/topic-router.js +12 -2
  47. package/dist/src/cohort-topic/topic-router.js.map +1 -1
  48. package/dist/src/dispute/client.d.ts.map +1 -1
  49. package/dist/src/dispute/client.js +15 -3
  50. package/dist/src/dispute/client.js.map +1 -1
  51. package/dist/src/index.d.ts +5 -0
  52. package/dist/src/index.d.ts.map +1 -1
  53. package/dist/src/index.js +5 -0
  54. package/dist/src/index.js.map +1 -1
  55. package/dist/src/libp2p-key-network.d.ts +78 -17
  56. package/dist/src/libp2p-key-network.d.ts.map +1 -1
  57. package/dist/src/libp2p-key-network.js +241 -147
  58. package/dist/src/libp2p-key-network.js.map +1 -1
  59. package/dist/src/libp2p-node-base.d.ts +35 -1
  60. package/dist/src/libp2p-node-base.d.ts.map +1 -1
  61. package/dist/src/libp2p-node-base.js +151 -26
  62. package/dist/src/libp2p-node-base.js.map +1 -1
  63. package/dist/src/libp2p-node-rn.d.ts +2 -2
  64. package/dist/src/libp2p-node-rn.d.ts.map +1 -1
  65. package/dist/src/libp2p-node-rn.js.map +1 -1
  66. package/dist/src/libp2p-node.d.ts +2 -2
  67. package/dist/src/libp2p-node.d.ts.map +1 -1
  68. package/dist/src/libp2p-node.js.map +1 -1
  69. package/dist/src/logger.d.ts.map +1 -1
  70. package/dist/src/logger.js +19 -7
  71. package/dist/src/logger.js.map +1 -1
  72. package/dist/src/matchmaking/query-transport.d.ts +3 -4
  73. package/dist/src/matchmaking/query-transport.d.ts.map +1 -1
  74. package/dist/src/matchmaking/query-transport.js +9 -9
  75. package/dist/src/matchmaking/query-transport.js.map +1 -1
  76. package/dist/src/network/network-manager-service.d.ts +10 -5
  77. package/dist/src/network/network-manager-service.d.ts.map +1 -1
  78. package/dist/src/network/network-manager-service.js +11 -2
  79. package/dist/src/network/network-manager-service.js.map +1 -1
  80. package/dist/src/network/relay-reservation.d.ts +170 -0
  81. package/dist/src/network/relay-reservation.d.ts.map +1 -0
  82. package/dist/src/network/relay-reservation.js +527 -0
  83. package/dist/src/network/relay-reservation.js.map +1 -0
  84. package/dist/src/pend-validation.d.ts +3 -2
  85. package/dist/src/pend-validation.d.ts.map +1 -1
  86. package/dist/src/pend-validation.js +3 -2
  87. package/dist/src/pend-validation.js.map +1 -1
  88. package/dist/src/protocol-client.d.ts +8 -1
  89. package/dist/src/protocol-client.d.ts.map +1 -1
  90. package/dist/src/protocol-client.js +11 -0
  91. package/dist/src/protocol-client.js.map +1 -1
  92. package/dist/src/reactivity/forwarder-host.d.ts +2 -1
  93. package/dist/src/reactivity/forwarder-host.d.ts.map +1 -1
  94. package/dist/src/reactivity/forwarder-host.js +2 -1
  95. package/dist/src/reactivity/forwarder-host.js.map +1 -1
  96. package/dist/src/reactivity/origination-manager.d.ts +3 -3
  97. package/dist/src/reactivity/origination-manager.js +3 -3
  98. package/dist/src/reactivity/recover-transport.d.ts +22 -14
  99. package/dist/src/reactivity/recover-transport.d.ts.map +1 -1
  100. package/dist/src/reactivity/recover-transport.js +32 -19
  101. package/dist/src/reactivity/recover-transport.js.map +1 -1
  102. package/dist/src/reactivity/subscription-manager.d.ts +2 -2
  103. package/dist/src/reactivity/topic-bytes.d.ts +6 -7
  104. package/dist/src/reactivity/topic-bytes.d.ts.map +1 -1
  105. package/dist/src/reactivity/topic-bytes.js +8 -9
  106. package/dist/src/reactivity/topic-bytes.js.map +1 -1
  107. package/dist/src/repo/client.d.ts +5 -2
  108. package/dist/src/repo/client.d.ts.map +1 -1
  109. package/dist/src/repo/client.js +49 -40
  110. package/dist/src/repo/client.js.map +1 -1
  111. package/dist/src/repo/cluster-coordinator.d.ts +119 -5
  112. package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
  113. package/dist/src/repo/cluster-coordinator.js +181 -29
  114. package/dist/src/repo/cluster-coordinator.js.map +1 -1
  115. package/dist/src/repo/coordinator-repo.d.ts +259 -76
  116. package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
  117. package/dist/src/repo/coordinator-repo.js +716 -328
  118. package/dist/src/repo/coordinator-repo.js.map +1 -1
  119. package/dist/src/repo/i-under-replication-ledger.d.ts +87 -0
  120. package/dist/src/repo/i-under-replication-ledger.d.ts.map +1 -0
  121. package/dist/src/repo/i-under-replication-ledger.js +2 -0
  122. package/dist/src/repo/i-under-replication-ledger.js.map +1 -0
  123. package/dist/src/repo/kv-under-replication-ledger.d.ts +91 -0
  124. package/dist/src/repo/kv-under-replication-ledger.d.ts.map +1 -0
  125. package/dist/src/repo/kv-under-replication-ledger.js +287 -0
  126. package/dist/src/repo/kv-under-replication-ledger.js.map +1 -0
  127. package/dist/src/repo/responsibility.d.ts +37 -0
  128. package/dist/src/repo/responsibility.d.ts.map +1 -0
  129. package/dist/src/repo/responsibility.js +37 -0
  130. package/dist/src/repo/responsibility.js.map +1 -0
  131. package/dist/src/repo/service.d.ts +44 -15
  132. package/dist/src/repo/service.d.ts.map +1 -1
  133. package/dist/src/repo/service.js +64 -27
  134. package/dist/src/repo/service.js.map +1 -1
  135. package/dist/src/repo/stuck-reservation.d.ts +127 -0
  136. package/dist/src/repo/stuck-reservation.d.ts.map +1 -0
  137. package/dist/src/repo/stuck-reservation.js +179 -0
  138. package/dist/src/repo/stuck-reservation.js.map +1 -0
  139. package/dist/src/repo/under-replication-drain.d.ts +208 -0
  140. package/dist/src/repo/under-replication-drain.d.ts.map +1 -0
  141. package/dist/src/repo/under-replication-drain.js +461 -0
  142. package/dist/src/repo/under-replication-drain.js.map +1 -0
  143. package/dist/src/rn.d.ts +5 -0
  144. package/dist/src/rn.d.ts.map +1 -1
  145. package/dist/src/rn.js +5 -0
  146. package/dist/src/rn.js.map +1 -1
  147. package/dist/src/storage/block-latch.d.ts +3 -2
  148. package/dist/src/storage/block-latch.d.ts.map +1 -1
  149. package/dist/src/storage/block-latch.js +7 -4
  150. package/dist/src/storage/block-latch.js.map +1 -1
  151. package/dist/src/storage/block-storage.d.ts +45 -3
  152. package/dist/src/storage/block-storage.d.ts.map +1 -1
  153. package/dist/src/storage/block-storage.js +156 -9
  154. package/dist/src/storage/block-storage.js.map +1 -1
  155. package/dist/src/storage/i-block-storage.d.ts +57 -5
  156. package/dist/src/storage/i-block-storage.d.ts.map +1 -1
  157. package/dist/src/storage/i-block-storage.js.map +1 -1
  158. package/dist/src/storage/pending-claim.d.ts +167 -0
  159. package/dist/src/storage/pending-claim.d.ts.map +1 -0
  160. package/dist/src/storage/pending-claim.js +156 -0
  161. package/dist/src/storage/pending-claim.js.map +1 -0
  162. package/dist/src/storage/raw-store-codec.d.ts +13 -0
  163. package/dist/src/storage/raw-store-codec.d.ts.map +1 -1
  164. package/dist/src/storage/raw-store-codec.js +24 -3
  165. package/dist/src/storage/raw-store-codec.js.map +1 -1
  166. package/dist/src/storage/restoration-coordinator.d.ts.map +1 -1
  167. package/dist/src/storage/restoration-coordinator.js +2 -1
  168. package/dist/src/storage/restoration-coordinator.js.map +1 -1
  169. package/dist/src/storage/ring-shift-coordinator.d.ts.map +1 -1
  170. package/dist/src/storage/ring-shift-coordinator.js +3 -3
  171. package/dist/src/storage/ring-shift-coordinator.js.map +1 -1
  172. package/dist/src/storage/storage-repo.d.ts +137 -5
  173. package/dist/src/storage/storage-repo.d.ts.map +1 -1
  174. package/dist/src/storage/storage-repo.js +273 -68
  175. package/dist/src/storage/storage-repo.js.map +1 -1
  176. package/dist/src/storage/struct.d.ts +58 -0
  177. package/dist/src/storage/struct.d.ts.map +1 -1
  178. package/dist/src/testing/mesh-harness.d.ts +92 -11
  179. package/dist/src/testing/mesh-harness.d.ts.map +1 -1
  180. package/dist/src/testing/mesh-harness.js +246 -103
  181. package/dist/src/testing/mesh-harness.js.map +1 -1
  182. package/package.json +2 -2
  183. package/readme.md +27 -3
  184. package/src/cluster/block-transfer-service.ts +132 -4
  185. package/src/cluster/block-transfer.ts +79 -84
  186. package/src/cluster/client.ts +115 -119
  187. package/src/cluster/cluster-policy.ts +2 -1
  188. package/src/cluster/cluster-repo.ts +3115 -2671
  189. package/src/cluster/commit-proof.ts +2 -2
  190. package/src/cluster/race-resolution.ts +76 -7
  191. package/src/cluster/rebalance-monitor.ts +677 -539
  192. package/src/cluster/spread-on-churn.ts +34 -54
  193. package/src/cohort-topic/host.ts +8 -6
  194. package/src/cohort-topic/membership-source.ts +1 -1
  195. package/src/cohort-topic/peer-codec.ts +34 -31
  196. package/src/cohort-topic/stream-util.ts +39 -4
  197. package/src/cohort-topic/topic-router.ts +12 -2
  198. package/src/dispute/client.ts +17 -7
  199. package/src/index.ts +64 -59
  200. package/src/libp2p-key-network.ts +1377 -1253
  201. package/src/libp2p-node-base.ts +186 -27
  202. package/src/libp2p-node-rn.ts +31 -30
  203. package/src/libp2p-node.ts +37 -36
  204. package/src/logger.ts +20 -7
  205. package/src/matchmaking/query-transport.ts +9 -9
  206. package/src/network/network-manager-service.ts +16 -5
  207. package/src/network/relay-reservation.ts +636 -0
  208. package/src/pend-validation.ts +3 -2
  209. package/src/protocol-client.ts +13 -1
  210. package/src/reactivity/forwarder-host.ts +2 -1
  211. package/src/reactivity/origination-manager.ts +3 -3
  212. package/src/reactivity/recover-transport.ts +39 -22
  213. package/src/reactivity/subscription-manager.ts +2 -2
  214. package/src/reactivity/topic-bytes.ts +35 -37
  215. package/src/repo/client.ts +172 -173
  216. package/src/repo/cluster-coordinator.ts +1296 -1113
  217. package/src/repo/coordinator-repo.ts +3087 -2687
  218. package/src/repo/i-under-replication-ledger.ts +89 -0
  219. package/src/repo/kv-under-replication-ledger.ts +302 -0
  220. package/src/repo/responsibility.ts +48 -0
  221. package/src/repo/service.ts +76 -39
  222. package/src/repo/stuck-reservation.ts +224 -0
  223. package/src/repo/under-replication-drain.ts +582 -0
  224. package/src/rn.ts +74 -69
  225. package/src/storage/block-latch.ts +7 -4
  226. package/src/storage/block-storage.ts +161 -10
  227. package/src/storage/i-block-storage.ts +60 -5
  228. package/src/storage/pending-claim.ts +200 -0
  229. package/src/storage/raw-store-codec.ts +26 -3
  230. package/src/storage/restoration-coordinator.ts +212 -211
  231. package/src/storage/ring-shift-coordinator.ts +271 -271
  232. package/src/storage/storage-repo.ts +311 -73
  233. package/src/storage/struct.ts +58 -0
  234. package/src/testing/mesh-harness.ts +329 -115
@@ -1,5 +1,5 @@
1
- import { LruMap, blockIdsForTransforms, highestStaleAt, isConflictFailure, isOwnRevision, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
2
- import { ClusterCoordinator, ConflictRaceLostError, ValidatorRejectionError } from "./cluster-coordinator.js";
1
+ import { LruMap, blockIdsForTransforms, transformForBlockId, highestStaleAt, isConflictFailure, isOwnRevision, DEFAULT_SUPER_MAJORITY_THRESHOLD, routingKeyForBlock, localDurability, unroutedDurability } from "@optimystic/db-core";
2
+ import { BlocksHeldError, ClusterCoordinator, ConflictRaceLostError, ValidatorRejectionError } from "./cluster-coordinator.js";
3
3
  import { peerIdFromString } from "@libp2p/peer-id";
4
4
  import { createLogger } from '../logger.js';
5
5
  import { PenaltyReason } from "../reputation/types.js";
@@ -7,8 +7,11 @@ import { quorumSize, corroboratorCapacity, selectQuorumRev, certifiedEquivocatio
7
7
  import { certifyClaim, isAttributableProofFailure, proofThresholds } from "../cluster/certified-claims.js";
8
8
  import { DEFAULT_CLUSTER_SIZE, resolveRepairCorroborationClusterSize } from "../cluster/cluster-policy.js";
9
9
  import { RECONCILE_TIMEOUT_MS } from "../cluster/reconcile-block.js";
10
- import { isMissingBaseRevisionFailure, MISSING_BASE_REVISION_REASON } from "../storage/storage-repo.js";
10
+ import { isMissingBaseRevisionFailure, COMMIT_NOT_DURABLE_REASON, MISSING_BASE_REVISION_REASON } from "../storage/storage-repo.js";
11
+ import { isReservationAgainst, reservationRequestFor, cohortCanMissAPend } from "../storage/pending-claim.js";
11
12
  import { buildBlockCommitProof } from "../cluster/commit-proof.js";
13
+ import { RESPONSIBILITY_TTL_MS, ResponsibilityRefusalError } from "./responsibility.js";
14
+ import { StuckReservationTracker } from "./stuck-reservation.js";
12
15
  /** How long one cohort peer gets to answer the latest-revision consult before it counts as silent. */
13
16
  const LATEST_QUERY_TIMEOUT_MS = 1000;
14
17
  /** True when a freshly-read local revision is strictly ahead of the baseline the repair started from. */
@@ -76,80 +79,8 @@ function soleHolderMessage(cohortPeers) {
76
79
  `second voter — reaching this message means the one holder attached no such proof, or one that did not ` +
77
80
  `verify.)`;
78
81
  }
79
- /**
80
- * How many DISTINCT later actions one unchanged holder must refuse on a block before the refusals are
81
- * named as a stuck reservation rather than as an ordinary lost race.
82
- *
83
- * **Why a count of distinct actions and not something else.** Elapsed time answers the wrong question
84
- * — a slow writer is not a stuck one, and a holder legitimately keeps its reservation for as long as
85
- * its own commit takes. A raw refusal count answers the wrong question too: a single writer retrying
86
- * a lost race produces a run of refusals under ONE action id (see {@link StuckReservationWatch.refused}).
87
- * What no healthy holder can produce is an unbounded stream of *different* writers all losing to it,
88
- * because a healthy holder's reservation lasts one pend-to-commit window.
89
- *
90
- * **Why 8.** The bound to clear is how many distinct actions can honestly be refused inside one such
91
- * window. Measured on the in-process mesh, in the healthy-contention arm of
92
- * `test/stuck-reservation-named.spec.ts`: a holder that pends, is raced by other writers, and then
93
- * commits refuses **2** distinct actions per episode — the two rivals — and the count resets on every
94
- * holder change. `concurrent-diary-append-acknowledgement.spec.ts` races three writers at one diary
95
- * and cannot exceed that either, for the same reason: at most (writers - 1) rivals can lose to one
96
- * winner. 8 is four times the measured healthy figure, and it is a floor a genuinely stuck block
97
- * clears trivially (the field instance refused hundreds).
98
- *
99
- * **The bound stated exactly.** It is distinct SYNC CYCLES, not distinct writers: one writer that
100
- * exhausts a sync's retry budget and is re-driven by its caller mints a fresh id for the next cycle,
101
- * so it can contribute more than one. That does not widen the window much — a cycle only ends in
102
- * exhaustion after `DefaultMaxAttempts` (10) attempts of backoff, roughly 21s (see the exhaustion
103
- * NOTE in `Collection.syncAttempts`), so a lone writer needs a holder to keep the block for upwards
104
- * of two and a half minutes before it reaches 8 by itself, which is not a healthy holder.
105
- *
106
- * **What the margin does NOT cover, stated honestly.** A block with more than 8 distinct writers
107
- * racing it inside a single pend-to-commit round trip could reach 8 with a perfectly healthy holder.
108
- * That is a diagnostic false positive on a log line and nothing else — this counter never refuses,
109
- * expires, or deletes anything (see {@link CoordinatorRepo.noteStuckReservation}) — and the remedy if
110
- * a deployment ever hits it is to raise this number, not to add a control path. Raising it costs
111
- * detection latency on low-traffic blocks, which need this many distinct write ATTEMPTS before the
112
- * condition can be named at all.
113
- */
114
- const STUCK_RESERVATION_DISTINCT_ACTIONS = 8;
115
- /** Whether two sorted holder lists name the same reservation — i.e. whether a refusal continues an
116
- * existing episode or starts a new one. Both sides come from the same sort, so this is a plain
117
- * element-wise comparison; a block normally has exactly one holder, since a member's own pend refuses
118
- * a second one (`ClusterMember.validatePendOperations`). */
119
- function sameHolders(a, b) {
120
- return a.length === b.length && a.every((id, i) => id === b[i]);
121
- }
122
- /**
123
- * The stuck-reservation wording: written for an operator reading logs, in the same register as
124
- * {@link cohortTooSmallMessage} and {@link soleHolderMessage} — what is stuck, what will and will not
125
- * clear it, and what to do next.
126
- *
127
- * The claim is deliberately about the RESERVATION, not about the writer's intent: this node cannot
128
- * see whether the holding process is alive, only that it has held the block across enough unrelated
129
- * later actions that no retry is going to win. So the line says what is provable (the block accepts
130
- * no writes while this record stands, and nothing on the node removes it) and points at the one check
131
- * that settles the rest.
132
- */
133
- function stuckReservationMessage(holders, refusedActions) {
134
- const held = holders.join(', ');
135
- return `This block is WEDGED BEHIND A PENDING WRITE THAT IS NOT COMPLETING, and retrying will never ` +
136
- `clear it: action(s) ${held} reserved the block and have now refused ${refusedActions} DISTINCT, ` +
137
- `unrelated later actions. Each of those refusals on its own looks exactly like an ordinary ` +
138
- `optimistic-concurrency loss, which is normal and healthy — the repetition is what is not. A ` +
139
- `healthy rival holds a block only for its own pend-to-commit window and then releases it by ` +
140
- `committing or cancelling; a reservation that keeps refusing NEW writers is holding the block ` +
141
- `against every writer on every machine, and each of them loses again identically. EXACTLY TWO ` +
142
- `THINGS CLEAR IT: a cancel for action(s) ${held} on this block (route it through the cohort so ` +
143
- `every member drops the record), or that same action's own commit landing. Nothing on the node ` +
144
- `expires it — there is no sweep for abandoned pending records — so until one of those two happens ` +
145
- `the block takes NO writes while continuing to serve reads and to look healthy in every other ` +
146
- `respect. The usual cause is a writer that went away between a failed or half-applied commit and ` +
147
- `the cancel it owed, so check whether whatever ran ${held} still exists before cancelling on its ` +
148
- `behalf. This line is a diagnosis and nothing more: this node does not expire, refuse, or delete ` +
149
- `the record on the strength of it.`;
150
- }
151
82
  export function coordinatorRepo(keyNetwork, createClusterClient, cfg, fretService, reputation, stateStore) {
152
- return (components) => new CoordinatorRepo(keyNetwork, createClusterClient, components.storageRepo, cfg, components.localCluster, components.localPeerId, fretService, components.clusterLatestCallback, reputation, stateStore, components.acquireBlockFromCohort, components.proofAnchoring);
83
+ return (components) => new CoordinatorRepo(keyNetwork, createClusterClient, components.storageRepo, cfg, components.localCluster, components.localPeerId, fretService, components.clusterLatestCallback, reputation, stateStore, components.acquireBlockFromCohort, components.proofAnchoring, components.underReplicationLedger, components.onCommittedHolders);
153
84
  }
154
85
  /** Cluster coordination repo - uses local store, as well as distributes changes to other nodes using cluster consensus. */
155
86
  export class CoordinatorRepo {
@@ -160,11 +91,12 @@ export class CoordinatorRepo {
160
91
  clusterLatestCallback;
161
92
  acquireBlockFromCohort;
162
93
  proofAnchoring;
94
+ underReplicationLedger;
95
+ onCommittedHolders;
163
96
  coordinator;
164
97
  DEFAULT_TIMEOUT = 30000; // 30 seconds default timeout
165
98
  localPeerId;
166
99
  responsibilityCache = new LruMap(1000);
167
- static RESPONSIBILITY_TTL_MS = 60_000;
168
100
  lastSeenCommitMs = new LruMap(1000);
169
101
  /** Per block, what earlier repair passes left unresolved — see {@link AheadClaimState}.
170
102
  * Outlives the consult on purpose: the read-repair window skips consults for blocks checked
@@ -174,14 +106,14 @@ export class CoordinatorRepo {
174
106
  * {@link reportRepairDeadlock} say its piece a second time. */
175
107
  unsettledAheadClaims = new LruMap(1000);
176
108
  /**
177
- * Per block, what its pending-conflict refusals have added up to — see {@link StuckReservationWatch}
178
- * and {@link noteStuckReservation}. Deliberately its OWN map rather than a third fact hung off
179
- * {@link unsettledAheadClaims}: that entry belongs to the read-repair path and is cleared by a block
180
- * converging on a revision, whereas this one belongs to the write path and is cleared by the block
181
- * accepting a write. Sharing the entry would mean teaching both of those lifetimes about a fact
182
- * neither owns (backlog `debt-freshness-state-scattered-across-coordinator-repo` is the standing
183
- * argument for collapsing all of this per-block state behind one collaborator; adding a fourth
184
- * carve-out to the freshness entry would have made that harder, not easier).
109
+ * Per block, what its pending-conflict refusals have added up to — see {@link StuckReservationTracker}
110
+ * and {@link noteStuckReservation}. Deliberately its OWN collaborator rather than a third fact hung
111
+ * off {@link unsettledAheadClaims}: that entry belongs to the read-repair path and is cleared by a
112
+ * block converging on a revision, whereas this one belongs to the write path and is cleared by the
113
+ * block accepting a write. Sharing the entry would mean teaching both of those lifetimes about a
114
+ * fact neither owns (backlog `debt-freshness-state-scattered-across-coordinator-repo` is the
115
+ * standing argument for collapsing all of this per-block state behind one collaborator; adding a
116
+ * fourth carve-out to the freshness entry would have made that harder, not easier).
185
117
  *
186
118
  * NOTE: LRU-bounded like its siblings. An eviction under >1000 conflicted blocks loses an episode's
187
119
  * say-once flag, so the line can repeat once for that block — the same bounded duplication
@@ -190,12 +122,11 @@ export class CoordinatorRepo {
190
122
  * NOTE: per COORDINATOR, and a block's coordinator is whichever peer the writer's key lookup
191
123
  * resolved — normally stable, but cohort churn or a routing change moves it. When it moves, the
192
124
  * count restarts on the new coordinator (the condition is named later) and the old one may name the
193
- * same episode again (the condition is named twice). Fine while a wedged block is diagnosed by
194
- * searching for its id; if a churning deployment ever makes duplicate lines the noisy failure this
195
- * one exists to replace, the say-once state has to move to where the record lives (the member's own
196
- * storage) rather than to where the refusal was classified.
125
+ * same episode again (the condition is named twice). Each MEMBER now keeps its own tracker as well
126
+ * (`ClusterMember.validatePendOperations`), fed by its own `held` votes, so the episode is also
127
+ * named where the record lives, by a count that does not move with the coordinator.
197
128
  */
198
- stuckReservations = new LruMap(1000);
129
+ stuckReservations = new StuckReservationTracker(1000);
199
130
  readRepairMode;
200
131
  readRepairWindowMs;
201
132
  readRepairSampleRate;
@@ -217,7 +148,7 @@ export class CoordinatorRepo {
217
148
  now = () => Date.now();
218
149
  /** Test seam: overridable RNG (0..1) for sample-rate gating. */
219
150
  rand = () => Math.random();
220
- constructor(keyNetwork, createClusterClient, storageRepo, cfg, localCluster, localPeerId, fretService, clusterLatestCallback, reputation, stateStore, acquireBlockFromCohort, proofAnchoring) {
151
+ constructor(keyNetwork, createClusterClient, storageRepo, cfg, localCluster, localPeerId, fretService, clusterLatestCallback, reputation, stateStore, acquireBlockFromCohort, proofAnchoring, underReplicationLedger, onCommittedHolders) {
221
152
  this.keyNetwork = keyNetwork;
222
153
  this.createClusterClient = createClusterClient;
223
154
  this.storageRepo = storageRepo;
@@ -225,6 +156,8 @@ export class CoordinatorRepo {
225
156
  this.clusterLatestCallback = clusterLatestCallback;
226
157
  this.acquireBlockFromCohort = acquireBlockFromCohort;
227
158
  this.proofAnchoring = proofAnchoring;
159
+ this.underReplicationLedger = underReplicationLedger;
160
+ this.onCommittedHolders = onCommittedHolders;
228
161
  this.localPeerId = localPeerId;
229
162
  this.log = createLogger('coordinator-repo', localPeerId?.toString());
230
163
  const policy = {
@@ -291,67 +224,103 @@ export class CoordinatorRepo {
291
224
  await this.coordinator.recoverTransactions();
292
225
  }
293
226
  /**
294
- * Check if this node is in the cluster for a given block.
295
- * Uses findCluster membership in the real network layer, self is always
296
- * included in the cohort when this node is responsible. This serves as a
297
- * defense-in-depth guard for requests that arrive at the wrong node.
298
- * Returns true if localPeerId is not set (backward compat for single-node/test setups).
227
+ * Whether this node is in the cohort for a block. Uses `findCluster` membership — in the real network
228
+ * layer, self is in the cohort exactly when this node is among the nearest `clusterSize` serving peers
229
+ * for the block, so a request that reaches a node outside it was sent to the wrong machine.
230
+ *
231
+ * Three answers, because a lookup that THREW is not an answer: `undetermined` is never cached (routing
232
+ * can recover on the next request) and never read as `responsible` — what each caller does with it is
233
+ * that caller's posture (see {@link verifyResponsibility} and `get`).
234
+ *
235
+ * With no `localPeerId` the check is skipped and the answer is `responsible`. That bypass exists for
236
+ * wiring without an identity (direct constructors, single-node and test setups), never for production,
237
+ * where `libp2p-node-base` always passes the node's peer id.
238
+ *
239
+ * NOTE: a cached answer stands for up to {@link RESPONSIBILITY_TTL_MS} after the cohort changes, so a
240
+ * node can still say `responsible` for a minute after it stops being so. {@link soloCohortDurability}'s
241
+ * `unrouted` class and `ClusterCoordinator.executeClusterTransaction`'s cohort-membership guard are what
242
+ * keep that window honest; if churn ever makes the window matter, shorten the TTL rather than
243
+ * re-looking-up on every write.
299
244
  */
300
- async isResponsibleForBlock(blockId) {
245
+ async responsibilityFor(blockId) {
301
246
  if (!this.localPeerId)
302
- return true;
247
+ return 'responsible';
303
248
  const cached = this.responsibilityCache.get(blockId);
304
249
  if (cached && cached.expires > Date.now()) {
305
- return cached.inCluster;
250
+ return cached.inCluster ? 'responsible' : 'not-responsible';
306
251
  }
307
- const blockIdBytes = new TextEncoder().encode(blockId);
308
252
  let inCluster;
309
253
  try {
310
- const peers = await this.keyNetwork.findCluster(blockIdBytes);
254
+ const peers = await this.keyNetwork.findCluster(routingKeyForBlock(blockId));
311
255
  inCluster = this.localPeerId.toString() in peers;
312
256
  }
313
257
  catch (err) {
314
258
  this.log('proximity:check-error', { blockId, error: err.message });
315
- // On failure, assume responsible to avoid false rejections
316
- return true;
259
+ return 'undetermined';
317
260
  }
318
- this.responsibilityCache.set(blockId, { inCluster, expires: Date.now() + CoordinatorRepo.RESPONSIBILITY_TTL_MS });
261
+ this.responsibilityCache.set(blockId, { inCluster, expires: Date.now() + RESPONSIBILITY_TTL_MS });
319
262
  this.log('proximity:checked', { blockId, inCluster });
320
- return inCluster;
263
+ return inCluster ? 'responsible' : 'not-responsible';
321
264
  }
322
265
  /**
323
- * Verify this node is responsible for all given block IDs. Throws if not.
266
+ * The write path's gate (`pend`, `cancel`, `commit`): refuse unless this node is responsible for EVERY
267
+ * block, and refuse too when it cannot tell — fail CLOSED. A write accepted on a thrown lookup commits
268
+ * where the network was never consulted (GitHub #19); refusing hands it back to the writer's transactor,
269
+ * which excludes this peer and re-picks inside the cohort.
270
+ *
271
+ * Every block is checked so the error names all of them. When the blocks split between the two kinds,
272
+ * `not-responsible` is the one thrown: it is a settled answer that the request is misrouted, which no
273
+ * recovery of the lookup would change; the undetermined blocks are still logged.
274
+ *
275
+ * For `cancel`, a refusal on a transient lookup fault leaves the pending record standing on this node
276
+ * until the writer's cancel retry lands on another cohort member — and this node is still discharged
277
+ * then, since as a member it judges the cancel against the record's own `peers`, not against a lookup.
324
278
  */
325
279
  async verifyResponsibility(blockIds) {
326
280
  const notResponsible = [];
281
+ const undetermined = [];
327
282
  for (const blockId of blockIds) {
328
- if (!await this.isResponsibleForBlock(blockId)) {
283
+ const verdict = await this.responsibilityFor(blockId);
284
+ if (verdict === 'not-responsible')
329
285
  notResponsible.push(blockId);
330
- }
331
- }
332
- if (notResponsible.length > 0) {
333
- this.log('proximity:rejected', { blockIds: notResponsible });
334
- throw new Error(`Not responsible for block(s): ${notResponsible.join(', ')}`);
286
+ else if (verdict === 'undetermined')
287
+ undetermined.push(blockId);
335
288
  }
289
+ if (notResponsible.length === 0 && undetermined.length === 0)
290
+ return;
291
+ this.log('proximity:rejected', { notResponsible, undetermined });
292
+ throw notResponsible.length > 0
293
+ ? new ResponsibilityRefusalError('not-responsible', notResponsible)
294
+ : new ResponsibilityRefusalError('undetermined', undetermined, 'cohort lookup failed; refusing the write rather than accepting it unrouted');
336
295
  }
337
296
  async get(blockGets, options) {
338
- // Soft proximity check — warn but still serve reads for graceful degradation
339
- // NOTE: a soft-served read now also *acquires* the block durably (see restoreCorroborated), where
340
- // before it could at most promote a pending this node already held. So a soft serve leaves behind
341
- // a replica of a block this node is not responsible for, and nothing sweeps those: ring-shift
342
- // sheds a keyspace RANGE, not "blocks outside my cohort". Fine while soft serves are what they
343
- // are meant to be — a rare degradation during routing churn since routing already placed this
344
- // node near the block. If they ever become routine, gate acquisition (not the serve itself) on
345
- // isResponsibleForBlock.
297
+ // Soft proximity check — warn but still serve reads for graceful degradation. Unlike the write
298
+ // path it stays OPEN when the lookup throws: a read is best-effort, and the consult below flags
299
+ // whatever it could not confirm (`unavailable`, `unconfirmedAheadRev`) rather than posing as sure.
300
+ // NOTE: a soft-served read also *acquires* the block durably (see restoreCorroborated), so a soft
301
+ // serve leaves behind a replica of a block this node is not responsible for, and nothing sweeps
302
+ // those: ring-shift sheds a keyspace RANGE, not "blocks outside my cohort". Soft serves are now
303
+ // confined to the responsibility caches' staleness window: a remote read for a block this node is
304
+ // not responsible for is redirected by `RepoService.checkRedirect` before it gets here, and this
305
+ // node's own transactor routes a read here only when self is in the block's cohort. If soft serves
306
+ // ever become routine (steady `proximity:get-warning` lines while the network's membership is not
307
+ // changing), gate acquisition (not the serve itself) on `responsibilityFor`.
308
+ //
309
+ // NOTE: accepted tradeoff — this check and `fetchBlockFromCluster` below each run their own
310
+ // `findCluster` for the same block, so a cold read costs two cohort lookups where one shared
311
+ // lookup would do. Declined 2026-09-11: at 0.009 ms per solo lookup there is nothing to win,
312
+ // and this check's future is open (blocked `writer-and-servers-disagree-on-where-a-block-lives`,
313
+ // option D2(b)). Revisit if a device profile shows `findCluster` as material.
346
314
  for (const blockId of blockGets.blockIds) {
347
- if (!await this.isResponsibleForBlock(blockId)) {
315
+ if (await this.responsibilityFor(blockId) === 'not-responsible') {
348
316
  this.log('proximity:get-warning', { blockId, msg: 'serving read for non-responsible block' });
349
317
  }
350
318
  }
351
319
  // First try local storage
352
320
  const localResult = await this.storageRepo.get(blockGets, options);
353
321
  // Decide per-block whether to consult cluster peers. Two triggers:
354
- // (a) Missing — block isn't present locally at all (legacy behavior).
322
+ // (a) Missing — block isn't present locally at all. Consults on every read: an absence is
323
+ // never remembered (see the accepted-tradeoff NOTE below).
355
324
  // (b) Stale-by-policy — block is present but read-repair policy says verify.
356
325
  // Skip cluster fetch if this is already a sync request (to prevent recursive queries).
357
326
  // A sync read is also never marked `unavailable` here — the consult it skips is the
@@ -361,7 +330,8 @@ export class CoordinatorRepo {
361
330
  const skipClusterFetch = options?.skipClusterFetch;
362
331
  // NOTE: NetworkTransactor.get treats an authoritative "absent" ({ state: {} })
363
332
  // as final and no longer retries it (ticket txn-perf-authoritative-notfound),
364
- // relying on this cluster reconciliation to have already run. When the consult
333
+ // relying on this cluster reconciliation to have already run for THIS read: an
334
+ // unflagged absent means the cohort was consulted before answering. When the consult
365
335
  // FAILS outright — or runs without ruling the block out and the block stays
366
336
  // missing — the entry is flagged `unavailable` below with a reason naming what
367
337
  // the consult established (see AbsenceVerdict and the mapping in the loop body),
@@ -371,6 +341,17 @@ export class CoordinatorRepo {
371
341
  // no flag and no transactor-level retry to compensate. That is fine (such a
372
342
  // coordinator has no cluster to reconcile against), but keep this coupling in
373
343
  // mind if a partial-cluster read path is added.
344
+ //
345
+ // NOTE: accepted tradeoff — a block missing locally consults on EVERY read, even on a cohort
346
+ // of one, where the consult is one `findCluster` (0.009 ms measured) and a once-per-window log
347
+ // line (see the solo-self exit). Remembering an absence was tried and removed (GitHub issue
348
+ // #20, ticket drop-the-settled-absence-memo): a cohort-of-one memo outlived the view it was
349
+ // settled under, and a node whose view then grew served a block another coordinator had just
350
+ // committed as never created. Cohort-member writes (`ClusterRepo` → storage) never pass
351
+ // through this class, so nothing here could clear such a memo either. Revisit only with a memo
352
+ // bound to the cohort view it was settled under AND cleared by every writer of local storage;
353
+ // `coordinator-repo-absence-write-bypass.spec.ts` is the gate, and backlog
354
+ // `feat-a-cohort-member-remembers-a-settled-absence` holds the design notes.
374
355
  if (this.clusterLatestCallback && !skipClusterFetch) {
375
356
  for (const blockId of blockGets.blockIds) {
376
357
  const localEntry = localResult[blockId];
@@ -397,7 +378,12 @@ export class CoordinatorRepo {
397
378
  }
398
379
  try {
399
380
  const { absence, currency } = await this.fetchBlockFromCluster(blockId, blockGets.context, localRev);
400
- const refreshed = await this.storageRepo.get({ blockIds: [blockId], context: blockGets.context }, options);
381
+ // `lineageOf` rides along: the refreshed entry REPLACES the local one below, and an
382
+ // entry that lost its lineage answer would read to the asker as "could not say".
383
+ const refreshed = await this.storageRepo.get({
384
+ blockIds: [blockId], context: blockGets.context,
385
+ ...(blockGets.lineageOf === undefined ? {} : { lineageOf: blockGets.lineageOf })
386
+ }, options);
401
387
  const newRev = refreshed[blockId]?.state?.latest?.rev;
402
388
  if (refreshed[blockId]) {
403
389
  localResult[blockId] = refreshed[blockId];
@@ -667,6 +653,28 @@ export class CoordinatorRepo {
667
653
  const lastSeen = this.lastSeenCommitMs.get(blockId);
668
654
  return lastSeen == null ? undefined : this.now() - lastSeen;
669
655
  }
656
+ /**
657
+ * Whether the solo-self exit of {@link fetchBlockFromCluster} has already named `blockId` inside
658
+ * the current read-repair window, for a block this node does NOT hold (`localRev` undefined). Such
659
+ * a block reaches that exit on every read, because `get` never remembers an absence (GitHub
660
+ * issue #20). So its `cluster-fetch:solo-self-skip` line and its stamp are both skipped while the
661
+ * stamp is at most one `readRepairWindowMs` old. An unconditional line would bring back issue #8's
662
+ * log volume for a probed but never-written collection. Re-stamping on every read would keep the
663
+ * stamp fresh and silence the line for good on a block read more often than once a window.
664
+ *
665
+ * Withholding the stamp never skips a consult: `get` asks {@link shouldReadRepair} only about
666
+ * present blocks. A HELD block reaches the exit only when read-repair already chose to consult, so
667
+ * this is always false for one. Its line and stamp are unchanged, and the line keeps pairing with
668
+ * `cluster-tx:read-repair-triggered`.
669
+ *
670
+ * NOTE: the stamp is shared with every exit that marks a block seen, so a missing block one of
671
+ * them stamped inside the window is not named until that stamp lapses. Fine for a diagnostic
672
+ * line; if anything ever needs "first answer from a self-only view" exactly, give the line its own
673
+ * per-block stamp.
674
+ */
675
+ soloAbsenceNamedThisWindow(blockId, localRev) {
676
+ return localRev === undefined && (this.ageMs(blockId) ?? Infinity) <= this.readRepairWindowMs;
677
+ }
670
678
  /** Mark blocks as freshly observed from cluster authority (post-commit or post-fetch). */
671
679
  markBlocksSeen(blockIds) {
672
680
  const now = this.now();
@@ -733,17 +741,21 @@ export class CoordinatorRepo {
733
741
  * retire it when the memo's claimants are among the peers that answered), `no-evidence`
734
742
  * leaves it standing untouched. Required, not optional, so an exit added later has to say
735
743
  * which it means.
744
+ *
745
+ * Nothing is remembered about an absence: `get` runs this pass on every read of a block missing
746
+ * locally (see the accepted-tradeoff NOTE there).
736
747
  */
737
748
  async fetchBlockFromCluster(blockId, context, localRev) {
738
749
  if (!this.clusterLatestCallback)
739
750
  return { absence: 'confirmed', currency: { kind: 'no-evidence' } };
740
- const blockIdBytes = new TextEncoder().encode(blockId);
741
- const peers = await this.keyNetwork.findCluster(blockIdBytes);
751
+ const peers = await this.keyNetwork.findCluster(routingKeyForBlock(blockId));
742
752
  const peerIds = peers ? Object.keys(peers) : [];
743
753
  // NOTE: deliberately does NOT call `markBlocksSeen` — unlike the solo-self exit below.
744
- // An empty cohort is a ROUTING FAILURE, not a settled answer, and `Libp2pKeyPeerNetwork`
745
- // cannot even produce one (its `findCluster` always includes self); the only producer
746
- // today is the mesh harness's injected `findClusterFails`. Arming the read-repair window
754
+ // An empty cohort is a ROUTING FAILURE, not a settled answer. A serving node's
755
+ // `Libp2pKeyPeerNetwork.findCluster` cannot produce one: it always holds at least this
756
+ // node when no nearer serving peer is known. Only a node that serves no storage (a
757
+ // client-only libp2p node, which runs no CoordinatorRepo today) can see an empty cohort;
758
+ // the only producer here is the mesh harness's injected `findClusterFails`. Arming the read-repair window
747
759
  // here would suppress a genuine repair for a whole `readRepairWindowMs` after a transient
748
760
  // blip, and re-entering costs no network work beyond the `findCluster` the read already
749
761
  // makes. Do not "fix" this by symmetry with the solo-self exit.
@@ -757,7 +769,11 @@ export class CoordinatorRepo {
757
769
  if (peerIds.length === 1
758
770
  && this.localPeerId
759
771
  && peerIds[0] === this.localPeerId.toString()) {
760
- this.log('cluster-fetch:solo-self-skip', { blockId });
772
+ // The line and the stamp below go together: both are skipped for a missing block this exit
773
+ // already named inside the window (see `soloAbsenceNamedThisWindow`).
774
+ const namedThisWindow = this.soloAbsenceNamedThisWindow(blockId, localRev);
775
+ if (!namedThisWindow)
776
+ this.log('cluster-fetch:solo-self-skip', { blockId });
761
777
  // Arm the lazy read-repair window. Without this the window is never stamped,
762
778
  // `shouldReadRepair` reads `lastSeen == null` forever, and every read re-enters this
763
779
  // exit — read -> stale -> consult -> solo skip -> no-op -> still stale, unbounded
@@ -775,7 +791,8 @@ export class CoordinatorRepo {
775
791
  // Deliberately opposite to the commit-side rule that withholds arming from a commit
776
792
  // whose quorum proves nothing about rivals: that damps nothing, this bounds an
777
793
  // otherwise unbounded loop. Landing both, keep both — see the specs for each.
778
- this.markBlocksSeen([blockId]);
794
+ if (!namedThisWindow)
795
+ this.markBlocksSeen([blockId]);
779
796
  // Currency: this exit queried NOBODY, so it refutes nothing — an earlier pass's unsettled
780
797
  // claim survives it. Note the coupling with the arming just above: retained doubt now
781
798
  // persists for up to `readRepairWindowMs` before a consult can refute it. That is correct
@@ -949,12 +966,12 @@ export class CoordinatorRepo {
949
966
  // unsettled claim (`recordAheadClaim`) and keeps stamping reads served below it while the
950
967
  // window suppresses the retry — the window damps repair effort, not honesty.
951
968
  // NOTE: that damping covers only a block this node holds at an OLDER revision. A block entirely
952
- // missing locally never consults the window (`get` triggers on `isMissing` before
953
- // `shouldReadRepair`), so a persistently failing acquisition — e.g. a two-node deployment that
954
- // never set `assumedClusterSize`, where the content quorum can never be met — re-fetches an
969
+ // missing locally never consults the window (`get` consults on every read of a missing block,
970
+ // before `shouldReadRepair`), so a persistently failing acquisition — e.g. a two-node deployment
971
+ // that never set `assumedClusterSize`, where the content quorum can never be met — re-fetches an
955
972
  // archive on every read of that block. Correct, and self-limiting once the cohort can agree; if
956
973
  // it ever shows as read amplification, gate the acquisition step (not the latest-query) on the
957
- // same window rather than widening `isMissing`.
974
+ // same window rather than remembering the absence (GitHub issue #20).
958
975
  this.markBlocksSeen([blockId]);
959
976
  // Converged: the corroboration is itself the evidence that nothing is ahead, and it came from
960
977
  // peers that answered — the shared verdict resolves to `nothing-ahead`, and the memo retires
@@ -1514,12 +1531,14 @@ export class CoordinatorRepo {
1514
1531
  return result;
1515
1532
  }
1516
1533
  /** The cluster half of {@link pend}, after responsibility is verified: consensus, the local-verdict
1517
- * arms, and the two optimistic-concurrency classifiers a rejection is run through. */
1534
+ * arms, and the catch that turns each optimistic-concurrency refusal into a retryable answer
1535
+ * the two signed-evidence ones ({@link ConflictRaceLostError}, {@link BlocksHeldError}) directly,
1536
+ * and a validator rejection only through {@link classifyStaleRejection}. */
1518
1537
  async pendThroughCluster(request, allBlockIds, options) {
1519
1538
  const coordinatingBlockIds = options?.coordinatingBlockIds ?? allBlockIds;
1520
- const peerCount = await this.coordinator.getClusterSize(coordinatingBlockIds[0]);
1521
- if (peerCount <= 1) {
1522
- return await this.storageRepo.pend(request, options);
1539
+ const cohort = await this.coordinator.resolveCohort(coordinatingBlockIds[0]);
1540
+ if (!cohort.resolved || cohort.peerIds.length <= 1) {
1541
+ return await this.pendSolo(request, cohort, options);
1523
1542
  }
1524
1543
  const message = {
1525
1544
  operations: [{ pend: request }],
@@ -1527,7 +1546,13 @@ export class CoordinatorRepo {
1527
1546
  coordinatingBlockIds
1528
1547
  };
1529
1548
  try {
1530
- const { localExecuted, localPendResult, cohortPendRefusals } = await this.coordinator.executeClusterTransaction(coordinatingBlockIds[0], message, options);
1549
+ const { record, localExecuted, localPendResult, cohortPendRefusals } = await this.coordinator.executeClusterTransaction(coordinatingBlockIds[0], message, options);
1550
+ // Who ACCEPTED the pending record — the cohort members whose promise vote approved it, plus
1551
+ // or minus this node's own member per exit below. A pend's `confirmed` is not a commit's:
1552
+ // accepting a pending record confers no storage durability (a pend that reached
1553
+ // pend-consensus may still have been stored by nobody — see the local-verdict arm below),
1554
+ // so the two numbers are never comparable, and the field says so.
1555
+ const pendDurability = (selfAccepted) => pendCohortDurability(record, selfAccepted, this.localPeerId?.toString());
1531
1556
  // The first cohort refusal in peer-id order, so two coordinators facing the same cohort
1532
1557
  // answer with the same one. Which refusal is reported does not change the outcome — every
1533
1558
  // entry is conflict-shaped and every one means "rebase and retry" — only which `pending` /
@@ -1578,7 +1603,9 @@ export class CoordinatorRepo {
1578
1603
  hasMissing: !!result.missing?.length,
1579
1604
  hasPending: !!result.pending?.length
1580
1605
  });
1581
- return answerWithCohortRefusal(result);
1606
+ // Storage's own answer is `local`; the cohort's answer replaces it — this node accepted
1607
+ // the record iff its fallback pend succeeded.
1608
+ return answerWithCohortRefusal(result.success ? { ...result, durability: pendDurability(true) } : result);
1582
1609
  }
1583
1610
  // Local cluster already executed during consensus — return storage's own verdict rather
1584
1611
  // than fabricating a success (the peerCount <= 1 path above returns storage's real result
@@ -1591,7 +1618,10 @@ export class CoordinatorRepo {
1591
1618
  // commit that reached commit-consensus IS the authoritative commit (Theorem 9), whereas a
1592
1619
  // pend that reached pend-consensus may still have been stored by nobody.
1593
1620
  if (localPendResult !== undefined) {
1594
- if (localPendResult.success || isConflictFailure(localPendResult)) {
1621
+ if (localPendResult.success) {
1622
+ return answerWithCohortRefusal({ ...localPendResult, durability: pendDurability(true) });
1623
+ }
1624
+ if (isConflictFailure(localPendResult)) {
1595
1625
  return answerWithCohortRefusal(localPendResult);
1596
1626
  }
1597
1627
  // A bare-reason refusal (no pending/missing — e.g. a local validation-hook fault)
@@ -1605,11 +1635,14 @@ export class CoordinatorRepo {
1605
1635
  }
1606
1636
  // No verdict retained (member predates retention, restart, or TTL): the prior shape,
1607
1637
  // still subject to the cohort-refusal rule — a lost local verdict must not resurrect the
1608
- // fabricated success this ticket exists to remove.
1638
+ // fabricated success this ticket exists to remove. For the durability class, a tolerated
1639
+ // local fault means this node did NOT accept (`false`: its approve vote is withdrawn from the
1640
+ // count), and an absent verdict says nothing either way (`undefined`: the vote stands).
1609
1641
  return answerWithCohortRefusal({
1610
1642
  success: true,
1611
1643
  pending: [],
1612
- blockIds: allBlockIds
1644
+ blockIds: allBlockIds,
1645
+ durability: pendDurability(localPendResult === undefined ? undefined : false)
1613
1646
  });
1614
1647
  }
1615
1648
  catch (error) {
@@ -1631,6 +1664,14 @@ export class CoordinatorRepo {
1631
1664
  // each member compares the rivals as IT holds them, so a member that already approved X
1632
1665
  // keeps X while a member that approved Y first keeps Y, and no rival reaches a promise
1633
1666
  // supermajority. (The hash tie-break is already symmetric — it cannot be the fix.)
1667
+ // TWO contenders coordinated by different nodes reach an all-lose round by another route,
1668
+ // and not by this catch: both collect their promises in parallel, so BOTH reach pend
1669
+ // consensus; each coordinator's own member applies its own pend first (local-first
1670
+ // delivery), every other member's storage refuses whichever arrived second, and each writer
1671
+ // is downgraded by `answerWithCohortRefusal` above. Measured by
1672
+ // `test/transaction-node-count-sweep.spec.ts` on the in-process mesh: round one lost by both
1673
+ // writers in 20 of 20 races across two to five machines (2026-09-16). Same cost and same
1674
+ // revisit condition as below.
1634
1675
  // Fine as it stands: since the torn-action fixes landed, an all-lose round costs one
1635
1676
  // retry cycle rather than wedging, and the contenders are separated next round by the
1636
1677
  // jittered backoff plus the aged retry priority carried on the re-pend
@@ -1644,13 +1685,49 @@ export class CoordinatorRepo {
1644
1685
  if (error instanceof ConflictRaceLostError) {
1645
1686
  return { success: false, conflict: true, reason: error.message };
1646
1687
  }
1647
- const stale = await this.classifyStaleRejection(error, request, allBlockIds)
1648
- ?? await this.classifyPendingConflictRejection(error, request, allBlockIds);
1688
+ if (error instanceof BlocksHeldError) {
1689
+ return await this.answerBlocksHeld(error, request, allBlockIds, cohort.peerIds.length);
1690
+ }
1691
+ const stale = await this.classifyStaleRejection(error, request, allBlockIds);
1649
1692
  if (stale)
1650
1693
  return stale;
1651
1694
  throw error;
1652
1695
  }
1653
1696
  }
1697
+ /**
1698
+ * The solo short-circuit of {@link pend}: a cohort of at most one peer runs no consensus, and the
1699
+ * pend goes straight to local storage exactly as before. What changes is the answer's class —
1700
+ * see {@link soloCohortDurability} for the four-way split. A refusal is returned untouched.
1701
+ */
1702
+ async pendSolo(request, cohort, options) {
1703
+ const result = await this.storageRepo.pend(request, options);
1704
+ return result.success ? { ...result, durability: this.soloCohortDurability(cohort) } : result;
1705
+ }
1706
+ /**
1707
+ * The class of a write that took the solo short-circuit, from the cohort resolution that put it
1708
+ * there. Policy-free: the write already happened, this only says what it was.
1709
+ * - not resolved (the lookup threw, or named nobody) → `unrouted`;
1710
+ * - resolved to exactly this node → `local`, a correct and complete one-machine write;
1711
+ * - resolved to exactly one peer that is NOT this node → `unrouted` — this node wrote somewhere
1712
+ * the cohort does not look, which is a write whose destination is wrong, not a solo write.
1713
+ * With no local peer id (direct constructors, single-node and test wiring) a resolved cohort of
1714
+ * one reads as `local`: the same posture `responsibilityFor` takes for a node with no
1715
+ * identity, and a node with no identity cannot be in any cohort but its own.
1716
+ *
1717
+ * Both `unrouted` arms are defence in depth now, not the main guard: {@link verifyResponsibility}
1718
+ * refuses a write whose lookup throws and a write for a cohort that excludes this node before the
1719
+ * short-circuit is reached. A write lands here unrouted only inside the responsibility cache's
1720
+ * staleness window, or when a second lookup fails moments after the first succeeded — and then it
1721
+ * still reports honestly rather than posing as `local`.
1722
+ */
1723
+ soloCohortDurability(cohort) {
1724
+ if (!cohort.resolved)
1725
+ return unroutedDurability();
1726
+ const sole = cohort.peerIds[0];
1727
+ if (this.localPeerId !== undefined && sole !== this.localPeerId.toString())
1728
+ return unroutedDurability();
1729
+ return localDurability(sole);
1730
+ }
1654
1731
  /**
1655
1732
  * Decide whether a cluster validator rejection was an optimistic-concurrency loss — the block
1656
1733
  * already advanced past the requested revision — rather than a genuine validation fault.
@@ -1722,51 +1799,56 @@ export class CoordinatorRepo {
1722
1799
  };
1723
1800
  }
1724
1801
  // NOTE: conservative — when only remote members saw the newer revision (local storage still
1725
- // behind), staleness can't be confirmed locally and the rejection stays a throw. If that
1726
- // shows up in practice, extend confirmation with a quorum read; never trust the reject text.
1727
- // `staleAt` is absent on this path for the same reason, and deliberately so there is no
1728
- // confirmed number to report, and the field's contract forbids inferring one from that text.
1802
+ // behind), staleness can't be confirmed locally and the rejection stays a throw. That revisit
1803
+ // condition HAS now tripped, for this method's former sibling: the pending-conflict refusal hit
1804
+ // exactly this window under delivery latency (the refusing member ahead of the coordinator) and
1805
+ // escaped as a permanent-looking throw. Its cure was not the quorum read suggested here but a
1806
+ // signed `held` vote — the member's own refusal, carried as evidence, so retryability stopped
1807
+ // depending on a local re-read at all (see {@link answerBlocksHeld}).
1808
+ //
1809
+ // The stale arm is deliberately LEFT on local corroboration. Its refusal is a revision claim,
1810
+ // and `staleAt` — the only place a losing writer learns the revision it lost to — can only be
1811
+ // reported from a revision this node read itself. Moving it to a non-counting vote would take
1812
+ // that number away, so it is its own ticket, not a rider on this one. Never trust the reject
1813
+ // text; `staleAt` is absent on the unconfirmed path for the same reason, and deliberately so.
1729
1814
  return undefined;
1730
1815
  }
1731
1816
  /**
1732
- * Sibling of {@link classifyStaleRejection} for the OTHER optimistic-concurrency refusal shape:
1733
- * the promise-phase pending-conflict vote (`validatePendOperations` rejecting a pend whose
1734
- * blocks are held by a different unresolved pending action). That vote surfaces here as a
1735
- * {@link ValidatorRejectionError}, and without classification it would escape as a throw —
1736
- * splitting multi-tree pends mid-batch instead of taking the retry path a lost race deserves.
1817
+ * Answer a pend the cohort refused with signed `held` votes: the blocks are reserved by a different
1818
+ * unresolved action on one or more members ({@link BlocksHeldError}). Always a retryable
1819
+ * {@link StaleFailure} with `conflict: true`, so the caller's normal retry path
1820
+ * (`Collection.sync`, and the multi-collection pendPhase via `isConflictFailure`) absorbs it
1821
+ * instead of a thrown error escaping mid-batch and splitting multi-tree pends.
1737
1822
  *
1738
- * Same confirmation discipline as the stale classifier: purely local. Re-read the affected
1739
- * blocks from our own storage and require some block's `state.pendings` to carry a rival
1740
- * actionId; the signed reject text is never consulted. A confirmed rival returns a
1741
- * {@link StaleFailure} with `conflict: true` and the rivals as `pending` (`ActionPending`
1742
- * without `transform` the type allows it, and no consumer rebases from it). Unconfirmed
1743
- * including read errors during confirmation stays a throw, preserving fail-fast for genuine
1744
- * validation faults. Checked after `classifyStaleRejection` so a confirmed committed loss
1745
- * (which carries the sharper `staleAt`) wins when both hold.
1823
+ * The local re-read is an ENRICHER, not a gate. It used to be the gate — this refusal arrived as a
1824
+ * `ValidatorRejectionError` and was only converted when the coordinator's own storage corroborated
1825
+ * the rival — and under latency the refusing member is routinely ahead of the coordinator, so the
1826
+ * corroboration missed and a transient refusal escaped as a permanent-looking throw. The `held`
1827
+ * vote is already signed evidence from the member that holds the rival, so retryability no longer
1828
+ * depends on a local re-read at all. What the re-read still buys, when it succeeds, is the concrete
1829
+ * rival list for {@link StaleFailure.pending} and the input {@link noteStuckReservation} needs to
1830
+ * name a block wedged behind a reservation that will never clear.
1831
+ *
1832
+ * NOTE: `error.heldBy` (peerId → holding action id) is dropped at this boundary, the same deliberate
1833
+ * drop the {@link ConflictRaceLostError} arm of {@link pendThroughCluster} documents: `StaleFailure`
1834
+ * has no field for it, and the members' action ids are not the same claim as the rivals this node
1835
+ * read out of its own storage, so they must not be folded into `pending`. A caller that wants to
1836
+ * WAIT on the holder rather than re-race it needs a typed field added here; never recover the ids
1837
+ * by parsing `reason`.
1746
1838
  */
1747
- async classifyPendingConflictRejection(error, request, blockIds) {
1748
- if (!(error instanceof ValidatorRejectionError))
1749
- return undefined;
1750
- let results;
1751
- try {
1752
- results = await this.storageRepo.get({ blockIds });
1753
- }
1754
- catch (readError) {
1755
- this.log('coordinator-repo:pend-conflict-classify-read-error', {
1839
+ async answerBlocksHeld(error, request, blockIds, peerCount) {
1840
+ const pending = await this.corroborateHeldBlocks(request, blockIds, peerCount);
1841
+ if (pending.length === 0) {
1842
+ // The member that refused is ahead of us — the normal shape under latency. Still a conflict:
1843
+ // the refusal is signed, and the un-enriched answer is exactly the shape the lost-race arm
1844
+ // returns. `noteStuckReservation` is deliberately not fed here; it counts refusals whose
1845
+ // holder THIS node can name, and a guess would poison its holder comparison.
1846
+ this.log('coordinator-repo:pend-held-uncorroborated', {
1756
1847
  actionId: request.actionId,
1757
- error: readError.message
1848
+ heldBy: error.heldBy
1758
1849
  });
1759
- return undefined;
1760
- }
1761
- const pending = [];
1762
- for (const blockId of blockIds) {
1763
- for (const actionId of results[blockId]?.state?.pendings ?? []) {
1764
- if (actionId !== request.actionId)
1765
- pending.push({ blockId, actionId });
1766
- }
1850
+ return { success: false, conflict: true, reason: error.message };
1767
1851
  }
1768
- if (pending.length === 0)
1769
- return undefined;
1770
1852
  // Counted as its own statement, never inside the log payload below: this call is the detection
1771
1853
  // mechanism, not a formatting step, and payload expressions in this repo are fair game to wrap
1772
1854
  // in an `enabled` gate (`Collection.advanceContext` does exactly that). A gate added there
@@ -1791,45 +1873,86 @@ export class CoordinatorRepo {
1791
1873
  };
1792
1874
  }
1793
1875
  /**
1794
- * Count one confirmed pending-conflict refusal against the holder(s) of each block it names, and say
1795
- * ONCE in words, at the moment it becomes provable when a block is wedged behind a reservation
1796
- * that is not going to clear.
1797
- *
1798
- * **Why this needs saying at all.** Every individual refusal here is indistinguishable from an
1799
- * ordinary lost race, which is a normal and healthy event, so the logs of a permanently wedged block
1800
- * read exactly like the logs of a busy one. Finding the difference today means noticing that the
1801
- * SAME rival action id keeps appearing across unrelated writers for as long as the process lives —
1802
- * a pattern nothing points at, and one that cost a downstream project several tickets and weeks to
1803
- * re-derive from raw traces. The node has the fact in hand at every refusal; this makes it sayable.
1876
+ * The rival unresolved actions THIS node's own storage says hold `blockIds`, excluding the request's
1877
+ * own action (a redelivered pend must not corroborate against itself). Empty when nothing is found
1878
+ * including when the read itself fails, which is reported and then treated as "nothing to add",
1879
+ * since the caller's answer does not depend on it.
1804
1880
  *
1805
- * **The signal, and the two things that are NOT the signal.** The discriminator is repetition
1806
- * against an unchanged holder — see {@link STUCK_RESERVATION_DISTINCT_ACTIONS} for why distinct
1807
- * refused actions is the right counter and for the measured threshold. Two cheaper-looking tests
1808
- * were tried and do not work: the members' in-memory reservation table
1809
- * (`ClusterMember.activeTransactions`) clears the moment a rival's pend reaches consensus, so a
1810
- * perfectly healthy rival inside its pend-to-commit window is absent from it too and absence there
1811
- * says nothing; and "the block already passed this pending record's revision" catches a different
1812
- * orphan class entirely — in the verified instance the wedged block sat at revision 1 while the
1813
- * orphaned record was for revision 2, still nominally promotable.
1881
+ * Carries no `transform`, which {@link ActionPending} allows and no consumer rebases from.
1814
1882
  *
1815
- * **Never a control path.** This classifies and logs; it never refuses, expires, or deletes
1816
- * anything. Deciding when a durable pending record may be removed is precisely the hard problem
1817
- * backlog `debt-unpromotable-pending-records-need-a-sweep` exists for — deleting a live reservation
1818
- * is worse than the leak — and a counter accurate enough for a log line is not evidence enough to
1819
- * destroy state.
1883
+ * Diagnostic enrichment only, never a gate: the refusal it corroborates was already cast and signed
1884
+ * by a member, and nothing here can turn an admission into a refusal or the reverse.
1885
+ */
1886
+ async corroborateHeldBlocks(request, blockIds, peerCount) {
1887
+ const pending = [];
1888
+ // The refusing member reads the request's base only in a cohort that can leave a member out
1889
+ // (`ClusterMember.reservationRequestOf`); read it here under the same condition.
1890
+ const readsBase = cohortCanMissAPend(peerCount, this.superMajorityThreshold);
1891
+ try {
1892
+ for (const blockId of blockIds) {
1893
+ // The request's base for the block, read exactly as the refusing member read it (the
1894
+ // malformed-base log line is that member's, so none is repeated here).
1895
+ const reservation = readsBase
1896
+ ? reservationRequestFor(request, blockId, transformForBlockId(request.transforms, blockId)).request
1897
+ : { rev: request.rev };
1898
+ for (const claim of await this.pendingClaimsOf(blockId)) {
1899
+ // The same rule, on the same inputs, as the refusing member (`isReservationAgainst`): a
1900
+ // record this request's writer has built on is not the rival it was refused on, and
1901
+ // naming it would feed a superseded action into the writer's `pending` list and into
1902
+ // the stuck-reservation holder comparison; and, where the base is read, a record
1903
+ // claiming a slot past the request's declared base IS one, even when the requested
1904
+ // revision has moved beyond it — the revision rule alone would leave that refusal
1905
+ // uncorroborated.
1906
+ if (claim.actionId !== request.actionId && isReservationAgainst(claim, reservation)) {
1907
+ pending.push({ blockId, actionId: claim.actionId });
1908
+ }
1909
+ }
1910
+ }
1911
+ }
1912
+ catch (readError) {
1913
+ this.log('coordinator-repo:pend-conflict-classify-read-error', {
1914
+ actionId: request.actionId,
1915
+ error: readError.message
1916
+ });
1917
+ return [];
1918
+ }
1919
+ return pending;
1920
+ }
1921
+ /**
1922
+ * This node's pending records on `blockId`, each with the slot it claims when storage can say
1923
+ * (`IPendingClaimReader`, which `StorageRepo` implements). A repo without the capability answers
1924
+ * with `state.pendings` alone, every record read as an unknown claim — which reserves, so the
1925
+ * corroboration can only over-name, never under-name.
1926
+ */
1927
+ async pendingClaimsOf(blockId) {
1928
+ const reader = this.storageRepo;
1929
+ if (typeof reader.listPendingClaims === 'function') {
1930
+ return await reader.listPendingClaims(blockId);
1931
+ }
1932
+ const results = await this.storageRepo.get({ blockIds: [blockId] });
1933
+ return (results[blockId]?.state?.pendings ?? []).map(actionId => ({ actionId }));
1934
+ }
1935
+ /**
1936
+ * Count one confirmed pending-conflict refusal against the holder(s) of each block it names, and say
1937
+ * ONCE — in words, at the moment it becomes provable — when a block is wedged behind a reservation
1938
+ * that is not going to clear. The counting, the threshold and the wording live in
1939
+ * {@link StuckReservationTracker}; this node's part is the corroborated holder list and the tag.
1820
1940
  *
1821
1941
  * Returns the highest distinct-refusal count any of this refusal's blocks has now reached, for the
1822
1942
  * classification line to carry; it saturates at the threshold once an episode has been reported,
1823
1943
  * since the ids are dropped at that point.
1824
1944
  *
1825
- * NOTE: fed only by {@link classifyPendingConflictRejection}, i.e. by refusals that arrive as a
1826
- * cohort-wide validator rejection. A block only PART of whose cohort holds the stranded record can
1945
+ * NOTE: fed only by {@link answerBlocksHeld}, i.e. by cohort-wide `held`-answered refusals that this
1946
+ * node's own storage can corroborate. A block only PART of whose cohort holds the stranded record can
1827
1947
  * still reach approval super-majority, and its refusal then comes back through the retained local
1828
- * apply verdict (`getExecutedPendResult`) instead, which this never sees so a partially wedged
1829
- * block goes unnamed. That is the weaker condition (the write does land on the healthy members),
1830
- * and instrumenting the second path would count a refusal that the cohort as a whole did not make.
1831
- * If partial strands ever turn out to be the common shape in the field, the counter belongs on the
1832
- * member side (`ClusterMember.validatePendOperations`), where each member sees its own votes.
1948
+ * apply verdict (`getExecutedPendResult`) instead, which this never sees. The un-corroborated arm of
1949
+ * {@link answerBlocksHeld} is likewise unfed: this node cannot name the holder, and feeding it a
1950
+ * guess would poison the holder comparison, which is what separates a wedge from healthy
1951
+ * contention. Neither gap goes unnamed any more: every MEMBER keeps its own tracker, fed by its own
1952
+ * `held` votes (`ClusterMember.validatePendOperations`, tag `cluster-member:stuck-reservation`), so
1953
+ * a reservation only some members hold — the shape `a-member-that-missed-a-commit-refuses-every-
1954
+ * later-write` measured, before the member stopped refusing on a superseded record at all — is
1955
+ * named by the members that hold it.
1833
1956
  */
1834
1957
  noteStuckReservation(pending, refusedActionId) {
1835
1958
  const rivalsByBlock = new Map();
@@ -1840,38 +1963,9 @@ export class CoordinatorRepo {
1840
1963
  else
1841
1964
  rivalsByBlock.set(blockId, [actionId]);
1842
1965
  }
1843
- let highest = 0;
1844
- for (const [blockId, rivals] of rivalsByBlock) {
1845
- const holders = [...new Set(rivals)].sort();
1846
- const prior = this.stuckReservations.get(blockId);
1847
- // A different holder set is a DIFFERENT episode — the block changed hands, which is the
1848
- // healthy cycle — so the count starts over and the new holder gets its own chance to speak.
1849
- const watch = prior !== undefined && sameHolders(prior.holders, holders)
1850
- ? prior
1851
- : { holders, refused: new Set(), reported: false };
1852
- if (watch !== prior)
1853
- this.stuckReservations.set(blockId, watch);
1854
- if (watch.reported) {
1855
- highest = Math.max(highest, STUCK_RESERVATION_DISTINCT_ACTIONS);
1856
- continue;
1857
- }
1858
- watch.refused.add(refusedActionId);
1859
- highest = Math.max(highest, watch.refused.size);
1860
- if (watch.refused.size < STUCK_RESERVATION_DISTINCT_ACTIONS)
1861
- continue;
1862
- this.log('coordinator-repo:stuck-reservation', {
1863
- blockId,
1864
- // The ids an operator needs to grep for and to cancel, kept as data beside the prose so a
1865
- // log search finds the block and the action without parsing English.
1866
- holdingActionIds: holders,
1867
- distinctRefusedActions: watch.refused.size,
1868
- message: stuckReservationMessage(holders, watch.refused.size)
1869
- });
1870
- watch.reported = true;
1871
- // Said once per episode: from here the flag alone suppresses, and the ids have done their
1872
- // work (their count is in the line above), so drop them rather than growing a set for the
1873
- // unbounded remainder of a permanent condition.
1874
- watch.refused.clear();
1966
+ const { highest, named } = this.stuckReservations.note(rivalsByBlock, refusedActionId);
1967
+ for (const episode of named) {
1968
+ this.log('coordinator-repo:stuck-reservation', episode);
1875
1969
  }
1876
1970
  return highest;
1877
1971
  }
@@ -1889,11 +1983,7 @@ export class CoordinatorRepo {
1889
1983
  * episode holding an LRU slot can only evict a live one.
1890
1984
  */
1891
1985
  clearStuckReservations(blockIds, holderActionId) {
1892
- for (const blockId of blockIds) {
1893
- if (holderActionId !== undefined && !this.stuckReservations.peek(blockId)?.holders.includes(holderActionId))
1894
- continue;
1895
- this.stuckReservations.delete(blockId);
1896
- }
1986
+ this.stuckReservations.forget(blockIds, holderActionId);
1897
1987
  }
1898
1988
  async cancel(actionRef, options) {
1899
1989
  const blockIds = actionRef.blockIds;
@@ -1914,11 +2004,12 @@ export class CoordinatorRepo {
1914
2004
  //
1915
2005
  // NOTE: `getClusterSize` is a second `findCluster` for the same key that
1916
2006
  // `executeClusterTransaction` is about to look up again, so a cancel over N blocks now
1917
- // costs 2N cohort lookups instead of N. Same shape `pend` and `commit` already pay, but
1918
- // they pay it once (they only ever consult `blockIds[0]`) where this scales with N. Fine
1919
- // while cancels span a handful of blocks; if wide multi-block cancels ever show up hot,
1920
- // have `executeClusterTransaction` return the cohort it already fetched (or own the
1921
- // short-circuit itself) rather than adding a cache here.
2007
+ // costs 2N cohort lookups instead of N. `pend` and `commit` pay the same doubled lookup
2008
+ // for a multi-peer cohort, but theirs is CONSTANT — they only ever consult `blockIds[0]`
2009
+ // — where this scales with N. Fine while cancels span a handful of blocks; if wide
2010
+ // multi-block cancels ever show up hot, have `executeClusterTransaction` return the
2011
+ // cohort it already fetched (or own the short-circuit itself) rather than adding a cache
2012
+ // here.
1922
2013
  const results = await Promise.all(blockIds.map(async (blockId) => {
1923
2014
  const peerCount = await this.coordinator.getClusterSize(blockId);
1924
2015
  if (peerCount <= 1)
@@ -1945,57 +2036,17 @@ export class CoordinatorRepo {
1945
2036
  async commit(request, options) {
1946
2037
  const blockIds = request.blockIds;
1947
2038
  await this.verifyResponsibility(blockIds);
1948
- const cohortPeerIds = await this.coordinator.getClusterPeerIds(blockIds[0]);
1949
- const peerCount = cohortPeerIds.length;
1950
- if (peerCount <= 1) {
1951
- // Solo cohort: consensus never runs, so no ClusterRecord exists to project a proof from —
1952
- // the lone member self-signs a one-peer proof instead (mintSoloCommitProof), which is what
1953
- // lets a block born on a cohort of one ever gain a second holder under the certified-push
1954
- // default (handlePush refuses a proof-less block). Minted even when peerCount is 0 or the
1955
- // sole peer is not self — findCluster failing (getClusterPeerIds returns []) puts the
1956
- // DEGRADED-ROUTING case in this same branch, and self genuinely committed these bytes
1957
- // either way; a proof's peer list is already not evidence of cohort membership by design
1958
- // (caller obligation #1 on verifyBlockCommitProofClaim), so gating the mint on cohort
1959
- // composition would buy no safety while opening a silent no-proof hole exactly when
1960
- // routing is degraded. The log line is how an operator tells a real cohort of one
1961
- // (cohortSize 1, soleIsSelf true) from a routing failure (cohortSize 0, or a sole peer
1962
- // that is not this node).
1963
- this.log('commit:solo-cohort', {
1964
- blockId: blockIds[0],
1965
- cohortSize: peerCount,
1966
- soleIsSelf: peerCount === 1 && this.localPeerId !== undefined
1967
- && cohortPeerIds[0] === this.localPeerId.toString()
1968
- });
1969
- // Same message shape the multi-peer path produces — executeClusterTransaction stamps
1970
- // coordinatingBlockIds at its choke point, so the solo artifact must carry it too or a
1971
- // solo proof's message is distinguishable from every other proof's.
1972
- const message = {
1973
- operations: [{ commit: request }],
1974
- coordinatingBlockIds: [blockIds[0]],
1975
- expiration: options?.expiration ?? Date.now() + this.DEFAULT_TIMEOUT
1976
- };
1977
- // `undefined` when no local cluster is wired (direct constructors, unit-test doubles) —
1978
- // then the commit lands proof-less, exactly the pre-mint behavior. The cast is the named
1979
- // ICommitProofPersister contract; a plain IRepo double ignores the extra argument.
1980
- const proof = await this.localCluster?.mintSoloCommitProof?.(message);
1981
- const result = await this.storageRepo.commit(request, options, proof);
1982
- // One self-approval arms the read-repair window only where the DECLARED cohort is also one
1983
- // — then no rival quorum can exist to be missed. At any larger declared size (including
1984
- // an undeclared one, which resolves to the replication factor, and including degraded
1985
- // routing where peerCount is 0) this commit proves nothing about rival quorums — see
1986
- // commitQuorumRulesOutRivals — so the window stays unarmed and the read path's
1987
- // solo-self-skip exit re-arms it once per consult instead (which keeps GitHub issue #8's
1988
- // consult storm bounded at one per window).
1989
- if (result.success && this.commitQuorumRulesOutRivals(1, peerCount))
1990
- this.markBlocksSeen(blockIds);
1991
- return result;
2039
+ const cohort = await this.coordinator.resolveCohort(blockIds[0]);
2040
+ if (!cohort.resolved || cohort.peerIds.length <= 1) {
2041
+ return await this.commitSolo(request, blockIds, cohort, options);
1992
2042
  }
2043
+ const peerCount = cohort.peerIds.length;
1993
2044
  const message = {
1994
2045
  operations: [{ commit: request }],
1995
2046
  expiration: options?.expiration ?? Date.now() + this.DEFAULT_TIMEOUT
1996
2047
  };
1997
2048
  try {
1998
- const { record, localExecuted, localCommitResult } = await this.coordinator.executeClusterTransaction(blockIds[0], message, options);
2049
+ const { record, localExecuted, localCommitResult, cohortCommitOutcomes } = await this.coordinator.executeClusterTransaction(blockIds[0], message, options);
1999
2050
  // Decided once for every success shape below (local-executed, local fallback, tolerated
2000
2051
  // divergence): whether this commit's quorum is freshness evidence or merely a commit.
2001
2052
  // NOTE: one verdict covers every block in `blockIds`, though it is measured against
@@ -2004,20 +2055,54 @@ export class CoordinatorRepo {
2004
2055
  // quorum that never voted. If commits ever coordinate per-block cohorts separately (see
2005
2056
  // `debt-sender-side-coordinating-block-binding-is-unchecked`), this must follow them.
2006
2057
  const armFreshness = this.commitQuorumRulesOutRivals(countApprovingCommitVotes(record), peerCount);
2058
+ // THE DURABILITY GATE. A commit is acknowledged to the writer only when more than half of
2059
+ // the cohort it ran on (`record.peers`) reports that its storage durably holds the
2060
+ // committed revision under this action, each member measured after its own reconcile.
2061
+ // Consensus votes are necessary for the commit to be authoritative; they were never
2062
+ // evidence that it was stored — every member can sign, then refuse at apply (no base for
2063
+ // the block, a missed pend), and the coordinator used to count the votes as success. The
2064
+ // same strict majority `clusterReachedCommitConsensus` measures votes against; the
2065
+ // freshness verdict above stays tied to the vote count, the two quorums are not
2066
+ // conflated. Other members' reports arrive as `cohortCommitOutcomes`; this node's own
2067
+ // member is added per arm below — its retained verdict when it executed, its fallback
2068
+ // commit when it did not.
2069
+ //
2070
+ // NOTE: a refusal here means "not confirmed durable at a quorum", never "guaranteed
2071
+ // absent" — the two-phase ambiguity. Members that refused with `missing-base-revision`
2072
+ // have already dropped their pending records, so the writer must cancel and re-drive at a
2073
+ // fresh revision, which is what the `conflict: true` shape makes `Collection.syncAttempts`
2074
+ // and the multi-collection `pendPhase` do. But a member reached only by
2075
+ // `scheduleCommitRetry` can still land the refused revision later, and this node's own
2076
+ // member (or its fallback commit) may hold it already. When the refused block is the
2077
+ // action's LOG TAIL, that matters beyond this block: `NetworkTransactor.commit` stops at a
2078
+ // refused tail and never sweeps the action's other blocks, so the writer can then read
2079
+ // its own log entry back while none of the data it describes was committed anywhere.
2080
+ // The writer's retry with the SAME action id is what converges that, and only because it
2081
+ // does two things: `isOwnRevision` in `StorageRepo.pend` and `commit` treats an
2082
+ // already-landed own revision as satisfied, and `Collection.completeOwnEntry` — on
2083
+ // finding that entry — re-sends the refused attempt at the same revision to land the
2084
+ // blocks left behind BEFORE the write is reported saved. Seeing its own log entry is
2085
+ // never, on its own, the writer's proof of durability.
2086
+ const durability = cohortDurability(record, cohortCommitOutcomes);
2007
2087
  if (localExecuted) {
2008
2088
  // Our own member applied this commit during consensus. Its retained storage verdict is
2009
2089
  // the one honest signal we have about durability: the member-side apply tolerates an
2010
- // "ahead" refusal as divergence (see the NOTE in ClusterMember.applyConsensusOperation),
2090
+ // "ahead" refusal as divergence (see the NOTE in ClusterMember.applyCommitToStorage),
2011
2091
  // which is correct for a redelivered or lagging commit — but when the refusal's real
2012
- // cause is a RIVAL action holding the requested revision, that tolerance turns a commit
2013
- // no member durably stored into a fabricated success. This is the
2092
+ // cause is a RIVAL action holding the requested revision, that tolerance would turn a
2093
+ // commit no member durably stored into a fabricated success. This is the
2014
2094
  // signed-but-not-yet-applied window: two commits for one revision can BOTH assemble
2015
2095
  // consensus when every member signs the second after signing (but before applying) the
2016
2096
  // first, because signing drops the member's reservation. Confirm the rival against local
2017
2097
  // storage (never the verdict's prose) and answer the writer with a retryable conflict so
2018
- // it re-drives at a fresh revision. Own-action or unconfirmed refusals keep the
2019
- // prior fabricated-success shape: consensus is authoritative and this member converges
2020
- // via replication.
2098
+ // it re-drives at a fresh revision. An own-action confirmation only clears the conflict
2099
+ // answer; it does NOT make this node a holder for the gate below. The confirmation fires
2100
+ // on the FIRST block found held under this action, and a multi-block commit torn locally
2101
+ // (`StorageRepo.commit` lands blocks in order and stops at the first refusal) holds some
2102
+ // blocks and not others — whereas the member's retained verdict is already a success
2103
+ // whenever EVERY block is held (`ClusterMember.durableCommitVerdict`, measured after its
2104
+ // reconcile). So the retained verdict is the only local contribution the gate counts;
2105
+ // a refused one, confirmed or not, leaves it to the rest of the cohort.
2021
2106
  //
2022
2107
  // NOTE: a CONFIRMED rival is trusted over the consensus outcome here. That is right in
2023
2108
  // the window this closes (the cohort refused the loser too), but it inverts if the two
@@ -2028,6 +2113,7 @@ export class CoordinatorRepo {
2028
2113
  // after a fork; that is partition-healing scope (docs/partition-healing.md). If forks
2029
2114
  // are ever observed here, weigh the retained verdict against the cohort's votes instead
2030
2115
  // of trusting the local re-read alone.
2116
+ const localDurable = localCommitResult?.success === true;
2031
2117
  if (localCommitResult !== undefined && !localCommitResult.success) {
2032
2118
  const rival = await this.confirmCommitRivalAgainstLocal(request);
2033
2119
  if (typeof rival === 'object')
@@ -2038,23 +2124,45 @@ export class CoordinatorRepo {
2038
2124
  reason: localCommitResult.reason
2039
2125
  });
2040
2126
  }
2127
+ // An absent verdict (a member that predates retention, a restart, the TTL) is not
2128
+ // evidence of holding anything: it is simply not counted — so the class below reads
2129
+ // LOWER than reality for such a member, never higher.
2130
+ const durableHolders = durability.remoteHolders.length + (localDurable ? 1 : 0);
2131
+ if (!isDurableMajority(durableHolders, durability.cohortPeerIds.length)) {
2132
+ return this.refuseCommitNotDurable(request, durableHolders, durability, 'local-executed');
2133
+ }
2041
2134
  if (armFreshness)
2042
2135
  this.markBlocksSeen(blockIds);
2043
- return { success: true };
2136
+ return await this.acknowledgeCommit(request, { success: true, durability: this.cohortWriteDurability(durability, localDurable) }, localDurable, record);
2044
2137
  }
2045
2138
  // Local cluster didn't execute during consensus. Attempt a local commit, but tolerate
2046
- // local divergence when the cluster already reached consensus this coordinator was
2047
- // likely picked for commit after missing the pend phase (unreachable during pend, fresh
2048
- // join, etc.). The cluster's majority is authoritative; this peer catches up via sync.
2139
+ // local divergence when the cluster already reached consensus AND a durable majority of
2140
+ // the cohort reports holding the revision this coordinator was likely picked for commit
2141
+ // after missing the pend phase (unreachable during pend, fresh join, etc.), and it catches
2142
+ // up via replication.
2049
2143
  //
2050
- // Divergence reaches us in BOTH shapes and both must be tolerated identically:
2144
+ // Divergence reaches us in BOTH shapes and both are handled identically:
2051
2145
  // - a THROW ("Pending action … not found"), when we never saw the pend;
2052
2146
  // - a RETURNED `success:false` carrying `missing-base-revision`, when we saw the pend
2053
2147
  // but not the revision that created the block (see StorageRepo.internalCommit).
2054
- // Only the throw was tolerated before the refusal existed. Reporting the refusal to the
2055
- // caller instead would surface a committed transaction as a stale loss: db-core's
2056
- // commitPhase treats any returned `success:false` as a permanent stale failure, so the
2057
- // client would retry an action the cluster already landed until it exhausted its budget.
2148
+ // Reporting a tolerated divergence to the caller as the raw refusal would surface a
2149
+ // committed transaction as a stale loss: db-core's commitPhase treats any returned
2150
+ // `success:false` as a stale loss, and its coordinator re-drives the refused collection
2151
+ // (alone, even after a sibling collection committed) until its retry budget is spent. So
2152
+ // a divergence on a durable majority is reported as success, and one WITHOUT a durable
2153
+ // majority as the durability gate's retryable refusal (`tolerateLocalCommitDivergence`),
2154
+ // which that re-drive is what clears.
2155
+ //
2156
+ // The gate is evaluated BEFORE the local fallback commit, and a failing gate skips it.
2157
+ // This node counts toward the majority only when it is in the cohort the commit ran on;
2158
+ // an off-cohort coordinator's copy is a lone holder no cohort member will ever reconcile
2159
+ // from (members reconcile from `record.peers`), and a refused commit must not create one —
2160
+ // that lone off-cohort copy is exactly the seed of the "revision exists on one node that
2161
+ // is not responsible for it" placement the durability gate exists to prevent.
2162
+ const selfInCohort = this.localPeerId !== undefined && this.localPeerId.toString() in record.peers;
2163
+ if (!isDurableMajority(durability.remoteHolders.length + (selfInCohort ? 1 : 0), durability.cohortPeerIds.length)) {
2164
+ return this.refuseCommitNotDurable(request, durability.remoteHolders.length, durability, 'fallback-skipped');
2165
+ }
2058
2166
  //
2059
2167
  // Deliberately NOT self-signed here (unlike the solo short-circuit above): consensus for
2060
2168
  // this commit ran on the cohort, so a one-peer minted proof would be a FALSE statement
@@ -2068,18 +2176,24 @@ export class CoordinatorRepo {
2068
2176
  try {
2069
2177
  const result = await this.storageRepo.commit(request, options, consensusProof);
2070
2178
  if (result.success) {
2179
+ // The gate above already admitted this shape: the remote holders plus this node
2180
+ // (when it is in the cohort) form the majority, and this node now holds it.
2181
+ // Storage's own `local` answer is replaced by the cohort's — this node counts exactly
2182
+ // as the gate counted it, i.e. only when it is a cohort member.
2071
2183
  if (armFreshness)
2072
2184
  this.markBlocksSeen(blockIds);
2073
- return result;
2185
+ // The fallback commit landed every block, so this node holds them — whether or not
2186
+ // it is a cohort member — and can source a push to whoever the class names as missing.
2187
+ return await this.acknowledgeCommit(request, { ...result, durability: this.cohortWriteDurability(durability, selfInCohort) }, true, record);
2074
2188
  }
2075
2189
  if (isMissingBaseRevisionFailure(result) && clusterReachedCommitConsensus(record)) {
2076
- return this.tolerateLocalCommitDivergence(request, blockIds, result.reason ?? MISSING_BASE_REVISION_REASON, armFreshness);
2190
+ return await this.tolerateLocalCommitDivergence(request, blockIds, result.reason ?? MISSING_BASE_REVISION_REASON, armFreshness, durability);
2077
2191
  }
2078
2192
  return result;
2079
2193
  }
2080
2194
  catch (err) {
2081
2195
  if (clusterReachedCommitConsensus(record)) {
2082
- return this.tolerateLocalCommitDivergence(request, blockIds, err.message, armFreshness);
2196
+ return await this.tolerateLocalCommitDivergence(request, blockIds, err.message, armFreshness, durability);
2083
2197
  }
2084
2198
  throw err;
2085
2199
  }
@@ -2224,20 +2338,294 @@ export class CoordinatorRepo {
2224
2338
  };
2225
2339
  }
2226
2340
  /**
2227
- * Report success for a commit the cluster carried but this peer could not apply locally.
2228
- * Convergence comes from replication (cohort reconcile, or read-driven acquisition), not from
2229
- * replay here. `armFreshness` says whether the commit's quorum was strong enough
2230
- * ({@link commitQuorumRulesOutRivals}) for the read path to treat the blocks as
2231
- * freshness-checked; a divergence tolerated on a downsized quorum leaves the window unarmed
2232
- * this peer is known to be behind here, the last place a self-referential freshness stamp
2233
- * belongs.
2341
+ * A commit the cluster carried but this peer could not apply locally. Reported as success only
2342
+ * when a durable majority of the OTHER cohort members holds it (this node, having diverged, is
2343
+ * not a holder) convergence then comes from replication (cohort reconcile, or read-driven
2344
+ * acquisition), not from replay here. Without that majority it is the durability gate's
2345
+ * retryable refusal: nothing is known to hold the revision. `armFreshness` says whether the
2346
+ * commit's quorum was strong enough ({@link commitQuorumRulesOutRivals}) for the read path to
2347
+ * treat the blocks as freshness-checked; a divergence tolerated on a downsized quorum leaves the
2348
+ * window unarmed — this peer is known to be behind here, the last place a self-referential
2349
+ * freshness stamp belongs.
2234
2350
  */
2235
- tolerateLocalCommitDivergence(request, blockIds, detail, armFreshness) {
2351
+ async tolerateLocalCommitDivergence(request, blockIds, detail, armFreshness, durability) {
2352
+ if (!isDurableMajority(durability.remoteHolders.length, durability.cohortPeerIds.length)) {
2353
+ return this.refuseCommitNotDurable(request, durability.remoteHolders.length, durability, `fallback-diverged: ${detail}`);
2354
+ }
2236
2355
  this.log('coordinator-repo:commit-local-failed-cluster-succeeded', { actionId: request.actionId, error: detail });
2237
2356
  if (armFreshness)
2238
2357
  this.markBlocksSeen(blockIds);
2239
- return { success: true };
2358
+ // This node holds nothing: it is an UNCONFIRMED member when it is in the cohort (so the class
2359
+ // is `majority`), and simply absent from the count when it is not (so the class can be `full`
2360
+ // when every member confirmed). Both fall out of the one rule in `cohortWriteDurability`.
2361
+ // Holding nothing, it is also no source for a push, so no shortfall is recorded here.
2362
+ return await this.acknowledgeCommit(request, { success: true, durability: this.cohortWriteDurability(durability, false) }, false);
2363
+ }
2364
+ /**
2365
+ * The one way a success leaves {@link commit}: record the replication shortfall the answer
2366
+ * describes, then return the answer unchanged. Every success exit — solo, local-executed, local
2367
+ * fallback, tolerated divergence — runs after the durability gate admitted it, so a refused
2368
+ * commit never reaches the ledger.
2369
+ *
2370
+ * `localHolds` is whether this node's own storage durably holds every block of the commit, which
2371
+ * is what makes it a source for the missing copies — not whether the class counted it. `record`
2372
+ * is the consensus the commit ran on, absent for the solo short-circuit (no cohort ran, so no
2373
+ * holders to report).
2374
+ */
2375
+ async acknowledgeCommit(request, answer, localHolds, record) {
2376
+ if (localHolds && record !== undefined)
2377
+ this.reportCommittedHolders(request, answer.durability, record);
2378
+ await this.noteReplicationShortfall(request, answer.durability, localHolds);
2379
+ return answer;
2380
+ }
2381
+ /**
2382
+ * Tell the {@link CommittedHoldersSink} who holds a cohort commit this node holds, so its rebalance
2383
+ * monitor does not push the blocks back to members that stored them. A holder is a member the
2384
+ * durability class confirmed AND that signed an approving commit vote: the confirmation rests on
2385
+ * the member's unsigned apply report, and requiring the signature too means a member lying in that
2386
+ * report can only misstate its own copy — a member already trusted with the commit. The members
2387
+ * the class names unconfirmed are reported as such, which withdraws any earlier record of them
2388
+ * (including the signer list this node's own member reported at apply), so they are pushed the
2389
+ * block. Never throws: the commit is already acknowledged.
2390
+ */
2391
+ reportCommittedHolders(request, durability, record) {
2392
+ if (!this.onCommittedHolders)
2393
+ return;
2394
+ const self = this.localPeerId?.toString();
2395
+ const unconfirmed = durability.unconfirmed ?? [];
2396
+ const holders = (durability.cohortPeerIds ?? [])
2397
+ .filter(peerId => !unconfirmed.includes(peerId))
2398
+ .filter(peerId => peerId === self || record.commits[peerId]?.type === 'approve');
2399
+ try {
2400
+ this.onCommittedHolders({ blockIds: request.blockIds, holders, unconfirmed });
2401
+ }
2402
+ catch (err) {
2403
+ this.log('coordinator-repo:committed-holders-sink-error', { actionId: request.actionId, error: err.message });
2404
+ }
2405
+ }
2406
+ /**
2407
+ * Write down who is still missing an acknowledged commit, while this node still knows — the
2408
+ * cohort's answer is gone the moment the writer is answered, and the in-memory commit retry does
2409
+ * not survive a restart. One entry per block, all sharing this commit's one cohort answer
2410
+ * (consensus ran on `blockIds[0]`'s cohort for the whole request):
2411
+ * - `full` settles any older entry for the block, whether or not this node holds it;
2412
+ * - `majority` records the unconfirmed members by name;
2413
+ * - `local` and `unrouted` record an EMPTY missing set, meaning "unknown" — nobody could be named.
2414
+ * A shortfall is recorded only when `localHolds`: a node holding nothing has nothing to push, and
2415
+ * the remote majority that admitted the commit already holds it.
2416
+ *
2417
+ * NEVER throws, and that asymmetry is deliberate: by the time this runs the commit is already
2418
+ * durable at the class the answer states, so a ledger fault must not turn an acknowledged write
2419
+ * into a reported failure. A failure is logged per block and the answer still goes out; the
2420
+ * block's copy is then owed with nobody tracking it, exactly as before the ledger existed.
2421
+ *
2422
+ * NOTE: one ledger read and write per block per below-full commit, awaited before the writer is
2423
+ * answered — on a node that is genuinely alone that is every commit (a `full` commit settles from
2424
+ * the ledger's in-memory index and touches the store only when an entry exists). If it shows in
2425
+ * commit latency, coalesce the writes rather than answering first: an entry written after the
2426
+ * answer can be lost to a crash in between, which is the one case the ledger exists for.
2427
+ */
2428
+ async noteReplicationShortfall(request, durability, localHolds) {
2429
+ const ledger = this.underReplicationLedger;
2430
+ if (ledger === undefined)
2431
+ return;
2432
+ // Blocks a torn commit abandoned were cancelled and hold nothing at this revision: there is no
2433
+ // copy to push, and an entry would have a drain repeatedly read a block that is not there.
2434
+ // No answer built in this class names `torn` today — the writer's transactor derives it after
2435
+ // every coordinator has answered — so this guards against a future producer, not a current one.
2436
+ const torn = new Set(durability.torn ?? []);
2437
+ const blockIds = request.blockIds.filter(blockId => !torn.has(blockId));
2438
+ const outcomes = await Promise.allSettled(blockIds.map(blockId => this.noteBlockShortfall(ledger, request, blockId, durability, localHolds)));
2439
+ outcomes.forEach((outcome, i) => {
2440
+ if (outcome.status === 'fulfilled')
2441
+ return;
2442
+ this.log('coordinator-repo:under-replication-record-failed', {
2443
+ actionId: request.actionId,
2444
+ blockId: blockIds[i],
2445
+ rev: request.rev,
2446
+ quorum: durability.quorum,
2447
+ error: outcome.reason instanceof Error ? outcome.reason.message : String(outcome.reason)
2448
+ });
2449
+ });
2450
+ }
2451
+ async noteBlockShortfall(ledger, request, blockId, durability, localHolds) {
2452
+ if (durability.quorum === 'full') {
2453
+ await ledger.settle(blockId, request.rev);
2454
+ return;
2455
+ }
2456
+ if (!localHolds)
2457
+ return;
2458
+ await ledger.record({
2459
+ blockId,
2460
+ rev: request.rev,
2461
+ actionId: request.actionId,
2462
+ quorum: durability.quorum,
2463
+ missingPeerIds: durability.quorum === 'majority' ? durability.unconfirmed ?? [] : [],
2464
+ recordedAt: this.now(),
2465
+ attempts: 0
2466
+ });
2467
+ }
2468
+ /**
2469
+ * The durability class beside the gate: derived from the SAME sets {@link isDurableMajority} was
2470
+ * just measured on, with this node's own contribution passed per arm exactly as the gate counted
2471
+ * it, so the class and the gate can never disagree. One rule for all three success exits of
2472
+ * {@link commit}: `full` iff every cohort member is confirmed, otherwise `majority` (the gate
2473
+ * already refused anything below that). Never `local` or `unrouted` here — those belong to the
2474
+ * solo short-circuit, where no cohort ran.
2475
+ */
2476
+ cohortWriteDurability(durability, selfHolds) {
2477
+ return cohortWriteDurability(durability, selfHolds, this.localPeerId?.toString());
2478
+ }
2479
+ /**
2480
+ * The solo short-circuit of {@link commit}: consensus never runs, so no ClusterRecord exists to
2481
+ * project a proof from — the lone member self-signs a one-peer proof instead (mintSoloCommitProof),
2482
+ * which is what lets a block born on a cohort of one ever gain a second holder under the
2483
+ * certified-push default (handlePush refuses a proof-less block). Minted even when the cohort did
2484
+ * not resolve or the sole peer is not self — a failed `findCluster` puts the DEGRADED-ROUTING case
2485
+ * in this same branch, and self genuinely committed these bytes either way; a proof's peer list is
2486
+ * already not evidence of cohort membership by design (caller obligation #1 on
2487
+ * verifyBlockCommitProofClaim), so gating the mint on cohort composition would buy no safety while
2488
+ * opening a silent no-proof hole exactly when routing is degraded.
2489
+ *
2490
+ * The policy is unchanged — every write accepted here before is accepted now. What changes is
2491
+ * that the answer says which situation it was: the `commit:solo-cohort` log line (how an operator
2492
+ * tells a real cohort of one — cohortSize 1, soleIsSelf true — from a routing failure) now also
2493
+ * carries the `quorum` the writer is told, and both come from {@link soloCohortDurability}, so the
2494
+ * log and the answer cannot disagree.
2495
+ */
2496
+ async commitSolo(request, blockIds, cohort, options) {
2497
+ const cohortPeerIds = cohort.resolved ? cohort.peerIds : [];
2498
+ const peerCount = cohortPeerIds.length;
2499
+ const soloDurability = this.soloCohortDurability(cohort);
2500
+ this.log('commit:solo-cohort', {
2501
+ blockId: blockIds[0],
2502
+ cohortSize: peerCount,
2503
+ soleIsSelf: peerCount === 1 && this.localPeerId !== undefined
2504
+ && cohortPeerIds[0] === this.localPeerId.toString(),
2505
+ quorum: soloDurability.quorum,
2506
+ ...(cohort.resolved ? {} : { reason: cohort.reason })
2507
+ });
2508
+ // Same message shape the multi-peer path produces — executeClusterTransaction stamps
2509
+ // coordinatingBlockIds at its choke point, so the solo artifact must carry it too or a
2510
+ // solo proof's message is distinguishable from every other proof's.
2511
+ const message = {
2512
+ operations: [{ commit: request }],
2513
+ coordinatingBlockIds: [blockIds[0]],
2514
+ expiration: options?.expiration ?? Date.now() + this.DEFAULT_TIMEOUT
2515
+ };
2516
+ // `undefined` when no local cluster is wired (direct constructors, unit-test doubles) —
2517
+ // then the commit lands proof-less, exactly the pre-mint behavior. The cast is the named
2518
+ // ICommitProofPersister contract; a plain IRepo double ignores the extra argument.
2519
+ const proof = await this.localCluster?.mintSoloCommitProof?.(message);
2520
+ const result = await this.storageRepo.commit(request, options, proof);
2521
+ if (!result.success)
2522
+ return result;
2523
+ // One self-approval arms the read-repair window only where the DECLARED cohort is also one
2524
+ // — then no rival quorum can exist to be missed. At any larger declared size (including
2525
+ // an undeclared one, which resolves to the replication factor, and including degraded
2526
+ // routing where peerCount is 0) this commit proves nothing about rival quorums — see
2527
+ // commitQuorumRulesOutRivals — so the window stays unarmed and the read path's
2528
+ // solo-self-skip exit re-arms it once per consult instead (which keeps GitHub issue #8's
2529
+ // consult storm bounded at one per window).
2530
+ if (this.commitQuorumRulesOutRivals(1, peerCount))
2531
+ this.markBlocksSeen(blockIds);
2532
+ return await this.acknowledgeCommit(request, { ...result, durability: soloDurability }, true);
2240
2533
  }
2534
+ /**
2535
+ * The durability gate's answer: a retryable refusal ({@link COMMIT_NOT_DURABLE_REASON}) naming how
2536
+ * many of the cohort reported holding the revision. `conflict: true` because the members that
2537
+ * refused have dropped their pending records, so the writer's only way forward is to cancel and
2538
+ * re-drive at a fresh revision — the path a conflict-shaped answer already puts it on. The
2539
+ * read-repair window is deliberately NOT armed: nothing about this commit is freshness evidence.
2540
+ */
2541
+ refuseCommitNotDurable(request, durableHolders, durability, arm) {
2542
+ const cohortSize = durability.cohortPeerIds.length;
2543
+ this.log('coordinator-repo:commit-not-durable', {
2544
+ actionId: request.actionId,
2545
+ rev: request.rev,
2546
+ durableHolders,
2547
+ cohortSize,
2548
+ remoteHolders: durability.remoteHolders,
2549
+ remoteRefusals: durability.remoteRefusals,
2550
+ arm
2551
+ });
2552
+ // A failure carries no durability, and no "unknown" class stands in for one.
2553
+ return {
2554
+ success: false,
2555
+ conflict: true,
2556
+ reason: `${COMMIT_NOT_DURABLE_REASON}: ${durableHolders} of ${cohortSize} cohort member(s) report holding rev ${request.rev} of action ${request.actionId} (${arm})`
2557
+ };
2558
+ }
2559
+ }
2560
+ function cohortDurability(record, outcomes) {
2561
+ const reports = Object.entries(outcomes ?? {});
2562
+ return {
2563
+ cohortPeerIds: Object.keys(record.peers),
2564
+ remoteHolders: reports.filter(([, report]) => report.success).map(([peerId]) => peerId),
2565
+ remoteRefusals: reports.filter(([, report]) => !report.success).map(([peerId]) => peerId)
2566
+ };
2567
+ }
2568
+ /**
2569
+ * The commit-tier durability class, from the gate's own sets. `confirmed` is counted exactly as the
2570
+ * gate counted it (remote holders, plus one for this node when `selfHolds`), and `unconfirmed` is
2571
+ * every cohort member outside that set — so a `full` answer names nobody as missing, and a
2572
+ * `majority` answer names exactly who. The class is `full` iff no cohort member is unconfirmed;
2573
+ * anything below `majority` never reaches here (the gate refused it).
2574
+ */
2575
+ function cohortWriteDurability(durability, selfHolds, selfPeerId) {
2576
+ const holders = new Set(durability.remoteHolders);
2577
+ if (selfHolds && selfPeerId !== undefined)
2578
+ holders.add(selfPeerId);
2579
+ const unconfirmed = durability.cohortPeerIds.filter(peerId => !holders.has(peerId));
2580
+ return {
2581
+ quorum: unconfirmed.length === 0 ? 'full' : 'majority',
2582
+ confirmed: durability.remoteHolders.length + (selfHolds ? 1 : 0),
2583
+ cohort: durability.cohortPeerIds.length,
2584
+ unconfirmed,
2585
+ cohortPeerIds: durability.cohortPeerIds
2586
+ };
2587
+ }
2588
+ /**
2589
+ * The pend-tier durability class: who ACCEPTED the pending record, read off the record's approving
2590
+ * promise votes. This node's own member is added when `selfAccepted` is true, removed when it is
2591
+ * false (its member applied and refused, so its approve vote no longer describes what it holds),
2592
+ * and left to its vote when `undefined` (no retained verdict — the vote is the only evidence there
2593
+ * is). This node counts only when it is IN the cohort, as the commit tier counts it: a coordinator
2594
+ * outside `record.peers` whose fallback pend landed holds a copy no cohort member will ever look
2595
+ * for, and counting it would let `confirmed` exceed `cohort`. NOT comparable to the commit-tier
2596
+ * count: accepting a pending record confers no storage durability, and the field's own
2597
+ * documentation says so.
2598
+ * NOTE: a REMOTE member's vote is never withdrawn here. The pend arm of `ClusterRecord.applyOutcomes`
2599
+ * reports only conflict-shaped refusals (which downgrade the whole pend to a retryable conflict
2600
+ * before this runs), so a remote member that promised and then faulted at apply keeps its approve
2601
+ * vote and a pend's `full` can overstate by that member. Harmless while nothing reads a pend's
2602
+ * class for repair; if that changes, widen the member's report to carry bare faults too.
2603
+ */
2604
+ function pendCohortDurability(record, selfAccepted, selfPeerId) {
2605
+ const cohortPeerIds = Object.keys(record.peers);
2606
+ const accepted = new Set(Object.entries(record.promises).filter(([, vote]) => vote.type === 'approve').map(([peerId]) => peerId));
2607
+ if (selfPeerId !== undefined && cohortPeerIds.includes(selfPeerId)) {
2608
+ if (selfAccepted === true)
2609
+ accepted.add(selfPeerId);
2610
+ if (selfAccepted === false)
2611
+ accepted.delete(selfPeerId);
2612
+ }
2613
+ const unconfirmed = cohortPeerIds.filter(peerId => !accepted.has(peerId));
2614
+ return {
2615
+ quorum: unconfirmed.length === 0 ? 'full' : 'majority',
2616
+ confirmed: accepted.size,
2617
+ cohort: cohortPeerIds.length,
2618
+ unconfirmed,
2619
+ cohortPeerIds
2620
+ };
2621
+ }
2622
+ /**
2623
+ * The durability gate's rule: a STRICT majority of the cohort the commit ran on — the same majority
2624
+ * {@link clusterReachedCommitConsensus} measures votes against, applied to durable holders instead
2625
+ * of approving votes.
2626
+ */
2627
+ function isDurableMajority(durableHolders, cohortSize) {
2628
+ return cohortSize > 0 && durableHolders > cohortSize / 2;
2241
2629
  }
2242
2630
  /** True if a simple majority of cluster peers signed an approving commit. */
2243
2631
  function clusterReachedCommitConsensus(record) {