@optimystic/db-p2p 1.0.0-beta.3 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (234) hide show
  1. package/dist/src/cluster/block-transfer-service.d.ts +73 -1
  2. package/dist/src/cluster/block-transfer-service.d.ts.map +1 -1
  3. package/dist/src/cluster/block-transfer-service.js +70 -3
  4. package/dist/src/cluster/block-transfer-service.js.map +1 -1
  5. package/dist/src/cluster/block-transfer.d.ts +16 -1
  6. package/dist/src/cluster/block-transfer.d.ts.map +1 -1
  7. package/dist/src/cluster/block-transfer.js +73 -72
  8. package/dist/src/cluster/block-transfer.js.map +1 -1
  9. package/dist/src/cluster/client.d.ts +4 -6
  10. package/dist/src/cluster/client.d.ts.map +1 -1
  11. package/dist/src/cluster/client.js +8 -13
  12. package/dist/src/cluster/client.js.map +1 -1
  13. package/dist/src/cluster/cluster-policy.d.ts +2 -1
  14. package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
  15. package/dist/src/cluster/cluster-policy.js.map +1 -1
  16. package/dist/src/cluster/cluster-repo.d.ts +155 -9
  17. package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
  18. package/dist/src/cluster/cluster-repo.js +446 -58
  19. package/dist/src/cluster/cluster-repo.js.map +1 -1
  20. package/dist/src/cluster/commit-proof.js +2 -2
  21. package/dist/src/cluster/race-resolution.d.ts +51 -7
  22. package/dist/src/cluster/race-resolution.d.ts.map +1 -1
  23. package/dist/src/cluster/race-resolution.js +71 -7
  24. package/dist/src/cluster/race-resolution.js.map +1 -1
  25. package/dist/src/cluster/rebalance-monitor.d.ts +67 -5
  26. package/dist/src/cluster/rebalance-monitor.d.ts.map +1 -1
  27. package/dist/src/cluster/rebalance-monitor.js +120 -26
  28. package/dist/src/cluster/rebalance-monitor.js.map +1 -1
  29. package/dist/src/cluster/spread-on-churn.d.ts.map +1 -1
  30. package/dist/src/cluster/spread-on-churn.js +34 -47
  31. package/dist/src/cluster/spread-on-churn.js.map +1 -1
  32. package/dist/src/cohort-topic/host.d.ts.map +1 -1
  33. package/dist/src/cohort-topic/host.js +8 -6
  34. package/dist/src/cohort-topic/host.js.map +1 -1
  35. package/dist/src/cohort-topic/membership-source.js +1 -1
  36. package/dist/src/cohort-topic/membership-source.js.map +1 -1
  37. package/dist/src/cohort-topic/peer-codec.d.ts.map +1 -1
  38. package/dist/src/cohort-topic/peer-codec.js +5 -2
  39. package/dist/src/cohort-topic/peer-codec.js.map +1 -1
  40. package/dist/src/cohort-topic/stream-util.d.ts +24 -3
  41. package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
  42. package/dist/src/cohort-topic/stream-util.js +36 -3
  43. package/dist/src/cohort-topic/stream-util.js.map +1 -1
  44. package/dist/src/cohort-topic/topic-router.d.ts +9 -0
  45. package/dist/src/cohort-topic/topic-router.d.ts.map +1 -1
  46. package/dist/src/cohort-topic/topic-router.js +12 -2
  47. package/dist/src/cohort-topic/topic-router.js.map +1 -1
  48. package/dist/src/dispute/client.d.ts.map +1 -1
  49. package/dist/src/dispute/client.js +15 -3
  50. package/dist/src/dispute/client.js.map +1 -1
  51. package/dist/src/index.d.ts +5 -0
  52. package/dist/src/index.d.ts.map +1 -1
  53. package/dist/src/index.js +5 -0
  54. package/dist/src/index.js.map +1 -1
  55. package/dist/src/libp2p-key-network.d.ts +78 -17
  56. package/dist/src/libp2p-key-network.d.ts.map +1 -1
  57. package/dist/src/libp2p-key-network.js +234 -147
  58. package/dist/src/libp2p-key-network.js.map +1 -1
  59. package/dist/src/libp2p-node-base.d.ts +35 -1
  60. package/dist/src/libp2p-node-base.d.ts.map +1 -1
  61. package/dist/src/libp2p-node-base.js +151 -26
  62. package/dist/src/libp2p-node-base.js.map +1 -1
  63. package/dist/src/libp2p-node-rn.d.ts +2 -2
  64. package/dist/src/libp2p-node-rn.d.ts.map +1 -1
  65. package/dist/src/libp2p-node-rn.js.map +1 -1
  66. package/dist/src/libp2p-node.d.ts +2 -2
  67. package/dist/src/libp2p-node.d.ts.map +1 -1
  68. package/dist/src/libp2p-node.js.map +1 -1
  69. package/dist/src/logger.d.ts.map +1 -1
  70. package/dist/src/logger.js +6 -1
  71. package/dist/src/logger.js.map +1 -1
  72. package/dist/src/matchmaking/query-transport.d.ts +3 -4
  73. package/dist/src/matchmaking/query-transport.d.ts.map +1 -1
  74. package/dist/src/matchmaking/query-transport.js +9 -9
  75. package/dist/src/matchmaking/query-transport.js.map +1 -1
  76. package/dist/src/network/network-manager-service.d.ts +10 -5
  77. package/dist/src/network/network-manager-service.d.ts.map +1 -1
  78. package/dist/src/network/network-manager-service.js +11 -2
  79. package/dist/src/network/network-manager-service.js.map +1 -1
  80. package/dist/src/network/relay-reservation.d.ts +170 -0
  81. package/dist/src/network/relay-reservation.d.ts.map +1 -0
  82. package/dist/src/network/relay-reservation.js +527 -0
  83. package/dist/src/network/relay-reservation.js.map +1 -0
  84. package/dist/src/pend-validation.d.ts +3 -2
  85. package/dist/src/pend-validation.d.ts.map +1 -1
  86. package/dist/src/pend-validation.js +3 -2
  87. package/dist/src/pend-validation.js.map +1 -1
  88. package/dist/src/protocol-client.d.ts +8 -1
  89. package/dist/src/protocol-client.d.ts.map +1 -1
  90. package/dist/src/protocol-client.js +11 -0
  91. package/dist/src/protocol-client.js.map +1 -1
  92. package/dist/src/reactivity/forwarder-host.d.ts +2 -1
  93. package/dist/src/reactivity/forwarder-host.d.ts.map +1 -1
  94. package/dist/src/reactivity/forwarder-host.js +2 -1
  95. package/dist/src/reactivity/forwarder-host.js.map +1 -1
  96. package/dist/src/reactivity/origination-manager.d.ts +3 -3
  97. package/dist/src/reactivity/origination-manager.js +3 -3
  98. package/dist/src/reactivity/recover-transport.d.ts +22 -14
  99. package/dist/src/reactivity/recover-transport.d.ts.map +1 -1
  100. package/dist/src/reactivity/recover-transport.js +32 -19
  101. package/dist/src/reactivity/recover-transport.js.map +1 -1
  102. package/dist/src/reactivity/subscription-manager.d.ts +2 -2
  103. package/dist/src/reactivity/topic-bytes.d.ts +6 -7
  104. package/dist/src/reactivity/topic-bytes.d.ts.map +1 -1
  105. package/dist/src/reactivity/topic-bytes.js +8 -9
  106. package/dist/src/reactivity/topic-bytes.js.map +1 -1
  107. package/dist/src/repo/client.d.ts +5 -2
  108. package/dist/src/repo/client.d.ts.map +1 -1
  109. package/dist/src/repo/client.js +49 -40
  110. package/dist/src/repo/client.js.map +1 -1
  111. package/dist/src/repo/cluster-coordinator.d.ts +96 -11
  112. package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
  113. package/dist/src/repo/cluster-coordinator.js +164 -25
  114. package/dist/src/repo/cluster-coordinator.js.map +1 -1
  115. package/dist/src/repo/coordinator-repo.d.ts +240 -122
  116. package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
  117. package/dist/src/repo/coordinator-repo.js +627 -473
  118. package/dist/src/repo/coordinator-repo.js.map +1 -1
  119. package/dist/src/repo/i-under-replication-ledger.d.ts +87 -0
  120. package/dist/src/repo/i-under-replication-ledger.d.ts.map +1 -0
  121. package/dist/src/repo/i-under-replication-ledger.js +2 -0
  122. package/dist/src/repo/i-under-replication-ledger.js.map +1 -0
  123. package/dist/src/repo/kv-under-replication-ledger.d.ts +91 -0
  124. package/dist/src/repo/kv-under-replication-ledger.d.ts.map +1 -0
  125. package/dist/src/repo/kv-under-replication-ledger.js +287 -0
  126. package/dist/src/repo/kv-under-replication-ledger.js.map +1 -0
  127. package/dist/src/repo/responsibility.d.ts +37 -0
  128. package/dist/src/repo/responsibility.d.ts.map +1 -0
  129. package/dist/src/repo/responsibility.js +37 -0
  130. package/dist/src/repo/responsibility.js.map +1 -0
  131. package/dist/src/repo/service.d.ts +44 -15
  132. package/dist/src/repo/service.d.ts.map +1 -1
  133. package/dist/src/repo/service.js +64 -27
  134. package/dist/src/repo/service.js.map +1 -1
  135. package/dist/src/repo/stuck-reservation.d.ts +127 -0
  136. package/dist/src/repo/stuck-reservation.d.ts.map +1 -0
  137. package/dist/src/repo/stuck-reservation.js +179 -0
  138. package/dist/src/repo/stuck-reservation.js.map +1 -0
  139. package/dist/src/repo/under-replication-drain.d.ts +208 -0
  140. package/dist/src/repo/under-replication-drain.d.ts.map +1 -0
  141. package/dist/src/repo/under-replication-drain.js +461 -0
  142. package/dist/src/repo/under-replication-drain.js.map +1 -0
  143. package/dist/src/rn.d.ts +5 -0
  144. package/dist/src/rn.d.ts.map +1 -1
  145. package/dist/src/rn.js +5 -0
  146. package/dist/src/rn.js.map +1 -1
  147. package/dist/src/storage/block-latch.d.ts +3 -2
  148. package/dist/src/storage/block-latch.d.ts.map +1 -1
  149. package/dist/src/storage/block-latch.js +7 -4
  150. package/dist/src/storage/block-latch.js.map +1 -1
  151. package/dist/src/storage/block-storage.d.ts +45 -3
  152. package/dist/src/storage/block-storage.d.ts.map +1 -1
  153. package/dist/src/storage/block-storage.js +156 -9
  154. package/dist/src/storage/block-storage.js.map +1 -1
  155. package/dist/src/storage/i-block-storage.d.ts +57 -5
  156. package/dist/src/storage/i-block-storage.d.ts.map +1 -1
  157. package/dist/src/storage/i-block-storage.js.map +1 -1
  158. package/dist/src/storage/pending-claim.d.ts +167 -0
  159. package/dist/src/storage/pending-claim.d.ts.map +1 -0
  160. package/dist/src/storage/pending-claim.js +156 -0
  161. package/dist/src/storage/pending-claim.js.map +1 -0
  162. package/dist/src/storage/raw-store-codec.d.ts +13 -0
  163. package/dist/src/storage/raw-store-codec.d.ts.map +1 -1
  164. package/dist/src/storage/raw-store-codec.js +24 -3
  165. package/dist/src/storage/raw-store-codec.js.map +1 -1
  166. package/dist/src/storage/restoration-coordinator.d.ts.map +1 -1
  167. package/dist/src/storage/restoration-coordinator.js +2 -1
  168. package/dist/src/storage/restoration-coordinator.js.map +1 -1
  169. package/dist/src/storage/ring-shift-coordinator.d.ts.map +1 -1
  170. package/dist/src/storage/ring-shift-coordinator.js +3 -3
  171. package/dist/src/storage/ring-shift-coordinator.js.map +1 -1
  172. package/dist/src/storage/storage-repo.d.ts +122 -5
  173. package/dist/src/storage/storage-repo.d.ts.map +1 -1
  174. package/dist/src/storage/storage-repo.js +245 -68
  175. package/dist/src/storage/storage-repo.js.map +1 -1
  176. package/dist/src/storage/struct.d.ts +58 -0
  177. package/dist/src/storage/struct.d.ts.map +1 -1
  178. package/dist/src/testing/mesh-harness.d.ts +77 -11
  179. package/dist/src/testing/mesh-harness.d.ts.map +1 -1
  180. package/dist/src/testing/mesh-harness.js +226 -100
  181. package/dist/src/testing/mesh-harness.js.map +1 -1
  182. package/package.json +2 -2
  183. package/readme.md +27 -3
  184. package/src/cluster/block-transfer-service.ts +132 -4
  185. package/src/cluster/block-transfer.ts +79 -84
  186. package/src/cluster/client.ts +115 -119
  187. package/src/cluster/cluster-policy.ts +2 -1
  188. package/src/cluster/cluster-repo.ts +493 -65
  189. package/src/cluster/commit-proof.ts +2 -2
  190. package/src/cluster/race-resolution.ts +76 -7
  191. package/src/cluster/rebalance-monitor.ts +677 -539
  192. package/src/cluster/spread-on-churn.ts +34 -54
  193. package/src/cohort-topic/host.ts +8 -6
  194. package/src/cohort-topic/membership-source.ts +1 -1
  195. package/src/cohort-topic/peer-codec.ts +34 -31
  196. package/src/cohort-topic/stream-util.ts +39 -4
  197. package/src/cohort-topic/topic-router.ts +12 -2
  198. package/src/dispute/client.ts +17 -7
  199. package/src/index.ts +64 -59
  200. package/src/libp2p-key-network.ts +1377 -1260
  201. package/src/libp2p-node-base.ts +186 -27
  202. package/src/libp2p-node-rn.ts +31 -30
  203. package/src/libp2p-node.ts +37 -36
  204. package/src/logger.ts +6 -1
  205. package/src/matchmaking/query-transport.ts +9 -9
  206. package/src/network/network-manager-service.ts +16 -5
  207. package/src/network/relay-reservation.ts +636 -0
  208. package/src/pend-validation.ts +3 -2
  209. package/src/protocol-client.ts +13 -1
  210. package/src/reactivity/forwarder-host.ts +2 -1
  211. package/src/reactivity/origination-manager.ts +3 -3
  212. package/src/reactivity/recover-transport.ts +39 -22
  213. package/src/reactivity/subscription-manager.ts +2 -2
  214. package/src/reactivity/topic-bytes.ts +35 -37
  215. package/src/repo/client.ts +172 -173
  216. package/src/repo/cluster-coordinator.ts +183 -25
  217. package/src/repo/coordinator-repo.ts +656 -506
  218. package/src/repo/i-under-replication-ledger.ts +89 -0
  219. package/src/repo/kv-under-replication-ledger.ts +302 -0
  220. package/src/repo/responsibility.ts +48 -0
  221. package/src/repo/service.ts +76 -39
  222. package/src/repo/stuck-reservation.ts +224 -0
  223. package/src/repo/under-replication-drain.ts +582 -0
  224. package/src/rn.ts +74 -69
  225. package/src/storage/block-latch.ts +7 -4
  226. package/src/storage/block-storage.ts +161 -10
  227. package/src/storage/i-block-storage.ts +60 -5
  228. package/src/storage/pending-claim.ts +200 -0
  229. package/src/storage/raw-store-codec.ts +26 -3
  230. package/src/storage/restoration-coordinator.ts +212 -211
  231. package/src/storage/ring-shift-coordinator.ts +271 -271
  232. package/src/storage/storage-repo.ts +280 -72
  233. package/src/storage/struct.ts +58 -0
  234. package/src/testing/mesh-harness.ts +293 -112
@@ -1,5 +1,5 @@
1
- import { LruMap, blockIdsForTransforms, highestStaleAt, isConflictFailure, isOwnRevision, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
2
- import { ClusterCoordinator, ConflictRaceLostError, ValidatorRejectionError } from "./cluster-coordinator.js";
1
+ import { LruMap, blockIdsForTransforms, transformForBlockId, highestStaleAt, isConflictFailure, isOwnRevision, DEFAULT_SUPER_MAJORITY_THRESHOLD, routingKeyForBlock, localDurability, unroutedDurability } from "@optimystic/db-core";
2
+ import { BlocksHeldError, ClusterCoordinator, ConflictRaceLostError, ValidatorRejectionError } from "./cluster-coordinator.js";
3
3
  import { peerIdFromString } from "@libp2p/peer-id";
4
4
  import { createLogger } from '../logger.js';
5
5
  import { PenaltyReason } from "../reputation/types.js";
@@ -8,7 +8,10 @@ import { certifyClaim, isAttributableProofFailure, proofThresholds } from "../cl
8
8
  import { DEFAULT_CLUSTER_SIZE, resolveRepairCorroborationClusterSize } from "../cluster/cluster-policy.js";
9
9
  import { RECONCILE_TIMEOUT_MS } from "../cluster/reconcile-block.js";
10
10
  import { isMissingBaseRevisionFailure, COMMIT_NOT_DURABLE_REASON, MISSING_BASE_REVISION_REASON } from "../storage/storage-repo.js";
11
+ import { isReservationAgainst, reservationRequestFor, cohortCanMissAPend } from "../storage/pending-claim.js";
11
12
  import { buildBlockCommitProof } from "../cluster/commit-proof.js";
13
+ import { RESPONSIBILITY_TTL_MS, ResponsibilityRefusalError } from "./responsibility.js";
14
+ import { StuckReservationTracker } from "./stuck-reservation.js";
12
15
  /** How long one cohort peer gets to answer the latest-revision consult before it counts as silent. */
13
16
  const LATEST_QUERY_TIMEOUT_MS = 1000;
14
17
  /** True when a freshly-read local revision is strictly ahead of the baseline the repair started from. */
@@ -76,80 +79,8 @@ function soleHolderMessage(cohortPeers) {
76
79
  `second voter — reaching this message means the one holder attached no such proof, or one that did not ` +
77
80
  `verify.)`;
78
81
  }
79
- /**
80
- * How many DISTINCT later actions one unchanged holder must refuse on a block before the refusals are
81
- * named as a stuck reservation rather than as an ordinary lost race.
82
- *
83
- * **Why a count of distinct actions and not something else.** Elapsed time answers the wrong question
84
- * — a slow writer is not a stuck one, and a holder legitimately keeps its reservation for as long as
85
- * its own commit takes. A raw refusal count answers the wrong question too: a single writer retrying
86
- * a lost race produces a run of refusals under ONE action id (see {@link StuckReservationWatch.refused}).
87
- * What no healthy holder can produce is an unbounded stream of *different* writers all losing to it,
88
- * because a healthy holder's reservation lasts one pend-to-commit window.
89
- *
90
- * **Why 8.** The bound to clear is how many distinct actions can honestly be refused inside one such
91
- * window. Measured on the in-process mesh, in the healthy-contention arm of
92
- * `test/stuck-reservation-named.spec.ts`: a holder that pends, is raced by other writers, and then
93
- * commits refuses **2** distinct actions per episode — the two rivals — and the count resets on every
94
- * holder change. `concurrent-diary-append-acknowledgement.spec.ts` races three writers at one diary
95
- * and cannot exceed that either, for the same reason: at most (writers - 1) rivals can lose to one
96
- * winner. 8 is four times the measured healthy figure, and it is a floor a genuinely stuck block
97
- * clears trivially (the field instance refused hundreds).
98
- *
99
- * **The bound stated exactly.** It is distinct SYNC CYCLES, not distinct writers: one writer that
100
- * exhausts a sync's retry budget and is re-driven by its caller mints a fresh id for the next cycle,
101
- * so it can contribute more than one. That does not widen the window much — a cycle only ends in
102
- * exhaustion after `DefaultMaxAttempts` (10) attempts of backoff, roughly 21s (see the exhaustion
103
- * NOTE in `Collection.syncAttempts`), so a lone writer needs a holder to keep the block for upwards
104
- * of two and a half minutes before it reaches 8 by itself, which is not a healthy holder.
105
- *
106
- * **What the margin does NOT cover, stated honestly.** A block with more than 8 distinct writers
107
- * racing it inside a single pend-to-commit round trip could reach 8 with a perfectly healthy holder.
108
- * That is a diagnostic false positive on a log line and nothing else — this counter never refuses,
109
- * expires, or deletes anything (see {@link CoordinatorRepo.noteStuckReservation}) — and the remedy if
110
- * a deployment ever hits it is to raise this number, not to add a control path. Raising it costs
111
- * detection latency on low-traffic blocks, which need this many distinct write ATTEMPTS before the
112
- * condition can be named at all.
113
- */
114
- const STUCK_RESERVATION_DISTINCT_ACTIONS = 8;
115
- /** Whether two sorted holder lists name the same reservation — i.e. whether a refusal continues an
116
- * existing episode or starts a new one. Both sides come from the same sort, so this is a plain
117
- * element-wise comparison; a block normally has exactly one holder, since a member's own pend refuses
118
- * a second one (`ClusterMember.validatePendOperations`). */
119
- function sameHolders(a, b) {
120
- return a.length === b.length && a.every((id, i) => id === b[i]);
121
- }
122
- /**
123
- * The stuck-reservation wording: written for an operator reading logs, in the same register as
124
- * {@link cohortTooSmallMessage} and {@link soleHolderMessage} — what is stuck, what will and will not
125
- * clear it, and what to do next.
126
- *
127
- * The claim is deliberately about the RESERVATION, not about the writer's intent: this node cannot
128
- * see whether the holding process is alive, only that it has held the block across enough unrelated
129
- * later actions that no retry is going to win. So the line says what is provable (the block accepts
130
- * no writes while this record stands, and nothing on the node removes it) and points at the one check
131
- * that settles the rest.
132
- */
133
- function stuckReservationMessage(holders, refusedActions) {
134
- const held = holders.join(', ');
135
- return `This block is WEDGED BEHIND A PENDING WRITE THAT IS NOT COMPLETING, and retrying will never ` +
136
- `clear it: action(s) ${held} reserved the block and have now refused ${refusedActions} DISTINCT, ` +
137
- `unrelated later actions. Each of those refusals on its own looks exactly like an ordinary ` +
138
- `optimistic-concurrency loss, which is normal and healthy — the repetition is what is not. A ` +
139
- `healthy rival holds a block only for its own pend-to-commit window and then releases it by ` +
140
- `committing or cancelling; a reservation that keeps refusing NEW writers is holding the block ` +
141
- `against every writer on every machine, and each of them loses again identically. EXACTLY TWO ` +
142
- `THINGS CLEAR IT: a cancel for action(s) ${held} on this block (route it through the cohort so ` +
143
- `every member drops the record), or that same action's own commit landing. Nothing on the node ` +
144
- `expires it — there is no sweep for abandoned pending records — so until one of those two happens ` +
145
- `the block takes NO writes while continuing to serve reads and to look healthy in every other ` +
146
- `respect. The usual cause is a writer that went away between a failed or half-applied commit and ` +
147
- `the cancel it owed, so check whether whatever ran ${held} still exists before cancelling on its ` +
148
- `behalf. This line is a diagnosis and nothing more: this node does not expire, refuse, or delete ` +
149
- `the record on the strength of it.`;
150
- }
151
82
  export function coordinatorRepo(keyNetwork, createClusterClient, cfg, fretService, reputation, stateStore) {
152
- return (components) => new CoordinatorRepo(keyNetwork, createClusterClient, components.storageRepo, cfg, components.localCluster, components.localPeerId, fretService, components.clusterLatestCallback, reputation, stateStore, components.acquireBlockFromCohort, components.proofAnchoring);
83
+ return (components) => new CoordinatorRepo(keyNetwork, createClusterClient, components.storageRepo, cfg, components.localCluster, components.localPeerId, fretService, components.clusterLatestCallback, reputation, stateStore, components.acquireBlockFromCohort, components.proofAnchoring, components.underReplicationLedger, components.onCommittedHolders);
153
84
  }
154
85
  /** Cluster coordination repo - uses local store, as well as distributes changes to other nodes using cluster consensus. */
155
86
  export class CoordinatorRepo {
@@ -160,37 +91,13 @@ export class CoordinatorRepo {
160
91
  clusterLatestCallback;
161
92
  acquireBlockFromCohort;
162
93
  proofAnchoring;
94
+ underReplicationLedger;
95
+ onCommittedHolders;
163
96
  coordinator;
164
97
  DEFAULT_TIMEOUT = 30000; // 30 seconds default timeout
165
98
  localPeerId;
166
99
  responsibilityCache = new LruMap(1000);
167
- static RESPONSIBILITY_TTL_MS = 60_000;
168
100
  lastSeenCommitMs = new LruMap(1000);
169
- /**
170
- * Per block, when (`this.now()` at the consult's start) a consult last SETTLED that this node's
171
- * absence of it is the cohort's answer too — see `absenceSettled` on {@link fetchBlockFromCluster}'s
172
- * result. Only the solo-self exit settles today: when this node is the block's whole cohort, every
173
- * acknowledged commit of it lands in this node's storage before the writer hears success, so the
174
- * block reads present and the memo dies. Read only for a block still missing locally
175
- * ({@link absenceIsSettled}); cleared by any pend or commit this node coordinates for the block, by
176
- * seeing it present, and by any later consult of it that does not settle.
177
- *
178
- * Deliberately its OWN map rather than `lastSeenCommitMs`: that one is also stamped for missing
179
- * blocks at exits whose verdict is NOT a confirmed absence (the `cohort-too-small` arm when a peer
180
- * claimed a revision, the post-restore arm when acquisition failed), so reading it here would turn
181
- * a `claimed-elsewhere` absence into an authoritative one for a whole window.
182
- *
183
- * NOTE: accepted tradeoff — a cohort-of-one view can be transient (peers still mid-identify; see
184
- * the solo exit), and a block another node creates through the cohort it sees is then reported
185
- * absent here for up to one `readRepairWindowMs` after this node's view grows — the same one-window
186
- * cost the held-block window pays for a growing cohort. Revisit if a caller ever needs
187
- * create-visibility across coordinators tighter than one window — such a caller needs a revision
188
- * floor (backlog `feat-refresh-can-demand-a-revision-floor`), not a shorter window.
189
- *
190
- * NOTE: LRU-bounded like its siblings; an eviction under >1000 settled blocks loses a memo and
191
- * costs one extra consult — the safe direction.
192
- */
193
- settledAbsences = new LruMap(1000);
194
101
  /** Per block, what earlier repair passes left unresolved — see {@link AheadClaimState}.
195
102
  * Outlives the consult on purpose: the read-repair window skips consults for blocks checked
196
103
  * recently, and a doubt dropped there is a stale answer served as confirmed again.
@@ -199,14 +106,14 @@ export class CoordinatorRepo {
199
106
  * {@link reportRepairDeadlock} say its piece a second time. */
200
107
  unsettledAheadClaims = new LruMap(1000);
201
108
  /**
202
- * Per block, what its pending-conflict refusals have added up to — see {@link StuckReservationWatch}
203
- * and {@link noteStuckReservation}. Deliberately its OWN map rather than a third fact hung off
204
- * {@link unsettledAheadClaims}: that entry belongs to the read-repair path and is cleared by a block
205
- * converging on a revision, whereas this one belongs to the write path and is cleared by the block
206
- * accepting a write. Sharing the entry would mean teaching both of those lifetimes about a fact
207
- * neither owns (backlog `debt-freshness-state-scattered-across-coordinator-repo` is the standing
208
- * argument for collapsing all of this per-block state behind one collaborator; adding a fourth
209
- * carve-out to the freshness entry would have made that harder, not easier).
109
+ * Per block, what its pending-conflict refusals have added up to — see {@link StuckReservationTracker}
110
+ * and {@link noteStuckReservation}. Deliberately its OWN collaborator rather than a third fact hung
111
+ * off {@link unsettledAheadClaims}: that entry belongs to the read-repair path and is cleared by a
112
+ * block converging on a revision, whereas this one belongs to the write path and is cleared by the
113
+ * block accepting a write. Sharing the entry would mean teaching both of those lifetimes about a
114
+ * fact neither owns (backlog `debt-freshness-state-scattered-across-coordinator-repo` is the
115
+ * standing argument for collapsing all of this per-block state behind one collaborator; adding a
116
+ * fourth carve-out to the freshness entry would have made that harder, not easier).
210
117
  *
211
118
  * NOTE: LRU-bounded like its siblings. An eviction under >1000 conflicted blocks loses an episode's
212
119
  * say-once flag, so the line can repeat once for that block — the same bounded duplication
@@ -215,12 +122,11 @@ export class CoordinatorRepo {
215
122
  * NOTE: per COORDINATOR, and a block's coordinator is whichever peer the writer's key lookup
216
123
  * resolved — normally stable, but cohort churn or a routing change moves it. When it moves, the
217
124
  * count restarts on the new coordinator (the condition is named later) and the old one may name the
218
- * same episode again (the condition is named twice). Fine while a wedged block is diagnosed by
219
- * searching for its id; if a churning deployment ever makes duplicate lines the noisy failure this
220
- * one exists to replace, the say-once state has to move to where the record lives (the member's own
221
- * storage) rather than to where the refusal was classified.
125
+ * same episode again (the condition is named twice). Each MEMBER now keeps its own tracker as well
126
+ * (`ClusterMember.validatePendOperations`), fed by its own `held` votes, so the episode is also
127
+ * named where the record lives, by a count that does not move with the coordinator.
222
128
  */
223
- stuckReservations = new LruMap(1000);
129
+ stuckReservations = new StuckReservationTracker(1000);
224
130
  readRepairMode;
225
131
  readRepairWindowMs;
226
132
  readRepairSampleRate;
@@ -242,7 +148,7 @@ export class CoordinatorRepo {
242
148
  now = () => Date.now();
243
149
  /** Test seam: overridable RNG (0..1) for sample-rate gating. */
244
150
  rand = () => Math.random();
245
- constructor(keyNetwork, createClusterClient, storageRepo, cfg, localCluster, localPeerId, fretService, clusterLatestCallback, reputation, stateStore, acquireBlockFromCohort, proofAnchoring) {
151
+ constructor(keyNetwork, createClusterClient, storageRepo, cfg, localCluster, localPeerId, fretService, clusterLatestCallback, reputation, stateStore, acquireBlockFromCohort, proofAnchoring, underReplicationLedger, onCommittedHolders) {
246
152
  this.keyNetwork = keyNetwork;
247
153
  this.createClusterClient = createClusterClient;
248
154
  this.storageRepo = storageRepo;
@@ -250,6 +156,8 @@ export class CoordinatorRepo {
250
156
  this.clusterLatestCallback = clusterLatestCallback;
251
157
  this.acquireBlockFromCohort = acquireBlockFromCohort;
252
158
  this.proofAnchoring = proofAnchoring;
159
+ this.underReplicationLedger = underReplicationLedger;
160
+ this.onCommittedHolders = onCommittedHolders;
253
161
  this.localPeerId = localPeerId;
254
162
  this.log = createLogger('coordinator-repo', localPeerId?.toString());
255
163
  const policy = {
@@ -299,7 +207,8 @@ export class CoordinatorRepo {
299
207
  peerId: localPeerId,
300
208
  wasTransactionExecuted: localCluster.wasTransactionExecuted?.bind(localCluster),
301
209
  getExecutedPendResult: localCluster.getExecutedPendResult?.bind(localCluster),
302
- getExecutedCommitResult: localCluster.getExecutedCommitResult?.bind(localCluster)
210
+ getExecutedCommitResult: localCluster.getExecutedCommitResult?.bind(localCluster),
211
+ reconcileRefusedCommit: localCluster.reconcileRefusedCommit?.bind(localCluster)
303
212
  } : undefined;
304
213
  this.coordinator = new ClusterCoordinator(keyNetwork, createClusterClient, policy, localClusterRef, fretService, reputation, stateStore);
305
214
  }
@@ -316,58 +225,87 @@ export class CoordinatorRepo {
316
225
  await this.coordinator.recoverTransactions();
317
226
  }
318
227
  /**
319
- * Check if this node is in the cluster for a given block.
320
- * Uses findCluster membership in the real network layer, self is always
321
- * included in the cohort when this node is responsible. This serves as a
322
- * defense-in-depth guard for requests that arrive at the wrong node.
323
- * Returns true if localPeerId is not set (backward compat for single-node/test setups).
228
+ * Whether this node is in the cohort for a block. Uses `findCluster` membership — in the real network
229
+ * layer, self is in the cohort exactly when this node is among the nearest `clusterSize` serving peers
230
+ * for the block, so a request that reaches a node outside it was sent to the wrong machine.
231
+ *
232
+ * Three answers, because a lookup that THREW is not an answer: `undetermined` is never cached (routing
233
+ * can recover on the next request) and never read as `responsible` — what each caller does with it is
234
+ * that caller's posture (see {@link verifyResponsibility} and `get`).
235
+ *
236
+ * With no `localPeerId` the check is skipped and the answer is `responsible`. That bypass exists for
237
+ * wiring without an identity (direct constructors, single-node and test setups), never for production,
238
+ * where `libp2p-node-base` always passes the node's peer id.
239
+ *
240
+ * NOTE: a cached answer stands for up to {@link RESPONSIBILITY_TTL_MS} after the cohort changes, so a
241
+ * node can still say `responsible` for a minute after it stops being so. {@link soloCohortDurability}'s
242
+ * `unrouted` class and `ClusterCoordinator.executeClusterTransaction`'s cohort-membership guard are what
243
+ * keep that window honest; if churn ever makes the window matter, shorten the TTL rather than
244
+ * re-looking-up on every write.
324
245
  */
325
- async isResponsibleForBlock(blockId) {
246
+ async responsibilityFor(blockId) {
326
247
  if (!this.localPeerId)
327
- return true;
248
+ return 'responsible';
328
249
  const cached = this.responsibilityCache.get(blockId);
329
250
  if (cached && cached.expires > Date.now()) {
330
- return cached.inCluster;
251
+ return cached.inCluster ? 'responsible' : 'not-responsible';
331
252
  }
332
- const blockIdBytes = new TextEncoder().encode(blockId);
333
253
  let inCluster;
334
254
  try {
335
- const peers = await this.keyNetwork.findCluster(blockIdBytes);
255
+ const peers = await this.keyNetwork.findCluster(routingKeyForBlock(blockId));
336
256
  inCluster = this.localPeerId.toString() in peers;
337
257
  }
338
258
  catch (err) {
339
259
  this.log('proximity:check-error', { blockId, error: err.message });
340
- // On failure, assume responsible to avoid false rejections
341
- return true;
260
+ return 'undetermined';
342
261
  }
343
- this.responsibilityCache.set(blockId, { inCluster, expires: Date.now() + CoordinatorRepo.RESPONSIBILITY_TTL_MS });
262
+ this.responsibilityCache.set(blockId, { inCluster, expires: Date.now() + RESPONSIBILITY_TTL_MS });
344
263
  this.log('proximity:checked', { blockId, inCluster });
345
- return inCluster;
264
+ return inCluster ? 'responsible' : 'not-responsible';
346
265
  }
347
266
  /**
348
- * Verify this node is responsible for all given block IDs. Throws if not.
267
+ * The write path's gate (`pend`, `cancel`, `commit`): refuse unless this node is responsible for EVERY
268
+ * block, and refuse too when it cannot tell — fail CLOSED. A write accepted on a thrown lookup commits
269
+ * where the network was never consulted (GitHub #19); refusing hands it back to the writer's transactor,
270
+ * which excludes this peer and re-picks inside the cohort.
271
+ *
272
+ * Every block is checked so the error names all of them. When the blocks split between the two kinds,
273
+ * `not-responsible` is the one thrown: it is a settled answer that the request is misrouted, which no
274
+ * recovery of the lookup would change; the undetermined blocks are still logged.
275
+ *
276
+ * For `cancel`, a refusal on a transient lookup fault leaves the pending record standing on this node
277
+ * until the writer's cancel retry lands on another cohort member — and this node is still discharged
278
+ * then, since as a member it judges the cancel against the record's own `peers`, not against a lookup.
349
279
  */
350
280
  async verifyResponsibility(blockIds) {
351
281
  const notResponsible = [];
282
+ const undetermined = [];
352
283
  for (const blockId of blockIds) {
353
- if (!await this.isResponsibleForBlock(blockId)) {
284
+ const verdict = await this.responsibilityFor(blockId);
285
+ if (verdict === 'not-responsible')
354
286
  notResponsible.push(blockId);
355
- }
356
- }
357
- if (notResponsible.length > 0) {
358
- this.log('proximity:rejected', { blockIds: notResponsible });
359
- throw new Error(`Not responsible for block(s): ${notResponsible.join(', ')}`);
287
+ else if (verdict === 'undetermined')
288
+ undetermined.push(blockId);
360
289
  }
290
+ if (notResponsible.length === 0 && undetermined.length === 0)
291
+ return;
292
+ this.log('proximity:rejected', { notResponsible, undetermined });
293
+ throw notResponsible.length > 0
294
+ ? new ResponsibilityRefusalError('not-responsible', notResponsible)
295
+ : new ResponsibilityRefusalError('undetermined', undetermined, 'cohort lookup failed; refusing the write rather than accepting it unrouted');
361
296
  }
362
297
  async get(blockGets, options) {
363
- // Soft proximity check — warn but still serve reads for graceful degradation
364
- // NOTE: a soft-served read now also *acquires* the block durably (see restoreCorroborated), where
365
- // before it could at most promote a pending this node already held. So a soft serve leaves behind
366
- // a replica of a block this node is not responsible for, and nothing sweeps those: ring-shift
367
- // sheds a keyspace RANGE, not "blocks outside my cohort". Fine while soft serves are what they
368
- // are meant to be — a rare degradation during routing churn since routing already placed this
369
- // node near the block. If they ever become routine, gate acquisition (not the serve itself) on
370
- // isResponsibleForBlock.
298
+ // Soft proximity check — warn but still serve reads for graceful degradation. Unlike the write
299
+ // path it stays OPEN when the lookup throws: a read is best-effort, and the consult below flags
300
+ // whatever it could not confirm (`unavailable`, `unconfirmedAheadRev`) rather than posing as sure.
301
+ // NOTE: a soft-served read also *acquires* the block durably (see restoreCorroborated), so a soft
302
+ // serve leaves behind a replica of a block this node is not responsible for, and nothing sweeps
303
+ // those: ring-shift sheds a keyspace RANGE, not "blocks outside my cohort". Soft serves are now
304
+ // confined to the responsibility caches' staleness window: a remote read for a block this node is
305
+ // not responsible for is redirected by `RepoService.checkRedirect` before it gets here, and this
306
+ // node's own transactor routes a read here only when self is in the block's cohort. If soft serves
307
+ // ever become routine (steady `proximity:get-warning` lines while the network's membership is not
308
+ // changing), gate acquisition (not the serve itself) on `responsibilityFor`.
371
309
  //
372
310
  // NOTE: accepted tradeoff — this check and `fetchBlockFromCluster` below each run their own
373
311
  // `findCluster` for the same block, so a cold read costs two cohort lookups where one shared
@@ -375,16 +313,15 @@ export class CoordinatorRepo {
375
313
  // and this check's future is open (blocked `writer-and-servers-disagree-on-where-a-block-lives`,
376
314
  // option D2(b)). Revisit if a device profile shows `findCluster` as material.
377
315
  for (const blockId of blockGets.blockIds) {
378
- if (!await this.isResponsibleForBlock(blockId)) {
316
+ if (await this.responsibilityFor(blockId) === 'not-responsible') {
379
317
  this.log('proximity:get-warning', { blockId, msg: 'serving read for non-responsible block' });
380
318
  }
381
319
  }
382
320
  // First try local storage
383
321
  const localResult = await this.storageRepo.get(blockGets, options);
384
322
  // Decide per-block whether to consult cluster peers. Two triggers:
385
- // (a) Missing — block isn't present locally at all. Consults unless an earlier consult
386
- // SETTLED its absence within the last `readRepairWindowMs` (see `settledAbsences` and
387
- // `absenceIsSettled`) — today only when this node is the block's whole cohort.
323
+ // (a) Missing — block isn't present locally at all. Consults on every read: an absence is
324
+ // never remembered (see the accepted-tradeoff NOTE below).
388
325
  // (b) Stale-by-policy — block is present but read-repair policy says verify.
389
326
  // Skip cluster fetch if this is already a sync request (to prevent recursive queries).
390
327
  // A sync read is also never marked `unavailable` here — the consult it skips is the
@@ -394,38 +331,33 @@ export class CoordinatorRepo {
394
331
  const skipClusterFetch = options?.skipClusterFetch;
395
332
  // NOTE: NetworkTransactor.get treats an authoritative "absent" ({ state: {} })
396
333
  // as final and no longer retries it (ticket txn-perf-authoritative-notfound),
397
- // relying on this cluster reconciliation to have already run — within the last
398
- // `readRepairWindowMs`: an unflagged absent means the cohort confirmed this absence
399
- // at most one window ago (a read inside the window serves the memo of that
400
- // confirmation, see `settledAbsences`), the same currency guarantee a held block's
401
- // content already carries. When the consult FAILS outright or runs without ruling
402
- // the block out and the block stays missing the entry is flagged `unavailable`
403
- // below with a reason naming what the consult established (see AbsenceVerdict and
404
- // the mapping in the loop body), which re-enables the transactor-level retry against
405
- // a different peer; none of those outcomes is ever remembered. If a coordinator is
406
- // configured WITHOUT clusterLatestCallback, there is no cohort to consult and the
407
- // local answer IS the whole truth it stays authoritative, with no flag and no
408
- // transactor-level retry to compensate. That is fine (such a coordinator has no
409
- // cluster to reconcile against), but keep this coupling in mind if a partial-cluster
410
- // read path is added.
334
+ // relying on this cluster reconciliation to have already run — for THIS read: an
335
+ // unflagged absent means the cohort was consulted before answering. When the consult
336
+ // FAILS outright or runs without ruling the block out and the block stays
337
+ // missing the entry is flagged `unavailable` below with a reason naming what
338
+ // the consult established (see AbsenceVerdict and the mapping in the loop body),
339
+ // which re-enables the transactor-level retry against a different peer. If a
340
+ // coordinator is configured WITHOUT clusterLatestCallback, there is no cohort to
341
+ // consult and the local answer IS the whole truth it stays authoritative, with
342
+ // no flag and no transactor-level retry to compensate. That is fine (such a
343
+ // coordinator has no cluster to reconcile against), but keep this coupling in
344
+ // mind if a partial-cluster read path is added.
345
+ //
346
+ // NOTE: accepted tradeoff a block missing locally consults on EVERY read, even on a cohort
347
+ // of one, where the consult is one `findCluster` (0.009 ms measured) and a once-per-window log
348
+ // line (see the solo-self exit). Remembering an absence was tried and removed (GitHub issue
349
+ // #20, ticket drop-the-settled-absence-memo): a cohort-of-one memo outlived the view it was
350
+ // settled under, and a node whose view then grew served a block another coordinator had just
351
+ // committed as never created. Cohort-member writes (`ClusterRepo` → storage) never pass
352
+ // through this class, so nothing here could clear such a memo either. Revisit only with a memo
353
+ // bound to the cohort view it was settled under AND cleared by every writer of local storage;
354
+ // `coordinator-repo-absence-write-bypass.spec.ts` is the gate, and backlog
355
+ // `feat-a-cohort-member-remembers-a-settled-absence` holds the design notes.
411
356
  if (this.clusterLatestCallback && !skipClusterFetch) {
412
357
  for (const blockId of blockGets.blockIds) {
413
358
  const localEntry = localResult[blockId];
414
359
  const localRev = localEntry?.state?.latest?.rev;
415
360
  const isMissing = !localEntry?.state?.latest;
416
- if (!isMissing) {
417
- // Present: any remembered absence is dead. Dropped now rather than left to age out,
418
- // so a block that vanishes again inside the same window consults instead of being
419
- // served the old memo.
420
- this.settledAbsences.delete(blockId);
421
- }
422
- else if (this.absenceIsSettled(blockId)) {
423
- // An earlier consult settled this absence within one window: serve the local entry
424
- // as it stands — an authoritative absent, or pending-only content — with no flag and
425
- // no log line. The held-block skip below logs nothing either, and a per-read line
426
- // here would recreate the volume this skip exists to remove.
427
- continue;
428
- }
429
361
  const isStale = !isMissing && this.shouldReadRepair(blockId);
430
362
  if (!isMissing && !isStale) {
431
363
  // No consult this pass — the read-repair window says this block was checked
@@ -445,33 +377,18 @@ export class CoordinatorRepo {
445
377
  localRev
446
378
  });
447
379
  }
448
- // Evidence gathered during the consult is at least this old, so a memo stamped with it never
449
- // outlives "confirmed within one window" by the consult's own duration.
450
- const consultStartedAt = this.now();
451
380
  try {
452
- const { absence, currency, absenceSettled } = await this.fetchBlockFromCluster(blockId, blockGets.context, localRev);
453
- const refreshed = await this.storageRepo.get({ blockIds: [blockId], context: blockGets.context }, options);
381
+ const { absence, currency } = await this.fetchBlockFromCluster(blockId, blockGets.context, localRev);
382
+ // `lineageOf` rides along: the refreshed entry REPLACES the local one below, and an
383
+ // entry that lost its lineage answer would read to the asker as "could not say".
384
+ const refreshed = await this.storageRepo.get({
385
+ blockIds: [blockId], context: blockGets.context,
386
+ ...(blockGets.lineageOf === undefined ? {} : { lineageOf: blockGets.lineageOf })
387
+ }, options);
454
388
  const newRev = refreshed[blockId]?.state?.latest?.rev;
455
389
  if (refreshed[blockId]) {
456
390
  localResult[blockId] = refreshed[blockId];
457
391
  }
458
- // Remember a settled absence for one window; forget it after every other outcome. A
459
- // consult that ended `unconfirmed`, `isolated` or `claimed`, that asked a multi-peer
460
- // cohort, or that rested on no real cohort view, leaves no memo, so the next read consults (and flags) exactly as before —
461
- // including after a sampled or paranoid re-consult of a block whose memo was still fresh.
462
- // NOTE: a local pend/commit that clears the memo WHILE this consult is in flight is undone
463
- // by the stamp below — one read inside the window can then serve an absent the refused
464
- // writer knows is wrong. Bounded: that writer's next pend clears the memo again. If it ever
465
- // shows as a stuck writer, skip this stamp when a clear of this block landed after
466
- // `consultStartedAt`.
467
- if (isMissing) {
468
- if (!refreshed[blockId]?.state?.latest && absence === 'confirmed' && absenceSettled) {
469
- this.settledAbsences.set(blockId, consultStartedAt);
470
- }
471
- else {
472
- this.settledAbsences.delete(blockId);
473
- }
474
- }
475
392
  if (isStale) {
476
393
  if (typeof newRev === 'number' && typeof localRev === 'number' && newRev > localRev) {
477
394
  this.log('cluster-tx:read-repair-applied', { blockId, oldRev: localRev, newRev });
@@ -523,9 +440,6 @@ export class CoordinatorRepo {
523
440
  // rather than return a stale cohort view, revisit — that would put the
524
441
  // isolated case back under this vaguer reason.
525
442
  if (isMissing) {
526
- // A consult that threw settled nothing — not even a memo that was still fresh
527
- // before a sampled or paranoid re-consult.
528
- this.settledAbsences.delete(blockId);
529
443
  this.flagUnconfirmedAbsence(localResult, blockId, 'peers-unreachable');
530
444
  }
531
445
  else {
@@ -735,33 +649,33 @@ export class CoordinatorRepo {
735
649
  }
736
650
  }
737
651
  }
738
- /**
739
- * Decide whether a block MISSING locally may skip its consult: an earlier consult settled its
740
- * absence within one read-repair window (see {@link settledAbsences}). The absence counterpart of
741
- * {@link shouldReadRepair}, giving absence the guarantee the window already gives content —
742
- * "checked with the cohort within one window" — and nothing weaker.
743
- *
744
- * Modes: `paranoid` never skips ("verify every read" means every read). `lazy` uses the window and
745
- * the sample rate, exactly as {@link shouldReadRepair} does. `off` uses the window but not the
746
- * sample rate (a `lazy`-only knob, as it is for content): `off` disables STALE-CONTENT repair, and
747
- * the absence consult is not that — it is what makes an absent answer authoritative at all — so
748
- * leaving it unbounded would make the mode meant to do less network work do more.
749
- */
750
- absenceIsSettled(blockId) {
751
- if (this.readRepairMode === 'paranoid')
752
- return false;
753
- const at = this.settledAbsences.get(blockId);
754
- if (at == null || this.now() - at > this.readRepairWindowMs)
755
- return false;
756
- if (this.readRepairMode === 'lazy' && this.readRepairSampleRate > 0 && this.rand() < this.readRepairSampleRate)
757
- return false;
758
- return true;
759
- }
760
652
  /** Milliseconds since we last marked this block fresh, or undefined if never. */
761
653
  ageMs(blockId) {
762
654
  const lastSeen = this.lastSeenCommitMs.get(blockId);
763
655
  return lastSeen == null ? undefined : this.now() - lastSeen;
764
656
  }
657
+ /**
658
+ * Whether the solo-self exit of {@link fetchBlockFromCluster} has already named `blockId` inside
659
+ * the current read-repair window, for a block this node does NOT hold (`localRev` undefined). Such
660
+ * a block reaches that exit on every read, because `get` never remembers an absence (GitHub
661
+ * issue #20). So its `cluster-fetch:solo-self-skip` line and its stamp are both skipped while the
662
+ * stamp is at most one `readRepairWindowMs` old. An unconditional line would bring back issue #8's
663
+ * log volume for a probed but never-written collection. Re-stamping on every read would keep the
664
+ * stamp fresh and silence the line for good on a block read more often than once a window.
665
+ *
666
+ * Withholding the stamp never skips a consult: `get` asks {@link shouldReadRepair} only about
667
+ * present blocks. A HELD block reaches the exit only when read-repair already chose to consult, so
668
+ * this is always false for one. Its line and stamp are unchanged, and the line keeps pairing with
669
+ * `cluster-tx:read-repair-triggered`.
670
+ *
671
+ * NOTE: the stamp is shared with every exit that marks a block seen, so a missing block one of
672
+ * them stamped inside the window is not named until that stamp lapses. Fine for a diagnostic
673
+ * line; if anything ever needs "first answer from a self-only view" exactly, give the line its own
674
+ * per-block stamp.
675
+ */
676
+ soloAbsenceNamedThisWindow(blockId, localRev) {
677
+ return localRev === undefined && (this.ageMs(blockId) ?? Infinity) <= this.readRepairWindowMs;
678
+ }
765
679
  /** Mark blocks as freshly observed from cluster authority (post-commit or post-fetch). */
766
680
  markBlocksSeen(blockIds) {
767
681
  const now = this.now();
@@ -769,16 +683,6 @@ export class CoordinatorRepo {
769
683
  this.lastSeenCommitMs.set(id, now);
770
684
  }
771
685
  }
772
- /**
773
- * Drop any settled-absence memo for `blockIds` ({@link settledAbsences}) — called at the top of a
774
- * local pend or commit, before routing and whatever the outcome. Clearing is always the safe
775
- * direction: the most it can cost is one extra consult.
776
- */
777
- forgetSettledAbsences(blockIds) {
778
- for (const id of blockIds) {
779
- this.settledAbsences.delete(id);
780
- }
781
- }
782
686
  /**
783
687
  * True when a commit's approve votes form a strict majority of the FULL cohort — the only case
784
688
  * where "this node committed" is evidence that no rival commit moved past it. Two strict
@@ -838,35 +742,27 @@ export class CoordinatorRepo {
838
742
  * retire it when the memo's claimants are among the peers that answered), `no-evidence`
839
743
  * leaves it standing untouched. Required, not optional, so an exit added later has to say
840
744
  * which it means.
841
- * - `absenceSettled` — whether this pass SETTLED the block's absence for one read-repair window:
842
- * no write of it this node has not seen can exist. Only the solo-self exit can say that today
843
- * (this node is the whole cohort, so every acknowledged commit lands here). `get` remembers a
844
- * settled absence ({@link settledAbsences}) and skips the consult while the block stays missing.
845
- * `absence === 'confirmed'` alone is not enough — the empty-cohort exit reports it without
846
- * asking anyone, and a multi-peer cohort's unanimous "nothing" can be overtaken by a commit
847
- * still on its way to this node. Required, like
848
- * `currency`, so an exit added later has to say which side it falls on rather than
849
- * remembering (or forgetting) to arm something.
745
+ *
746
+ * Nothing is remembered about an absence: `get` runs this pass on every read of a block missing
747
+ * locally (see the accepted-tradeoff NOTE there).
850
748
  */
851
749
  async fetchBlockFromCluster(blockId, context, localRev) {
852
- // Unreachable from `get` (it guards on the callback); nothing to settle against.
853
750
  if (!this.clusterLatestCallback)
854
- return { absence: 'confirmed', currency: { kind: 'no-evidence' }, absenceSettled: false };
855
- const blockIdBytes = new TextEncoder().encode(blockId);
856
- const peers = await this.keyNetwork.findCluster(blockIdBytes);
751
+ return { absence: 'confirmed', currency: { kind: 'no-evidence' } };
752
+ const peers = await this.keyNetwork.findCluster(routingKeyForBlock(blockId));
857
753
  const peerIds = peers ? Object.keys(peers) : [];
858
754
  // NOTE: deliberately does NOT call `markBlocksSeen` — unlike the solo-self exit below.
859
- // An empty cohort is a ROUTING FAILURE, not a settled answer, and `Libp2pKeyPeerNetwork`
860
- // cannot even produce one (its `findCluster` always includes self); the only producer
861
- // today is the mesh harness's injected `findClusterFails`. Arming the read-repair window
755
+ // An empty cohort is a ROUTING FAILURE, not a settled answer. A serving node's
756
+ // `Libp2pKeyPeerNetwork.findCluster` cannot produce one: it always holds at least this
757
+ // node when no nearer serving peer is known. Only a node that serves no storage (a
758
+ // client-only libp2p node, which runs no CoordinatorRepo today) can see an empty cohort;
759
+ // the only producer here is the mesh harness's injected `findClusterFails`. Arming the read-repair window
862
760
  // here would suppress a genuine repair for a whole `readRepairWindowMs` after a transient
863
761
  // blip, and re-entering costs no network work beyond the `findCluster` the read already
864
762
  // makes. Do not "fix" this by symmetry with the solo-self exit.
865
- // For the same reason it returns `absenceSettled: false`: remembering the absence would serve an
866
- // authoritative absent for a whole window on the strength of a lookup that returned nobody.
867
763
  // Currency: nobody was asked, so nothing was refuted — an earlier pass's unsettled claim stands.
868
764
  if (peerIds.length === 0)
869
- return { absence: 'confirmed', currency: { kind: 'no-evidence' }, absenceSettled: false };
765
+ return { absence: 'confirmed', currency: { kind: 'no-evidence' } };
870
766
  // Solo-cluster short-circuit: the only responsible peer is us. There is no
871
767
  // remote to sync from, so skip the callback entirely. Querying ourselves
872
768
  // would dial self via SyncClient — pointless at best, and on nodes without
@@ -874,7 +770,11 @@ export class CoordinatorRepo {
874
770
  if (peerIds.length === 1
875
771
  && this.localPeerId
876
772
  && peerIds[0] === this.localPeerId.toString()) {
877
- this.log('cluster-fetch:solo-self-skip', { blockId });
773
+ // The line and the stamp below go together: both are skipped for a missing block this exit
774
+ // already named inside the window (see `soloAbsenceNamedThisWindow`).
775
+ const namedThisWindow = this.soloAbsenceNamedThisWindow(blockId, localRev);
776
+ if (!namedThisWindow)
777
+ this.log('cluster-fetch:solo-self-skip', { blockId });
878
778
  // Arm the lazy read-repair window. Without this the window is never stamped,
879
779
  // `shouldReadRepair` reads `lastSeen == null` forever, and every read re-enters this
880
780
  // exit — read -> stale -> consult -> solo skip -> no-op -> still stale, unbounded
@@ -892,18 +792,15 @@ export class CoordinatorRepo {
892
792
  // Deliberately opposite to the commit-side rule that withholds arming from a commit
893
793
  // whose quorum proves nothing about rivals: that damps nothing, this bounds an
894
794
  // otherwise unbounded loop. Landing both, keep both — see the specs for each.
895
- this.markBlocksSeen([blockId]);
795
+ if (!namedThisWindow)
796
+ this.markBlocksSeen([blockId]);
896
797
  // Currency: this exit queried NOBODY, so it refutes nothing — an earlier pass's unsettled
897
798
  // claim survives it. Note the coupling with the arming just above: retained doubt now
898
799
  // persists for up to `readRepairWindowMs` before a consult can refute it. That is correct
899
800
  // and deliberate — the window damps repair EFFORT, not honesty — and it is the same
900
801
  // coupling the comment at the final exit below describes. Arming the window and keeping
901
802
  // the memo are answers to different questions; do not collapse them.
902
- // Absence: settled. Nobody else could hold the block, and re-asking inside one window learns
903
- // nothing the next `findCluster` would not — the same argument, with the same one-window
904
- // self-heal when a cohort appears, as the arming above. (That arming's stamp is NOT what
905
- // suppresses the next read of a missing block: `get` reads `settledAbsences` for those.)
906
- return { absence: 'confirmed', currency: { kind: 'no-evidence' }, absenceSettled: true };
803
+ return { absence: 'confirmed', currency: { kind: 'no-evidence' } };
907
804
  }
908
805
  const { corroborated, corroboration, local, silent, answered, claims, uncorroboratedRev, deadlock } = await this.queryClusterForLatest(peerIds, blockId, context);
909
806
  // Any silence taints the WHOLE consult, not a fraction of it (fail-closed): one silent
@@ -991,14 +888,7 @@ export class CoordinatorRepo {
991
888
  if (deadlock === 'cohort-too-small') {
992
889
  this.markBlocksSeen([blockId]);
993
890
  }
994
- // Absence NOT settled, even when every cohort member answered "I hold nothing". This node
995
- // takes part in other coordinators' writes as a cohort member through ClusterRepo, straight
996
- // into storage and never through this class's pend/commit, so nothing here would clear the
997
- // memo; and a commit acknowledged at super-majority reaches the remaining members in the
998
- // background. Remembering this answer served a writer's own create as absent through a
999
- // lagging member (fresh-node-ddl-multi Scenario B, 5 of 20 runs). Only the solo exit
1000
- // settles — see backlog `feat-a-cohort-member-remembers-a-settled-absence`.
1001
- return { absence, currency, absenceSettled: false };
891
+ return { absence, currency };
1002
892
  }
1003
893
  // The self answer is the sharper baseline (same storage, same context, read alongside the
1004
894
  // cohort's), but it exists only when `findCluster` returned this node. A soft serve for a
@@ -1049,8 +939,7 @@ export class CoordinatorRepo {
1049
939
  // ahead. Only reachable when a peer answered (a corroboration requires claims), so the
1050
940
  // shared verdict resolves to `nothing-ahead` and a memo whose claimants are among those
1051
941
  // answers is cleared.
1052
- // `absenceSettled`: never read for a held block; false for the same reason as the exit above.
1053
- return { absence: silenceVerdict, currency: nothingAheadVerdict, absenceSettled: false };
942
+ return { absence: silenceVerdict, currency: nothingAheadVerdict };
1054
943
  }
1055
944
  // Corroborated revision is ahead of ours — converge onto it.
1056
945
  const rev = await this.restoreCorroborated(blockId, corroborated, baselineRev, peerIds);
@@ -1078,13 +967,12 @@ export class CoordinatorRepo {
1078
967
  // unsettled claim (`recordAheadClaim`) and keeps stamping reads served below it while the
1079
968
  // window suppresses the retry — the window damps repair effort, not honesty.
1080
969
  // NOTE: that damping covers only a block this node holds at an OLDER revision. A block entirely
1081
- // missing locally is suppressed only by its own absence memo (`settledAbsences`), and this exit
1082
- // never arms it (`absenceSettled: false` below a failed acquisition is `claimed`). So a
1083
- // persistently failing acquisition e.g. a two-node deployment that never set
1084
- // `assumedClusterSize`, where the content quorum can never be met still re-fetches an archive
1085
- // on every read of that block. Correct, and self-limiting once the cohort can agree; if it ever
1086
- // shows as read amplification, gate the acquisition step (not the latest-query) on the same
1087
- // window rather than letting a `claimed` absence settle.
970
+ // missing locally never consults the window (`get` consults on every read of a missing block,
971
+ // before `shouldReadRepair`), so a persistently failing acquisition e.g. a two-node deployment
972
+ // that never set `assumedClusterSize`, where the content quorum can never be met — re-fetches an
973
+ // archive on every read of that block. Correct, and self-limiting once the cohort can agree; if
974
+ // it ever shows as read amplification, gate the acquisition step (not the latest-query) on the
975
+ // same window rather than remembering the absence (GitHub issue #20).
1088
976
  this.markBlocksSeen([blockId]);
1089
977
  // Converged: the corroboration is itself the evidence that nothing is ahead, and it came from
1090
978
  // peers that answered — the shared verdict resolves to `nothing-ahead`, and the memo retires
@@ -1092,9 +980,7 @@ export class CoordinatorRepo {
1092
980
  const currency = converged
1093
981
  ? nothingAheadVerdict
1094
982
  : { kind: 'unsettled-claim', rev: corroborated.rev, claimants: claimantsAtOrAbove(corroborated.rev), silent };
1095
- // Never settles an absence: restored → the block is now present (nothing to remember); not
1096
- // restored → `claimed`.
1097
- return { absence, currency, absenceSettled: false };
983
+ return { absence, currency };
1098
984
  }
1099
985
  /**
1100
986
  * Bring this node up to the cohort-corroborated `corroborated`, returning the revision it holds
@@ -1632,12 +1518,6 @@ export class CoordinatorRepo {
1632
1518
  }
1633
1519
  async pend(request, options) {
1634
1520
  const allBlockIds = blockIdsForTransforms(request.transforms);
1635
- // Forget any settled absence of these blocks FIRST — before responsibility, routing, or the
1636
- // outcome is known. A pend refused because a block already exists somewhere is the strongest
1637
- // evidence there is that the memo was wrong, and the writer's retry re-reads: that read must
1638
- // consult (and restore) rather than serve the memo's authoritative absent for the rest of the
1639
- // window. After a success the block is present and the memo is dead anyway.
1640
- this.forgetSettledAbsences(allBlockIds);
1641
1521
  await this.verifyResponsibility(allBlockIds);
1642
1522
  const result = await this.pendThroughCluster(request, allBlockIds, options);
1643
1523
  // A pend the blocks ACCEPTED is the proof that no reservation is holding them any more — the
@@ -1652,12 +1532,14 @@ export class CoordinatorRepo {
1652
1532
  return result;
1653
1533
  }
1654
1534
  /** The cluster half of {@link pend}, after responsibility is verified: consensus, the local-verdict
1655
- * arms, and the two optimistic-concurrency classifiers a rejection is run through. */
1535
+ * arms, and the catch that turns each optimistic-concurrency refusal into a retryable answer
1536
+ * the two signed-evidence ones ({@link ConflictRaceLostError}, {@link BlocksHeldError}) directly,
1537
+ * and a validator rejection only through {@link classifyStaleRejection}. */
1656
1538
  async pendThroughCluster(request, allBlockIds, options) {
1657
1539
  const coordinatingBlockIds = options?.coordinatingBlockIds ?? allBlockIds;
1658
- const peerCount = await this.coordinator.getClusterSize(coordinatingBlockIds[0]);
1659
- if (peerCount <= 1) {
1660
- return await this.storageRepo.pend(request, options);
1540
+ const cohort = await this.coordinator.resolveCohort(coordinatingBlockIds[0]);
1541
+ if (!cohort.resolved || cohort.peerIds.length <= 1) {
1542
+ return await this.pendSolo(request, cohort, options);
1661
1543
  }
1662
1544
  const message = {
1663
1545
  operations: [{ pend: request }],
@@ -1665,7 +1547,13 @@ export class CoordinatorRepo {
1665
1547
  coordinatingBlockIds
1666
1548
  };
1667
1549
  try {
1668
- const { localExecuted, localPendResult, cohortPendRefusals } = await this.coordinator.executeClusterTransaction(coordinatingBlockIds[0], message, options);
1550
+ const { record, localExecuted, localPendResult, cohortPendRefusals } = await this.coordinator.executeClusterTransaction(coordinatingBlockIds[0], message, options);
1551
+ // Who ACCEPTED the pending record — the cohort members whose promise vote approved it, plus
1552
+ // or minus this node's own member per exit below. A pend's `confirmed` is not a commit's:
1553
+ // accepting a pending record confers no storage durability (a pend that reached
1554
+ // pend-consensus may still have been stored by nobody — see the local-verdict arm below),
1555
+ // so the two numbers are never comparable, and the field says so.
1556
+ const pendDurability = (selfAccepted) => pendCohortDurability(record, selfAccepted, this.localPeerId?.toString());
1669
1557
  // The first cohort refusal in peer-id order, so two coordinators facing the same cohort
1670
1558
  // answer with the same one. Which refusal is reported does not change the outcome — every
1671
1559
  // entry is conflict-shaped and every one means "rebase and retry" — only which `pending` /
@@ -1716,7 +1604,9 @@ export class CoordinatorRepo {
1716
1604
  hasMissing: !!result.missing?.length,
1717
1605
  hasPending: !!result.pending?.length
1718
1606
  });
1719
- return answerWithCohortRefusal(result);
1607
+ // Storage's own answer is `local`; the cohort's answer replaces it — this node accepted
1608
+ // the record iff its fallback pend succeeded.
1609
+ return answerWithCohortRefusal(result.success ? { ...result, durability: pendDurability(true) } : result);
1720
1610
  }
1721
1611
  // Local cluster already executed during consensus — return storage's own verdict rather
1722
1612
  // than fabricating a success (the peerCount <= 1 path above returns storage's real result
@@ -1729,7 +1619,10 @@ export class CoordinatorRepo {
1729
1619
  // commit that reached commit-consensus IS the authoritative commit (Theorem 9), whereas a
1730
1620
  // pend that reached pend-consensus may still have been stored by nobody.
1731
1621
  if (localPendResult !== undefined) {
1732
- if (localPendResult.success || isConflictFailure(localPendResult)) {
1622
+ if (localPendResult.success) {
1623
+ return answerWithCohortRefusal({ ...localPendResult, durability: pendDurability(true) });
1624
+ }
1625
+ if (isConflictFailure(localPendResult)) {
1733
1626
  return answerWithCohortRefusal(localPendResult);
1734
1627
  }
1735
1628
  // A bare-reason refusal (no pending/missing — e.g. a local validation-hook fault)
@@ -1743,11 +1636,14 @@ export class CoordinatorRepo {
1743
1636
  }
1744
1637
  // No verdict retained (member predates retention, restart, or TTL): the prior shape,
1745
1638
  // still subject to the cohort-refusal rule — a lost local verdict must not resurrect the
1746
- // fabricated success this ticket exists to remove.
1639
+ // fabricated success this ticket exists to remove. For the durability class, a tolerated
1640
+ // local fault means this node did NOT accept (`false`: its approve vote is withdrawn from the
1641
+ // count), and an absent verdict says nothing either way (`undefined`: the vote stands).
1747
1642
  return answerWithCohortRefusal({
1748
1643
  success: true,
1749
1644
  pending: [],
1750
- blockIds: allBlockIds
1645
+ blockIds: allBlockIds,
1646
+ durability: pendDurability(localPendResult === undefined ? undefined : false)
1751
1647
  });
1752
1648
  }
1753
1649
  catch (error) {
@@ -1769,6 +1665,14 @@ export class CoordinatorRepo {
1769
1665
  // each member compares the rivals as IT holds them, so a member that already approved X
1770
1666
  // keeps X while a member that approved Y first keeps Y, and no rival reaches a promise
1771
1667
  // supermajority. (The hash tie-break is already symmetric — it cannot be the fix.)
1668
+ // TWO contenders coordinated by different nodes reach an all-lose round by another route,
1669
+ // and not by this catch: both collect their promises in parallel, so BOTH reach pend
1670
+ // consensus; each coordinator's own member applies its own pend first (local-first
1671
+ // delivery), every other member's storage refuses whichever arrived second, and each writer
1672
+ // is downgraded by `answerWithCohortRefusal` above. Measured by
1673
+ // `test/transaction-node-count-sweep.spec.ts` on the in-process mesh: round one lost by both
1674
+ // writers in 20 of 20 races across two to five machines (2026-09-16). Same cost and same
1675
+ // revisit condition as below.
1772
1676
  // Fine as it stands: since the torn-action fixes landed, an all-lose round costs one
1773
1677
  // retry cycle rather than wedging, and the contenders are separated next round by the
1774
1678
  // jittered backoff plus the aged retry priority carried on the re-pend
@@ -1782,13 +1686,49 @@ export class CoordinatorRepo {
1782
1686
  if (error instanceof ConflictRaceLostError) {
1783
1687
  return { success: false, conflict: true, reason: error.message };
1784
1688
  }
1785
- const stale = await this.classifyStaleRejection(error, request, allBlockIds)
1786
- ?? await this.classifyPendingConflictRejection(error, request, allBlockIds);
1689
+ if (error instanceof BlocksHeldError) {
1690
+ return await this.answerBlocksHeld(error, request, allBlockIds, cohort.peerIds.length);
1691
+ }
1692
+ const stale = await this.classifyStaleRejection(error, request, allBlockIds);
1787
1693
  if (stale)
1788
1694
  return stale;
1789
1695
  throw error;
1790
1696
  }
1791
1697
  }
1698
+ /**
1699
+ * The solo short-circuit of {@link pend}: a cohort of at most one peer runs no consensus, and the
1700
+ * pend goes straight to local storage exactly as before. What changes is the answer's class —
1701
+ * see {@link soloCohortDurability} for the four-way split. A refusal is returned untouched.
1702
+ */
1703
+ async pendSolo(request, cohort, options) {
1704
+ const result = await this.storageRepo.pend(request, options);
1705
+ return result.success ? { ...result, durability: this.soloCohortDurability(cohort) } : result;
1706
+ }
1707
+ /**
1708
+ * The class of a write that took the solo short-circuit, from the cohort resolution that put it
1709
+ * there. Policy-free: the write already happened, this only says what it was.
1710
+ * - not resolved (the lookup threw, or named nobody) → `unrouted`;
1711
+ * - resolved to exactly this node → `local`, a correct and complete one-machine write;
1712
+ * - resolved to exactly one peer that is NOT this node → `unrouted` — this node wrote somewhere
1713
+ * the cohort does not look, which is a write whose destination is wrong, not a solo write.
1714
+ * With no local peer id (direct constructors, single-node and test wiring) a resolved cohort of
1715
+ * one reads as `local`: the same posture `responsibilityFor` takes for a node with no
1716
+ * identity, and a node with no identity cannot be in any cohort but its own.
1717
+ *
1718
+ * Both `unrouted` arms are defence in depth now, not the main guard: {@link verifyResponsibility}
1719
+ * refuses a write whose lookup throws and a write for a cohort that excludes this node before the
1720
+ * short-circuit is reached. A write lands here unrouted only inside the responsibility cache's
1721
+ * staleness window, or when a second lookup fails moments after the first succeeded — and then it
1722
+ * still reports honestly rather than posing as `local`.
1723
+ */
1724
+ soloCohortDurability(cohort) {
1725
+ if (!cohort.resolved)
1726
+ return unroutedDurability();
1727
+ const sole = cohort.peerIds[0];
1728
+ if (this.localPeerId !== undefined && sole !== this.localPeerId.toString())
1729
+ return unroutedDurability();
1730
+ return localDurability(sole);
1731
+ }
1792
1732
  /**
1793
1733
  * Decide whether a cluster validator rejection was an optimistic-concurrency loss — the block
1794
1734
  * already advanced past the requested revision — rather than a genuine validation fault.
@@ -1860,51 +1800,56 @@ export class CoordinatorRepo {
1860
1800
  };
1861
1801
  }
1862
1802
  // NOTE: conservative — when only remote members saw the newer revision (local storage still
1863
- // behind), staleness can't be confirmed locally and the rejection stays a throw. If that
1864
- // shows up in practice, extend confirmation with a quorum read; never trust the reject text.
1865
- // `staleAt` is absent on this path for the same reason, and deliberately so there is no
1866
- // confirmed number to report, and the field's contract forbids inferring one from that text.
1803
+ // behind), staleness can't be confirmed locally and the rejection stays a throw. That revisit
1804
+ // condition HAS now tripped, for this method's former sibling: the pending-conflict refusal hit
1805
+ // exactly this window under delivery latency (the refusing member ahead of the coordinator) and
1806
+ // escaped as a permanent-looking throw. Its cure was not the quorum read suggested here but a
1807
+ // signed `held` vote — the member's own refusal, carried as evidence, so retryability stopped
1808
+ // depending on a local re-read at all (see {@link answerBlocksHeld}).
1809
+ //
1810
+ // The stale arm is deliberately LEFT on local corroboration. Its refusal is a revision claim,
1811
+ // and `staleAt` — the only place a losing writer learns the revision it lost to — can only be
1812
+ // reported from a revision this node read itself. Moving it to a non-counting vote would take
1813
+ // that number away, so it is its own ticket, not a rider on this one. Never trust the reject
1814
+ // text; `staleAt` is absent on the unconfirmed path for the same reason, and deliberately so.
1867
1815
  return undefined;
1868
1816
  }
1869
1817
  /**
1870
- * Sibling of {@link classifyStaleRejection} for the OTHER optimistic-concurrency refusal shape:
1871
- * the promise-phase pending-conflict vote (`validatePendOperations` rejecting a pend whose
1872
- * blocks are held by a different unresolved pending action). That vote surfaces here as a
1873
- * {@link ValidatorRejectionError}, and without classification it would escape as a throw —
1874
- * splitting multi-tree pends mid-batch instead of taking the retry path a lost race deserves.
1818
+ * Answer a pend the cohort refused with signed `held` votes: the blocks are reserved by a different
1819
+ * unresolved action on one or more members ({@link BlocksHeldError}). Always a retryable
1820
+ * {@link StaleFailure} with `conflict: true`, so the caller's normal retry path
1821
+ * (`Collection.sync`, and the multi-collection pendPhase via `isConflictFailure`) absorbs it
1822
+ * instead of a thrown error escaping mid-batch and splitting multi-tree pends.
1823
+ *
1824
+ * The local re-read is an ENRICHER, not a gate. It used to be the gate — this refusal arrived as a
1825
+ * `ValidatorRejectionError` and was only converted when the coordinator's own storage corroborated
1826
+ * the rival — and under latency the refusing member is routinely ahead of the coordinator, so the
1827
+ * corroboration missed and a transient refusal escaped as a permanent-looking throw. The `held`
1828
+ * vote is already signed evidence from the member that holds the rival, so retryability no longer
1829
+ * depends on a local re-read at all. What the re-read still buys, when it succeeds, is the concrete
1830
+ * rival list for {@link StaleFailure.pending} and the input {@link noteStuckReservation} needs to
1831
+ * name a block wedged behind a reservation that will never clear.
1875
1832
  *
1876
- * Same confirmation discipline as the stale classifier: purely local. Re-read the affected
1877
- * blocks from our own storage and require some block's `state.pendings` to carry a rival
1878
- * actionId; the signed reject text is never consulted. A confirmed rival returns a
1879
- * {@link StaleFailure} with `conflict: true` and the rivals as `pending` (`ActionPending`
1880
- * without `transform` the type allows it, and no consumer rebases from it). Unconfirmed
1881
- * including read errors during confirmation — stays a throw, preserving fail-fast for genuine
1882
- * validation faults. Checked after `classifyStaleRejection` so a confirmed committed loss
1883
- * (which carries the sharper `staleAt`) wins when both hold.
1833
+ * NOTE: `error.heldBy` (peerId holding action id) is dropped at this boundary, the same deliberate
1834
+ * drop the {@link ConflictRaceLostError} arm of {@link pendThroughCluster} documents: `StaleFailure`
1835
+ * has no field for it, and the members' action ids are not the same claim as the rivals this node
1836
+ * read out of its own storage, so they must not be folded into `pending`. A caller that wants to
1837
+ * WAIT on the holder rather than re-race it needs a typed field added here; never recover the ids
1838
+ * by parsing `reason`.
1884
1839
  */
1885
- async classifyPendingConflictRejection(error, request, blockIds) {
1886
- if (!(error instanceof ValidatorRejectionError))
1887
- return undefined;
1888
- let results;
1889
- try {
1890
- results = await this.storageRepo.get({ blockIds });
1891
- }
1892
- catch (readError) {
1893
- this.log('coordinator-repo:pend-conflict-classify-read-error', {
1840
+ async answerBlocksHeld(error, request, blockIds, peerCount) {
1841
+ const pending = await this.corroborateHeldBlocks(request, blockIds, peerCount);
1842
+ if (pending.length === 0) {
1843
+ // The member that refused is ahead of us — the normal shape under latency. Still a conflict:
1844
+ // the refusal is signed, and the un-enriched answer is exactly the shape the lost-race arm
1845
+ // returns. `noteStuckReservation` is deliberately not fed here; it counts refusals whose
1846
+ // holder THIS node can name, and a guess would poison its holder comparison.
1847
+ this.log('coordinator-repo:pend-held-uncorroborated', {
1894
1848
  actionId: request.actionId,
1895
- error: readError.message
1849
+ heldBy: error.heldBy
1896
1850
  });
1897
- return undefined;
1898
- }
1899
- const pending = [];
1900
- for (const blockId of blockIds) {
1901
- for (const actionId of results[blockId]?.state?.pendings ?? []) {
1902
- if (actionId !== request.actionId)
1903
- pending.push({ blockId, actionId });
1904
- }
1851
+ return { success: false, conflict: true, reason: error.message };
1905
1852
  }
1906
- if (pending.length === 0)
1907
- return undefined;
1908
1853
  // Counted as its own statement, never inside the log payload below: this call is the detection
1909
1854
  // mechanism, not a formatting step, and payload expressions in this repo are fair game to wrap
1910
1855
  // in an `enabled` gate (`Collection.advanceContext` does exactly that). A gate added there
@@ -1929,45 +1874,86 @@ export class CoordinatorRepo {
1929
1874
  };
1930
1875
  }
1931
1876
  /**
1932
- * Count one confirmed pending-conflict refusal against the holder(s) of each block it names, and say
1933
- * ONCE in words, at the moment it becomes provable when a block is wedged behind a reservation
1934
- * that is not going to clear.
1877
+ * The rival unresolved actions THIS node's own storage says hold `blockIds`, excluding the request's
1878
+ * own action (a redelivered pend must not corroborate against itself). Empty when nothing is found
1879
+ * including when the read itself fails, which is reported and then treated as "nothing to add",
1880
+ * since the caller's answer does not depend on it.
1935
1881
  *
1936
- * **Why this needs saying at all.** Every individual refusal here is indistinguishable from an
1937
- * ordinary lost race, which is a normal and healthy event, so the logs of a permanently wedged block
1938
- * read exactly like the logs of a busy one. Finding the difference today means noticing that the
1939
- * SAME rival action id keeps appearing across unrelated writers for as long as the process lives —
1940
- * a pattern nothing points at, and one that cost a downstream project several tickets and weeks to
1941
- * re-derive from raw traces. The node has the fact in hand at every refusal; this makes it sayable.
1882
+ * Carries no `transform`, which {@link ActionPending} allows and no consumer rebases from.
1942
1883
  *
1943
- * **The signal, and the two things that are NOT the signal.** The discriminator is repetition
1944
- * against an unchanged holder see {@link STUCK_RESERVATION_DISTINCT_ACTIONS} for why distinct
1945
- * refused actions is the right counter and for the measured threshold. Two cheaper-looking tests
1946
- * were tried and do not work: the members' in-memory reservation table
1947
- * (`ClusterMember.activeTransactions`) clears the moment a rival's pend reaches consensus, so a
1948
- * perfectly healthy rival inside its pend-to-commit window is absent from it too and absence there
1949
- * says nothing; and "the block already passed this pending record's revision" catches a different
1950
- * orphan class entirely — in the verified instance the wedged block sat at revision 1 while the
1951
- * orphaned record was for revision 2, still nominally promotable.
1952
- *
1953
- * **Never a control path.** This classifies and logs; it never refuses, expires, or deletes
1954
- * anything. Deciding when a durable pending record may be removed is precisely the hard problem
1955
- * backlog `debt-unpromotable-pending-records-need-a-sweep` exists for — deleting a live reservation
1956
- * is worse than the leak — and a counter accurate enough for a log line is not evidence enough to
1957
- * destroy state.
1884
+ * Diagnostic enrichment only, never a gate: the refusal it corroborates was already cast and signed
1885
+ * by a member, and nothing here can turn an admission into a refusal or the reverse.
1886
+ */
1887
+ async corroborateHeldBlocks(request, blockIds, peerCount) {
1888
+ const pending = [];
1889
+ // The refusing member reads the request's base only in a cohort that can leave a member out
1890
+ // (`ClusterMember.reservationRequestOf`); read it here under the same condition.
1891
+ const readsBase = cohortCanMissAPend(peerCount, this.superMajorityThreshold);
1892
+ try {
1893
+ for (const blockId of blockIds) {
1894
+ // The request's base for the block, read exactly as the refusing member read it (the
1895
+ // malformed-base log line is that member's, so none is repeated here).
1896
+ const reservation = readsBase
1897
+ ? reservationRequestFor(request, blockId, transformForBlockId(request.transforms, blockId)).request
1898
+ : { rev: request.rev };
1899
+ for (const claim of await this.pendingClaimsOf(blockId)) {
1900
+ // The same rule, on the same inputs, as the refusing member (`isReservationAgainst`): a
1901
+ // record this request's writer has built on is not the rival it was refused on, and
1902
+ // naming it would feed a superseded action into the writer's `pending` list and into
1903
+ // the stuck-reservation holder comparison; and, where the base is read, a record
1904
+ // claiming a slot past the request's declared base IS one, even when the requested
1905
+ // revision has moved beyond it — the revision rule alone would leave that refusal
1906
+ // uncorroborated.
1907
+ if (claim.actionId !== request.actionId && isReservationAgainst(claim, reservation)) {
1908
+ pending.push({ blockId, actionId: claim.actionId });
1909
+ }
1910
+ }
1911
+ }
1912
+ }
1913
+ catch (readError) {
1914
+ this.log('coordinator-repo:pend-conflict-classify-read-error', {
1915
+ actionId: request.actionId,
1916
+ error: readError.message
1917
+ });
1918
+ return [];
1919
+ }
1920
+ return pending;
1921
+ }
1922
+ /**
1923
+ * This node's pending records on `blockId`, each with the slot it claims when storage can say
1924
+ * (`IPendingClaimReader`, which `StorageRepo` implements). A repo without the capability answers
1925
+ * with `state.pendings` alone, every record read as an unknown claim — which reserves, so the
1926
+ * corroboration can only over-name, never under-name.
1927
+ */
1928
+ async pendingClaimsOf(blockId) {
1929
+ const reader = this.storageRepo;
1930
+ if (typeof reader.listPendingClaims === 'function') {
1931
+ return await reader.listPendingClaims(blockId);
1932
+ }
1933
+ const results = await this.storageRepo.get({ blockIds: [blockId] });
1934
+ return (results[blockId]?.state?.pendings ?? []).map(actionId => ({ actionId }));
1935
+ }
1936
+ /**
1937
+ * Count one confirmed pending-conflict refusal against the holder(s) of each block it names, and say
1938
+ * ONCE — in words, at the moment it becomes provable — when a block is wedged behind a reservation
1939
+ * that is not going to clear. The counting, the threshold and the wording live in
1940
+ * {@link StuckReservationTracker}; this node's part is the corroborated holder list and the tag.
1958
1941
  *
1959
1942
  * Returns the highest distinct-refusal count any of this refusal's blocks has now reached, for the
1960
1943
  * classification line to carry; it saturates at the threshold once an episode has been reported,
1961
1944
  * since the ids are dropped at that point.
1962
1945
  *
1963
- * NOTE: fed only by {@link classifyPendingConflictRejection}, i.e. by refusals that arrive as a
1964
- * cohort-wide validator rejection. A block only PART of whose cohort holds the stranded record can
1946
+ * NOTE: fed only by {@link answerBlocksHeld}, i.e. by cohort-wide `held`-answered refusals that this
1947
+ * node's own storage can corroborate. A block only PART of whose cohort holds the stranded record can
1965
1948
  * still reach approval super-majority, and its refusal then comes back through the retained local
1966
- * apply verdict (`getExecutedPendResult`) instead, which this never sees so a partially wedged
1967
- * block goes unnamed. That is the weaker condition (the write does land on the healthy members),
1968
- * and instrumenting the second path would count a refusal that the cohort as a whole did not make.
1969
- * If partial strands ever turn out to be the common shape in the field, the counter belongs on the
1970
- * member side (`ClusterMember.validatePendOperations`), where each member sees its own votes.
1949
+ * apply verdict (`getExecutedPendResult`) instead, which this never sees. The un-corroborated arm of
1950
+ * {@link answerBlocksHeld} is likewise unfed: this node cannot name the holder, and feeding it a
1951
+ * guess would poison the holder comparison, which is what separates a wedge from healthy
1952
+ * contention. Neither gap goes unnamed any more: every MEMBER keeps its own tracker, fed by its own
1953
+ * `held` votes (`ClusterMember.validatePendOperations`, tag `cluster-member:stuck-reservation`), so
1954
+ * a reservation only some members hold — the shape `a-member-that-missed-a-commit-refuses-every-
1955
+ * later-write` measured, before the member stopped refusing on a superseded record at all — is
1956
+ * named by the members that hold it.
1971
1957
  */
1972
1958
  noteStuckReservation(pending, refusedActionId) {
1973
1959
  const rivalsByBlock = new Map();
@@ -1978,38 +1964,9 @@ export class CoordinatorRepo {
1978
1964
  else
1979
1965
  rivalsByBlock.set(blockId, [actionId]);
1980
1966
  }
1981
- let highest = 0;
1982
- for (const [blockId, rivals] of rivalsByBlock) {
1983
- const holders = [...new Set(rivals)].sort();
1984
- const prior = this.stuckReservations.get(blockId);
1985
- // A different holder set is a DIFFERENT episode — the block changed hands, which is the
1986
- // healthy cycle — so the count starts over and the new holder gets its own chance to speak.
1987
- const watch = prior !== undefined && sameHolders(prior.holders, holders)
1988
- ? prior
1989
- : { holders, refused: new Set(), reported: false };
1990
- if (watch !== prior)
1991
- this.stuckReservations.set(blockId, watch);
1992
- if (watch.reported) {
1993
- highest = Math.max(highest, STUCK_RESERVATION_DISTINCT_ACTIONS);
1994
- continue;
1995
- }
1996
- watch.refused.add(refusedActionId);
1997
- highest = Math.max(highest, watch.refused.size);
1998
- if (watch.refused.size < STUCK_RESERVATION_DISTINCT_ACTIONS)
1999
- continue;
2000
- this.log('coordinator-repo:stuck-reservation', {
2001
- blockId,
2002
- // The ids an operator needs to grep for and to cancel, kept as data beside the prose so a
2003
- // log search finds the block and the action without parsing English.
2004
- holdingActionIds: holders,
2005
- distinctRefusedActions: watch.refused.size,
2006
- message: stuckReservationMessage(holders, watch.refused.size)
2007
- });
2008
- watch.reported = true;
2009
- // Said once per episode: from here the flag alone suppresses, and the ids have done their
2010
- // work (their count is in the line above), so drop them rather than growing a set for the
2011
- // unbounded remainder of a permanent condition.
2012
- watch.refused.clear();
1967
+ const { highest, named } = this.stuckReservations.note(rivalsByBlock, refusedActionId);
1968
+ for (const episode of named) {
1969
+ this.log('coordinator-repo:stuck-reservation', episode);
2013
1970
  }
2014
1971
  return highest;
2015
1972
  }
@@ -2027,11 +1984,7 @@ export class CoordinatorRepo {
2027
1984
  * episode holding an LRU slot can only evict a live one.
2028
1985
  */
2029
1986
  clearStuckReservations(blockIds, holderActionId) {
2030
- for (const blockId of blockIds) {
2031
- if (holderActionId !== undefined && !this.stuckReservations.peek(blockId)?.holders.includes(holderActionId))
2032
- continue;
2033
- this.stuckReservations.delete(blockId);
2034
- }
1987
+ this.stuckReservations.forget(blockIds, holderActionId);
2035
1988
  }
2036
1989
  async cancel(actionRef, options) {
2037
1990
  const blockIds = actionRef.blockIds;
@@ -2083,54 +2036,12 @@ export class CoordinatorRepo {
2083
2036
  }
2084
2037
  async commit(request, options) {
2085
2038
  const blockIds = request.blockIds;
2086
- // Same as `pend`: forget any settled absence of these blocks before routing, whatever the outcome.
2087
- this.forgetSettledAbsences(blockIds);
2088
2039
  await this.verifyResponsibility(blockIds);
2089
- const cohortPeerIds = await this.coordinator.getClusterPeerIds(blockIds[0]);
2090
- const peerCount = cohortPeerIds.length;
2091
- if (peerCount <= 1) {
2092
- // Solo cohort: consensus never runs, so no ClusterRecord exists to project a proof from —
2093
- // the lone member self-signs a one-peer proof instead (mintSoloCommitProof), which is what
2094
- // lets a block born on a cohort of one ever gain a second holder under the certified-push
2095
- // default (handlePush refuses a proof-less block). Minted even when peerCount is 0 or the
2096
- // sole peer is not self — findCluster failing (getClusterPeerIds returns []) puts the
2097
- // DEGRADED-ROUTING case in this same branch, and self genuinely committed these bytes
2098
- // either way; a proof's peer list is already not evidence of cohort membership by design
2099
- // (caller obligation #1 on verifyBlockCommitProofClaim), so gating the mint on cohort
2100
- // composition would buy no safety while opening a silent no-proof hole exactly when
2101
- // routing is degraded. The log line is how an operator tells a real cohort of one
2102
- // (cohortSize 1, soleIsSelf true) from a routing failure (cohortSize 0, or a sole peer
2103
- // that is not this node).
2104
- this.log('commit:solo-cohort', {
2105
- blockId: blockIds[0],
2106
- cohortSize: peerCount,
2107
- soleIsSelf: peerCount === 1 && this.localPeerId !== undefined
2108
- && cohortPeerIds[0] === this.localPeerId.toString()
2109
- });
2110
- // Same message shape the multi-peer path produces — executeClusterTransaction stamps
2111
- // coordinatingBlockIds at its choke point, so the solo artifact must carry it too or a
2112
- // solo proof's message is distinguishable from every other proof's.
2113
- const message = {
2114
- operations: [{ commit: request }],
2115
- coordinatingBlockIds: [blockIds[0]],
2116
- expiration: options?.expiration ?? Date.now() + this.DEFAULT_TIMEOUT
2117
- };
2118
- // `undefined` when no local cluster is wired (direct constructors, unit-test doubles) —
2119
- // then the commit lands proof-less, exactly the pre-mint behavior. The cast is the named
2120
- // ICommitProofPersister contract; a plain IRepo double ignores the extra argument.
2121
- const proof = await this.localCluster?.mintSoloCommitProof?.(message);
2122
- const result = await this.storageRepo.commit(request, options, proof);
2123
- // One self-approval arms the read-repair window only where the DECLARED cohort is also one
2124
- // — then no rival quorum can exist to be missed. At any larger declared size (including
2125
- // an undeclared one, which resolves to the replication factor, and including degraded
2126
- // routing where peerCount is 0) this commit proves nothing about rival quorums — see
2127
- // commitQuorumRulesOutRivals — so the window stays unarmed and the read path's
2128
- // solo-self-skip exit re-arms it once per consult instead (which keeps GitHub issue #8's
2129
- // consult storm bounded at one per window).
2130
- if (result.success && this.commitQuorumRulesOutRivals(1, peerCount))
2131
- this.markBlocksSeen(blockIds);
2132
- return result;
2040
+ const cohort = await this.coordinator.resolveCohort(blockIds[0]);
2041
+ if (!cohort.resolved || cohort.peerIds.length <= 1) {
2042
+ return await this.commitSolo(request, blockIds, cohort, options);
2133
2043
  }
2044
+ const peerCount = cohort.peerIds.length;
2134
2045
  const message = {
2135
2046
  operations: [{ commit: request }],
2136
2047
  expiration: options?.expiration ?? Date.now() + this.DEFAULT_TIMEOUT
@@ -2163,10 +2074,16 @@ export class CoordinatorRepo {
2163
2074
  // fresh revision, which is what the `conflict: true` shape makes `Collection.syncAttempts`
2164
2075
  // and the multi-collection `pendPhase` do. But a member reached only by
2165
2076
  // `scheduleCommitRetry` can still land the refused revision later, and this node's own
2166
- // member (or its fallback commit) may hold it already. The writer's retry with the SAME
2167
- // action id converges either way: `isOwnRevision` in `StorageRepo.pend` and `commit`
2168
- // treats an already-landed own revision as satisfied, and `inFlightActionId` in
2169
- // `Collection.updateInternal` keeps the retry on the same action.
2077
+ // member (or its fallback commit) may hold it already. When the refused block is the
2078
+ // action's LOG TAIL, that matters beyond this block: `NetworkTransactor.commit` stops at a
2079
+ // refused tail and never sweeps the action's other blocks, so the writer can then read
2080
+ // its own log entry back while none of the data it describes was committed anywhere.
2081
+ // The writer's retry with the SAME action id is what converges that, and only because it
2082
+ // does two things: `isOwnRevision` in `StorageRepo.pend` and `commit` treats an
2083
+ // already-landed own revision as satisfied, and `Collection.completeOwnEntry` — on
2084
+ // finding that entry — re-sends the refused attempt at the same revision to land the
2085
+ // blocks left behind BEFORE the write is reported saved. Seeing its own log entry is
2086
+ // never, on its own, the writer's proof of durability.
2170
2087
  const durability = cohortDurability(record, cohortCommitOutcomes);
2171
2088
  if (localExecuted) {
2172
2089
  // Our own member applied this commit during consensus. Its retained storage verdict is
@@ -2209,14 +2126,15 @@ export class CoordinatorRepo {
2209
2126
  });
2210
2127
  }
2211
2128
  // An absent verdict (a member that predates retention, a restart, the TTL) is not
2212
- // evidence of holding anything: it is simply not counted.
2213
- const durableHolders = durability.remoteHolders + (localDurable ? 1 : 0);
2214
- if (!isDurableMajority(durableHolders, durability.cohortSize)) {
2129
+ // evidence of holding anything: it is simply not counted — so the class below reads
2130
+ // LOWER than reality for such a member, never higher.
2131
+ const durableHolders = durability.remoteHolders.length + (localDurable ? 1 : 0);
2132
+ if (!isDurableMajority(durableHolders, durability.cohortPeerIds.length)) {
2215
2133
  return this.refuseCommitNotDurable(request, durableHolders, durability, 'local-executed');
2216
2134
  }
2217
2135
  if (armFreshness)
2218
2136
  this.markBlocksSeen(blockIds);
2219
- return { success: true };
2137
+ return await this.acknowledgeCommit(request, { success: true, durability: this.cohortWriteDurability(durability, localDurable) }, localDurable, record);
2220
2138
  }
2221
2139
  // Local cluster didn't execute during consensus. Attempt a local commit, but tolerate
2222
2140
  // local divergence when the cluster already reached consensus AND a durable majority of
@@ -2230,9 +2148,11 @@ export class CoordinatorRepo {
2230
2148
  // but not the revision that created the block (see StorageRepo.internalCommit).
2231
2149
  // Reporting a tolerated divergence to the caller as the raw refusal would surface a
2232
2150
  // committed transaction as a stale loss: db-core's commitPhase treats any returned
2233
- // `success:false` as a permanent stale failure. So a divergence on a durable majority is
2234
- // reported as success, and one WITHOUT a durable majority as the durability gate's
2235
- // retryable refusal (`tolerateLocalCommitDivergence`).
2151
+ // `success:false` as a stale loss, and its coordinator re-drives the refused collection
2152
+ // (alone, even after a sibling collection committed) until its retry budget is spent. So
2153
+ // a divergence on a durable majority is reported as success, and one WITHOUT a durable
2154
+ // majority as the durability gate's retryable refusal (`tolerateLocalCommitDivergence`),
2155
+ // which that re-drive is what clears.
2236
2156
  //
2237
2157
  // The gate is evaluated BEFORE the local fallback commit, and a failing gate skips it.
2238
2158
  // This node counts toward the majority only when it is in the cohort the commit ran on;
@@ -2241,8 +2161,8 @@ export class CoordinatorRepo {
2241
2161
  // that lone off-cohort copy is exactly the seed of the "revision exists on one node that
2242
2162
  // is not responsible for it" placement the durability gate exists to prevent.
2243
2163
  const selfInCohort = this.localPeerId !== undefined && this.localPeerId.toString() in record.peers;
2244
- if (!isDurableMajority(durability.remoteHolders + (selfInCohort ? 1 : 0), durability.cohortSize)) {
2245
- return this.refuseCommitNotDurable(request, durability.remoteHolders, durability, 'fallback-skipped');
2164
+ if (!isDurableMajority(durability.remoteHolders.length + (selfInCohort ? 1 : 0), durability.cohortPeerIds.length)) {
2165
+ return this.refuseCommitNotDurable(request, durability.remoteHolders.length, durability, 'fallback-skipped');
2246
2166
  }
2247
2167
  //
2248
2168
  // Deliberately NOT self-signed here (unlike the solo short-circuit above): consensus for
@@ -2259,18 +2179,22 @@ export class CoordinatorRepo {
2259
2179
  if (result.success) {
2260
2180
  // The gate above already admitted this shape: the remote holders plus this node
2261
2181
  // (when it is in the cohort) form the majority, and this node now holds it.
2182
+ // Storage's own `local` answer is replaced by the cohort's — this node counts exactly
2183
+ // as the gate counted it, i.e. only when it is a cohort member.
2262
2184
  if (armFreshness)
2263
2185
  this.markBlocksSeen(blockIds);
2264
- return result;
2186
+ // The fallback commit landed every block, so this node holds them — whether or not
2187
+ // it is a cohort member — and can source a push to whoever the class names as missing.
2188
+ return await this.acknowledgeCommit(request, { ...result, durability: this.cohortWriteDurability(durability, selfInCohort) }, true, record);
2265
2189
  }
2266
2190
  if (isMissingBaseRevisionFailure(result) && clusterReachedCommitConsensus(record)) {
2267
- return this.tolerateLocalCommitDivergence(request, blockIds, result.reason ?? MISSING_BASE_REVISION_REASON, armFreshness, durability);
2191
+ return await this.tolerateLocalCommitDivergence(request, blockIds, result.reason ?? MISSING_BASE_REVISION_REASON, armFreshness, durability);
2268
2192
  }
2269
2193
  return result;
2270
2194
  }
2271
2195
  catch (err) {
2272
2196
  if (clusterReachedCommitConsensus(record)) {
2273
- return this.tolerateLocalCommitDivergence(request, blockIds, err.message, armFreshness, durability);
2197
+ return await this.tolerateLocalCommitDivergence(request, blockIds, err.message, armFreshness, durability);
2274
2198
  }
2275
2199
  throw err;
2276
2200
  }
@@ -2425,14 +2349,188 @@ export class CoordinatorRepo {
2425
2349
  * window unarmed — this peer is known to be behind here, the last place a self-referential
2426
2350
  * freshness stamp belongs.
2427
2351
  */
2428
- tolerateLocalCommitDivergence(request, blockIds, detail, armFreshness, durability) {
2429
- if (!isDurableMajority(durability.remoteHolders, durability.cohortSize)) {
2430
- return this.refuseCommitNotDurable(request, durability.remoteHolders, durability, `fallback-diverged: ${detail}`);
2352
+ async tolerateLocalCommitDivergence(request, blockIds, detail, armFreshness, durability) {
2353
+ if (!isDurableMajority(durability.remoteHolders.length, durability.cohortPeerIds.length)) {
2354
+ return this.refuseCommitNotDurable(request, durability.remoteHolders.length, durability, `fallback-diverged: ${detail}`);
2431
2355
  }
2432
2356
  this.log('coordinator-repo:commit-local-failed-cluster-succeeded', { actionId: request.actionId, error: detail });
2433
2357
  if (armFreshness)
2434
2358
  this.markBlocksSeen(blockIds);
2435
- return { success: true };
2359
+ // This node holds nothing: it is an UNCONFIRMED member when it is in the cohort (so the class
2360
+ // is `majority`), and simply absent from the count when it is not (so the class can be `full`
2361
+ // when every member confirmed). Both fall out of the one rule in `cohortWriteDurability`.
2362
+ // Holding nothing, it is also no source for a push, so no shortfall is recorded here.
2363
+ return await this.acknowledgeCommit(request, { success: true, durability: this.cohortWriteDurability(durability, false) }, false);
2364
+ }
2365
+ /**
2366
+ * The one way a success leaves {@link commit}: record the replication shortfall the answer
2367
+ * describes, then return the answer unchanged. Every success exit — solo, local-executed, local
2368
+ * fallback, tolerated divergence — runs after the durability gate admitted it, so a refused
2369
+ * commit never reaches the ledger.
2370
+ *
2371
+ * `localHolds` is whether this node's own storage durably holds every block of the commit, which
2372
+ * is what makes it a source for the missing copies — not whether the class counted it. `record`
2373
+ * is the consensus the commit ran on, absent for the solo short-circuit (no cohort ran, so no
2374
+ * holders to report).
2375
+ */
2376
+ async acknowledgeCommit(request, answer, localHolds, record) {
2377
+ if (localHolds && record !== undefined)
2378
+ this.reportCommittedHolders(request, answer.durability, record);
2379
+ await this.noteReplicationShortfall(request, answer.durability, localHolds);
2380
+ return answer;
2381
+ }
2382
+ /**
2383
+ * Tell the {@link CommittedHoldersSink} who holds a cohort commit this node holds, so its rebalance
2384
+ * monitor does not push the blocks back to members that stored them. A holder is a member the
2385
+ * durability class confirmed AND that signed an approving commit vote: the confirmation rests on
2386
+ * the member's unsigned apply report, and requiring the signature too means a member lying in that
2387
+ * report can only misstate its own copy — a member already trusted with the commit. The members
2388
+ * the class names unconfirmed are reported as such, which withdraws any earlier record of them
2389
+ * (including the signer list this node's own member reported at apply), so they are pushed the
2390
+ * block. Never throws: the commit is already acknowledged.
2391
+ */
2392
+ reportCommittedHolders(request, durability, record) {
2393
+ if (!this.onCommittedHolders)
2394
+ return;
2395
+ const self = this.localPeerId?.toString();
2396
+ const unconfirmed = durability.unconfirmed ?? [];
2397
+ const holders = (durability.cohortPeerIds ?? [])
2398
+ .filter(peerId => !unconfirmed.includes(peerId))
2399
+ .filter(peerId => peerId === self || record.commits[peerId]?.type === 'approve');
2400
+ try {
2401
+ this.onCommittedHolders({ blockIds: request.blockIds, holders, unconfirmed });
2402
+ }
2403
+ catch (err) {
2404
+ this.log('coordinator-repo:committed-holders-sink-error', { actionId: request.actionId, error: err.message });
2405
+ }
2406
+ }
2407
+ /**
2408
+ * Write down who is still missing an acknowledged commit, while this node still knows — the
2409
+ * cohort's answer is gone the moment the writer is answered, and the in-memory commit retry does
2410
+ * not survive a restart. One entry per block, all sharing this commit's one cohort answer
2411
+ * (consensus ran on `blockIds[0]`'s cohort for the whole request):
2412
+ * - `full` settles any older entry for the block, whether or not this node holds it;
2413
+ * - `majority` records the unconfirmed members by name;
2414
+ * - `local` and `unrouted` record an EMPTY missing set, meaning "unknown" — nobody could be named.
2415
+ * A shortfall is recorded only when `localHolds`: a node holding nothing has nothing to push, and
2416
+ * the remote majority that admitted the commit already holds it.
2417
+ *
2418
+ * NEVER throws, and that asymmetry is deliberate: by the time this runs the commit is already
2419
+ * durable at the class the answer states, so a ledger fault must not turn an acknowledged write
2420
+ * into a reported failure. A failure is logged per block and the answer still goes out; the
2421
+ * block's copy is then owed with nobody tracking it, exactly as before the ledger existed.
2422
+ *
2423
+ * NOTE: one ledger read and write per block per below-full commit, awaited before the writer is
2424
+ * answered — on a node that is genuinely alone that is every commit (a `full` commit settles from
2425
+ * the ledger's in-memory index and touches the store only when an entry exists). If it shows in
2426
+ * commit latency, coalesce the writes rather than answering first: an entry written after the
2427
+ * answer can be lost to a crash in between, which is the one case the ledger exists for.
2428
+ */
2429
+ async noteReplicationShortfall(request, durability, localHolds) {
2430
+ const ledger = this.underReplicationLedger;
2431
+ if (ledger === undefined)
2432
+ return;
2433
+ // Blocks a torn commit abandoned were cancelled and hold nothing at this revision: there is no
2434
+ // copy to push, and an entry would have a drain repeatedly read a block that is not there.
2435
+ // No answer built in this class names `torn` today — the writer's transactor derives it after
2436
+ // every coordinator has answered — so this guards against a future producer, not a current one.
2437
+ const torn = new Set(durability.torn ?? []);
2438
+ const blockIds = request.blockIds.filter(blockId => !torn.has(blockId));
2439
+ const outcomes = await Promise.allSettled(blockIds.map(blockId => this.noteBlockShortfall(ledger, request, blockId, durability, localHolds)));
2440
+ outcomes.forEach((outcome, i) => {
2441
+ if (outcome.status === 'fulfilled')
2442
+ return;
2443
+ this.log('coordinator-repo:under-replication-record-failed', {
2444
+ actionId: request.actionId,
2445
+ blockId: blockIds[i],
2446
+ rev: request.rev,
2447
+ quorum: durability.quorum,
2448
+ error: outcome.reason instanceof Error ? outcome.reason.message : String(outcome.reason)
2449
+ });
2450
+ });
2451
+ }
2452
+ async noteBlockShortfall(ledger, request, blockId, durability, localHolds) {
2453
+ if (durability.quorum === 'full') {
2454
+ await ledger.settle(blockId, request.rev);
2455
+ return;
2456
+ }
2457
+ if (!localHolds)
2458
+ return;
2459
+ await ledger.record({
2460
+ blockId,
2461
+ rev: request.rev,
2462
+ actionId: request.actionId,
2463
+ quorum: durability.quorum,
2464
+ missingPeerIds: durability.quorum === 'majority' ? durability.unconfirmed ?? [] : [],
2465
+ recordedAt: this.now(),
2466
+ attempts: 0
2467
+ });
2468
+ }
2469
+ /**
2470
+ * The durability class beside the gate: derived from the SAME sets {@link isDurableMajority} was
2471
+ * just measured on, with this node's own contribution passed per arm exactly as the gate counted
2472
+ * it, so the class and the gate can never disagree. One rule for all three success exits of
2473
+ * {@link commit}: `full` iff every cohort member is confirmed, otherwise `majority` (the gate
2474
+ * already refused anything below that). Never `local` or `unrouted` here — those belong to the
2475
+ * solo short-circuit, where no cohort ran.
2476
+ */
2477
+ cohortWriteDurability(durability, selfHolds) {
2478
+ return cohortWriteDurability(durability, selfHolds, this.localPeerId?.toString());
2479
+ }
2480
+ /**
2481
+ * The solo short-circuit of {@link commit}: consensus never runs, so no ClusterRecord exists to
2482
+ * project a proof from — the lone member self-signs a one-peer proof instead (mintSoloCommitProof),
2483
+ * which is what lets a block born on a cohort of one ever gain a second holder under the
2484
+ * certified-push default (handlePush refuses a proof-less block). Minted even when the cohort did
2485
+ * not resolve or the sole peer is not self — a failed `findCluster` puts the DEGRADED-ROUTING case
2486
+ * in this same branch, and self genuinely committed these bytes either way; a proof's peer list is
2487
+ * already not evidence of cohort membership by design (caller obligation #1 on
2488
+ * verifyBlockCommitProofClaim), so gating the mint on cohort composition would buy no safety while
2489
+ * opening a silent no-proof hole exactly when routing is degraded.
2490
+ *
2491
+ * The policy is unchanged — every write accepted here before is accepted now. What changes is
2492
+ * that the answer says which situation it was: the `commit:solo-cohort` log line (how an operator
2493
+ * tells a real cohort of one — cohortSize 1, soleIsSelf true — from a routing failure) now also
2494
+ * carries the `quorum` the writer is told, and both come from {@link soloCohortDurability}, so the
2495
+ * log and the answer cannot disagree.
2496
+ */
2497
+ async commitSolo(request, blockIds, cohort, options) {
2498
+ const cohortPeerIds = cohort.resolved ? cohort.peerIds : [];
2499
+ const peerCount = cohortPeerIds.length;
2500
+ const soloDurability = this.soloCohortDurability(cohort);
2501
+ this.log('commit:solo-cohort', {
2502
+ blockId: blockIds[0],
2503
+ cohortSize: peerCount,
2504
+ soleIsSelf: peerCount === 1 && this.localPeerId !== undefined
2505
+ && cohortPeerIds[0] === this.localPeerId.toString(),
2506
+ quorum: soloDurability.quorum,
2507
+ ...(cohort.resolved ? {} : { reason: cohort.reason })
2508
+ });
2509
+ // Same message shape the multi-peer path produces — executeClusterTransaction stamps
2510
+ // coordinatingBlockIds at its choke point, so the solo artifact must carry it too or a
2511
+ // solo proof's message is distinguishable from every other proof's.
2512
+ const message = {
2513
+ operations: [{ commit: request }],
2514
+ coordinatingBlockIds: [blockIds[0]],
2515
+ expiration: options?.expiration ?? Date.now() + this.DEFAULT_TIMEOUT
2516
+ };
2517
+ // `undefined` when no local cluster is wired (direct constructors, unit-test doubles) —
2518
+ // then the commit lands proof-less, exactly the pre-mint behavior. The cast is the named
2519
+ // ICommitProofPersister contract; a plain IRepo double ignores the extra argument.
2520
+ const proof = await this.localCluster?.mintSoloCommitProof?.(message);
2521
+ const result = await this.storageRepo.commit(request, options, proof);
2522
+ if (!result.success)
2523
+ return result;
2524
+ // One self-approval arms the read-repair window only where the DECLARED cohort is also one
2525
+ // — then no rival quorum can exist to be missed. At any larger declared size (including
2526
+ // an undeclared one, which resolves to the replication factor, and including degraded
2527
+ // routing where peerCount is 0) this commit proves nothing about rival quorums — see
2528
+ // commitQuorumRulesOutRivals — so the window stays unarmed and the read path's
2529
+ // solo-self-skip exit re-arms it once per consult instead (which keeps GitHub issue #8's
2530
+ // consult storm bounded at one per window).
2531
+ if (this.commitQuorumRulesOutRivals(1, peerCount))
2532
+ this.markBlocksSeen(blockIds);
2533
+ return await this.acknowledgeCommit(request, { ...result, durability: soloDurability }, true);
2436
2534
  }
2437
2535
  /**
2438
2536
  * The durability gate's answer: a retryable refusal ({@link COMMIT_NOT_DURABLE_REASON}) naming how
@@ -2442,28 +2540,84 @@ export class CoordinatorRepo {
2442
2540
  * read-repair window is deliberately NOT armed: nothing about this commit is freshness evidence.
2443
2541
  */
2444
2542
  refuseCommitNotDurable(request, durableHolders, durability, arm) {
2543
+ const cohortSize = durability.cohortPeerIds.length;
2445
2544
  this.log('coordinator-repo:commit-not-durable', {
2446
2545
  actionId: request.actionId,
2447
2546
  rev: request.rev,
2448
2547
  durableHolders,
2449
- cohortSize: durability.cohortSize,
2548
+ cohortSize,
2450
2549
  remoteHolders: durability.remoteHolders,
2451
2550
  remoteRefusals: durability.remoteRefusals,
2452
2551
  arm
2453
2552
  });
2553
+ // A failure carries no durability, and no "unknown" class stands in for one.
2454
2554
  return {
2455
2555
  success: false,
2456
2556
  conflict: true,
2457
- reason: `${COMMIT_NOT_DURABLE_REASON}: ${durableHolders} of ${durability.cohortSize} cohort member(s) report holding rev ${request.rev} of action ${request.actionId} (${arm})`
2557
+ reason: `${COMMIT_NOT_DURABLE_REASON}: ${durableHolders} of ${cohortSize} cohort member(s) report holding rev ${request.rev} of action ${request.actionId} (${arm})`
2458
2558
  };
2459
2559
  }
2460
2560
  }
2461
2561
  function cohortDurability(record, outcomes) {
2462
- const reports = Object.values(outcomes ?? {});
2562
+ const reports = Object.entries(outcomes ?? {});
2563
+ return {
2564
+ cohortPeerIds: Object.keys(record.peers),
2565
+ remoteHolders: reports.filter(([, report]) => report.success).map(([peerId]) => peerId),
2566
+ remoteRefusals: reports.filter(([, report]) => !report.success).map(([peerId]) => peerId)
2567
+ };
2568
+ }
2569
+ /**
2570
+ * The commit-tier durability class, from the gate's own sets. `confirmed` is counted exactly as the
2571
+ * gate counted it (remote holders, plus one for this node when `selfHolds`), and `unconfirmed` is
2572
+ * every cohort member outside that set — so a `full` answer names nobody as missing, and a
2573
+ * `majority` answer names exactly who. The class is `full` iff no cohort member is unconfirmed;
2574
+ * anything below `majority` never reaches here (the gate refused it).
2575
+ */
2576
+ function cohortWriteDurability(durability, selfHolds, selfPeerId) {
2577
+ const holders = new Set(durability.remoteHolders);
2578
+ if (selfHolds && selfPeerId !== undefined)
2579
+ holders.add(selfPeerId);
2580
+ const unconfirmed = durability.cohortPeerIds.filter(peerId => !holders.has(peerId));
2581
+ return {
2582
+ quorum: unconfirmed.length === 0 ? 'full' : 'majority',
2583
+ confirmed: durability.remoteHolders.length + (selfHolds ? 1 : 0),
2584
+ cohort: durability.cohortPeerIds.length,
2585
+ unconfirmed,
2586
+ cohortPeerIds: durability.cohortPeerIds
2587
+ };
2588
+ }
2589
+ /**
2590
+ * The pend-tier durability class: who ACCEPTED the pending record, read off the record's approving
2591
+ * promise votes. This node's own member is added when `selfAccepted` is true, removed when it is
2592
+ * false (its member applied and refused, so its approve vote no longer describes what it holds),
2593
+ * and left to its vote when `undefined` (no retained verdict — the vote is the only evidence there
2594
+ * is). This node counts only when it is IN the cohort, as the commit tier counts it: a coordinator
2595
+ * outside `record.peers` whose fallback pend landed holds a copy no cohort member will ever look
2596
+ * for, and counting it would let `confirmed` exceed `cohort`. NOT comparable to the commit-tier
2597
+ * count: accepting a pending record confers no storage durability, and the field's own
2598
+ * documentation says so.
2599
+ * NOTE: a REMOTE member's vote is never withdrawn here. The pend arm of `ClusterRecord.applyOutcomes`
2600
+ * reports only conflict-shaped refusals (which downgrade the whole pend to a retryable conflict
2601
+ * before this runs), so a remote member that promised and then faulted at apply keeps its approve
2602
+ * vote and a pend's `full` can overstate by that member. Harmless while nothing reads a pend's
2603
+ * class for repair; if that changes, widen the member's report to carry bare faults too.
2604
+ */
2605
+ function pendCohortDurability(record, selfAccepted, selfPeerId) {
2606
+ const cohortPeerIds = Object.keys(record.peers);
2607
+ const accepted = new Set(Object.entries(record.promises).filter(([, vote]) => vote.type === 'approve').map(([peerId]) => peerId));
2608
+ if (selfPeerId !== undefined && cohortPeerIds.includes(selfPeerId)) {
2609
+ if (selfAccepted === true)
2610
+ accepted.add(selfPeerId);
2611
+ if (selfAccepted === false)
2612
+ accepted.delete(selfPeerId);
2613
+ }
2614
+ const unconfirmed = cohortPeerIds.filter(peerId => !accepted.has(peerId));
2463
2615
  return {
2464
- cohortSize: Object.keys(record.peers).length,
2465
- remoteHolders: reports.filter(report => report.success).length,
2466
- remoteRefusals: reports.filter(report => !report.success).length
2616
+ quorum: unconfirmed.length === 0 ? 'full' : 'majority',
2617
+ confirmed: accepted.size,
2618
+ cohort: cohortPeerIds.length,
2619
+ unconfirmed,
2620
+ cohortPeerIds
2467
2621
  };
2468
2622
  }
2469
2623
  /**