@optimystic/db-p2p 0.24.1 → 0.25.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (257) hide show
  1. package/{readme.md → README.md} +688 -668
  2. package/dist/src/cluster/block-transfer-service.d.ts +133 -10
  3. package/dist/src/cluster/block-transfer-service.d.ts.map +1 -1
  4. package/dist/src/cluster/block-transfer-service.js +155 -9
  5. package/dist/src/cluster/block-transfer-service.js.map +1 -1
  6. package/dist/src/cluster/block-transfer.d.ts +67 -7
  7. package/dist/src/cluster/block-transfer.d.ts.map +1 -1
  8. package/dist/src/cluster/block-transfer.js +127 -24
  9. package/dist/src/cluster/block-transfer.js.map +1 -1
  10. package/dist/src/cluster/certified-claims.d.ts +170 -0
  11. package/dist/src/cluster/certified-claims.d.ts.map +1 -0
  12. package/dist/src/cluster/certified-claims.js +200 -0
  13. package/dist/src/cluster/certified-claims.js.map +1 -0
  14. package/dist/src/cluster/client-signature-verifier.d.ts +23 -0
  15. package/dist/src/cluster/client-signature-verifier.d.ts.map +1 -0
  16. package/dist/src/cluster/client-signature-verifier.js +33 -0
  17. package/dist/src/cluster/client-signature-verifier.js.map +1 -0
  18. package/dist/src/cluster/cluster-policy.d.ts +9 -3
  19. package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
  20. package/dist/src/cluster/cluster-policy.js +111 -27
  21. package/dist/src/cluster/cluster-policy.js.map +1 -1
  22. package/dist/src/cluster/cluster-repo.d.ts +140 -11
  23. package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
  24. package/dist/src/cluster/cluster-repo.js +485 -29
  25. package/dist/src/cluster/cluster-repo.js.map +1 -1
  26. package/dist/src/cluster/commit-proof.d.ts +134 -0
  27. package/dist/src/cluster/commit-proof.d.ts.map +1 -0
  28. package/dist/src/cluster/commit-proof.js +263 -0
  29. package/dist/src/cluster/commit-proof.js.map +1 -0
  30. package/dist/src/cluster/quorum-restore.d.ts +95 -8
  31. package/dist/src/cluster/quorum-restore.d.ts.map +1 -1
  32. package/dist/src/cluster/quorum-restore.js +113 -14
  33. package/dist/src/cluster/quorum-restore.js.map +1 -1
  34. package/dist/src/cluster/rebalance-monitor.d.ts +84 -0
  35. package/dist/src/cluster/rebalance-monitor.d.ts.map +1 -1
  36. package/dist/src/cluster/rebalance-monitor.js +207 -5
  37. package/dist/src/cluster/rebalance-monitor.js.map +1 -1
  38. package/dist/src/cluster/reconcile-block.d.ts +27 -7
  39. package/dist/src/cluster/reconcile-block.d.ts.map +1 -1
  40. package/dist/src/cluster/reconcile-block.js +179 -38
  41. package/dist/src/cluster/reconcile-block.js.map +1 -1
  42. package/dist/src/cluster/service.d.ts +21 -4
  43. package/dist/src/cluster/service.d.ts.map +1 -1
  44. package/dist/src/cluster/service.js +28 -17
  45. package/dist/src/cluster/service.js.map +1 -1
  46. package/dist/src/cluster/spread-on-churn.d.ts +8 -2
  47. package/dist/src/cluster/spread-on-churn.d.ts.map +1 -1
  48. package/dist/src/cluster/spread-on-churn.js +7 -7
  49. package/dist/src/cluster/spread-on-churn.js.map +1 -1
  50. package/dist/src/cohort-topic/host.d.ts.map +1 -1
  51. package/dist/src/cohort-topic/host.js +8 -7
  52. package/dist/src/cohort-topic/host.js.map +1 -1
  53. package/dist/src/cohort-topic/stream-util.d.ts +7 -6
  54. package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
  55. package/dist/src/cohort-topic/stream-util.js +14 -41
  56. package/dist/src/cohort-topic/stream-util.js.map +1 -1
  57. package/dist/src/dispute/cascade.d.ts +7 -8
  58. package/dist/src/dispute/cascade.d.ts.map +1 -1
  59. package/dist/src/dispute/cascade.js +24 -7
  60. package/dist/src/dispute/cascade.js.map +1 -1
  61. package/dist/src/dispute/dispute-service.d.ts +5 -0
  62. package/dist/src/dispute/dispute-service.d.ts.map +1 -1
  63. package/dist/src/dispute/dispute-service.js +13 -3
  64. package/dist/src/dispute/dispute-service.js.map +1 -1
  65. package/dist/src/dispute/invalidation.d.ts +33 -15
  66. package/dist/src/dispute/invalidation.d.ts.map +1 -1
  67. package/dist/src/dispute/invalidation.js +127 -34
  68. package/dist/src/dispute/invalidation.js.map +1 -1
  69. package/dist/src/dispute/service.d.ts.map +1 -1
  70. package/dist/src/dispute/service.js +2 -1
  71. package/dist/src/dispute/service.js.map +1 -1
  72. package/dist/src/index.d.ts +7 -0
  73. package/dist/src/index.d.ts.map +1 -1
  74. package/dist/src/index.js +7 -0
  75. package/dist/src/index.js.map +1 -1
  76. package/dist/src/libp2p-key-network.d.ts +78 -6
  77. package/dist/src/libp2p-key-network.d.ts.map +1 -1
  78. package/dist/src/libp2p-key-network.js +136 -57
  79. package/dist/src/libp2p-key-network.js.map +1 -1
  80. package/dist/src/libp2p-node-base.d.ts +38 -7
  81. package/dist/src/libp2p-node-base.d.ts.map +1 -1
  82. package/dist/src/libp2p-node-base.js +150 -62
  83. package/dist/src/libp2p-node-base.js.map +1 -1
  84. package/dist/src/logger.d.ts +2 -2
  85. package/dist/src/logger.js +2 -2
  86. package/dist/src/network/open-protocol-stream.d.ts +56 -0
  87. package/dist/src/network/open-protocol-stream.d.ts.map +1 -0
  88. package/dist/src/network/open-protocol-stream.js +60 -0
  89. package/dist/src/network/open-protocol-stream.js.map +1 -0
  90. package/dist/src/network/register-protocol-handler.d.ts +66 -0
  91. package/dist/src/network/register-protocol-handler.d.ts.map +1 -0
  92. package/dist/src/network/register-protocol-handler.js +51 -0
  93. package/dist/src/network/register-protocol-handler.js.map +1 -0
  94. package/dist/src/peer-address-book.d.ts +128 -1
  95. package/dist/src/peer-address-book.d.ts.map +1 -1
  96. package/dist/src/peer-address-book.js +198 -15
  97. package/dist/src/peer-address-book.js.map +1 -1
  98. package/dist/src/pend-validation.d.ts +65 -0
  99. package/dist/src/pend-validation.d.ts.map +1 -0
  100. package/dist/src/pend-validation.js +69 -0
  101. package/dist/src/pend-validation.js.map +1 -0
  102. package/dist/src/reactivity/notify-transport.d.ts.map +1 -1
  103. package/dist/src/reactivity/notify-transport.js +2 -1
  104. package/dist/src/reactivity/notify-transport.js.map +1 -1
  105. package/dist/src/reactivity/push-state-gossip.d.ts.map +1 -1
  106. package/dist/src/reactivity/push-state-gossip.js +2 -1
  107. package/dist/src/reactivity/push-state-gossip.js.map +1 -1
  108. package/dist/src/repo/cluster-coordinator.d.ts +32 -6
  109. package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
  110. package/dist/src/repo/cluster-coordinator.js +44 -10
  111. package/dist/src/repo/cluster-coordinator.js.map +1 -1
  112. package/dist/src/repo/coordinator-repo.d.ts +181 -25
  113. package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
  114. package/dist/src/repo/coordinator-repo.js +650 -62
  115. package/dist/src/repo/coordinator-repo.js.map +1 -1
  116. package/dist/src/repo/served-repo-proxy.d.ts +30 -0
  117. package/dist/src/repo/served-repo-proxy.d.ts.map +1 -0
  118. package/dist/src/repo/served-repo-proxy.js +50 -0
  119. package/dist/src/repo/served-repo-proxy.js.map +1 -0
  120. package/dist/src/repo/service.d.ts +22 -1
  121. package/dist/src/repo/service.d.ts.map +1 -1
  122. package/dist/src/repo/service.js +25 -13
  123. package/dist/src/repo/service.js.map +1 -1
  124. package/dist/src/reputation/types.d.ts +3 -1
  125. package/dist/src/reputation/types.d.ts.map +1 -1
  126. package/dist/src/reputation/types.js +3 -1
  127. package/dist/src/reputation/types.js.map +1 -1
  128. package/dist/src/rn.d.ts +24 -2
  129. package/dist/src/rn.d.ts.map +1 -1
  130. package/dist/src/rn.js +33 -4
  131. package/dist/src/rn.js.map +1 -1
  132. package/dist/src/routing/libp2p-known-peers.d.ts.map +1 -1
  133. package/dist/src/routing/libp2p-known-peers.js +5 -0
  134. package/dist/src/routing/libp2p-known-peers.js.map +1 -1
  135. package/dist/src/storage/block-archive.d.ts +153 -0
  136. package/dist/src/storage/block-archive.d.ts.map +1 -0
  137. package/dist/src/storage/block-archive.js +194 -0
  138. package/dist/src/storage/block-archive.js.map +1 -0
  139. package/dist/src/storage/block-latch.d.ts +92 -0
  140. package/dist/src/storage/block-latch.d.ts.map +1 -0
  141. package/dist/src/storage/block-latch.js +132 -0
  142. package/dist/src/storage/block-latch.js.map +1 -0
  143. package/dist/src/storage/block-storage.d.ts +169 -12
  144. package/dist/src/storage/block-storage.d.ts.map +1 -1
  145. package/dist/src/storage/block-storage.js +415 -152
  146. package/dist/src/storage/block-storage.js.map +1 -1
  147. package/dist/src/storage/cached-raw-storage.d.ts +5 -1
  148. package/dist/src/storage/cached-raw-storage.d.ts.map +1 -1
  149. package/dist/src/storage/cached-raw-storage.js +16 -1
  150. package/dist/src/storage/cached-raw-storage.js.map +1 -1
  151. package/dist/src/storage/cached-store-driver.d.ts +14 -4
  152. package/dist/src/storage/cached-store-driver.d.ts.map +1 -1
  153. package/dist/src/storage/cached-store-driver.js +39 -5
  154. package/dist/src/storage/cached-store-driver.js.map +1 -1
  155. package/dist/src/storage/i-block-storage.d.ts +70 -19
  156. package/dist/src/storage/i-block-storage.d.ts.map +1 -1
  157. package/dist/src/storage/i-block-storage.js +21 -1
  158. package/dist/src/storage/i-block-storage.js.map +1 -1
  159. package/dist/src/storage/i-raw-storage.d.ts +38 -0
  160. package/dist/src/storage/i-raw-storage.d.ts.map +1 -1
  161. package/dist/src/storage/kv-raw-storage.d.ts +14 -8
  162. package/dist/src/storage/kv-raw-storage.d.ts.map +1 -1
  163. package/dist/src/storage/kv-raw-storage.js +29 -9
  164. package/dist/src/storage/kv-raw-storage.js.map +1 -1
  165. package/dist/src/storage/memory-store-driver.d.ts +4 -1
  166. package/dist/src/storage/memory-store-driver.d.ts.map +1 -1
  167. package/dist/src/storage/memory-store-driver.js +22 -2
  168. package/dist/src/storage/memory-store-driver.js.map +1 -1
  169. package/dist/src/storage/raw-store-codec.js +2 -2
  170. package/dist/src/storage/raw-store-driver.d.ts +44 -3
  171. package/dist/src/storage/raw-store-driver.d.ts.map +1 -1
  172. package/dist/src/storage/shared-cache-pool.d.ts +60 -3
  173. package/dist/src/storage/shared-cache-pool.d.ts.map +1 -1
  174. package/dist/src/storage/shared-cache-pool.js +68 -4
  175. package/dist/src/storage/shared-cache-pool.js.map +1 -1
  176. package/dist/src/storage/storage-repo.d.ts +204 -26
  177. package/dist/src/storage/storage-repo.d.ts.map +1 -1
  178. package/dist/src/storage/storage-repo.js +422 -119
  179. package/dist/src/storage/storage-repo.js.map +1 -1
  180. package/dist/src/storage/store-identity.d.ts +21 -0
  181. package/dist/src/storage/store-identity.d.ts.map +1 -0
  182. package/dist/src/storage/store-identity.js +33 -0
  183. package/dist/src/storage/store-identity.js.map +1 -0
  184. package/dist/src/storage/struct.d.ts +15 -0
  185. package/dist/src/storage/struct.d.ts.map +1 -1
  186. package/dist/src/storage/with-read-cache.d.ts +92 -0
  187. package/dist/src/storage/with-read-cache.d.ts.map +1 -0
  188. package/dist/src/storage/with-read-cache.js +132 -0
  189. package/dist/src/storage/with-read-cache.js.map +1 -0
  190. package/dist/src/sync/service.d.ts +13 -3
  191. package/dist/src/sync/service.d.ts.map +1 -1
  192. package/dist/src/sync/service.js +7 -30
  193. package/dist/src/sync/service.js.map +1 -1
  194. package/dist/src/testing/mesh-harness.d.ts +75 -2
  195. package/dist/src/testing/mesh-harness.d.ts.map +1 -1
  196. package/dist/src/testing/mesh-harness.js +193 -81
  197. package/dist/src/testing/mesh-harness.js.map +1 -1
  198. package/dist/src/testing/raw-storage-conformance.d.ts +8 -0
  199. package/dist/src/testing/raw-storage-conformance.d.ts.map +1 -1
  200. package/dist/src/testing/raw-storage-conformance.js +97 -2
  201. package/dist/src/testing/raw-storage-conformance.js.map +1 -1
  202. package/package.json +5 -6
  203. package/src/cluster/block-transfer-service.ts +528 -312
  204. package/src/cluster/block-transfer.ts +547 -424
  205. package/src/cluster/certified-claims.ts +319 -0
  206. package/src/cluster/client-signature-verifier.ts +34 -0
  207. package/src/cluster/cluster-policy.ts +294 -203
  208. package/src/cluster/cluster-repo.ts +2501 -2034
  209. package/src/cluster/commit-proof.ts +344 -0
  210. package/src/cluster/quorum-restore.ts +348 -223
  211. package/src/cluster/rebalance-monitor.ts +539 -254
  212. package/src/cluster/reconcile-block.ts +404 -203
  213. package/src/cluster/service.ts +308 -293
  214. package/src/cluster/spread-on-churn.ts +352 -346
  215. package/src/cohort-topic/host.ts +8 -7
  216. package/src/cohort-topic/stream-util.ts +14 -43
  217. package/src/dispute/cascade.ts +31 -15
  218. package/src/dispute/dispute-service.ts +706 -696
  219. package/src/dispute/invalidation.ts +162 -52
  220. package/src/dispute/service.ts +126 -125
  221. package/src/index.ts +7 -0
  222. package/src/libp2p-key-network.ts +142 -58
  223. package/src/libp2p-node-base.ts +1793 -1675
  224. package/src/logger.ts +27 -27
  225. package/src/network/open-protocol-stream.ts +87 -0
  226. package/src/network/register-protocol-handler.ts +85 -0
  227. package/src/peer-address-book.ts +360 -149
  228. package/src/pend-validation.ts +89 -0
  229. package/src/reactivity/notify-transport.ts +145 -144
  230. package/src/reactivity/push-state-gossip.ts +292 -291
  231. package/src/repo/cluster-coordinator.ts +65 -11
  232. package/src/repo/coordinator-repo.ts +729 -71
  233. package/src/repo/served-repo-proxy.ts +55 -0
  234. package/src/repo/service.ts +332 -313
  235. package/src/reputation/types.ts +3 -1
  236. package/src/rn.ts +34 -4
  237. package/src/routing/libp2p-known-peers.ts +31 -26
  238. package/src/storage/block-archive.ts +242 -0
  239. package/src/storage/block-latch.ts +144 -0
  240. package/src/storage/block-storage.ts +787 -498
  241. package/src/storage/cached-raw-storage.ts +21 -1
  242. package/src/storage/cached-store-driver.ts +898 -859
  243. package/src/storage/i-block-storage.ts +172 -116
  244. package/src/storage/i-raw-storage.ts +40 -0
  245. package/src/storage/kv-raw-storage.ts +34 -9
  246. package/src/storage/memory-store-driver.ts +26 -2
  247. package/src/storage/raw-store-codec.ts +42 -42
  248. package/src/storage/raw-store-driver.ts +45 -3
  249. package/src/storage/shared-cache-pool.ts +516 -452
  250. package/src/storage/storage-repo.ts +1421 -1014
  251. package/src/storage/store-identity.ts +41 -0
  252. package/src/storage/struct.ts +52 -34
  253. package/src/storage/with-read-cache.ts +182 -0
  254. package/src/sync/service.ts +178 -198
  255. package/src/testing/cohort-topic-mesh-harness.ts +673 -673
  256. package/src/testing/mesh-harness.ts +524 -344
  257. package/src/testing/raw-storage-conformance.ts +114 -2
@@ -1,9 +1,10 @@
1
- import { LruMap, blockIdsForTransforms, highestStaleAt, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
1
+ import { LruMap, blockIdsForTransforms, highestStaleAt, isConflictFailure, isOwnRevision, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
2
2
  import { ClusterCoordinator, ConflictRaceLostError, ValidatorRejectionError } from "./cluster-coordinator.js";
3
3
  import { peerIdFromString } from "@libp2p/peer-id";
4
4
  import { createLogger } from '../logger.js';
5
5
  import { PenaltyReason } from "../reputation/types.js";
6
- import { quorumSize, corroboratorCapacity, selectQuorumRev } from "../cluster/quorum-restore.js";
6
+ import { quorumSize, corroboratorCapacity, selectQuorumRev, certifiedEquivocation, CORROBORATION_FLOOR } from "../cluster/quorum-restore.js";
7
+ import { certifyClaim, isAttributableProofFailure, proofThresholds } from "../cluster/certified-claims.js";
7
8
  import { DEFAULT_CLUSTER_SIZE } from "../cluster/cluster-policy.js";
8
9
  import { RECONCILE_TIMEOUT_MS } from "../cluster/reconcile-block.js";
9
10
  import { isMissingBaseRevisionFailure, MISSING_BASE_REVISION_REASON } from "../storage/storage-repo.js";
@@ -27,8 +28,52 @@ function withDeadline(promise, ms, label) {
27
28
  clearTimeout(timer);
28
29
  });
29
30
  }
31
+ /** The `cohort-too-small` wording: the cohort cannot field the quorum however healthy its peers are. */
32
+ function cohortTooSmallMessage(cohortPeers, claimants, requiredEvenIfAllAnswered, repairCorroborationClusterSize) {
33
+ return `Block repair cannot converge for this block and the condition is PERMANENT, not transient: ` +
34
+ `this node's cohort has ${cohortPeers} peer(s) besides itself, all of them answered ` +
35
+ `(${claimants} hold the block), but accepting a revision would need ${requiredEvenIfAllAnswered} ` +
36
+ `agreeing peers even if every one of those ${cohortPeers} answered and agreed. No later pass can reach ` +
37
+ `that, however healthy every peer is, so this node's copy of the block stays as it is. Repair needs ` +
38
+ `${CORROBORATION_FLOOR} cohort peers BESIDES the reader to answer and agree, relaxed to 1 only for a ` +
39
+ `cohort that DECLARES it is smaller; repairCorroborationClusterSize currently resolves to ` +
40
+ `${repairCorroborationClusterSize}. Two things produce this, and this node cannot tell them ` +
41
+ `apart: (1) the deployment really does run this few machines — set clusterPolicy.assumedClusterSize ` +
42
+ `to the number you actually run (it does not lower clusterSize / the replication factor), or set an ` +
43
+ `honest clusterSize, and run at least ${CORROBORATION_FLOOR + 2} machines for any tolerance of one ` +
44
+ `unreachable peer; or (2) this node's view of the cohort has shrunk below the real deployment — a ` +
45
+ `partition or a routing problem, which configuration will not fix. Check the peer count above ` +
46
+ `against the machines you run before changing anything.`;
47
+ }
48
+ /**
49
+ * The `sole-holder` wording: the cohort is big enough, but only one of its peers holds the block.
50
+ *
51
+ * Every claim here is scoped to THIS NODE'S COHORT PEERS, which is the whole of what the pass
52
+ * observed. It deliberately does not say "only one machine in the deployment holds this block": this
53
+ * node's own copy is excluded from the claim set (it cannot corroborate the revision it is trying to
54
+ * repair), so a reader that holds the block itself would make that reading false — and a scary
55
+ * all-caps line an operator can disprove by looking at their own disks is worth less than no line.
56
+ * For the same reason the remedy is "another COHORT PEER holding it" rather than "a second copy":
57
+ * with the reader holding one, a second copy already exists and is still not enough.
58
+ */
59
+ function soleHolderMessage(cohortPeers) {
60
+ return `Block repair cannot converge for this block and the condition is PERMANENT, not transient: ` +
61
+ `ONLY ONE COHORT PEER HOLDS THIS BLOCK. Of this node's ${cohortPeers} cohort peers, 1 reports holding ` +
62
+ `it and the other ${cohortPeers - 1} answered that they hold NOTHING — an answer, not silence, so this ` +
63
+ `is the whole picture and not a slow pass. Repair adopts a revision only when ${CORROBORATION_FLOOR} ` +
64
+ `peers BESIDES this node agree on it, and a lone holder cannot second itself, so every later pass ` +
65
+ `declines identically. This node's own copy, if it has one, is the copy being repaired and does not ` +
66
+ `count toward that number. MORE MACHINES DO NOT FIX THIS, and neither does any cluster-size setting — ` +
67
+ `what is missing is ANOTHER COHORT PEER HOLDING THE BLOCK. The usual cause is data written while the ` +
68
+ `deployment (or this block's cohort) was smaller: a block that had one holder then still has one holder ` +
69
+ `now, because the two paths that would replicate it — read-repair and reconcile — both decline on this ` +
70
+ `same rule. Committing any new revision of the block writes it to the current cohort and clears this. ` +
71
+ `(A lone holder whose answer carries a valid cohort commit proof for its revision IS adopted without a ` +
72
+ `second voter — reaching this message means the one holder attached no such proof, or one that did not ` +
73
+ `verify.)`;
74
+ }
30
75
  export function coordinatorRepo(keyNetwork, createClusterClient, cfg, fretService, reputation, stateStore) {
31
- return (components) => new CoordinatorRepo(keyNetwork, createClusterClient, components.storageRepo, cfg, components.localCluster, components.localPeerId, fretService, components.clusterLatestCallback, reputation, stateStore, components.acquireBlockFromCohort);
76
+ return (components) => new CoordinatorRepo(keyNetwork, createClusterClient, components.storageRepo, cfg, components.localCluster, components.localPeerId, fretService, components.clusterLatestCallback, reputation, stateStore, components.acquireBlockFromCohort, components.proofAnchoring);
32
77
  }
33
78
  /** Cluster coordination repo - uses local store, as well as distributes changes to other nodes using cluster consensus. */
34
79
  export class CoordinatorRepo {
@@ -37,18 +82,19 @@ export class CoordinatorRepo {
37
82
  storageRepo;
38
83
  clusterLatestCallback;
39
84
  acquireBlockFromCohort;
85
+ proofAnchoring;
40
86
  coordinator;
41
87
  DEFAULT_TIMEOUT = 30000; // 30 seconds default timeout
42
88
  localPeerId;
43
89
  responsibilityCache = new LruMap(1000);
44
90
  static RESPONSIBILITY_TTL_MS = 60_000;
45
91
  lastSeenCommitMs = new LruMap(1000);
46
- /** Per block, the cohort-claimed revision the last freshness consult could not settle — the
47
- * doubt {@link flagUnconfirmedCurrency} stamps onto reads served below it. Outlives the
48
- * consult on purpose: the read-repair window skips consults for blocks checked recently, and
49
- * a doubt dropped there is a stale answer served as confirmed again.
92
+ /** Per block, what earlier repair passes left unresolved see {@link AheadClaimState}.
93
+ * Outlives the consult on purpose: the read-repair window skips consults for blocks checked
94
+ * recently, and a doubt dropped there is a stale answer served as confirmed again.
50
95
  * NOTE: LRU-bounded like `lastSeenCommitMs`; an eviction under >1000 doubted blocks loses the
51
- * doubt until the next consult re-derives it (one read-repair window later, at worst). */
96
+ * doubt until the next consult re-derives it (one read-repair window later, at worst) and lets
97
+ * {@link reportRepairDeadlock} say its piece a second time. */
52
98
  unsettledAheadClaims = new LruMap(1000);
53
99
  readRepairMode;
54
100
  readRepairWindowMs;
@@ -71,12 +117,13 @@ export class CoordinatorRepo {
71
117
  now = () => Date.now();
72
118
  /** Test seam: overridable RNG (0..1) for sample-rate gating. */
73
119
  rand = () => Math.random();
74
- constructor(keyNetwork, createClusterClient, storageRepo, cfg, localCluster, localPeerId, fretService, clusterLatestCallback, reputation, stateStore, acquireBlockFromCohort) {
120
+ constructor(keyNetwork, createClusterClient, storageRepo, cfg, localCluster, localPeerId, fretService, clusterLatestCallback, reputation, stateStore, acquireBlockFromCohort, proofAnchoring) {
75
121
  this.keyNetwork = keyNetwork;
76
122
  this.createClusterClient = createClusterClient;
77
123
  this.storageRepo = storageRepo;
78
124
  this.clusterLatestCallback = clusterLatestCallback;
79
125
  this.acquireBlockFromCohort = acquireBlockFromCohort;
126
+ this.proofAnchoring = proofAnchoring;
80
127
  this.localPeerId = localPeerId;
81
128
  this.log = createLogger('coordinator-repo', localPeerId?.toString());
82
129
  const policy = {
@@ -122,7 +169,9 @@ export class CoordinatorRepo {
122
169
  const localClusterRef = localCluster && localPeerId ? {
123
170
  update: localCluster.update.bind(localCluster),
124
171
  peerId: localPeerId,
125
- wasTransactionExecuted: localCluster.wasTransactionExecuted?.bind(localCluster)
172
+ wasTransactionExecuted: localCluster.wasTransactionExecuted?.bind(localCluster),
173
+ getExecutedPendResult: localCluster.getExecutedPendResult?.bind(localCluster),
174
+ getExecutedCommitResult: localCluster.getExecutedCommitResult?.bind(localCluster)
126
175
  } : undefined;
127
176
  this.coordinator = new ClusterCoordinator(keyNetwork, createClusterClient, policy, localClusterRef, fretService, reputation, stateStore);
128
177
  }
@@ -342,10 +391,22 @@ export class CoordinatorRepo {
342
391
  * claimed revision (see {@link flagUnconfirmedCurrency}), which is what bounds the map.
343
392
  */
344
393
  recordAheadClaim(blockId, claimedRev) {
345
- if (claimedRev === undefined)
346
- this.unsettledAheadClaims.delete(blockId);
347
- else
348
- this.unsettledAheadClaims.set(blockId, claimedRev);
394
+ const prior = this.unsettledAheadClaims.get(blockId);
395
+ if (claimedRev === undefined) {
396
+ // The consult is the authority on the CLAIM, and only on the claim. A recorded deadlock is
397
+ // not about any revision — it is about how many machines this deployment can field, or how
398
+ // many of them hold the block — so it outlives the claim that first exposed it and is
399
+ // dropped only when the block converges (see {@link flagUnconfirmedCurrency}).
400
+ if (prior?.deadlocksReported)
401
+ this.unsettledAheadClaims.set(blockId, { deadlocksReported: prior.deadlocksReported });
402
+ else
403
+ this.unsettledAheadClaims.delete(blockId);
404
+ return;
405
+ }
406
+ this.unsettledAheadClaims.set(blockId, {
407
+ rev: claimedRev,
408
+ ...(prior?.deadlocksReported ? { deadlocksReported: prior.deadlocksReported } : {})
409
+ });
349
410
  }
350
411
  /**
351
412
  * Stamp {@link GetBlockResult.unconfirmedAheadRev} on an entry sitting behind an unsettled
@@ -372,11 +433,11 @@ export class CoordinatorRepo {
372
433
  * from the tail block's `latest.rev`, and `syncInternal` commits every block of an action at
373
434
  * `context.rev + 1` — so `context.rev >= claimedRev` is a well-defined comparison. `state.latest`
374
435
  * is this node's newest revision for the block even on a pinned read (StorageRepo reports the
375
- * content's own revision separately as `materializedRev`), which is exactly the number "is this
436
+ * content's own revision separately as `materialized`), which is exactly the number "is this
376
437
  * node behind the claim?" asks about.
377
438
  */
378
439
  flagUnconfirmedCurrency(results, blockId, context) {
379
- const claimedRev = this.unsettledAheadClaims.get(blockId);
440
+ const claimedRev = this.unsettledAheadClaims.get(blockId)?.rev;
380
441
  if (claimedRev === undefined)
381
442
  return;
382
443
  const entry = results[blockId];
@@ -386,7 +447,9 @@ export class CoordinatorRepo {
386
447
  if (typeof servedRev !== 'number')
387
448
  return;
388
449
  if (servedRev >= claimedRev) {
389
- // Caught up — by this pass's repair or by a commit that landed since. Nothing to doubt.
450
+ // Caught up — by this pass's repair or by a commit that landed since. Nothing to doubt, and
451
+ // nothing deadlocked either: repair demonstrably converged for this block, so a later
452
+ // non-convergence is a new episode and gets to say so again.
390
453
  this.unsettledAheadClaims.delete(blockId);
391
454
  return;
392
455
  }
@@ -582,7 +645,7 @@ export class CoordinatorRepo {
582
645
  }
583
646
  try {
584
647
  // Bounded: a stalled cohort peer must not hold up the caller's read. Persisting happens
585
- // inside the callback via `saveReplicatedBlock`, which takes the per-block commit latch —
648
+ // inside the callback via `saveReplicatedBlock`, which takes the block write latch —
586
649
  // safe to call from here because the read path holds no latch of its own (`StorageRepo.get`
587
650
  // acquires and releases it around the promotion above, and nothing wraps this method).
588
651
  // NOTE: `get` walks its block ids sequentially, so the bound is per block, not per call — a
@@ -606,7 +669,8 @@ export class CoordinatorRepo {
606
669
  * the repair. Returns the local revision afterwards.
607
670
  *
608
671
  * A pending-only block (metadata seeded by `savePendingTransaction`, no committed revision) asked
609
- * for a forward revision no promotion can reach used to throw out of `BlockStorage.ensureRevision`.
672
+ * for a forward revision no promotion can reach used to throw out of the restore step (now
673
+ * `BlockStorage.restoreRevision`, driven by `StorageRepo.get`'s healing helper).
610
674
  * It no longer does: "no committed base here" is an absence, so that read comes back as a plain
611
675
  * unflagged `{ state: {} }` and this method simply returns `undefined` — acquisition then supplies
612
676
  * the revision. The `unavailable` arm below still fires for the shapes that ARE a guess (a `latest`
@@ -652,11 +716,14 @@ export class CoordinatorRepo {
652
716
  * repair. It is returned separately so the caller can compare, not vote.
653
717
  *
654
718
  * NOTE: the quorum is corroboration-of-a-claim, NOT Sybil-resistant cohort
655
- * membership — a peer minting fresh keypairs still casts a vote, and the claims
656
- * themselves are bare assertions (a `BlockArchive` carries no commit certificate, so
657
- * there is nothing here to verify a `(rev, actionId)` against). Commit-cert +
658
- * membership anchoring is deferred to backlog
659
- * `debt-read-repair-commit-cert-verification`.
719
+ * membership — a peer minting fresh keypairs still casts a vote. A claim that arrives
720
+ * with a cohort commit proof is additionally VERIFIED here (`certifyClaim`,
721
+ * `cluster/certified-claims.ts`); when the proof holds, the claim is certified and
722
+ * {@link selectQuorumRev} accepts it without a second voter — the cohort's signature set
723
+ * is its corroboration. What a passing proof does NOT prove is that its signers are the
724
+ * block's responsible cohort (anyone controlling N keys can sign their own N-peer
725
+ * proof); anchoring the signer set to topology is the optional, observational-only
726
+ * {@link ProofAnchoring} layer, unwired in production today.
660
727
  */
661
728
  async queryClusterForLatest(peerIds, blockId, context) {
662
729
  // Query peers in parallel for their latest revision. Each query is DEADLINED (rejects), not
@@ -705,62 +772,278 @@ export class CoordinatorRepo {
705
772
  }
706
773
  if (!value)
707
774
  continue; // responded, holds nothing — an absent claim, not silence
708
- claims.push({ peerId: peerIdStr, rev: value.rev, actionId: value.actionId });
775
+ // The proof rides along here and is verified BELOW (certifyClaim) before selection reads
776
+ // the claim set: presence proves nothing — the peer chose what to attach — but a proof
777
+ // that verifies certifies the claim, and a certified claim needs no second voter.
778
+ claims.push({
779
+ peerId: peerIdStr, rev: value.rev, actionId: value.actionId,
780
+ ...(value.proof ? { proof: value.proof } : {})
781
+ });
709
782
  }
710
783
  if (silent.length > 0) {
711
784
  this.log('cluster-fetch:peers-silent', { blockId, silent: silent.length, consulted: peerIds.length });
712
785
  }
786
+ // Verify every attached proof, in parallel, BEFORE selection — and penalize provable proof
787
+ // misbehavior HERE, at verification time, independent of what selection later does with the
788
+ // claim. Only attributable failures (isAttributableProofFailure) are penalized: a failure
789
+ // whose signer identities were never proven — unknown/non-ed25519 signer, malformed
790
+ // signature or proof, a legacy record, the oversized-cohort cap — could have been authored
791
+ // by anyone in the chain, and penalizing on it would let an attacker frame a peer (the same
792
+ // discipline as VerifyOutcome.penalize in cluster-repo.ts). A claim whose proof fails stays
793
+ // in the claim set UNCERTIFIED: it still corroborates by distinct-peer count exactly as a
794
+ // proof-less claim does — a peer that could fabricate a bad proof could equally have sent no
795
+ // proof, so dropping the vote would buy nothing.
796
+ // NOTE: cost is one verification pass per proof-carrying answer per consult, each bounded by
797
+ // MAX_PROOF_SIGNERS (256) signature checks. In `lazy` mode consults are rate-limited by the
798
+ // read-repair window; in `paranoid` mode every read of every block pays cohort-width
799
+ // verifications. Fine at deployment cohort sizes (~10) — if paranoid readers ever show CPU
800
+ // time in `certifyClaim`, cache verdicts per (blockId, rev, actionId, proof hash) rather than
801
+ // skipping verification.
802
+ await Promise.all(claims.map(async (claim) => {
803
+ if (!claim.proof)
804
+ return;
805
+ const verdict = await certifyClaim(claim.proof, { blockId, rev: claim.rev, actionId: claim.actionId },
806
+ // Shared with the reconcile path, so the two cannot drift on what the members actually
807
+ // enforced — see `proofThresholds` for why the simple-majority term is not
808
+ // this.simpleMajorityThreshold.
809
+ proofThresholds(this.superMajorityThreshold), this.proofAnchoring);
810
+ if (verdict.certified) {
811
+ claim.certified = true;
812
+ return;
813
+ }
814
+ this.log('cluster-fetch:proof-uncertified', {
815
+ blockId, peerId: claim.peerId, rev: claim.rev, failure: verdict.failure
816
+ });
817
+ if (isAttributableProofFailure(verdict.failure)) {
818
+ this.penalizeProofService(claim.peerId, blockId);
819
+ }
820
+ }));
713
821
  const nonSelfCount = peerIds.filter(id => id !== selfId).length;
714
822
  const answered = nonSelfCount - silent.length;
715
823
  const capacity = corroboratorCapacity(nonSelfCount, this.repairCorroborationClusterSize);
824
+ const required = quorumSize(claims.length, this.simpleMajorityThreshold, capacity);
716
825
  const selected = selectQuorumRev(claims, this.simpleMajorityThreshold, capacity);
717
826
  if (!selected) {
827
+ // A decline can be the certified path REFUSING to pick a side: two distinct actions each
828
+ // carrying a verified cohort proof for the same top revision. Name that apart from the
829
+ // routine no-quorum — the cohort (or whoever holds its keys) provably signed both sides,
830
+ // an incident rather than a shortage of answers. Neither claimant is penalized: both
831
+ // proofs verified, so which side is "wrong" is exactly what this node cannot know.
832
+ const equivocation = certifiedEquivocation(claims);
833
+ if (equivocation) {
834
+ this.log('cluster-fetch:certified-equivocation', {
835
+ blockId, rev: equivocation.rev, actionIds: equivocation.actionIds
836
+ });
837
+ }
838
+ // The three populations are reported SEPARATELY, never rolled into one "responders" count:
839
+ // "1 of 2 responded" and "1 holder, 1 confirmed non-holder, 0 silent" call for completely
840
+ // different operator actions — the first says wait or fix reachability, the second says the
841
+ // block has only one copy and no amount of waiting produces a second.
718
842
  this.log('cluster-fetch:no-quorum', {
719
843
  blockId,
720
- responders: claims.length,
721
- required: quorumSize(claims.length, this.simpleMajorityThreshold, capacity),
844
+ cohortPeers: nonSelfCount,
845
+ holders: claims.length,
846
+ absent: answered - claims.length,
847
+ silent: silent.length,
848
+ required,
722
849
  repairCorroborationClusterSize: this.repairCorroborationClusterSize
723
850
  });
851
+ // ...and, when this decline is provably permanent rather than transient, say THAT once,
852
+ // in words. The `no-quorum` line above fires on every pass and cannot tell the two apart.
853
+ this.reportRepairDeadlock({
854
+ blockId, claims, silentCount: silent.length, cohortPeers: nonSelfCount, answered, required, capacity
855
+ });
724
856
  // The claims themselves must not drive restoration — but their existence is
725
857
  // evidence the caller needs: an answer served below the highest claim cannot be
726
858
  // confirmed current (see ClusterLatestQuery.uncorroboratedRev).
727
- // NOTE: ONE claim is enough to raise that doubt, and a claim is a bare assertion
728
- // (no commit certificate to verify it against). So a single lying cohort peer can
729
- // deny unpinned reads of a block by claiming a revision nobody else holds — an
730
- // availability lever it did not have while uncorroborated claims were discarded.
731
- // Deliberate for now: the alternative is the silent stale serve this marker exists
732
- // to end, and the same liar can already force a silent-treated absence by staying
733
- // quiet. Revisit if claims become attestable (backlog
734
- // `debt-read-repair-commit-cert-verification`) then gate the stamp on a verified
735
- // certificate rather than on the bare claim.
859
+ // NOTE: ONE claim is enough to raise that doubt, and the claims reaching this branch
860
+ // are unverified assertions a certified claim converges above instead of declining
861
+ // (the only certified shape that lands here is the equivocation decline). So a single
862
+ // lying cohort peer can deny unpinned reads of a block by claiming a revision nobody
863
+ // else holds an availability lever it did not have while uncorroborated claims were
864
+ // discarded. Deliberate for now: the alternative is the silent stale serve this marker
865
+ // exists to end, and the same liar can already force a silent-treated absence by
866
+ // staying quiet. If the lever is ever exercised, gate the stamp on a certified claim
867
+ // (the verification machinery now exists) rather than on the bare assertion — at the
868
+ // cost of re-opening the stale-serve window for the proof-less honest majority.
736
869
  const uncorroboratedRev = claims.length > 0 ? Math.max(...claims.map(c => c.rev)) : undefined;
737
870
  return { local, silent, answered, ...(uncorroboratedRev !== undefined ? { uncorroboratedRev } : {}) };
738
871
  }
739
- // Best-effort: penalize peers whose claim contradicts the corroborated pair
740
- // (an inflated rev the quorum outvoted, or conflicting content at the agreed
741
- // rev). A lower rev is just lag, never penalized. Never let this throw.
872
+ if (selected.certified) {
873
+ // Which rule won matters when reading a repair log: a certified selection may rest on a
874
+ // SINGLE claimant whose corroboration is the cohort's signature set, not other voters.
875
+ this.log('cluster-fetch:certified-selected', {
876
+ blockId, rev: selected.rev, claimants: selected.supporters.length
877
+ });
878
+ }
879
+ // Best-effort: penalize peers whose claim contradicts a CORROBORATED selection — a different
880
+ // action at the very same revision. A higher rev may be honest leadership and a lower rev is
881
+ // just lag; neither is penalized, nor is anything contradicting a certified-only selection
882
+ // (an unanchored proof must not be able to convict the honest cohort). Never let this throw.
742
883
  this.penalizeContradictingRevClaims(claims, selected, blockId);
743
884
  return { corroborated: { actionId: selected.actionId, rev: selected.rev }, local, silent, answered };
744
885
  }
745
886
  /**
746
- * Report peers whose reported latest contradicts the quorum-corroborated pair. Best-effort.
887
+ * Say ONCE per block, in words, when a corroboration decline is provably PERMANENT rather than a
888
+ * transient shortage of answers. There are exactly TWO permanent shapes, and they send the operator
889
+ * to different places, so each gets its own `reason` and its own wording:
890
+ *
891
+ * - `cohort-too-small` — this node's cohort has fewer peers than the quorum would demand even if
892
+ * every one of them answered and agreed. The remedy is machines or an honest declared size.
893
+ * - `sole-holder` — the cohort is big enough, but exactly ONE of its peers holds the block at all
894
+ * and every other peer answered that it holds nothing. The remedy is another cohort peer
895
+ * holding the block; machines and configuration are both irrelevant. Note the scope: this node's
896
+ * own copy is excluded from the claim set, so a reader that holds the block itself still sees
897
+ * `sole-holder` — the message says "cohort peer", never "machine in the deployment".
898
+ *
899
+ * **What makes `cohort-too-small` provable.** Not "this pass fell short" — a pass falls short
900
+ * whenever some peer simply does not hold the block *yet*. The decisive question is whether the
901
+ * cohort could supply the quorum AT ALL: ask what would be required if every cohort peer answered
902
+ * and agreed — the best case any later pass can reach without new machines — and compare it to how
903
+ * many peers the cohort has. Short of that best case the shortfall is not the machine count, and
904
+ * saying PERMANENT would send the operator to change a number that was never the problem. Twelve
905
+ * days of log archaeology went into re-deriving the real condition from a thousand identical
906
+ * `cluster-fetch:no-quorum` lines; the node knows it at the moment of each decline.
907
+ *
908
+ * **What makes `sole-holder` provable.** Note it is only reachable for a lone UNCERTIFIED holder:
909
+ * a lone holder whose cohort commit proof verified is selected by the certified path and converges
910
+ * before any decline — so the wording's "a lone holder cannot second itself" stays accurate for
911
+ * every claim that gets here. "That peer will hold it later" is an assumption, and for a
912
+ * peer that ANSWERED "I hold nothing" it is false: the only two mechanisms that would turn a
913
+ * non-holder into a holder — `queryClusterForLatest` (read-repair) and `createReconcileBlock`
914
+ * (reconcile) — consume this very decision, so they decline for exactly the same reason on that
915
+ * peer. Every peer answered, one holds the block, the rest hold nothing, and no later pass changes
916
+ * any of that. What DOES change it is a new copy: a commit that writes the block again pushes it to
917
+ * the current cohort. (Sibling work `replicate-owned-blocks-when-the-cohort-grows` makes that
918
+ * automatic; until it lands the operator has to cause the write.)
919
+ *
920
+ * **What is deliberately NOT reported.** A cohort that answers unanimously "I hold nothing" — an
921
+ * agreed absence is an answer, not a failed repair. A pass with any silent peer: silence cannot
922
+ * change the arithmetic (`cohortPeers` counts silent peers too), but it does mean this node saw less
923
+ * than the whole picture, and the next clean pass says the same thing at no cost. Note there is
924
+ * deliberately NO "the claims disagreed" exemption for `cohort-too-small`: a cohort too small to
925
+ * reach quorum stays too small whether its peers agree or not, so disagreement would suppress a line
926
+ * that is still true. Two or more disagreeing holders DO suppress `sole-holder`, because that is a
927
+ * cohort with two copies whose peers have not settled yet — a later pass can settle it.
747
928
  *
748
- * NOTE: the `rev > selected.rev` branch is NOT provable misbehavior an honest
749
- * peer legitimately ahead of the sampled quorum (an in-flight commit it durably
750
- * stored, or other honest holders dropped from the sample by the 1s per-peer
751
- * timeout) reports a higher rev and gets penalized (weight 30 immediate
752
- * deprioritize at threshold 20). Distinguishing a liar from an honest leader
753
- * needs the commit-cert verification tracked in backlog
754
- * `debt-read-repair-penalty-provable-only` / `debt-read-repair-commit-cert-verification`.
929
+ * **Never a lever.** This only classifies and logs; it never relaxes a floor. Which is also why the
930
+ * `cohort-too-small` message names *two* readings of the same numbers a deployment that genuinely
931
+ * runs this few machines, or a cohort view shrunk below the real deployment by a partition or by an
932
+ * attacker with routing influence. `corroboratorCapacity` keeps the shrunken view out of the relaxed
933
+ * branch, but this node cannot tell the two apart from the inside, and an operator sent to fix the
934
+ * wrong one is the failure this line exists to end.
935
+ *
936
+ * NOTE: the reader is still told only "this may be stale" — `BlockPossiblyStaleError` implies a
937
+ * retry might help, which is wrong advice for a block whose repair is deadlocked as configured.
938
+ * Carrying this condition into the error needs a new field on `GetBlockResult` plus a change to
939
+ * that error's documented contract; deliberately out of scope here (see the ticket
940
+ * `repair-deadlock-is-never-named`, *Not this ticket*).
941
+ */
942
+ reportRepairDeadlock(pass) {
943
+ const { blockId, claims, silentCount, cohortPeers, answered, required, capacity } = pass;
944
+ // An incomplete picture proves nothing about the deployment; the next clean pass says it.
945
+ if (silentCount > 0)
946
+ return;
947
+ // Nobody claimed anything: the cohort agrees the block is absent, which is an answer, not a
948
+ // deadlock.
949
+ if (claims.length === 0)
950
+ return;
951
+ // The decisive test for the first shape. `requiredEvenIfAllAnswered` is the quorum this cohort
952
+ // would face with every one of its peers answering and agreeing — the best case reachable
953
+ // without adding machines. A cohort that can meet it is not too small.
954
+ const requiredEvenIfAllAnswered = quorumSize(cohortPeers, this.simpleMajorityThreshold, capacity);
955
+ const cohortTooSmall = cohortPeers < requiredEvenIfAllAnswered;
956
+ // The second shape: exactly one cohort peer holds the block AT ALL, and — since a claim is one
957
+ // peer's latest, so a single claim is a single distinct (rev, actionId) group with a single
958
+ // supporter — every other cohort peer answered that it holds nothing. `answered === cohortPeers`
959
+ // is already implied by the silence guard above; it is stated because the two counts arrive as
960
+ // independent parameters and "everybody answered" is half of what makes this provable.
961
+ //
962
+ // NOTE: there is a narrow window where `sole-holder` is true of the instant but not of the
963
+ // deployment — a commit that has landed on one cohort member and has not yet been pushed to the
964
+ // rest presents exactly this shape. Calling it PERMANENT is defensible even there (repair
965
+ // genuinely cannot converge until the push lands, and the once-per-episode flag clears the
966
+ // moment the block converges, so the line does not repeat), and widening the window is what the
967
+ // push path's own threat model decides — see
968
+ // `tickets/blocked/repair-floor-defends-a-door-the-push-path-leaves-open`. If commit-to-push
969
+ // latency ever grows enough that operators see `sole-holder` on blocks that heal moments later,
970
+ // gate the line on the block having been quiet for longer than that latency rather than
971
+ // softening the wording.
972
+ const soleHolder = claims.length === 1 && answered === cohortPeers;
973
+ if (!cohortTooSmall && !soleHolder)
974
+ return;
975
+ // Both shapes can hold at once (an undeclared two-machine deployment whose single peer holds the
976
+ // block is both). `cohort-too-small` is reported in preference because its remedy is the one
977
+ // that actually works there: declaring the real size makes the floor reachable, after which the
978
+ // lone peer's claim IS adopted — so calling it a sole-holder problem would send the operator
979
+ // looking for a copy they do not need.
980
+ const reason = cohortTooSmall ? 'cohort-too-small' : 'sole-holder';
981
+ const state = this.unsettledAheadClaims.get(blockId);
982
+ const alreadySaid = state?.deadlocksReported ?? [];
983
+ // Suppressed per REASON, not once outright: an episode that starts as `cohort-too-small` and
984
+ // becomes `sole-holder` — the operator added the machines that reason asked for, and the block
985
+ // is still stuck — has a second thing to say, and a silent log there is the failure this line
986
+ // exists to end. Neither reason repeats within an episode.
987
+ if (alreadySaid.includes(reason))
988
+ return;
989
+ this.log('cluster-fetch:repair-deadlock', {
990
+ blockId,
991
+ reason,
992
+ cohortPeers,
993
+ answered,
994
+ claimants: claims.length,
995
+ required,
996
+ requiredEvenIfAllAnswered,
997
+ repairCorroborationClusterSize: this.repairCorroborationClusterSize,
998
+ message: cohortTooSmall
999
+ ? cohortTooSmallMessage(cohortPeers, claims.length, requiredEvenIfAllAnswered, this.repairCorroborationClusterSize)
1000
+ : soleHolderMessage(cohortPeers)
1001
+ });
1002
+ // Hung off the existing per-block freshness entry rather than a fourth per-block map. The entry
1003
+ // survives `recordAheadClaim` clearing its `rev`, and is dropped wholesale once the block
1004
+ // converges — so each reason is said once per non-convergence episode, not once per pass.
1005
+ // NOTE: per BLOCK, though the condition is a property of the cohort, not of any block — so a node
1006
+ // in this state that reads N distinct blocks emits N lines. Deliberate: the operator wants to
1007
+ // know which blocks are stuck, and N is bounded by blocks actually read (1821 lines for a single
1008
+ // block was the defect). If a deployment in this state ever makes this the noisy line again, add
1009
+ // a node-level once-flag keyed on (cohortPeers, requiredEvenIfAllAnswered) and let the per-block
1010
+ // entry only suppress repeats.
1011
+ this.unsettledAheadClaims.set(blockId, { ...(state ?? {}), deadlocksReported: [...alreadySaid, reason] });
1012
+ }
1013
+ /**
1014
+ * Report peers whose reported latest PROVABLY contradicts a CORROBORATED selection: the same
1015
+ * revision under a different actionId. Two actions cannot both be the commit at one revision,
1016
+ * and the pair a quorum of distinct peers agreed on is the one this node can stand behind, so
1017
+ * the disagreeing claimant is wrong. Best-effort.
1018
+ *
1019
+ * A CERTIFIED selection is deliberately excluded — no claim is penalized against it. A passing
1020
+ * proof shows the cohort it names signed the commit, never that those signers are the block's
1021
+ * responsible cohort: anyone holding N keys can mint a proof that verifies (see caller
1022
+ * obligation #1 in `cluster/commit-proof.ts`, and the unwired {@link ProofAnchoring} layer).
1023
+ * Penalizing here would therefore hand one forged proof a lever it must not have — every honest
1024
+ * peer holding the real action at that revision reported for InvalidRestoration (weight 30,
1025
+ * above the deprioritize threshold of 20), on every consult. Losing the selection to the proof
1026
+ * is already the accepted cost of the certified path; deprioritizing the honest cohort on top of
1027
+ * it is not. Revisit when certification is anchored to the block's derived cohort
1028
+ * (`feat-cluster-membership-threshold-cert-anchoring`): a gated proof makes the contradiction
1029
+ * provable again.
1030
+ *
1031
+ * A claim at a HIGHER rev than the selection is deliberately NOT penalized: a peer can honestly
1032
+ * be ahead of the sampled quorum — an in-flight commit it durably stored before the rest of the
1033
+ * cohort, or other honest holders dropped from the sample by the 1s per-peer consult deadline —
1034
+ * and the InvalidRestoration weight (30) sits above the deprioritize threshold (20), so a single
1035
+ * false hit used to deprioritize an honest, up-to-date peer. Declining to RESTORE from the
1036
+ * uncorroborated higher claim already happens in selection; the affirmative penalty on that
1037
+ * ambiguous evidence is what this method no longer applies. Provably-bad proof SERVICE is
1038
+ * penalized at verification time instead (the certifyClaim pass in
1039
+ * {@link queryClusterForLatest}).
755
1040
  */
756
1041
  penalizeContradictingRevClaims(claims, selected, blockId) {
757
- if (!this.reputation)
1042
+ if (!this.reputation || selected.certified)
758
1043
  return;
759
1044
  try {
760
1045
  for (const c of claims) {
761
- const contradicts = c.rev > selected.rev
762
- || (c.rev === selected.rev && c.actionId !== selected.actionId);
763
- if (contradicts) {
1046
+ if (c.rev === selected.rev && c.actionId !== selected.actionId) {
764
1047
  this.reputation.reportPeer(c.peerId, PenaltyReason.InvalidRestoration, `read-repair:${blockId}`);
765
1048
  }
766
1049
  }
@@ -769,6 +1052,22 @@ export class CoordinatorRepo {
769
1052
  this.log('cluster-fetch:penalize-error', { blockId, error: err.message });
770
1053
  }
771
1054
  }
1055
+ /**
1056
+ * Best-effort penalty for a peer whose SERVED PROOF provably lies or provably does not cover the
1057
+ * claim it was attached to (see the attributability classification in
1058
+ * `cluster/certified-claims.ts`). Never throws — mirrors
1059
+ * {@link penalizeContradictingRevClaims}.
1060
+ */
1061
+ penalizeProofService(peerId, blockId) {
1062
+ if (!this.reputation)
1063
+ return;
1064
+ try {
1065
+ this.reputation.reportPeer(peerId, PenaltyReason.InvalidRestoration, `read-repair:${blockId}`);
1066
+ }
1067
+ catch (err) {
1068
+ this.log('cluster-fetch:penalize-error', { blockId, error: err.message });
1069
+ }
1070
+ }
772
1071
  async pend(request, options) {
773
1072
  const allBlockIds = blockIdsForTransforms(request.transforms);
774
1073
  await this.verifyResponsibility(allBlockIds);
@@ -783,10 +1082,13 @@ export class CoordinatorRepo {
783
1082
  coordinatingBlockIds
784
1083
  };
785
1084
  try {
786
- const { localExecuted } = await this.coordinator.executeClusterTransaction(coordinatingBlockIds[0], message, options);
1085
+ const { localExecuted, localPendResult } = await this.coordinator.executeClusterTransaction(coordinatingBlockIds[0], message, options);
787
1086
  this.log('coordinator-repo:pend-cluster-complete', {
788
1087
  actionId: request.actionId,
789
- localExecuted
1088
+ localExecuted,
1089
+ localVerdict: localPendResult === undefined ? 'none'
1090
+ : localPendResult.success ? 'success'
1091
+ : isConflictFailure(localPendResult) ? 'conflict' : 'fault'
790
1092
  });
791
1093
  // Only call storageRepo if local cluster didn't already execute during consensus
792
1094
  if (!localExecuted) {
@@ -799,7 +1101,29 @@ export class CoordinatorRepo {
799
1101
  });
800
1102
  return result;
801
1103
  }
802
- // Local cluster already executed - return success
1104
+ // Local cluster already executed during consensus — return storage's own verdict rather
1105
+ // than fabricating a success (the peerCount <= 1 path above returns storage's real result
1106
+ // verbatim; the cluster path must never answer differently). Pend-consensus confers no
1107
+ // durability: a refusal carrying `pending` (a rival's unresolved action holds the blocks)
1108
+ // or `missing` (the requested revision is already committed) is the optimistic-concurrency
1109
+ // verdict — the same scan every member runs, not a local fault — and must reach the writer
1110
+ // as a retryable conflict so NetworkTransactor.pendPhase cancels the partial pend and the
1111
+ // writer rebases. This deliberately differs from `commit`'s divergence split below: a
1112
+ // commit that reached commit-consensus IS the authoritative commit (Theorem 9), whereas a
1113
+ // pend that reached pend-consensus may still have been stored by nobody.
1114
+ if (localPendResult !== undefined) {
1115
+ if (localPendResult.success || isConflictFailure(localPendResult)) {
1116
+ return localPendResult;
1117
+ }
1118
+ // A bare-reason refusal (no pending/missing — e.g. a local validation-hook fault)
1119
+ // stays tolerated local divergence: consensus is authoritative and the pend may well
1120
+ // have landed on the rest of the cohort.
1121
+ this.log('coordinator-repo:pend-local-fault-tolerated', {
1122
+ actionId: request.actionId,
1123
+ reason: localPendResult.reason
1124
+ });
1125
+ }
1126
+ // No verdict retained (member predates retention, restart, or TTL): the prior shape.
803
1127
  return {
804
1128
  success: true,
805
1129
  pending: [],
@@ -818,10 +1142,28 @@ export class CoordinatorRepo {
818
1142
  // has no field for it and the retry loop only needs "retryable". If a caller ever needs to
819
1143
  // know WHICH transaction won (e.g. to wait on it rather than re-race it), add a typed field
820
1144
  // for it; never recover it by parsing `reason`.
1145
+ //
1146
+ // NOTE: with three or more contenders the members can split so that EVERY contender is
1147
+ // told it lost the race — an all-lose round where nobody wins and each writer retries.
1148
+ // The cause is `ClusterMember.resolveRace`'s approvals-first rule, not its tie-break:
1149
+ // each member compares the rivals as IT holds them, so a member that already approved X
1150
+ // keeps X while a member that approved Y first keeps Y, and no rival reaches a promise
1151
+ // supermajority. (The hash tie-break is already symmetric — it cannot be the fix.)
1152
+ // Fine as it stands: since the torn-action fixes landed, an all-lose round costs one
1153
+ // retry cycle rather than wedging, and the contenders are separated next round by the
1154
+ // jittered backoff plus the aged retry priority carried on the re-pend
1155
+ // (`clampPriority(consecutiveFailures)` in `Collection.syncInternal`), which out-ranks
1156
+ // fresh priority-0 rivals at EQUAL approval counts — priority sits below the approval
1157
+ // count in `resolveRace`, so it does not displace a more-progressed rival. If a
1158
+ // high-contention workload ever shows syncs exhausting `maxAttempts` on repeated
1159
+ // all-lose rounds, the fix is reserve/defer at pend time (backlog
1160
+ // `feat-occ-priority-reservation`, which `resolveRace`'s own residual-fairness NOTE
1161
+ // already points at) rather than raising maxAttempts.
821
1162
  if (error instanceof ConflictRaceLostError) {
822
1163
  return { success: false, conflict: true, reason: error.message };
823
1164
  }
824
- const stale = await this.classifyStaleRejection(error, request, allBlockIds);
1165
+ const stale = await this.classifyStaleRejection(error, request, allBlockIds)
1166
+ ?? await this.classifyPendingConflictRejection(error, request, allBlockIds);
825
1167
  if (stale)
826
1168
  return stale;
827
1169
  throw error;
@@ -867,7 +1209,18 @@ export class CoordinatorRepo {
867
1209
  // number and the reason prose name that block, so they never disagree.
868
1210
  const staleAt = highestStaleAt(blockIds.map(blockId => {
869
1211
  const latest = results[blockId]?.state.latest;
870
- return latest && latest.rev >= requestedRev ? { blockId, rev: latest.rev } : undefined;
1212
+ if (!latest || latest.rev < requestedRev)
1213
+ return undefined;
1214
+ // Per-block self-exclusion (see {@link isOwnRevision}): our own durable half of a torn
1215
+ // action is not a confirmed loss. Deliberately per-block, NOT the bail-entirely
1216
+ // 'own-durable' shape of confirmCommitRivalAgainstLocal — a confirmed rival on ANOTHER
1217
+ // block still confirms, and when none is confirmed anywhere the rejection stays a throw
1218
+ // exactly as before. With the two pend-tier sites upstream fixed (StorageRepo.pend and
1219
+ // ClusterMember.validatePendOperations) this shape should not reach here; mirrored so
1220
+ // all three pend-tier checks agree.
1221
+ if (isOwnRevision(latest, requestedRev, request.actionId))
1222
+ return undefined;
1223
+ return { blockId, rev: latest.rev };
871
1224
  }));
872
1225
  if (staleAt) {
873
1226
  this.log('coordinator-repo:pend-stale-classified', {
@@ -893,6 +1246,56 @@ export class CoordinatorRepo {
893
1246
  // confirmed number to report, and the field's contract forbids inferring one from that text.
894
1247
  return undefined;
895
1248
  }
1249
+ /**
1250
+ * Sibling of {@link classifyStaleRejection} for the OTHER optimistic-concurrency refusal shape:
1251
+ * the promise-phase pending-conflict vote (`validatePendOperations` rejecting a pend whose
1252
+ * blocks are held by a different unresolved pending action). That vote surfaces here as a
1253
+ * {@link ValidatorRejectionError}, and without classification it would escape as a throw —
1254
+ * splitting multi-tree pends mid-batch instead of taking the retry path a lost race deserves.
1255
+ *
1256
+ * Same confirmation discipline as the stale classifier: purely local. Re-read the affected
1257
+ * blocks from our own storage and require some block's `state.pendings` to carry a rival
1258
+ * actionId; the signed reject text is never consulted. A confirmed rival returns a
1259
+ * {@link StaleFailure} with `conflict: true` and the rivals as `pending` (`ActionPending`
1260
+ * without `transform` — the type allows it, and no consumer rebases from it). Unconfirmed —
1261
+ * including read errors during confirmation — stays a throw, preserving fail-fast for genuine
1262
+ * validation faults. Checked after `classifyStaleRejection` so a confirmed committed loss
1263
+ * (which carries the sharper `staleAt`) wins when both hold.
1264
+ */
1265
+ async classifyPendingConflictRejection(error, request, blockIds) {
1266
+ if (!(error instanceof ValidatorRejectionError))
1267
+ return undefined;
1268
+ let results;
1269
+ try {
1270
+ results = await this.storageRepo.get({ blockIds });
1271
+ }
1272
+ catch (readError) {
1273
+ this.log('coordinator-repo:pend-conflict-classify-read-error', {
1274
+ actionId: request.actionId,
1275
+ error: readError.message
1276
+ });
1277
+ return undefined;
1278
+ }
1279
+ const pending = [];
1280
+ for (const blockId of blockIds) {
1281
+ for (const actionId of results[blockId]?.state?.pendings ?? []) {
1282
+ if (actionId !== request.actionId)
1283
+ pending.push({ blockId, actionId });
1284
+ }
1285
+ }
1286
+ if (pending.length === 0)
1287
+ return undefined;
1288
+ this.log('coordinator-repo:pend-conflict-classified', {
1289
+ actionId: request.actionId,
1290
+ rivals: pending.map(p => `${p.blockId}:${p.actionId}`)
1291
+ });
1292
+ return {
1293
+ success: false,
1294
+ conflict: true,
1295
+ pending,
1296
+ reason: `pending conflict: block(s) held by unresolved rival action(s) ${[...new Set(pending.map(p => p.actionId))].join(', ')}`
1297
+ };
1298
+ }
896
1299
  async cancel(actionRef, options) {
897
1300
  const blockIds = actionRef.blockIds;
898
1301
  await this.verifyResponsibility(blockIds);
@@ -902,12 +1305,30 @@ export class CoordinatorRepo {
902
1305
  expiration: options?.expiration ?? Date.now() + this.DEFAULT_TIMEOUT
903
1306
  };
904
1307
  try {
905
- // For each block ID, execute a cluster transaction
906
- const clusterPromises = blockIds.map(blockId => this.coordinator.executeClusterTransaction(blockId, message, options));
907
- // Wait for all cluster transactions to complete
908
- const results = await Promise.all(clusterPromises);
1308
+ // One cluster transaction per block ID but a block whose cohort is just this node
1309
+ // short-circuits to local storage, exactly as `pend` and `commit` do above. Without the
1310
+ // short-circuit a solo cohort enters `executeTransaction`, fails `minAbsoluteClusterSize`
1311
+ // (2), and throws `Cluster size 1 below minimum 2 and not validated` — so a single-peer
1312
+ // deployment could pend and commit but never cancel, unless the operator had opened the
1313
+ // `allowUnvalidatedSmallCluster` hatch. Decided per block rather than once for
1314
+ // `blockIds[0]`, because a multi-block cancel can span cohorts of different sizes.
1315
+ //
1316
+ // NOTE: `getClusterSize` is a second `findCluster` for the same key that
1317
+ // `executeClusterTransaction` is about to look up again, so a cancel over N blocks now
1318
+ // costs 2N cohort lookups instead of N. Same shape `pend` and `commit` already pay, but
1319
+ // they pay it once (they only ever consult `blockIds[0]`) where this scales with N. Fine
1320
+ // while cancels span a handful of blocks; if wide multi-block cancels ever show up hot,
1321
+ // have `executeClusterTransaction` return the cohort it already fetched (or own the
1322
+ // short-circuit itself) rather than adding a cache here.
1323
+ const results = await Promise.all(blockIds.map(async (blockId) => {
1324
+ const peerCount = await this.coordinator.getClusterSize(blockId);
1325
+ if (peerCount <= 1)
1326
+ return false;
1327
+ const { localExecuted } = await this.coordinator.executeClusterTransaction(blockId, message, options);
1328
+ return localExecuted;
1329
+ }));
909
1330
  // Only call storageRepo if local cluster didn't already execute during consensus
910
- const anyLocalExecuted = results.some(r => r.localExecuted);
1331
+ const anyLocalExecuted = results.some(Boolean);
911
1332
  if (!anyLocalExecuted) {
912
1333
  await this.storageRepo.cancel(actionRef, options);
913
1334
  }
@@ -932,8 +1353,41 @@ export class CoordinatorRepo {
932
1353
  expiration: options?.expiration ?? Date.now() + this.DEFAULT_TIMEOUT
933
1354
  };
934
1355
  try {
935
- const { record, localExecuted } = await this.coordinator.executeClusterTransaction(blockIds[0], message, options);
1356
+ const { record, localExecuted, localCommitResult } = await this.coordinator.executeClusterTransaction(blockIds[0], message, options);
936
1357
  if (localExecuted) {
1358
+ // Our own member applied this commit during consensus. Its retained storage verdict is
1359
+ // the one honest signal we have about durability: the member-side apply tolerates an
1360
+ // "ahead" refusal as divergence (see the NOTE in ClusterMember.applyConsensusOperation),
1361
+ // which is correct for a redelivered or lagging commit — but when the refusal's real
1362
+ // cause is a RIVAL action holding the requested revision, that tolerance turns a commit
1363
+ // no member durably stored into a fabricated success. This is the
1364
+ // signed-but-not-yet-applied window: two commits for one revision can BOTH assemble
1365
+ // consensus when every member signs the second after signing (but before applying) the
1366
+ // first, because signing drops the member's reservation. Confirm the rival against local
1367
+ // storage (never the verdict's prose) and answer the writer with a retryable conflict so
1368
+ // it re-drives at a fresh revision. Own-action or unconfirmed refusals keep the
1369
+ // prior fabricated-success shape: consensus is authoritative and this member converges
1370
+ // via replication.
1371
+ //
1372
+ // NOTE: a CONFIRMED rival is trusted over the consensus outcome here. That is right in
1373
+ // the window this closes (the cohort refused the loser too), but it inverts if the two
1374
+ // ever disagree — a local rival at the requested revision while a super-majority
1375
+ // approved OUR commit means this node is on a forked lineage, and refusing then tells a
1376
+ // writer whose write did land to re-drive it (a duplicate entry). Members holding the
1377
+ // rival reject at the promise round, so consensus and a local rival can only disagree
1378
+ // after a fork; that is partition-healing scope (docs/partition-healing.md). If forks
1379
+ // are ever observed here, weigh the retained verdict against the cohort's votes instead
1380
+ // of trusting the local re-read alone.
1381
+ if (localCommitResult !== undefined && !localCommitResult.success) {
1382
+ const rival = await this.confirmCommitRivalAgainstLocal(request);
1383
+ if (typeof rival === 'object')
1384
+ return rival;
1385
+ this.log('coordinator-repo:commit-local-refusal-tolerated', {
1386
+ actionId: request.actionId,
1387
+ confirmation: rival ?? 'unconfirmed',
1388
+ reason: localCommitResult.reason
1389
+ });
1390
+ }
937
1391
  this.markBlocksSeen(blockIds);
938
1392
  return { success: true };
939
1393
  }
@@ -970,9 +1424,143 @@ export class CoordinatorRepo {
970
1424
  }
971
1425
  catch (error) {
972
1426
  this.log('coordinator-repo:commit-error', { actionId: request.actionId, error: error.message });
1427
+ // A lost commit-consensus race is an optimistic-concurrency loss, not a fault — mirror
1428
+ // `pend`'s conversion above. At the moment this is thrown, zero members approved and the
1429
+ // members hold the winner: nothing of the loser landed, so a retryable-conflict answer is
1430
+ // truthful. Returning it (rather than rethrowing) matters more here than on the pend path:
1431
+ // db-core's `commitCollection` retries a THROWN commit error verbatim up to 3 times, and by
1432
+ // the retry the members have applied the winner and cleared its reservation — the re-driven
1433
+ // commit can then assemble a consensus no member will durably store (the writer's append
1434
+ // fulfills, the entry exists on no node). A RETURNED `success:false` is instead surfaced
1435
+ // immediately as a stale loss; the writer cancels the pend, re-reads, and re-drives the
1436
+ // whole pend+commit at a fresh revision. `staleAt` stays absent for the same reason as
1437
+ // pend's: it is confirmed-only, and a lost race is a rival commit racing the same revision,
1438
+ // not a locally-confirmed revision claim.
1439
+ if (error instanceof ConflictRaceLostError) {
1440
+ return { success: false, conflict: true, reason: error.message };
1441
+ }
1442
+ // A promise-phase stale-commit reject (`ClusterMember.validateCommitRevisions` — a member
1443
+ // holds the requested revision under a different action) surfaces here as a
1444
+ // ValidatorRejectionError; classify it against local storage the way `pend` does, so the
1445
+ // writer gets a clean retryable conflict instead of three verbatim re-drives and a hard
1446
+ // failure.
1447
+ const stale = await this.classifyCommitStaleRejection(error, request);
1448
+ if (stale)
1449
+ return stale;
973
1450
  throw error;
974
1451
  }
975
1452
  }
1453
+ /**
1454
+ * Commit-shaped sibling of {@link classifyStaleRejection}: decide whether a cluster validator
1455
+ * rejection of a COMMIT was an optimistic-concurrency loss — the requested revision is already
1456
+ * committed under a different action — rather than a genuine validation fault. A confirmed loss
1457
+ * returns a {@link StaleFailure} with `conflict: true` so db-core's `commitCollection` surfaces
1458
+ * it immediately as a stale loss (no verbatim retry) and the writer re-drives at a fresh
1459
+ * revision.
1460
+ *
1461
+ * Same confirmation discipline as the pend classifiers: purely local re-read; the signed reject
1462
+ * text is never consulted. One commit-specific delta — confirmation must EXCLUDE the
1463
+ * own-action-at-rev case: a block whose requested revision is held by THIS action is already
1464
+ * durable, and answering `conflict` for it would make the writer rebase and re-append an
1465
+ * already-committed action at a new revision — a duplicate entry. So:
1466
+ * - `latest.rev === request.rev` → compare `latest.actionId`: ours ⇒ bail (stays a throw),
1467
+ * a rival's ⇒ confirmed loss;
1468
+ * - `latest.rev > request.rev` → ask the {@link IRevisionActionReader} capability who holds
1469
+ * `request.rev`: ours ⇒ bail, a rival's ⇒ confirmed loss, unknown/absent/fault ⇒ unconfirmed;
1470
+ * - anything unconfirmed (including read errors) stays a throw — fail-fast for genuine faults.
1471
+ */
1472
+ async classifyCommitStaleRejection(error, request) {
1473
+ if (!(error instanceof ValidatorRejectionError))
1474
+ return undefined;
1475
+ const rival = await this.confirmCommitRivalAgainstLocal(request);
1476
+ // 'own-durable' and unconfirmed both stay a throw here: fail-fast for genuine faults, and a
1477
+ // commit already durable under this action must never be answered `conflict` (the writer
1478
+ // would rebase and re-append it — a duplicate entry).
1479
+ return typeof rival === 'object' ? rival : undefined;
1480
+ }
1481
+ /**
1482
+ * Shared confirmation core for the two commit-tier conversion sites ({@link classifyCommitStaleRejection}
1483
+ * and the locally-executed refusal check in {@link commit}): decide, from LOCAL storage only, who
1484
+ * holds the requested revision.
1485
+ * - a confirmed RIVAL → the {@link StaleFailure} conflict answer (with `staleAt` = highest
1486
+ * confirmed holder);
1487
+ * - our OWN action durable at the requested revision → `'own-durable'` (callers must not answer
1488
+ * `conflict` — the writer would rebase an already-landed action into a duplicate entry);
1489
+ * - anything else (behind, truncated history, read faults, capability absent) → `undefined`,
1490
+ * unconfirmed.
1491
+ * The signed reject text / retained verdict prose is never consulted.
1492
+ */
1493
+ async confirmCommitRivalAgainstLocal(request) {
1494
+ const blockIds = request.blockIds;
1495
+ let results;
1496
+ try {
1497
+ results = await this.storageRepo.get({ blockIds });
1498
+ }
1499
+ catch (readError) {
1500
+ this.log('coordinator-repo:commit-stale-classify-read-error', {
1501
+ actionId: request.actionId,
1502
+ error: readError.message
1503
+ });
1504
+ return undefined;
1505
+ }
1506
+ const reader = this.storageRepo;
1507
+ // Scan EVERY block (same rule as the pend classifier): report the highest confirmed rival
1508
+ // revision, but bail the moment any block shows OUR action durable at the requested revision.
1509
+ const rivalStales = [];
1510
+ for (const blockId of blockIds) {
1511
+ const latest = results[blockId]?.state?.latest;
1512
+ if (!latest || latest.rev < request.rev)
1513
+ continue;
1514
+ if (latest.rev === request.rev) {
1515
+ if (latest.actionId === request.actionId) {
1516
+ this.log('coordinator-repo:commit-stale-classify-own-action', {
1517
+ actionId: request.actionId, blockId, rev: request.rev
1518
+ });
1519
+ return 'own-durable';
1520
+ }
1521
+ rivalStales.push({ blockId, rev: latest.rev });
1522
+ continue;
1523
+ }
1524
+ // latest.rev > request.rev — latest can no longer name who took request.rev.
1525
+ if (typeof reader.getRevisionAction !== 'function')
1526
+ continue;
1527
+ let takenBy;
1528
+ try {
1529
+ takenBy = await reader.getRevisionAction(blockId, request.rev);
1530
+ }
1531
+ catch (readError) {
1532
+ this.log('coordinator-repo:commit-stale-classify-revision-read-error', {
1533
+ actionId: request.actionId, blockId, rev: request.rev,
1534
+ error: readError.message
1535
+ });
1536
+ continue;
1537
+ }
1538
+ if (takenBy === request.actionId) {
1539
+ this.log('coordinator-repo:commit-stale-classify-own-action', {
1540
+ actionId: request.actionId, blockId, rev: request.rev, latestRev: latest.rev
1541
+ });
1542
+ return 'own-durable';
1543
+ }
1544
+ if (takenBy !== undefined)
1545
+ rivalStales.push({ blockId, rev: latest.rev });
1546
+ // takenBy undefined (truncated history): unconfirmed for this block.
1547
+ }
1548
+ const staleAt = highestStaleAt(rivalStales);
1549
+ if (!staleAt)
1550
+ return undefined;
1551
+ this.log('coordinator-repo:commit-stale-classified', {
1552
+ actionId: request.actionId,
1553
+ blockId: staleAt.blockId,
1554
+ latestRev: staleAt.rev,
1555
+ requestedRev: request.rev
1556
+ });
1557
+ return {
1558
+ success: false,
1559
+ conflict: true,
1560
+ reason: `stale commit: block ${staleAt.blockId} at rev ${staleAt.rev}, requested rev ${request.rev}`,
1561
+ staleAt
1562
+ };
1563
+ }
976
1564
  /**
977
1565
  * Report success for a commit the cluster carried but this peer could not apply locally. The
978
1566
  * blocks are marked seen so the read path treats them as freshness-checked; convergence comes