@optimystic/db-p2p 0.24.2 → 0.25.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (249) hide show
  1. package/{readme.md → README.md} +688 -668
  2. package/dist/src/cluster/block-transfer-service.d.ts +133 -10
  3. package/dist/src/cluster/block-transfer-service.d.ts.map +1 -1
  4. package/dist/src/cluster/block-transfer-service.js +155 -9
  5. package/dist/src/cluster/block-transfer-service.js.map +1 -1
  6. package/dist/src/cluster/block-transfer.d.ts +67 -7
  7. package/dist/src/cluster/block-transfer.d.ts.map +1 -1
  8. package/dist/src/cluster/block-transfer.js +127 -24
  9. package/dist/src/cluster/block-transfer.js.map +1 -1
  10. package/dist/src/cluster/certified-claims.d.ts +170 -0
  11. package/dist/src/cluster/certified-claims.d.ts.map +1 -0
  12. package/dist/src/cluster/certified-claims.js +200 -0
  13. package/dist/src/cluster/certified-claims.js.map +1 -0
  14. package/dist/src/cluster/client-signature-verifier.d.ts +23 -0
  15. package/dist/src/cluster/client-signature-verifier.d.ts.map +1 -0
  16. package/dist/src/cluster/client-signature-verifier.js +33 -0
  17. package/dist/src/cluster/client-signature-verifier.js.map +1 -0
  18. package/dist/src/cluster/cluster-policy.d.ts +9 -3
  19. package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
  20. package/dist/src/cluster/cluster-policy.js +111 -27
  21. package/dist/src/cluster/cluster-policy.js.map +1 -1
  22. package/dist/src/cluster/cluster-repo.d.ts +140 -11
  23. package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
  24. package/dist/src/cluster/cluster-repo.js +485 -29
  25. package/dist/src/cluster/cluster-repo.js.map +1 -1
  26. package/dist/src/cluster/commit-proof.d.ts +134 -0
  27. package/dist/src/cluster/commit-proof.d.ts.map +1 -0
  28. package/dist/src/cluster/commit-proof.js +263 -0
  29. package/dist/src/cluster/commit-proof.js.map +1 -0
  30. package/dist/src/cluster/quorum-restore.d.ts +95 -8
  31. package/dist/src/cluster/quorum-restore.d.ts.map +1 -1
  32. package/dist/src/cluster/quorum-restore.js +113 -14
  33. package/dist/src/cluster/quorum-restore.js.map +1 -1
  34. package/dist/src/cluster/rebalance-monitor.d.ts +84 -0
  35. package/dist/src/cluster/rebalance-monitor.d.ts.map +1 -1
  36. package/dist/src/cluster/rebalance-monitor.js +207 -5
  37. package/dist/src/cluster/rebalance-monitor.js.map +1 -1
  38. package/dist/src/cluster/reconcile-block.d.ts +27 -7
  39. package/dist/src/cluster/reconcile-block.d.ts.map +1 -1
  40. package/dist/src/cluster/reconcile-block.js +179 -38
  41. package/dist/src/cluster/reconcile-block.js.map +1 -1
  42. package/dist/src/cluster/service.d.ts +13 -4
  43. package/dist/src/cluster/service.d.ts.map +1 -1
  44. package/dist/src/cluster/service.js +17 -18
  45. package/dist/src/cluster/service.js.map +1 -1
  46. package/dist/src/cluster/spread-on-churn.d.ts +8 -2
  47. package/dist/src/cluster/spread-on-churn.d.ts.map +1 -1
  48. package/dist/src/cluster/spread-on-churn.js +7 -7
  49. package/dist/src/cluster/spread-on-churn.js.map +1 -1
  50. package/dist/src/cohort-topic/host.d.ts.map +1 -1
  51. package/dist/src/cohort-topic/host.js +8 -7
  52. package/dist/src/cohort-topic/host.js.map +1 -1
  53. package/dist/src/cohort-topic/stream-util.d.ts +7 -6
  54. package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
  55. package/dist/src/cohort-topic/stream-util.js +14 -41
  56. package/dist/src/cohort-topic/stream-util.js.map +1 -1
  57. package/dist/src/dispute/cascade.d.ts +7 -8
  58. package/dist/src/dispute/cascade.d.ts.map +1 -1
  59. package/dist/src/dispute/cascade.js +24 -7
  60. package/dist/src/dispute/cascade.js.map +1 -1
  61. package/dist/src/dispute/dispute-service.d.ts +5 -0
  62. package/dist/src/dispute/dispute-service.d.ts.map +1 -1
  63. package/dist/src/dispute/dispute-service.js +13 -3
  64. package/dist/src/dispute/dispute-service.js.map +1 -1
  65. package/dist/src/dispute/invalidation.d.ts +33 -15
  66. package/dist/src/dispute/invalidation.d.ts.map +1 -1
  67. package/dist/src/dispute/invalidation.js +127 -34
  68. package/dist/src/dispute/invalidation.js.map +1 -1
  69. package/dist/src/dispute/service.d.ts.map +1 -1
  70. package/dist/src/dispute/service.js +2 -1
  71. package/dist/src/dispute/service.js.map +1 -1
  72. package/dist/src/index.d.ts +7 -0
  73. package/dist/src/index.d.ts.map +1 -1
  74. package/dist/src/index.js +7 -0
  75. package/dist/src/index.js.map +1 -1
  76. package/dist/src/libp2p-key-network.d.ts +14 -10
  77. package/dist/src/libp2p-key-network.d.ts.map +1 -1
  78. package/dist/src/libp2p-key-network.js +28 -58
  79. package/dist/src/libp2p-key-network.js.map +1 -1
  80. package/dist/src/libp2p-node-base.d.ts +38 -7
  81. package/dist/src/libp2p-node-base.d.ts.map +1 -1
  82. package/dist/src/libp2p-node-base.js +148 -63
  83. package/dist/src/libp2p-node-base.js.map +1 -1
  84. package/dist/src/network/open-protocol-stream.d.ts +56 -0
  85. package/dist/src/network/open-protocol-stream.d.ts.map +1 -0
  86. package/dist/src/network/open-protocol-stream.js +60 -0
  87. package/dist/src/network/open-protocol-stream.js.map +1 -0
  88. package/dist/src/network/register-protocol-handler.d.ts +66 -0
  89. package/dist/src/network/register-protocol-handler.d.ts.map +1 -0
  90. package/dist/src/network/register-protocol-handler.js +51 -0
  91. package/dist/src/network/register-protocol-handler.js.map +1 -0
  92. package/dist/src/peer-address-book.d.ts +60 -2
  93. package/dist/src/peer-address-book.d.ts.map +1 -1
  94. package/dist/src/peer-address-book.js +89 -1
  95. package/dist/src/peer-address-book.js.map +1 -1
  96. package/dist/src/pend-validation.d.ts +65 -0
  97. package/dist/src/pend-validation.d.ts.map +1 -0
  98. package/dist/src/pend-validation.js +69 -0
  99. package/dist/src/pend-validation.js.map +1 -0
  100. package/dist/src/reactivity/notify-transport.d.ts.map +1 -1
  101. package/dist/src/reactivity/notify-transport.js +2 -1
  102. package/dist/src/reactivity/notify-transport.js.map +1 -1
  103. package/dist/src/reactivity/push-state-gossip.d.ts.map +1 -1
  104. package/dist/src/reactivity/push-state-gossip.js +2 -1
  105. package/dist/src/reactivity/push-state-gossip.js.map +1 -1
  106. package/dist/src/repo/cluster-coordinator.d.ts +32 -6
  107. package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
  108. package/dist/src/repo/cluster-coordinator.js +44 -10
  109. package/dist/src/repo/cluster-coordinator.js.map +1 -1
  110. package/dist/src/repo/coordinator-repo.d.ts +181 -25
  111. package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
  112. package/dist/src/repo/coordinator-repo.js +650 -62
  113. package/dist/src/repo/coordinator-repo.js.map +1 -1
  114. package/dist/src/repo/served-repo-proxy.d.ts +30 -0
  115. package/dist/src/repo/served-repo-proxy.d.ts.map +1 -0
  116. package/dist/src/repo/served-repo-proxy.js +50 -0
  117. package/dist/src/repo/served-repo-proxy.js.map +1 -0
  118. package/dist/src/repo/service.d.ts +16 -1
  119. package/dist/src/repo/service.d.ts.map +1 -1
  120. package/dist/src/repo/service.js +18 -16
  121. package/dist/src/repo/service.js.map +1 -1
  122. package/dist/src/reputation/types.d.ts +3 -1
  123. package/dist/src/reputation/types.d.ts.map +1 -1
  124. package/dist/src/reputation/types.js +3 -1
  125. package/dist/src/reputation/types.js.map +1 -1
  126. package/dist/src/rn.d.ts +24 -2
  127. package/dist/src/rn.d.ts.map +1 -1
  128. package/dist/src/rn.js +33 -4
  129. package/dist/src/rn.js.map +1 -1
  130. package/dist/src/storage/block-archive.d.ts +153 -0
  131. package/dist/src/storage/block-archive.d.ts.map +1 -0
  132. package/dist/src/storage/block-archive.js +194 -0
  133. package/dist/src/storage/block-archive.js.map +1 -0
  134. package/dist/src/storage/block-latch.d.ts +92 -0
  135. package/dist/src/storage/block-latch.d.ts.map +1 -0
  136. package/dist/src/storage/block-latch.js +132 -0
  137. package/dist/src/storage/block-latch.js.map +1 -0
  138. package/dist/src/storage/block-storage.d.ts +169 -12
  139. package/dist/src/storage/block-storage.d.ts.map +1 -1
  140. package/dist/src/storage/block-storage.js +415 -152
  141. package/dist/src/storage/block-storage.js.map +1 -1
  142. package/dist/src/storage/cached-raw-storage.d.ts +5 -1
  143. package/dist/src/storage/cached-raw-storage.d.ts.map +1 -1
  144. package/dist/src/storage/cached-raw-storage.js +16 -1
  145. package/dist/src/storage/cached-raw-storage.js.map +1 -1
  146. package/dist/src/storage/cached-store-driver.d.ts +14 -4
  147. package/dist/src/storage/cached-store-driver.d.ts.map +1 -1
  148. package/dist/src/storage/cached-store-driver.js +39 -5
  149. package/dist/src/storage/cached-store-driver.js.map +1 -1
  150. package/dist/src/storage/i-block-storage.d.ts +70 -19
  151. package/dist/src/storage/i-block-storage.d.ts.map +1 -1
  152. package/dist/src/storage/i-block-storage.js +21 -1
  153. package/dist/src/storage/i-block-storage.js.map +1 -1
  154. package/dist/src/storage/i-raw-storage.d.ts +38 -0
  155. package/dist/src/storage/i-raw-storage.d.ts.map +1 -1
  156. package/dist/src/storage/kv-raw-storage.d.ts +14 -8
  157. package/dist/src/storage/kv-raw-storage.d.ts.map +1 -1
  158. package/dist/src/storage/kv-raw-storage.js +29 -9
  159. package/dist/src/storage/kv-raw-storage.js.map +1 -1
  160. package/dist/src/storage/memory-store-driver.d.ts +4 -1
  161. package/dist/src/storage/memory-store-driver.d.ts.map +1 -1
  162. package/dist/src/storage/memory-store-driver.js +22 -2
  163. package/dist/src/storage/memory-store-driver.js.map +1 -1
  164. package/dist/src/storage/raw-store-codec.js +2 -2
  165. package/dist/src/storage/raw-store-driver.d.ts +44 -3
  166. package/dist/src/storage/raw-store-driver.d.ts.map +1 -1
  167. package/dist/src/storage/shared-cache-pool.d.ts +60 -3
  168. package/dist/src/storage/shared-cache-pool.d.ts.map +1 -1
  169. package/dist/src/storage/shared-cache-pool.js +68 -4
  170. package/dist/src/storage/shared-cache-pool.js.map +1 -1
  171. package/dist/src/storage/storage-repo.d.ts +204 -26
  172. package/dist/src/storage/storage-repo.d.ts.map +1 -1
  173. package/dist/src/storage/storage-repo.js +422 -119
  174. package/dist/src/storage/storage-repo.js.map +1 -1
  175. package/dist/src/storage/store-identity.d.ts +21 -0
  176. package/dist/src/storage/store-identity.d.ts.map +1 -0
  177. package/dist/src/storage/store-identity.js +33 -0
  178. package/dist/src/storage/store-identity.js.map +1 -0
  179. package/dist/src/storage/struct.d.ts +15 -0
  180. package/dist/src/storage/struct.d.ts.map +1 -1
  181. package/dist/src/storage/with-read-cache.d.ts +92 -0
  182. package/dist/src/storage/with-read-cache.d.ts.map +1 -0
  183. package/dist/src/storage/with-read-cache.js +132 -0
  184. package/dist/src/storage/with-read-cache.js.map +1 -0
  185. package/dist/src/sync/service.d.ts +13 -3
  186. package/dist/src/sync/service.d.ts.map +1 -1
  187. package/dist/src/sync/service.js +7 -30
  188. package/dist/src/sync/service.js.map +1 -1
  189. package/dist/src/testing/mesh-harness.d.ts +75 -2
  190. package/dist/src/testing/mesh-harness.d.ts.map +1 -1
  191. package/dist/src/testing/mesh-harness.js +193 -81
  192. package/dist/src/testing/mesh-harness.js.map +1 -1
  193. package/dist/src/testing/raw-storage-conformance.d.ts +8 -0
  194. package/dist/src/testing/raw-storage-conformance.d.ts.map +1 -1
  195. package/dist/src/testing/raw-storage-conformance.js +97 -2
  196. package/dist/src/testing/raw-storage-conformance.js.map +1 -1
  197. package/package.json +5 -6
  198. package/src/cluster/block-transfer-service.ts +528 -312
  199. package/src/cluster/block-transfer.ts +547 -424
  200. package/src/cluster/certified-claims.ts +319 -0
  201. package/src/cluster/client-signature-verifier.ts +34 -0
  202. package/src/cluster/cluster-policy.ts +294 -203
  203. package/src/cluster/cluster-repo.ts +2501 -2034
  204. package/src/cluster/commit-proof.ts +344 -0
  205. package/src/cluster/quorum-restore.ts +348 -223
  206. package/src/cluster/rebalance-monitor.ts +539 -254
  207. package/src/cluster/reconcile-block.ts +404 -203
  208. package/src/cluster/service.ts +23 -20
  209. package/src/cluster/spread-on-churn.ts +352 -346
  210. package/src/cohort-topic/host.ts +8 -7
  211. package/src/cohort-topic/stream-util.ts +14 -43
  212. package/src/dispute/cascade.ts +31 -15
  213. package/src/dispute/dispute-service.ts +706 -696
  214. package/src/dispute/invalidation.ts +162 -52
  215. package/src/dispute/service.ts +126 -125
  216. package/src/index.ts +7 -0
  217. package/src/libp2p-key-network.ts +1204 -1235
  218. package/src/libp2p-node-base.ts +185 -70
  219. package/src/network/open-protocol-stream.ts +87 -0
  220. package/src/network/register-protocol-handler.ts +85 -0
  221. package/src/peer-address-book.ts +96 -2
  222. package/src/pend-validation.ts +89 -0
  223. package/src/reactivity/notify-transport.ts +2 -1
  224. package/src/reactivity/push-state-gossip.ts +2 -1
  225. package/src/repo/cluster-coordinator.ts +65 -11
  226. package/src/repo/coordinator-repo.ts +729 -71
  227. package/src/repo/served-repo-proxy.ts +55 -0
  228. package/src/repo/service.ts +25 -16
  229. package/src/reputation/types.ts +3 -1
  230. package/src/rn.ts +34 -4
  231. package/src/storage/block-archive.ts +242 -0
  232. package/src/storage/block-latch.ts +144 -0
  233. package/src/storage/block-storage.ts +787 -498
  234. package/src/storage/cached-raw-storage.ts +21 -1
  235. package/src/storage/cached-store-driver.ts +898 -859
  236. package/src/storage/i-block-storage.ts +172 -116
  237. package/src/storage/i-raw-storage.ts +40 -0
  238. package/src/storage/kv-raw-storage.ts +34 -9
  239. package/src/storage/memory-store-driver.ts +26 -2
  240. package/src/storage/raw-store-codec.ts +42 -42
  241. package/src/storage/raw-store-driver.ts +45 -3
  242. package/src/storage/shared-cache-pool.ts +516 -452
  243. package/src/storage/storage-repo.ts +1421 -1014
  244. package/src/storage/store-identity.ts +41 -0
  245. package/src/storage/struct.ts +52 -34
  246. package/src/storage/with-read-cache.ts +182 -0
  247. package/src/sync/service.ts +178 -198
  248. package/src/testing/mesh-harness.ts +524 -344
  249. package/src/testing/raw-storage-conformance.ts +114 -2
@@ -1,1014 +1,1421 @@
1
- import type {
2
- IRepo, MessageOptions, BlockId, CommitRequest, CommitResult, GetBlockResults, PendRequest, PendResult, ActionBlocks,
3
- ActionId, BlockGets, ActionPending, PendSuccess, ActionTransform, ActionTransforms,
4
- GetBlockResult, IBlock, ActionRev, BlockUnavailableReason,
5
- PendValidationHook,
6
- CollectionId, IBlockChangeNotifier, CollectionChangeListener, CollectionChangeEvent,
7
- StaleFailure
8
- } from "@optimystic/db-core";
9
- import {
10
- Latches, transformForBlockId, applyTransform, groupBy, concatTransform, emptyTransforms,
11
- blockIdsForTransforms, transformsFromTransform, highestStaleAt
12
- } from "@optimystic/db-core";
13
- import { asyncIteratorToArray } from "../it-utility.js";
14
- import type { IBlockStorage } from "./i-block-storage.js";
15
- import type { IBlockReplicaStore } from "../cluster/block-transfer-service.js";
16
- import { createLogger } from "../logger.js";
17
-
18
- const log = createLogger('storage-repo');
19
-
20
- /**
21
- * Single source of truth for the per-block commit latch key. Held by {@link StorageRepo.commit} and
22
- * {@link StorageRepo.saveReplicatedBlock}, and — through an injected runner ({@link withBlockCommitLatch})
23
- * — by the invalidation-apply path. Every out-of-band writer of a block's `meta.latest` must serialize
24
- * on this key against a concurrent local commit on the same block; keeping all call sites on this helper
25
- * is what prevents the key from drifting between them.
26
- */
27
- export const commitLatchKey = (blockId: BlockId): string => `StorageRepo.commit:${blockId}`;
28
-
29
- /**
30
- * Runs `fn` while holding the per-block commit latch {@link commitLatchKey}. This is the capability the
31
- * dispute module's `applyInvalidation` is handed (through its context) so its compensating
32
- * `saveReplica`/`saveDeletion` read-modify-write of `meta.latest` is mutually exclusive with a concurrent
33
- * {@link StorageRepo.commit} on the same block otherwise an invalidation advancing `latest` outside
34
- * that latch is invisible to commit's staleness guard and can be clobbered (a non-monotonic regression).
35
- *
36
- * Acquire/release is per call, so a caller holds at most one block latch at any instant and cannot
37
- * deadlock against commit's sorted, up-front multi-latch acquisition.
38
- */
39
- export async function withBlockCommitLatch<T>(blockId: BlockId, fn: () => Promise<T>): Promise<T> {
40
- const release = await Latches.acquire(commitLatchKey(blockId));
41
- try {
42
- return await fn();
43
- } finally {
44
- release();
45
- }
46
- }
47
-
48
- /**
49
- * Stable, greppable prefix on the failure reason a commit carries when this node cannot materialize
50
- * the revision it was asked to record. It is a STRING marker rather than only an error class because
51
- * {@link StorageRepo.commit} reports per-block faults as `StaleFailure.reason` (a plain string that
52
- * also crosses the wire), so the class identity is lost by the time a caller inspects the result.
53
- */
54
- export const MISSING_BASE_REVISION_REASON = 'missing-base-revision';
55
-
56
- /**
57
- * This node was asked to commit revision N of a block it holds no materializable base for, so
58
- * applying the transform would materialize nothing while `latest` advanced to N — a block that is
59
- * then unreadable locally, unservable to peers, and that rejects every later write (see
60
- * {@link StorageRepo.internalCommit}). The commit is refused instead; the caller heals the node
61
- * out-of-band (`ClusterMember` pulls the committed revision from a cohort peer) and retries.
62
- */
63
- export class MissingBaseRevisionError extends Error {
64
- constructor(readonly blockId: BlockId, readonly rev: number, detail: string) {
65
- super(`${MISSING_BASE_REVISION_REASON}: block ${blockId} cannot materialize rev ${rev} — ${detail}`);
66
- this.name = 'MissingBaseRevisionError';
67
- }
68
- }
69
-
70
- /**
71
- * True when a {@link CommitResult} failed because this node holds no materializable base for one of
72
- * the committed blocks. Distinguishes that recoverable divergence (heal by fetching the block from a
73
- * cohort peer) from a genuine storage fault, which must still propagate.
74
- */
75
- export function isMissingBaseRevisionFailure(result: CommitResult): boolean {
76
- return !result.success && (result.reason?.startsWith(MISSING_BASE_REVISION_REASON) ?? false);
77
- }
78
-
79
- export type StorageRepoOptions = {
80
- /** Optional hook to validate transactions in PendRequests */
81
- validatePend?: PendValidationHook;
82
- };
83
-
84
- export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaStore {
85
- private readonly validatePend?: PendValidationHook;
86
- /** Per-collection change listeners; empty sets are pruned on unsubscribe. */
87
- private readonly changeListeners = new Map<CollectionId, Set<CollectionChangeListener>>();
88
- /** Catch-all change listeners fire for EVERY collection's commit on this node. */
89
- private readonly anyChangeListeners = new Set<CollectionChangeListener>();
90
-
91
- constructor(
92
- private readonly createBlockStorage: (blockId: BlockId) => IBlockStorage,
93
- options?: StorageRepoOptions
94
- ) {
95
- this.validatePend = options?.validatePend;
96
- }
97
-
98
- /**
99
- * Subscribe to commits that mutate `collectionId`'s blocks on this node.
100
- * Returns an idempotent unsubscribe. See {@link IBlockChangeNotifier}.
101
- */
102
- onCollectionChange(collectionId: CollectionId, listener: CollectionChangeListener): () => void {
103
- let set = this.changeListeners.get(collectionId);
104
- if (!set) {
105
- set = new Set();
106
- this.changeListeners.set(collectionId, set);
107
- }
108
- set.add(listener);
109
- let unsubscribed = false;
110
- return () => {
111
- if (unsubscribed) return;
112
- unsubscribed = true;
113
- const current = this.changeListeners.get(collectionId);
114
- if (current) {
115
- current.delete(listener);
116
- if (current.size === 0) {
117
- this.changeListeners.delete(collectionId);
118
- }
119
- }
120
- };
121
- }
122
-
123
- /**
124
- * Subscribe to commits mutating ANY collection on this node the catch-all feed the
125
- * cohort-topic origination bridge consumes (it cannot enumerate collection ids ahead of time,
126
- * so a per-collection {@link onCollectionChange} subscription cannot see every commit). Fires for
127
- * the same `(pending → committed)` transitions as {@link onCollectionChange}, but across every
128
- * collection. Returns an idempotent unsubscribe; a throwing listener is isolated + logged.
129
- */
130
- onAnyCollectionChange(listener: CollectionChangeListener): () => void {
131
- this.anyChangeListeners.add(listener);
132
- let unsubscribed = false;
133
- return () => {
134
- if (unsubscribed) return;
135
- unsubscribed = true;
136
- this.anyChangeListeners.delete(listener);
137
- };
138
- }
139
-
140
- /**
141
- * Fire one {@link CollectionChangeEvent} per distinct collection that was
142
- * newly committed. Called AFTER the commit critical section (locks released),
143
- * fire-and-forget synchronous; a throwing listener is isolated and logged. Each event reaches
144
- * both that collection's {@link onCollectionChange} subscribers and every
145
- * {@link onAnyCollectionChange} catch-all subscriber.
146
- *
147
- * `tailId` is the `CommitRequest.tailId` on the commit path; `undefined` on read-driven
148
- * promotions (the get/emitPromotions path has no commit request). A single commit is for one
149
- * collection's chain in practice, so all events from one commit share the same `tailId`.
150
- */
151
- private emitCollectionChanges(collectionBlocks: Map<CollectionId, BlockId[]>, actionId: ActionId, rev: number, tailId?: BlockId): void {
152
- const hasCatchAll = this.anyChangeListeners.size > 0;
153
- for (const [collectionId, blockIds] of collectionBlocks) {
154
- const listeners = this.changeListeners.get(collectionId);
155
- if ((!listeners || listeners.size === 0) && !hasCatchAll) {
156
- continue;
157
- }
158
- const event: CollectionChangeEvent = { collectionId, blockIds, actionId, rev, tailId };
159
- if (listeners && listeners.size > 0) {
160
- this.fireChangeListeners(listeners, event);
161
- }
162
- if (hasCatchAll) {
163
- this.fireChangeListeners(this.anyChangeListeners, event);
164
- }
165
- }
166
- }
167
-
168
- /** Dispatch `event` to a snapshot of `listeners` (safe under mid-emit (un)subscribe), isolating + logging any throw. */
169
- private fireChangeListeners(listeners: Set<CollectionChangeListener>, event: CollectionChangeEvent): void {
170
- for (const listener of Array.from(listeners)) {
171
- try {
172
- listener(event);
173
- } catch (err) {
174
- log('onCollectionChange listener threw for collection=%s: %o', event.collectionId, err);
175
- }
176
- }
177
- }
178
-
179
- async get({ blockIds, context }: BlockGets, _options?: MessageOptions): Promise<GetBlockResults> {
180
- const distinctBlockIds = Array.from(new Set(blockIds));
181
- log('get blockIds=%d', distinctBlockIds.length);
182
- // Read-driven promotions that land durably here, captured so we can emit a
183
- // change event per durable landing after the parallel reads complete (mirrors
184
- // commit's "emit after the work" ordering). The array is shared across the
185
- // parallel map closures below safe because each push happens synchronously
186
- // between awaits (single-threaded), never concurrently.
187
- const promotions: { collectionId: CollectionId, blockId: BlockId, actionId: ActionId, rev: number }[] = [];
188
- const results = await Promise.all(distinctBlockIds.map(async (blockId) => {
189
- const blockStorage = this.createBlockStorage(blockId);
190
- // Set when this node KNOWS its answer for the block is a guess: the promotion
191
- // below refused for a missing base, or getBlock() threw (truncated history /
192
- // failed restore). An absent-reading block then reports `unavailable` instead of
193
- // posing as an authoritative "never existed" — see BlockUnavailableReason.
194
- let unavailable: BlockUnavailableReason | undefined;
195
-
196
- // Ensure that all outstanding transactions in the context are committed.
197
- // This promotes a landed-elsewhere pending via internalCommit, which mutates
198
- // meta.latest — the same read-modify-write commit()/saveReplicatedBlock guard
199
- // with the per-block commit latch. It MUST hold that latch too, or a promotion
200
- // racing a concurrent commit on the block regresses latest non-monotonically /
201
- // cross-writes a revision. Cheap unlatched pre-scan first so the common
202
- // contextless read and no-pending read never pay for latch acquisition; the
203
- // authoritative decision is re-made inside the latch.
204
- if (context) {
205
- const preLatest = await blockStorage.getLatest();
206
- const preMissing = preLatest
207
- ? context.committed.filter(c => c.rev > preLatest.rev)
208
- : context.committed;
209
- if (preMissing.length > 0) {
210
- await withBlockCommitLatch(blockId, async () => {
211
- // Re-read authoritative state under the latch: a concurrent commit may have
212
- // promoted or superseded a pending between the unlatched pre-scan and here.
213
- // Recompute which committed entries are still ahead of `latest` (drops the
214
- // superseded, rev <= latest.rev) and re-fetch each pending inside the loop
215
- // (skips the already-promoted, pending gone). This makes read-driven
216
- // promotion idempotent under races, mirroring commit()'s alreadyDone/stale
217
- // partitioning.
218
- const latest = await blockStorage.getLatest();
219
- const missing = latest
220
- ? context.committed.filter(c => c.rev > latest.rev)
221
- : context.committed;
222
- // Sort a COPY: when `latest` is undefined, `missing` aliases the caller's
223
- // `context.committed` array, and an in-place `.sort()` would reorder the shared
224
- // request context under the caller's feet.
225
- try {
226
- for (const { actionId, rev } of [...missing].sort((a, b) => a.rev - b.rev)) {
227
- const pending = await blockStorage.getPendingTransaction(actionId);
228
- if (pending) {
229
- const collectionId = await this.internalCommit(blockId, actionId, rev, blockStorage);
230
- if (collectionId !== undefined) {
231
- promotions.push({ collectionId, blockId, actionId, rev });
232
- }
233
- }
234
- }
235
- } catch (err) {
236
- // This node holds no materializable base for the block, so NO context revision
237
- // can be promoted here (each builds on the one before). Leave `latest` where it
238
- // is the invariant internalCommit just enforced and let the commit-path
239
- // healing supply the content; a read must not fail for it. Every other fault
240
- // still propagates.
241
- if (!(err instanceof MissingBaseRevisionError)) {
242
- throw err;
243
- }
244
- // This node holds records PROVING the block exists (a pending it could not
245
- // promote); if the block then reads as absent below, the answer is a guess,
246
- // not an authoritative "never existed".
247
- unavailable = 'unmaterializable';
248
- log('get:promote-skipped-missing-base blockId=%s rev=%d reason=%s', blockId, err.rev, err.message);
249
- }
250
- });
251
- }
252
- }
253
-
254
- // NOTE: a Crash-D3 block (durably promoted + revision saved, but the setLatest lost so
255
- // meta.latest is stale and the pending record is gone) reads as empty/stale here — a
256
- // context-driven get skips promotion (pending gone) and a default getBlock() sees the
257
- // stale latest. It is soft-wedged (stale), not hard-wedged: the next commit-retry for
258
- // (actionId, rev) self-heals it via storage.recover() in commit(). Not repaired lazily on
259
- // the read path because get() holds no commit latch; if stale reads on unwritten blocks
260
- // ever become a problem, add a latched lazy recover() here.
261
- //
262
- // getBlock() THROWS when this node holds a `latest` it cannot materialize (truncated
263
- // history: "Failed to find materialized block", or a failed restore). Caught PER BLOCK so
264
- // one broken block cannot fail the whole batch's Promise.all and take healthy siblings
265
- // down with it. The read still fails for THIS block — TransactorSource throws
266
- // BlockUnavailableError on the flagged entry — so nothing is swallowed.
267
- let blockRev: Awaited<ReturnType<IBlockStorage['getBlock']>>;
268
- try {
269
- blockRev = await blockStorage.getBlock(context?.rev);
270
- } catch (err) {
271
- // NOTE: the entry drops `state.latest`, which this node does know (getLatest() does not
272
- // materialize, so it does not throw). Empty state is what makes CoordinatorRepo treat the
273
- // block as missing and consult the cohort exactly the repair this block needs. If a
274
- // consumer ever needs the revision behind an unavailable answer (e.g. to ask the cohort
275
- // for a specific rev instead of the whole block), carry `latest` here and widen the
276
- // coordinator's consult trigger to `isMissing || unavailable` so repair still fires.
277
- log('get:unmaterializable blockId=%s error=%s', blockId,
278
- err instanceof Error ? err.message : String(err));
279
- return [blockId, { state: {}, unavailable: 'unmaterializable' } as GetBlockResult];
280
- }
281
-
282
- // Include pending action if requested, applying the pending transform over whatever
283
- // committed base getBlock() resolved (possibly none — a pending-only insert has no
284
- // committed revision under it and getBlock reports that as an absent base, not a fault).
285
- if (context?.actionId !== undefined) {
286
- const pendingTransform = await blockStorage.getPendingTransaction(context.actionId);
287
- if (!pendingTransform) {
288
- if (unavailable !== undefined) {
289
- // The promotion refusal above deleted this very pending record
290
- // (`refuseMissingBase` drops the pending it cannot promote). This node DID hold
291
- // the record and dropped it, so the honest answer is an availability one — not
292
- // a caller-contract violation, and never a throw that would fail the whole batch.
293
- return [blockId, { state: {}, unavailable } as GetBlockResult];
294
- }
295
- // Caller-contract violation (the caller asserted a pending this repo never had, or
296
- // cancelled) an error, not an availability question. Deliberately NOT `unavailable`.
297
- //
298
- // It is NOT the only way to reach here. A context that both PROVES its own action
299
- // (`committed` names it) and names it as the pending overlay (`actionId`) is
300
- // self-contradictory, and the two halves of that contradiction land differently: if
301
- // the read-driven promotion above REFUSED, the arm above answers gracefully; if it
302
- // SUCCEEDED, `promotePendingTransaction` moved the record and we throw here — failing
303
- // the whole batch for a request the refusal path tolerates. No production code sets
304
- // `ActionContext.actionId` at all today, so neither is reachable except from tests or
305
- // a peer that crafts the field on the wire. See
306
- // tickets/blocked/repo-pending-overlay-has-no-producer.
307
- throw new Error(`Pending action ${context.actionId} not found`);
308
- }
309
- const block = applyTransform(blockRev?.block, pendingTransform);
310
- return [blockId, {
311
- block,
312
- state: {
313
- latest: await blockStorage.getLatest(),
314
- pendings: [context.actionId]
315
- },
316
- // The COMMITTED revision underneath the pending overlay. A pending has no revision
317
- // of its own, so the honest answer is the base it was applied to. Absent when there
318
- // was no base at all a pending-only insert served over an absent committed base,
319
- // where fabricating a revision would claim content this node never committed.
320
- ...(blockRev ? { materializedRev: blockRev.actionRev.rev } : {}),
321
- // A pending applied to a missing base can materialize nothing (applyTransform drops
322
- // updates with no block to apply them to) that absence is a guess, and is flagged.
323
- // A materialized block is a real answer regardless of the earlier refusal. TWO ways
324
- // an empty result is a guess: the promotion refusal fired (`unavailable` set), or
325
- // there was no committed base under the overlay at all (`blockRev === undefined`)
326
- // this node holds a pending record PROVING the block exists and produced nothing.
327
- // The second clause's ABSENCE in the other direction is equally load-bearing: a
328
- // pending DELETE over a real committed base also lands here with no block, and that
329
- // is an authoritative tombstone which must stay unflagged.
330
- ...(block === undefined && (unavailable !== undefined || blockRev === undefined)
331
- ? { unavailable: unavailable ?? 'unmaterializable' }
332
- : {})
333
- } as GetBlockResult];
334
- }
335
-
336
- if (!blockRev) {
337
- // `unavailable` distinguishes "never existed" (the common insert-probe case, no flag)
338
- // from "this node cannot reconstruct it" (the promotion above refused for a missing
339
- // base). A tombstoned block also lands here with meta.latest set, but it never enters
340
- // the missing-base catch, so it stays an authoritative absent — keyed off the explicit
341
- // flag, not off "no block".
342
- return [blockId, { state: {}, ...(unavailable !== undefined ? { unavailable } : {}) } as GetBlockResult];
343
- }
344
-
345
- const pendings = await asyncIteratorToArray(blockStorage.listPendingTransactions());
346
- return [blockId, {
347
- block: blockRev.block,
348
- // `getBlock(context?.rev)` materialized the content at the highest committed revision
349
- // at or below the pin, and reports it as `actionRev` report THAT alongside the
350
- // content. `state.latest` deliberately stays the node's newest revision for the block
351
- // (StorageRepo.get's own promotion pre-scan and CoordinatorRepo's read-repair compare
352
- // against it), so the two disagree exactly when a pinned read is serving older content.
353
- materializedRev: blockRev.actionRev.rev,
354
- state: {
355
- latest: await blockStorage.getLatest(),
356
- pendings
357
- }
358
- }];
359
- }));
360
-
361
- // Emit per durable read-driven landing (Option A emit eagerly). Done after the
362
- // parallel reads complete so emission stays outside the per-block work, matching
363
- // commit's ordering. No-op when nothing was promoted.
364
- this.emitPromotions(promotions);
365
-
366
- return Object.fromEntries(results);
367
- }
368
-
369
- /**
370
- * Emit a {@link CollectionChangeEvent} for each read-driven promotion that landed
371
- * during a {@link get}. A single get() can promote multiple distinct actions, each
372
- * at its own `(actionId, rev)`, so group by `(actionId, rev)` and route each group
373
- * through {@link emitCollectionChanges} once.
374
- */
375
- private emitPromotions(promotions: { collectionId: CollectionId, blockId: BlockId, actionId: ActionId, rev: number }[]): void {
376
- if (promotions.length === 0) {
377
- return;
378
- }
379
- const groups = new Map<string, { actionId: ActionId, rev: number, collectionBlocks: Map<CollectionId, BlockId[]> }>();
380
- for (const { collectionId, blockId, actionId, rev } of promotions) {
381
- const key = `${actionId} ${rev}`;
382
- let group = groups.get(key);
383
- if (!group) {
384
- group = { actionId, rev, collectionBlocks: new Map() };
385
- groups.set(key, group);
386
- }
387
- const list = group.collectionBlocks.get(collectionId) ?? [];
388
- list.push(blockId);
389
- group.collectionBlocks.set(collectionId, list);
390
- }
391
- for (const { actionId, rev, collectionBlocks } of groups.values()) {
392
- this.emitCollectionChanges(collectionBlocks, actionId, rev);
393
- }
394
- }
395
-
396
- async pend(request: PendRequest, _options?: MessageOptions): Promise<PendResult> {
397
- // Validate transaction if present and validation hook is configured
398
- if (this.validatePend && request.transaction && request.operationsHash) {
399
- const validationResult = await this.validatePend(request.transaction, request.operationsHash);
400
- if (!validationResult.valid) {
401
- // Hard rejection: the transaction itself is invalid, so no `conflict` flag a
402
- // re-read and re-pend would fail the same way and only burn the retry budget.
403
- return {
404
- success: false,
405
- reason: validationResult.reason ?? 'Transaction validation failed'
406
- };
407
- }
408
- }
409
-
410
- const blockIds = blockIdsForTransforms(request.transforms);
411
- log('pend actionId=%s blockIds=%d rev=%s', request.actionId, blockIds.length, request.rev);
412
- const pendings: ActionPending[] = [];
413
- const missing: ActionTransforms[] = [];
414
- // Highest revision this node confirms holding among the blocks that are at or past the
415
- // requested one — reported as StaleFailure.staleAt so a losing writer learns the number
416
- // instead of parsing prose. Confirmed-local only: we read it from our own storage below.
417
- let staleAt: StaleFailure['staleAt'];
418
-
419
- // Potential race condition: A concurrent commit operation could complete
420
- // between the conflict checks (latest.rev, listPendingTransactions) and the
421
- // savePendingTransaction call below. This pend operation might succeed based on
422
- // stale information, but the subsequent commit for this pend would likely
423
- // fail correctly later if a conflict arose. Locking here could make the initial
424
- // check more accurate but adds overhead. The current approach prioritizes
425
- // letting the commit be the final arbiter.
426
- for (const blockId of blockIds) {
427
- const blockStorage = this.createBlockStorage(blockId);
428
- const transforms = transformForBlockId(request.transforms, blockId);
429
-
430
- // First handle any pending actions
431
- const pending = await asyncIteratorToArray(blockStorage.listPendingTransactions());
432
- pendings.push(...pending.map(actionId => ({ blockId, actionId })));
433
-
434
- // Handle any conflicting revisions
435
- if (request.rev !== undefined || transforms.insert) {
436
- const latest = await blockStorage.getLatest();
437
- if (latest && latest.rev >= (request.rev ?? 0)) {
438
- // Only a real revision race yields a meaningful `staleAt`. When `request.rev` is
439
- // undefined this same branch fires for an insert collision (the comparison degrades
440
- // to `latest.rev >= 0`, true for any existing block), and reporting that block's
441
- // revision would be a number that answers a question nobody asked.
442
- if (request.rev !== undefined) {
443
- staleAt = highestStaleAt([staleAt, { blockId, rev: latest.rev }]);
444
- }
445
- const transforms = await asyncIteratorToArray(blockStorage.listRevisions(request.rev ?? 0, latest.rev));
446
- for (const actionRev of transforms) {
447
- const transform = await blockStorage.getTransaction(actionRev.actionId);
448
- if (!transform) {
449
- throw new Error(`Missing action ${actionRev.actionId} for block ${blockId}`);
450
- }
451
- missing.push({
452
- actionId: actionRev.actionId,
453
- rev: actionRev.rev,
454
- transforms: transformsFromTransform(transform, blockId)
455
- });
456
- }
457
- }
458
- }
459
- }
460
-
461
- if (missing.length) {
462
- log('pend:stale actionId=%s missing=%d', request.actionId, missing.length);
463
- return {
464
- success: false,
465
- conflict: true,
466
- missing,
467
- ...(staleAt === undefined ? {} : { staleAt })
468
- };
469
- }
470
-
471
- if (pendings.length > 0) {
472
- if (request.policy === 'f') { // Fail on pending actions
473
- return { success: false, conflict: true, pending: pendings };
474
- } else if (request.policy === 'r') { // Return populated pending actions
475
- return {
476
- success: false,
477
- conflict: true,
478
- pending: await Promise.all(pendings.map(async action => {
479
- const blockStorage = this.createBlockStorage(action.blockId);
480
- return {
481
- blockId: action.blockId,
482
- actionId: action.actionId,
483
- transform: (await blockStorage.getPendingTransaction(action.actionId))
484
- ?? (await blockStorage.getTransaction(action.actionId))! // Possible that since enumeration, the action has been promoted
485
- }
486
- }))
487
- };
488
- }
489
- }
490
-
491
-
492
- // Simultaneously save pending action for each block
493
- // Note: that this is not atomic, after we checked for conflicts and pending actions
494
- // new pending or committed actions may have been added. This is okay, because
495
- // this check during pend is conservative.
496
- await Promise.all(blockIds.map(blockId => {
497
- const blockStorage = this.createBlockStorage(blockId);
498
- const blockTransform = transformForBlockId(request.transforms, blockId);
499
- return blockStorage.savePendingTransaction(request.actionId, blockTransform);
500
- }));
501
-
502
- return {
503
- success: true,
504
- pending: pendings,
505
- blockIds
506
- } as PendSuccess;
507
- }
508
-
509
- async cancel(actionRef: ActionBlocks, _options?: MessageOptions): Promise<void> {
510
- log('cancel actionId=%s blockIds=%d', actionRef.actionId, actionRef.blockIds.length);
511
- await Promise.all(actionRef.blockIds.map(blockId => {
512
- const blockStorage = this.createBlockStorage(blockId);
513
- return blockStorage.deletePendingTransaction(actionRef.actionId);
514
- }));
515
- }
516
-
517
- /**
518
- * Commit a previously-pended action across its blocks, under the per-block commit latches.
519
- *
520
- * **Divergence vs genuine fault.** When the batch cannot be completed, the reason decides what
521
- * happens to the pending records the pend left behind. `ClusterMember.applyConsensusOperation`
522
- * makes the same split one layer up — it *tolerates* a divergence (and reconciles every
523
- * `commit.blockIds` entry from a cohort peer) but *propagates* a genuine fault for retry — so this
524
- * method must agree with it:
525
- *
526
- * - **Divergence** this node is behind the agreed history, either because it holds no
527
- * materializable base ({@link MissingBaseRevisionError}) or because it never received the pend
528
- * (the `Pending action not found` throw). Reconcile is guaranteed to follow and will advance
529
- * every block in the batch past `request.rev`, so no pending record here can ever be promoted:
530
- * {@link dropUnpromotablePendings} deletes them (see {@link refuseMissingBase}, which already
531
- * accepts this tradeoff for the single refusing block).
532
- * - **Genuine fault** any other throw out of {@link internalCommit} (a raw-storage error, …).
533
- * `ClusterMember` propagates it and the commit is retried, and a retry can still replay the
534
- * pendings, so they are KEPT.
535
- *
536
- * The stale/`missedCommits` early return (this node is AHEAD — it already holds a revision at or
537
- * past `request.rev`, committed under a different action) deliberately keeps pendings too, and its
538
- * cure is the losing client's `cancel`: `CoordinatorRepo.cancel` runs through consensus, so every
539
- * member drops the record, not just the coordinator. Replication cannot be the cure here — this
540
- * node is already ahead, and a later forward write carries a DIFFERENT action id, which is not
541
- * what `BlockStorage.saveForwardRevision` deletes. A client that dies between the stale result and
542
- * its `cancel` therefore still strands the record; that is pre-existing and orthogonal to the
543
- * divergence split above.
544
- */
545
- async commit(request: CommitRequest, _options?: MessageOptions): Promise<CommitResult> {
546
- log('commit actionId=%s rev=%d blockIds=%d', request.actionId, request.rev, request.blockIds.length);
547
- const uniqueBlockIds = Array.from(new Set(request.blockIds)).sort();
548
- const releases: (() => void)[] = [];
549
- // Collects the blocks newly committed in this call, grouped by collection,
550
- // so we can emit change events once locks are released. Blocks that land before
551
- // a mid-loop failure stay here and are still emitted (Option A emit eagerly):
552
- // they are durably committed and a retry rolls the remainder forward.
553
- const collectionBlocks = new Map<CollectionId, BlockId[]>();
554
- // Captured when internalCommit throws mid-loop; we break (rather than return)
555
- // so locks release and accumulated landings still emit before we report failure.
556
- let failure: { reason: string } | undefined;
557
-
558
- try {
559
- // Acquire locks sequentially based on sorted IDs to prevent deadlocks
560
- for (const id of uniqueBlockIds) {
561
- const lockId = commitLatchKey(id);
562
- const release = await Latches.acquire(lockId);
563
- releases.push(release);
564
- }
565
-
566
- // --- Start of Critical Section ---
567
-
568
- const blockStorages = request.blockIds.map(blockId => ({
569
- blockId,
570
- storage: this.createBlockStorage(blockId)
571
- }));
572
-
573
- // Partition blocks into:
574
- // - alreadyDone: latest.rev === request.rev && latest.actionId === request.actionId
575
- // (idempotent retry — a prior commit of this same action already landed here;
576
- // skip rather than treat as a conflict. Needed to rollforward stranded blocks
577
- // after a mid-batch crash committed some but not all blocks.)
578
- // - missedCommits: latest.rev >= request.rev but not the same actionId → real stale conflict.
579
- // - toCommit: latest.rev < request.rev or no latest yet → run internalCommit.
580
- const toCommit: { blockId: BlockId, storage: IBlockStorage }[] = [];
581
- const missedCommits: { blockId: BlockId, transforms: ActionTransform[] }[] = [];
582
- // Highest revision among the blocks confirmed lost to a newer one — reported as
583
- // StaleFailure.staleAt. The idempotent-retry `continue` below is a no-op, not a loss,
584
- // so it never seeds this.
585
- let staleAt: StaleFailure['staleAt'];
586
- for (const entry of blockStorages) {
587
- const { blockId, storage } = entry;
588
- const latest = await storage.getLatest();
589
- if (latest && latest.rev >= request.rev) {
590
- if (latest.rev === request.rev && latest.actionId === request.actionId) {
591
- // Idempotent no-op for this block — already committed with this exact (actionId, rev).
592
- continue;
593
- }
594
- staleAt = highestStaleAt([staleAt, { blockId, rev: latest.rev }]);
595
- const transforms: ActionTransform[] = [];
596
- for await (const actionRev of storage.listRevisions(request.rev, latest.rev)) {
597
- const transform = await storage.getTransaction(actionRev.actionId);
598
- if (!transform) {
599
- throw new Error(`Missing action ${actionRev.actionId} for block ${blockId}`);
600
- }
601
- transforms.push({
602
- actionId: actionRev.actionId,
603
- rev: actionRev.rev,
604
- transform
605
- });
606
- }
607
- missedCommits.push({ blockId, transforms }); // Push, even if transforms is empty, because we want to reject the older version
608
- continue;
609
- }
610
- toCommit.push(entry);
611
- }
612
-
613
- if (missedCommits.length) {
614
- log('commit:stale actionId=%s missed=%d', request.actionId, missedCommits.length);
615
- return { // Return directly, locks will be released in finally
616
- success: false,
617
- missing: perBlockActionTransformsToPerAction(missedCommits),
618
- ...(staleAt === undefined ? {} : { staleAt })
619
- };
620
- }
621
-
622
- // Check for missing pending actions only on blocks that still need to commit.
623
- // Already-done blocks will have had their pending promoted, so skipping them here
624
- // is what makes the idempotent rollforward work.
625
- //
626
- // A toCommit block whose pending is absent is one of two states:
627
- // - Crash-D3: the action was durably promoted and its revision saved, but the crash
628
- // lost the setLatest, so meta.latest is still < request.rev and the pending record
629
- // is gone. getTransaction(actionId) returns the promoted transform. Self-heal here
630
- // via storage.recover() (redoes the lost setLatest, advancing latest to the highest
631
- // contiguous promoted rev, >= request.rev). recover() is idempotent + monotonic, so
632
- // calling it under the already-held commit latch is safe. Recovered blocks are then
633
- // excluded from the internalCommit loop below — their pending is gone, so
634
- // internalCommit would throw.
635
- // - Genuine missing pend: the action was never promoted (getTransaction undefined),
636
- // so the pend is truly missing. Throw exactly as before.
637
- // Crash-D2 never reaches this branch: its pending record is still present.
638
- const missingPends: { blockId: BlockId, actionId: ActionId }[] = [];
639
- const recovered = new Set<BlockId>();
640
- for (const { blockId, storage } of toCommit) {
641
- const pendingAction = await storage.getPendingTransaction(request.actionId);
642
- if (pendingAction) {
643
- continue;
644
- }
645
- const promoted = await storage.getTransaction(request.actionId);
646
- if (!promoted) {
647
- missingPends.push({ blockId, actionId: request.actionId });
648
- continue;
649
- }
650
- // Crash-D3 signature (pending absent + action durably promoted). Redo the lost setLatest.
651
- const result = await storage.recover();
652
- if (result.latest !== undefined && result.latest.rev >= request.rev) {
653
- recovered.add(blockId);
654
- } else {
655
- // Torn/partial state: recover() could not advance latest to request.rev (metadata
656
- // absent, or a revision entry missing despite the promoted transaction). Fall back
657
- // to treating the block as a genuine missing-pend error rather than silently succeeding.
658
- missingPends.push({ blockId, actionId: request.actionId });
659
- }
660
- }
661
-
662
- // NOTE: if a batch ever held BOTH a recovered D3 block and a genuine missing-pend block,
663
- // this throw fires after recover() already advanced the D3 block durably, so that block's
664
- // change event is skipped (the retry then treats it as alreadyDone and never re-emits;
665
- // durable state stays correct, only the emit is lost). Judged unreachable today: a single
666
- // crash mid-internalCommit leaves exactly one D3 block, with the rest alreadyDone or
667
- // pending-present a never-pended block cannot coexist with it in one retry. If a path
668
- // ever produces that mix, emit recovered blocks' events before throwing here.
669
- if (missingPends.length) {
670
- // Divergence (this node is behind): `ClusterMember` treats this throw as the canonical
671
- // "behind" signal and reconciles EVERY block in the batch, advancing each past
672
- // `request.rev`. Nothing can promote the pendings the other blocks still hold, so drop
673
- // them here while the latches are still held before reporting. The thrown message
674
- // must stay byte-identical: `ClusterMember.isMissingPendingActionError` matches on it.
675
- await this.dropUnpromotablePendings(toCommit, request.actionId);
676
- throw new Error(`Pending action ${request.actionId} not found for block(s): ${missingPends.map(p => p.blockId).join(', ')}`);
677
- }
678
-
679
- // The original commit crashed before setLatest, so it also never emitted a change event
680
- // for a recovered (Crash-D3) block. Now that recover() has committed it at request.rev,
681
- // report its collection so downstream watchers wakemirroring internalCommit. Resolve
682
- // the collectionId from the now-materialized block; a delete materializes to a tombstone
683
- // (getBlock undefined), so fall back to the prior materialized block's header exactly as
684
- // internalCommit does otherwise a recovered delete would silently fail to wake watchers.
685
- // Only when neither resolves (a delete-only block with no prior materialization) is the
686
- // emit skipped, the same terminal fallback internalCommit uses.
687
- for (const { blockId, storage } of toCommit) {
688
- if (!recovered.has(blockId)) {
689
- continue;
690
- }
691
- const collectionId = (await storage.getBlock(request.rev))?.block.header.collectionId
692
- ?? (await storage.getBlock(request.rev - 1))?.block.header.collectionId;
693
- if (collectionId !== undefined) {
694
- const list = collectionBlocks.get(collectionId) ?? [];
695
- list.push(blockId);
696
- collectionBlocks.set(collectionId, list);
697
- }
698
- }
699
-
700
- // Commit the action for each block that still needs it.
701
- // This loop will execute atomically for all blocks due to the acquired locks.
702
- // Recovered (Crash-D3) blocks are already committed at request.rev and their pending is
703
- // gone, so skip them internalCommit would throw on the missing pending record.
704
- //
705
- // Set when the mid-loop failure was a divergence rather than a genuine fault the split
706
- // documented on commit() above, which decides the fate of the batch's pending records.
707
- let divergentFailure = false;
708
- for (const { blockId, storage } of toCommit) {
709
- if (recovered.has(blockId)) {
710
- continue;
711
- }
712
- try {
713
- // internalCommit will throw if it encounters an issue
714
- const collectionId = await this.internalCommit(blockId, request.actionId, request.rev, storage);
715
- if (collectionId !== undefined) {
716
- const list = collectionBlocks.get(collectionId) ?? [];
717
- list.push(blockId);
718
- collectionBlocks.set(collectionId, list);
719
- }
720
- } catch (err) {
721
- // Partial-commit recovery: blocks already in collectionBlocks DID land
722
- // durably and must still emit; a retry with the same (actionId, rev)
723
- // treats them as idempotent no-ops and advances the remainder. Break
724
- // instead of returning so locks release and those landings emit below.
725
- failure = { reason: err instanceof Error ? err.message : 'Unknown error during commit' };
726
- divergentFailure = err instanceof MissingBaseRevisionError;
727
- break;
728
- }
729
- }
730
-
731
- // The break left every not-yet-reached block still holding its pending record. Whether
732
- // that record is still usable depends ENTIRELY on why we stopped — see the table on
733
- // commit() above. Runs inside the try, so the per-block latches are still held.
734
- // NOTE: a non-divergence fault deliberately KEEPS the batch's pendings so a retry can
735
- // replay them. If ClusterMember ever stops retrying propagated commit faults, this arm
736
- // becomes dead weight and the discriminator can collapse to "always drop".
737
- if (divergentFailure) {
738
- await this.dropUnpromotablePendings(toCommit, request.actionId);
739
- }
740
- }
741
- finally {
742
- // Release locks in reverse order of acquisition
743
- releases.reverse().forEach(release => release());
744
- }
745
-
746
- // Notify after the critical section, for every block newly committed here —
747
- // including those that landed before a mid-loop failure (alreadyDone / stale
748
- // partitions never reach `collectionBlocks`).
749
- this.emitCollectionChanges(collectionBlocks, request.actionId, request.rev, request.tailId);
750
-
751
- return failure ? { success: false, reason: failure.reason } : { success: true };
752
- }
753
-
754
- /**
755
- * Delete `actionId`'s pending record from every given block, tolerating absence.
756
- *
757
- * Called by {@link commit} when it abandons a batch **because this node has diverged from the
758
- * agreed history** — the caller has already made that determination; this helper does not
759
- * re-derive it. Once `ClusterMember` reconciles the batch, every one of these blocks sits at or
760
- * past `request.rev`, so a commit retry partitions them as already-done/stale and never revisits
761
- * their pendings; left in place they are reported as phantom conflicting actions by {@link pend}
762
- * for every later write to the block (under `policy: 'f'`, forever).
763
- *
764
- * No special-casing is needed for blocks that already landed (record promoted), that were
765
- * `recovered` (record already gone), or for the refusing block itself
766
- * ({@link refuseMissingBase} deleted its record): deleting an absent pending record is a no-op on
767
- * every backend.
768
- *
769
- * Per-block failures are logged and swallowed rather than propagated: this cleanup must never
770
- * replace the failure the caller is about to report — the pre-loop throw's message is pattern-
771
- * matched by `ClusterMember.isMissingPendingActionError`, and a swapped error would misroute
772
- * consensus. A leftover record only degrades this node's participation in that one block.
773
- */
774
- private async dropUnpromotablePendings(
775
- blocks: { blockId: BlockId, storage: IBlockStorage }[],
776
- actionId: ActionId
777
- ): Promise<void> {
778
- if (blocks.length === 0) {
779
- return;
780
- }
781
- log('commit:drop-unpromotable-pendings actionId=%s blockIds=%d', actionId, blocks.length);
782
- await Promise.all(blocks.map(async ({ blockId, storage }) => {
783
- try {
784
- await storage.deletePendingTransaction(actionId);
785
- } catch (err) {
786
- log('commit:drop-unpromotable-pending-failed blockId=%s actionId=%s error=%s', blockId, actionId,
787
- err instanceof Error ? err.message : String(err));
788
- }
789
- }));
790
- }
791
-
792
- /**
793
- * Reconciles `metadata.latest` for a single block with the highest contiguous
794
- * fully-promoted revision in durable storage. Use after a crash between
795
- * `promotePendingTransaction` and `setLatest` when retry-commit cannot help
796
- * (the pending record is already gone) but the revision and committed-log entry
797
- * are durable. Idempotent and monotonic.
798
- */
799
- async recoverBlock(blockId: BlockId): Promise<void> {
800
- log('recoverBlock blockId=%s', blockId);
801
- const storage = this.createBlockStorage(blockId);
802
- // Hold the per-block commit latch: recover() is a read-modify-write of meta.latest that
803
- // blindly writes back the metadata object it read, so its "advance only" guard is TOCTOU —
804
- // racing a concurrent commit()/saveReplicatedBlock that advanced latest in between would
805
- // clobber it (a non-monotonic regression). Same latching invariant as every other
806
- // latest-mutating site. commit() calls storage.recover() directly under its own held latch,
807
- // so it never routes through here no double-acquire / deadlock.
808
- await withBlockCommitLatch(blockId, () => storage.recover());
809
- }
810
-
811
- /**
812
- * Persist a replica of a block received out-of-band (churn re-replication) into
813
- * local storage. Distinct from the {@link IRepo} commit funnel: the block arrives
814
- * already materialized from a departing owner, not as a pend/commit. See
815
- * {@link IBlockStorage.saveReplica} for the durability/monotonicity contract.
816
- *
817
- * Held under the same `StorageRepo.commit:<id>` latch as {@link commit} so the
818
- * replica's read-modify-write of `latest` is mutually exclusive with a concurrent
819
- * local commit on the same block — otherwise `saveReplica`'s monotonic guard could
820
- * read a stale `latest` and clobber a commit that advanced it in between.
821
- */
822
- async saveReplicatedBlock(blockId: BlockId, block: IBlock, source?: ActionRev): Promise<void> {
823
- log('saveReplicatedBlock blockId=%s rev=%s', blockId, source?.rev);
824
- const storage = this.createBlockStorage(blockId);
825
- const release = await Latches.acquire(commitLatchKey(blockId));
826
- // Captured under the latch; emitted after release to match commit's ordering.
827
- let landed: { collectionId: CollectionId, actionId: ActionId, rev: number } | undefined;
828
- try {
829
- const priorLatest = await storage.getLatest();
830
- const effective = await storage.saveReplica(block, source);
831
- // Advanced iff there was no prior revision or the effective rev moved past it. On the
832
- // monotonic no-op, saveReplica returns the held latest unchanged effective.rev === priorLatest.rev.
833
- const advanced = priorLatest === undefined || effective.rev > priorLatest.rev;
834
- const collectionId = block.header?.collectionId;
835
- if (advanced && collectionId !== undefined) {
836
- landed = { collectionId, actionId: effective.actionId, rev: effective.rev };
837
- }
838
- } finally {
839
- release();
840
- }
841
- // Replica-persist has no CommitRequest, hence no tailId — like a read-driven promotion,
842
- // this wakes local onCollectionChange watchers but is cert-gated out of cohort-topic
843
- // re-origination downstream (change-bridge selfIsCohortMember treats a tail-less event as
844
- // never a member).
845
- if (landed) {
846
- this.emitCollectionChanges(
847
- new Map([[landed.collectionId, [blockId]]]),
848
- landed.actionId,
849
- landed.rev,
850
- );
851
- }
852
- }
853
-
854
- private async internalCommit(blockId: BlockId, actionId: ActionId, rev: number, storage: IBlockStorage): Promise<CollectionId | undefined> {
855
- // Note: This method is called under the per-block commit latch by commit() (within its
856
- // locked critical section) and by the read-driven promotion in get() (which now takes the
857
- // same latch). So, operations like getPendingTransaction, getLatest, getBlock,
858
- // saveMaterializedBlock, saveRevision, promotePendingTransaction, setLatest are protected
859
- // against concurrent commits for the *same blockId*.
860
-
861
- const transform = await storage.getPendingTransaction(actionId);
862
- // No need to check if !transform here, as the caller (commit) already verified this.
863
- // If it's null here, it indicates a logic error or race condition bypassed the lock (unlikely).
864
- if (!transform) {
865
- throw new Error(`Consistency Error: Pending action ${actionId} disappeared for block ${blockId} within critical section.`);
866
- }
867
-
868
- // Get prior materialized block if it exists
869
- const latest = await storage.getLatest();
870
- const priorBlock = await this.readCommitBase(blockId, actionId, rev, storage, latest);
871
-
872
- // Apply transform and save materialized block
873
- // applyTransform handles undefined priorBlock correctly for inserts
874
- const newBlock = applyTransform(priorBlock, transform);
875
-
876
- // INVARIANT: `latest` must never advance past a revision this node can materialize.
877
- // `applyTransform` silently drops `updates` when there is no block to apply them to, so a
878
- // member that missed the block's CREATING revision would otherwise record rev N while storing
879
- // nothing to serve it from. `latest === undefined` is precisely the "nothing below to fall
880
- // back to" case: materializeBlock's descending walk needs some materialization at or below the
881
- // target, and with no prior revision there is none. With a prior `latest` an absent newBlock is
882
- // a legitimate tombstone (the walk resolves to an earlier materialization), so it stays allowed.
883
- if (!newBlock && latest === undefined) {
884
- return await this.refuseMissingBase(blockId, actionId, rev, storage,
885
- 'no committed revision to apply the transform to');
886
- }
887
-
888
- if (newBlock) {
889
- await storage.saveMaterializedBlock(actionId, newBlock);
890
- }
891
-
892
- // Save revision and promote action *before* updating latest
893
- // This ensures that if the process crashes between these steps,
894
- // the 'latest' pointer doesn't point to a revision that hasn't been fully recorded.
895
- await storage.saveRevision(rev, actionId);
896
- await storage.promotePendingTransaction(actionId);
897
-
898
- // Update latest revision *last*
899
- await storage.setLatest({ actionId, rev });
900
-
901
- // Prune the now-superseded prior materialization (checkpoint retention). Runs LAST after the
902
- // new rev's materialization + revision + transform + setLatest are all durable — so no crash
903
- // point can leave a rev unrecoverable: a crash BEFORE this leaves a redundant (harmless)
904
- // materialization the next commit's prune reclaims; a crash AFTER is fully consistent. The prune
905
- // only ever deletes a materialization reconstructible from the retained floor + transforms. Runs
906
- // under the per-block commit latch already held here, so it serializes against concurrent commits.
907
- // NOTE: prune targets ONLY the immediate prior. A crash between setLatest and this call leaves that
908
- // one prior materialization un-pruned; since a later commit prunes ITS OWN prior (never the earlier
909
- // leaked rev), that copy is NOT auto-reclaimed — a bounded (≤1 block-copy per crash), harmless leak
910
- // (state stays consistent + reconstructible). If crash-before-prune leaks ever accumulate materially,
911
- // add a bounded look-back (prune non-retained mats in [rev-checkpointInterval, rev)) here, or a
912
- // periodic reconciliation sweep do NOT reintroduce a per-read re-cache.
913
- if (latest !== undefined) {
914
- await storage.pruneSupersededMaterialization(latest);
915
- }
916
-
917
- // Report the affected collection for change-event routing. For a delete the
918
- // materialized block is undefined, so fall back to the prior block's header.
919
- // Either may be absent only for a malformed/headerless block — return
920
- // undefined so the caller skips it rather than emitting a bogus event.
921
- return newBlock?.header.collectionId ?? priorBlock?.header.collectionId;
922
- }
923
-
924
- /**
925
- * The materialization this commit builds on: the block at `latest`, or `undefined` when the block
926
- * holds no committed revision yet (the normal insert case).
927
- *
928
- * `getBlock` THROWS when this node holds a `latest` it cannot materialize — a block already wedged
929
- * by a pre-fix commit, or by truncated history. That is the same divergence as having no base at
930
- * all, so it is translated into {@link MissingBaseRevisionError} rather than surfacing as an opaque
931
- * storage fault: the healing path can then repair the block instead of the fault resetting the
932
- * cluster stream, and a wedged node recovers on the next write touching the block.
933
- *
934
- * The catch is deliberately UNNARROWED — it also absorbs a transient fault (a raw-storage read
935
- * error, a `restoreCallback` timeout on a block whose `ranges` do not cover its own `latest`).
936
- * BlockStorage reports every one of these as a bare `Error`, so they cannot be told apart here,
937
- * and treating them as divergence is the safe default: this node genuinely cannot materialize the
938
- * base right now, and the cluster's policy is to heal rather than throw out of consensus. The
939
- * price is that a transient fault ALSO drops pending records — this block's (see
940
- * {@link refuseMissingBase}) AND, because {@link commit} keys its cleanup off the same error type,
941
- * every not-yet-reached block in the same batch — so those blocks converge by replication instead
942
- * of by a replay the retry could have done. That is a wider blast radius than the per-block
943
- * refusal alone, and it is why the discriminator must NOT be loosened beyond this error type.
944
- * Narrowing this would require typed faults out of BlockStorage; until then, prefer the tolerant
945
- * reading.
946
- */
947
- private async readCommitBase(
948
- blockId: BlockId,
949
- actionId: ActionId,
950
- rev: number,
951
- storage: IBlockStorage,
952
- latest: ActionRev | undefined
953
- ): Promise<IBlock | undefined> {
954
- if (!latest) {
955
- return undefined;
956
- }
957
- try {
958
- return (await storage.getBlock(latest.rev))?.block;
959
- } catch (err) {
960
- log('commit:unmaterializable-base blockId=%s baseRev=%d error=%s', blockId, latest.rev,
961
- err instanceof Error ? err.message : String(err));
962
- return await this.refuseMissingBase(blockId, actionId, rev, storage,
963
- `local rev ${latest.rev} is not materializable here`);
964
- }
965
- }
966
-
967
- /**
968
- * Refuse a commit this node cannot materialize. Always throws {@link MissingBaseRevisionError};
969
- * nothing durable has been written at this point, so the block is left exactly as it was minus the
970
- * pending record.
971
- *
972
- * The pending is dropped because it can never be promoted here: promotion needs a base this node
973
- * must obtain out-of-band, and once the healing path lands that revision `latest` is already >= rev,
974
- * so a commit retry partitions the block as already-done/stale and never revisits the pending.
975
- * Leaving it would also report a phantom conflicting action from {@link pend} for every later write.
976
- */
977
- private async refuseMissingBase(
978
- blockId: BlockId,
979
- actionId: ActionId,
980
- rev: number,
981
- storage: IBlockStorage,
982
- detail: string
983
- ): Promise<never> {
984
- await storage.deletePendingTransaction(actionId);
985
- log('commit:missing-base blockId=%s rev=%d actionId=%s detail=%s', blockId, rev, actionId, detail);
986
- throw new MissingBaseRevisionError(blockId, rev, detail);
987
- }
988
- }
989
-
990
- /**
991
- * Converts list of missing actions per block into a list of missing actions across blocks.
992
- *
993
- * NOTE: relies on each (actionId, blockId) pair appearing at most once — one revision per action
994
- * per block. If a block ever records two revisions under the same actionId, concatTransform now
995
- * concatenates both revisions' ops into one array rather than dropping the earlier one — still
996
- * wrong, since ops from distinct revisions are not composable against a single base, but loud
997
- * rather than silent. Group by (actionId, rev) instead if that case becomes reachable.
998
- */
999
- function perBlockActionTransformsToPerAction(missing: { blockId: BlockId; transforms: ActionTransform[]; }[]) {
1000
- const missingFlat = missing.flatMap(({ blockId, transforms }) =>
1001
- transforms.map(transform => ({ blockId, transform }))
1002
- );
1003
- const missingByActionId = groupBy(missingFlat, ({ transform }) => transform.actionId);
1004
- return Object.entries(missingByActionId).map(([actionId, items]) =>
1005
- items.reduce((acc, { blockId, transform }) => {
1006
- acc.transforms = concatTransform(acc.transforms, blockId, transform.transform);
1007
- return acc;
1008
- }, {
1009
- actionId: actionId as ActionId,
1010
- rev: items[0]!.transform.rev, // Assumption: an action commits at one revision, so every block's entry for this actionId agrees. Distinct actionIds may still carry distinct revs.
1011
- transforms: emptyTransforms()
1012
- })
1013
- );
1014
- }
1
+ import type {
2
+ IRepo, MessageOptions, BlockId, CommitRequest, CommitResult, GetBlockResults, PendRequest, PendResult, ActionBlocks,
3
+ ActionId, BlockGets, ActionPending, PendSuccess, ActionTransform, ActionTransforms,
4
+ GetBlockResult, IBlock, ActionRev, BlockUnavailableReason,
5
+ PendValidationHook, UnvalidatablePendPolicy,
6
+ CollectionId, IBlockChangeNotifier, CollectionChangeListener, CollectionChangeEvent,
7
+ StaleFailure
8
+ } from "@optimystic/db-core";
9
+ import {
10
+ transformForBlockId, applyTransform, groupBy, concatTransform, emptyTransforms,
11
+ blockIdsForTransforms, transformsFromTransform, highestStaleAt, isOwnRevision, canonicalBlockHash
12
+ } from "@optimystic/db-core";
13
+ import { asyncIteratorToArray } from "../it-utility.js";
14
+ import type { IBlockStorage } from "./i-block-storage.js";
15
+ import type { IBlockReplicaStore } from "../cluster/block-transfer-service.js";
16
+ import { proofDeclaredDigest, type BlockCommitProof } from "../cluster/commit-proof.js";
17
+ import { RevisionNotCoveredError } from "./i-block-storage.js";
18
+ import { acquireBlockWriteLatches, withBlockWriteLatch, type BlockWriteLatch } from "./block-latch.js";
19
+ import { createLogger } from "../logger.js";
20
+ import { checkPendValidation } from "../pend-validation.js";
21
+
22
+ const log = createLogger('storage-repo');
23
+
24
+ /**
25
+ * Stable, greppable prefix on the failure reason a commit carries when this node cannot materialize
26
+ * the revision it was asked to record. It is a STRING marker rather than only an error class because
27
+ * {@link StorageRepo.commit} reports per-block faults as `StaleFailure.reason` (a plain string that
28
+ * also crosses the wire), so the class identity is lost by the time a caller inspects the result.
29
+ */
30
+ export const MISSING_BASE_REVISION_REASON = 'missing-base-revision';
31
+
32
+ /**
33
+ * The two stable reject-reason prefixes a validating receiver emits, re-exported here (and from
34
+ * `cluster/cluster-repo.ts`) next to their siblings so a caller inspecting a `PendResult` reason
35
+ * need not know which module defines them. Both tiers refuse with the same prefixes because both
36
+ * run the same {@link checkPendValidation}.
37
+ */
38
+ export { PEND_NOT_VALIDATABLE, VALIDATOR_FAULT } from "../pend-validation.js";
39
+
40
+ /**
41
+ * This node was asked to commit revision N of a block it holds no materializable base for, so
42
+ * applying the transform would materialize nothing while `latest` advanced to N — a block that is
43
+ * then unreadable locally, unservable to peers, and that rejects every later write (see
44
+ * {@link StorageRepo.internalCommit}). The commit is refused instead; the caller heals the node
45
+ * out-of-band (`ClusterMember` pulls the committed revision from a cohort peer) and retries.
46
+ */
47
+ export class MissingBaseRevisionError extends Error {
48
+ constructor(readonly blockId: BlockId, readonly rev: number, detail: string) {
49
+ super(`${MISSING_BASE_REVISION_REASON}: block ${blockId} cannot materialize rev ${rev} ${detail}`);
50
+ this.name = 'MissingBaseRevisionError';
51
+ }
52
+ }
53
+
54
+ /**
55
+ * True when a {@link CommitResult} failed because this node holds no materializable base for one of
56
+ * the committed blocks. Distinguishes that recoverable divergence (heal by fetching the block from a
57
+ * cohort peer) from a genuine storage fault, which must still propagate.
58
+ */
59
+ export function isMissingBaseRevisionFailure(result: CommitResult): boolean {
60
+ return !result.success && (result.reason?.startsWith(MISSING_BASE_REVISION_REASON) ?? false);
61
+ }
62
+
63
+ export type StorageRepoOptions = {
64
+ /** Optional hook to validate transactions in PendRequests */
65
+ validatePend?: PendValidationHook;
66
+ /**
67
+ * What this repo does — when a `validatePend` hook IS configured — with a pend that carries no
68
+ * `validation` payload and therefore nothing to re-check. Default 'accept'; see
69
+ * {@link UnvalidatablePendPolicy}. The cluster tier's mirror of this knob is
70
+ * `ClusterConsensusConfig.unvalidatablePendPolicy`, and both are enforced by the one
71
+ * `checkPendValidation`.
72
+ *
73
+ * NOTE: the two tiers are configured INDEPENDENTLY, so a node set to 'accept' at the cluster tier
74
+ * and 'reject' here would vote approve on a pend its own storage then refuses at apply — burning
75
+ * a consensus round to reach a verdict it already knew. Harmless today because no composition
76
+ * root supplies a checker at either tier (backlog
77
+ * `feat-no-deployment-validates-transactions-at-pend`); when one does, resolve both knobs from a
78
+ * single operator field rather than letting a deployment set them apart.
79
+ */
80
+ unvalidatablePendPolicy?: UnvalidatablePendPolicy;
81
+ };
82
+
83
+ /**
84
+ * What {@link StorageRepo.previewCommitDigest} predicts a commit would materialize. `digest` is the
85
+ * {@link canonicalBlockHash} of the materialized content, or `undefined` when the transform
86
+ * materializes nothing (a delete/tombstone, updates with no base to apply them to) or the base
87
+ * exists but cannot be materialized locally. `baseRev` is the local committed revision the preview
88
+ * was computed against (absent when there is none, or when the transform is base-independent and no
89
+ * base was read). `baseIndependent` is true when the pended transform carries an `insert`, making
90
+ * the result identical on every member regardless of what base it holds.
91
+ */
92
+ export type CommitDigestPreview = {
93
+ digest?: string;
94
+ baseRev?: number;
95
+ baseIndependent: boolean;
96
+ };
97
+
98
+ /**
99
+ * The capability {@link ClusterMember.validateCommitOperations} probes its `storageRepo` for. Named
100
+ * (rather than written inline at the probe) so there is ONE definition of the contract and so a repo
101
+ * decorator wrapping the member's storage seam has something to `implements` and forward — a wrapper
102
+ * that drops the method silently disables the commit content-digest check on that node.
103
+ */
104
+ export interface ICommitDigestPreviewer {
105
+ previewCommitDigest(blockId: BlockId, actionId: ActionId, rev: number): Promise<CommitDigestPreview | undefined>;
106
+ }
107
+
108
+ /**
109
+ * The capability `ClusterMember.applyConsensusOperation` casts its `storageRepo` to when handing a
110
+ * {@link BlockCommitProof} down the commit path. Named for the same reason as
111
+ * {@link ICommitDigestPreviewer}: one definition of the widened contract, and something a repo
112
+ * decorator can `implements` and forward. `IRepo.commit` takes two arguments; the third is
113
+ * harmless at runtime for a plain `IRepo` implementation (the extra argument is ignored), so
114
+ * callers cast rather than structurally probe — but a decorator that narrows back to `IRepo`
115
+ * silently stops persisting proofs on that node.
116
+ */
117
+ export interface ICommitProofPersister {
118
+ commit(request: CommitRequest, options?: MessageOptions, proof?: BlockCommitProof): Promise<CommitResult>;
119
+ }
120
+
121
+ /**
122
+ * The capability that answers "which action committed revision N of this block?" — the question the
123
+ * commit-tier stale checks need when local `latest` has already advanced PAST a contested revision,
124
+ * so `latest.actionId` alone can no longer distinguish "my commit landed and history moved on"
125
+ * (abstain / not a conflict) from "a rival took my revision" (reject / retryable conflict).
126
+ * Consumed by the cluster member's promise-round stale-commit check
127
+ * (`ClusterMember.validateCommitRevisions`) and by `CoordinatorRepo`'s commit rejection classifier.
128
+ * Named (rather than probed inline) for the same reason as {@link ICommitDigestPreviewer}: one
129
+ * definition of the contract, and something a repo decorator can `implements` and forward — a
130
+ * wrapper that drops the method silently degrades both checks to an abstain on that node.
131
+ */
132
+ export interface IRevisionActionReader {
133
+ /**
134
+ * The action id recorded for `rev` of `blockId`, or `undefined` when this node holds no revision
135
+ * record for it (never seen, or history truncated below `rev`). Read-only; never takes the block
136
+ * write latch (callers are on vote/classification paths and must treat a throw as "unknown").
137
+ */
138
+ getRevisionAction(blockId: BlockId, rev: number): Promise<ActionId | undefined>;
139
+ }
140
+
141
+ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaStore, ICommitDigestPreviewer, ICommitProofPersister, IRevisionActionReader {
142
+ private readonly validatePend?: PendValidationHook;
143
+ private readonly unvalidatablePendPolicy: UnvalidatablePendPolicy;
144
+ /** Per-collection change listeners; empty sets are pruned on unsubscribe. */
145
+ private readonly changeListeners = new Map<CollectionId, Set<CollectionChangeListener>>();
146
+ /** Catch-all change listeners — fire for EVERY collection's commit on this node. */
147
+ private readonly anyChangeListeners = new Set<CollectionChangeListener>();
148
+
149
+ constructor(
150
+ private readonly createBlockStorage: (blockId: BlockId) => IBlockStorage,
151
+ options?: StorageRepoOptions
152
+ ) {
153
+ this.validatePend = options?.validatePend;
154
+ this.unvalidatablePendPolicy = options?.unvalidatablePendPolicy ?? 'accept';
155
+ }
156
+
157
+ /**
158
+ * Subscribe to commits that mutate `collectionId`'s blocks on this node.
159
+ * Returns an idempotent unsubscribe. See {@link IBlockChangeNotifier}.
160
+ */
161
+ onCollectionChange(collectionId: CollectionId, listener: CollectionChangeListener): () => void {
162
+ let set = this.changeListeners.get(collectionId);
163
+ if (!set) {
164
+ set = new Set();
165
+ this.changeListeners.set(collectionId, set);
166
+ }
167
+ set.add(listener);
168
+ let unsubscribed = false;
169
+ return () => {
170
+ if (unsubscribed) return;
171
+ unsubscribed = true;
172
+ const current = this.changeListeners.get(collectionId);
173
+ if (current) {
174
+ current.delete(listener);
175
+ if (current.size === 0) {
176
+ this.changeListeners.delete(collectionId);
177
+ }
178
+ }
179
+ };
180
+ }
181
+
182
+ /**
183
+ * Subscribe to commits mutating ANY collection on this node — the catch-all feed the
184
+ * cohort-topic origination bridge consumes (it cannot enumerate collection ids ahead of time,
185
+ * so a per-collection {@link onCollectionChange} subscription cannot see every commit). Fires for
186
+ * the same `(pending → committed)` transitions as {@link onCollectionChange}, but across every
187
+ * collection. Returns an idempotent unsubscribe; a throwing listener is isolated + logged.
188
+ */
189
+ onAnyCollectionChange(listener: CollectionChangeListener): () => void {
190
+ this.anyChangeListeners.add(listener);
191
+ let unsubscribed = false;
192
+ return () => {
193
+ if (unsubscribed) return;
194
+ unsubscribed = true;
195
+ this.anyChangeListeners.delete(listener);
196
+ };
197
+ }
198
+
199
+ /**
200
+ * Fire one {@link CollectionChangeEvent} per distinct collection that was
201
+ * newly committed. Called AFTER the commit critical section (locks released),
202
+ * fire-and-forget synchronous; a throwing listener is isolated and logged. Each event reaches
203
+ * both that collection's {@link onCollectionChange} subscribers and every
204
+ * {@link onAnyCollectionChange} catch-all subscriber.
205
+ *
206
+ * `tailId` is the `CommitRequest.tailId` on the commit path; `undefined` on read-driven
207
+ * promotions (the get/emitPromotions path has no commit request). A single commit is for one
208
+ * collection's chain in practice, so all events from one commit share the same `tailId`.
209
+ */
210
+ private emitCollectionChanges(collectionBlocks: Map<CollectionId, BlockId[]>, actionId: ActionId, rev: number, tailId?: BlockId): void {
211
+ const hasCatchAll = this.anyChangeListeners.size > 0;
212
+ for (const [collectionId, blockIds] of collectionBlocks) {
213
+ const listeners = this.changeListeners.get(collectionId);
214
+ if ((!listeners || listeners.size === 0) && !hasCatchAll) {
215
+ continue;
216
+ }
217
+ const event: CollectionChangeEvent = { collectionId, blockIds, actionId, rev, tailId };
218
+ if (listeners && listeners.size > 0) {
219
+ this.fireChangeListeners(listeners, event);
220
+ }
221
+ if (hasCatchAll) {
222
+ this.fireChangeListeners(this.anyChangeListeners, event);
223
+ }
224
+ }
225
+ }
226
+
227
+ /** Dispatch `event` to a snapshot of `listeners` (safe under mid-emit (un)subscribe), isolating + logging any throw. */
228
+ private fireChangeListeners(listeners: Set<CollectionChangeListener>, event: CollectionChangeEvent): void {
229
+ for (const listener of Array.from(listeners)) {
230
+ try {
231
+ listener(event);
232
+ } catch (err) {
233
+ log('onCollectionChange listener threw for collection=%s: %o', event.collectionId, err);
234
+ }
235
+ }
236
+ }
237
+
238
+ async get({ blockIds, context }: BlockGets, _options?: MessageOptions): Promise<GetBlockResults> {
239
+ const distinctBlockIds = Array.from(new Set(blockIds));
240
+ log('get blockIds=%d', distinctBlockIds.length);
241
+ // Read-driven promotions that land durably here, captured so we can emit a
242
+ // change event per durable landing after the parallel reads complete (mirrors
243
+ // commit's "emit after the work" ordering). The array is shared across the
244
+ // parallel map closures below safe because each push happens synchronously
245
+ // between awaits (single-threaded), never concurrently.
246
+ const promotions: { collectionId: CollectionId, blockId: BlockId, actionId: ActionId, rev: number }[] = [];
247
+ const results = await Promise.all(distinctBlockIds.map(async (blockId) => {
248
+ const blockStorage = this.createBlockStorage(blockId);
249
+ // Set when this node KNOWS its answer for the block is a guess: the promotion
250
+ // below refused for a missing base, or getBlock() threw (truncated history /
251
+ // failed restore). An absent-reading block then reports `unavailable` instead of
252
+ // posing as an authoritative "never existed" — see BlockUnavailableReason.
253
+ let unavailable: BlockUnavailableReason | undefined;
254
+
255
+ // Ensure that all outstanding transactions in the context are committed.
256
+ // This promotes a landed-elsewhere pending via internalCommit, which writes the
257
+ // block's metadata the same read-modify-write commit()/saveReplicatedBlock guard
258
+ // with the per-block write latch. It MUST hold that latch too, or a promotion
259
+ // racing a concurrent commit on the block regresses latest non-monotonically /
260
+ // cross-writes a revision. Cheap unlatched pre-scan first so the common
261
+ // contextless read and no-pending read never pay for latch acquisition; the
262
+ // authoritative decision is re-made inside the latch.
263
+ if (context) {
264
+ const preLatest = await blockStorage.getLatest();
265
+ const preMissing = preLatest
266
+ ? context.committed.filter(c => c.rev > preLatest.rev)
267
+ : context.committed;
268
+ if (preMissing.length > 0) {
269
+ await withBlockWriteLatch(blockId, async (latch) => {
270
+ // Re-read authoritative state under the latch: a concurrent commit may have
271
+ // promoted or superseded a pending between the unlatched pre-scan and here.
272
+ // Recompute which committed entries are still ahead of `latest` (drops the
273
+ // superseded, rev <= latest.rev) and re-fetch each pending inside the loop
274
+ // (skips the already-promoted, pending gone). This makes read-driven
275
+ // promotion idempotent under races, mirroring commit()'s alreadyDone/stale
276
+ // partitioning.
277
+ const latest = await blockStorage.getLatest();
278
+ const missing = latest
279
+ ? context.committed.filter(c => c.rev > latest.rev)
280
+ : context.committed;
281
+ // Sort a COPY: when `latest` is undefined, `missing` aliases the caller's
282
+ // `context.committed` array, and an in-place `.sort()` would reorder the shared
283
+ // request context under the caller's feet.
284
+ try {
285
+ for (const { actionId, rev } of [...missing].sort((a, b) => a.rev - b.rev)) {
286
+ const pending = await blockStorage.getPendingTransaction(actionId);
287
+ if (pending) {
288
+ const collectionId = await this.internalCommit(blockId, actionId, rev, blockStorage, latch);
289
+ if (collectionId !== undefined) {
290
+ promotions.push({ collectionId, blockId, actionId, rev });
291
+ }
292
+ }
293
+ }
294
+ } catch (err) {
295
+ // This node holds no materializable base for the block, so NO context revision
296
+ // can be promoted here (each builds on the one before). Leave `latest` where it
297
+ // is — the invariant internalCommit just enforced — and let the commit-path
298
+ // healing supply the content; a read must not fail for it. Every other fault
299
+ // still propagates.
300
+ if (!(err instanceof MissingBaseRevisionError)) {
301
+ throw err;
302
+ }
303
+ // This node holds records PROVING the block exists (a pending it could not
304
+ // promote); if the block then reads as absent below, the answer is a guess,
305
+ // not an authoritative "never existed".
306
+ unavailable = 'unmaterializable';
307
+ log('get:promote-skipped-missing-base blockId=%s rev=%d reason=%s', blockId, err.rev, err.message);
308
+ }
309
+ });
310
+ }
311
+ }
312
+
313
+ // NOTE: a Crash-D3 block (durably promoted + revision saved, but the setLatest lost so
314
+ // meta.latest is stale and the pending record is gone) reads as empty/stale here — a
315
+ // context-driven get skips promotion (pending gone) and a default getBlock() sees the
316
+ // stale latest. It is soft-wedged (stale), not hard-wedged: the next commit-retry for
317
+ // (actionId, rev) self-heals it via storage.recover() in commit(). Not repaired lazily on
318
+ // the read path because the plain read below holds no write latch; if stale reads on
319
+ // unwritten blocks ever become a problem, add a latched lazy recover() here.
320
+ //
321
+ // readBlockHealing() THROWS when this node holds a `latest` it cannot materialize
322
+ // (truncated history: "Failed to find materialized block", or a failed restore). Caught
323
+ // PER BLOCK so one broken block cannot fail the whole batch's Promise.all and take healthy
324
+ // siblings down with it. The read still fails for THIS block TransactorSource throws
325
+ // BlockUnavailableError on the flagged entry so nothing is swallowed.
326
+ let blockRev: Awaited<ReturnType<IBlockStorage['getBlock']>>;
327
+ try {
328
+ blockRev = await this.readBlockHealing(blockId, blockStorage, context?.rev);
329
+ } catch (err) {
330
+ // NOTE: the entry drops `state.latest`, which this node does know (getLatest() does not
331
+ // materialize, so it does not throw). Empty state is what makes CoordinatorRepo treat the
332
+ // block as missing and consult the cohort — exactly the repair this block needs. If a
333
+ // consumer ever needs the revision behind an unavailable answer (e.g. to ask the cohort
334
+ // for a specific rev instead of the whole block), carry `latest` here and widen the
335
+ // coordinator's consult trigger to `isMissing || unavailable` so repair still fires.
336
+ log('get:unmaterializable blockId=%s error=%s', blockId,
337
+ err instanceof Error ? err.message : String(err));
338
+ return [blockId, { state: {}, unavailable: 'unmaterializable' } as GetBlockResult];
339
+ }
340
+
341
+ // Include pending action if requested, applying the pending transform over whatever
342
+ // committed base getBlock() resolved (possibly none a pending-only insert has no
343
+ // committed revision under it and getBlock reports that as an absent base, not a fault).
344
+ if (context?.actionId !== undefined) {
345
+ const pendingTransform = await blockStorage.getPendingTransaction(context.actionId);
346
+ if (!pendingTransform) {
347
+ if (unavailable !== undefined) {
348
+ // The promotion refusal above deleted this very pending record
349
+ // (`refuseMissingBase` drops the pending it cannot promote). This node DID hold
350
+ // the record and dropped it, so the honest answer is an availability one — not
351
+ // a caller-contract violation, and never a throw that would fail the whole batch.
352
+ return [blockId, { state: {}, unavailable } as GetBlockResult];
353
+ }
354
+ // Caller-contract violation (the caller asserted a pending this repo never had, or
355
+ // cancelled) — an error, not an availability question. Deliberately NOT `unavailable`.
356
+ //
357
+ // It is NOT the only way to reach here. A context that both PROVES its own action
358
+ // (`committed` names it) and names it as the pending overlay (`actionId`) is
359
+ // self-contradictory, and the two halves of that contradiction land differently: if
360
+ // the read-driven promotion above REFUSED, the arm above answers gracefully; if it
361
+ // SUCCEEDED, `promotePendingTransaction` moved the record and we throw here failing
362
+ // the whole batch for a request the refusal path tolerates. No production code sets
363
+ // `ActionContext.actionId` at all today, so neither is reachable except from tests or
364
+ // a peer that crafts the field on the wire. See
365
+ // tickets/blocked/repo-pending-overlay-has-no-producer.
366
+ throw new Error(`Pending action ${context.actionId} not found`);
367
+ }
368
+ const block = applyTransform(blockRev?.block, pendingTransform);
369
+ return [blockId, {
370
+ block,
371
+ state: {
372
+ latest: await blockStorage.getLatest(),
373
+ pendings: [context.actionId]
374
+ },
375
+ // The COMMITTED revision underneath the pending overlay. A pending has no revision
376
+ // of its own, so the honest answer is the base it was applied to. Absent when there
377
+ // was no base at all — a pending-only insert served over an absent committed base,
378
+ // where fabricating a revision would claim content this node never committed.
379
+ ...(blockRev ? { materialized: blockRev.actionRev } : {}),
380
+ // A pending applied to a missing base can materialize nothing (applyTransform drops
381
+ // updates with no block to apply them to) — that absence is a guess, and is flagged.
382
+ // A materialized block is a real answer regardless of the earlier refusal. TWO ways
383
+ // an empty result is a guess: the promotion refusal fired (`unavailable` set), or
384
+ // there was no committed base under the overlay at all (`blockRev === undefined`)
385
+ // this node holds a pending record PROVING the block exists and produced nothing.
386
+ // The second clause's ABSENCE in the other direction is equally load-bearing: a
387
+ // pending DELETE over a real committed base also lands here with no block, and that
388
+ // is an authoritative tombstone which must stay unflagged.
389
+ ...(block === undefined && (unavailable !== undefined || blockRev === undefined)
390
+ ? { unavailable: unavailable ?? 'unmaterializable' }
391
+ : {})
392
+ } as GetBlockResult];
393
+ }
394
+
395
+ if (!blockRev) {
396
+ // `unavailable` distinguishes "never existed" (the common insert-probe case, no flag)
397
+ // from "this node cannot reconstruct it" (the promotion above refused for a missing
398
+ // base). A tombstoned block also lands here with meta.latest set, but it never enters
399
+ // the missing-base catch, so it stays an authoritative absent — keyed off the explicit
400
+ // flag, not off "no block".
401
+ return [blockId, { state: {}, ...(unavailable !== undefined ? { unavailable } : {}) } as GetBlockResult];
402
+ }
403
+
404
+ const pendings = await asyncIteratorToArray(blockStorage.listPendingTransactions());
405
+ return [blockId, {
406
+ block: blockRev.block,
407
+ // `getBlock(context?.rev)` materialized the content at the highest committed revision
408
+ // at or below the pin, and reports it as `actionRev` — report THAT alongside the
409
+ // content. `state.latest` deliberately stays the node's newest revision for the block
410
+ // (StorageRepo.get's own promotion pre-scan and CoordinatorRepo's read-repair compare
411
+ // against it), so the two disagree exactly when a pinned read is serving older content.
412
+ materialized: blockRev.actionRev,
413
+ state: {
414
+ latest: await blockStorage.getLatest(),
415
+ pendings
416
+ }
417
+ }];
418
+ }));
419
+
420
+ // Emit per durable read-driven landing (Option A — emit eagerly). Done after the
421
+ // parallel reads complete so emission stays outside the per-block work, matching
422
+ // commit's ordering. No-op when nothing was promoted.
423
+ this.emitPromotions(promotions);
424
+
425
+ return Object.fromEntries(results);
426
+ }
427
+
428
+ /**
429
+ * The one place a local coverage gap is healed from a peer. `getBlock` is local-only; when it
430
+ * reports the target revision as not covered ({@link RevisionNotCoveredError}) this fetches it
431
+ * through `restoreRevision` under the block's write latch — the restore writes revision records
432
+ * and merges coverage into the metadata blob, so it must serialize against every other writer of
433
+ * the block — and re-reads. Only the restore is latched; the reads on either side are not, and
434
+ * the latch is never held across the two.
435
+ *
436
+ * A restore that fails on a **pending-only** block (metadata seeded by a pend, no committed
437
+ * revision) reads as ABSENT, not as a fault: the named revision was a guess about content this
438
+ * node never held, and the caller's insert-probe / pending-overlay logic already treats an absent
439
+ * base as "nothing committed here". A failed restore on a block that DOES hold a `latest` is a
440
+ * real fault (a `latest` this node cannot serve) and propagates, so the caller reports the block
441
+ * as unavailable. Any throw from the second read (records restored but nothing materializable
442
+ * under them) propagates the same way.
443
+ */
444
+ private async readBlockHealing(
445
+ blockId: BlockId,
446
+ storage: IBlockStorage,
447
+ rev: number | undefined
448
+ ): Promise<{ block: IBlock, actionRev: ActionRev } | undefined> {
449
+ try {
450
+ return await storage.getBlock(rev);
451
+ } catch (err) {
452
+ if (!(err instanceof RevisionNotCoveredError)) {
453
+ throw err;
454
+ }
455
+ try {
456
+ // NOTE: the peer fetch inside restoreRevision runs UNDER the block's write latch, so a
457
+ // slow restore queues every commit/pend/replica on this block behind one network
458
+ // round-trip. Fine at today's restore rates (a gap is healed once, then served
459
+ // locally); if restore latency ever shows up delaying commits, fetch + vet OUTSIDE the
460
+ // latch and take it only to write, re-checking coverage inside.
461
+ await withBlockWriteLatch(blockId, latch => storage.restoreRevision(err.rev, latch));
462
+ } catch (restoreErr) {
463
+ if (await storage.getLatest() === undefined) {
464
+ log('get:restore-failed-pending-only blockId=%s rev=%d error=%s', blockId, err.rev,
465
+ restoreErr instanceof Error ? restoreErr.message : String(restoreErr));
466
+ return undefined;
467
+ }
468
+ throw restoreErr;
469
+ }
470
+ return await storage.getBlock(rev);
471
+ }
472
+ }
473
+
474
+ /**
475
+ * Emit a {@link CollectionChangeEvent} for each read-driven promotion that landed
476
+ * during a {@link get}. A single get() can promote multiple distinct actions, each
477
+ * at its own `(actionId, rev)`, so group by `(actionId, rev)` and route each group
478
+ * through {@link emitCollectionChanges} once.
479
+ */
480
+ private emitPromotions(promotions: { collectionId: CollectionId, blockId: BlockId, actionId: ActionId, rev: number }[]): void {
481
+ if (promotions.length === 0) {
482
+ return;
483
+ }
484
+ const groups = new Map<string, { actionId: ActionId, rev: number, collectionBlocks: Map<CollectionId, BlockId[]> }>();
485
+ for (const { collectionId, blockId, actionId, rev } of promotions) {
486
+ const key = `${actionId} ${rev}`;
487
+ let group = groups.get(key);
488
+ if (!group) {
489
+ group = { actionId, rev, collectionBlocks: new Map() };
490
+ groups.set(key, group);
491
+ }
492
+ const list = group.collectionBlocks.get(collectionId) ?? [];
493
+ list.push(blockId);
494
+ group.collectionBlocks.set(collectionId, list);
495
+ }
496
+ for (const { actionId, rev, collectionBlocks } of groups.values()) {
497
+ this.emitCollectionChanges(collectionBlocks, actionId, rev);
498
+ }
499
+ }
500
+
501
+ async pend(request: PendRequest, _options?: MessageOptions): Promise<PendResult> {
502
+ // Re-check the transaction when a validation hook is configured — the unvalidatable-pend
503
+ // policy and the throwing-hook catch both live in the shared `checkPendValidation`, so this
504
+ // tier and the cluster tier cannot drift apart on what they refuse.
505
+ const hook = this.validatePend;
506
+ const validation = await checkPendValidation(
507
+ request,
508
+ hook && (({ transaction, operationsHash }) => hook(transaction, operationsHash)),
509
+ this.unvalidatablePendPolicy,
510
+ event => event.kind === 'unvalidatable'
511
+ ? log('pend-unvalidatable actionId=%s policy=%s', request.actionId, event.policy)
512
+ : log('pend validator-fault actionId=%s error=%s', request.actionId, event.error)
513
+ );
514
+ if (!validation.valid) {
515
+ // Hard rejection: no `conflict` flag, because re-driving the same request fails the same
516
+ // way and would only burn the writer's retry budget.
517
+ return {
518
+ success: false,
519
+ reason: validation.reason ?? 'Transaction validation failed'
520
+ };
521
+ }
522
+
523
+ const blockIds = blockIdsForTransforms(request.transforms);
524
+ log('pend actionId=%s blockIds=%d rev=%s', request.actionId, blockIds.length, request.rev);
525
+ const pendings: ActionPending[] = [];
526
+ const missing: ActionTransforms[] = [];
527
+ // Highest revision this node confirms holding among the blocks that are at or past the
528
+ // requested one reported as StaleFailure.staleAt so a losing writer learns the number
529
+ // instead of parsing prose. Confirmed-local only: we read it from our own storage below.
530
+ let staleAt: StaleFailure['staleAt'];
531
+ // Blocks this action ALREADY committed at exactly the requested revision — the durable half
532
+ // of a torn action whose retry reuses the same actionId. Sibling of the `alreadyDone`
533
+ // partition in `commit` below: satisfied, not merely non-stale, so no pending is recorded
534
+ // for them (see the fan-out at the end of this method).
535
+ const satisfied = new Set<BlockId>();
536
+
537
+ // Potential race condition: A concurrent commit operation could complete
538
+ // between the conflict checks (latest.rev, listPendingTransactions) and the
539
+ // savePendingTransaction call below. This pend operation might succeed based on
540
+ // stale information, but the subsequent commit for this pend would likely
541
+ // fail correctly later if a conflict arose. Locking here could make the initial
542
+ // check more accurate but adds overhead. The current approach prioritizes
543
+ // letting the commit be the final arbiter.
544
+ for (const blockId of blockIds) {
545
+ const blockStorage = this.createBlockStorage(blockId);
546
+ const transforms = transformForBlockId(request.transforms, blockId);
547
+
548
+ // Handle any conflicting revisions FIRST: a block this same action already committed at
549
+ // exactly the requested revision is satisfied, and skips both this check and the
550
+ // pending-action listing below.
551
+ if (request.rev !== undefined || transforms.insert) {
552
+ const latest = await blockStorage.getLatest();
553
+ // Our own already-durable work, met again by a retry (see {@link isOwnRevision}):
554
+ // treating it as a stale rival would refuse the writer with its own commit.
555
+ // NOTE: a rev-less pend (`request.rev === undefined`, an insert-only claim) can
556
+ // never match, so a torn action retried WITHOUT a revision is still refused by its
557
+ // own insert. No production caller sends one — `TransactorSource.transact` and the
558
+ // multi-collection coordinator both require a rev — so this is unreachable today;
559
+ // if a rev-less write path ever appears, match on `latest.actionId` alone here.
560
+ if (isOwnRevision(latest, request.rev, request.actionId)) {
561
+ satisfied.add(blockId);
562
+ continue;
563
+ }
564
+ if (latest && latest.rev >= (request.rev ?? 0)) {
565
+ // Only a real revision race yields a meaningful `staleAt`. When `request.rev` is
566
+ // undefined this same branch fires for an insert collision (the comparison degrades
567
+ // to `latest.rev >= 0`, true for any existing block), and reporting that block's
568
+ // revision would be a number that answers a question nobody asked.
569
+ if (request.rev !== undefined) {
570
+ staleAt = highestStaleAt([staleAt, { blockId, rev: latest.rev }]);
571
+ }
572
+ const transforms = await asyncIteratorToArray(blockStorage.listRevisions(request.rev ?? 0, latest.rev));
573
+ for (const actionRev of transforms) {
574
+ const transform = await blockStorage.getTransaction(actionRev.actionId);
575
+ if (!transform) {
576
+ throw new Error(`Missing action ${actionRev.actionId} for block ${blockId}`);
577
+ }
578
+ missing.push({
579
+ actionId: actionRev.actionId,
580
+ rev: actionRev.rev,
581
+ transforms: transformsFromTransform(transform, blockId)
582
+ });
583
+ }
584
+ }
585
+ }
586
+
587
+ // Then handle any pending actions
588
+ const pending = await asyncIteratorToArray(blockStorage.listPendingTransactions());
589
+ pendings.push(...pending.map(actionId => ({ blockId, actionId })));
590
+ }
591
+
592
+ if (missing.length) {
593
+ log('pend:stale actionId=%s missing=%d', request.actionId, missing.length);
594
+ return {
595
+ success: false,
596
+ conflict: true,
597
+ missing,
598
+ ...(staleAt === undefined ? {} : { staleAt })
599
+ };
600
+ }
601
+
602
+ if (pendings.length > 0) {
603
+ if (request.policy === 'f') { // Fail on pending actions
604
+ return { success: false, conflict: true, pending: pendings };
605
+ } else if (request.policy === 'r') { // Return populated pending actions
606
+ return {
607
+ success: false,
608
+ conflict: true,
609
+ pending: await Promise.all(pendings.map(async action => {
610
+ const blockStorage = this.createBlockStorage(action.blockId);
611
+ return {
612
+ blockId: action.blockId,
613
+ actionId: action.actionId,
614
+ transform: (await blockStorage.getPendingTransaction(action.actionId))
615
+ ?? (await blockStorage.getTransaction(action.actionId))! // Possible that since enumeration, the action has been promoted
616
+ }
617
+ }))
618
+ };
619
+ }
620
+ }
621
+
622
+
623
+ // Simultaneously save pending action for each block
624
+ // Note: that this is not atomic, after we checked for conflicts and pending actions
625
+ // new pending or committed actions may have been added. This is okay, because
626
+ // this check during pend is conservative.
627
+ //
628
+ // Each block's pending write runs under THAT block's write latch, one latch per branch and
629
+ // never nested: savePendingTransaction seeds the block's metadata blob when it has none, and
630
+ // an unlatched seed racing a concurrent commit/replica on a fresh block erases the `latest`
631
+ // the other writer just landed. Never more than one block latch is held by a branch, so this
632
+ // cannot deadlock against commit's sorted multi-latch acquisition.
633
+ //
634
+ // `satisfied` blocks are skipped: `commit`'s `alreadyDone` arm skips `internalCommit`, the
635
+ // only thing that promotes (and thereby removes) a pending record, so a pending saved here
636
+ // would never clear a permanent durable reservation that the rival-pending checks (this
637
+ // method's listPendingTransactions scan, and `ClusterMember.validatePendOperations`) refuse
638
+ // every future writer against, a worse wedge than the one this carve-out fixes. They still
639
+ // ride in the returned `blockIds` so `cancel` covers them (deleting an absent pending is a
640
+ // no-op that writes no metadata).
641
+ await Promise.all(blockIds.filter(blockId => !satisfied.has(blockId)).map(blockId => {
642
+ const blockStorage = this.createBlockStorage(blockId);
643
+ const blockTransform = transformForBlockId(request.transforms, blockId);
644
+ return withBlockWriteLatch(blockId, latch => blockStorage.savePendingTransaction(request.actionId, blockTransform, latch));
645
+ }));
646
+
647
+ return {
648
+ success: true,
649
+ pending: pendings,
650
+ blockIds
651
+ } as PendSuccess;
652
+ }
653
+
654
+ async cancel(actionRef: ActionBlocks, _options?: MessageOptions): Promise<void> {
655
+ log('cancel actionId=%s blockIds=%d', actionRef.actionId, actionRef.blockIds.length);
656
+ await Promise.all(actionRef.blockIds.map(blockId => {
657
+ const blockStorage = this.createBlockStorage(blockId);
658
+ return withBlockWriteLatch(blockId, latch => blockStorage.deletePendingTransaction(actionRef.actionId, latch));
659
+ }));
660
+ }
661
+
662
+ /**
663
+ * Commit a previously-pended action across its blocks, under the block write latches.
664
+ *
665
+ * **Divergence vs genuine fault.** When the batch cannot be completed, the reason decides what
666
+ * happens to the pending records the pend left behind. `ClusterMember.applyConsensusOperation`
667
+ * makes the same split one layer up it *tolerates* a divergence (and reconciles every
668
+ * `commit.blockIds` entry from a cohort peer) but *propagates* a genuine fault for retry — so this
669
+ * method must agree with it:
670
+ *
671
+ * - **Divergence** this node is behind the agreed history, either because it holds no
672
+ * materializable base ({@link MissingBaseRevisionError}) or because it never received the pend
673
+ * (the `Pending action not found` throw). Reconcile is guaranteed to follow and will advance
674
+ * every block in the batch past `request.rev`, so no pending record here can ever be promoted:
675
+ * {@link dropUnpromotablePendings} deletes them (see {@link refuseMissingBase}, which already
676
+ * accepts this tradeoff for the single refusing block).
677
+ * - **Genuine fault** — any other throw out of {@link internalCommit} (a raw-storage error, …).
678
+ * `ClusterMember` propagates it and the commit is retried, and a retry can still replay the
679
+ * pendings, so they are KEPT.
680
+ *
681
+ * The stale/`missedCommits` early return (this node is AHEAD it already holds a revision at or
682
+ * past `request.rev`, committed under a different action) deliberately keeps pendings too, and its
683
+ * cure is the losing client's `cancel`: `CoordinatorRepo.cancel` runs through consensus, so every
684
+ * member drops the record, not just the coordinator. Replication cannot be the cure here — this
685
+ * node is already ahead, and a later forward write carries a DIFFERENT action id, which is not
686
+ * what `BlockStorage.saveForwardRevision` deletes. A client that dies between the stale result and
687
+ * its `cancel` therefore still strands the record; that is pre-existing and orthogonal to the
688
+ * divergence split above.
689
+ */
690
+ async commit(request: CommitRequest, _options?: MessageOptions, proof?: BlockCommitProof): Promise<CommitResult> {
691
+ log('commit actionId=%s rev=%d blockIds=%d', request.actionId, request.rev, request.blockIds.length);
692
+ // Deduped ONCE, in request order — the order blocks are committed and reported in. The latches
693
+ // are acquired in sorted order by `acquireBlockWriteLatches` over this same set, so every
694
+ // `latches.get(blockId)!` below resolves.
695
+ const blockIds = Array.from(new Set(request.blockIds));
696
+ // Collects the blocks newly committed in this call, grouped by collection,
697
+ // so we can emit change events once locks are released. Blocks that land before
698
+ // a mid-loop failure stay here and are still emitted (Option A — emit eagerly):
699
+ // they are durably committed and a retry rolls the remainder forward.
700
+ const collectionBlocks = new Map<CollectionId, BlockId[]>();
701
+ // Captured when internalCommit throws mid-loop; we break (rather than return)
702
+ // so locks release and accumulated landings still emit before we report failure.
703
+ let failure: { reason: string } | undefined;
704
+
705
+ // Every block's token is kept so each write below can prove it runs inside that block's latch.
706
+ const { latches, release } = await acquireBlockWriteLatches(blockIds);
707
+
708
+ try {
709
+ // --- Start of Critical Section ---
710
+
711
+ // Request order, deduped (NOT the sorted acquisition order): the order here is the order
712
+ // blocks are committed and reported in change events, which callers may observe.
713
+ const blockStorages = blockIds.map(blockId => ({
714
+ blockId,
715
+ storage: this.createBlockStorage(blockId),
716
+ latch: latches.get(blockId)!
717
+ }));
718
+
719
+ // Partition blocks into:
720
+ // - alreadyDone: latest.rev === request.rev && latest.actionId === request.actionId
721
+ // (idempotent retry — a prior commit of this same action already landed here;
722
+ // skip rather than treat as a conflict. Needed to rollforward stranded blocks
723
+ // after a mid-batch crash committed some but not all blocks.)
724
+ // - missedCommits: latest.rev >= request.rev but not the same actionId real stale conflict.
725
+ // - toCommit: latest.rev < request.rev or no latest yet run internalCommit.
726
+ const toCommit: { blockId: BlockId, storage: IBlockStorage, latch: BlockWriteLatch }[] = [];
727
+ const missedCommits: { blockId: BlockId, transforms: ActionTransform[] }[] = [];
728
+ // Highest revision among the blocks confirmed lost to a newer one — reported as
729
+ // StaleFailure.staleAt. The idempotent-retry `continue` below is a no-op, not a loss,
730
+ // so it never seeds this.
731
+ let staleAt: StaleFailure['staleAt'];
732
+ for (const entry of blockStorages) {
733
+ const { blockId, storage, latch } = entry;
734
+ const latest = await storage.getLatest();
735
+ if (latest && latest.rev >= request.rev) {
736
+ if (isOwnRevision(latest, request.rev, request.actionId)) {
737
+ // Idempotent no-op for this block — already committed with this exact (actionId, rev).
738
+ // A retry can carry a proof the original commit lacked (or crashed before writing):
739
+ // back-fill it, strictly additively, under the same digest-match retention rule the
740
+ // original commit applies. Runs inside the latched critical section.
741
+ await this.backFillProof(blockId, storage, request.rev, request.actionId, proof, latch);
742
+ continue;
743
+ }
744
+ staleAt = highestStaleAt([staleAt, { blockId, rev: latest.rev }]);
745
+ const transforms: ActionTransform[] = [];
746
+ for await (const actionRev of storage.listRevisions(request.rev, latest.rev)) {
747
+ const transform = await storage.getTransaction(actionRev.actionId);
748
+ if (!transform) {
749
+ throw new Error(`Missing action ${actionRev.actionId} for block ${blockId}`);
750
+ }
751
+ transforms.push({
752
+ actionId: actionRev.actionId,
753
+ rev: actionRev.rev,
754
+ transform
755
+ });
756
+ }
757
+ missedCommits.push({ blockId, transforms }); // Push, even if transforms is empty, because we want to reject the older version
758
+ continue;
759
+ }
760
+ toCommit.push(entry);
761
+ }
762
+
763
+ if (missedCommits.length) {
764
+ log('commit:stale actionId=%s missed=%d', request.actionId, missedCommits.length);
765
+ return { // Return directly, locks will be released in finally
766
+ success: false,
767
+ missing: perBlockActionTransformsToPerAction(missedCommits),
768
+ ...(staleAt === undefined ? {} : { staleAt })
769
+ };
770
+ }
771
+
772
+ // Check for missing pending actions only on blocks that still need to commit.
773
+ // Already-done blocks will have had their pending promoted, so skipping them here
774
+ // is what makes the idempotent rollforward work.
775
+ //
776
+ // A toCommit block whose pending is absent is one of two states:
777
+ // - Crash-D3: the action was durably promoted and its revision saved, but the crash
778
+ // lost the setLatest, so meta.latest is still < request.rev and the pending record
779
+ // is gone. getTransaction(actionId) returns the promoted transform. Self-heal here
780
+ // via storage.recover() (redoes the lost setLatest, advancing latest to the highest
781
+ // contiguous promoted rev, >= request.rev). recover() is idempotent + monotonic, so
782
+ // calling it under the already-held block write latch is safe. Recovered blocks are then
783
+ // excluded from the internalCommit loop below — their pending is gone, so
784
+ // internalCommit would throw.
785
+ // - Genuine missing pend: the action was never promoted (getTransaction → undefined),
786
+ // so the pend is truly missing. Throw exactly as before.
787
+ // Crash-D2 never reaches this branch: its pending record is still present.
788
+ const missingPends: { blockId: BlockId, actionId: ActionId }[] = [];
789
+ const recovered = new Set<BlockId>();
790
+ for (const { blockId, storage, latch } of toCommit) {
791
+ const pendingAction = await storage.getPendingTransaction(request.actionId);
792
+ if (pendingAction) {
793
+ continue;
794
+ }
795
+ const promoted = await storage.getTransaction(request.actionId);
796
+ if (!promoted) {
797
+ missingPends.push({ blockId, actionId: request.actionId });
798
+ continue;
799
+ }
800
+ // Crash-D3 signature (pending absent + action durably promoted). Redo the lost setLatest.
801
+ const result = await storage.recover(latch);
802
+ if (result.latest !== undefined && result.latest.rev >= request.rev) {
803
+ recovered.add(blockId);
804
+ } else {
805
+ // Torn/partial state: recover() could not advance latest to request.rev (metadata
806
+ // absent, or a revision entry missing despite the promoted transaction). Fall back
807
+ // to treating the block as a genuine missing-pend error rather than silently succeeding.
808
+ missingPends.push({ blockId, actionId: request.actionId });
809
+ }
810
+ }
811
+
812
+ // NOTE: if a batch ever held BOTH a recovered D3 block and a genuine missing-pend block,
813
+ // this throw fires after recover() already advanced the D3 block durably, so that block's
814
+ // change event is skipped (the retry then treats it as alreadyDone and never re-emits;
815
+ // durable state stays correct, only the emit is lost). Judged unreachable today: a single
816
+ // crash mid-internalCommit leaves exactly one D3 block, with the rest alreadyDone or
817
+ // pending-present a never-pended block cannot coexist with it in one retry. If a path
818
+ // ever produces that mix, emit recovered blocks' events before throwing here.
819
+ if (missingPends.length) {
820
+ // Divergence (this node is behind): `ClusterMember` treats this throw as the canonical
821
+ // "behind" signal and reconciles EVERY block in the batch, advancing each past
822
+ // `request.rev`. Nothing can promote the pendings the other blocks still hold, so drop
823
+ // them here while the latches are still held — before reporting. The thrown message
824
+ // must stay byte-identical: `ClusterMember.isMissingPendingActionError` matches on it.
825
+ await this.dropUnpromotablePendings(toCommit, request.actionId);
826
+ throw new Error(`Pending action ${request.actionId} not found for block(s): ${missingPends.map(p => p.blockId).join(', ')}`);
827
+ }
828
+
829
+ // The original commit crashed before setLatest, so it also never emitted a change event
830
+ // for a recovered (Crash-D3) block. Now that recover() has committed it at request.rev,
831
+ // report its collection so downstream watchers wake mirroring internalCommit. Resolve
832
+ // the collectionId from the now-materialized block; a delete materializes to a tombstone
833
+ // (getBlock undefined), so fall back to the prior materialized block's header exactly as
834
+ // internalCommit does — otherwise a recovered delete would silently fail to wake watchers.
835
+ // Only when neither resolves (a delete-only block with no prior materialization) is the
836
+ // emit skipped, the same terminal fallback internalCommit uses.
837
+ for (const { blockId, storage, latch } of toCommit) {
838
+ if (!recovered.has(blockId)) {
839
+ continue;
840
+ }
841
+ const collectionId = (await storage.getBlock(request.rev))?.block.header.collectionId
842
+ ?? (await storage.getBlock(request.rev - 1))?.block.header.collectionId;
843
+ if (collectionId !== undefined) {
844
+ const list = collectionBlocks.get(collectionId) ?? [];
845
+ list.push(blockId);
846
+ collectionBlocks.set(collectionId, list);
847
+ }
848
+ // The recovered block IS committed at request.rev, but it is excluded from the
849
+ // internalCommit loop below — so without this it would be the one landing path that
850
+ // never retains the cohort's proof, even though this very call is carrying it.
851
+ await this.backFillProof(blockId, storage, request.rev, request.actionId, proof, latch);
852
+ }
853
+
854
+ // Commit the action for each block that still needs it.
855
+ // This loop will execute atomically for all blocks due to the acquired locks.
856
+ // Recovered (Crash-D3) blocks are already committed at request.rev and their pending is
857
+ // gone, so skip them internalCommit would throw on the missing pending record.
858
+ //
859
+ // Set when the mid-loop failure was a divergence rather than a genuine fault — the split
860
+ // documented on commit() above, which decides the fate of the batch's pending records.
861
+ let divergentFailure = false;
862
+ for (const { blockId, storage, latch } of toCommit) {
863
+ if (recovered.has(blockId)) {
864
+ continue;
865
+ }
866
+ try {
867
+ // internalCommit will throw if it encounters an issue
868
+ const collectionId = await this.internalCommit(blockId, request.actionId, request.rev, storage, latch, proof);
869
+ if (collectionId !== undefined) {
870
+ const list = collectionBlocks.get(collectionId) ?? [];
871
+ list.push(blockId);
872
+ collectionBlocks.set(collectionId, list);
873
+ }
874
+ } catch (err) {
875
+ // Partial-commit recovery: blocks already in collectionBlocks DID land
876
+ // durably and must still emit; a retry with the same (actionId, rev)
877
+ // treats them as idempotent no-ops and advances the remainder. Break
878
+ // instead of returning so locks release and those landings emit below.
879
+ failure = { reason: err instanceof Error ? err.message : 'Unknown error during commit' };
880
+ divergentFailure = err instanceof MissingBaseRevisionError;
881
+ break;
882
+ }
883
+ }
884
+
885
+ // The break left every not-yet-reached block still holding its pending record. Whether
886
+ // that record is still usable depends ENTIRELY on why we stopped — see the table on
887
+ // commit() above. Runs inside the try, so the per-block latches are still held.
888
+ // NOTE: a non-divergence fault deliberately KEEPS the batch's pendings so a retry can
889
+ // replay them. If ClusterMember ever stops retrying propagated commit faults, this arm
890
+ // becomes dead weight and the discriminator can collapse to "always drop".
891
+ if (divergentFailure) {
892
+ await this.dropUnpromotablePendings(toCommit, request.actionId);
893
+ }
894
+ }
895
+ finally {
896
+ // Releases every block latch, in reverse acquisition order.
897
+ release();
898
+ }
899
+
900
+ // Notify after the critical section, for every block newly committed here —
901
+ // including those that landed before a mid-loop failure (alreadyDone / stale
902
+ // partitions never reach `collectionBlocks`).
903
+ this.emitCollectionChanges(collectionBlocks, request.actionId, request.rev, request.tailId);
904
+
905
+ return failure ? { success: false, reason: failure.reason } : { success: true };
906
+ }
907
+
908
+ /**
909
+ * Delete `actionId`'s pending record from every given block, tolerating absence.
910
+ *
911
+ * Called by {@link commit} when it abandons a batch **because this node has diverged from the
912
+ * agreed history**the caller has already made that determination; this helper does not
913
+ * re-derive it. Once `ClusterMember` reconciles the batch, every one of these blocks sits at or
914
+ * past `request.rev`, so a commit retry partitions them as already-done/stale and never revisits
915
+ * their pendings; left in place they are reported as phantom conflicting actions by {@link pend}
916
+ * for every later write to the block (under `policy: 'f'`, forever).
917
+ *
918
+ * No special-casing is needed for blocks that already landed (record promoted), that were
919
+ * `recovered` (record already gone), or for the refusing block itself
920
+ * ({@link refuseMissingBase} deleted its record): deleting an absent pending record is a no-op on
921
+ * every backend.
922
+ *
923
+ * Per-block failures are logged and swallowed rather than propagated: this cleanup must never
924
+ * replace the failure the caller is about to report — the pre-loop throw's message is pattern-
925
+ * matched by `ClusterMember.isMissingPendingActionError`, and a swapped error would misroute
926
+ * consensus. A leftover record only degrades this node's participation in that one block.
927
+ */
928
+ private async dropUnpromotablePendings(
929
+ blocks: { blockId: BlockId, storage: IBlockStorage, latch: BlockWriteLatch }[],
930
+ actionId: ActionId
931
+ ): Promise<void> {
932
+ if (blocks.length === 0) {
933
+ return;
934
+ }
935
+ log('commit:drop-unpromotable-pendings actionId=%s blockIds=%d', actionId, blocks.length);
936
+ await Promise.all(blocks.map(async ({ blockId, storage, latch }) => {
937
+ try {
938
+ await storage.deletePendingTransaction(actionId, latch);
939
+ } catch (err) {
940
+ log('commit:drop-unpromotable-pending-failed blockId=%s actionId=%s error=%s', blockId, actionId,
941
+ err instanceof Error ? err.message : String(err));
942
+ }
943
+ }));
944
+ }
945
+
946
+ /**
947
+ * Reconciles `metadata.latest` for a single block with the highest contiguous
948
+ * fully-promoted revision in durable storage. Use after a crash between
949
+ * `promotePendingTransaction` and `setLatest` when retry-commit cannot help
950
+ * (the pending record is already gone) but the revision and committed-log entry
951
+ * are durable. Idempotent and monotonic.
952
+ */
953
+ async recoverBlock(blockId: BlockId): Promise<void> {
954
+ log('recoverBlock blockId=%s', blockId);
955
+ const storage = this.createBlockStorage(blockId);
956
+ // Hold the block write latch: recover() is a read-modify-write of the metadata blob that
957
+ // blindly writes back the object it read, so its "advance only" guard is TOCTOU — racing a
958
+ // concurrent commit()/saveReplicatedBlock that advanced latest in between would clobber it
959
+ // (a non-monotonic regression). Same latching invariant as every other metadata writer.
960
+ // commit() calls storage.recover(latch) directly under its own held latch, so it never
961
+ // routes through here no double-acquire / deadlock.
962
+ await withBlockWriteLatch(blockId, latch => storage.recover(latch));
963
+ }
964
+
965
+ /**
966
+ * Persist a replica of a block received out-of-band (churn re-replication) into
967
+ * local storage. Distinct from the {@link IRepo} commit funnel: the block arrives
968
+ * already materialized from a departing owner, not as a pend/commit. See
969
+ * {@link IBlockStorage.saveReplica} for the durability/monotonicity contract.
970
+ *
971
+ * Held under the same block write latch as {@link commit} so the replica's
972
+ * read-modify-write of the metadata blob is mutually exclusive with a concurrent
973
+ * local commit on the same block otherwise `saveReplica`'s monotonic guard could
974
+ * read a stale `latest` and clobber a commit that advanced it in between.
975
+ *
976
+ * `verifiedProof` is retained when supplied: both the reconcile path
977
+ * (`cluster/reconcile-block.ts`) and the certified push path (`BlockTransferService.handlePush`)
978
+ * pass the {@link BlockCommitProof} they verified against these exact bytes (`certifyContent`'s
979
+ * digest check), so a repaired replica serves the proof onward and certification no longer decays
980
+ * across repair hops.
981
+ *
982
+ * When the push does NOT advance `latest` (this node already holds that revision), `saveReplica`
983
+ * is a no-op and persists nothing — so the proof is back-filled here instead, through
984
+ * {@link backFillProof}'s digest-match rule. It is deliberately NOT persisted inside
985
+ * `saveReplica`: the proof was verified against the PUSHED bytes, while a back-fill attaches it to
986
+ * this node's HELD materialization, and a diverged holder's bytes at the same `(rev, actionId)`
987
+ * may differ. Storing a proof whose declared digest contradicts local content would make this node
988
+ * serve content that fails its own proof — `digest-mismatch` is an ATTRIBUTABLE fault in
989
+ * `certified-claims.ts`, so every receiver would penalize it.
990
+ */
991
+ async saveReplicatedBlock(blockId: BlockId, block: IBlock, source?: ActionRev, verifiedProof?: BlockCommitProof): Promise<void> {
992
+ log('saveReplicatedBlock blockId=%s rev=%s', blockId, source?.rev);
993
+ const storage = this.createBlockStorage(blockId);
994
+ // Captured under the latch; emitted after release to match commit's ordering.
995
+ let landed: { collectionId: CollectionId, actionId: ActionId, rev: number } | undefined;
996
+ await withBlockWriteLatch(blockId, async (latch) => {
997
+ const priorLatest = await storage.getLatest();
998
+ const effective = await storage.saveReplica(block, source, verifiedProof, latch);
999
+ // Advanced iff there was no prior revision or the effective rev moved past it. On the
1000
+ // monotonic no-op, saveReplica returns the held latest unchanged → effective.rev === priorLatest.rev.
1001
+ const advanced = priorLatest === undefined || effective.rev > priorLatest.rev;
1002
+ const collectionId = block.header?.collectionId;
1003
+ if (advanced && collectionId !== undefined) {
1004
+ landed = { collectionId, actionId: effective.actionId, rev: effective.rev };
1005
+ }
1006
+ if (!advanced && verifiedProof !== undefined && source !== undefined
1007
+ && effective.rev === source.rev && effective.actionId === source.actionId) {
1008
+ // The push named exactly the revision this node already holds, and carried a verified
1009
+ // proof for it. Back-fill so a proof-lessly-landed revision stops being corroboration-only
1010
+ // the moment valid evidence for it arrives. Requires agreement on BOTH rev and actionId:
1011
+ // same rev under a different action is a divergence, not the same revision.
1012
+ //
1013
+ // A held revision NEWER than the pushed one is deliberately not back-filled: `servableProof`
1014
+ // only ever serves the proof for `latest.rev`, so the proof would be keyed to a revision
1015
+ // this node will never serve, for content it may not even materialize.
1016
+ //
1017
+ // Runs under the block write latch already held here — the same latch the commit-path
1018
+ // back-fill sites hold, so no new latch interaction. `backFillProof` never throws: the
1019
+ // revision is already durable, and a proof-persist fault must not turn a no-op into a
1020
+ // failure.
1021
+ //
1022
+ // NOTE: once a proof IS retained this costs one key lookup per duplicate push
1023
+ // (`backFillProof` returns before materializing). A holder whose bytes diverge from the
1024
+ // cohort's never retains one, so it re-materializes and re-hashes the block on EVERY
1025
+ // certified push of that revision. Bounded by push frequency and fine at spread-on-churn
1026
+ // rates; if a diverged holder under repeated push ever shows up in a profile, remember the
1027
+ // withheld `(rev, actionId)` and skip the re-check.
1028
+ await this.backFillProof(blockId, storage, effective.rev, effective.actionId, verifiedProof, latch);
1029
+ }
1030
+ });
1031
+ // Replica-persist has no CommitRequest, hence no tailId — like a read-driven promotion,
1032
+ // this wakes local onCollectionChange watchers but is cert-gated out of cohort-topic
1033
+ // re-origination downstream (change-bridge selfIsCohortMember treats a tail-less event as
1034
+ // never a member).
1035
+ if (landed) {
1036
+ this.emitCollectionChanges(
1037
+ new Map([[landed.collectionId, [blockId]]]),
1038
+ landed.actionId,
1039
+ landed.rev,
1040
+ );
1041
+ }
1042
+ }
1043
+
1044
+ /**
1045
+ * The digest the block WOULD materialize to if `actionId`'s pending transform committed at `rev`,
1046
+ * plus the base revision it was computed from. Read-only: touches no durable state and takes no
1047
+ * block write latch.
1048
+ *
1049
+ * Mirrors {@link internalCommit}'s reads (pending transform → latest → base → applyTransform) so
1050
+ * the prediction and the eventual commit cannot drift. Consumed by the cluster member's
1051
+ * promise-round content-digest check (`ClusterMember.validateCommitOperations`), which compares it
1052
+ * against the digest the transaction author declared on the commit request.
1053
+ *
1054
+ * Deliberately does NOT take the block write latch: this runs on the vote path, ahead of the
1055
+ * commit that will take it, so taking it here would serialize voting behind commits and risks
1056
+ * deadlocking against commit's sorted up-front multi-block latch acquisition. The price is that a
1057
+ * concurrent commit can move `latest` mid-preview; the caller's checkable rule (base-independent,
1058
+ * or `baseRev` agreement) makes a torn read at worst an abstain, never a false reject of honest
1059
+ * content.
1060
+ *
1061
+ * `rev` is accepted for parity/logging with the commit that would follow; materialization does not
1062
+ * depend on it (internalCommit only records it).
1063
+ *
1064
+ * Returns `undefined` when this node holds no pending transform for the action (it never saw the
1065
+ * pend) — distinct from a defined preview with `digest: undefined` (see {@link CommitDigestPreview}).
1066
+ */
1067
+ async previewCommitDigest(blockId: BlockId, actionId: ActionId, rev: number): Promise<CommitDigestPreview | undefined> {
1068
+ const storage = this.createBlockStorage(blockId);
1069
+ const transform = await storage.getPendingTransaction(actionId);
1070
+ if (!transform) {
1071
+ return undefined;
1072
+ }
1073
+
1074
+ // An insert replaces the block wholesale before updates apply, so the result is the same on
1075
+ // every member no matter what base it holds — do not read a base at all (the block may even be
1076
+ // locally wedged/unmaterializable, which must not degrade a base-independent preview).
1077
+ const baseIndependent = transform.insert !== undefined;
1078
+ let base: IBlock | undefined;
1079
+ let baseRev: number | undefined;
1080
+ if (!baseIndependent) {
1081
+ const latest = await storage.getLatest();
1082
+ if (latest) {
1083
+ baseRev = latest.rev;
1084
+ try {
1085
+ base = (await storage.getBlock(latest.rev))?.block;
1086
+ } catch (err) {
1087
+ // This node holds a `latest` it cannot materialize (see readCommitBase). That is a
1088
+ // local deficiency, not a content mismatch — report "cannot check" so the caller
1089
+ // abstains. Unlike the commit path's refuseMissingBase, this must NOT delete the
1090
+ // pending record or throw: preview is read-only and runs before any commit exists.
1091
+ log('previewCommitDigest:unmaterializable-base blockId=%s baseRev=%d rev=%d error=%s',
1092
+ blockId, latest.rev, rev, err instanceof Error ? err.message : String(err));
1093
+ return { baseIndependent: false, baseRev, digest: undefined };
1094
+ }
1095
+ }
1096
+ }
1097
+
1098
+ // Clone both: applyTransform assigns `transform.insert` into the result by reference and
1099
+ // applyOperations mutates the block in place, so materializing on live storage/pending objects
1100
+ // would corrupt them for the real commit that follows.
1101
+ const newBlock = applyTransform(structuredClone(base), structuredClone(transform));
1102
+ // `undefined` covers the tombstone (delete transform) and updates-with-no-base (applyTransform
1103
+ // drops updates when there is no block to apply them to) — both materialize nothing.
1104
+ const digest = newBlock ? await canonicalBlockHash(newBlock) : undefined;
1105
+ return { digest, baseRev, baseIndependent };
1106
+ }
1107
+
1108
+ /**
1109
+ * See {@link IRevisionActionReader}. Reads the block's revision index directly
1110
+ * (`listRevisions(rev, rev)` — both bounds inclusive per the `IBlockStorage` contract); an empty
1111
+ * range means this node holds no record for that revision.
1112
+ */
1113
+ async getRevisionAction(blockId: BlockId, rev: number): Promise<ActionId | undefined> {
1114
+ const storage = this.createBlockStorage(blockId);
1115
+ for await (const actionRev of storage.listRevisions(rev, rev)) {
1116
+ return actionRev.actionId;
1117
+ }
1118
+ return undefined;
1119
+ }
1120
+
1121
+ /**
1122
+ * The {@link BlockCommitProof} this node retained for `blockId` at `rev`, or `undefined` when it
1123
+ * kept none — a revision committed before proofs were persisted, a member whose materialization
1124
+ * diverged from the declared digest (see {@link persistProofIfContentMatches}), or simply a
1125
+ * revision this node never landed.
1126
+ *
1127
+ * Public because a peer answering a block-repair fetch serves the proof alongside the revision
1128
+ * (`serveBlockArchive`), which is the only way a requester can check a lone holder's claim
1129
+ * without a second holder to corroborate it. Read-only and unlatched: a proof is written once
1130
+ * and never mutated, so a concurrent commit can only make this return a proof for a revision
1131
+ * that just became stale — which the caller pairs with the revision it actually read.
1132
+ */
1133
+ async getBlockProof(blockId: BlockId, rev: number): Promise<BlockCommitProof | undefined> {
1134
+ return await this.createBlockStorage(blockId).getBlockProof(rev);
1135
+ }
1136
+
1137
+ private async internalCommit(blockId: BlockId, actionId: ActionId, rev: number, storage: IBlockStorage, latch: BlockWriteLatch, proof?: BlockCommitProof): Promise<CollectionId | undefined> {
1138
+ // Note: This method is called under the block write latch — by commit() (within its locked
1139
+ // critical section) and by the read-driven promotion in get() (which takes the same latch);
1140
+ // `latch` is the proof of that. So, operations like getPendingTransaction, getLatest,
1141
+ // getBlock, saveMaterializedBlock, saveRevision, promotePendingTransaction, setLatest are
1142
+ // protected against concurrent writers for the *same blockId*.
1143
+ //
1144
+ // `getBlock` here (via readCommitBase) is LOCAL-ONLY: the commit path never fetches from a
1145
+ // peer while holding N block latches. A coverage gap reads as a missing base, which the
1146
+ // healing path repairs by replication instead.
1147
+
1148
+ const transform = await storage.getPendingTransaction(actionId);
1149
+ // No need to check if !transform here, as the caller (commit) already verified this.
1150
+ // If it's null here, it indicates a logic error or race condition bypassed the lock (unlikely).
1151
+ if (!transform) {
1152
+ throw new Error(`Consistency Error: Pending action ${actionId} disappeared for block ${blockId} within critical section.`);
1153
+ }
1154
+
1155
+ // Get prior materialized block if it exists
1156
+ const latest = await storage.getLatest();
1157
+ const priorBlock = await this.readCommitBase(blockId, actionId, rev, storage, latest, latch);
1158
+
1159
+ // Apply transform and save materialized block
1160
+ // applyTransform handles undefined priorBlock correctly for inserts
1161
+ const newBlock = applyTransform(priorBlock, transform);
1162
+
1163
+ // INVARIANT: `latest` must never advance past a revision this node can materialize.
1164
+ // `applyTransform` silently drops `updates` when there is no block to apply them to, so a
1165
+ // member that missed the block's CREATING revision would otherwise record rev N while storing
1166
+ // nothing to serve it from. `latest === undefined` is precisely the "nothing below to fall
1167
+ // back to" case: materializeBlock's descending walk needs some materialization at or below the
1168
+ // target, and with no prior revision there is none. With a prior `latest` an absent newBlock is
1169
+ // a legitimate tombstone (the walk resolves to an earlier materialization), so it stays allowed.
1170
+ if (!newBlock && latest === undefined) {
1171
+ return await this.refuseMissingBase(blockId, actionId, rev, storage, latch,
1172
+ 'no committed revision to apply the transform to');
1173
+ }
1174
+
1175
+ if (newBlock) {
1176
+ await storage.saveMaterializedBlock(actionId, newBlock, latch);
1177
+ }
1178
+
1179
+ // Save revision and promote action *before* updating latest
1180
+ // This ensures that if the process crashes between these steps,
1181
+ // the 'latest' pointer doesn't point to a revision that hasn't been fully recorded.
1182
+ await storage.saveRevision(rev, actionId, latch);
1183
+ await storage.promotePendingTransaction(actionId, latch);
1184
+
1185
+ // Update latest revision *last*
1186
+ await storage.setLatest({ actionId, rev }, latch);
1187
+
1188
+ // Persist the cohort's commit proof AFTER the commit is durably latest — the proof is
1189
+ // evidence about a landed revision, never a precondition of landing it. The retention rule
1190
+ // (persist only when the LOCAL materialization matches the digest the commit op declared)
1191
+ // and its failure logging live in the shared helper; a proof-persist fault must not fail a
1192
+ // commit that already landed, so the helper never throws.
1193
+ if (proof !== undefined) {
1194
+ await this.persistProofIfContentMatches(blockId, actionId, rev, storage, proof, newBlock, latch);
1195
+ }
1196
+
1197
+ // Prune the now-superseded prior materialization (checkpoint retention). Runs LAST — after the
1198
+ // new rev's materialization + revision + transform + setLatest are all durable — so no crash
1199
+ // point can leave a rev unrecoverable: a crash BEFORE this leaves a redundant (harmless)
1200
+ // materialization the next commit's prune reclaims; a crash AFTER is fully consistent. The prune
1201
+ // only ever deletes a materialization reconstructible from the retained floor + transforms. Runs
1202
+ // under the block write latch already held here, so it serializes against concurrent commits.
1203
+ // NOTE: prune targets ONLY the immediate prior. A crash between setLatest and this call leaves that
1204
+ // one prior materialization un-pruned; since a later commit prunes ITS OWN prior (never the earlier
1205
+ // leaked rev), that copy is NOT auto-reclaimed — a bounded (≤1 block-copy per crash), harmless leak
1206
+ // (state stays consistent + reconstructible). If crash-before-prune leaks ever accumulate materially,
1207
+ // add a bounded look-back (prune non-retained mats in [rev-checkpointInterval, rev)) here, or a
1208
+ // periodic reconciliation sweep — do NOT reintroduce a per-read re-cache.
1209
+ if (latest !== undefined) {
1210
+ await storage.pruneSupersededMaterialization(latest, latch);
1211
+ }
1212
+
1213
+ // Report the affected collection for change-event routing. For a delete the
1214
+ // materialized block is undefined, so fall back to the prior block's header.
1215
+ // Either may be absent only for a malformed/headerless block — return
1216
+ // undefined so the caller skips it rather than emitting a bogus event.
1217
+ return newBlock?.header.collectionId ?? priorBlock?.header.collectionId;
1218
+ }
1219
+
1220
+ /**
1221
+ * Retain `proof` for a block this call found ALREADY committed at `(rev, actionId)` — the paths
1222
+ * that land (or find already landed) a revision without running {@link internalCommit}, and would
1223
+ * otherwise never retain a proof: the idempotent re-commit partition, the Crash-D3 `recover()`
1224
+ * partition, and {@link saveReplicatedBlock}'s monotonic no-op on a certified push. Strictly
1225
+ * additive: an existing proof is left alone, and the same digest-match rule as the fresh-commit
1226
+ * site decides retention.
1227
+ *
1228
+ * `rev`/`actionId` are passed separately rather than as a `CommitRequest` because the replica
1229
+ * caller has no commit request — it has the `(rev, actionId)` the push and the held revision
1230
+ * agree on.
1231
+ *
1232
+ * Callers must hold the block's write latch (`latch`). `getBlock` is local-only and can throw on
1233
+ * an unmaterializable or uncovered base — treated as "no local content", i.e. the proof is withheld.
1234
+ */
1235
+ private async backFillProof(
1236
+ blockId: BlockId, storage: IBlockStorage, rev: number, actionId: ActionId, proof: BlockCommitProof | undefined,
1237
+ latch: BlockWriteLatch
1238
+ ): Promise<void> {
1239
+ if (proof === undefined || await storage.getBlockProof(rev) !== undefined) {
1240
+ return;
1241
+ }
1242
+ let committedBlock: IBlock | undefined;
1243
+ try {
1244
+ committedBlock = (await storage.getBlock(rev))?.block;
1245
+ } catch {
1246
+ committedBlock = undefined;
1247
+ }
1248
+ await this.persistProofIfContentMatches(blockId, actionId, rev, storage, proof, committedBlock, latch);
1249
+ }
1250
+
1251
+ /**
1252
+ * The single retention rule for {@link BlockCommitProof}s, shared by the fresh-commit site
1253
+ * ({@link internalCommit}, after `setLatest`) and the already-landed back-fill
1254
+ * ({@link backFillProof}):
1255
+ *
1256
+ * > **A member persists the proof only when its own materialization matches the digest the
1257
+ * > commit operation declared for this block.**
1258
+ *
1259
+ * One rule covers every awkward case without a second flag: a DIVERGED member (committed onto a
1260
+ * lagging base) computes a different hash, stores no proof, and falls back to corroboration
1261
+ * exactly as today — the `commit:proof-digest-mismatch` log line is also the first signal this
1262
+ * system has ever had that a member diverged. A member that abstained at vote time still checks
1263
+ * here (by commit time it HAS materialized) and legitimately keeps the proof on agreement. A
1264
+ * tombstone (no `block`) and a commit with no `blockDigests` (pre-upgrade client) declare no
1265
+ * digest and store no proof (`commit:proof-undeclared`).
1266
+ *
1267
+ * Never throws: the commit this proof describes already durably landed, so a proof-persist
1268
+ * fault must not turn `commit()` into `success:false` for a landed commit — it is logged and
1269
+ * the proof simply is not retained (repair falls back to corroboration).
1270
+ *
1271
+ * NOTE: one commit of N blocks stores the SAME proof under each block's `(blockId, rev)` proofs-store key, and
1272
+ * the proof itself carries the commit op's N `blockIds`/`blockDigests` — so bytes retained per
1273
+ * commit grow with N². Measured base cost is ~4.6 KB for a 10-peer 2-block commit
1274
+ * (`test/commit-proof.spec.ts` "size"), and nothing today bounds `CommitRequest.blockIds`. Fine
1275
+ * at the handful-of-blocks batches the transactor produces now; if per-coordinator batches ever
1276
+ * grow large, store the proof once under its `messageHash` and key each revision to a pointer.
1277
+ */
1278
+ private async persistProofIfContentMatches(
1279
+ blockId: BlockId,
1280
+ actionId: ActionId,
1281
+ rev: number,
1282
+ storage: IBlockStorage,
1283
+ proof: BlockCommitProof,
1284
+ block: IBlock | undefined,
1285
+ latch: BlockWriteLatch
1286
+ ): Promise<void> {
1287
+ try {
1288
+ const declaredDigest = proofDeclaredDigest(proof, { blockId, rev, actionId });
1289
+ if (declaredDigest === undefined) {
1290
+ log('commit:proof-undeclared blockId=%s rev=%d actionId=%s', blockId, rev, actionId);
1291
+ return;
1292
+ }
1293
+ // A digest was declared but this node materialized nothing (tombstone / unmaterializable
1294
+ // read on the back-fill path): the local content provably is not the declared content.
1295
+ const localDigest = block === undefined ? undefined : await canonicalBlockHash(block);
1296
+ if (localDigest !== declaredDigest) {
1297
+ log('commit:proof-digest-mismatch blockId=%s rev=%d actionId=%s declared=%s local=%s',
1298
+ blockId, rev, actionId, declaredDigest, localDigest);
1299
+ return;
1300
+ }
1301
+ await storage.saveBlockProof(rev, proof, latch);
1302
+ } catch (err) {
1303
+ log('commit:proof-persist-failed blockId=%s rev=%d actionId=%s error=%s', blockId, rev, actionId,
1304
+ err instanceof Error ? err.message : String(err));
1305
+ }
1306
+ }
1307
+
1308
+ /**
1309
+ * The materialization this commit builds on: the block at `latest`, or `undefined` when the block
1310
+ * holds no committed revision yet (the normal insert case).
1311
+ *
1312
+ * `getBlock` THROWS when this node holds a `latest` it cannot materialize — a block already wedged
1313
+ * by a pre-fix commit, or by truncated history. That is the same divergence as having no base at
1314
+ * all, so it is translated into {@link MissingBaseRevisionError} rather than surfacing as an opaque
1315
+ * storage fault: the healing path can then repair the block instead of the fault resetting the
1316
+ * cluster stream, and a wedged node recovers on the next write touching the block.
1317
+ *
1318
+ * The catch is deliberately UNNARROWED — it also absorbs a transient fault (a raw-storage read
1319
+ * error, a `restoreCallback` timeout on a block whose `ranges` do not cover its own `latest`).
1320
+ * BlockStorage reports every one of these as a bare `Error`, so they cannot be told apart here,
1321
+ * and treating them as divergence is the safe default: this node genuinely cannot materialize the
1322
+ * base right now, and the cluster's policy is to heal rather than throw out of consensus. The
1323
+ * price is that a transient fault ALSO drops pending records — this block's (see
1324
+ * {@link refuseMissingBase}) AND, because {@link commit} keys its cleanup off the same error type,
1325
+ * every not-yet-reached block in the same batch — so those blocks converge by replication instead
1326
+ * of by a replay the retry could have done. That is a wider blast radius than the per-block
1327
+ * refusal alone, and it is why the discriminator must NOT be loosened beyond this error type.
1328
+ * Narrowing this would require typed faults out of BlockStorage; until then, prefer the tolerant
1329
+ * reading.
1330
+ */
1331
+ private async readCommitBase(
1332
+ blockId: BlockId,
1333
+ actionId: ActionId,
1334
+ rev: number,
1335
+ storage: IBlockStorage,
1336
+ latest: ActionRev | undefined,
1337
+ latch: BlockWriteLatch
1338
+ ): Promise<IBlock | undefined> {
1339
+ if (!latest) {
1340
+ return undefined;
1341
+ }
1342
+ // NOTE: this read is deliberately LOCAL-ONLY and does not heal. `getBlock` no longer restores
1343
+ // from a peer (that moved to the explicit `restoreRevision`, which `StorageRepo.get` calls), so a
1344
+ // base this node cannot materialize locally raises {@link MissingBaseRevisionError} here instead
1345
+ // of being fetched in line. The reason is the calling context, not the cost of a fetch: `commit`
1346
+ // holds the write latch of EVERY block in the batch across this call, and network I/O inside that
1347
+ // critical section makes one unreachable peer stall every writer of every block in the batch for
1348
+ // the length of a round trip. Healing is out-of-band instead — cohort reconcile supplies the
1349
+ // revision (`ClusterMember` → `saveReplicatedBlock`) and the action is retried, by which point
1350
+ // this read succeeds locally. Pinned by `test/storage-repo.spec.ts` "commit reads its base
1351
+ // locally", which wires a restore callback that would have answered and asserts it is never
1352
+ // called. Do not reintroduce a restore on this path; if a commit ever genuinely needs one, fetch
1353
+ // BEFORE taking the latches, not underneath them.
1354
+ //
1355
+ // NOTE: `latest.rev` is always inside `meta.ranges` today — every writer of `latest`
1356
+ // (`setLatest`, `saveForwardRevision`, `recover`) merges an open-ended range anchored at or
1357
+ // below the new latest in the same `saveMetadata` — so the RevisionNotCoveredError arm below
1358
+ // is unreachable from here and only truncated-history corruption lands in the catch. If a
1359
+ // future change can leave `latest` uncovered, the ordering in `get` becomes load-bearing: the
1360
+ // read-driven promotion runs BEFORE `readBlockHealing`, so a coverage gap under `latest` would
1361
+ // make `refuseMissingBase` delete the pending record moments before the healing read would
1362
+ // have restored it. Heal before refusing if that day comes.
1363
+ try {
1364
+ return (await storage.getBlock(latest.rev))?.block;
1365
+ } catch (err) {
1366
+ log('commit:unmaterializable-base blockId=%s baseRev=%d error=%s', blockId, latest.rev,
1367
+ err instanceof Error ? err.message : String(err));
1368
+ return await this.refuseMissingBase(blockId, actionId, rev, storage, latch,
1369
+ `local rev ${latest.rev} is not materializable here`);
1370
+ }
1371
+ }
1372
+
1373
+ /**
1374
+ * Refuse a commit this node cannot materialize. Always throws {@link MissingBaseRevisionError};
1375
+ * nothing durable has been written at this point, so the block is left exactly as it was minus the
1376
+ * pending record.
1377
+ *
1378
+ * The pending is dropped because it can never be promoted here: promotion needs a base this node
1379
+ * must obtain out-of-band, and once the healing path lands that revision `latest` is already >= rev,
1380
+ * so a commit retry partitions the block as already-done/stale and never revisits the pending.
1381
+ * Leaving it would also report a phantom conflicting action from {@link pend} for every later write.
1382
+ */
1383
+ private async refuseMissingBase(
1384
+ blockId: BlockId,
1385
+ actionId: ActionId,
1386
+ rev: number,
1387
+ storage: IBlockStorage,
1388
+ latch: BlockWriteLatch,
1389
+ detail: string
1390
+ ): Promise<never> {
1391
+ await storage.deletePendingTransaction(actionId, latch);
1392
+ log('commit:missing-base blockId=%s rev=%d actionId=%s detail=%s', blockId, rev, actionId, detail);
1393
+ throw new MissingBaseRevisionError(blockId, rev, detail);
1394
+ }
1395
+ }
1396
+
1397
+ /**
1398
+ * Converts list of missing actions per block into a list of missing actions across blocks.
1399
+ *
1400
+ * NOTE: relies on each (actionId, blockId) pair appearing at most once — one revision per action
1401
+ * per block. If a block ever records two revisions under the same actionId, concatTransform now
1402
+ * concatenates both revisions' ops into one array rather than dropping the earlier one — still
1403
+ * wrong, since ops from distinct revisions are not composable against a single base, but loud
1404
+ * rather than silent. Group by (actionId, rev) instead if that case becomes reachable.
1405
+ */
1406
+ function perBlockActionTransformsToPerAction(missing: { blockId: BlockId; transforms: ActionTransform[]; }[]) {
1407
+ const missingFlat = missing.flatMap(({ blockId, transforms }) =>
1408
+ transforms.map(transform => ({ blockId, transform }))
1409
+ );
1410
+ const missingByActionId = groupBy(missingFlat, ({ transform }) => transform.actionId);
1411
+ return Object.entries(missingByActionId).map(([actionId, items]) =>
1412
+ items.reduce((acc, { blockId, transform }) => {
1413
+ acc.transforms = concatTransform(acc.transforms, blockId, transform.transform);
1414
+ return acc;
1415
+ }, {
1416
+ actionId: actionId as ActionId,
1417
+ rev: items[0]!.transform.rev, // Assumption: an action commits at one revision, so every block's entry for this actionId agrees. Distinct actionIds may still carry distinct revs.
1418
+ transforms: emptyTransforms()
1419
+ })
1420
+ );
1421
+ }