@optimystic/db-p2p 1.0.0-beta.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (234) hide show
  1. package/dist/src/cluster/block-transfer-service.d.ts +73 -1
  2. package/dist/src/cluster/block-transfer-service.d.ts.map +1 -1
  3. package/dist/src/cluster/block-transfer-service.js +70 -3
  4. package/dist/src/cluster/block-transfer-service.js.map +1 -1
  5. package/dist/src/cluster/block-transfer.d.ts +16 -1
  6. package/dist/src/cluster/block-transfer.d.ts.map +1 -1
  7. package/dist/src/cluster/block-transfer.js +73 -72
  8. package/dist/src/cluster/block-transfer.js.map +1 -1
  9. package/dist/src/cluster/client.d.ts +4 -6
  10. package/dist/src/cluster/client.d.ts.map +1 -1
  11. package/dist/src/cluster/client.js +8 -13
  12. package/dist/src/cluster/client.js.map +1 -1
  13. package/dist/src/cluster/cluster-policy.d.ts +2 -1
  14. package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
  15. package/dist/src/cluster/cluster-policy.js.map +1 -1
  16. package/dist/src/cluster/cluster-repo.d.ts +172 -17
  17. package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
  18. package/dist/src/cluster/cluster-repo.js +553 -149
  19. package/dist/src/cluster/cluster-repo.js.map +1 -1
  20. package/dist/src/cluster/commit-proof.js +2 -2
  21. package/dist/src/cluster/race-resolution.d.ts +51 -7
  22. package/dist/src/cluster/race-resolution.d.ts.map +1 -1
  23. package/dist/src/cluster/race-resolution.js +71 -7
  24. package/dist/src/cluster/race-resolution.js.map +1 -1
  25. package/dist/src/cluster/rebalance-monitor.d.ts +67 -5
  26. package/dist/src/cluster/rebalance-monitor.d.ts.map +1 -1
  27. package/dist/src/cluster/rebalance-monitor.js +120 -26
  28. package/dist/src/cluster/rebalance-monitor.js.map +1 -1
  29. package/dist/src/cluster/spread-on-churn.d.ts.map +1 -1
  30. package/dist/src/cluster/spread-on-churn.js +34 -47
  31. package/dist/src/cluster/spread-on-churn.js.map +1 -1
  32. package/dist/src/cohort-topic/host.d.ts.map +1 -1
  33. package/dist/src/cohort-topic/host.js +8 -6
  34. package/dist/src/cohort-topic/host.js.map +1 -1
  35. package/dist/src/cohort-topic/membership-source.js +1 -1
  36. package/dist/src/cohort-topic/membership-source.js.map +1 -1
  37. package/dist/src/cohort-topic/peer-codec.d.ts.map +1 -1
  38. package/dist/src/cohort-topic/peer-codec.js +5 -2
  39. package/dist/src/cohort-topic/peer-codec.js.map +1 -1
  40. package/dist/src/cohort-topic/stream-util.d.ts +24 -3
  41. package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
  42. package/dist/src/cohort-topic/stream-util.js +36 -3
  43. package/dist/src/cohort-topic/stream-util.js.map +1 -1
  44. package/dist/src/cohort-topic/topic-router.d.ts +9 -0
  45. package/dist/src/cohort-topic/topic-router.d.ts.map +1 -1
  46. package/dist/src/cohort-topic/topic-router.js +12 -2
  47. package/dist/src/cohort-topic/topic-router.js.map +1 -1
  48. package/dist/src/dispute/client.d.ts.map +1 -1
  49. package/dist/src/dispute/client.js +15 -3
  50. package/dist/src/dispute/client.js.map +1 -1
  51. package/dist/src/index.d.ts +5 -0
  52. package/dist/src/index.d.ts.map +1 -1
  53. package/dist/src/index.js +5 -0
  54. package/dist/src/index.js.map +1 -1
  55. package/dist/src/libp2p-key-network.d.ts +78 -17
  56. package/dist/src/libp2p-key-network.d.ts.map +1 -1
  57. package/dist/src/libp2p-key-network.js +241 -147
  58. package/dist/src/libp2p-key-network.js.map +1 -1
  59. package/dist/src/libp2p-node-base.d.ts +35 -1
  60. package/dist/src/libp2p-node-base.d.ts.map +1 -1
  61. package/dist/src/libp2p-node-base.js +151 -26
  62. package/dist/src/libp2p-node-base.js.map +1 -1
  63. package/dist/src/libp2p-node-rn.d.ts +2 -2
  64. package/dist/src/libp2p-node-rn.d.ts.map +1 -1
  65. package/dist/src/libp2p-node-rn.js.map +1 -1
  66. package/dist/src/libp2p-node.d.ts +2 -2
  67. package/dist/src/libp2p-node.d.ts.map +1 -1
  68. package/dist/src/libp2p-node.js.map +1 -1
  69. package/dist/src/logger.d.ts.map +1 -1
  70. package/dist/src/logger.js +19 -7
  71. package/dist/src/logger.js.map +1 -1
  72. package/dist/src/matchmaking/query-transport.d.ts +3 -4
  73. package/dist/src/matchmaking/query-transport.d.ts.map +1 -1
  74. package/dist/src/matchmaking/query-transport.js +9 -9
  75. package/dist/src/matchmaking/query-transport.js.map +1 -1
  76. package/dist/src/network/network-manager-service.d.ts +10 -5
  77. package/dist/src/network/network-manager-service.d.ts.map +1 -1
  78. package/dist/src/network/network-manager-service.js +11 -2
  79. package/dist/src/network/network-manager-service.js.map +1 -1
  80. package/dist/src/network/relay-reservation.d.ts +170 -0
  81. package/dist/src/network/relay-reservation.d.ts.map +1 -0
  82. package/dist/src/network/relay-reservation.js +527 -0
  83. package/dist/src/network/relay-reservation.js.map +1 -0
  84. package/dist/src/pend-validation.d.ts +3 -2
  85. package/dist/src/pend-validation.d.ts.map +1 -1
  86. package/dist/src/pend-validation.js +3 -2
  87. package/dist/src/pend-validation.js.map +1 -1
  88. package/dist/src/protocol-client.d.ts +8 -1
  89. package/dist/src/protocol-client.d.ts.map +1 -1
  90. package/dist/src/protocol-client.js +11 -0
  91. package/dist/src/protocol-client.js.map +1 -1
  92. package/dist/src/reactivity/forwarder-host.d.ts +2 -1
  93. package/dist/src/reactivity/forwarder-host.d.ts.map +1 -1
  94. package/dist/src/reactivity/forwarder-host.js +2 -1
  95. package/dist/src/reactivity/forwarder-host.js.map +1 -1
  96. package/dist/src/reactivity/origination-manager.d.ts +3 -3
  97. package/dist/src/reactivity/origination-manager.js +3 -3
  98. package/dist/src/reactivity/recover-transport.d.ts +22 -14
  99. package/dist/src/reactivity/recover-transport.d.ts.map +1 -1
  100. package/dist/src/reactivity/recover-transport.js +32 -19
  101. package/dist/src/reactivity/recover-transport.js.map +1 -1
  102. package/dist/src/reactivity/subscription-manager.d.ts +2 -2
  103. package/dist/src/reactivity/topic-bytes.d.ts +6 -7
  104. package/dist/src/reactivity/topic-bytes.d.ts.map +1 -1
  105. package/dist/src/reactivity/topic-bytes.js +8 -9
  106. package/dist/src/reactivity/topic-bytes.js.map +1 -1
  107. package/dist/src/repo/client.d.ts +5 -2
  108. package/dist/src/repo/client.d.ts.map +1 -1
  109. package/dist/src/repo/client.js +49 -40
  110. package/dist/src/repo/client.js.map +1 -1
  111. package/dist/src/repo/cluster-coordinator.d.ts +119 -5
  112. package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
  113. package/dist/src/repo/cluster-coordinator.js +181 -29
  114. package/dist/src/repo/cluster-coordinator.js.map +1 -1
  115. package/dist/src/repo/coordinator-repo.d.ts +259 -76
  116. package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
  117. package/dist/src/repo/coordinator-repo.js +716 -328
  118. package/dist/src/repo/coordinator-repo.js.map +1 -1
  119. package/dist/src/repo/i-under-replication-ledger.d.ts +87 -0
  120. package/dist/src/repo/i-under-replication-ledger.d.ts.map +1 -0
  121. package/dist/src/repo/i-under-replication-ledger.js +2 -0
  122. package/dist/src/repo/i-under-replication-ledger.js.map +1 -0
  123. package/dist/src/repo/kv-under-replication-ledger.d.ts +91 -0
  124. package/dist/src/repo/kv-under-replication-ledger.d.ts.map +1 -0
  125. package/dist/src/repo/kv-under-replication-ledger.js +287 -0
  126. package/dist/src/repo/kv-under-replication-ledger.js.map +1 -0
  127. package/dist/src/repo/responsibility.d.ts +37 -0
  128. package/dist/src/repo/responsibility.d.ts.map +1 -0
  129. package/dist/src/repo/responsibility.js +37 -0
  130. package/dist/src/repo/responsibility.js.map +1 -0
  131. package/dist/src/repo/service.d.ts +44 -15
  132. package/dist/src/repo/service.d.ts.map +1 -1
  133. package/dist/src/repo/service.js +64 -27
  134. package/dist/src/repo/service.js.map +1 -1
  135. package/dist/src/repo/stuck-reservation.d.ts +127 -0
  136. package/dist/src/repo/stuck-reservation.d.ts.map +1 -0
  137. package/dist/src/repo/stuck-reservation.js +179 -0
  138. package/dist/src/repo/stuck-reservation.js.map +1 -0
  139. package/dist/src/repo/under-replication-drain.d.ts +208 -0
  140. package/dist/src/repo/under-replication-drain.d.ts.map +1 -0
  141. package/dist/src/repo/under-replication-drain.js +461 -0
  142. package/dist/src/repo/under-replication-drain.js.map +1 -0
  143. package/dist/src/rn.d.ts +5 -0
  144. package/dist/src/rn.d.ts.map +1 -1
  145. package/dist/src/rn.js +5 -0
  146. package/dist/src/rn.js.map +1 -1
  147. package/dist/src/storage/block-latch.d.ts +3 -2
  148. package/dist/src/storage/block-latch.d.ts.map +1 -1
  149. package/dist/src/storage/block-latch.js +7 -4
  150. package/dist/src/storage/block-latch.js.map +1 -1
  151. package/dist/src/storage/block-storage.d.ts +45 -3
  152. package/dist/src/storage/block-storage.d.ts.map +1 -1
  153. package/dist/src/storage/block-storage.js +156 -9
  154. package/dist/src/storage/block-storage.js.map +1 -1
  155. package/dist/src/storage/i-block-storage.d.ts +57 -5
  156. package/dist/src/storage/i-block-storage.d.ts.map +1 -1
  157. package/dist/src/storage/i-block-storage.js.map +1 -1
  158. package/dist/src/storage/pending-claim.d.ts +167 -0
  159. package/dist/src/storage/pending-claim.d.ts.map +1 -0
  160. package/dist/src/storage/pending-claim.js +156 -0
  161. package/dist/src/storage/pending-claim.js.map +1 -0
  162. package/dist/src/storage/raw-store-codec.d.ts +13 -0
  163. package/dist/src/storage/raw-store-codec.d.ts.map +1 -1
  164. package/dist/src/storage/raw-store-codec.js +24 -3
  165. package/dist/src/storage/raw-store-codec.js.map +1 -1
  166. package/dist/src/storage/restoration-coordinator.d.ts.map +1 -1
  167. package/dist/src/storage/restoration-coordinator.js +2 -1
  168. package/dist/src/storage/restoration-coordinator.js.map +1 -1
  169. package/dist/src/storage/ring-shift-coordinator.d.ts.map +1 -1
  170. package/dist/src/storage/ring-shift-coordinator.js +3 -3
  171. package/dist/src/storage/ring-shift-coordinator.js.map +1 -1
  172. package/dist/src/storage/storage-repo.d.ts +137 -5
  173. package/dist/src/storage/storage-repo.d.ts.map +1 -1
  174. package/dist/src/storage/storage-repo.js +273 -68
  175. package/dist/src/storage/storage-repo.js.map +1 -1
  176. package/dist/src/storage/struct.d.ts +58 -0
  177. package/dist/src/storage/struct.d.ts.map +1 -1
  178. package/dist/src/testing/mesh-harness.d.ts +92 -11
  179. package/dist/src/testing/mesh-harness.d.ts.map +1 -1
  180. package/dist/src/testing/mesh-harness.js +246 -103
  181. package/dist/src/testing/mesh-harness.js.map +1 -1
  182. package/package.json +2 -2
  183. package/readme.md +27 -3
  184. package/src/cluster/block-transfer-service.ts +132 -4
  185. package/src/cluster/block-transfer.ts +79 -84
  186. package/src/cluster/client.ts +115 -119
  187. package/src/cluster/cluster-policy.ts +2 -1
  188. package/src/cluster/cluster-repo.ts +3115 -2671
  189. package/src/cluster/commit-proof.ts +2 -2
  190. package/src/cluster/race-resolution.ts +76 -7
  191. package/src/cluster/rebalance-monitor.ts +677 -539
  192. package/src/cluster/spread-on-churn.ts +34 -54
  193. package/src/cohort-topic/host.ts +8 -6
  194. package/src/cohort-topic/membership-source.ts +1 -1
  195. package/src/cohort-topic/peer-codec.ts +34 -31
  196. package/src/cohort-topic/stream-util.ts +39 -4
  197. package/src/cohort-topic/topic-router.ts +12 -2
  198. package/src/dispute/client.ts +17 -7
  199. package/src/index.ts +64 -59
  200. package/src/libp2p-key-network.ts +1377 -1253
  201. package/src/libp2p-node-base.ts +186 -27
  202. package/src/libp2p-node-rn.ts +31 -30
  203. package/src/libp2p-node.ts +37 -36
  204. package/src/logger.ts +20 -7
  205. package/src/matchmaking/query-transport.ts +9 -9
  206. package/src/network/network-manager-service.ts +16 -5
  207. package/src/network/relay-reservation.ts +636 -0
  208. package/src/pend-validation.ts +3 -2
  209. package/src/protocol-client.ts +13 -1
  210. package/src/reactivity/forwarder-host.ts +2 -1
  211. package/src/reactivity/origination-manager.ts +3 -3
  212. package/src/reactivity/recover-transport.ts +39 -22
  213. package/src/reactivity/subscription-manager.ts +2 -2
  214. package/src/reactivity/topic-bytes.ts +35 -37
  215. package/src/repo/client.ts +172 -173
  216. package/src/repo/cluster-coordinator.ts +1296 -1113
  217. package/src/repo/coordinator-repo.ts +3087 -2687
  218. package/src/repo/i-under-replication-ledger.ts +89 -0
  219. package/src/repo/kv-under-replication-ledger.ts +302 -0
  220. package/src/repo/responsibility.ts +48 -0
  221. package/src/repo/service.ts +76 -39
  222. package/src/repo/stuck-reservation.ts +224 -0
  223. package/src/repo/under-replication-drain.ts +582 -0
  224. package/src/rn.ts +74 -69
  225. package/src/storage/block-latch.ts +7 -4
  226. package/src/storage/block-storage.ts +161 -10
  227. package/src/storage/i-block-storage.ts +60 -5
  228. package/src/storage/pending-claim.ts +200 -0
  229. package/src/storage/raw-store-codec.ts +26 -3
  230. package/src/storage/restoration-coordinator.ts +212 -211
  231. package/src/storage/ring-shift-coordinator.ts +271 -271
  232. package/src/storage/storage-repo.ts +311 -73
  233. package/src/storage/struct.ts +58 -0
  234. package/src/testing/mesh-harness.ts +329 -115
@@ -1,9 +1,11 @@
1
- import { transformForBlockId, applyTransform, groupBy, concatTransform, emptyTransforms, blockIdsForTransforms, transformsFromTransform, highestStaleAt, isOwnRevision, canonicalBlockHash } from "@optimystic/db-core";
1
+ import { transformForBlockId, applyTransform, groupBy, concatTransform, emptyTransforms, blockIdsForTransforms, transformsFromTransform, highestStaleAt, isOwnRevision, canonicalBlockHash, localDurability } from "@optimystic/db-core";
2
2
  import { asyncIteratorToArray } from "../it-utility.js";
3
+ import { isReservationAgainst, isBaseIndependent, declaredBaseFor } from "./pending-claim.js";
3
4
  import { proofDeclaredDigest } from "../cluster/commit-proof.js";
4
5
  import { RevisionNotCoveredError } from "./i-block-storage.js";
5
6
  import { acquireBlockWriteLatches, withBlockWriteLatch } from "./block-latch.js";
6
7
  import { createLogger } from "../logger.js";
8
+ import { cloneDecoded } from "./raw-store-codec.js";
7
9
  import { checkPendValidation } from "../pend-validation.js";
8
10
  const log = createLogger('storage-repo');
9
11
  /**
@@ -45,6 +47,23 @@ export class MissingBaseRevisionError extends Error {
45
47
  export function isMissingBaseRevisionFailure(result) {
46
48
  return !result.success && (result.reason?.startsWith(MISSING_BASE_REVISION_REASON) ?? false);
47
49
  }
50
+ /**
51
+ * Stable, greppable prefix on the failure reason `CoordinatorRepo.commit` answers with when a commit
52
+ * assembled consensus but FEWER than a majority of the cohort reported durably holding the committed
53
+ * revision afterwards. Same convention as {@link MISSING_BASE_REVISION_REASON}: a string marker,
54
+ * because the reason crosses the wire as `StaleFailure.reason` prose. The refusal is retryable
55
+ * (`conflict: true`) and means "not confirmed durable at a quorum" — never "guaranteed absent"; see
56
+ * the durability gate in `CoordinatorRepo.commit` for the two-phase ambiguity that wording covers.
57
+ */
58
+ export const COMMIT_NOT_DURABLE_REASON = 'commit-not-durable';
59
+ /**
60
+ * True when a {@link CommitResult} was refused by the coordinator's durability gate — consensus was
61
+ * reached but no durable majority reported holding the revision. Sibling of
62
+ * {@link isMissingBaseRevisionFailure}, for callers that need to tell this refusal from a stale loss.
63
+ */
64
+ export function isCommitNotDurableFailure(result) {
65
+ return !result.success && (result.reason?.startsWith(COMMIT_NOT_DURABLE_REASON) ?? false);
66
+ }
48
67
  export class StorageRepo {
49
68
  createBlockStorage;
50
69
  validatePend;
@@ -53,6 +72,9 @@ export class StorageRepo {
53
72
  changeListeners = new Map();
54
73
  /** Catch-all change listeners — fire for EVERY collection's commit on this node. */
55
74
  anyChangeListeners = new Set();
75
+ /** Full-replication listeners — fire when a block this node acknowledged below `full` has
76
+ * reached every cohort member. See {@link IBlockDurabilityNotifier}. */
77
+ durabilityListeners = new Set();
56
78
  constructor(createBlockStorage, options) {
57
79
  this.createBlockStorage = createBlockStorage;
58
80
  this.validatePend = options?.validatePend;
@@ -138,7 +160,34 @@ export class StorageRepo {
138
160
  }
139
161
  }
140
162
  }
141
- async get({ blockIds, context }, _options) {
163
+ /** Subscribe to full-replication events. See {@link IBlockDurabilityNotifier}. */
164
+ onBlockDurabilityReached(listener) {
165
+ this.durabilityListeners.add(listener);
166
+ let unsubscribed = false;
167
+ return () => {
168
+ if (unsubscribed)
169
+ return;
170
+ unsubscribed = true;
171
+ this.durabilityListeners.delete(listener);
172
+ };
173
+ }
174
+ /**
175
+ * Fire one {@link BlockDurabilityReachedEvent} to every subscriber. The producer is the
176
+ * under-replication drain, which reaches this through a one-method sink the node hands it and
177
+ * calls it only AFTER the block's ledger entry is gone. Same listener isolation as
178
+ * {@link fireChangeListeners}: a throwing listener is logged and the rest still run.
179
+ */
180
+ emitBlockDurabilityReached(event) {
181
+ for (const listener of Array.from(this.durabilityListeners)) {
182
+ try {
183
+ listener(event);
184
+ }
185
+ catch (err) {
186
+ log('onBlockDurabilityReached listener threw for blocks=%o: %o', event.blockIds, err);
187
+ }
188
+ }
189
+ }
190
+ async get({ blockIds, context, lineageOf }, _options) {
142
191
  const distinctBlockIds = Array.from(new Set(blockIds));
143
192
  log('get blockIds=%d', distinctBlockIds.length);
144
193
  // Read-driven promotions that land durably here, captured so we can emit a
@@ -184,22 +233,38 @@ export class StorageRepo {
184
233
  // `context.committed` array, and an in-place `.sort()` would reorder the shared
185
234
  // request context under the caller's feet.
186
235
  //
187
- // NOTE: this loop SKIPS an entry whose pending it does not hold and promotes the
188
- // next one anyway so a member missing an intermediate revision that touched THIS
189
- // block forks it exactly as an un-guarded commit would. `internalCommit`'s declared-
190
- // base guard cannot help here: `context.committed` is a collection-level list of
191
- // (actionId, rev) with no per-block base, and "no pending for that action" is the
192
- // normal case for the many actions that never touched this block. Closing it needs
193
- // the authored base stored WITH the pended transform tracked by
194
- // `backlog/bug-a-pended-transform-does-not-carry-its-base`.
236
+ // The loop skips an entry whose pending record it does not hold the normal case
237
+ // for the many actions that never touched this block so on its own it would
238
+ // promote the record after a missed change straight over the stale copy. What
239
+ // stops that is the base each record's pend carried (`PendingClaim.baseRev`):
240
+ // `mayPromoteOnRead` applies a record only to the exact revision its operations
241
+ // were computed against and DECLINES otherwise, leaving the record and `latest`
242
+ // untouched and ending the walk for this block (each later entry builds on this
243
+ // one). No commit declaration is needed, which is the point: there is no commit
244
+ // request on this path.
195
245
  try {
196
246
  for (const { actionId, rev } of [...missing].sort((a, b) => a.rev - b.rev)) {
197
247
  const pending = await blockStorage.getPendingTransaction(actionId);
198
- if (pending) {
199
- const collectionId = await this.internalCommit(blockId, actionId, rev, blockStorage, latch);
200
- if (collectionId !== undefined) {
201
- promotions.push({ collectionId, blockId, actionId, rev });
248
+ if (!pending) {
249
+ continue;
250
+ }
251
+ // Re-read per entry: the previous iteration may have just promoted the base this one needs.
252
+ const held = await blockStorage.getLatest();
253
+ if (!(await this.mayPromoteOnRead(blockId, blockStorage, actionId, pending, held))) {
254
+ // A decline is not a refusal: the record stays, and the committed content
255
+ // served below is real, merely behind — the reader's floors and the
256
+ // coordinator's read-repair own "behind", so no flag. The one exception is
257
+ // a block this node holds NO committed revision of: the answer below would
258
+ // be an absent that this node's own record contradicts, so it is flagged as
259
+ // a guess rather than posing as "never existed".
260
+ if (held === undefined) {
261
+ unavailable = 'unmaterializable';
202
262
  }
263
+ break;
264
+ }
265
+ const collectionId = await this.internalCommit(blockId, actionId, rev, blockStorage, latch);
266
+ if (collectionId !== undefined) {
267
+ promotions.push({ collectionId, blockId, actionId, rev });
203
268
  }
204
269
  }
205
270
  }
@@ -208,7 +273,9 @@ export class StorageRepo {
208
273
  // can be promoted here (each builds on the one before). Leave `latest` where it
209
274
  // is — the invariant internalCommit just enforced — and let the commit-path
210
275
  // healing supply the content; a read must not fail for it. Every other fault
211
- // still propagates.
276
+ // still propagates. Reached only by a base-independent record now (an update-only
277
+ // one is declined above, never refused here): a delete over no committed
278
+ // revision, or an insert whose held `latest` is unmaterializable.
212
279
  if (!(err instanceof MissingBaseRevisionError)) {
213
280
  throw err;
214
281
  }
@@ -275,6 +342,11 @@ export class StorageRepo {
275
342
  // tickets/blocked/repo-pending-overlay-has-no-producer.
276
343
  throw new Error(`Pending action ${context.actionId} not found`);
277
344
  }
345
+ // A record the promotion above DECLINED (its base not reached here) is still present, so
346
+ // it is overlaid on whatever committed content this node holds — content older than the
347
+ // base its operations were computed against. Tolerated on this branch alone: the caller
348
+ // asserted its own pending, no production code sets `actionId` (the blocked ticket
349
+ // above), and the no-base case is still flagged by the clauses below.
278
350
  const block = applyTransform(blockRev?.block, pendingTransform);
279
351
  return [blockId, {
280
352
  block,
@@ -328,8 +400,31 @@ export class StorageRepo {
328
400
  // parallel reads complete so emission stays outside the per-block work, matching
329
401
  // commit's ordering. No-op when nothing was promoted.
330
402
  this.emitPromotions(promotions);
403
+ if (lineageOf !== undefined) {
404
+ await this.answerLineage(results, lineageOf);
405
+ }
331
406
  return Object.fromEntries(results);
332
407
  }
408
+ /**
409
+ * Answers {@link BlockGets.lineageOf} on every entry, from this node's own records (see
410
+ * {@link IBlockStorage.lineageOf}). Runs after the block reads, so it describes storage at least
411
+ * as new as the content served beside it; every fact it reads only ever moves forward, so a
412
+ * commit landing in between cannot make the answer wrong, only early.
413
+ *
414
+ * A read fault answers `unknown` rather than failing the batch: the asker reads that as "this
415
+ * node could not say", which is exactly what happened.
416
+ */
417
+ async answerLineage(results, target) {
418
+ await Promise.all(results.map(async ([blockId, entry]) => {
419
+ try {
420
+ entry.lineage = await this.createBlockStorage(blockId).lineageOf(target);
421
+ }
422
+ catch (err) {
423
+ log('get:lineage-unreadable blockId=%s rev=%d error=%s', blockId, target.rev, err instanceof Error ? err.message : String(err));
424
+ entry.lineage = 'unknown';
425
+ }
426
+ }));
427
+ }
333
428
  /**
334
429
  * The one place a local coverage gap is healed from a peer. `getBlock` is local-only; when it
335
430
  * reports the target revision as not covered ({@link RevisionNotCoveredError}) this fetches it
@@ -513,9 +608,33 @@ export class StorageRepo {
513
608
  }
514
609
  }
515
610
  }
516
- // Then handle any pending actions
517
- const pending = await asyncIteratorToArray(blockStorage.listPendingTransactions());
518
- pendings.push(...pending.map(actionId => ({ blockId, actionId })));
611
+ // NOTE: a pend of an update-only transform for a block this node holds NO revision of
612
+ // falls through here and is recorded (`latest` is undefined, so there is nothing to be
613
+ // stale against). It can never be promoted on this node without a reconcile —
614
+ // `internalCommit`'s fork guard refuses it (`missing-base-revision`) and drops the
615
+ // record — so the pend round it wins is one this member could not honour on its own.
616
+ // Harmless today: the commit-tier durability gate (`CoordinatorRepo.commit`) refuses
617
+ // the acknowledgement unless a majority of the cohort holds the revision after
618
+ // reconcile, and the coordinating member's proof-carrying copy is what a behind member
619
+ // reconciles from. If pend-time refusals ever become worth their cost (one wasted
620
+ // consensus round per such write), refuse at `ClusterMember.validatePendOperations`
621
+ // instead of here.
622
+ // Then the pending records that RESERVE the block against this request. A record claiming a
623
+ // slot the collection has already moved past is not one of them (the revision rule of
624
+ // `isReservationAgainst`): counting it refused every later writer on the strength of a
625
+ // commit this node merely missed. Deliberately NOT fed the pend's declared base: the base
626
+ // arm is the promise vote's alone, and only in a cohort that can leave a member out
627
+ // (`ClusterMember.reservingRivals`), so this scan is never stricter than the vote — a pend
628
+ // the cohort approved is not then refused here at apply, and a member that voted `held` on
629
+ // a stray record but was outvoted still stores the pend, whose commit then sweeps the record.
630
+ for (const claim of await blockStorage.listPendingClaims()) {
631
+ if (isReservationAgainst(claim, { rev: request.rev })) {
632
+ pendings.push({ blockId, actionId: claim.actionId });
633
+ }
634
+ else {
635
+ log('pend:superseded-claim actionId=%s blockId=%s rival=%s claimedRev=%d requestedRev=%d', request.actionId, blockId, claim.actionId, claim.rev, request.rev);
636
+ }
637
+ }
519
638
  }
520
639
  // Every refusal below returns having written ZERO pending records — that is what pass 1
521
640
  // finishing before pass 2 begins buys.
@@ -565,12 +684,15 @@ export class StorageRepo {
565
684
  }
566
685
  const blockStorage = this.createBlockStorage(blockId);
567
686
  const blockTransform = transformForBlockId(request.transforms, blockId);
568
- await blockStorage.savePendingTransaction(request.actionId, blockTransform, request.rev, latches.get(blockId));
687
+ await blockStorage.savePendingTransaction(request.actionId, blockTransform, request.rev, declaredBaseFor(request.baseRevs, blockId, blockTransform), latches.get(blockId));
569
688
  }
689
+ // This layer answers for one machine's storage and nothing else: `local`, with no cohort
690
+ // view. The coordinator above it replaces this with the cohort's answer on every cluster path.
570
691
  return {
571
692
  success: true,
572
693
  pending: pendings,
573
- blockIds
694
+ blockIds,
695
+ durability: localDurability()
574
696
  };
575
697
  }
576
698
  finally {
@@ -823,7 +945,8 @@ export class StorageRepo {
823
945
  // including those that landed before a mid-loop failure (alreadyDone / stale
824
946
  // partitions never reach `collectionBlocks`).
825
947
  this.emitCollectionChanges(collectionBlocks, request.actionId, request.rev, request.tailId);
826
- return failure ? { success: false, reason: failure.reason } : { success: true };
948
+ // `local`, as in `pend`: a single machine's verdict about its own storage.
949
+ return failure ? { success: false, reason: failure.reason } : { success: true, durability: localDurability() };
827
950
  }
828
951
  /**
829
952
  * Delete `actionId`'s pending record from every given block, tolerating absence.
@@ -1005,8 +1128,13 @@ export class StorageRepo {
1005
1128
  }
1006
1129
  // Clone both: applyTransform assigns `transform.insert` into the result by reference and
1007
1130
  // applyOperations mutates the block in place, so materializing on live storage/pending objects
1008
- // would corrupt them for the real commit that follows.
1009
- const newBlock = applyTransform(structuredClone(base), structuredClone(transform));
1131
+ // would corrupt them for the real commit that follows. `cloneDecoded` (a JSON round-trip) rather
1132
+ // than `structuredClone`, which Hermes lacks; lossless here because both values were just decoded
1133
+ // from JSON by the store (every `IRawStorage` in this repo is the JSON-coded `KvRawStorage`).
1134
+ // NOTE: if an `IRawStorage` that hands out live, never-serialized objects is ever wired in, this
1135
+ // preview can drift from internalCommit (which applies to the uncloned values): an update op
1136
+ // setting a field to `undefined` clones to `null`, which canonical JSON hashes differently.
1137
+ const newBlock = applyTransform(cloneDecoded(base), cloneDecoded(transform));
1010
1138
  // `undefined` covers the tombstone (delete transform) and updates-with-no-base (applyTransform
1011
1139
  // drops updates when there is no block to apply them to) — both materialize nothing.
1012
1140
  const digest = newBlock ? await canonicalBlockHash(newBlock) : undefined;
@@ -1024,6 +1152,14 @@ export class StorageRepo {
1024
1152
  }
1025
1153
  return undefined;
1026
1154
  }
1155
+ /** See {@link IPendingClaimReader}. */
1156
+ async listPendingClaims(blockId) {
1157
+ return await this.createBlockStorage(blockId).listPendingClaims();
1158
+ }
1159
+ /** See {@link IPendingClaimReader}. */
1160
+ async pendingClaimOf(blockId, actionId) {
1161
+ return await this.createBlockStorage(blockId).pendingClaimOf(actionId);
1162
+ }
1027
1163
  /**
1028
1164
  * The {@link BlockCommitProof} this node retained for `blockId` at `rev`, or `undefined` when it
1029
1165
  * kept none — a revision committed before proofs were persisted, a member whose materialization
@@ -1042,8 +1178,10 @@ export class StorageRepo {
1042
1178
  /**
1043
1179
  * @param declaredBaseRev The committed revision of the base the WRITER applied this block's
1044
1180
  * transform to, as declared in the commit op's `blockDigests[blockId].baseRev`. Untrusted wire
1045
- * data, so it is typed `unknown` and validated below. Absent from the read-driven promotion in
1046
- * {@link get}, which has no commit request and therefore nothing to compare against.
1181
+ * data, so it is typed `unknown` and validated in {@link guardCommitBase} where it is the
1182
+ * FALLBACK, not the primary check: the base the record's own pend carried (`PendingClaim.baseRev`)
1183
+ * is read first. Absent from the read-driven promotion in {@link get}, which has no commit request
1184
+ * and has already judged the stored base (`mayPromoteOnRead`).
1047
1185
  */
1048
1186
  async internalCommit(blockId, actionId, rev, storage, latch, proof, declaredBaseRev) {
1049
1187
  // Note: This method is called under the block write latch — by commit() (within its locked
@@ -1063,47 +1201,8 @@ export class StorageRepo {
1063
1201
  }
1064
1202
  // Get prior materialized block if it exists
1065
1203
  const latest = await storage.getLatest();
1066
- // FORK GUARD: apply an update-only transform ONLY to the base the writer actually read.
1067
- // Revisions are allocated per COLLECTION, not per block, so `rev - 1` is meaningless here — a
1068
- // member legitimately holds block X at rev 1 and receives a commit of X at rev 7 when revs 2-6
1069
- // touched other blocks. The only sound discriminator is the writer's own per-block declaration:
1070
- // it read the block at `declaredBaseRev`, so a member holding anything else would be applying
1071
- // the transform to different bytes and silently forking the block's content at this revision.
1072
- //
1073
- // Each clause is deliberate:
1074
- // - `typeof declaredBaseRev === 'number'` — `blockDigests` is untrusted wire data with no
1075
- // ingress schema (same rule as ClusterMember.validateCommitOperations). Missing, malformed,
1076
- // or absent-by-design declarations ABSTAIN, preserving today's behavior for pre-upgrade
1077
- // writers, undeclarable blocks, and the read-driven promotion in get().
1078
- // - `!transform.insert` — an insert-carrying transform is base-independent, so there is nothing
1079
- // to fork. Keyed on the member's OWN pended transform (as previewCommitDigest does), never on
1080
- // the declaration, so a hostile writer cannot flip the arm by attaching a bogus baseRev.
1081
- // - `latest?.rev !== declaredBaseRev` covers all three unsafe states: BEHIND the declared base
1082
- // (missed updates — the fork case), AHEAD of it (this member holds a revision the writer
1083
- // never saw — divergent history), and no local revision at all against a numeric declaration.
1084
- //
1085
- // Refusing is cheap and self-healing: refuseMissingBase throws MissingBaseRevisionError, which
1086
- // commit() classifies as divergence and ClusterMember.applyConsensusOperation maps to "behind",
1087
- // running reconcileDivergentCommit to pull the committed revision from a cohort peer. The
1088
- // writer's retry then lands on a healed base. A hostile writer declaring a junk numeric baseRev
1089
- // can force refusals and reconcile churn, but never a fork.
1090
- //
1091
- // NOTE: the AHEAD case is reported as "behind" divergence like every other missing-base
1092
- // refusal, so a cohort where nobody holds `rev` reconciles, fails `no-rev-quorum`, and logs
1093
- // that rather than a clean stale failure. Correct outcome — the writer read a base the cohort
1094
- // has moved past, and its retry re-reads — but the log reads as lag when it is the opposite.
1095
- // If those lines ever have to be triaged in volume, give the ahead arm its own reason string.
1096
- //
1097
- // NOTE: this guard only reaches what the writer declared, so two arms of the same fork still
1098
- // stand — both tracked by `backlog/bug-a-pended-transform-does-not-carry-its-base`:
1099
- // (1) a commit whose block declares NO digest — pre-upgrade writer, undeclarable block
1100
- // (read-far-then-update eviction, see db-core transform/digest.ts), or a delete-only transform —
1101
- // gap-applies exactly as before; (2) the read-driven promotion in `get`, which reaches this
1102
- // method with no commit request at all (`declaredBaseRev` undefined) and so cannot check. If
1103
- // forked-content reports persist, those are the residuals to look at.
1104
- if (typeof declaredBaseRev === 'number' && !transform.insert && latest?.rev !== declaredBaseRev) {
1105
- return await this.refuseMissingBase(blockId, actionId, rev, storage, latch, `local latest ${latest?.rev ?? 'none'} is not the declared base ${declaredBaseRev} of rev ${rev}`);
1106
- }
1204
+ // FORK GUARD: apply an update-only transform ONLY to the base its author computed it against.
1205
+ await this.guardCommitBase(blockId, actionId, rev, storage, latch, transform, latest, declaredBaseRev);
1107
1206
  const priorBlock = await this.readCommitBase(blockId, actionId, rev, storage, latest, latch);
1108
1207
  // Apply transform and save materialized block
1109
1208
  // applyTransform handles undefined priorBlock correctly for inserts
@@ -1126,8 +1225,9 @@ export class StorageRepo {
1126
1225
  // the 'latest' pointer doesn't point to a revision that hasn't been fully recorded.
1127
1226
  await storage.saveRevision(rev, actionId, latch);
1128
1227
  await storage.promotePendingTransaction(actionId, latch);
1129
- // Update latest revision *last*
1130
- await storage.setLatest({ actionId, rev }, latch);
1228
+ // Update latest revision *last*. An insert replaced the block wholesale, so its content was
1229
+ // not built on what this node held before (see BlockMetadata.lineageFloor).
1230
+ await storage.setLatest({ actionId, rev }, transform.insert === undefined, latch);
1131
1231
  // Persist the cohort's commit proof AFTER the commit is durably latest — the proof is
1132
1232
  // evidence about a landed revision, never a precondition of landing it. The retention rule
1133
1233
  // (persist only when the LOCAL materialization matches the digest the commit op declared)
@@ -1157,6 +1257,111 @@ export class StorageRepo {
1157
1257
  // undefined so the caller skips it rather than emitting a bogus event.
1158
1258
  return newBlock?.header.collectionId ?? priorBlock?.header.collectionId;
1159
1259
  }
1260
+ /**
1261
+ * The fork guard: an update-only transform is applied ONLY to the base its author computed it
1262
+ * against. Revisions are allocated per COLLECTION, not per block, so `rev - 1` is meaningless here —
1263
+ * a member legitimately holds block X at rev 1 and receives a commit of X at rev 7 when revs 2-6
1264
+ * touched other blocks (the retired decision `st-commit-contiguity-guard-premise`). The only sound
1265
+ * discriminator is what the author said the base was, and the author says it twice:
1266
+ *
1267
+ * - `stored` — the base the record's own PEND carried for this block (`PendRequest.baseRevs`, kept
1268
+ * as `PendingClaim.baseRev`). PRIMARY, because it was recorded with the very operations it
1269
+ * describes and is present on every path that applies the record, commit message or not.
1270
+ * - `declared` — `blockDigests[blockId].baseRev` on the commit. The FALLBACK, for a record whose
1271
+ * pend named no base: a sender running older code, or a drift-blind source (test doubles).
1272
+ * Untrusted wire data with no ingress schema (same rule as ClusterMember.validateCommitOperations):
1273
+ * anything but a number abstains rather than being coerced into a comparison.
1274
+ *
1275
+ * Three steps, in order:
1276
+ *
1277
+ * 1. Both present and unequal → refuse, with its own detail and log line. An honest writer never
1278
+ * does this (the base pinned at staging is the one pended and the one declared — `Tracker`), so
1279
+ * the shape it closes is a member holding a STALE pending record from an earlier attempt of a
1280
+ * retried action — the retry's pend never reached this member — that receives the retry's
1281
+ * commit: the old record's operations were computed against a different base, and a guard that
1282
+ * read only the declaration would apply them wherever this member's latest happened to equal
1283
+ * the new declaration.
1284
+ * 2. `effective = stored ?? declared`; a number, and `latest?.rev !== effective` → refuse. Covers
1285
+ * all three unsafe states: BEHIND the base (missed updates — the fork case), AHEAD of it (this
1286
+ * member holds a revision the writer never saw — divergent history), and no local revision at
1287
+ * all against a numeric base.
1288
+ * 3. Neither present → apply as before the guard existed, logged as `commit:base-undeclared` so the
1289
+ * residual is countable. This is the one arm left open, BY CHOICE, for senders that name no
1290
+ * base anywhere: refusing a base-less pend outright would turn every such writer's write into a
1291
+ * hard failure on a release that may run mixed versions for a while.
1292
+ *
1293
+ * Base-independent transforms — an insert (replaces the block wholesale) or a delete (materializes
1294
+ * to nothing) — are never guarded, keyed on the member's OWN pended transform and never on a
1295
+ * declaration, so a hostile writer cannot flip the arm by attaching a bogus base.
1296
+ *
1297
+ * Refusing is cheap and self-healing: refuseMissingBase throws MissingBaseRevisionError, which
1298
+ * commit() classifies as divergence and ClusterMember.applyConsensusOperation maps to "behind",
1299
+ * running reconcileDivergentCommit to pull the committed revision from a cohort peer. The writer's
1300
+ * retry then lands on a healed base. A hostile writer naming a junk numeric base — on the pend or
1301
+ * on the commit — can force refusals and reconcile churn, but never a fork.
1302
+ *
1303
+ * NOTE: the AHEAD case is reported as "behind" divergence like every other missing-base refusal, so
1304
+ * a cohort where nobody holds `rev` reconciles, fails `no-rev-quorum`, and logs that rather than a
1305
+ * clean stale failure. Correct outcome — the writer read a base the cohort has moved past, and its
1306
+ * retry re-reads — but the log reads as lag when it is the opposite. If those lines ever have to be
1307
+ * triaged in volume, give the ahead arm its own reason string.
1308
+ */
1309
+ async guardCommitBase(blockId, actionId, rev, storage, latch, transform, latest, declaredBaseRev) {
1310
+ if (isBaseIndependent(transform)) {
1311
+ return;
1312
+ }
1313
+ // NOTE: `pendingClaimOf` re-reads the record `internalCommit` already holds (to prove the claim
1314
+ // is live) plus the metadata — two local KV gets per update-only commit, unmeasured. If the
1315
+ // commit path ever shows them in a profile, read the metadata alone here: the caller's record
1316
+ // read is the liveness proof.
1317
+ const stored = (await storage.pendingClaimOf(actionId))?.baseRev;
1318
+ const declared = typeof declaredBaseRev === 'number' ? declaredBaseRev : undefined;
1319
+ if (stored !== undefined && declared !== undefined && stored !== declared) {
1320
+ log('commit:base-disagreement blockId=%s rev=%d actionId=%s stored=%d declared=%d', blockId, rev, actionId, stored, declared);
1321
+ return await this.refuseMissingBase(blockId, actionId, rev, storage, latch, `stored base ${stored} disagrees with declared base ${declared} of rev ${rev}`);
1322
+ }
1323
+ const effective = stored ?? declared;
1324
+ if (effective === undefined) {
1325
+ // NOTE: debug level only, so the base-less residual is countable but not visible in production
1326
+ // logs; every bare test-double pend lands here, so a visible level would drown the suites. If
1327
+ // a mixed-version fleet ever needs the count, give this one line its own logger.
1328
+ log('commit:base-undeclared blockId=%s rev=%d actionId=%s latest=%s', blockId, rev, actionId, latest?.rev ?? 'none');
1329
+ return;
1330
+ }
1331
+ if (latest?.rev !== effective) {
1332
+ return await this.refuseMissingBase(blockId, actionId, rev, storage, latch, `local latest ${latest?.rev ?? 'none'} is not the ${stored !== undefined ? 'stored' : 'declared'} base ${effective} of rev ${rev}`);
1333
+ }
1334
+ }
1335
+ /**
1336
+ * Whether the read-driven promotion in {@link get} may apply `actionId`'s pending record here: a
1337
+ * base-independent record (an insert or a delete) always; an update-only one only when the base
1338
+ * its pend carried (`PendingClaim.baseRev`) is a number equal to this node's `latest`. Anything
1339
+ * else — a base this node has not reached, one it is past, or none stored at all — DECLINES, and
1340
+ * the caller leaves the record and `latest` untouched.
1341
+ *
1342
+ * Declining is deliberately distinct from {@link refuseMissingBase}, which deletes the record
1343
+ * because it can never be promoted here. A declined record is not dead: this node's latest reaches
1344
+ * the stored base only through a replica or reconcile, and when that lands `sweepDeadClaims`
1345
+ * removes the record if its slot is passed, or a later context read promotes it if not. A record
1346
+ * whose pend named no base is declined too — the promotion must not apply a change whose base it
1347
+ * cannot establish, and block repair supplies the version instead. The cost falls on base-less
1348
+ * senders alone: their held-but-missed records no longer come current on a read, only through the
1349
+ * next commit's reconcile or the coordinator's read-repair.
1350
+ *
1351
+ * {@link guardCommitBase} still runs inside `internalCommit` afterwards; under the latch the
1352
+ * caller holds, this check is exactly what makes it pass.
1353
+ */
1354
+ async mayPromoteOnRead(blockId, storage, actionId, pending, latest) {
1355
+ if (isBaseIndependent(pending)) {
1356
+ return true;
1357
+ }
1358
+ const stored = (await storage.pendingClaimOf(actionId))?.baseRev;
1359
+ if (stored !== undefined && latest?.rev === stored) {
1360
+ return true;
1361
+ }
1362
+ log('get:promote-declined blockId=%s actionId=%s storedBase=%s latest=%s', blockId, actionId, stored ?? 'none', latest?.rev ?? 'none');
1363
+ return false;
1364
+ }
1160
1365
  /**
1161
1366
  * Retain `proof` for a block this call found ALREADY committed at `(rev, actionId)` — the paths
1162
1367
  * that land (or find already landed) a revision without running {@link internalCommit}, and would