@optimystic/db-p2p 0.21.0 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/dist/src/cluster/cluster-policy.d.ts +13 -2
  2. package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
  3. package/dist/src/cluster/cluster-policy.js +51 -4
  4. package/dist/src/cluster/cluster-policy.js.map +1 -1
  5. package/dist/src/cluster/cluster-repo.d.ts +3 -3
  6. package/dist/src/cluster/cluster-repo.js +3 -3
  7. package/dist/src/cluster/cluster-size-coupling.d.ts +28 -0
  8. package/dist/src/cluster/cluster-size-coupling.d.ts.map +1 -0
  9. package/dist/src/cluster/cluster-size-coupling.js +35 -0
  10. package/dist/src/cluster/cluster-size-coupling.js.map +1 -0
  11. package/dist/src/cluster/quorum-restore.d.ts +6 -0
  12. package/dist/src/cluster/quorum-restore.d.ts.map +1 -1
  13. package/dist/src/cluster/quorum-restore.js +1 -1
  14. package/dist/src/cluster/quorum-restore.js.map +1 -1
  15. package/dist/src/cluster/reconcile-block.d.ts.map +1 -1
  16. package/dist/src/cluster/reconcile-block.js +15 -3
  17. package/dist/src/cluster/reconcile-block.js.map +1 -1
  18. package/dist/src/index.d.ts +2 -0
  19. package/dist/src/index.d.ts.map +1 -1
  20. package/dist/src/index.js +2 -0
  21. package/dist/src/index.js.map +1 -1
  22. package/dist/src/libp2p-key-network.d.ts +46 -5
  23. package/dist/src/libp2p-key-network.d.ts.map +1 -1
  24. package/dist/src/libp2p-key-network.js +40 -9
  25. package/dist/src/libp2p-key-network.js.map +1 -1
  26. package/dist/src/libp2p-node-base.d.ts +3 -2
  27. package/dist/src/libp2p-node-base.d.ts.map +1 -1
  28. package/dist/src/libp2p-node-base.js +834 -777
  29. package/dist/src/libp2p-node-base.js.map +1 -1
  30. package/dist/src/libp2p-node-rn.d.ts +2 -2
  31. package/dist/src/libp2p-node-rn.d.ts.map +1 -1
  32. package/dist/src/libp2p-node-rn.js.map +1 -1
  33. package/dist/src/libp2p-node.d.ts +2 -2
  34. package/dist/src/libp2p-node.d.ts.map +1 -1
  35. package/dist/src/libp2p-node.js.map +1 -1
  36. package/dist/src/network/network-manager-service.d.ts +2 -0
  37. package/dist/src/network/network-manager-service.d.ts.map +1 -1
  38. package/dist/src/network/network-manager-service.js +4 -0
  39. package/dist/src/network/network-manager-service.js.map +1 -1
  40. package/dist/src/optimystic-node.d.ts +35 -0
  41. package/dist/src/optimystic-node.d.ts.map +1 -0
  42. package/dist/src/optimystic-node.js +2 -0
  43. package/dist/src/optimystic-node.js.map +1 -0
  44. package/dist/src/repo/coordinator-repo.d.ts +16 -5
  45. package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
  46. package/dist/src/repo/coordinator-repo.js +24 -8
  47. package/dist/src/repo/coordinator-repo.js.map +1 -1
  48. package/dist/src/rn.d.ts +2 -0
  49. package/dist/src/rn.d.ts.map +1 -1
  50. package/dist/src/rn.js +2 -0
  51. package/dist/src/rn.js.map +1 -1
  52. package/dist/src/storage/block-storage.d.ts.map +1 -1
  53. package/dist/src/storage/block-storage.js +57 -5
  54. package/dist/src/storage/block-storage.js.map +1 -1
  55. package/dist/src/storage/i-block-storage.d.ts +20 -1
  56. package/dist/src/storage/i-block-storage.d.ts.map +1 -1
  57. package/dist/src/storage/storage-repo.d.ts +56 -3
  58. package/dist/src/storage/storage-repo.d.ts.map +1 -1
  59. package/dist/src/storage/storage-repo.js +124 -18
  60. package/dist/src/storage/storage-repo.js.map +1 -1
  61. package/dist/src/testing/raw-storage-conformance.d.ts.map +1 -1
  62. package/dist/src/testing/raw-storage-conformance.js +17 -0
  63. package/dist/src/testing/raw-storage-conformance.js.map +1 -1
  64. package/package.json +2 -2
  65. package/readme.md +41 -26
  66. package/src/cluster/cluster-policy.ts +55 -4
  67. package/src/cluster/cluster-repo.ts +3 -3
  68. package/src/cluster/cluster-size-coupling.ts +45 -0
  69. package/src/cluster/quorum-restore.ts +1 -1
  70. package/src/cluster/reconcile-block.ts +15 -3
  71. package/src/index.ts +2 -0
  72. package/src/libp2p-key-network.ts +41 -9
  73. package/src/libp2p-node-base.ts +907 -847
  74. package/src/libp2p-node-rn.ts +2 -2
  75. package/src/libp2p-node.ts +2 -2
  76. package/src/network/network-manager-service.ts +5 -0
  77. package/src/optimystic-node.ts +36 -0
  78. package/src/repo/coordinator-repo.ts +24 -8
  79. package/src/rn.ts +2 -0
  80. package/src/storage/block-storage.ts +59 -6
  81. package/src/storage/i-block-storage.ts +20 -1
  82. package/src/storage/storage-repo.ts +129 -18
  83. package/src/testing/raw-storage-conformance.ts +20 -0
@@ -1,10 +1,10 @@
1
- import type { Libp2p } from 'libp2p';
2
1
  import {
3
2
  createLibp2pNodeBase,
4
3
  type Libp2pTransports,
5
4
  type NodeOptions,
6
5
  type RawStorageProvider,
7
6
  } from './libp2p-node-base.js';
7
+ import type { OptimysticNode } from './optimystic-node.js';
8
8
 
9
9
  export type { Libp2pTransports, NodeOptions, RawStorageProvider };
10
10
 
@@ -14,7 +14,7 @@ export type { Libp2pTransports, NodeOptions, RawStorageProvider };
14
14
  * This entrypoint intentionally does not import Node-only transports (like `@libp2p/tcp`).
15
15
  * Callers must provide `options.transports` (and typically `options.listenAddrs`).
16
16
  */
17
- export async function createLibp2pNode(options: NodeOptions): Promise<Libp2p> {
17
+ export async function createLibp2pNode(options: NodeOptions): Promise<OptimysticNode> {
18
18
  const transports = options.transports;
19
19
  if (!transports || transports.length === 0) {
20
20
  throw new Error(
@@ -1,4 +1,3 @@
1
- import type { Libp2p } from 'libp2p';
2
1
  import { tcp } from '@libp2p/tcp';
3
2
  import { webSockets } from '@libp2p/websockets';
4
3
  import { circuitRelayTransport } from '@libp2p/circuit-relay-v2';
@@ -8,10 +7,11 @@ import {
8
7
  type NodeOptions,
9
8
  type RawStorageProvider,
10
9
  } from './libp2p-node-base.js';
10
+ import type { OptimysticNode } from './optimystic-node.js';
11
11
 
12
12
  export type { Libp2pTransports, NodeOptions, RawStorageProvider };
13
13
 
14
- export async function createLibp2pNode(options: NodeOptions): Promise<Libp2p> {
14
+ export async function createLibp2pNode(options: NodeOptions): Promise<OptimysticNode> {
15
15
  const port = options.port ?? 0;
16
16
  const wsHost = options.wsHost ?? '0.0.0.0';
17
17
 
@@ -58,6 +58,11 @@ export class NetworkManagerService implements Startable {
58
58
  }
59
59
  }
60
60
 
61
+ /** The cluster size this instance actually resolved to, for `assertClusterSizeCoupling`. */
62
+ get effectiveClusterSize(): number {
63
+ return this.cfg.clusterSize;
64
+ }
65
+
61
66
  setLibp2p(libp2p: Libp2p): void {
62
67
  this.libp2pRef = libp2p;
63
68
  }
@@ -0,0 +1,36 @@
1
+ import type { Libp2p, PrivateKey } from '@libp2p/interface';
2
+ import type { IBlockChangeNotifier, IRepo } from '@optimystic/db-core';
3
+ import type { DisputeService } from './dispute/dispute-service.js';
4
+ import type { Libp2pKeyPeerNetwork } from './libp2p-key-network.js';
5
+ import type { PeerReputationService } from './reputation/peer-reputation.js';
6
+ import type { StorageRepo } from './storage/storage-repo.js';
7
+
8
+ /**
9
+ * The handles `createLibp2pNodeBase` attaches to the libp2p node it returns. This is the
10
+ * sanctioned in-process surface a host reads — declared once here so reaching it does not
11
+ * require a cast, and so a host cannot silently rebuild a component the node already owns.
12
+ *
13
+ * Deliberately NOT the full set of `(node as any).*` attachments made in `libp2p-node-base.ts`:
14
+ * the churn/rebalance/ring-shift monitors, the cohort-topic host and the reactivity registries
15
+ * are node-internal wiring, not a host-facing surface, and typing them is a separate job.
16
+ */
17
+ export interface OptimysticNodeAttachments {
18
+ /**
19
+ * The node's ONE key network — built from its resolved cluster policy, network-namespaced
20
+ * protocol prefix, reputation tracker and persistence. A host that needs key/coordinator
21
+ * lookup uses THIS; constructing a second one gives peer selection a different cohort
22
+ * width and coordinator than the node's own consensus path uses for the same key.
23
+ */
24
+ keyNetwork: Libp2pKeyPeerNetwork;
25
+ coordinatedRepo: IRepo;
26
+ storageRepo: StorageRepo;
27
+ /** Per-collection change origin. Replaced by the cohort-topic bridge notifier when enabled. */
28
+ blockChangeNotifier: IBlockChangeNotifier;
29
+ reputation: PeerReputationService;
30
+ /** Present only when the dispute subsystem is configured. */
31
+ disputeService?: DisputeService;
32
+ /** The node's libp2p Ed25519 identity key, for hosts binding a client-transaction signer. */
33
+ peerPrivateKey: PrivateKey;
34
+ }
35
+
36
+ export type OptimysticNode = Libp2p & OptimysticNodeAttachments;
@@ -9,6 +9,7 @@ import type { IPeerReputation } from "../reputation/types.js";
9
9
  import { PenaltyReason } from "../reputation/types.js";
10
10
  import type { ITransactionStateStore } from "../cluster/i-transaction-state-store.js";
11
11
  import { quorumSize, corroboratorCapacity, selectQuorumRev, type RevClaim, type QuorumRev } from "../cluster/quorum-restore.js";
12
+ import { DEFAULT_CLUSTER_SIZE } from "../cluster/cluster-policy.js";
12
13
  import { RECONCILE_TIMEOUT_MS } from "../cluster/reconcile-block.js";
13
14
  import { isMissingBaseRevisionFailure, MISSING_BASE_REVISION_REASON } from "../storage/storage-repo.js";
14
15
  import type { ReconcileBlockCallback } from "../cluster/cluster-repo.js";
@@ -191,7 +192,10 @@ export class CoordinatorRepo implements IRepo {
191
192
  ) {
192
193
  this.localPeerId = localPeerId;
193
194
  const policy: ClusterConsensusConfig & { clusterSize: number } = {
194
- clusterSize: cfg?.clusterSize ?? 10,
195
+ // Same constant `resolveClusterPolicy` gives a node that declares no clusterSize, not a
196
+ // second literal: a direct constructor (the readme's manual-wiring path) and the node
197
+ // assembly must land on the same width or the two disagree about the same key's cohort.
198
+ clusterSize: cfg?.clusterSize ?? DEFAULT_CLUSTER_SIZE,
195
199
  assumedClusterSize: cfg?.assumedClusterSize,
196
200
  superMajorityThreshold: cfg?.superMajorityThreshold ?? DEFAULT_SUPER_MAJORITY_THRESHOLD,
197
201
  simpleMajorityThreshold: cfg?.simpleMajorityThreshold ?? 0.51,
@@ -393,12 +397,21 @@ export class CoordinatorRepo implements IRepo {
393
397
  *
394
398
  * No-op once the entry carries a real answer (the consult restored the block) or a sharper flag
395
399
  * (storage's `'unmaterializable'`), so callers only need to establish that the answer is a guess.
400
+ *
401
+ * "Carries a real answer" is tested as `entry.block !== undefined`, NOT as `state.latest` being
402
+ * set. The two used to move together, so `state.latest` read as a serviceable proxy — but a
403
+ * pending-only insert (pended, not yet committed) served through the pending overlay has real
404
+ * CONTENT and no committed revision at all, so its `state.latest` is undefined. Flagging that
405
+ * entry would mark a block this node is positively holding as an unconfirmed absence, and
406
+ * `NetworkTransactor`'s `isAuthoritative` keys off the flag alone — the read would burn its
407
+ * retry budget re-asking other peers for content it already has. `state.latest` stays in the
408
+ * test as well so a stale-but-real committed answer is likewise never downgraded.
396
409
  */
397
410
  private flagUnconfirmedAbsence(results: GetBlockResults, blockId: BlockId): void {
398
411
  const entry = results[blockId];
399
412
  if (!entry) {
400
413
  results[blockId] = { state: {}, unavailable: 'peers-unreachable' };
401
- } else if (!entry.state?.latest && entry.unavailable === undefined) {
414
+ } else if (entry.block === undefined && !entry.state?.latest && entry.unavailable === undefined) {
402
415
  entry.unavailable = 'peers-unreachable';
403
416
  }
404
417
  }
@@ -603,11 +616,13 @@ export class CoordinatorRepo implements IRepo {
603
616
  * the repair. Returns the local revision afterwards.
604
617
  *
605
618
  * A pending-only block (metadata seeded by `savePendingTransaction`, no committed revision) asked
606
- * for a forward revision no promotion can reach used to throw out of `BlockStorage.ensureRevision`;
607
- * `StorageRepo.get` now reports it as an entry flagged `unavailable` instead (ticket
608
- * repo-reports-unavailable-vs-absent). On THIS path either shape is an absence, not a read failure
609
- * acquisition is precisely the mechanism that can supply the revision so both are logged as
610
- * `promote-unavailable` and stepped over rather than short-circuiting the caller.
619
+ * for a forward revision no promotion can reach used to throw out of `BlockStorage.ensureRevision`.
620
+ * It no longer does: "no committed base here" is an absence, so that read comes back as a plain
621
+ * unflagged `{ state: {} }` and this method simply returns `undefined` acquisition then supplies
622
+ * the revision. The `unavailable` arm below still fires for the shapes that ARE a guess (a `latest`
623
+ * this node cannot materialize, a missing-base promotion refusal); on THIS path those are an
624
+ * absence too rather than a read failure, so they are logged as `promote-unavailable` and stepped
625
+ * over rather than short-circuiting the caller. The catch stays for any other fault, same reason.
611
626
  */
612
627
  private async promoteCorroborated(blockId: BlockId, corroborated: ActionRev): Promise<number | undefined> {
613
628
  try {
@@ -715,7 +730,8 @@ export class CoordinatorRepo implements IRepo {
715
730
  log('cluster-fetch:no-quorum', {
716
731
  blockId,
717
732
  responders: claims.length,
718
- required: quorumSize(claims.length, this.simpleMajorityThreshold, capacity)
733
+ required: quorumSize(claims.length, this.simpleMajorityThreshold, capacity),
734
+ repairCorroborationClusterSize: this.repairCorroborationClusterSize
719
735
  });
720
736
  return { local, silent };
721
737
  }
package/src/rn.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  export * from './cluster/client.js';
2
+ export * from './cluster/cluster-policy.js';
2
3
  export * from './cluster/cluster-repo.js';
3
4
  export * from './cluster/service.js';
4
5
  export * from './protocol-client.js';
@@ -25,6 +26,7 @@ export * from './sync/service.js';
25
26
  export * from './it-utility.js';
26
27
  export * from './libp2p-key-network.js';
27
28
  export * from './libp2p-node-rn.js';
29
+ export * from './optimystic-node.js';
28
30
  export * from './routing/responsibility.js';
29
31
  export * from './routing/libp2p-known-peers.js';
30
32
  export * from './network/network-manager-service.js';
@@ -50,14 +50,44 @@ export class BlockStorage implements IBlockStorage {
50
50
  return undefined;
51
51
  }
52
52
 
53
- // Pending-only state: metadata was seeded by savePendingTransaction but no
54
- // revision has been committed yet. Treat as "doesn't exist" for the default
55
- // request pathmatches StorageRepo.get()'s contract that undefined => empty.
56
- if (rev === undefined && meta.latest === undefined) {
57
- return undefined;
53
+ // Pending-only state: metadata was seeded by savePendingTransaction but no revision has been
54
+ // committed yet. "No committed base here" is an ABSENCE, not a fault — nothing is being
55
+ // FAILED to reconstruct so both arms below answer `undefined` rather than throwing, whether
56
+ // or not the caller named a revision. StorageRepo.get then applies any pending overlay over
57
+ // that absent base; a throw here would instead be caught into `unavailable: 'unmaterializable'`
58
+ // and a writer reading back its own not-yet-committed insert would be told it is unreadable.
59
+ // `unmaterializable` must keep its one meaning: records prove the block exists and this node
60
+ // cannot reconstruct it.
61
+ if (meta.latest === undefined) {
62
+ if (rev === undefined) {
63
+ return undefined;
64
+ }
65
+ // A named rev still ATTEMPTS the restore: `restoreCallback` may be able to supply that
66
+ // revision even though nothing is committed locally, and a successful restore serves real
67
+ // content with `latest` still undefined. That capability is pinned by the 'getBlock for an
68
+ // absent revision fires restoreCallback (restore not short-circuited)' test in
69
+ // test/block-storage.spec.ts — do not short-circuit it away.
70
+ //
71
+ // Only ensureRevision's FAILURE is swallowed (no callback wired, or restore could not
72
+ // supply the rev): that is precisely the "no committed base here" absence. materializeBlock
73
+ // below is deliberately OUTSIDE the try — a throw from there means revision records exist
74
+ // with no materialization anywhere under them, which is genuine corruption and must keep
75
+ // reading as `unmaterializable`.
76
+ //
77
+ // NOTE: a contextful read of a pending-only block still attempts a network restore before
78
+ // falling back to absent (same cost as the pre-fix throw path); if pending-only read-backs
79
+ // ever show as hot, short-circuit when ranges are empty.
80
+ try {
81
+ await this.ensureRevision(meta, rev);
82
+ } catch (err) {
83
+ log('getBlock:no-committed-base blockId=%s rev=%d error=%s', this.blockId, rev,
84
+ err instanceof Error ? err.message : String(err));
85
+ return undefined;
86
+ }
87
+ return await this.materializeBlock(meta, rev);
58
88
  }
59
89
 
60
- const targetRev = rev ?? meta.latest!.rev;
90
+ const targetRev = rev ?? meta.latest.rev;
61
91
  await this.ensureRevision(meta, targetRev);
62
92
  return await this.materializeBlock(meta, targetRev);
63
93
  }
@@ -271,6 +301,29 @@ export class BlockStorage implements IBlockStorage {
271
301
  };
272
302
  await this.saveRestored(archive);
273
303
 
304
+ // INVARIANT P: a block never holds a pending record AND a committed record for the same
305
+ // action id. On the commit path `promotePendingTransaction` maintains it by MOVING the
306
+ // record atomically; this forward path writes the committed transform directly (via
307
+ // saveRestored above), so it owes the deletion itself. Without it, a node that pended the
308
+ // action but diverged before committing keeps a record nothing can ever promote — reported
309
+ // as a phantom conflicting action by every later `pend` on the block, which under
310
+ // `policy: 'f'` refuses that node's participation in the block's writes permanently.
311
+ //
312
+ // Deliberately on the WRITE path only: the monotonic guard above returns before here, and
313
+ // that early return must stay a true no-op (the earlier call that wrote the revision is the
314
+ // one that owed the deletion). Deliberately here rather than in `saveRestored`, which is
315
+ // also reached from ensureRevision's historical restore under a different latch, where a
316
+ // deletion could race a concurrent promotePendingTransaction; this path holds
317
+ // `BlockStorage.saveReplica:<id>` and (via StorageRepo.saveReplicatedBlock) the per-block
318
+ // commit latch, so it is already mutually exclusive with a live commit.
319
+ //
320
+ // NOTE: deletes only this revision's actionId, not every pending whose action is already
321
+ // committed. A broader sweep would repair records orphaned by routes that do not carry the
322
+ // committing actionId; if orphaned pendings ever show up in the field on blocks whose
323
+ // committing action id differs, widen to a sweep over listPendingTransactions filtered by
324
+ // getTransaction.
325
+ await this.storage.deletePendingTransaction(this.blockId, actionId);
326
+
274
327
  // Seed metadata when absent, advance latest, and merge the covered range.
275
328
  const prevRev = meta?.latest?.rev;
276
329
  if (!meta) {
@@ -47,7 +47,18 @@ export interface IBlockStorage {
47
47
  /** Saves a revision */
48
48
  saveRevision(rev: number, actionId: ActionId): Promise<void>;
49
49
 
50
- /** Promotes a pending action to committed */
50
+ /**
51
+ * Promotes a pending action to committed, MOVING the record from the pending namespace to the
52
+ * committed one in a single atomic step.
53
+ *
54
+ * **Invariant P** — a block never holds a pending record and a committed record for the same
55
+ * action id at the same time. This method maintains it on the commit path; every OTHER writer of
56
+ * a committed transform for a block ({@link saveReplica}, {@link saveDeletion}, and any forward
57
+ * path added later) must maintain it too, by deleting that action's pending record when it
58
+ * writes the committed one. A pending record left beside a committed one can never be promoted,
59
+ * and is reported as a phantom conflicting action by `StorageRepo.pend` on every later write to
60
+ * the block.
61
+ */
51
62
  promotePendingTransaction(actionId: ActionId): Promise<void>;
52
63
 
53
64
  /** Sets the latest revision information */
@@ -62,6 +73,11 @@ export interface IBlockStorage {
62
73
  * it falls back to `rev = 1` and a deterministic `actionId` derived from the block
63
74
  * (so retries stay idempotent — never random).
64
75
  *
76
+ * Maintains **Invariant P** (see {@link promotePendingTransaction}): writing the committed
77
+ * transform for `actionId` also deletes that action's pending record on this block, so a node
78
+ * that pended the action but diverged before committing it does not keep an unpromotable record.
79
+ * Only on the write path — the monotonic no-op below deletes nothing.
80
+ *
65
81
  * No-op (still durable) when an equal-or-newer revision is already present: `latest`
66
82
  * is never downgraded. Idempotent for a fixed `(rev, actionId)`. Returns the
67
83
  * effective latest `ActionRev`.
@@ -75,6 +91,9 @@ export interface IBlockStorage {
75
91
  * materialization as a deletion, so a `getBlock()` after a tombstone reads back as *absent*
76
92
  * (`undefined`) while a historical `getBlock(creationRev)` still materializes the created content.
77
93
  *
94
+ * Maintains **Invariant P** (see {@link promotePendingTransaction}) on the write path, exactly as
95
+ * {@link saveReplica} does: the tombstone's `actionId` loses its pending record on this block.
96
+ *
78
97
  * Idempotent for a fixed `(rev, actionId)`; never downgrades `latest` (a no-op — still durable —
79
98
  * when an equal-or-newer revision is already present). Returns the effective latest `ActionRev`.
80
99
  */
@@ -280,19 +280,30 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
280
280
  }
281
281
 
282
282
  // Include pending action if requested, applying the pending transform over whatever
283
- // committed base getBlock() resolved (possibly none).
284
- // NOTE: the `blockRev == null` arm is currently DEAD, so a pending-only insert is NOT
285
- // served here despite what this branch is shaped to do: `ActionContext.rev` is required,
286
- // and BlockStorage.getBlock only tolerates a missing committed base when `rev` is
287
- // undefined — so a contextful read of a block with no committed revision throws above and
288
- // is caught into `unavailable: 'unmaterializable'` before reaching here. Asserted in
289
- // storage-repo.spec.ts ('KNOWN GAP: a pending-only insert read WITH a context ...'); see
290
- // tickets/backlog/debt-pending-only-insert-unreadable-with-context.
283
+ // committed base getBlock() resolved (possibly none — a pending-only insert has no
284
+ // committed revision under it and getBlock reports that as an absent base, not a fault).
291
285
  if (context?.actionId !== undefined) {
292
286
  const pendingTransform = await blockStorage.getPendingTransaction(context.actionId);
293
287
  if (!pendingTransform) {
288
+ if (unavailable !== undefined) {
289
+ // The promotion refusal above deleted this very pending record
290
+ // (`refuseMissingBase` drops the pending it cannot promote). This node DID hold
291
+ // the record and dropped it, so the honest answer is an availability one — not
292
+ // a caller-contract violation, and never a throw that would fail the whole batch.
293
+ return [blockId, { state: {}, unavailable } as GetBlockResult];
294
+ }
294
295
  // Caller-contract violation (the caller asserted a pending this repo never had, or
295
296
  // cancelled) — an error, not an availability question. Deliberately NOT `unavailable`.
297
+ //
298
+ // It is NOT the only way to reach here. A context that both PROVES its own action
299
+ // (`committed` names it) and names it as the pending overlay (`actionId`) is
300
+ // self-contradictory, and the two halves of that contradiction land differently: if
301
+ // the read-driven promotion above REFUSED, the arm above answers gracefully; if it
302
+ // SUCCEEDED, `promotePendingTransaction` moved the record and we throw here — failing
303
+ // the whole batch for a request the refusal path tolerates. No production code sets
304
+ // `ActionContext.actionId` at all today, so neither is reachable except from tests or
305
+ // a peer that crafts the field on the wire. See
306
+ // tickets/blocked/repo-pending-overlay-has-no-producer.
296
307
  throw new Error(`Pending action ${context.actionId} not found`);
297
308
  }
298
309
  const block = applyTransform(blockRev?.block, pendingTransform);
@@ -304,13 +315,21 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
304
315
  },
305
316
  // The COMMITTED revision underneath the pending overlay. A pending has no revision
306
317
  // of its own, so the honest answer is the base it was applied to. Absent when there
307
- // was no base unreachable today (see the NOTE above), kept so the field stays
308
- // correct rather than fabricating a revision if that arm is ever reopened.
318
+ // was no base at all a pending-only insert served over an absent committed base,
319
+ // where fabricating a revision would claim content this node never committed.
309
320
  ...(blockRev ? { materializedRev: blockRev.actionRev.rev } : {}),
310
321
  // A pending applied to a missing base can materialize nothing (applyTransform drops
311
322
  // updates with no block to apply them to) — that absence is a guess, and is flagged.
312
- // A materialized block is a real answer regardless of the earlier refusal.
313
- ...(unavailable !== undefined && block === undefined ? { unavailable } : {})
323
+ // A materialized block is a real answer regardless of the earlier refusal. TWO ways
324
+ // an empty result is a guess: the promotion refusal fired (`unavailable` set), or
325
+ // there was no committed base under the overlay at all (`blockRev === undefined`) —
326
+ // this node holds a pending record PROVING the block exists and produced nothing.
327
+ // The second clause's ABSENCE in the other direction is equally load-bearing: a
328
+ // pending DELETE over a real committed base also lands here with no block, and that
329
+ // is an authoritative tombstone which must stay unflagged.
330
+ ...(block === undefined && (unavailable !== undefined || blockRev === undefined)
331
+ ? { unavailable: unavailable ?? 'unmaterializable' }
332
+ : {})
314
333
  } as GetBlockResult];
315
334
  }
316
335
 
@@ -495,6 +514,34 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
495
514
  }));
496
515
  }
497
516
 
517
+ /**
518
+ * Commit a previously-pended action across its blocks, under the per-block commit latches.
519
+ *
520
+ * **Divergence vs genuine fault.** When the batch cannot be completed, the reason decides what
521
+ * happens to the pending records the pend left behind. `ClusterMember.applyConsensusOperation`
522
+ * makes the same split one layer up — it *tolerates* a divergence (and reconciles every
523
+ * `commit.blockIds` entry from a cohort peer) but *propagates* a genuine fault for retry — so this
524
+ * method must agree with it:
525
+ *
526
+ * - **Divergence** — this node is behind the agreed history, either because it holds no
527
+ * materializable base ({@link MissingBaseRevisionError}) or because it never received the pend
528
+ * (the `Pending action … not found` throw). Reconcile is guaranteed to follow and will advance
529
+ * every block in the batch past `request.rev`, so no pending record here can ever be promoted:
530
+ * {@link dropUnpromotablePendings} deletes them (see {@link refuseMissingBase}, which already
531
+ * accepts this tradeoff for the single refusing block).
532
+ * - **Genuine fault** — any other throw out of {@link internalCommit} (a raw-storage error, …).
533
+ * `ClusterMember` propagates it and the commit is retried, and a retry can still replay the
534
+ * pendings, so they are KEPT.
535
+ *
536
+ * The stale/`missedCommits` early return (this node is AHEAD — it already holds a revision at or
537
+ * past `request.rev`, committed under a different action) deliberately keeps pendings too, and its
538
+ * cure is the losing client's `cancel`: `CoordinatorRepo.cancel` runs through consensus, so every
539
+ * member drops the record, not just the coordinator. Replication cannot be the cure here — this
540
+ * node is already ahead, and a later forward write carries a DIFFERENT action id, which is not
541
+ * what `BlockStorage.saveForwardRevision` deletes. A client that dies between the stale result and
542
+ * its `cancel` therefore still strands the record; that is pre-existing and orthogonal to the
543
+ * divergence split above.
544
+ */
498
545
  async commit(request: CommitRequest, _options?: MessageOptions): Promise<CommitResult> {
499
546
  log('commit actionId=%s rev=%d blockIds=%d', request.actionId, request.rev, request.blockIds.length);
500
547
  const uniqueBlockIds = Array.from(new Set(request.blockIds)).sort();
@@ -620,6 +667,12 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
620
667
  // pending-present — a never-pended block cannot coexist with it in one retry. If a path
621
668
  // ever produces that mix, emit recovered blocks' events before throwing here.
622
669
  if (missingPends.length) {
670
+ // Divergence (this node is behind): `ClusterMember` treats this throw as the canonical
671
+ // "behind" signal and reconciles EVERY block in the batch, advancing each past
672
+ // `request.rev`. Nothing can promote the pendings the other blocks still hold, so drop
673
+ // them here — while the latches are still held — before reporting. The thrown message
674
+ // must stay byte-identical: `ClusterMember.isMissingPendingActionError` matches on it.
675
+ await this.dropUnpromotablePendings(toCommit, request.actionId);
623
676
  throw new Error(`Pending action ${request.actionId} not found for block(s): ${missingPends.map(p => p.blockId).join(', ')}`);
624
677
  }
625
678
 
@@ -648,6 +701,10 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
648
701
  // This loop will execute atomically for all blocks due to the acquired locks.
649
702
  // Recovered (Crash-D3) blocks are already committed at request.rev and their pending is
650
703
  // gone, so skip them — internalCommit would throw on the missing pending record.
704
+ //
705
+ // Set when the mid-loop failure was a divergence rather than a genuine fault — the split
706
+ // documented on commit() above, which decides the fate of the batch's pending records.
707
+ let divergentFailure = false;
651
708
  for (const { blockId, storage } of toCommit) {
652
709
  if (recovered.has(blockId)) {
653
710
  continue;
@@ -666,9 +723,20 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
666
723
  // treats them as idempotent no-ops and advances the remainder. Break
667
724
  // instead of returning so locks release and those landings emit below.
668
725
  failure = { reason: err instanceof Error ? err.message : 'Unknown error during commit' };
726
+ divergentFailure = err instanceof MissingBaseRevisionError;
669
727
  break;
670
728
  }
671
729
  }
730
+
731
+ // The break left every not-yet-reached block still holding its pending record. Whether
732
+ // that record is still usable depends ENTIRELY on why we stopped — see the table on
733
+ // commit() above. Runs inside the try, so the per-block latches are still held.
734
+ // NOTE: a non-divergence fault deliberately KEEPS the batch's pendings so a retry can
735
+ // replay them. If ClusterMember ever stops retrying propagated commit faults, this arm
736
+ // becomes dead weight and the discriminator can collapse to "always drop".
737
+ if (divergentFailure) {
738
+ await this.dropUnpromotablePendings(toCommit, request.actionId);
739
+ }
672
740
  }
673
741
  finally {
674
742
  // Release locks in reverse order of acquisition
@@ -683,6 +751,44 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
683
751
  return failure ? { success: false, reason: failure.reason } : { success: true };
684
752
  }
685
753
 
754
+ /**
755
+ * Delete `actionId`'s pending record from every given block, tolerating absence.
756
+ *
757
+ * Called by {@link commit} when it abandons a batch **because this node has diverged from the
758
+ * agreed history** — the caller has already made that determination; this helper does not
759
+ * re-derive it. Once `ClusterMember` reconciles the batch, every one of these blocks sits at or
760
+ * past `request.rev`, so a commit retry partitions them as already-done/stale and never revisits
761
+ * their pendings; left in place they are reported as phantom conflicting actions by {@link pend}
762
+ * for every later write to the block (under `policy: 'f'`, forever).
763
+ *
764
+ * No special-casing is needed for blocks that already landed (record promoted), that were
765
+ * `recovered` (record already gone), or for the refusing block itself
766
+ * ({@link refuseMissingBase} deleted its record): deleting an absent pending record is a no-op on
767
+ * every backend.
768
+ *
769
+ * Per-block failures are logged and swallowed rather than propagated: this cleanup must never
770
+ * replace the failure the caller is about to report — the pre-loop throw's message is pattern-
771
+ * matched by `ClusterMember.isMissingPendingActionError`, and a swapped error would misroute
772
+ * consensus. A leftover record only degrades this node's participation in that one block.
773
+ */
774
+ private async dropUnpromotablePendings(
775
+ blocks: { blockId: BlockId, storage: IBlockStorage }[],
776
+ actionId: ActionId
777
+ ): Promise<void> {
778
+ if (blocks.length === 0) {
779
+ return;
780
+ }
781
+ log('commit:drop-unpromotable-pendings actionId=%s blockIds=%d', actionId, blocks.length);
782
+ await Promise.all(blocks.map(async ({ blockId, storage }) => {
783
+ try {
784
+ await storage.deletePendingTransaction(actionId);
785
+ } catch (err) {
786
+ log('commit:drop-unpromotable-pending-failed blockId=%s actionId=%s error=%s', blockId, actionId,
787
+ err instanceof Error ? err.message : String(err));
788
+ }
789
+ }));
790
+ }
791
+
686
792
  /**
687
793
  * Reconciles `metadata.latest` for a single block with the highest contiguous
688
794
  * fully-promoted revision in durable storage. Use after a crash between
@@ -830,9 +936,13 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
830
936
  * BlockStorage reports every one of these as a bare `Error`, so they cannot be told apart here,
831
937
  * and treating them as divergence is the safe default: this node genuinely cannot materialize the
832
938
  * base right now, and the cluster's policy is to heal rather than throw out of consensus. The
833
- * price is that a transient fault ALSO drops the pending (see {@link refuseMissingBase}), so the
834
- * block converges by replication instead of by a replay the retry could have done. Narrowing this
835
- * would require typed faults out of BlockStorage; until then, prefer the tolerant reading.
939
+ * price is that a transient fault ALSO drops pending records this block's (see
940
+ * {@link refuseMissingBase}) AND, because {@link commit} keys its cleanup off the same error type,
941
+ * every not-yet-reached block in the same batch so those blocks converge by replication instead
942
+ * of by a replay the retry could have done. That is a wider blast radius than the per-block
943
+ * refusal alone, and it is why the discriminator must NOT be loosened beyond this error type.
944
+ * Narrowing this would require typed faults out of BlockStorage; until then, prefer the tolerant
945
+ * reading.
836
946
  */
837
947
  private async readCommitBase(
838
948
  blockId: BlockId,
@@ -881,9 +991,10 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
881
991
  * Converts list of missing actions per block into a list of missing actions across blocks.
882
992
  *
883
993
  * NOTE: relies on each (actionId, blockId) pair appearing at most once — one revision per action
884
- * per block. If a block ever records two revisions under the same actionId, concatTransform's
885
- * last-wins merge would silently drop the earlier revision's ops for that block; group by
886
- * (actionId, rev) instead. See debt-concat-transform-overlapping-updates.
994
+ * per block. If a block ever records two revisions under the same actionId, concatTransform now
995
+ * concatenates both revisions' ops into one array rather than dropping the earlier one — still
996
+ * wrong, since ops from distinct revisions are not composable against a single base, but loud
997
+ * rather than silent. Group by (actionId, rev) instead if that case becomes reachable.
887
998
  */
888
999
  function perBlockActionTransformsToPerAction(missing: { blockId: BlockId; transforms: ActionTransform[]; }[]) {
889
1000
  const missingFlat = missing.flatMap(({ blockId, transforms }) =>
@@ -194,6 +194,26 @@ export function runRawStorageConformance(
194
194
  expect(await collect(storage.listPendingTransactions(blockId))).to.deep.equal([]);
195
195
  });
196
196
 
197
+ it('deletePendingTransaction on an absent entry is a tolerated no-op', async () => {
198
+ // Load-bearing across backends: the commit path deletes a pending record it has NOT
199
+ // checked for presence — `StorageRepo.dropUnpromotablePendings` over a whole batch, and
200
+ // `BlockStorage.saveForwardRevision` maintaining Invariant P — so a backend that threw
201
+ // (or rejected) on a miss would turn every ordinary commit into a logged failure.
202
+ const blockId = 'pend-del-absent' as BlockId;
203
+ await storage.deletePendingTransaction(blockId, 'never-pended' as ActionId);
204
+
205
+ // Also a no-op beside a LIVE pending for a different action: the delete must be scoped
206
+ // to its action id, never a per-block clear.
207
+ await storage.savePendingTransaction(blockId, 'a1' as ActionId, { delete: true });
208
+ await storage.deletePendingTransaction(blockId, 'other' as ActionId);
209
+ expect(await storage.getPendingTransaction(blockId, 'a1' as ActionId)).to.deep.equal({ delete: true });
210
+
211
+ // Idempotent: deleting the same entry twice is fine.
212
+ await storage.deletePendingTransaction(blockId, 'a1' as ActionId);
213
+ await storage.deletePendingTransaction(blockId, 'a1' as ActionId);
214
+ expect(await collect(storage.listPendingTransactions(blockId))).to.deep.equal([]);
215
+ });
216
+
197
217
  it('listPendingTransactions tolerates awaits interleaved between yields (drain-before-yield)', async () => {
198
218
  const blockId = 'drain-pend' as BlockId;
199
219
  await storage.savePendingTransaction(blockId, 'a1' as ActionId, { delete: true });