@optimystic/db-core 1.12.1 → 1.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/dist/src/cluster/i-cluster.d.ts +16 -1
  2. package/dist/src/cluster/i-cluster.d.ts.map +1 -1
  3. package/dist/src/cluster/structs.d.ts +60 -6
  4. package/dist/src/cluster/structs.d.ts.map +1 -1
  5. package/dist/src/cluster/structs.js +5 -0
  6. package/dist/src/cluster/structs.js.map +1 -1
  7. package/dist/src/collection/collection.d.ts.map +1 -1
  8. package/dist/src/collection/collection.js +5 -3
  9. package/dist/src/collection/collection.js.map +1 -1
  10. package/dist/src/index.d.ts +1 -0
  11. package/dist/src/index.d.ts.map +1 -1
  12. package/dist/src/index.js +1 -0
  13. package/dist/src/index.js.map +1 -1
  14. package/dist/src/network/durability.d.ts +17 -4
  15. package/dist/src/network/durability.d.ts.map +1 -1
  16. package/dist/src/network/durability.js +60 -15
  17. package/dist/src/network/durability.js.map +1 -1
  18. package/dist/src/network/i-key-network.d.ts +9 -0
  19. package/dist/src/network/i-key-network.d.ts.map +1 -1
  20. package/dist/src/network/struct.d.ts +45 -5
  21. package/dist/src/network/struct.d.ts.map +1 -1
  22. package/dist/src/network/struct.js.map +1 -1
  23. package/dist/src/transaction/coordinator.d.ts +23 -14
  24. package/dist/src/transaction/coordinator.d.ts.map +1 -1
  25. package/dist/src/transaction/coordinator.js +84 -48
  26. package/dist/src/transaction/coordinator.js.map +1 -1
  27. package/dist/src/transaction/durability.d.ts +31 -0
  28. package/dist/src/transaction/durability.d.ts.map +1 -0
  29. package/dist/src/transaction/durability.js +36 -0
  30. package/dist/src/transaction/durability.js.map +1 -0
  31. package/dist/src/transaction/errors.d.ts +14 -6
  32. package/dist/src/transaction/errors.d.ts.map +1 -1
  33. package/dist/src/transaction/errors.js +16 -5
  34. package/dist/src/transaction/errors.js.map +1 -1
  35. package/dist/src/transaction/index.d.ts +2 -0
  36. package/dist/src/transaction/index.d.ts.map +1 -1
  37. package/dist/src/transaction/index.js +1 -0
  38. package/dist/src/transaction/index.js.map +1 -1
  39. package/dist/src/transaction/session.d.ts +2 -0
  40. package/dist/src/transaction/session.d.ts.map +1 -1
  41. package/dist/src/transaction/session.js +4 -2
  42. package/dist/src/transaction/session.js.map +1 -1
  43. package/dist/src/transaction/transaction.d.ts +9 -0
  44. package/dist/src/transaction/transaction.d.ts.map +1 -1
  45. package/dist/src/transaction/transaction.js.map +1 -1
  46. package/dist/src/transactor/change-notifier.d.ts +20 -8
  47. package/dist/src/transactor/change-notifier.d.ts.map +1 -1
  48. package/dist/src/transactor/change-notifier.js.map +1 -1
  49. package/dist/src/transactor/network-transactor.d.ts +48 -5
  50. package/dist/src/transactor/network-transactor.d.ts.map +1 -1
  51. package/dist/src/transactor/network-transactor.js +98 -22
  52. package/dist/src/transactor/network-transactor.js.map +1 -1
  53. package/dist/src/transactor/owed-cancels.d.ts +96 -0
  54. package/dist/src/transactor/owed-cancels.d.ts.map +1 -0
  55. package/dist/src/transactor/owed-cancels.js +211 -0
  56. package/dist/src/transactor/owed-cancels.js.map +1 -0
  57. package/dist/src/transactor/transactor-source.d.ts +5 -4
  58. package/dist/src/transactor/transactor-source.d.ts.map +1 -1
  59. package/dist/src/transactor/transactor-source.js +11 -6
  60. package/dist/src/transactor/transactor-source.js.map +1 -1
  61. package/dist/src/utility/unref-timer.d.ts +9 -0
  62. package/dist/src/utility/unref-timer.d.ts.map +1 -0
  63. package/dist/src/utility/unref-timer.js +15 -0
  64. package/dist/src/utility/unref-timer.js.map +1 -0
  65. package/package.json +1 -1
  66. package/src/cluster/i-cluster.ts +13 -2
  67. package/src/cluster/structs.ts +62 -6
  68. package/src/collection/collection.ts +5 -3
  69. package/src/index.ts +1 -0
  70. package/src/network/durability.ts +63 -15
  71. package/src/network/i-key-network.ts +10 -0
  72. package/src/network/struct.ts +45 -5
  73. package/src/transaction/coordinator.ts +130 -79
  74. package/src/transaction/durability.ts +60 -0
  75. package/src/transaction/errors.ts +18 -3
  76. package/src/transaction/index.ts +2 -0
  77. package/src/transaction/session.ts +4 -2
  78. package/src/transaction/transaction.ts +9 -0
  79. package/src/transactor/change-notifier.ts +20 -8
  80. package/src/transactor/network-transactor.ts +120 -29
  81. package/src/transactor/owed-cancels.ts +252 -0
  82. package/src/transactor/transactor-source.ts +11 -6
  83. package/src/utility/unref-timer.ts +15 -0
@@ -2,6 +2,7 @@ import { randomBytes } from "@noble/hashes/utils.js";
2
2
  import { toString as uint8ArrayToString } from "uint8arrays/to-string";
3
3
  import type { BlockId } from "../blocks/index.js";
4
4
  import type { CollectionId } from "../collection/index.js";
5
+ import type { TransactionDurability } from "./durability.js";
5
6
  import { hashString } from "../utility/hash-string.js";
6
7
 
7
8
  /**
@@ -386,6 +387,14 @@ export type ExecutionResult = {
386
387
  committedCollections?: CollectionId[];
387
388
  /** On a partial multi-collection commit failure, the collections that failed to commit. */
388
389
  failedCollections?: CollectionId[];
390
+ /**
391
+ * Who holds what this commit wrote, present whenever something was written: on success, every
392
+ * collection the commit wrote; on a partial failure, exactly {@link committedCollections}. Absent
393
+ * when nothing was written — a failure where nothing landed, or a success with nothing staged.
394
+ * Test it through `isFullyDurable(durability.overall)`; an absent `overall` means a collection is
395
+ * saved but who holds it is unknown, so show the change as pending.
396
+ */
397
+ durability?: TransactionDurability;
389
398
  };
390
399
 
391
400
  /**
@@ -85,25 +85,37 @@ export function isBlockChangeNotifier(x: unknown): x is IBlockChangeNotifier {
85
85
  * acknowledged below `full` (`WriteDurability.quorum`) is pending until these events cover its
86
86
  * blocks, at which point a host can move it from "pending" to "saved".
87
87
  *
88
+ * **Fired only on the node whose ledger recorded the shortfall** — the one a below-`full` answer
89
+ * names as `WriteDurability.recordedBy`. An answer without `recordedBy` is a write no node is
90
+ * tracking, and no event should be expected for it. That node fires the event when its record of a
91
+ * block's shortfall is removed, which happens two ways, each firing exactly once per record: the
92
+ * node's push of the missing copies empties it (the event carries the record's own `rev` and
93
+ * `actionId`), or a later commit of the block that every member confirmed settles it (the event
94
+ * carries that later commit's `rev` and `actionId`).
95
+ *
88
96
  * **Keyed by block, deliberately, never by action.** The node's record of what is still owed (the
89
97
  * under-replication ledger) keeps ONE entry per block, at the block's highest under-replicated
90
98
  * revision: a block written again by a LATER action replaces the earlier entry, so the earlier
91
99
  * action's shortfall is cleared by the later action's copies and reported under the later
92
100
  * action's `actionId` — the earlier action never fires an event of its own. A host that keys its
93
- * pending state by action alone therefore waits forever for a superseded write. Hold the block ids
94
- * the write's result names (`PendSuccess.blockIds`) and clear each as an event covers it; `rev`
95
- * and `actionId` say which write's copies did the covering.
101
+ * pending state by action alone therefore waits forever for a superseded write. Hold the blocks the
102
+ * write's durability names (`WriteDurability.blockRevs`, each with the revision it committed at) and
103
+ * clear a block when an event names it at that revision or a later one; `rev` and `actionId` say
104
+ * which write's copies did the covering. An event can arrive before the write's answer has reached
105
+ * the host, so a host that registers on the answer must also remember events it has already seen.
96
106
  */
97
107
  export type BlockDurabilityReachedEvent = {
98
- /** Blocks that reached full replication in this round. Today's producer reports one block per
99
- * event; the list shape lets a producer report several blocks of one action together. */
108
+ /** Blocks that reached full replication in this round. Both of today's producers report one block
109
+ * per event; the list shape lets a producer report several blocks of one action together. */
100
110
  readonly blockIds: readonly BlockId[];
101
111
  /** The revision every cohort member now holds — the block's highest under-replicated revision,
102
- * which supersedes every lower one. */
112
+ * or the fully replicated commit that settled it, either of which supersedes every lower one. */
103
113
  readonly rev: number;
104
- /** The action whose recorded shortfall cleared. See the note above on superseded writes. */
114
+ /** The action whose copies covered the block: the one whose recorded shortfall cleared, or the
115
+ * fully replicated commit that settled it. See the note above on superseded writes. */
105
116
  readonly actionId: ActionId;
106
- /** Best-effort; absent when the collection could not be resolved from the block's header. */
117
+ /** Best-effort; absent when the collection could not be resolved from the block's header, and
118
+ * always when a settling commit fired the event (the coordinator does not read the block). */
107
119
  readonly collectionId?: CollectionId;
108
120
  };
109
121
 
@@ -17,6 +17,7 @@ import { routingKeyForBlock } from "../network/routing-key.js";
17
17
  import { isRecordEmpty } from "../utility/is-record-empty.js";
18
18
  import { type CoordinatorBatch, makeBatchesByPeer, incompleteBatches, everyBatch, allBatches, mergeBlocks, processBatches, createBatchesForPayload } from "../utility/batch-coordinator.js";
19
19
  import { abortableDelay, jitteredBackoffMs } from "../utility/backoff.js";
20
+ import { OwedCancels, type OwedCancelClock } from "./owed-cancels.js";
20
21
  import { createLogger, verbose } from "../logger.js";
21
22
 
22
23
  const log = createLogger('network-transactor');
@@ -54,6 +55,12 @@ type NetworkTransactorInit = {
54
55
  * which is the nearest.
55
56
  */
56
57
  localPeerId?: PeerId;
58
+ /**
59
+ * Time for the re-sends of cancels this transactor could not discharge ({@link OwedCancels}).
60
+ * Omit in production (`Date.now` and an unref'd `setTimeout`); a test injects a fake to fire the
61
+ * re-send without waiting on the wall clock.
62
+ */
63
+ owedCancelClock?: OwedCancelClock;
57
64
  }
58
65
 
59
66
  /**
@@ -95,6 +102,12 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
95
102
  private readonly txnCoordinatorCache = new Map<ActionId, { coordinators: Map<BlockId, PeerId>; expires: number }>();
96
103
  private static readonly MAX_TXN_COORDINATOR_CACHE_ENTRIES = 1000;
97
104
 
105
+ /**
106
+ * Cancels {@link dischargeCancel} gave up on, kept and re-sent before this transactor's next pend
107
+ * of their blocks and on a backoff timer, for this transactor's life — see {@link OwedCancels}.
108
+ */
109
+ private readonly owedCancels: OwedCancels;
110
+
98
111
  constructor(
99
112
  init: NetworkTransactorInit,
100
113
  ) {
@@ -109,6 +122,28 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
109
122
  this.getRepo = init.getRepo;
110
123
  this.localChangeNotifier = init.localChangeNotifier;
111
124
  this.localPeerId = init.localPeerId?.toString();
125
+ this.owedCancels = new OwedCancels({
126
+ attempt: actionRef => this.resendOwedCancel(actionRef),
127
+ clock: init.owedCancelClock
128
+ });
129
+ }
130
+
131
+ /**
132
+ * Every action this transactor owes a cancel for — one it sent, could not get answered, and is
133
+ * still re-sending — with the blocks still owed. A snapshot, for diagnostics: an embedder about to
134
+ * drop this transactor can see what it would leave standing.
135
+ */
136
+ owedCancelBlocks(): Map<ActionId, BlockId[]> {
137
+ return this.owedCancels.snapshot();
138
+ }
139
+
140
+ /**
141
+ * Stops re-sending owed cancels; whatever is still owed is logged (`cancel:owed-abandoned`) and
142
+ * left standing. Call when the node this transactor sends through stops, or its re-send timer keeps
143
+ * the transactor alive, retrying against a node that can no longer answer, for the life of the process.
144
+ */
145
+ stop(): void {
146
+ this.owedCancels.stop();
112
147
  }
113
148
 
114
149
  /**
@@ -648,6 +683,9 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
648
683
  : transformsFromTransform(filteredTransform, blockId);
649
684
  };
650
685
  const blockIds = blockIdsForTransforms(blockAction.transforms);
686
+ // A cancel this transactor still owes on any of these blocks goes out first: its records would
687
+ // otherwise refuse this pend — or, for this same action id, the late cancel would delete it.
688
+ await this.owedCancels.beforePend(blockAction.actionId, blockIds);
651
689
  const batches = await this.consolidateCoordinators(blockIds, blockAction.transforms, transformForBlock);
652
690
  log('pend actionId=%s blockIds=%d batches=%d', blockAction.actionId, blockIds.length, batches.length);
653
691
  if (verbose) {
@@ -1015,8 +1053,9 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
1015
1053
  * Confirmed blocks are excluded only to skip a pointless consensus round — cancelling one would
1016
1054
  * also be a no-op — so an over-broad `confirmed` set costs latency, never correctness.
1017
1055
  *
1018
- * Residuals: the cancel is retried and checked ({@link dischargeCancel}) but still bounded, so a
1019
- * fault outlasting its budget leaves the record wedged until a node-side backstop sweep exists (backlog:
1056
+ * Residuals: the cancel is retried and checked ({@link dischargeCancel}), and one that outlasts its
1057
+ * budget is kept and re-sent for this transactor's life ({@link owedCancels}), so only a writer that
1058
+ * exits first leaves the record wedged until a node-side backstop sweep exists (backlog:
1020
1059
  * `debt-unpromotable-pending-records-need-a-sweep`); and this covers only the sweep's abandonment
1021
1060
  * — `StorageRepo.commit`'s genuine-fault arm deliberately KEEPS a failed batch's pendings for a
1022
1061
  * retry, so it is a second producer of the same durable state whenever that retry never comes.
@@ -1046,7 +1085,7 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
1046
1085
  try {
1047
1086
  await this.cancel({ actionId, blockIds: abandoned });
1048
1087
  } catch (cancelError) {
1049
- try { log('WARN: cancel of abandoned sweep blocks failed — pending records may wedge until the node-side sweep lands: %o', cancelError); } catch { /* ignore */ }
1088
+ try { log('WARN: cancel of abandoned sweep blocks failed — kept and re-sent until it lands (cancel:owed): %o', cancelError); } catch { /* ignore */ }
1050
1089
  }
1051
1090
  return abandoned;
1052
1091
  }
@@ -1280,8 +1319,14 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
1280
1319
  * cost latency only. Each round narrows to the blocks still outstanding, so a partially
1281
1320
  * successful round does not re-issue the cancels that already landed.
1282
1321
  *
1283
- * NOTE: a fault that outlasts these bounds still strands the record, and nothing node-side
1284
- * reclaims it. That residual is the backstop tracked in backlog
1322
+ * **What happens when the bounds run out.** The blocks still outstanding are handed to
1323
+ * {@link owedCancels} before the throw, which re-sends their cancel before this transactor's next
1324
+ * pend of any of them and on a backoff timer, for the transactor's life — a fault that outlasts
1325
+ * this loop costs the records' holders some refused writes, not the block. Recording it here,
1326
+ * from this loop's own outstanding set, is what spares every caller from having to remember to.
1327
+ *
1328
+ * NOTE: a writer that exits (or stops this transactor, {@link stop}) while a cancel is still owed
1329
+ * strands the record, and nothing node-side reclaims it. That residual is the backstop tracked in backlog
1285
1330
  * `debt-unpromotable-pending-records-need-a-sweep`.
1286
1331
  *
1287
1332
  * @param seedBatches Batches to use for round 0 only (see {@link cancelBatch}). Omitted → round
@@ -1291,7 +1336,8 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
1291
1336
  actionRef: ActionBlocks,
1292
1337
  seedBatches?: CoordinatorBatch<BlockId[], void>[]
1293
1338
  ): Promise<void> {
1294
- let outstanding = Array.from(new Set(actionRef.blockIds));
1339
+ const requested = Array.from(new Set(actionRef.blockIds));
1340
+ let outstanding = requested;
1295
1341
  if (outstanding.length === 0) {
1296
1342
  return;
1297
1343
  }
@@ -1299,30 +1345,15 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
1299
1345
  let roundBatches: CoordinatorBatch<BlockId[], void>[] = [];
1300
1346
  let lastError: Error | undefined;
1301
1347
  for (let round = 0; ; ++round) {
1302
- roundBatches = [];
1303
- try {
1304
- // Batch construction can throw (coordinator lookup); `processBatches` cannot.
1305
- roundBatches = round === 0 && seedBatches
1306
- ? seedBatches
1307
- : await this.batchesForPayload<BlockId[], void>(outstanding, outstanding, mergeBlocks, []);
1308
- await processBatches(
1309
- roundBatches,
1310
- (batch) => this.getRepo(batch.peerId).cancel({ actionId: actionRef.actionId, blockIds: batch.payload }, { expiration: deadline, dialTimeoutMs: this.dialTimeoutMs }),
1311
- batch => batch.payload,
1312
- mergeBlocks,
1313
- deadline,
1314
- (blockId, options) => this.keyNetwork.findCoordinator(routingKeyForBlock(blockId), options)
1315
- );
1316
- } catch (e) {
1317
- lastError = asError(e);
1318
- }
1319
-
1320
- const discharged = dischargedBlocks(roundBatches);
1321
- outstanding = outstanding.filter(bid => !discharged.has(bid));
1348
+ const result = await this.cancelRound(actionRef.actionId, outstanding, deadline, round === 0 ? seedBatches : undefined);
1349
+ roundBatches = result.batches;
1350
+ lastError = result.error ?? lastError;
1351
+ outstanding = outstanding.filter(bid => !result.discharged.has(bid));
1322
1352
  if (outstanding.length === 0) {
1323
1353
  if (round > 0) {
1324
1354
  log('cancel:discharged actionId=%s rounds=%d', actionRef.actionId, round + 1);
1325
1355
  }
1356
+ this.owedCancels.discharged(actionRef.actionId, requested);
1326
1357
  return;
1327
1358
  }
1328
1359
 
@@ -1341,6 +1372,9 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
1341
1372
  return `${b.peerId.toString()}[block:${b.blockId}](${status})${errMsg}`;
1342
1373
  });
1343
1374
  const rootCause = firstBatchError(roundBatches) ?? lastError;
1375
+ const stillHeld = new Set(outstanding);
1376
+ this.owedCancels.discharged(actionRef.actionId, requested.filter(bid => !stillHeld.has(bid)));
1377
+ this.owedCancels.record(actionRef.actionId, outstanding, rootCause?.message);
1344
1378
  const aggregate = new Error(`Cancel of action ${actionRef.actionId} did not discharge ${outstanding.length} block(s): ${outstanding.join(', ')}`
1345
1379
  + (details ? `; peers: ${details}` : '')
1346
1380
  + (rootCause ? `; root: ${rootCause.message}` : ''));
@@ -1349,6 +1383,49 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
1349
1383
  throw aggregate;
1350
1384
  }
1351
1385
 
1386
+ /**
1387
+ * One cancel round for `blockIds`: batches built from `seedBatches` or resolved live, sent, and
1388
+ * judged by which blocks some peer answered for. Never throws — a failed lookup is returned as
1389
+ * `error` beside an empty discharged set.
1390
+ */
1391
+ private async cancelRound(
1392
+ actionId: ActionId,
1393
+ blockIds: BlockId[],
1394
+ deadline: number,
1395
+ seedBatches?: CoordinatorBatch<BlockId[], void>[]
1396
+ ): Promise<{ batches: CoordinatorBatch<BlockId[], void>[]; discharged: Set<BlockId>; error?: Error }> {
1397
+ let batches: CoordinatorBatch<BlockId[], void>[] = [];
1398
+ let error: Error | undefined;
1399
+ try {
1400
+ // Batch construction can throw (coordinator lookup); `processBatches` cannot.
1401
+ batches = seedBatches ?? await this.batchesForPayload<BlockId[], void>(blockIds, blockIds, mergeBlocks, []);
1402
+ await processBatches(
1403
+ batches,
1404
+ (batch) => this.getRepo(batch.peerId).cancel({ actionId, blockIds: batch.payload }, { expiration: deadline, dialTimeoutMs: this.dialTimeoutMs }),
1405
+ batch => batch.payload,
1406
+ mergeBlocks,
1407
+ deadline,
1408
+ (blockId, options) => this.keyNetwork.findCoordinator(routingKeyForBlock(blockId), options)
1409
+ );
1410
+ } catch (e) {
1411
+ error = asError(e);
1412
+ }
1413
+ return { batches, discharged: dischargedBlocks(batches), error };
1414
+ }
1415
+
1416
+ /**
1417
+ * One re-send of a cancel {@link dischargeCancel} gave up on, for {@link owedCancels}: a single
1418
+ * round within `abortOrCancelTimeoutMs`, never the full loop — it runs ahead of a pend, and a
1419
+ * cohort that still cannot approve the cancel cannot approve the pend either.
1420
+ */
1421
+ private async resendOwedCancel(actionRef: ActionBlocks): Promise<ReadonlySet<BlockId>> {
1422
+ const { discharged, error } = await this.cancelRound(actionRef.actionId, actionRef.blockIds, Date.now() + this.abortOrCancelTimeoutMs);
1423
+ if (discharged.size < actionRef.blockIds.length) {
1424
+ log('cancel:owed-retry-short actionId=%s answered=%d of %d%s', actionRef.actionId, discharged.size, actionRef.blockIds.length, error ? ` error=${error.message}` : '');
1425
+ }
1426
+ return discharged;
1427
+ }
1428
+
1352
1429
  /** Renders the batches worth naming in an aggregate error: the ones that never completed, or —
1353
1430
  * when every batch completed and the aggregate is about a non-success RESPONSE rather than a
1354
1431
  * transport fault — all of them. */
@@ -1481,10 +1558,24 @@ function baseRevsFor(all: BlockBaseRevs | undefined, batchBlockIds: BlockId[]):
1481
1558
  /** `action` re-scoped to one coordinator batch: the batch's own transforms, and the bases for the
1482
1559
  * blocks in it alone (see {@link PendRequest.baseRevs}). The action-wide `baseRevs` key is dropped
1483
1560
  * BEFORE the subset is spread, so a batch none of whose blocks names a base carries no key at all
1484
- * rather than the whole action's map — and no cohort signs for a block it is not responsible for. */
1561
+ * rather than the whole action's map — and no cohort signs for a block it is not responsible for.
1562
+ *
1563
+ * `soleReservation` survives only on a batch carrying every block of the action: a pend split across
1564
+ * coordinators — by `consolidateCoordinators`, or by `processBatches` re-batching a failed batch — is
1565
+ * several reservations, which the field must never claim to be. Decided per send, like the bases. */
1485
1566
  function pendRequestForBatch(action: PendRequest, payload: Transforms): PendRequest {
1486
- const { baseRevs, ...rest } = action;
1487
- return { ...rest, transforms: payload, ...baseRevsFor(baseRevs, blockIdsForTransforms(payload)) };
1567
+ const { baseRevs, soleReservation, ...rest } = action;
1568
+ const batchBlockIds = blockIdsForTransforms(payload);
1569
+ const whole = (): boolean => {
1570
+ const inBatch = new Set(batchBlockIds);
1571
+ return blockIdsForTransforms(action.transforms).every(blockId => inBatch.has(blockId));
1572
+ };
1573
+ return {
1574
+ ...rest,
1575
+ transforms: payload,
1576
+ ...baseRevsFor(baseRevs, batchBlockIds),
1577
+ ...(soleReservation === true && whole() ? { soleReservation } : {})
1578
+ };
1488
1579
  }
1489
1580
 
1490
1581
  /** The read one coordinator batch sends downstream: the batch's ids, the caller's context, and the
@@ -0,0 +1,252 @@
1
+ import type { ActionBlocks } from "../network/struct.js";
2
+ import type { ActionId } from "../collection/action.js";
3
+ import type { BlockId } from "../blocks/structs.js";
4
+ import { jitteredBackoffMs } from "../utility/backoff.js";
5
+ import { armUnrefTimer } from "../utility/unref-timer.js";
6
+ import { createLogger } from "../logger.js";
7
+
8
+ const log = createLogger('network-transactor');
9
+
10
+ export type OwedCancelTimerCancel = () => void;
11
+
12
+ /**
13
+ * Optional injection seam for deterministic time, the shape `ClusterCoordinatorClock` in
14
+ * `packages/db-p2p/src/repo/cluster-coordinator.ts` gives its retry timer. Production leaves both
15
+ * undefined and gets `Date.now` plus an unref'd `setTimeout`, so an owed cancel never keeps an
16
+ * otherwise-idle process alive.
17
+ */
18
+ export interface OwedCancelClock {
19
+ /** Clock (Unix ms). Defaults to `Date.now`. */
20
+ now?: () => number;
21
+ /** Schedule a one-shot timer, returning its cancel. Defaults to an unref'd `setTimeout`. */
22
+ setTimer?: (fn: () => void, delayMs: number) => OwedCancelTimerCancel;
23
+ }
24
+
25
+ export interface OwedCancelsInit {
26
+ /**
27
+ * One bounded cancel attempt for `actionRef`, resolving with the blocks some peer ANSWERED for —
28
+ * the same discharged set the transactor's own cancel loop judges by. A throw counts as nothing
29
+ * answered.
30
+ */
31
+ attempt: (actionRef: ActionBlocks) => Promise<ReadonlySet<BlockId>>;
32
+ clock?: OwedCancelClock;
33
+ /** Most actions remembered at once. Default {@link DEFAULT_MAX_OWED_ACTIONS}. */
34
+ maxActions?: number;
35
+ }
36
+
37
+ /**
38
+ * Bounds the memory a writer spends on cancels it cannot land. Evicting an entry re-opens the wedge
39
+ * that entry was kept to close, so it is logged; a writer that owes this many is cut off from its
40
+ * cohorts, and its records are the node-side backstop's (backlog
41
+ * `debt-unpromotable-pending-records-need-a-sweep`).
42
+ */
43
+ const DEFAULT_MAX_OWED_ACTIONS = 256;
44
+
45
+ /** The backoff between re-sends of one owed cancel: a second at first, doubling to half a minute. */
46
+ const RETRY_BACKOFF = { baseMs: 1_000, capMs: 30_000 };
47
+
48
+ interface OwedEntry {
49
+ readonly blockIds: Set<BlockId>;
50
+ /** When the cancel first fell short, for the discharge log. */
51
+ readonly since: number;
52
+ /** Re-sends made so far — the backoff's exponent. */
53
+ attempts: number;
54
+ nextAttemptAt: number;
55
+ }
56
+
57
+ /**
58
+ * The cancels a writer could not discharge, kept and re-sent until they land.
59
+ *
60
+ * A cancel whose rounds all fall short — every one of them is a cluster transaction, so the member
61
+ * whose silence sank the commit sinks the cancel too — leaves pending records standing that make the
62
+ * members refuse every later write to those blocks, from any writer. Nothing else removes them while
63
+ * their slot is unreached (docs/repository.md, "A pending record's lifetime is bounded by its
64
+ * writer"), so the writer keeps the cancel and sends it again:
65
+ *
66
+ * - **before its own next pend of any of those blocks** ({@link beforePend}), which is what clears
67
+ * the leftover in front of the writer's next write to the block;
68
+ * - **on a backoff timer** while anything is owed, so other machines' writes, which the records also
69
+ * refuse, are freed without this writer writing the block again.
70
+ *
71
+ * Re-sending is safe whenever it arrives: a cancel deletes only the cancelled action's pending records
72
+ * and never moves a revision, so it commutes with every other action. The one exception is a re-pend
73
+ * of the SAME action id, whose new record a late cancel would delete — so a pend of an owed action
74
+ * supersedes the owed cancel for every block it names, and waits out any re-send of it already in
75
+ * flight before it goes out.
76
+ */
77
+ export class OwedCancels {
78
+ private readonly entries = new Map<ActionId, OwedEntry>();
79
+ private readonly inFlight = new Map<ActionId, Promise<void>>();
80
+ private readonly attempt: OwedCancelsInit['attempt'];
81
+ private readonly now: () => number;
82
+ private readonly setTimer: (fn: () => void, delayMs: number) => OwedCancelTimerCancel;
83
+ private readonly maxActions: number;
84
+ private timer: { cancel: OwedCancelTimerCancel; dueAt: number } | undefined;
85
+ private stopped = false;
86
+
87
+ constructor(init: OwedCancelsInit) {
88
+ this.attempt = init.attempt;
89
+ this.now = init.clock?.now ?? ((): number => Date.now());
90
+ this.setTimer = init.clock?.setTimer ?? armUnrefTimer;
91
+ this.maxActions = init.maxActions ?? DEFAULT_MAX_OWED_ACTIONS;
92
+ }
93
+
94
+ /** Every owed action with the blocks it still owes a cancel for. A snapshot. */
95
+ snapshot(): Map<ActionId, BlockId[]> {
96
+ return new Map(Array.from(this.entries, ([actionId, entry]) => [actionId, Array.from(entry.blockIds)]));
97
+ }
98
+
99
+ /** A cancel for `actionId` ran out of budget with `blockIds` still unanswered. */
100
+ record(actionId: ActionId, blockIds: readonly BlockId[], why: string | undefined): void {
101
+ if (blockIds.length === 0) return;
102
+ if (this.stopped) {
103
+ log('cancel:owed-abandoned actionId=%s blocks=%o why=stopped', actionId, blockIds);
104
+ return;
105
+ }
106
+ const existing = this.entries.get(actionId);
107
+ if (existing) {
108
+ for (const bid of blockIds) existing.blockIds.add(bid);
109
+ existing.nextAttemptAt = this.now() + this.backoffMs(existing.attempts);
110
+ } else {
111
+ const now = this.now();
112
+ this.entries.set(actionId, { blockIds: new Set(blockIds), since: now, attempts: 0, nextAttemptAt: now + this.backoffMs(0) });
113
+ this.evictOverflow();
114
+ }
115
+ log('cancel:owed actionId=%s blocks=%o why=%s', actionId, blockIds, why ?? 'unknown');
116
+ this.arm();
117
+ }
118
+
119
+ /** A cancel for `actionId` was answered for `blockIds` — whoever sent it, they are owed no longer. */
120
+ discharged(actionId: ActionId, blockIds: Iterable<BlockId>): void {
121
+ const settled = this.remove(actionId, blockIds);
122
+ if (settled) {
123
+ log('cancel:owed-discharged actionId=%s attempts=%d stoodMs=%d', actionId, settled.attempts, this.now() - settled.since);
124
+ }
125
+ }
126
+
127
+ /**
128
+ * Re-sends nothing from here on: the timer is disarmed and every cancel still owed is logged
129
+ * (`cancel:owed-abandoned`) and forgotten, its records left to the node-side backstop. For when the
130
+ * node the re-sends go through stops — they could never land, and the timer would otherwise keep
131
+ * the transactor, and everything it reaches, alive for the life of the process.
132
+ */
133
+ stop(): void {
134
+ this.stopped = true;
135
+ this.timer?.cancel();
136
+ this.timer = undefined;
137
+ for (const [actionId, entry] of this.entries) {
138
+ log('cancel:owed-abandoned actionId=%s blocks=%o stoodMs=%d', actionId, Array.from(entry.blockIds), this.now() - entry.since);
139
+ }
140
+ this.entries.clear();
141
+ }
142
+
143
+ /**
144
+ * Runs ahead of every pend of `actionId` over `blockIds`: supersedes this action's own owed cancel
145
+ * for those blocks, then makes one attempt at every OTHER owed cancel that names any of them. A
146
+ * failed attempt stays owed and the pend proceeds; a cohort still unable to approve the cancel
147
+ * cannot approve the pend either, so one attempt — not a loop — is what the pend should wait for.
148
+ */
149
+ async beforePend(actionId: ActionId, blockIds: readonly BlockId[]): Promise<void> {
150
+ if (this.entries.size === 0 && this.inFlight.size === 0) return;
151
+ await this.supersede(actionId, blockIds);
152
+ const named = new Set(blockIds);
153
+ const overlapping = Array.from(this.entries)
154
+ .filter(([owedId, entry]) => owedId !== actionId && Array.from(entry.blockIds).some(bid => named.has(bid)))
155
+ .map(([owedId]) => owedId);
156
+ await Promise.all(overlapping.map(owedId => this.attemptOnce(owedId)));
157
+ }
158
+
159
+ /**
160
+ * A pend of `actionId` writes a fresh record on every block it names, which a late cancel for the
161
+ * same action id would delete. So those blocks leave the owed entry first — synchronously, so no
162
+ * re-send started from here on can carry them — and a re-send already in flight is waited out, so
163
+ * it lands before the pend rather than after it.
164
+ */
165
+ private async supersede(actionId: ActionId, blockIds: readonly BlockId[]): Promise<void> {
166
+ const entry = this.entries.get(actionId);
167
+ const superseded = entry ? blockIds.filter(bid => entry.blockIds.has(bid)) : [];
168
+ if (superseded.length > 0) {
169
+ log('cancel:owed-superseded actionId=%s blocks=%o', actionId, superseded);
170
+ this.remove(actionId, superseded);
171
+ }
172
+ await this.inFlight.get(actionId);
173
+ }
174
+
175
+ /** Takes `blockIds` off `actionId`'s entry, returning the entry when that emptied — and so removed — it. */
176
+ private remove(actionId: ActionId, blockIds: Iterable<BlockId>): OwedEntry | undefined {
177
+ const entry = this.entries.get(actionId);
178
+ if (!entry) return undefined;
179
+ for (const bid of blockIds) entry.blockIds.delete(bid);
180
+ if (entry.blockIds.size > 0) return undefined;
181
+ this.entries.delete(actionId);
182
+ this.arm();
183
+ return entry;
184
+ }
185
+
186
+ /** One re-send of `actionId`'s owed cancel, joining the one already in flight if there is one. */
187
+ private attemptOnce(actionId: ActionId): Promise<void> {
188
+ const running = this.inFlight.get(actionId);
189
+ if (running) return running;
190
+ const entry = this.entries.get(actionId);
191
+ if (!entry) return Promise.resolve();
192
+ ++entry.attempts;
193
+ const run = this.resend(actionId, Array.from(entry.blockIds))
194
+ .finally(() => {
195
+ if (this.inFlight.get(actionId) === run) this.inFlight.delete(actionId);
196
+ this.arm();
197
+ });
198
+ this.inFlight.set(actionId, run);
199
+ return run;
200
+ }
201
+
202
+ private async resend(actionId: ActionId, blockIds: BlockId[]): Promise<void> {
203
+ let answered: ReadonlySet<BlockId>;
204
+ try {
205
+ answered = await this.attempt({ actionId, blockIds });
206
+ } catch (error) {
207
+ log('cancel:owed-retry-failed actionId=%s blocks=%d error=%s', actionId, blockIds.length, (error as Error)?.message ?? String(error));
208
+ answered = new Set();
209
+ }
210
+ this.discharged(actionId, answered);
211
+ const entry = this.entries.get(actionId);
212
+ if (entry) {
213
+ entry.nextAttemptAt = this.now() + this.backoffMs(entry.attempts);
214
+ }
215
+ }
216
+
217
+ /** Arms the one timer at the earliest re-send due among the entries not already being re-sent, or disarms it. */
218
+ private arm(): void {
219
+ let dueAt: number | undefined;
220
+ for (const [actionId, entry] of this.entries) {
221
+ if (this.inFlight.has(actionId)) continue;
222
+ if (dueAt === undefined || entry.nextAttemptAt < dueAt) dueAt = entry.nextAttemptAt;
223
+ }
224
+ if (this.timer && (dueAt === undefined || this.timer.dueAt > dueAt)) {
225
+ this.timer.cancel();
226
+ this.timer = undefined;
227
+ }
228
+ if (dueAt === undefined || this.timer) return;
229
+ this.timer = { cancel: this.setTimer(() => this.fire(), Math.max(0, dueAt - this.now())), dueAt };
230
+ }
231
+
232
+ private fire(): void {
233
+ this.timer = undefined;
234
+ const now = this.now();
235
+ for (const [actionId, entry] of this.entries) {
236
+ if (entry.nextAttemptAt <= now) void this.attemptOnce(actionId);
237
+ }
238
+ this.arm();
239
+ }
240
+
241
+ private backoffMs(attempts: number): number {
242
+ return jitteredBackoffMs(attempts, RETRY_BACKOFF);
243
+ }
244
+
245
+ private evictOverflow(): void {
246
+ while (this.entries.size > this.maxActions) {
247
+ const [oldestId, oldest] = this.entries.entries().next().value as [ActionId, OwedEntry];
248
+ this.entries.delete(oldestId);
249
+ log('cancel:owed-evicted actionId=%s blocks=%o stoodMs=%d', oldestId, Array.from(oldest.blockIds), this.now() - oldest.since);
250
+ }
251
+ }
252
+ }
@@ -2,6 +2,7 @@ import { randomBytes } from '@noble/hashes/utils.js'
2
2
  import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
3
3
  import type { IBlock, BlockId, BlockHeader, ITransactor, ActionId, CommitResult, ActionContext, BlockType, BlockSource, ReadPurpose, Transforms, BlockContentDigests, BlockBaseRevs, GetBlockResult } from "../index.js";
4
4
  import { BlockUnavailableError, BlockPossiblyStaleError } from "../network/struct.js";
5
+ import { withBlockRevs } from "../network/durability.js";
5
6
  import type { ReadDependency } from "../transaction/transaction.js";
6
7
  import { ReadDependencyCollector } from "../transaction/read-dependency-collector.js";
7
8
  import { blockDigestsField, baseRevsField } from "../transform/digest.js";
@@ -228,17 +229,20 @@ export class TransactorSource<TBlock extends IBlock> implements BlockSource<TBlo
228
229
  * committed revision each update-only block's operations were computed against, from the same tracker
229
230
  * (`Tracker.stagedBaseRevs`). Omitted from the pend request when empty, for the same preimage reason.
230
231
  * @returns The transactor's own verdict, unflattened: a {@link CommitSuccess} carrying the
231
- * {@link WriteDurability} of the committed revision, or a {@link StaleFailure} if the pend or the commit
232
- * was refused. Success is deliberately NOT collapsed to `undefined` — the durability is the only thing
233
- * that tells a write every machine holds from one only this machine holds, and a caller that wants the
234
- * old boolean reads `result.success`. Test "is this completely saved" through `isFullyDurable`, never
232
+ * {@link WriteDurability} of the committed revision — naming every block it committed at `rev`
233
+ * (`blockRevs`) — or a {@link StaleFailure} if the pend or the commit was refused. Success is
234
+ * deliberately NOT collapsed to `undefined` — the durability is the only thing that tells a write every
235
+ * machine holds from one only this machine holds, and a caller that wants the old boolean reads
236
+ * `result.success`. Test "is this completely saved" through `isFullyDurable`, never
235
237
  * by comparing `quorum` (see {@link WriteDurability}).
236
238
  */
237
239
  async transact(transform: Transforms, actionId: ActionId, rev: number, headerId: BlockId, tailId: BlockId, priority = 0, blockDigests?: BlockContentDigests, baseRevs?: BlockBaseRevs): Promise<CommitResult> {
238
240
  const pendResult = await this.transactor.pend({
239
241
  transforms: transform, actionId, rev, policy: 'r',
240
242
  ...(priority > 0 ? { priority } : {}),
241
- ...baseRevsField(baseRevs)
243
+ ...baseRevsField(baseRevs),
244
+ // One collection, pended whole and then committed: the shape `PendRequest.soleReservation` names.
245
+ soleReservation: true
242
246
  });
243
247
  if (!pendResult.success) {
244
248
  return pendResult;
@@ -259,8 +263,9 @@ export class TransactorSource<TBlock extends IBlock> implements BlockSource<TBlo
259
263
  // rebase — letting the cancel's own failure throw over it would turn a routine,
260
264
  // recoverable race into a hard failure. So the cancel fault is logged, not raised.
261
265
  await this.dischargePend(actionId, pendResult.blockIds);
266
+ return commitResult;
262
267
  }
263
- return commitResult;
268
+ return { ...commitResult, durability: withBlockRevs(commitResult.durability, pendResult.blockIds, rev) };
264
269
  } catch (e) {
265
270
  // `e` is the real cause — a transport fault, the thing the caller needs to see. A cancel
266
271
  // that also fails must not silently take its place, but it must not be lost either: the
@@ -0,0 +1,15 @@
1
+ /**
2
+ * Keep a pending timer from holding a Node process open. Browsers and React Native return a number
3
+ * from `setTimeout` / `setInterval`, which has no `unref`, so the call is guarded. Every first-party
4
+ * `unref` goes through here; the `NO_TIMER_UNREF` lint rule in `eslint.config.js` enforces that.
5
+ */
6
+ export function unrefTimer<T>(handle: T): T {
7
+ (handle as { unref?: () => void }).unref?.();
8
+ return handle;
9
+ }
10
+
11
+ /** Arm a one-shot timer that never holds a process open; returns its cancel. The default behind the injectable `setTimer` seams. */
12
+ export function armUnrefTimer(fn: () => void, ms: number): () => void {
13
+ const handle = unrefTimer(setTimeout(fn, ms));
14
+ return (): void => clearTimeout(handle);
15
+ }