@optimystic/db-core 1.13.0 → 1.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/dist/src/cluster/structs.d.ts +12 -0
  2. package/dist/src/cluster/structs.d.ts.map +1 -1
  3. package/dist/src/index.d.ts +1 -0
  4. package/dist/src/index.d.ts.map +1 -1
  5. package/dist/src/index.js +1 -0
  6. package/dist/src/index.js.map +1 -1
  7. package/dist/src/network/struct.d.ts +20 -0
  8. package/dist/src/network/struct.d.ts.map +1 -1
  9. package/dist/src/network/struct.js.map +1 -1
  10. package/dist/src/transaction/coordinator.d.ts.map +1 -1
  11. package/dist/src/transaction/coordinator.js +11 -3
  12. package/dist/src/transaction/coordinator.js.map +1 -1
  13. package/dist/src/transactor/network-transactor.d.ts +48 -5
  14. package/dist/src/transactor/network-transactor.d.ts.map +1 -1
  15. package/dist/src/transactor/network-transactor.js +98 -22
  16. package/dist/src/transactor/network-transactor.js.map +1 -1
  17. package/dist/src/transactor/owed-cancels.d.ts +96 -0
  18. package/dist/src/transactor/owed-cancels.d.ts.map +1 -0
  19. package/dist/src/transactor/owed-cancels.js +211 -0
  20. package/dist/src/transactor/owed-cancels.js.map +1 -0
  21. package/dist/src/transactor/transactor-source.d.ts.map +1 -1
  22. package/dist/src/transactor/transactor-source.js +3 -1
  23. package/dist/src/transactor/transactor-source.js.map +1 -1
  24. package/dist/src/utility/unref-timer.d.ts +9 -0
  25. package/dist/src/utility/unref-timer.d.ts.map +1 -0
  26. package/dist/src/utility/unref-timer.js +15 -0
  27. package/dist/src/utility/unref-timer.js.map +1 -0
  28. package/package.json +1 -1
  29. package/src/cluster/structs.ts +12 -0
  30. package/src/index.ts +1 -0
  31. package/src/network/struct.ts +20 -0
  32. package/src/transaction/coordinator.ts +11 -3
  33. package/src/transactor/network-transactor.ts +120 -29
  34. package/src/transactor/owed-cancels.ts +252 -0
  35. package/src/transactor/transactor-source.ts +3 -1
  36. package/src/utility/unref-timer.ts +15 -0
@@ -17,6 +17,7 @@ import { routingKeyForBlock } from "../network/routing-key.js";
17
17
  import { isRecordEmpty } from "../utility/is-record-empty.js";
18
18
  import { type CoordinatorBatch, makeBatchesByPeer, incompleteBatches, everyBatch, allBatches, mergeBlocks, processBatches, createBatchesForPayload } from "../utility/batch-coordinator.js";
19
19
  import { abortableDelay, jitteredBackoffMs } from "../utility/backoff.js";
20
+ import { OwedCancels, type OwedCancelClock } from "./owed-cancels.js";
20
21
  import { createLogger, verbose } from "../logger.js";
21
22
 
22
23
  const log = createLogger('network-transactor');
@@ -54,6 +55,12 @@ type NetworkTransactorInit = {
54
55
  * which is the nearest.
55
56
  */
56
57
  localPeerId?: PeerId;
58
+ /**
59
+ * Time for the re-sends of cancels this transactor could not discharge ({@link OwedCancels}).
60
+ * Omit in production (`Date.now` and an unref'd `setTimeout`); a test injects a fake to fire the
61
+ * re-send without waiting on the wall clock.
62
+ */
63
+ owedCancelClock?: OwedCancelClock;
57
64
  }
58
65
 
59
66
  /**
@@ -95,6 +102,12 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
95
102
  private readonly txnCoordinatorCache = new Map<ActionId, { coordinators: Map<BlockId, PeerId>; expires: number }>();
96
103
  private static readonly MAX_TXN_COORDINATOR_CACHE_ENTRIES = 1000;
97
104
 
105
+ /**
106
+ * Cancels {@link dischargeCancel} gave up on, kept and re-sent before this transactor's next pend
107
+ * of their blocks and on a backoff timer, for this transactor's life — see {@link OwedCancels}.
108
+ */
109
+ private readonly owedCancels: OwedCancels;
110
+
98
111
  constructor(
99
112
  init: NetworkTransactorInit,
100
113
  ) {
@@ -109,6 +122,28 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
109
122
  this.getRepo = init.getRepo;
110
123
  this.localChangeNotifier = init.localChangeNotifier;
111
124
  this.localPeerId = init.localPeerId?.toString();
125
+ this.owedCancels = new OwedCancels({
126
+ attempt: actionRef => this.resendOwedCancel(actionRef),
127
+ clock: init.owedCancelClock
128
+ });
129
+ }
130
+
131
+ /**
132
+ * Every action this transactor owes a cancel for — one it sent, could not get answered, and is
133
+ * still re-sending — with the blocks still owed. A snapshot, for diagnostics: an embedder about to
134
+ * drop this transactor can see what it would leave standing.
135
+ */
136
+ owedCancelBlocks(): Map<ActionId, BlockId[]> {
137
+ return this.owedCancels.snapshot();
138
+ }
139
+
140
+ /**
141
+ * Stops re-sending owed cancels; whatever is still owed is logged (`cancel:owed-abandoned`) and
142
+ * left standing. Call when the node this transactor sends through stops, or its re-send timer keeps
143
+ * the transactor alive, retrying against a node that can no longer answer, for the life of the process.
144
+ */
145
+ stop(): void {
146
+ this.owedCancels.stop();
112
147
  }
113
148
 
114
149
  /**
@@ -648,6 +683,9 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
648
683
  : transformsFromTransform(filteredTransform, blockId);
649
684
  };
650
685
  const blockIds = blockIdsForTransforms(blockAction.transforms);
686
+ // A cancel this transactor still owes on any of these blocks goes out first: its records would
687
+ // otherwise refuse this pend — or, for this same action id, the late cancel would delete it.
688
+ await this.owedCancels.beforePend(blockAction.actionId, blockIds);
651
689
  const batches = await this.consolidateCoordinators(blockIds, blockAction.transforms, transformForBlock);
652
690
  log('pend actionId=%s blockIds=%d batches=%d', blockAction.actionId, blockIds.length, batches.length);
653
691
  if (verbose) {
@@ -1015,8 +1053,9 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
1015
1053
  * Confirmed blocks are excluded only to skip a pointless consensus round — cancelling one would
1016
1054
  * also be a no-op — so an over-broad `confirmed` set costs latency, never correctness.
1017
1055
  *
1018
- * Residuals: the cancel is retried and checked ({@link dischargeCancel}) but still bounded, so a
1019
- * fault outlasting its budget leaves the record wedged until a node-side backstop sweep exists (backlog:
1056
+ * Residuals: the cancel is retried and checked ({@link dischargeCancel}), and one that outlasts its
1057
+ * budget is kept and re-sent for this transactor's life ({@link owedCancels}), so only a writer that
1058
+ * exits first leaves the record wedged until a node-side backstop sweep exists (backlog:
1020
1059
  * `debt-unpromotable-pending-records-need-a-sweep`); and this covers only the sweep's abandonment
1021
1060
  * — `StorageRepo.commit`'s genuine-fault arm deliberately KEEPS a failed batch's pendings for a
1022
1061
  * retry, so it is a second producer of the same durable state whenever that retry never comes.
@@ -1046,7 +1085,7 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
1046
1085
  try {
1047
1086
  await this.cancel({ actionId, blockIds: abandoned });
1048
1087
  } catch (cancelError) {
1049
- try { log('WARN: cancel of abandoned sweep blocks failed — pending records may wedge until the node-side sweep lands: %o', cancelError); } catch { /* ignore */ }
1088
+ try { log('WARN: cancel of abandoned sweep blocks failed — kept and re-sent until it lands (cancel:owed): %o', cancelError); } catch { /* ignore */ }
1050
1089
  }
1051
1090
  return abandoned;
1052
1091
  }
@@ -1280,8 +1319,14 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
1280
1319
  * cost latency only. Each round narrows to the blocks still outstanding, so a partially
1281
1320
  * successful round does not re-issue the cancels that already landed.
1282
1321
  *
1283
- * NOTE: a fault that outlasts these bounds still strands the record, and nothing node-side
1284
- * reclaims it. That residual is the backstop tracked in backlog
1322
+ * **What happens when the bounds run out.** The blocks still outstanding are handed to
1323
+ * {@link owedCancels} before the throw, which re-sends their cancel before this transactor's next
1324
+ * pend of any of them and on a backoff timer, for the transactor's life — a fault that outlasts
1325
+ * this loop costs the records' holders some refused writes, not the block. Recording it here,
1326
+ * from this loop's own outstanding set, is what spares every caller from having to remember to.
1327
+ *
1328
+ * NOTE: a writer that exits (or stops this transactor, {@link stop}) while a cancel is still owed
1329
+ * strands the record, and nothing node-side reclaims it. That residual is the backstop tracked in backlog
1285
1330
  * `debt-unpromotable-pending-records-need-a-sweep`.
1286
1331
  *
1287
1332
  * @param seedBatches Batches to use for round 0 only (see {@link cancelBatch}). Omitted → round
@@ -1291,7 +1336,8 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
1291
1336
  actionRef: ActionBlocks,
1292
1337
  seedBatches?: CoordinatorBatch<BlockId[], void>[]
1293
1338
  ): Promise<void> {
1294
- let outstanding = Array.from(new Set(actionRef.blockIds));
1339
+ const requested = Array.from(new Set(actionRef.blockIds));
1340
+ let outstanding = requested;
1295
1341
  if (outstanding.length === 0) {
1296
1342
  return;
1297
1343
  }
@@ -1299,30 +1345,15 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
1299
1345
  let roundBatches: CoordinatorBatch<BlockId[], void>[] = [];
1300
1346
  let lastError: Error | undefined;
1301
1347
  for (let round = 0; ; ++round) {
1302
- roundBatches = [];
1303
- try {
1304
- // Batch construction can throw (coordinator lookup); `processBatches` cannot.
1305
- roundBatches = round === 0 && seedBatches
1306
- ? seedBatches
1307
- : await this.batchesForPayload<BlockId[], void>(outstanding, outstanding, mergeBlocks, []);
1308
- await processBatches(
1309
- roundBatches,
1310
- (batch) => this.getRepo(batch.peerId).cancel({ actionId: actionRef.actionId, blockIds: batch.payload }, { expiration: deadline, dialTimeoutMs: this.dialTimeoutMs }),
1311
- batch => batch.payload,
1312
- mergeBlocks,
1313
- deadline,
1314
- (blockId, options) => this.keyNetwork.findCoordinator(routingKeyForBlock(blockId), options)
1315
- );
1316
- } catch (e) {
1317
- lastError = asError(e);
1318
- }
1319
-
1320
- const discharged = dischargedBlocks(roundBatches);
1321
- outstanding = outstanding.filter(bid => !discharged.has(bid));
1348
+ const result = await this.cancelRound(actionRef.actionId, outstanding, deadline, round === 0 ? seedBatches : undefined);
1349
+ roundBatches = result.batches;
1350
+ lastError = result.error ?? lastError;
1351
+ outstanding = outstanding.filter(bid => !result.discharged.has(bid));
1322
1352
  if (outstanding.length === 0) {
1323
1353
  if (round > 0) {
1324
1354
  log('cancel:discharged actionId=%s rounds=%d', actionRef.actionId, round + 1);
1325
1355
  }
1356
+ this.owedCancels.discharged(actionRef.actionId, requested);
1326
1357
  return;
1327
1358
  }
1328
1359
 
@@ -1341,6 +1372,9 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
1341
1372
  return `${b.peerId.toString()}[block:${b.blockId}](${status})${errMsg}`;
1342
1373
  });
1343
1374
  const rootCause = firstBatchError(roundBatches) ?? lastError;
1375
+ const stillHeld = new Set(outstanding);
1376
+ this.owedCancels.discharged(actionRef.actionId, requested.filter(bid => !stillHeld.has(bid)));
1377
+ this.owedCancels.record(actionRef.actionId, outstanding, rootCause?.message);
1344
1378
  const aggregate = new Error(`Cancel of action ${actionRef.actionId} did not discharge ${outstanding.length} block(s): ${outstanding.join(', ')}`
1345
1379
  + (details ? `; peers: ${details}` : '')
1346
1380
  + (rootCause ? `; root: ${rootCause.message}` : ''));
@@ -1349,6 +1383,49 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
1349
1383
  throw aggregate;
1350
1384
  }
1351
1385
 
1386
+ /**
1387
+ * One cancel round for `blockIds`: batches built from `seedBatches` or resolved live, sent, and
1388
+ * judged by which blocks some peer answered for. Never throws — a failed lookup is returned as
1389
+ * `error` beside an empty discharged set.
1390
+ */
1391
+ private async cancelRound(
1392
+ actionId: ActionId,
1393
+ blockIds: BlockId[],
1394
+ deadline: number,
1395
+ seedBatches?: CoordinatorBatch<BlockId[], void>[]
1396
+ ): Promise<{ batches: CoordinatorBatch<BlockId[], void>[]; discharged: Set<BlockId>; error?: Error }> {
1397
+ let batches: CoordinatorBatch<BlockId[], void>[] = [];
1398
+ let error: Error | undefined;
1399
+ try {
1400
+ // Batch construction can throw (coordinator lookup); `processBatches` cannot.
1401
+ batches = seedBatches ?? await this.batchesForPayload<BlockId[], void>(blockIds, blockIds, mergeBlocks, []);
1402
+ await processBatches(
1403
+ batches,
1404
+ (batch) => this.getRepo(batch.peerId).cancel({ actionId, blockIds: batch.payload }, { expiration: deadline, dialTimeoutMs: this.dialTimeoutMs }),
1405
+ batch => batch.payload,
1406
+ mergeBlocks,
1407
+ deadline,
1408
+ (blockId, options) => this.keyNetwork.findCoordinator(routingKeyForBlock(blockId), options)
1409
+ );
1410
+ } catch (e) {
1411
+ error = asError(e);
1412
+ }
1413
+ return { batches, discharged: dischargedBlocks(batches), error };
1414
+ }
1415
+
1416
+ /**
1417
+ * One re-send of a cancel {@link dischargeCancel} gave up on, for {@link owedCancels}: a single
1418
+ * round within `abortOrCancelTimeoutMs`, never the full loop — it runs ahead of a pend, and a
1419
+ * cohort that still cannot approve the cancel cannot approve the pend either.
1420
+ */
1421
+ private async resendOwedCancel(actionRef: ActionBlocks): Promise<ReadonlySet<BlockId>> {
1422
+ const { discharged, error } = await this.cancelRound(actionRef.actionId, actionRef.blockIds, Date.now() + this.abortOrCancelTimeoutMs);
1423
+ if (discharged.size < actionRef.blockIds.length) {
1424
+ log('cancel:owed-retry-short actionId=%s answered=%d of %d%s', actionRef.actionId, discharged.size, actionRef.blockIds.length, error ? ` error=${error.message}` : '');
1425
+ }
1426
+ return discharged;
1427
+ }
1428
+
1352
1429
  /** Renders the batches worth naming in an aggregate error: the ones that never completed, or —
1353
1430
  * when every batch completed and the aggregate is about a non-success RESPONSE rather than a
1354
1431
  * transport fault — all of them. */
@@ -1481,10 +1558,24 @@ function baseRevsFor(all: BlockBaseRevs | undefined, batchBlockIds: BlockId[]):
1481
1558
  /** `action` re-scoped to one coordinator batch: the batch's own transforms, and the bases for the
1482
1559
  * blocks in it alone (see {@link PendRequest.baseRevs}). The action-wide `baseRevs` key is dropped
1483
1560
  * BEFORE the subset is spread, so a batch none of whose blocks names a base carries no key at all
1484
- * rather than the whole action's map — and no cohort signs for a block it is not responsible for. */
1561
+ * rather than the whole action's map — and no cohort signs for a block it is not responsible for.
1562
+ *
1563
+ * `soleReservation` survives only on a batch carrying every block of the action: a pend split across
1564
+ * coordinators — by `consolidateCoordinators`, or by `processBatches` re-batching a failed batch — is
1565
+ * several reservations, which the field must never claim to be. Decided per send, like the bases. */
1485
1566
  function pendRequestForBatch(action: PendRequest, payload: Transforms): PendRequest {
1486
- const { baseRevs, ...rest } = action;
1487
- return { ...rest, transforms: payload, ...baseRevsFor(baseRevs, blockIdsForTransforms(payload)) };
1567
+ const { baseRevs, soleReservation, ...rest } = action;
1568
+ const batchBlockIds = blockIdsForTransforms(payload);
1569
+ const whole = (): boolean => {
1570
+ const inBatch = new Set(batchBlockIds);
1571
+ return blockIdsForTransforms(action.transforms).every(blockId => inBatch.has(blockId));
1572
+ };
1573
+ return {
1574
+ ...rest,
1575
+ transforms: payload,
1576
+ ...baseRevsFor(baseRevs, batchBlockIds),
1577
+ ...(soleReservation === true && whole() ? { soleReservation } : {})
1578
+ };
1488
1579
  }
1489
1580
 
1490
1581
  /** The read one coordinator batch sends downstream: the batch's ids, the caller's context, and the
@@ -0,0 +1,252 @@
1
+ import type { ActionBlocks } from "../network/struct.js";
2
+ import type { ActionId } from "../collection/action.js";
3
+ import type { BlockId } from "../blocks/structs.js";
4
+ import { jitteredBackoffMs } from "../utility/backoff.js";
5
+ import { armUnrefTimer } from "../utility/unref-timer.js";
6
+ import { createLogger } from "../logger.js";
7
+
8
+ const log = createLogger('network-transactor');
9
+
10
+ export type OwedCancelTimerCancel = () => void;
11
+
12
+ /**
13
+ * Optional injection seam for deterministic time, the shape `ClusterCoordinatorClock` in
14
+ * `packages/db-p2p/src/repo/cluster-coordinator.ts` gives its retry timer. Production leaves both
15
+ * undefined and gets `Date.now` plus an unref'd `setTimeout`, so an owed cancel never keeps an
16
+ * otherwise-idle process alive.
17
+ */
18
+ export interface OwedCancelClock {
19
+ /** Clock (Unix ms). Defaults to `Date.now`. */
20
+ now?: () => number;
21
+ /** Schedule a one-shot timer, returning its cancel. Defaults to an unref'd `setTimeout`. */
22
+ setTimer?: (fn: () => void, delayMs: number) => OwedCancelTimerCancel;
23
+ }
24
+
25
+ export interface OwedCancelsInit {
26
+ /**
27
+ * One bounded cancel attempt for `actionRef`, resolving with the blocks some peer ANSWERED for —
28
+ * the same discharged set the transactor's own cancel loop judges by. A throw counts as nothing
29
+ * answered.
30
+ */
31
+ attempt: (actionRef: ActionBlocks) => Promise<ReadonlySet<BlockId>>;
32
+ clock?: OwedCancelClock;
33
+ /** Most actions remembered at once. Default {@link DEFAULT_MAX_OWED_ACTIONS}. */
34
+ maxActions?: number;
35
+ }
36
+
37
+ /**
38
+ * Bounds the memory a writer spends on cancels it cannot land. Evicting an entry re-opens the wedge
39
+ * that entry was kept to close, so it is logged; a writer that owes this many is cut off from its
40
+ * cohorts, and its records are the node-side backstop's (backlog
41
+ * `debt-unpromotable-pending-records-need-a-sweep`).
42
+ */
43
+ const DEFAULT_MAX_OWED_ACTIONS = 256;
44
+
45
+ /** The backoff between re-sends of one owed cancel: a second at first, doubling to half a minute. */
46
+ const RETRY_BACKOFF = { baseMs: 1_000, capMs: 30_000 };
47
+
48
+ interface OwedEntry {
49
+ readonly blockIds: Set<BlockId>;
50
+ /** When the cancel first fell short, for the discharge log. */
51
+ readonly since: number;
52
+ /** Re-sends made so far — the backoff's exponent. */
53
+ attempts: number;
54
+ nextAttemptAt: number;
55
+ }
56
+
57
+ /**
58
+ * The cancels a writer could not discharge, kept and re-sent until they land.
59
+ *
60
+ * A cancel whose rounds all fall short — every one of them is a cluster transaction, so the member
61
+ * whose silence sank the commit sinks the cancel too — leaves pending records standing that make the
62
+ * members refuse every later write to those blocks, from any writer. Nothing else removes them while
63
+ * their slot is unreached (docs/repository.md, "A pending record's lifetime is bounded by its
64
+ * writer"), so the writer keeps the cancel and sends it again:
65
+ *
66
+ * - **before its own next pend of any of those blocks** ({@link beforePend}), which is what clears
67
+ * the leftover in front of the writer's next write to the block;
68
+ * - **on a backoff timer** while anything is owed, so other machines' writes, which the records also
69
+ * refuse, are freed without this writer writing the block again.
70
+ *
71
+ * Re-sending is safe whenever it arrives: a cancel deletes only the cancelled action's pending records
72
+ * and never moves a revision, so it commutes with every other action. The one exception is a re-pend
73
+ * of the SAME action id, whose new record a late cancel would delete — so a pend of an owed action
74
+ * supersedes the owed cancel for every block it names, and waits out any re-send of it already in
75
+ * flight before it goes out.
76
+ */
77
+ export class OwedCancels {
78
+ private readonly entries = new Map<ActionId, OwedEntry>();
79
+ private readonly inFlight = new Map<ActionId, Promise<void>>();
80
+ private readonly attempt: OwedCancelsInit['attempt'];
81
+ private readonly now: () => number;
82
+ private readonly setTimer: (fn: () => void, delayMs: number) => OwedCancelTimerCancel;
83
+ private readonly maxActions: number;
84
+ private timer: { cancel: OwedCancelTimerCancel; dueAt: number } | undefined;
85
+ private stopped = false;
86
+
87
+ constructor(init: OwedCancelsInit) {
88
+ this.attempt = init.attempt;
89
+ this.now = init.clock?.now ?? ((): number => Date.now());
90
+ this.setTimer = init.clock?.setTimer ?? armUnrefTimer;
91
+ this.maxActions = init.maxActions ?? DEFAULT_MAX_OWED_ACTIONS;
92
+ }
93
+
94
+ /** Every owed action with the blocks it still owes a cancel for. A snapshot. */
95
+ snapshot(): Map<ActionId, BlockId[]> {
96
+ return new Map(Array.from(this.entries, ([actionId, entry]) => [actionId, Array.from(entry.blockIds)]));
97
+ }
98
+
99
+ /** A cancel for `actionId` ran out of budget with `blockIds` still unanswered. */
100
+ record(actionId: ActionId, blockIds: readonly BlockId[], why: string | undefined): void {
101
+ if (blockIds.length === 0) return;
102
+ if (this.stopped) {
103
+ log('cancel:owed-abandoned actionId=%s blocks=%o why=stopped', actionId, blockIds);
104
+ return;
105
+ }
106
+ const existing = this.entries.get(actionId);
107
+ if (existing) {
108
+ for (const bid of blockIds) existing.blockIds.add(bid);
109
+ existing.nextAttemptAt = this.now() + this.backoffMs(existing.attempts);
110
+ } else {
111
+ const now = this.now();
112
+ this.entries.set(actionId, { blockIds: new Set(blockIds), since: now, attempts: 0, nextAttemptAt: now + this.backoffMs(0) });
113
+ this.evictOverflow();
114
+ }
115
+ log('cancel:owed actionId=%s blocks=%o why=%s', actionId, blockIds, why ?? 'unknown');
116
+ this.arm();
117
+ }
118
+
119
+ /** A cancel for `actionId` was answered for `blockIds` — whoever sent it, they are owed no longer. */
120
+ discharged(actionId: ActionId, blockIds: Iterable<BlockId>): void {
121
+ const settled = this.remove(actionId, blockIds);
122
+ if (settled) {
123
+ log('cancel:owed-discharged actionId=%s attempts=%d stoodMs=%d', actionId, settled.attempts, this.now() - settled.since);
124
+ }
125
+ }
126
+
127
+ /**
128
+ * Re-sends nothing from here on: the timer is disarmed and every cancel still owed is logged
129
+ * (`cancel:owed-abandoned`) and forgotten, its records left to the node-side backstop. For when the
130
+ * node the re-sends go through stops — they could never land, and the timer would otherwise keep
131
+ * the transactor, and everything it reaches, alive for the life of the process.
132
+ */
133
+ stop(): void {
134
+ this.stopped = true;
135
+ this.timer?.cancel();
136
+ this.timer = undefined;
137
+ for (const [actionId, entry] of this.entries) {
138
+ log('cancel:owed-abandoned actionId=%s blocks=%o stoodMs=%d', actionId, Array.from(entry.blockIds), this.now() - entry.since);
139
+ }
140
+ this.entries.clear();
141
+ }
142
+
143
+ /**
144
+ * Runs ahead of every pend of `actionId` over `blockIds`: supersedes this action's own owed cancel
145
+ * for those blocks, then makes one attempt at every OTHER owed cancel that names any of them. A
146
+ * failed attempt stays owed and the pend proceeds; a cohort still unable to approve the cancel
147
+ * cannot approve the pend either, so one attempt — not a loop — is what the pend should wait for.
148
+ */
149
+ async beforePend(actionId: ActionId, blockIds: readonly BlockId[]): Promise<void> {
150
+ if (this.entries.size === 0 && this.inFlight.size === 0) return;
151
+ await this.supersede(actionId, blockIds);
152
+ const named = new Set(blockIds);
153
+ const overlapping = Array.from(this.entries)
154
+ .filter(([owedId, entry]) => owedId !== actionId && Array.from(entry.blockIds).some(bid => named.has(bid)))
155
+ .map(([owedId]) => owedId);
156
+ await Promise.all(overlapping.map(owedId => this.attemptOnce(owedId)));
157
+ }
158
+
159
+ /**
160
+ * A pend of `actionId` writes a fresh record on every block it names, which a late cancel for the
161
+ * same action id would delete. So those blocks leave the owed entry first — synchronously, so no
162
+ * re-send started from here on can carry them — and a re-send already in flight is waited out, so
163
+ * it lands before the pend rather than after it.
164
+ */
165
+ private async supersede(actionId: ActionId, blockIds: readonly BlockId[]): Promise<void> {
166
+ const entry = this.entries.get(actionId);
167
+ const superseded = entry ? blockIds.filter(bid => entry.blockIds.has(bid)) : [];
168
+ if (superseded.length > 0) {
169
+ log('cancel:owed-superseded actionId=%s blocks=%o', actionId, superseded);
170
+ this.remove(actionId, superseded);
171
+ }
172
+ await this.inFlight.get(actionId);
173
+ }
174
+
175
+ /** Takes `blockIds` off `actionId`'s entry, returning the entry when that emptied — and so removed — it. */
176
+ private remove(actionId: ActionId, blockIds: Iterable<BlockId>): OwedEntry | undefined {
177
+ const entry = this.entries.get(actionId);
178
+ if (!entry) return undefined;
179
+ for (const bid of blockIds) entry.blockIds.delete(bid);
180
+ if (entry.blockIds.size > 0) return undefined;
181
+ this.entries.delete(actionId);
182
+ this.arm();
183
+ return entry;
184
+ }
185
+
186
+ /** One re-send of `actionId`'s owed cancel, joining the one already in flight if there is one. */
187
+ private attemptOnce(actionId: ActionId): Promise<void> {
188
+ const running = this.inFlight.get(actionId);
189
+ if (running) return running;
190
+ const entry = this.entries.get(actionId);
191
+ if (!entry) return Promise.resolve();
192
+ ++entry.attempts;
193
+ const run = this.resend(actionId, Array.from(entry.blockIds))
194
+ .finally(() => {
195
+ if (this.inFlight.get(actionId) === run) this.inFlight.delete(actionId);
196
+ this.arm();
197
+ });
198
+ this.inFlight.set(actionId, run);
199
+ return run;
200
+ }
201
+
202
+ private async resend(actionId: ActionId, blockIds: BlockId[]): Promise<void> {
203
+ let answered: ReadonlySet<BlockId>;
204
+ try {
205
+ answered = await this.attempt({ actionId, blockIds });
206
+ } catch (error) {
207
+ log('cancel:owed-retry-failed actionId=%s blocks=%d error=%s', actionId, blockIds.length, (error as Error)?.message ?? String(error));
208
+ answered = new Set();
209
+ }
210
+ this.discharged(actionId, answered);
211
+ const entry = this.entries.get(actionId);
212
+ if (entry) {
213
+ entry.nextAttemptAt = this.now() + this.backoffMs(entry.attempts);
214
+ }
215
+ }
216
+
217
+ /** Arms the one timer at the earliest re-send due among the entries not already being re-sent, or disarms it. */
218
+ private arm(): void {
219
+ let dueAt: number | undefined;
220
+ for (const [actionId, entry] of this.entries) {
221
+ if (this.inFlight.has(actionId)) continue;
222
+ if (dueAt === undefined || entry.nextAttemptAt < dueAt) dueAt = entry.nextAttemptAt;
223
+ }
224
+ if (this.timer && (dueAt === undefined || this.timer.dueAt > dueAt)) {
225
+ this.timer.cancel();
226
+ this.timer = undefined;
227
+ }
228
+ if (dueAt === undefined || this.timer) return;
229
+ this.timer = { cancel: this.setTimer(() => this.fire(), Math.max(0, dueAt - this.now())), dueAt };
230
+ }
231
+
232
+ private fire(): void {
233
+ this.timer = undefined;
234
+ const now = this.now();
235
+ for (const [actionId, entry] of this.entries) {
236
+ if (entry.nextAttemptAt <= now) void this.attemptOnce(actionId);
237
+ }
238
+ this.arm();
239
+ }
240
+
241
+ private backoffMs(attempts: number): number {
242
+ return jitteredBackoffMs(attempts, RETRY_BACKOFF);
243
+ }
244
+
245
+ private evictOverflow(): void {
246
+ while (this.entries.size > this.maxActions) {
247
+ const [oldestId, oldest] = this.entries.entries().next().value as [ActionId, OwedEntry];
248
+ this.entries.delete(oldestId);
249
+ log('cancel:owed-evicted actionId=%s blocks=%o stoodMs=%d', oldestId, Array.from(oldest.blockIds), this.now() - oldest.since);
250
+ }
251
+ }
252
+ }
@@ -240,7 +240,9 @@ export class TransactorSource<TBlock extends IBlock> implements BlockSource<TBlo
240
240
  const pendResult = await this.transactor.pend({
241
241
  transforms: transform, actionId, rev, policy: 'r',
242
242
  ...(priority > 0 ? { priority } : {}),
243
- ...baseRevsField(baseRevs)
243
+ ...baseRevsField(baseRevs),
244
+ // One collection, pended whole and then committed: the shape `PendRequest.soleReservation` names.
245
+ soleReservation: true
244
246
  });
245
247
  if (!pendResult.success) {
246
248
  return pendResult;
@@ -0,0 +1,15 @@
1
+ /**
2
+ * Keep a pending timer from holding a Node process open. Browsers and React Native return a number
3
+ * from `setTimeout` / `setInterval`, which has no `unref`, so the call is guarded. Every first-party
4
+ * `unref` goes through here; the `NO_TIMER_UNREF` lint rule in `eslint.config.js` enforces that.
5
+ */
6
+ export function unrefTimer<T>(handle: T): T {
7
+ (handle as { unref?: () => void }).unref?.();
8
+ return handle;
9
+ }
10
+
11
+ /** Arm a one-shot timer that never holds a process open; returns its cancel. The default behind the injectable `setTimer` seams. */
12
+ export function armUnrefTimer(fn: () => void, ms: number): () => void {
13
+ const handle = unrefTimer(setTimeout(fn, ms));
14
+ return (): void => clearTimeout(handle);
15
+ }