@optimystic/db-core 1.13.0 → 1.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/structs.d.ts +12 -0
- package/dist/src/cluster/structs.d.ts.map +1 -1
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +1 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/network/struct.d.ts +20 -0
- package/dist/src/network/struct.d.ts.map +1 -1
- package/dist/src/network/struct.js.map +1 -1
- package/dist/src/transaction/coordinator.d.ts.map +1 -1
- package/dist/src/transaction/coordinator.js +11 -3
- package/dist/src/transaction/coordinator.js.map +1 -1
- package/dist/src/transactor/network-transactor.d.ts +48 -5
- package/dist/src/transactor/network-transactor.d.ts.map +1 -1
- package/dist/src/transactor/network-transactor.js +98 -22
- package/dist/src/transactor/network-transactor.js.map +1 -1
- package/dist/src/transactor/owed-cancels.d.ts +96 -0
- package/dist/src/transactor/owed-cancels.d.ts.map +1 -0
- package/dist/src/transactor/owed-cancels.js +211 -0
- package/dist/src/transactor/owed-cancels.js.map +1 -0
- package/dist/src/transactor/transactor-source.d.ts.map +1 -1
- package/dist/src/transactor/transactor-source.js +3 -1
- package/dist/src/transactor/transactor-source.js.map +1 -1
- package/dist/src/utility/unref-timer.d.ts +9 -0
- package/dist/src/utility/unref-timer.d.ts.map +1 -0
- package/dist/src/utility/unref-timer.js +15 -0
- package/dist/src/utility/unref-timer.js.map +1 -0
- package/package.json +1 -1
- package/src/cluster/structs.ts +12 -0
- package/src/index.ts +1 -0
- package/src/network/struct.ts +20 -0
- package/src/transaction/coordinator.ts +11 -3
- package/src/transactor/network-transactor.ts +120 -29
- package/src/transactor/owed-cancels.ts +252 -0
- package/src/transactor/transactor-source.ts +3 -1
- package/src/utility/unref-timer.ts +15 -0
|
@@ -17,6 +17,7 @@ import { routingKeyForBlock } from "../network/routing-key.js";
|
|
|
17
17
|
import { isRecordEmpty } from "../utility/is-record-empty.js";
|
|
18
18
|
import { type CoordinatorBatch, makeBatchesByPeer, incompleteBatches, everyBatch, allBatches, mergeBlocks, processBatches, createBatchesForPayload } from "../utility/batch-coordinator.js";
|
|
19
19
|
import { abortableDelay, jitteredBackoffMs } from "../utility/backoff.js";
|
|
20
|
+
import { OwedCancels, type OwedCancelClock } from "./owed-cancels.js";
|
|
20
21
|
import { createLogger, verbose } from "../logger.js";
|
|
21
22
|
|
|
22
23
|
const log = createLogger('network-transactor');
|
|
@@ -54,6 +55,12 @@ type NetworkTransactorInit = {
|
|
|
54
55
|
* which is the nearest.
|
|
55
56
|
*/
|
|
56
57
|
localPeerId?: PeerId;
|
|
58
|
+
/**
|
|
59
|
+
* Time for the re-sends of cancels this transactor could not discharge ({@link OwedCancels}).
|
|
60
|
+
* Omit in production (`Date.now` and an unref'd `setTimeout`); a test injects a fake to fire the
|
|
61
|
+
* re-send without waiting on the wall clock.
|
|
62
|
+
*/
|
|
63
|
+
owedCancelClock?: OwedCancelClock;
|
|
57
64
|
}
|
|
58
65
|
|
|
59
66
|
/**
|
|
@@ -95,6 +102,12 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
95
102
|
private readonly txnCoordinatorCache = new Map<ActionId, { coordinators: Map<BlockId, PeerId>; expires: number }>();
|
|
96
103
|
private static readonly MAX_TXN_COORDINATOR_CACHE_ENTRIES = 1000;
|
|
97
104
|
|
|
105
|
+
/**
|
|
106
|
+
* Cancels {@link dischargeCancel} gave up on, kept and re-sent before this transactor's next pend
|
|
107
|
+
* of their blocks and on a backoff timer, for this transactor's life — see {@link OwedCancels}.
|
|
108
|
+
*/
|
|
109
|
+
private readonly owedCancels: OwedCancels;
|
|
110
|
+
|
|
98
111
|
constructor(
|
|
99
112
|
init: NetworkTransactorInit,
|
|
100
113
|
) {
|
|
@@ -109,6 +122,28 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
109
122
|
this.getRepo = init.getRepo;
|
|
110
123
|
this.localChangeNotifier = init.localChangeNotifier;
|
|
111
124
|
this.localPeerId = init.localPeerId?.toString();
|
|
125
|
+
this.owedCancels = new OwedCancels({
|
|
126
|
+
attempt: actionRef => this.resendOwedCancel(actionRef),
|
|
127
|
+
clock: init.owedCancelClock
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Every action this transactor owes a cancel for — one it sent, could not get answered, and is
|
|
133
|
+
* still re-sending — with the blocks still owed. A snapshot, for diagnostics: an embedder about to
|
|
134
|
+
* drop this transactor can see what it would leave standing.
|
|
135
|
+
*/
|
|
136
|
+
owedCancelBlocks(): Map<ActionId, BlockId[]> {
|
|
137
|
+
return this.owedCancels.snapshot();
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Stops re-sending owed cancels; whatever is still owed is logged (`cancel:owed-abandoned`) and
|
|
142
|
+
* left standing. Call when the node this transactor sends through stops, or its re-send timer keeps
|
|
143
|
+
* the transactor alive, retrying against a node that can no longer answer, for the life of the process.
|
|
144
|
+
*/
|
|
145
|
+
stop(): void {
|
|
146
|
+
this.owedCancels.stop();
|
|
112
147
|
}
|
|
113
148
|
|
|
114
149
|
/**
|
|
@@ -648,6 +683,9 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
648
683
|
: transformsFromTransform(filteredTransform, blockId);
|
|
649
684
|
};
|
|
650
685
|
const blockIds = blockIdsForTransforms(blockAction.transforms);
|
|
686
|
+
// A cancel this transactor still owes on any of these blocks goes out first: its records would
|
|
687
|
+
// otherwise refuse this pend — or, for this same action id, the late cancel would delete it.
|
|
688
|
+
await this.owedCancels.beforePend(blockAction.actionId, blockIds);
|
|
651
689
|
const batches = await this.consolidateCoordinators(blockIds, blockAction.transforms, transformForBlock);
|
|
652
690
|
log('pend actionId=%s blockIds=%d batches=%d', blockAction.actionId, blockIds.length, batches.length);
|
|
653
691
|
if (verbose) {
|
|
@@ -1015,8 +1053,9 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
1015
1053
|
* Confirmed blocks are excluded only to skip a pointless consensus round — cancelling one would
|
|
1016
1054
|
* also be a no-op — so an over-broad `confirmed` set costs latency, never correctness.
|
|
1017
1055
|
*
|
|
1018
|
-
* Residuals: the cancel is retried and checked ({@link dischargeCancel})
|
|
1019
|
-
*
|
|
1056
|
+
* Residuals: the cancel is retried and checked ({@link dischargeCancel}), and one that outlasts its
|
|
1057
|
+
* budget is kept and re-sent for this transactor's life ({@link owedCancels}), so only a writer that
|
|
1058
|
+
* exits first leaves the record wedged until a node-side backstop sweep exists (backlog:
|
|
1020
1059
|
* `debt-unpromotable-pending-records-need-a-sweep`); and this covers only the sweep's abandonment
|
|
1021
1060
|
* — `StorageRepo.commit`'s genuine-fault arm deliberately KEEPS a failed batch's pendings for a
|
|
1022
1061
|
* retry, so it is a second producer of the same durable state whenever that retry never comes.
|
|
@@ -1046,7 +1085,7 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
1046
1085
|
try {
|
|
1047
1086
|
await this.cancel({ actionId, blockIds: abandoned });
|
|
1048
1087
|
} catch (cancelError) {
|
|
1049
|
-
try { log('WARN: cancel of abandoned sweep blocks failed —
|
|
1088
|
+
try { log('WARN: cancel of abandoned sweep blocks failed — kept and re-sent until it lands (cancel:owed): %o', cancelError); } catch { /* ignore */ }
|
|
1050
1089
|
}
|
|
1051
1090
|
return abandoned;
|
|
1052
1091
|
}
|
|
@@ -1280,8 +1319,14 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
1280
1319
|
* cost latency only. Each round narrows to the blocks still outstanding, so a partially
|
|
1281
1320
|
* successful round does not re-issue the cancels that already landed.
|
|
1282
1321
|
*
|
|
1283
|
-
*
|
|
1284
|
-
*
|
|
1322
|
+
* **What happens when the bounds run out.** The blocks still outstanding are handed to
|
|
1323
|
+
* {@link owedCancels} before the throw, which re-sends their cancel before this transactor's next
|
|
1324
|
+
* pend of any of them and on a backoff timer, for the transactor's life — a fault that outlasts
|
|
1325
|
+
* this loop costs the records' holders some refused writes, not the block. Recording it here,
|
|
1326
|
+
* from this loop's own outstanding set, is what spares every caller from having to remember to.
|
|
1327
|
+
*
|
|
1328
|
+
* NOTE: a writer that exits (or stops this transactor, {@link stop}) while a cancel is still owed
|
|
1329
|
+
* strands the record, and nothing node-side reclaims it. That residual is the backstop tracked in backlog
|
|
1285
1330
|
* `debt-unpromotable-pending-records-need-a-sweep`.
|
|
1286
1331
|
*
|
|
1287
1332
|
* @param seedBatches Batches to use for round 0 only (see {@link cancelBatch}). Omitted → round
|
|
@@ -1291,7 +1336,8 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
1291
1336
|
actionRef: ActionBlocks,
|
|
1292
1337
|
seedBatches?: CoordinatorBatch<BlockId[], void>[]
|
|
1293
1338
|
): Promise<void> {
|
|
1294
|
-
|
|
1339
|
+
const requested = Array.from(new Set(actionRef.blockIds));
|
|
1340
|
+
let outstanding = requested;
|
|
1295
1341
|
if (outstanding.length === 0) {
|
|
1296
1342
|
return;
|
|
1297
1343
|
}
|
|
@@ -1299,30 +1345,15 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
1299
1345
|
let roundBatches: CoordinatorBatch<BlockId[], void>[] = [];
|
|
1300
1346
|
let lastError: Error | undefined;
|
|
1301
1347
|
for (let round = 0; ; ++round) {
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
? seedBatches
|
|
1307
|
-
: await this.batchesForPayload<BlockId[], void>(outstanding, outstanding, mergeBlocks, []);
|
|
1308
|
-
await processBatches(
|
|
1309
|
-
roundBatches,
|
|
1310
|
-
(batch) => this.getRepo(batch.peerId).cancel({ actionId: actionRef.actionId, blockIds: batch.payload }, { expiration: deadline, dialTimeoutMs: this.dialTimeoutMs }),
|
|
1311
|
-
batch => batch.payload,
|
|
1312
|
-
mergeBlocks,
|
|
1313
|
-
deadline,
|
|
1314
|
-
(blockId, options) => this.keyNetwork.findCoordinator(routingKeyForBlock(blockId), options)
|
|
1315
|
-
);
|
|
1316
|
-
} catch (e) {
|
|
1317
|
-
lastError = asError(e);
|
|
1318
|
-
}
|
|
1319
|
-
|
|
1320
|
-
const discharged = dischargedBlocks(roundBatches);
|
|
1321
|
-
outstanding = outstanding.filter(bid => !discharged.has(bid));
|
|
1348
|
+
const result = await this.cancelRound(actionRef.actionId, outstanding, deadline, round === 0 ? seedBatches : undefined);
|
|
1349
|
+
roundBatches = result.batches;
|
|
1350
|
+
lastError = result.error ?? lastError;
|
|
1351
|
+
outstanding = outstanding.filter(bid => !result.discharged.has(bid));
|
|
1322
1352
|
if (outstanding.length === 0) {
|
|
1323
1353
|
if (round > 0) {
|
|
1324
1354
|
log('cancel:discharged actionId=%s rounds=%d', actionRef.actionId, round + 1);
|
|
1325
1355
|
}
|
|
1356
|
+
this.owedCancels.discharged(actionRef.actionId, requested);
|
|
1326
1357
|
return;
|
|
1327
1358
|
}
|
|
1328
1359
|
|
|
@@ -1341,6 +1372,9 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
1341
1372
|
return `${b.peerId.toString()}[block:${b.blockId}](${status})${errMsg}`;
|
|
1342
1373
|
});
|
|
1343
1374
|
const rootCause = firstBatchError(roundBatches) ?? lastError;
|
|
1375
|
+
const stillHeld = new Set(outstanding);
|
|
1376
|
+
this.owedCancels.discharged(actionRef.actionId, requested.filter(bid => !stillHeld.has(bid)));
|
|
1377
|
+
this.owedCancels.record(actionRef.actionId, outstanding, rootCause?.message);
|
|
1344
1378
|
const aggregate = new Error(`Cancel of action ${actionRef.actionId} did not discharge ${outstanding.length} block(s): ${outstanding.join(', ')}`
|
|
1345
1379
|
+ (details ? `; peers: ${details}` : '')
|
|
1346
1380
|
+ (rootCause ? `; root: ${rootCause.message}` : ''));
|
|
@@ -1349,6 +1383,49 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
1349
1383
|
throw aggregate;
|
|
1350
1384
|
}
|
|
1351
1385
|
|
|
1386
|
+
/**
|
|
1387
|
+
* One cancel round for `blockIds`: batches built from `seedBatches` or resolved live, sent, and
|
|
1388
|
+
* judged by which blocks some peer answered for. Never throws — a failed lookup is returned as
|
|
1389
|
+
* `error` beside an empty discharged set.
|
|
1390
|
+
*/
|
|
1391
|
+
private async cancelRound(
|
|
1392
|
+
actionId: ActionId,
|
|
1393
|
+
blockIds: BlockId[],
|
|
1394
|
+
deadline: number,
|
|
1395
|
+
seedBatches?: CoordinatorBatch<BlockId[], void>[]
|
|
1396
|
+
): Promise<{ batches: CoordinatorBatch<BlockId[], void>[]; discharged: Set<BlockId>; error?: Error }> {
|
|
1397
|
+
let batches: CoordinatorBatch<BlockId[], void>[] = [];
|
|
1398
|
+
let error: Error | undefined;
|
|
1399
|
+
try {
|
|
1400
|
+
// Batch construction can throw (coordinator lookup); `processBatches` cannot.
|
|
1401
|
+
batches = seedBatches ?? await this.batchesForPayload<BlockId[], void>(blockIds, blockIds, mergeBlocks, []);
|
|
1402
|
+
await processBatches(
|
|
1403
|
+
batches,
|
|
1404
|
+
(batch) => this.getRepo(batch.peerId).cancel({ actionId, blockIds: batch.payload }, { expiration: deadline, dialTimeoutMs: this.dialTimeoutMs }),
|
|
1405
|
+
batch => batch.payload,
|
|
1406
|
+
mergeBlocks,
|
|
1407
|
+
deadline,
|
|
1408
|
+
(blockId, options) => this.keyNetwork.findCoordinator(routingKeyForBlock(blockId), options)
|
|
1409
|
+
);
|
|
1410
|
+
} catch (e) {
|
|
1411
|
+
error = asError(e);
|
|
1412
|
+
}
|
|
1413
|
+
return { batches, discharged: dischargedBlocks(batches), error };
|
|
1414
|
+
}
|
|
1415
|
+
|
|
1416
|
+
/**
|
|
1417
|
+
* One re-send of a cancel {@link dischargeCancel} gave up on, for {@link owedCancels}: a single
|
|
1418
|
+
* round within `abortOrCancelTimeoutMs`, never the full loop — it runs ahead of a pend, and a
|
|
1419
|
+
* cohort that still cannot approve the cancel cannot approve the pend either.
|
|
1420
|
+
*/
|
|
1421
|
+
private async resendOwedCancel(actionRef: ActionBlocks): Promise<ReadonlySet<BlockId>> {
|
|
1422
|
+
const { discharged, error } = await this.cancelRound(actionRef.actionId, actionRef.blockIds, Date.now() + this.abortOrCancelTimeoutMs);
|
|
1423
|
+
if (discharged.size < actionRef.blockIds.length) {
|
|
1424
|
+
log('cancel:owed-retry-short actionId=%s answered=%d of %d%s', actionRef.actionId, discharged.size, actionRef.blockIds.length, error ? ` error=${error.message}` : '');
|
|
1425
|
+
}
|
|
1426
|
+
return discharged;
|
|
1427
|
+
}
|
|
1428
|
+
|
|
1352
1429
|
/** Renders the batches worth naming in an aggregate error: the ones that never completed, or —
|
|
1353
1430
|
* when every batch completed and the aggregate is about a non-success RESPONSE rather than a
|
|
1354
1431
|
* transport fault — all of them. */
|
|
@@ -1481,10 +1558,24 @@ function baseRevsFor(all: BlockBaseRevs | undefined, batchBlockIds: BlockId[]):
|
|
|
1481
1558
|
/** `action` re-scoped to one coordinator batch: the batch's own transforms, and the bases for the
|
|
1482
1559
|
* blocks in it alone (see {@link PendRequest.baseRevs}). The action-wide `baseRevs` key is dropped
|
|
1483
1560
|
* BEFORE the subset is spread, so a batch none of whose blocks names a base carries no key at all
|
|
1484
|
-
* rather than the whole action's map — and no cohort signs for a block it is not responsible for.
|
|
1561
|
+
* rather than the whole action's map — and no cohort signs for a block it is not responsible for.
|
|
1562
|
+
*
|
|
1563
|
+
* `soleReservation` survives only on a batch carrying every block of the action: a pend split across
|
|
1564
|
+
* coordinators — by `consolidateCoordinators`, or by `processBatches` re-batching a failed batch — is
|
|
1565
|
+
* several reservations, which the field must never claim to be. Decided per send, like the bases. */
|
|
1485
1566
|
function pendRequestForBatch(action: PendRequest, payload: Transforms): PendRequest {
|
|
1486
|
-
const { baseRevs, ...rest } = action;
|
|
1487
|
-
|
|
1567
|
+
const { baseRevs, soleReservation, ...rest } = action;
|
|
1568
|
+
const batchBlockIds = blockIdsForTransforms(payload);
|
|
1569
|
+
const whole = (): boolean => {
|
|
1570
|
+
const inBatch = new Set(batchBlockIds);
|
|
1571
|
+
return blockIdsForTransforms(action.transforms).every(blockId => inBatch.has(blockId));
|
|
1572
|
+
};
|
|
1573
|
+
return {
|
|
1574
|
+
...rest,
|
|
1575
|
+
transforms: payload,
|
|
1576
|
+
...baseRevsFor(baseRevs, batchBlockIds),
|
|
1577
|
+
...(soleReservation === true && whole() ? { soleReservation } : {})
|
|
1578
|
+
};
|
|
1488
1579
|
}
|
|
1489
1580
|
|
|
1490
1581
|
/** The read one coordinator batch sends downstream: the batch's ids, the caller's context, and the
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
import type { ActionBlocks } from "../network/struct.js";
|
|
2
|
+
import type { ActionId } from "../collection/action.js";
|
|
3
|
+
import type { BlockId } from "../blocks/structs.js";
|
|
4
|
+
import { jitteredBackoffMs } from "../utility/backoff.js";
|
|
5
|
+
import { armUnrefTimer } from "../utility/unref-timer.js";
|
|
6
|
+
import { createLogger } from "../logger.js";
|
|
7
|
+
|
|
8
|
+
const log = createLogger('network-transactor');
|
|
9
|
+
|
|
10
|
+
export type OwedCancelTimerCancel = () => void;
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Optional injection seam for deterministic time, the shape `ClusterCoordinatorClock` in
|
|
14
|
+
* `packages/db-p2p/src/repo/cluster-coordinator.ts` gives its retry timer. Production leaves both
|
|
15
|
+
* undefined and gets `Date.now` plus an unref'd `setTimeout`, so an owed cancel never keeps an
|
|
16
|
+
* otherwise-idle process alive.
|
|
17
|
+
*/
|
|
18
|
+
export interface OwedCancelClock {
|
|
19
|
+
/** Clock (Unix ms). Defaults to `Date.now`. */
|
|
20
|
+
now?: () => number;
|
|
21
|
+
/** Schedule a one-shot timer, returning its cancel. Defaults to an unref'd `setTimeout`. */
|
|
22
|
+
setTimer?: (fn: () => void, delayMs: number) => OwedCancelTimerCancel;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface OwedCancelsInit {
|
|
26
|
+
/**
|
|
27
|
+
* One bounded cancel attempt for `actionRef`, resolving with the blocks some peer ANSWERED for —
|
|
28
|
+
* the same discharged set the transactor's own cancel loop judges by. A throw counts as nothing
|
|
29
|
+
* answered.
|
|
30
|
+
*/
|
|
31
|
+
attempt: (actionRef: ActionBlocks) => Promise<ReadonlySet<BlockId>>;
|
|
32
|
+
clock?: OwedCancelClock;
|
|
33
|
+
/** Most actions remembered at once. Default {@link DEFAULT_MAX_OWED_ACTIONS}. */
|
|
34
|
+
maxActions?: number;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Bounds the memory a writer spends on cancels it cannot land. Evicting an entry re-opens the wedge
|
|
39
|
+
* that entry was kept to close, so it is logged; a writer that owes this many is cut off from its
|
|
40
|
+
* cohorts, and its records are the node-side backstop's (backlog
|
|
41
|
+
* `debt-unpromotable-pending-records-need-a-sweep`).
|
|
42
|
+
*/
|
|
43
|
+
const DEFAULT_MAX_OWED_ACTIONS = 256;
|
|
44
|
+
|
|
45
|
+
/** The backoff between re-sends of one owed cancel: a second at first, doubling to half a minute. */
|
|
46
|
+
const RETRY_BACKOFF = { baseMs: 1_000, capMs: 30_000 };
|
|
47
|
+
|
|
48
|
+
interface OwedEntry {
|
|
49
|
+
readonly blockIds: Set<BlockId>;
|
|
50
|
+
/** When the cancel first fell short, for the discharge log. */
|
|
51
|
+
readonly since: number;
|
|
52
|
+
/** Re-sends made so far — the backoff's exponent. */
|
|
53
|
+
attempts: number;
|
|
54
|
+
nextAttemptAt: number;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* The cancels a writer could not discharge, kept and re-sent until they land.
|
|
59
|
+
*
|
|
60
|
+
* A cancel whose rounds all fall short — every one of them is a cluster transaction, so the member
|
|
61
|
+
* whose silence sank the commit sinks the cancel too — leaves pending records standing that make the
|
|
62
|
+
* members refuse every later write to those blocks, from any writer. Nothing else removes them while
|
|
63
|
+
* their slot is unreached (docs/repository.md, "A pending record's lifetime is bounded by its
|
|
64
|
+
* writer"), so the writer keeps the cancel and sends it again:
|
|
65
|
+
*
|
|
66
|
+
* - **before its own next pend of any of those blocks** ({@link beforePend}), which is what clears
|
|
67
|
+
* the leftover in front of the writer's next write to the block;
|
|
68
|
+
* - **on a backoff timer** while anything is owed, so other machines' writes, which the records also
|
|
69
|
+
* refuse, are freed without this writer writing the block again.
|
|
70
|
+
*
|
|
71
|
+
* Re-sending is safe whenever it arrives: a cancel deletes only the cancelled action's pending records
|
|
72
|
+
* and never moves a revision, so it commutes with every other action. The one exception is a re-pend
|
|
73
|
+
* of the SAME action id, whose new record a late cancel would delete — so a pend of an owed action
|
|
74
|
+
* supersedes the owed cancel for every block it names, and waits out any re-send of it already in
|
|
75
|
+
* flight before it goes out.
|
|
76
|
+
*/
|
|
77
|
+
export class OwedCancels {
|
|
78
|
+
private readonly entries = new Map<ActionId, OwedEntry>();
|
|
79
|
+
private readonly inFlight = new Map<ActionId, Promise<void>>();
|
|
80
|
+
private readonly attempt: OwedCancelsInit['attempt'];
|
|
81
|
+
private readonly now: () => number;
|
|
82
|
+
private readonly setTimer: (fn: () => void, delayMs: number) => OwedCancelTimerCancel;
|
|
83
|
+
private readonly maxActions: number;
|
|
84
|
+
private timer: { cancel: OwedCancelTimerCancel; dueAt: number } | undefined;
|
|
85
|
+
private stopped = false;
|
|
86
|
+
|
|
87
|
+
constructor(init: OwedCancelsInit) {
|
|
88
|
+
this.attempt = init.attempt;
|
|
89
|
+
this.now = init.clock?.now ?? ((): number => Date.now());
|
|
90
|
+
this.setTimer = init.clock?.setTimer ?? armUnrefTimer;
|
|
91
|
+
this.maxActions = init.maxActions ?? DEFAULT_MAX_OWED_ACTIONS;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** Every owed action with the blocks it still owes a cancel for. A snapshot. */
|
|
95
|
+
snapshot(): Map<ActionId, BlockId[]> {
|
|
96
|
+
return new Map(Array.from(this.entries, ([actionId, entry]) => [actionId, Array.from(entry.blockIds)]));
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** A cancel for `actionId` ran out of budget with `blockIds` still unanswered. */
|
|
100
|
+
record(actionId: ActionId, blockIds: readonly BlockId[], why: string | undefined): void {
|
|
101
|
+
if (blockIds.length === 0) return;
|
|
102
|
+
if (this.stopped) {
|
|
103
|
+
log('cancel:owed-abandoned actionId=%s blocks=%o why=stopped', actionId, blockIds);
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
const existing = this.entries.get(actionId);
|
|
107
|
+
if (existing) {
|
|
108
|
+
for (const bid of blockIds) existing.blockIds.add(bid);
|
|
109
|
+
existing.nextAttemptAt = this.now() + this.backoffMs(existing.attempts);
|
|
110
|
+
} else {
|
|
111
|
+
const now = this.now();
|
|
112
|
+
this.entries.set(actionId, { blockIds: new Set(blockIds), since: now, attempts: 0, nextAttemptAt: now + this.backoffMs(0) });
|
|
113
|
+
this.evictOverflow();
|
|
114
|
+
}
|
|
115
|
+
log('cancel:owed actionId=%s blocks=%o why=%s', actionId, blockIds, why ?? 'unknown');
|
|
116
|
+
this.arm();
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** A cancel for `actionId` was answered for `blockIds` — whoever sent it, they are owed no longer. */
|
|
120
|
+
discharged(actionId: ActionId, blockIds: Iterable<BlockId>): void {
|
|
121
|
+
const settled = this.remove(actionId, blockIds);
|
|
122
|
+
if (settled) {
|
|
123
|
+
log('cancel:owed-discharged actionId=%s attempts=%d stoodMs=%d', actionId, settled.attempts, this.now() - settled.since);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Re-sends nothing from here on: the timer is disarmed and every cancel still owed is logged
|
|
129
|
+
* (`cancel:owed-abandoned`) and forgotten, its records left to the node-side backstop. For when the
|
|
130
|
+
* node the re-sends go through stops — they could never land, and the timer would otherwise keep
|
|
131
|
+
* the transactor, and everything it reaches, alive for the life of the process.
|
|
132
|
+
*/
|
|
133
|
+
stop(): void {
|
|
134
|
+
this.stopped = true;
|
|
135
|
+
this.timer?.cancel();
|
|
136
|
+
this.timer = undefined;
|
|
137
|
+
for (const [actionId, entry] of this.entries) {
|
|
138
|
+
log('cancel:owed-abandoned actionId=%s blocks=%o stoodMs=%d', actionId, Array.from(entry.blockIds), this.now() - entry.since);
|
|
139
|
+
}
|
|
140
|
+
this.entries.clear();
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Runs ahead of every pend of `actionId` over `blockIds`: supersedes this action's own owed cancel
|
|
145
|
+
* for those blocks, then makes one attempt at every OTHER owed cancel that names any of them. A
|
|
146
|
+
* failed attempt stays owed and the pend proceeds; a cohort still unable to approve the cancel
|
|
147
|
+
* cannot approve the pend either, so one attempt — not a loop — is what the pend should wait for.
|
|
148
|
+
*/
|
|
149
|
+
async beforePend(actionId: ActionId, blockIds: readonly BlockId[]): Promise<void> {
|
|
150
|
+
if (this.entries.size === 0 && this.inFlight.size === 0) return;
|
|
151
|
+
await this.supersede(actionId, blockIds);
|
|
152
|
+
const named = new Set(blockIds);
|
|
153
|
+
const overlapping = Array.from(this.entries)
|
|
154
|
+
.filter(([owedId, entry]) => owedId !== actionId && Array.from(entry.blockIds).some(bid => named.has(bid)))
|
|
155
|
+
.map(([owedId]) => owedId);
|
|
156
|
+
await Promise.all(overlapping.map(owedId => this.attemptOnce(owedId)));
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* A pend of `actionId` writes a fresh record on every block it names, which a late cancel for the
|
|
161
|
+
* same action id would delete. So those blocks leave the owed entry first — synchronously, so no
|
|
162
|
+
* re-send started from here on can carry them — and a re-send already in flight is waited out, so
|
|
163
|
+
* it lands before the pend rather than after it.
|
|
164
|
+
*/
|
|
165
|
+
private async supersede(actionId: ActionId, blockIds: readonly BlockId[]): Promise<void> {
|
|
166
|
+
const entry = this.entries.get(actionId);
|
|
167
|
+
const superseded = entry ? blockIds.filter(bid => entry.blockIds.has(bid)) : [];
|
|
168
|
+
if (superseded.length > 0) {
|
|
169
|
+
log('cancel:owed-superseded actionId=%s blocks=%o', actionId, superseded);
|
|
170
|
+
this.remove(actionId, superseded);
|
|
171
|
+
}
|
|
172
|
+
await this.inFlight.get(actionId);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/** Takes `blockIds` off `actionId`'s entry, returning the entry when that emptied — and so removed — it. */
|
|
176
|
+
private remove(actionId: ActionId, blockIds: Iterable<BlockId>): OwedEntry | undefined {
|
|
177
|
+
const entry = this.entries.get(actionId);
|
|
178
|
+
if (!entry) return undefined;
|
|
179
|
+
for (const bid of blockIds) entry.blockIds.delete(bid);
|
|
180
|
+
if (entry.blockIds.size > 0) return undefined;
|
|
181
|
+
this.entries.delete(actionId);
|
|
182
|
+
this.arm();
|
|
183
|
+
return entry;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/** One re-send of `actionId`'s owed cancel, joining the one already in flight if there is one. */
|
|
187
|
+
private attemptOnce(actionId: ActionId): Promise<void> {
|
|
188
|
+
const running = this.inFlight.get(actionId);
|
|
189
|
+
if (running) return running;
|
|
190
|
+
const entry = this.entries.get(actionId);
|
|
191
|
+
if (!entry) return Promise.resolve();
|
|
192
|
+
++entry.attempts;
|
|
193
|
+
const run = this.resend(actionId, Array.from(entry.blockIds))
|
|
194
|
+
.finally(() => {
|
|
195
|
+
if (this.inFlight.get(actionId) === run) this.inFlight.delete(actionId);
|
|
196
|
+
this.arm();
|
|
197
|
+
});
|
|
198
|
+
this.inFlight.set(actionId, run);
|
|
199
|
+
return run;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
private async resend(actionId: ActionId, blockIds: BlockId[]): Promise<void> {
|
|
203
|
+
let answered: ReadonlySet<BlockId>;
|
|
204
|
+
try {
|
|
205
|
+
answered = await this.attempt({ actionId, blockIds });
|
|
206
|
+
} catch (error) {
|
|
207
|
+
log('cancel:owed-retry-failed actionId=%s blocks=%d error=%s', actionId, blockIds.length, (error as Error)?.message ?? String(error));
|
|
208
|
+
answered = new Set();
|
|
209
|
+
}
|
|
210
|
+
this.discharged(actionId, answered);
|
|
211
|
+
const entry = this.entries.get(actionId);
|
|
212
|
+
if (entry) {
|
|
213
|
+
entry.nextAttemptAt = this.now() + this.backoffMs(entry.attempts);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/** Arms the one timer at the earliest re-send due among the entries not already being re-sent, or disarms it. */
|
|
218
|
+
private arm(): void {
|
|
219
|
+
let dueAt: number | undefined;
|
|
220
|
+
for (const [actionId, entry] of this.entries) {
|
|
221
|
+
if (this.inFlight.has(actionId)) continue;
|
|
222
|
+
if (dueAt === undefined || entry.nextAttemptAt < dueAt) dueAt = entry.nextAttemptAt;
|
|
223
|
+
}
|
|
224
|
+
if (this.timer && (dueAt === undefined || this.timer.dueAt > dueAt)) {
|
|
225
|
+
this.timer.cancel();
|
|
226
|
+
this.timer = undefined;
|
|
227
|
+
}
|
|
228
|
+
if (dueAt === undefined || this.timer) return;
|
|
229
|
+
this.timer = { cancel: this.setTimer(() => this.fire(), Math.max(0, dueAt - this.now())), dueAt };
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
private fire(): void {
|
|
233
|
+
this.timer = undefined;
|
|
234
|
+
const now = this.now();
|
|
235
|
+
for (const [actionId, entry] of this.entries) {
|
|
236
|
+
if (entry.nextAttemptAt <= now) void this.attemptOnce(actionId);
|
|
237
|
+
}
|
|
238
|
+
this.arm();
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
private backoffMs(attempts: number): number {
|
|
242
|
+
return jitteredBackoffMs(attempts, RETRY_BACKOFF);
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
private evictOverflow(): void {
|
|
246
|
+
while (this.entries.size > this.maxActions) {
|
|
247
|
+
const [oldestId, oldest] = this.entries.entries().next().value as [ActionId, OwedEntry];
|
|
248
|
+
this.entries.delete(oldestId);
|
|
249
|
+
log('cancel:owed-evicted actionId=%s blocks=%o stoodMs=%d', oldestId, Array.from(oldest.blockIds), this.now() - oldest.since);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
}
|
|
@@ -240,7 +240,9 @@ export class TransactorSource<TBlock extends IBlock> implements BlockSource<TBlo
|
|
|
240
240
|
const pendResult = await this.transactor.pend({
|
|
241
241
|
transforms: transform, actionId, rev, policy: 'r',
|
|
242
242
|
...(priority > 0 ? { priority } : {}),
|
|
243
|
-
...baseRevsField(baseRevs)
|
|
243
|
+
...baseRevsField(baseRevs),
|
|
244
|
+
// One collection, pended whole and then committed: the shape `PendRequest.soleReservation` names.
|
|
245
|
+
soleReservation: true
|
|
244
246
|
});
|
|
245
247
|
if (!pendResult.success) {
|
|
246
248
|
return pendResult;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Keep a pending timer from holding a Node process open. Browsers and React Native return a number
|
|
3
|
+
* from `setTimeout` / `setInterval`, which has no `unref`, so the call is guarded. Every first-party
|
|
4
|
+
* `unref` goes through here; the `NO_TIMER_UNREF` lint rule in `eslint.config.js` enforces that.
|
|
5
|
+
*/
|
|
6
|
+
export function unrefTimer<T>(handle: T): T {
|
|
7
|
+
(handle as { unref?: () => void }).unref?.();
|
|
8
|
+
return handle;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
/** Arm a one-shot timer that never holds a process open; returns its cancel. The default behind the injectable `setTimer` seams. */
|
|
12
|
+
export function armUnrefTimer(fn: () => void, ms: number): () => void {
|
|
13
|
+
const handle = unrefTimer(setTimeout(fn, ms));
|
|
14
|
+
return (): void => clearTimeout(handle);
|
|
15
|
+
}
|