@optimystic/db-core 1.15.0 → 1.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/structs.d.ts +11 -1
- package/dist/src/cluster/structs.d.ts.map +1 -1
- package/dist/src/cluster/structs.js.map +1 -1
- package/dist/src/collection/collection.d.ts +80 -32
- package/dist/src/collection/collection.d.ts.map +1 -1
- package/dist/src/collection/collection.js +121 -54
- package/dist/src/collection/collection.js.map +1 -1
- package/dist/src/network/i-repo.d.ts +13 -0
- package/dist/src/network/i-repo.d.ts.map +1 -1
- package/dist/src/network/repo-protocol.d.ts +28 -15
- package/dist/src/network/repo-protocol.d.ts.map +1 -1
- package/dist/src/network/stale-failure.d.ts +10 -7
- package/dist/src/network/stale-failure.d.ts.map +1 -1
- package/dist/src/network/stale-failure.js +10 -7
- package/dist/src/network/stale-failure.js.map +1 -1
- package/dist/src/transactor/block-floors.d.ts +2 -2
- package/dist/src/transactor/network-transactor.d.ts +21 -1
- package/dist/src/transactor/network-transactor.d.ts.map +1 -1
- package/dist/src/transactor/network-transactor.js +45 -13
- package/dist/src/transactor/network-transactor.js.map +1 -1
- package/dist/src/utility/batch-coordinator.d.ts +55 -1
- package/dist/src/utility/batch-coordinator.d.ts.map +1 -1
- package/dist/src/utility/batch-coordinator.js +82 -12
- package/dist/src/utility/batch-coordinator.js.map +1 -1
- package/dist/src/utility/unref-timer.d.ts +2 -0
- package/dist/src/utility/unref-timer.d.ts.map +1 -1
- package/dist/src/utility/unref-timer.js +4 -0
- package/dist/src/utility/unref-timer.js.map +1 -1
- package/package.json +1 -1
- package/src/cluster/structs.ts +11 -1
- package/src/collection/collection.ts +125 -55
- package/src/network/i-repo.ts +13 -0
- package/src/network/repo-protocol.ts +22 -7
- package/src/network/stale-failure.ts +10 -7
- package/src/transactor/block-floors.ts +2 -2
- package/src/transactor/network-transactor.ts +67 -15
- package/src/utility/batch-coordinator.ts +124 -18
- package/src/utility/unref-timer.ts +5 -0
|
@@ -1,10 +1,88 @@
|
|
|
1
1
|
import type { PeerId } from "../network/types.js";
|
|
2
2
|
import type { BlockId } from "../index.js";
|
|
3
3
|
import { Pending } from "./pending.js";
|
|
4
|
+
import { unrefDelay } from "./unref-timer.js";
|
|
4
5
|
import { createLogger } from "../logger.js";
|
|
5
6
|
|
|
6
7
|
const log = createLogger('batch-coordinator');
|
|
7
8
|
|
|
9
|
+
/**
|
|
10
|
+
* How a transactor paces the retries it sends after a request went unanswered. A request given up
|
|
11
|
+
* on at a deadline was usually not lost: its frames are still in this node's send queue to that
|
|
12
|
+
* peer, so a retry sent at once through the same queue only lengthens the backlog that made its
|
|
13
|
+
* predecessor late. See {@link retryWaitMs}.
|
|
14
|
+
*/
|
|
15
|
+
export type RetryBackoff = {
|
|
16
|
+
/** First wait in ms — one declared link round trip. Not above 0 disables every wait (no declared round trip). */
|
|
17
|
+
baseMs: number;
|
|
18
|
+
/** The peer this transactor runs on, when it runs on one (`NetworkTransactorInit.localPeerId`). */
|
|
19
|
+
localPeerId?: PeerId;
|
|
20
|
+
/** Test seam; default an unref'd `setTimeout` promise. */
|
|
21
|
+
sleep?: (ms: number) => Promise<void>;
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
/** The retry {@link retryWaitMs} is asked to pace. */
|
|
25
|
+
export type RetryAttempt = {
|
|
26
|
+
/** 1-based: how many coordinators this retry's chain has already tried — the retry batch's `excludedPeers.length`. */
|
|
27
|
+
attempt: number;
|
|
28
|
+
/** The coordinator whose request failed. */
|
|
29
|
+
failedPeer: PeerId;
|
|
30
|
+
/** How long that request was outstanding before it failed, ms. */
|
|
31
|
+
outstandingMs: number;
|
|
32
|
+
/** The coordinators the retry will be sent to. */
|
|
33
|
+
retryPeers: readonly PeerId[];
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* How long to wait before sending a retry: one base for the first retry of an operation, doubling
|
|
38
|
+
* for each further coordinator its chain tries, and never less than the failed request was
|
|
39
|
+
* outstanding when the retry goes out through this node's own queue to the peer whose request is
|
|
40
|
+
* still in it — when the retry re-picks the failed peer, or when this node coordinates the retry
|
|
41
|
+
* (the failed peer was picked from the block's cohort, so this node's consensus round sends to it).
|
|
42
|
+
* A retry to a third coordinator goes over a different connection and takes no such floor. A fast
|
|
43
|
+
* failure (connection refused) was outstanding briefly, so its floor is small too.
|
|
44
|
+
*/
|
|
45
|
+
export function retryWaitMs(backoff: RetryBackoff, retry: RetryAttempt): number {
|
|
46
|
+
if (!(backoff.baseMs > 0)) return 0;
|
|
47
|
+
const doubled = backoff.baseMs * 2 ** Math.max(0, retry.attempt - 1);
|
|
48
|
+
// NOTE: the floor is as large as the failure was slow — a request that ran out a whole RPC dial
|
|
49
|
+
// deadline (33 s at a declared 3 s round trip, of a 132 s transaction budget) holds a retry this
|
|
50
|
+
// node coordinates back that long. `retryWaitWithinBudget` keeps the wait inside the budget, not
|
|
51
|
+
// small within it. If field logs show `retry ... waitMs` near the dial deadline costing operations
|
|
52
|
+
// their answer, cap the floor at a few round trips rather than dropping it.
|
|
53
|
+
return sendsThroughFailedPeersQueue(backoff, retry) ? Math.max(doubled, retry.outstandingMs) : doubled;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function sendsThroughFailedPeersQueue(backoff: RetryBackoff, retry: RetryAttempt): boolean {
|
|
57
|
+
const failed = retry.failedPeer.toString();
|
|
58
|
+
const local = backoff.localPeerId?.toString();
|
|
59
|
+
return retry.retryPeers.some(peer => {
|
|
60
|
+
const id = peer.toString();
|
|
61
|
+
return id === failed || id === local;
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* The wait before a retry, or `undefined` when a positive wait would reach `expiration` — the retry
|
|
67
|
+
* is then not sent (logged `retry:skipped-budget`) and the caller keeps the original failure. The
|
|
68
|
+
* check runs before any timer is armed, which also keeps a doubled wait from reaching `setTimeout`
|
|
69
|
+
* above its 32-bit limit: the transaction budget is far below it. A zero wait is returned as is,
|
|
70
|
+
* leaving the caller's own expiration checks exactly as they were without a backoff.
|
|
71
|
+
*/
|
|
72
|
+
export function retryWaitWithinBudget(backoff: RetryBackoff | undefined, retry: RetryAttempt, expiration: number): number | undefined {
|
|
73
|
+
const waitMs = backoff ? retryWaitMs(backoff, retry) : 0;
|
|
74
|
+
if (waitMs > 0 && Date.now() + waitMs >= expiration) {
|
|
75
|
+
log('retry:skipped-budget peer=%s waitMs=%d remainingMs=%d', retry.failedPeer.toString(), waitMs, expiration - Date.now());
|
|
76
|
+
return undefined;
|
|
77
|
+
}
|
|
78
|
+
return waitMs;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** The backoff's sleep. Callers await it only for a positive wait, so a zero wait adds no yield and ordering is as it was without one. */
|
|
82
|
+
export function retrySleep(backoff: RetryBackoff | undefined): (ms: number) => Promise<void> {
|
|
83
|
+
return backoff?.sleep ?? unrefDelay;
|
|
84
|
+
}
|
|
85
|
+
|
|
8
86
|
/**
|
|
9
87
|
* Represents a batch of operations for a specific block coordinated by a peer
|
|
10
88
|
*/
|
|
@@ -19,6 +97,14 @@ export type CoordinatorBatch<TPayload, TResponse> = {
|
|
|
19
97
|
excludedPeers?: PeerId[];
|
|
20
98
|
/** When a batch coordinates multiple blocks (via cluster intersection), lists all block IDs */
|
|
21
99
|
coordinatingBlockIds?: BlockId[];
|
|
100
|
+
/** How long this batch's request was outstanding before it failed, ms — set by {@link processBatches} on failure only. */
|
|
101
|
+
outstandingMs?: number;
|
|
102
|
+
/**
|
|
103
|
+
* Set by {@link processBatches} when it re-batched this batch's failure onto other coordinators. The
|
|
104
|
+
* retries sit in the ROOT batch's `subsumedBy` (not this one's) and carry this batch's peer in their
|
|
105
|
+
* `excludedPeers`, so the chain has already moved past it.
|
|
106
|
+
*/
|
|
107
|
+
rebatched?: true;
|
|
22
108
|
}
|
|
23
109
|
|
|
24
110
|
/**
|
|
@@ -107,6 +193,7 @@ export function mergeBlocks(_payload: BlockId[], blockId: BlockId, mergeWithPayl
|
|
|
107
193
|
* @param getBlockPayload - The function to call to get the payload given a parent payload and block id, and optionally merge with an existing payload
|
|
108
194
|
* @param expiration - The expiration time for the operation
|
|
109
195
|
* @param findCoordinator - The function to call to find a coordinator for a block id
|
|
196
|
+
* @param backoff - How long to wait before each retry ({@link retryWaitMs}); absent, retries go out at once
|
|
110
197
|
*/
|
|
111
198
|
export async function processBatches<TPayload, TResponse>(
|
|
112
199
|
batches: CoordinatorBatch<TPayload, TResponse>[],
|
|
@@ -114,38 +201,57 @@ export async function processBatches<TPayload, TResponse>(
|
|
|
114
201
|
getBlockIds: (batch: CoordinatorBatch<TPayload, TResponse>) => BlockId[],
|
|
115
202
|
getBlockPayload: (payload: TPayload, blockId: BlockId, mergeWithPayload: TPayload | undefined) => TPayload,
|
|
116
203
|
expiration: number,
|
|
117
|
-
findCoordinator: (blockId: BlockId, options: { excludedPeers: PeerId[] }) => Promise<PeerId
|
|
204
|
+
findCoordinator: (blockId: BlockId, options: { excludedPeers: PeerId[] }) => Promise<PeerId>,
|
|
205
|
+
backoff?: RetryBackoff
|
|
118
206
|
): Promise<void> {
|
|
119
207
|
// Root-map ensures retries are recorded on the original batch to avoid deep trees
|
|
120
208
|
const rootOf = new WeakMap<CoordinatorBatch<TPayload, TResponse>, CoordinatorBatch<TPayload, TResponse>>();
|
|
121
209
|
for (const b of batches) rootOf.set(b, b);
|
|
122
210
|
|
|
211
|
+
// Re-batch a failed batch onto the next coordinators and process them, after the backoff's wait.
|
|
212
|
+
// The wait is ONE for the whole retry set, decided once its coordinators are known; a wait that
|
|
213
|
+
// would reach the expiration sends no retry, leaving the batch's original failure as its outcome.
|
|
214
|
+
const retry = async (batch: CoordinatorBatch<TPayload, TResponse>, outstandingMs: number) => {
|
|
215
|
+
const excludedPeers = [batch.peerId, ...(batch.excludedPeers ?? [])];
|
|
216
|
+
const retries = await createBatchesForPayload<TPayload, TResponse>(
|
|
217
|
+
getBlockIds(batch),
|
|
218
|
+
batch.payload,
|
|
219
|
+
getBlockPayload,
|
|
220
|
+
excludedPeers,
|
|
221
|
+
findCoordinator
|
|
222
|
+
);
|
|
223
|
+
if (retries.length === 0) return;
|
|
224
|
+
const waitMs = retryWaitWithinBudget(backoff, {
|
|
225
|
+
attempt: excludedPeers.length,
|
|
226
|
+
failedPeer: batch.peerId,
|
|
227
|
+
outstandingMs,
|
|
228
|
+
retryPeers: retries.map(r => r.peerId)
|
|
229
|
+
}, expiration);
|
|
230
|
+
if (waitMs === undefined) return;
|
|
231
|
+
log('retry peer=%s excluded=%d waitMs=%d outstandingMs=%d', batch.peerId.toString(), excludedPeers.length, waitMs, outstandingMs);
|
|
232
|
+
if (waitMs > 0) await retrySleep(backoff)(waitMs);
|
|
233
|
+
if (expiration <= Date.now()) return;
|
|
234
|
+
const root = rootOf.get(batch) ?? batch;
|
|
235
|
+
root.subsumedBy = [...(root.subsumedBy ?? []), ...retries];
|
|
236
|
+
for (const r of retries) rootOf.set(r, root);
|
|
237
|
+
batch.rebatched = true;
|
|
238
|
+
// Process retries, but ensure further failures also attach to the same root
|
|
239
|
+
await processSet(retries);
|
|
240
|
+
};
|
|
241
|
+
|
|
123
242
|
// Process a set of batches concurrently and enqueue retries flatly onto the root's subsumedBy list
|
|
124
|
-
const processSet = async (set: CoordinatorBatch<TPayload, TResponse>[]) => {
|
|
243
|
+
const processSet = async (set: CoordinatorBatch<TPayload, TResponse>[]): Promise<void> => {
|
|
125
244
|
await Promise.all(set.map(async (batch) => {
|
|
245
|
+
const sentAt = Date.now();
|
|
126
246
|
batch.request = new Pending(process(batch)
|
|
127
247
|
.catch(async e => {
|
|
248
|
+
batch.outstandingMs = Date.now() - sentAt;
|
|
128
249
|
// Always rethrow the ORIGINAL first-attempt error `e` so batch.request.error
|
|
129
250
|
// preserves the root cause. If retry setup itself fails (e.g., findCoordinator
|
|
130
251
|
// throws "self-exhausted" on a solo node), that retry error MUST NOT mask `e`.
|
|
131
252
|
if (expiration > Date.now()) {
|
|
132
|
-
const excludedPeers = [batch.peerId, ...(batch.excludedPeers ?? [])];
|
|
133
|
-
log('retry peer=%s excluded=%d', batch.peerId.toString(), excludedPeers.length);
|
|
134
253
|
try {
|
|
135
|
-
|
|
136
|
-
getBlockIds(batch),
|
|
137
|
-
batch.payload,
|
|
138
|
-
getBlockPayload,
|
|
139
|
-
excludedPeers,
|
|
140
|
-
findCoordinator
|
|
141
|
-
);
|
|
142
|
-
if (retries.length > 0 && expiration > Date.now()) {
|
|
143
|
-
const root = rootOf.get(batch) ?? batch;
|
|
144
|
-
root.subsumedBy = [...(root.subsumedBy ?? []), ...retries];
|
|
145
|
-
for (const r of retries) rootOf.set(r, root);
|
|
146
|
-
// Process retries, but ensure further failures also attach to the same root
|
|
147
|
-
await processSet(retries);
|
|
148
|
-
}
|
|
254
|
+
await retry(batch, batch.outstandingMs);
|
|
149
255
|
} catch (retryErr) {
|
|
150
256
|
log('retry:setup-failed peer=%s original=%o retry=%o',
|
|
151
257
|
batch.peerId.toString(), e, retryErr);
|
|
@@ -13,3 +13,8 @@ export function armUnrefTimer(fn: () => void, ms: number): () => void {
|
|
|
13
13
|
const handle = unrefTimer(setTimeout(fn, ms));
|
|
14
14
|
return (): void => clearTimeout(handle);
|
|
15
15
|
}
|
|
16
|
+
|
|
17
|
+
/** Resolve after `ms` on a timer that never holds a process open. The default behind the injectable `sleep` seams. */
|
|
18
|
+
export function unrefDelay(ms: number): Promise<void> {
|
|
19
|
+
return new Promise<void>(resolve => { armUnrefTimer(resolve, ms); });
|
|
20
|
+
}
|