@optimystic/db-core 1.15.0 → 1.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/dist/src/cluster/structs.d.ts +11 -1
  2. package/dist/src/cluster/structs.d.ts.map +1 -1
  3. package/dist/src/cluster/structs.js.map +1 -1
  4. package/dist/src/collection/collection.d.ts +80 -32
  5. package/dist/src/collection/collection.d.ts.map +1 -1
  6. package/dist/src/collection/collection.js +121 -54
  7. package/dist/src/collection/collection.js.map +1 -1
  8. package/dist/src/network/i-repo.d.ts +13 -0
  9. package/dist/src/network/i-repo.d.ts.map +1 -1
  10. package/dist/src/network/repo-protocol.d.ts +28 -15
  11. package/dist/src/network/repo-protocol.d.ts.map +1 -1
  12. package/dist/src/network/stale-failure.d.ts +10 -7
  13. package/dist/src/network/stale-failure.d.ts.map +1 -1
  14. package/dist/src/network/stale-failure.js +10 -7
  15. package/dist/src/network/stale-failure.js.map +1 -1
  16. package/dist/src/transactor/block-floors.d.ts +2 -2
  17. package/dist/src/transactor/network-transactor.d.ts +21 -1
  18. package/dist/src/transactor/network-transactor.d.ts.map +1 -1
  19. package/dist/src/transactor/network-transactor.js +45 -13
  20. package/dist/src/transactor/network-transactor.js.map +1 -1
  21. package/dist/src/utility/batch-coordinator.d.ts +55 -1
  22. package/dist/src/utility/batch-coordinator.d.ts.map +1 -1
  23. package/dist/src/utility/batch-coordinator.js +82 -12
  24. package/dist/src/utility/batch-coordinator.js.map +1 -1
  25. package/dist/src/utility/unref-timer.d.ts +2 -0
  26. package/dist/src/utility/unref-timer.d.ts.map +1 -1
  27. package/dist/src/utility/unref-timer.js +4 -0
  28. package/dist/src/utility/unref-timer.js.map +1 -1
  29. package/package.json +1 -1
  30. package/src/cluster/structs.ts +11 -1
  31. package/src/collection/collection.ts +125 -55
  32. package/src/network/i-repo.ts +13 -0
  33. package/src/network/repo-protocol.ts +22 -7
  34. package/src/network/stale-failure.ts +10 -7
  35. package/src/transactor/block-floors.ts +2 -2
  36. package/src/transactor/network-transactor.ts +67 -15
  37. package/src/utility/batch-coordinator.ts +124 -18
  38. package/src/utility/unref-timer.ts +5 -0
@@ -1,10 +1,88 @@
1
1
  import type { PeerId } from "../network/types.js";
2
2
  import type { BlockId } from "../index.js";
3
3
  import { Pending } from "./pending.js";
4
+ import { unrefDelay } from "./unref-timer.js";
4
5
  import { createLogger } from "../logger.js";
5
6
 
6
7
  const log = createLogger('batch-coordinator');
7
8
 
9
+ /**
10
+ * How a transactor paces the retries it sends after a request went unanswered. A request given up
11
+ * on at a deadline was usually not lost: its frames are still in this node's send queue to that
12
+ * peer, so a retry sent at once through the same queue only lengthens the backlog that made its
13
+ * predecessor late. See {@link retryWaitMs}.
14
+ */
15
+ export type RetryBackoff = {
16
+ /** First wait in ms — one declared link round trip. Not above 0 disables every wait (no declared round trip). */
17
+ baseMs: number;
18
+ /** The peer this transactor runs on, when it runs on one (`NetworkTransactorInit.localPeerId`). */
19
+ localPeerId?: PeerId;
20
+ /** Test seam; default an unref'd `setTimeout` promise. */
21
+ sleep?: (ms: number) => Promise<void>;
22
+ };
23
+
24
+ /** The retry {@link retryWaitMs} is asked to pace. */
25
+ export type RetryAttempt = {
26
+ /** 1-based: how many coordinators this retry's chain has already tried — the retry batch's `excludedPeers.length`. */
27
+ attempt: number;
28
+ /** The coordinator whose request failed. */
29
+ failedPeer: PeerId;
30
+ /** How long that request was outstanding before it failed, ms. */
31
+ outstandingMs: number;
32
+ /** The coordinators the retry will be sent to. */
33
+ retryPeers: readonly PeerId[];
34
+ };
35
+
36
+ /**
37
+ * How long to wait before sending a retry: one base for the first retry of an operation, doubling
38
+ * for each further coordinator its chain tries, and never less than the failed request was
39
+ * outstanding when the retry goes out through this node's own queue to the peer whose request is
40
+ * still in it — when the retry re-picks the failed peer, or when this node coordinates the retry
41
+ * (the failed peer was picked from the block's cohort, so this node's consensus round sends to it).
42
+ * A retry to a third coordinator goes over a different connection and takes no such floor. A fast
43
+ * failure (connection refused) was outstanding briefly, so its floor is small too.
44
+ */
45
+ export function retryWaitMs(backoff: RetryBackoff, retry: RetryAttempt): number {
46
+ if (!(backoff.baseMs > 0)) return 0;
47
+ const doubled = backoff.baseMs * 2 ** Math.max(0, retry.attempt - 1);
48
+ // NOTE: the floor is as large as the failure was slow — a request that ran out a whole RPC dial
49
+ // deadline (33 s at a declared 3 s round trip, of a 132 s transaction budget) holds a retry this
50
+ // node coordinates back that long. `retryWaitWithinBudget` keeps the wait inside the budget, not
51
+ // small within it. If field logs show `retry ... waitMs` near the dial deadline costing operations
52
+ // their answer, cap the floor at a few round trips rather than dropping it.
53
+ return sendsThroughFailedPeersQueue(backoff, retry) ? Math.max(doubled, retry.outstandingMs) : doubled;
54
+ }
55
+
56
+ function sendsThroughFailedPeersQueue(backoff: RetryBackoff, retry: RetryAttempt): boolean {
57
+ const failed = retry.failedPeer.toString();
58
+ const local = backoff.localPeerId?.toString();
59
+ return retry.retryPeers.some(peer => {
60
+ const id = peer.toString();
61
+ return id === failed || id === local;
62
+ });
63
+ }
64
+
65
+ /**
66
+ * The wait before a retry, or `undefined` when a positive wait would reach `expiration` — the retry
67
+ * is then not sent (logged `retry:skipped-budget`) and the caller keeps the original failure. The
68
+ * check runs before any timer is armed, which also keeps a doubled wait from reaching `setTimeout`
69
+ * above its 32-bit limit: the transaction budget is far below it. A zero wait is returned as is,
70
+ * leaving the caller's own expiration checks exactly as they were without a backoff.
71
+ */
72
+ export function retryWaitWithinBudget(backoff: RetryBackoff | undefined, retry: RetryAttempt, expiration: number): number | undefined {
73
+ const waitMs = backoff ? retryWaitMs(backoff, retry) : 0;
74
+ if (waitMs > 0 && Date.now() + waitMs >= expiration) {
75
+ log('retry:skipped-budget peer=%s waitMs=%d remainingMs=%d', retry.failedPeer.toString(), waitMs, expiration - Date.now());
76
+ return undefined;
77
+ }
78
+ return waitMs;
79
+ }
80
+
81
+ /** The backoff's sleep. Callers await it only for a positive wait, so a zero wait adds no yield and ordering is as it was without one. */
82
+ export function retrySleep(backoff: RetryBackoff | undefined): (ms: number) => Promise<void> {
83
+ return backoff?.sleep ?? unrefDelay;
84
+ }
85
+
8
86
  /**
9
87
  * Represents a batch of operations for a specific block coordinated by a peer
10
88
  */
@@ -19,6 +97,14 @@ export type CoordinatorBatch<TPayload, TResponse> = {
19
97
  excludedPeers?: PeerId[];
20
98
  /** When a batch coordinates multiple blocks (via cluster intersection), lists all block IDs */
21
99
  coordinatingBlockIds?: BlockId[];
100
+ /** How long this batch's request was outstanding before it failed, ms — set by {@link processBatches} on failure only. */
101
+ outstandingMs?: number;
102
+ /**
103
+ * Set by {@link processBatches} when it re-batched this batch's failure onto other coordinators. The
104
+ * retries sit in the ROOT batch's `subsumedBy` (not this one's) and carry this batch's peer in their
105
+ * `excludedPeers`, so the chain has already moved past it.
106
+ */
107
+ rebatched?: true;
22
108
  }
23
109
 
24
110
  /**
@@ -107,6 +193,7 @@ export function mergeBlocks(_payload: BlockId[], blockId: BlockId, mergeWithPayl
107
193
  * @param getBlockPayload - The function to call to get the payload given a parent payload and block id, and optionally merge with an existing payload
108
194
  * @param expiration - The expiration time for the operation
109
195
  * @param findCoordinator - The function to call to find a coordinator for a block id
196
+ * @param backoff - How long to wait before each retry ({@link retryWaitMs}); absent, retries go out at once
110
197
  */
111
198
  export async function processBatches<TPayload, TResponse>(
112
199
  batches: CoordinatorBatch<TPayload, TResponse>[],
@@ -114,38 +201,57 @@ export async function processBatches<TPayload, TResponse>(
114
201
  getBlockIds: (batch: CoordinatorBatch<TPayload, TResponse>) => BlockId[],
115
202
  getBlockPayload: (payload: TPayload, blockId: BlockId, mergeWithPayload: TPayload | undefined) => TPayload,
116
203
  expiration: number,
117
- findCoordinator: (blockId: BlockId, options: { excludedPeers: PeerId[] }) => Promise<PeerId>
204
+ findCoordinator: (blockId: BlockId, options: { excludedPeers: PeerId[] }) => Promise<PeerId>,
205
+ backoff?: RetryBackoff
118
206
  ): Promise<void> {
119
207
  // Root-map ensures retries are recorded on the original batch to avoid deep trees
120
208
  const rootOf = new WeakMap<CoordinatorBatch<TPayload, TResponse>, CoordinatorBatch<TPayload, TResponse>>();
121
209
  for (const b of batches) rootOf.set(b, b);
122
210
 
211
+ // Re-batch a failed batch onto the next coordinators and process them, after the backoff's wait.
212
+ // The wait is ONE for the whole retry set, decided once its coordinators are known; a wait that
213
+ // would reach the expiration sends no retry, leaving the batch's original failure as its outcome.
214
+ const retry = async (batch: CoordinatorBatch<TPayload, TResponse>, outstandingMs: number) => {
215
+ const excludedPeers = [batch.peerId, ...(batch.excludedPeers ?? [])];
216
+ const retries = await createBatchesForPayload<TPayload, TResponse>(
217
+ getBlockIds(batch),
218
+ batch.payload,
219
+ getBlockPayload,
220
+ excludedPeers,
221
+ findCoordinator
222
+ );
223
+ if (retries.length === 0) return;
224
+ const waitMs = retryWaitWithinBudget(backoff, {
225
+ attempt: excludedPeers.length,
226
+ failedPeer: batch.peerId,
227
+ outstandingMs,
228
+ retryPeers: retries.map(r => r.peerId)
229
+ }, expiration);
230
+ if (waitMs === undefined) return;
231
+ log('retry peer=%s excluded=%d waitMs=%d outstandingMs=%d', batch.peerId.toString(), excludedPeers.length, waitMs, outstandingMs);
232
+ if (waitMs > 0) await retrySleep(backoff)(waitMs);
233
+ if (expiration <= Date.now()) return;
234
+ const root = rootOf.get(batch) ?? batch;
235
+ root.subsumedBy = [...(root.subsumedBy ?? []), ...retries];
236
+ for (const r of retries) rootOf.set(r, root);
237
+ batch.rebatched = true;
238
+ // Process retries, but ensure further failures also attach to the same root
239
+ await processSet(retries);
240
+ };
241
+
123
242
  // Process a set of batches concurrently and enqueue retries flatly onto the root's subsumedBy list
124
- const processSet = async (set: CoordinatorBatch<TPayload, TResponse>[]) => {
243
+ const processSet = async (set: CoordinatorBatch<TPayload, TResponse>[]): Promise<void> => {
125
244
  await Promise.all(set.map(async (batch) => {
245
+ const sentAt = Date.now();
126
246
  batch.request = new Pending(process(batch)
127
247
  .catch(async e => {
248
+ batch.outstandingMs = Date.now() - sentAt;
128
249
  // Always rethrow the ORIGINAL first-attempt error `e` so batch.request.error
129
250
  // preserves the root cause. If retry setup itself fails (e.g., findCoordinator
130
251
  // throws "self-exhausted" on a solo node), that retry error MUST NOT mask `e`.
131
252
  if (expiration > Date.now()) {
132
- const excludedPeers = [batch.peerId, ...(batch.excludedPeers ?? [])];
133
- log('retry peer=%s excluded=%d', batch.peerId.toString(), excludedPeers.length);
134
253
  try {
135
- const retries = await createBatchesForPayload<TPayload, TResponse>(
136
- getBlockIds(batch),
137
- batch.payload,
138
- getBlockPayload,
139
- excludedPeers,
140
- findCoordinator
141
- );
142
- if (retries.length > 0 && expiration > Date.now()) {
143
- const root = rootOf.get(batch) ?? batch;
144
- root.subsumedBy = [...(root.subsumedBy ?? []), ...retries];
145
- for (const r of retries) rootOf.set(r, root);
146
- // Process retries, but ensure further failures also attach to the same root
147
- await processSet(retries);
148
- }
254
+ await retry(batch, batch.outstandingMs);
149
255
  } catch (retryErr) {
150
256
  log('retry:setup-failed peer=%s original=%o retry=%o',
151
257
  batch.peerId.toString(), e, retryErr);
@@ -13,3 +13,8 @@ export function armUnrefTimer(fn: () => void, ms: number): () => void {
13
13
  const handle = unrefTimer(setTimeout(fn, ms));
14
14
  return (): void => clearTimeout(handle);
15
15
  }
16
+
17
+ /** Resolve after `ms` on a timer that never holds a process open. The default behind the injectable `sleep` seams. */
18
+ export function unrefDelay(ms: number): Promise<void> {
19
+ return new Promise<void>(resolve => { armUnrefTimer(resolve, ms); });
20
+ }