@optimystic/db-core 1.5.0 → 1.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/README.md +336 -336
  2. package/dist/src/collection/collection.d.ts +44 -4
  3. package/dist/src/collection/collection.d.ts.map +1 -1
  4. package/dist/src/collection/collection.js +95 -14
  5. package/dist/src/collection/collection.js.map +1 -1
  6. package/dist/src/log/log.d.ts +41 -8
  7. package/dist/src/log/log.d.ts.map +1 -1
  8. package/dist/src/log/log.js +15 -10
  9. package/dist/src/log/log.js.map +1 -1
  10. package/dist/src/testing/test-transactor.d.ts +36 -0
  11. package/dist/src/testing/test-transactor.d.ts.map +1 -1
  12. package/dist/src/testing/test-transactor.js +49 -0
  13. package/dist/src/testing/test-transactor.js.map +1 -1
  14. package/dist/src/transaction/coordinator.d.ts.map +1 -1
  15. package/dist/src/transaction/coordinator.js +26 -9
  16. package/dist/src/transaction/coordinator.js.map +1 -1
  17. package/package.json +1 -1
  18. package/src/blocks/block-store.ts +31 -31
  19. package/src/blocks/block-types.ts +11 -11
  20. package/src/blocks/index.ts +5 -5
  21. package/src/blocks/structs.ts +17 -17
  22. package/src/btree/btree.ts +872 -872
  23. package/src/btree/independent-trunk.ts +53 -53
  24. package/src/btree/index.ts +5 -5
  25. package/src/btree/key-range.ts +15 -15
  26. package/src/btree/keyset.ts +6 -6
  27. package/src/btree/nodes.ts +25 -25
  28. package/src/btree/path.ts +37 -37
  29. package/src/btree/tree-block.ts +11 -11
  30. package/src/btree/trunk.ts +14 -14
  31. package/src/chain/chain-nodes.ts +26 -26
  32. package/src/chain/chain.ts +349 -349
  33. package/src/chain/index.ts +3 -3
  34. package/src/cluster/i-cluster.ts +6 -6
  35. package/src/cluster/index.ts +3 -3
  36. package/src/cohort-topic/addressing.ts +120 -120
  37. package/src/cohort-topic/antidos/bootstrap-evidence.ts +106 -106
  38. package/src/cohort-topic/antidos/index.ts +5 -5
  39. package/src/cohort-topic/antidos/rate-limiter.ts +210 -210
  40. package/src/cohort-topic/antidos/replay-guard.ts +146 -146
  41. package/src/cohort-topic/antidos/topic-budget.ts +160 -160
  42. package/src/cohort-topic/antiflood/index.ts +2 -2
  43. package/src/cohort-topic/antiflood/invariants.ts +108 -108
  44. package/src/cohort-topic/antiflood/jitter.ts +117 -117
  45. package/src/cohort-topic/coldstart.ts +237 -237
  46. package/src/cohort-topic/dmax.ts +88 -88
  47. package/src/cohort-topic/gossip/bus.ts +254 -254
  48. package/src/cohort-topic/gossip/index.ts +3 -3
  49. package/src/cohort-topic/gossip/records.ts +45 -45
  50. package/src/cohort-topic/gossip/view.ts +91 -91
  51. package/src/cohort-topic/index.ts +20 -20
  52. package/src/cohort-topic/load/barometer.ts +134 -134
  53. package/src/cohort-topic/load/index.ts +1 -1
  54. package/src/cohort-topic/member-engine.ts +430 -430
  55. package/src/cohort-topic/membership/index.ts +3 -3
  56. package/src/cohort-topic/membership/publisher.ts +163 -163
  57. package/src/cohort-topic/membership/source.ts +41 -41
  58. package/src/cohort-topic/ports.ts +157 -157
  59. package/src/cohort-topic/promotion.ts +477 -477
  60. package/src/cohort-topic/registration/bytes.ts +37 -37
  61. package/src/cohort-topic/registration/handoff.ts +154 -154
  62. package/src/cohort-topic/registration/index.ts +6 -6
  63. package/src/cohort-topic/registration/renewal.ts +495 -495
  64. package/src/cohort-topic/registration/sharding.ts +61 -61
  65. package/src/cohort-topic/registration/store.ts +81 -81
  66. package/src/cohort-topic/registration/types.ts +91 -91
  67. package/src/cohort-topic/ring-hash.ts +50 -50
  68. package/src/cohort-topic/service.ts +416 -416
  69. package/src/cohort-topic/sig/index.ts +2 -2
  70. package/src/cohort-topic/sig/payloads.ts +59 -59
  71. package/src/cohort-topic/sig/threshold.ts +64 -64
  72. package/src/cohort-topic/tiers.ts +74 -74
  73. package/src/cohort-topic/traffic.ts +233 -233
  74. package/src/cohort-topic/walk.ts +326 -326
  75. package/src/cohort-topic/willingness.ts +237 -237
  76. package/src/cohort-topic/wire/index.ts +18 -18
  77. package/src/cohort-topic/wire/payloads.ts +126 -126
  78. package/src/collection/action.ts +48 -48
  79. package/src/collection/collection-type-registry.ts +37 -37
  80. package/src/collection/collection.ts +2203 -2118
  81. package/src/collection/index.ts +4 -4
  82. package/src/collections/diary/index.ts +2 -2
  83. package/src/collections/diary/struct.ts +3 -3
  84. package/src/collections/index.ts +2 -2
  85. package/src/collections/tree/collection-trunk.ts +25 -25
  86. package/src/collections/tree/index.ts +2 -2
  87. package/src/collections/tree/readme.md +55 -55
  88. package/src/log/index.ts +2 -2
  89. package/src/log/log.ts +304 -268
  90. package/src/log/struct.ts +165 -165
  91. package/src/matchmaking/capability-filter.ts +45 -45
  92. package/src/matchmaking/config.ts +98 -98
  93. package/src/matchmaking/index.ts +21 -21
  94. package/src/matchmaking/multi-cohort-seeker.ts +234 -234
  95. package/src/matchmaking/provider.ts +123 -123
  96. package/src/matchmaking/query-eval.ts +105 -105
  97. package/src/matchmaking/seeker-walk.ts +127 -127
  98. package/src/matchmaking/seeker.ts +86 -86
  99. package/src/matchmaking/topic-anchor.ts +90 -90
  100. package/src/matchmaking/voting-quorum.ts +394 -394
  101. package/src/network/i-peer-network.ts +25 -25
  102. package/src/network/repo-protocol.ts +13 -13
  103. package/src/network/types.ts +37 -37
  104. package/src/reactivity/backfill.ts +220 -220
  105. package/src/reactivity/backpressure.ts +191 -191
  106. package/src/reactivity/checkpoint.ts +308 -308
  107. package/src/reactivity/config.ts +172 -172
  108. package/src/reactivity/dedupe.ts +132 -132
  109. package/src/reactivity/forwarder.ts +87 -87
  110. package/src/reactivity/index.ts +34 -34
  111. package/src/reactivity/notification.ts +123 -123
  112. package/src/reactivity/policy.ts +79 -79
  113. package/src/reactivity/push-state.ts +310 -310
  114. package/src/reactivity/recover.ts +153 -153
  115. package/src/reactivity/replay-buffer.ts +141 -141
  116. package/src/reactivity/resume.ts +549 -549
  117. package/src/reactivity/rotation.ts +415 -415
  118. package/src/reactivity/subscriber.ts +132 -132
  119. package/src/reactivity/subscription.ts +66 -66
  120. package/src/reactivity/topic-anchor.ts +71 -71
  121. package/src/reactivity/verify.ts +73 -73
  122. package/src/reactivity/wire-validate.ts +13 -13
  123. package/src/testing/async-wait.ts +65 -65
  124. package/src/testing/index.ts +2 -2
  125. package/src/testing/test-transactor.ts +62 -0
  126. package/src/transaction/actions-engine.ts +68 -68
  127. package/src/transaction/coordinator.ts +28 -11
  128. package/src/transaction/index.ts +52 -52
  129. package/src/transaction/operations-hash.ts +196 -196
  130. package/src/transaction/read-dependency-collector.ts +78 -78
  131. package/src/transaction/session.ts +238 -238
  132. package/src/transaction/transaction.ts +442 -442
  133. package/src/transaction/validator.ts +197 -197
  134. package/src/transactor/block-floors.ts +126 -126
  135. package/src/transactor/index.ts +6 -6
  136. package/src/transactor/network-transactor.ts +1545 -1545
  137. package/src/transform/atomic-proxy.ts +92 -92
  138. package/src/transform/cache-source.ts +370 -370
  139. package/src/transform/helpers.ts +164 -164
  140. package/src/transform/index.ts +7 -7
  141. package/src/transform/struct.ts +21 -21
  142. package/src/utility/actor.ts +62 -62
  143. package/src/utility/backoff.ts +95 -95
  144. package/src/utility/batch-coordinator.ts +191 -191
  145. package/src/utility/ensured.ts +32 -32
  146. package/src/utility/groupby.ts +18 -18
  147. package/src/utility/hash-string.ts +14 -14
  148. package/src/utility/is-record-empty.ts +5 -5
  149. package/src/utility/latches.ts +42 -42
  150. package/src/utility/lru-map.ts +60 -60
  151. package/src/utility/nameof.ts +7 -7
  152. package/src/utility/pending.ts +43 -43
@@ -1,95 +1,95 @@
1
- import { randomBytes } from '@noble/hashes/utils.js';
2
-
3
- /**
4
- * Jittered exponential backoff shared by the retry loops in db-core
5
- * ({@link Collection.sync} and {@link TransactionCoordinator.commit}).
6
- *
7
- * The point of the jitter is fairness under contention: when many clients lose the SAME
8
- * optimistic-concurrency race at t=0, a bare exponential curve makes them all re-attempt at the
9
- * identical next tick — a thundering herd that just re-collides. Multiplying the exponential value
10
- * by a random factor spreads those re-attempts across a window so offered load per tick sheds
11
- * instead of cascading. This is the same shape as `cohort-topic`'s `backoffRetryMs`, with the
12
- * proportional-jitter term added.
13
- */
14
-
15
- /** A source of uniform randomness in [0, 1). Injected in tests so a deterministic sequence can be
16
- * asserted; production uses {@link cryptoRand}. */
17
- export type RandFn = () => number;
18
-
19
- /** Uniform value in [0, 1) drawn from the same CSPRNG (`@noble/hashes` `randomBytes`) the rest of
20
- * db-core uses — deliberately NOT `Math.random`, so backoff jitter shares the package's randomness
21
- * source rather than introducing a second, weaker one. */
22
- export function cryptoRand(): number {
23
- const b = randomBytes(4);
24
- // Assemble a uint32 (>>> 0 clears the sign bit the shifts would otherwise set), then normalise.
25
- const u = (((b[0]! << 24) | (b[1]! << 16) | (b[2]! << 8) | b[3]!) >>> 0);
26
- return u / 0x1_0000_0000;
27
- }
28
-
29
- export interface JitteredBackoffConfig {
30
- /** Base delay (ms) for attempt 0, before jitter. */
31
- baseMs: number;
32
- /** Hard ceiling (ms) on the pre-jitter exponential value. */
33
- capMs: number;
34
- /** Geometric growth per attempt. Default 2 (doubling). */
35
- factor?: number;
36
- /** Fraction of the exponential value that jitter may subtract, in [0, 1]. Default 0.5, so the
37
- * returned delay lands uniformly in `((1 - jitterFraction) · exp, exp]` — i.e. `(0.5·exp, exp]`. */
38
- jitterFraction?: number;
39
- }
40
-
41
- /**
42
- * Delay (ms) for retry `attempt` (0-based):
43
- *
44
- * exp = min(baseMs · factor^attempt, capMs)
45
- * delay = exp · (1 - jitterFraction · rand()) // rand() ∈ [0, 1)
46
- *
47
- * With the defaults (factor 2, jitterFraction 0.5) the delay is uniform in `(0.5·exp, exp]`: never
48
- * zero (as long as `baseMs > 0`) and never above `capMs`. `attempt` must be a non-negative integer
49
- * — pass `consecutiveFailures - 1` so the first retry backs off by ~`baseMs`.
50
- */
51
- export function jitteredBackoffMs(
52
- attempt: number,
53
- config: JitteredBackoffConfig,
54
- rand: RandFn = cryptoRand,
55
- ): number {
56
- if (!Number.isInteger(attempt) || attempt < 0) {
57
- throw new RangeError(`attempt must be a non-negative integer, got ${attempt}`);
58
- }
59
- const factor = config.factor ?? 2;
60
- const jitterFraction = config.jitterFraction ?? 0.5;
61
- const exp = Math.min(config.baseMs * factor ** attempt, config.capMs);
62
- return exp * (1 - jitterFraction * rand());
63
- }
64
-
65
- /** Build an AbortError for a cooperatively-aborted wait. Prefers the signal's own reason when it is
66
- * an Error (so callers who passed a custom abort reason see it), otherwise a `name='AbortError'`
67
- * Error. */
68
- export function makeAbortError(signal?: AbortSignal): Error {
69
- if (signal && signal.reason instanceof Error) {
70
- return signal.reason;
71
- }
72
- const err = new Error('The operation was aborted');
73
- err.name = 'AbortError';
74
- return err;
75
- }
76
-
77
- /** Sleep for `ms`, resolving early (rejecting with an AbortError) if `signal` aborts — so a retry
78
- * loop waiting out a backoff rejects promptly instead of finishing the sleep. */
79
- export function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
80
- return new Promise<void>((resolve, reject) => {
81
- if (signal?.aborted) {
82
- reject(makeAbortError(signal));
83
- return;
84
- }
85
- const onAbort = () => {
86
- clearTimeout(timer);
87
- reject(makeAbortError(signal!));
88
- };
89
- const timer = setTimeout(() => {
90
- signal?.removeEventListener('abort', onAbort);
91
- resolve();
92
- }, ms);
93
- signal?.addEventListener('abort', onAbort, { once: true });
94
- });
95
- }
1
+ import { randomBytes } from '@noble/hashes/utils.js';
2
+
3
+ /**
4
+ * Jittered exponential backoff shared by the retry loops in db-core
5
+ * ({@link Collection.sync} and {@link TransactionCoordinator.commit}).
6
+ *
7
+ * The point of the jitter is fairness under contention: when many clients lose the SAME
8
+ * optimistic-concurrency race at t=0, a bare exponential curve makes them all re-attempt at the
9
+ * identical next tick — a thundering herd that just re-collides. Multiplying the exponential value
10
+ * by a random factor spreads those re-attempts across a window so offered load per tick sheds
11
+ * instead of cascading. This is the same shape as `cohort-topic`'s `backoffRetryMs`, with the
12
+ * proportional-jitter term added.
13
+ */
14
+
15
+ /** A source of uniform randomness in [0, 1). Injected in tests so a deterministic sequence can be
16
+ * asserted; production uses {@link cryptoRand}. */
17
+ export type RandFn = () => number;
18
+
19
+ /** Uniform value in [0, 1) drawn from the same CSPRNG (`@noble/hashes` `randomBytes`) the rest of
20
+ * db-core uses — deliberately NOT `Math.random`, so backoff jitter shares the package's randomness
21
+ * source rather than introducing a second, weaker one. */
22
+ export function cryptoRand(): number {
23
+ const b = randomBytes(4);
24
+ // Assemble a uint32 (>>> 0 clears the sign bit the shifts would otherwise set), then normalise.
25
+ const u = (((b[0]! << 24) | (b[1]! << 16) | (b[2]! << 8) | b[3]!) >>> 0);
26
+ return u / 0x1_0000_0000;
27
+ }
28
+
29
+ export interface JitteredBackoffConfig {
30
+ /** Base delay (ms) for attempt 0, before jitter. */
31
+ baseMs: number;
32
+ /** Hard ceiling (ms) on the pre-jitter exponential value. */
33
+ capMs: number;
34
+ /** Geometric growth per attempt. Default 2 (doubling). */
35
+ factor?: number;
36
+ /** Fraction of the exponential value that jitter may subtract, in [0, 1]. Default 0.5, so the
37
+ * returned delay lands uniformly in `((1 - jitterFraction) · exp, exp]` — i.e. `(0.5·exp, exp]`. */
38
+ jitterFraction?: number;
39
+ }
40
+
41
+ /**
42
+ * Delay (ms) for retry `attempt` (0-based):
43
+ *
44
+ * exp = min(baseMs · factor^attempt, capMs)
45
+ * delay = exp · (1 - jitterFraction · rand()) // rand() ∈ [0, 1)
46
+ *
47
+ * With the defaults (factor 2, jitterFraction 0.5) the delay is uniform in `(0.5·exp, exp]`: never
48
+ * zero (as long as `baseMs > 0`) and never above `capMs`. `attempt` must be a non-negative integer
49
+ * — pass `consecutiveFailures - 1` so the first retry backs off by ~`baseMs`.
50
+ */
51
+ export function jitteredBackoffMs(
52
+ attempt: number,
53
+ config: JitteredBackoffConfig,
54
+ rand: RandFn = cryptoRand,
55
+ ): number {
56
+ if (!Number.isInteger(attempt) || attempt < 0) {
57
+ throw new RangeError(`attempt must be a non-negative integer, got ${attempt}`);
58
+ }
59
+ const factor = config.factor ?? 2;
60
+ const jitterFraction = config.jitterFraction ?? 0.5;
61
+ const exp = Math.min(config.baseMs * factor ** attempt, config.capMs);
62
+ return exp * (1 - jitterFraction * rand());
63
+ }
64
+
65
+ /** Build an AbortError for a cooperatively-aborted wait. Prefers the signal's own reason when it is
66
+ * an Error (so callers who passed a custom abort reason see it), otherwise a `name='AbortError'`
67
+ * Error. */
68
+ export function makeAbortError(signal?: AbortSignal): Error {
69
+ if (signal && signal.reason instanceof Error) {
70
+ return signal.reason;
71
+ }
72
+ const err = new Error('The operation was aborted');
73
+ err.name = 'AbortError';
74
+ return err;
75
+ }
76
+
77
+ /** Sleep for `ms`, resolving early (rejecting with an AbortError) if `signal` aborts — so a retry
78
+ * loop waiting out a backoff rejects promptly instead of finishing the sleep. */
79
+ export function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
80
+ return new Promise<void>((resolve, reject) => {
81
+ if (signal?.aborted) {
82
+ reject(makeAbortError(signal));
83
+ return;
84
+ }
85
+ const onAbort = () => {
86
+ clearTimeout(timer);
87
+ reject(makeAbortError(signal!));
88
+ };
89
+ const timer = setTimeout(() => {
90
+ signal?.removeEventListener('abort', onAbort);
91
+ resolve();
92
+ }, ms);
93
+ signal?.addEventListener('abort', onAbort, { once: true });
94
+ });
95
+ }
@@ -1,191 +1,191 @@
1
- import type { PeerId } from "../network/types.js";
2
- import type { BlockId } from "../index.js";
3
- import { Pending } from "./pending.js";
4
- import { createLogger } from "../logger.js";
5
-
6
- const log = createLogger('batch-coordinator');
7
-
8
- /**
9
- * Represents a batch of operations for a specific block coordinated by a peer
10
- */
11
- export type CoordinatorBatch<TPayload, TResponse> = {
12
- peerId: PeerId;
13
- blockId: BlockId;
14
- payload: TPayload;
15
- request?: Pending<TResponse>;
16
- /** Whether this batch has been subsumed by other successful batches */
17
- subsumedBy?: CoordinatorBatch<TPayload, TResponse>[];
18
- /** Peers that have already been tried (and failed) */
19
- excludedPeers?: PeerId[];
20
- /** When a batch coordinates multiple blocks (via cluster intersection), lists all block IDs */
21
- coordinatingBlockIds?: BlockId[];
22
- }
23
-
24
- /**
25
- * Creates batches for a given payload, grouped by the coordinating peer for each block id
26
- */
27
- export function makeBatchesByPeer<TPayload, TResponse>(
28
- blockPeers: (readonly [BlockId, PeerId])[],
29
- payload: TPayload,
30
- getBlockPayload: (payload: TPayload, blockId: BlockId, mergeWithPayload: TPayload | undefined) => TPayload,
31
- excludedPeers?: PeerId[]
32
- ): CoordinatorBatch<TPayload, TResponse>[] {
33
- const groups = blockPeers.reduce((acc, [blockId, peerId]) => {
34
- const peerId_str = peerId.toString();
35
- const coordinator = acc.get(peerId_str) ?? { peerId, blockId, excludedPeers } as Partial<CoordinatorBatch<TPayload, TResponse>>;
36
- acc.set(peerId_str, { ...coordinator, payload: getBlockPayload(payload, blockId, coordinator.payload) } as CoordinatorBatch<TPayload, TResponse>);
37
- return acc;
38
- }, new Map<string, CoordinatorBatch<TPayload, TResponse>>());
39
- return Array.from(groups.values());
40
- }
41
-
42
- /**
43
- * Iterates over all batches that have not completed, whether subsumed or not
44
- */
45
- export function* incompleteBatches<TPayload, TResponse>(batches: CoordinatorBatch<TPayload, TResponse>[]): IterableIterator<CoordinatorBatch<TPayload, TResponse>> {
46
- const stack: CoordinatorBatch<TPayload, TResponse>[] = [...batches];
47
- while (stack.length > 0) {
48
- const batch = stack.pop()!;
49
- if (!batch.request || !batch.request.isResponse) {
50
- yield batch;
51
- }
52
- if (batch.subsumedBy && batch.subsumedBy.length) {
53
- stack.push(...batch.subsumedBy);
54
- }
55
- }
56
- }
57
-
58
- /**
59
- * Checks if all completed batches (ignoring failures) satisfy a predicate
60
- */
61
- export function everyBatch<TPayload, TResponse>(batches: CoordinatorBatch<TPayload, TResponse>[], predicate: (batch: CoordinatorBatch<TPayload, TResponse>) => boolean): boolean {
62
- // For each root batch require that SOME node in its retry tree satisfies the predicate.
63
- // Use iterative DFS to avoid recursion depth and minimize allocations.
64
- for (const root of batches) {
65
- let found = false;
66
- const stack: CoordinatorBatch<TPayload, TResponse>[] = [root];
67
- while (stack.length > 0) {
68
- const node = stack.pop()!;
69
- if (predicate(node)) { found = true; break; }
70
- if (node.subsumedBy && node.subsumedBy.length) {
71
- for (let i = 0; i < node.subsumedBy.length; i++) stack.push(node.subsumedBy[i]!);
72
- }
73
- }
74
- if (!found) return false;
75
- }
76
- return true;
77
- }
78
-
79
- /**
80
- * Iterates over all batches that satisfy an optional predicate, whether subsumed or not
81
- */
82
- export function* allBatches<TPayload, TResponse>(batches: CoordinatorBatch<TPayload, TResponse>[], predicate?: (batch: CoordinatorBatch<TPayload, TResponse>) => boolean): IterableIterator<CoordinatorBatch<TPayload, TResponse>> {
83
- const stack: CoordinatorBatch<TPayload, TResponse>[] = [...batches];
84
- while (stack.length > 0) {
85
- const batch = stack.pop()!;
86
- if (!predicate || predicate(batch)) {
87
- yield batch;
88
- }
89
- if (batch.subsumedBy && batch.subsumedBy.length) {
90
- stack.push(...batch.subsumedBy);
91
- }
92
- }
93
- }
94
-
95
- /**
96
- * Returns a new blockId list payload with the given block id appended
97
- */
98
- export function mergeBlocks(_payload: BlockId[], blockId: BlockId, mergeWithPayload: BlockId[] | undefined): BlockId[] {
99
- return [...(mergeWithPayload ?? []), blockId];
100
- }
101
-
102
- /**
103
- * Processes a set of batches, retrying any failures until success or expiration
104
- * @param batches - The batches to process - each represents a group of blocks centered on a coordinating peer
105
- * @param process - The function to call for a given batch
106
- * @param getBlockIds - The function to call to get the block ids for a given batch
107
- * @param getBlockPayload - The function to call to get the payload given a parent payload and block id, and optionally merge with an existing payload
108
- * @param expiration - The expiration time for the operation
109
- * @param findCoordinator - The function to call to find a coordinator for a block id
110
- */
111
- export async function processBatches<TPayload, TResponse>(
112
- batches: CoordinatorBatch<TPayload, TResponse>[],
113
- process: (batch: CoordinatorBatch<TPayload, TResponse>) => Promise<TResponse>,
114
- getBlockIds: (batch: CoordinatorBatch<TPayload, TResponse>) => BlockId[],
115
- getBlockPayload: (payload: TPayload, blockId: BlockId, mergeWithPayload: TPayload | undefined) => TPayload,
116
- expiration: number,
117
- findCoordinator: (blockId: BlockId, options: { excludedPeers: PeerId[] }) => Promise<PeerId>
118
- ): Promise<void> {
119
- // Root-map ensures retries are recorded on the original batch to avoid deep trees
120
- const rootOf = new WeakMap<CoordinatorBatch<TPayload, TResponse>, CoordinatorBatch<TPayload, TResponse>>();
121
- for (const b of batches) rootOf.set(b, b);
122
-
123
- // Process a set of batches concurrently and enqueue retries flatly onto the root's subsumedBy list
124
- const processSet = async (set: CoordinatorBatch<TPayload, TResponse>[]) => {
125
- await Promise.all(set.map(async (batch) => {
126
- batch.request = new Pending(process(batch)
127
- .catch(async e => {
128
- // Always rethrow the ORIGINAL first-attempt error `e` so batch.request.error
129
- // preserves the root cause. If retry setup itself fails (e.g., findCoordinator
130
- // throws "self-exhausted" on a solo node), that retry error MUST NOT mask `e`.
131
- if (expiration > Date.now()) {
132
- const excludedPeers = [batch.peerId, ...(batch.excludedPeers ?? [])];
133
- log('retry peer=%s excluded=%d', batch.peerId.toString(), excludedPeers.length);
134
- try {
135
- const retries = await createBatchesForPayload<TPayload, TResponse>(
136
- getBlockIds(batch),
137
- batch.payload,
138
- getBlockPayload,
139
- excludedPeers,
140
- findCoordinator
141
- );
142
- if (retries.length > 0 && expiration > Date.now()) {
143
- const root = rootOf.get(batch) ?? batch;
144
- root.subsumedBy = [...(root.subsumedBy ?? []), ...retries];
145
- for (const r of retries) rootOf.set(r, root);
146
- // Process retries, but ensure further failures also attach to the same root
147
- await processSet(retries);
148
- }
149
- } catch (retryErr) {
150
- log('retry:setup-failed peer=%s original=%o retry=%o',
151
- batch.peerId.toString(), e, retryErr);
152
- // Swallow retryErr; the original `e` is authoritative.
153
- }
154
- }
155
- throw e;
156
- }));
157
- }));
158
-
159
- // Wait for all in this set to settle
160
- await Promise.all(set.map(b => b.request?.result().catch(() => { /* ignore */ })));
161
- };
162
-
163
- await processSet(batches);
164
- }
165
-
166
- /**
167
- * Creates batches for a given payload, grouped by the coordinating peer for each block id
168
- * This is a placeholder function that will be implemented by the caller
169
- */
170
- export async function createBatchesForPayload<TPayload, TResponse>(
171
- blockIds: BlockId[],
172
- payload: TPayload,
173
- getBlockPayload: (payload: TPayload, blockId: BlockId, mergeWithPayload: TPayload | undefined) => TPayload,
174
- excludedPeers: PeerId[],
175
- findCoordinator: (blockId: BlockId, options: { excludedPeers: PeerId[] }) => Promise<PeerId>
176
- ): Promise<CoordinatorBatch<TPayload, TResponse>[]> {
177
- // Group by block id
178
- const distinctBlockIds = new Set(blockIds);
179
-
180
- // Find coordinator for each key
181
- const blockIdPeerId = await Promise.all(
182
- Array.from(distinctBlockIds).map(async (bid) =>
183
- [bid, await findCoordinator(bid, { excludedPeers })] as const
184
- )
185
- );
186
-
187
- // Group blocks around their coordinating peers
188
- const batches = makeBatchesByPeer<TPayload, TResponse>(blockIdPeerId, payload, getBlockPayload, excludedPeers);
189
- log('createBatches blockIds=%d batches=%d excluded=%d', distinctBlockIds.size, batches.length, excludedPeers.length);
190
- return batches;
191
- }
1
+ import type { PeerId } from "../network/types.js";
2
+ import type { BlockId } from "../index.js";
3
+ import { Pending } from "./pending.js";
4
+ import { createLogger } from "../logger.js";
5
+
6
+ const log = createLogger('batch-coordinator');
7
+
8
+ /**
9
+ * Represents a batch of operations for a specific block coordinated by a peer
10
+ */
11
+ export type CoordinatorBatch<TPayload, TResponse> = {
12
+ peerId: PeerId;
13
+ blockId: BlockId;
14
+ payload: TPayload;
15
+ request?: Pending<TResponse>;
16
+ /** Whether this batch has been subsumed by other successful batches */
17
+ subsumedBy?: CoordinatorBatch<TPayload, TResponse>[];
18
+ /** Peers that have already been tried (and failed) */
19
+ excludedPeers?: PeerId[];
20
+ /** When a batch coordinates multiple blocks (via cluster intersection), lists all block IDs */
21
+ coordinatingBlockIds?: BlockId[];
22
+ }
23
+
24
+ /**
25
+ * Creates batches for a given payload, grouped by the coordinating peer for each block id
26
+ */
27
+ export function makeBatchesByPeer<TPayload, TResponse>(
28
+ blockPeers: (readonly [BlockId, PeerId])[],
29
+ payload: TPayload,
30
+ getBlockPayload: (payload: TPayload, blockId: BlockId, mergeWithPayload: TPayload | undefined) => TPayload,
31
+ excludedPeers?: PeerId[]
32
+ ): CoordinatorBatch<TPayload, TResponse>[] {
33
+ const groups = blockPeers.reduce((acc, [blockId, peerId]) => {
34
+ const peerId_str = peerId.toString();
35
+ const coordinator = acc.get(peerId_str) ?? { peerId, blockId, excludedPeers } as Partial<CoordinatorBatch<TPayload, TResponse>>;
36
+ acc.set(peerId_str, { ...coordinator, payload: getBlockPayload(payload, blockId, coordinator.payload) } as CoordinatorBatch<TPayload, TResponse>);
37
+ return acc;
38
+ }, new Map<string, CoordinatorBatch<TPayload, TResponse>>());
39
+ return Array.from(groups.values());
40
+ }
41
+
42
+ /**
43
+ * Iterates over all batches that have not completed, whether subsumed or not
44
+ */
45
+ export function* incompleteBatches<TPayload, TResponse>(batches: CoordinatorBatch<TPayload, TResponse>[]): IterableIterator<CoordinatorBatch<TPayload, TResponse>> {
46
+ const stack: CoordinatorBatch<TPayload, TResponse>[] = [...batches];
47
+ while (stack.length > 0) {
48
+ const batch = stack.pop()!;
49
+ if (!batch.request || !batch.request.isResponse) {
50
+ yield batch;
51
+ }
52
+ if (batch.subsumedBy && batch.subsumedBy.length) {
53
+ stack.push(...batch.subsumedBy);
54
+ }
55
+ }
56
+ }
57
+
58
+ /**
59
+ * Checks if all completed batches (ignoring failures) satisfy a predicate
60
+ */
61
+ export function everyBatch<TPayload, TResponse>(batches: CoordinatorBatch<TPayload, TResponse>[], predicate: (batch: CoordinatorBatch<TPayload, TResponse>) => boolean): boolean {
62
+ // For each root batch require that SOME node in its retry tree satisfies the predicate.
63
+ // Use iterative DFS to avoid recursion depth and minimize allocations.
64
+ for (const root of batches) {
65
+ let found = false;
66
+ const stack: CoordinatorBatch<TPayload, TResponse>[] = [root];
67
+ while (stack.length > 0) {
68
+ const node = stack.pop()!;
69
+ if (predicate(node)) { found = true; break; }
70
+ if (node.subsumedBy && node.subsumedBy.length) {
71
+ for (let i = 0; i < node.subsumedBy.length; i++) stack.push(node.subsumedBy[i]!);
72
+ }
73
+ }
74
+ if (!found) return false;
75
+ }
76
+ return true;
77
+ }
78
+
79
+ /**
80
+ * Iterates over all batches that satisfy an optional predicate, whether subsumed or not
81
+ */
82
+ export function* allBatches<TPayload, TResponse>(batches: CoordinatorBatch<TPayload, TResponse>[], predicate?: (batch: CoordinatorBatch<TPayload, TResponse>) => boolean): IterableIterator<CoordinatorBatch<TPayload, TResponse>> {
83
+ const stack: CoordinatorBatch<TPayload, TResponse>[] = [...batches];
84
+ while (stack.length > 0) {
85
+ const batch = stack.pop()!;
86
+ if (!predicate || predicate(batch)) {
87
+ yield batch;
88
+ }
89
+ if (batch.subsumedBy && batch.subsumedBy.length) {
90
+ stack.push(...batch.subsumedBy);
91
+ }
92
+ }
93
+ }
94
+
95
+ /**
96
+ * Returns a new blockId list payload with the given block id appended
97
+ */
98
+ export function mergeBlocks(_payload: BlockId[], blockId: BlockId, mergeWithPayload: BlockId[] | undefined): BlockId[] {
99
+ return [...(mergeWithPayload ?? []), blockId];
100
+ }
101
+
102
+ /**
103
+ * Processes a set of batches, retrying any failures until success or expiration
104
+ * @param batches - The batches to process - each represents a group of blocks centered on a coordinating peer
105
+ * @param process - The function to call for a given batch
106
+ * @param getBlockIds - The function to call to get the block ids for a given batch
107
+ * @param getBlockPayload - The function to call to get the payload given a parent payload and block id, and optionally merge with an existing payload
108
+ * @param expiration - The expiration time for the operation
109
+ * @param findCoordinator - The function to call to find a coordinator for a block id
110
+ */
111
+ export async function processBatches<TPayload, TResponse>(
112
+ batches: CoordinatorBatch<TPayload, TResponse>[],
113
+ process: (batch: CoordinatorBatch<TPayload, TResponse>) => Promise<TResponse>,
114
+ getBlockIds: (batch: CoordinatorBatch<TPayload, TResponse>) => BlockId[],
115
+ getBlockPayload: (payload: TPayload, blockId: BlockId, mergeWithPayload: TPayload | undefined) => TPayload,
116
+ expiration: number,
117
+ findCoordinator: (blockId: BlockId, options: { excludedPeers: PeerId[] }) => Promise<PeerId>
118
+ ): Promise<void> {
119
+ // Root-map ensures retries are recorded on the original batch to avoid deep trees
120
+ const rootOf = new WeakMap<CoordinatorBatch<TPayload, TResponse>, CoordinatorBatch<TPayload, TResponse>>();
121
+ for (const b of batches) rootOf.set(b, b);
122
+
123
+ // Process a set of batches concurrently and enqueue retries flatly onto the root's subsumedBy list
124
+ const processSet = async (set: CoordinatorBatch<TPayload, TResponse>[]) => {
125
+ await Promise.all(set.map(async (batch) => {
126
+ batch.request = new Pending(process(batch)
127
+ .catch(async e => {
128
+ // Always rethrow the ORIGINAL first-attempt error `e` so batch.request.error
129
+ // preserves the root cause. If retry setup itself fails (e.g., findCoordinator
130
+ // throws "self-exhausted" on a solo node), that retry error MUST NOT mask `e`.
131
+ if (expiration > Date.now()) {
132
+ const excludedPeers = [batch.peerId, ...(batch.excludedPeers ?? [])];
133
+ log('retry peer=%s excluded=%d', batch.peerId.toString(), excludedPeers.length);
134
+ try {
135
+ const retries = await createBatchesForPayload<TPayload, TResponse>(
136
+ getBlockIds(batch),
137
+ batch.payload,
138
+ getBlockPayload,
139
+ excludedPeers,
140
+ findCoordinator
141
+ );
142
+ if (retries.length > 0 && expiration > Date.now()) {
143
+ const root = rootOf.get(batch) ?? batch;
144
+ root.subsumedBy = [...(root.subsumedBy ?? []), ...retries];
145
+ for (const r of retries) rootOf.set(r, root);
146
+ // Process retries, but ensure further failures also attach to the same root
147
+ await processSet(retries);
148
+ }
149
+ } catch (retryErr) {
150
+ log('retry:setup-failed peer=%s original=%o retry=%o',
151
+ batch.peerId.toString(), e, retryErr);
152
+ // Swallow retryErr; the original `e` is authoritative.
153
+ }
154
+ }
155
+ throw e;
156
+ }));
157
+ }));
158
+
159
+ // Wait for all in this set to settle
160
+ await Promise.all(set.map(b => b.request?.result().catch(() => { /* ignore */ })));
161
+ };
162
+
163
+ await processSet(batches);
164
+ }
165
+
166
+ /**
167
+ * Creates batches for a given payload, grouped by the coordinating peer for each block id
168
+ * This is a placeholder function that will be implemented by the caller
169
+ */
170
+ export async function createBatchesForPayload<TPayload, TResponse>(
171
+ blockIds: BlockId[],
172
+ payload: TPayload,
173
+ getBlockPayload: (payload: TPayload, blockId: BlockId, mergeWithPayload: TPayload | undefined) => TPayload,
174
+ excludedPeers: PeerId[],
175
+ findCoordinator: (blockId: BlockId, options: { excludedPeers: PeerId[] }) => Promise<PeerId>
176
+ ): Promise<CoordinatorBatch<TPayload, TResponse>[]> {
177
+ // Group by block id
178
+ const distinctBlockIds = new Set(blockIds);
179
+
180
+ // Find coordinator for each key
181
+ const blockIdPeerId = await Promise.all(
182
+ Array.from(distinctBlockIds).map(async (bid) =>
183
+ [bid, await findCoordinator(bid, { excludedPeers })] as const
184
+ )
185
+ );
186
+
187
+ // Group blocks around their coordinating peers
188
+ const batches = makeBatchesByPeer<TPayload, TResponse>(blockIdPeerId, payload, getBlockPayload, excludedPeers);
189
+ log('createBatches blockIds=%d batches=%d excluded=%d', distinctBlockIds.size, batches.length, excludedPeers.length);
190
+ return batches;
191
+ }
@@ -1,32 +1,32 @@
1
- // Retrieves a value from a record, generating an entry if none exists
2
- export function ensured<K extends string | number | symbol, V>(
3
- map: Record<K, V>,
4
- key: K,
5
- makeNew: () => Exclude<V, undefined>,
6
- existing?: (existing: Exclude<V, undefined>) => void
7
- ): Exclude<V, undefined> {
8
- let v = map[key];
9
- if (typeof v === 'undefined') {
10
- v = makeNew();
11
- map[key] = v;
12
- } else if (existing) {
13
- existing(v as Exclude<V, undefined>);
14
- }
15
- return v as Exclude<V, undefined>;
16
- }
17
-
18
- export function ensuredMap<K extends string | number | symbol, V>(
19
- map: Map<K, V>,
20
- key: K,
21
- makeNew: () => Exclude<V, undefined>,
22
- existing?: (existing: Exclude<V, undefined>) => void
23
- ): Exclude<V, undefined> {
24
- let v = map.get(key);
25
- if (typeof v === 'undefined') {
26
- v = makeNew();
27
- map.set(key, v);
28
- } else if (existing) {
29
- existing(v as Exclude<V, undefined>);
30
- }
31
- return v as Exclude<V, undefined>;
32
- }
1
+ // Retrieves a value from a record, generating an entry if none exists
2
+ export function ensured<K extends string | number | symbol, V>(
3
+ map: Record<K, V>,
4
+ key: K,
5
+ makeNew: () => Exclude<V, undefined>,
6
+ existing?: (existing: Exclude<V, undefined>) => void
7
+ ): Exclude<V, undefined> {
8
+ let v = map[key];
9
+ if (typeof v === 'undefined') {
10
+ v = makeNew();
11
+ map[key] = v;
12
+ } else if (existing) {
13
+ existing(v as Exclude<V, undefined>);
14
+ }
15
+ return v as Exclude<V, undefined>;
16
+ }
17
+
18
+ export function ensuredMap<K extends string | number | symbol, V>(
19
+ map: Map<K, V>,
20
+ key: K,
21
+ makeNew: () => Exclude<V, undefined>,
22
+ existing?: (existing: Exclude<V, undefined>) => void
23
+ ): Exclude<V, undefined> {
24
+ let v = map.get(key);
25
+ if (typeof v === 'undefined') {
26
+ v = makeNew();
27
+ map.set(key, v);
28
+ } else if (existing) {
29
+ existing(v as Exclude<V, undefined>);
30
+ }
31
+ return v as Exclude<V, undefined>;
32
+ }