@optimystic/db-core 1.4.0 → 1.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/README.md +336 -336
  2. package/dist/src/collection/collection.d.ts +44 -4
  3. package/dist/src/collection/collection.d.ts.map +1 -1
  4. package/dist/src/collection/collection.js +95 -14
  5. package/dist/src/collection/collection.js.map +1 -1
  6. package/dist/src/collections/tree/struct.d.ts +9 -4
  7. package/dist/src/collections/tree/struct.d.ts.map +1 -1
  8. package/dist/src/collections/tree/struct.js +9 -4
  9. package/dist/src/collections/tree/struct.js.map +1 -1
  10. package/dist/src/log/log.d.ts +41 -8
  11. package/dist/src/log/log.d.ts.map +1 -1
  12. package/dist/src/log/log.js +15 -10
  13. package/dist/src/log/log.js.map +1 -1
  14. package/dist/src/testing/test-transactor.d.ts +36 -0
  15. package/dist/src/testing/test-transactor.d.ts.map +1 -1
  16. package/dist/src/testing/test-transactor.js +49 -0
  17. package/dist/src/testing/test-transactor.js.map +1 -1
  18. package/dist/src/transaction/coordinator.d.ts.map +1 -1
  19. package/dist/src/transaction/coordinator.js +26 -9
  20. package/dist/src/transaction/coordinator.js.map +1 -1
  21. package/package.json +1 -1
  22. package/src/blocks/block-store.ts +31 -31
  23. package/src/blocks/block-types.ts +11 -11
  24. package/src/blocks/index.ts +5 -5
  25. package/src/blocks/structs.ts +17 -17
  26. package/src/btree/btree.ts +872 -872
  27. package/src/btree/independent-trunk.ts +53 -53
  28. package/src/btree/index.ts +5 -5
  29. package/src/btree/key-range.ts +15 -15
  30. package/src/btree/keyset.ts +6 -6
  31. package/src/btree/nodes.ts +25 -25
  32. package/src/btree/path.ts +37 -37
  33. package/src/btree/tree-block.ts +11 -11
  34. package/src/btree/trunk.ts +14 -14
  35. package/src/chain/chain-nodes.ts +26 -26
  36. package/src/chain/chain.ts +349 -349
  37. package/src/chain/index.ts +3 -3
  38. package/src/cluster/i-cluster.ts +6 -6
  39. package/src/cluster/index.ts +3 -3
  40. package/src/cohort-topic/addressing.ts +120 -120
  41. package/src/cohort-topic/antidos/bootstrap-evidence.ts +106 -106
  42. package/src/cohort-topic/antidos/index.ts +5 -5
  43. package/src/cohort-topic/antidos/rate-limiter.ts +210 -210
  44. package/src/cohort-topic/antidos/replay-guard.ts +146 -146
  45. package/src/cohort-topic/antidos/topic-budget.ts +160 -160
  46. package/src/cohort-topic/antiflood/index.ts +2 -2
  47. package/src/cohort-topic/antiflood/invariants.ts +108 -108
  48. package/src/cohort-topic/antiflood/jitter.ts +117 -117
  49. package/src/cohort-topic/coldstart.ts +237 -237
  50. package/src/cohort-topic/dmax.ts +88 -88
  51. package/src/cohort-topic/gossip/bus.ts +254 -254
  52. package/src/cohort-topic/gossip/index.ts +3 -3
  53. package/src/cohort-topic/gossip/records.ts +45 -45
  54. package/src/cohort-topic/gossip/view.ts +91 -91
  55. package/src/cohort-topic/index.ts +20 -20
  56. package/src/cohort-topic/load/barometer.ts +134 -134
  57. package/src/cohort-topic/load/index.ts +1 -1
  58. package/src/cohort-topic/member-engine.ts +430 -430
  59. package/src/cohort-topic/membership/index.ts +3 -3
  60. package/src/cohort-topic/membership/publisher.ts +163 -163
  61. package/src/cohort-topic/membership/source.ts +41 -41
  62. package/src/cohort-topic/ports.ts +157 -157
  63. package/src/cohort-topic/promotion.ts +477 -477
  64. package/src/cohort-topic/registration/bytes.ts +37 -37
  65. package/src/cohort-topic/registration/handoff.ts +154 -154
  66. package/src/cohort-topic/registration/index.ts +6 -6
  67. package/src/cohort-topic/registration/renewal.ts +495 -495
  68. package/src/cohort-topic/registration/sharding.ts +61 -61
  69. package/src/cohort-topic/registration/store.ts +81 -81
  70. package/src/cohort-topic/registration/types.ts +91 -91
  71. package/src/cohort-topic/ring-hash.ts +50 -50
  72. package/src/cohort-topic/service.ts +416 -416
  73. package/src/cohort-topic/sig/index.ts +2 -2
  74. package/src/cohort-topic/sig/payloads.ts +59 -59
  75. package/src/cohort-topic/sig/threshold.ts +64 -64
  76. package/src/cohort-topic/tiers.ts +74 -74
  77. package/src/cohort-topic/traffic.ts +233 -233
  78. package/src/cohort-topic/walk.ts +326 -326
  79. package/src/cohort-topic/willingness.ts +237 -237
  80. package/src/cohort-topic/wire/index.ts +18 -18
  81. package/src/cohort-topic/wire/payloads.ts +126 -126
  82. package/src/collection/action.ts +48 -48
  83. package/src/collection/collection-type-registry.ts +37 -37
  84. package/src/collection/collection.ts +2203 -2118
  85. package/src/collection/index.ts +4 -4
  86. package/src/collections/diary/index.ts +2 -2
  87. package/src/collections/diary/struct.ts +3 -3
  88. package/src/collections/index.ts +2 -2
  89. package/src/collections/tree/collection-trunk.ts +25 -25
  90. package/src/collections/tree/index.ts +2 -2
  91. package/src/collections/tree/readme.md +55 -55
  92. package/src/collections/tree/struct.ts +9 -4
  93. package/src/log/index.ts +2 -2
  94. package/src/log/log.ts +304 -268
  95. package/src/log/struct.ts +165 -165
  96. package/src/matchmaking/capability-filter.ts +45 -45
  97. package/src/matchmaking/config.ts +98 -98
  98. package/src/matchmaking/index.ts +21 -21
  99. package/src/matchmaking/multi-cohort-seeker.ts +234 -234
  100. package/src/matchmaking/provider.ts +123 -123
  101. package/src/matchmaking/query-eval.ts +105 -105
  102. package/src/matchmaking/seeker-walk.ts +127 -127
  103. package/src/matchmaking/seeker.ts +86 -86
  104. package/src/matchmaking/topic-anchor.ts +90 -90
  105. package/src/matchmaking/voting-quorum.ts +394 -394
  106. package/src/network/i-peer-network.ts +25 -25
  107. package/src/network/repo-protocol.ts +13 -13
  108. package/src/network/types.ts +37 -37
  109. package/src/reactivity/backfill.ts +220 -220
  110. package/src/reactivity/backpressure.ts +191 -191
  111. package/src/reactivity/checkpoint.ts +308 -308
  112. package/src/reactivity/config.ts +172 -172
  113. package/src/reactivity/dedupe.ts +132 -132
  114. package/src/reactivity/forwarder.ts +87 -87
  115. package/src/reactivity/index.ts +34 -34
  116. package/src/reactivity/notification.ts +123 -123
  117. package/src/reactivity/policy.ts +79 -79
  118. package/src/reactivity/push-state.ts +310 -310
  119. package/src/reactivity/recover.ts +153 -153
  120. package/src/reactivity/replay-buffer.ts +141 -141
  121. package/src/reactivity/resume.ts +549 -549
  122. package/src/reactivity/rotation.ts +415 -415
  123. package/src/reactivity/subscriber.ts +132 -132
  124. package/src/reactivity/subscription.ts +66 -66
  125. package/src/reactivity/topic-anchor.ts +71 -71
  126. package/src/reactivity/verify.ts +73 -73
  127. package/src/reactivity/wire-validate.ts +13 -13
  128. package/src/testing/async-wait.ts +65 -65
  129. package/src/testing/index.ts +2 -2
  130. package/src/testing/test-transactor.ts +62 -0
  131. package/src/transaction/actions-engine.ts +68 -68
  132. package/src/transaction/coordinator.ts +28 -11
  133. package/src/transaction/index.ts +52 -52
  134. package/src/transaction/operations-hash.ts +196 -196
  135. package/src/transaction/read-dependency-collector.ts +78 -78
  136. package/src/transaction/session.ts +238 -238
  137. package/src/transaction/transaction.ts +442 -442
  138. package/src/transaction/validator.ts +197 -197
  139. package/src/transactor/block-floors.ts +126 -126
  140. package/src/transactor/index.ts +6 -6
  141. package/src/transactor/network-transactor.ts +1545 -1545
  142. package/src/transform/atomic-proxy.ts +92 -92
  143. package/src/transform/cache-source.ts +370 -370
  144. package/src/transform/helpers.ts +164 -164
  145. package/src/transform/index.ts +7 -7
  146. package/src/transform/struct.ts +21 -21
  147. package/src/utility/actor.ts +62 -62
  148. package/src/utility/backoff.ts +95 -95
  149. package/src/utility/batch-coordinator.ts +191 -191
  150. package/src/utility/ensured.ts +32 -32
  151. package/src/utility/groupby.ts +18 -18
  152. package/src/utility/hash-string.ts +14 -14
  153. package/src/utility/is-record-empty.ts +5 -5
  154. package/src/utility/latches.ts +42 -42
  155. package/src/utility/lru-map.ts +60 -60
  156. package/src/utility/nameof.ts +7 -7
  157. package/src/utility/pending.ts +43 -43
@@ -1,95 +1,95 @@
1
- import { randomBytes } from '@noble/hashes/utils.js';
2
-
3
- /**
4
- * Jittered exponential backoff shared by the retry loops in db-core
5
- * ({@link Collection.sync} and {@link TransactionCoordinator.commit}).
6
- *
7
- * The point of the jitter is fairness under contention: when many clients lose the SAME
8
- * optimistic-concurrency race at t=0, a bare exponential curve makes them all re-attempt at the
9
- * identical next tick — a thundering herd that just re-collides. Multiplying the exponential value
10
- * by a random factor spreads those re-attempts across a window so offered load per tick sheds
11
- * instead of cascading. This is the same shape as `cohort-topic`'s `backoffRetryMs`, with the
12
- * proportional-jitter term added.
13
- */
14
-
15
- /** A source of uniform randomness in [0, 1). Injected in tests so a deterministic sequence can be
16
- * asserted; production uses {@link cryptoRand}. */
17
- export type RandFn = () => number;
18
-
19
- /** Uniform value in [0, 1) drawn from the same CSPRNG (`@noble/hashes` `randomBytes`) the rest of
20
- * db-core uses — deliberately NOT `Math.random`, so backoff jitter shares the package's randomness
21
- * source rather than introducing a second, weaker one. */
22
- export function cryptoRand(): number {
23
- const b = randomBytes(4);
24
- // Assemble a uint32 (>>> 0 clears the sign bit the shifts would otherwise set), then normalise.
25
- const u = (((b[0]! << 24) | (b[1]! << 16) | (b[2]! << 8) | b[3]!) >>> 0);
26
- return u / 0x1_0000_0000;
27
- }
28
-
29
- export interface JitteredBackoffConfig {
30
- /** Base delay (ms) for attempt 0, before jitter. */
31
- baseMs: number;
32
- /** Hard ceiling (ms) on the pre-jitter exponential value. */
33
- capMs: number;
34
- /** Geometric growth per attempt. Default 2 (doubling). */
35
- factor?: number;
36
- /** Fraction of the exponential value that jitter may subtract, in [0, 1]. Default 0.5, so the
37
- * returned delay lands uniformly in `((1 - jitterFraction) · exp, exp]` — i.e. `(0.5·exp, exp]`. */
38
- jitterFraction?: number;
39
- }
40
-
41
- /**
42
- * Delay (ms) for retry `attempt` (0-based):
43
- *
44
- * exp = min(baseMs · factor^attempt, capMs)
45
- * delay = exp · (1 - jitterFraction · rand()) // rand() ∈ [0, 1)
46
- *
47
- * With the defaults (factor 2, jitterFraction 0.5) the delay is uniform in `(0.5·exp, exp]`: never
48
- * zero (as long as `baseMs > 0`) and never above `capMs`. `attempt` must be a non-negative integer
49
- * — pass `consecutiveFailures - 1` so the first retry backs off by ~`baseMs`.
50
- */
51
- export function jitteredBackoffMs(
52
- attempt: number,
53
- config: JitteredBackoffConfig,
54
- rand: RandFn = cryptoRand,
55
- ): number {
56
- if (!Number.isInteger(attempt) || attempt < 0) {
57
- throw new RangeError(`attempt must be a non-negative integer, got ${attempt}`);
58
- }
59
- const factor = config.factor ?? 2;
60
- const jitterFraction = config.jitterFraction ?? 0.5;
61
- const exp = Math.min(config.baseMs * factor ** attempt, config.capMs);
62
- return exp * (1 - jitterFraction * rand());
63
- }
64
-
65
- /** Build an AbortError for a cooperatively-aborted wait. Prefers the signal's own reason when it is
66
- * an Error (so callers who passed a custom abort reason see it), otherwise a `name='AbortError'`
67
- * Error. */
68
- export function makeAbortError(signal?: AbortSignal): Error {
69
- if (signal && signal.reason instanceof Error) {
70
- return signal.reason;
71
- }
72
- const err = new Error('The operation was aborted');
73
- err.name = 'AbortError';
74
- return err;
75
- }
76
-
77
- /** Sleep for `ms`, resolving early (rejecting with an AbortError) if `signal` aborts — so a retry
78
- * loop waiting out a backoff rejects promptly instead of finishing the sleep. */
79
- export function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
80
- return new Promise<void>((resolve, reject) => {
81
- if (signal?.aborted) {
82
- reject(makeAbortError(signal));
83
- return;
84
- }
85
- const onAbort = () => {
86
- clearTimeout(timer);
87
- reject(makeAbortError(signal!));
88
- };
89
- const timer = setTimeout(() => {
90
- signal?.removeEventListener('abort', onAbort);
91
- resolve();
92
- }, ms);
93
- signal?.addEventListener('abort', onAbort, { once: true });
94
- });
95
- }
1
+ import { randomBytes } from '@noble/hashes/utils.js';
2
+
3
+ /**
4
+ * Jittered exponential backoff shared by the retry loops in db-core
5
+ * ({@link Collection.sync} and {@link TransactionCoordinator.commit}).
6
+ *
7
+ * The point of the jitter is fairness under contention: when many clients lose the SAME
8
+ * optimistic-concurrency race at t=0, a bare exponential curve makes them all re-attempt at the
9
+ * identical next tick — a thundering herd that just re-collides. Multiplying the exponential value
10
+ * by a random factor spreads those re-attempts across a window so offered load per tick sheds
11
+ * instead of cascading. This is the same shape as `cohort-topic`'s `backoffRetryMs`, with the
12
+ * proportional-jitter term added.
13
+ */
14
+
15
+ /** A source of uniform randomness in [0, 1). Injected in tests so a deterministic sequence can be
16
+ * asserted; production uses {@link cryptoRand}. */
17
+ export type RandFn = () => number;
18
+
19
+ /** Uniform value in [0, 1) drawn from the same CSPRNG (`@noble/hashes` `randomBytes`) the rest of
20
+ * db-core uses — deliberately NOT `Math.random`, so backoff jitter shares the package's randomness
21
+ * source rather than introducing a second, weaker one. */
22
+ export function cryptoRand(): number {
23
+ const b = randomBytes(4);
24
+ // Assemble a uint32 (>>> 0 clears the sign bit the shifts would otherwise set), then normalise.
25
+ const u = (((b[0]! << 24) | (b[1]! << 16) | (b[2]! << 8) | b[3]!) >>> 0);
26
+ return u / 0x1_0000_0000;
27
+ }
28
+
29
+ export interface JitteredBackoffConfig {
30
+ /** Base delay (ms) for attempt 0, before jitter. */
31
+ baseMs: number;
32
+ /** Hard ceiling (ms) on the pre-jitter exponential value. */
33
+ capMs: number;
34
+ /** Geometric growth per attempt. Default 2 (doubling). */
35
+ factor?: number;
36
+ /** Fraction of the exponential value that jitter may subtract, in [0, 1]. Default 0.5, so the
37
+ * returned delay lands uniformly in `((1 - jitterFraction) · exp, exp]` — i.e. `(0.5·exp, exp]`. */
38
+ jitterFraction?: number;
39
+ }
40
+
41
+ /**
42
+ * Delay (ms) for retry `attempt` (0-based):
43
+ *
44
+ * exp = min(baseMs · factor^attempt, capMs)
45
+ * delay = exp · (1 - jitterFraction · rand()) // rand() ∈ [0, 1)
46
+ *
47
+ * With the defaults (factor 2, jitterFraction 0.5) the delay is uniform in `(0.5·exp, exp]`: never
48
+ * zero (as long as `baseMs > 0`) and never above `capMs`. `attempt` must be a non-negative integer
49
+ * — pass `consecutiveFailures - 1` so the first retry backs off by ~`baseMs`.
50
+ */
51
+ export function jitteredBackoffMs(
52
+ attempt: number,
53
+ config: JitteredBackoffConfig,
54
+ rand: RandFn = cryptoRand,
55
+ ): number {
56
+ if (!Number.isInteger(attempt) || attempt < 0) {
57
+ throw new RangeError(`attempt must be a non-negative integer, got ${attempt}`);
58
+ }
59
+ const factor = config.factor ?? 2;
60
+ const jitterFraction = config.jitterFraction ?? 0.5;
61
+ const exp = Math.min(config.baseMs * factor ** attempt, config.capMs);
62
+ return exp * (1 - jitterFraction * rand());
63
+ }
64
+
65
+ /** Build an AbortError for a cooperatively-aborted wait. Prefers the signal's own reason when it is
66
+ * an Error (so callers who passed a custom abort reason see it), otherwise a `name='AbortError'`
67
+ * Error. */
68
+ export function makeAbortError(signal?: AbortSignal): Error {
69
+ if (signal && signal.reason instanceof Error) {
70
+ return signal.reason;
71
+ }
72
+ const err = new Error('The operation was aborted');
73
+ err.name = 'AbortError';
74
+ return err;
75
+ }
76
+
77
+ /** Sleep for `ms`, resolving early (rejecting with an AbortError) if `signal` aborts — so a retry
78
+ * loop waiting out a backoff rejects promptly instead of finishing the sleep. */
79
+ export function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
80
+ return new Promise<void>((resolve, reject) => {
81
+ if (signal?.aborted) {
82
+ reject(makeAbortError(signal));
83
+ return;
84
+ }
85
+ const onAbort = () => {
86
+ clearTimeout(timer);
87
+ reject(makeAbortError(signal!));
88
+ };
89
+ const timer = setTimeout(() => {
90
+ signal?.removeEventListener('abort', onAbort);
91
+ resolve();
92
+ }, ms);
93
+ signal?.addEventListener('abort', onAbort, { once: true });
94
+ });
95
+ }
@@ -1,191 +1,191 @@
1
- import type { PeerId } from "../network/types.js";
2
- import type { BlockId } from "../index.js";
3
- import { Pending } from "./pending.js";
4
- import { createLogger } from "../logger.js";
5
-
6
- const log = createLogger('batch-coordinator');
7
-
8
- /**
9
- * Represents a batch of operations for a specific block coordinated by a peer
10
- */
11
- export type CoordinatorBatch<TPayload, TResponse> = {
12
- peerId: PeerId;
13
- blockId: BlockId;
14
- payload: TPayload;
15
- request?: Pending<TResponse>;
16
- /** Whether this batch has been subsumed by other successful batches */
17
- subsumedBy?: CoordinatorBatch<TPayload, TResponse>[];
18
- /** Peers that have already been tried (and failed) */
19
- excludedPeers?: PeerId[];
20
- /** When a batch coordinates multiple blocks (via cluster intersection), lists all block IDs */
21
- coordinatingBlockIds?: BlockId[];
22
- }
23
-
24
- /**
25
- * Creates batches for a given payload, grouped by the coordinating peer for each block id
26
- */
27
- export function makeBatchesByPeer<TPayload, TResponse>(
28
- blockPeers: (readonly [BlockId, PeerId])[],
29
- payload: TPayload,
30
- getBlockPayload: (payload: TPayload, blockId: BlockId, mergeWithPayload: TPayload | undefined) => TPayload,
31
- excludedPeers?: PeerId[]
32
- ): CoordinatorBatch<TPayload, TResponse>[] {
33
- const groups = blockPeers.reduce((acc, [blockId, peerId]) => {
34
- const peerId_str = peerId.toString();
35
- const coordinator = acc.get(peerId_str) ?? { peerId, blockId, excludedPeers } as Partial<CoordinatorBatch<TPayload, TResponse>>;
36
- acc.set(peerId_str, { ...coordinator, payload: getBlockPayload(payload, blockId, coordinator.payload) } as CoordinatorBatch<TPayload, TResponse>);
37
- return acc;
38
- }, new Map<string, CoordinatorBatch<TPayload, TResponse>>());
39
- return Array.from(groups.values());
40
- }
41
-
42
- /**
43
- * Iterates over all batches that have not completed, whether subsumed or not
44
- */
45
- export function* incompleteBatches<TPayload, TResponse>(batches: CoordinatorBatch<TPayload, TResponse>[]): IterableIterator<CoordinatorBatch<TPayload, TResponse>> {
46
- const stack: CoordinatorBatch<TPayload, TResponse>[] = [...batches];
47
- while (stack.length > 0) {
48
- const batch = stack.pop()!;
49
- if (!batch.request || !batch.request.isResponse) {
50
- yield batch;
51
- }
52
- if (batch.subsumedBy && batch.subsumedBy.length) {
53
- stack.push(...batch.subsumedBy);
54
- }
55
- }
56
- }
57
-
58
- /**
59
- * Checks if all completed batches (ignoring failures) satisfy a predicate
60
- */
61
- export function everyBatch<TPayload, TResponse>(batches: CoordinatorBatch<TPayload, TResponse>[], predicate: (batch: CoordinatorBatch<TPayload, TResponse>) => boolean): boolean {
62
- // For each root batch require that SOME node in its retry tree satisfies the predicate.
63
- // Use iterative DFS to avoid recursion depth and minimize allocations.
64
- for (const root of batches) {
65
- let found = false;
66
- const stack: CoordinatorBatch<TPayload, TResponse>[] = [root];
67
- while (stack.length > 0) {
68
- const node = stack.pop()!;
69
- if (predicate(node)) { found = true; break; }
70
- if (node.subsumedBy && node.subsumedBy.length) {
71
- for (let i = 0; i < node.subsumedBy.length; i++) stack.push(node.subsumedBy[i]!);
72
- }
73
- }
74
- if (!found) return false;
75
- }
76
- return true;
77
- }
78
-
79
- /**
80
- * Iterates over all batches that satisfy an optional predicate, whether subsumed or not
81
- */
82
- export function* allBatches<TPayload, TResponse>(batches: CoordinatorBatch<TPayload, TResponse>[], predicate?: (batch: CoordinatorBatch<TPayload, TResponse>) => boolean): IterableIterator<CoordinatorBatch<TPayload, TResponse>> {
83
- const stack: CoordinatorBatch<TPayload, TResponse>[] = [...batches];
84
- while (stack.length > 0) {
85
- const batch = stack.pop()!;
86
- if (!predicate || predicate(batch)) {
87
- yield batch;
88
- }
89
- if (batch.subsumedBy && batch.subsumedBy.length) {
90
- stack.push(...batch.subsumedBy);
91
- }
92
- }
93
- }
94
-
95
- /**
96
- * Returns a new blockId list payload with the given block id appended
97
- */
98
- export function mergeBlocks(_payload: BlockId[], blockId: BlockId, mergeWithPayload: BlockId[] | undefined): BlockId[] {
99
- return [...(mergeWithPayload ?? []), blockId];
100
- }
101
-
102
- /**
103
- * Processes a set of batches, retrying any failures until success or expiration
104
- * @param batches - The batches to process - each represents a group of blocks centered on a coordinating peer
105
- * @param process - The function to call for a given batch
106
- * @param getBlockIds - The function to call to get the block ids for a given batch
107
- * @param getBlockPayload - The function to call to get the payload given a parent payload and block id, and optionally merge with an existing payload
108
- * @param expiration - The expiration time for the operation
109
- * @param findCoordinator - The function to call to find a coordinator for a block id
110
- */
111
- export async function processBatches<TPayload, TResponse>(
112
- batches: CoordinatorBatch<TPayload, TResponse>[],
113
- process: (batch: CoordinatorBatch<TPayload, TResponse>) => Promise<TResponse>,
114
- getBlockIds: (batch: CoordinatorBatch<TPayload, TResponse>) => BlockId[],
115
- getBlockPayload: (payload: TPayload, blockId: BlockId, mergeWithPayload: TPayload | undefined) => TPayload,
116
- expiration: number,
117
- findCoordinator: (blockId: BlockId, options: { excludedPeers: PeerId[] }) => Promise<PeerId>
118
- ): Promise<void> {
119
- // Root-map ensures retries are recorded on the original batch to avoid deep trees
120
- const rootOf = new WeakMap<CoordinatorBatch<TPayload, TResponse>, CoordinatorBatch<TPayload, TResponse>>();
121
- for (const b of batches) rootOf.set(b, b);
122
-
123
- // Process a set of batches concurrently and enqueue retries flatly onto the root's subsumedBy list
124
- const processSet = async (set: CoordinatorBatch<TPayload, TResponse>[]) => {
125
- await Promise.all(set.map(async (batch) => {
126
- batch.request = new Pending(process(batch)
127
- .catch(async e => {
128
- // Always rethrow the ORIGINAL first-attempt error `e` so batch.request.error
129
- // preserves the root cause. If retry setup itself fails (e.g., findCoordinator
130
- // throws "self-exhausted" on a solo node), that retry error MUST NOT mask `e`.
131
- if (expiration > Date.now()) {
132
- const excludedPeers = [batch.peerId, ...(batch.excludedPeers ?? [])];
133
- log('retry peer=%s excluded=%d', batch.peerId.toString(), excludedPeers.length);
134
- try {
135
- const retries = await createBatchesForPayload<TPayload, TResponse>(
136
- getBlockIds(batch),
137
- batch.payload,
138
- getBlockPayload,
139
- excludedPeers,
140
- findCoordinator
141
- );
142
- if (retries.length > 0 && expiration > Date.now()) {
143
- const root = rootOf.get(batch) ?? batch;
144
- root.subsumedBy = [...(root.subsumedBy ?? []), ...retries];
145
- for (const r of retries) rootOf.set(r, root);
146
- // Process retries, but ensure further failures also attach to the same root
147
- await processSet(retries);
148
- }
149
- } catch (retryErr) {
150
- log('retry:setup-failed peer=%s original=%o retry=%o',
151
- batch.peerId.toString(), e, retryErr);
152
- // Swallow retryErr; the original `e` is authoritative.
153
- }
154
- }
155
- throw e;
156
- }));
157
- }));
158
-
159
- // Wait for all in this set to settle
160
- await Promise.all(set.map(b => b.request?.result().catch(() => { /* ignore */ })));
161
- };
162
-
163
- await processSet(batches);
164
- }
165
-
166
- /**
167
- * Creates batches for a given payload, grouped by the coordinating peer for each block id
168
- * This is a placeholder function that will be implemented by the caller
169
- */
170
- export async function createBatchesForPayload<TPayload, TResponse>(
171
- blockIds: BlockId[],
172
- payload: TPayload,
173
- getBlockPayload: (payload: TPayload, blockId: BlockId, mergeWithPayload: TPayload | undefined) => TPayload,
174
- excludedPeers: PeerId[],
175
- findCoordinator: (blockId: BlockId, options: { excludedPeers: PeerId[] }) => Promise<PeerId>
176
- ): Promise<CoordinatorBatch<TPayload, TResponse>[]> {
177
- // Group by block id
178
- const distinctBlockIds = new Set(blockIds);
179
-
180
- // Find coordinator for each key
181
- const blockIdPeerId = await Promise.all(
182
- Array.from(distinctBlockIds).map(async (bid) =>
183
- [bid, await findCoordinator(bid, { excludedPeers })] as const
184
- )
185
- );
186
-
187
- // Group blocks around their coordinating peers
188
- const batches = makeBatchesByPeer<TPayload, TResponse>(blockIdPeerId, payload, getBlockPayload, excludedPeers);
189
- log('createBatches blockIds=%d batches=%d excluded=%d', distinctBlockIds.size, batches.length, excludedPeers.length);
190
- return batches;
191
- }
1
+ import type { PeerId } from "../network/types.js";
2
+ import type { BlockId } from "../index.js";
3
+ import { Pending } from "./pending.js";
4
+ import { createLogger } from "../logger.js";
5
+
6
+ const log = createLogger('batch-coordinator');
7
+
8
+ /**
9
+ * Represents a batch of operations for a specific block coordinated by a peer
10
+ */
11
+ export type CoordinatorBatch<TPayload, TResponse> = {
12
+ peerId: PeerId;
13
+ blockId: BlockId;
14
+ payload: TPayload;
15
+ request?: Pending<TResponse>;
16
+ /** Whether this batch has been subsumed by other successful batches */
17
+ subsumedBy?: CoordinatorBatch<TPayload, TResponse>[];
18
+ /** Peers that have already been tried (and failed) */
19
+ excludedPeers?: PeerId[];
20
+ /** When a batch coordinates multiple blocks (via cluster intersection), lists all block IDs */
21
+ coordinatingBlockIds?: BlockId[];
22
+ }
23
+
24
+ /**
25
+ * Creates batches for a given payload, grouped by the coordinating peer for each block id
26
+ */
27
+ export function makeBatchesByPeer<TPayload, TResponse>(
28
+ blockPeers: (readonly [BlockId, PeerId])[],
29
+ payload: TPayload,
30
+ getBlockPayload: (payload: TPayload, blockId: BlockId, mergeWithPayload: TPayload | undefined) => TPayload,
31
+ excludedPeers?: PeerId[]
32
+ ): CoordinatorBatch<TPayload, TResponse>[] {
33
+ const groups = blockPeers.reduce((acc, [blockId, peerId]) => {
34
+ const peerId_str = peerId.toString();
35
+ const coordinator = acc.get(peerId_str) ?? { peerId, blockId, excludedPeers } as Partial<CoordinatorBatch<TPayload, TResponse>>;
36
+ acc.set(peerId_str, { ...coordinator, payload: getBlockPayload(payload, blockId, coordinator.payload) } as CoordinatorBatch<TPayload, TResponse>);
37
+ return acc;
38
+ }, new Map<string, CoordinatorBatch<TPayload, TResponse>>());
39
+ return Array.from(groups.values());
40
+ }
41
+
42
+ /**
43
+ * Iterates over all batches that have not completed, whether subsumed or not
44
+ */
45
+ export function* incompleteBatches<TPayload, TResponse>(batches: CoordinatorBatch<TPayload, TResponse>[]): IterableIterator<CoordinatorBatch<TPayload, TResponse>> {
46
+ const stack: CoordinatorBatch<TPayload, TResponse>[] = [...batches];
47
+ while (stack.length > 0) {
48
+ const batch = stack.pop()!;
49
+ if (!batch.request || !batch.request.isResponse) {
50
+ yield batch;
51
+ }
52
+ if (batch.subsumedBy && batch.subsumedBy.length) {
53
+ stack.push(...batch.subsumedBy);
54
+ }
55
+ }
56
+ }
57
+
58
+ /**
59
+ * Checks if all completed batches (ignoring failures) satisfy a predicate
60
+ */
61
+ export function everyBatch<TPayload, TResponse>(batches: CoordinatorBatch<TPayload, TResponse>[], predicate: (batch: CoordinatorBatch<TPayload, TResponse>) => boolean): boolean {
62
+ // For each root batch require that SOME node in its retry tree satisfies the predicate.
63
+ // Use iterative DFS to avoid recursion depth and minimize allocations.
64
+ for (const root of batches) {
65
+ let found = false;
66
+ const stack: CoordinatorBatch<TPayload, TResponse>[] = [root];
67
+ while (stack.length > 0) {
68
+ const node = stack.pop()!;
69
+ if (predicate(node)) { found = true; break; }
70
+ if (node.subsumedBy && node.subsumedBy.length) {
71
+ for (let i = 0; i < node.subsumedBy.length; i++) stack.push(node.subsumedBy[i]!);
72
+ }
73
+ }
74
+ if (!found) return false;
75
+ }
76
+ return true;
77
+ }
78
+
79
+ /**
80
+ * Iterates over all batches that satisfy an optional predicate, whether subsumed or not
81
+ */
82
+ export function* allBatches<TPayload, TResponse>(batches: CoordinatorBatch<TPayload, TResponse>[], predicate?: (batch: CoordinatorBatch<TPayload, TResponse>) => boolean): IterableIterator<CoordinatorBatch<TPayload, TResponse>> {
83
+ const stack: CoordinatorBatch<TPayload, TResponse>[] = [...batches];
84
+ while (stack.length > 0) {
85
+ const batch = stack.pop()!;
86
+ if (!predicate || predicate(batch)) {
87
+ yield batch;
88
+ }
89
+ if (batch.subsumedBy && batch.subsumedBy.length) {
90
+ stack.push(...batch.subsumedBy);
91
+ }
92
+ }
93
+ }
94
+
95
+ /**
96
+ * Returns a new blockId list payload with the given block id appended
97
+ */
98
+ export function mergeBlocks(_payload: BlockId[], blockId: BlockId, mergeWithPayload: BlockId[] | undefined): BlockId[] {
99
+ return [...(mergeWithPayload ?? []), blockId];
100
+ }
101
+
102
+ /**
103
+ * Processes a set of batches, retrying any failures until success or expiration
104
+ * @param batches - The batches to process - each represents a group of blocks centered on a coordinating peer
105
+ * @param process - The function to call for a given batch
106
+ * @param getBlockIds - The function to call to get the block ids for a given batch
107
+ * @param getBlockPayload - The function to call to get the payload given a parent payload and block id, and optionally merge with an existing payload
108
+ * @param expiration - The expiration time for the operation
109
+ * @param findCoordinator - The function to call to find a coordinator for a block id
110
+ */
111
+ export async function processBatches<TPayload, TResponse>(
112
+ batches: CoordinatorBatch<TPayload, TResponse>[],
113
+ process: (batch: CoordinatorBatch<TPayload, TResponse>) => Promise<TResponse>,
114
+ getBlockIds: (batch: CoordinatorBatch<TPayload, TResponse>) => BlockId[],
115
+ getBlockPayload: (payload: TPayload, blockId: BlockId, mergeWithPayload: TPayload | undefined) => TPayload,
116
+ expiration: number,
117
+ findCoordinator: (blockId: BlockId, options: { excludedPeers: PeerId[] }) => Promise<PeerId>
118
+ ): Promise<void> {
119
+ // Root-map ensures retries are recorded on the original batch to avoid deep trees
120
+ const rootOf = new WeakMap<CoordinatorBatch<TPayload, TResponse>, CoordinatorBatch<TPayload, TResponse>>();
121
+ for (const b of batches) rootOf.set(b, b);
122
+
123
+ // Process a set of batches concurrently and enqueue retries flatly onto the root's subsumedBy list
124
+ const processSet = async (set: CoordinatorBatch<TPayload, TResponse>[]) => {
125
+ await Promise.all(set.map(async (batch) => {
126
+ batch.request = new Pending(process(batch)
127
+ .catch(async e => {
128
+ // Always rethrow the ORIGINAL first-attempt error `e` so batch.request.error
129
+ // preserves the root cause. If retry setup itself fails (e.g., findCoordinator
130
+ // throws "self-exhausted" on a solo node), that retry error MUST NOT mask `e`.
131
+ if (expiration > Date.now()) {
132
+ const excludedPeers = [batch.peerId, ...(batch.excludedPeers ?? [])];
133
+ log('retry peer=%s excluded=%d', batch.peerId.toString(), excludedPeers.length);
134
+ try {
135
+ const retries = await createBatchesForPayload<TPayload, TResponse>(
136
+ getBlockIds(batch),
137
+ batch.payload,
138
+ getBlockPayload,
139
+ excludedPeers,
140
+ findCoordinator
141
+ );
142
+ if (retries.length > 0 && expiration > Date.now()) {
143
+ const root = rootOf.get(batch) ?? batch;
144
+ root.subsumedBy = [...(root.subsumedBy ?? []), ...retries];
145
+ for (const r of retries) rootOf.set(r, root);
146
+ // Process retries, but ensure further failures also attach to the same root
147
+ await processSet(retries);
148
+ }
149
+ } catch (retryErr) {
150
+ log('retry:setup-failed peer=%s original=%o retry=%o',
151
+ batch.peerId.toString(), e, retryErr);
152
+ // Swallow retryErr; the original `e` is authoritative.
153
+ }
154
+ }
155
+ throw e;
156
+ }));
157
+ }));
158
+
159
+ // Wait for all in this set to settle
160
+ await Promise.all(set.map(b => b.request?.result().catch(() => { /* ignore */ })));
161
+ };
162
+
163
+ await processSet(batches);
164
+ }
165
+
166
+ /**
167
+ * Creates batches for a given payload, grouped by the coordinating peer for each block id
168
+ * This is a placeholder function that will be implemented by the caller
169
+ */
170
+ export async function createBatchesForPayload<TPayload, TResponse>(
171
+ blockIds: BlockId[],
172
+ payload: TPayload,
173
+ getBlockPayload: (payload: TPayload, blockId: BlockId, mergeWithPayload: TPayload | undefined) => TPayload,
174
+ excludedPeers: PeerId[],
175
+ findCoordinator: (blockId: BlockId, options: { excludedPeers: PeerId[] }) => Promise<PeerId>
176
+ ): Promise<CoordinatorBatch<TPayload, TResponse>[]> {
177
+ // Group by block id
178
+ const distinctBlockIds = new Set(blockIds);
179
+
180
+ // Find coordinator for each key
181
+ const blockIdPeerId = await Promise.all(
182
+ Array.from(distinctBlockIds).map(async (bid) =>
183
+ [bid, await findCoordinator(bid, { excludedPeers })] as const
184
+ )
185
+ );
186
+
187
+ // Group blocks around their coordinating peers
188
+ const batches = makeBatchesByPeer<TPayload, TResponse>(blockIdPeerId, payload, getBlockPayload, excludedPeers);
189
+ log('createBatches blockIds=%d batches=%d excluded=%d', distinctBlockIds.size, batches.length, excludedPeers.length);
190
+ return batches;
191
+ }
@@ -1,32 +1,32 @@
1
- // Retrieves a value from a record, generating an entry if none exists
2
- export function ensured<K extends string | number | symbol, V>(
3
- map: Record<K, V>,
4
- key: K,
5
- makeNew: () => Exclude<V, undefined>,
6
- existing?: (existing: Exclude<V, undefined>) => void
7
- ): Exclude<V, undefined> {
8
- let v = map[key];
9
- if (typeof v === 'undefined') {
10
- v = makeNew();
11
- map[key] = v;
12
- } else if (existing) {
13
- existing(v as Exclude<V, undefined>);
14
- }
15
- return v as Exclude<V, undefined>;
16
- }
17
-
18
- export function ensuredMap<K extends string | number | symbol, V>(
19
- map: Map<K, V>,
20
- key: K,
21
- makeNew: () => Exclude<V, undefined>,
22
- existing?: (existing: Exclude<V, undefined>) => void
23
- ): Exclude<V, undefined> {
24
- let v = map.get(key);
25
- if (typeof v === 'undefined') {
26
- v = makeNew();
27
- map.set(key, v);
28
- } else if (existing) {
29
- existing(v as Exclude<V, undefined>);
30
- }
31
- return v as Exclude<V, undefined>;
32
- }
1
+ // Retrieves a value from a record, generating an entry if none exists
2
+ export function ensured<K extends string | number | symbol, V>(
3
+ map: Record<K, V>,
4
+ key: K,
5
+ makeNew: () => Exclude<V, undefined>,
6
+ existing?: (existing: Exclude<V, undefined>) => void
7
+ ): Exclude<V, undefined> {
8
+ let v = map[key];
9
+ if (typeof v === 'undefined') {
10
+ v = makeNew();
11
+ map[key] = v;
12
+ } else if (existing) {
13
+ existing(v as Exclude<V, undefined>);
14
+ }
15
+ return v as Exclude<V, undefined>;
16
+ }
17
+
18
+ export function ensuredMap<K extends string | number | symbol, V>(
19
+ map: Map<K, V>,
20
+ key: K,
21
+ makeNew: () => Exclude<V, undefined>,
22
+ existing?: (existing: Exclude<V, undefined>) => void
23
+ ): Exclude<V, undefined> {
24
+ let v = map.get(key);
25
+ if (typeof v === 'undefined') {
26
+ v = makeNew();
27
+ map.set(key, v);
28
+ } else if (existing) {
29
+ existing(v as Exclude<V, undefined>);
30
+ }
31
+ return v as Exclude<V, undefined>;
32
+ }