@effectstream/runtime 0.100.22 → 0.100.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,73 @@
1
+ /**
2
+ * Bounded hand-off queue between the merge (producer) and the apply loop (consumer).
3
+ *
4
+ * The merge sends finalized blocks into an effection channel; the apply loop drains
5
+ * them. effection channel subscribers buffer unbounded, so during deep catch-up the
6
+ * merge (no I/O) outpaces the apply loop and the queue grows toward the whole backlog
7
+ * → OOM (sync/CLAUDE.md Finding #1) — especially with empty-block coalescing removing
8
+ * the per-block apply brake.
9
+ *
10
+ * This wraps the channel with backpressure: the producer blocks once `cap` produced
11
+ * blocks are still unconsumed, and the consumer wakes it as it drains. The cap only
12
+ * ever throttles the producer (the consumer always keeps pulling), so it can't
13
+ * deadlock. `inFlight = produced − consumed` is the live queue depth, surfaced on
14
+ * /debug/metrics.
15
+ */
16
+ import { type Channel, createChannel, type Operation } from "effection";
17
+ import { conditionVariable } from "@effectstream/utils";
18
+ import type { ChainBlock } from "@effectstream/sync";
19
+ import {
20
+ finalizedStreamStatus,
21
+ recordConsumed,
22
+ recordProduced,
23
+ } from "./api/stream-status.ts";
24
+ import type { FinalizedBlockSubscription } from "./coalesce.ts";
25
+
26
+ export type BoundedFinalizedStream = {
27
+ /** Send side — hand to `startMerge`. `send` blocks the producer at the cap. */
28
+ stream: Channel<ChainBlock, void>;
29
+ /** Read side — hand to the coalescer / apply loop. Counts drains and wakes the producer. */
30
+ subscription: FinalizedBlockSubscription;
31
+ };
32
+
33
+ export function* createBoundedFinalizedStream(
34
+ cap: number,
35
+ ): Operation<BoundedFinalizedStream> {
36
+ const channel = createChannel<ChainBlock>();
37
+ // Subscribe BEFORE the caller spawns the merge: effection channels drop sends
38
+ // with no subscriber, so a fast restart could emit blocks before we subscribe
39
+ // and stall at the pre-restart height.
40
+ const subscription = yield* channel;
41
+
42
+ const drained = conditionVariable<void>();
43
+ const inFlight = () =>
44
+ finalizedStreamStatus.produced - finalizedStreamStatus.consumed;
45
+
46
+ // `startMerge` only uses `.send`; gate it. check-then-wait is atomic under
47
+ // effection's cooperative scheduling, so there's no lost wakeup.
48
+ const stream: Channel<ChainBlock, void> = {
49
+ ...channel,
50
+ *send(block: ChainBlock) {
51
+ while (inFlight() >= cap) {
52
+ yield* drained.wait();
53
+ }
54
+ recordProduced();
55
+ yield* channel.send(block);
56
+ },
57
+ };
58
+
59
+ // Count consumed at the pull point (exact channel depth, decoupled from the
60
+ // coalescer's run length) and wake the merge once the queue drops below cap.
61
+ const countedSubscription: FinalizedBlockSubscription = {
62
+ *next() {
63
+ const result = yield* subscription.next();
64
+ if (!result.done) {
65
+ recordConsumed(1);
66
+ if (inFlight() < cap) drained.wake();
67
+ }
68
+ return result;
69
+ },
70
+ };
71
+
72
+ return { stream, subscription: countedSubscription };
73
+ }
package/src/main.ts CHANGED
@@ -23,16 +23,21 @@ import {
23
23
  import { startMerge, startSync } from "@effectstream/sync";
24
24
  import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
25
25
  import {
26
- createChannel,
27
- each,
28
26
  type Operation,
29
27
  sleep,
30
28
  spawn,
31
29
  until,
32
30
  } from "effection";
33
31
  import { initTelemetry } from "./telemetry.ts";
34
- import { type PendingEvent, processFinalizedBlock } from "./process-blocks.ts";
32
+ import {
33
+ type PendingEvent,
34
+ processFinalizedBlockWithRetry,
35
+ } from "./process-blocks.ts";
36
+ import { createEmptyBlockCoalescer } from "./coalesce.ts";
35
37
  import { startHttpServer } from "./api/http-server.ts";
38
+ import { recordAppliedBlock } from "./api/apply-status.ts";
39
+ import { recordCoalesced } from "./api/stream-status.ts";
40
+ import { createBoundedFinalizedStream } from "./finalized-stream.ts";
36
41
  import type { StartConfig } from "./types.ts";
37
42
  import type { Client } from "pg";
38
43
  import type { EffectstreamBlockHash } from "@effectstream/utils";
@@ -91,13 +96,20 @@ export function* start(config: StartConfig): Operation<void> {
91
96
  );
92
97
  });
93
98
 
99
+ // 10× main clock block time (NTP if present, else protocol 0). Undefined disables lag gating.
94
100
  const ntpConfig = syncInfo.find(s => s.networkType === ConfigNetworkType.NTP);
95
- const ntpBlockTimeMS = (ntpConfig?.network as { blockTimeMS: number } | undefined)?.blockTimeMS;
96
- const lagThresholdMs = ntpBlockTimeMS != null ? ntpBlockTimeMS * 10 : undefined;
101
+ const clockBlockTimeMS =
102
+ (ntpConfig?.network as { blockTimeMS?: number } | undefined)?.blockTimeMS ??
103
+ (syncInfo[0]?.network as { blockTimeMS?: number } | undefined)?.blockTimeMS;
104
+ const lagThresholdMs = clockBlockTimeMS != null ? clockBlockTimeMS * 10 : undefined;
97
105
 
98
- const finalizedBlockStream = createChannel<ChainBlock>();
106
+ // Bounded hand-off queue between the merge and the apply loop. Backpressure caps
107
+ // the in-memory queue so deep catch-up can't grow it toward the whole backlog
108
+ // (see finalized-stream.ts / sync/CLAUDE.md Finding #1).
109
+ const { stream: finalizedStream, subscription: finalizedBlocks } =
110
+ yield* createBoundedFinalizedStream(ENV.EFFECTSTREAM_FINALIZED_STREAM_CAP);
99
111
 
100
- yield* spawn(() => startMerge(syncProtocols, finalizedBlockStream));
112
+ yield* spawn(() => startMerge(syncProtocols, finalizedStream));
101
113
 
102
114
  const heartbeatIntervalMs = 60_000;
103
115
  yield* spawn(function* () {
@@ -139,55 +151,66 @@ export function* start(config: StartConfig): Operation<void> {
139
151
  yield* spawn(() => runSnapshotLoop(config.snapshotConfig!));
140
152
  }
141
153
 
142
- for (const value of yield* each(finalizedBlockStream)) {
143
- // We request a dbClient for a non-shared dbConn object.
144
- // For PGLite, this is not enough, as the can only be one connection at a time.
145
- // So we request a DBMutex as well.
146
- let dbClient: Client | undefined;
147
- // Custom app events emitted by STFs during this block. Collected by
148
- // processFinalizedBlock and flushed below ONLY after it returns — i.e.
149
- // strictly after the block's COMMIT. See plan invariant I1.
150
- let blockAppEvents: PendingEvent[] = [];
151
- try {
152
- yield* acquireDBMutex(`processing-blocks:${value.blockNumber}`);
153
- dbClient = yield* until((dbConn as any).connect()); // Client,
154
-
155
- const result = yield* processFinalizedBlock(
156
- value,
157
- config,
158
- dbClient as any, // Client,
159
- blockHash,
154
+ const coalescer = createEmptyBlockCoalescer({
155
+ enabled: ENV.EFFECTSTREAM_COALESCE_EMPTY_BLOCKS,
156
+ subscription: finalizedBlocks,
157
+ pool: dbConn as any, // Pool,
158
+ migrations: config.migrations,
159
+ lagThresholdMs,
160
+ getPreviousBlockHash: () => blockHash,
161
+ onFlush: (endpoint, length) => {
162
+ // `consumed` is counted at the subscription pull point (above), not here, so
163
+ // it reflects the channel depth regardless of how many blocks a run folds.
164
+ recordCoalesced(length);
165
+ recordAppliedBlock(endpoint);
166
+ emitLatestBlocks(
167
+ endpoint.blockNumber,
168
+ endpoint.timestamp,
169
+ getRangesForSyncProtocols(endpoint),
170
+ );
171
+ log.local(
172
+ ComponentNames.EFFECTSTREAM_SYNC,
173
+ "block-merge",
174
+ SeverityNumber.INFO,
175
+ (l) =>
176
+ l(
177
+ `coalesced ${length} empty block(s) → block ${endpoint.blockNumber}`,
178
+ ),
160
179
  );
161
- const resultHash = result.blockHash;
162
- blockAppEvents = result.events;
163
- if (resultHash !== "0x0") {
164
- blockHash = resultHash;
165
- }
166
- } finally {
167
- releaseDBMutex(`processing-blocks:${value.blockNumber}`);
168
- if (dbClient) {
169
- (dbClient as any).release(); // Client,
170
- }
180
+ },
181
+ });
182
+
183
+ while (true) {
184
+ const value = yield* coalescer.advance();
185
+ if (value === undefined) break;
186
+
187
+ // Owns connection checkout, the per-block DB mutex (PGLite), and
188
+ // transient-pg retry/backoff. App events flush below only after it
189
+ // returns — strictly after the block's COMMIT.
190
+ const result = yield* processFinalizedBlockWithRetry(
191
+ value,
192
+ config,
193
+ dbConn as any, // Pool,
194
+ blockHash,
195
+ );
196
+ const blockAppEvents: PendingEvent[] = result.events;
197
+ if (result.blockHash !== "0x0") {
198
+ blockHash = result.blockHash;
171
199
  }
172
200
 
201
+ recordAppliedBlock(value); // apply-stage liveness for /debug/metrics
202
+
173
203
  // Used to emit & log the block range for each protocol.
174
204
  const contentBlocksForProtocol = getRangesForSyncProtocols(value);
175
205
 
176
- // Fire-and-forget: do not stall the sync loop on broker acks.
206
+ // Fire-and-forget: don't stall the sync loop on broker acks. Ordering is
207
+ // safe without a wrapper queue — Opifex assigns packet IDs synchronously in
208
+ // publish() and net.Socket serializes writes in call order, so MQTT 3.1.1
209
+ // §4.6 in-order delivery holds per (publisher, topic, QoS).
177
210
  //
178
- // Opifex's TcpClient assigns packet IDs synchronously in client.publish()
179
- // and writes to the socket via Node's net.Socket, which has its own write
180
- // queue — so concurrent publish() calls land on the wire in call order.
181
- // MQTT 3.1.1 §4.6 then guarantees in-order delivery to each subscriber
182
- // per (publisher, topic, QoS). We don't need a wrapper queue.
183
- //
184
- // TODO(scaling): the broker runs in-process (see `new EventBroker(...)`
185
- // above). Fan-out to N subscribers consumes the same event loop as block
186
- // processing. At ~thousands of subscribers, consider:
187
- // 1) moving the broker to a worker / separate process,
188
- // 2) a publisher-side circuit breaker if ctx.unresolvedPublish.size > N
189
- // (drop events with a counter for observability).
190
- // Today: localhost, in-process, expected O(10²) subscribers — non-issue.
211
+ // TODO(scaling): the broker is in-process, so fan-out shares this event
212
+ // loop. At ~thousands of subscribers, move it to a worker and/or add a
213
+ // publisher-side circuit breaker. Today: localhost, O(10²) subs — non-issue.
191
214
  emitLatestBlocks(
192
215
  value.blockNumber,
193
216
  value.timestamp,
@@ -220,7 +243,7 @@ export function* start(config: StartConfig): Operation<void> {
220
243
  }...${lagSuffix} | ${JSON.stringify(contentBlocksForProtocol)}`,
221
244
  ),
222
245
  );
223
- yield* each.next();
246
+ if (config.dev?.applyDelayMs) yield* sleep(config.dev.applyDelayMs);
224
247
  }
225
248
  }
226
249
 
@@ -239,13 +262,9 @@ function getRangesForSyncProtocols(value: ChainBlock): Record<string, [number, n
239
262
  }
240
263
 
241
264
  /**
242
- * Publish built-in block-level events.
243
- *
244
- * Fire-and-forget for the same reasons as the app-event flush above
245
- * (do not stall the sync loop; ordering preserved by Opifex + MQTT 3.1.1).
246
- *
247
- * Errors are caught locally so they don't escape to the global
248
- * `unhandledrejection` handler without context.
265
+ * Publish built-in block-level events. Fire-and-forget like the app-event flush
266
+ * above (don't stall the sync loop; ordering held by Opifex + MQTT 3.1.1).
267
+ * Errors are caught locally so they don't reach the global handler contextless.
249
268
  */
250
269
  function emitLatestBlocks(
251
270
  rollUpBlockHeight: number,
@@ -296,45 +315,48 @@ function* startup(
296
315
  });
297
316
  });
298
317
 
299
-
300
318
  yield* acquireDBMutex(`startup-node`);
319
+ // `finally` releases the mutex on error/cancellation too; a throw in any step
320
+ // below would otherwise leak it and deadlock every later DB operation.
321
+ try {
322
+ // Dev-only reset of user-owned public tables
323
+ if (config.dev?.resetPublicData) {
324
+ yield* resetPublicTables(dbConn as any); // Client,
325
+ }
301
326
 
302
- // Dev-only reset of user-owned public tables
303
- if (config.dev?.resetPublicData) {
304
- yield* resetPublicTables(dbConn as any); // Client,
305
- }
306
-
307
- // When the node is started, we apply system migrations.
308
- // Either system initial migrations, or migrations given a Paima Engine Update.
309
- yield* applySystemMigrations(
310
- config.appVersion,
311
- versionInfo,
312
- lastBlockHeight,
313
- dbConn,
314
- config.migrations,
315
- );
327
+ // When the node is started, we apply system migrations.
328
+ // Either system initial migrations, or migrations given a Paima Engine Update.
329
+ yield* applySystemMigrations(
330
+ config.appVersion,
331
+ versionInfo,
332
+ lastBlockHeight,
333
+ dbConn,
334
+ config.migrations,
335
+ );
316
336
 
317
- // Validate that immutable config fields (e.g. NTP startTime, startBlockHeight)
318
- // have not changed since the last run. Persists a snapshot on first start.
319
- // Must run after system migrations so the snapshot table exists.
320
- yield* validateAndSnapshotConfig(syncInfo, dbConn);
337
+ // Validate that immutable config fields (e.g. NTP startTime, startBlockHeight)
338
+ // have not changed since the last run. Persists a snapshot on first start.
339
+ // Must run after system migrations so the snapshot table exists.
340
+ yield* validateAndSnapshotConfig(syncInfo, dbConn);
321
341
 
322
- const syncProtocols = yield* genSyncProtocols(dbConn as any, // Client,
323
- syncInfo);
342
+ const syncProtocols = yield* genSyncProtocols(dbConn as any, // Client,
343
+ syncInfo);
324
344
 
325
- const capabilities = yield* until(detectCapabilities(dbConn as any));
326
- const viewStrategy = selectViewStrategy(capabilities);
345
+ const capabilities = yield* until(detectCapabilities(dbConn as any));
346
+ const viewStrategy = selectViewStrategy(capabilities);
327
347
 
328
- yield* createDynamicTables(
329
- versionInfo,
330
- lastBlockHeight,
331
- dbConn as any, // Client,
332
- syncProtocols,
333
- viewStrategy,
334
- );
348
+ yield* createDynamicTables(
349
+ versionInfo,
350
+ lastBlockHeight,
351
+ dbConn as any, // Client,
352
+ syncProtocols,
353
+ viewStrategy,
354
+ );
335
355
 
336
- releaseDBMutex(`startup-node`);
337
- return syncProtocols;
356
+ return syncProtocols;
357
+ } finally {
358
+ releaseDBMutex(`startup-node`);
359
+ }
338
360
  }
339
361
 
340
362
  // Convert the primitive config to the final primitive instance
@@ -1,6 +1,6 @@
1
1
  import type { ChainBlock } from "@effectstream/sync";
2
- import { call, type Operation, until } from "effection";
3
- import type { Pool } from "pg";
2
+ import { call, type Operation, sleep, until } from "effection";
3
+ import type { Pool, PoolClient } from "pg";
4
4
  import { type BaseStfInput, type EmitFn, primitiveTransitionFunction } from "@effectstream/sm";
5
5
  import { PreparedQuery } from "@pgtyped/runtime";
6
6
  import type {
@@ -9,19 +9,24 @@ import type {
9
9
  SyncStateUpdateStream,
10
10
  } from "@effectstream/coroutine";
11
11
  import {
12
+ acquireDBMutex,
12
13
  blockHeightDone,
13
14
  deleteEmptyBlocks,
14
15
  deleteScheduled,
15
16
  getFutureGameInputByBlockHeight,
16
17
  getFutureGameInputByMaxTimestamp,
17
18
  insertGameInputResult,
19
+ isTransientPgError,
20
+ poolErrors,
21
+ releaseDBMutex,
18
22
  removePages,
19
23
  saveLastBlock,
24
+ upsertPage,
20
25
  } from "@effectstream/db";
21
26
  import { Buffer } from "node:buffer";
22
27
  import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
23
28
  import type { StartConfig } from "./types.ts";
24
- import type { BlockNumber, EffectstreamBlockHash } from "@effectstream/utils";
29
+ import type { EffectstreamBlockHash } from "@effectstream/utils";
25
30
  import { generateEffectstreamBlockHash, Prando } from "@effectstream/crypto";
26
31
  import { applyUserMigrations } from "./version-migrations.ts";
27
32
  import type { EventPathAndDef } from "@effectstream/event-client";
@@ -308,36 +313,53 @@ export function* processFinalizedBlock(
308
313
  }, dbConn)
309
314
  );
310
315
 
311
- /* STEP 7: Update page state for all sync protocols consumed
312
- * This is to mark what blocks where consumed by this specific
313
- * block, to avoid re-processing the same blocks more than once
314
- * if the sync protocol is restarted.
316
+ /* STEP 7: Persist the block-accurate resume marker per protocol, inside this
317
+ * block's transaction (the runtime is the sole writer of the resume
318
+ * position). Upsert at page_number = ownBlockNumber, then prune
319
+ * everything below → one row per protocol = the committed watermark.
320
+ * Full mechanism: @effectstream/sync CLAUDE.md, design idea #5.
315
321
  */
316
-
317
- // Get latest block for each sync protocol.
318
- const latestBlocks: Record<string, BlockNumber> = {};
319
- for (const blockInfo of value.blockInfo) {
320
- if (
321
- !latestBlocks[blockInfo.protocol_name] ||
322
- latestBlocks[blockInfo.protocol_name] < blockInfo.block_number
323
- ) {
324
- latestBlocks[blockInfo.protocol_name] = blockInfo.block_number;
325
- }
326
- }
327
-
328
- for (const [protocolName, blockNumber] of Object.entries(latestBlocks)) {
322
+ for (const { protocol_name, lastPage } of value.resumePages) {
323
+ yield* until(
324
+ upsertPage.run({
325
+ protocol_name,
326
+ page_number: lastPage.ownBlockNumber,
327
+ page: JSON.stringify(lastPage),
328
+ }, dbConn),
329
+ );
329
330
  yield* until(
330
331
  removePages.run({
331
- protocol_name: protocolName,
332
- page_number: blockNumber,
332
+ protocol_name,
333
+ page_number: lastPage.ownBlockNumber,
333
334
  }, dbConn),
334
335
  );
335
336
  }
336
337
 
337
338
  /* STEP 8: Commit the transaction. */
338
339
  yield* until(dbConn.query("COMMIT"));
340
+ // A successful COMMIT proves the DB is reachable end-to-end — reset the
341
+ // pool failure clock so the write path also feeds the health tracker
342
+ // (it otherwise only sees floating pool errors and the read-side API).
343
+ poolErrors.markHealthy();
339
344
  } catch (err) {
340
- yield* until(dbConn.query("ROLLBACK"));
345
+ // Best-effort rollback. If the socket is dead the ROLLBACK throws too;
346
+ // swallow that so it doesn't mask the original error — the caller's retry
347
+ // loop keys off the original (transient) error, not a rollback failure.
348
+ try {
349
+ yield* until(dbConn.query("ROLLBACK"));
350
+ } catch (rollbackErr) {
351
+ log.remote(
352
+ ComponentNames.EFFECTSTREAM_SYNC,
353
+ "block-processing",
354
+ SeverityNumber.WARN,
355
+ (log) =>
356
+ log(
357
+ `ROLLBACK failed for block ${value.blockNumber} (connection likely dead): ${
358
+ String(rollbackErr)
359
+ }`,
360
+ ),
361
+ );
362
+ }
341
363
  log.remote(
342
364
  ComponentNames.EFFECTSTREAM_SYNC,
343
365
  "block-processing",
@@ -345,12 +367,89 @@ export function* processFinalizedBlock(
345
367
  (log) =>
346
368
  log(`Error processing block ${value.blockNumber}: ${String(err)}`),
347
369
  );
348
- // We cannot recover from this error.
370
+ // Re-throw: `processFinalizedBlockWithRetry` decides whether this is a
371
+ // retryable transient pg failure or a fatal bug.
349
372
  throw err;
350
373
  }
351
374
 
352
- return {
353
- blockHash: hasOperations ? blockHash : "0x0" as PaimaBlockHash,
354
- events: blockEvents
375
+ return {
376
+ blockHash: hasOperations ? blockHash : "0x0" as PaimaBlockHash,
377
+ events: blockEvents
355
378
  };
356
379
  }
380
+
381
+ /** Caps the exponential backoff between block-retry attempts. */
382
+ const MAX_RETRY_BACKOFF_MS = 5_000;
383
+
384
+ /**
385
+ * Run {@link processFinalizedBlock} against a freshly checked-out pool client,
386
+ * retrying the whole block on transient pg failures (Neon cold-starts,
387
+ * idle-socket evictions, TLS blips).
388
+ *
389
+ * The previous socket is dead after such a failure, so each attempt
390
+ * re-acquires a client and reprocesses the block from `BEGIN` — safe because
391
+ * the failed tx never committed and the caller only advances the block stream
392
+ * after this returns.
393
+ *
394
+ * `poolErrors` governs escalation: it logs WARN→ERROR as the outage persists
395
+ * and calls `process.exit(1)` once it crosses the fatal threshold, letting the
396
+ * orchestrator restart us. A non-transient error (a real SQL/logic bug) is
397
+ * re-thrown and crashes, as before.
398
+ *
399
+ * Owns the per-block DB mutex (for PGLite) and client lifecycle so the caller
400
+ * doesn't have to.
401
+ */
402
+ export function* processFinalizedBlockWithRetry(
403
+ value: ChainBlock,
404
+ config: StartConfig,
405
+ pool: Pool,
406
+ previousBlockHash: EffectstreamBlockHash | null,
407
+ ): Operation<ProcessFinalizedBlockResult> {
408
+ const lockName = `processing-blocks:${value.blockNumber}`;
409
+ let attempt = 0;
410
+ while (true) {
411
+ let dbClient: PoolClient | undefined;
412
+ let transientErr: unknown;
413
+ try {
414
+ yield* acquireDBMutex(lockName);
415
+ dbClient = yield* until(pool.connect());
416
+ return yield* processFinalizedBlock(
417
+ value,
418
+ config,
419
+ dbClient as unknown as Pool, // a checked-out client, used as the tx conn
420
+ previousBlockHash,
421
+ );
422
+ } catch (err) {
423
+ if (!isTransientPgError(err)) throw err;
424
+ transientErr = err;
425
+ } finally {
426
+ releaseDBMutex(lockName);
427
+ if (dbClient) {
428
+ // Passing the error destroys the client instead of recycling it, so
429
+ // the pool never hands the dead socket back out. On success
430
+ // `transientErr` is undefined and the client returns to the pool.
431
+ try {
432
+ dbClient.release(transientErr as Error | undefined);
433
+ } catch {
434
+ /* already released/destroyed */
435
+ }
436
+ }
437
+ }
438
+
439
+ attempt++;
440
+ poolErrors.log(transientErr, ["sync-write"]);
441
+ const backoffMs = Math.min(500 * 2 ** (attempt - 1), MAX_RETRY_BACKOFF_MS);
442
+ log.local(
443
+ ComponentNames.EFFECTSTREAM_SYNC,
444
+ "block-processing",
445
+ SeverityNumber.WARN,
446
+ (l) =>
447
+ l(
448
+ `transient db error on block ${value.blockNumber}; retry #${attempt} in ${backoffMs}ms: ${
449
+ String(transientErr)
450
+ }`,
451
+ ),
452
+ );
453
+ yield* sleep(backoffMs);
454
+ }
455
+ }
package/src/types.ts CHANGED
@@ -79,5 +79,12 @@ export type StartConfig = {
79
79
  * state such as `bufferedData.size()`. No effect in production.
80
80
  */
81
81
  onStarted?: (ctx: { syncProtocols: AllSyncProtocols[] }) => void;
82
+ /**
83
+ * Test/diagnostic only: sleep this many ms after each applied block, slowing
84
+ * the drain so the per-chain fetch buffers fill to their cap and the
85
+ * backpressure fix becomes observable (otherwise drain ≈ fetch and the buffer
86
+ * never grows). No effect in production (leave unset / 0).
87
+ */
88
+ applyDelayMs?: number;
82
89
  };
83
90
  };
@@ -0,0 +1,69 @@
1
+ import { expect, test } from "bun:test";
2
+ import {
3
+ type EmptyRunBoundaries,
4
+ isCoalescableEmptyBlock,
5
+ } from "../src/coalesce.ts";
6
+
7
+ const NO_BOUNDARIES: EmptyRunBoundaries = {
8
+ minScheduledBlockHeight: null,
9
+ minScheduledTimestampMs: null,
10
+ };
11
+
12
+ function block(
13
+ blockNumber: number,
14
+ timestamp: number,
15
+ primitives: unknown[] = [],
16
+ ): any {
17
+ return { blockNumber, timestamp, blockInfo: [], resumePages: [], primitives };
18
+ }
19
+
20
+ test("empty block with no boundaries is coalescable", () => {
21
+ expect(isCoalescableEmptyBlock(block(10, 10_000), NO_BOUNDARIES, undefined))
22
+ .toBe(true);
23
+ });
24
+
25
+ test("block with on-chain content is NOT coalescable", () => {
26
+ expect(
27
+ isCoalescableEmptyBlock(block(10, 10_000, [{}]), NO_BOUNDARIES, undefined),
28
+ ).toBe(false);
29
+ });
30
+
31
+ test("block where a migration fires is NOT coalescable", () => {
32
+ const migrations = [{ name: "m1", sql: "", blockHeight: 10 }];
33
+ expect(isCoalescableEmptyBlock(block(10, 10_000), NO_BOUNDARIES, migrations))
34
+ .toBe(false);
35
+ expect(isCoalescableEmptyBlock(block(9, 9_000), NO_BOUNDARIES, migrations))
36
+ .toBe(true);
37
+ });
38
+
39
+ test("block at/after a pending block-height-scheduled input is NOT coalescable", () => {
40
+ const boundaries: EmptyRunBoundaries = {
41
+ minScheduledBlockHeight: 10,
42
+ minScheduledTimestampMs: null,
43
+ };
44
+ expect(isCoalescableEmptyBlock(block(9, 9_000), boundaries, undefined)).toBe(
45
+ true,
46
+ );
47
+ expect(isCoalescableEmptyBlock(block(10, 10_000), boundaries, undefined)).toBe(
48
+ false,
49
+ );
50
+ expect(isCoalescableEmptyBlock(block(11, 11_000), boundaries, undefined)).toBe(
51
+ false,
52
+ );
53
+ });
54
+
55
+ test("block at/after a due timestamp-scheduled input is NOT coalescable", () => {
56
+ const boundaries: EmptyRunBoundaries = {
57
+ minScheduledBlockHeight: null,
58
+ minScheduledTimestampMs: 10_000,
59
+ };
60
+ expect(isCoalescableEmptyBlock(block(9, 9_999), boundaries, undefined)).toBe(
61
+ true,
62
+ );
63
+ expect(isCoalescableEmptyBlock(block(10, 10_000), boundaries, undefined)).toBe(
64
+ false,
65
+ );
66
+ expect(isCoalescableEmptyBlock(block(11, 10_001), boundaries, undefined)).toBe(
67
+ false,
68
+ );
69
+ });