@effectstream/runtime 0.100.21 → 0.100.23

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@effectstream/runtime",
-  "version": "0.100.21",
+  "version": "0.100.23",
   "license": "MIT",
   "homepage": "https://effectstream.github.io/docs/",
   "repository": {
@@ -13,17 +13,17 @@
   },
   "exports": "./src/mod.ts",
   "dependencies": {
-    "@effectstream/concise": "0.100.21",
-    "@effectstream/config": "0.100.21",
-    "@effectstream/coroutine": "0.100.21",
-    "@effectstream/crypto": "0.100.21",
-    "@effectstream/db": "0.100.21",
-    "@effectstream/event-client": "0.100.21",
-    "@effectstream/event-server": "0.100.21",
-    "@effectstream/log": "0.100.21",
-    "@effectstream/sm": "0.100.21",
-    "@effectstream/sync": "0.100.21",
-    "@effectstream/utils": "0.100.21",
+    "@effectstream/concise": "0.100.23",
+    "@effectstream/config": "0.100.23",
+    "@effectstream/coroutine": "0.100.23",
+    "@effectstream/crypto": "0.100.23",
+    "@effectstream/db": "0.100.23",
+    "@effectstream/event-client": "0.100.23",
+    "@effectstream/event-server": "0.100.23",
+    "@effectstream/log": "0.100.23",
+    "@effectstream/sm": "0.100.23",
+    "@effectstream/sync": "0.100.23",
+    "@effectstream/utils": "0.100.23",
     "@fastify/cors": "^11.0.1",
     "@metamask/json-rpc-engine": "^10.0.3",
     "pg": "8.21.0",

package/src/api/http-server.ts

@@ -1,6 +1,7 @@
 import fastify, { type FastifyInstance, type FastifyReply, type FastifyRequest } from "fastify";
 import { evmRpcEngine } from "./rpc-evm/eip1193.ts";
 import { appliedBlockStatus } from "./apply-status.ts";
+import { finalizedStreamStatus } from "./stream-status.ts";
 import type { Pool } from "pg";
 import cors from "@fastify/cors";
 import { run, until } from "effection";
@@ -509,6 +510,14 @@ export const startHttpServer = function* (
           mergeWaiting: p.mergeWaitingForPage,
           mergeDemandRoot: p.mergeDemandRoot ?? null,
         })),
+        // Finalized-block stream backlog: blocks the merge has produced but the
+        // apply loop hasn't drained. `inFlight` is the otherwise-unobservable
+        // subscriber-queue depth, bounded by backpressure (sync/CLAUDE.md Finding #1).
+        // `coalesced` = empty blocks folded away during catch-up.
+        finalizedStream: {
+          inFlight: finalizedStreamStatus.produced - finalizedStreamStatus.consumed,
+          coalesced: finalizedStreamStatus.coalesced,
+        },
         // Apply-stage lag: how old the last-applied block is. Unlike `buf` (fetch
         // backlog) this stays high when the node is write/apply-bound.
         applied: {

package/src/api/stream-status.ts

@@ -0,0 +1,35 @@
+/**
+ * Finalized-block stream backlog for /debug/metrics.
+ *
+ * `finalized-stream.ts` bumps `produced` when the merge sends a block and `consumed`
+ * when the apply loop drains one; their difference is the live subscriber-queue
+ * depth that drives backpressure and is surfaced as `inFlight` on /debug/metrics
+ * (the one structure flagged unbounded in sync/CLAUDE.md Finding #1). `coalesced`
+ * counts empty blocks folded away, so a catch-up test can prove coalescing fired.
+ *
+ * Process-global: one node runs per process, so a module-level singleton avoids
+ * threading a ref through startMerge and the coalescer.
+ */
+export type FinalizedStreamStatus = {
+  produced: number;
+  consumed: number;
+  coalesced: number;
+};
+
+export const finalizedStreamStatus: FinalizedStreamStatus = {
+  produced: 0,
+  consumed: 0,
+  coalesced: 0,
+};
+
+export function recordProduced(): void {
+  finalizedStreamStatus.produced++;
+}
+
+export function recordConsumed(n: number): void {
+  finalizedStreamStatus.consumed += n;
+}
+
+export function recordCoalesced(n: number): void {
+  finalizedStreamStatus.coalesced += n;
+}

package/src/finalized-stream.ts

@@ -0,0 +1,73 @@
+/**
+ * Bounded hand-off queue between the merge (producer) and the apply loop (consumer).
+ *
+ * The merge sends finalized blocks into an effection channel; the apply loop drains
+ * them. effection channel subscribers buffer unbounded, so during deep catch-up the
+ * merge (no I/O) outpaces the apply loop and the queue grows toward the whole backlog
+ * → OOM (sync/CLAUDE.md Finding #1) — especially with empty-block coalescing removing
+ * the per-block apply brake.
+ *
+ * This wraps the channel with backpressure: the producer blocks once `cap` produced
+ * blocks are still unconsumed, and the consumer wakes it as it drains. The cap only
+ * ever throttles the producer (the consumer always keeps pulling), so it can't
+ * deadlock. `inFlight = produced − consumed` is the live queue depth, surfaced on
+ * /debug/metrics.
+ */
+import { type Channel, createChannel, type Operation } from "effection";
+import { conditionVariable } from "@effectstream/utils";
+import type { ChainBlock } from "@effectstream/sync";
+import {
+  finalizedStreamStatus,
+  recordConsumed,
+  recordProduced,
+} from "./api/stream-status.ts";
+import type { FinalizedBlockSubscription } from "./coalesce.ts";
+
+export type BoundedFinalizedStream = {
+  /** Send side — hand to `startMerge`. `send` blocks the producer at the cap. */
+  stream: Channel<ChainBlock, void>;
+  /** Read side — hand to the coalescer / apply loop. Counts drains and wakes the producer. */
+  subscription: FinalizedBlockSubscription;
+};
+
+export function* createBoundedFinalizedStream(
+  cap: number,
+): Operation<BoundedFinalizedStream> {
+  const channel = createChannel<ChainBlock>();
+  // Subscribe BEFORE the caller spawns the merge: effection channels drop sends
+  // with no subscriber, so a fast restart could emit blocks before we subscribe
+  // and stall at the pre-restart height.
+  const subscription = yield* channel;
+
+  const drained = conditionVariable<void>();
+  const inFlight = () =>
+    finalizedStreamStatus.produced - finalizedStreamStatus.consumed;
+
+  // `startMerge` only uses `.send`; gate it. check-then-wait is atomic under
+  // effection's cooperative scheduling, so there's no lost wakeup.
+  const stream: Channel<ChainBlock, void> = {
+    ...channel,
+    *send(block: ChainBlock) {
+      while (inFlight() >= cap) {
+        yield* drained.wait();
+      }
+      recordProduced();
+      yield* channel.send(block);
+    },
+  };
+
+  // Count consumed at the pull point (exact channel depth, decoupled from the
+  // coalescer's run length) and wake the merge once the queue drops below cap.
+  const countedSubscription: FinalizedBlockSubscription = {
+    *next() {
+      const result = yield* subscription.next();
+      if (!result.done) {
+        recordConsumed(1);
+        if (inFlight() < cap) drained.wake();
+      }
+      return result;
+    },
+  };
+
+  return { stream, subscription: countedSubscription };
+}

package/src/main.ts

@@ -23,7 +23,6 @@ import {
 import { startMerge, startSync } from "@effectstream/sync";
 import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
 import {
-  createChannel,
   type Operation,
   sleep,
   spawn,
@@ -37,6 +36,8 @@ import {
 import { createEmptyBlockCoalescer } from "./coalesce.ts";
 import { startHttpServer } from "./api/http-server.ts";
 import { recordAppliedBlock } from "./api/apply-status.ts";
+import { recordCoalesced } from "./api/stream-status.ts";
+import { createBoundedFinalizedStream } from "./finalized-stream.ts";
 import type { StartConfig } from "./types.ts";
 import type { Client } from "pg";
 import type { EffectstreamBlockHash } from "@effectstream/utils";
@@ -102,16 +103,13 @@ export function* start(config: StartConfig): Operation<void> {
     (syncInfo[0]?.network as { blockTimeMS?: number } | undefined)?.blockTimeMS;
   const lagThresholdMs = clockBlockTimeMS != null ? clockBlockTimeMS * 10 : undefined;
 
-  const finalizedBlockStream = createChannel<ChainBlock>();
+  // Bounded hand-off queue between the merge and the apply loop. Backpressure caps
+  // the in-memory queue so deep catch-up can't grow it toward the whole backlog
+  // (see finalized-stream.ts / sync/CLAUDE.md Finding #1).
+  const { stream: finalizedStream, subscription: finalizedBlocks } =
+    yield* createBoundedFinalizedStream(ENV.EFFECTSTREAM_FINALIZED_STREAM_CAP);
 
-  // Subscribe BEFORE spawning the merge: effection channels drop sends with no
-  // subscriber, so a fast restart could emit blocks before we subscribe and
-  // stall at the pre-restart height.
-  const finalizedBlocks = yield* finalizedBlockStream;
-  // NOTE: this subscriber's queue is unbounded — during deep catch-up the merge
-  // can outpace block processing and grow it (Fix C backpressure, sync/CLAUDE.md).
-
-  yield* spawn(() => startMerge(syncProtocols, finalizedBlockStream));
+  yield* spawn(() => startMerge(syncProtocols, finalizedStream));
 
   const heartbeatIntervalMs = 60_000;
   yield* spawn(function* () {
@@ -161,6 +159,9 @@ export function* start(config: StartConfig): Operation<void> {
     lagThresholdMs,
     getPreviousBlockHash: () => blockHash,
     onFlush: (endpoint, length) => {
+      // `consumed` is counted at the subscription pull point (above), not here, so
+      // it reflects the channel depth regardless of how many blocks a run folds.
+      recordCoalesced(length);
       recordAppliedBlock(endpoint);
       emitLatestBlocks(
         endpoint.blockNumber,

package/test/reproduction/coalesce-memory.test.ts

@@ -0,0 +1,181 @@
+/**
+ * Memory / queue-depth characterization for empty-block coalescing during a deep
+ * catch-up (sync/CLAUDE.md Finding #1 + Fix E).
+ *
+ * Coalescing is a consumer-side speedup: folding empty catch-up blocks removes the
+ * per-block STF + DB-commit bottleneck that otherwise acts as accidental
+ * backpressure. The worry is that, with that brake gone, the fetchers race to tip
+ * and either (a) balloon the uncapped `finalizedBlockStream` subscriber queue or
+ * (b) keep a parallel chain in the cap-lifted merge-demand exemption long enough to
+ * grow its Deque toward the whole backlog.
+ *
+ * This test drives a deep, all-empty catch-up with coalescing ON and asserts hard
+ * ceilings on the stream backlog (`inFlight`) and per-protocol buffers, plus a soft
+ * memory guard and a liveness check. Green → coalescing keeps everything bounded
+ * (regression guard). Red on the backlog/buffer bound → reproduces the leak, and the
+ * captured peak series localizes it.
+ *
+ * Postgres-only (chosen backend): higher-fidelity apply throughput than PGLite.
+ * Skips when no Postgres binary is on PATH.
+ */
+import { afterEach, beforeEach, expect, test } from "bun:test";
+import {
+  ConfigBuilder,
+  ConfigNetworkType,
+  ConfigSyncProtocolType,
+} from "@effectstream/config";
+import { TestChainControl } from "@effectstream/sync";
+import {
+  bufferSize,
+  type Harness,
+  latestFinalizedHeight,
+  postgresAvailable,
+  type RunningNode,
+  setupHarness,
+  sleep,
+  TEST_PRIMITIVE_TYPE,
+} from "./harness.ts";
+
+const RUN = postgresAvailable();
+
+const START_TIME = 1_700_000_000_000; // ~Nov 2023: every catch-up block is "old"
+const MAIN_BLOCK_MS = 1000;
+const PAR_BLOCK_MS = 100; // 10 parallel blocks per root slot
+const STEP_SIZE = 1000;
+const CAP = 3000;
+
+// A deep backlog: ~15k root slots the one-block-per-txn merge could never drain to
+// in time, so without coalescing the node would crawl — and any unbounded structure
+// would grow toward the whole backlog. Coalescing must fold the empties and keep
+// memory flat.
+const MAIN_TIP = 15_000;
+const PAR_TIP = MAIN_TIP * (MAIN_BLOCK_MS / PAR_BLOCK_MS); // cover the same wall range
+
+// Buffer settles in [~CAP, CAP + stepSize]; allow one overshoot chunk + slack.
+const BUFFER_BOUND = CAP + STEP_SIZE + 200;
+// The stream backlog must stay far below the whole-backlog (~MAIN_TIP) failure mode.
+// A healthy fast consumer keeps this in the low hundreds; we allow generous slack.
+const QUEUE_BOUND = 5_000;
+// Liveness: the node must catch up essentially to the tip (no deadlock / no crawl).
+const LIVE_TARGET = Math.floor(MAIN_TIP * 0.9);
+
+const MAIN = "memMainClock";
+const PAR = "memParallelP";
+
+function buildConfig() {
+  return new ConfigBuilder()
+    .setNamespace((b) => b.setSecurityNamespace("sync-coalesce-mem"))
+    .buildNetworks((b) =>
+      b
+        .addNetwork({
+          name: "clock",
+          type: ConfigNetworkType.TEST,
+          startTime: START_TIME,
+          blockTimeMS: MAIN_BLOCK_MS,
+        })
+        .addNetwork({
+          name: "chainP",
+          type: ConfigNetworkType.TEST,
+          startTime: START_TIME,
+          blockTimeMS: PAR_BLOCK_MS,
+        })
+    )
+    .buildDeployments((b) => b)
+    .buildSyncProtocols((b) =>
+      b
+        .addMain(
+          (n) => n.clock,
+          () => ({
+            name: MAIN,
+            type: ConfigSyncProtocolType.TEST_MAIN,
+            startBlockHeight: 1,
+            pollingInterval: 30,
+            stepSize: STEP_SIZE,
+            maxBufferedPages: CAP,
+          }),
+        )
+        .addParallel(
+          (n) => (n as any).chainP,
+          () => ({
+            name: PAR,
+            type: ConfigSyncProtocolType.TEST_PARALLEL,
+            startBlockHeight: 1,
+            pollingInterval: 30,
+            stepSize: STEP_SIZE,
+            delayMs: 0,
+            events: [], // no primitives → every block is coalescable
+            maxBufferedPages: CAP,
+          }),
+        )
+    )
+    .buildPrimitives((b) =>
+      b.addPrimitive(
+        (sp) => (sp as any)[PAR],
+        () => ({ name: "noEvtMem", type: TEST_PRIMITIVE_TYPE, startBlockHeight: 1 }),
+      )
+    )
+    .build();
+}
+
+let h: Harness;
+beforeEach(async () => {
+  if (!RUN) return;
+  h = await setupHarness({ backend: "postgres" });
+});
+afterEach(async () => {
+  if (!RUN) return;
+  await h?.teardown();
+});
+
+test("coalescing keeps stream backlog & buffers bounded during a deep empty catch-up", async () => {
+  if (!RUN) {
+    console.warn("coalesce-memory: skipped (no Postgres binary on PATH)");
+    return;
+  }
+
+  TestChainControl.setTip(MAIN, MAIN_TIP);
+  TestChainControl.setTip(PAR, PAR_TIP);
+
+  const node: RunningNode = await h.runNode({
+    config: buildConfig(),
+    apiPort: 19181,
+    coalesce: true,
+  });
+
+  let peakMain = 0;
+  let peakPar = 0;
+  let peakInFlight = 0;
+  let lastCoalesced = 0;
+  let lastApplied = 0;
+
+  try {
+    const startMs = Date.now();
+    const deadline = startMs + 110_000;
+    while (Date.now() < deadline) {
+      peakMain = Math.max(peakMain, bufferSize(node, MAIN));
+      peakPar = Math.max(peakPar, bufferSize(node, PAR));
+      peakInFlight = Math.max(peakInFlight, node.inFlight);
+      lastCoalesced = node.coalescedBlocks;
+      lastApplied = (await latestFinalizedHeight(h.pool)) ?? lastApplied;
+      if (lastApplied >= LIVE_TARGET) break; // caught up — bounded steady state captured
+      await sleep(50);
+    }
+  } finally {
+    console.log(
+      `coalesce-memory: peakInFlight=${peakInFlight} mainBuf=${peakMain} ` +
+        `parBuf=${peakPar} coalesced=${lastCoalesced} applied=${lastApplied}`,
+    );
+    await node.stop();
+  }
+
+  // 1. Coalescing actually fired (else this is a no-op test).
+  expect(lastCoalesced).toBeGreaterThan(1000);
+  // 2. Stream backlog stayed bounded — the merge did NOT outrun the consumer
+  //    toward the whole backlog (the uncapped subscriber queue, Finding #1). This
+  //    bounded queue is what keeps memory flat; no separate RSS check is needed.
+  expect(peakInFlight).toBeLessThanOrEqual(QUEUE_BOUND);
+  // 3. Per-protocol buffers stayed at the cap, even under coalescing's fast consumer.
+  expect(Math.max(peakMain, peakPar)).toBeLessThanOrEqual(BUFFER_BOUND);
+  // 4. Liveness: the node caught up to the tip (no deadlock / no crawl).
+  expect(lastApplied).toBeGreaterThanOrEqual(LIVE_TARGET);
+}, 120_000);

package/test/reproduction/harness.ts

@@ -32,6 +32,8 @@ import { toSyncProtocolWithNetwork } from "@effectstream/config";
 export type RunNodeOpts = {
   config: any;
   apiPort: number;
+  /** Enable/disable empty-block coalescing for this long-lived node. */
+  coalesce?: boolean;
 };
 
 export type RunningNode = {
@@ -40,6 +42,10 @@ export type RunningNode = {
   bufferSizes: Record<string, number>;
   pauses: Record<string, number>;
   ownBlocks: Record<string, number | null>;
+  /** finalizedBlockStream backlog (produced − consumed); 0 until first scrape. */
+  inFlight: number;
+  /** Empty blocks folded away by the coalescer so far. */
+  coalescedBlocks: number;
   stop: () => Promise<void>;
   syncProtocols: () => any[];
 };
@@ -230,8 +236,6 @@ function ensureCluster(): Promise<Cluster> {
     const dataDir = mkdtempSync(join(tmpdir(), "repro-pg-"));
     const port = await freePort();
 
-    writeFileSync(join(dataDir, "repro-owner.pid"), String(process.pid));
-
     execFileSync(join(bin, "initdb"), [
       "-D", dataDir,
       "-U", "postgres",
@@ -240,6 +244,10 @@ function ensureCluster(): Promise<Cluster> {
       "--encoding=UTF8",
       "--no-instructions",
     ], { env, stdio: "ignore" });
+
+    // Write our owner marker AFTER initdb — it refuses a non-empty data dir, so
+    // this must not exist beforehand. Read later by cleanupOrphanedClusters.
+    writeFileSync(join(dataDir, "repro-owner.pid"), String(process.pid));
     appendFileSync(join(dataDir, "postgresql.conf"), `\nport = ${port}\n`);
     execFileSync(join(bin, "pg_ctl"), [
       "-D", dataDir,
@@ -384,6 +392,7 @@ function makeRunNode(
       target: 999999,
       apiPort: opts.apiPort,
       config: opts.config,
+      coalesce: opts.coalesce,
     };
 
     const child = spawn("bun", [RUNNER, JSON.stringify(spec)], {
@@ -401,6 +410,8 @@ function makeRunNode(
       bufferSizes: {},
       pauses: {},
       ownBlocks: {},
+      inFlight: 0,
+      coalescedBlocks: 0,
       syncProtocols() {
         return Object.entries(this.bufferSizes).map(([name, buf]) => ({
           name,
@@ -433,6 +444,10 @@ function makeRunNode(
                 node.ownBlocks[p.name] = p.ownBlockNumber;
               }
             }
+            if (data?.finalizedStream) {
+              node.inFlight = data.finalizedStream.inFlight ?? 0;
+              node.coalescedBlocks = data.finalizedStream.coalesced ?? 0;
+            }
           }
         } catch {
           // ignore