@effectstream/runtime 0.100.20 → 0.100.21

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@effectstream/runtime",
-  "version": "0.100.20",
+  "version": "0.100.21",
   "license": "MIT",
   "homepage": "https://effectstream.github.io/docs/",
   "repository": {
@@ -13,20 +13,20 @@
   },
   "exports": "./src/mod.ts",
   "dependencies": {
-    "@effectstream/concise": "0.100.20",
-    "@effectstream/config": "0.100.20",
-    "@effectstream/coroutine": "0.100.20",
-    "@effectstream/crypto": "0.100.20",
-    "@effectstream/db": "0.100.20",
-    "@effectstream/event-client": "0.100.20",
-    "@effectstream/event-server": "0.100.20",
-    "@effectstream/log": "0.100.20",
-    "@effectstream/sm": "0.100.20",
-    "@effectstream/sync": "0.100.20",
-    "@effectstream/utils": "0.100.20",
+    "@effectstream/concise": "0.100.21",
+    "@effectstream/config": "0.100.21",
+    "@effectstream/coroutine": "0.100.21",
+    "@effectstream/crypto": "0.100.21",
+    "@effectstream/db": "0.100.21",
+    "@effectstream/event-client": "0.100.21",
+    "@effectstream/event-server": "0.100.21",
+    "@effectstream/log": "0.100.21",
+    "@effectstream/sm": "0.100.21",
+    "@effectstream/sync": "0.100.21",
+    "@effectstream/utils": "0.100.21",
     "@fastify/cors": "^11.0.1",
     "@metamask/json-rpc-engine": "^10.0.3",
-    "pg": "8.17.2",
+    "pg": "8.21.0",
     "effection": "^3.5.0",
     "@opentelemetry/api": "^1.9.0",
     "@opentelemetry/auto-instrumentations-node": "^0.54.0",

package/src/api/http-server.ts

@@ -504,6 +504,10 @@ export const startHttpServer = function* (
           pausedNow: p.pausedNow,
           pauses: p.backpressurePauses,
           pausedMs: p.backpressurePausedMs,
+          // Merge-demand exemption: when true the merge is gated on this chain's
+          // page and the cap is lifted.
+          mergeWaiting: p.mergeWaitingForPage,
+          mergeDemandRoot: p.mergeDemandRoot ?? null,
         })),
         // Apply-stage lag: how old the last-applied block is. Unlike `buf` (fetch
         // backlog) this stays high when the node is write/apply-bound.

package/src/main.ts

@@ -6,11 +6,13 @@ import {
 import {
   acquireDBMutex,
   createDynamicTables,
+  detectCapabilities,
   getConnection,
   getLastNonEmptyBlockHash,
   releaseDBMutex,
   resetPublicTables,
   runSnapshotLoop,
+  selectViewStrategy,
 } from "@effectstream/db";
 import { EventBroker } from "@effectstream/event-server";
 import { ENV } from "@effectstream/utils/node-env";
@@ -103,12 +105,11 @@ export function* start(config: StartConfig): Operation<void> {
   const finalizedBlockStream = createChannel<ChainBlock>();
 
   // Subscribe BEFORE spawning the merge: effection channels drop sends with no
-  // active subscriber, so a fast restart (in-memory state, no RPC wait) could
-  // emit blocks before we subscribe and silently stall at the pre-restart height.
+  // subscriber, so a fast restart could emit blocks before we subscribe and
+  // stall at the pre-restart height.
   const finalizedBlocks = yield* finalizedBlockStream;
-  // NOTE: this subscriber's queue is unbounded. During deep catch-up the merge
-  // can outpace the per-block processing below and grow it without bound — the
-  // downstream side of the Fix C backpressure issue (see sync/CLAUDE.md).
+  // NOTE: this subscriber's queue is unbounded — during deep catch-up the merge
+  // can outpace block processing and grow it (Fix C backpressure, sync/CLAUDE.md).
 
   yield* spawn(() => startMerge(syncProtocols, finalizedBlockStream));
 
@@ -182,10 +183,9 @@ export function* start(config: StartConfig): Operation<void> {
     const value = yield* coalescer.advance();
     if (value === undefined) break;
 
-    // processFinalizedBlockWithRetry owns connection checkout, the per-block
-    // DB mutex (PGLite), and transient-pg retry/backoff. App events it
-    // collects are flushed below ONLY after it returns — i.e. strictly after
-    // the block's COMMIT.
+    // Owns connection checkout, the per-block DB mutex (PGLite), and
+    // transient-pg retry/backoff. App events flush below only after it
+    // returns — strictly after the block's COMMIT.
     const result = yield* processFinalizedBlockWithRetry(
       value,
       config,
@@ -202,21 +202,14 @@ export function* start(config: StartConfig): Operation<void> {
     // Used to emit & log the block range for each protocol.
     const contentBlocksForProtocol = getRangesForSyncProtocols(value);
 
-    // Fire-and-forget: do not stall the sync loop on broker acks.
+    // Fire-and-forget: don't stall the sync loop on broker acks. Ordering is
+    // safe without a wrapper queue — Opifex assigns packet IDs synchronously in
+    // publish() and net.Socket serializes writes in call order, so MQTT 3.1.1
+    // §4.6 in-order delivery holds per (publisher, topic, QoS).
     //
-    // Opifex's TcpClient assigns packet IDs synchronously in client.publish()
-    // and writes to the socket via Node's net.Socket, which has its own write
-    // queue — so concurrent publish() calls land on the wire in call order.
-    // MQTT 3.1.1 §4.6 then guarantees in-order delivery to each subscriber
-    // per (publisher, topic, QoS). We don't need a wrapper queue.
-    //
-    // TODO(scaling): the broker runs in-process (see `new EventBroker(...)`
-    // above). Fan-out to N subscribers consumes the same event loop as block
-    // processing. At ~thousands of subscribers, consider:
-    //   1) moving the broker to a worker / separate process,
-    //   2) a publisher-side circuit breaker if ctx.unresolvedPublish.size > N
-    //      (drop events with a counter for observability).
-    // Today: localhost, in-process, expected O(10²) subscribers — non-issue.
+    // TODO(scaling): the broker is in-process, so fan-out shares this event
+    // loop. At ~thousands of subscribers, move it to a worker and/or add a
+    // publisher-side circuit breaker. Today: localhost, O(10²) subs — non-issue.
     emitLatestBlocks(
       value.blockNumber,
       value.timestamp,
@@ -268,13 +261,9 @@ function getRangesForSyncProtocols(value: ChainBlock): Record<string, [number, n
 }
 
 /**
- * Publish built-in block-level events.
- *
- * Fire-and-forget for the same reasons as the app-event flush above
- * (do not stall the sync loop; ordering preserved by Opifex + MQTT 3.1.1).
- *
- * Errors are caught locally so they don't escape to the global
- * `unhandledrejection` handler without context.
+ * Publish built-in block-level events. Fire-and-forget like the app-event flush
+ * above (don't stall the sync loop; ordering held by Opifex + MQTT 3.1.1).
+ * Errors are caught locally so they don't reach the global handler contextless.
  */
 function emitLatestBlocks(
   rollUpBlockHeight: number,
@@ -325,41 +314,48 @@ function* startup(
     });
   });
 
-  
   yield* acquireDBMutex(`startup-node`);
+  // `finally` releases the mutex on error/cancellation too; a throw in any step
+  // below would otherwise leak it and deadlock every later DB operation.
+  try {
+    // Dev-only reset of user-owned public tables
+    if (config.dev?.resetPublicData) {
+      yield* resetPublicTables(dbConn as any); // Client,
+    }
 
-  // Dev-only reset of user-owned public tables
-  if (config.dev?.resetPublicData) {
-    yield* resetPublicTables(dbConn as any); // Client,
-  }
+    // When the node is started, we apply system migrations.
+    // Either system initial migrations, or migrations given a Paima Engine Update.
+    yield* applySystemMigrations(
+      config.appVersion,
+      versionInfo,
+      lastBlockHeight,
+      dbConn,
+      config.migrations,
+    );
 
-  // When the node is started, we apply system migrations.
-  // Either system initial migrations, or migrations given a Paima Engine Update.
-  yield* applySystemMigrations(
-    config.appVersion,
-    versionInfo,
-    lastBlockHeight,
-    dbConn,
-    config.migrations,
-  );
+    // Validate that immutable config fields (e.g. NTP startTime, startBlockHeight)
+    // have not changed since the last run. Persists a snapshot on first start.
+    // Must run after system migrations so the snapshot table exists.
+    yield* validateAndSnapshotConfig(syncInfo, dbConn);
 
-  // Validate that immutable config fields (e.g. NTP startTime, startBlockHeight)
-  // have not changed since the last run. Persists a snapshot on first start.
-  // Must run after system migrations so the snapshot table exists.
-  yield* validateAndSnapshotConfig(syncInfo, dbConn);
+    const syncProtocols = yield* genSyncProtocols(dbConn as any, // Client,
+      syncInfo);
 
-  const syncProtocols = yield* genSyncProtocols(dbConn as any, // Client,
-    syncInfo);
+    const capabilities = yield* until(detectCapabilities(dbConn as any));
+    const viewStrategy = selectViewStrategy(capabilities);
 
-  yield* createDynamicTables(
-    versionInfo,
-    lastBlockHeight,
-    dbConn as any, // Client,
-    syncProtocols,
-  );
+    yield* createDynamicTables(
+      versionInfo,
+      lastBlockHeight,
+      dbConn as any, // Client,
+      syncProtocols,
+      viewStrategy,
+    );
 
-  releaseDBMutex(`startup-node`);
-  return syncProtocols;
+    return syncProtocols;
+  } finally {
+    releaseDBMutex(`startup-node`);
+  }
 }
 
 // Convert the primitive config to the final primitive instance

package/test/reproduction/buffering.test.ts

@@ -8,6 +8,10 @@
  *  1b — head-of-line blocking: a stalled slow chain STILL halts block production
  *       (the merge gate is correct), but the fast chain's buffer is now bounded by
  *       the cap instead of ballooning behind the head-of-line block.
+ *  1c — skip-ahead: the merge-demand exemption lifts the cap while the merge is gated
+ *       on a chain whose page is far behind the (skipped-ahead) root, so it advances
+ *       instead of deadlocking.
+ *  1d — density: the same exemption when a parallel chain is finer-grained than the cap.
  *
  * Before the fix these buffers reached ~49k–50k (see
  * e2e/perf/results/BACKPRESSURE-BASELINE.md); these tests assert they now
@@ -177,7 +181,7 @@ test("1a: backpressure caps the buffer while the node keeps applying blocks", as
   let mainPauses = 0;
   try {
     const startMs = Date.now();
-    const deadline = startMs + 6000;
+    const deadline = startMs + 30_000;
     while (Date.now() < deadline) {
       const wall = Date.now();
       const mainBuf = bufferSize(node, MAIN_1A);
@@ -327,7 +331,8 @@ test("1b: a stalled chain still halts production, but the fast chain's buffer st
   let fastPauses = 0;
   try {
     const startMs = Date.now();
-    const deadline = startMs + 15_000;
+    // Generous for slow CI; exits early once the stall + capped sibling are captured.
+    const deadline = startMs + 30_000;
     let lastHeight = -1;
     let lastChange = Date.now();
     while (Date.now() < deadline) {
@@ -387,3 +392,259 @@ test("1b: a stalled chain still halts production, but the fast chain's buffer st
   // ...and the cap actually fired on the fast chain (observability metric populated).
   expect(fastPauses).toBeGreaterThan(0);
 }, 60_000);
+
+// ── 1c — skip-ahead: merge-demand exemption keeps the cap from deadlocking ────────
+//
+// The root (main clock) starts far ahead of a parallel chain that starts at block 1.
+// To produce the first root block at τ (main page SKIP_START) the merge needs the
+// parallel page PAST τ — so the fetcher must scan ~SKIP_START blocks, more than the
+// cap. The exemption lifts the cap while the merge is gated on this chain, so it
+// reaches τ and production proceeds (without it the node wedges on the first block).
+
+const MAIN_1C = "mainClock1c";
+const PAR_1C = "parallelP1c";
+// Main genesis page (skip-ahead); cap far below it so the parallel chain must buffer
+// past the cap to reach τ.
+const SKIP_START = 300;
+const SMALL_STEP = 50;
+const SMALL_CAP = 100;
+// Liveness: past the first non-empty block at SKIP_START.
+const LIVE_TARGET_1C = SKIP_START + 10;
+// Bounded by necessity: at most the skip window + one in-flight chunk.
+const SKIP_BUFFER_BOUND = SKIP_START + SMALL_STEP + 200;
+
+function buildConfig1c() {
+  return new ConfigBuilder()
+    .setNamespace((b) => b.setSecurityNamespace("sync-buffering-1c"))
+    .buildNetworks((b) =>
+      b
+        .addNetwork({
+          name: "clock",
+          type: ConfigNetworkType.TEST,
+          startTime: START_TIME,
+          blockTimeMS: BLOCK_TIME_MS,
+        })
+        .addNetwork({
+          name: "chainP",
+          type: ConfigNetworkType.TEST,
+          startTime: START_TIME,
+          blockTimeMS: BLOCK_TIME_MS,
+        })
+    )
+    .buildDeployments((b) => b)
+    .buildSyncProtocols((b) =>
+      b
+        .addMain(
+          (n) => n.clock,
+          () => ({
+            name: MAIN_1C,
+            type: ConfigSyncProtocolType.TEST_MAIN,
+            // Skipped far ahead of the parallel chain (the recovery skip-ahead).
+            startBlockHeight: SKIP_START,
+            pollingInterval: 30,
+            stepSize: SMALL_STEP,
+            maxBufferedPages: SMALL_CAP,
+          }),
+        )
+        .addParallel(
+          (n) => (n as any).chainP,
+          () => ({
+            name: PAR_1C,
+            type: ConfigSyncProtocolType.TEST_PARALLEL,
+            startBlockHeight: 1,
+            pollingInterval: 30,
+            stepSize: SMALL_STEP,
+            delayMs: 0,
+            events: [],
+            maxBufferedPages: SMALL_CAP,
+          }),
+        )
+    )
+    .buildPrimitives((b) =>
+      b.addPrimitive(
+        (sp) => (sp as any)[PAR_1C],
+        () => ({ name: "noEvt1c", type: TEST_PRIMITIVE_TYPE, startBlockHeight: 1 }),
+      )
+    )
+    .build();
+}
+
+test("1c: a skipped-ahead root does NOT deadlock the merge on the first non-empty block", async () => {
+  // Main skipped to SKIP_START; parallel starts at 1 with a high tip.
+  TestChainControl.setTip(MAIN_1C, SKIP_START + 100);
+  TestChainControl.setTip(PAR_1C, 10_000);
+
+  const node = await h.runNode({ config: buildConfig1c(), apiPort: 19143 });
+  const samples: BufSample[] = [];
+  let peakPar = 0;
+  let lastApplied = 0;
+  try {
+    const startMs = Date.now();
+    // Generous for slow CI; exits early once the liveness target is reached.
+    const deadline = startMs + 30_000;
+    while (Date.now() < deadline) {
+      const wall = Date.now();
+      const mainBuf = bufferSize(node, MAIN_1C);
+      const parBuf = bufferSize(node, PAR_1C);
+      const appliedBlock = await latestFinalizedHeight(h.pool);
+      samples.push({
+        t: wall - startMs,
+        wall,
+        mainBuf,
+        evmBuf: parBuf,
+        mainOwnBlock: ownBlock(node, MAIN_1C),
+        evmOwnBlock: ownBlock(node, PAR_1C),
+        rss: process.memoryUsage().rss,
+        appliedBlock,
+      });
+      peakPar = Math.max(peakPar, parBuf);
+      lastApplied = appliedBlock ?? lastApplied;
+      // Got past the first non-empty block → the deadlock is broken; stop early.
+      if (lastApplied >= LIVE_TARGET_1C) break;
+      await sleep(50);
+    }
+  } finally {
+    writeArtifact("buffering-1c", samples, {
+      mode: "1c-skip-ahead-deadlock",
+      stepSize: SMALL_STEP,
+      skipStart: SKIP_START,
+      cap: SMALL_CAP,
+      liveTarget: LIVE_TARGET_1C,
+      peakParBuf: peakPar,
+      lastApplied,
+    });
+    await node.stop();
+  }
+
+  // Liveness: production advanced past the first non-empty block (no wedge).
+  expect(lastApplied).toBeGreaterThanOrEqual(LIVE_TARGET_1C);
+  // Bounded by necessity: only the skip window + one chunk, not the whole tip.
+  expect(peakPar).toBeLessThanOrEqual(SKIP_BUFFER_BOUND);
+}, 60_000);
+
+// ── 1d — density: same exemption when a chain is finer-grained than the cap ───────
+//
+// Same circular wait without a skip-ahead: a parallel chain far finer than the root
+// (100 parallel blocks per root slot). Each root block needs the parallel page past
+// the slot boundary (~PER_SLOT buffered items), but the cap is below that. The
+// exemption lets the per-slot window buffer and drain each slot (without it the node
+// wedges on the first block).
+
+const MAIN_1D = "mainClock1d";
+const PAR_1D = "parallelP1d";
+const DENSE_BLOCK_MS = 10; // 100 parallel blocks per 1000ms root slot
+const PER_SLOT = BLOCK_TIME_MS / DENSE_BLOCK_MS; // 100 parallel items per root slot
+// Step + cap small enough that cap + one overshoot chunk (~40) stays below PER_SLOT
+// (100), so the fetcher pauses before its page can cross the slot boundary.
+const DENSE_STEP = 10;
+const CAP_1D = 30;
+const LIVE_TARGET_1D = 20;
+// Bounded by the per-slot need plus one chunk — not the whole tip.
+const DENSE_BUFFER_BOUND = PER_SLOT + DENSE_STEP + 150;
+
+function buildConfig1d() {
+  return new ConfigBuilder()
+    .setNamespace((b) => b.setSecurityNamespace("sync-buffering-1d"))
+    .buildNetworks((b) =>
+      b
+        .addNetwork({
+          name: "clock",
+          type: ConfigNetworkType.TEST,
+          startTime: START_TIME,
+          blockTimeMS: BLOCK_TIME_MS,
+        })
+        .addNetwork({
+          name: "chainDense",
+          type: ConfigNetworkType.TEST,
+          startTime: START_TIME,
+          blockTimeMS: DENSE_BLOCK_MS,
+        })
+    )
+    .buildDeployments((b) => b)
+    .buildSyncProtocols((b) =>
+      b
+        .addMain(
+          (n) => n.clock,
+          () => ({
+            name: MAIN_1D,
+            type: ConfigSyncProtocolType.TEST_MAIN,
+            startBlockHeight: 1,
+            pollingInterval: 30,
+            stepSize: DENSE_STEP,
+            maxBufferedPages: 200,
+          }),
+        )
+        .addParallel(
+          (n) => (n as any).chainDense,
+          () => ({
+            name: PAR_1D,
+            type: ConfigSyncProtocolType.TEST_PARALLEL,
+            startBlockHeight: 1,
+            pollingInterval: 30,
+            stepSize: DENSE_STEP,
+            delayMs: 0,
+            events: [],
+            maxBufferedPages: CAP_1D,
+          }),
+        )
+    )
+    .buildPrimitives((b) =>
+      b.addPrimitive(
+        (sp) => (sp as any)[PAR_1D],
+        () => ({ name: "noEvt1d", type: TEST_PRIMITIVE_TYPE, startBlockHeight: 1 }),
+      )
+    )
+    .build();
+}
+
+test("1d: a parallel chain denser than the cap does NOT deadlock the merge", async () => {
+  TestChainControl.setTip(MAIN_1D, 60);
+  TestChainControl.setTip(PAR_1D, 10_000);
+
+  const node = await h.runNode({ config: buildConfig1d(), apiPort: 19144 });
+  const samples: BufSample[] = [];
+  let peakPar = 0;
+  let lastApplied = 0;
+  try {
+    const startMs = Date.now();
+    // Generous for slow CI; exits early once the liveness target is reached.
+    const deadline = startMs + 30_000;
+    while (Date.now() < deadline) {
+      const wall = Date.now();
+      const mainBuf = bufferSize(node, MAIN_1D);
+      const parBuf = bufferSize(node, PAR_1D);
+      const appliedBlock = await latestFinalizedHeight(h.pool);
+      samples.push({
+        t: wall - startMs,
+        wall,
+        mainBuf,
+        evmBuf: parBuf,
+        mainOwnBlock: ownBlock(node, MAIN_1D),
+        evmOwnBlock: ownBlock(node, PAR_1D),
+        rss: process.memoryUsage().rss,
+        appliedBlock,
+      });
+      peakPar = Math.max(peakPar, parBuf);
+      lastApplied = appliedBlock ?? lastApplied;
+      if (lastApplied >= LIVE_TARGET_1D) break;
+      await sleep(50);
+    }
+  } finally {
+    writeArtifact("buffering-1d", samples, {
+      mode: "1d-density-deadlock",
+      stepSize: SMALL_STEP,
+      denseBlockMs: DENSE_BLOCK_MS,
+      perSlot: PER_SLOT,
+      cap: CAP_1D,
+      liveTarget: LIVE_TARGET_1D,
+      peakParBuf: peakPar,
+      lastApplied,
+    });
+    await node.stop();
+  }
+
+  // Liveness: the node produces blocks (no wedge on block 1).
+  expect(lastApplied).toBeGreaterThanOrEqual(LIVE_TARGET_1D);
+  // Bounded by the per-slot need: ~one slot of parallel data plus one chunk.
+  expect(peakPar).toBeLessThanOrEqual(DENSE_BUFFER_BOUND);
+}, 60_000);