@effectstream/runtime 0.100.18 → 0.100.20

package/src/main.ts

@@ -22,15 +22,19 @@ import { startMerge, startSync } from "@effectstream/sync";
 import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
 import {
   createChannel,
-  each,
   type Operation,
   sleep,
   spawn,
   until,
 } from "effection";
 import { initTelemetry } from "./telemetry.ts";
-import { type PendingEvent, processFinalizedBlock } from "./process-blocks.ts";
+import {
+  type PendingEvent,
+  processFinalizedBlockWithRetry,
+} from "./process-blocks.ts";
+import { createEmptyBlockCoalescer } from "./coalesce.ts";
 import { startHttpServer } from "./api/http-server.ts";
+import { recordAppliedBlock } from "./api/apply-status.ts";
 import type { StartConfig } from "./types.ts";
 import type { Client } from "pg";
 import type { EffectstreamBlockHash } from "@effectstream/utils";
@@ -62,6 +66,9 @@ export function* start(config: StartConfig): Operation<void> {
   const syncProtocols = yield* startup(dbConn as any, // Client,
     syncInfo, config);
 
+  // Test-only: surface live sync protocols (e.g. for buffer-size assertions).
+  config.dev?.onStarted?.({ syncProtocols });
+
   log.remote(
     ComponentNames.EFFECTSTREAM_RUNTIME,
     [],
@@ -86,12 +93,23 @@ export function* start(config: StartConfig): Operation<void> {
     );
   });
 
+  // 10× main clock block time (NTP if present, else protocol 0). Undefined disables lag gating.
   const ntpConfig = syncInfo.find(s => s.networkType === ConfigNetworkType.NTP);
-  const ntpBlockTimeMS = (ntpConfig?.network as { blockTimeMS: number } | undefined)?.blockTimeMS;
-  const lagThresholdMs = ntpBlockTimeMS != null ? ntpBlockTimeMS * 10 : undefined;
+  const clockBlockTimeMS =
+    (ntpConfig?.network as { blockTimeMS?: number } | undefined)?.blockTimeMS ??
+    (syncInfo[0]?.network as { blockTimeMS?: number } | undefined)?.blockTimeMS;
+  const lagThresholdMs = clockBlockTimeMS != null ? clockBlockTimeMS * 10 : undefined;
 
   const finalizedBlockStream = createChannel<ChainBlock>();
 
+  // Subscribe BEFORE spawning the merge: effection channels drop sends with no
+  // active subscriber, so a fast restart (in-memory state, no RPC wait) could
+  // emit blocks before we subscribe and silently stall at the pre-restart height.
+  const finalizedBlocks = yield* finalizedBlockStream;
+  // NOTE: this subscriber's queue is unbounded. During deep catch-up the merge
+  // can outpace the per-block processing below and grow it without bound — the
+  // downstream side of the Fix C backpressure issue (see sync/CLAUDE.md).
+
   yield* spawn(() => startMerge(syncProtocols, finalizedBlockStream));
 
   const heartbeatIntervalMs = 60_000;
@@ -134,37 +152,53 @@ export function* start(config: StartConfig): Operation<void> {
     yield* spawn(() => runSnapshotLoop(config.snapshotConfig!));
   }
 
-  for (const value of yield* each(finalizedBlockStream)) {
-    // We request a dbClient for a non-shared dbConn object.
-    // For PGLite, this is not enough, as the can only be one connection at a time.
-    // So we request a DBMutex as well.
-    let dbClient: Client | undefined;
-    // Custom app events emitted by STFs during this block. Collected by
-    // processFinalizedBlock and flushed below ONLY after it returns — i.e.
-    // strictly after the block's COMMIT. See plan invariant I1.
-    let blockAppEvents: PendingEvent[] = [];
-    try {
-      yield* acquireDBMutex(`processing-blocks:${value.blockNumber}`);
-      dbClient = yield* until((dbConn as any).connect()); // Client,
-
-      const result = yield* processFinalizedBlock(
-        value,
-        config,
-        dbClient as any, // Client,
-        blockHash,
+  const coalescer = createEmptyBlockCoalescer({
+    enabled: ENV.EFFECTSTREAM_COALESCE_EMPTY_BLOCKS,
+    subscription: finalizedBlocks,
+    pool: dbConn as any, // Pool,
+    migrations: config.migrations,
+    lagThresholdMs,
+    getPreviousBlockHash: () => blockHash,
+    onFlush: (endpoint, length) => {
+      recordAppliedBlock(endpoint);
+      emitLatestBlocks(
+        endpoint.blockNumber,
+        endpoint.timestamp,
+        getRangesForSyncProtocols(endpoint),
+      );
+      log.local(
+        ComponentNames.EFFECTSTREAM_SYNC,
+        "block-merge",
+        SeverityNumber.INFO,
+        (l) =>
+          l(
+            `coalesced ${length} empty block(s) → block ${endpoint.blockNumber}`,
+          ),
       );
-      const resultHash = result.blockHash;
-      blockAppEvents = result.events;
-      if (resultHash !== "0x0") {
-        blockHash = resultHash;
-      }
-    } finally {
-      releaseDBMutex(`processing-blocks:${value.blockNumber}`);
-      if (dbClient) {
-        (dbClient as any).release(); // Client,
-      }
+    },
+  });
+
+  while (true) {
+    const value = yield* coalescer.advance();
+    if (value === undefined) break;
+
+    // processFinalizedBlockWithRetry owns connection checkout, the per-block
+    // DB mutex (PGLite), and transient-pg retry/backoff. App events it
+    // collects are flushed below ONLY after it returns — i.e. strictly after
+    // the block's COMMIT.
+    const result = yield* processFinalizedBlockWithRetry(
+      value,
+      config,
+      dbConn as any, // Pool,
+      blockHash,
+    );
+    const blockAppEvents: PendingEvent[] = result.events;
+    if (result.blockHash !== "0x0") {
+      blockHash = result.blockHash;
     }
 
+    recordAppliedBlock(value); // apply-stage liveness for /debug/metrics
+
     // Used to emit & log the block range for each protocol.
     const contentBlocksForProtocol = getRangesForSyncProtocols(value);
 
@@ -215,7 +249,7 @@ export function* start(config: StartConfig): Operation<void> {
           }...${lagSuffix} | ${JSON.stringify(contentBlocksForProtocol)}`,
         ),
     );
-    yield* each.next();
+    if (config.dev?.applyDelayMs) yield* sleep(config.dev.applyDelayMs);
   }
 }
 

package/src/process-blocks.ts

@@ -1,6 +1,6 @@
 import type { ChainBlock } from "@effectstream/sync";
-import { call, type Operation, until } from "effection";
-import type { Pool } from "pg";
+import { call, type Operation, sleep, until } from "effection";
+import type { Pool, PoolClient } from "pg";
 import { type BaseStfInput, type EmitFn, primitiveTransitionFunction } from "@effectstream/sm";
 import { PreparedQuery } from "@pgtyped/runtime";
 import type {
@@ -9,19 +9,24 @@ import type {
   SyncStateUpdateStream,
 } from "@effectstream/coroutine";
 import {
+  acquireDBMutex,
   blockHeightDone,
   deleteEmptyBlocks,
   deleteScheduled,
   getFutureGameInputByBlockHeight,
   getFutureGameInputByMaxTimestamp,
   insertGameInputResult,
+  isTransientPgError,
+  poolErrors,
+  releaseDBMutex,
   removePages,
   saveLastBlock,
+  upsertPage,
 } from "@effectstream/db";
 import { Buffer } from "node:buffer";
 import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
 import type { StartConfig } from "./types.ts";
-import type { BlockNumber, EffectstreamBlockHash } from "@effectstream/utils";
+import type { EffectstreamBlockHash } from "@effectstream/utils";
 import { generateEffectstreamBlockHash, Prando } from "@effectstream/crypto";
 import { applyUserMigrations } from "./version-migrations.ts";
 import type { EventPathAndDef } from "@effectstream/event-client";
@@ -308,36 +313,53 @@ export function* processFinalizedBlock(
       }, dbConn)
     );
 
-    /* STEP 7: Update page state for all sync protocols consumed
-    *          This is to mark what blocks where consumed by this specific
-    *          block, to avoid re-processing the same blocks more than once
-    *          if the sync protocol is restarted.
+    /* STEP 7: Persist the block-accurate resume marker per protocol, inside this
+    *          block's transaction (the runtime is the sole writer of the resume
+    *          position). Upsert at page_number = ownBlockNumber, then prune
+    *          everything below → one row per protocol = the committed watermark.
+    *          Full mechanism: @effectstream/sync CLAUDE.md, design idea #5.
     */
-
-    // Get latest block for each sync protocol.
-    const latestBlocks: Record<string, BlockNumber> = {};
-    for (const blockInfo of value.blockInfo) {
-      if (
-        !latestBlocks[blockInfo.protocol_name] ||
-        latestBlocks[blockInfo.protocol_name] < blockInfo.block_number
-      ) {
-        latestBlocks[blockInfo.protocol_name] = blockInfo.block_number;
-      }
-    }
-
-    for (const [protocolName, blockNumber] of Object.entries(latestBlocks)) {
+    for (const { protocol_name, lastPage } of value.resumePages) {
+      yield* until(
+        upsertPage.run({
+          protocol_name,
+          page_number: lastPage.ownBlockNumber,
+          page: JSON.stringify(lastPage),
+        }, dbConn),
+      );
       yield* until(
         removePages.run({
-          protocol_name: protocolName,
-          page_number: blockNumber,
+          protocol_name,
+          page_number: lastPage.ownBlockNumber,
         }, dbConn),
       );
     }
 
     /* STEP 8: Commit the transaction. */
     yield* until(dbConn.query("COMMIT"));
+    // A successful COMMIT proves the DB is reachable end-to-end — reset the
+    // pool failure clock so the write path also feeds the health tracker
+    // (it otherwise only sees floating pool errors and the read-side API).
+    poolErrors.markHealthy();
   } catch (err) {
-    yield* until(dbConn.query("ROLLBACK"));
+    // Best-effort rollback. If the socket is dead the ROLLBACK throws too;
+    // swallow that so it doesn't mask the original error — the caller's retry
+    // loop keys off the original (transient) error, not a rollback failure.
+    try {
+      yield* until(dbConn.query("ROLLBACK"));
+    } catch (rollbackErr) {
+      log.remote(
+        ComponentNames.EFFECTSTREAM_SYNC,
+        "block-processing",
+        SeverityNumber.WARN,
+        (log) =>
+          log(
+            `ROLLBACK failed for block ${value.blockNumber} (connection likely dead): ${
+              String(rollbackErr)
+            }`,
+          ),
+      );
+    }
     log.remote(
       ComponentNames.EFFECTSTREAM_SYNC,
       "block-processing",
@@ -345,12 +367,89 @@ export function* processFinalizedBlock(
       (log) =>
         log(`Error processing block ${value.blockNumber}: ${String(err)}`),
     );
-    // We cannot recover from this error.
+    // Re-throw: `processFinalizedBlockWithRetry` decides whether this is a
+    // retryable transient pg failure or a fatal bug.
     throw err;
   }
 
-  return { 
-    blockHash: hasOperations ? blockHash : "0x0" as PaimaBlockHash, 
-    events: blockEvents 
+  return {
+    blockHash: hasOperations ? blockHash : "0x0" as PaimaBlockHash,
+    events: blockEvents
   };
 }
+
+/** Caps the exponential backoff between block-retry attempts. */
+const MAX_RETRY_BACKOFF_MS = 5_000;
+
+/**
+ * Run {@link processFinalizedBlock} against a freshly checked-out pool client,
+ * retrying the whole block on transient pg failures (Neon cold-starts,
+ * idle-socket evictions, TLS blips).
+ *
+ * The previous socket is dead after such a failure, so each attempt
+ * re-acquires a client and reprocesses the block from `BEGIN` — safe because
+ * the failed tx never committed and the caller only advances the block stream
+ * after this returns.
+ *
+ * `poolErrors` governs escalation: it logs WARN→ERROR as the outage persists
+ * and calls `process.exit(1)` once it crosses the fatal threshold, letting the
+ * orchestrator restart us. A non-transient error (a real SQL/logic bug) is
+ * re-thrown and crashes, as before.
+ *
+ * Owns the per-block DB mutex (for PGLite) and client lifecycle so the caller
+ * doesn't have to.
+ */
+export function* processFinalizedBlockWithRetry(
+  value: ChainBlock,
+  config: StartConfig,
+  pool: Pool,
+  previousBlockHash: EffectstreamBlockHash | null,
+): Operation<ProcessFinalizedBlockResult> {
+  const lockName = `processing-blocks:${value.blockNumber}`;
+  let attempt = 0;
+  while (true) {
+    let dbClient: PoolClient | undefined;
+    let transientErr: unknown;
+    try {
+      yield* acquireDBMutex(lockName);
+      dbClient = yield* until(pool.connect());
+      return yield* processFinalizedBlock(
+        value,
+        config,
+        dbClient as unknown as Pool, // a checked-out client, used as the tx conn
+        previousBlockHash,
+      );
+    } catch (err) {
+      if (!isTransientPgError(err)) throw err;
+      transientErr = err;
+    } finally {
+      releaseDBMutex(lockName);
+      if (dbClient) {
+        // Passing the error destroys the client instead of recycling it, so
+        // the pool never hands the dead socket back out. On success
+        // `transientErr` is undefined and the client returns to the pool.
+        try {
+          dbClient.release(transientErr as Error | undefined);
+        } catch {
+          /* already released/destroyed */
+        }
+      }
+    }
+
+    attempt++;
+    poolErrors.log(transientErr, ["sync-write"]);
+    const backoffMs = Math.min(500 * 2 ** (attempt - 1), MAX_RETRY_BACKOFF_MS);
+    log.local(
+      ComponentNames.EFFECTSTREAM_SYNC,
+      "block-processing",
+      SeverityNumber.WARN,
+      (l) =>
+        l(
+          `transient db error on block ${value.blockNumber}; retry #${attempt} in ${backoffMs}ms: ${
+            String(transientErr)
+          }`,
+        ),
+    );
+    yield* sleep(backoffMs);
+  }
+}

package/src/types.ts

@@ -1,4 +1,5 @@
 import type { Operation } from "effection";
+import type { AllSyncProtocols } from "@effectstream/sync";
 import type { SyncProtocolWithNetwork } from "@effectstream/config";
 import type { AppEvents, BaseStfInput, BaseStfOutput, Primitive } from "@effectstream/sm";
 import type { FastifyInstance } from "fastify";
@@ -72,5 +73,18 @@ export type StartConfig = {
   dev?: {
     /** Reset public-schema tables on each sync reset. For local testing only. */
     resetPublicData?: boolean;
+    /**
+     * Test-only hook invoked once the sync protocols are instantiated (after
+     * `genSyncProtocols`, before the merge starts). Lets tests observe live
+     * state such as `bufferedData.size()`. No effect in production.
+     */
+    onStarted?: (ctx: { syncProtocols: AllSyncProtocols[] }) => void;
+    /**
+     * Test/diagnostic only: sleep this many ms after each applied block, slowing
+     * the drain so the per-chain fetch buffers fill to their cap and the
+     * backpressure fix becomes observable (otherwise drain ≈ fetch and the buffer
+     * never grows). No effect in production (leave unset / 0).
+     */
+    applyDelayMs?: number;
   };
 };

package/test/coalesce.unit.test.ts

@@ -0,0 +1,69 @@
+import { expect, test } from "bun:test";
+import {
+  type EmptyRunBoundaries,
+  isCoalescableEmptyBlock,
+} from "../src/coalesce.ts";
+
+const NO_BOUNDARIES: EmptyRunBoundaries = {
+  minScheduledBlockHeight: null,
+  minScheduledTimestampMs: null,
+};
+
+function block(
+  blockNumber: number,
+  timestamp: number,
+  primitives: unknown[] = [],
+): any {
+  return { blockNumber, timestamp, blockInfo: [], resumePages: [], primitives };
+}
+
+test("empty block with no boundaries is coalescable", () => {
+  expect(isCoalescableEmptyBlock(block(10, 10_000), NO_BOUNDARIES, undefined))
+    .toBe(true);
+});
+
+test("block with on-chain content is NOT coalescable", () => {
+  expect(
+    isCoalescableEmptyBlock(block(10, 10_000, [{}]), NO_BOUNDARIES, undefined),
+  ).toBe(false);
+});
+
+test("block where a migration fires is NOT coalescable", () => {
+  const migrations = [{ name: "m1", sql: "", blockHeight: 10 }];
+  expect(isCoalescableEmptyBlock(block(10, 10_000), NO_BOUNDARIES, migrations))
+    .toBe(false);
+  expect(isCoalescableEmptyBlock(block(9, 9_000), NO_BOUNDARIES, migrations))
+    .toBe(true);
+});
+
+test("block at/after a pending block-height-scheduled input is NOT coalescable", () => {
+  const boundaries: EmptyRunBoundaries = {
+    minScheduledBlockHeight: 10,
+    minScheduledTimestampMs: null,
+  };
+  expect(isCoalescableEmptyBlock(block(9, 9_000), boundaries, undefined)).toBe(
+    true,
+  );
+  expect(isCoalescableEmptyBlock(block(10, 10_000), boundaries, undefined)).toBe(
+    false,
+  );
+  expect(isCoalescableEmptyBlock(block(11, 11_000), boundaries, undefined)).toBe(
+    false,
+  );
+});
+
+test("block at/after a due timestamp-scheduled input is NOT coalescable", () => {
+  const boundaries: EmptyRunBoundaries = {
+    minScheduledBlockHeight: null,
+    minScheduledTimestampMs: 10_000,
+  };
+  expect(isCoalescableEmptyBlock(block(9, 9_999), boundaries, undefined)).toBe(
+    true,
+  );
+  expect(isCoalescableEmptyBlock(block(10, 10_000), boundaries, undefined)).toBe(
+    false,
+  );
+  expect(isCoalescableEmptyBlock(block(11, 10_001), boundaries, undefined)).toBe(
+    false,
+  );
+});