@effectstream/runtime 0.100.20 → 0.100.22

package/README.md

@@ -69,40 +69,6 @@ exercised when this loop is running. The `StartConfig` shape determines
 which DB migrations apply, which state machines fire, and whether the
 runtime exposes a Fastify router.
 
-## Empty-block coalescing (catch-up)
-
-The runtime replays one Effectstream block per main-clock tick, each in its
-own DB transaction. During a deep catch-up over an idle gap (e.g. an NTP main
-chain at one block/second across days), most of those blocks are **empty** — no
-on-chain content, no scheduled input, no migration — yet still cost a full
-`BEGIN…COMMIT` round-trip.
-Empty-block coalescing folds a run of consecutive empty blocks into a **single
-transaction** that writes only the run's endpoint. It is **opt-in** and off by
-default:
-
-```bash
-EFFECTSTREAM_COALESCE_EMPTY_BLOCKS=true
-```
-
-Behavior and guarantees:
-
-- **Catch-up only.** Coalescing engages only while behind the chain tip
-  (`now - block.timestamp` greater than 10× the main clock's block time) and
-  disengages at the tip, so steady-state stays one block per transaction.
-- **Identical database.** A coalesced sync produces a byte-for-byte identical
-  database to a non-coalesced one. Empty blocks already never advance the block
-  hash or RNG seed, so only the endpoint's block row and the per-protocol resume
-  watermark are written — exactly as the normal path would leave them.
-- **Never skips work.** A run is flushed before any block that has on-chain
-  content, a migration at its height, or a due block-height/timestamp-scheduled
-  input — so nothing that mutates state is ever folded away.
-- **Tip stays visible.** If the producer goes quiet on an empty tail, the run is
-  flushed on a short idle timeout so the latest height still commits.
-- A run is also capped (`MAX_EMPTY_RUN`) to bound transaction size and recovery.
-
-Implementation: [`src/coalesce.ts`](./src/coalesce.ts) (the `createEmptyBlockCoalescer`
-coordinator the main loop drives).
-
 ## Key exports
 
 - `init()` - `Operation<void>`. One-shot setup: OpenTelemetry, config validation, version pinning. Call before `start`.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@effectstream/runtime",
-  "version": "0.100.20",
+  "version": "0.100.22",
   "license": "MIT",
   "homepage": "https://effectstream.github.io/docs/",
   "repository": {
@@ -13,20 +13,20 @@
   },
   "exports": "./src/mod.ts",
   "dependencies": {
-    "@effectstream/concise": "0.100.20",
-    "@effectstream/config": "0.100.20",
-    "@effectstream/coroutine": "0.100.20",
-    "@effectstream/crypto": "0.100.20",
-    "@effectstream/db": "0.100.20",
-    "@effectstream/event-client": "0.100.20",
-    "@effectstream/event-server": "0.100.20",
-    "@effectstream/log": "0.100.20",
-    "@effectstream/sm": "0.100.20",
-    "@effectstream/sync": "0.100.20",
-    "@effectstream/utils": "0.100.20",
+    "@effectstream/concise": "0.100.22",
+    "@effectstream/config": "0.100.22",
+    "@effectstream/coroutine": "0.100.22",
+    "@effectstream/crypto": "0.100.22",
+    "@effectstream/db": "0.100.22",
+    "@effectstream/event-client": "0.100.22",
+    "@effectstream/event-server": "0.100.22",
+    "@effectstream/log": "0.100.22",
+    "@effectstream/sm": "0.100.22",
+    "@effectstream/sync": "0.100.22",
+    "@effectstream/utils": "0.100.22",
     "@fastify/cors": "^11.0.1",
     "@metamask/json-rpc-engine": "^10.0.3",
-    "pg": "8.17.2",
+    "pg": "8.21.0",
     "effection": "^3.5.0",
     "@opentelemetry/api": "^1.9.0",
     "@opentelemetry/auto-instrumentations-node": "^0.54.0",

package/src/api/http-server.ts

@@ -1,6 +1,5 @@
 import fastify, { type FastifyInstance, type FastifyReply, type FastifyRequest } from "fastify";
 import { evmRpcEngine } from "./rpc-evm/eip1193.ts";
-import { appliedBlockStatus } from "./apply-status.ts";
 import type { Pool } from "pg";
 import cors from "@fastify/cors";
 import { run, until } from "effection";
@@ -14,7 +13,6 @@ import {
   getTableSchema,
   type IGetAllAddressesResult,
   type IGetAllTableNamesResult,
-  poolErrors,
   releaseDBMutex,
   runPreparedQuery,
   waitUntilFree,
@@ -366,29 +364,19 @@ export const startHttpServer = function* (
     yield* until(apiRouter(server, dbConn));
   }
 
-  const HealthDbSchema = Type.Object({
-    consecutive: Type.Number(),
-    firstFailureAt: Type.Number(),
-    sustainedDurationMs: Type.Number(),
-    sustained: Type.Boolean(),
-  });
-  const HealthResponseSchema = Type.Object({
-    status: Type.String(),
-    db: HealthDbSchema,
-  });
   server.get("/health", {
     schema: {
       tags: ["status"],
       response: {
-        200: HealthResponseSchema,
-        503: HealthResponseSchema,
+        200: Type.Object({
+          status: Type.String(),
+        }),
       },
     },
-  }, (_req, reply) => {
-    const db = poolErrors.state();
-    return reply
-      .status(db.sustained ? 503 : 200)
-      .send({ status: db.sustained ? "db-unreachable" : "ok", db });
+  }, () => {
+    return {
+      status: "ok",
+    };
   });
 
   server.get("/addresses", {
@@ -481,42 +469,6 @@ export const startHttpServer = function* (
   });
 
   if (ENV.ENABLE_DEV_AND_DEBUG_ENDPOINTS) {
-    server.get("/debug/metrics", {
-      schema: {
-        tags: ["developer"],
-        response: {
-          200: Type.Object({}, { additionalProperties: true }),
-        },
-      },
-    }, () => {
-      const appliedTs = appliedBlockStatus.timestamp;
-      return {
-        timestamp: Date.now(),
-        uptimeSeconds: process.uptime(),
-        memory: process.memoryUsage(),
-        protocols: syncProtocols.map((p) => ({
-          name: p.name,
-          buf: p.bufferedData.size(),
-          ownBlockNumber: p.lastPage?.ownBlockNumber ?? null,
-          // Backpressure observability
-          cap: p.bufferCap,
-          bufHighWater: p.bufferHighWater,
-          pausedNow: p.pausedNow,
-          pauses: p.backpressurePauses,
-          pausedMs: p.backpressurePausedMs,
-        })),
-        // Apply-stage lag: how old the last-applied block is. Unlike `buf` (fetch
-        // backlog) this stays high when the node is write/apply-bound.
-        applied: {
-          blockNumber: appliedBlockStatus.blockNumber,
-          timestamp: appliedTs,
-          lagSeconds: appliedTs != null
-            ? Math.round((Date.now() - appliedTs) / 100) / 10
-            : null,
-        },
-      };
-    });
-
     server.get("/debug/sync-protocols", {
       schema: {
         tags: ["developer"],

package/src/main.ts

@@ -6,11 +6,13 @@ import {
 import {
   acquireDBMutex,
   createDynamicTables,
+  detectCapabilities,
   getConnection,
   getLastNonEmptyBlockHash,
   releaseDBMutex,
   resetPublicTables,
   runSnapshotLoop,
+  selectViewStrategy,
 } from "@effectstream/db";
 import { EventBroker } from "@effectstream/event-server";
 import { ENV } from "@effectstream/utils/node-env";
@@ -22,19 +24,15 @@ import { startMerge, startSync } from "@effectstream/sync";
 import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
 import {
   createChannel,
+  each,
   type Operation,
   sleep,
   spawn,
   until,
 } from "effection";
 import { initTelemetry } from "./telemetry.ts";
-import {
-  type PendingEvent,
-  processFinalizedBlockWithRetry,
-} from "./process-blocks.ts";
-import { createEmptyBlockCoalescer } from "./coalesce.ts";
+import { type PendingEvent, processFinalizedBlock } from "./process-blocks.ts";
 import { startHttpServer } from "./api/http-server.ts";
-import { recordAppliedBlock } from "./api/apply-status.ts";
 import type { StartConfig } from "./types.ts";
 import type { Client } from "pg";
 import type { EffectstreamBlockHash } from "@effectstream/utils";
@@ -93,23 +91,12 @@ export function* start(config: StartConfig): Operation<void> {
     );
   });
 
-  // 10× main clock block time (NTP if present, else protocol 0). Undefined disables lag gating.
   const ntpConfig = syncInfo.find(s => s.networkType === ConfigNetworkType.NTP);
-  const clockBlockTimeMS =
-    (ntpConfig?.network as { blockTimeMS?: number } | undefined)?.blockTimeMS ??
-    (syncInfo[0]?.network as { blockTimeMS?: number } | undefined)?.blockTimeMS;
-  const lagThresholdMs = clockBlockTimeMS != null ? clockBlockTimeMS * 10 : undefined;
+  const ntpBlockTimeMS = (ntpConfig?.network as { blockTimeMS: number } | undefined)?.blockTimeMS;
+  const lagThresholdMs = ntpBlockTimeMS != null ? ntpBlockTimeMS * 10 : undefined;
 
   const finalizedBlockStream = createChannel<ChainBlock>();
 
-  // Subscribe BEFORE spawning the merge: effection channels drop sends with no
-  // active subscriber, so a fast restart (in-memory state, no RPC wait) could
-  // emit blocks before we subscribe and silently stall at the pre-restart height.
-  const finalizedBlocks = yield* finalizedBlockStream;
-  // NOTE: this subscriber's queue is unbounded. During deep catch-up the merge
-  // can outpace the per-block processing below and grow it without bound — the
-  // downstream side of the Fix C backpressure issue (see sync/CLAUDE.md).
-
   yield* spawn(() => startMerge(syncProtocols, finalizedBlockStream));
 
   const heartbeatIntervalMs = 60_000;
@@ -152,53 +139,37 @@ export function* start(config: StartConfig): Operation<void> {
     yield* spawn(() => runSnapshotLoop(config.snapshotConfig!));
   }
 
-  const coalescer = createEmptyBlockCoalescer({
-    enabled: ENV.EFFECTSTREAM_COALESCE_EMPTY_BLOCKS,
-    subscription: finalizedBlocks,
-    pool: dbConn as any, // Pool,
-    migrations: config.migrations,
-    lagThresholdMs,
-    getPreviousBlockHash: () => blockHash,
-    onFlush: (endpoint, length) => {
-      recordAppliedBlock(endpoint);
-      emitLatestBlocks(
-        endpoint.blockNumber,
-        endpoint.timestamp,
-        getRangesForSyncProtocols(endpoint),
-      );
-      log.local(
-        ComponentNames.EFFECTSTREAM_SYNC,
-        "block-merge",
-        SeverityNumber.INFO,
-        (l) =>
-          l(
-            `coalesced ${length} empty block(s) → block ${endpoint.blockNumber}`,
-          ),
+  for (const value of yield* each(finalizedBlockStream)) {
+    // We request a dbClient for a non-shared dbConn object.
+    // For PGLite, this is not enough, as the can only be one connection at a time.
+    // So we request a DBMutex as well.
+    let dbClient: Client | undefined;
+    // Custom app events emitted by STFs during this block. Collected by
+    // processFinalizedBlock and flushed below ONLY after it returns — i.e.
+    // strictly after the block's COMMIT. See plan invariant I1.
+    let blockAppEvents: PendingEvent[] = [];
+    try {
+      yield* acquireDBMutex(`processing-blocks:${value.blockNumber}`);
+      dbClient = yield* until((dbConn as any).connect()); // Client,
+
+      const result = yield* processFinalizedBlock(
+        value,
+        config,
+        dbClient as any, // Client,
+        blockHash,
       );
-    },
-  });
-
-  while (true) {
-    const value = yield* coalescer.advance();
-    if (value === undefined) break;
-
-    // processFinalizedBlockWithRetry owns connection checkout, the per-block
-    // DB mutex (PGLite), and transient-pg retry/backoff. App events it
-    // collects are flushed below ONLY after it returns — i.e. strictly after
-    // the block's COMMIT.
-    const result = yield* processFinalizedBlockWithRetry(
-      value,
-      config,
-      dbConn as any, // Pool,
-      blockHash,
-    );
-    const blockAppEvents: PendingEvent[] = result.events;
-    if (result.blockHash !== "0x0") {
-      blockHash = result.blockHash;
+      const resultHash = result.blockHash;
+      blockAppEvents = result.events;
+      if (resultHash !== "0x0") {
+        blockHash = resultHash;
+      }
+    } finally {
+      releaseDBMutex(`processing-blocks:${value.blockNumber}`);
+      if (dbClient) {
+        (dbClient as any).release(); // Client,
+      }
     }
 
-    recordAppliedBlock(value); // apply-stage liveness for /debug/metrics
-
     // Used to emit & log the block range for each protocol.
     const contentBlocksForProtocol = getRangesForSyncProtocols(value);
 
@@ -249,7 +220,7 @@ export function* start(config: StartConfig): Operation<void> {
           }...${lagSuffix} | ${JSON.stringify(contentBlocksForProtocol)}`,
         ),
     );
-    if (config.dev?.applyDelayMs) yield* sleep(config.dev.applyDelayMs);
+    yield* each.next();
   }
 }
 
@@ -351,11 +322,15 @@ function* startup(
   const syncProtocols = yield* genSyncProtocols(dbConn as any, // Client,
     syncInfo);
 
+  const capabilities = yield* until(detectCapabilities(dbConn as any));
+  const viewStrategy = selectViewStrategy(capabilities);
+
   yield* createDynamicTables(
     versionInfo,
     lastBlockHeight,
     dbConn as any, // Client,
     syncProtocols,
+    viewStrategy,
   );
 
   releaseDBMutex(`startup-node`);

package/src/process-blocks.ts

@@ -1,6 +1,6 @@
 import type { ChainBlock } from "@effectstream/sync";
-import { call, type Operation, sleep, until } from "effection";
-import type { Pool, PoolClient } from "pg";
+import { call, type Operation, until } from "effection";
+import type { Pool } from "pg";
 import { type BaseStfInput, type EmitFn, primitiveTransitionFunction } from "@effectstream/sm";
 import { PreparedQuery } from "@pgtyped/runtime";
 import type {
@@ -9,24 +9,19 @@ import type {
   SyncStateUpdateStream,
 } from "@effectstream/coroutine";
 import {
-  acquireDBMutex,
   blockHeightDone,
   deleteEmptyBlocks,
   deleteScheduled,
   getFutureGameInputByBlockHeight,
   getFutureGameInputByMaxTimestamp,
   insertGameInputResult,
-  isTransientPgError,
-  poolErrors,
-  releaseDBMutex,
   removePages,
   saveLastBlock,
-  upsertPage,
 } from "@effectstream/db";
 import { Buffer } from "node:buffer";
 import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
 import type { StartConfig } from "./types.ts";
-import type { EffectstreamBlockHash } from "@effectstream/utils";
+import type { BlockNumber, EffectstreamBlockHash } from "@effectstream/utils";
 import { generateEffectstreamBlockHash, Prando } from "@effectstream/crypto";
 import { applyUserMigrations } from "./version-migrations.ts";
 import type { EventPathAndDef } from "@effectstream/event-client";
@@ -313,53 +308,36 @@ export function* processFinalizedBlock(
       }, dbConn)
     );
 
-    /* STEP 7: Persist the block-accurate resume marker per protocol, inside this
-    *          block's transaction (the runtime is the sole writer of the resume
-    *          position). Upsert at page_number = ownBlockNumber, then prune
-    *          everything below → one row per protocol = the committed watermark.
-    *          Full mechanism: @effectstream/sync CLAUDE.md, design idea #5.
+    /* STEP 7: Update page state for all sync protocols consumed
+    *          This is to mark what blocks where consumed by this specific
+    *          block, to avoid re-processing the same blocks more than once
+    *          if the sync protocol is restarted.
     */
-    for (const { protocol_name, lastPage } of value.resumePages) {
-      yield* until(
-        upsertPage.run({
-          protocol_name,
-          page_number: lastPage.ownBlockNumber,
-          page: JSON.stringify(lastPage),
-        }, dbConn),
-      );
+
+    // Get latest block for each sync protocol.
+    const latestBlocks: Record<string, BlockNumber> = {};
+    for (const blockInfo of value.blockInfo) {
+      if (
+        !latestBlocks[blockInfo.protocol_name] ||
+        latestBlocks[blockInfo.protocol_name] < blockInfo.block_number
+      ) {
+        latestBlocks[blockInfo.protocol_name] = blockInfo.block_number;
+      }
+    }
+
+    for (const [protocolName, blockNumber] of Object.entries(latestBlocks)) {
       yield* until(
         removePages.run({
-          protocol_name,
-          page_number: lastPage.ownBlockNumber,
+          protocol_name: protocolName,
+          page_number: blockNumber,
         }, dbConn),
       );
     }
 
     /* STEP 8: Commit the transaction. */
     yield* until(dbConn.query("COMMIT"));
-    // A successful COMMIT proves the DB is reachable end-to-end — reset the
-    // pool failure clock so the write path also feeds the health tracker
-    // (it otherwise only sees floating pool errors and the read-side API).
-    poolErrors.markHealthy();
   } catch (err) {
-    // Best-effort rollback. If the socket is dead the ROLLBACK throws too;
-    // swallow that so it doesn't mask the original error — the caller's retry
-    // loop keys off the original (transient) error, not a rollback failure.
-    try {
-      yield* until(dbConn.query("ROLLBACK"));
-    } catch (rollbackErr) {
-      log.remote(
-        ComponentNames.EFFECTSTREAM_SYNC,
-        "block-processing",
-        SeverityNumber.WARN,
-        (log) =>
-          log(
-            `ROLLBACK failed for block ${value.blockNumber} (connection likely dead): ${
-              String(rollbackErr)
-            }`,
-          ),
-      );
-    }
+    yield* until(dbConn.query("ROLLBACK"));
     log.remote(
       ComponentNames.EFFECTSTREAM_SYNC,
       "block-processing",
@@ -367,89 +345,12 @@ export function* processFinalizedBlock(
       (log) =>
         log(`Error processing block ${value.blockNumber}: ${String(err)}`),
     );
-    // Re-throw: `processFinalizedBlockWithRetry` decides whether this is a
-    // retryable transient pg failure or a fatal bug.
+    // We cannot recover from this error.
     throw err;
   }
 
-  return {
-    blockHash: hasOperations ? blockHash : "0x0" as PaimaBlockHash,
-    events: blockEvents
+  return { 
+    blockHash: hasOperations ? blockHash : "0x0" as PaimaBlockHash, 
+    events: blockEvents 
   };
 }
-
-/** Caps the exponential backoff between block-retry attempts. */
-const MAX_RETRY_BACKOFF_MS = 5_000;
-
-/**
- * Run {@link processFinalizedBlock} against a freshly checked-out pool client,
- * retrying the whole block on transient pg failures (Neon cold-starts,
- * idle-socket evictions, TLS blips).
- *
- * The previous socket is dead after such a failure, so each attempt
- * re-acquires a client and reprocesses the block from `BEGIN` — safe because
- * the failed tx never committed and the caller only advances the block stream
- * after this returns.
- *
- * `poolErrors` governs escalation: it logs WARN→ERROR as the outage persists
- * and calls `process.exit(1)` once it crosses the fatal threshold, letting the
- * orchestrator restart us. A non-transient error (a real SQL/logic bug) is
- * re-thrown and crashes, as before.
- *
- * Owns the per-block DB mutex (for PGLite) and client lifecycle so the caller
- * doesn't have to.
- */
-export function* processFinalizedBlockWithRetry(
-  value: ChainBlock,
-  config: StartConfig,
-  pool: Pool,
-  previousBlockHash: EffectstreamBlockHash | null,
-): Operation<ProcessFinalizedBlockResult> {
-  const lockName = `processing-blocks:${value.blockNumber}`;
-  let attempt = 0;
-  while (true) {
-    let dbClient: PoolClient | undefined;
-    let transientErr: unknown;
-    try {
-      yield* acquireDBMutex(lockName);
-      dbClient = yield* until(pool.connect());
-      return yield* processFinalizedBlock(
-        value,
-        config,
-        dbClient as unknown as Pool, // a checked-out client, used as the tx conn
-        previousBlockHash,
-      );
-    } catch (err) {
-      if (!isTransientPgError(err)) throw err;
-      transientErr = err;
-    } finally {
-      releaseDBMutex(lockName);
-      if (dbClient) {
-        // Passing the error destroys the client instead of recycling it, so
-        // the pool never hands the dead socket back out. On success
-        // `transientErr` is undefined and the client returns to the pool.
-        try {
-          dbClient.release(transientErr as Error | undefined);
-        } catch {
-          /* already released/destroyed */
-        }
-      }
-    }
-
-    attempt++;
-    poolErrors.log(transientErr, ["sync-write"]);
-    const backoffMs = Math.min(500 * 2 ** (attempt - 1), MAX_RETRY_BACKOFF_MS);
-    log.local(
-      ComponentNames.EFFECTSTREAM_SYNC,
-      "block-processing",
-      SeverityNumber.WARN,
-      (l) =>
-        l(
-          `transient db error on block ${value.blockNumber}; retry #${attempt} in ${backoffMs}ms: ${
-            String(transientErr)
-          }`,
-        ),
-    );
-    yield* sleep(backoffMs);
-  }
-}

package/src/types.ts

@@ -79,12 +79,5 @@ export type StartConfig = {
      * state such as `bufferedData.size()`. No effect in production.
      */
     onStarted?: (ctx: { syncProtocols: AllSyncProtocols[] }) => void;
-    /**
-     * Test/diagnostic only: sleep this many ms after each applied block, slowing
-     * the drain so the per-chain fetch buffers fill to their cap and the
-     * backpressure fix becomes observable (otherwise drain ≈ fetch and the buffer
-     * never grows). No effect in production (leave unset / 0).
-     */
-    applyDelayMs?: number;
   };
 };