@effectstream/sync 0.100.20 → 0.100.22

package/CLAUDE.md

@@ -35,7 +35,7 @@ config (syncProtocols.main + syncProtocols.parallel)
 │  PER-CHAIN FETCH LOOPS              one spawn per protocol             │
 │  startSync(state):  stateToInput → readData → updateState             │
 │    → push Output to bufferedData (Deque)  + wake newData/newPageCondVar│
-│    → lastPage kept IN-MEMORY only (no DB write; runtime owns resume)   │
+│    → upsertPage(page_number = chunk_end) into sync_protocol_pagination │
 │    → producerChannel.send(...)  ← NOTE: nothing consumes this today    │
 └────────────────────────────────────────┬─────────────────────────────┘
                                           ▼
@@ -47,9 +47,8 @@ config (syncProtocols.main + syncProtocols.parallel)
 └────────────────────────────────────────┬─────────────────────────────┘
                                           ▼
    runtime/src/main.ts:  for (block of each(finalizedBlockStream))
-     acquire DB mutex → processFinalizedBlock (STF)
-     → upsertPage(block.resumePages) + removePages(< committed) [in txn] → COMMIT
-     → fire-and-forget MQTT events → each.next()
+     acquire DB mutex → processFinalizedBlock (STF) → COMMIT
+     → removePages(< merged block) → fire-and-forget MQTT events → each.next()
 ```
 
 ## The three per-chain abstractions (`src/sync-protocols/<chain>/`)
@@ -113,34 +112,12 @@ iteration it walks protocols in order:
    a parallel chain's timestamp so older blocks merge into more-recent root slots.
 4. **Mutate-after-commit.** `OutputAndCleanup` keeps data in the Deque until the block is
    safely produced; the consumer commits to Postgres *before* emitting events.
-5. **Durable, block-accurate resume queue.** *(Single source of truth for how restart/resume
-   works — code comments point here; see also Finding #2 for the bug this fixes.)*
-
-   `effectstream.sync_protocol_pagination` is keyed `(protocol_name, page_number)` and the
-   **runtime is the sole writer** of the resume position (Fix D). End to end:
-   - **Fetch loop** (`base/state.ts:updateState`) keeps `lastPage` in memory only — it does
-     **not** persist (it runs ahead of commits, so its chunk-end page is not a safe resume
-     point).
-   - **Merge** (`orchestration/merge.ts`) tags every `ChainBlock` with `resumePages`: one
-     marker per protocol = `outputToLastPage(lastConsumed)`, the `LastPage`
-     (`{ own, ownBlockNumber, root }`) of the **highest datum merged into the block**. Each
-     protocol implements `outputToLastPage` (`base/state.ts` + per-chain `state.ts`) because
-     `own` is a structured page for object-paged chains and can't be rebuilt from the block
-     number. Pending data above the committed timestamp is **not** marked, so it is always
-     re-fetched, never skipped.
-   - **Commit** (`runtime/process-blocks.ts`, STEP 7) writes each marker **inside the block's
-     transaction**: `upsertPage(page_number = ownBlockNumber)` then
-     `removePages(page_number < ownBlockNumber)` — leaving exactly one row per protocol = the
-     committed watermark, atomic with the block.
-   - **Boot** (`restoreState`) reads `getPage` = `ORDER BY page_number ASC LIMIT 1` and
-     resumes from `nextInterval(own).from` = the next uncommitted block.
-
-   **Re-scan on restart is bounded by `stepSize`, not a chain's whole quiet tail.** Every
-   fetched chunk's boundary page (`data.to`) is buffered and consumed even when empty, so the
-   marker advances to within one `stepSize` of the committed frontier — and reaches the
-   fetched tip exactly at full catch-up. So a long-quiet chain re-fetches at most ~`stepSize`
-   blocks after a restart, regardless of how long ago its last data was. Trade-off:
-   `getSyncAndLastPage` now reports `synced_page == fetched_page`.
+5. **Durable, resumable page queue.** `effectstream.sync_protocol_pagination` is keyed
+   `(protocol_name, page_number)`. Fetch appends a row per chunk (`upsertPage`,
+   `page_number = chunk_end`); on commit the consumer deletes consumed chunks
+   (`removePages(page_number < merged_block)`, `runtime/process-blocks.ts:328`); on boot
+   `restoreState` resumes from `getPage` = `ORDER BY page_number ASC LIMIT 1` (oldest
+   remaining chunk).
 
 ---
 
@@ -169,20 +146,24 @@ Consequences during catch-up:
 
 → Fix: **(C) backpressure** — cap `bufferedData` and pause the fetch loop until the merge drains.
 
-### 2. Silent data gap on restart (correctness) — ✅ FIXED (D)
+### 2. Silent data gap on restart (correctness)
 
-**The bug (historical).** The page queue was **chunk-granular** but commits are
-**single-block-granular**: the fetch loop persisted `upsertPage(page_number = chunk_end)`,
-commit deleted `removePages(page_number < merged_block_N)`, and restart resumed from
-`getPage` = oldest remaining chunk at `nextInterval(own).from = chunk_end + 1`. During
-catch-up chunks are full `stepSize`, so the committed watermark `N` sat **inside** the
-oldest retained chunk `[start, end]` (`end ≥ N ⇒ not deleted`); resume jumped to `end + 1`,
-**silently skipping `(N, end]`** → missing primitives → divergent app state, with no error.
+The page queue is **chunk-granular** but commits are **single-block-granular**:
+- fetch persists `upsertPage(page_number = chunk_end)` (`base/state.ts:132`; `own = ownBlockNumber = Number(data.to)`),
+- commit deletes `removePages(page_number < merged_block_N)` (`process-blocks.ts:328`),
+- restart resumes from `getPage` = oldest remaining chunk, at `nextInterval(own).from = chunk_end + 1`.
 
-**The fix (D) — block-accurate resume, runtime is the sole writer.** The runtime persists a
-block-accurate marker per protocol on commit instead of the fetch loop persisting chunk-end
-pages. Full mechanism in **design idea #5** above. Regression test: the
-`consistency.test.ts` "mid-chunk restart" case (Postgres-only).
+During catch-up chunks are full `stepSize`, so at restart the committed watermark `N` sits
+**inside** the oldest retained chunk `[start, end]` (`start ≤ N < end ⇒ end ≥ N ⇒ not deleted`).
+Resume jumps to `end + 1`, **silently skipping source blocks `(N, end]`** (up to
+`stepSize - 1`). The in-memory Deque held them; they're gone on restart; the effectstream
+height continues seamlessly with **missing primitives → divergent app state**. There is
+**no error** and **no restart/resume test** (the only tests in `test/examples.test.ts` are
+export smoke-checks). The existing `getSyncAndLastPage` `synced_page` (`MIN(page_number)`)
+is itself chunk-granular and does not reflect the true committed block.
+
+→ Fix: **(D) block-accurate resume** — persist the committed per-protocol watermark and
+resume from `watermark + 1`, reconciling stale fetched-ahead page rows.
 
 ### 3. Serial replay is the catch-up bottleneck
 
@@ -201,6 +182,6 @@ block-hash chain.
 > survives a restart. See `docs/ADDING-A-SYNC-PROTOCOL.md` for how to add a chain
 > (with `test` as the worked example).
 >
-> Deterministic reproductions of #1/#2/#3 and their fixes land alongside each fix:
-> **#2 → block-accurate resume is DONE** (`consistency.test.ts`). Still in follow-up
-> PRs: #1 → fetch backpressure; #3 → opt-in empty-block commit batching during catch-up.
+> Deterministic reproductions of #1/#2/#3 and their fixes land in **follow-up
+> PRs**, each test alongside its fix: #1 → fetch backpressure; #2 →
+> block-accurate resume; #3 → opt-in empty-block commit batching during catch-up.

package/README.md

@@ -46,55 +46,6 @@ const protocols = await genSyncProtocols(config);
 // protocols.parallelEvmRPC_fast.runOne()  // poll one block
 ```
 
-## Backpressure (`maxBufferedPages`)
-
-During deep catch-up a chain's fetch loop races to its tip far faster than the
-merge can drain (the merge applies one block per DB transaction). Without a bound
-the in-memory buffer (`SyncState.bufferedData`) grows toward the **entire backlog**
-— hundreds of thousands of block objects — which is an OOM risk.
-
-**The cap.** Every chain's `stateToInput` calls `bufferAtCap(state, syncProtocol)`
-first (`sync-protocols/common/page-helpers.ts`): when
-`bufferedData.size() >= maxBufferedPages` it returns `undefined`, so the chain stops
-fetching — exactly as if it had caught up to the tip — and the polling loop sleeps
-`pollingInterval` and retries. The merge keeps draining the buffer; once it drops
-below the cap the next poll resumes. Peak in-memory buffering is therefore bounded
-to **≈ `maxBufferedPages + stepSize`** per chain (one in-flight chunk can overshoot
-the cap), instead of the whole backlog.
-
-**Config.** `maxBufferedPages` is an optional field on every sync-protocol config
-(declared once on the shared `PollingSyncProtocol` schema). When unset it defaults to
-**`4 × stepSize`** (the `MAX_BUFFER_MULTIPLE` constant in `page-helpers.ts`), always
-clamped to **`≥ stepSize + 1`** so a chain can always fetch at least one chunk to feed
-the merge. ~4 chunks of look-ahead never starves the merge while keeping memory
-bounded; raise it to trade memory for more fetch look-ahead. The cap only bites during
-catch-up — in steady state the buffer sits near zero and the cap is never reached.
-
-**Deadlock-safety.** Pausing a full chain cannot deadlock: a chain's page only
-advances via a fetch (`updateState`), and the merge drains a chain's buffered data
-*before* it waits on that chain's page — so the paused (full) chain is never the one
-the merge is blocked on without a path to resume.
-
-**Scope.** The guard runs in **every** chain's `stateToInput`, so all sync chains are
-covered: EVM, NTP, Bitcoin, Avail, Celestia, NEAR, Midnight, Cardano (UTXO-RPC), and
-the synthetic `test` chain. Two notes:
-- **Cardano (UTXO-RPC)** has no `stepSize` (it streams one block per pass), so the cap
-  falls back to a default chunk size of 1000 (⇒ default cap 4000); set `maxBufferedPages`
-  explicitly to tune it.
-- The cap bounds `SyncState.bufferedData` (the merge-facing Deque the backpressure
-  feature protects). The UTXO-RPC fetcher additionally keeps its own internal FollowTip stream
-  buffer; pausing `stateToInput` stops draining it into `bufferedData`, but bounding
-  that lower-level stream is a separate, fetcher-specific concern.
-
-**Observability.** Each `SyncState` tracks, and the runtime's `/debug/metrics`
-endpoint reports per protocol: `cap` (resolved `maxBufferedPages`), `buf` (current
-size), `bufHighWater` (peak since boot — catches spikes between samples), `pausedNow`,
-`pauses` (rising-edge count — **the "backpressure engaged" signal**), and `pausedMs`
-(total time paused). `pauses > 0` means the cap actively bounded memory; `0` means it
-was never needed in that run. Steady-state these sit at `0`; during a real deep
-catch-up (or under the perf harness's `PERF_APPLY_DELAY_MS` drain throttle) they climb
-as `buf` pins to `cap`.
-
 ## Key exports
 
 - `genSyncProtocols(dbConn, syncInfo)` - Effection generator that instantiates a runtime fetcher + state pair for every protocol in `syncInfo` (from `config.syncProtocols`). Called from the runtime's process-blocks loop.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@effectstream/sync",
-  "version": "0.100.20",
+  "version": "0.100.22",
   "license": "MIT",
   "homepage": "https://effectstream.github.io/docs/",
   "repository": {
@@ -13,12 +13,12 @@
   },
   "exports": "./src/mod.ts",
   "dependencies": {
-    "@effectstream/config": "0.100.20",
-    "@effectstream/db": "0.100.20",
-    "@effectstream/log": "0.100.20",
-    "@effectstream/utils": "0.100.20",
+    "@effectstream/config": "0.100.22",
+    "@effectstream/db": "0.100.22",
+    "@effectstream/log": "0.100.22",
+    "@effectstream/utils": "0.100.22",
     "@midnight-ntwrk/ledger-v8": "8.0.3",
-    "@midnight-ntwrk/midnight-js-indexer-public-data-provider": "4.0.2",
+    "@midnight-ntwrk/midnight-js-indexer-public-data-provider": "4.0.4",
     "@midnight-ntwrk/onchain-runtime": "npm:@midnight-ntwrk/onchain-runtime-v3@3.0.0",
     "effection": "3.5.0",
     "graphql-ws": "^6.0.6",

package/src/sync-protocols/avail/state.ts

@@ -10,7 +10,6 @@ import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/c
 import { getPage } from "@effectstream/db";
 import { AvailClient } from "./AvailClient.ts";
 import { applyDelay } from "../common/utils.ts";
-import { bufferAtCap } from "../common/page-helpers.ts";
 
 export class AvailSyncState extends SyncState<
   Input,
@@ -64,7 +63,6 @@ export class AvailSyncState extends SyncState<
 
   @bound
   override *stateToInput(): Operation<Input | undefined> {
-    if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
     const latestHeight = yield* call(() => this.client.getLatestBlockHeight());
     const startHeight = this.lastPage?.own.height ??
       this.config.syncProtocol.startBlockHeight - 1;
@@ -101,15 +99,6 @@ export class AvailSyncState extends SyncState<
     rootOutput.primitives.push(...primitives);
   }
 
-  @bound
-  override outputToLastPage(data: Output): LastPage<Page, RootPage> {
-    return {
-      own: { height: data.raw.number, hash: data.raw.hash },
-      ownBlockNumber: data.raw.number,
-      root: this.toRootPage(data),
-    };
-  }
-
   @bound
   override getNamespace(): string[] {
     return [this.config.network.name, this.config.syncProtocol.name];

package/src/sync-protocols/base/state.ts

@@ -1,4 +1,4 @@
-import { type Operation } from "effection";
+import { type Operation, until } from "effection";
 import Deque from "denque";
 import {
   type BlockNumber,
@@ -11,6 +11,7 @@ import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
 import type { BaseDataFetcher, DataFetched } from "./fetcher.ts";
 import type { PageRelation } from "./page.ts";
 import type { PoolClient } from "pg";
+import { acquireDBMutex, releaseDBMutex, upsertPage } from "@effectstream/db";
 
 export type LastPage<Page, RootPage> = {
   own: Page;
@@ -74,20 +75,6 @@ export abstract class SyncState<
   /** Timestamp of the most recent error, or 0 if no error has occurred. */
   public lastErrorTimestamp: number = 0;
 
-  // ── Backpressure observability (see common/page-helpers.ts + README) ──
-  /** Resolved fetch cap (`maxBufferedPages`); set on each backpressure check, 0 until the first. */
-  public bufferCap: number = 0;
-  /** Peak `bufferedData.size()` seen since boot — catches spikes the periodic sampler misses. */
-  public bufferHighWater: number = 0;
-  /** Whether fetching is currently paused because the buffer is at/over the cap. */
-  public pausedNow: boolean = false;
-  /** How many times the cap engaged (rising edges) — the "backpressure fired" counter. */
-  public backpressurePauses: number = 0;
-  /** Total time fetching has been paused by the cap, in ms. */
-  public backpressurePausedMs: number = 0;
-  /** Start of the current pause, for accumulating `backpressurePausedMs`. */
-  private pauseStartMs: number = 0;
-
   constructor(
     public readonly name: string,
     lastPage: undefined | LastPage<Page, RootPage>,
@@ -119,24 +106,6 @@ export abstract class SyncState<
    */
   abstract mergeDatum(ourOutput: Output, rootOutput: RootOutput): void;
 
-  recordBackpressure(atCap: boolean, cap: number): void {
-    this.bufferCap = cap;
-    const now = Date.now();
-    if (atCap && !this.pausedNow) {
-      this.pausedNow = true;
-      this.pauseStartMs = now;
-      this.backpressurePauses += 1;
-    } else if (!atCap && this.pausedNow) {
-      this.pausedNow = false;
-      this.backpressurePausedMs += now - this.pauseStartMs;
-    }
-  }
-  /**
-   * Build the resume marker for a single output: the {@link LastPage} to resume
-   * fetching *after* it. See CLAUDE.md design idea #5 (restart/resume).
-   */
-  abstract outputToLastPage(data: Output): LastPage<Page, RootPage>;
-
   @bound
   *updateState(
     _input: Input,
@@ -146,8 +115,6 @@ export abstract class SyncState<
       for (const datum of data.output) {
         this.bufferedData.push(datum);
       }
-      // Peak right after a push — the local max before the merge drains it.
-      this.bufferHighWater = Math.max(this.bufferHighWater, this.bufferedData.size());
       this.newDataCondVar.wake();
     }
 
@@ -162,8 +129,13 @@ export abstract class SyncState<
           ),
       );
 
-      // lastPage is in-memory only; the runtime is the sole writer of the
-      // persisted resume marker (see CLAUDE.md design idea #5).
+      yield* acquireDBMutex(`update-state-${this.name}`);
+      yield* until(upsertPage.run({
+        protocol_name: this.name,
+        page_number: data.lastPage.ownBlockNumber,
+        page: JSON.stringify(data.lastPage),
+      }, this.dbConn as any)); // Client,
+      releaseDBMutex(`update-state-${this.name}`);
       this.lastPage = data.lastPage;
       this.newPageCondVar.wake();
     }

package/src/sync-protocols/bitcoin/state.ts

@@ -16,7 +16,7 @@ import {
   type Output,
   type Page,
 } from "./types.ts";
-import { bufferAtCap, genInputRange } from "../common/page-helpers.ts";
+import { genInputRange } from "../common/page-helpers.ts";
 import type {
   ConfigNetworkType,
   SyncProtocolWithNetwork,
@@ -58,7 +58,6 @@ export class BitcoinSyncState extends SyncState<
         block_number: Number(data.raw.height),
         blockHash: hash,
       })),
-      resumePages: [],
       primitives: data.primitives.map((primitive) => ({
         ...primitive,
         source: this.config.syncProtocol.name,
@@ -76,7 +75,6 @@ export class BitcoinSyncState extends SyncState<
 
   @bound
   override *stateToInput(): Operation<Input | undefined> {
-    if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
     return yield* genInputRange(
       this as BitcoinSyncState,
       this.config.syncProtocol.startBlockHeight as Page,
@@ -114,16 +112,6 @@ export class BitcoinSyncState extends SyncState<
     rootOutput.blockInfo.push(...blockInfo);
   }
 
-  @bound
-  override outputToLastPage(data: Output): LastPage<Page, RootPage> {
-    const block = Number(data.raw.height);
-    return {
-      own: block,
-      ownBlockNumber: block,
-      root: this.toRootPage(data),
-    };
-  }
-
   @bound
   override getNamespace(): string[] {
     return [this.config.network.name, this.config.syncProtocol.name];

package/src/sync-protocols/celestia/state.ts

@@ -10,7 +10,6 @@ import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/c
 import { getPage } from "@effectstream/db";
 import { CelestiaClient } from "./CelestiaClient.ts";
 import { applyDelay } from "../common/utils.ts";
-import { bufferAtCap } from "../common/page-helpers.ts";
 
 export class CelestiaSyncState extends SyncState<
   Input,
@@ -62,7 +61,6 @@ export class CelestiaSyncState extends SyncState<
 
   @bound
   override *stateToInput(): Operation<Input | undefined> {
-    if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
     const latestHeight = yield* call(() =>
       this.client.getLatestBlockHeight()
     );
@@ -101,15 +99,6 @@ export class CelestiaSyncState extends SyncState<
     rootOutput.primitives.push(...primitives);
   }
 
-  @bound
-  override outputToLastPage(data: Output): LastPage<Page, RootPage> {
-    return {
-      own: { height: data.height, hash: data.hash },
-      ownBlockNumber: data.height,
-      root: this.toRootPage(data),
-    };
-  }
-
   @bound
   override getNamespace(): string[] {
     return [this.config.network.name, this.config.syncProtocol.name];

package/src/sync-protocols/common/page-helpers.ts

@@ -5,43 +5,6 @@ import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
 import type { PaginatedSyncProtocols } from "../types.ts";
 import type { PageTypeOf } from "../base/state.ts";
 
-// Fetch backpressure. See README "Backpressure (`maxBufferedPages`)".
-const MAX_BUFFER_MULTIPLE = 4;
-// Fallback chunk size for chains that fetch without an explicit stepSize
-// (e.g. utxorpc streams one block per pass).
-const DEFAULT_STEP_SIZE = 1000;
-
-/** Resolved fetch-backpressure cap for a chain. See README "Backpressure". */
-export function bufferCapFor(
-  syncProtocol: { maxBufferedPages?: number; stepSize?: number },
-): number {
-  const step = syncProtocol.stepSize ?? DEFAULT_STEP_SIZE;
-  return Math.max(
-    syncProtocol.maxBufferedPages ?? MAX_BUFFER_MULTIPLE * step,
-    step + 1,
-  );
-}
-
-/**
- * Whether a chain should pause fetching because its in-memory buffer is at/over
- * the cap. Each chain's `stateToInput` calls this first and returns `undefined`
- * when true, so the fetch loop sleeps + retries while the merge drains. Also
- * records backpressure metrics on `state` (for `/debug/metrics`).
- * See README "Backpressure (`maxBufferedPages`)".
- */
-export function bufferAtCap(
-  state: {
-    bufferedData: { size(): number };
-    recordBackpressure(atCap: boolean, cap: number): void;
-  },
-  syncProtocol: { maxBufferedPages?: number; stepSize?: number },
-): boolean {
-  const cap = bufferCapFor(syncProtocol);
-  const atCap = state.bufferedData.size() >= cap;
-  state.recordBackpressure(atCap, cap);
-  return atCap;
-}
-
 type ConfigSubset<Page> = {
   name: string;
   /**

package/src/sync-protocols/common/root.ts

@@ -5,7 +5,6 @@ import type {
   TimestampMs,
 } from "@effectstream/utils";
 import type { PageRelation } from "../base/page.ts";
-import type { LastPage } from "../base/state.ts";
 import type {
   ConfigSyncProtocolType,
   FlattenSyncProtocolIOFor,
@@ -20,14 +19,6 @@ export type ChainBlock = {
     block_number: BlockNumber;
     blockHash: BlockHash;
   }[];
-  /**
-   * Per-protocol block-accurate resume markers the runtime persists on commit.
-   * See `@effectstream/sync` CLAUDE.md, design idea #5 (restart/resume).
-   */
-  resumePages: {
-    protocol_name: string;
-    lastPage: LastPage<unknown, unknown>;
-  }[];
   primitives: (
     & FlattenSyncProtocolIOFor<
       ConfigSyncProtocolType

package/src/sync-protocols/evm/state.ts

@@ -7,7 +7,7 @@ import { bound, type EvmBlockNumber } from "@effectstream/utils";
 import { type LastPage, SyncState } from "../base/state.ts";
 import type { RootOutput, RootPage } from "../types.ts";
 import type { EvmFetcher } from "./fetcher.ts";
-import { bufferAtCap, genInputRange } from "../common/page-helpers.ts";
+import { genInputRange } from "../common/page-helpers.ts";
 import type { Input, Output, Page } from "./types.ts";
 import { toMsTimestamp } from "./types.ts";
 import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/config";
@@ -53,7 +53,6 @@ export class EvmSyncState extends SyncState<
 
   @bound
   override *stateToInput(): Operation<Input | undefined> {
-    if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
     return yield* genInputRange(
       this as EvmSyncState,
       this.config.syncProtocol.startBlockHeight as EvmBlockNumber,
@@ -91,16 +90,6 @@ export class EvmSyncState extends SyncState<
     rootOutput.blockInfo.push(...blockInfo);
   }
 
-  @bound
-  override outputToLastPage(data: Output): LastPage<Page, RootPage> {
-    const block = Number(data.raw.number);
-    return {
-      own: block,
-      ownBlockNumber: block,
-      root: this.toRootPage(data),
-    };
-  }
-
   @bound
   override getNamespace(): string[] {
     return [this.config.network.name, this.config.syncProtocol.name];

package/src/sync-protocols/midnight/MidnightClient.ts

@@ -32,6 +32,14 @@ export interface MidnightGqlBlockState {
         outputIndex: number;
         owner:       string;
       }[];
+      unshieldedCreatedOutputs?: {
+        intentHash:  string;
+        outputIndex: number;
+        owner:       string;
+      }[];
+      // Present only on RegularTransaction (selected via inline fragment): the
+      // coin-commitment Merkle tree root after this transaction.
+      zswapMerkleTreeRoot?: string;
     }[];
   };
 };
@@ -43,6 +51,10 @@ export interface BlockFetchOptions {
   zswapLedgerEvents?: boolean;
   /** Include unshieldedSpentOutputs (needed for MidnightUnshieldedSpendPrimitive). Default: false */
   unshieldedSpentOutputs?: boolean;
+  /** Include unshieldedCreatedOutputs (needed for MidnightUnshieldedCreatePrimitive). Default: false */
+  unshieldedCreatedOutputs?: boolean;
+  /** Include `... on RegularTransaction { zswapMerkleTreeRoot }` (needed for MidnightZswapRootPrimitive). Default: false */
+  zswapRoots?: boolean;
 }
 
 type PublicDataProvider = ReturnType<typeof indexerPublicDataProvider>;
@@ -168,6 +180,8 @@ export class MidnightClient {
       contractActions = true,
       zswapLedgerEvents = true,
       unshieldedSpentOutputs = false,
+      unshieldedCreatedOutputs = false,
+      zswapRoots = false,
     } = options;
     const contractActionsField = contractActions
       ? `contractActions { address state }`
@@ -178,6 +192,14 @@ export class MidnightClient {
     const unshieldedSpentField = unshieldedSpentOutputs
       ? `unshieldedSpentOutputs { intentHash outputIndex owner }`
       : "";
+    const unshieldedCreatedField = unshieldedCreatedOutputs
+      ? `unshieldedCreatedOutputs { intentHash outputIndex owner }`
+      : "";
+    // zswapMerkleTreeRoot lives on RegularTransaction, not the Transaction
+    // interface, so it must be selected through an inline fragment.
+    const zswapRootFragment = zswapRoots
+      ? `... on RegularTransaction { zswapMerkleTreeRoot }`
+      : "";
     const query = `query {
       block(offset: { height: ${blockHeight} }) {
         hash
@@ -192,6 +214,8 @@ export class MidnightClient {
           ${contractActionsField}
           ${zswapField}
           ${unshieldedSpentField}
+          ${unshieldedCreatedField}
+          ${zswapRootFragment}
         }
       }
     }`;

package/src/sync-protocols/midnight/fetcher.ts

@@ -84,7 +84,9 @@ export class MidnightFetcher extends BaseDataFetcher<
       contractActions: this.config.primitives.some(
         (p) =>
           p.primitive.type !== "Midnight:Nullifier" &&
-          p.primitive.type !== "Midnight:UnshieldedSpend",
+          p.primitive.type !== "Midnight:UnshieldedSpend" &&
+          p.primitive.type !== "Midnight:UnshieldedCreate" &&
+          p.primitive.type !== "Midnight:ZswapRoot",
       ),
       zswapLedgerEvents: this.config.primitives.some(
         (p) => p.primitive.type === "Midnight:Nullifier",
@@ -92,6 +94,12 @@ export class MidnightFetcher extends BaseDataFetcher<
       unshieldedSpentOutputs: this.config.primitives.some(
         (p) => p.primitive.type === "Midnight:UnshieldedSpend",
       ),
+      unshieldedCreatedOutputs: this.config.primitives.some(
+        (p) => p.primitive.type === "Midnight:UnshieldedCreate",
+      ),
+      zswapRoots: this.config.primitives.some(
+        (p) => p.primitive.type === "Midnight:ZswapRoot",
+      ),
     };
     const self = this;
     const heights = Array.from(
@@ -105,14 +113,28 @@ export class MidnightFetcher extends BaseDataFetcher<
     const fetched = yield* all(
       heights.map(function* (height) {
         const signal = yield* useAbortSignal();
-        const result: MidnightGqlBlockState = yield* call(() =>
-          self.client.fetchBlock(height, blockFetchOptions, signal)
-        );
-        const primitives = yield* self.readPrimitives(
-          height,
-          result,
-          self.config.primitives,
-        );
+        let result: MidnightGqlBlockState;
+        let primitives;
+        try {
+          result = yield* call(() =>
+            self.client.fetchBlock(height, blockFetchOptions, signal)
+          );
+          primitives = yield* self.readPrimitives(
+            height,
+            result,
+            self.config.primitives,
+          );
+        } catch (e) {
+          // Surface per-height failures: the outer fetch loop swallows errors
+          // and retries silently, which previously hid a permanent failure
+          // (e.g. a wasm-instance mismatch in a contract ledger() call) as an
+          // ever-stuck page with no log output.
+          console.error(
+            `[Midnight] readData failed at height ${height}:`,
+            (e as Error)?.stack ?? e,
+          );
+          throw e;
+        }
         return {
           output: {
             raw: result.block as unknown as Block,
@@ -166,6 +188,10 @@ export class MidnightFetcher extends BaseDataFetcher<
         syncResults.push(...this.fetchNullifiers(height, primitiveEntry, block));
       } else if (primitiveEntry.primitive.type === "Midnight:UnshieldedSpend") {
         syncResults.push(...this.fetchUnshieldedSpends(height, primitiveEntry, block));
+      } else if (primitiveEntry.primitive.type === "Midnight:UnshieldedCreate") {
+        syncResults.push(...this.fetchUnshieldedCreates(height, primitiveEntry, block));
+      } else if (primitiveEntry.primitive.type === "Midnight:ZswapRoot") {
+        syncResults.push(...this.fetchZswapRoots(height, primitiveEntry, block));
       } else {
         asyncOps.push(
           this.fetchContractState(height, client, primitiveEntry, block),
@@ -243,6 +269,76 @@ export class MidnightFetcher extends BaseDataFetcher<
     return results;
   }
 
+  // Mirror of fetchUnshieldedSpends over `unshieldedCreatedOutputs` — every
+  // unshielded UTXO created in the block (regular AND system transactions).
+  // One state-machine input per created output.
+  @bound
+  fetchUnshieldedCreates(
+    height: number,
+    primitiveEntry: PrimitiveEntryType,
+    block: MidnightGqlBlockState,
+  ): PrimitiveType[] {
+    const results: PrimitiveType[] = [];
+    for (const tx of block.block.transactions) {
+      for (const created of tx.unshieldedCreatedOutputs ?? []) {
+        results.push({
+          syncProtocol: {
+            name: primitiveEntry.syncProtocol,
+            blockNumber: height,
+            transactionHash: tx.hash,
+            contractAddress: "",
+          },
+          primitive: primitiveEntry.primitive.name,
+          output: {
+            payloadType: "midnight-unshielded-create",
+            payload: {
+              owner: created.owner,
+              intentHash: created.intentHash,
+              outputIndex: created.outputIndex,
+              txHash: tx.hash,
+            },
+          },
+        });
+      }
+    }
+    return results;
+  }
+
+  // Emit the latest zswap coin-commitment Merkle tree root for the block: the
+  // `zswapMerkleTreeRoot` of the last RegularTransaction that carries one.
+  // Blocks with no regular transactions emit nothing (the root is unchanged).
+  @bound
+  fetchZswapRoots(
+    height: number,
+    primitiveEntry: PrimitiveEntryType,
+    block: MidnightGqlBlockState,
+  ): PrimitiveType[] {
+    let root: string | undefined;
+    let txHash = "";
+    for (const tx of block.block.transactions) {
+      if (tx.zswapMerkleTreeRoot) {
+        root = tx.zswapMerkleTreeRoot;
+        txHash = tx.hash;
+      }
+    }
+    if (!root) return [];
+    return [
+      {
+        syncProtocol: {
+          name: primitiveEntry.syncProtocol,
+          blockNumber: height,
+          transactionHash: txHash,
+          contractAddress: "",
+        },
+        primitive: primitiveEntry.primitive.name,
+        output: {
+          payloadType: "midnight-zswap-root",
+          payload: { root, txHash },
+        },
+      },
+    ];
+  }
+
   @bound
   *fetchContractState(
     height: number,
@@ -275,17 +371,21 @@ export class MidnightFetcher extends BaseDataFetcher<
         // Addresses length can be padded by 0's
         return c.address.padStart(longest, '0') === contractAddress.padStart(longest, '0');
       })!.state!;
-      const byteState = new Uint8Array(rawState.match(/.{1,2}/g)!.map(byte => parseInt(byte, 16)));
       const primitive = primitiveEntry.primitive;
-      const contractState = ContractState.deserialize(byteState);
-      const stateValue = contractState.data.state;
-      // Parse additional fields from the schema (defined per-contract on the primitive)
-      const additionalFields = primitive.parseAdditionalLedgerFields?.(stateValue) ?? {};
-      // Keep generated ledger fields, but let schema-parsed fields override.
-      // This is important for map-like fields where contract.ledger() exposes
-      // iterable wrappers that are not directly JSON-serializable.
       const f = primitive.contract ?? { ledger: (_: StateValue): Record<string, any> => ({}) };
-      const state = { ...f.ledger(stateValue), ...additionalFields };
+      const ledgerFromTxStateHex = (f as {
+        ledgerFromTxStateHex?: (rawHexState: string) => Record<string, any>;
+      }).ledgerFromTxStateHex;
+      let state: Record<string, any>;
+      if (typeof ledgerFromTxStateHex === "function") {
+        state = ledgerFromTxStateHex(rawState);
+      } else {
+        const byteState = new Uint8Array(rawState.match(/.{1,2}/g)!.map(byte => parseInt(byte, 16)));
+        const contractState = ContractState.deserialize(byteState);
+        const stateValue = contractState.data.state;
+        const additionalFields = primitive.parseAdditionalLedgerFields?.(stateValue) ?? {};
+        state = { ...f.ledger(stateValue), ...additionalFields };
+      }
       return {
         syncProtocol: {
           name: primitiveEntry.syncProtocol,

package/src/sync-protocols/midnight/state.ts

@@ -10,7 +10,6 @@ import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/c
 import { getPage } from "@effectstream/db";
 import { MidnightClient } from "./MidnightClient.ts";
 import { applyDelay } from "../common/utils.ts";
-import { bufferAtCap } from "../common/page-helpers.ts";
 
 type LatestBlock = {
   block: {
@@ -73,7 +72,6 @@ export class MidnightSyncState extends SyncState<
 
   @bound
   override *stateToInput(): Operation<Input | undefined> {
-    if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
     const latestBlockResult = yield* call(() => this.client.fetchLatestBlock());
     const latestHeight = latestBlockResult.block.height;
 
@@ -112,15 +110,6 @@ export class MidnightSyncState extends SyncState<
     rootOutput.primitives.push(...primitives);
   }
 
-  @bound
-  override outputToLastPage(data: Output): LastPage<Page, RootPage> {
-    return {
-      own: { height: data.raw.height, hash: data.raw.hash },
-      ownBlockNumber: data.raw.height,
-      root: this.toRootPage(data),
-    };
-  }
-
   @bound
   override getNamespace(): string[] {
     return [this.config.network.name, this.config.syncProtocol.name];
@@ -146,12 +135,7 @@ export class MidnightSyncState extends SyncState<
       page,
       config,
       fetcher,
-      // TODO: The urls should be part of the config.
-      new MidnightClient(
-        config.syncProtocol.indexer,
-        config.syncProtocol.indexerWS ??
-          "ws://127.0.0.1:8088/api/v3/graphql/ws",
-      ),
+      fetcher.client,
       dbConn,
     );
   }

package/src/sync-protocols/near/state.ts

@@ -10,7 +10,6 @@ import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/c
 import { getPage } from "@effectstream/db";
 import { NearClient } from "./NearClient.ts";
 import { applyDelay } from "../common/utils.ts";
-import { bufferAtCap } from "../common/page-helpers.ts";
 
 export class NearSyncState extends SyncState<
   Input,
@@ -61,7 +60,6 @@ export class NearSyncState extends SyncState<
 
   @bound
   override *stateToInput(): Operation<Input | undefined> {
-    if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
     const latestHeight = yield* call(() =>
       this.client.getLatestBlockHeight()
     );
@@ -100,15 +98,6 @@ export class NearSyncState extends SyncState<
     rootOutput.primitives.push(...primitives);
   }
 
-  @bound
-  override outputToLastPage(data: Output): LastPage<Page, RootPage> {
-    return {
-      own: { height: data.height, hash: data.hash },
-      ownBlockNumber: data.height,
-      root: this.toRootPage(data),
-    };
-  }
-
   @bound
   override getNamespace(): string[] {
     return [this.config.network.name, this.config.syncProtocol.name];

package/src/sync-protocols/ntp/state.ts

@@ -7,7 +7,7 @@ import { bound, type NtpBlockNumber } from "@effectstream/utils";
 import { type LastPage, SyncState } from "../base/state.ts";
 import type { RootOutput, RootPage } from "../types.ts";
 import type { NtpFetcher } from "./fetcher.ts";
-import { bufferAtCap, genInputRange } from "../common/page-helpers.ts";
+import { genInputRange } from "../common/page-helpers.ts";
 import type { Input, Output, Page } from "./types.ts";
 import { toMsTimestamp } from "./types.ts";
 import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/config";
@@ -48,7 +48,6 @@ export class NtpSyncState extends SyncState<
       })),
       blockNumber: Number(data.raw.blockNumber),
       timestamp: this.toRootPage(data),
-      resumePages: [],
       primitives: [],
     };
   }
@@ -60,7 +59,6 @@ export class NtpSyncState extends SyncState<
 
   @bound
   override *stateToInput(): Operation<Input | undefined> {
-    if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
     return yield* genInputRange(
       this as NtpSyncState,
       1, // NTP Pages start from 1, and the fetcher offsets using the start page (date)
@@ -88,16 +86,6 @@ export class NtpSyncState extends SyncState<
     throw new Error("Only parallel chains merge into root");
   }
 
-  @bound
-  override outputToLastPage(data: Output): LastPage<Page, RootPage> {
-    const block = Number(data.raw.blockNumber);
-    return {
-      own: block,
-      ownBlockNumber: block,
-      root: this.toRootPage(data),
-    };
-  }
-
   @bound
   override getNamespace(): string[] {
     return [this.config.network.name, this.config.syncProtocol.name];

package/src/sync-protocols/orchestration/merge.ts

@@ -124,11 +124,6 @@ export function* mergeIntoRoot<SyncProtocol extends AllSyncProtocols>(
     }
     const output = iState.bufferedData.peekAt(0)!;
     const newRoot = iState.toRootOutput(output.output);
-    // Resume marker: the single datum this root block is built from (CLAUDE.md #5).
-    newRoot.resumePages.push({
-      protocol_name: state.name,
-      lastPage: iState.outputToLastPage(output.output),
-    });
     return {
       updateCache: () => {
         output.cleanup();
@@ -173,17 +168,6 @@ export function* mergeIntoRoot<SyncProtocol extends AllSyncProtocols>(
     iState.mergeDatum,
   );
 
-  // Resume marker: the last (highest) datum merged this iteration, still in the
-  // Deque at `cleanups.length - 1` (cleanup runs after send). Nothing consumed →
-  // no marker. See CLAUDE.md #5.
-  if (cleanups.length > 0) {
-    const lastConsumed = iState.bufferedData.peekAt(cleanups.length - 1)!;
-    rootInfo.value.resumePages.push({
-      protocol_name: state.name,
-      lastPage: iState.outputToLastPage(lastConsumed.output),
-    });
-  }
-
   return {
     updateCache: () => {
       state.bufferedData.remove(0, cleanups.length);

package/src/sync-protocols/test/state.ts

@@ -7,7 +7,7 @@ import { bound, type BlockNumber } from "@effectstream/utils";
 import { type LastPage, SyncState } from "../base/state.ts";
 import type { RootOutput, RootPage } from "../types.ts";
 import type { TestFetcher } from "./fetcher.ts";
-import { bufferAtCap, genInputRange } from "../common/page-helpers.ts";
+import { genInputRange } from "../common/page-helpers.ts";
 import type { Input, Output, Page } from "./types.ts";
 import { toMsTimestamp } from "./types.ts";
 import type {
@@ -60,7 +60,6 @@ export class TestSyncState extends SyncState<
       })),
       blockNumber: Number(data.raw.blockNumber),
       timestamp: this.toRootPage(data),
-      resumePages: [],
       primitives: data.primitives.map((p) => ({
         ...p,
         source: this.config.syncProtocol.name,
@@ -78,7 +77,6 @@ export class TestSyncState extends SyncState<
 
   @bound
   override *stateToInput(): Operation<Input | undefined> {
-    if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
     return yield* genInputRange(
       this as TestSyncState,
       this.config.syncProtocol.startBlockHeight as BlockNumber,
@@ -116,16 +114,6 @@ export class TestSyncState extends SyncState<
     rootOutput.blockInfo.push(...blockInfo);
   }
 
-  @bound
-  override outputToLastPage(data: Output): LastPage<Page, RootPage> {
-    const block = Number(data.raw.blockNumber);
-    return {
-      own: block,
-      ownBlockNumber: block as BlockNumber,
-      root: this.toRootPage(data),
-    };
-  }
-
   @bound
   override getNamespace(): string[] {
     return [this.config.network.name, this.config.syncProtocol.name];

package/src/sync-protocols/utxorpc/WatchMultiplexer.ts

@@ -40,15 +40,13 @@ export class WatchMultiplexer {
   ) {
     for (const [key, { predicate }] of primitivesByPredicate) {
       const effective = isEffectiveServerPredicate(predicate);
-      // TODO(dolos): when Dolos implements has_certificate, remove this fallback
       const sdkPredicate = effective
         ? configPredicateToSdkPredicate(predicate)
         : {};
 
       if (!effective) {
         console.warn(
-          `[WatchMultiplexer] Predicate for key "${key}" cannot be server-side filtered ` +
-          `(Dolos doesn't support has_certificate). Falling back to match-all with client-side filtering.`
+          `[WatchMultiplexer] Predicate for key "${key}" has no server-filterable fields. Falling back to match-all with client-side filtering.`
         );
       }
 

package/src/sync-protocols/utxorpc/state.ts

@@ -1,5 +1,4 @@
 import { applyDelay } from "../common/utils.ts";
-import { bufferAtCap } from "../common/page-helpers.ts";
 import type { Operation } from "effection";
 import { call } from "effection";
 import { getPage } from "@effectstream/db";
@@ -72,7 +71,6 @@ export class UtxoRpcSyncState extends SyncState<
 
   @bound
   override *stateToInput(): Operation<Input | undefined> {
-    if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
     // Get the value of the latest block height.
     const tipHeight = yield* call(() => this.fetcher.lastHeight());
     if (!tipHeight) {
@@ -159,22 +157,6 @@ export class UtxoRpcSyncState extends SyncState<
     rootOutput.blockInfo.push(...blockInfo);
   }
 
-  @bound
-  override outputToLastPage(data: Output): LastPage<Page, RootPage> {
-    // Mirror the fetcher's `own`: slot/height from the block header, and the
-    // block hash from `blockHashes` (the fetcher builds it as `[completed.hash]`,
-    // the same value it stored in `own.hash`).
-    return {
-      own: {
-        slot: Number(data.raw.header!.slot),
-        height: Number(data.raw.header!.height),
-        hash: data.blockHashes[0],
-      },
-      ownBlockNumber: Number(data.raw.header!.height),
-      root: this.toRootPage(data),
-    };
-  }
-
   @bound
   override getNamespace(): string[] {
     return [this.config.network.name, this.config.syncProtocol.name];

package/src/sync-protocols/utxorpc/utils.ts

@@ -1,5 +1,5 @@
 import type { cardano } from '@utxorpc/spec';
-import type { UtxorpcAddressPattern, UtxorpcAnyChainTxPattern, UtxorpcAssetPattern, UtxorpcTxOutputPattern, UtxorpcTxPredicate } from "@effectstream/config";
+import type { UtxorpcAddressPattern, UtxorpcAnyChainTxPattern, UtxorpcAssetPattern, UtxorpcCertificatePattern, UtxorpcTxOutputPattern, UtxorpcTxPredicate } from "@effectstream/config";
 import { hexStringToUint8Array } from "@effectstream/utils";
 import { Cardano } from '@cardano-sdk/core';
 import { HexBlob } from '@cardano-sdk/util';
@@ -111,14 +111,24 @@ function convertAssetPattern(pattern: UtxorpcAssetPattern): Record<string, Uint8
   return result;
 }
 
+function convertCertificatePattern(pattern: boolean | UtxorpcCertificatePattern): Record<string, unknown> {
+  if (typeof pattern === "boolean") return {};
+  if (pattern.stake_delegation) {
+    const value: Record<string, unknown> = {};
+    if (pattern.stake_delegation.pool_keyhash) {
+      value.poolKeyhash = hexStringToUint8Array(pattern.stake_delegation.pool_keyhash);
+    }
+    return { certificateType: { case: "stakeDelegation", value } };
+  }
+  if (pattern.any_pool_keyhash) {
+    return { certificateType: { case: "anyPoolKeyhash", value: hexStringToUint8Array(pattern.any_pool_keyhash) } };
+  }
+  return {};
+}
+
 /**
  * Convert a config-layer UtxorpcTxPredicate (snake_case) to the SDK's TxPredicate (camelCase).
- *
- * NOTE: `has_certificate` is in the UTxORPC spec but Dolos does NOT implement server-side
- * filtering for it. Predicates using has_certificate will be sent to the server but silently
- * ignored — client-side filtering in matchesPredicate() still catches them.
  */
-// TODO(dolos): when Dolos implements has_certificate filtering, this conversion will just work
 export function configPredicateToSdkPredicate(config: UtxorpcTxPredicate): TxPredicate {
   const result: TxPredicate = {};
 
@@ -128,7 +138,7 @@ export function configPredicateToSdkPredicate(config: UtxorpcTxPredicate): TxPre
     if (c.has_address) match.hasAddress = convertAddressPattern(c.has_address);
     if (c.moves_asset) match.movesAsset = convertAssetPattern(c.moves_asset);
     if (c.mints_asset) match.mintsAsset = convertAssetPattern(c.mints_asset);
-    if (c.has_certificate) match.hasCertificate = {};
+    if (c.has_certificate) match.hasCertificate = convertCertificatePattern(c.has_certificate);
     result.match = match as TxPredicate["match"];
   }
 
@@ -148,13 +158,10 @@ export function predicateKey(predicate: UtxorpcTxPredicate): string {
 }
 
 /**
- * Returns true if this predicate can be effectively filtered server-side by Dolos.
- * Currently Dolos Watch ignores `has_certificate`, so predicates relying solely on it
- * are not effective and should fall back to match-all.
+ * Returns true if this predicate can be effectively filtered server-side.
  */
-// TODO(dolos): remove this when Dolos implements has_certificate filtering
 export function isEffectiveServerPredicate(predicate: UtxorpcTxPredicate): boolean {
   if (!predicate.match?.cardano) return false;
   const c = predicate.match.cardano;
-  return !!(c.has_address || c.moves_asset || c.mints_asset);
+  return !!(c.has_address || c.moves_asset || c.mints_asset || c.has_certificate);
 }