@effectstream/sync 0.100.12 → 0.100.14

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@effectstream/sync",
-  "version": "0.100.12",
+  "version": "0.100.14",
   "license": "MIT",
   "homepage": "https://effectstream.github.io/docs/",
   "repository": {
@@ -13,10 +13,10 @@
   },
   "exports": "./src/mod.ts",
   "dependencies": {
-    "@effectstream/config": "0.100.12",
-    "@effectstream/db": "0.100.12",
-    "@effectstream/log": "0.100.12",
-    "@effectstream/utils": "0.100.12",
+    "@effectstream/config": "0.100.14",
+    "@effectstream/db": "0.100.14",
+    "@effectstream/log": "0.100.14",
+    "@effectstream/utils": "0.100.14",
     "@midnight-ntwrk/ledger-v8": "8.0.3",
     "@midnight-ntwrk/midnight-js-indexer-public-data-provider": "4.0.2",
     "@midnight-ntwrk/onchain-runtime": "npm:@midnight-ntwrk/onchain-runtime-v3@3.0.0",

package/src/sync-protocols/avail/fetcher.ts

@@ -48,11 +48,16 @@ export class AvailFetcher extends BaseDataFetcher<
     lastPage: LastPage<Page, RootPage> | undefined,
   ): Operation<DataFetched<Output, Page, RootPage>> {
     const outputs: OutputAndCleanup<Output>[] = [];
+    const fetchAllBlocks = this.config.primitives.some(
+      (p) => p.primitive.getAllBlockHeaders,
+    );
     console.log(
       `[Avail] Fetching blocks from ${data.from} to ${data.to}. ${
         data.isPresync ? "[presync]" : ""
       }`,
     );
+
+    let lastHeader: Output["raw"] | undefined;
     for (let height = data.from; height <= data.to; height++) {
       let header;
       try {
@@ -60,26 +65,28 @@ export class AvailFetcher extends BaseDataFetcher<
           this.client.getBlockHeaderFromHeight(height)
         );
       } catch (_e) {
-        // Likely reached the tip; stop here
         break;
       }
 
+      lastHeader = header;
       const primitives = yield* this.readPrimitives(
         height,
         header,
         this.config.primitives,
       );
 
-      outputs.push({
-        output: {
-          raw: header,
-          primitives,
-        },
-        cleanup: () => {},
-      });
+      if (fetchAllBlocks || primitives.length > 0 || height === data.to) {
+        outputs.push({
+          output: {
+            raw: header,
+            primitives,
+          },
+          cleanup: () => {},
+        });
+      }
     }
 
-    if (outputs.length === 0) {
+    if (!lastHeader) {
       if (!lastPage) {
         throw new Error(
           `Could not fetch any blocks from ${data.from} to ${data.to} and no previous page was found.`,
@@ -91,16 +98,18 @@ export class AvailFetcher extends BaseDataFetcher<
       };
     }
 
-    const lastOutput = outputs[outputs.length - 1].output;
     return {
       output: outputs,
       lastPage: {
-        ownBlockNumber: lastOutput.raw.number,
+        ownBlockNumber: lastHeader.number,
         own: {
-          height: lastOutput.raw.number,
-          hash: lastOutput.raw.hash,
+          height: lastHeader.number,
+          hash: lastHeader.hash,
         },
-        root: rootConversion.toRootPage(lastOutput),
+        root: rootConversion.toRootPage({
+          raw: lastHeader,
+          primitives: [],
+        }),
       },
     };
   }

package/src/sync-protocols/bitcoin/fetcher.ts

@@ -111,24 +111,31 @@ export class BitcoinFetcher
     );
     const groupedByPage = this.groupByPage(primitives);
 
+    const fetchAllBlocks = this.config.primitives.some(
+      (p) => p.primitive.getAllBlockHeaders,
+    );
+
     const outputs: OutputAndCleanup<Output>[] = [];
+    let lastBlock: BitcoinBlock | undefined;
     for (let height = Number(data.from); height <= Number(data.to); height++) {
       const blockHeight = height as Page;
       const block = yield* call(() => pageFetcher(blockHeight));
-      outputs.push({
-        output: {
-          raw: block,
-          primitives: groupedByPage[String(height)] ?? [],
-          blockHashes: [block.hash],
-        },
-        cleanup: () => {},
-      });
+      lastBlock = block;
+      const blockPrimitives = groupedByPage[String(height)] ?? [];
+      if (fetchAllBlocks || blockPrimitives.length > 0 || height === Number(data.to)) {
+        outputs.push({
+          output: {
+            raw: block,
+            primitives: blockPrimitives,
+            blockHashes: [block.hash],
+          },
+          cleanup: () => {},
+        });
+      }
     }
-    const lastOutput = outputs[outputs.length - 1];
-    if (!lastOutput) {
+    if (!lastBlock) {
       throw new Error("BitcoinFetcher: no block data fetched for interval");
     }
-    const lastBlock = lastOutput.output.raw;
     const lastPageRef = this.asPage(Number(data.to));
     return {
       output: outputs,

package/src/sync-protocols/celestia/CelestiaClient.ts

@@ -53,6 +53,7 @@ const CELESTIA_NODE_URL = ENV.getString(
   "CELESTIA_NODE_URL",
   "http://localhost:26658",
 );
+const CELESTIA_AUTH_TOKEN = ENV.getString("CELESTIA_AUTH_TOKEN", "");
 
 /**
  * Converts a hex-encoded Celestia namespace to a base64-encoded 29-byte array.
@@ -81,9 +82,11 @@ export class CelestiaClient {
   private async rpc<T>(method: string, params: unknown[]): Promise<T | null> {
     let res: Response;
     try {
+      const headers: Record<string, string> = { "Content-Type": "application/json" };
+      if (CELESTIA_AUTH_TOKEN) headers["Authorization"] = `Bearer ${CELESTIA_AUTH_TOKEN}`;
       res = await fetch(this.rpcUrl, {
         method: "POST",
-        headers: { "Content-Type": "application/json" },
+        headers,
         body: JSON.stringify({ jsonrpc: "2.0", id: 1, method, params }),
       });
     } catch (err: unknown) {

package/src/sync-protocols/celestia/fetcher.ts

@@ -47,6 +47,9 @@ export class CelestiaFetcher extends BaseDataFetcher<
     lastPage: LastPage<Page, RootPage> | undefined,
   ): Operation<DataFetched<Output, Page, RootPage>> {
     const outputs: OutputAndCleanup<Output>[] = [];
+    const fetchAllBlocks = this.config.primitives.some(
+      (p) => p.primitive.getAllBlockHeaders,
+    );
 
     console.log(
       `[Celestia] Fetching blocks from ${data.from} to ${data.to}.${
@@ -54,6 +57,7 @@ export class CelestiaFetcher extends BaseDataFetcher<
       }`,
     );
 
+    let lastFetchedOutput: Output | undefined;
     for (let height = data.from; height <= data.to; height++) {
       let extHeader;
       try {
@@ -61,7 +65,6 @@ export class CelestiaFetcher extends BaseDataFetcher<
           this.client.getHeaderAtHeight(height)
         );
       } catch (_e) {
-        // Likely reached the chain tip; stop here
         break;
       }
 
@@ -73,18 +76,23 @@ export class CelestiaFetcher extends BaseDataFetcher<
         this.config.primitives,
       );
 
-      outputs.push({
-        output: {
-          timestamp: extHeader.header.time,
-          height,
-          hash: blockHash,
-          primitives,
-        },
-        cleanup: () => {},
-      });
+      const output: Output = {
+        timestamp: extHeader.header.time,
+        height,
+        hash: blockHash,
+        primitives,
+      };
+      lastFetchedOutput = output;
+
+      if (fetchAllBlocks || primitives.length > 0 || height === data.to) {
+        outputs.push({
+          output,
+          cleanup: () => {},
+        });
+      }
     }
 
-    if (outputs.length === 0) {
+    if (!lastFetchedOutput) {
       if (!lastPage) {
         throw new Error(
           `[Celestia] Could not fetch any blocks from ${data.from} to ${data.to} and no previous page was found.`,
@@ -96,16 +104,15 @@ export class CelestiaFetcher extends BaseDataFetcher<
       };
     }
 
-    const lastOutput = outputs[outputs.length - 1].output;
     return {
       output: outputs,
       lastPage: {
-        ownBlockNumber: lastOutput.height,
+        ownBlockNumber: lastFetchedOutput.height,
         own: {
-          height: lastOutput.height,
-          hash: lastOutput.hash,
+          height: lastFetchedOutput.height,
+          hash: lastFetchedOutput.hash,
         },
-        root: rootConversion.toRootPage(lastOutput),
+        root: rootConversion.toRootPage(lastFetchedOutput),
       },
     };
   }

package/src/sync-protocols/evm/fetcher.ts

@@ -25,7 +25,7 @@ import type {
 import { PageSchema } from "./types.ts";
 import {
   fetchNewestPage,
-  genImmediatePageRequests,
+  genOnDemandPageRequests,
   type PageRange,
   type PageRequest,
 } from "../base/page.ts";
@@ -62,15 +62,17 @@ export class EvmFetcher
     data: Input,
     rootConversion: RootConversion<Output, RootOutput, RootPage>,
   ): Operation<DataFetched<Output, Page, RootPage>> {
-    const allPages: Page[] = [];
-    for (let p = data.from; p <= data.to; p++) allPages.push(p);
-    const pageFetcher = genImmediatePageRequests(
-      allPages,
+    const fetchAllBlocks = this.config.primitives.some(
+      (p) => p.primitive.getAllBlockHeaders,
+    );
+
+    const pageFetcher = genOnDemandPageRequests(
+      data.from,
+      data.to,
       (page) => this.client.getBlock({ blockNumber: BigInt(page) }),
+      blockNumberRelation,
     );
 
-    // TODO: if we expect multiple primitives per block
-    //       it can, depending on the chain, be faster to just download the full block and parse it locally
     const primitives = yield* this.readPrimitives(
       data,
       pageFetcher,
@@ -92,40 +94,43 @@ export class EvmFetcher
             };
           }),
         ),
-        // we always need to get the last page in the parallel case
-        // so we can use it for pagination
+        // we always need the last page for pagination
         call(() => pageFetcher(Number(data.to))),
       ],
     );
 
-    // Build an output with all page info, as we need all the hashes to build the effectstream-block-hash
-    // All blocks are already pre-fetched in parallel by genImmediatePageRequests above
     const allOutputs: Output[] = [];
     for (let page = data.from; page <= data.to; page++) {
       const _output = output.find((o) => o.raw.number === BigInt(page));
       if (_output) {
         allOutputs.push(_output);
-      } else {
+      } else if (fetchAllBlocks) {
         const raw = yield* call(() => pageFetcher(page));
         allOutputs.push({
           raw,
           primitives: [],
           blockHashes: [raw.hash] as EvmBlockHash[],
         });
+      } else if (page === data.to) {
+        allOutputs.push({
+          raw: lastPage,
+          primitives: [],
+          blockHashes: [lastPage.hash] as EvmBlockHash[],
+        });
       }
     }
 
     return {
       output: allOutputs.map((o) => ({
         output: o,
-        cleanup: () => {}, // no cleanup required
+        cleanup: () => {},
       })),
       lastPage: {
         own: Number(data.to),
         ownBlockNumber: Number(data.to),
         root: rootConversion.toRootPage({
           blockHashes: [lastPage.hash] as EvmBlockHash[],
-          primitives: [], // unused in toRootPage
+          primitives: [],
           raw: lastPage,
         }),
       },

package/src/sync-protocols/midnight/MidnightClient.ts

@@ -27,6 +27,11 @@ export interface MidnightGqlBlockState {
         raw:   string;
         maxId: number;
       }[];
+      unshieldedSpentOutputs?: {
+        intentHash:  string;
+        outputIndex: number;
+        owner:       string;
+      }[];
     }[];
   };
 };
@@ -36,6 +41,8 @@ export interface BlockFetchOptions {
   contractActions?: boolean;
   /** Include zswapLedgerEvents in the transaction fields (needed for MidnightNullifierPrimitive). Default: true */
   zswapLedgerEvents?: boolean;
+  /** Include unshieldedSpentOutputs (needed for MidnightUnshieldedSpendPrimitive). Default: false */
+  unshieldedSpentOutputs?: boolean;
 }
 
 type PublicDataProvider = ReturnType<typeof indexerPublicDataProvider>;
@@ -157,13 +164,20 @@ export class MidnightClient {
     options: BlockFetchOptions = {},
     signal?: AbortSignal,
   ): Promise<MidnightGqlBlockState> {
-    const { contractActions = true, zswapLedgerEvents = true } = options;
+    const {
+      contractActions = true,
+      zswapLedgerEvents = true,
+      unshieldedSpentOutputs = false,
+    } = options;
     const contractActionsField = contractActions
       ? `contractActions { address state }`
       : "";
     const zswapField = zswapLedgerEvents
       ? `zswapLedgerEvents { id raw maxId }`
       : "";
+    const unshieldedSpentField = unshieldedSpentOutputs
+      ? `unshieldedSpentOutputs { intentHash outputIndex owner }`
+      : "";
     const query = `query {
       block(offset: { height: ${blockHeight} }) {
         hash
@@ -177,6 +191,7 @@ export class MidnightClient {
           hash
           ${contractActionsField}
           ${zswapField}
+          ${unshieldedSpentField}
         }
       }
     }`;

package/src/sync-protocols/midnight/fetcher.ts

@@ -82,22 +82,27 @@ export class MidnightFetcher extends BaseDataFetcher<
     );
     const blockFetchOptions: BlockFetchOptions = {
       contractActions: this.config.primitives.some(
-        (p) => p.primitive.type !== "Midnight:Nullifier",
+        (p) =>
+          p.primitive.type !== "Midnight:Nullifier" &&
+          p.primitive.type !== "Midnight:UnshieldedSpend",
       ),
       zswapLedgerEvents: this.config.primitives.some(
         (p) => p.primitive.type === "Midnight:Nullifier",
       ),
+      unshieldedSpentOutputs: this.config.primitives.some(
+        (p) => p.primitive.type === "Midnight:UnshieldedSpend",
+      ),
     };
     const self = this;
     const heights = Array.from(
       { length: data.to - data.from + 1 },
       (_, i) => i + data.from,
     );
+    const fetchAllBlocks = this.config.primitives.some(
+      (p) => p.primitive.getAllBlockHeaders,
+    );
+
     const fetched = yield* all(
-      // The endpoint does not support range requests,
-      // so we fetch each block individually in parallel.
-      // useAbortSignal() ties each fetch's lifetime to its Effection scope,
-      // so cancellation by all() cleanly aborts the underlying HTTP request.
       heights.map(function* (height) {
         const signal = yield* useAbortSignal();
         const result: MidnightGqlBlockState = yield* call(() =>
@@ -117,9 +122,22 @@ export class MidnightFetcher extends BaseDataFetcher<
         };
       }),
     );
-    outputs.push(...fetched);
 
-    const lastOutput = outputs[outputs.length - 1].output;
+    const lastFetched = fetched[fetched.length - 1];
+    if (fetchAllBlocks) {
+      outputs.push(...fetched);
+    } else {
+      for (const item of fetched) {
+        if (item.output.primitives.length > 0) {
+          outputs.push(item);
+        }
+      }
+      if (outputs.length === 0 || outputs[outputs.length - 1] !== lastFetched) {
+        outputs.push(lastFetched);
+      }
+    }
+
+    const lastOutput = lastFetched.output;
     return {
       output: outputs,
       lastPage: {
@@ -146,6 +164,8 @@ export class MidnightFetcher extends BaseDataFetcher<
     for (const primitiveEntry of primitiveEntries) {
       if (primitiveEntry.primitive.type === "Midnight:Nullifier") {
         syncResults.push(...this.fetchNullifiers(height, primitiveEntry, block));
+      } else if (primitiveEntry.primitive.type === "Midnight:UnshieldedSpend") {
+        syncResults.push(...this.fetchUnshieldedSpends(height, primitiveEntry, block));
       } else {
         asyncOps.push(
           this.fetchContractState(height, client, primitiveEntry, block),
@@ -191,6 +211,38 @@ export class MidnightFetcher extends BaseDataFetcher<
     return results;
   }
 
+  @bound
+  fetchUnshieldedSpends(
+    height: number,
+    primitiveEntry: PrimitiveEntryType,
+    block: MidnightGqlBlockState,
+  ): PrimitiveType[] {
+    const results: PrimitiveType[] = [];
+    for (const tx of block.block.transactions) {
+      for (const spend of tx.unshieldedSpentOutputs ?? []) {
+        results.push({
+          syncProtocol: {
+            name: primitiveEntry.syncProtocol,
+            blockNumber: height,
+            transactionHash: tx.hash,
+            contractAddress: "",
+          },
+          primitive: primitiveEntry.primitive.name,
+          output: {
+            payloadType: "midnight-unshielded-spend",
+            payload: {
+              owner: spend.owner,
+              intentHash: spend.intentHash,
+              outputIndex: spend.outputIndex,
+              txHash: tx.hash,
+            },
+          },
+        });
+      }
+    }
+    return results;
+  }
+
   @bound
   *fetchContractState(
     height: number,

package/src/sync-protocols/utxorpc/WatchMultiplexer.ts

@@ -0,0 +1,300 @@
+import { type CardanoSyncClient, CardanoWatchClient, type CardanoTxEvent, type ClientBuilderOptions } from "@utxorpc/sdk";
+import { cardano } from "@utxorpc/spec";
+import type { watch } from "@utxorpc/spec";
+import type { OutputAndCleanup } from "../base/state.ts";
+import Deque from "denque";
+import type { BlockNumber } from "@effectstream/utils";
+import { conditionVariable, hexStringToUint8Array } from "@effectstream/utils";
+import type { ChainPoint, CompletedBlock, PrimitiveEntryType } from "./types.ts";
+import type { UtxorpcTxPredicate } from "@effectstream/config";
+import { configPredicateToSdkPredicate, isEffectiveServerPredicate, predicateKey } from "./utils.ts";
+import { Buffer } from "node:buffer";
+
+type WatcherPendingBlock = {
+  txs: cardano.Tx[];
+  block: cardano.Block | undefined;
+  idle: boolean;
+  idleRef: watch.BlockRef | undefined;
+};
+
+type WatcherState = {
+  predicateKey: string;
+  sdkPredicate: ReturnType<typeof configPredicateToSdkPredicate>;
+  pendingBlocks: Map<number, WatcherPendingBlock>;
+  highestSeenHeight: number;
+};
+
+export class WatchMultiplexer {
+  private readonly watchers: Map<string, WatcherState> = new Map();
+  private readonly completedBuffer: Deque<OutputAndCleanup<CompletedBlock>> = new Deque();
+  private readonly newDataCondVar = conditionVariable<void>();
+  private highestConfirmedHeight: number | undefined = undefined;
+  private lowestIncompleteHeight: number = 0;
+  private slotOriginTimestamp: bigint | undefined = undefined;
+
+  constructor(
+    private readonly clientOptions: ClientBuilderOptions,
+    primitivesByPredicate: Map<string, { predicate: UtxorpcTxPredicate; primitiveEntries: PrimitiveEntryType[] }>,
+    private readonly finality: BlockNumber,
+    private readonly syncClient: CardanoSyncClient,
+  ) {
+    for (const [key, { predicate }] of primitivesByPredicate) {
+      const effective = isEffectiveServerPredicate(predicate);
+      // TODO(dolos): when Dolos implements has_certificate, remove this fallback
+      const sdkPredicate = effective
+        ? configPredicateToSdkPredicate(predicate)
+        : {};
+
+      if (!effective) {
+        console.warn(
+          `[WatchMultiplexer] Predicate for key "${key}" cannot be server-side filtered ` +
+          `(Dolos doesn't support has_certificate). Falling back to match-all with client-side filtering.`
+        );
+      }
+
+      this.watchers.set(key, {
+        predicateKey: key,
+        sdkPredicate,
+        pendingBlocks: new Map(),
+        highestSeenHeight: 0,
+      });
+    }
+  }
+
+  public async resolveOrigin(): Promise<ChainPoint> {
+    const blocks = await this.syncClient.fetchHistory(undefined, 1);
+    if (!blocks.length) throw new Error("No blocks found on chain");
+    const header = blocks[0].parsedBlock.header!;
+    return {
+      slot: Number(header.slot),
+      hash: Buffer.from(header.hash).toString("hex"),
+    };
+  }
+
+  public async start(point?: ChainPoint): Promise<void> {
+    if (this.watchers.size === 0) {
+      throw new Error("No watchers configured");
+    }
+
+    if (this.slotOriginTimestamp === undefined) {
+      try {
+        const blocks = await this.syncClient.fetchHistory(undefined, 1);
+        if (blocks.length > 0) {
+          const blk = blocks[0].parsedBlock;
+          if (blk.timestamp && blk.header?.slot != null) {
+            this.slotOriginTimestamp = blk.timestamp - blk.header.slot;
+          }
+        }
+      } catch {
+        // Chain may not be ready yet — slotOriginTimestamp will be set on the first Apply event
+      }
+    }
+
+    const intersect = point ? [point] : [];
+    const promises: Promise<void>[] = [];
+    for (const [key, watcher] of this.watchers) {
+      const watchClient = new CardanoWatchClient(this.clientOptions);
+      promises.push(this.runWatcher(key, watcher, watchClient, intersect));
+    }
+
+    console.log(`[WatchMultiplexer] Started ${this.watchers.size} watcher(s)`);
+    await Promise.all(promises);
+  }
+
+  private async runWatcher(
+    key: string,
+    watcher: WatcherState,
+    watchClient: CardanoWatchClient,
+    intersect: ChainPoint[],
+  ): Promise<void> {
+    const stream = watchClient.watchTxByPredicate(watcher.sdkPredicate, intersect);
+    for await (const event of stream) {
+      this.handleEvent(key, watcher, event);
+    }
+    console.warn(`[WatchMultiplexer] Watcher "${key}" stream ended unexpectedly`);
+  }
+
+  private handleEvent(
+    key: string,
+    watcher: WatcherState,
+    event: CardanoTxEvent,
+  ): void {
+    if (event.action === "apply") {
+      const height = Number(event.Block.header!.height);
+      let pending = watcher.pendingBlocks.get(height);
+      if (!pending) {
+        pending = { txs: [], block: undefined, idle: false, idleRef: undefined };
+        watcher.pendingBlocks.set(height, pending);
+      }
+      pending.txs.push(event.Tx);
+      pending.block = event.Block;
+      pending.idle = false;
+      if (this.slotOriginTimestamp === undefined) {
+        const blk = event.Block;
+        this.slotOriginTimestamp = blk.timestamp - blk.header!.slot;
+      }
+      if (height > watcher.highestSeenHeight) {
+        watcher.highestSeenHeight = height;
+      }
+      this.tryCompleteBlocks();
+    } else if (event.action === "idle") {
+      const height = Number(event.BlockRef.height);
+      let pending = watcher.pendingBlocks.get(height);
+      if (!pending) {
+        pending = { txs: [], block: undefined, idle: true, idleRef: event.BlockRef };
+        watcher.pendingBlocks.set(height, pending);
+      } else {
+        pending.idle = true;
+        pending.idleRef = event.BlockRef;
+      }
+      if (height > watcher.highestSeenHeight) {
+        watcher.highestSeenHeight = height;
+      }
+      this.tryCompleteBlocks();
+    } else if (event.action === "undo") {
+      const height = Number(event.Block.header!.height);
+      this.handleUndo(height);
+    }
+  }
+
+  private handleUndo(fromHeight: number): void {
+    // Pop all completed blocks at or above the undo height
+    while (this.completedBuffer.length > 0) {
+      const last = this.completedBuffer.peekBack()!;
+      if (last.output.height < fromHeight) break;
+      this.completedBuffer.pop();
+    }
+
+    // Clear pending blocks at or above the undo height for all watchers
+    for (const [, watcher] of this.watchers) {
+      for (const [h] of watcher.pendingBlocks) {
+        if (h >= fromHeight) {
+          watcher.pendingBlocks.delete(h);
+        }
+      }
+      if (watcher.highestSeenHeight >= fromHeight) {
+        watcher.highestSeenHeight = fromHeight - 1;
+      }
+    }
+
+    // Reset confirmed height
+    if (this.highestConfirmedHeight != null && this.highestConfirmedHeight >= fromHeight) {
+      this.highestConfirmedHeight = fromHeight - 1;
+    }
+    if (this.lowestIncompleteHeight >= fromHeight) {
+      this.lowestIncompleteHeight = fromHeight;
+    }
+  }
+
+  private tryCompleteBlocks(): void {
+    const minHighest = Math.min(...[...this.watchers.values()].map(w => w.highestSeenHeight));
+    if (minHighest <= 0) return;
+
+    let height = this.lowestIncompleteHeight;
+    if (height === 0) {
+      let lowest = Infinity;
+      for (const [, watcher] of this.watchers) {
+        for (const [h] of watcher.pendingBlocks) {
+          if (h < lowest) lowest = h;
+        }
+      }
+      if (lowest === Infinity) return;
+      height = lowest;
+    }
+
+    // A block at `height` is only safe to process once every watcher has
+    // advanced past it (received an event for height+1 or later).  This
+    // guarantees all Apply events for that height have been delivered —
+    // the Watch protocol can send multiple Apply events per block.
+    while (height < minHighest) {
+      const completed = this.buildCompletedBlock(height);
+      if (completed) {
+        const finalityThreshold = minHighest - this.finality;
+        if (height <= finalityThreshold) {
+          this.completedBuffer.push({
+            output: completed,
+            cleanup: () => this.completedBuffer.shift(),
+          });
+          this.highestConfirmedHeight = height;
+          this.newDataCondVar.wake();
+        } else {
+          break;
+        }
+      }
+
+      for (const [, watcher] of this.watchers) {
+        watcher.pendingBlocks.delete(height);
+      }
+
+      height++;
+    }
+    this.lowestIncompleteHeight = height;
+  }
+
+  private buildCompletedBlock(height: number): CompletedBlock | null {
+    let block: cardano.Block | undefined;
+    let slot = 0;
+    let hash = "";
+    const matchedTxsByPredicate = new Map<string, cardano.Tx[]>();
+
+    for (const [key, watcher] of this.watchers) {
+      const pending = watcher.pendingBlocks.get(height);
+      if (!pending) return null;
+
+      if (pending.block) {
+        block = pending.block;
+        slot = Number(pending.block.header!.slot);
+        hash = Buffer.from(pending.block.header!.hash).toString("hex");
+      } else if (pending.idleRef && !block) {
+        slot = Number(pending.idleRef.slot);
+        hash = Buffer.from(pending.idleRef.hash).toString("hex");
+      }
+
+      if (pending.txs.length > 0) {
+        matchedTxsByPredicate.set(key, pending.txs);
+      }
+    }
+
+    if (!block) {
+      const idleSlot = BigInt(slot);
+      const idleHash = hexStringToUint8Array(hash);
+      const estimatedTimestamp = this.slotOriginTimestamp != null
+        ? this.slotOriginTimestamp + idleSlot
+        : idleSlot;
+      block = new cardano.Block({
+        header: new cardano.BlockHeader({
+          slot: idleSlot,
+          hash: idleHash,
+          height: BigInt(height),
+        }),
+        body: new cardano.BlockBody({ tx: [] }),
+        timestamp: estimatedTimestamp,
+      });
+    }
+
+    return { height, slot, hash, block, matchedTxsByPredicate };
+  }
+
+  public fetchBlocks(
+    from: BlockNumber,
+    to: BlockNumber,
+  ): OutputAndCleanup<CompletedBlock>[] {
+    const blocks: OutputAndCleanup<CompletedBlock>[] = [];
+    for (let i = 0; i < this.completedBuffer.length; i++) {
+      const entry = this.completedBuffer.peekAt(i)!;
+      if (entry.output.height >= from) {
+        blocks.push(entry);
+      }
+      if (entry.output.height >= to) {
+        break;
+      }
+    }
+    return blocks;
+  }
+
+  public lastHeight(): bigint | undefined {
+    if (this.highestConfirmedHeight == null) return undefined;
+    return BigInt(this.highestConfirmedHeight);
+  }
+
+}

package/src/sync-protocols/utxorpc/fetcher.ts

@@ -2,31 +2,30 @@ import { BaseDataFetcher, type DataFetched } from "../base/fetcher.ts";
 import type { RootOutput, RootPage } from "../types.ts";
 import type { Operation } from "effection";
 import { bound, uint8ArrayToHexString } from "@effectstream/utils";
-import type { ChainPoint, ConfigType, Input, Output, Page, PrimitiveType } from "./types.ts";
+import type { ChainPoint, CompletedBlock, ConfigType, Input, Output, Page, PrimitiveType } from "./types.ts";
 import type { OutputAndCleanup, RootConversion } from "../base/state.ts";
-import type { BufferedRpc } from "./BufferedRpc.ts";
-import { Buffer } from "node:buffer";
+import type { WatchMultiplexer } from "./WatchMultiplexer.ts";
 import type { cardano } from "@utxorpc/spec";
-import { hashEqual, matchesPredicate } from "./utils.ts";
+import { matchesPredicate, predicateKey } from "./utils.ts";
 
 export class UtxoRpcFetcher
-  extends BaseDataFetcher<Input, Output, RootOutput, Page, RootPage> // PrimitiveFetcher<Input, Page, cardano.Block, PrimitiveType>,
+  extends BaseDataFetcher<Input, Output, RootOutput, Page, RootPage>
 {
   constructor(
     readonly config: ConfigType,
-    readonly client: BufferedRpc,
+    readonly multiplexer: WatchMultiplexer,
   ) {
     super(config.syncProtocol.name);
   }
 
   @bound
   async startAsync(point?: ChainPoint) {
-    await this.client.start(point);
+    await this.multiplexer.start(point);
   }
 
   @bound
   async resolveOrigin(): Promise<ChainPoint> {
-    return this.client.resolveOrigin();
+    return this.multiplexer.resolveOrigin();
   }
 
   @bound
@@ -40,63 +39,79 @@ export class UtxoRpcFetcher
       }`,
     );
     const outputs: OutputAndCleanup<Output>[] = [];
-    const blocks = this.client.fetchBlocks(data.from, data.to);
-    for (const block of blocks) {
-      outputs.push({
-        output: {
-          // TODO: What is the correct block hash?
-          blockHashes: [String(block.output.header?.hash)],
-          raw: block.output,
-          primitives: this.findPrimitives(block.output),
-        },
-        cleanup: block.cleanup,
-      });
+    const blocks = this.multiplexer.fetchBlocks(data.from, data.to);
+    let lastEntry: typeof blocks[number] | undefined;
+    for (const entry of blocks) {
+      lastEntry = entry;
+      const completed = entry.output;
+      const primitives = this.findPrimitivesFromWatchData(completed);
+      if (primitives.length > 0) {
+        outputs.push({
+          output: {
+            blockHashes: [completed.hash],
+            raw: completed.block,
+            primitives,
+          },
+          cleanup: entry.cleanup,
+        });
+      }
     }
+    const lastCompleted = lastEntry!.output;
     return {
       output: outputs,
       lastPage: {
         ownBlockNumber: Number(data.to),
         own: {
-          slot: Number(
-            outputs[outputs.length - 1].output.raw.header!.slot,
-          ),
-          height: Number(
-            outputs[outputs.length - 1].output.raw.header!.height,
-          ),
-          hash: Buffer.from(
-            outputs[outputs.length - 1].output.raw.header!.hash,
-          )
-            .toString("hex"),
+          slot: Number(lastCompleted.block.header!.slot),
+          height: Number(lastCompleted.block.header!.height),
+          hash: lastCompleted.hash,
         },
         root: rootConversion.toRootPage({
           blockHashes: [],
-          primitives: [], // TODO: I think this can be left empty here
-          raw: outputs[outputs.length - 1].output.raw,
+          primitives: [],
+          raw: lastCompleted.block,
         }),
       },
     };
   }
 
   @bound
-  findPrimitives(block: cardano.Block): PrimitiveType[] {
+  findPrimitivesFromWatchData(completed: CompletedBlock): PrimitiveType[] {
     const primitiveResponses: PrimitiveType[] = [];
-    for (const tx of block.body?.tx ?? []) {
+    const block = completed.block;
+    const txIndexCache = new Map<string, number>();
+
+    for (const primitiveEntry of this.config.primitives) {
+      const pKey = predicateKey(primitiveEntry.primitive.predicate);
+      const matchedTxs = completed.matchedTxsByPredicate.get(pKey)
+        ?? this.findFallbackTxs(completed, primitiveEntry.primitive.predicate);
+
       let logIndex = 0;
-      for (const primitveEntry of this.config.primitives) {
-        if (!matchesPredicate(tx, primitveEntry.primitive.predicate)) {
+      for (const tx of matchedTxs) {
+        if (!matchesPredicate(tx, primitiveEntry.primitive.predicate)) {
           continue;
         }
-        const transactionIndex = block.body?.tx.findIndex(t => hashEqual(t.hash, tx.hash));
+
+        const txHashHex = uint8ArrayToHexString(tx.hash);
+        let transactionIndex = txIndexCache.get(txHashHex);
+        if (transactionIndex === undefined) {
+          transactionIndex = block.body?.tx.findIndex(t =>
+            t.hash.length === tx.hash.length && t.hash.every((b, i) => b === tx.hash[i])
+          ) ?? -1;
+          txIndexCache.set(txHashHex, transactionIndex);
+        }
+
         primitiveResponses.push({
           syncProtocol: {
-            name: primitveEntry.syncProtocol,
+            name: primitiveEntry.syncProtocol,
             blockNumber: Number(block.header!.height),
-            transactionHash: uint8ArrayToHexString(tx.hash),
+            transactionHash: txHashHex,
             transactionIndex,
             contractAddress: '',
             logIndex,
+            absoluteSlot: Number(block.header!.slot),
           },
-          primitive: primitveEntry.primitive.name,
+          primitive: primitiveEntry.primitive.name,
           output: {
             payloadType: 'utxorpc-response',
             payload: { tx }
@@ -108,9 +123,30 @@ export class UtxoRpcFetcher
     return primitiveResponses;
   }
 
+  /**
+   * When a primitive's predicate key doesn't exactly match any watcher key
+   * (e.g., has_certificate primitives merged into match-all watcher),
+   * fall back to scanning all matched txs across all watchers.
+   */
+  private findFallbackTxs(
+    completed: CompletedBlock,
+    _predicate: unknown,
+  ): cardano.Tx[] {
+    const seen = new Set<string>();
+    const txs: cardano.Tx[] = [];
+    for (const [, watcherTxs] of completed.matchedTxsByPredicate) {
+      for (const tx of watcherTxs) {
+        const hash = uint8ArrayToHexString(tx.hash);
+        if (!seen.has(hash)) {
+          seen.add(hash);
+          txs.push(tx);
+        }
+      }
+    }
+    return txs;
+  }
+
   lastHeight(): bigint | undefined {
-    // Calling this.client.syncClient.getTip(); 
-    // returns the hash and slot, but not the block height
-    return this.client.lastHeight();
+    return this.multiplexer.lastHeight();
   }
 }

package/src/sync-protocols/utxorpc/types.ts

@@ -51,3 +51,11 @@ export type ConfigType = Extract<
   SyncProtocolWithNetwork,
   { syncProtocolType: ConfigSyncProtocolType.CARDANO_UTXORPC_PARALLEL }
 >;
+
+export type CompletedBlock = {
+  height: number;
+  slot: number;
+  hash: string;
+  block: cardano.Block;
+  matchedTxsByPredicate: Map<string, cardano.Tx[]>;
+};

package/src/sync-protocols/utxorpc/utils.ts

@@ -3,6 +3,8 @@ import type { UtxorpcAddressPattern, UtxorpcAnyChainTxPattern, UtxorpcAssetPatte
 import { hexStringToUint8Array } from "@effectstream/utils";
 import { Cardano } from '@cardano-sdk/core';
 import { HexBlob } from '@cardano-sdk/util';
+import type { CardanoTxPredicate as TxPredicate } from "@utxorpc/sdk";
+import stableStringify from "json-stable-stringify";
 
 export function hashEqual(lhs: Uint8Array, rhs: Uint8Array): boolean {
   if (lhs.length !== rhs.length) {
@@ -38,9 +40,10 @@ function matchesPattern(tx: cardano.Tx, pattern: UtxorpcAnyChainTxPattern): bool
   const outputs = tx.outputs;
   const inputs = tx.inputs.map(input => input.asOutput!).filter(x => x);
   const matchHasAddress = !cardanoPattern.has_address || matchesAddress(outputs, cardanoPattern.has_address) || matchesAddress(inputs, cardanoPattern.has_address);
-  const matchMovesAsset = !cardanoPattern.moves_asset || matchesAsset(inputs, cardanoPattern.moves_asset);
+  const matchMovesAsset = !cardanoPattern.moves_asset || matchesAsset(outputs, cardanoPattern.moves_asset) || matchesAsset(inputs, cardanoPattern.moves_asset);
   const matchMintsAsset = !cardanoPattern.mints_asset || matchesAsset(outputs, cardanoPattern.mints_asset);
-  return matchHasAddress && matchMovesAsset && matchMintsAsset;
+  const matchHasCertificate = !cardanoPattern.has_certificate || tx.certificates.length > 0;
+  return matchHasAddress && matchMovesAsset && matchMintsAsset && matchHasCertificate;
 }
 
 function matchesOutputPattern(outputs: cardano.TxOutput[], pattern: UtxorpcTxOutputPattern): boolean {
@@ -77,4 +80,81 @@ function matchesAsset(outputs: cardano.TxOutput[], pattern: UtxorpcAssetPattern)
     }
     return ma.assets.some(a => !assetName || hashEqual(a.name, assetName));
   }));
+}
+
+// =============================================
+// Config → SDK predicate conversion
+// =============================================
+
+function convertAddressPattern(pattern: UtxorpcAddressPattern): Record<string, Uint8Array> {
+  const result: Record<string, Uint8Array> = {};
+  if (pattern.exact_address) {
+    result.exactAddress = Uint8Array.from(atob(pattern.exact_address), c => c.charCodeAt(0));
+  }
+  if (pattern.payment_part) {
+    result.paymentPart = hexStringToUint8Array(pattern.payment_part);
+  }
+  if (pattern.delegation_part) {
+    result.delegationPart = hexStringToUint8Array(pattern.delegation_part);
+  }
+  return result;
+}
+
+function convertAssetPattern(pattern: UtxorpcAssetPattern): Record<string, Uint8Array> {
+  const result: Record<string, Uint8Array> = {};
+  if (pattern.policy_id) {
+    result.policyId = hexStringToUint8Array(pattern.policy_id);
+  }
+  if (pattern.asset_name) {
+    result.assetName = hexStringToUint8Array(pattern.asset_name);
+  }
+  return result;
+}
+
+/**
+ * Convert a config-layer UtxorpcTxPredicate (snake_case) to the SDK's TxPredicate (camelCase).
+ *
+ * NOTE: `has_certificate` is in the UTxORPC spec but Dolos does NOT implement server-side
+ * filtering for it. Predicates using has_certificate will be sent to the server but silently
+ * ignored — client-side filtering in matchesPredicate() still catches them.
+ */
+// TODO(dolos): when Dolos implements has_certificate filtering, this conversion will just work
+export function configPredicateToSdkPredicate(config: UtxorpcTxPredicate): TxPredicate {
+  const result: TxPredicate = {};
+
+  if (config.match?.cardano) {
+    const c = config.match.cardano;
+    const match: Record<string, unknown> = {};
+    if (c.has_address) match.hasAddress = convertAddressPattern(c.has_address);
+    if (c.moves_asset) match.movesAsset = convertAssetPattern(c.moves_asset);
+    if (c.mints_asset) match.mintsAsset = convertAssetPattern(c.mints_asset);
+    if (c.has_certificate) match.hasCertificate = {};
+    result.match = match as TxPredicate["match"];
+  }
+
+  if (config.not) result.not = config.not.map(configPredicateToSdkPredicate);
+  if (config.all_of) result.allOf = config.all_of.map(configPredicateToSdkPredicate);
+  if (config.any_of) result.anyOf = config.any_of.map(configPredicateToSdkPredicate);
+
+  return result;
+}
+
+/**
+ * Deterministic string key for a predicate, used to deduplicate watchers.
+ * Predicates that produce the same key share a single WatchClient stream.
+ */
+export function predicateKey(predicate: UtxorpcTxPredicate): string {
+  return stableStringify(predicate) ?? "{}";
+}
+
+/**
+ * Returns true if this predicate can be effectively filtered server-side by Dolos.
+ * Currently Dolos Watch ignores `has_certificate`, so predicates relying solely on it
+ * are not effective and should fall back to match-all.
+ */
+// TODO(dolos): remove this when Dolos implements has_certificate filtering
+export function isEffectiveServerPredicate(predicate: UtxorpcTxPredicate): boolean {
+  if (!predicate.match?.cardano) return false;
+  const c = predicate.match.cardano;
+  return !!(c.has_address || c.moves_asset || c.mints_asset);
 }

package/src/syncProtocolFactory.ts

@@ -11,9 +11,12 @@ import {
   getViemNetwork,
 } from "@effectstream/config";
 import { CardanoSyncClient } from "@utxorpc/sdk";
-import { BufferedRpc } from "./sync-protocols/utxorpc/BufferedRpc.ts";
+import { WatchMultiplexer } from "./sync-protocols/utxorpc/WatchMultiplexer.ts";
 import { UtxoRpcFetcher } from "./sync-protocols/utxorpc/fetcher.ts";
 import { UtxoRpcSyncState } from "./sync-protocols/utxorpc/state.ts";
+import { isEffectiveServerPredicate, predicateKey } from "./sync-protocols/utxorpc/utils.ts";
+import type { UtxorpcTxPredicate } from "@effectstream/config";
+import type { PrimitiveEntryType } from "./sync-protocols/utxorpc/types.ts";
 import { MidnightFetcher, MidnightSyncState } from "@effectstream/sync";
 import { AvailFetcher } from "./sync-protocols/avail/fetcher.ts";
 import { AvailSyncState } from "./sync-protocols/avail/state.ts";
@@ -67,17 +70,23 @@ export function* genSyncProtocols(
       ) {
         throw new Error("CARP not supported yet");
       }
-      const syncClient = new CardanoSyncClient({
+      const clientOptions = {
         uri: entry.syncProtocol.rpcUrl,
         headers: entry.syncProtocol.headers,
-      });
-      const bufferedRpc = new BufferedRpc(
-        syncClient,
+      };
+      const syncClient = new CardanoSyncClient(clientOptions);
+      const primitivesByPredicate = groupPrimitivesByPredicate(
+        entry.primitives as PrimitiveEntryType[],
+      );
+      const multiplexer = new WatchMultiplexer(
+        clientOptions,
+        primitivesByPredicate,
         entry.syncProtocol.confirmationDepth,
+        syncClient,
       );
       const fetcher = new UtxoRpcFetcher(
         entry,
-        bufferedRpc,
+        multiplexer,
       );
       const state = yield* UtxoRpcSyncState.restoreState(
         dbConn,
@@ -147,3 +156,27 @@ export function* genSyncProtocols(
 
   return result;
 }
+
+function groupPrimitivesByPredicate(
+  primitives: PrimitiveEntryType[],
+): Map<string, { predicate: UtxorpcTxPredicate; primitiveEntries: PrimitiveEntryType[] }> {
+  const groups = new Map<string, { predicate: UtxorpcTxPredicate; primitiveEntries: PrimitiveEntryType[] }>();
+  const emptyKey = predicateKey({});
+
+  for (const entry of primitives) {
+    const predicate = entry.primitive.predicate;
+    // TODO(dolos): when Dolos implements has_certificate, remove this fallback to empty predicate
+    const effective = isEffectiveServerPredicate(predicate);
+    const key = effective ? predicateKey(predicate) : emptyKey;
+    const effectivePredicate = effective ? predicate : {};
+
+    let group = groups.get(key);
+    if (!group) {
+      group = { predicate: effectivePredicate, primitiveEntries: [] };
+      groups.set(key, group);
+    }
+    group.primitiveEntries.push(entry);
+  }
+
+  return groups;
+}