@effectstream/sync 0.100.21 → 0.100.23

package/CLAUDE.md

@@ -171,6 +171,15 @@ Consequences during catch-up:
 `bufferedData` and pauses the fetch loop (each chain's `stateToInput` returns `undefined`)
 until the merge drains. See README "Backpressure (`maxBufferedPages`)".
 
+> **Merge→apply queue cap (Fix C′) — ✅ SHIPPED.** The Deque cap above bounds *fetch*, but the
+> merge then sends finalized blocks into an effection channel whose subscriber queue buffers
+> unbounded — so the merge (no I/O) could still race ahead of the serial apply loop and grow that
+> queue toward the whole backlog, especially with empty-block coalescing (Fix E) removing the
+> per-block apply brake. The runtime now gates the merge's `send` (`runtime/src/main.ts`): it blocks
+> once `EFFECTSTREAM_FINALIZED_STREAM_CAP` (default 2048) produced blocks are unconsumed, and the
+> apply loop wakes it as it drains. `inFlight = produced − consumed` is surfaced on
+> `/debug/metrics`. Repro/guard: `runtime/test/reproduction/coalesce-memory.test.ts`.
+
 > **Merge-demand exemption (safeguard).** A naive cap can deadlock the merge: it drains a
 > parallel chain's buffer only *after* its page passes the root timestamp `τ`, so if the cap
 > pauses the fetcher while its page is still `≤ τ` (a far skip-ahead, or a chain finer-grained

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@effectstream/sync",
-  "version": "0.100.21",
+  "version": "0.100.23",
   "license": "MIT",
   "homepage": "https://effectstream.github.io/docs/",
   "repository": {
@@ -13,10 +13,10 @@
   },
   "exports": "./src/mod.ts",
   "dependencies": {
-    "@effectstream/config": "0.100.21",
-    "@effectstream/db": "0.100.21",
-    "@effectstream/log": "0.100.21",
-    "@effectstream/utils": "0.100.21",
+    "@effectstream/config": "0.100.23",
+    "@effectstream/db": "0.100.23",
+    "@effectstream/log": "0.100.23",
+    "@effectstream/utils": "0.100.23",
     "@midnight-ntwrk/ledger-v8": "8.0.3",
     "@midnight-ntwrk/midnight-js-indexer-public-data-provider": "4.0.4",
     "@midnight-ntwrk/onchain-runtime": "npm:@midnight-ntwrk/onchain-runtime-v3@3.0.0",

package/src/sync-protocols/midnight/MidnightClient.ts

@@ -32,6 +32,22 @@ export interface MidnightGqlBlockState {
         outputIndex: number;
         owner:       string;
       }[];
+      unshieldedCreatedOutputs?: {
+        intentHash:  string;
+        outputIndex: number;
+        owner:       string;
+      }[];
+      // Present only on RegularTransaction (selected via inline fragment): the
+      // coin-commitment Merkle tree root after this transaction.
+      zswapMerkleTreeRoot?: string;
+      // Selected only for MidnightTokenMintPrimitive: the serialized
+      // transaction bytes (Transaction interface) and the apply result
+      // (RegularTransaction only — absent on system transactions).
+      raw?: string;
+      transactionResult?: {
+        status: "SUCCESS" | "PARTIAL_SUCCESS" | "FAILURE";
+        segments?: { id: number; success: boolean }[] | null;
+      };
     }[];
   };
 };
@@ -43,6 +59,12 @@ export interface BlockFetchOptions {
   zswapLedgerEvents?: boolean;
   /** Include unshieldedSpentOutputs (needed for MidnightUnshieldedSpendPrimitive). Default: false */
   unshieldedSpentOutputs?: boolean;
+  /** Include unshieldedCreatedOutputs (needed for MidnightUnshieldedCreatePrimitive). Default: false */
+  unshieldedCreatedOutputs?: boolean;
+  /** Include `... on RegularTransaction { zswapMerkleTreeRoot }` (needed for MidnightZswapRootPrimitive). Default: false */
+  zswapRoots?: boolean;
+  /** Include tx `raw` + `... on RegularTransaction { transactionResult }` (needed for MidnightTokenMintPrimitive). Default: false */
+  tokenMints?: boolean;
 }
 
 type PublicDataProvider = ReturnType<typeof indexerPublicDataProvider>;
@@ -168,6 +190,9 @@ export class MidnightClient {
       contractActions = true,
       zswapLedgerEvents = true,
       unshieldedSpentOutputs = false,
+      unshieldedCreatedOutputs = false,
+      zswapRoots = false,
+      tokenMints = false,
     } = options;
     const contractActionsField = contractActions
       ? `contractActions { address state }`
@@ -178,6 +203,22 @@ export class MidnightClient {
     const unshieldedSpentField = unshieldedSpentOutputs
       ? `unshieldedSpentOutputs { intentHash outputIndex owner }`
       : "";
+    const unshieldedCreatedField = unshieldedCreatedOutputs
+      ? `unshieldedCreatedOutputs { intentHash outputIndex owner }`
+      : "";
+    // zswapMerkleTreeRoot lives on RegularTransaction, not the Transaction
+    // interface, so it must be selected through an inline fragment.
+    const zswapRootFragment = zswapRoots
+      ? `... on RegularTransaction { zswapMerkleTreeRoot }`
+      : "";
+    // `raw` is on the Transaction interface; `transactionResult` is
+    // RegularTransaction-only (system transactions have neither result nor
+    // contract calls, so they are skipped downstream). Duplicate inline
+    // fragments merge legally in GraphQL.
+    const tokenMintFields = tokenMints
+      ? `raw
+         ... on RegularTransaction { transactionResult { status segments { id success } } }`
+      : "";
     const query = `query {
       block(offset: { height: ${blockHeight} }) {
         hash
@@ -192,6 +233,9 @@ export class MidnightClient {
           ${contractActionsField}
           ${zswapField}
           ${unshieldedSpentField}
+          ${unshieldedCreatedField}
+          ${zswapRootFragment}
+          ${tokenMintFields}
         }
       }
     }`;

package/src/sync-protocols/midnight/fetcher.ts

@@ -19,6 +19,7 @@ import { bound } from "@effectstream/utils";
 import { MidnightClient, type BlockFetchOptions, type MidnightGqlBlockState } from "./MidnightClient.ts";
 import { ContractState, StateValue } from "@midnight-ntwrk/onchain-runtime";
 import { decodeZswapInputEvent } from "./zswap-decoder.ts";
+import { decodeTokenMints } from "./mint-decoder.ts";
 
 export class MidnightFetcher extends BaseDataFetcher<
   Input,
@@ -84,7 +85,10 @@ export class MidnightFetcher extends BaseDataFetcher<
       contractActions: this.config.primitives.some(
         (p) =>
           p.primitive.type !== "Midnight:Nullifier" &&
-          p.primitive.type !== "Midnight:UnshieldedSpend",
+          p.primitive.type !== "Midnight:UnshieldedSpend" &&
+          p.primitive.type !== "Midnight:UnshieldedCreate" &&
+          p.primitive.type !== "Midnight:ZswapRoot" &&
+          p.primitive.type !== "Midnight:TokenMint",
       ),
       zswapLedgerEvents: this.config.primitives.some(
         (p) => p.primitive.type === "Midnight:Nullifier",
@@ -92,6 +96,15 @@ export class MidnightFetcher extends BaseDataFetcher<
       unshieldedSpentOutputs: this.config.primitives.some(
         (p) => p.primitive.type === "Midnight:UnshieldedSpend",
       ),
+      unshieldedCreatedOutputs: this.config.primitives.some(
+        (p) => p.primitive.type === "Midnight:UnshieldedCreate",
+      ),
+      zswapRoots: this.config.primitives.some(
+        (p) => p.primitive.type === "Midnight:ZswapRoot",
+      ),
+      tokenMints: this.config.primitives.some(
+        (p) => p.primitive.type === "Midnight:TokenMint",
+      ),
     };
     const self = this;
     const heights = Array.from(
@@ -105,14 +118,28 @@ export class MidnightFetcher extends BaseDataFetcher<
     const fetched = yield* all(
       heights.map(function* (height) {
         const signal = yield* useAbortSignal();
-        const result: MidnightGqlBlockState = yield* call(() =>
-          self.client.fetchBlock(height, blockFetchOptions, signal)
-        );
-        const primitives = yield* self.readPrimitives(
-          height,
-          result,
-          self.config.primitives,
-        );
+        let result: MidnightGqlBlockState;
+        let primitives;
+        try {
+          result = yield* call(() =>
+            self.client.fetchBlock(height, blockFetchOptions, signal)
+          );
+          primitives = yield* self.readPrimitives(
+            height,
+            result,
+            self.config.primitives,
+          );
+        } catch (e) {
+          // Surface per-height failures: the outer fetch loop swallows errors
+          // and retries silently, which previously hid a permanent failure
+          // (e.g. a wasm-instance mismatch in a contract ledger() call) as an
+          // ever-stuck page with no log output.
+          console.error(
+            `[Midnight] readData failed at height ${height}:`,
+            (e as Error)?.stack ?? e,
+          );
+          throw e;
+        }
         return {
           output: {
             raw: result.block as unknown as Block,
@@ -166,6 +193,12 @@ export class MidnightFetcher extends BaseDataFetcher<
         syncResults.push(...this.fetchNullifiers(height, primitiveEntry, block));
       } else if (primitiveEntry.primitive.type === "Midnight:UnshieldedSpend") {
         syncResults.push(...this.fetchUnshieldedSpends(height, primitiveEntry, block));
+      } else if (primitiveEntry.primitive.type === "Midnight:UnshieldedCreate") {
+        syncResults.push(...this.fetchUnshieldedCreates(height, primitiveEntry, block));
+      } else if (primitiveEntry.primitive.type === "Midnight:ZswapRoot") {
+        syncResults.push(...this.fetchZswapRoots(height, primitiveEntry, block));
+      } else if (primitiveEntry.primitive.type === "Midnight:TokenMint") {
+        syncResults.push(...this.fetchTokenMints(height, primitiveEntry, block));
       } else {
         asyncOps.push(
           this.fetchContractState(height, client, primitiveEntry, block),
@@ -243,6 +276,112 @@ export class MidnightFetcher extends BaseDataFetcher<
     return results;
   }
 
+  // Mirror of fetchUnshieldedSpends over `unshieldedCreatedOutputs` — every
+  // unshielded UTXO created in the block (regular AND system transactions).
+  // One state-machine input per created output.
+  @bound
+  fetchUnshieldedCreates(
+    height: number,
+    primitiveEntry: PrimitiveEntryType,
+    block: MidnightGqlBlockState,
+  ): PrimitiveType[] {
+    const results: PrimitiveType[] = [];
+    for (const tx of block.block.transactions) {
+      for (const created of tx.unshieldedCreatedOutputs ?? []) {
+        results.push({
+          syncProtocol: {
+            name: primitiveEntry.syncProtocol,
+            blockNumber: height,
+            transactionHash: tx.hash,
+            contractAddress: "",
+          },
+          primitive: primitiveEntry.primitive.name,
+          output: {
+            payloadType: "midnight-unshielded-create",
+            payload: {
+              owner: created.owner,
+              intentHash: created.intentHash,
+              outputIndex: created.outputIndex,
+              txHash: tx.hash,
+            },
+          },
+        });
+      }
+    }
+    return results;
+  }
+
+  // Emit the latest zswap coin-commitment Merkle tree root for the block: the
+  // `zswapMerkleTreeRoot` of the last RegularTransaction that carries one.
+  // Blocks with no regular transactions emit nothing (the root is unchanged).
+  @bound
+  fetchZswapRoots(
+    height: number,
+    primitiveEntry: PrimitiveEntryType,
+    block: MidnightGqlBlockState,
+  ): PrimitiveType[] {
+    let root: string | undefined;
+    let txHash = "";
+    for (const tx of block.block.transactions) {
+      if (tx.zswapMerkleTreeRoot) {
+        root = tx.zswapMerkleTreeRoot;
+        txHash = tx.hash;
+      }
+    }
+    if (!root) return [];
+    return [
+      {
+        syncProtocol: {
+          name: primitiveEntry.syncProtocol,
+          blockNumber: height,
+          transactionHash: txHash,
+          contractAddress: "",
+        },
+        primitive: primitiveEntry.primitive.name,
+        output: {
+          payloadType: "midnight-zswap-root",
+          payload: { root, txHash },
+        },
+      },
+    ];
+  }
+
+  // Decode custom token mints out of each regular transaction's raw bytes
+  // (ledger-v8 deserialize → contract call transcript effects). One input per
+  // (tx, call, domainSep, kind) that actually applied on chain — see
+  // mint-decoder.ts for the segment semantics. System transactions carry no
+  // transactionResult and are skipped.
+  @bound
+  fetchTokenMints(
+    height: number,
+    primitiveEntry: PrimitiveEntryType,
+    block: MidnightGqlBlockState,
+  ): PrimitiveType[] {
+    const results: PrimitiveType[] = [];
+    for (const tx of block.block.transactions) {
+      if (!tx.raw || !tx.transactionResult) continue;
+      for (const mint of decodeTokenMints(tx.raw, tx.transactionResult)) {
+        results.push({
+          syncProtocol: {
+            name: primitiveEntry.syncProtocol,
+            blockNumber: height,
+            transactionHash: tx.hash,
+            contractAddress: mint.contractAddress,
+          },
+          primitive: primitiveEntry.primitive.name,
+          output: {
+            payloadType: "midnight-token-mint",
+            payload: {
+              ...mint,
+              txHash: tx.hash,
+            },
+          },
+        });
+      }
+    }
+    return results;
+  }
+
   @bound
   *fetchContractState(
     height: number,

package/src/sync-protocols/midnight/mint-decoder.ts

@@ -0,0 +1,151 @@
+import { Buffer } from "node:buffer";
+import * as ledger from "@midnight-ntwrk/ledger-v8";
+
+// Decodes custom token mints out of a raw Midnight transaction. A contract
+// call's transcript effects record mints as `domain_sep → amount` maps
+// (`shieldedMints` / `unshieldedMints`); the resulting token type ("color")
+// is `rawTokenType(domain_sep, contract_address)`. Neither the indexer's
+// GraphQL nor the wallet expose this mapping, so the only route is
+// deserializing the transaction bytes with ledger-v8 — same approach as
+// zswap-decoder.ts for nullifier events.
+//
+// Mirrors zswap-decoder.ts tolerance: never throws — an undecodable
+// transaction logs a warning and contributes nothing.
+
+export interface TokenMintRecord {
+  /** Address of the minting contract, hex */
+  contractAddress: string;
+  /** 32-byte domain separator pre-image, hex */
+  domainSep: string;
+  /** Derived token type ("color") = rawTokenType(domainSep, contractAddress), hex */
+  rawTokenType: string;
+  kind: "shielded" | "unshielded";
+  /** Total minted for this (call, domainSep, kind), as a decimal string (u64 exceeds MAX_SAFE_INTEGER) */
+  amount: string;
+  /** Contract entry point that minted, when printable */
+  entryPoint?: string;
+}
+
+export interface TxResultLike {
+  status: "SUCCESS" | "PARTIAL_SUCCESS" | "FAILURE";
+  segments?: { id: number; success: boolean }[] | null;
+}
+
+function hexToBytes(hex: string): Uint8Array {
+  const clean = hex.startsWith("0x") ? hex.slice(2) : hex;
+  return Uint8Array.from(Buffer.from(clean, "hex"));
+}
+
+function bytesToHex(bytes: Uint8Array): string {
+  return Array.from(bytes).map((b) => b.toString(16).padStart(2, "0")).join("");
+}
+
+function normalizeHex(value: string): string {
+  return value.replace(/^0x/, "").toLowerCase();
+}
+
+/**
+ * Extract every applied token mint from a raw serialized transaction.
+ *
+ * Segment semantics (failed segments roll back on chain):
+ * - no `result` (system transactions) or status FAILURE → nothing applied;
+ * - SUCCESS → guaranteed + all fallible transcripts applied;
+ * - PARTIAL_SUCCESS → guaranteed transcripts applied, fallible only where
+ *   the intent's segment (the `intents` map key) succeeded.
+ */
+export function decodeTokenMints(
+  rawHex: string,
+  result?: TxResultLike | null,
+): TokenMintRecord[] {
+  if (!result || result.status === "FAILURE") return [];
+
+  let tx: any;
+  try {
+    tx = ledger.Transaction.deserialize(
+      "signature",
+      "proof",
+      "binding",
+      hexToBytes(rawHex),
+    );
+  } catch (e) {
+    console.warn(
+      `[mint-decoder] undecodable transaction, skipping: ${
+        (e as Error)?.message ?? e
+      }`,
+    );
+    return [];
+  }
+
+  const records: TokenMintRecord[] = [];
+  for (const [segmentId, intent] of tx.intents ?? new Map()) {
+    const fallibleApplied = result.status === "SUCCESS" ||
+      (result.segments?.some((s) => s.id === Number(segmentId) && s.success) ??
+        false);
+    for (const action of intent.actions ?? []) {
+      // Only contract calls carry transcripts; deploys and maintenance
+      // updates cannot mint.
+      if (!(action instanceof ledger.ContractCall)) continue;
+      try {
+        records.push(...mintsOfCall(action, fallibleApplied));
+      } catch (e) {
+        console.warn(
+          `[mint-decoder] failed to read mints of call to ${
+            (action as any)?.address ?? "?"
+          }: ${(e as Error)?.message ?? e}`,
+        );
+      }
+    }
+  }
+  return records;
+}
+
+function mintsOfCall(
+  call: any,
+  fallibleApplied: boolean,
+): TokenMintRecord[] {
+  // Accumulate domainSep|kind → amount across the applied transcripts. The
+  // guaranteed transcript applied whenever the transaction wasn't a FAILURE
+  // (the caller filters those); the fallible transcript only if its segment
+  // succeeded.
+  const totals = new Map<string, bigint>();
+  const transcripts = [
+    call.guaranteedTranscript,
+    ...(fallibleApplied ? [call.fallibleTranscript] : []),
+  ];
+  for (const transcript of transcripts) {
+    const effects = transcript?.effects;
+    if (!effects) continue;
+    for (
+      const [kind, mints] of [
+        ["shielded", effects.shieldedMints],
+        ["unshielded", effects.unshieldedMints],
+      ] as const
+    ) {
+      for (const [sepHex, amount] of mints ?? new Map()) {
+        const key = `${normalizeHex(String(sepHex))}|${kind}`;
+        totals.set(key, (totals.get(key) ?? 0n) + BigInt(amount));
+      }
+    }
+  }
+
+  const records: TokenMintRecord[] = [];
+  for (const [key, amount] of totals) {
+    const [domainSep, kind] = key.split("|") as [
+      string,
+      "shielded" | "unshielded",
+    ];
+    records.push({
+      contractAddress: normalizeHex(String(call.address)),
+      domainSep,
+      rawTokenType: normalizeHex(
+        String(ledger.rawTokenType(hexToBytes(domainSep), call.address)),
+      ),
+      kind,
+      amount: amount.toString(),
+      entryPoint: typeof call.entryPoint === "string"
+        ? call.entryPoint
+        : bytesToHex(call.entryPoint),
+    });
+  }
+  return records;
+}

package/test/mint-decoder.test.ts

@@ -0,0 +1,28 @@
+import { describe, expect, test } from "bun:test";
+
+import { decodeTokenMints } from "../src/sync-protocols/midnight/mint-decoder.ts";
+
+// Tolerance contract: the decoder must never throw — an undecodable or
+// inapplicable transaction contributes nothing (the fetcher's per-height
+// error path would otherwise wedge the sync loop).
+describe("decodeTokenMints — tolerance", () => {
+  test("no transactionResult (system tx) → []", () => {
+    expect(decodeTokenMints("deadbeef", undefined)).toEqual([]);
+    expect(decodeTokenMints("deadbeef", null)).toEqual([]);
+  });
+
+  test("status FAILURE → [] (everything rolled back, incl. guaranteed)", () => {
+    expect(
+      decodeTokenMints("deadbeef", { status: "FAILURE", segments: [] }),
+    ).toEqual([]);
+  });
+
+  test("garbage bytes with SUCCESS status → [] without throwing", () => {
+    expect(
+      decodeTokenMints("not-even-hex", { status: "SUCCESS" }),
+    ).toEqual([]);
+    expect(
+      decodeTokenMints("00".repeat(64), { status: "SUCCESS" }),
+    ).toEqual([]);
+  });
+});