@effectstream/sync 0.100.19 → 0.100.21

package/CLAUDE.md

@@ -167,7 +167,18 @@ Consequences during catch-up:
   (`mergeIntoRoot`, `merge.ts`), so a slow/stalled chain stalls all block production while
   the others keep ballooning (head-of-line blocking).
 
-→ Fix: **(C) backpressure** — cap `bufferedData` and pause the fetch loop until the merge drains.
+→ Fix: **(C) backpressure** — ✅ SHIPPED. `bufferAtCap` (`common/page-helpers.ts`) caps
+`bufferedData` and pauses the fetch loop (each chain's `stateToInput` returns `undefined`)
+until the merge drains. See README "Backpressure (`maxBufferedPages`)".
+
+> **Merge-demand exemption (safeguard).** A naive cap can deadlock the merge: it drains a
+> parallel chain's buffer only *after* its page passes the root timestamp `τ`, so if the cap
+> pauses the fetcher while its page is still `≤ τ` (a far skip-ahead, or a chain finer-grained
+> than the cap), merge waits for the page ↔ the page waits for a fetch ↔ the fetch waits for
+> the buffer to drain — circular. Guard: `mergeIntoRoot` sets `SyncState.mergeWaitingForPage`
+> while blocked on a chain's page, and `bufferAtCap` returns `false` for that chain while the
+> flag is set — so the fetcher advances past `τ`. Bounded by necessity: only
+> `(lastPage.root, τ]` is buffered above the cap. Repros: `buffering.test.ts` 1c/1d.
 
 ### 2. Silent data gap on restart (correctness) — ✅ FIXED (D)
 
@@ -202,5 +213,7 @@ block-hash chain.
 > (with `test` as the worked example).
 >
 > Deterministic reproductions of #1/#2/#3 and their fixes land alongside each fix:
-> **#2 → block-accurate resume is DONE** (`consistency.test.ts`). Still in follow-up
-> PRs: #1 → fetch backpressure; #3 → opt-in empty-block commit batching during catch-up.
+> **#2 → block-accurate resume is DONE** (`consistency.test.ts`); **#1 → fetch
+> backpressure is DONE** (`buffering.test.ts`, incl. 1c/1d for the merge-demand
+> exemption). Still in follow-up PRs: #3 → opt-in empty-block commit batching during
+> catch-up.

package/README.md

@@ -70,10 +70,21 @@ the merge. ~4 chunks of look-ahead never starves the merge while keeping memory
 bounded; raise it to trade memory for more fetch look-ahead. The cap only bites during
 catch-up — in steady state the buffer sits near zero and the cap is never reached.
 
-**Deadlock-safety.** Pausing a full chain cannot deadlock: a chain's page only
-advances via a fetch (`updateState`), and the merge drains a chain's buffered data
-*before* it waits on that chain's page — so the paused (full) chain is never the one
-the merge is blocked on without a path to resume.
+**Deadlock-safety (merge-demand exemption).** A naive cap can deadlock the merge: it
+produces a root block at timestamp `τ` only once a parallel chain's page passes `τ`, and
+drains that chain's buffer only *after* the page passes `τ`. If the cap pauses the
+fetcher while its page is still `≤ τ` (the data up to `τ` exceeds the cap — e.g. a far
+skip-ahead or a parallel chain finer-grained than the cap), neither side can proceed: a
+circular wait, not a lost wakeup.
+
+The safeguard is a **merge-demand exemption**: while the merge is blocked on a chain's
+page (`mergeIntoRoot` sets `SyncState.mergeWaitingForPage`), `bufferAtCap` returns
+`false` for that chain so the fetcher advances until its page crosses `τ`. Memory stays
+**bounded by necessity** — only `(lastPage.root, τ]` is buffered above the cap, exactly
+what the merge must hold to build block `τ`; once the page passes `τ` the normal cap
+re-engages. A chain that is merely tip-limited (caught up, not gated by the merge) is
+*not* exempted, so steady-state and head-of-line bounds are unaffected. Reproductions:
+`buffering.test.ts` 1c (skip-ahead) and 1d (density).
 
 **Scope.** The guard runs in **every** chain's `stateToInput`, so all sync chains are
 covered: EVM, NTP, Bitcoin, Avail, Celestia, NEAR, Midnight, Cardano (UTXO-RPC), and
@@ -89,8 +100,10 @@ the synthetic `test` chain. Two notes:
 **Observability.** Each `SyncState` tracks, and the runtime's `/debug/metrics`
 endpoint reports per protocol: `cap` (resolved `maxBufferedPages`), `buf` (current
 size), `bufHighWater` (peak since boot — catches spikes between samples), `pausedNow`,
-`pauses` (rising-edge count — **the "backpressure engaged" signal**), and `pausedMs`
-(total time paused). `pauses > 0` means the cap actively bounded memory; `0` means it
+`pauses` (rising-edge count — **the "backpressure engaged" signal**), `pausedMs`
+(total time paused), and `mergeWaiting`/`mergeDemandRoot` (merge-demand exemption:
+`mergeWaiting=true` means the merge is gated on this chain's page and the cap is lifted).
+`pauses > 0` means the cap actively bounded memory; `0` means it
 was never needed in that run. Steady-state these sit at `0`; during a real deep
 catch-up (or under the perf harness's `PERF_APPLY_DELAY_MS` drain throttle) they climb
 as `buf` pins to `cap`.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@effectstream/sync",
-  "version": "0.100.19",
+  "version": "0.100.21",
   "license": "MIT",
   "homepage": "https://effectstream.github.io/docs/",
   "repository": {
@@ -13,12 +13,12 @@
   },
   "exports": "./src/mod.ts",
   "dependencies": {
-    "@effectstream/config": "0.100.19",
-    "@effectstream/db": "0.100.19",
-    "@effectstream/log": "0.100.19",
-    "@effectstream/utils": "0.100.19",
+    "@effectstream/config": "0.100.21",
+    "@effectstream/db": "0.100.21",
+    "@effectstream/log": "0.100.21",
+    "@effectstream/utils": "0.100.21",
     "@midnight-ntwrk/ledger-v8": "8.0.3",
-    "@midnight-ntwrk/midnight-js-indexer-public-data-provider": "4.0.2",
+    "@midnight-ntwrk/midnight-js-indexer-public-data-provider": "4.0.4",
     "@midnight-ntwrk/onchain-runtime": "npm:@midnight-ntwrk/onchain-runtime-v3@3.0.0",
     "effection": "3.5.0",
     "graphql-ws": "^6.0.6",

package/src/sync-protocols/base/state.ts

@@ -88,6 +88,18 @@ export abstract class SyncState<
   /** Start of the current pause, for accumulating `backpressurePausedMs`. */
   private pauseStartMs: number = 0;
 
+  // ── Merge-demand exemption (see common/page-helpers.ts + README) ──
+  /**
+   * True while the merge is blocked waiting for THIS chain's page to pass the
+   * current root timestamp. The cap must not pause the fetcher here: the merge
+   * drains this chain's buffer only after its page passes the target, so pausing
+   * would be a circular wait. While exempt the fetcher buffers only the data the
+   * merge must hold to build that block anyway (bounded by necessity).
+   */
+  public mergeWaitingForPage: boolean = false;
+  /** Root page (ms timestamp) the merge is waiting to exceed — observability only. */
+  public mergeDemandRoot: RootPage | undefined = undefined;
+
   constructor(
     public readonly name: string,
     lastPage: undefined | LastPage<Page, RootPage>,

package/src/sync-protocols/common/page-helpers.ts

@@ -33,11 +33,15 @@ export function bufferAtCap(
   state: {
     bufferedData: { size(): number };
     recordBackpressure(atCap: boolean, cap: number): void;
+    mergeWaitingForPage: boolean;
   },
   syncProtocol: { maxBufferedPages?: number; stepSize?: number },
 ): boolean {
   const cap = bufferCapFor(syncProtocol);
-  const atCap = state.bufferedData.size() >= cap;
+  // Exempt this chain from the cap while the merge is blocked on its page (pausing
+  // there would deadlock — see SyncState.mergeWaitingForPage). Reporting
+  // atCap=false while exempt is correct: the chain is genuinely not paused.
+  const atCap = !state.mergeWaitingForPage && state.bufferedData.size() >= cap;
   state.recordBackpressure(atCap, cap);
   return atCap;
 }

package/src/sync-protocols/midnight/fetcher.ts

@@ -275,17 +275,21 @@ export class MidnightFetcher extends BaseDataFetcher<
         // Addresses length can be padded by 0's
         return c.address.padStart(longest, '0') === contractAddress.padStart(longest, '0');
       })!.state!;
-      const byteState = new Uint8Array(rawState.match(/.{1,2}/g)!.map(byte => parseInt(byte, 16)));
       const primitive = primitiveEntry.primitive;
-      const contractState = ContractState.deserialize(byteState);
-      const stateValue = contractState.data.state;
-      // Parse additional fields from the schema (defined per-contract on the primitive)
-      const additionalFields = primitive.parseAdditionalLedgerFields?.(stateValue) ?? {};
-      // Keep generated ledger fields, but let schema-parsed fields override.
-      // This is important for map-like fields where contract.ledger() exposes
-      // iterable wrappers that are not directly JSON-serializable.
       const f = primitive.contract ?? { ledger: (_: StateValue): Record<string, any> => ({}) };
-      const state = { ...f.ledger(stateValue), ...additionalFields };
+      const ledgerFromTxStateHex = (f as {
+        ledgerFromTxStateHex?: (rawHexState: string) => Record<string, any>;
+      }).ledgerFromTxStateHex;
+      let state: Record<string, any>;
+      if (typeof ledgerFromTxStateHex === "function") {
+        state = ledgerFromTxStateHex(rawState);
+      } else {
+        const byteState = new Uint8Array(rawState.match(/.{1,2}/g)!.map(byte => parseInt(byte, 16)));
+        const contractState = ContractState.deserialize(byteState);
+        const stateValue = contractState.data.state;
+        const additionalFields = primitive.parseAdditionalLedgerFields?.(stateValue) ?? {};
+        state = { ...f.ledger(stateValue), ...additionalFields };
+      }
       return {
         syncProtocol: {
           name: primitiveEntry.syncProtocol,

package/src/sync-protocols/midnight/state.ts

@@ -146,12 +146,7 @@ export class MidnightSyncState extends SyncState<
       page,
       config,
       fetcher,
-      // TODO: The urls should be part of the config.
-      new MidnightClient(
-        config.syncProtocol.indexer,
-        config.syncProtocol.indexerWS ??
-          "ws://127.0.0.1:8088/api/v3/graphql/ws",
-      ),
+      fetcher.client,
       dbConn,
     );
   }

package/src/sync-protocols/orchestration/merge.ts

@@ -146,6 +146,11 @@ export function* mergeIntoRoot<SyncProtocol extends AllSyncProtocols>(
       rootInfo.toPage(rootInfo.value),
     ) <= 0
   ) {
+    // Publish the demand before waiting so the cap exempts this chain while the
+    // merge is gated on its page (see SyncState.mergeWaitingForPage). No lock
+    // needed: the fetch loop re-polls stateToInput every pollingInterval.
+    state.mergeWaitingForPage = true;
+    state.mergeDemandRoot = rootInfo.toPage(rootInfo.value);
     log.remote(
       ComponentNames.EFFECTSTREAM_SYNC,
       state.getNamespace(),
@@ -160,6 +165,8 @@ export function* mergeIntoRoot<SyncProtocol extends AllSyncProtocols>(
       (log) => log("Merge unblocked (missing parallel data): got data"),
     );
   }
+  state.mergeWaitingForPage = false;
+  state.mergeDemandRoot = undefined;
   const cleanups = getFromBuffer(
     {
       data: iState.bufferedData,

package/src/sync-protocols/utxorpc/WatchMultiplexer.ts

@@ -40,15 +40,13 @@ export class WatchMultiplexer {
   ) {
     for (const [key, { predicate }] of primitivesByPredicate) {
       const effective = isEffectiveServerPredicate(predicate);
-      // TODO(dolos): when Dolos implements has_certificate, remove this fallback
       const sdkPredicate = effective
         ? configPredicateToSdkPredicate(predicate)
         : {};
 
       if (!effective) {
         console.warn(
-          `[WatchMultiplexer] Predicate for key "${key}" cannot be server-side filtered ` +
-          `(Dolos doesn't support has_certificate). Falling back to match-all with client-side filtering.`
+          `[WatchMultiplexer] Predicate for key "${key}" has no server-filterable fields. Falling back to match-all with client-side filtering.`
         );
       }
 

package/src/sync-protocols/utxorpc/utils.ts

@@ -1,5 +1,5 @@
 import type { cardano } from '@utxorpc/spec';
-import type { UtxorpcAddressPattern, UtxorpcAnyChainTxPattern, UtxorpcAssetPattern, UtxorpcTxOutputPattern, UtxorpcTxPredicate } from "@effectstream/config";
+import type { UtxorpcAddressPattern, UtxorpcAnyChainTxPattern, UtxorpcAssetPattern, UtxorpcCertificatePattern, UtxorpcTxOutputPattern, UtxorpcTxPredicate } from "@effectstream/config";
 import { hexStringToUint8Array } from "@effectstream/utils";
 import { Cardano } from '@cardano-sdk/core';
 import { HexBlob } from '@cardano-sdk/util';
@@ -111,14 +111,24 @@ function convertAssetPattern(pattern: UtxorpcAssetPattern): Record<string, Uint8
   return result;
 }
 
+function convertCertificatePattern(pattern: boolean | UtxorpcCertificatePattern): Record<string, unknown> {
+  if (typeof pattern === "boolean") return {};
+  if (pattern.stake_delegation) {
+    const value: Record<string, unknown> = {};
+    if (pattern.stake_delegation.pool_keyhash) {
+      value.poolKeyhash = hexStringToUint8Array(pattern.stake_delegation.pool_keyhash);
+    }
+    return { certificateType: { case: "stakeDelegation", value } };
+  }
+  if (pattern.any_pool_keyhash) {
+    return { certificateType: { case: "anyPoolKeyhash", value: hexStringToUint8Array(pattern.any_pool_keyhash) } };
+  }
+  return {};
+}
+
 /**
  * Convert a config-layer UtxorpcTxPredicate (snake_case) to the SDK's TxPredicate (camelCase).
- *
- * NOTE: `has_certificate` is in the UTxORPC spec but Dolos does NOT implement server-side
- * filtering for it. Predicates using has_certificate will be sent to the server but silently
- * ignored — client-side filtering in matchesPredicate() still catches them.
  */
-// TODO(dolos): when Dolos implements has_certificate filtering, this conversion will just work
 export function configPredicateToSdkPredicate(config: UtxorpcTxPredicate): TxPredicate {
   const result: TxPredicate = {};
 
@@ -128,7 +138,7 @@ export function configPredicateToSdkPredicate(config: UtxorpcTxPredicate): TxPre
     if (c.has_address) match.hasAddress = convertAddressPattern(c.has_address);
     if (c.moves_asset) match.movesAsset = convertAssetPattern(c.moves_asset);
     if (c.mints_asset) match.mintsAsset = convertAssetPattern(c.mints_asset);
-    if (c.has_certificate) match.hasCertificate = {};
+    if (c.has_certificate) match.hasCertificate = convertCertificatePattern(c.has_certificate);
     result.match = match as TxPredicate["match"];
   }
 
@@ -148,13 +158,10 @@ export function predicateKey(predicate: UtxorpcTxPredicate): string {
 }
 
 /**
- * Returns true if this predicate can be effectively filtered server-side by Dolos.
- * Currently Dolos Watch ignores `has_certificate`, so predicates relying solely on it
- * are not effective and should fall back to match-all.
+ * Returns true if this predicate can be effectively filtered server-side.
  */
-// TODO(dolos): remove this when Dolos implements has_certificate filtering
 export function isEffectiveServerPredicate(predicate: UtxorpcTxPredicate): boolean {
   if (!predicate.match?.cardano) return false;
   const c = predicate.match.cardano;
-  return !!(c.has_address || c.moves_asset || c.mints_asset);
+  return !!(c.has_address || c.moves_asset || c.mints_asset || c.has_certificate);
 }