@effectstream/sync 0.100.25 → 0.100.27

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@effectstream/sync",
-  "version": "0.100.25",
+  "version": "0.100.27",
   "license": "MIT",
   "homepage": "https://effectstream.github.io/docs/",
   "repository": {
@@ -13,10 +13,10 @@
   },
   "exports": "./src/mod.ts",
   "dependencies": {
-    "@effectstream/config": "0.100.25",
-    "@effectstream/db": "0.100.25",
-    "@effectstream/log": "0.100.25",
-    "@effectstream/utils": "0.100.25",
+    "@effectstream/config": "0.100.27",
+    "@effectstream/db": "0.100.27",
+    "@effectstream/log": "0.100.27",
+    "@effectstream/utils": "0.100.27",
     "@midnight-ntwrk/ledger-v8": "8.0.3",
     "@midnight-ntwrk/midnight-js-indexer-public-data-provider": "4.0.4",
     "@midnight-ntwrk/onchain-runtime": "npm:@midnight-ntwrk/onchain-runtime-v3@3.0.0",

package/src/sync-protocols/common/root.ts

@@ -34,6 +34,7 @@ export type ChainBlock = {
     >
     & { source: string }
   )[];
+  coalescedCount?: number;
 };
 
 export const chainPageRelation: PageRelation<ChainPage> = {

package/src/sync-protocols/midnight/MidnightClient.ts

@@ -1,4 +1,3 @@
-import { indexerPublicDataProvider } from "@midnight-ntwrk/midnight-js-indexer-public-data-provider";
 import type { Block } from "./types.ts";
 import type { ExecutionResult } from "graphql-ws/client";
 
@@ -67,24 +66,16 @@ export interface BlockFetchOptions {
   tokenMints?: boolean;
 }
 
-type PublicDataProvider = ReturnType<typeof indexerPublicDataProvider>;
 export class MidnightClient {
   private readonly queryURL: string;
-  private readonly subscriptionURL: string;
-  private publicDataProvider: PublicDataProvider;
   private readonly networkId?: string;
   /** Timeout in milliseconds for individual HTTP requests to the indexer. */
   private readonly requestTimeoutMs: number;
 
-  constructor(queryURL: string, subscriptionURL: string, networkId?: string, requestTimeoutMs = 30_000) {
+  constructor(queryURL: string, networkId?: string, requestTimeoutMs = 30_000) {
     this.queryURL = queryURL;
-    this.subscriptionURL = subscriptionURL;
     this.networkId = networkId;
     this.requestTimeoutMs = requestTimeoutMs;
-    this.publicDataProvider = indexerPublicDataProvider(
-      queryURL,
-      subscriptionURL,
-    );
     console.log(
       `[MidnightClient] Using indexer ${queryURL} (network: ${
         networkId ?? "unknown"
@@ -92,48 +83,6 @@ export class MidnightClient {
     );
   }
 
-  /**
-   * Recreate the publicDataProvider to recover from stale internal state.
-   * The Midnight JS library can get into a state where all queryContractState
-   * calls fail with {"json":{}} until the provider is recreated.
-   */
-  resetProvider(): void {
-    console.log(
-      `[MidnightClient] Resetting publicDataProvider for ${this.queryURL}`,
-    );
-    this.publicDataProvider = indexerPublicDataProvider(
-      this.queryURL,
-      this.subscriptionURL,
-    );
-  }
-
-  async fetchContractState(contractAddress: string, blockHeight: number) {
-    try {
-      const state = await Promise.race([
-        this.publicDataProvider.queryContractState(
-          contractAddress,
-          {
-            type: "blockHeight",
-            blockHeight,
-          },
-        ),
-        new Promise<never>((_, reject) => {
-          setTimeout(
-            () => reject(new Error(`fetchContractState timed out after ${this.requestTimeoutMs}ms (contract=${contractAddress}, height=${blockHeight})`)),
-            this.requestTimeoutMs,
-          );
-        }),
-      ]);
-      return state;
-    } catch (error) {
-      // The Midnight JS publicDataProvider can enter a broken state where all
-      // subsequent calls fail (e.g. {"json":{}}).  Recreate it so the next
-      // retry starts with a fresh connection.
-      this.resetProvider();
-      throw error;
-    }
-  }
-
   async gqlQuery(query: string, signal?: AbortSignal): Promise<any> {
     const timeoutSignal = AbortSignal.timeout(this.requestTimeoutMs);
     const combinedSignal = signal

package/src/sync-protocols/midnight/fetcher.ts

@@ -35,16 +35,9 @@ export class MidnightFetcher extends BaseDataFetcher<
   ) {
     super(config.syncProtocol.name);
     const indexerHttp = config.syncProtocol.indexer;
-    const indexerWs =
-      (config.syncProtocol as any).indexerWS ??
-      (config.syncProtocol as any).indexerWs;
-    if (!indexerHttp || !indexerWs) {
+    if (!indexerHttp) {
       throw new Error(
-        `Midnight sync protocol "${
-          config.syncProtocol.name
-        }" requires both indexer and indexerWS URLs. Received indexer=${
-          indexerHttp ?? "undefined"
-        }, indexerWS=${indexerWs ?? "undefined"}.`,
+        `Midnight sync protocol "${config.syncProtocol.name}" requires an indexer URL.`,
       );
     }
     this.networkId = config.network?.networkId ??
@@ -65,7 +58,6 @@ export class MidnightFetcher extends BaseDataFetcher<
 
     this.client = new MidnightClient(
       indexerHttp,
-      indexerWs,
       this.networkId,
     );
   }
@@ -385,19 +377,17 @@ export class MidnightFetcher extends BaseDataFetcher<
   @bound
   *fetchContractState(
     height: number,
-    client: MidnightClient,
+    _client: MidnightClient,
     primitiveEntry: PrimitiveEntryType,
     block: MidnightGqlBlockState,
   ): Operation<PrimitiveType[]> {
     const contractAddress = primitiveEntry.primitive.contractAddress!;
-    const blockFinalState = yield* call(() =>
-      client.fetchContractState(contractAddress, height)
-    );
-    // The state returns null if the contract was not called in the block height
-    if (!blockFinalState) {
-      return [];
-    }
 
+    // Determine whether the contract was called in this block by inspecting
+    // the contractActions already present in the fetched block data — no extra
+    // network round-trip needed. The per-transaction state is also read from
+    // contractActions below, so queryContractState would be redundant and its
+    // accumulating internal WS subscriptions caused saturation during catch-up.
     const transactions = block.block.transactions.filter((t) => {
       return (t.contractActions ?? []).some((c) => {
         const longest = Math.max(contractAddress.length, c.address.length);
@@ -406,6 +396,9 @@ export class MidnightFetcher extends BaseDataFetcher<
       });
     });
 
+    if (transactions.length === 0) {
+      return [];
+    }
 
     return transactions.map(t => {
       // TODO: What does it mean if t.contractActions has more than one element?

package/src/sync-protocols/midnight/state.ts

@@ -12,11 +12,6 @@ import { MidnightClient } from "./MidnightClient.ts";
 import { applyDelay } from "../common/utils.ts";
 import { bufferAtCap } from "../common/page-helpers.ts";
 
-type LatestBlock = {
-  block: {
-    height: number;
-  };
-};
 
 export class MidnightSyncState extends SyncState<
   Input,
@@ -27,6 +22,13 @@ export class MidnightSyncState extends SyncState<
   MidnightFetcher
 > {
   private readonly url: string;
+  private cachedLatestHeight: number | undefined;
+  private cacheTimestampMs: number = 0;
+  // Midnight produces ~1 block every 6 s. Re-query the real tip once this many
+  // estimated chain blocks have elapsed, giving periodic lag info without a
+  // round-trip on every batch.
+  private static readonly CHAIN_BLOCK_TIME_MS = 6_000;
+  private static readonly REFRESH_EVERY_CHAIN_BLOCKS = 100;
 
   constructor(
     lastPage: LastPage<Page, RootPage> | undefined,
@@ -74,12 +76,29 @@ export class MidnightSyncState extends SyncState<
   @bound
   override *stateToInput(): Operation<Input | undefined> {
     if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
-    const latestBlockResult = yield* call(() => this.client.fetchLatestBlock());
-    const latestHeight = latestBlockResult.block.height;
 
     const startHeight = this.lastPage?.own.height ??
       this.config.syncProtocol.startBlockHeight - 1;
 
+    // Extrapolate how many new chain blocks have likely arrived since the last
+    // real query, then re-query only when that estimate exceeds the threshold.
+    // This bounds indexer round-trips to O(catch-up_time / (REFRESH * block_time))
+    // rather than one per batch, while still refreshing periodically so lag
+    // metrics stay accurate.
+    const estimatedNewBlocks = this.cachedLatestHeight !== undefined
+      ? Math.floor((Date.now() - this.cacheTimestampMs) / MidnightSyncState.CHAIN_BLOCK_TIME_MS)
+      : 0;
+    if (
+      this.cachedLatestHeight === undefined ||
+      startHeight >= this.cachedLatestHeight ||
+      estimatedNewBlocks >= MidnightSyncState.REFRESH_EVERY_CHAIN_BLOCKS
+    ) {
+      const latestBlockResult = yield* call(() => this.client.fetchLatestBlock());
+      this.cachedLatestHeight = latestBlockResult.block.height;
+      this.cacheTimestampMs = Date.now();
+    }
+    const latestHeight = this.cachedLatestHeight;
+
     if (startHeight >= latestHeight) {
       return undefined;
     }

package/src/sync-protocols/orchestration/merge.ts

@@ -11,11 +11,98 @@ import type Deque from "denque";
 import type { ChainBlock } from "../common/root.ts";
 import { chainPageRelation } from "../common/root.ts";
 import type { CacheCleanup, OutputAndCleanup } from "../base/state.ts";
+import { ENV } from "@effectstream/utils/node-env";
+
+export const mergeCoalescingBoundaries = {
+  minScheduledBlockHeight: null as number | null,
+  minScheduledTimestampMs: null as number | null,
+  migrationBlockHeights: [] as number[],
+};
+
+function isCoalescableEmpty(block: ChainBlock): boolean {
+  if (block.primitives.length > 0) return false;
+
+  const bh = block.blockNumber;
+  if (mergeCoalescingBoundaries.migrationBlockHeights.includes(bh)) {
+    return false;
+  }
+
+  if (
+    mergeCoalescingBoundaries.minScheduledBlockHeight != null &&
+    bh >= mergeCoalescingBoundaries.minScheduledBlockHeight
+  ) {
+    return false;
+  }
+
+  if (
+    mergeCoalescingBoundaries.minScheduledTimestampMs != null &&
+    block.timestamp >= mergeCoalescingBoundaries.minScheduledTimestampMs
+  ) {
+    return false;
+  }
+
+  return true;
+}
+
+function mergeBlockInfo(
+  target: ChainBlock["blockInfo"],
+  source: ChainBlock["blockInfo"],
+): ChainBlock["blockInfo"] {
+  const map = new Map<string, ChainBlock["blockInfo"][number]>();
+  for (const item of target) {
+    map.set(item.protocol_name, item);
+  }
+  for (const item of source) {
+    map.set(item.protocol_name, item);
+  }
+  return Array.from(map.values());
+}
+
+function mergeResumePages(
+  target: ChainBlock["resumePages"],
+  source: ChainBlock["resumePages"],
+): ChainBlock["resumePages"] {
+  const map = new Map<string, ChainBlock["resumePages"][number]>();
+  for (const item of target) {
+    map.set(item.protocol_name, item);
+  }
+  for (const item of source) {
+    map.set(item.protocol_name, item);
+  }
+  return Array.from(map.values());
+}
 
 export function* startMerge(
   syncProtocols: AllSyncProtocols[],
   finalizedBlockStream: Channel<ChainBlock, void>,
 ): Operation<ChainBlock> {
+  let pendingEmptyRun: {
+    endpoint: ChainBlock;
+    count: number;
+  } | null = null;
+
+  const flushPendingRun = function* (): Operation<void> {
+    if (pendingEmptyRun == null) return;
+    const { endpoint } = pendingEmptyRun;
+    endpoint.coalescedCount = pendingEmptyRun.count;
+
+    log.remote(
+      ComponentNames.EFFECTSTREAM_SYNC,
+      "block-merge",
+      SeverityNumber.INFO,
+      (log) =>
+        log(
+          `producing coalesced empty block run of length ${pendingEmptyRun!.count} ending at block ${endpoint.blockNumber}`,
+        ),
+    );
+
+    yield* finalizedBlockStream.send(endpoint);
+    pendingEmptyRun = null;
+  };
+
+  const enabled = ENV.EFFECTSTREAM_COALESCE_EMPTY_BLOCKS;
+  const MAX_PRODUCER_COALESCE = 1000;
+
   while (true) {
     let newBlock: ChainBlock | undefined = undefined;
 
@@ -44,17 +131,66 @@ export function* startMerge(
       }
     }
     if (newBlock == null) continue;
-    log.remote(
-      ComponentNames.EFFECTSTREAM_SYNC,
-      "block-merge",
-      SeverityNumber.INFO,
-      (log) => log(`producing block ${newBlock?.blockNumber}`),
-    );
 
-    yield* finalizedBlockStream.send(newBlock);
+    if (enabled && isCoalescableEmpty(newBlock)) {
+      if (pendingEmptyRun == null) {
+        pendingEmptyRun = {
+          endpoint: newBlock,
+          count: 1,
+        };
+      } else {
+        pendingEmptyRun.endpoint.blockNumber = newBlock.blockNumber;
+        pendingEmptyRun.endpoint.timestamp = newBlock.timestamp;
+        pendingEmptyRun.endpoint.blockInfo = mergeBlockInfo(
+          pendingEmptyRun.endpoint.blockInfo,
+          newBlock.blockInfo,
+        );
+        pendingEmptyRun.endpoint.resumePages = mergeResumePages(
+          pendingEmptyRun.endpoint.resumePages,
+          newBlock.resumePages,
+        );
+        pendingEmptyRun.count++;
+      }
+
+      // Run cleanups eagerly so the consumed datum leaves the Deque now, instead
+      // of deferring to flush. This (a) keeps the main chain's Deque from growing
+      // by one entry per coalesced block across a long run, and (b) makes the
+      // tip check below meaningful: with the consumed datum removed,
+      // bufferedData.size() === 0 means the buffer is drained and we're at the
+      // tip. Safe because, during an empty run, every cleanup is a no-op — the
+      // main (NTP) datum's cleanup is `() => {}` and parallel chains merge
+      // nothing (cleanups.length === 0). Resume durability is unaffected: the
+      // runtime is the sole writer of the persisted resume marker, which only
+      // lands when the coalesced block commits downstream (CLAUDE.md #5).
+      for (const fn of cleanup) {
+        fn();
+      }
 
-    for (const fn of cleanup) {
-      fn();
+      // At the tip (steady state) the main chain produces one slot per
+      // blockTimeMS, so the Deque drains to empty after each one — flush
+      // immediately so trickle-in blocks aren't held back. While catching up the
+      // Deque stays non-empty and the run keeps accumulating until the cap.
+      const ntpState = syncProtocols[0];
+      const atTip = ntpState.bufferedData.size() === 0;
+      if (pendingEmptyRun.count >= MAX_PRODUCER_COALESCE || atTip) {
+        yield* flushPendingRun();
+      }
+    } else {
+      if (pendingEmptyRun != null) {
+        yield* flushPendingRun();
+      }
+      log.remote(
+        ComponentNames.EFFECTSTREAM_SYNC,
+        "block-merge",
+        SeverityNumber.INFO,
+        (log) => log(`producing block ${newBlock?.blockNumber}`),
+      );
+
+      yield* finalizedBlockStream.send(newBlock);
+
+      for (const fn of cleanup) {
+        fn();
+      }
     }
   }
 }