@effectstream/sync 0.100.24 → 0.100.26

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@effectstream/sync",
-  "version": "0.100.24",
+  "version": "0.100.26",
   "license": "MIT",
   "homepage": "https://effectstream.github.io/docs/",
   "repository": {
@@ -13,10 +13,10 @@
   },
   "exports": "./src/mod.ts",
   "dependencies": {
-    "@effectstream/config": "0.100.24",
-    "@effectstream/db": "0.100.24",
-    "@effectstream/log": "0.100.24",
-    "@effectstream/utils": "0.100.24",
+    "@effectstream/config": "0.100.26",
+    "@effectstream/db": "0.100.26",
+    "@effectstream/log": "0.100.26",
+    "@effectstream/utils": "0.100.26",
     "@midnight-ntwrk/ledger-v8": "8.0.3",
     "@midnight-ntwrk/midnight-js-indexer-public-data-provider": "4.0.4",
     "@midnight-ntwrk/onchain-runtime": "npm:@midnight-ntwrk/onchain-runtime-v3@3.0.0",

package/src/sync-protocols/midnight/MidnightClient.ts

@@ -1,4 +1,3 @@
-import { indexerPublicDataProvider } from "@midnight-ntwrk/midnight-js-indexer-public-data-provider";
 import type { Block } from "./types.ts";
 import type { ExecutionResult } from "graphql-ws/client";
 
@@ -67,24 +66,16 @@ export interface BlockFetchOptions {
   tokenMints?: boolean;
 }
 
-type PublicDataProvider = ReturnType<typeof indexerPublicDataProvider>;
 export class MidnightClient {
   private readonly queryURL: string;
-  private readonly subscriptionURL: string;
-  private publicDataProvider: PublicDataProvider;
   private readonly networkId?: string;
   /** Timeout in milliseconds for individual HTTP requests to the indexer. */
   private readonly requestTimeoutMs: number;
 
-  constructor(queryURL: string, subscriptionURL: string, networkId?: string, requestTimeoutMs = 30_000) {
+  constructor(queryURL: string, networkId?: string, requestTimeoutMs = 30_000) {
     this.queryURL = queryURL;
-    this.subscriptionURL = subscriptionURL;
     this.networkId = networkId;
     this.requestTimeoutMs = requestTimeoutMs;
-    this.publicDataProvider = indexerPublicDataProvider(
-      queryURL,
-      subscriptionURL,
-    );
     console.log(
       `[MidnightClient] Using indexer ${queryURL} (network: ${
         networkId ?? "unknown"
@@ -92,48 +83,6 @@ export class MidnightClient {
     );
   }
 
-  /**
-   * Recreate the publicDataProvider to recover from stale internal state.
-   * The Midnight JS library can get into a state where all queryContractState
-   * calls fail with {"json":{}} until the provider is recreated.
-   */
-  resetProvider(): void {
-    console.log(
-      `[MidnightClient] Resetting publicDataProvider for ${this.queryURL}`,
-    );
-    this.publicDataProvider = indexerPublicDataProvider(
-      this.queryURL,
-      this.subscriptionURL,
-    );
-  }
-
-  async fetchContractState(contractAddress: string, blockHeight: number) {
-    try {
-      const state = await Promise.race([
-        this.publicDataProvider.queryContractState(
-          contractAddress,
-          {
-            type: "blockHeight",
-            blockHeight,
-          },
-        ),
-        new Promise<never>((_, reject) => {
-          setTimeout(
-            () => reject(new Error(`fetchContractState timed out after ${this.requestTimeoutMs}ms (contract=${contractAddress}, height=${blockHeight})`)),
-            this.requestTimeoutMs,
-          );
-        }),
-      ]);
-      return state;
-    } catch (error) {
-      // The Midnight JS publicDataProvider can enter a broken state where all
-      // subsequent calls fail (e.g. {"json":{}}).  Recreate it so the next
-      // retry starts with a fresh connection.
-      this.resetProvider();
-      throw error;
-    }
-  }
-
   async gqlQuery(query: string, signal?: AbortSignal): Promise<any> {
     const timeoutSignal = AbortSignal.timeout(this.requestTimeoutMs);
     const combinedSignal = signal

package/src/sync-protocols/midnight/fetcher.ts

@@ -35,16 +35,9 @@ export class MidnightFetcher extends BaseDataFetcher<
   ) {
     super(config.syncProtocol.name);
     const indexerHttp = config.syncProtocol.indexer;
-    const indexerWs =
-      (config.syncProtocol as any).indexerWS ??
-      (config.syncProtocol as any).indexerWs;
-    if (!indexerHttp || !indexerWs) {
+    if (!indexerHttp) {
       throw new Error(
-        `Midnight sync protocol "${
-          config.syncProtocol.name
-        }" requires both indexer and indexerWS URLs. Received indexer=${
-          indexerHttp ?? "undefined"
-        }, indexerWS=${indexerWs ?? "undefined"}.`,
+        `Midnight sync protocol "${config.syncProtocol.name}" requires an indexer URL.`,
       );
     }
     this.networkId = config.network?.networkId ??
@@ -65,7 +58,6 @@ export class MidnightFetcher extends BaseDataFetcher<
 
     this.client = new MidnightClient(
       indexerHttp,
-      indexerWs,
       this.networkId,
     );
   }
@@ -385,19 +377,17 @@ export class MidnightFetcher extends BaseDataFetcher<
   @bound
   *fetchContractState(
     height: number,
-    client: MidnightClient,
+    _client: MidnightClient,
     primitiveEntry: PrimitiveEntryType,
     block: MidnightGqlBlockState,
   ): Operation<PrimitiveType[]> {
     const contractAddress = primitiveEntry.primitive.contractAddress!;
-    const blockFinalState = yield* call(() =>
-      client.fetchContractState(contractAddress, height)
-    );
-    // The state returns null if the contract was not called in the block height
-    if (!blockFinalState) {
-      return [];
-    }
 
+    // Determine whether the contract was called in this block by inspecting
+    // the contractActions already present in the fetched block data — no extra
+    // network round-trip needed. The per-transaction state is also read from
+    // contractActions below, so queryContractState would be redundant and its
+    // accumulating internal WS subscriptions caused saturation during catch-up.
     const transactions = block.block.transactions.filter((t) => {
       return (t.contractActions ?? []).some((c) => {
         const longest = Math.max(contractAddress.length, c.address.length);
@@ -406,6 +396,9 @@ export class MidnightFetcher extends BaseDataFetcher<
       });
     });
 
+    if (transactions.length === 0) {
+      return [];
+    }
 
     return transactions.map(t => {
       // TODO: What does it mean if t.contractActions has more than one element?

package/src/sync-protocols/midnight/state.ts

@@ -12,11 +12,6 @@ import { MidnightClient } from "./MidnightClient.ts";
 import { applyDelay } from "../common/utils.ts";
 import { bufferAtCap } from "../common/page-helpers.ts";
 
-type LatestBlock = {
-  block: {
-    height: number;
-  };
-};
 
 export class MidnightSyncState extends SyncState<
   Input,
@@ -27,6 +22,13 @@ export class MidnightSyncState extends SyncState<
   MidnightFetcher
 > {
   private readonly url: string;
+  private cachedLatestHeight: number | undefined;
+  private cacheTimestampMs: number = 0;
+  // Midnight produces ~1 block every 6 s. Re-query the real tip once this many
+  // estimated chain blocks have elapsed, giving periodic lag info without a
+  // round-trip on every batch.
+  private static readonly CHAIN_BLOCK_TIME_MS = 6_000;
+  private static readonly REFRESH_EVERY_CHAIN_BLOCKS = 100;
 
   constructor(
     lastPage: LastPage<Page, RootPage> | undefined,
@@ -74,12 +76,29 @@ export class MidnightSyncState extends SyncState<
   @bound
   override *stateToInput(): Operation<Input | undefined> {
     if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
-    const latestBlockResult = yield* call(() => this.client.fetchLatestBlock());
-    const latestHeight = latestBlockResult.block.height;
 
     const startHeight = this.lastPage?.own.height ??
       this.config.syncProtocol.startBlockHeight - 1;
 
+    // Extrapolate how many new chain blocks have likely arrived since the last
+    // real query, then re-query only when that estimate exceeds the threshold.
+    // This bounds indexer round-trips to O(catch-up_time / (REFRESH * block_time))
+    // rather than one per batch, while still refreshing periodically so lag
+    // metrics stay accurate.
+    const estimatedNewBlocks = this.cachedLatestHeight !== undefined
+      ? Math.floor((Date.now() - this.cacheTimestampMs) / MidnightSyncState.CHAIN_BLOCK_TIME_MS)
+      : 0;
+    if (
+      this.cachedLatestHeight === undefined ||
+      startHeight >= this.cachedLatestHeight ||
+      estimatedNewBlocks >= MidnightSyncState.REFRESH_EVERY_CHAIN_BLOCKS
+    ) {
+      const latestBlockResult = yield* call(() => this.client.fetchLatestBlock());
+      this.cachedLatestHeight = latestBlockResult.block.height;
+      this.cacheTimestampMs = Date.now();
+    }
+    const latestHeight = this.cachedLatestHeight;
+
     if (startHeight >= latestHeight) {
       return undefined;
     }