@effectstream/sync 0.100.27 → 0.101.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@effectstream/sync",
-  "version": "0.100.27",
+  "version": "0.101.0",
   "license": "MIT",
   "homepage": "https://effectstream.github.io/docs/",
   "repository": {
@@ -13,10 +13,10 @@
   },
   "exports": "./src/mod.ts",
   "dependencies": {
-    "@effectstream/config": "0.100.27",
-    "@effectstream/db": "0.100.27",
-    "@effectstream/log": "0.100.27",
-    "@effectstream/utils": "0.100.27",
+    "@effectstream/config": "0.101.0",
+    "@effectstream/db": "0.101.0",
+    "@effectstream/log": "0.101.0",
+    "@effectstream/utils": "0.101.0",
     "@midnight-ntwrk/ledger-v8": "8.0.3",
     "@midnight-ntwrk/midnight-js-indexer-public-data-provider": "4.0.4",
     "@midnight-ntwrk/onchain-runtime": "npm:@midnight-ntwrk/onchain-runtime-v3@3.0.0",

package/src/sync-protocols/celestia/CelestiaClient.ts

@@ -54,6 +54,9 @@ const CELESTIA_NODE_URL = ENV.getString(
   "http://localhost:26658",
 );
 const CELESTIA_AUTH_TOKEN = ENV.getString("CELESTIA_AUTH_TOKEN", "");
+const CELESTIA_RPC_TIMEOUT_MS = parseInt(
+  ENV.getString("CELESTIA_RPC_TIMEOUT_MS", "15000"),
+);
 
 /**
  * Converts a hex-encoded Celestia namespace to a base64-encoded 29-byte array.
@@ -80,29 +83,38 @@ export class CelestiaClient {
   }
 
   private async rpc<T>(method: string, params: unknown[]): Promise<T | null> {
-    let res: Response;
-    try {
-      const headers: Record<string, string> = { "Content-Type": "application/json" };
-      if (CELESTIA_AUTH_TOKEN) headers["Authorization"] = `Bearer ${CELESTIA_AUTH_TOKEN}`;
-      res = await fetch(this.rpcUrl, {
-        method: "POST",
-        headers,
-        body: JSON.stringify({ jsonrpc: "2.0", id: 1, method, params }),
-      });
-    } catch (err: unknown) {
-      const msg = err instanceof Error ? err.message : String(err);
-      console.error(`[Celestia] RPC fetch error [${method}]: ${msg}`);
-      return null;
-    }
-
-    const json = await res.json();
-    if (json.error) {
-      const errMsg: string = json.error.message ?? String(json.error);
-      if (errMsg.includes("blob: not found")) return null;
-      console.error(`[Celestia] RPC error [${method}]:`, errMsg);
-      return null;
+    // Retry forever — a transient stall must never cause a block to be skipped.
+    // The only exit paths are: (a) success, (b) "blob: not found" (empty namespace).
+    // Everything else (network error, timeout, RPC error) loops with backoff.
+    let attempt = 0;
+    while (true) {
+      try {
+        const headers: Record<string, string> = { "Content-Type": "application/json" };
+        if (CELESTIA_AUTH_TOKEN) headers["Authorization"] = `Bearer ${CELESTIA_AUTH_TOKEN}`;
+        const res = await fetch(this.rpcUrl, {
+          method: "POST",
+          headers,
+          body: JSON.stringify({ jsonrpc: "2.0", id: 1, method, params }),
+          signal: AbortSignal.timeout(CELESTIA_RPC_TIMEOUT_MS),
+        });
+        const json = await res.json();
+        if (json.error) {
+          const errMsg: string = json.error.message ?? String(json.error);
+          if (errMsg.includes("blob: not found")) return null;
+          throw new Error(`[Celestia] RPC error [${method}]: ${errMsg}`);
+        }
+        return json.result as T;
+      } catch (err: unknown) {
+        attempt++;
+        const backoffMs = Math.min(1000 * 2 ** Math.min(attempt - 1, 5), 30_000);
+        console.warn(
+          `[Celestia] ${method} attempt ${attempt} failed (retrying in ${backoffMs}ms): ${
+            err instanceof Error ? err.message : String(err)
+          }`,
+        );
+        await new Promise((resolve) => setTimeout(resolve, backoffMs));
+      }
     }
-    return json.result as T;
   }
 
   async getLatestBlockHeight(): Promise<number> {

package/src/sync-protocols/celestia/fetcher.ts

@@ -51,44 +51,49 @@ export class CelestiaFetcher extends BaseDataFetcher<
       (p) => p.primitive.getAllBlockHeaders,
     );
 
+    const heights: number[] = [];
+    for (let h = data.from; h <= data.to; h++) heights.push(h);
+
     console.log(
-      `[Celestia] Fetching blocks from ${data.from} to ${data.to}.${
+      `[Celestia] Fetching blocks from ${data.from} to ${data.to} (${heights.length} blocks, concurrency=${(this.config.syncProtocol as any).concurrency ?? 1}).${
         data.isPresync ? " [presync]" : ""
       }`,
     );
 
-    let lastFetchedOutput: Output | undefined;
-    for (let height = data.from; height <= data.to; height++) {
-      let extHeader;
-      try {
-        extHeader = yield* call(() =>
-          this.client.getHeaderAtHeight(height)
-        );
-      } catch (_e) {
-        break;
-      }
+    const results: (Output | undefined)[] = new Array(heights.length);
+    let cursor = 0;
+    const concurrency: number = (this.config.syncProtocol as any).concurrency ?? 1;
+    const self = this;
 
-      const blockHash = extHeader.commit.block_id.hash;
+    // Each worker pulls the next unstarted height from `cursor` and stores the
+    // result at the corresponding index so output order is always height-ascending.
+    // The cursor increment is cooperative-race-free in effection: it runs
+    // synchronously before the first yield in each worker iteration.
+    function* worker(): Operation<void> {
+      while (true) {
+        const i = cursor++;
+        if (i >= heights.length) return;
+        const height = heights[i];
+        const extHeader = yield* call(() => self.client.getHeaderAtHeight(height));
+        const blockHash = extHeader.commit.block_id.hash;
+        const primitives = yield* self.readPrimitives(height, blockHash, self.config.primitives);
+        results[i] = { timestamp: extHeader.header.time, height, hash: blockHash, primitives };
+      }
+    }
 
-      const primitives = yield* this.readPrimitives(
-        height,
-        blockHash,
-        this.config.primitives,
+    if (heights.length > 0) {
+      yield* all(
+        Array.from({ length: Math.min(concurrency, heights.length) }, () => worker()),
       );
+    }
 
-      const output: Output = {
-        timestamp: extHeader.header.time,
-        height,
-        hash: blockHash,
-        primitives,
-      };
+    let lastFetchedOutput: Output | undefined;
+    for (let i = 0; i < results.length; i++) {
+      const output = results[i];
+      if (!output) continue;
       lastFetchedOutput = output;
-
-      if (fetchAllBlocks || primitives.length > 0 || height === data.to) {
-        outputs.push({
-          output,
-          cleanup: () => {},
-        });
+      if (fetchAllBlocks || output.primitives.length > 0 || output.height === data.to) {
+        outputs.push({ output, cleanup: () => {} });
       }
     }