@effectstream/sync 0.100.13 → 0.100.15

package/README.md

@@ -0,0 +1,68 @@
+# @effectstream/sync
+
+The blockchain-sync service inside an EffectStream node. Reads finalized
+blocks from every chain you've configured (EVM, Bitcoin, Cardano, Midnight,
+Avail, Celestia, NEAR…), normalizes them into a single rollup ordering,
+and stages the inputs the state machine consumes.
+
+## Install
+
+```bash
+bun add @effectstream/sync
+# or
+npm install @effectstream/sync
+```
+
+## Usage
+
+This package pairs with [`@effectstream/runtime`](https://www.npmjs.com/package/@effectstream/runtime),
+which boots sync as part of `start()`: it calls `genSyncProtocols(...)`
+against the `syncProtocols` section of your
+[`@effectstream/config`](https://www.npmjs.com/package/@effectstream/config),
+then drives the resulting fetcher + state pairs every block. As an app
+author you declare which protocols to sync in your config; everything
+else runs automatically.
+
+If you're building a new chain integration, implement the sync-protocol
+interfaces in [`src/sync-protocols/`](https://github.com/PaimaStudios/paima-engine/tree/main/packages/node-sdk/sync/src/sync-protocols).
+
+## Inside EffectStream
+
+`genSyncProtocols(config)` is what the runtime calls during boot. It walks
+the `syncProtocols` section of your `@effectstream/config`, instantiates a
+fetcher for each chain (viem for EVM, UTXORpc for Cardano, the Midnight
+SDK for Midnight, etc.), and writes paginated blocks into PostgreSQL
+through `@effectstream/db`. The state machine then drains that queue.
+
+```typescript
+import { genSyncProtocols } from "@effectstream/sync";
+// inside the runtime startup path:
+const protocols = await genSyncProtocols(config);
+// protocols.parallelEvmRPC_fast.runOne()  // poll one block
+```
+
+## Key exports
+
+- `genSyncProtocols(dbConn, syncInfo)` — Effection generator that instantiates a runtime fetcher + state pair for every protocol in `syncInfo` (from `config.syncProtocols`). Called from the runtime's process-blocks loop.
+- `AllSyncProtocols` — union type covering every supported protocol; useful when authoring config that fans out.
+- `ChainBlock`, plus base `Fetcher`/`State` types from `sync-protocols/base/` — the wire shape per chain.
+
+Per-chain `Fetcher` / `SyncState` classes (`EvmFetcher`,
+`BitcoinFetcher`, `MidnightFetcher`, `AvailFetcher`, `UtxoRpcFetcher`,
+`NtpFetcher`, `CelestiaFetcher`, `NearFetcher`, and matching `*SyncState`
+classes) are exported but are internal to the factory wiring —
+application code drives them through `genSyncProtocols` rather than
+instantiating them directly. Reach for them only if you're writing a
+custom orchestration layer.
+
+## Examples
+
+End-to-end sync test (boots a node, reads blocks, asserts the DB):
+[`e2e/evm/sync/`](https://github.com/PaimaStudios/paima-engine/tree/main/e2e/evm/sync).
+
+Runnable: [`test/examples.test.ts`](./test/examples.test.ts).
+
+## Links
+
+- Docs: https://effectstream.github.io/docs/packages/node/sync
+- Source: https://github.com/PaimaStudios/paima-engine/tree/main/packages/node-sdk/sync

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@effectstream/sync",
-  "version": "0.100.13",
+  "version": "0.100.15",
   "license": "MIT",
   "homepage": "https://effectstream.github.io/docs/",
   "repository": {
@@ -13,10 +13,10 @@
   },
   "exports": "./src/mod.ts",
   "dependencies": {
-    "@effectstream/config": "0.100.13",
-    "@effectstream/db": "0.100.13",
-    "@effectstream/log": "0.100.13",
-    "@effectstream/utils": "0.100.13",
+    "@effectstream/config": "0.100.15",
+    "@effectstream/db": "0.100.15",
+    "@effectstream/log": "0.100.15",
+    "@effectstream/utils": "0.100.15",
     "@midnight-ntwrk/ledger-v8": "8.0.3",
     "@midnight-ntwrk/midnight-js-indexer-public-data-provider": "4.0.2",
     "@midnight-ntwrk/onchain-runtime": "npm:@midnight-ntwrk/onchain-runtime-v3@3.0.0",

package/src/sync-protocols/avail/fetcher.ts

@@ -48,11 +48,16 @@ export class AvailFetcher extends BaseDataFetcher<
     lastPage: LastPage<Page, RootPage> | undefined,
   ): Operation<DataFetched<Output, Page, RootPage>> {
     const outputs: OutputAndCleanup<Output>[] = [];
+    const fetchAllBlocks = this.config.primitives.some(
+      (p) => p.primitive.getAllBlockHeaders,
+    );
     console.log(
       `[Avail] Fetching blocks from ${data.from} to ${data.to}. ${
         data.isPresync ? "[presync]" : ""
       }`,
     );
+
+    let lastHeader: Output["raw"] | undefined;
     for (let height = data.from; height <= data.to; height++) {
       let header;
       try {
@@ -60,26 +65,28 @@ export class AvailFetcher extends BaseDataFetcher<
           this.client.getBlockHeaderFromHeight(height)
         );
       } catch (_e) {
-        // Likely reached the tip; stop here
         break;
       }
 
+      lastHeader = header;
       const primitives = yield* this.readPrimitives(
         height,
         header,
         this.config.primitives,
       );
 
-      outputs.push({
-        output: {
-          raw: header,
-          primitives,
-        },
-        cleanup: () => {},
-      });
+      if (fetchAllBlocks || primitives.length > 0 || height === data.to) {
+        outputs.push({
+          output: {
+            raw: header,
+            primitives,
+          },
+          cleanup: () => {},
+        });
+      }
     }
 
-    if (outputs.length === 0) {
+    if (!lastHeader) {
       if (!lastPage) {
         throw new Error(
           `Could not fetch any blocks from ${data.from} to ${data.to} and no previous page was found.`,
@@ -91,16 +98,18 @@ export class AvailFetcher extends BaseDataFetcher<
       };
     }
 
-    const lastOutput = outputs[outputs.length - 1].output;
     return {
       output: outputs,
       lastPage: {
-        ownBlockNumber: lastOutput.raw.number,
+        ownBlockNumber: lastHeader.number,
         own: {
-          height: lastOutput.raw.number,
-          hash: lastOutput.raw.hash,
+          height: lastHeader.number,
+          hash: lastHeader.hash,
         },
-        root: rootConversion.toRootPage(lastOutput),
+        root: rootConversion.toRootPage({
+          raw: lastHeader,
+          primitives: [],
+        }),
       },
     };
   }

package/src/sync-protocols/bitcoin/fetcher.ts

@@ -111,24 +111,31 @@ export class BitcoinFetcher
     );
     const groupedByPage = this.groupByPage(primitives);
 
+    const fetchAllBlocks = this.config.primitives.some(
+      (p) => p.primitive.getAllBlockHeaders,
+    );
+
     const outputs: OutputAndCleanup<Output>[] = [];
+    let lastBlock: BitcoinBlock | undefined;
     for (let height = Number(data.from); height <= Number(data.to); height++) {
       const blockHeight = height as Page;
       const block = yield* call(() => pageFetcher(blockHeight));
-      outputs.push({
-        output: {
-          raw: block,
-          primitives: groupedByPage[String(height)] ?? [],
-          blockHashes: [block.hash],
-        },
-        cleanup: () => {},
-      });
+      lastBlock = block;
+      const blockPrimitives = groupedByPage[String(height)] ?? [];
+      if (fetchAllBlocks || blockPrimitives.length > 0 || height === Number(data.to)) {
+        outputs.push({
+          output: {
+            raw: block,
+            primitives: blockPrimitives,
+            blockHashes: [block.hash],
+          },
+          cleanup: () => {},
+        });
+      }
     }
-    const lastOutput = outputs[outputs.length - 1];
-    if (!lastOutput) {
+    if (!lastBlock) {
       throw new Error("BitcoinFetcher: no block data fetched for interval");
     }
-    const lastBlock = lastOutput.output.raw;
     const lastPageRef = this.asPage(Number(data.to));
     return {
       output: outputs,

package/src/sync-protocols/celestia/fetcher.ts

@@ -47,6 +47,9 @@ export class CelestiaFetcher extends BaseDataFetcher<
     lastPage: LastPage<Page, RootPage> | undefined,
   ): Operation<DataFetched<Output, Page, RootPage>> {
     const outputs: OutputAndCleanup<Output>[] = [];
+    const fetchAllBlocks = this.config.primitives.some(
+      (p) => p.primitive.getAllBlockHeaders,
+    );
 
     console.log(
       `[Celestia] Fetching blocks from ${data.from} to ${data.to}.${
@@ -54,6 +57,7 @@ export class CelestiaFetcher extends BaseDataFetcher<
       }`,
     );
 
+    let lastFetchedOutput: Output | undefined;
     for (let height = data.from; height <= data.to; height++) {
       let extHeader;
       try {
@@ -61,7 +65,6 @@ export class CelestiaFetcher extends BaseDataFetcher<
           this.client.getHeaderAtHeight(height)
         );
       } catch (_e) {
-        // Likely reached the chain tip; stop here
         break;
       }
 
@@ -73,18 +76,23 @@ export class CelestiaFetcher extends BaseDataFetcher<
         this.config.primitives,
       );
 
-      outputs.push({
-        output: {
-          timestamp: extHeader.header.time,
-          height,
-          hash: blockHash,
-          primitives,
-        },
-        cleanup: () => {},
-      });
+      const output: Output = {
+        timestamp: extHeader.header.time,
+        height,
+        hash: blockHash,
+        primitives,
+      };
+      lastFetchedOutput = output;
+
+      if (fetchAllBlocks || primitives.length > 0 || height === data.to) {
+        outputs.push({
+          output,
+          cleanup: () => {},
+        });
+      }
     }
 
-    if (outputs.length === 0) {
+    if (!lastFetchedOutput) {
       if (!lastPage) {
         throw new Error(
           `[Celestia] Could not fetch any blocks from ${data.from} to ${data.to} and no previous page was found.`,
@@ -96,16 +104,15 @@ export class CelestiaFetcher extends BaseDataFetcher<
       };
     }
 
-    const lastOutput = outputs[outputs.length - 1].output;
     return {
       output: outputs,
       lastPage: {
-        ownBlockNumber: lastOutput.height,
+        ownBlockNumber: lastFetchedOutput.height,
         own: {
-          height: lastOutput.height,
-          hash: lastOutput.hash,
+          height: lastFetchedOutput.height,
+          hash: lastFetchedOutput.hash,
         },
-        root: rootConversion.toRootPage(lastOutput),
+        root: rootConversion.toRootPage(lastFetchedOutput),
       },
     };
   }

package/src/sync-protocols/evm/fetcher.ts

@@ -25,7 +25,7 @@ import type {
 import { PageSchema } from "./types.ts";
 import {
   fetchNewestPage,
-  genImmediatePageRequests,
+  genOnDemandPageRequests,
   type PageRange,
   type PageRequest,
 } from "../base/page.ts";
@@ -62,15 +62,17 @@ export class EvmFetcher
     data: Input,
     rootConversion: RootConversion<Output, RootOutput, RootPage>,
   ): Operation<DataFetched<Output, Page, RootPage>> {
-    const allPages: Page[] = [];
-    for (let p = data.from; p <= data.to; p++) allPages.push(p);
-    const pageFetcher = genImmediatePageRequests(
-      allPages,
+    const fetchAllBlocks = this.config.primitives.some(
+      (p) => p.primitive.getAllBlockHeaders,
+    );
+
+    const pageFetcher = genOnDemandPageRequests(
+      data.from,
+      data.to,
       (page) => this.client.getBlock({ blockNumber: BigInt(page) }),
+      blockNumberRelation,
     );
 
-    // TODO: if we expect multiple primitives per block
-    //       it can, depending on the chain, be faster to just download the full block and parse it locally
     const primitives = yield* this.readPrimitives(
       data,
       pageFetcher,
@@ -92,40 +94,43 @@ export class EvmFetcher
             };
           }),
         ),
-        // we always need to get the last page in the parallel case
-        // so we can use it for pagination
+        // we always need the last page for pagination
         call(() => pageFetcher(Number(data.to))),
       ],
     );
 
-    // Build an output with all page info, as we need all the hashes to build the effectstream-block-hash
-    // All blocks are already pre-fetched in parallel by genImmediatePageRequests above
     const allOutputs: Output[] = [];
     for (let page = data.from; page <= data.to; page++) {
       const _output = output.find((o) => o.raw.number === BigInt(page));
       if (_output) {
         allOutputs.push(_output);
-      } else {
+      } else if (fetchAllBlocks) {
         const raw = yield* call(() => pageFetcher(page));
         allOutputs.push({
           raw,
           primitives: [],
           blockHashes: [raw.hash] as EvmBlockHash[],
         });
+      } else if (page === data.to) {
+        allOutputs.push({
+          raw: lastPage,
+          primitives: [],
+          blockHashes: [lastPage.hash] as EvmBlockHash[],
+        });
       }
     }
 
     return {
       output: allOutputs.map((o) => ({
         output: o,
-        cleanup: () => {}, // no cleanup required
+        cleanup: () => {},
       })),
       lastPage: {
         own: Number(data.to),
         ownBlockNumber: Number(data.to),
         root: rootConversion.toRootPage({
           blockHashes: [lastPage.hash] as EvmBlockHash[],
-          primitives: [], // unused in toRootPage
+          primitives: [],
           raw: lastPage,
         }),
       },

package/src/sync-protocols/midnight/fetcher.ts

@@ -98,11 +98,11 @@ export class MidnightFetcher extends BaseDataFetcher<
       { length: data.to - data.from + 1 },
       (_, i) => i + data.from,
     );
+    const fetchAllBlocks = this.config.primitives.some(
+      (p) => p.primitive.getAllBlockHeaders,
+    );
+
     const fetched = yield* all(
-      // The endpoint does not support range requests,
-      // so we fetch each block individually in parallel.
-      // useAbortSignal() ties each fetch's lifetime to its Effection scope,
-      // so cancellation by all() cleanly aborts the underlying HTTP request.
       heights.map(function* (height) {
         const signal = yield* useAbortSignal();
         const result: MidnightGqlBlockState = yield* call(() =>
@@ -122,9 +122,22 @@ export class MidnightFetcher extends BaseDataFetcher<
         };
       }),
     );
-    outputs.push(...fetched);
 
-    const lastOutput = outputs[outputs.length - 1].output;
+    const lastFetched = fetched[fetched.length - 1];
+    if (fetchAllBlocks) {
+      outputs.push(...fetched);
+    } else {
+      for (const item of fetched) {
+        if (item.output.primitives.length > 0) {
+          outputs.push(item);
+        }
+      }
+      if (outputs.length === 0 || outputs[outputs.length - 1] !== lastFetched) {
+        outputs.push(lastFetched);
+      }
+    }
+
+    const lastOutput = lastFetched.output;
     return {
       output: outputs,
       lastPage: {

package/src/sync-protocols/utxorpc/WatchMultiplexer.ts

@@ -76,6 +76,20 @@ export class WatchMultiplexer {
       throw new Error("No watchers configured");
     }
 
+    if (this.slotOriginTimestamp === undefined) {
+      try {
+        const blocks = await this.syncClient.fetchHistory(undefined, 1);
+        if (blocks.length > 0) {
+          const blk = blocks[0].parsedBlock;
+          if (blk.timestamp && blk.header?.slot != null) {
+            this.slotOriginTimestamp = blk.timestamp - blk.header.slot;
+          }
+        }
+      } catch {
+        // Chain may not be ready yet — slotOriginTimestamp will be set on the first Apply event
+      }
+    }
+
     const intersect = point ? [point] : [];
     const promises: Promise<void>[] = [];
     for (const [key, watcher] of this.watchers) {

package/src/sync-protocols/utxorpc/fetcher.ts

@@ -40,34 +40,36 @@ export class UtxoRpcFetcher
     );
     const outputs: OutputAndCleanup<Output>[] = [];
     const blocks = this.multiplexer.fetchBlocks(data.from, data.to);
+    let lastEntry: typeof blocks[number] | undefined;
     for (const entry of blocks) {
+      lastEntry = entry;
       const completed = entry.output;
-      outputs.push({
-        output: {
-          blockHashes: [completed.hash],
-          raw: completed.block,
-          primitives: this.findPrimitivesFromWatchData(completed),
-        },
-        cleanup: entry.cleanup,
-      });
+      const primitives = this.findPrimitivesFromWatchData(completed);
+      if (primitives.length > 0) {
+        outputs.push({
+          output: {
+            blockHashes: [completed.hash],
+            raw: completed.block,
+            primitives,
+          },
+          cleanup: entry.cleanup,
+        });
+      }
     }
+    const lastCompleted = lastEntry!.output;
     return {
       output: outputs,
       lastPage: {
         ownBlockNumber: Number(data.to),
         own: {
-          slot: Number(
-            outputs[outputs.length - 1].output.raw.header!.slot,
-          ),
-          height: Number(
-            outputs[outputs.length - 1].output.raw.header!.height,
-          ),
-          hash: outputs[outputs.length - 1].output.blockHashes[0],
+          slot: Number(lastCompleted.block.header!.slot),
+          height: Number(lastCompleted.block.header!.height),
+          hash: lastCompleted.hash,
         },
         root: rootConversion.toRootPage({
           blockHashes: [],
           primitives: [],
-          raw: outputs[outputs.length - 1].output.raw,
+          raw: lastCompleted.block,
         }),
       },
     };

package/test/examples.test.ts

@@ -0,0 +1,18 @@
+// Examples for the README — verify public exports resolve.
+
+import { test, expect } from "bun:test";
+import * as sync from "../src/mod.ts";
+
+test("README: genSyncProtocols is exported as a generator factory", () => {
+  expect(typeof sync.genSyncProtocols).toBe("function");
+});
+
+test("README: per-chain fetchers and states are exported", () => {
+  // A sampling of the protocols supported by the package.
+  expect("EvmFetcher" in sync).toBe(true);
+  expect("BitcoinFetcher" in sync).toBe(true);
+  expect("MidnightFetcher" in sync).toBe(true);
+  expect("AvailFetcher" in sync).toBe(true);
+  expect("UtxoRpcFetcher" in sync).toBe(true);
+  expect("CelestiaFetcher" in sync).toBe(true);
+});