@effectstream/sync 0.100.0

package/src/sync-protocols/common/page-helpers.ts

@@ -0,0 +1,65 @@
+import type { Operation } from "effection";
+import type { PageRange } from "../base/page.ts";
+import { narrowResult } from "@effectstream/utils";
+import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
+import type { PaginatedSyncProtocols } from "../types.ts";
+import type { PageTypeOf } from "../base/state.ts";
+
+type ConfigSubset<Page> = {
+  name: string;
+  /**
+   * inclusive
+   */
+  startPage: Page;
+};
+
+export type PageSyncRange<Page> = PageRange<Page> & { isPresync: boolean };
+
+export function* genInputRange<SyncProtocol extends PaginatedSyncProtocols>(
+  state: SyncProtocol,
+  genesisPage: PageTypeOf<SyncProtocol>,
+  config: ConfigSubset<PageTypeOf<SyncProtocol>>,
+  namespace: string[],
+): Operation<undefined | PageSyncRange<PageTypeOf<SyncProtocol>>> {
+  const nextPage = state.lastPage == null
+    ? genesisPage
+    : state.fetcher.nextInterval(state.lastPage.own).from;
+  // TODO This will always be false as the genesis page is the startPage
+  const isPresync = state.pageRelation.compare(nextPage, config.startPage) < 0;
+
+  const maxPage = isPresync
+    ? state.fetcher.previousInterval(config.startPage).to
+    : yield* state.fetcher.getLatestPage(state.lastPage?.own);
+  const clampedMax = state.pageRelation.min(
+    maxPage,
+    state.fetcher.intervalFromStart(nextPage).to,
+  );
+  // this can happen if the genesis is not the exact start of the system
+  // and (esp. on local networks) the system hasn't reached the genesis page yet
+  if (state.pageRelation.compare(nextPage, clampedMax) > 0) {
+    return undefined;
+  }
+
+  const pageRange = state.pageRelation.equals(clampedMax, nextPage)
+    ? `[${nextPage}]`
+    : `[${nextPage}, ${clampedMax}]`;
+  log.remote(
+    ComponentNames.EFFECTSTREAM_SYNC,
+    namespace,
+    SeverityNumber.INFO,
+    (log) => log(pageRange),
+  );
+  return {
+    from: narrowResult<
+      PageTypeOf<PaginatedSyncProtocols>,
+      PageTypeOf<PaginatedSyncProtocols>,
+      PageTypeOf<SyncProtocol>
+    >(nextPage),
+    to: narrowResult<
+      PageTypeOf<PaginatedSyncProtocols>,
+      PageTypeOf<PaginatedSyncProtocols>,
+      PageTypeOf<SyncProtocol>
+    >(clampedMax),
+    isPresync,
+  };
+}

package/src/sync-protocols/common/root.ts

@@ -0,0 +1,35 @@
+import type {
+  BlockHash,
+  BlockNumber,
+  EffectstreamBlockNumber,
+  TimestampMs,
+} from "@effectstream/utils";
+import type { PageRelation } from "../base/page.ts";
+import type {
+  ConfigSyncProtocolType,
+  FlattenSyncProtocolIOFor,
+} from "@effectstream/config";
+
+export type ChainPage = TimestampMs;
+export type ChainBlock = {
+  blockNumber: EffectstreamBlockNumber;
+  timestamp: TimestampMs;
+  blockInfo: {
+    protocol_name: string;
+    block_number: BlockNumber;
+    blockHash: BlockHash;
+  }[];
+  primitives: (
+    & FlattenSyncProtocolIOFor<
+      ConfigSyncProtocolType
+    >
+    & { source: string }
+  )[];
+};
+
+export const chainPageRelation: PageRelation<ChainPage> = {
+  compare: (p1, p2) => p1 - p2,
+  equals: (p1, p2) => p1 === p2,
+  min: (p1, p2) => (p1 < p2 ? p1 : p2),
+  max: (p1, p2) => (p1 > p2 ? p1 : p2),
+};

package/src/sync-protocols/common/utils.ts

@@ -0,0 +1,29 @@
+import type {
+  AbsoluteSlotNumber,
+  BlockNumber,
+  IntervalMs,
+  TimestampMs,
+} from "@effectstream/utils";
+import type { PageRelation } from "../base/page.ts";
+
+export function applyDelay(
+  timestamp: TimestampMs,
+  delayMs: IntervalMs,
+): TimestampMs {
+  // note: plus ("+") here makes it look like this chain is further along than it is
+  //       this is exactly what we want to make old blocks get merged into recent root data
+  return Math.max(0, timestamp + delayMs);
+}
+
+export const blockNumberRelation: PageRelation<BlockNumber> = {
+  compare: (p1, p2) => p1 - p2,
+  equals: (p1, p2) => p1 === p2,
+  min: (p1, p2) => (p1 < p2 ? p1 : p2),
+  max: (p1, p2) => (p1 > p2 ? p1 : p2),
+};
+export const absSlotRelation: PageRelation<AbsoluteSlotNumber> = {
+  compare: (p1, p2) => p1 - p2,
+  equals: (p1, p2) => p1 === p2,
+  min: (p1, p2) => (p1 < p2 ? p1 : p2),
+  max: (p1, p2) => (p1 > p2 ? p1 : p2),
+};

package/src/sync-protocols/evm/fetcher.ts

@@ -0,0 +1,290 @@
+import {
+  BaseDataFetcher,
+  type DataFetched,
+  type PaginatedFetcher,
+} from "../base/fetcher.ts";
+import type { RootOutput, RootPage } from "../types.ts";
+import type { AbiEvent, Chain, GetBlockReturnType, PublicClient } from "viem";
+import type { Operation } from "effection";
+import { all, call } from "effection";
+import {
+  bound,
+  type EvmBlockHash,
+  type EvmRpcPageJson,
+  keysOf,
+} from "@effectstream/utils";
+import { blockNumberRelation } from "../common/utils.ts";
+import type {
+  ConfigType,
+  Input,
+  Output,
+  Page,
+  PrimitiveEntryType,
+  PrimitiveType,
+} from "./types.ts";
+import { PageSchema } from "./types.ts";
+import {
+  fetchNewestPage,
+  genImmediatePageRequests,
+  type PageRange,
+  type PageRequest,
+} from "../base/page.ts";
+import type { PrimitiveFetcher } from "../base/primitive.ts";
+import type { RootConversion } from "../base/state.ts";
+import type {
+  ConfigNetworkType,
+  PrimitiveEntry,
+  SyncProtocolWithNetwork,
+} from "@effectstream/config";
+import type { ConfigSyncProtocolType } from "@effectstream/config";
+import { Value } from "@sinclair/typebox/value";
+
+export class EvmFetcher
+  extends BaseDataFetcher<Input, Output, RootOutput, Page, RootPage>
+  implements
+    PrimitiveFetcher<
+      Input,
+      Page,
+      GetBlockReturnType<Chain>,
+      PrimitiveType,
+      ConfigSyncProtocolType.EVM_RPC_PARALLEL
+    >,
+    PaginatedFetcher<Page> {
+  constructor(
+    readonly config: ConfigType,
+    readonly client: PublicClient<any, Chain, any, any>,
+  ) {
+    super(config.syncProtocol.name);
+  }
+
+  @bound
+  override *readData(
+    data: Input,
+    rootConversion: RootConversion<Output, RootOutput, RootPage>,
+  ): Operation<DataFetched<Output, Page, RootPage>> {
+    const allPages: Page[] = [];
+    for (let p = data.from; p <= data.to; p++) allPages.push(p);
+    const pageFetcher = genImmediatePageRequests(
+      allPages,
+      (page) => this.client.getBlock({ blockNumber: BigInt(page) }),
+    );
+
+    // TODO: if we expect multiple primitives per block
+    //       it can, depending on the chain, be faster to just download the full block and parse it locally
+    const primitives = yield* this.readPrimitives(
+      data,
+      pageFetcher,
+      this.config.primitives,
+    );
+    const groupedByPage = this.groupByPage(primitives);
+
+    const [output, lastPage] = yield* all(
+      [
+        all(
+          keysOf(groupedByPage).map(function* (pageJson) {
+            const page = Value.Encode(PageSchema, pageJson);
+            const raw = yield* call(() => pageFetcher(page));
+            const blockHashes = [raw.hash] as EvmBlockHash[];
+            return {
+              raw,
+              primitives: groupedByPage[pageJson],
+              blockHashes,
+            };
+          }),
+        ),
+        // we always need to get the last page in the parallel case
+        // so we can use it for pagination
+        call(() => pageFetcher(Number(data.to))),
+      ],
+    );
+
+    // Build an output with all page info, as we need all the hashes to build the effectstream-block-hash
+    // All blocks are already pre-fetched in parallel by genImmediatePageRequests above
+    const allOutputs: Output[] = [];
+    for (let page = data.from; page <= data.to; page++) {
+      const _output = output.find((o) => o.raw.number === BigInt(page));
+      if (_output) {
+        allOutputs.push(_output);
+      } else {
+        const raw = yield* call(() => pageFetcher(page));
+        allOutputs.push({
+          raw,
+          primitives: [],
+          blockHashes: [raw.hash] as EvmBlockHash[],
+        });
+      }
+    }
+
+    return {
+      output: allOutputs.map((o) => ({
+        output: o,
+        cleanup: () => {}, // no cleanup required
+      })),
+      lastPage: {
+        own: Number(data.to),
+        ownBlockNumber: Number(data.to),
+        root: rootConversion.toRootPage({
+          blockHashes: [lastPage.hash] as EvmBlockHash[],
+          primitives: [], // unused in toRootPage
+          raw: lastPage,
+        }),
+      },
+    };
+  }
+
+  @bound
+  groupByPage(
+    primitives: PrimitiveType[],
+  ): Record<EvmRpcPageJson, PrimitiveType[]> {
+    return primitives.reduce((acc, primitive) => {
+      const key = Value.Decode(
+        PageSchema,
+        Number(primitive.syncProtocol.blockNumber),
+      );
+      (acc[key] ??= []).push(
+        primitive,
+      );
+      return acc;
+    }, {} as Record<EvmRpcPageJson, PrimitiveType[]>);
+  }
+
+  fetchLogsAndExtractPrimitiveData = function* (
+    fromBlock: bigint,
+    toBlock: bigint,
+    client: PublicClient<any, Chain, any, any>,
+    primitiveEntry: PrimitiveEntryType,
+    abi: AbiEvent,
+    pageRequest: PageRequest<Page, GetBlockReturnType<Chain>>,
+  ): Operation<
+    PrimitiveType[]
+  > {
+    // Fetch range of event-logs
+    const logs = yield* call(() =>
+      client.getLogs({
+        address: primitiveEntry.primitive.contractAddress as `0x${string}`,
+        event: abi,
+        fromBlock,
+        toBlock,
+        strict: true,
+      })
+    );
+
+    // Create PrimitiveType for each event.
+    const primitiveResponses: PrimitiveType[] = [];
+    for (const log of logs) {
+      // const block = yield* call(() => pageRequest(Number(log.blockNumber)));
+      const args = (log as unknown as {
+        args: Record<string, any>;
+      }).args;
+      const primitiveResponse: PrimitiveType = {
+        syncProtocol: {
+          name: primitiveEntry.syncProtocol,
+          blockNumber: Number(log.blockNumber),
+          transactionHash: log.transactionHash,
+          transactionIndex: log.transactionIndex,
+          contractAddress: log.address,
+          logIndex: log.logIndex,
+        },
+        primitive: primitiveEntry.primitive.name,
+        output: {
+          payloadType: abi.name,
+          payload: args,
+        },
+      };
+
+      primitiveResponses.push(primitiveResponse);
+    }
+
+    return primitiveResponses;
+  };
+
+  @bound
+  *readPrimitives(
+    data: Input,
+    pageRequest: PageRequest<Page, GetBlockReturnType<Chain>>,
+    primitiveEntries: Extract<
+      PrimitiveEntry,
+      { syncProtocol: ConfigSyncProtocolType.EVM_RPC_PARALLEL }
+    >[],
+  ): Operation<PrimitiveType[]> {
+    const client = this.client;
+    const allOperations: Operation<PrimitiveType[]>[] = [];
+
+    const fromBlock = yield* call(() => pageRequest(data.from));
+    const toBlock = yield* call(() => pageRequest(data.to));
+    for (const primitive of primitiveEntries) {
+      if (Array.isArray(primitive.primitive.abi)) {
+        const abis = primitive.primitive.abi as AbiEvent[];
+        for (const abi of abis) {
+          allOperations.push(
+            this.fetchLogsAndExtractPrimitiveData(
+              fromBlock.number,
+              toBlock.number,
+              client,
+              primitive,
+              abi,
+              pageRequest,
+            ),
+          );
+        }
+      } else {
+        allOperations.push(
+          this.fetchLogsAndExtractPrimitiveData(
+            fromBlock.number,
+            toBlock.number,
+            client,
+            primitive,
+            primitive.primitive.abi as AbiEvent,
+            pageRequest,
+          ),
+        );
+      }
+    }
+
+    return (yield* all(allOperations)).flat();
+  }
+
+  @bound
+  *getLatestPage(
+    knownLastPage: undefined | Page,
+  ): Operation<Page> {
+    return yield* fetchNewestPage<Page>(
+      blockNumberRelation,
+      // override Viem's cache logic with out own
+      () =>
+        call(() =>
+          this.client.getBlockNumber({ cacheTime: 0 }).then(Number).then(
+            (height) => {
+              const config = this.config.syncProtocol;
+              if ("confirmationDepth" in config) {
+                return height - config.confirmationDepth;
+              }
+              return height;
+            },
+          )
+        ),
+      knownLastPage,
+      this.config.syncProtocol.pollingInterval,
+    );
+  }
+
+  @bound
+  previousInterval(nextIntervalStart: Page): PageRange<Page> {
+    return this.intervalFromStart(
+      nextIntervalStart - this.config.syncProtocol.stepSize - 1,
+    );
+  }
+
+  @bound
+  nextInterval(prevIntervalEnd: Page): PageRange<Page> {
+    return this.intervalFromStart(prevIntervalEnd + 1);
+  }
+
+  @bound
+  intervalFromStart(start: Page): PageRange<Page> {
+    return {
+      from: start,
+      to: start + this.config.syncProtocol.stepSize,
+    };
+  }
+}

package/src/sync-protocols/evm/state.ts

@@ -0,0 +1,122 @@
+import { applyDelay, blockNumberRelation } from "../common/utils.ts";
+import type { Operation } from "effection";
+import { call } from "effection";
+import { getPage } from "@effectstream/db";
+import type { PoolClient } from "pg";
+import { bound, type EvmBlockNumber } from "@effectstream/utils";
+import { type LastPage, SyncState } from "../base/state.ts";
+import type { RootOutput, RootPage } from "../types.ts";
+import type { EvmFetcher } from "./fetcher.ts";
+import { genInputRange } from "../common/page-helpers.ts";
+import type { Input, Output, Page } from "./types.ts";
+import { toMsTimestamp } from "./types.ts";
+import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/config";
+
+export class EvmSyncState extends SyncState<
+  Input,
+  Output,
+  Page,
+  RootOutput,
+  RootPage,
+  EvmFetcher
+> {
+  constructor(
+    lastPage: LastPage<Page, RootPage>,
+    readonly config: Extract<
+      SyncProtocolWithNetwork,
+      { networkType: ConfigNetworkType.EVM }
+    >,
+    fetcher: EvmFetcher,
+    dbConn: PoolClient,
+  ) {
+    super(
+      config.syncProtocol.name,
+      lastPage,
+      fetcher,
+      blockNumberRelation,
+      dbConn,
+    );
+  }
+
+  @bound
+  override toRootOutput(data: Output): RootOutput {
+    throw new Error("Only main chains create root outputs");
+  }
+
+  @bound
+  override toRootPage(data: Output): RootPage {
+    return applyDelay(
+      toMsTimestamp(data.raw.timestamp),
+      this.config.syncProtocol.delayMs,
+    );
+  }
+
+  @bound
+  override *stateToInput(): Operation<Input | undefined> {
+    return yield* genInputRange(
+      this as EvmSyncState,
+      this.config.syncProtocol.startBlockHeight as EvmBlockNumber,
+      {
+        name: this.config.syncProtocol.name,
+        startPage: this.config.syncProtocol.startBlockHeight as EvmBlockNumber,
+      },
+      this.getNamespace(),
+    );
+  }
+
+  @bound
+  override toPage(
+    input: Input,
+    data: Output[],
+  ): Page {
+    return Number(input.to);
+  }
+
+  @bound
+  override mergeDatum(
+    ourOutput: Output,
+    rootOutput: RootOutput,
+  ): void {
+    const primitives = ourOutput.primitives.map((p) => ({
+      ...p,
+      source: this.config.syncProtocol.name,
+    }));
+    const blockInfo = ourOutput.blockHashes.map((h) => ({
+      protocol_name: this.config.syncProtocol.name,
+      block_number: Number(ourOutput.raw.number),
+      blockHash: h,
+    }));
+    rootOutput.primitives.push(...primitives);
+    rootOutput.blockInfo.push(...blockInfo);
+  }
+
+  @bound
+  override getNamespace(): string[] {
+    return [this.config.network.name, this.config.syncProtocol.name];
+  }
+
+  static *restoreState(
+    dbConn: PoolClient,
+    config: Extract<
+      SyncProtocolWithNetwork,
+      { networkType: ConfigNetworkType.EVM }
+    >,
+    fetcher: EvmFetcher,
+  ): Operation<EvmSyncState> {
+    // TODO: move this DB query into page-helpers?
+    const [result] = yield* call(async () =>
+      await getPage.run({
+        protocol_name: config.syncProtocol.name,
+      }, dbConn)
+    );
+    const page: any = result
+      ? result.page as unknown as LastPage<Page, RootPage>
+      : undefined;
+    return new EvmSyncState(
+      page,
+      config,
+      fetcher,
+      dbConn,
+    );
+  }
+}

package/src/sync-protocols/evm/types.ts

@@ -0,0 +1,53 @@
+import type {
+  BlockNumber,
+  EvmBlockHash,
+  EvmRpcPageJson,
+  TimestampMs,
+} from "@effectstream/utils";
+import { TypeboxHelpers } from "@effectstream/utils";
+import type { Chain, GetBlockReturnType } from "viem";
+import type { PageSyncRange } from "../common/page-helpers.ts";
+import { Type } from "@sinclair/typebox";
+import type {
+  ConfigNetworkType,
+  ConfigSyncProtocolType,
+  FlattenSyncProtocolIOFor,
+  PrimitiveEntry,
+  SyncProtocolWithNetwork,
+} from "@effectstream/config";
+
+export type Page = BlockNumber;
+const PageJsonSchema = Type.Unsafe<
+  EvmRpcPageJson
+>(Type.String());
+export const PageSchema = TypeboxHelpers.SerializeObjAsJson<
+  Page,
+  typeof PageJsonSchema
+>();
+
+export type ConfigType = Extract<
+  SyncProtocolWithNetwork,
+  { networkType: ConfigNetworkType.EVM }
+>;
+
+export type PrimitiveEntryType = Extract<
+  PrimitiveEntry,
+  { syncProtocol: ConfigSyncProtocolType.EVM_RPC_PARALLEL }
+>;
+
+export type PrimitiveType = FlattenSyncProtocolIOFor<
+  ConfigSyncProtocolType.EVM_RPC_PARALLEL
+>;
+export type Input = PageSyncRange<Page>;
+export type Output = {
+  raw: GetBlockReturnType<Chain>;
+  primitives: PrimitiveType[];
+  blockHashes: EvmBlockHash[];
+};
+
+/**
+ * recall: EVM blocks use second resolution, but we want ms resolution
+ */
+export function toMsTimestamp(timestamp: bigint): TimestampMs {
+  return Number(timestamp) * 1000;
+}