@effectstream/sync 0.100.0

package/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "@effectstream/sync",
+  "version": "0.100.0",
+  "license": "MIT",
+  "exports": "./src/mod.ts",
+  "dependencies": {
+    "@effectstream/config": "0.100.0",
+    "@effectstream/db": "0.100.0",
+    "@effectstream/log": "0.100.0",
+    "@effectstream/utils": "0.100.0",
+    "@midnight-ntwrk/ledger-v8": "8.0.3",
+    "@midnight-ntwrk/midnight-js-indexer-public-data-provider": "4.0.2",
+    "@midnight-ntwrk/onchain-runtime": "npm:@midnight-ntwrk/onchain-runtime-v3@3.0.0",
+    "effection": "3.5.0",
+    "graphql-ws": "^6.0.6",
+    "ntp-time-sync": "^0.5.0",
+    "viem": "2.37.3",
+    "denque": "^2.1.0",
+    "pg": "^8.14.0",
+    "json-stable-stringify": "^1.2.1",
+    "@cardano-sdk/core": "^0.46.11",
+    "@cardano-sdk/util": "^0.17.1",
+    "@utxorpc/sdk": "^0.8.0",
+    "@utxorpc/spec": "^0.18.1",
+    "@sinclair/typebox": "0.34.41",
+    "avail-js-sdk": "^0.4.2",
+    "buffer": "^6.0.3"
+  }
+}

package/src/mod.ts

@@ -0,0 +1,3 @@
+export * from "./sync-protocols/mod.ts";
+export * from "./syncProtocolFactory.ts";
+export type { AllSyncProtocols } from "./sync-protocols/types.ts";

package/src/sync-protocols/avail/AvailClient.ts

@@ -0,0 +1,120 @@
+import { SDK } from "avail-js-sdk";
+import type {
+  AvailBlock,
+  AvailBlockDataItem,
+  AvailBlockHeader,
+  AvailStatus,
+} from "./types.ts";
+
+const AVAIL_CLIENT_DEFAULT_URL = "http://localhost:7007";
+const AVAIL_NODE_DEFAULT_URL = "ws://localhost:9955/ws";
+
+export class AvailClient {
+  private readonly url: string;
+  private readonly sdk: Promise<SDK>;
+  constructor(nodeUrl: string, lightClientUrl: string) {
+    this.url = lightClientUrl ?? AVAIL_CLIENT_DEFAULT_URL;
+    this.sdk = SDK.New(nodeUrl ?? AVAIL_NODE_DEFAULT_URL);
+  }
+
+  async getStatus(): Promise<AvailStatus> {
+    const response = await fetch(`${this.url}/v2/status`);
+    if (!response.ok) {
+      throw new Error(
+        `Failed to get status from light client, status: ${response.status}`,
+      );
+    }
+    const status: AvailStatus = await response.json();
+    if (!status) {
+      throw new Error(`Failed to get status from light client, no status found`);
+    }
+    return status;
+  }
+
+  async getLatestBlockHeight(): Promise<number> {
+    const api = (await this.sdk).client.api;
+    // Block from avail node is slightly above the light client
+    // const header = await api.rpc.chain.getHeader();
+    const status: AvailStatus = await this.getStatus();
+
+    // If the available field is present, this is the real latest block 
+    // synced by the light client for the app-id
+    // otherwise the app id is yet not available.
+    if (status.blocks.available) {
+      return status.blocks.available.last;
+    } else {
+      return status.blocks.latest - 1;
+    }
+  }
+
+  async getBlockFromHash(hash: string): Promise<AvailBlock> {
+    const block = await (await this.sdk).client.api.rpc.chain.getBlock(hash);
+    return {
+      header: block.block.header as unknown as AvailBlockHeader,
+      extrinsics: block.block.extrinsics as unknown as string[],
+    };
+  }
+
+  async getBlockHeaderFromHeight(height: number): Promise<AvailBlockHeader> {
+    const status = await this.getStatus();
+    if (!status.blocks.available || height < status.blocks.available.first) {
+      // This is before the app is deployed; return a dummy header
+      // At this point in time, the app is still not deployed
+      return { 
+        hash: "0x" + String(height) as `0x${string}`,
+        parent_hash: "0x0",
+        number: height,
+        state_root: "0x0",
+        extrinsics_root: "0x0",
+        extension: {
+          rows: 0,
+          cols: 0,
+          data_root: "0x0",
+          commitments: [],
+          app_lookup: {
+            size: 0,
+            index: [],
+          },
+        },
+        received_at: 0,
+      }
+    }
+
+    const response = await fetch(`${this.url}/v2/blocks/${height}/header`);
+    if (!response.ok) {
+      throw new Error(
+        `Failed to get block header from height ${height}, status: ${response.status}`,
+      );
+    }
+    const blockHeader = await response.json() as AvailBlockHeader;
+    if (!blockHeader) {
+      throw new Error(
+        `Failed to get block header from height ${height}, no header found`,
+      );
+    }
+    return blockHeader;
+  }
+
+  async getBlockDataFromHeight(height: number): Promise<AvailBlockDataItem> {
+    const status = await this.getStatus();
+    if (!status.blocks.available || height < status.blocks.available.first) {
+      // The app has no data or has not been deployed yet
+      // So we return a dummy empty data item
+      return { block_number: height, data_transactions: [] }
+    }
+
+    const response = await fetch(`${this.url}/v2/blocks/${height}/data`);
+    if (!response.ok) {
+      throw new Error(
+        `Failed to get block data from height ${height}, status: ${response.status}`,
+      );
+    }
+    const blockData = await response.json() as AvailBlockDataItem;
+    if (!blockData) {
+      throw new Error(
+        `Failed to get block data from height ${height}, no data found`,
+      );
+    }
+    return blockData;
+  }
+}

package/src/sync-protocols/avail/fetcher.ts

@@ -0,0 +1,174 @@
+import {
+  ConfigSyncProtocolType,
+  type PrimitiveEntry,
+  type SyncProtocolWithNetwork,
+} from "@effectstream/config";
+import { BaseDataFetcher } from "../base/fetcher.ts";
+import type { DataFetched } from "../base/fetcher.ts";
+import type {
+  LastPage,
+  OutputAndCleanup,
+  RootConversion,
+} from "../base/state.ts";
+import type { RootOutput, RootPage } from "../types.ts";
+import type {
+  ConfigType,
+  Input,
+  Output,
+  Page,
+  PrimitiveType,
+} from "./types.ts";
+import { AvailClient } from "./AvailClient.ts";
+import { all, call, type Operation, sleep } from "effection";
+import { bound } from "@effectstream/utils";
+
+export class AvailFetcher extends BaseDataFetcher<
+  Input,
+  Output,
+  RootOutput,
+  Page,
+  RootPage
+> {
+  readonly client: AvailClient;
+
+  constructor(
+    readonly config: ConfigType,
+  ) {
+    super(config.syncProtocol.name);
+    this.client = new AvailClient(
+      config.syncProtocol.rpc,
+      config.syncProtocol.lightClient,
+    );
+  }
+
+  @bound
+  override *readData(
+    data: Input,
+    rootConversion: RootConversion<Output, RootOutput, RootPage>,
+    lastPage: LastPage<Page, RootPage> | undefined,
+  ): Operation<DataFetched<Output, Page, RootPage>> {
+    const outputs: OutputAndCleanup<Output>[] = [];
+    console.log(
+      `[Avail] Fetching blocks from ${data.from} to ${data.to}. ${
+        data.isPresync ? "[presync]" : ""
+      }`,
+    );
+    for (let height = data.from; height <= data.to; height++) {
+      let header;
+      try {
+        header = yield* call(() =>
+          this.client.getBlockHeaderFromHeight(height)
+        );
+      } catch (_e) {
+        // Likely reached the tip; stop here
+        break;
+      }
+
+      const primitives = yield* this.readPrimitives(
+        height,
+        header,
+        this.config.primitives,
+      );
+
+      outputs.push({
+        output: {
+          raw: header,
+          primitives,
+        },
+        cleanup: () => {},
+      });
+    }
+
+    if (outputs.length === 0) {
+      if (!lastPage) {
+        throw new Error(
+          `Could not fetch any blocks from ${data.from} to ${data.to} and no previous page was found.`,
+        );
+      }
+      return {
+        output: [],
+        lastPage,
+      };
+    }
+
+    const lastOutput = outputs[outputs.length - 1].output;
+    return {
+      output: outputs,
+      lastPage: {
+        ownBlockNumber: lastOutput.raw.number,
+        own: {
+          height: lastOutput.raw.number,
+          hash: lastOutput.raw.hash,
+        },
+        root: rootConversion.toRootPage(lastOutput),
+      },
+    };
+  }
+
+  @bound
+  *readPrimitives(
+    height: number,
+    header: Output["raw"],
+    primitiveEntries: Extract<
+      PrimitiveEntry,
+      { syncProtocol: ConfigSyncProtocolType.AVAIL_PARALLEL }
+    >[],
+  ): Operation<PrimitiveType[]> {
+    const allOperations: Operation<PrimitiveType | undefined>[] = [];
+    for (const primitiveEntry of primitiveEntries) {
+      allOperations.push(
+        this.fetchData(height, primitiveEntry, header),
+      );
+    }
+    return (yield* all(allOperations)).flat().filter(
+      Boolean,
+    ) as PrimitiveType[];
+  }
+
+  @bound
+  *fetchData(
+    height: number,
+    primitiveEntry: Extract<
+      PrimitiveEntry,
+      { syncProtocol: ConfigSyncProtocolType.AVAIL_PARALLEL }
+    >,
+    header: Output["raw"],
+  ): Operation<PrimitiveType | undefined> {
+    // For now we only read blocks; if the appId appears in the block header
+    // we will invoke the data endpoint, but we will not emit a primitive yet.
+    const appId = primitiveEntry.primitive.appId;
+    if (typeof appId !== "number") return undefined;
+
+    const isPresent = header.extension.app_lookup.index.some((e) =>
+      e.appId === appId
+    );
+    if (!isPresent) return undefined;
+    // Read block data for the corresponding height to unblock future parsing.
+    // Intentionally ignore the result for now.
+    const blockData = yield* call(() =>
+      this.client.getBlockDataFromHeight(height)
+    );
+    // TODO: handle multiple data transactions
+    const dataTransaction = blockData.data_transactions.at(0);
+    if (!dataTransaction) return undefined;
+    // TODO: is always a JSON string?
+    const data: string = atob(dataTransaction.data);
+    return {
+      syncProtocol: {
+        name: primitiveEntry.syncProtocol,
+        blockNumber: height,
+        transactionHash: header.hash,
+        contractAddress: primitiveEntry.primitive.applicationKey,
+      },
+      primitive: primitiveEntry.primitive.name,
+      output: {
+        payloadType: "avail-app-state",
+        payload: {
+          inputData: height.toString(),
+          inputNonce: "0x", // TODO replace this with something meaningful
+          suppliedValue: data,
+        },
+      },
+    };
+  }
+}

package/src/sync-protocols/avail/state.ts

@@ -0,0 +1,134 @@
+import { call, type Operation } from "effection";
+import { bound, type TimestampMs } from "@effectstream/utils";
+import type { PoolClient } from "pg";
+import { type LastPage, SyncState } from "../base/state.ts";
+import type { RootOutput, RootPage } from "../types.ts";
+import type { Input, Output, Page } from "./types.ts";
+import { pageRelation } from "./types.ts";
+import type { AvailFetcher } from "./fetcher.ts";
+import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/config";
+import { getPage } from "@effectstream/db";
+import { AvailClient } from "./AvailClient.ts";
+import { applyDelay } from "../common/utils.ts";
+
+export class AvailSyncState extends SyncState<
+  Input,
+  Output,
+  Page,
+  RootOutput,
+  RootPage,
+  AvailFetcher
+> {
+  constructor(
+    lastPage: LastPage<Page, RootPage> | undefined,
+    readonly config: Extract<
+      SyncProtocolWithNetwork,
+      { networkType: ConfigNetworkType.AVAIL }
+    >,
+    fetcher: AvailFetcher,
+    public readonly client: AvailClient,
+    dbConn: PoolClient,
+  ) {
+    super(
+      config.syncProtocol.name,
+      lastPage,
+      fetcher,
+      pageRelation,
+      dbConn,
+    );
+  }
+
+  @bound
+  override toPage(input: Input, data: Output[]): Page {
+    const lastBlock = data[data.length - 1];
+    return {
+      height: lastBlock.raw.number,
+      hash: lastBlock.raw.hash,
+    };
+  }
+
+  @bound
+  override toRootPage(data: Output): RootPage {
+    // Avail headers include `received_at` (sec). Use it as the chain page.
+    return applyDelay(
+      data.raw.received_at * 1000,
+      this.config.syncProtocol.delayMs,
+    );
+  }
+
+  @bound
+  override toRootOutput(_data: Output): RootOutput {
+    throw new Error("Only main chains create root outputs");
+  }
+
+  @bound
+  override *stateToInput(): Operation<Input | undefined> {
+    const latestHeight = yield* call(() => this.client.getLatestBlockHeight());
+    const startHeight = this.lastPage?.own.height ??
+      this.config.syncProtocol.startBlockHeight - 1;
+
+    if (startHeight >= latestHeight) {
+      return undefined;
+    }
+
+    const from = startHeight + 1;
+    const to = Math.min(
+      from + this.config.syncProtocol.stepSize - 1,
+      latestHeight,
+    );
+
+    return {
+      from,
+      to,
+      isPresync: false,
+    };
+  }
+
+  @bound
+  override mergeDatum(ourOutput: Output, rootOutput: RootOutput): void {
+    const primitives = ourOutput.primitives.map((p) => ({
+      ...p,
+      source: this.config.syncProtocol.name,
+    }));
+    const blockInfo = [{
+      protocol_name: this.config.syncProtocol.name,
+      block_number: ourOutput.raw.number,
+      blockHash: ourOutput.raw.hash,
+    }];
+    rootOutput.blockInfo.push(...blockInfo);
+    rootOutput.primitives.push(...primitives);
+  }
+
+  @bound
+  override getNamespace(): string[] {
+    return [this.config.network.name, this.config.syncProtocol.name];
+  }
+
+  static *restoreState(
+    dbConn: PoolClient,
+    config: Extract<
+      SyncProtocolWithNetwork,
+      { networkType: ConfigNetworkType.AVAIL }
+    >,
+    fetcher: AvailFetcher,
+  ): Operation<AvailSyncState> {
+    const [result] = yield* call(async () =>
+      await getPage.run({
+        protocol_name: config.syncProtocol.name,
+      }, dbConn)
+    );
+    const page = result
+      ? result.page as unknown as LastPage<Page, RootPage>
+      : undefined;
+    return new AvailSyncState(
+      page,
+      config,
+      fetcher,
+      new AvailClient(
+        config.syncProtocol.rpc,
+        config.syncProtocol.lightClient,
+      ),
+      dbConn,
+    );
+  }
+}

package/src/sync-protocols/avail/types.ts

@@ -0,0 +1,92 @@
+import type {
+  ConfigSyncProtocolType,
+  FlattenSyncProtocolIOFor,
+  SyncProtocolWithNetwork,
+} from "@effectstream/config";
+import type { BlockNumber } from "@effectstream/utils";
+import type { PageSyncRange } from "../common/page-helpers.ts";
+import type { PageRelation } from "../base/page.ts";
+
+export type AvailStatus = {
+  modes: string[];
+  app_id: number;
+  genesis_hash: string;
+  network: string;
+  blocks: {
+    latest: number;
+    available?: {
+      first: number;
+      last: number;
+    };
+    app_data?: {
+      first: number;
+      last: number;
+    };
+  };
+}
+
+export type AvailBlockHeader = {
+  hash: `0x${string}`;
+  parent_hash: `0x${string}`;
+  number: number;
+  state_root: `0x${string}`;
+  extrinsics_root: `0x${string}`;
+  extension: {
+    rows: number;
+    cols: number;
+    data_root: `0x${string}`;
+    commitments: string[];
+    app_lookup: {
+      size: number;
+      index: {
+        appId: number;
+        start: number;
+      }[];
+    };
+  };
+  received_at: number;
+};
+
+export type AvailBlockDataItem = {
+  block_number: number;
+  data_transactions: {
+    data: string; // BASE64 encoded data
+    extrinsic: string;
+  }[];
+};
+
+export type AvailBlock = {
+  header: AvailBlockHeader;
+  extrinsics: string[];
+};
+
+// =====================
+// Sync protocol typings
+// =====================
+
+export type Page = {
+  height: BlockNumber;
+  hash: `0x${string}`;
+};
+
+export type PrimitiveType = FlattenSyncProtocolIOFor<
+  ConfigSyncProtocolType.AVAIL_PARALLEL
+>;
+
+export type Input = PageSyncRange<BlockNumber>;
+export type Output = {
+  raw: AvailBlockHeader;
+  primitives: PrimitiveType[];
+};
+
+export const pageRelation: PageRelation<Page> = {
+  compare: (p1, p2) => p1.height - p2.height,
+  equals: (p1, p2) => p1.height === p2.height,
+  min: (p1, p2) => (p1.height < p2.height ? p1 : p2),
+  max: (p1, p2) => (p1.height > p2.height ? p1 : p2),
+};
+
+export type ConfigType = Extract<
+  SyncProtocolWithNetwork,
+  { syncProtocolType: ConfigSyncProtocolType.AVAIL_PARALLEL }
+>;

package/src/sync-protocols/base/fetcher.ts

@@ -0,0 +1,43 @@
+import type { Channel, Operation } from "effection";
+import { createChannel } from "effection";
+import type { PageRange } from "./page.ts";
+import type { LastPage, OutputAndCleanup, RootConversion } from "./state.ts";
+
+export type DataFetched<Output, Page, RootPage> = {
+  output: OutputAndCleanup<Output>[];
+  lastPage: LastPage<Page, RootPage>;
+};
+
+export abstract class BaseDataFetcher<
+  Input,
+  Output,
+  RootOutput,
+  Page,
+  RootPage,
+> {
+  producerChannel: Channel<Output, void>;
+
+  constructor(
+    readonly name: string,
+  ) {
+    this.producerChannel = createChannel<Output>();
+  }
+
+  abstract readData(
+    data: Input,
+    rootConversion: RootConversion<Output, RootOutput, RootPage>,
+    // TODO: Search for a better strategy so it's not needed. For now its just for mindnight
+    lastPage: LastPage<Page, RootPage> | undefined,
+  ): Operation<DataFetched<Output, Page, RootPage>>;
+}
+
+/**
+ * Useful interface when pagination is done in fixed predictable chunks
+ */
+export interface PaginatedFetcher<Page> {
+  getLatestPage(knownLastPage: undefined | Page): Operation<Page>;
+
+  previousInterval(nextIntervalStart: Page): PageRange<Page>;
+  nextInterval(prevIntervalEnd: Page): PageRange<Page>;
+  intervalFromStart(start: Page): PageRange<Page>;
+}

package/src/sync-protocols/base/page.ts

@@ -0,0 +1,89 @@
+import type { Operation } from "effection";
+import { all, lift, sleep } from "effection";
+import { retry } from "@effectstream/utils";
+import type { IntervalMs, Json } from "@effectstream/utils";
+import stableStringify from "json-stable-stringify";
+
+export type PageRange<Page> = {
+  from: Page;
+  to: Page;
+};
+export interface PageRelation<T> {
+  /**
+   * -1 → p1 < p2 \
+   *  0 → p1 = p2 \
+   * +1 → p1 > p2
+   */
+  compare: (p1: T, p2: T) => number;
+  equals: (p1: T, p2: T) => boolean;
+  min: (p1: T, p2: T) => T;
+  max: (p1: T, p2: T) => T;
+}
+
+// ===========================
+//   Page request generation
+// (shared between primitives)
+// ===========================
+
+export type PageRequest<Page, Result> = (page: Page) => Promise<Result>;
+
+export function genImmediatePageRequests<
+  Page extends string | number,
+  Result,
+>(
+  pages: Page[],
+  fetchPage: (page: Page) => Promise<Result>,
+): PageRequest<Page, Result> {
+  const fetchedPages = Promise.all(pages.map((page) => fetchPage(page)));
+  const result = {} as Record<Page, Promise<Result>>;
+  for (let i = 0; i < pages.length; i++) {
+    result[pages[i]] = fetchedPages.then((fetched) => fetched[i]);
+  }
+  return (page: Page) => result[page];
+}
+export function genOnDemandPageRequests<
+  Page extends Json,
+  Result,
+>(
+  fromPage: Page,
+  toPage: Page,
+  fetchPage: (page: Page) => Promise<Result>,
+  pageRelation: PageRelation<Page>,
+): PageRequest<Page, Result> {
+  const promises = {} as Record<string, Promise<Result>>;
+  const pageRequest = async (
+    page: Page,
+  ): Promise<Result> => {
+    if (
+      pageRelation.compare(page, fromPage) < 0 ||
+      pageRelation.compare(page, toPage) > 0
+    ) {
+      throw new Error(
+        `Block out of range. ${page} not in [${fromPage}, ${toPage}]`,
+      );
+    }
+    const jsonKey: string = stableStringify(page)!;
+    if (promises[jsonKey] != null) {
+      return await promises[jsonKey];
+    }
+    promises[jsonKey] = fetchPage(page);
+    return await promises[jsonKey];
+  };
+  return pageRequest;
+}
+
+export function* fetchNewestPage<Page>(
+  pageRelation: PageRelation<Page>,
+  getLatestPage: () => Operation<Page>,
+  knownLastPage: undefined | Page,
+  pollingInterval: IntervalMs,
+): Operation<Page> {
+  return yield* retry<Page>(
+    getLatestPage,
+    (res) =>
+      knownLastPage == null
+        ? true
+        : pageRelation.compare(res, knownLastPage) > 0,
+    () => sleep(pollingInterval),
+  );
+}