@effectstream/sync 0.100.0

package/src/sync-protocols/bitcoin/state.ts

@@ -0,0 +1,144 @@
+import { applyDelay, blockNumberRelation } from "../common/utils.ts";
+import type { Operation } from "effection";
+import { call } from "effection";
+import { getPage } from "@effectstream/db";
+import type { PoolClient } from "pg";
+import { bound } from "@effectstream/utils";
+import {
+  type LastPage,
+  SyncState,
+} from "../base/state.ts";
+import type { RootOutput, RootPage } from "../types.ts";
+import type { BitcoinFetcher } from "./fetcher.ts";
+import {
+  toMsTimestamp,
+  type Input,
+  type Output,
+  type Page,
+} from "./types.ts";
+import { genInputRange } from "../common/page-helpers.ts";
+import type {
+  ConfigNetworkType,
+  SyncProtocolWithNetwork,
+} from "@effectstream/config";
+
+export class BitcoinSyncState extends SyncState<
+  Input,
+  Output,
+  Page,
+  RootOutput,
+  RootPage,
+  BitcoinFetcher
+> {
+  constructor(
+    lastPage: LastPage<Page, RootPage> | undefined,
+    readonly config: Extract<
+      SyncProtocolWithNetwork,
+      { networkType: ConfigNetworkType.BITCOIN }
+    >,
+    fetcher: BitcoinFetcher,
+    dbConn: PoolClient,
+  ) {
+    super(
+      config.syncProtocol.name,
+      lastPage,
+      fetcher,
+      blockNumberRelation,
+      dbConn,
+    );
+  }
+
+  @bound
+  override toRootOutput(data: Output): RootOutput {
+    return {
+      blockNumber: Number(data.raw.height),
+      timestamp: this.toRootPage(data),
+      blockInfo: data.blockHashes.map((hash) => ({
+        protocol_name: this.config.syncProtocol.name,
+        block_number: Number(data.raw.height),
+        blockHash: hash,
+      })),
+      primitives: data.primitives.map((primitive) => ({
+        ...primitive,
+        source: this.config.syncProtocol.name,
+      })),
+    };
+  }
+
+  @bound
+  override toRootPage(data: Output): RootPage {
+    return applyDelay(
+      toMsTimestamp(data.raw.time),
+      this.config.syncProtocol.delayMs ?? 0,
+    );
+  }
+
+  @bound
+  override *stateToInput(): Operation<Input | undefined> {
+    return yield* genInputRange(
+      this as BitcoinSyncState,
+      this.config.syncProtocol.startBlockHeight as Page,
+      {
+        name: this.config.syncProtocol.name,
+        startPage: this.config.syncProtocol.startBlockHeight as Page,
+      },
+      this.getNamespace(),
+    );
+  }
+
+  @bound
+  override toPage(
+    input: Input,
+    _data: Output[],
+  ): Page {
+    return input.to;
+  }
+
+  @bound
+  override mergeDatum(
+    ourOutput: Output,
+    rootOutput: RootOutput,
+  ): void {
+    const primitives = ourOutput.primitives.map((primitive) => ({
+      ...primitive,
+      source: this.config.syncProtocol.name,
+    }));
+    const blockInfo = ourOutput.blockHashes.map((hash) => ({
+      protocol_name: this.config.syncProtocol.name,
+      block_number: Number(ourOutput.raw.height),
+      blockHash: hash,
+    }));
+    rootOutput.primitives.push(...primitives);
+    rootOutput.blockInfo.push(...blockInfo);
+  }
+
+  @bound
+  override getNamespace(): string[] {
+    return [this.config.network.name, this.config.syncProtocol.name];
+  }
+
+  static *restoreState(
+    dbConn: PoolClient,
+    config: Extract<
+      SyncProtocolWithNetwork,
+      { networkType: ConfigNetworkType.BITCOIN }
+    >,
+    fetcher: BitcoinFetcher,
+  ): Operation<BitcoinSyncState> {
+    const [result] = yield* call(async () =>
+      await getPage.run({
+        protocol_name: config.syncProtocol.name,
+      }, dbConn)
+    );
+    const page = result
+      ? result.page as unknown as LastPage<Page, RootPage>
+      : undefined;
+    return new BitcoinSyncState(
+      page,
+      config,
+      fetcher,
+      dbConn,
+    );
+  }
+}
+

package/src/sync-protocols/bitcoin/types.ts

@@ -0,0 +1,92 @@
+import type {
+  BitcoinBlockHash,
+  BitcoinTxId,
+  BlockNumber,
+  TimestampMs,
+} from "@effectstream/utils";
+import type { PageSyncRange } from "../common/page-helpers.ts";
+import type {
+  ConfigNetworkType,
+  ConfigSyncProtocolType,
+  FlattenSyncProtocolIOFor,
+  PrimitiveEntry,
+  SyncProtocolWithNetwork,
+} from "@effectstream/config";
+
+export type Page = BlockNumber;
+
+export type ConfigType = Extract<
+  SyncProtocolWithNetwork,
+  { networkType: ConfigNetworkType.BITCOIN }
+>;
+
+export type PrimitiveEntryType = Extract<
+  PrimitiveEntry,
+  { syncProtocol: ConfigSyncProtocolType.BITCOIN_RPC_PARALLEL }
+>;
+
+export type PrimitiveType = FlattenSyncProtocolIOFor<
+  ConfigSyncProtocolType.BITCOIN_RPC_PARALLEL
+>;
+
+export type Input = PageSyncRange<Page>;
+
+export type BitcoinScriptPubKey = {
+  asm: string;
+  hex: string;
+  type?: string;
+  address?: string;
+  addresses?: string[];
+};
+
+export type BitcoinVout = {
+  value: number;
+  n: number;
+  scriptPubKey: BitcoinScriptPubKey;
+};
+
+export type BitcoinVin = {
+  txid?: BitcoinTxId;
+  vout?: number;
+  coinbase?: string;
+  sequence: number;
+};
+
+export type BitcoinTransaction = {
+  txid: BitcoinTxId;
+  hash: BitcoinTxId;
+  version: number;
+  size: number;
+  vsize: number;
+  weight: number;
+  locktime: number;
+  vin: BitcoinVin[];
+  vout: BitcoinVout[];
+};
+
+export type BitcoinBlock = {
+  hash: BitcoinBlockHash;
+  confirmations: number;
+  size: number;
+  height: Page;
+  version: number;
+  time: number;
+  mediantime: number;
+  nonce: number;
+  bits: string;
+  difficulty: number;
+  previousblockhash?: BitcoinBlockHash;
+  nextblockhash?: BitcoinBlockHash;
+  tx: BitcoinTransaction[];
+};
+
+export type Output = {
+  raw: BitcoinBlock;
+  primitives: PrimitiveType[];
+  blockHashes: BitcoinBlockHash[];
+};
+
+export function toMsTimestamp(timestamp: number): TimestampMs {
+  return timestamp * 1000 as TimestampMs;
+}
+

package/src/sync-protocols/celestia/CelestiaClient.ts

@@ -0,0 +1,139 @@
+export type CelestiaBlob = {
+  /** Base64-encoded 29-byte namespace */
+  namespace: string;
+  /** Base64-encoded blob data */
+  data: string;
+  share_version: number;
+  /** Base64-encoded blob commitment */
+  commitment: string;
+  signer: string;
+  index: number;
+};
+
+export type CelestiaHeader = {
+  version: { block: string; app: string };
+  chain_id: string;
+  /** String-encoded block height */
+  height: string;
+  /** ISO 8601 timestamp */
+  time: string;
+  last_block_id: {
+    hash: string;
+    parts: { total: number; hash: string };
+  };
+  last_commit_hash: string;
+  data_hash: string;
+  validators_hash: string;
+  next_validators_hash: string;
+  consensus_hash: string;
+  app_hash: string;
+  last_results_hash: string;
+  evidence_hash: string;
+  proposer_address: string;
+};
+
+export type CelestiaCommit = {
+  height: string;
+  block_id: {
+    hash: string;
+    parts: { total: number; hash: string };
+  };
+};
+
+export type CelestiaExtendedHeader = {
+  header: CelestiaHeader;
+  commit: CelestiaCommit;
+  validator_set: unknown;
+  dah: unknown;
+};
+
+import { ENV } from "@effectstream/utils/node-env";
+
+const CELESTIA_NODE_URL = ENV.getString(
+  "CELESTIA_NODE_URL",
+  "http://localhost:26658",
+);
+
+/**
+ * Converts a hex-encoded Celestia namespace to a base64-encoded 29-byte array.
+ * Format: 1 byte version (0x00) + 28 bytes namespace ID, right-aligned.
+ */
+export function celestiaNamespaceToBase64(hex: string): string {
+  const cleanHex = hex.replace(/^0x/, "").padStart(56, "0"); // 28 bytes = 56 hex chars
+  const bytes = new Uint8Array(29); // version byte (0) + 28-byte ID
+  for (let i = 0; i < 28; i++) {
+    bytes[i + 1] = parseInt(cleanHex.slice(i * 2, i * 2 + 2), 16);
+  }
+  let binary = "";
+  for (const byte of bytes) {
+    binary += String.fromCharCode(byte);
+  }
+  return btoa(binary);
+}
+
+export class CelestiaClient {
+  private readonly rpcUrl: string;
+
+  constructor(rpcUrl: string = CELESTIA_NODE_URL) {
+    this.rpcUrl = rpcUrl;
+  }
+
+  private async rpc<T>(method: string, params: unknown[]): Promise<T | null> {
+    let res: Response;
+    try {
+      res = await fetch(this.rpcUrl, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ jsonrpc: "2.0", id: 1, method, params }),
+      });
+    } catch (err: unknown) {
+      const msg = err instanceof Error ? err.message : String(err);
+      console.error(`[Celestia] RPC fetch error [${method}]: ${msg}`);
+      return null;
+    }
+
+    const json = await res.json();
+    if (json.error) {
+      const errMsg: string = json.error.message ?? String(json.error);
+      if (errMsg.includes("blob: not found")) return null;
+      console.error(`[Celestia] RPC error [${method}]:`, errMsg);
+      return null;
+    }
+    return json.result as T;
+  }
+
+  async getLatestBlockHeight(): Promise<number> {
+    const head = await this.rpc<CelestiaExtendedHeader>(
+      "header.LocalHead",
+      [],
+    );
+    if (!head) {
+      throw new Error("[Celestia] Could not retrieve latest block header");
+    }
+    return parseInt(head.header.height, 10);
+  }
+
+  async getHeaderAtHeight(height: number): Promise<CelestiaExtendedHeader> {
+    const header = await this.rpc<CelestiaExtendedHeader>(
+      "header.GetByHeight",
+      [height],
+    );
+    if (!header) {
+      throw new Error(
+        `[Celestia] Could not retrieve header at height ${height}`,
+      );
+    }
+    return header;
+  }
+
+  async getBlobsAtHeight(
+    height: number,
+    namespaceB64: string,
+  ): Promise<CelestiaBlob[]> {
+    const blobs = await this.rpc<CelestiaBlob[]>("blob.GetAll", [
+      height,
+      [namespaceB64],
+    ]);
+    return blobs ?? [];
+  }
+}

package/src/sync-protocols/celestia/fetcher.ts

@@ -0,0 +1,165 @@
+import {
+  ConfigSyncProtocolType,
+  type PrimitiveEntry,
+} from "@effectstream/config";
+import { BaseDataFetcher } from "../base/fetcher.ts";
+import type { DataFetched } from "../base/fetcher.ts";
+import type {
+  LastPage,
+  OutputAndCleanup,
+  RootConversion,
+} from "../base/state.ts";
+import type { RootOutput, RootPage } from "../types.ts";
+import type {
+  ConfigType,
+  Input,
+  Output,
+  Page,
+  PrimitiveType,
+} from "./types.ts";
+import {
+  CelestiaClient,
+  celestiaNamespaceToBase64,
+} from "./CelestiaClient.ts";
+import { all, call, type Operation } from "effection";
+import { bound } from "@effectstream/utils";
+
+export class CelestiaFetcher extends BaseDataFetcher<
+  Input,
+  Output,
+  RootOutput,
+  Page,
+  RootPage
+> {
+  readonly client: CelestiaClient;
+
+  constructor(
+    readonly config: ConfigType,
+  ) {
+    super(config.syncProtocol.name);
+    this.client = new CelestiaClient(config.network.rpcUrl);
+  }
+
+  @bound
+  override *readData(
+    data: Input,
+    rootConversion: RootConversion<Output, RootOutput, RootPage>,
+    lastPage: LastPage<Page, RootPage> | undefined,
+  ): Operation<DataFetched<Output, Page, RootPage>> {
+    const outputs: OutputAndCleanup<Output>[] = [];
+
+    console.log(
+      `[Celestia] Fetching blocks from ${data.from} to ${data.to}.${
+        data.isPresync ? " [presync]" : ""
+      }`,
+    );
+
+    for (let height = data.from; height <= data.to; height++) {
+      let extHeader;
+      try {
+        extHeader = yield* call(() =>
+          this.client.getHeaderAtHeight(height)
+        );
+      } catch (_e) {
+        // Likely reached the chain tip; stop here
+        break;
+      }
+
+      const blockHash = extHeader.commit.block_id.hash;
+
+      const primitives = yield* this.readPrimitives(
+        height,
+        blockHash,
+        this.config.primitives,
+      );
+
+      outputs.push({
+        output: {
+          timestamp: extHeader.header.time,
+          height,
+          hash: blockHash,
+          primitives,
+        },
+        cleanup: () => {},
+      });
+    }
+
+    if (outputs.length === 0) {
+      if (!lastPage) {
+        throw new Error(
+          `[Celestia] Could not fetch any blocks from ${data.from} to ${data.to} and no previous page was found.`,
+        );
+      }
+      return {
+        output: [],
+        lastPage,
+      };
+    }
+
+    const lastOutput = outputs[outputs.length - 1].output;
+    return {
+      output: outputs,
+      lastPage: {
+        ownBlockNumber: lastOutput.height,
+        own: {
+          height: lastOutput.height,
+          hash: lastOutput.hash,
+        },
+        root: rootConversion.toRootPage(lastOutput),
+      },
+    };
+  }
+
+  @bound
+  *readPrimitives(
+    height: number,
+    blockHash: string,
+    primitiveEntries: Extract<
+      PrimitiveEntry,
+      { syncProtocol: ConfigSyncProtocolType.CELESTIA_PARALLEL }
+    >[],
+  ): Operation<PrimitiveType[]> {
+    const allOperations: Operation<PrimitiveType[]>[] = primitiveEntries.map(
+      (entry) => this.fetchBlobsForPrimitive(height, blockHash, entry),
+    );
+    const results = yield* all(allOperations);
+    return results.flat();
+  }
+
+  @bound
+  *fetchBlobsForPrimitive(
+    height: number,
+    blockHash: string,
+    primitiveEntry: Extract<
+      PrimitiveEntry,
+      { syncProtocol: ConfigSyncProtocolType.CELESTIA_PARALLEL }
+    >,
+  ): Operation<PrimitiveType[]> {
+    const namespace = primitiveEntry.primitive.namespace;
+    const namespaceB64 = celestiaNamespaceToBase64(namespace);
+
+    const blobs = yield* call(() =>
+      this.client.getBlobsAtHeight(height, namespaceB64)
+    );
+
+    return blobs.map((blob): PrimitiveType => ({
+      syncProtocol: {
+        name: primitiveEntry.syncProtocol,
+        blockNumber: height,
+        transactionHash: blockHash,
+        contractAddress: namespace,
+        logIndex: blob.index,
+      },
+      primitive: primitiveEntry.primitive.name,
+      output: {
+        payloadType: "celestia-blob",
+        payload: {
+          suppliedValue: atob(blob.data),
+          namespace,
+          commitment: blob.commitment,
+          blobIndex: blob.index,
+        },
+      },
+    }));
+  }
+}

package/src/sync-protocols/celestia/state.ts

@@ -0,0 +1,131 @@
+import { call, type Operation } from "effection";
+import { bound } from "@effectstream/utils";
+import type { PoolClient } from "pg";
+import { type LastPage, SyncState } from "../base/state.ts";
+import type { RootOutput, RootPage } from "../types.ts";
+import type { Input, Output, Page } from "./types.ts";
+import { pageRelation } from "./types.ts";
+import type { CelestiaFetcher } from "./fetcher.ts";
+import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/config";
+import { getPage } from "@effectstream/db";
+import { CelestiaClient } from "./CelestiaClient.ts";
+import { applyDelay } from "../common/utils.ts";
+
+export class CelestiaSyncState extends SyncState<
+  Input,
+  Output,
+  Page,
+  RootOutput,
+  RootPage,
+  CelestiaFetcher
+> {
+  constructor(
+    lastPage: LastPage<Page, RootPage> | undefined,
+    readonly config: Extract<
+      SyncProtocolWithNetwork,
+      { networkType: ConfigNetworkType.CELESTIA }
+    >,
+    fetcher: CelestiaFetcher,
+    public readonly client: CelestiaClient,
+    dbConn: PoolClient,
+  ) {
+    super(
+      config.syncProtocol.name,
+      lastPage,
+      fetcher,
+      pageRelation,
+      dbConn,
+    );
+  }
+
+  @bound
+  override toPage(_input: Input, data: Output[]): Page {
+    const lastBlock = data[data.length - 1];
+    return {
+      height: lastBlock.height,
+      hash: lastBlock.hash,
+    };
+  }
+
+  @bound
+  override toRootPage(data: Output): RootPage {
+    // Celestia headers use ISO 8601 timestamps; convert to ms.
+    const timestampMs = new Date(data.timestamp).getTime();
+    return applyDelay(timestampMs, this.config.syncProtocol.delayMs);
+  }
+
+  @bound
+  override toRootOutput(_data: Output): RootOutput {
+    throw new Error("Only main chains create root outputs");
+  }
+
+  @bound
+  override *stateToInput(): Operation<Input | undefined> {
+    const latestHeight = yield* call(() =>
+      this.client.getLatestBlockHeight()
+    );
+    const startHeight = this.lastPage?.own.height ??
+      this.config.syncProtocol.startBlockHeight - 1;
+
+    if (startHeight >= latestHeight) {
+      return undefined;
+    }
+
+    const from = startHeight + 1;
+    const to = Math.min(
+      from + this.config.syncProtocol.stepSize - 1,
+      latestHeight,
+    );
+
+    return {
+      from,
+      to,
+      isPresync: false,
+    };
+  }
+
+  @bound
+  override mergeDatum(ourOutput: Output, rootOutput: RootOutput): void {
+    const primitives = ourOutput.primitives.map((p) => ({
+      ...p,
+      source: this.config.syncProtocol.name,
+    }));
+    const blockInfo = [{
+      protocol_name: this.config.syncProtocol.name,
+      block_number: ourOutput.height,
+      blockHash: ourOutput.hash,
+    }];
+    rootOutput.blockInfo.push(...blockInfo);
+    rootOutput.primitives.push(...primitives);
+  }
+
+  @bound
+  override getNamespace(): string[] {
+    return [this.config.network.name, this.config.syncProtocol.name];
+  }
+
+  static *restoreState(
+    dbConn: PoolClient,
+    config: Extract<
+      SyncProtocolWithNetwork,
+      { networkType: ConfigNetworkType.CELESTIA }
+    >,
+    fetcher: CelestiaFetcher,
+  ): Operation<CelestiaSyncState> {
+    const [result] = yield* call(async () =>
+      await getPage.run({
+        protocol_name: config.syncProtocol.name,
+      }, dbConn)
+    );
+    const page = result
+      ? result.page as unknown as LastPage<Page, RootPage>
+      : undefined;
+    return new CelestiaSyncState(
+      page,
+      config,
+      fetcher,
+      new CelestiaClient(config.network.rpcUrl),
+      dbConn,
+    );
+  }
+}

package/src/sync-protocols/celestia/types.ts

@@ -0,0 +1,46 @@
+import type {
+  ConfigSyncProtocolType,
+  FlattenSyncProtocolIOFor,
+  SyncProtocolWithNetwork,
+} from "@effectstream/config";
+import type { BlockNumber } from "@effectstream/utils";
+import type { PageSyncRange } from "../common/page-helpers.ts";
+import type { PageRelation } from "../base/page.ts";
+
+// =====================
+// Sync protocol typings
+// =====================
+
+/** The page identifier for Celestia: a block height + hash pair */
+export type Page = {
+  height: BlockNumber;
+  hash: string;
+};
+
+export type PrimitiveType = FlattenSyncProtocolIOFor<
+  ConfigSyncProtocolType.CELESTIA_PARALLEL
+>;
+
+export type Input = PageSyncRange<BlockNumber>;
+
+export type Output = {
+  /** ISO 8601 timestamp from the block header */
+  timestamp: string;
+  /** Block height */
+  height: number;
+  /** Hex block hash from last_block_id */
+  hash: string;
+  primitives: PrimitiveType[];
+};
+
+export const pageRelation: PageRelation<Page> = {
+  compare: (p1, p2) => p1.height - p2.height,
+  equals: (p1, p2) => p1.height === p2.height,
+  min: (p1, p2) => (p1.height < p2.height ? p1 : p2),
+  max: (p1, p2) => (p1.height > p2.height ? p1 : p2),
+};
+
+export type ConfigType = Extract<
+  SyncProtocolWithNetwork,
+  { syncProtocolType: ConfigSyncProtocolType.CELESTIA_PARALLEL }
+>;