@effectstream/sync 0.100.0

package/src/sync-protocols/utxorpc/fetcher.ts

@@ -0,0 +1,116 @@
+import { BaseDataFetcher, type DataFetched } from "../base/fetcher.ts";
+import type { RootOutput, RootPage } from "../types.ts";
+import type { Operation } from "effection";
+import { bound, uint8ArrayToHexString } from "@effectstream/utils";
+import type { ChainPoint, ConfigType, Input, Output, Page, PrimitiveType } from "./types.ts";
+import type { OutputAndCleanup, RootConversion } from "../base/state.ts";
+import type { BufferedRpc } from "./BufferedRpc.ts";
+import { Buffer } from "node:buffer";
+import type { cardano } from "@utxorpc/spec";
+import { hashEqual, matchesPredicate } from "./utils.ts";
+
+export class UtxoRpcFetcher
+  extends BaseDataFetcher<Input, Output, RootOutput, Page, RootPage> // PrimitiveFetcher<Input, Page, cardano.Block, PrimitiveType>,
+{
+  constructor(
+    readonly config: ConfigType,
+    readonly client: BufferedRpc,
+  ) {
+    super(config.syncProtocol.name);
+  }
+
+  @bound
+  async startAsync(point?: ChainPoint) {
+    await this.client.start(point);
+  }
+
+  @bound
+  async resolveOrigin(): Promise<ChainPoint> {
+    return this.client.resolveOrigin();
+  }
+
+  @bound
+  override *readData(
+    data: Input,
+    rootConversion: RootConversion<Output, RootOutput, RootPage>,
+  ): Operation<DataFetched<Output, Page, RootPage>> {
+    console.log(
+      `[UTXORPC] Fetching blocks from ${data.from} to ${data.to}. ${
+        data.isPresync ? "[presync]" : ""
+      }`,
+    );
+    const outputs: OutputAndCleanup<Output>[] = [];
+    const blocks = this.client.fetchBlocks(data.from, data.to);
+    for (const block of blocks) {
+      outputs.push({
+        output: {
+          // TODO: What is the correct block hash?
+          blockHashes: [String(block.output.header?.hash)],
+          raw: block.output,
+          primitives: this.findPrimitives(block.output),
+        },
+        cleanup: block.cleanup,
+      });
+    }
+    return {
+      output: outputs,
+      lastPage: {
+        ownBlockNumber: Number(data.to),
+        own: {
+          slot: Number(
+            outputs[outputs.length - 1].output.raw.header!.slot,
+          ),
+          height: Number(
+            outputs[outputs.length - 1].output.raw.header!.height,
+          ),
+          hash: Buffer.from(
+            outputs[outputs.length - 1].output.raw.header!.hash,
+          )
+            .toString("hex"),
+        },
+        root: rootConversion.toRootPage({
+          blockHashes: [],
+          primitives: [], // TODO: I think this can be left empty here
+          raw: outputs[outputs.length - 1].output.raw,
+        }),
+      },
+    };
+  }
+
+  @bound
+  findPrimitives(block: cardano.Block): PrimitiveType[] {
+    const primitiveResponses: PrimitiveType[] = [];
+    for (const tx of block.body?.tx ?? []) {
+      let logIndex = 0;
+      for (const primitveEntry of this.config.primitives) {
+        if (!matchesPredicate(tx, primitveEntry.primitive.predicate)) {
+          continue;
+        }
+        const transactionIndex = block.body?.tx.findIndex(t => hashEqual(t.hash, tx.hash));
+        primitiveResponses.push({
+          syncProtocol: {
+            name: primitveEntry.syncProtocol,
+            blockNumber: Number(block.header!.height),
+            transactionHash: uint8ArrayToHexString(tx.hash),
+            transactionIndex,
+            contractAddress: '',
+            logIndex,
+          },
+          primitive: primitveEntry.primitive.name,
+          output: {
+            payloadType: 'utxorpc-response',
+            payload: { tx }
+          }
+        });
+        ++logIndex;
+      }
+    }
+    return primitiveResponses;
+  }
+
+  lastHeight(): bigint | undefined {
+    // Calling this.client.syncClient.getTip(); 
+    // returns the hash and slot, but not the block height
+    return this.client.lastHeight();
+  }
+}

package/src/sync-protocols/utxorpc/state.ts

@@ -0,0 +1,186 @@
+import { applyDelay } from "../common/utils.ts";
+import type { Operation } from "effection";
+import { call } from "effection";
+import { getPage } from "@effectstream/db";
+import type { PoolClient } from "pg";
+import { bound, type TimestampMs } from "@effectstream/utils";
+import { type LastPage, SyncState } from "../base/state.ts";
+import type { RootOutput, RootPage } from "../types.ts";
+import type { UtxoRpcFetcher } from "./fetcher.ts";
+import type { ConfigType, Input, Output, Page } from "./types.ts";
+import { chainPointRelation } from "./types.ts";
+import { Buffer } from "node:buffer";
+
+export class UtxoRpcSyncState extends SyncState<
+  Input,
+  Output,
+  Page,
+  RootOutput,
+  RootPage,
+  UtxoRpcFetcher
+> {
+  constructor(
+    lastPage: undefined | LastPage<Page, RootPage>,
+    readonly config: ConfigType,
+    fetcher: UtxoRpcFetcher,
+    dbConn: PoolClient,
+  ) {
+    super(
+      config.syncProtocol.name,
+      lastPage,
+      fetcher,
+      chainPointRelation,
+      dbConn,
+    );
+  }
+
+  @bound
+  override *startAsync(): Operation<void> {
+    if (this.lastPage == null) {
+      const cp = this.config.syncProtocol.startChainPoint;
+      if (cp === "origin") {
+        const origin = yield* call(() => this.fetcher.resolveOrigin());
+        yield* call(() => this.fetcher.startAsync(origin));
+      } else if (cp === "tip") {
+        yield* call(() => this.fetcher.startAsync(undefined));
+      } else {
+        yield* call(() => this.fetcher.startAsync({ slot: cp.slot, hash: cp.hash }));
+      }
+      return;
+    }
+    yield* call(() =>
+      this.fetcher.startAsync({
+        slot: this.lastPage!.own.slot,
+        hash: this.lastPage!.own.hash,
+      })
+    );
+  }
+
+  @bound
+  override toRootOutput(data: Output): RootOutput {
+    throw new Error("Only main chains create root outputs");
+  }
+
+  @bound
+  override toRootPage(data: Output): RootPage {
+    return applyDelay(
+      Number(data.raw.timestamp * 1000n),
+      this.config.syncProtocol.delayMs,
+    );
+  }
+
+  @bound
+  override *stateToInput(): Operation<Input | undefined> {
+    // Get the value of the latest block height.
+    const tipHeight = yield* call(() => this.fetcher.lastHeight());
+    if (!tipHeight) {
+      return undefined;
+    }
+
+    // Using 0 as the startHeight for both "origin" and a specified ChainPoint is safe because:
+    // the FollowTip stream only buffers blocks starting from the intersection,
+    // so fetchBlocks(from=1) will match the first block, regardless of it's height.
+    // After that first block, lastPage.own.height will give a real height value.
+    const cp = this.config.syncProtocol.startChainPoint;
+    const startHeight = this.lastPage?.own.height ??
+      (cp === "tip"
+        ? Number(tipHeight) - 1
+        : 0);
+
+    if (BigInt(startHeight) >= tipHeight) {
+      return undefined;
+    }
+
+    // TODO We need to check how to handle the presync process.
+    // const storedBlocks = yield* this.fetcher.client.storedBlocks(
+    //   this.lastPage?.own.height,
+    // );
+    // if (storedBlocks.from.slot < this.config.syncProtocol.startSlot) {
+    //   return {
+    //     from: storedBlocks.from.height,
+    //     to: Math.max(
+    //       storedBlocks.to.height,
+    //       // TODO: this should be a height and not a slot number
+    //       //       either we need to resolve the actual block at that slot
+    //       //       (blocked by https://github.com/utxorpc/spec/issues/148)
+    //       //       or we need to guess and refine if wrong
+    //       this.config.syncProtocol.startSlot - 1,
+    //     ),
+    //     isPresync: true,
+    //   };
+    // }
+
+    // TODO Is this correct?
+    // EVM Handles this with `genInputRange`
+    const from = startHeight + 1;
+    const to = tipHeight ? Math.min(
+      from + 1,
+      Number(tipHeight),
+    ) : from + 1;
+
+    return {
+      from,
+      to,
+      isPresync: false, // TODO: handle presync
+    };
+  }
+
+  @bound
+  override toPage(
+    input: Input,
+    data: Output[],
+  ): Page {
+    return {
+      slot: Number(data[data.length - 1].raw.header!.slot),
+      height: Number(data[data.length - 1].raw.header!.height),
+      hash: Buffer.from(data[data.length - 1].raw.header!.hash).toString(
+        "hex",
+      ),
+    };
+  }
+
+  @bound
+  override mergeDatum(
+    ourOutput: Output,
+    rootOutput: RootOutput,
+  ): void {
+    const primitives = ourOutput.primitives.map((p) => ({
+      ...p,
+      source: this.config.syncProtocol.name,
+    }));
+    const blockInfo = ourOutput.blockHashes.map((h) => ({
+      protocol_name: this.config.syncProtocol.name,
+      block_number: Number(ourOutput.raw.header!.height),
+      blockHash: h,
+    }));
+    rootOutput.primitives.push(...primitives);
+    rootOutput.blockInfo.push(...blockInfo);
+  }
+
+  @bound
+  override getNamespace(): string[] {
+    return [this.config.network.name, this.config.syncProtocol.name];
+  }
+
+  static *restoreState(
+    dbConn: PoolClient,
+    config: ConfigType,
+    fetcher: UtxoRpcFetcher,
+  ): Operation<UtxoRpcSyncState> {
+    // TODO: move this DB query into page-helpers?
+    const [result] = yield* call(async () =>
+      await getPage.run({
+        protocol_name: config.syncProtocol.name,
+      }, dbConn)
+    );
+    const page = result
+      ? result.page as unknown as LastPage<Page, RootPage>
+      : undefined;
+    return new UtxoRpcSyncState(
+      page,
+      config,
+      fetcher,
+      dbConn,
+    );
+  }
+}

package/src/sync-protocols/utxorpc/types.ts

@@ -0,0 +1,53 @@
+import type {
+  AbsoluteSlotNumber,
+  BlockNumber,
+  CardanoBlockHash,
+  CardanoBlockNumber,
+  TimestampMs,
+} from "@effectstream/utils";
+import type { PageRelation } from "../base/page.ts";
+import type { cardano } from "@utxorpc/spec";
+import type { PageSyncRange } from "../common/page-helpers.ts";
+import type {
+  ConfigSyncProtocolType,
+  FlattenSyncProtocolIOFor,
+  PrimitiveEntry,
+  SyncProtocolWithNetwork,
+} from "@effectstream/config";
+
+import type { CardanoChainPoint } from "@utxorpc/sdk";
+export type ChainPoint = CardanoChainPoint;
+
+export type Page = {
+  slot: AbsoluteSlotNumber;
+  height: CardanoBlockNumber;
+  hash: CardanoBlockHash;
+};
+
+export type PrimitiveEntryType = Extract<
+  PrimitiveEntry,
+  { syncProtocol: ConfigSyncProtocolType.CARDANO_UTXORPC_PARALLEL }
+>;
+
+// TODO: blocked on https://github.com/utxorpc/spec/issues/135
+export type PrimitiveType = FlattenSyncProtocolIOFor<
+  ConfigSyncProtocolType.CARDANO_UTXORPC_PARALLEL
+>;
+export type Input = PageSyncRange<BlockNumber>;
+export type Output = {
+  raw: cardano.Block;
+  primitives: PrimitiveType[];
+  blockHashes: CardanoBlockHash[];
+};
+
+export const chainPointRelation: PageRelation<Page> = {
+  compare: (p1, p2) => p1.slot - p2.slot,
+  equals: (p1, p2) => p1.slot === p2.slot,
+  min: (p1, p2) => (p1.slot < p2.slot ? p1 : p2),
+  max: (p1, p2) => (p1.slot > p2.slot ? p1 : p2),
+};
+
+export type ConfigType = Extract<
+  SyncProtocolWithNetwork,
+  { syncProtocolType: ConfigSyncProtocolType.CARDANO_UTXORPC_PARALLEL }
+>;

package/src/sync-protocols/utxorpc/utils.ts

@@ -0,0 +1,80 @@
+import type { cardano } from '@utxorpc/spec';
+import type { UtxorpcAddressPattern, UtxorpcAnyChainTxPattern, UtxorpcAssetPattern, UtxorpcTxOutputPattern, UtxorpcTxPredicate } from "@effectstream/config";
+import { hexStringToUint8Array } from "@effectstream/utils";
+import { Cardano } from '@cardano-sdk/core';
+import { HexBlob } from '@cardano-sdk/util';
+
+export function hashEqual(lhs: Uint8Array, rhs: Uint8Array): boolean {
+  if (lhs.length !== rhs.length) {
+    return false;
+  }
+  for (let i = 0; i < lhs.length; ++i) {
+    if (lhs[i] !== rhs[i]) {
+      return false;
+    }
+  }
+  return true;
+}
+
+export function matchesPredicate(tx: cardano.Tx, predicate: UtxorpcTxPredicate): boolean {
+  if (predicate.match && !matchesPattern(tx, predicate.match)) {
+    return false;
+  }
+  if (predicate.not && predicate.not.some(p => matchesPredicate(tx, p))) {
+    return false;
+  }
+  if (predicate.all_of && !predicate.all_of.every(p => matchesPredicate(tx, p))) {
+    return false;
+  }
+  if (predicate.any_of && !predicate.any_of.some(p => matchesPredicate(tx, p))) {
+    return false;
+  }
+  return true;
+}
+
+function matchesPattern(tx: cardano.Tx, pattern: UtxorpcAnyChainTxPattern): boolean {
+  const cardanoPattern = pattern.cardano;
+  if (!cardanoPattern) return false;
+  const outputs = tx.outputs;
+  const inputs = tx.inputs.map(input => input.asOutput!).filter(x => x);
+  const matchHasAddress = !cardanoPattern.has_address || matchesAddress(outputs, cardanoPattern.has_address) || matchesAddress(inputs, cardanoPattern.has_address);
+  const matchMovesAsset = !cardanoPattern.moves_asset || matchesAsset(inputs, cardanoPattern.moves_asset);
+  const matchMintsAsset = !cardanoPattern.mints_asset || matchesAsset(outputs, cardanoPattern.mints_asset);
+  return matchHasAddress && matchMovesAsset && matchMintsAsset;
+}
+
+function matchesOutputPattern(outputs: cardano.TxOutput[], pattern: UtxorpcTxOutputPattern): boolean {
+  const matchAddress = !pattern.address || matchesAddress(outputs, pattern.address);
+  const matchAsset = !pattern.asset || matchesAsset(outputs, pattern.asset);
+  return matchAddress && matchAsset;
+}
+
+function matchesAddress(outputs: cardano.TxOutput[], pattern: UtxorpcAddressPattern): boolean {
+  if (pattern.exact_address) {
+    const address = Uint8Array.from(atob(pattern.exact_address), c => c.charCodeAt(0));
+    if (!outputs.some(o => hashEqual(o.address, address))) {
+      return false;
+    }
+  }
+  if (pattern.payment_part || pattern.delegation_part) {
+    const addresses = outputs.map(o => Cardano.Address.fromBytes(HexBlob.fromBytes(o.address)));
+    if (pattern.payment_part && !addresses.some(a => a.getProps()?.paymentPart?.hash === pattern.payment_part)) {
+      return false;
+    }
+    if (pattern.delegation_part && !addresses.some(a => a.getProps()?.delegationPart?.hash === pattern.delegation_part)) {
+      return false;
+    }
+  }
+  return true;
+}
+
+function matchesAsset(outputs: cardano.TxOutput[], pattern: UtxorpcAssetPattern): boolean {
+  const policyId = pattern.policy_id ? hexStringToUint8Array(pattern.policy_id) : null;
+  const assetName = pattern.asset_name ? hexStringToUint8Array(pattern.asset_name) : null;
+  return outputs.some(o => o.assets.some(ma => {
+    if (policyId && !hashEqual(policyId, ma.policyId)) {
+      return false;
+    }
+    return ma.assets.some(a => !assetName || hashEqual(a.name, assetName));
+  }));
+}

package/src/syncProtocolFactory.ts

@@ -0,0 +1,149 @@
+import { EvmSyncState } from "./sync-protocols/evm/state.ts";
+import { EvmFetcher } from "./sync-protocols/evm/fetcher.ts";
+import type { Operation } from "effection";
+import type { PoolClient } from "pg";
+import type { AllSyncProtocols } from "./sync-protocols/types.ts";
+import { createViemPublicClient } from "@effectstream/utils";
+import type { SyncProtocolWithNetwork } from "@effectstream/config";
+import {
+  ConfigNetworkType,
+  ConfigSyncProtocolType,
+  getViemNetwork,
+} from "@effectstream/config";
+import { CardanoSyncClient } from "@utxorpc/sdk";
+import { BufferedRpc } from "./sync-protocols/utxorpc/BufferedRpc.ts";
+import { UtxoRpcFetcher } from "./sync-protocols/utxorpc/fetcher.ts";
+import { UtxoRpcSyncState } from "./sync-protocols/utxorpc/state.ts";
+import { MidnightFetcher, MidnightSyncState } from "@effectstream/sync";
+import { AvailFetcher } from "./sync-protocols/avail/fetcher.ts";
+import { AvailSyncState } from "./sync-protocols/avail/state.ts";
+import { NtpFetcher } from "./sync-protocols/ntp/fetcher.ts";
+import { NtpSyncState } from "./sync-protocols/ntp/state.ts";
+import {
+  BitcoinFetcher,
+  BitcoinRpcClient,
+} from "./sync-protocols/bitcoin/fetcher.ts";
+import { BitcoinSyncState } from "./sync-protocols/bitcoin/state.ts";
+import { CelestiaFetcher } from "./sync-protocols/celestia/fetcher.ts";
+import { CelestiaSyncState } from "./sync-protocols/celestia/state.ts";
+import { NearFetcher } from "./sync-protocols/near/fetcher.ts";
+import { NearSyncState } from "./sync-protocols/near/state.ts";
+
+export function* genSyncProtocols(
+  dbConn: PoolClient,
+  syncInfo: SyncProtocolWithNetwork[],
+): Operation<AllSyncProtocols[]> {
+  const result: AllSyncProtocols[] = [];
+  for (const entry of syncInfo) {
+    if (entry.networkType === ConfigNetworkType.NTP) {
+      const fetcher = new NtpFetcher(entry);
+      const state = yield* NtpSyncState.restoreState(
+        dbConn,
+        entry,
+        fetcher,
+      );
+      result.push(state);
+    } else if (
+      entry.networkType === ConfigNetworkType.EVM
+    ) {
+      const viemNetwork = yield* getViemNetwork(entry.network);
+      const fetcher = new EvmFetcher(
+        entry,
+        createViemPublicClient(viemNetwork, {
+          cacheTime: entry.syncProtocol.pollingInterval,
+        }),
+      );
+      const state = yield* EvmSyncState.restoreState(
+        dbConn,
+        entry,
+        fetcher,
+      );
+      result.push(state);
+    } else if (
+      entry.networkType === ConfigNetworkType.CARDANO
+    ) {
+      if (
+        entry.syncProtocolType === ConfigSyncProtocolType.CARDANO_CARP_PARALLEL
+      ) {
+        throw new Error("CARP not supported yet");
+      }
+      const syncClient = new CardanoSyncClient({
+        uri: entry.syncProtocol.rpcUrl,
+        headers: entry.syncProtocol.headers,
+      });
+      const bufferedRpc = new BufferedRpc(
+        syncClient,
+        entry.syncProtocol.confirmationDepth,
+      );
+      const fetcher = new UtxoRpcFetcher(
+        entry,
+        bufferedRpc,
+      );
+      const state = yield* UtxoRpcSyncState.restoreState(
+        dbConn,
+        entry,
+        fetcher,
+      );
+      result.push(state);
+    } else if (
+      entry.networkType === ConfigNetworkType.MIDNIGHT
+    ) {
+      const fetcher = new MidnightFetcher(entry);
+      const state = yield* MidnightSyncState.restoreState(
+        dbConn,
+        entry,
+        fetcher,
+      );
+      result.push(state);
+    } else if (
+      entry.networkType === ConfigNetworkType.AVAIL
+    ) {
+      const fetcher = new AvailFetcher(entry);
+      const state = yield* AvailSyncState.restoreState(
+        dbConn,
+        entry,
+        fetcher,
+      );
+      result.push(state);
+    } else if (
+      entry.networkType === ConfigNetworkType.BITCOIN
+    ) {
+      const rpcClient = new BitcoinRpcClient({
+        url: entry.network.rpcUrl,
+        username: entry.network.rpcAuth?.username ?? null,
+        password: entry.network.rpcAuth?.password ?? null,
+      });
+      const fetcher = new BitcoinFetcher(entry, rpcClient);
+      const state = yield* BitcoinSyncState.restoreState(
+        dbConn,
+        entry,
+        fetcher,
+      );
+      result.push(state);
+    } else if (
+      entry.networkType === ConfigNetworkType.CELESTIA
+    ) {
+      const fetcher = new CelestiaFetcher(entry);
+      const state = yield* CelestiaSyncState.restoreState(
+        dbConn,
+        entry,
+        fetcher,
+      );
+      result.push(state);
+    } else if (
+      entry.networkType === ConfigNetworkType.NEAR
+    ) {
+      const fetcher = new NearFetcher(entry);
+      const state = yield* NearSyncState.restoreState(
+        dbConn,
+        entry,
+        fetcher,
+      );
+      result.push(state);
+    } else {
+      throw new Error(`Unsupported network type: ${entry.network.type}`);
+    }
+  }
+
+  return result;
+}