@effectstream/sync 0.100.0

package/src/sync-protocols/ntp/state.ts

@@ -0,0 +1,118 @@
+import { blockNumberRelation } from "../common/utils.ts";
+import type { Operation } from "effection";
+import { call } from "effection";
+import { getPage } from "@effectstream/db";
+import type { PoolClient } from "pg";
+import { bound, type NtpBlockNumber } from "@effectstream/utils";
+import { type LastPage, SyncState } from "../base/state.ts";
+import type { RootOutput, RootPage } from "../types.ts";
+import type { NtpFetcher } from "./fetcher.ts";
+import { genInputRange } from "../common/page-helpers.ts";
+import type { Input, Output, Page } from "./types.ts";
+import { toMsTimestamp } from "./types.ts";
+import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/config";
+
+export class NtpSyncState extends SyncState<
+  Input,
+  Output,
+  Page,
+  RootOutput,
+  RootPage,
+  NtpFetcher
+> {
+  constructor(
+    lastPage: LastPage<Page, RootPage>,
+    readonly config: Extract<
+      SyncProtocolWithNetwork,
+      { networkType: ConfigNetworkType.NTP }
+    >,
+    fetcher: NtpFetcher,
+    dbConn: PoolClient,
+  ) {
+    super(
+      config.syncProtocol.name,
+      lastPage,
+      fetcher,
+      blockNumberRelation,
+      dbConn,
+    );
+  }
+
+  @bound
+  override toRootOutput(data: Output): RootOutput {
+    return {
+      blockInfo: data.blockHashes.map((h) => ({
+        protocol_name: this.config.syncProtocol.name,
+        block_number: Number(data.raw.blockNumber),
+        blockHash: h,
+      })),
+      blockNumber: Number(data.raw.blockNumber),
+      timestamp: this.toRootPage(data),
+      primitives: [],
+    };
+  }
+
+  @bound
+  override toRootPage(data: Output): RootPage {
+    return toMsTimestamp(data.raw.timestamp);
+  }
+
+  @bound
+  override *stateToInput(): Operation<Input | undefined> {
+    return yield* genInputRange(
+      this as NtpSyncState,
+      1, // NTP Pages start from 1, and the fetcher offsets using the start page (date)
+      {
+        name: this.config.syncProtocol.name,
+        startPage: this.config.syncProtocol.startBlockHeight as NtpBlockNumber,
+      },
+      this.getNamespace(),
+    );
+  }
+
+  @bound
+  override toPage(
+    input: Input,
+    data: Output[],
+  ): Page {
+    return Number(input.to);
+  }
+
+  @bound
+  override mergeDatum(
+    ourOutput: Output,
+    rootOutput: RootOutput,
+  ): void {
+    throw new Error("Only parallel chains merge into root");
+  }
+
+  @bound
+  override getNamespace(): string[] {
+    return [this.config.network.name, this.config.syncProtocol.name];
+  }
+
+  static *restoreState(
+    dbConn: PoolClient,
+    config: Extract<
+      SyncProtocolWithNetwork,
+      { networkType: ConfigNetworkType.NTP }
+    >,
+    fetcher: NtpFetcher,
+  ): Operation<NtpSyncState> {
+    // TODO: move this DB query into page-helpers?
+    const [result] = yield* call(async () =>
+      await getPage.run({
+        protocol_name: config.syncProtocol.name,
+      }, dbConn)
+    );
+    const page: any = result
+      ? result.page as unknown as LastPage<Page, RootPage>
+      : undefined;
+    return new NtpSyncState(
+      page,
+      config,
+      fetcher,
+      dbConn,
+    );
+  }
+}

package/src/sync-protocols/ntp/types.ts

@@ -0,0 +1,40 @@
+import type {
+  BlockNumber,
+  NtpBlockHash,
+  NtpBlockNumber,
+  NtpPageJson,
+  TimestampMs,
+} from "@effectstream/utils";
+import { TypeboxHelpers } from "@effectstream/utils";
+import type { PageSyncRange } from "../common/page-helpers.ts";
+import { Type } from "@sinclair/typebox";
+import type {
+  ConfigSyncProtocolType,
+  FlattenSyncProtocolIOFor,
+} from "@effectstream/config";
+
+export type Page = BlockNumber;
+const PageJsonSchema = Type.Unsafe<
+  NtpPageJson
+>(Type.String());
+export const PageSchema = TypeboxHelpers.SerializeObjAsJson<
+  Page,
+  typeof PageJsonSchema
+>();
+
+export type PrimitiveType = FlattenSyncProtocolIOFor<
+  ConfigSyncProtocolType.NTP_MAIN
+>;
+export type Input = PageSyncRange<Page>;
+export type Output = {
+  raw: {
+    timestamp: bigint;
+    hash: NtpBlockHash;
+    blockNumber: number;
+  };
+  blockHashes: NtpBlockHash[];
+};
+
+export function toMsTimestamp(timestamp: bigint): TimestampMs {
+  return Number(timestamp);
+}

package/src/sync-protocols/orchestration/merge.ts

@@ -0,0 +1,215 @@
+import type { Channel, Operation } from "effection";
+import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
+import type {
+  AllSyncProtocols,
+  ISyncProtocol,
+  RootOutput,
+  RootPage,
+} from "../types.ts";
+import type { PageRelation } from "../base/page.ts";
+import type Deque from "denque";
+import type { ChainBlock } from "../common/root.ts";
+import { chainPageRelation } from "../common/root.ts";
+import type { CacheCleanup, OutputAndCleanup } from "../base/state.ts";
+
+export function* startMerge(
+  syncProtocols: AllSyncProtocols[],
+  finalizedBlockStream: Channel<ChainBlock, void>,
+): Operation<ChainBlock> {
+  while (true) {
+    let newBlock: ChainBlock | undefined = undefined;
+
+    const cleanup: CacheCleanup[] = [];
+
+    for (let i = 0; i < syncProtocols.length; i++) {
+      const mergeInfo: MergeResult<RootOutput> = yield* mergeIntoRoot(
+        syncProtocols[i],
+        {
+          value: newBlock,
+          toPage: (block) => block.timestamp,
+          comparePage: chainPageRelation,
+        },
+      );
+      cleanup.push(mergeInfo.updateCache);
+      if (mergeInfo.newRoot != null) {
+        newBlock = mergeInfo.newRoot;
+      }
+      if (i === 0) {
+        log.remote(
+          ComponentNames.EFFECTSTREAM_SYNC,
+          "block-merge",
+          SeverityNumber.DEBUG,
+          (log) => log(`new block ${mergeInfo.newRoot?.blockNumber}`),
+        );
+      }
+    }
+    if (newBlock == null) continue;
+    log.remote(
+      ComponentNames.EFFECTSTREAM_SYNC,
+      "block-merge",
+      SeverityNumber.INFO,
+      (log) => log(`producing block ${newBlock?.blockNumber}`),
+    );
+
+    yield* finalizedBlockStream.send(newBlock);
+
+    for (const fn of cleanup) {
+      fn();
+    }
+  }
+}
+
+export type MergeResult<RootOutput> = {
+  updateCache: () => void;
+  /**
+   * undefined → replace whatever root might exist previously with this one.
+   *             this is primarily useful for setting the initial root
+   */
+  newRoot: undefined | RootOutput;
+};
+
+/**
+ * Merge as much of the data into the root as possible\
+ * Note: it does *not* modify the child data.
+ *       Instead, it returns a callback to remove elements that were merged.
+ *       This is to help with error handling
+ *       since you can avoid removing elements until you're sure no errors have happened
+ *       given there is no way to recover these elements if an error happens after they get removed.
+ *
+ * @param value the data being constructed. `undefined` if this is the first piece of data in the chain
+ */
+export function* mergeIntoRoot<SyncProtocol extends AllSyncProtocols>(
+  state: SyncProtocol,
+  rootInfo: {
+    value: undefined | RootOutput;
+    toPage: (data: RootOutput) => RootPage;
+    comparePage: PageRelation<RootPage>;
+  },
+): Operation<MergeResult<RootOutput>> {
+  const iState = state as ISyncProtocol;
+
+  // 1) If we've never made a single query yet, wait
+  while (state.lastPage == null) {
+    log.remote(
+      ComponentNames.EFFECTSTREAM_SYNC,
+      state.getNamespace(),
+      SeverityNumber.DEBUG,
+      (log) => log("Merge blocked (first sync): waiting for page"),
+    );
+    yield* state.newPageCondVar.wait();
+    log.remote(
+      ComponentNames.EFFECTSTREAM_SYNC,
+      state.getNamespace(),
+      SeverityNumber.DEBUG,
+      (log) => log("Merge unblocked (first sync): got page"),
+    );
+  }
+
+  // 2) Handle case where this is the first in the list of states to merge data from
+  if (rootInfo.value == null) {
+    while (state.bufferedData.size() === 0) {
+      log.remote(
+        ComponentNames.EFFECTSTREAM_SYNC,
+        state.getNamespace(),
+        SeverityNumber.DEBUG,
+        (log) => log("Merge blocked (missing main data): waiting for data"),
+      );
+      yield* state.newDataCondVar.wait();
+      log.remote(
+        ComponentNames.EFFECTSTREAM_SYNC,
+        state.getNamespace(),
+        SeverityNumber.DEBUG,
+        (log) => log("Merge unblocked (missing main data): got data"),
+      );
+    }
+    const output = iState.bufferedData.peekAt(0)!;
+    const newRoot = iState.toRootOutput(output.output);
+    return {
+      updateCache: () => {
+        output.cleanup();
+        state.bufferedData.remove(0, 1);
+      },
+      newRoot,
+    };
+  }
+
+  // 3) If a previous state created data, keep merging data from this state into it
+  //    until we've exceeded the time range
+  while (
+    rootInfo.comparePage.compare(
+      state.lastPage.root,
+      rootInfo.toPage(rootInfo.value),
+    ) <= 0
+  ) {
+    log.remote(
+      ComponentNames.EFFECTSTREAM_SYNC,
+      state.getNamespace(),
+      SeverityNumber.DEBUG,
+      (log) => log("Merge blocked (missing parallel data): waiting for data"),
+    );
+    yield* state.newPageCondVar.wait();
+    log.remote(
+      ComponentNames.EFFECTSTREAM_SYNC,
+      state.getNamespace(),
+      SeverityNumber.DEBUG,
+      (log) => log("Merge unblocked (missing parallel data): got data"),
+    );
+  }
+  const cleanups = getFromBuffer(
+    {
+      data: iState.bufferedData,
+      toPage: iState.toRootPage,
+    },
+    {
+      data: rootInfo.value,
+      toPage: rootInfo.toPage,
+    },
+    rootInfo.comparePage,
+    iState.mergeDatum,
+  );
+
+  return {
+    updateCache: () => {
+      state.bufferedData.remove(0, cleanups.length);
+      for (const fn of cleanups) {
+        fn();
+      }
+    },
+    newRoot: undefined,
+  };
+}
+
+/**
+ * Merge as much of the data into the root as possible
+ * Note: it does *not* modify the child data
+ *
+ * @returns the amount of elements that can be added to the root data
+ */
+function getFromBuffer<Output, RootOutput, RootPage>(
+  child: {
+    data: Deque<OutputAndCleanup<Output>>;
+    toPage: (data: Output) => RootPage;
+  },
+  root: {
+    data: RootOutput;
+    toPage: (data: RootOutput) => RootPage;
+  },
+  pageRelation: PageRelation<RootPage>,
+  mergeData: (child: Output, root: RootOutput) => void,
+): CacheCleanup[] {
+  const cleanups: CacheCleanup[] = [];
+  let nextElem = child.data.peekAt(cleanups.length);
+
+  while (
+    nextElem != null &&
+    pageRelation.compare(
+        child.toPage(nextElem.output),
+        root.toPage(root.data),
+      ) <= 0
+  ) {
+    mergeData(nextElem.output, root.data);
+    cleanups.push(nextElem.cleanup);
+    nextElem = child.data.peekAt(cleanups.length);
+  }
+  return cleanups;
+}

package/src/sync-protocols/orchestration/sync.ts

@@ -0,0 +1,74 @@
+import type { Operation } from "effection";
+import { sleep, spawn } from "effection";
+import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
+import type { AllSyncProtocols, ISyncProtocol } from "../types.ts";
+import { tryYield } from "@effectstream/utils";
+
+export function* startSync(
+  state: AllSyncProtocols,
+): Operation<void> {
+  const iState: ISyncProtocol = state as ISyncProtocol;
+
+  // spawn async task
+  yield* spawn(function* () {
+    yield* iState.startAsync();
+  });
+
+  // spawn polling task
+  yield* spawn(function* () {
+    while (true) {
+      const inputResult = yield* tryYield(iState.stateToInput());
+      if (inputResult.error != null) {
+        state.consecutiveErrors++;
+        state.lastErrorTimestamp = Date.now();
+        log.remote(
+          ComponentNames.EFFECTSTREAM_SYNC,
+          [...state.getNamespace(), "stateToInput"],
+          SeverityNumber.ERROR,
+          (log) => log(inputResult.error),
+        );
+        const { config } = state.fetcher;
+        if ("pollingInterval" in config.syncProtocol) {
+          yield* sleep(config.syncProtocol.pollingInterval);
+        }
+        continue;
+      }
+      const input = inputResult.data;
+      const { config } = state.fetcher;
+      if (input == null) {
+        if ("pollingInterval" in config.syncProtocol) {
+          yield* sleep(config.syncProtocol.pollingInterval);
+        }
+        continue;
+      }
+
+      const result = yield* tryYield(iState.fetcher.readData(input, iState));
+      if (result.error != null) {
+        state.consecutiveErrors++;
+        state.lastErrorTimestamp = Date.now();
+        log.remote(
+          ComponentNames.EFFECTSTREAM_SYNC,
+          [...state.getNamespace(), "readData"],
+          SeverityNumber.ERROR,
+          (l) => l(result.error),
+        );
+        if ("pollingInterval" in config.syncProtocol) {
+          yield* sleep(config.syncProtocol.pollingInterval);
+        }
+        continue;
+      }
+      state.consecutiveErrors = 0;
+      state.lastSuccessfulFetchMs = Date.now();
+      log.remote(
+        ComponentNames.EFFECTSTREAM_SYNC,
+        [...state.getNamespace(), "data"],
+        SeverityNumber.TRACE,
+        (log) => log(result.data),
+      );
+      yield* iState.updateState(input, result.data);
+      for (const datum of result.data.output) {
+        yield* iState.fetcher.producerChannel.send(datum.output);
+      }
+    }
+  });
+}

package/src/sync-protocols/types.ts

@@ -0,0 +1,32 @@
+import type { ChainBlock, ChainPage } from "./common/root.ts";
+import type { EvmSyncState } from "./evm/state.ts";
+import type { SyncState } from "./base/state.ts";
+import type { UtxoRpcSyncState } from "./utxorpc/state.ts";
+import type { PaginatedFetcher } from "./base/fetcher.ts";
+import type { UnionToIntersection } from "@effectstream/utils";
+import type { MidnightSyncState } from "./midnight/state.ts";
+import type { NtpSyncState } from "./ntp/state.ts";
+import type { AvailSyncState } from "./avail/state.ts";
+import type { BitcoinSyncState } from "./bitcoin/state.ts";
+import type { CelestiaSyncState } from "./celestia/state.ts";
+import type { NearSyncState } from "./near/state.ts";
+
+// TODO: move folders
+export type RootOutput = ChainBlock;
+export type RootPage = ChainPage;
+
+// TODO: map config types to sync protocols
+export type AllSyncProtocols =
+  | NtpSyncState
+  | EvmSyncState
+  | UtxoRpcSyncState
+  | MidnightSyncState
+  | AvailSyncState
+  | BitcoinSyncState
+  | CelestiaSyncState
+  | NearSyncState;
+export type ISyncProtocol = UnionToIntersection<AllSyncProtocols>;
+
+type toPaginated<T> = T extends { fetcher: PaginatedFetcher<infer Page> } ? T
+  : never;
+export type PaginatedSyncProtocols = toPaginated<AllSyncProtocols>;

package/src/sync-protocols/utxorpc/BufferedRpc.ts

@@ -0,0 +1,169 @@
+import type { CardanoSyncClient } from "@utxorpc/sdk";
+import type { cardano } from "@utxorpc/spec";
+import type { OutputAndCleanup } from "../base/state.ts";
+import Deque from "denque";
+import type { BlockNumber } from "@effectstream/utils";
+import type { Operation } from "effection";
+import { conditionVariable } from "@effectstream/utils";
+import type { ChainPoint, Page } from "./types.ts";
+import { Buffer } from "node:buffer";
+
+export class BufferedRpc {
+  private readonly buffer: Deque<OutputAndCleanup<cardano.Block>> =
+    new Deque();
+  private readonly newDataCondVar = conditionVariable<void>();
+
+  constructor(
+    public readonly syncClient: CardanoSyncClient,
+    /**
+     * Finality should ensure that we never expose data that could be affected by `undo`
+     * recall: finality is in terms of blocks created (not in terms of slots)
+     */
+    private readonly finality: BlockNumber,
+  ) {}
+
+  static lastHeight: bigint | undefined = undefined;
+  static initialized = false;
+
+  public lastHeight(): bigint | undefined {
+    return BufferedRpc.lastHeight;
+  }
+
+  public async resolveOrigin(): Promise<ChainPoint> {
+    const blocks = await this.syncClient.fetchHistory(undefined, 1);
+    if (!blocks.length) throw new Error("No blocks found on chain");
+    const header = blocks[0].parsedBlock.header!;
+    return {
+      slot: Number(header.slot),
+      hash: Buffer.from(header.hash).toString("hex"),
+    };
+  }
+
+  public async start(point?: ChainPoint): Promise<void> {
+    if (BufferedRpc.initialized) {
+      // We do not want to follow the tip again.
+      throw new Error("BufferedRpc already initialized");
+    }
+    BufferedRpc.initialized = true;
+    const intersect = point ? [point] : [];
+    const blockEvents = this.syncClient.followTip(intersect);
+
+    let seenReset = false;
+
+    for await (const blockEvent of blockEvents) {
+      if (blockEvent.action === "apply") {
+        if (blockEvent.block.header?.height) {
+          BufferedRpc.lastHeight = blockEvent.block.header?.height;
+        }
+        // This works ONLY because the buffer is processed FIFO
+        this.buffer.push({
+          output: blockEvent.block,
+          cleanup: () => this.buffer.shift(),
+        });
+        this.newDataCondVar.wake();
+      } else if (blockEvent.action === "undo") {
+        if (blockEvent.block.header?.height) {
+          BufferedRpc.lastHeight = blockEvent.block.header?.height;
+        }
+        this.buffer.pop();
+      } else if (blockEvent.action === "reset") {
+        if (!seenReset) {
+          seenReset = true;
+          continue;
+        }
+
+        const lastBlock = this.buffer.peekBack();
+        const lastPoint = lastBlock ? JSON.stringify({
+          slot: Number(lastBlock.output.header!.slot),
+          hash: Buffer.from(lastBlock.output.header!.hash).toString("hex"),
+        }) : "<none>";
+        throw new Error(
+          `Paima node stuck on fork. Currently at point ${lastPoint}`,
+        );
+      }
+    }
+  }
+
+  public *storedBlocks(
+    height: undefined | BlockNumber,
+  ): Operation<{ from: Page; to: Page }> {
+    let lastElement = this.buffer.peekBack();
+
+    if (height == null) {
+      height = 0;
+    }
+
+    // if we have nothing in our buffer yet, or if we're asking for a future height
+    while (
+      this.buffer.length == 0 ||
+      lastElement == null ||
+      height >= Number(lastElement.output.header!.height) - this.finality
+    ) {
+      yield* this.newDataCondVar.wait();
+      lastElement = this.buffer.peekBack();
+    }
+
+    const endIndex = (() => {
+      let endIndex = this.buffer.length - 1;
+      const highestValid =
+        Number(this.buffer.peekBack()!.output.header!.height) -
+        this.finality;
+      while (
+        endIndex > 0 &&
+        Number(this.buffer.peekAt(endIndex)!.output.header!.height) >
+          highestValid
+      ) {
+        endIndex--;
+      }
+
+      return endIndex;
+    })();
+    const startIndex = (() => {
+      let startIndex = endIndex;
+      while (
+        startIndex > 0 &&
+        Number(
+            this.buffer.peekAt(startIndex - 1)!.output.header!.height,
+          ) >
+          height
+      ) {
+        startIndex--;
+      }
+      return startIndex;
+    })();
+    return {
+      from: toPage(this.buffer.peekAt(startIndex)!.output),
+      to: toPage(this.buffer.peekAt(endIndex)!.output),
+    };
+  }
+
+  /**
+   * Return an array of blocks
+   * This is preferred over `fetchBlock` where there may be gaps in blockHeight
+   * Or when you don't want a blocking call
+   */
+  public fetchBlocks(
+    from: BlockNumber, // (inclusive)
+    to: BlockNumber, // (inclusive)
+  ): OutputAndCleanup<cardano.Block>[] {
+    const blocks: OutputAndCleanup<cardano.Block>[] = [];
+    for (let i = 0; i < this.buffer.length; i++) {
+      const block = this.buffer.peekAt(i)!;
+      if (Number(block.output.header!.height) >= from) {
+        blocks.push(block);
+      }
+      if (Number(block.output.header!.height) >= to) {
+        break;
+      }
+    }
+    return blocks;
+  }
+}
+
+function toPage(block: cardano.Block): Page {
+  return {
+    slot: Number(block.header!.slot),
+    hash: Buffer.from(block.header!.hash).toString("hex"),
+    height: Number(block.header!.height),
+  };
+}