@effectstream/sync 0.100.0

package/src/sync-protocols/near/fetcher.ts

@@ -0,0 +1,274 @@
+import {
+  ConfigSyncProtocolType,
+  type PrimitiveEntry,
+} from "@effectstream/config";
+import { BaseDataFetcher } from "../base/fetcher.ts";
+import type { DataFetched } from "../base/fetcher.ts";
+import type {
+  LastPage,
+  OutputAndCleanup,
+  RootConversion,
+} from "../base/state.ts";
+import type { RootOutput, RootPage } from "../types.ts";
+import type {
+  ConfigType,
+  Input,
+  Output,
+  Page,
+  PrimitiveType,
+} from "./types.ts";
+import {
+  NearClient,
+  parseNep297Log,
+  type Nep297Event,
+  type NearExecutionOutcomeWithId,
+} from "./NearClient.ts";
+import { call, type Operation } from "effection";
+import { bound } from "@effectstream/utils";
+
+export class NearFetcher extends BaseDataFetcher<
+  Input,
+  Output,
+  RootOutput,
+  Page,
+  RootPage
+> {
+  readonly client: NearClient;
+
+  constructor(
+    readonly config: ConfigType,
+  ) {
+    super(config.syncProtocol.name);
+    this.client = new NearClient(config.network.rpcUrl);
+  }
+
+  @bound
+  override *readData(
+    data: Input,
+    rootConversion: RootConversion<Output, RootOutput, RootPage>,
+    lastPage: LastPage<Page, RootPage> | undefined,
+  ): Operation<DataFetched<Output, Page, RootPage>> {
+    const outputs: OutputAndCleanup<Output>[] = [];
+
+    console.log(
+      `[NEAR] Fetching blocks from ${data.from} to ${data.to}.${
+        data.isPresync ? " [presync]" : ""
+      }`,
+    );
+
+    for (let height = data.from; height <= data.to; height++) {
+      let block;
+      try {
+        block = yield* call(() =>
+          this.client.getBlock({ block_id: height })
+        );
+      } catch (_e) {
+        // Likely reached the chain tip; stop here
+        break;
+      }
+
+      const blockHash = block.header.hash;
+      // NEAR timestamps are in nanoseconds — convert to milliseconds
+      const timestampMs = Math.floor(
+        Number(block.header.timestamp_nanosec ?? block.header.timestamp) / 1_000_000,
+      );
+
+      const primitives = yield* this.readPrimitives(
+        height,
+        blockHash,
+        block.chunks,
+        this.config.primitives,
+      );
+
+      outputs.push({
+        output: {
+          timestamp: timestampMs,
+          height,
+          hash: blockHash,
+          primitives,
+        },
+        cleanup: () => {},
+      });
+    }
+
+    if (outputs.length === 0) {
+      if (!lastPage) {
+        throw new Error(
+          `[NEAR] Could not fetch any blocks from ${data.from} to ${data.to} and no previous page was found.`,
+        );
+      }
+      return {
+        output: [],
+        lastPage,
+      };
+    }
+
+    const lastOutput = outputs[outputs.length - 1].output;
+    return {
+      output: outputs,
+      lastPage: {
+        ownBlockNumber: lastOutput.height,
+        own: {
+          height: lastOutput.height,
+          hash: lastOutput.hash,
+        },
+        root: rootConversion.toRootPage(lastOutput),
+      },
+    };
+  }
+
+  @bound
+  *readPrimitives(
+    height: number,
+    blockHash: string,
+    chunkHeaders: { chunk_hash: string; shard_id: number }[],
+    primitiveEntries: Extract<
+      PrimitiveEntry,
+      { syncProtocol: ConfigSyncProtocolType.NEAR_RPC_PARALLEL }
+    >[],
+  ): Operation<PrimitiveType[]> {
+    if (primitiveEntries.length === 0) return [];
+
+    const allPrimitives: PrimitiveType[] = [];
+
+    // Fetch each chunk and extract events from transactions
+    for (const chunkHeader of chunkHeaders) {
+      let chunk;
+      try {
+        chunk = yield* call(() =>
+          this.client.getChunk(chunkHeader.chunk_hash)
+        );
+      } catch (_e) {
+        // Chunk may be empty or unavailable
+        continue;
+      }
+
+      for (const tx of chunk.transactions) {
+        let txStatus;
+        try {
+          txStatus = yield* call(() =>
+            this.client.getTxStatus(tx.hash, tx.signer_id)
+          );
+        } catch (_e) {
+          continue;
+        }
+
+        // Collect all execution outcomes (transaction outcome + receipt outcomes)
+        const allOutcomes: NearExecutionOutcomeWithId[] = [
+          txStatus.transaction_outcome,
+          ...txStatus.receipts_outcome,
+        ];
+
+        // Event-based primitives (Generic, Intent, NEP-141/171/245) match
+        // NEP-297 logs emitted during execution.
+        for (const outcomeWithId of allOutcomes) {
+          const { outcome } = outcomeWithId;
+
+          for (const log of outcome.logs) {
+            const event = parseNep297Log(log);
+            if (!event) continue;
+
+            for (const entry of primitiveEntries) {
+              const prim = entry.primitive;
+              // AccountWatch primitives have no event metadata; skip them here.
+              if (prim.eventStandard == null) continue;
+              if (!this.matchesPrimitive(outcome.executor_id, event, prim)) {
+                continue;
+              }
+
+              for (let i = 0; i < event.data.length; i++) {
+                allPrimitives.push({
+                  syncProtocol: {
+                    name: entry.syncProtocol,
+                    blockNumber: height,
+                    transactionHash: tx.hash,
+                    contractAddress: outcome.executor_id,
+                    logIndex: i,
+                  },
+                  primitive: prim.name,
+                  output: {
+                    payloadType: `${event.standard}:${event.event}`,
+                    payload: {
+                      ...event.data[i] as Record<string, unknown>,
+                      _standard: event.standard,
+                      _version: event.version,
+                      _event: event.event,
+                    },
+                  },
+                });
+              }
+            }
+          }
+        }
+
+        // AccountWatch primitives capture raw execution-outcome data for any
+        // FunctionCall whose receiver matches the watched contractId, whether
+        // or not a NEP-297 event was logged. This is the non-event path.
+        const txStatusKind = txStatus.transaction_outcome.outcome.status;
+        const txSucceeded =
+          (txStatusKind as { SuccessValue?: string }).SuccessValue !== undefined ||
+          (txStatusKind as { SuccessReceiptId?: string }).SuccessReceiptId !== undefined;
+        const txStatusStr = txSucceeded ? "success" : "failure";
+
+        for (const entry of primitiveEntries) {
+          const prim = entry.primitive;
+          // Identify outcome-based primitives by the absence of event metadata.
+          if (prim.eventStandard != null) continue;
+          if (prim.contractId !== tx.receiver_id) continue;
+
+          for (let actionIndex = 0; actionIndex < tx.actions.length; actionIndex++) {
+            const action = tx.actions[actionIndex];
+            const fc = action.FunctionCall;
+            if (!fc) continue;
+
+            let argsDecoded = "";
+            try {
+              argsDecoded = Buffer.from(fc.args, "base64").toString("utf8");
+            } catch {
+              argsDecoded = fc.args;
+            }
+
+            allPrimitives.push({
+              syncProtocol: {
+                name: entry.syncProtocol,
+                blockNumber: height,
+                transactionHash: tx.hash,
+                contractAddress: tx.receiver_id,
+                logIndex: actionIndex,
+              },
+              primitive: prim.name,
+              output: {
+                payloadType: "account-watch:function-call",
+                payload: {
+                  signer_id: tx.signer_id,
+                  receiver_id: tx.receiver_id,
+                  method_name: fc.method_name,
+                  args: argsDecoded,
+                  deposit: fc.deposit,
+                  status: txStatusStr,
+                },
+              },
+            });
+          }
+        }
+      }
+    }
+
+    return allPrimitives;
+  }
+
+  private matchesPrimitive(
+    executorId: string,
+    event: Nep297Event,
+    prim: {
+      contractId: string;
+      eventStandard?: string;
+      eventType?: string;
+    },
+  ): boolean {
+    if (prim.contractId !== executorId) return false;
+    if (prim.eventStandard && prim.eventStandard !== event.standard) return false;
+    if (prim.eventType && prim.eventType !== event.event) return false;
+    return true;
+  }
+}

package/src/sync-protocols/near/state.ts

@@ -0,0 +1,130 @@
+import { call, type Operation } from "effection";
+import { bound } from "@effectstream/utils";
+import type { PoolClient } from "pg";
+import { type LastPage, SyncState } from "../base/state.ts";
+import type { RootOutput, RootPage } from "../types.ts";
+import type { Input, Output, Page } from "./types.ts";
+import { pageRelation } from "./types.ts";
+import type { NearFetcher } from "./fetcher.ts";
+import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/config";
+import { getPage } from "@effectstream/db";
+import { NearClient } from "./NearClient.ts";
+import { applyDelay } from "../common/utils.ts";
+
+export class NearSyncState extends SyncState<
+  Input,
+  Output,
+  Page,
+  RootOutput,
+  RootPage,
+  NearFetcher
+> {
+  constructor(
+    lastPage: LastPage<Page, RootPage> | undefined,
+    readonly config: Extract<
+      SyncProtocolWithNetwork,
+      { networkType: ConfigNetworkType.NEAR }
+    >,
+    fetcher: NearFetcher,
+    public readonly client: NearClient,
+    dbConn: PoolClient,
+  ) {
+    super(
+      config.syncProtocol.name,
+      lastPage,
+      fetcher,
+      pageRelation,
+      dbConn,
+    );
+  }
+
+  @bound
+  override toPage(_input: Input, data: Output[]): Page {
+    const lastBlock = data[data.length - 1];
+    return {
+      height: lastBlock.height,
+      hash: lastBlock.hash,
+    };
+  }
+
+  @bound
+  override toRootPage(data: Output): RootPage {
+    // NEAR timestamps are already in ms (converted in fetcher)
+    return applyDelay(data.timestamp, this.config.syncProtocol.delayMs);
+  }
+
+  @bound
+  override toRootOutput(_data: Output): RootOutput {
+    throw new Error("Only main chains create root outputs");
+  }
+
+  @bound
+  override *stateToInput(): Operation<Input | undefined> {
+    const latestHeight = yield* call(() =>
+      this.client.getLatestBlockHeight()
+    );
+    const startHeight = this.lastPage?.own.height ??
+      this.config.syncProtocol.startBlockHeight - 1;
+
+    if (startHeight >= latestHeight) {
+      return undefined;
+    }
+
+    const from = startHeight + 1;
+    const to = Math.min(
+      from + this.config.syncProtocol.stepSize - 1,
+      latestHeight,
+    );
+
+    return {
+      from,
+      to,
+      isPresync: false,
+    };
+  }
+
+  @bound
+  override mergeDatum(ourOutput: Output, rootOutput: RootOutput): void {
+    const primitives = ourOutput.primitives.map((p) => ({
+      ...p,
+      source: this.config.syncProtocol.name,
+    }));
+    const blockInfo = [{
+      protocol_name: this.config.syncProtocol.name,
+      block_number: ourOutput.height,
+      blockHash: ourOutput.hash,
+    }];
+    rootOutput.blockInfo.push(...blockInfo);
+    rootOutput.primitives.push(...primitives);
+  }
+
+  @bound
+  override getNamespace(): string[] {
+    return [this.config.network.name, this.config.syncProtocol.name];
+  }
+
+  static *restoreState(
+    dbConn: PoolClient,
+    config: Extract<
+      SyncProtocolWithNetwork,
+      { networkType: ConfigNetworkType.NEAR }
+    >,
+    fetcher: NearFetcher,
+  ): Operation<NearSyncState> {
+    const [result] = yield* call(async () =>
+      await getPage.run({
+        protocol_name: config.syncProtocol.name,
+      }, dbConn)
+    );
+    const page = result
+      ? result.page as unknown as LastPage<Page, RootPage>
+      : undefined;
+    return new NearSyncState(
+      page,
+      config,
+      fetcher,
+      new NearClient(config.network.rpcUrl),
+      dbConn,
+    );
+  }
+}

package/src/sync-protocols/near/types.ts

@@ -0,0 +1,46 @@
+import type {
+  ConfigSyncProtocolType,
+  FlattenSyncProtocolIOFor,
+  SyncProtocolWithNetwork,
+} from "@effectstream/config";
+import type { BlockNumber } from "@effectstream/utils";
+import type { PageSyncRange } from "../common/page-helpers.ts";
+import type { PageRelation } from "../base/page.ts";
+
+// =====================
+// Sync protocol typings
+// =====================
+
+/** The page identifier for NEAR: a block height + hash pair */
+export type Page = {
+  height: BlockNumber;
+  hash: string;
+};
+
+export type PrimitiveType = FlattenSyncProtocolIOFor<
+  ConfigSyncProtocolType.NEAR_RPC_PARALLEL
+>;
+
+export type Input = PageSyncRange<BlockNumber>;
+
+export type Output = {
+  /** Block timestamp in milliseconds (converted from NEAR nanoseconds) */
+  timestamp: number;
+  /** Block height */
+  height: number;
+  /** Base58 block hash */
+  hash: string;
+  primitives: PrimitiveType[];
+};
+
+export const pageRelation: PageRelation<Page> = {
+  compare: (p1, p2) => p1.height - p2.height,
+  equals: (p1, p2) => p1.height === p2.height,
+  min: (p1, p2) => (p1.height < p2.height ? p1 : p2),
+  max: (p1, p2) => (p1.height > p2.height ? p1 : p2),
+};
+
+export type ConfigType = Extract<
+  SyncProtocolWithNetwork,
+  { syncProtocolType: ConfigSyncProtocolType.NEAR_RPC_PARALLEL }
+>;

package/src/sync-protocols/ntp/fetcher.ts

@@ -0,0 +1,199 @@
+import {
+  BaseDataFetcher,
+  type DataFetched,
+  type PaginatedFetcher,
+} from "../base/fetcher.ts";
+import type { RootOutput, RootPage } from "../types.ts";
+import type { Operation } from "effection";
+import { all, call, until } from "effection";
+import { bound, type NtpBlockHash, type NtpPageJson } from "@effectstream/utils";
+import { blockNumberRelation } from "../common/utils.ts";
+import type { Input, Output, Page, PrimitiveType } from "./types.ts";
+import {
+  fetchNewestPage,
+  genImmediatePageRequests,
+  genOnDemandPageRequests,
+  type PageRange,
+  type PageRequest,
+} from "../base/page.ts";
+import type { PrimitiveFetcher } from "../base/primitive.ts";
+import type { RootConversion } from "../base/state.ts";
+import type {
+  ConfigNetworkType,
+  PrimitiveEntry,
+  SyncProtocolWithNetwork,
+} from "@effectstream/config";
+import type { ConfigSyncProtocolType } from "@effectstream/config";
+import { NtpTimeSync, type NtpTimeSyncDefaultOptions } from "ntp-time-sync";
+
+type RecursivePartial<T> = {
+  [P in keyof T]?: RecursivePartial<T[P]>;
+};
+
+export class NtpFetcher
+  extends BaseDataFetcher<Input, Output, RootOutput, Page, RootPage>
+  implements
+    PrimitiveFetcher<
+      Input,
+      Page,
+      { blockNumber: number; timestamp: bigint; hash: string },
+      PrimitiveType,
+      ConfigSyncProtocolType.NTP_MAIN
+    >,
+    PaginatedFetcher<Page> {
+  readonly ntpTimeSync: NtpTimeSync;
+  constructor(
+    readonly config: Extract<
+      SyncProtocolWithNetwork,
+      { networkType: ConfigNetworkType.NTP }
+    >,
+  ) {
+    super(config.syncProtocol.name);
+
+    const options: RecursivePartial<typeof NtpTimeSyncDefaultOptions> = {
+      ntpDefaults: {
+        minPoll: 10,
+      },
+    };
+    if (config.network.servers && config.network.servers.length) {
+      options.servers = config.network.servers;
+    }
+    // Cannot select options
+    // https://github.com/buffcode/ntp-time-sync/issues/104
+    this.ntpTimeSync = NtpTimeSync.getInstance();
+  }
+
+  @bound
+  override *readData(
+    data: Input,
+    rootConversion: RootConversion<Output, RootOutput, RootPage>,
+  ): Operation<DataFetched<Output, Page, RootPage>> {
+    const pageFetcher = (() => {
+      if (data.isPresync) {
+        return genOnDemandPageRequests(
+          data.from,
+          data.to,
+          async (page) => {
+            const diff = BigInt(this.config.network.blockTimeMS * page);
+            const timestamp = BigInt(this.config.network.startTime) + diff;
+            return {
+              blockNumber: page,
+              timestamp,
+              hash: `0x${timestamp.toString(16)}`,
+            };
+          },
+          blockNumberRelation,
+        );
+      }
+      return genImmediatePageRequests(
+        Array.from(
+          { length: data.to - data.from + 1 },
+          (_, i) => i + data.from,
+        ),
+        async (page) => {
+          const diff = BigInt(this.config.network.blockTimeMS * page);
+          const timestamp = BigInt(this.config.network.startTime) + diff;
+          return {
+            blockNumber: page,
+            timestamp,
+            hash: `0x${timestamp.toString(16)}`,
+          };
+        },
+      );
+    })();
+
+    const output = yield* all(
+      Array.from(
+        { length: data.to - data.from + 1 },
+        (_, i) => i + data.from,
+      ).map(function* (page: Page) {
+        const raw = yield* call(() => pageFetcher(page));
+        const blockHashes = [raw.hash] as NtpBlockHash[];
+        return {
+          raw,
+          blockHashes,
+        };
+      }),
+    );
+    return {
+      output: output.map((o) => ({
+        output: o,
+        cleanup: () => {}, // no cleanup required
+      })),
+      lastPage: {
+        ownBlockNumber: Number(data.to),
+        own: Number(data.to),
+        root: rootConversion.toRootPage({
+          blockHashes: [] as NtpBlockHash[],
+          raw: (yield* call(() => pageFetcher(Number(data.to)))),
+        }),
+      },
+    };
+  }
+
+  @bound
+  groupByPage(
+    primitives: PrimitiveType[],
+  ): Record<NtpPageJson, PrimitiveType[]> {
+    // Not used in NTP
+    return {};
+  }
+
+  @bound
+  *readPrimitives(
+    data: Input,
+    pageRequest: PageRequest<Page, { timestamp: bigint; hash: string }>,
+    primitiveEntries: Extract<
+      PrimitiveEntry,
+      { syncProtocol: ConfigSyncProtocolType.NTP_MAIN }
+    >[],
+  ): Operation<PrimitiveType[]> {
+    return [];
+  }
+
+  @bound
+  *getLatestPage(
+    knownLastPage: undefined | Page,
+  ): Operation<Page> {
+    const self = this;
+    return yield* fetchNewestPage<Page>(
+      blockNumberRelation,
+      function* (): Operation<Page> {
+        const result = yield* until(self.ntpTimeSync.getTime());
+        if (result.offset > 5000) {
+          console.error(
+            `NTP offset is higher than 5[s]. Please check your NTP server & system time configuration.
+NTP time: ${result.now}
+Local time: ${new Date()}
+Offset [ms]: ${result.offset}`,
+          );
+        }
+        const diff = result.now.getTime() - self.config.network.startTime;
+        const blockNumber = Math.floor(diff / self.config.network.blockTimeMS);
+        return blockNumber;
+      },
+      knownLastPage,
+      this.config.syncProtocol.pollingInterval,
+    );
+  }
+
+  @bound
+  previousInterval(nextIntervalStart: Page): PageRange<Page> {
+    return this.intervalFromStart(
+      nextIntervalStart - this.config.syncProtocol.stepSize - 1,
+    );
+  }
+
+  @bound
+  nextInterval(prevIntervalEnd: Page): PageRange<Page> {
+    return this.intervalFromStart(prevIntervalEnd + 1);
+  }
+
+  @bound
+  intervalFromStart(start: Page): PageRange<Page> {
+    return {
+      from: start,
+      to: start + this.config.syncProtocol.stepSize,
+    };
+  }
+}