@effectstream/sync 0.100.0

package/src/sync-protocols/midnight/MidnightClient.ts

@@ -0,0 +1,210 @@
+import { indexerPublicDataProvider } from "@midnight-ntwrk/midnight-js-indexer-public-data-provider";
+import type { Block } from "./types.ts";
+import type { ExecutionResult } from "graphql-ws/client";
+
+type MidnightGqlBlock = {
+  block: Block;
+};
+
+
+export interface MidnightGqlBlockState {
+  block: {
+    hash:            string;
+    height:          number;
+    protocolVersion: number;
+    timestamp:       number;
+    parent:          {
+      hash: string;
+    };
+    transactions:    {
+      hash:            string;
+      contractActions: {
+        address:    string;
+        state:      string;
+      }[];
+      zswapLedgerEvents?: {
+        id:    number;
+        raw:   string;
+        maxId: number;
+      }[];
+    }[];
+  };
+};
+
+export interface BlockFetchOptions {
+  /** Include contractActions in the transaction fields (needed for MidnightGenericPrimitive). Default: true */
+  contractActions?: boolean;
+  /** Include zswapLedgerEvents in the transaction fields (needed for MidnightNullifierPrimitive). Default: true */
+  zswapLedgerEvents?: boolean;
+}
+
+type PublicDataProvider = ReturnType<typeof indexerPublicDataProvider>;
+export class MidnightClient {
+  private readonly queryURL: string;
+  private readonly subscriptionURL: string;
+  private publicDataProvider: PublicDataProvider;
+  private readonly networkId?: string;
+  /** Timeout in milliseconds for individual HTTP requests to the indexer. */
+  private readonly requestTimeoutMs: number;
+
+  constructor(queryURL: string, subscriptionURL: string, networkId?: string, requestTimeoutMs = 30_000) {
+    this.queryURL = queryURL;
+    this.subscriptionURL = subscriptionURL;
+    this.networkId = networkId;
+    this.requestTimeoutMs = requestTimeoutMs;
+    this.publicDataProvider = indexerPublicDataProvider(
+      queryURL,
+      subscriptionURL,
+    );
+    console.log(
+      `[MidnightClient] Using indexer ${queryURL} (network: ${
+        networkId ?? "unknown"
+      })`,
+    );
+  }
+
+  /**
+   * Recreate the publicDataProvider to recover from stale internal state.
+   * The Midnight JS library can get into a state where all queryContractState
+   * calls fail with {"json":{}} until the provider is recreated.
+   */
+  resetProvider(): void {
+    console.log(
+      `[MidnightClient] Resetting publicDataProvider for ${this.queryURL}`,
+    );
+    this.publicDataProvider = indexerPublicDataProvider(
+      this.queryURL,
+      this.subscriptionURL,
+    );
+  }
+
+  async fetchContractState(contractAddress: string, blockHeight: number) {
+    try {
+      const state = await Promise.race([
+        this.publicDataProvider.queryContractState(
+          contractAddress,
+          {
+            type: "blockHeight",
+            blockHeight,
+          },
+        ),
+        new Promise<never>((_, reject) => {
+          setTimeout(
+            () => reject(new Error(`fetchContractState timed out after ${this.requestTimeoutMs}ms (contract=${contractAddress}, height=${blockHeight})`)),
+            this.requestTimeoutMs,
+          );
+        }),
+      ]);
+      return state;
+    } catch (error) {
+      // The Midnight JS publicDataProvider can enter a broken state where all
+      // subsequent calls fail (e.g. {"json":{}}).  Recreate it so the next
+      // retry starts with a fresh connection.
+      this.resetProvider();
+      throw error;
+    }
+  }
+
+  async gqlQuery(query: string, signal?: AbortSignal): Promise<any> {
+    const timeoutSignal = AbortSignal.timeout(this.requestTimeoutMs);
+    const combinedSignal = signal
+      ? AbortSignal.any([signal, timeoutSignal])
+      : timeoutSignal;
+    const response = await fetch(this.queryURL, {
+      method: "POST",
+      body: JSON.stringify({ query }),
+      headers: {
+        "Content-Type": "application/json",
+      },
+      signal: combinedSignal,
+    });
+
+    if (!response.ok) {
+      console.error(`Failed to fetch ${this.queryURL}`);
+      console.error("Query:", JSON.stringify(query));
+      console.error("Response:", await response.text());
+      throw new GraphQLError(
+        `Server returned ${response.status} ${response.statusText}`,
+      );
+    }
+
+    const body = await response.json();
+    if ("errors" in body) {
+      throw new GraphQLError(
+        "Server returned errors",
+        body.errors as GraphQLErrorDetail[],
+      );
+    }
+    if ("data" in body) {
+      return body.data;
+    }
+    throw new GraphQLError("Server returned nothing");
+  }
+
+  handleGqlWsError<T>(
+    ex: IteratorResult<ExecutionResult<T, unknown>, unknown>,
+  ): T {
+    if (ex.done) throw new GraphQLError("Subscription ended");
+    if (ex.value.errors) {
+      throw new GraphQLError("Subscription errored", ex.value.errors);
+    }
+    if (!ex.value.data) throw new GraphQLError("Server returned nothing");
+    return ex.value.data;
+  }
+
+  async fetchBlock(
+    blockHeight: number,
+    options: BlockFetchOptions = {},
+    signal?: AbortSignal,
+  ): Promise<MidnightGqlBlockState> {
+    const { contractActions = true, zswapLedgerEvents = true } = options;
+    const contractActionsField = contractActions
+      ? `contractActions { address state }`
+      : "";
+    const zswapField = zswapLedgerEvents
+      ? `zswapLedgerEvents { id raw maxId }`
+      : "";
+    const query = `query {
+      block(offset: { height: ${blockHeight} }) {
+        hash
+        height
+        protocolVersion
+        timestamp
+        parent {
+          hash
+        }
+        transactions {
+          hash
+          ${contractActionsField}
+          ${zswapField}
+        }
+      }
+    }`;
+    return await this.gqlQuery(query, signal);
+  }
+
+  async fetchLatestBlock() {
+    const query = `query {
+      block {
+        height
+      }
+    }`;
+    return await this.gqlQuery(query);
+  }
+}
+
+interface GraphQLErrorDetail {
+  message: string;
+  locations?: readonly { line: number; column: number }[];
+  path?: readonly (string | number)[];
+  extensions?: object;
+}
+
+class GraphQLError extends Error {
+  errors?: readonly GraphQLErrorDetail[];
+
+  constructor(message: string, errors?: readonly GraphQLErrorDetail[]) {
+    super(message);
+    this.errors = errors;
+  }
+}

package/src/sync-protocols/midnight/fetcher.ts

@@ -0,0 +1,252 @@
+import { BaseDataFetcher } from "../base/fetcher.ts";
+import type {
+  Block,
+  ConfigType,
+  Input,
+  Output,
+  Page,
+  PrimitiveEntryType,
+  PrimitiveType,
+} from "./types.ts";
+import { all, call, type Operation, useAbortSignal } from "effection";
+import type { DataFetched } from "../base/fetcher.ts";
+import type {
+  OutputAndCleanup,
+  RootConversion,
+} from "../base/state.ts";
+import type { RootOutput, RootPage } from "../types.ts";
+import { bound } from "@effectstream/utils";
+import { MidnightClient, type BlockFetchOptions, type MidnightGqlBlockState } from "./MidnightClient.ts";
+import { ContractState, StateValue } from "@midnight-ntwrk/onchain-runtime";
+import { decodeZswapInputEvent } from "./zswap-decoder.ts";
+
+export class MidnightFetcher extends BaseDataFetcher<
+  Input,
+  Output,
+  RootOutput,
+  Page,
+  RootPage
+> {
+  readonly client: MidnightClient;
+  private readonly networkId?: string;
+  constructor(
+    readonly config: ConfigType,
+  ) {
+    super(config.syncProtocol.name);
+    const indexerHttp = config.syncProtocol.indexer;
+    const indexerWs =
+      (config.syncProtocol as any).indexerWS ??
+      (config.syncProtocol as any).indexerWs;
+    if (!indexerHttp || !indexerWs) {
+      throw new Error(
+        `Midnight sync protocol "${
+          config.syncProtocol.name
+        }" requires both indexer and indexerWS URLs. Received indexer=${
+          indexerHttp ?? "undefined"
+        }, indexerWS=${indexerWs ?? "undefined"}.`,
+      );
+    }
+    this.networkId = config.network?.networkId ??
+      (config.network as any)?.id;
+
+    if (this.networkId != null) {
+      for (const entry of config.primitives) {
+        const primitiveNetworkId = (entry.primitive as any).networkId;
+        if (primitiveNetworkId != null && primitiveNetworkId !== this.networkId) {
+          throw new Error(
+            `Midnight primitive "${entry.primitive.name}" has networkId "${primitiveNetworkId}" ` +
+            `but the network "${config.network.name}" uses networkId "${this.networkId}". ` +
+            `These must match.`,
+          );
+        }
+      }
+    }
+
+    this.client = new MidnightClient(
+      indexerHttp,
+      indexerWs,
+      this.networkId,
+    );
+  }
+
+  @bound
+  override *readData(
+    data: Input,
+    rootConversion: RootConversion<Output, RootOutput, RootPage>,
+  ): Operation<DataFetched<Output, Page, RootPage>> {
+    const outputs: OutputAndCleanup<Output>[] = [];
+    console.log(
+      `[Midnight${this.networkId ? `:${this.networkId}` : ""}] Fetching blocks from ${data.from} to ${data.to}. ${
+        data.isPresync ? "[presync]" : ""
+      }`,
+    );
+    const blockFetchOptions: BlockFetchOptions = {
+      contractActions: this.config.primitives.some(
+        (p) => p.primitive.type !== "Midnight:Nullifier",
+      ),
+      zswapLedgerEvents: this.config.primitives.some(
+        (p) => p.primitive.type === "Midnight:Nullifier",
+      ),
+    };
+    const self = this;
+    const heights = Array.from(
+      { length: data.to - data.from + 1 },
+      (_, i) => i + data.from,
+    );
+    const fetched = yield* all(
+      // The endpoint does not support range requests,
+      // so we fetch each block individually in parallel.
+      // useAbortSignal() ties each fetch's lifetime to its Effection scope,
+      // so cancellation by all() cleanly aborts the underlying HTTP request.
+      heights.map(function* (height) {
+        const signal = yield* useAbortSignal();
+        const result: MidnightGqlBlockState = yield* call(() =>
+          self.client.fetchBlock(height, blockFetchOptions, signal)
+        );
+        const primitives = yield* self.readPrimitives(
+          height,
+          result,
+          self.config.primitives,
+        );
+        return {
+          output: {
+            raw: result.block as unknown as Block,
+            primitives,
+          },
+          cleanup: () => {},
+        };
+      }),
+    );
+    outputs.push(...fetched);
+
+    const lastOutput = outputs[outputs.length - 1].output;
+    return {
+      output: outputs,
+      lastPage: {
+        ownBlockNumber: lastOutput.raw.height,
+        own: {
+          height: lastOutput.raw.height,
+          hash: lastOutput.raw.hash,
+        },
+        root: rootConversion.toRootPage(lastOutput),
+      },
+    };
+  }
+
+  @bound
+  *readPrimitives(
+    height: number,
+    block: MidnightGqlBlockState,
+    primitiveEntries: PrimitiveEntryType[],
+  ): Operation<PrimitiveType[]> {
+    const client = this.client;
+    const asyncOps: Operation<PrimitiveType[]>[] = [];
+    const syncResults: PrimitiveType[] = [];
+
+    for (const primitiveEntry of primitiveEntries) {
+      if (primitiveEntry.primitive.type === "Midnight:Nullifier") {
+        syncResults.push(...this.fetchNullifiers(height, primitiveEntry, block));
+      } else {
+        asyncOps.push(
+          this.fetchContractState(height, client, primitiveEntry, block),
+        );
+      }
+    }
+
+    const asyncResults = (yield* all(asyncOps)).flat();
+    return [...syncResults, ...asyncResults].filter(Boolean) as PrimitiveType[];
+  }
+
+  @bound
+  fetchNullifiers(
+    height: number,
+    primitiveEntry: PrimitiveEntryType,
+    block: MidnightGqlBlockState,
+  ): PrimitiveType[] {
+    const results: PrimitiveType[] = [];
+    for (const tx of block.block.transactions) {
+      for (const event of tx.zswapLedgerEvents ?? []) {
+        const decoded = decodeZswapInputEvent(event.raw);
+        if (!decoded) continue;
+        results.push({
+          syncProtocol: {
+            name: primitiveEntry.syncProtocol,
+            blockNumber: height,
+            transactionHash: tx.hash,
+            contractAddress: "",
+          },
+          primitive: primitiveEntry.primitive.name,
+          output: {
+            payloadType: "midnight-nullifier",
+            payload: {
+              nullifier: decoded.nullifier,
+              txHash: decoded.txHash,
+              eventId: event.id,
+              logicalSegment: decoded.logicalSegment,
+            },
+          },
+        });
+      }
+    }
+    return results;
+  }
+
+  @bound
+  *fetchContractState(
+    height: number,
+    client: MidnightClient,
+    primitiveEntry: PrimitiveEntryType,
+    block: MidnightGqlBlockState,
+  ): Operation<PrimitiveType[]> {
+    const contractAddress = primitiveEntry.primitive.contractAddress!;
+    const blockFinalState = yield* call(() =>
+      client.fetchContractState(contractAddress, height)
+    );
+    // The state returns null if the contract was not called in the block height
+    if (!blockFinalState) {
+      return [];
+    }
+
+    const transactions = block.block.transactions.filter((t) => {
+      return (t.contractActions ?? []).some((c) => {
+        const longest = Math.max(contractAddress.length, c.address.length);
+        // Addresses length can be padded by 0's
+        return c.address.padStart(longest, '0') === contractAddress.padStart(longest, '0');
+      });
+    });
+
+
+    return transactions.map(t => {
+      // TODO: What does it mean if t.contractActions has more than one element?
+      const rawState: string = t.contractActions.find((c) => {
+        const longest = Math.max(contractAddress.length, c.address.length);
+        // Addresses length can be padded by 0's
+        return c.address.padStart(longest, '0') === contractAddress.padStart(longest, '0');
+      })!.state!;
+      const byteState = new Uint8Array(rawState.match(/.{1,2}/g)!.map(byte => parseInt(byte, 16)));
+      const primitive = primitiveEntry.primitive;
+      const contractState = ContractState.deserialize(byteState);
+      const stateValue = contractState.data.state;
+      // Parse additional fields from the schema (defined per-contract on the primitive)
+      const additionalFields = primitive.parseAdditionalLedgerFields?.(stateValue) ?? {};
+      // Keep generated ledger fields, but let schema-parsed fields override.
+      // This is important for map-like fields where contract.ledger() exposes
+      // iterable wrappers that are not directly JSON-serializable.
+      const f = primitive.contract ?? { ledger: (_: StateValue): Record<string, any> => ({}) };
+      const state = { ...f.ledger(stateValue), ...additionalFields };
+      return {
+        syncProtocol: {
+          name: primitiveEntry.syncProtocol,
+          blockNumber: height,
+          transactionHash: t.hash,
+          contractAddress: contractAddress,
+        },
+        primitive: primitiveEntry.primitive.name,
+        output: {
+          payloadType: "midnight-contract-state",
+          payload: state,
+        },
+      }
+    });
+  }
+}

package/src/sync-protocols/midnight/state.ts

@@ -0,0 +1,147 @@
+import { call, type Operation } from "effection";
+import { bound, type TimestampMs } from "@effectstream/utils";
+import type { PoolClient } from "pg";
+import { type LastPage, SyncState } from "../base/state.ts";
+import type { RootOutput, RootPage } from "../types.ts";
+import type { Input, Output, Page } from "./types.ts";
+import { pageRelation } from "./types.ts";
+import type { MidnightFetcher } from "./fetcher.ts";
+import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/config";
+import { getPage } from "@effectstream/db";
+import { MidnightClient } from "./MidnightClient.ts";
+import { applyDelay } from "../common/utils.ts";
+
+type LatestBlock = {
+  block: {
+    height: number;
+  };
+};
+
+export class MidnightSyncState extends SyncState<
+  Input,
+  Output,
+  Page,
+  RootOutput,
+  RootPage,
+  MidnightFetcher
+> {
+  private readonly url: string;
+
+  constructor(
+    lastPage: LastPage<Page, RootPage> | undefined,
+    readonly config: Extract<
+      SyncProtocolWithNetwork,
+      { networkType: ConfigNetworkType.MIDNIGHT }
+    >,
+    fetcher: MidnightFetcher,
+    public readonly client: MidnightClient,
+    dbConn: PoolClient,
+  ) {
+    super(
+      config.syncProtocol.name,
+      lastPage,
+      fetcher,
+      pageRelation,
+      dbConn,
+    );
+    this.url = config.syncProtocol.indexer;
+  }
+
+  @bound
+  override toPage(input: Input, data: Output[]): Page {
+    const lastBlock = data[data.length - 1];
+    return {
+      height: lastBlock.raw.height,
+      hash: lastBlock.raw.hash,
+    };
+  }
+
+  @bound
+  override toRootPage(data: Output): RootPage {
+    const time = data.raw.timestamp as unknown as TimestampMs;
+    return applyDelay(
+      time,
+      this.config.syncProtocol.delayMs,
+    );
+  }
+
+  @bound
+  override toRootOutput(data: Output): RootOutput {
+    throw new Error("Only main chains create root outputs");
+  }
+
+  @bound
+  override *stateToInput(): Operation<Input | undefined> {
+    const latestBlockResult = yield* call(() => this.client.fetchLatestBlock());
+    const latestHeight = latestBlockResult.block.height;
+
+    const startHeight = this.lastPage?.own.height ??
+      this.config.syncProtocol.startBlockHeight - 1;
+
+    if (startHeight >= latestHeight) {
+      return undefined;
+    }
+
+    const from = startHeight + 1;
+    const to = Math.min(
+      from + this.config.syncProtocol.stepSize - 1,
+      latestHeight,
+    );
+
+    return {
+      from,
+      to,
+      isPresync: false, // TODO: handle presync
+    };
+  }
+
+  @bound
+  override mergeDatum(ourOutput: Output, rootOutput: RootOutput): void {
+    const primitives = ourOutput.primitives.map((p) => ({
+      ...p,
+      source: this.config.syncProtocol.name,
+    }));
+    const blockInfo = [{
+      protocol_name: this.config.syncProtocol.name,
+      block_number: ourOutput.raw.height,
+      blockHash: ourOutput.raw.hash,
+    }];
+    rootOutput.blockInfo.push(...blockInfo);
+    rootOutput.primitives.push(...primitives);
+  }
+
+  @bound
+  override getNamespace(): string[] {
+    return [this.config.network.name, this.config.syncProtocol.name];
+  }
+
+  static *restoreState(
+    dbConn: PoolClient,
+    config: Extract<
+      SyncProtocolWithNetwork,
+      { networkType: ConfigNetworkType.MIDNIGHT }
+    >,
+    fetcher: MidnightFetcher,
+  ): Operation<MidnightSyncState> {
+    const [result] = yield* call(async () =>
+      await getPage.run({
+        protocol_name: config.syncProtocol.name,
+      }, dbConn)
+    );
+    const page = result
+      ? result.page as unknown as LastPage<Page, RootPage>
+      : undefined;
+    return new MidnightSyncState(
+      page,
+      config,
+      fetcher,
+      // TODO: The urls should be part of the config.
+      new MidnightClient(
+        config.syncProtocol.indexer,
+        config.syncProtocol.indexerWS ??
+          "ws://127.0.0.1:8088/api/v3/graphql/ws",
+      ),
+      dbConn,
+    );
+  }
+}