@unlink-xyz/core 0.1.0 → 0.1.2

package/transactions/note-sync.ts

@@ -0,0 +1,452 @@
+import { poseidon } from "@railgun-community/circomlibjs";
+
+import type { ZkAccount } from "../account/zkAccount.js";
+import type { FetchLike } from "../clients/http.js";
+import { createIndexerClient } from "../clients/indexer.js";
+import type { CommitmentRecord } from "../clients/indexer.js";
+import { serviceConfig } from "../config.js";
+import { CoreError } from "../errors.js";
+import { ByteLength, ByteUtils } from "../key-derivation/bytes.js";
+import {
+  createMerkleTrees,
+  rebuildTreeFromStore,
+  type LeafRecord,
+  type LocalMerkleTrees,
+  type NoteRecord,
+  type RootRecord,
+} from "../state/index.js";
+import { formatUint256, parseHexToBigInt } from "../utils/bigint.js";
+import { decryptNote } from "../utils/crypto.js";
+import { ensureChainId } from "../utils/validators.js";
+import type { Ciphertext } from "./types.js";
+
+const DEFAULT_BATCH_SIZE = 256;
+
+export type NoteSyncStateStore = {
+  putLeaf(record: LeafRecord): Promise<void>;
+  getLeaf(chainId: number, index: number): Promise<LeafRecord | null>;
+  clearLeaves(chainId: number): Promise<void>;
+  putRoot(record: RootRecord): Promise<void>;
+  putNote(record: NoteRecord): Promise<void>;
+  getNote(chainId: number, index: number): Promise<NoteRecord | null>;
+  markNoteSpent(
+    chainId: number,
+    index: number,
+    spentAt?: number,
+  ): Promise<NoteRecord>;
+  putCiphertext(
+    chainId: number,
+    index: number,
+    payload: Uint8Array,
+  ): Promise<void>;
+  countNullifiers(chainId: number): Promise<number>;
+  putNullifier(record: {
+    chainId: number;
+    nullifier: string;
+    noteIndex?: number;
+  }): Promise<void>;
+};
+
+export type NoteSyncOptions = {
+  merkleTrees?: LocalMerkleTrees;
+  indexerClient?: ReturnType<typeof createIndexerClient>;
+  fetch?: FetchLike;
+  limit?: number;
+};
+
+type ChainSyncStatus = {
+  inFlight: boolean;
+  lastSuccess: number | null;
+  lastError?: string;
+};
+
+function buildCiphertext(bytes: [string, string, string]): Ciphertext {
+  return {
+    data: [
+      parseHexToBigInt(bytes[0]),
+      parseHexToBigInt(bytes[1]),
+      parseHexToBigInt(bytes[2]),
+    ],
+  };
+}
+
+function encodeCiphertext(ciphertext: Ciphertext): Uint8Array {
+  const payload = new Uint8Array(ByteLength.UINT_256 * ciphertext.data.length);
+  ciphertext.data.forEach((value, index) => {
+    const start = index * ByteLength.UINT_256;
+    payload.set(ByteUtils.hexStringToBytes(formatUint256(value)), start);
+  });
+  return payload;
+}
+
+export function createNoteSyncService(
+  stateStore: NoteSyncStateStore,
+  options: NoteSyncOptions = {},
+) {
+  const trees = options.merkleTrees ?? createMerkleTrees();
+  const limit = options.limit ?? DEFAULT_BATCH_SIZE;
+  const status = new Map<number, ChainSyncStatus>();
+
+  // Initialize indexer client
+  let indexerClient = options.indexerClient;
+  if (!indexerClient) {
+    const fetchImpl =
+      options.fetch ?? (typeof fetch === "function" ? fetch : undefined);
+    if (!fetchImpl) {
+      throw new Error(
+        "fetch dependency is required to sync indexer commitments",
+      );
+    }
+    indexerClient = createIndexerClient(serviceConfig.indexerBaseUrl, {
+      fetch: fetchImpl,
+    });
+  }
+
+  // Count-based optimization: avoid per-nullifier lookups when counts are unchanged
+  async function shouldQueryNullifiers(chainId: number) {
+    const [onChainCount, localCount] = await Promise.all([
+      indexerClient!.getNullifierCount(chainId),
+      stateStore.countNullifiers(chainId),
+    ]);
+    return onChainCount !== localCount;
+  }
+
+  function createNoteRecord(params: {
+    chainId: number;
+    index: number;
+    commitment: string;
+    token: string;
+    amount: bigint;
+    npk: bigint;
+    mpk: bigint;
+    random: bigint;
+    nullifier: string;
+    spentAt?: number;
+  }): NoteRecord {
+    const { chainId, index, commitment, token, amount, npk, mpk, random, nullifier, spentAt } =
+      params;
+    const base: NoteRecord = {
+      chainId,
+      index,
+      token,
+      value: amount.toString(),
+      commitment,
+      npk: formatUint256(npk),
+      mpk: formatUint256(mpk),
+      random: formatUint256(random),
+      nullifier,
+    };
+    return spentAt === undefined ? base : { ...base, spentAt };
+  }
+
+  async function recordCommitment(
+    chainId: number,
+    record: CommitmentRecord,
+    allowRelaxedOrder = false,
+  ) {
+    const value = parseHexToBigInt(record.commitment);
+    const currentLeafCount = trees.getLeafCount(chainId);
+    if (record.index >= currentLeafCount) {
+      const { index } = trees.addLeaf(chainId, value);
+      if (index !== record.index && !allowRelaxedOrder) {
+        throw new CoreError("indexed commitments out of order");
+      }
+      await stateStore.putLeaf({
+        chainId,
+        index: record.index,
+        commitment: record.commitment,
+      });
+    }
+    await stateStore.putRoot({ chainId, root: record.root });
+  }
+
+  async function tryStoreNote(
+    chainId: number,
+    record: CommitmentRecord,
+    account: ZkAccount,
+    shouldCheckNullifiers: boolean,
+  ) {
+    const ciphertext = buildCiphertext(record.ciphertext);
+    let note;
+    try {
+      note = decryptNote(ciphertext, account.masterPublicKey);
+    } catch (err) {
+      if (err instanceof CoreError) {
+        return null;
+      }
+      throw err;
+    }
+    const random = note.random;
+    const amount = note.amount;
+    const token = note.token;
+    const npk = poseidon([account.masterPublicKey, random]);
+    const commitment = poseidon([npk, BigInt(token), amount]);
+    if (commitment !== parseHexToBigInt(record.commitment)) {
+      return null;
+    }
+
+    const nullifier = poseidon([account.nullifyingKey, BigInt(record.index)]);
+    const nullifierHex = formatUint256(nullifier);
+
+    const ciphertextBytes = encodeCiphertext(ciphertext);
+    await stateStore.putCiphertext(chainId, record.index, ciphertextBytes);
+    const baseNoteParams = {
+      chainId,
+      index: record.index,
+      commitment: record.commitment,
+      token,
+      amount,
+      npk,
+      mpk: account.masterPublicKey,
+      random,
+      nullifier: nullifierHex,
+    };
+
+    // Check if this note already exists in local storage
+    const existingNote = await stateStore.getNote(chainId, record.index);
+
+    // If note is already marked as spent locally, don't query indexer
+    // to ask for nullifier because we already know it's spent
+    if (existingNote?.spentAt !== undefined) {
+      // Ensure nullifier is stored for count optimization (idempotent)
+      await stateStore.putNullifier({
+        chainId,
+        nullifier: nullifierHex,
+        noteIndex: record.index,
+      });
+      return existingNote;
+    }
+
+    // Skip indexer nullifier query if nullifier counts match
+    if (!shouldCheckNullifiers) {
+      if (existingNote) {
+        return existingNote;
+      }
+      const stored = createNoteRecord(baseNoteParams);
+      await stateStore.putNote(stored);
+      return stored;
+    }
+
+    // Counts mismatched: query individual nullifier to check if this note was spent
+    const onChainNullifier = await indexerClient!.getNullifier({
+      chainId,
+      nullifier: nullifierHex,
+    });
+
+    if (onChainNullifier) {
+      const spentAtMs = onChainNullifier.spentAt * 1000;
+
+      // Store nullifier
+      await stateStore.putNullifier({
+        chainId,
+        nullifier: nullifierHex,
+        noteIndex: record.index,
+      });
+
+      if (existingNote) {
+        // Update existing unspent note to mark as spent
+        await stateStore.markNoteSpent(chainId, record.index, spentAtMs);
+        return existingNote;
+      }
+      const stored = createNoteRecord({
+        ...baseNoteParams,
+        spentAt: spentAtMs,
+      });
+      await stateStore.putNote(stored);
+      return stored;
+    }
+
+    // Skip re-storing unspent notes that already exist
+    if (existingNote) {
+      return existingNote;
+    }
+
+    const stored = createNoteRecord(baseNoteParams);
+    await stateStore.putNote(stored);
+    return stored;
+  }
+
+  // Pulls commitments from the indexer starting at `start`, updating local tree and notes.
+  async function ingestFrom(
+    chainId: number,
+    start: number,
+    account: ZkAccount,
+  ) {
+    const shouldCheckNullifiers = await shouldQueryNullifiers(chainId);
+    let cursor = start;
+    for (;;) {
+      const batch = await indexerClient!.fetchCommitmentBatch({
+        chainId,
+        start: cursor,
+        limit,
+      });
+
+      if (batch.commitments.length === 0) {
+        break;
+      }
+
+      for (const record of batch.commitments) {
+        const expectedIndex = trees.getLeafCount(chainId);
+        if (record.index !== expectedIndex) {
+          throw new Error("indexed commitments out of order");
+        }
+
+        const value = ByteUtils.hexToBigInt(record.commitment);
+        const { index } = trees.addLeaf(chainId, value);
+        if (index !== record.index) {
+          throw new Error("local merkle tree desynchronized");
+        }
+
+        await stateStore.putLeaf({
+          chainId,
+          index: record.index,
+          commitment: record.commitment,
+        });
+
+        await stateStore.putRoot({ chainId, root: record.root });
+        await tryStoreNote(chainId, record, account, shouldCheckNullifiers);
+        cursor = record.index + 1;
+      }
+
+      if (batch.commitments.length < limit) {
+        break;
+      }
+    }
+  }
+
+  // Clears local state for a chain and rehydrates fully from the indexer.
+  async function fullResync(chainId: number, account: ZkAccount) {
+    trees.reset(chainId);
+    await stateStore.clearLeaves(chainId);
+    await ingestFrom(chainId, 0, account);
+  }
+
+  /**
+   * Syncs a single chain into local state, optionally forcing a full rebuild.
+   * Skips overlapping runs via `inFlight` and falls back to full resync on
+   * ordering/root inconsistencies.
+   */
+  async function syncChain(
+    chainId: number,
+    account: ZkAccount,
+    opts: { forceFullResync?: boolean } = {},
+  ): Promise<void> {
+    ensureChainId(chainId);
+    const current = status.get(chainId) ?? {
+      inFlight: false,
+      lastSuccess: null,
+    };
+    if (current.inFlight) {
+      return;
+    }
+    status.set(chainId, { ...current, inFlight: true, lastError: undefined });
+
+    try {
+      const start = opts.forceFullResync
+        ? 0
+        : await rebuildTreeFromStore({
+            chainId,
+            trees,
+            loadLeaf: stateStore.getLeaf.bind(stateStore),
+          });
+      if (opts.forceFullResync) {
+        await fullResync(chainId, account);
+      } else {
+        try {
+          await ingestFrom(chainId, start, account);
+        } catch (err) {
+          // Fallback to full resync on ordering/root mismatch.
+          await fullResync(chainId, account);
+        }
+      }
+      status.set(chainId, {
+        inFlight: false,
+        lastSuccess: Date.now(),
+      });
+    } catch (err) {
+      status.set(chainId, {
+        inFlight: false,
+        lastSuccess: current.lastSuccess ?? null,
+        lastError: err instanceof Error ? err.message : String(err),
+      });
+      throw err;
+    }
+  }
+
+  /**
+   * Syncs multiple chains sequentially.
+   */
+  async function syncChains(
+    chainIds: number[],
+    account: ZkAccount,
+    opts?: { forceFullResync?: boolean },
+  ): Promise<void> {
+    for (const chainId of chainIds) {
+      await syncChain(chainId, account, opts);
+    }
+  }
+
+  /**
+   * Legacy sync method that returns synced notes.
+   * Kept for backward compatibility.
+   */
+  async function sync(
+    chainId: number,
+    account: ZkAccount,
+    opts?: { start?: number; forceFullScan?: boolean },
+  ): Promise<NoteRecord[]> {
+    ensureChainId(chainId);
+    await rebuildTreeFromStore({
+      chainId,
+      trees,
+      loadLeaf: stateStore.getLeaf.bind(stateStore),
+    });
+    const synced: NoteRecord[] = [];
+    let start = opts?.start ?? 0;
+
+    const shouldCheckNullifiers = await shouldQueryNullifiers(chainId);
+    let keepFetching = true;
+    while (keepFetching) {
+      const batch = await indexerClient!.fetchCommitmentBatch({
+        chainId,
+        start,
+        limit,
+      });
+      if (batch.commitments.length === 0) {
+        keepFetching = false;
+        continue;
+      }
+
+      for (const record of batch.commitments) {
+        await recordCommitment(chainId, record, !!opts?.forceFullScan);
+        const storedNote = await tryStoreNote(
+          chainId,
+          record,
+          account,
+          shouldCheckNullifiers,
+        );
+        if (storedNote) {
+          synced.push(storedNote);
+        }
+        start = record.index + 1;
+      }
+
+      if (batch.commitments.length < limit) {
+        keepFetching = false;
+      }
+    }
+
+    return synced;
+  }
+
+  function getStatus() {
+    return status;
+  }
+
+  return {
+    sync,
+    syncChain,
+    syncChains,
+    getStatus,
+  };
+}

package/transactions/reconcile.ts

@@ -0,0 +1,73 @@
+import type {
+  JobStatus,
+  PendingJobKind,
+  PendingJobRecord,
+} from "../state/index.js";
+import type {
+  BaseStateStore,
+  DepositSyncResult,
+  TransactSyncResult,
+} from "./types.js";
+
+type DepositClient = {
+  syncPendingDeposit(relayId: string): Promise<DepositSyncResult>;
+};
+
+type TransactService = {
+  syncPendingTransact(relayId: string): Promise<TransactSyncResult>;
+};
+
+type ReconcilerDeps = {
+  stateStore: BaseStateStore;
+  depositClient: DepositClient;
+  transactService: TransactService;
+};
+
+const ACTIVE_STATUSES: JobStatus[] = ["pending", "submitted", "broadcasting"];
+
+export function createJobReconciler(deps: ReconcilerDeps) {
+  const { stateStore, depositClient, transactService } = deps;
+
+  async function reconcileJob(job: PendingJobRecord) {
+    if (job.kind === "deposit") {
+      return depositClient.syncPendingDeposit(job.relayId);
+    }
+    return transactService.syncPendingTransact(job.relayId);
+  }
+
+  async function reconcileRelay(relayId: string) {
+    const job = await stateStore.getPendingJob(relayId);
+    if (!job) {
+      throw new Error(`unknown relay ${relayId}`);
+    }
+    return reconcileJob(job);
+  }
+
+  async function reconcileAll(
+    filter: {
+      kind?: PendingJobKind;
+      statuses?: JobStatus[];
+    } = {},
+  ) {
+    const jobs = await stateStore.listPendingJobs({
+      kind: filter.kind,
+      statuses: filter.statuses ?? ACTIVE_STATUSES,
+    });
+    const results = [];
+    for (const job of jobs) {
+      try {
+        const res = await reconcileJob(job);
+        results.push(res);
+      } catch (err) {
+        // Allow caller to decide how to handle errors; we keep going.
+        results.push(err);
+      }
+    }
+    return results;
+  }
+
+  return {
+    reconcileRelay,
+    reconcileAll,
+  };
+}