@effectstream/sync 0.100.0

package/src/sync-protocols/midnight/types.ts

@@ -0,0 +1,128 @@
+import type {
+  BlockNumber,
+  MidnightAddress,
+  MidnightBlockHash,
+  MidnightBlockNumber,
+  MidnightTxHash,
+  TimestampIso8601,
+  TimestampMs,
+} from "@effectstream/utils";
+import type {
+  ConfigNetworkType,
+  ConfigSyncProtocolType,
+  FlattenSyncProtocolIOFor,
+  PrimitiveEntry,
+  SyncProtocolWithNetwork,
+} from "@effectstream/config";
+import type { MidnightGqlBlockState } from "./MidnightClient.ts";
+
+export type CommonFunnelArgs = {
+  /**
+   * TODO: right now the Midnight funnels works in an odd way
+   *       where we fetch every single block in one large graphql query
+   *       then parse into it as needed
+   *       Not clear this is the best approach
+   */
+  block: CachedBlock;
+  fromBlock: BlockNumber;
+  toBlock: BlockNumber;
+  toMainchainBlockNumber?: (block: BlockNumber) => BlockNumber;
+  toMainchainTimestamp?: (block: BlockNumber) => TimestampMs;
+  isPresync: boolean;
+};
+
+// =============
+// Graphql Block
+// =============
+
+// Interfaces approximated from GraphQL schema served by indexer v1.3.1
+
+interface ContractCallOrDeploy {
+  address: MidnightAddress;
+  state: string;
+  transaction: Transaction;
+  zswapChainState: string;
+}
+
+interface ContractCall extends ContractCallOrDeploy {
+  deploy: ContractCallOrDeploy;
+  operation: string;
+}
+
+interface ContractDeploy extends ContractCallOrDeploy {
+  definition: string;
+}
+
+interface Transaction {
+  block: Block;
+  hash: MidnightTxHash;
+  identifiers: string[];
+  contractCalls: (ContractCall | ContractDeploy)[];
+  raw: string;
+  applyStage: string;
+  merkleTreeRoot?: string;
+}
+
+export interface ZswapLedgerEvent {
+  id:    number;
+  raw:   string;
+  maxId: number;
+}
+
+export interface Block {
+  hash:            MidnightBlockHash;
+  height:          MidnightBlockNumber;
+  protocolVersion: number;
+  timestamp:       TimestampIso8601;
+  parent:          {
+    hash: MidnightBlockHash;
+  };
+  transactions:    {
+    hash:            MidnightTxHash;
+    contractActions: {
+      address:    MidnightAddress;
+      state:      string;
+    }[];
+    zswapLedgerEvents?: ZswapLedgerEvent[];
+  }[];
+}
+
+export type CachedBlock = Pick<Block, "height" | "hash" | "timestamp"> & {
+  transactions: (Pick<Transaction, "hash"> & {
+    contractCalls: Pick<ContractCall, "address" | "state">[];
+  })[];
+};
+
+import type { PageRelation } from "../base/page.ts";
+import type { PageSyncRange } from "../common/page-helpers.ts";
+
+export type Page = {
+  height: BlockNumber;
+  hash: MidnightBlockHash;
+};
+
+export type PrimitiveType = FlattenSyncProtocolIOFor<
+  ConfigSyncProtocolType.MIDNIGHT_PARALLEL
+>;
+export type Input = PageSyncRange<BlockNumber>;
+export type Output = {
+  raw: Block;
+  primitives: PrimitiveType[];
+};
+
+export const pageRelation: PageRelation<Page> = {
+  compare: (p1, p2) => p1.height - p2.height,
+  equals: (p1, p2) => p1.height === p2.height,
+  min: (p1, p2) => (p1.height < p2.height ? p1 : p2),
+  max: (p1, p2) => (p1.height > p2.height ? p1 : p2),
+};
+
+export type ConfigType = Extract<
+  SyncProtocolWithNetwork,
+  { networkType: ConfigNetworkType.MIDNIGHT }
+>;
+
+export type PrimitiveEntryType = Extract<
+  PrimitiveEntry,
+  { syncProtocol: ConfigSyncProtocolType.MIDNIGHT_PARALLEL }
+>;

package/src/sync-protocols/midnight/zswap-decoder.ts

@@ -0,0 +1,123 @@
+import { Buffer } from "node:buffer";
+import * as ledger from '@midnight-ntwrk/ledger-v8';
+
+// TODO This alternative implementation should be the native way to decode the event.
+function new_decodeZswapInputEvent(rawHex: string): ledger.Event | null {
+  try {
+    const bytes = Uint8Array.from(
+      Buffer.from(rawHex.replace(/^0x/, ""), "hex"),
+    );
+    const event = ledger.Event.deserialize(bytes);
+    console.log("event", event);
+    // if (event.type === "ZswapInput") {
+      // event.nullifier  — Uint8Array (32 bytes)
+      // event.source.transactionHash
+      // event.source.logicalSegment
+    // } else if (event.type === "ZswapOutput") {
+      // event.commitment, etc.
+    // }
+  } catch {
+    return null;
+  }
+  return null;
+}
+
+// ─── SCALE compact integer decoder ───────────────────────────────────────────
+//
+// Bottom 2 bits of the first byte are the mode:
+//   00 = 1-byte  (value in bits [7:2])
+//   01 = 2-byte
+//   10 = 4-byte
+//   11 = n-byte length prefix
+
+function readScaleU32(b: Uint8Array, pos: number): [number, number] {
+  const first = b[pos];
+  const mode = first & 0b11;
+  if (mode === 0b00) {
+    return [(first & 0xFC) >> 2, pos + 1];
+  } else if (mode === 0b01) {
+    const second = b[pos + 1];
+    const v0 = ((first & 0xFC) >> 2) | ((second & 0x03) << 6);
+    const v1 = (second & 0xFC) >> 2;
+    return [(v0 | (v1 << 8)) >>> 0, pos + 2];
+  } else if (mode === 0b10) {
+    const s = b[pos + 1], t = b[pos + 2], u = b[pos + 3];
+    const v0 = ((first & 0xFC) >> 2) | ((s & 0x03) << 6);
+    const v1 = ((s & 0xFC) >> 2) | ((t & 0x03) << 6);
+    const v2 = ((t & 0xFC) >> 2) | ((u & 0x03) << 6);
+    const v3 = (u & 0xFC) >> 2;
+    return [(v0 | (v1 << 8) | (v2 << 16) | (v3 << 24)) >>> 0, pos + 4];
+  } else {
+    const n = ((first & 0xFC) >> 2) + 4;
+    let val = 0;
+    for (let i = 0; i < n; i++) val |= b[pos + 1 + i] << (i * 8);
+    return [val >>> 0, pos + 1 + n];
+  }
+}
+
+function hexToBytes(hex: string): Uint8Array {
+  const clean = hex.startsWith("0x") ? hex.slice(2) : hex;
+  const arr = new Uint8Array(clean.length / 2);
+  for (let i = 0; i < arr.length; i++) {
+    arr[i] = parseInt(clean.slice(i * 2, i * 2 + 2), 16);
+  }
+  return arr;
+}
+
+function bytesToHex(bytes: Uint8Array): string {
+  return Array.from(bytes).map((b) => b.toString(16).padStart(2, "0")).join("");
+}
+
+// ─── Binary layout for "midnight:event[v9]:" ─────────────────────────────────
+//   19 bytes   outer tag "midnight:event[v9]:"
+//   32 bytes   EventSource.transaction_hash
+//    2 bytes   EventSource.logical_segment  (LE u16)
+//    2 bytes   EventSource.physical_segment (LE u16, skipped)
+//   SCALE u32  EventDetails variant: 0=ZswapInput, 1=ZswapOutput, 2+=other
+//
+//   ZswapInput: 4-byte contract_id + 32-byte CoinNullifier
+
+const OUTER_TAG = "midnight:event[v9]:";
+
+export interface DecodedZswapInput {
+  /** 32-byte transaction hash, hex */
+  txHash: string;
+  /** Logical segment index */
+  logicalSegment: number;
+  /** 32-byte nullifier, hex */
+  nullifier: string;
+}
+
+/**
+ * Decode a raw SCALE-encoded zswap ledger event.
+ * Returns the decoded data if the event is a ZswapInput (nullifier spend),
+ * or `null` for ZswapOutput / other variants.
+ */
+export function decodeZswapInputEvent(
+  rawHex: string,
+): DecodedZswapInput | null {
+  try {
+    const b = hexToBytes(rawHex);
+    let pos = OUTER_TAG.length;
+
+    const txHash = bytesToHex(b.slice(pos, pos + 32));
+    pos += 32;
+    const logicalSegment = b[pos] | (b[pos + 1] << 8);
+    pos += 2;
+    pos += 2; // skip physical segment
+
+    const [variant, p1] = readScaleU32(b, pos);
+    pos = p1;
+
+    if (variant !== 0) {
+      return null; // ZswapOutput or other — not a nullifier event
+    }
+
+    pos += 4; // skip 4-byte contract_id
+
+    const nullifier = bytesToHex(b.slice(pos, pos + 32));
+    return { txHash, logicalSegment, nullifier };
+  } catch {
+    return null;
+  }
+}

package/src/sync-protocols/mod.ts

@@ -0,0 +1,29 @@
+export * from "./evm/fetcher.ts";
+export * from "./evm/state.ts";
+
+export * from "./midnight/fetcher.ts";
+export * from "./midnight/state.ts";
+
+export * from "./avail/fetcher.ts";
+export * from "./avail/state.ts";
+
+export * from "./bitcoin/fetcher.ts";
+export * from "./bitcoin/state.ts";
+
+export * from "./ntp/fetcher.ts";
+export * from "./ntp/state.ts";
+
+export * from "./utxorpc/fetcher.ts";
+export * from "./utxorpc/state.ts";
+
+export * from "./celestia/fetcher.ts";
+export * from "./celestia/state.ts";
+
+export type * from "./base/fetcher.ts";
+export type * from "./base/state.ts";
+export * from "./orchestration/merge.ts";
+export * from "./orchestration/sync.ts";
+
+export { chainPageRelation } from "./common/root.ts";
+// TODO: remove
+export type { ChainBlock } from "./common/root.ts";

package/src/sync-protocols/near/NearClient.ts

@@ -0,0 +1,178 @@
+// ============================
+// NEAR JSON-RPC type responses
+// ============================
+
+export type NearBlockHeader = {
+  height: number;
+  hash: string;
+  /** Nanosecond timestamp */
+  timestamp: number;
+  /** Nanosecond timestamp as string in some APIs */
+  timestamp_nanosec: string;
+  chunks_included: number;
+};
+
+export type NearBlockChunkHeader = {
+  chunk_hash: string;
+  shard_id: number;
+  height_included: number;
+  tx_root: string;
+};
+
+export type NearBlock = {
+  header: NearBlockHeader;
+  chunks: NearBlockChunkHeader[];
+};
+
+export type NearChunkTransaction = {
+  hash: string;
+  signer_id: string;
+  receiver_id: string;
+  actions: NearAction[];
+};
+
+export type NearAction = {
+  FunctionCall?: {
+    method_name: string;
+    args: string; // base64-encoded
+    gas: number;
+    deposit: string;
+  };
+  Transfer?: {
+    deposit: string;
+  };
+  [key: string]: unknown;
+};
+
+export type NearChunk = {
+  header: {
+    chunk_hash: string;
+    shard_id: number;
+    height_included: number;
+  };
+  transactions: NearChunkTransaction[];
+};
+
+export type NearExecutionOutcome = {
+  receipt_ids: string[];
+  status: { SuccessValue?: string; SuccessReceiptId?: string; Failure?: unknown };
+  logs: string[];
+  executor_id: string;
+};
+
+export type NearExecutionOutcomeWithId = {
+  id: string;
+  outcome: NearExecutionOutcome;
+  block_hash: string;
+};
+
+export type NearTxStatusResult = {
+  transaction: {
+    hash: string;
+    signer_id: string;
+    receiver_id: string;
+    actions: NearAction[];
+  };
+  transaction_outcome: NearExecutionOutcomeWithId;
+  receipts_outcome: NearExecutionOutcomeWithId[];
+};
+
+// ==============
+// NEP-297 events
+// ==============
+
+export type Nep297Event = {
+  standard: string;
+  version: string;
+  event: string;
+  data: unknown[];
+};
+
+/**
+ * Parse a NEP-297 event from a NEAR execution log line.
+ * NEP-297 events have the format: `EVENT_JSON:{...}`
+ */
+export function parseNep297Log(log: string): Nep297Event | null {
+  if (!log.startsWith("EVENT_JSON:")) return null;
+  try {
+    return JSON.parse(log.slice("EVENT_JSON:".length));
+  } catch {
+    return null;
+  }
+}
+
+// ===========
+// RPC client
+// ===========
+
+export class NearClient {
+  private readonly rpcUrl: string;
+
+  constructor(rpcUrl: string) {
+    this.rpcUrl = rpcUrl;
+  }
+
+  private async rpc<T>(method: string, params: Record<string, unknown>): Promise<T> {
+    const res = await fetch(this.rpcUrl, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ jsonrpc: "2.0", id: 1, method, params }),
+    });
+
+    const json = await res.json();
+    if (json.error) {
+      throw new Error(
+        `[NEAR] RPC error [${method}]: ${json.error.cause?.name ?? json.error.message ?? JSON.stringify(json.error)}`,
+      );
+    }
+    return json.result as T;
+  }
+
+  async getBlock(
+    params: { finality: "final" | "optimistic" } | { block_id: number },
+  ): Promise<NearBlock> {
+    return this.rpc<NearBlock>("block", params);
+  }
+
+  async getChunk(chunkHash: string): Promise<NearChunk> {
+    return this.rpc<NearChunk>("chunk", { chunk_id: chunkHash });
+  }
+
+  async getTxStatus(
+    txHash: string,
+    senderAccountId: string,
+  ): Promise<NearTxStatusResult> {
+    return this.rpc<NearTxStatusResult>("EXPERIMENTAL_tx_status", {
+      tx_hash: txHash,
+      sender_account_id: senderAccountId,
+      wait_until: "EXECUTED",
+    });
+  }
+
+  async getLatestBlockHeight(): Promise<number> {
+    const block = await this.getBlock({ finality: "final" });
+    return block.header.height;
+  }
+
+  async viewFunction(
+    accountId: string,
+    methodName: string,
+    argsBase64: string = "",
+  ): Promise<unknown> {
+    const result = await this.rpc<{
+      result: number[];
+      logs: string[];
+      block_height: number;
+      block_hash: string;
+    }>("query", {
+      request_type: "call_function",
+      finality: "final",
+      account_id: accountId,
+      method_name: methodName,
+      args_base64: argsBase64,
+    });
+    const bytes = new Uint8Array(result.result);
+    const text = new TextDecoder().decode(bytes);
+    return JSON.parse(text);
+  }
+}