@apibara/evm-rpc 2.1.0-beta.41

package/src/filter.ts

@@ -0,0 +1,61 @@
+import type { LogFilter as DnaLogFilter, HeaderFilter } from "@apibara/evm";
+import type { ValidateFilterResult } from "@apibara/protocol/rpc";
+import { isHex } from "viem";
+
+export type Filter = {
+  header?: HeaderFilter;
+  logs: LogFilter[];
+};
+
+export type LogFilter = Pick<
+  DnaLogFilter,
+  "address" | "topics" | "strict" | "id"
+>;
+
+export function validateFilter(filter: Filter): ValidateFilterResult {
+  if (!filter.logs || filter.logs.length === 0) {
+    return { valid: false, error: "Missing logs filter" };
+  }
+
+  let logFilterIndex = 0;
+  for (const logFilter of filter.logs ?? []) {
+    if (
+      logFilter.address === undefined &&
+      (logFilter.topics?.length ?? 0) === 0
+    ) {
+      return {
+        valid: false,
+        error: `Must provide at least one address or topic in log filter at position ${logFilterIndex}`,
+      };
+    }
+
+    if (logFilter.address) {
+      if (!isHex(logFilter.address)) {
+        return {
+          valid: false,
+          error: "Invalid address format. Expected 0x-prefixed hex string",
+        };
+      }
+    }
+
+    if (logFilter.topics) {
+      for (let i = 0; i < logFilter.topics.length; i++) {
+        const topic = logFilter.topics[i];
+        if (topic === null) {
+          continue;
+        }
+
+        if (!isHex(topic)) {
+          return {
+            valid: false,
+            error: `Invalid topic at index ${i}: ${topic}. Must be null or a 0x-prefixed hex string`,
+          };
+        }
+      }
+    }
+
+    logFilterIndex++;
+  }
+
+  return { valid: true };
+}

package/src/index.ts

@@ -0,0 +1,3 @@
+export { EvmRpcStream } from "./stream-config";
+export type { Filter, LogFilter } from "./filter";
+export type { Block, BlockHeader, Log } from "./block";

package/src/log-fetcher.ts

@@ -0,0 +1,195 @@
+import type { Bytes } from "@apibara/protocol";
+import type { RpcLog } from "viem";
+import { hexToNumber, isHex, numberToHex, pad, trim } from "viem";
+import type { Log } from "./block";
+import type { Filter, LogFilter } from "./filter";
+import type { ViemRpcClient } from "./stream-config";
+import { viemRpcLogToDna } from "./transform";
+
+export async function fetchLogsByBlockHash({
+  client,
+  blockHash,
+  filter,
+}: {
+  client: ViemRpcClient;
+  blockHash: Bytes;
+  filter: Filter;
+}): Promise<{ logs: Log[] }> {
+  if (!filter.logs || filter.logs.length === 0) {
+    return { logs: [] };
+  }
+
+  const responses = await Promise.all(
+    filter.logs.map(async (logFilter) => {
+      const logs = await client.request({
+        method: "eth_getLogs",
+        params: [
+          {
+            blockHash,
+            address: logFilter.address,
+            topics: logFilter.topics ? [...logFilter.topics] : undefined,
+          },
+        ],
+      });
+      return { logs, logFilter };
+    }),
+  );
+
+  // Multiple calls may have produced the same log.
+  // We track all the logs (by their logIndex, which is unique within a block).
+  // logIndex -> position
+  const allLogs: Log[] = [];
+  const seenLogsByIndex: Record<number, number> = {};
+
+  for (const { logFilter, logs } of responses) {
+    for (const log of logs) {
+      if (log.blockNumber === null) {
+        throw new Error("Log block number is null");
+      }
+
+      const refinedLog = refineLog(log, logFilter);
+
+      if (refinedLog) {
+        const existingPosition = seenLogsByIndex[refinedLog.logIndex];
+
+        if (existingPosition !== undefined) {
+          const existingLog = allLogs[existingPosition];
+          (existingLog.filterIds as number[]).push(logFilter.id ?? 0);
+        } else {
+          (refinedLog.filterIds as number[]).push(logFilter.id ?? 0);
+
+          allLogs.push(refinedLog);
+          seenLogsByIndex[refinedLog.logIndex] = allLogs.length - 1;
+        }
+      }
+    }
+  }
+
+  return { logs: allLogs };
+}
+
+export async function fetchLogsForRange({
+  client,
+  fromBlock,
+  toBlock,
+  filter,
+}: {
+  client: ViemRpcClient;
+  fromBlock: bigint;
+  toBlock: bigint;
+  filter: Filter;
+}): Promise<{ logs: Record<number, Log[]>; blockNumbers: bigint[] }> {
+  const logsByBlock: Record<number, Log[]> = {};
+
+  if (!filter.logs || filter.logs.length === 0) {
+    return { logs: logsByBlock, blockNumbers: [] };
+  }
+
+  const responses = await Promise.all(
+    filter.logs.map(async (logFilter) => {
+      const logs = await client.request({
+        method: "eth_getLogs",
+        params: [
+          {
+            fromBlock: numberToHex(fromBlock),
+            toBlock: numberToHex(toBlock),
+            address: logFilter.address,
+            topics: logFilter.topics ? [...logFilter.topics] : undefined,
+          },
+        ],
+      });
+      return { logs, logFilter };
+    }),
+  );
+
+  const blockNumbers = new Set<bigint>();
+
+  // Multiple calls may have produced the same log.
+  // We track all the logs (by their logIndex, which is unique within a block).
+  // blockNumber -> logIndex -> position
+  const seenLogsByBlockNumberAndIndex: Record<
+    number,
+    Record<number, number>
+  > = {};
+
+  for (const { logFilter, logs } of responses) {
+    for (const log of logs) {
+      if (log.blockNumber === null) {
+        throw new Error("Log block number is null");
+      }
+
+      const refinedLog = refineLog(log, logFilter);
+
+      if (refinedLog) {
+        const blockNumber = hexToNumber(log.blockNumber);
+        blockNumbers.add(BigInt(blockNumber));
+
+        if (!logsByBlock[blockNumber]) {
+          logsByBlock[blockNumber] = [];
+        }
+
+        if (!seenLogsByBlockNumberAndIndex[blockNumber]) {
+          seenLogsByBlockNumberAndIndex[blockNumber] = {};
+        }
+
+        const existingPosition =
+          seenLogsByBlockNumberAndIndex[blockNumber][refinedLog.logIndex];
+
+        if (existingPosition !== undefined) {
+          const existingLog = logsByBlock[blockNumber][existingPosition];
+          (existingLog.filterIds as number[]).push(logFilter.id ?? 0);
+        } else {
+          (refinedLog.filterIds as number[]).push(logFilter.id ?? 0);
+
+          logsByBlock[blockNumber].push(refinedLog);
+          seenLogsByBlockNumberAndIndex[blockNumber][refinedLog.logIndex] =
+            logsByBlock[blockNumber].length - 1;
+        }
+      }
+    }
+  }
+
+  return { logs: logsByBlock, blockNumbers: Array.from(blockNumbers) };
+}
+
+function refineLog(log: RpcLog, filter: LogFilter): Log | null {
+  if (log.removed) {
+    return null;
+  }
+
+  const filterTopics = filter.topics ?? [];
+  // Strict mode
+  if (filter.strict && log.topics.length !== filterTopics.length) {
+    return null;
+  }
+
+  if (filterTopics.length === 0) {
+    return viemRpcLogToDna(log);
+  }
+
+  if (log.topics.length < filterTopics.length) {
+    return null;
+  }
+
+  for (let i = 0; i < filterTopics.length; i++) {
+    const filterTopic = filterTopics[i];
+    const logTopic = log.topics[i];
+
+    if (filterTopic === null) continue;
+
+    if (!logTopic) return null;
+
+    if (!isHex(filterTopic) || !isHex(logTopic)) {
+      return null;
+    }
+
+    const normalizedFilter = pad(trim(filterTopic), { size: 32 });
+    const normalizedLog = pad(trim(logTopic), { size: 32 });
+
+    if (normalizedFilter !== normalizedLog) {
+      return null;
+    }
+  }
+
+  return viemRpcLogToDna(log);
+}

package/src/range-oracle.ts

@@ -0,0 +1,61 @@
+const BLOCK_RANGE_ERROR_PATTERNS = ["invalid block range params"] as const;
+
+export type BlockRange = {
+  start: bigint;
+  end: bigint;
+};
+
+export type BlockRangeOracle = {
+  clampRange(original: BlockRange): BlockRange;
+  handleSuccess(): void;
+  handleError(error: unknown): { retry: boolean };
+};
+
+export function createBlockRangeOracle({
+  startingSize,
+  minSize = 1n,
+  maxSize = 10_000n,
+}: {
+  startingSize: bigint;
+  minSize?: bigint;
+  maxSize?: bigint;
+}): BlockRangeOracle {
+  let currentSize = startingSize;
+
+  return {
+    clampRange(original: BlockRange): BlockRange {
+      const start = original.start;
+      let end = original.end;
+
+      const newEnd = start + currentSize - 1n;
+      if (newEnd < end) {
+        end = newEnd;
+      }
+
+      return { start, end };
+    },
+    handleSuccess(): void {
+      // TODO: we can track how many successful requests and increase the size.
+      // Probably want to receive the number of logs as argument and have a "target"
+    },
+    handleError(error: unknown): { retry: boolean } {
+      if (error instanceof Error) {
+        const message = error.message.toLowerCase();
+        const isBlockRangeError = BLOCK_RANGE_ERROR_PATTERNS.some((pattern) =>
+          message.includes(pattern),
+        );
+
+        if (isBlockRangeError) {
+          if (currentSize > minSize) {
+            const newSize = currentSize / 2n;
+            currentSize = newSize > minSize ? newSize : minSize;
+          }
+
+          return { retry: true };
+        }
+      }
+
+      return { retry: false };
+    },
+  };
+}

package/src/stream-config.ts

@@ -0,0 +1,299 @@
+import type { Bytes } from "@apibara/protocol";
+import {
+  type BlockInfo,
+  type FetchBlockByNumberArgs,
+  type FetchBlockByNumberResult,
+  type FetchBlockRangeArgs,
+  type FetchBlockRangeResult,
+  type FetchBlockResult,
+  type FetchCursorArgs,
+  RpcStreamConfig,
+  type ValidateFilterResult,
+} from "@apibara/protocol/rpc";
+import {
+  type EIP1193Parameters,
+  type PublicRpcSchema,
+  type RpcBlock,
+  formatBlock,
+  numberToHex,
+  toHex,
+} from "viem";
+import type { Block, Log } from "./block";
+import { type Filter, validateFilter } from "./filter";
+import { fetchLogsByBlockHash, fetchLogsForRange } from "./log-fetcher";
+import { type BlockRangeOracle, createBlockRangeOracle } from "./range-oracle";
+import { rpcBlockHeaderToDna } from "./transform";
+
+export type RequestParameters = EIP1193Parameters<PublicRpcSchema>;
+
+export type RequestReturnType<method extends RequestParameters["method"]> =
+  Extract<PublicRpcSchema[number], { Method: method }>["ReturnType"];
+
+// Require just the bare minimum from the provided viem client.
+export type ViemRpcClient = {
+  request: <TParams extends RequestParameters>(
+    params: TParams,
+  ) => Promise<RequestReturnType<TParams["method"]>>;
+};
+
+export type EvmRpcStreamOptions = {
+  /** How many blocks to fetch in a single eth_getLogs call. */
+  getLogsRangeSize?: bigint;
+  /** How often to refresh the head block. */
+  headRefreshIntervalMs?: number;
+  /** How often to refresh the finalized block. */
+  finalizedRefreshIntervalMs?: number;
+};
+
+export class EvmRpcStream extends RpcStreamConfig<Filter, Block> {
+  private blockRangeOracle: BlockRangeOracle;
+
+  constructor(
+    private client: ViemRpcClient,
+    private options: EvmRpcStreamOptions = {},
+  ) {
+    super();
+
+    this.blockRangeOracle = createBlockRangeOracle({
+      startingSize: options.getLogsRangeSize ?? 1_000n,
+      // Use the provided size to limit the maximum range size
+      maxSize: options.getLogsRangeSize ? options.getLogsRangeSize : undefined,
+    });
+  }
+
+  headRefreshIntervalMs(): number {
+    return this.options.headRefreshIntervalMs ?? 3_000;
+  }
+
+  finalizedRefreshIntervalMs(): number {
+    return this.options.finalizedRefreshIntervalMs ?? 30_000;
+  }
+
+  validateFilter(filter: Filter): ValidateFilterResult {
+    return validateFilter(filter);
+  }
+
+  async fetchCursor(args: FetchCursorArgs): Promise<BlockInfo | null> {
+    let block: RpcBlock | null = null;
+    if (args.blockNumber !== undefined) {
+      const blockNumber = toHex(args.blockNumber);
+      block = await this.client.request({
+        method: "eth_getBlockByNumber",
+        params: [blockNumber, false],
+      });
+    } else if (args.blockTag) {
+      block = await this.client.request({
+        method: "eth_getBlockByNumber",
+        params: [args.blockTag, false],
+      });
+    } else if (args.blockHash) {
+      block = await this.client.request({
+        method: "eth_getBlockByHash",
+        params: [args.blockHash, false],
+      });
+    } else {
+      throw new Error(
+        "One of blockNumber, blockHash or blockTag must be provided",
+      );
+    }
+
+    if (!block) {
+      return null;
+    }
+
+    const formattedBlock = formatBlock(block);
+
+    if (formattedBlock.number === null) {
+      throw new Error("RPC block is missing required block number");
+    }
+
+    if (formattedBlock.hash === null) {
+      throw new Error("RPC block is missing required block hash");
+    }
+
+    return {
+      blockNumber: formattedBlock.number,
+      blockHash: formattedBlock.hash,
+      parentBlockHash: formattedBlock.parentHash,
+    };
+  }
+
+  async fetchBlockRange({
+    startBlock,
+    finalizedBlock,
+    filter,
+  }: FetchBlockRangeArgs<Filter>): Promise<FetchBlockRangeResult<Block>> {
+    const { start: fromBlock, end: toBlock } = this.blockRangeOracle.clampRange(
+      { start: startBlock, end: finalizedBlock },
+    );
+
+    // console.log("Fetching block range", fromBlock, toBlock, filter);
+
+    const { logs: logsByBlockNumber, blockNumbers } =
+      await this.fetchLogsForRangeWithRetry({
+        fromBlock,
+        toBlock,
+        filter,
+      });
+
+    // If the client needs all headers, we iterate over the range and fetch headers
+    // and then join them with the logs
+    // Otherwise, we drive the block number iteration from the fetched logs.
+    const data: FetchBlockResult<Block>[] = [];
+
+    // Fetch block headers in parallel to optimize batching.
+    const blockNumberResponses = [];
+    if (filter.header === "always") {
+      for (let blockNumber = fromBlock; blockNumber <= toBlock; blockNumber++) {
+        blockNumberResponses.push(
+          this.fetchBlockHeaderByNumberWithRetry({
+            blockNumber,
+          }),
+        );
+      }
+    } else {
+      for (const blockNumber of blockNumbers) {
+        blockNumberResponses.push(
+          this.fetchBlockHeaderByNumberWithRetry({
+            blockNumber,
+          }),
+        );
+      }
+    }
+
+    const blockNumbersWithHeader = await Promise.all(blockNumberResponses);
+    for (const { blockNumber, header } of blockNumbersWithHeader) {
+      const logs = logsByBlockNumber[Number(blockNumber)] ?? [];
+
+      data.push({
+        cursor: undefined,
+        endCursor: { orderKey: blockNumber },
+        block: { header, logs },
+      });
+    }
+
+    return { startBlock: fromBlock, endBlock: toBlock, data };
+  }
+
+  async fetchBlockByNumber({
+    blockNumber,
+    expectedParentBlockHash,
+    filter,
+  }: FetchBlockByNumberArgs<Filter>): Promise<FetchBlockByNumberResult<Block>> {
+    // Fetch block header and check it matches the expected parent block hash.
+    const { header } = await this.fetchBlockHeaderByNumberWithRetry({
+      blockNumber,
+    });
+
+    if (header.blockHash === undefined) {
+      throw new Error(`Block ${blockNumber} has no block hash`);
+    }
+
+    const blockInfo: BlockInfo = {
+      blockNumber: header.blockNumber,
+      blockHash: header.blockHash,
+      parentBlockHash: header.parentBlockHash,
+    };
+
+    if (header.parentBlockHash !== expectedParentBlockHash) {
+      return {
+        status: "reorg",
+        blockInfo,
+      };
+    }
+
+    // Use the hash from the current block to fetch logs in a reorg-safe way.
+    const { logs } = await this.fetchLogsByBlockHashWithRetry({
+      blockHash: header.blockHash,
+      filter,
+    });
+
+    let cursor = undefined;
+    if (blockNumber > 0n) {
+      cursor = {
+        orderKey: blockNumber - 1n,
+        uniqueKey: header.parentBlockHash,
+      };
+    }
+    const endCursor = {
+      orderKey: blockNumber,
+      uniqueKey: header.blockHash,
+    };
+
+    let block = null;
+
+    // TODO: handle header on new block.
+    if (filter.header === "always" || logs.length > 0) {
+      block = {
+        header,
+        logs,
+      };
+    }
+
+    return {
+      status: "success",
+      blockInfo,
+      data: {
+        cursor,
+        endCursor,
+        block,
+      },
+    };
+  }
+
+  private async fetchLogsForRangeWithRetry({
+    fromBlock,
+    toBlock,
+    filter,
+  }: {
+    fromBlock: bigint;
+    toBlock: bigint;
+    filter: Filter;
+  }): Promise<{ logs: Record<number, Log[]>; blockNumbers: bigint[] }> {
+    // TODO: implement retry
+    try {
+      return await fetchLogsForRange({
+        client: this.client,
+        fromBlock,
+        toBlock,
+        filter,
+      });
+    } catch (error) {
+      this.blockRangeOracle.handleError(error);
+      throw error;
+    }
+  }
+
+  private async fetchLogsByBlockHashWithRetry({
+    blockHash,
+    filter,
+  }: {
+    blockHash: Bytes;
+    filter: Filter;
+  }): Promise<{ logs: Log[] }> {
+    // TODO: implement retry
+    return await fetchLogsByBlockHash({
+      client: this.client,
+      blockHash,
+      filter,
+    });
+  }
+
+  private async fetchBlockHeaderByNumberWithRetry({
+    blockNumber,
+  }: {
+    blockNumber: bigint;
+  }) {
+    // TODO: implement retry
+    const block = await this.client.request({
+      method: "eth_getBlockByNumber",
+      params: [numberToHex(blockNumber), false],
+    });
+
+    if (block === null) {
+      throw new Error(`Block ${blockNumber} not found`);
+    }
+
+    return { header: rpcBlockHeaderToDna(block), blockNumber };
+  }
+}

package/src/transform.ts

@@ -0,0 +1,70 @@
+import {
+  type RpcBlock,
+  type RpcLog,
+  type Block as ViemBlock,
+  formatBlock,
+  hexToNumber,
+} from "viem";
+import type { BlockHeader as DnaBlockHeader, Log as DnaLog } from "./block";
+
+export function viemRpcLogToDna(viemLog: RpcLog): DnaLog {
+  if (
+    viemLog.logIndex === null ||
+    viemLog.transactionIndex === null ||
+    viemLog.transactionHash === null
+  ) {
+    throw new Error(
+      "Invalid log: missing required fields: logIndex, transactionIndex, transactionHash",
+    );
+  }
+
+  return {
+    filterIds: [],
+    logIndex: hexToNumber(viemLog.logIndex),
+    address: viemLog.address,
+    topics: viemLog.topics,
+    data: viemLog.data,
+    transactionIndex: hexToNumber(viemLog.transactionIndex),
+    transactionHash: viemLog.transactionHash,
+    transactionStatus: viemLog.removed ? "reverted" : "succeeded",
+  };
+}
+
+export function rpcBlockHeaderToDna(block: RpcBlock): DnaBlockHeader {
+  const formattedBlock = formatBlock(block);
+  return viemBlockHeaderToDna(formattedBlock);
+}
+
+export function viemBlockHeaderToDna(viemBlock: ViemBlock): DnaBlockHeader {
+  if (viemBlock.number === null || !viemBlock.hash) {
+    throw new Error(
+      `Invalid block: missing required fields (number: ${viemBlock.number}, hash: ${viemBlock.hash})`,
+    );
+  }
+
+  return {
+    blockNumber: viemBlock.number,
+    blockHash: viemBlock.hash,
+    parentBlockHash: viemBlock.parentHash,
+    unclesHash: viemBlock.sha3Uncles,
+    miner: viemBlock.miner ?? undefined,
+    stateRoot: viemBlock.stateRoot,
+    transactionsRoot: viemBlock.transactionsRoot,
+    receiptsRoot: viemBlock.receiptsRoot,
+    logsBloom: viemBlock.logsBloom ?? undefined,
+    difficulty: viemBlock.difficulty,
+    gasLimit: viemBlock.gasLimit,
+    gasUsed: viemBlock.gasUsed,
+    timestamp: new Date(Number(viemBlock.timestamp) * 1000),
+    extraData: viemBlock.extraData,
+    mixHash: viemBlock.mixHash ?? undefined,
+    nonce: viemBlock.nonce ? BigInt(viemBlock.nonce) : undefined,
+    baseFeePerGas: viemBlock.baseFeePerGas ?? undefined,
+    withdrawalsRoot: viemBlock.withdrawalsRoot ?? undefined,
+    totalDifficulty: viemBlock.totalDifficulty ?? undefined,
+    blobGasUsed: viemBlock.blobGasUsed ?? undefined,
+    excessBlobGas: viemBlock.excessBlobGas ?? undefined,
+    parentBeaconBlockRoot: viemBlock.parentBeaconBlockRoot ?? undefined,
+    requestsHash: undefined, // TODO: check
+  };
+}