@lodestar/prover 1.35.0-dev.8ea34e52ba → 1.35.0-dev.91dadf81de

package/src/utils/gitData/index.ts

@@ -0,0 +1,70 @@
+import {execSync} from "node:child_process";
+// This file is created in the build step and is distributed through NPM
+// MUST be in sync with `-/gitDataPath.ts` and `package.json` files.
+import {GitData, readGitDataFile} from "./gitDataPath.js";
+
+/** Reads git data from a persisted file or local git data at build time. */
+export function readAndGetGitData(): GitData {
+  try {
+    // Gets git data containing current branch and commit info from persistent file.
+    let persistedGitData: Partial<GitData>;
+    try {
+      persistedGitData = readGitDataFile();
+    } catch (_e) {
+      persistedGitData = {};
+    }
+
+    const currentGitData = getGitData();
+
+    return {
+      // If the CLI is run from source, prioritze current git data
+      // over `.git-data.json` file, which might be stale here.
+      branch:
+        currentGitData.branch && currentGitData.branch.length > 0
+          ? currentGitData.branch
+          : (persistedGitData.branch ?? ""),
+      commit:
+        currentGitData.commit && currentGitData.commit.length > 0
+          ? currentGitData.commit
+          : (persistedGitData.commit ?? ""),
+    };
+  } catch (_e) {
+    return {
+      branch: "",
+      commit: "",
+    };
+  }
+}
+
+/** Gets git data containing current branch and commit info from CLI. */
+export function getGitData(): GitData {
+  return {
+    branch: process.env.GIT_BRANCH ?? getBranch(),
+    commit: process.env.GIT_COMMIT ?? getCommit(),
+  };
+}
+
+/** Tries to get branch from git CLI. */
+function getBranch(): string {
+  try {
+    return shellSilent("git rev-parse --abbrev-ref HEAD");
+  } catch (_e) {
+    return "";
+  }
+}
+
+/** Tries to get commit from git from git CLI. */
+function getCommit(): string {
+  try {
+    return shellSilent("git rev-parse --verify HEAD");
+  } catch (_e) {
+    return "";
+  }
+}
+
+/** Silent shell that won't pollute stdout, or stderr */
+function shellSilent(cmd: string): string {
+  return execSync(cmd, {stdio: ["ignore", "pipe", "ignore"]})
+    .toString()
+    .trim();
+}

package/src/utils/gitData/writeGitData.ts

@@ -0,0 +1,10 @@
+#!/usr/bin/env node
+
+// For RATIONALE of this file, check packages/cli/src/util/gitData/gitDataPath.ts
+// Persist exact commit in NPM distributions for easier tracking of the build
+
+import {writeGitDataFile} from "./gitDataPath.js";
+import {getGitData} from "./index.js";
+
+// Script to write the git data file (json) used by the build procedures to persist git data.
+writeGitDataFile(getGitData());

package/src/utils/json_rpc.ts

@@ -0,0 +1,170 @@
+import {Logger} from "@lodestar/logger";
+import {VERIFICATION_FAILED_RESPONSE_CODE} from "../constants.js";
+import {
+  JsonRpcBatchRequest,
+  JsonRpcBatchResponse,
+  JsonRpcErrorPayload,
+  JsonRpcNotificationPayload,
+  JsonRpcRequest,
+  JsonRpcRequestOrBatch,
+  JsonRpcRequestPayload,
+  JsonRpcResponse,
+  JsonRpcResponseOrBatch,
+  JsonRpcResponseWithErrorPayload,
+  JsonRpcResponseWithResultPayload,
+} from "../types.js";
+import {isNullish} from "./validation.js";
+
+export function getResponseForRequest<P, R, E = unknown>(
+  payload: JsonRpcRequest<P>,
+  res?: R,
+  error?: JsonRpcErrorPayload<E>
+): JsonRpcResponse<R, E> {
+  // If it's a notification
+  if (!isRequest(payload)) {
+    throw new Error("Cannot generate response for notification");
+  }
+
+  if (!isNullish(res) && isNullish(error)) {
+    return {
+      jsonrpc: payload.jsonrpc,
+      id: payload.id,
+      result: res,
+    };
+  }
+
+  if (!isNullish(error)) {
+    return {
+      jsonrpc: payload.jsonrpc,
+      id: payload.id,
+      error,
+    };
+  }
+
+  throw new Error("Either result or error must be defined.");
+}
+
+export function getVerificationFailedMessage(method: string): string {
+  return `verification for '${method}' request failed.`;
+}
+
+export function isVerificationFailedError<P>(payload: JsonRpcResponseWithErrorPayload<P>): boolean {
+  return !isValidResponsePayload(payload) && payload.error.code === VERIFICATION_FAILED_RESPONSE_CODE;
+}
+
+export function getErrorResponseForRequestWithFailedVerification<P, D = unknown>(
+  payload: JsonRpcRequest<P>,
+  message: string,
+  data?: D
+): JsonRpcResponseWithErrorPayload<D> {
+  return isNullish(data)
+    ? (getResponseForRequest(payload, undefined, {
+        code: VERIFICATION_FAILED_RESPONSE_CODE,
+        message,
+      }) as JsonRpcResponseWithErrorPayload<D>)
+    : (getResponseForRequest(payload, undefined, {
+        code: VERIFICATION_FAILED_RESPONSE_CODE,
+        message,
+        data,
+      }) as JsonRpcResponseWithErrorPayload<D>);
+}
+
+function isValidResponsePayload<R, E>(
+  response: JsonRpcResponse<R, E> | undefined
+): response is JsonRpcResponseWithResultPayload<R> {
+  return !isNullish(response) && isNullish(response.error);
+}
+
+export function isValidResponse<R, E>(
+  response: JsonRpcResponseOrBatch<R, E> | undefined
+): response is JsonRpcResponseWithResultPayload<R> | JsonRpcResponseWithResultPayload<R>[] {
+  return Array.isArray(response) ? response.every(isValidResponsePayload) : isValidResponsePayload(response);
+}
+
+export function isValidBatchResponse<R, E>(
+  payload: JsonRpcBatchRequest,
+  response: JsonRpcBatchResponse<R, E>
+): response is JsonRpcBatchResponse<R, E> | JsonRpcResponseWithResultPayload<R>[] {
+  for (const [index, req] of payload.entries()) {
+    if (isRequest(req) && (response[index].id !== req.id || !isValidResponse(response[index]))) return false;
+  }
+  return true;
+}
+
+export function mergeBatchReqResp(
+  payload: JsonRpcBatchRequest,
+  response: JsonRpcBatchResponse
+): {request: JsonRpcRequest; response: JsonRpcResponse}[] {
+  const result = [];
+  for (const [index, req] of payload.entries()) {
+    if (isRequest(req)) {
+      // Some providers return raw json-rpc response, some return only result
+      // we need to just merge the result back based on the provider
+      result.push({request: req, response: response[index]});
+    }
+  }
+  return result;
+}
+
+export function isNotification<P>(payload: JsonRpcRequest<P>): payload is JsonRpcNotificationPayload<P> {
+  return !("id" in payload);
+}
+
+export function isRequest<P>(payload: JsonRpcRequest<P>): payload is JsonRpcRequestPayload<P> {
+  return "id" in payload;
+}
+
+export function isBatchRequest<P>(payload: JsonRpcRequestOrBatch<P>): payload is JsonRpcBatchRequest<P> {
+  return Array.isArray(payload);
+}
+
+export function isBatchResponse<R>(response: JsonRpcResponseOrBatch<R>): response is JsonRpcBatchResponse<R> {
+  return Array.isArray(response);
+}
+
+function logRequestPayload(payload: JsonRpcRequest, logger: Logger): void {
+  logger.debug("PR -> EL", {
+    id: isRequest(payload) ? payload.id : "notification",
+    method: payload.method,
+    params: JSON.stringify(payload.params),
+  });
+}
+
+export function logRequest(payload: JsonRpcRequestOrBatch | undefined | null, logger: Logger): void {
+  if (payload === undefined || payload === null) {
+    return;
+  }
+
+  for (const p of isBatchRequest(payload) ? payload : [payload]) {
+    logRequestPayload(p, logger);
+  }
+}
+
+function logResponsePayload(response: JsonRpcResponse | null | undefined, logger: Logger): void {
+  if (response === undefined || response === null) {
+    logger.debug("PR <- EL (empty response)");
+    return;
+  }
+
+  if (isValidResponse(response)) {
+    logger.debug("PR <- EL", {
+      id: response.id,
+      result: JSON.stringify(response.result),
+    });
+  } else {
+    logger.debug("PR <- E:", {
+      id: response.id,
+      error: JSON.stringify(response.error),
+    });
+  }
+}
+
+export function logResponse(response: JsonRpcResponseOrBatch | undefined, logger: Logger): void {
+  if (response === undefined || response === null) {
+    return;
+  }
+
+  for (const p of isBatchResponse(response) ? response : [response]) {
+    logResponsePayload(p, logger);
+  }
+}

package/src/utils/process.ts

@@ -0,0 +1,111 @@
+import {Logger} from "@lodestar/logger";
+import {ELVerifiedRequestHandler} from "../interfaces.js";
+import {ProofProvider} from "../proof_provider/proof_provider.js";
+import {JsonRpcBatchRequest, JsonRpcBatchResponse, JsonRpcRequestOrBatch, JsonRpcResponseOrBatch} from "../types.js";
+import {eth_call} from "../verified_requests/eth_call.js";
+import {eth_estimateGas} from "../verified_requests/eth_estimateGas.js";
+import {eth_getBalance} from "../verified_requests/eth_getBalance.js";
+import {eth_getBlockByHash} from "../verified_requests/eth_getBlockByHash.js";
+import {eth_getBlockByNumber} from "../verified_requests/eth_getBlockByNumber.js";
+import {eth_getCode} from "../verified_requests/eth_getCode.js";
+import {eth_getTransactionCount} from "../verified_requests/eth_getTransactionCount.js";
+import {getResponseForRequest, isBatchRequest, isRequest} from "./json_rpc.js";
+import {ELRpcProvider} from "./rpc_provider.js";
+import {isNullish} from "./validation.js";
+
+// biome-ignore lint/suspicious/noExplicitAny: We need to use `any` type here
+export const verifiableMethodHandlers: Record<string, ELVerifiedRequestHandler<any, any>> = {
+  eth_getBalance: eth_getBalance,
+  eth_getTransactionCount: eth_getTransactionCount,
+  eth_getBlockByHash: eth_getBlockByHash,
+  eth_getBlockByNumber: eth_getBlockByNumber,
+  eth_getCode: eth_getCode,
+  eth_call: eth_call,
+  eth_estimateGas: eth_estimateGas,
+};
+
+export const verifiableMethods = Object.keys(verifiableMethodHandlers);
+export const alwaysAllowedMethods = ["eth_subscribe", "eth_unsubscribe", "eth_getProof"];
+
+export function splitRequestsInChunks(
+  payload: JsonRpcRequestOrBatch,
+  unverifiedWhitelist?: string[]
+): {
+  verifiable: JsonRpcBatchRequest;
+  nonVerifiable: JsonRpcBatchRequest;
+  blocked: JsonRpcBatchRequest;
+} {
+  const verifiable: JsonRpcBatchRequest = [];
+  const nonVerifiable: JsonRpcBatchRequest = [];
+  const blocked: JsonRpcBatchRequest = [];
+
+  for (const pay of isBatchRequest(payload) ? payload : [payload]) {
+    if (isRequest(pay) && verifiableMethods.includes(pay.method)) {
+      verifiable.push(pay);
+      continue;
+    }
+
+    // If unverifiedWhitelist is not set that implies all methods are allowed
+    if ((isRequest(pay) && isNullish(unverifiedWhitelist)) || unverifiedWhitelist?.includes(pay.method)) {
+      nonVerifiable.push(pay);
+      continue;
+    }
+
+    if (alwaysAllowedMethods.includes(pay.method)) {
+      nonVerifiable.push(pay);
+      continue;
+    }
+
+    blocked.push(pay);
+  }
+
+  return {verifiable, nonVerifiable, blocked};
+}
+
+export async function processAndVerifyRequest({
+  payload,
+  rpc,
+  proofProvider,
+  logger,
+}: {
+  payload: JsonRpcRequestOrBatch;
+  rpc: ELRpcProvider;
+  proofProvider: ProofProvider;
+  logger: Logger;
+}): Promise<JsonRpcResponseOrBatch | undefined> {
+  await proofProvider.waitToBeReady();
+
+  const {verifiable, nonVerifiable, blocked} = splitRequestsInChunks(payload, proofProvider.opts.unverifiedWhitelist);
+  const verifiedResponses: JsonRpcBatchResponse = [];
+  const nonVerifiedResponses: JsonRpcBatchResponse = [];
+  const blockedResponses: JsonRpcBatchResponse = [];
+
+  for (const request of verifiable) {
+    logger.debug("Processing verifiable request", {
+      method: request.method,
+      params: JSON.stringify(request.params),
+    });
+    const verifiableRequestHandler = verifiableMethodHandlers[request.method];
+    const response = await verifiableRequestHandler({payload: request, rpc, proofProvider, logger});
+    verifiedResponses.push(response);
+  }
+
+  if (nonVerifiable.length > 0) {
+    logger.warn("Forwarding non-verifiable requests to EL provider.", {count: nonVerifiable.length});
+    const response = await rpc.batchRequest(nonVerifiable, {raiseError: false});
+    nonVerifiedResponses.push(...response.map((r) => r.response));
+  }
+
+  for (const request of blocked) {
+    blockedResponses.push(
+      getResponseForRequest(request, undefined, {message: `Method "${request.method}" not allowed.`})
+    );
+  }
+
+  const responses = [...verifiedResponses, ...nonVerifiedResponses, ...blockedResponses];
+
+  if (responses.length === 1) {
+    return responses[0];
+  }
+  return responses;
+}

package/src/utils/req_resp.ts

@@ -0,0 +1,34 @@
+import http from "node:http";
+import {JsonRpcRequestPayload, JsonRpcResponse} from "../types.js";
+
+export const fetchRequestPayload = async (req: http.IncomingMessage): Promise<JsonRpcRequestPayload> => {
+  return new Promise((resolve, reject) => {
+    let body = "";
+    req.on("data", (chunk) => {
+      body += chunk;
+    });
+    req.on("end", () => {
+      try {
+        resolve(JSON.parse(body) as JsonRpcRequestPayload);
+      } catch (err) {
+        reject(err);
+      }
+    });
+  });
+};
+
+export const fetchResponseBody = async (res: http.IncomingMessage): Promise<JsonRpcResponse> => {
+  return new Promise((resolve, reject) => {
+    let body = "";
+    res.on("data", (chunk) => {
+      body += chunk;
+    });
+    res.on("end", () => {
+      try {
+        resolve(JSON.parse(body) as JsonRpcResponse);
+      } catch (err) {
+        reject(err);
+      }
+    });
+  });
+};

package/src/utils/rpc_provider.ts

@@ -0,0 +1,117 @@
+import {Logger} from "@lodestar/logger";
+import {ZERO_ADDRESS} from "../constants.js";
+import {ELRequestHandler} from "../interfaces.js";
+import {
+  ELApi,
+  ELApiParams,
+  ELApiReturn,
+  JsonRpcBatchRequest,
+  JsonRpcBatchResponse,
+  JsonRpcRequest,
+  JsonRpcResponse,
+  JsonRpcResponseWithResultPayload,
+} from "../types.js";
+import {
+  isRequest,
+  isValidBatchResponse,
+  isValidResponse,
+  logRequest,
+  logResponse,
+  mergeBatchReqResp,
+} from "./json_rpc.js";
+import {isNullish} from "./validation.js";
+
+export type Optional<T, K extends keyof T> = Omit<T, K> & {[P in keyof T]?: T[P] | undefined};
+
+export class ELRpcProvider {
+  private handler: ELRequestHandler;
+  private logger: Logger;
+
+  private requestId = 0;
+
+  constructor(handler: ELRequestHandler, logger: Logger) {
+    this.handler = handler;
+    this.logger = logger;
+  }
+
+  /**
+   * Request the EL RPC Provider
+   *
+   * @template K
+   * @template E
+   * @param {K} method - RPC Method
+   * @param {ELApiParams[K]} params - RPC Params
+   * @param {{raiseError?: E}} [opts]
+   * @return {*}  {Promise<E extends false ? JsonRpcResponse<ELApiReturn[K]> : JsonRpcResponseWithResultPayload<ELApiReturn[K]>>}
+   * @memberof ELRpc
+   */
+  async request<K extends keyof ELApi, E extends boolean>(
+    method: K,
+    params: ELApiParams[K],
+    opts?: {raiseError?: E}
+  ): Promise<E extends false ? JsonRpcResponse<ELApiReturn[K]> : JsonRpcResponseWithResultPayload<ELApiReturn[K]>> {
+    const {raiseError} = opts ?? {raiseError: true};
+
+    const payload: JsonRpcRequest = {jsonrpc: "2.0", method, params, id: this.getRequestId()};
+    logRequest(payload, this.logger);
+
+    const response = await this.handler(payload);
+    logResponse(response, this.logger);
+
+    if (raiseError && !isValidResponse(response)) {
+      throw new Error(`Invalid response from RPC. method=${method} params=${JSON.stringify(params)}`);
+    }
+
+    return response as JsonRpcResponseWithResultPayload<ELApiReturn[K]>;
+  }
+
+  async batchRequest<E extends boolean>(
+    input: JsonRpcBatchRequest,
+    opts: {raiseError: E}
+  ): Promise<
+    E extends false
+      ? {request: JsonRpcRequest; response: JsonRpcResponse}[]
+      : {request: JsonRpcRequest; response: JsonRpcResponseWithResultPayload<unknown>}[]
+  > {
+    const payloads: JsonRpcBatchRequest = [];
+
+    for (const req of input) {
+      if (isRequest(req) && isNullish(req.id)) {
+        payloads.push({jsonrpc: "2.0", method: req.method, params: req.params, id: this.getRequestId()});
+      } else {
+        payloads.push(req);
+      }
+    }
+
+    logRequest(payloads, this.logger);
+    const response = await this.handler(payloads);
+    logResponse(response, this.logger);
+
+    if (isNullish(response)) {
+      throw new Error("Invalid empty response from server.");
+    }
+
+    if (opts.raiseError && !isValidBatchResponse(payloads, response as JsonRpcBatchResponse)) {
+      throw new Error(
+        `Invalid response from RPC. payload=${JSON.stringify(payloads)} response=${JSON.stringify(response)}}`
+      );
+    }
+
+    return mergeBatchReqResp(payloads, response as JsonRpcBatchResponse) as E extends false
+      ? {request: JsonRpcRequest; response: JsonRpcResponse}[]
+      : {request: JsonRpcRequest; response: JsonRpcResponseWithResultPayload<unknown>}[];
+  }
+
+  async verifyCompatibility(): Promise<void> {
+    try {
+      await this.request("eth_getProof", [ZERO_ADDRESS, [], "latest"], {raiseError: true});
+    } catch (err) {
+      this.logger.error("Execution compatibility failed.", undefined, err as Error);
+      throw new Error("RPC does not support 'eth_getProof', which is required for the prover to work properly.");
+    }
+  }
+
+  getRequestId(): string {
+    return (++this.requestId).toString();
+  }
+}

package/src/utils/validation.ts

@@ -0,0 +1,161 @@
+import {Block} from "@ethereumjs/block";
+import {RLP} from "@ethereumjs/rlp";
+import {Trie} from "@ethereumjs/trie";
+import {Account, KECCAK256_NULL_S} from "@ethereumjs/util";
+import {keccak256} from "ethereum-cryptography/keccak.js";
+import {ChainForkConfig} from "@lodestar/config";
+import {Bytes32, ExecutionPayload} from "@lodestar/types";
+import {Logger} from "@lodestar/utils";
+import {ELBlock, ELProof, ELStorageProof, HexString} from "../types.js";
+import {blockDataFromELBlock, bufferToHex, hexToBuffer, padLeft} from "./conversion.js";
+import {getChainCommon} from "./execution.js";
+
+const emptyAccountSerialize = new Account().serialize();
+const storageKeyLength = 32;
+
+export function isBlockNumber(block: number | string): boolean {
+  if (typeof block === "number") {
+    return true;
+  }
+
+  // If block is hex and less than 32 byte long it is a block number, else it's a block hash
+  return hexToBuffer(block).byteLength < 32;
+}
+
+export async function isValidAccount({
+  address,
+  stateRoot,
+  proof,
+  logger,
+}: {
+  address: HexString;
+  stateRoot: Bytes32;
+  proof: ELProof;
+  logger: Logger;
+}): Promise<boolean> {
+  const trie = await Trie.create();
+  const key = keccak256(hexToBuffer(address));
+
+  try {
+    const expectedAccountRLP = await trie.verifyProof(
+      Buffer.from(stateRoot),
+      Buffer.from(key),
+      proof.accountProof.map(hexToBuffer)
+    );
+
+    // Shresth Agrawal (2022) Patronum source code. https://github.com/lightclients/patronum
+    const account = Account.fromAccountData({
+      nonce: BigInt(proof.nonce),
+      balance: BigInt(proof.balance),
+      storageRoot: proof.storageHash,
+      codeHash: proof.codeHash,
+    });
+    return account.serialize().equals(expectedAccountRLP ? expectedAccountRLP : emptyAccountSerialize);
+  } catch (err) {
+    logger.error("Error verifying account proof", undefined, err as Error);
+    return false;
+  }
+}
+
+export async function isValidStorageKeys({
+  storageKeys,
+  proof,
+  logger,
+}: {
+  storageKeys: HexString[];
+  proof: ELStorageProof;
+  logger: Logger;
+}): Promise<boolean> {
+  const trie = await Trie.create();
+
+  for (let i = 0; i < storageKeys.length; i++) {
+    const sp = proof.storageProof[i];
+    const key = keccak256(padLeft(hexToBuffer(storageKeys[i]), storageKeyLength));
+    try {
+      const expectedStorageRLP = await trie.verifyProof(
+        hexToBuffer(proof.storageHash),
+        Buffer.from(key),
+        sp.proof.map(hexToBuffer)
+      );
+
+      // buffer.equals is not compatible with Uint8Array for browser
+      // so we need to convert the output of RLP.encode to Buffer first
+      const isStorageValid =
+        (!expectedStorageRLP && sp.value === "0x0") ||
+        (!!expectedStorageRLP && expectedStorageRLP.equals(Buffer.from(RLP.encode(sp.value))));
+      if (!isStorageValid) return false;
+    } catch (err) {
+      logger.error("Error verifying storage keys", undefined, err as Error);
+      return false;
+    }
+  }
+
+  return true;
+}
+
+export async function isValidBlock({
+  executionPayload,
+  block,
+  logger,
+  config,
+}: {
+  executionPayload: ExecutionPayload;
+  block: ELBlock;
+  logger: Logger;
+  config: ChainForkConfig;
+}): Promise<boolean> {
+  if (bufferToHex(executionPayload.blockHash) !== block.hash) {
+    logger.error("Block hash does not match", {
+      rpcBlockHash: block.hash,
+      beaconExecutionBlockHash: bufferToHex(executionPayload.blockHash),
+    });
+
+    return false;
+  }
+
+  if (bufferToHex(executionPayload.parentHash) !== block.parentHash) {
+    logger.error("Block parent hash does not match", {
+      rpcBlockHash: block.parentHash,
+      beaconExecutionBlockHash: bufferToHex(executionPayload.parentHash),
+    });
+
+    return false;
+  }
+
+  const common = getChainCommon(config.PRESET_BASE);
+  common.setHardforkByBlockNumber(executionPayload.blockNumber, undefined, executionPayload.timestamp);
+  const blockObject = Block.fromBlockData(blockDataFromELBlock(block), {common});
+
+  if (!(await blockObject.validateTransactionsTrie())) {
+    logger.error("Block transactions could not be verified.", {
+      blockHash: bufferToHex(blockObject.hash()),
+      blockNumber: blockObject.header.number,
+    });
+
+    return false;
+  }
+
+  return true;
+}
+
+export async function isValidCodeHash({
+  codeHash,
+  codeResponse,
+}: {
+  codeHash: string;
+  codeResponse: string;
+  logger: Logger;
+}): Promise<boolean> {
+  // if there is no code hash for that address
+  if (codeResponse === "0x" && codeHash === `0x${KECCAK256_NULL_S}`) return true;
+
+  return bufferToHex(keccak256(hexToBuffer(codeResponse))) === codeHash;
+}
+
+export function isNullish<T>(val: T | undefined | null): val is null | undefined {
+  return val === null || val === undefined;
+}
+
+export function isPresent<T>(val: T | undefined | null): val is T {
+  return !isNullish(val);
+}