@lodestar/prover 1.35.0-dev.e18102ed8c → 1.35.0-dev.f45a2be721

package/src/utils/evm.ts

@@ -0,0 +1,284 @@
+import {Block, BlockHeader} from "@ethereumjs/block";
+import {Blockchain} from "@ethereumjs/blockchain";
+import {TransactionFactory} from "@ethereumjs/tx";
+import {Account, Address} from "@ethereumjs/util";
+import {RunTxResult, VM} from "@ethereumjs/vm";
+import {NetworkName} from "@lodestar/config/networks";
+import {ExecutionPayload} from "@lodestar/types";
+import {Logger} from "@lodestar/utils";
+import {ZERO_ADDRESS} from "../constants.js";
+import {ProofProvider} from "../proof_provider/proof_provider.js";
+import {ELBlock, ELProof, ELTransaction, JsonRpcVersion} from "../types.js";
+import {bufferToHex, chunkIntoN, cleanObject, hexToBigInt, hexToBuffer, numberToHex, padLeft} from "./conversion.js";
+import {getChainCommon, getTxType} from "./execution.js";
+import {isValidResponse} from "./json_rpc.js";
+import {ELRpcProvider} from "./rpc_provider.js";
+import {isNullish, isValidAccount, isValidCodeHash, isValidStorageKeys} from "./validation.js";
+
+export async function createVM({proofProvider}: {proofProvider: ProofProvider}): Promise<VM> {
+  const common = getChainCommon(proofProvider.config.PRESET_BASE as string);
+  const blockchain = await Blockchain.create({common});
+
+  // Connect blockchain object with existing proof provider for block history
+  // biome-ignore lint/suspicious/noExplicitAny: We need to use `any` type here
+  (blockchain as any).getBlock = async (blockId: number) => {
+    const payload = await proofProvider.getExecutionPayload(blockId);
+    return {
+      hash: () => payload.blockHash,
+    };
+  };
+  const vm = await VM.create({common, blockchain});
+
+  return vm;
+}
+
+export async function getVMWithState({
+  rpc,
+  executionPayload,
+  tx,
+  vm,
+  logger,
+}: {
+  rpc: ELRpcProvider;
+  vm: VM;
+  executionPayload: ExecutionPayload;
+  tx: ELTransaction;
+  logger: Logger;
+}): Promise<VM> {
+  const {stateRoot, blockHash, gasLimit} = executionPayload;
+  const blockHashHex = bufferToHex(blockHash);
+
+  // If tx does not have a from address then it must be initiated via zero address
+  const from = tx.from ?? ZERO_ADDRESS;
+  const to = tx.to;
+
+  // Create Access List for the contract call
+  const accessListTx = cleanObject({
+    to,
+    from,
+    data: tx.input ? tx.input : tx.data,
+    value: tx.value,
+    gas: tx.gas ? tx.gas : numberToHex(gasLimit),
+    gasPrice: "0x0",
+  }) as ELTransaction;
+  const response = await rpc.request("eth_createAccessList", [accessListTx, blockHashHex], {raiseError: false});
+
+  if (!isValidResponse(response) || response.result.error) {
+    throw new Error(`Invalid response from RPC. method: eth_createAccessList, params: ${JSON.stringify(tx)}`);
+  }
+
+  const storageKeysMap: Record<string, string[]> = {};
+  for (const {address, storageKeys} of response.result.accessList) {
+    storageKeysMap[address] = storageKeys;
+  }
+
+  // If from address is not present then we have to fetch it for all keys
+  if (isNullish(storageKeysMap[from])) {
+    storageKeysMap[from] = [];
+  }
+
+  // If to address is not present then we have to fetch it with for all keys
+  if (to && isNullish(storageKeysMap[to])) {
+    storageKeysMap[to] = [];
+  }
+
+  const batchRequests = [];
+  for (const [address, storageKeys] of Object.entries(storageKeysMap)) {
+    batchRequests.push({
+      jsonrpc: "2.0" as JsonRpcVersion,
+      id: rpc.getRequestId(),
+      method: "eth_getProof",
+      params: [address, storageKeys, blockHashHex],
+    });
+
+    batchRequests.push({
+      jsonrpc: "2.0" as JsonRpcVersion,
+      id: rpc.getRequestId(),
+      method: "eth_getCode",
+      params: [address, blockHashHex],
+    });
+  }
+
+  // If all responses are valid then we will have even number of responses
+  // For each address, one response for eth_getProof and one for eth_getCode
+  const batchResponse = await rpc.batchRequest(batchRequests, {raiseError: true});
+  const batchResponseInChunks = chunkIntoN(batchResponse, 2);
+
+  const vmState: VMState = {};
+  for (const [proofResponse, codeResponse] of batchResponseInChunks) {
+    const addressHex = proofResponse.request.params[0] as string;
+    if (!isNullish(vmState[addressHex])) continue;
+
+    const proof = proofResponse.response.result as ELProof;
+    const storageKeys = proofResponse.request.params[1] as string[];
+    const code = codeResponse.response.result as string;
+
+    const validAccount = await isValidAccount({address: addressHex, proof, logger, stateRoot});
+    const validStorage = validAccount && (await isValidStorageKeys({storageKeys, proof, logger}));
+    if (!validAccount || !validStorage) {
+      throw new Error(`Invalid account: ${addressHex}`);
+    }
+
+    if (!(await isValidCodeHash({codeResponse: code, logger, codeHash: proof.codeHash}))) {
+      throw new Error(`Invalid code hash: ${addressHex}`);
+    }
+
+    vmState[addressHex] = {code, proof};
+  }
+
+  return updateVMWithState({vm, state: vmState, logger});
+}
+type VMState = Record<string, {code: string; proof: ELProof}>;
+export async function updateVMWithState({vm, state}: {logger: Logger; state: VMState; vm: VM}): Promise<VM> {
+  await vm.stateManager.checkpoint();
+  for (const [addressHex, {proof, code}] of Object.entries(state)) {
+    const address = Address.fromString(addressHex);
+    const codeBuffer = hexToBuffer(code);
+
+    const account = Account.fromAccountData({
+      nonce: BigInt(proof.nonce),
+      balance: BigInt(proof.balance),
+      codeHash: proof.codeHash,
+    });
+
+    await vm.stateManager.putAccount(address, account);
+
+    for (const {key, value} of proof.storageProof) {
+      await vm.stateManager.putContractStorage(address, padLeft(hexToBuffer(key), 32), padLeft(hexToBuffer(value), 32));
+    }
+    if (codeBuffer.byteLength !== 0) await vm.stateManager.putContractCode(address, codeBuffer);
+  }
+
+  await vm.stateManager.commit();
+  return vm;
+}
+
+export async function executeVMCall({
+  rpc,
+  tx,
+  vm,
+  executionPayload,
+  network,
+}: {
+  rpc: ELRpcProvider;
+  tx: ELTransaction;
+  vm: VM;
+  executionPayload: ExecutionPayload;
+  network: NetworkName;
+}): Promise<RunTxResult["execResult"]> {
+  const {from, to, gas, gasPrice, maxPriorityFeePerGas, value, data, input} = tx;
+  const blockHash = bufferToHex(executionPayload.blockHash);
+  const {result: block} = await rpc.request("eth_getBlockByHash", [blockHash, true], {
+    raiseError: true,
+  });
+
+  if (!block) {
+    throw new Error(`Block not found: ${blockHash}`);
+  }
+
+  const {execResult} = await vm.evm.runCall({
+    caller: from ? Address.fromString(from) : undefined,
+    to: to ? Address.fromString(to) : undefined,
+    gasLimit: hexToBigInt(gas ?? block.gasLimit),
+    gasPrice: hexToBigInt(gasPrice ?? maxPriorityFeePerGas ?? "0x0"),
+    value: hexToBigInt(value ?? "0x0"),
+    data: input ? hexToBuffer(input) : data ? hexToBuffer(data) : undefined,
+    block: {
+      header: getVMBlockHeaderFromELBlock(block, executionPayload, network),
+    },
+  });
+
+  if (execResult.exceptionError) {
+    throw new Error(execResult.exceptionError.error);
+  }
+
+  return execResult;
+}
+
+export async function executeVMTx({
+  rpc,
+  tx,
+  vm,
+  executionPayload,
+  network,
+}: {
+  rpc: ELRpcProvider;
+  tx: ELTransaction;
+  vm: VM;
+  executionPayload: ExecutionPayload;
+  network: NetworkName;
+}): Promise<RunTxResult> {
+  const {result: block} = await rpc.request("eth_getBlockByHash", [bufferToHex(executionPayload.blockHash), true], {
+    raiseError: true,
+  });
+
+  if (!block) {
+    throw new Error(`Block not found: ${bufferToHex(executionPayload.blockHash)}`);
+  }
+  const txType = getTxType(tx);
+  const from = tx.from ? Address.fromString(tx.from) : Address.zero();
+  const to = tx.to ? Address.fromString(tx.to) : undefined;
+
+  const txData = {
+    ...tx,
+    from,
+    to,
+    type: txType,
+    // If no gas limit is specified use the last block gas limit as an upper bound.
+    gasLimit: hexToBigInt(tx.gas ?? block.gasLimit),
+  };
+
+  if (txType === 2) {
+    // Handle EIP-1559 transactions
+    // To fix the vm error: Transaction's maxFeePerGas (0) is less than the block's baseFeePerGas
+    txData.maxFeePerGas = txData.maxFeePerGas ?? block.baseFeePerGas;
+  } else {
+    // Legacy transaction
+    txData.gasPrice = isNullish(txData.gasPrice) || txData.gasPrice === "0x0" ? block.baseFeePerGas : txData.gasPrice;
+  }
+
+  const txObject = TransactionFactory.fromTxData(txData, {common: getChainCommon(network), freeze: false});
+
+  // Override to avoid tx signature verification
+  txObject.getSenderAddress = () => (tx.from ? Address.fromString(tx.from) : Address.zero());
+
+  const result = await vm.runTx({
+    tx: txObject,
+    skipNonce: true,
+    skipBalance: true,
+    skipBlockGasLimitValidation: true,
+    skipHardForkValidation: true,
+    block: {
+      header: getVMBlockHeaderFromELBlock(block, executionPayload, network),
+    } as Block,
+  });
+
+  return result;
+}
+
+export function getVMBlockHeaderFromELBlock(
+  block: ELBlock,
+  executionPayload: ExecutionPayload,
+  network: NetworkName
+): BlockHeader {
+  const blockHeaderData = {
+    number: hexToBigInt(block.number),
+    cliqueSigner: () => Address.fromString(block.miner),
+    timestamp: hexToBigInt(block.timestamp),
+    difficulty: hexToBigInt(block.difficulty),
+    gasLimit: hexToBigInt(block.gasLimit),
+    baseFeePerGas: block.baseFeePerGas ? hexToBigInt(block.baseFeePerGas) : undefined,
+
+    // Use these values from the execution payload
+    // instead of the block values to ensure that
+    // the VM is using the verified values from the lightclient
+    prevRandao: Buffer.from(executionPayload.prevRandao),
+    stateRoot: Buffer.from(executionPayload.stateRoot),
+    parentHash: Buffer.from(executionPayload.parentHash),
+
+    // TODO: Fix the coinbase address
+    coinbase: Address.fromString(block.miner),
+  };
+
+  return BlockHeader.fromHeaderData(blockHeaderData, {common: getChainCommon(network)});
+}

package/src/utils/execution.ts

@@ -0,0 +1,76 @@
+import {Common, CustomChain, Hardfork} from "@ethereumjs/common";
+import {ELApiParams, ELApiReturn, ELTransaction} from "../types.js";
+import {isValidResponse} from "./json_rpc.js";
+import {ELRpcProvider} from "./rpc_provider.js";
+import {isBlockNumber, isPresent} from "./validation.js";
+
+export type Optional<T, K extends keyof T> = Omit<T, K> & {[P in keyof T]?: T[P] | undefined};
+
+export async function getELCode(
+  rpc: ELRpcProvider,
+  args: ELApiParams["eth_getCode"]
+): Promise<ELApiReturn["eth_getCode"]> {
+  const codeResult = await rpc.request("eth_getCode", args, {raiseError: false});
+
+  if (!isValidResponse(codeResult)) {
+    throw new Error(`Can not find code for address=${args[0]}`);
+  }
+
+  return codeResult.result;
+}
+
+export async function getELProof(
+  rpc: ELRpcProvider,
+  args: ELApiParams["eth_getProof"]
+): Promise<ELApiReturn["eth_getProof"]> {
+  const proof = await rpc.request("eth_getProof", args, {raiseError: false});
+  if (!isValidResponse(proof)) {
+    throw new Error(`Can not find proof for address=${args[0]}`);
+  }
+  return proof.result;
+}
+
+export async function getELBlock(
+  rpc: ELRpcProvider,
+  args: ELApiParams["eth_getBlockByNumber"]
+): Promise<ELApiReturn["eth_getBlockByNumber"]> {
+  const block = await rpc.request(isBlockNumber(args[0]) ? "eth_getBlockByNumber" : "eth_getBlockByHash", args, {
+    raiseError: false,
+  });
+
+  if (!isValidResponse(block)) {
+    throw new Error(`Can not find block. id=${args[0]}`);
+  }
+
+  return block.result;
+}
+
+export function getChainCommon(network: string): Common {
+  switch (network) {
+    case "mainnet":
+    case "sepolia":
+    case "holesky":
+    case "ephemery":
+      // TODO: Not sure how to detect the fork during runtime
+      return new Common({chain: network, hardfork: Hardfork.Shanghai});
+    case "minimal":
+      // TODO: Not sure how to detect the fork during runtime
+      return new Common({chain: "mainnet", hardfork: Hardfork.Shanghai});
+    case "gnosis":
+      return new Common({chain: CustomChain.xDaiChain});
+    default:
+      throw new Error(`Non supported network "${network}"`);
+  }
+}
+
+export function getTxType(tx: ELTransaction): number {
+  if (isPresent(tx.maxFeePerGas) || isPresent(tx.maxPriorityFeePerGas)) {
+    return 2;
+  }
+
+  if (isPresent(tx.accessList)) {
+    return 1;
+  }
+
+  return 0;
+}

package/src/utils/file.ts

@@ -0,0 +1,51 @@
+import fs from "node:fs";
+import path from "node:path";
+import yaml from "js-yaml";
+
+const {load, FAILSAFE_SCHEMA, Type} = yaml;
+
+enum FileFormat {
+  json = "json",
+  yaml = "yaml",
+  yml = "yml",
+  toml = "toml",
+}
+
+const yamlSchema = FAILSAFE_SCHEMA.extend({
+  implicit: [
+    new Type("tag:yaml.org,2002:str", {
+      kind: "scalar",
+      construct: function construct(data) {
+        return data !== null ? data : "";
+      },
+    }),
+  ],
+});
+
+/**
+ * Parse file contents as Json.
+ */
+function parse<T>(contents: string, fileFormat: FileFormat): T {
+  switch (fileFormat) {
+    case FileFormat.json:
+      return JSON.parse(contents) as T;
+    case FileFormat.yaml:
+    case FileFormat.yml:
+      return load(contents, {schema: yamlSchema}) as T;
+    default:
+      return contents as unknown as T;
+  }
+}
+
+/**
+ * Read a JSON serializable object from a file
+ *
+ * Parse either from json, yaml, or toml
+ * Optional acceptedFormats object can be passed which can be an array of accepted formats, in future can be extended to include parseFn for the accepted formats
+ */
+export function readFile<T>(filepath: string, acceptedFormats?: string[]): T {
+  const fileFormat = path.extname(filepath).substr(1);
+  if (acceptedFormats && !acceptedFormats.includes(fileFormat)) throw new Error(`UnsupportedFileFormat: ${filepath}`);
+  const contents = fs.readFileSync(filepath, "utf-8");
+  return parse(contents, fileFormat as FileFormat);
+}

package/src/utils/gitData/gitDataPath.ts

@@ -0,0 +1,48 @@
+import fs from "node:fs";
+import path from "node:path";
+import {fileURLToPath} from "node:url";
+
+// Global variable __dirname no longer available in ES6 modules.
+// Solutions: https://stackoverflow.com/questions/46745014/alternative-for-dirname-in-node-js-when-using-es6-modules
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+// Persist git data and distribute through NPM so CLI consumers can know exactly
+// at what commit was this src build. This is used in the metrics and to log initially.
+//
+// - For NPM release (stable): Only the version is persisted. Once must then track the version's tag
+//   in Github to resolve that version to a specific commit. While this is okay, git-data.json gives
+//   a gurantee of the exact commit at build time.
+//
+// - For NPM release (dev): canary commits include the commit, so this feature is not really
+//   necessary. However, it's more cumbersome to have conditional logic on stable / dev.
+//
+// - For build from source: .git folder is available in the context of the built code, so it can extract
+//   branch and commit directly without the need for .git-data.json.
+//
+// - For build from source dockerized: This feature is required to know the branch and commit, since
+//   git data is not persisted past the build. However, .dockerignore prevents .git folder from being
+//   copied into the container's context, so .git-data.json can't be generated.
+
+/**
+ * WARNING!! If you change this path make sure to update:
+ * - 'packages/cli/package.json' -> .files -> `".git-data.json"`
+ */
+export const gitDataPath = path.resolve(__dirname, "../../../.git-data.json");
+
+/** Git data type used to construct version information string and persistence. */
+export type GitData = {
+  /** "developer-feature" */
+  branch: string;
+  /** "80c248bb392f512cc115d95059e22239a17bbd7d" */
+  commit: string;
+};
+
+/** Writes a persistent git data file. */
+export function writeGitDataFile(gitData: GitData): void {
+  fs.writeFileSync(gitDataPath, JSON.stringify(gitData, null, 2));
+}
+
+/** Reads the persistent git data file. */
+export function readGitDataFile(): GitData {
+  return JSON.parse(fs.readFileSync(gitDataPath, "utf8")) as GitData;
+}

package/src/utils/gitData/index.ts

@@ -0,0 +1,70 @@
+import {execSync} from "node:child_process";
+// This file is created in the build step and is distributed through NPM
+// MUST be in sync with `-/gitDataPath.ts` and `package.json` files.
+import {GitData, readGitDataFile} from "./gitDataPath.js";
+
+/** Reads git data from a persisted file or local git data at build time. */
+export function readAndGetGitData(): GitData {
+  try {
+    // Gets git data containing current branch and commit info from persistent file.
+    let persistedGitData: Partial<GitData>;
+    try {
+      persistedGitData = readGitDataFile();
+    } catch (_e) {
+      persistedGitData = {};
+    }
+
+    const currentGitData = getGitData();
+
+    return {
+      // If the CLI is run from source, prioritze current git data
+      // over `.git-data.json` file, which might be stale here.
+      branch:
+        currentGitData.branch && currentGitData.branch.length > 0
+          ? currentGitData.branch
+          : (persistedGitData.branch ?? ""),
+      commit:
+        currentGitData.commit && currentGitData.commit.length > 0
+          ? currentGitData.commit
+          : (persistedGitData.commit ?? ""),
+    };
+  } catch (_e) {
+    return {
+      branch: "",
+      commit: "",
+    };
+  }
+}
+
+/** Gets git data containing current branch and commit info from CLI. */
+export function getGitData(): GitData {
+  return {
+    branch: process.env.GIT_BRANCH ?? getBranch(),
+    commit: process.env.GIT_COMMIT ?? getCommit(),
+  };
+}
+
+/** Tries to get branch from git CLI. */
+function getBranch(): string {
+  try {
+    return shellSilent("git rev-parse --abbrev-ref HEAD");
+  } catch (_e) {
+    return "";
+  }
+}
+
+/** Tries to get commit from git from git CLI. */
+function getCommit(): string {
+  try {
+    return shellSilent("git rev-parse --verify HEAD");
+  } catch (_e) {
+    return "";
+  }
+}
+
+/** Silent shell that won't pollute stdout, or stderr */
+function shellSilent(cmd: string): string {
+  return execSync(cmd, {stdio: ["ignore", "pipe", "ignore"]})
+    .toString()
+    .trim();
+}

package/src/utils/gitData/writeGitData.ts

@@ -0,0 +1,10 @@
+#!/usr/bin/env node
+
+// For RATIONALE of this file, check packages/cli/src/util/gitData/gitDataPath.ts
+// Persist exact commit in NPM distributions for easier tracking of the build
+
+import {writeGitDataFile} from "./gitDataPath.js";
+import {getGitData} from "./index.js";
+
+// Script to write the git data file (json) used by the build procedures to persist git data.
+writeGitDataFile(getGitData());