@lodestar/prover 1.35.0-dev.0c6f50771f → 1.35.0-dev.248b252a94

package/src/cli/cli.ts

@@ -0,0 +1,58 @@
+// Must not use `* as yargs`, see https://github.com/yargs/yargs/issues/1131
+import yargs, {Argv} from "yargs";
+import {hideBin} from "yargs/helpers";
+import {registerCommandToYargs} from "@lodestar/utils";
+import {getVersionData} from "../utils/version.js";
+import {cmds, proverProxyStartCommand} from "./cmds/index.js";
+import {globalOptions} from "./options.js";
+
+const {version} = getVersionData();
+const topBanner = `🌟 Lodestar Prover Proxy: Ethereum RPC proxy for RPC responses, verified against the trusted block hashes.
+  * Version: ${version}
+  * by ChainSafe Systems, 2018-${new Date().getFullYear()}`;
+const bottomBanner = `📖 For more information, check the CLI reference:
+  * https://chainsafe.github.io/lodestar/reference/cli
+
+✍️ Give feedback and report issues on GitHub:
+  * https://github.com/ChainSafe/lodestar`;
+
+export const yarg = yargs((hideBin as (args: string[]) => string[])(process.argv));
+
+/**
+ * Common factory for running the CLI and running integration tests
+ * The CLI must actually be executed in a different script
+ */
+export function getLodestarProverCli(): Argv {
+  const prover = yarg
+    .env("LODESTAR")
+    .parserConfiguration({
+      // As of yargs v16.1.0 dot-notation breaks strictOptions()
+      // Manually processing options is typesafe tho more verbose
+      "dot-notation": false,
+    })
+    .options(globalOptions)
+    // blank scriptName so that help text doesn't display the cli name before each command
+    .scriptName("")
+    .demandCommand(1)
+    // Control show help behaviour below on .fail()
+    .showHelpOnFail(false)
+    .usage(topBanner)
+    .epilogue(bottomBanner)
+    .version(topBanner)
+    .alias("h", "help")
+    .alias("v", "version")
+    .recommendCommands();
+
+  // yargs.command and all ./cmds
+  for (const cmd of cmds) {
+    registerCommandToYargs(prover, cmd);
+  }
+
+  // Register the proxy command as the default one
+  registerCommandToYargs(prover, {...proverProxyStartCommand, command: "*"});
+
+  // throw an error if we see an unrecognized cmd
+  prover.recommendCommands().strict();
+
+  return prover;
+}

package/src/cli/cmds/index.ts

@@ -0,0 +1,7 @@
+import {CliCommand} from "@lodestar/utils";
+import {GlobalArgs} from "../options.js";
+import {proverProxyStartCommand} from "./start/index.js";
+
+export {proverProxyStartCommand} from "./start/index.js";
+
+export const cmds: Required<CliCommand<GlobalArgs, Record<never, never>>>["subcommands"] = [proverProxyStartCommand];

package/src/cli/cmds/start/handler.ts

@@ -0,0 +1,27 @@
+import {ChainConfig, chainConfigFromJson} from "@lodestar/config";
+import {readFile} from "../../../utils/file.js";
+import {VerifiedProxyOptions, createVerifiedExecutionProxy} from "../../../web3_proxy.js";
+import {GlobalArgs, parseGlobalArgs} from "../../options.js";
+import {StartArgs, parseStartArgs} from "./options.js";
+
+/**
+ * Runs a beacon node.
+ */
+export async function proverProxyStartHandler(args: StartArgs & GlobalArgs): Promise<void> {
+  const {network, logLevel, paramsFile} = parseGlobalArgs(args);
+  const opts = parseStartArgs(args);
+
+  const config: Partial<ChainConfig> = paramsFile ? chainConfigFromJson(readFile(paramsFile)) : {};
+
+  const options: VerifiedProxyOptions = {
+    ...opts,
+    logLevel,
+    ...(network ? {network} : {config}),
+  };
+
+  const {server, proofProvider} = createVerifiedExecutionProxy(options);
+
+  server.listen(opts.port);
+
+  await proofProvider.waitToBeReady();
+}

package/src/cli/cmds/start/index.ts

@@ -0,0 +1,18 @@
+import {CliCommand} from "@lodestar/utils";
+import {GlobalArgs} from "../../options.js";
+import {proverProxyStartHandler} from "./handler.js";
+import {StartArgs, startOptions} from "./options.js";
+
+export const proverProxyStartCommand: CliCommand<StartArgs, GlobalArgs> = {
+  command: "proxy",
+  describe: "Start proxy server",
+  examples: [
+    {
+      command:
+        "start --network sepolia --execution-rpc https://lodestar-sepoliarpc.chainsafe.io --mode rest --beacon-rpc https://lodestar-sepolia.chainsafe.io",
+      description: "Start a proxy server and connect to the sepolia testnet",
+    },
+  ],
+  options: startOptions,
+  handler: proverProxyStartHandler,
+};

package/src/cli/cmds/start/options.ts

@@ -0,0 +1,85 @@
+import {CliCommandOptions} from "@lodestar/utils";
+import {DEFAULT_PROXY_REQUEST_TIMEOUT} from "../../../constants.js";
+import {LCTransport} from "../../../interfaces.js";
+import {alwaysAllowedMethods} from "../../../utils/process.js";
+
+export type StartArgs = {
+  port: number;
+  executionRpcUrl: string;
+  beaconUrls: string[];
+  wsCheckpoint?: string;
+  unverifiedWhitelist?: string[];
+  requestTimeout: number;
+};
+
+export type StartOptions = {
+  executionRpcUrl: string;
+  port: number;
+  wsCheckpoint?: string;
+  unverifiedWhitelist?: string[];
+  requestTimeout: number;
+} & {transport: LCTransport.Rest; urls: string[]};
+
+export const startOptions: CliCommandOptions<StartArgs> = {
+  port: {
+    description: "Port number to start the proxy.",
+    type: "number",
+    default: 8080,
+  },
+
+  executionRpcUrl: {
+    description: "RPC url for the execution node.",
+    type: "string",
+    demandOption: true,
+    group: "execution",
+  },
+
+  unverifiedWhitelist: {
+    description: `Methods which are allowed to forward. If not provided, all methods are allowed. ${alwaysAllowedMethods.join(
+      ","
+    )} are always allowed.`,
+    type: "array",
+    demandOption: false,
+    group: "execution",
+  },
+
+  requestTimeout: {
+    description: "Number of ms to wait for a response from the execution node.",
+    default: DEFAULT_PROXY_REQUEST_TIMEOUT,
+    type: "number",
+    demandOption: false,
+    group: "execution",
+  },
+
+  beaconUrls: {
+    description: "Urls of the beacon nodes to connect to.",
+    type: "array",
+    string: true,
+    coerce: (urls: string[]): string[] =>
+      // Parse ["url1,url2"] to ["url1", "url2"]
+      urls.flatMap((item) => item.split(",")),
+    demandOption: true,
+    group: "beacon",
+  },
+
+  wsCheckpoint: {
+    description:
+      "The trusted checkpoint root to start the lightclient. If not provided will initialize from the latest finalized slot. It shouldn't be older than weak subjectivity period",
+    type: "string",
+    demandOption: false,
+    group: "beacon",
+  },
+};
+
+export function parseStartArgs(args: StartArgs): StartOptions {
+  // Remove undefined values to allow deepmerge to inject default values downstream
+  return {
+    port: args.port,
+    executionRpcUrl: args.executionRpcUrl,
+    transport: LCTransport.Rest,
+    urls: args.beaconUrls ?? [],
+    wsCheckpoint: args.wsCheckpoint,
+    unverifiedWhitelist: args.unverifiedWhitelist,
+    requestTimeout: args.requestTimeout ?? DEFAULT_PROXY_REQUEST_TIMEOUT,
+  };
+}

package/src/cli/index.ts

@@ -0,0 +1,30 @@
+#!/usr/bin/env node
+
+// MUST import first to apply preset from args and set ssz hasher
+import "./applyPreset.js";
+
+import {YargsError} from "../utils/errors.js";
+import {getLodestarProverCli, yarg} from "./cli.js";
+import "source-map-support/register.js";
+
+const prover = getLodestarProverCli();
+
+void prover
+  .fail((msg, err) => {
+    if (msg?.includes("Not enough non-option arguments")) {
+      // Show command help message when no command is provided
+      yarg.showHelp();
+      // biome-ignore lint/suspicious/noConsole: This code will run only in browser so console will be available.
+      console.log("\n");
+    }
+
+    const errorMessage =
+      err !== undefined ? (err instanceof YargsError ? err.message : err.stack) : msg || "Unknown error";
+
+    // biome-ignore lint/suspicious/noConsole: We want to explicitly want to log error to console
+    console.error(` ✖ ${errorMessage}\n`);
+    process.exit(1);
+  })
+
+  // Execute CLI
+  .parse();

package/src/cli/options.ts

@@ -0,0 +1,73 @@
+import {NetworkName, networksChainConfig} from "@lodestar/config/networks";
+import {ACTIVE_PRESET} from "@lodestar/params";
+import {CliCommandOptions, LogLevel, LogLevels} from "@lodestar/utils";
+import {YargsError} from "../utils/errors.js";
+
+export type GlobalArgs = {
+  network?: string;
+  logLevel: string;
+  presetFile?: string;
+  preset: string;
+  paramsFile?: string;
+};
+
+export type GlobalOptions = {
+  logLevel: LogLevel;
+} & ({paramsFile: string; network?: never} | {network: NetworkName; paramsFile?: never});
+
+export const globalOptions: CliCommandOptions<GlobalArgs> = {
+  network: {
+    description: "Specify the network to connect.",
+    type: "string",
+    choices: [
+      ...Object.keys(networksChainConfig), // Leave always as last network. The order matters for the --help printout
+      "dev",
+    ],
+    conflicts: ["paramsFile"],
+  },
+
+  paramsFile: {
+    description: "Network configuration file",
+    type: "string",
+    conflicts: ["network"],
+  },
+
+  logLevel: {
+    description: "Set the log level.",
+    type: "string",
+    choices: LogLevels,
+    default: "info",
+  },
+
+  // hidden option to allow for LODESTAR_PRESET to be set
+  preset: {
+    hidden: true,
+    type: "string",
+    default: ACTIVE_PRESET,
+  },
+
+  presetFile: {
+    hidden: true,
+    description: "Preset configuration file to override the active preset with custom values",
+    type: "string",
+  },
+};
+
+export function parseGlobalArgs(args: GlobalArgs): GlobalOptions {
+  // Remove undefined values to allow deepmerge to inject default values downstream
+  if (args.network) {
+    return {
+      network: args.network as NetworkName,
+      logLevel: args.logLevel as LogLevel,
+    };
+  }
+
+  if (args.paramsFile) {
+    return {
+      logLevel: args.logLevel as LogLevel,
+      paramsFile: args.paramsFile,
+    };
+  }
+
+  throw new YargsError("Either --network or --paramsFile must be provided");
+}

package/src/constants.ts

@@ -0,0 +1,6 @@
+// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/p2p-interface.md#configuration
+export const MAX_REQUEST_LIGHT_CLIENT_UPDATES = 128;
+export const MAX_PAYLOAD_HISTORY = 32;
+export const VERIFICATION_FAILED_RESPONSE_CODE = -33091;
+export const ZERO_ADDRESS = "0x0000000000000000000000000000000000000000";
+export const DEFAULT_PROXY_REQUEST_TIMEOUT = 3000;

package/src/index.ts

@@ -0,0 +1,5 @@
+export * from "./interfaces.js";
+export * from "./proof_provider/index.js";
+export {isVerificationFailedError} from "./utils/json_rpc.js";
+export {createVerifiedExecutionProvider} from "./web3_provider.js";
+export {createVerifiedExecutionProxy} from "./web3_proxy.js";

package/src/interfaces.ts

@@ -0,0 +1,90 @@
+import {ChainConfig} from "@lodestar/config";
+import {NetworkName} from "@lodestar/config/networks";
+import {LogLevel, Logger} from "@lodestar/utils";
+import {ProofProvider} from "./proof_provider/proof_provider.js";
+import {JsonRpcRequest, JsonRpcRequestOrBatch, JsonRpcResponse, JsonRpcResponseOrBatch} from "./types.js";
+import {ELRpcProvider} from "./utils/rpc_provider.js";
+
+export type {NetworkName} from "@lodestar/config/networks";
+export enum LCTransport {
+  Rest = "Rest",
+  P2P = "P2P",
+}
+
+// Provide either network or config. This will be helpful to connect to a custom network
+export type NetworkOrConfig = {network: NetworkName; config?: never} | {network?: never; config: Partial<ChainConfig>};
+
+export type RootProviderInitOptions = ConsensusNodeOptions &
+  NetworkOrConfig & {
+    signal: AbortSignal;
+    logger: Logger;
+    wsCheckpoint?: string;
+    unverifiedWhitelist?: string[];
+  };
+
+// The `undefined` is necessary to match the types for the web3 1.x
+export type ELRequestHandler<Params = unknown[], Response = unknown> = (
+  payload: JsonRpcRequestOrBatch<Params>
+) => Promise<JsonRpcResponseOrBatch<Response> | undefined>;
+
+// biome-ignore lint/suspicious/noExplicitAny: We need to use `any` type here
+export type ELRequestHandlerAny = ELRequestHandler<any, any>;
+
+/**
+ * @deprecated Kept for backward compatibility. Use `AnyWeb3Provider` type instead.
+ */
+export type Web3Provider = object;
+
+export type ELVerifiedRequestHandlerOpts<Params = unknown[]> = {
+  payload: JsonRpcRequest<Params>;
+  rpc: ELRpcProvider;
+  proofProvider: ProofProvider;
+  logger: Logger;
+};
+
+export type ELVerifiedRequestHandler<Params = unknown[], Response = unknown> = (
+  opts: ELVerifiedRequestHandlerOpts<Params>
+) => Promise<JsonRpcResponse<Response>>;
+
+// Either a logger is provided by user or user specify a log level
+// If both are skipped then we don't log anything (useful for browser plugins)
+export type LogOptions = {logger?: Logger; logLevel?: never} | {logLevel?: LogLevel; logger?: never};
+
+export type ConsensusNodeOptions =
+  | {transport: LCTransport.Rest; urls: string[]}
+  | {transport: LCTransport.P2P; bootnodes: string[]};
+
+export type RootProviderOptions = {
+  signal?: AbortSignal;
+  wsCheckpoint?: string;
+  unverifiedWhitelist?: string[];
+};
+
+export type ProviderTypeOptions<T extends boolean | undefined> = {
+  /**
+   * If user specify custom provider types we will register those at the start in given order.
+   * So if you provider [custom1, custom2] and we already have [web3js, ethers] then final order
+   * of providers will be [custom1, custom2, web3js, ethers]
+   */
+  providerTypes?: Web3ProviderType<AnyWeb3Provider>[];
+  /**
+   * To keep the backward compatible behavior if this option is not set we consider `true` as default.
+   * In coming breaking release we may set this option default to `false`.
+   */
+  mutateProvider?: T;
+};
+
+export type VerifiedExecutionInitOptions<T extends boolean | undefined> = LogOptions &
+  ConsensusNodeOptions &
+  NetworkOrConfig &
+  RootProviderOptions &
+  ProviderTypeOptions<T>;
+
+export type AnyWeb3Provider = object;
+
+export interface Web3ProviderType<T extends AnyWeb3Provider> {
+  name: string;
+  matched: (provider: AnyWeb3Provider) => provider is T;
+  handler(provider: T): ELRpcProvider["handler"];
+  mutateProvider(provider: T, newHandler: ELRpcProvider["handler"]): void;
+}

package/src/proof_provider/index.ts

@@ -0,0 +1 @@
+export * from "./proof_provider.js";

package/src/proof_provider/ordered_map.ts

@@ -0,0 +1,25 @@
+export class OrderedMap<T> extends Map<number, T> {
+  private _min?: number;
+  private _max?: number;
+
+  get min(): number | undefined {
+    return this._min;
+  }
+
+  get max(): number | undefined {
+    return this._max;
+  }
+
+  set(key: number, value: T): this {
+    if (this._min === undefined || key < this._min) {
+      this._min = key;
+    }
+
+    if (this._max === undefined || key > this._max) {
+      this._max = key;
+    }
+
+    super.set(key, value);
+    return this;
+  }
+}

package/src/proof_provider/payload_store.ts

@@ -0,0 +1,223 @@
+import {ApiClient} from "@lodestar/api";
+import {ForkName} from "@lodestar/params";
+import {ExecutionPayload, LightClientHeader} from "@lodestar/types";
+import {Logger} from "@lodestar/utils";
+import {MAX_PAYLOAD_HISTORY} from "../constants.js";
+import {fetchBlock, getExecutionPayloadForBlockNumber} from "../utils/consensus.js";
+import {bufferToHex, hexToNumber} from "../utils/conversion.js";
+import {OrderedMap} from "./ordered_map.js";
+
+type BlockELRoot = string;
+type BlockELRootAndSlot = {
+  blockELRoot: BlockELRoot;
+  slot: number;
+};
+type BlockCLRoot = string;
+
+/**
+ * The in-memory store for the execution payloads to be used to verify the proofs
+ */
+export class PayloadStore {
+  // We store the block root from execution for finalized blocks
+  // As these blocks are finalized, so not to be worried about conflicting roots
+  private finalizedRoots = new OrderedMap<BlockELRootAndSlot>();
+
+  // Unfinalized blocks may change over time and may have conflicting roots
+  // We can receive multiple light-client headers for the same block of execution
+  // So we why store unfinalized payloads by their CL root, which is only used
+  // in processing the light-client headers
+  private unfinalizedRoots = new Map<BlockCLRoot, BlockELRoot>();
+
+  // Payloads store with BlockELRoot as key
+  private payloads = new Map<BlockELRoot, ExecutionPayload>();
+
+  private latestBlockRoot: BlockELRoot | null = null;
+
+  constructor(private opts: {api: ApiClient; logger: Logger}) {}
+
+  get finalized(): ExecutionPayload | undefined {
+    const maxBlockNumberForFinalized = this.finalizedRoots.max;
+
+    if (maxBlockNumberForFinalized === undefined) {
+      return undefined;
+    }
+
+    const finalizedMaxRoot = this.finalizedRoots.get(maxBlockNumberForFinalized);
+    if (finalizedMaxRoot) {
+      return this.payloads.get(finalizedMaxRoot.blockELRoot);
+    }
+
+    return undefined;
+  }
+
+  get latest(): ExecutionPayload | undefined {
+    if (this.latestBlockRoot) {
+      return this.payloads.get(this.latestBlockRoot);
+    }
+
+    return undefined;
+  }
+
+  async get(blockId: number | string): Promise<ExecutionPayload | undefined> {
+    // Given block id is a block hash in hex (32 bytes root takes 64 hex chars + 2 for 0x prefix)
+    if (typeof blockId === "string" && blockId.startsWith("0x") && blockId.length === 64 + 2) {
+      return this.payloads.get(blockId);
+    }
+
+    // Given block id is a block number in hex
+    if (typeof blockId === "string" && blockId.startsWith("0x")) {
+      return this.getOrFetchFinalizedPayload(hexToNumber(blockId));
+    }
+
+    // Given block id is a block number in decimal string
+    if (typeof blockId === "string" && !blockId.startsWith("0x")) {
+      return this.getOrFetchFinalizedPayload(parseInt(blockId, 10));
+    }
+
+    // Given block id is a block number in decimal
+    if (typeof blockId === "number") {
+      return this.getOrFetchFinalizedPayload(blockId);
+    }
+
+    return undefined;
+  }
+
+  protected async getOrFetchFinalizedPayload(blockNumber: number): Promise<ExecutionPayload | undefined> {
+    const maxBlockNumberForFinalized = this.finalizedRoots.max;
+    const minBlockNumberForFinalized = this.finalizedRoots.min;
+
+    if (maxBlockNumberForFinalized === undefined || minBlockNumberForFinalized === undefined) {
+      return;
+    }
+
+    if (blockNumber > maxBlockNumberForFinalized) {
+      throw new Error(
+        `Block number ${blockNumber} is higher than the latest finalized block number. We recommend to use block hash for unfinalized blocks.`
+      );
+    }
+
+    let blockELRoot = this.finalizedRoots.get(blockNumber);
+    // check if we have payload cached locally else fetch from api
+    if (!blockELRoot) {
+      const finalizedMaxRoot = this.finalizedRoots.get(maxBlockNumberForFinalized);
+      const slot = finalizedMaxRoot?.slot;
+      if (slot !== undefined) {
+        const payloads = await getExecutionPayloadForBlockNumber(this.opts.api, slot, blockNumber);
+        for (const [slot, payload] of payloads.entries()) {
+          this.set(payload, slot, true);
+        }
+      }
+    }
+
+    blockELRoot = this.finalizedRoots.get(blockNumber);
+    if (blockELRoot) {
+      return this.payloads.get(blockELRoot.blockELRoot);
+    }
+
+    return undefined;
+  }
+
+  set(payload: ExecutionPayload, slot: number, finalized: boolean): void {
+    const blockELRoot = bufferToHex(payload.blockHash);
+    this.payloads.set(blockELRoot, payload);
+
+    if (this.latestBlockRoot) {
+      const latestPayload = this.payloads.get(this.latestBlockRoot);
+      if (latestPayload && latestPayload.blockNumber < payload.blockNumber) {
+        this.latestBlockRoot = blockELRoot;
+      }
+    } else {
+      this.latestBlockRoot = blockELRoot;
+    }
+
+    if (finalized) {
+      this.finalizedRoots.set(payload.blockNumber, {blockELRoot, slot});
+    }
+  }
+
+  async processLCHeader(header: LightClientHeader<ForkName.capella>, finalized = false): Promise<void> {
+    const blockSlot = header.beacon.slot;
+    const blockNumber = header.execution.blockNumber;
+    const blockELRoot = bufferToHex(header.execution.blockHash);
+    const blockCLRoot = bufferToHex(header.beacon.stateRoot);
+    const existingELRoot = this.unfinalizedRoots.get(blockCLRoot);
+
+    // ==== Finalized blocks ====
+    // if the block is finalized, we need to update the finalizedRoots map
+    if (finalized) {
+      this.finalizedRoots.set(blockNumber, {blockELRoot, slot: blockSlot});
+
+      // If the block is finalized and we already have the payload
+      // We can remove it from the unfinalizedRoots map and do nothing else
+      if (existingELRoot) {
+        this.unfinalizedRoots.delete(blockCLRoot);
+      }
+
+      // If the block is finalized and we do not have the payload
+      // We need to fetch and set the payload
+      else {
+        const block = await fetchBlock(this.opts.api, blockSlot);
+        if (block) {
+          this.payloads.set(blockELRoot, block.message.body.executionPayload);
+        } else {
+          this.opts.logger.error("Failed to fetch block", blockSlot);
+        }
+      }
+
+      return;
+    }
+
+    // ==== Unfinalized blocks ====
+    // We already have the payload for this block
+    if (existingELRoot && existingELRoot === blockELRoot) {
+      return;
+    }
+
+    // Re-org happened, we need to update the payload
+    if (existingELRoot && existingELRoot !== blockELRoot) {
+      this.payloads.delete(existingELRoot);
+    }
+
+    // This is unfinalized header we need to store it's root related to cl root
+    this.unfinalizedRoots.set(blockCLRoot, blockELRoot);
+
+    // We do not have the payload for this block, we need to fetch it
+    const block = await fetchBlock(this.opts.api, blockSlot);
+    if (block) {
+      this.set(block.message.body.executionPayload, blockSlot, false);
+    } else {
+      this.opts.logger.error("Failed to fetch finalized block", blockSlot);
+    }
+    this.prune();
+  }
+
+  prune(): void {
+    if (this.finalizedRoots.size <= MAX_PAYLOAD_HISTORY) return;
+    // Store doe not have any finalized blocks means it's recently initialized
+    if (this.finalizedRoots.max === undefined || this.finalizedRoots.min === undefined) return;
+
+    for (
+      let blockNumber = this.finalizedRoots.max - MAX_PAYLOAD_HISTORY;
+      blockNumber >= this.finalizedRoots.min;
+      blockNumber--
+    ) {
+      const blockELRoot = this.finalizedRoots.get(blockNumber);
+      if (blockELRoot) {
+        this.payloads.delete(blockELRoot.blockELRoot);
+        this.finalizedRoots.delete(blockNumber);
+      }
+    }
+
+    for (const [clRoot, elRoot] of this.unfinalizedRoots) {
+      const payload = this.payloads.get(elRoot);
+      if (!payload) {
+        this.unfinalizedRoots.delete(clRoot);
+        continue;
+      }
+
+      if (payload.blockNumber < this.finalizedRoots.min) {
+        this.unfinalizedRoots.delete(clRoot);
+      }
+    }
+  }
+}