@chainsafe/lodestar 1.35.0-dev.8644a83c62 → 1.35.0-dev.894daf8b44

package/src/cmds/validator/options.ts

@@ -0,0 +1,463 @@
+import {WireFormat, defaultInit} from "@lodestar/api";
+import {CliCommandOptions} from "@lodestar/utils";
+import {defaultOptions} from "@lodestar/validator";
+import {coerceCors, enabledAllBashFriendly} from "../../options/beaconNodeOptions/api.js";
+import {LogArgs, logOptions} from "../../options/logOptions.js";
+import {ensure0xPrefix} from "../../util/index.js";
+import {keymanagerRestApiServerOptsDefault} from "./keymanager/server.js";
+import {defaultAccountPaths, defaultValidatorPaths} from "./paths.js";
+
+export type AccountValidatorArgs = {
+  keystoresDir?: string;
+  secretsDir?: string;
+  remoteKeysDir?: string;
+  proposerDir?: string;
+};
+
+export const validatorMetricsDefaultOptions = {
+  enabled: false,
+  port: 5064,
+  address: "127.0.0.1",
+};
+
+export const validatorMonitoringDefaultOptions = {
+  interval: 60_000,
+  initialDelay: 30_000,
+  requestTimeout: 10_000,
+  collectSystemStats: false,
+};
+
+// Defined as variable to not set yargs.default to an array
+export const DEFAULT_BEACON_NODE_URL = "";
+
+export type IValidatorCliArgs = AccountValidatorArgs &
+  KeymanagerArgs &
+  LogArgs & {
+    validatorsDbDir?: string;
+    beaconNodes: string[];
+    force?: boolean;
+    graffiti?: string;
+    afterBlockDelaySlotFraction?: number;
+    scAfterBlockDelaySlotFraction?: number;
+    suggestedFeeRecipient?: string;
+    proposerSettingsFile?: string;
+    strictFeeRecipientCheck?: boolean;
+    doppelgangerProtection?: boolean;
+    defaultGasLimit?: number;
+
+    builder?: boolean;
+    "builder.selection"?: string;
+    "builder.boostFactor"?: string;
+
+    /** @deprecated */
+    useProduceBlockV3?: boolean;
+    broadcastValidation?: string;
+    blindedLocal?: boolean;
+
+    importKeystores?: string[];
+    importKeystoresPassword?: string;
+    disableKeystoresThreadPool?: boolean;
+
+    "http.requestWireFormat"?: string;
+    "http.responseWireFormat"?: string;
+
+    "externalSigner.url"?: string;
+    "externalSigner.pubkeys"?: string[];
+    "externalSigner.fetch"?: boolean;
+    "externalSigner.fetchInterval"?: number;
+
+    distributed?: boolean;
+
+    interopIndexes?: string;
+    fromMnemonic?: string;
+    mnemonicIndexes?: string;
+
+    metrics?: boolean;
+    "metrics.port"?: number;
+    "metrics.address"?: string;
+
+    "monitoring.endpoint"?: string;
+    "monitoring.interval"?: number;
+    "monitoring.initialDelay"?: number;
+    "monitoring.requestTimeout"?: number;
+    "monitoring.collectSystemStats"?: boolean;
+  };
+
+export type KeymanagerArgs = {
+  keymanager?: boolean;
+  "keymanager.auth"?: boolean;
+  "keymanager.tokenFile"?: string;
+  "keymanager.port"?: number;
+  "keymanager.address"?: string;
+  "keymanager.cors"?: string;
+  "keymanager.headerLimit"?: number;
+  "keymanager.bodyLimit"?: number;
+  "keymanager.stacktraces"?: boolean;
+};
+
+export const keymanagerOptions: CliCommandOptions<KeymanagerArgs> = {
+  keymanager: {
+    type: "boolean",
+    description: "Enable key manager API server",
+    default: false,
+    group: "keymanager",
+  },
+  "keymanager.auth": {
+    alias: ["keymanager.authEnabled"],
+    type: "boolean",
+    description: "Enable token bearer authentication for key manager API server",
+    default: true,
+    group: "keymanager",
+  },
+  "keymanager.tokenFile": {
+    alias: ["tokenFile"],
+    type: "string",
+    description: "Path to file containing bearer token used for key manager API authentication",
+    group: "keymanager",
+  },
+  "keymanager.port": {
+    type: "number",
+    description: "Set port for key manager API",
+    defaultDescription: String(keymanagerRestApiServerOptsDefault.port),
+    group: "keymanager",
+  },
+  "keymanager.address": {
+    type: "string",
+    description: "Set host for key manager API",
+    defaultDescription: keymanagerRestApiServerOptsDefault.address,
+    group: "keymanager",
+  },
+  "keymanager.cors": {
+    type: "string",
+    description: `Configures the Access-Control-Allow-Origin CORS header for key manager API. Use '${enabledAllBashFriendly}' to allow all origins`,
+    defaultDescription: keymanagerRestApiServerOptsDefault.cors,
+    group: "keymanager",
+    coerce: coerceCors,
+  },
+  "keymanager.headerLimit": {
+    hidden: true,
+    type: "number",
+    description: "Defines the maximum length of request headers, in bytes, the server is allowed to accept",
+  },
+  "keymanager.bodyLimit": {
+    hidden: true,
+    type: "number",
+    description: "Defines the maximum payload, in bytes, the server is allowed to accept",
+  },
+  "keymanager.stacktraces": {
+    hidden: true,
+    type: "boolean",
+    description: "Return stacktraces in HTTP error responses",
+  },
+};
+
+export const validatorOptions: CliCommandOptions<IValidatorCliArgs> = {
+  ...logOptions,
+  ...keymanagerOptions,
+
+  keystoresDir: {
+    hidden: true,
+    description: "Directory for storing validator keystores.",
+    defaultDescription: defaultAccountPaths.keystoresDir,
+    type: "string",
+  },
+
+  secretsDir: {
+    hidden: true,
+    description: "Directory for storing validator keystore secrets.",
+    defaultDescription: defaultAccountPaths.secretsDir,
+    type: "string",
+  },
+
+  remoteKeysDir: {
+    hidden: true,
+    description: "Directory for storing validator remote key definitions.",
+    defaultDescription: defaultAccountPaths.keystoresDir,
+    type: "string",
+  },
+
+  proposerDir: {
+    hidden: true,
+    description: "Directory for storing keymanager's proposer configs for validators",
+    defaultDescription: defaultAccountPaths.proposerDir,
+    type: "string",
+  },
+
+  validatorsDbDir: {
+    hidden: true,
+    description: "Data directory for validator databases.",
+    defaultDescription: defaultValidatorPaths.validatorsDbDir,
+    type: "string",
+  },
+
+  beaconNodes: {
+    description: "Addresses to connect to BeaconNode",
+    default: ["http://127.0.0.1:9596"],
+    type: "array",
+    string: true,
+    coerce: (urls: string[]): string[] =>
+      // Parse ["url1,url2"] to ["url1", "url2"]
+      urls.flatMap((item) => item.split(",")),
+    alias: ["server"], // for backwards compatibility
+  },
+
+  force: {
+    description: "Open validators even if there's a lockfile. Use with caution",
+    type: "boolean",
+  },
+
+  graffiti: {
+    description: "Specify your custom graffiti to be included in blocks (plain UTF8 text, 32 characters max)",
+    type: "string",
+  },
+
+  afterBlockDelaySlotFraction: {
+    hidden: true,
+    description:
+      "Delay before publishing attestations if block comes early, as a fraction of SLOT_DURATION_MS (value is from 0 inclusive to 1 exclusive)",
+    type: "number",
+  },
+
+  scAfterBlockDelaySlotFraction: {
+    hidden: true,
+    description:
+      "Delay before publishing SyncCommitteeSignature if block comes early, as a fraction of SLOT_DURATION_MS (value is from 0 inclusive to 1 exclusive)",
+    type: "number",
+  },
+
+  proposerSettingsFile: {
+    description:
+      "A yaml file to specify detailed default and per validator public key customized proposer configs. PS: This feature and its format is in alpha and subject to change",
+    type: "string",
+  },
+
+  suggestedFeeRecipient: {
+    description:
+      "Specify fee recipient default for collecting the EL block fees and rewards (a hex string representing 20 bytes address: ^0x[a-fA-F0-9]{40}$). It would be possible (WIP) to override this per validator key using config or key manager API. Only used post merge.",
+    defaultDescription: defaultOptions.suggestedFeeRecipient,
+    type: "string",
+  },
+
+  strictFeeRecipientCheck: {
+    description: "Enable strict checking of the validator's `feeRecipient` with the one returned by engine",
+    type: "boolean",
+  },
+
+  defaultGasLimit: {
+    description: "Suggested gas limit to the engine/builder for building execution payloads. Only used post merge.",
+    defaultDescription: `${defaultOptions.defaultGasLimit}`,
+    type: "number",
+  },
+
+  builder: {
+    type: "boolean",
+    description: `An alias for \`--builder.selection ${defaultOptions.builderAliasSelection}\` for the builder flow, ignored if \`--builder.selection\` is explicitly provided`,
+    group: "builder",
+  },
+
+  "builder.selection": {
+    type: "string",
+    description:
+      "Builder block selection strategy `default`, `maxprofit`, `builderalways`, `builderonly`, `executionalways`, or `executiononly`",
+    defaultDescription: `${defaultOptions.builderSelection}`,
+    group: "builder",
+  },
+
+  "builder.boostFactor": {
+    type: "string",
+    description:
+      "Percentage multiplier the block producing beacon node must apply to boost (>100) or dampen (<100) builder block value for selection against execution block. The multiplier is ignored if `--builder.selection` is set to anything other than `maxprofit`",
+    defaultDescription: `${defaultOptions.builderBoostFactor}`,
+    group: "builder",
+  },
+
+  useProduceBlockV3: {
+    hidden: true,
+    deprecated: true,
+    type: "boolean",
+  },
+
+  broadcastValidation: {
+    type: "string",
+    description: "Validations to be run by beacon node for the signed block prior to publishing",
+    defaultDescription: `${defaultOptions.broadcastValidation}`,
+  },
+
+  blindedLocal: {
+    type: "boolean",
+    description: "Request fetching local block in blinded format for produceBlockV3",
+    defaultDescription: `${defaultOptions.blindedLocal}`,
+  },
+
+  importKeystores: {
+    alias: ["keystore"], // Backwards compatibility with old `validator import` cmdx
+    description: "Path(s) to a directory or single file path to validator keystores, i.e. Launchpad validators",
+    defaultDescription: "./keystores/*.json",
+    type: "array",
+  },
+
+  importKeystoresPassword: {
+    alias: ["passphraseFile"], // Backwards compatibility with old `validator import` cmd
+    description: "Path to a file with password to decrypt all keystores from `importKeystores` option",
+    defaultDescription: "./password.txt",
+    type: "string",
+  },
+
+  disableKeystoresThreadPool: {
+    hidden: true,
+    description:
+      "Disable thread pool and instead use main thread to decrypt keystores. This can speed up decryption in testing environments like Kurtosis",
+    type: "boolean",
+  },
+
+  doppelgangerProtection: {
+    alias: ["doppelgangerProtectionEnabled"],
+    description: "Enables Doppelganger protection",
+    default: false,
+    type: "boolean",
+  },
+
+  "http.requestWireFormat": {
+    type: "string",
+    description: `Wire format to use in HTTP requests to beacon node. Can be one of \`${WireFormat.json}\` or \`${WireFormat.ssz}\``,
+    defaultDescription: `${defaultInit.requestWireFormat}`,
+    group: "http",
+  },
+
+  "http.responseWireFormat": {
+    type: "string",
+    description: `Preferred wire format for HTTP responses from beacon node. Can be one of \`${WireFormat.json}\` or \`${WireFormat.ssz}\``,
+    defaultDescription: `${defaultInit.responseWireFormat}`,
+    group: "http",
+  },
+
+  // External signer
+
+  "externalSigner.url": {
+    description: "URL to connect to an external signing server",
+    type: "string",
+    group: "externalSigner",
+  },
+
+  "externalSigner.pubkeys": {
+    implies: ["externalSigner.url"],
+    description:
+      "List of validator public keys used by an external signer. May also provide a single string of comma-separated public keys",
+    type: "array",
+    string: true, // Ensures the pubkey string is not automatically converted to numbers
+    coerce: (pubkeys: string[]): string[] =>
+      // Parse ["0x11,0x22"] to ["0x11", "0x22"]
+      pubkeys
+        .flatMap((item) => item.split(","))
+        .map(ensure0xPrefix),
+    group: "externalSigner",
+  },
+
+  "externalSigner.fetch": {
+    implies: ["externalSigner.url"],
+    conflicts: ["externalSigner.pubkeys"],
+    description:
+      "Fetch the list of public keys to validate from an external signer. Cannot be used in combination with `--externalSigner.pubkeys`",
+    type: "boolean",
+    group: "externalSigner",
+  },
+
+  "externalSigner.fetchInterval": {
+    implies: ["externalSigner.fetch"],
+    description:
+      "Interval in milliseconds between fetching the list of public keys from external signer, once per epoch by default",
+    type: "number",
+    group: "externalSigner",
+  },
+
+  // Distributed validator
+
+  distributed: {
+    description: "Enables specific features required to run as part of a distributed validator cluster",
+    type: "boolean",
+  },
+
+  // Metrics
+
+  metrics: {
+    type: "boolean",
+    description: "Enable the Prometheus metrics HTTP server",
+    defaultDescription: String(validatorMetricsDefaultOptions.enabled),
+    group: "metrics",
+  },
+
+  "metrics.port": {
+    type: "number",
+    description: "Listen TCP port for the Prometheus metrics HTTP server",
+    defaultDescription: String(validatorMetricsDefaultOptions.port),
+    group: "metrics",
+  },
+
+  "metrics.address": {
+    type: "string",
+    description: "Listen address for the Prometheus metrics HTTP server",
+    defaultDescription: String(validatorMetricsDefaultOptions.address),
+    group: "metrics",
+  },
+
+  // Monitoring
+
+  "monitoring.endpoint": {
+    type: "string",
+    description:
+      "Enables monitoring service for sending clients stats to the specified endpoint of a remote service (e.g. beaconcha.in)",
+    group: "monitoring",
+  },
+
+  "monitoring.interval": {
+    type: "number",
+    description: "Interval in milliseconds between sending client stats to the remote service",
+    defaultDescription: String(validatorMonitoringDefaultOptions.interval),
+    group: "monitoring",
+  },
+
+  "monitoring.initialDelay": {
+    type: "number",
+    description: "Initial delay in milliseconds before client stats are sent to the remote service",
+    defaultDescription: String(validatorMonitoringDefaultOptions.initialDelay),
+    group: "monitoring",
+    hidden: true,
+  },
+
+  "monitoring.requestTimeout": {
+    type: "number",
+    description: "Timeout in milliseconds for sending client stats to the remote service",
+    defaultDescription: String(validatorMonitoringDefaultOptions.requestTimeout),
+    group: "monitoring",
+    hidden: true,
+  },
+
+  "monitoring.collectSystemStats": {
+    type: "boolean",
+    description:
+      "Enable collecting system stats. This should only be enabled if validator client and beacon node are running on different hosts.",
+    defaultDescription: String(validatorMonitoringDefaultOptions.collectSystemStats),
+    group: "monitoring",
+    hidden: true,
+  },
+
+  // For testing only
+
+  interopIndexes: {
+    hidden: true,
+    description: "Range (inclusive) of interop key indexes to validate with: 0..16",
+    type: "string",
+  },
+
+  fromMnemonic: {
+    hidden: true,
+    description: "UNSAFE. Run keys from a mnemonic. Requires mnemonicIndexes option",
+    type: "string",
+  },
+
+  mnemonicIndexes: {
+    hidden: true,
+    description: "UNSAFE. Range (inclusive) of mnemonic key indexes to validate with: 0..16",
+    type: "string",
+  },
+};

package/src/cmds/validator/paths.ts

@@ -0,0 +1,95 @@
+import path from "node:path";
+import {GlobalArgs} from "../../options/index.js";
+import {GlobalPaths, getGlobalPaths} from "../../paths/global.js";
+
+export type IValidatorPaths = {
+  validatorsDbDir: string;
+};
+
+export type AccountPaths = {
+  cacheDir: string;
+  keystoresDir: string;
+  secretsDir: string;
+  remoteKeysDir: string;
+  proposerDir: string;
+};
+
+/**
+ * Defines the path structure of the validator files
+ *
+ * ```bash
+ * $validatordataDir
+ * └── validator-db
+ *     └── (db files)
+ * ```
+ */
+export function getValidatorPaths(
+  args: Partial<IValidatorPaths> & Pick<GlobalArgs, "dataDir">,
+  network: string
+): IValidatorPaths & GlobalPaths {
+  // Compute global paths first
+  const globalPaths = getGlobalPaths(args, network);
+
+  const dataDir = globalPaths.dataDir;
+  const validatorsDbDir = args.validatorsDbDir ?? path.join(dataDir, "validator-db");
+
+  return {
+    ...globalPaths,
+    validatorsDbDir,
+  };
+}
+
+/**
+ * Constructs representations of the path structure to show in command's description
+ */
+export const defaultValidatorPaths = getValidatorPaths({dataDir: "$dataDir"}, "$network");
+
+/**
+ * Defines the path structure of the account files
+ *
+ * ```bash
+ * $accountsdataDir
+ * ├── secrets
+ * |   ├── 0x8e41b969493454318c27ec6fac90645769331c07ebc8db5037...
+ * |   └── 0xa329f988c16993768299643d918a2694892c012765d896a16f...
+ * ├── keystores
+ * |   ├── 0x8e41b969493454318c27ec6fac90645769331c07ebc8db5037...
+ * |   |   ├── eth1-deposit-data.rlp
+ * |   |   ├── eth1-deposit-gwei.txt
+ * |   |   └── voting-keystore.json
+ * |   └── 0xa329f988c16993768299643d918a2694892c012765d896a16f...
+ * |       ├── eth1-deposit-data.rlp
+ * |       ├── eth1-deposit-gwei.txt
+ * |       └── voting-keystore.json
+ * └── remoteKeys
+ *     └── 0xa329f988c16993768299643d918a2694892c012765d896a16f.json
+ * ```
+ */
+// Using Pick<GlobalArgs, "dataDir"> make changes in GlobalArgs throw a type error here
+export function getAccountPaths(
+  args: Partial<AccountPaths> & Pick<GlobalArgs, "dataDir">,
+  network: string
+): AccountPaths & GlobalPaths {
+  // Compute global paths first
+  const globalPaths = getGlobalPaths(args, network);
+
+  const dataDir = globalPaths.dataDir;
+  const cacheDir = args.cacheDir || path.join(dataDir, "cache");
+  const keystoresDir = args.keystoresDir || path.join(dataDir, "keystores");
+  const secretsDir = args.secretsDir || path.join(dataDir, "secrets");
+  const remoteKeysDir = args.remoteKeysDir || path.join(dataDir, "remoteKeys");
+  const proposerDir = args.proposerDir || path.join(dataDir, "proposerConfigs");
+  return {
+    ...globalPaths,
+    cacheDir,
+    keystoresDir,
+    secretsDir,
+    remoteKeysDir,
+    proposerDir,
+  };
+}
+
+/**
+ * Constructs representations of the path structure to show in command's description
+ */
+export const defaultAccountPaths = getAccountPaths({dataDir: "$dataDir"}, "$network");

package/src/cmds/validator/signers/importExternalKeystores.ts

@@ -0,0 +1,69 @@
+import inquirer from "inquirer";
+import {readPassphraseFile, recursiveLookup} from "../../../util/index.js";
+import {LocalKeystoreDefinition} from "../keymanager/interface.js";
+
+/**
+ * Imports keystores from un-controlled directories provided by the user.
+ * This directories are expected to contain other files, so some filtering in done to improve UX.
+ * @param args
+ */
+export function importKeystoreDefinitionsFromExternalDir(args: {
+  keystoresPath: string[];
+  password: string;
+}): LocalKeystoreDefinition[] {
+  const allFiles: string[] = [];
+
+  for (const keystorePath of args.keystoresPath) {
+    recursiveLookup(keystorePath, allFiles);
+  }
+
+  return allFiles
+    .filter((filepath) => isVotingKeystore(filepath))
+    .map((keystorePath) => ({
+      keystorePath,
+      password: args.password,
+    }));
+}
+
+export async function readPassphraseOrPrompt(args: {importKeystoresPassword?: string}): Promise<string> {
+  if (args.importKeystoresPassword) {
+    return readPassphraseFile(args.importKeystoresPassword);
+  }
+
+  const answers = await inquirer.prompt<{password: string}>([
+    {
+      name: "password",
+      type: "password",
+      message: "Enter the keystore(s) password",
+    },
+  ]);
+
+  return answers.password;
+}
+
+/**
+ * Returns `true` if we should consider the `filename` to represent a voting keystore.
+ */
+export function isVotingKeystore(filename: string): boolean {
+  // All formats end with `.json`.
+  return (
+    filename.endsWith(".json") &&
+    // The staking-deposit-cli tool outputs a deposit_data file in the directory users typically import from.
+    // Ignoring that file is very helpful for UX, and it's very unlikely that someone names their keystore that way.
+    !/deposit_data-\d+\.json$/gi.test(filename)
+    // Note: Previously this tool only imported the exact naming from the staking-deposit-cli tool.
+    //       However, that's too restrictive. Guide left here as a reference
+    //
+    // The format exported by the `staking-deposit-cli` library.
+    //
+    // Reference to function that generates keystores:
+    // https://github.com/ethereum/staking-deposit-cli/blob/7cebff15eac299b3b1b090c896dd3410c8463450/eth2deposit/credentials.py#L58-L62
+    //
+    // Since we include the key derivation path of `m/12381/3600/x/0/0` this should only ever match
+    // with a voting keystore and never a withdrawal keystore.
+    //
+    // Key derivation path reference:
+    //
+    // https://eips.ethereum.org/EIPS/eip-2334
+  );
+}