@chainsafe/lodestar 1.35.0-dev.fcf8d024ea → 1.35.0-dev.feed916580

package/src/cmds/validator/signers/index.ts

@@ -1,176 +0,0 @@
-import path from "node:path";
-import {deriveEth2ValidatorKeys, deriveKeyFromMnemonic} from "@chainsafe/bls-keygen";
-import {SecretKey} from "@chainsafe/blst";
-import {interopSecretKey} from "@lodestar/state-transition";
-import {LogLevel, Logger, isValidHttpUrl} from "@lodestar/utils";
-import {Signer, SignerType, externalSignerGetKeys} from "@lodestar/validator";
-import {GlobalArgs, defaultNetwork} from "../../../options/index.js";
-import {YargsError, assertValidPubkeysHex, parseRange} from "../../../util/index.js";
-import {showProgress} from "../../../util/progress.js";
-import {decryptKeystoreDefinitions} from "../keymanager/decryptKeystoreDefinitions.js";
-import {PersistedKeysBackend} from "../keymanager/persistedKeys.js";
-import {IValidatorCliArgs} from "../options.js";
-import {getAccountPaths} from "../paths.js";
-import {importKeystoreDefinitionsFromExternalDir, readPassphraseOrPrompt} from "./importExternalKeystores.js";
-
-const KEYSTORE_IMPORT_PROGRESS_MS = 10000;
-
-/**
- * Options processing hierarchy
- * --interopIndexes
- * --fromMnemonic, then requires --mnemonicIndexes
- * --importKeystores, then requires --importKeystoresPassword
- * --externalSigner.fetch, then requires --externalSigner.url
- * --externalSigner.pubkeys, then requires --externalSigner.url
- * else load from persisted
- * - both remote keys and local keystores
- *
- * @returns Signers =  an item capable of producing signatures. Two types exist:
- * - Local: a secret key capable of signing
- * - Remote: a URL that supports EIP-3030 (BLS Remote Signer HTTP API)
- *
- *  Local secret keys can be gathered from:
- * - Local keystores existent on disk
- * - Local keystores imported via keymanager api
- * - Derived from a mnemonic (TESTING ONLY)
- * - Derived from interop keys (TESTING ONLY)
- *
- * Remote signers need to pre-declare the list of pubkeys to validate with
- * - Via CLI argument
- * - Fetched directly from remote signer API
- * - Remote signer definition imported from keymanager api
- */
-export async function getSignersFromArgs(
-  args: IValidatorCliArgs & GlobalArgs,
-  network: string,
-  {logger, signal}: {logger: Pick<Logger, LogLevel.info | LogLevel.warn | LogLevel.debug>; signal: AbortSignal}
-): Promise<Signer[]> {
-  const accountPaths = getAccountPaths(args, network);
-
-  // ONLY USE FOR TESTNETS - Derive interop keys
-  if (args.interopIndexes) {
-    const indexes = parseRange(args.interopIndexes);
-    // Using a remote signer with TESTNETS
-    if (args["externalSigner.pubkeys"] || args["externalSigner.fetch"]) {
-      return getRemoteSigners(args);
-    }
-    return indexes.map((index) => ({type: SignerType.Local, secretKey: interopSecretKey(index)}));
-  }
-
-  // UNSAFE, ONLY USE FOR TESTNETS - Derive keys directly from a mnemonic
-  if (args.fromMnemonic) {
-    if (network === defaultNetwork) {
-      throw new YargsError("fromMnemonic must only be used in testnets");
-    }
-    if (!args.mnemonicIndexes) {
-      throw new YargsError("Must specify mnemonicIndexes with fromMnemonic");
-    }
-
-    const masterSK = deriveKeyFromMnemonic(args.fromMnemonic);
-    const indexes = parseRange(args.mnemonicIndexes);
-    return indexes.map((index) => ({
-      type: SignerType.Local,
-      secretKey: SecretKey.fromBytes(deriveEth2ValidatorKeys(masterSK, index).signing),
-    }));
-  }
-
-  // Import JSON keystores and run
-  if (args.importKeystores) {
-    const keystoreDefinitions = importKeystoreDefinitionsFromExternalDir({
-      keystoresPath: args.importKeystores,
-      password: await readPassphraseOrPrompt(args),
-    });
-
-    const needle = showProgress({
-      total: keystoreDefinitions.length,
-      frequencyMs: KEYSTORE_IMPORT_PROGRESS_MS,
-      signal,
-      progress: ({ratePerSec, percentage, current, total}) => {
-        logger.info(
-          `${percentage.toFixed(0)}% of keystores imported. current=${current} total=${total} rate=${(
-            ratePerSec * 60
-          ).toFixed(2)}keys/m`
-        );
-      },
-    });
-    return decryptKeystoreDefinitions(keystoreDefinitions, {
-      ignoreLockFile: args.force,
-      onDecrypt: needle,
-      cacheFilePath: path.join(accountPaths.cacheDir, "imported_keystores.cache"),
-      disableThreadPool: args.disableKeystoresThreadPool,
-      logger,
-      signal,
-    });
-  }
-
-  // Remote keys are declared manually or will be fetched from external signer
-  if (args["externalSigner.pubkeys"] || args["externalSigner.fetch"]) {
-    return getRemoteSigners(args);
-  }
-
-  // Read keys from local account manager
-  const persistedKeysBackend = new PersistedKeysBackend(accountPaths);
-
-  // Read and decrypt local keystores, imported via keymanager api or import cmd
-  const keystoreDefinitions = persistedKeysBackend.readAllKeystores();
-
-  const needle = showProgress({
-    total: keystoreDefinitions.length,
-    frequencyMs: KEYSTORE_IMPORT_PROGRESS_MS,
-    signal,
-    progress: ({ratePerSec, percentage, current, total}) => {
-      logger.info(
-        `${percentage.toFixed(0)}% of local keystores imported. current=${current} total=${total} rate=${(
-          ratePerSec * 60
-        ).toFixed(2)}keys/m`
-      );
-    },
-  });
-
-  const keystoreSigners = await decryptKeystoreDefinitions(keystoreDefinitions, {
-    ignoreLockFile: args.force,
-    onDecrypt: needle,
-    cacheFilePath: path.join(accountPaths.cacheDir, "local_keystores.cache"),
-    disableThreadPool: args.disableKeystoresThreadPool,
-    logger,
-    signal,
-  });
-
-  // Read local remote keys, imported via keymanager api
-  const signerDefinitions = persistedKeysBackend.readAllRemoteKeys();
-  const remoteSigners = signerDefinitions.map(({url, pubkey}): Signer => ({type: SignerType.Remote, url, pubkey}));
-
-  return [...keystoreSigners, ...remoteSigners];
-}
-
-export function getSignerPubkeyHex(signer: Signer): string {
-  switch (signer.type) {
-    case SignerType.Local:
-      return signer.secretKey.toPublicKey().toHex();
-
-    case SignerType.Remote:
-      return signer.pubkey;
-  }
-}
-
-async function getRemoteSigners(args: IValidatorCliArgs & GlobalArgs): Promise<Signer[]> {
-  const externalSignerUrl = args["externalSigner.url"];
-  if (!externalSignerUrl) {
-    throw new YargsError(
-      `Must set externalSigner.url with ${
-        args["externalSigner.pubkeys"] ? "externalSigner.pubkeys" : "externalSigner.fetch"
-      }`
-    );
-  }
-  if (!isValidHttpUrl(externalSignerUrl)) {
-    throw new YargsError(`Invalid external signer URL: ${externalSignerUrl}`);
-  }
-  if (args["externalSigner.pubkeys"] && args["externalSigner.pubkeys"].length === 0) {
-    throw new YargsError("externalSigner.pubkeys is set to an empty list");
-  }
-
-  const pubkeys = args["externalSigner.pubkeys"] ?? (await externalSignerGetKeys(externalSignerUrl));
-  assertValidPubkeysHex(pubkeys);
-
-  return pubkeys.map((pubkey) => ({type: SignerType.Remote, pubkey, url: externalSignerUrl}));
-}

package/src/cmds/validator/signers/logSigners.ts

@@ -1,81 +0,0 @@
-import {LogLevel, Logger, toPrintableUrl} from "@lodestar/utils";
-import {Signer, SignerLocal, SignerRemote, SignerType} from "@lodestar/validator";
-import {YargsError} from "../../../util/errors.js";
-import {IValidatorCliArgs} from "../options.js";
-
-/**
- * Log each pubkeys for auditing out keys are loaded from the logs
- */
-export function logSigners(logger: Pick<Logger, LogLevel.info>, signers: Signer[]): void {
-  const localSigners: SignerLocal[] = [];
-  const remoteSigners: SignerRemote[] = [];
-
-  for (const signer of signers) {
-    switch (signer.type) {
-      case SignerType.Local:
-        localSigners.push(signer);
-        break;
-      case SignerType.Remote:
-        remoteSigners.push(signer);
-        break;
-    }
-  }
-
-  if (localSigners.length > 0) {
-    logger.info(`${localSigners.length} local keystores`);
-    for (const signer of localSigners) {
-      logger.info(signer.secretKey.toPublicKey().toHex());
-    }
-  }
-
-  for (const {url, pubkeys} of groupRemoteSignersByUrl(remoteSigners)) {
-    logger.info(`Remote signers on URL: ${toPrintableUrl(url)}`);
-    for (const pubkey of pubkeys) {
-      logger.info(pubkey);
-    }
-  }
-}
-
-/**
- * Only used for logging remote signers grouped by URL
- */
-function groupRemoteSignersByUrl(remoteSigners: SignerRemote[]): {url: string; pubkeys: string[]}[] {
-  const byUrl = new Map<string, {url: string; pubkeys: string[]}>();
-
-  for (const remoteSigner of remoteSigners) {
-    let x = byUrl.get(remoteSigner.url);
-    if (!x) {
-      x = {url: remoteSigner.url, pubkeys: []};
-      byUrl.set(remoteSigner.url, x);
-    }
-    x.pubkeys.push(remoteSigner.pubkey);
-  }
-
-  return Array.from(byUrl.values());
-}
-
-/**
- * Notify user if there are no signers at startup, this might be intended but could also be due to
- * misconfiguration. It is possible that signers are added later via keymanager or if an external signer
- * is connected with fetching enabled, but otherwise exit the process and suggest a different configuration.
- */
-export function warnOrExitNoSigners(args: IValidatorCliArgs, logger: Pick<Logger, LogLevel.warn>): void {
-  if (args.keymanager && !args["externalSigner.fetch"]) {
-    logger.warn("No local keystores or remote keys found with current args, expecting to be added via keymanager");
-  } else if (!args.keymanager && args["externalSigner.fetch"]) {
-    logger.warn("No remote keys found with current args, expecting to be added to external signer and fetched later");
-  } else if (args.keymanager && args["externalSigner.fetch"]) {
-    logger.warn(
-      "No local keystores or remote keys found with current args, expecting to be added via keymanager or fetched from external signer later"
-    );
-  } else {
-    if (args["externalSigner.url"]) {
-      throw new YargsError(
-        "No remote keys found with current args, start with --externalSigner.fetch to automatically fetch from external signer"
-      );
-    }
-    throw new YargsError(
-      "No local keystores or remote keys found with current args, start with --keymanager if intending to add them later via keymanager"
-    );
-  }
-}

package/src/cmds/validator/slashingProtection/export.ts

@@ -1,110 +0,0 @@
-import path from "node:path";
-import {getNodeLogger} from "@lodestar/logger/node";
-import {CliCommand, toPubkeyHex} from "@lodestar/utils";
-import {InterchangeFormatVersion} from "@lodestar/validator";
-import {getBeaconConfigFromArgs} from "../../../config/index.js";
-import {GlobalArgs} from "../../../options/index.js";
-import {LogArgs} from "../../../options/logOptions.js";
-import {YargsError, ensure0xPrefix, isValidatePubkeyHex, writeFile600Perm} from "../../../util/index.js";
-import {parseLoggerArgs} from "../../../util/logger.js";
-import {AccountValidatorArgs} from "../options.js";
-import {getValidatorPaths} from "../paths.js";
-import {ISlashingProtectionArgs} from "./options.js";
-import {getGenesisValidatorsRoot, getSlashingProtection} from "./utils.js";
-
-type ExportArgs = {
-  file: string;
-  pubkeys?: string[];
-};
-
-export const exportCmd: CliCommand<ExportArgs, ISlashingProtectionArgs & AccountValidatorArgs & GlobalArgs & LogArgs> =
-  {
-    command: "export",
-
-    describe: "Export an interchange file.",
-
-    examples: [
-      {
-        command: "validator slashing-protection export --network hoodi --file interchange.json",
-        description: "Export an interchange JSON file for all validators in the slashing protection DB",
-      },
-    ],
-
-    options: {
-      file: {
-        description: "The slashing protection interchange file to export to (.json).",
-        demandOption: true,
-        type: "string",
-      },
-      pubkeys: {
-        description: "Export slashing protection data only for a given subset of public keys",
-        type: "array",
-        string: true, // Ensures the pubkey string is not automatically converted to numbers
-        coerce: (pubkeys: string[]): string[] =>
-          // Parse ["0x11,0x22"] to ["0x11", "0x22"]
-          pubkeys
-            .flatMap((item) => item.split(","))
-            .map(ensure0xPrefix),
-      },
-    },
-
-    handler: async (args) => {
-      const {file} = args;
-
-      const {config, network} = getBeaconConfigFromArgs(args);
-      const validatorPaths = getValidatorPaths(args, network);
-      // slashingProtection commands are fast so do not require logFile feature
-      const logger = getNodeLogger(
-        parseLoggerArgs(args, {defaultLogFilepath: path.join(validatorPaths.dataDir, "validator.log")}, config)
-      );
-
-      const {validatorsDbDir: dbPath} = getValidatorPaths(args, network);
-
-      const formatVersion: InterchangeFormatVersion = {version: "5"};
-      logger.info("Exporting slashing protection data", {...formatVersion, dbPath});
-
-      const {slashingProtection, metadata} = await getSlashingProtection(args, network, logger);
-
-      // When exporting validator DB should already have genesisValidatorsRoot persisted.
-      // For legacy node and general fallback, fetch from:
-      // - known genesis data from existing network
-      // - else fetch from beacon node
-      const genesisValidatorsRoot =
-        (await metadata.getGenesisValidatorsRoot()) ?? (await getGenesisValidatorsRoot(args));
-
-      logger.verbose("Fetching pubkeys from slashing protection db");
-      const allPubkeys = await slashingProtection.listPubkeys();
-      let pubkeysToExport = allPubkeys;
-
-      if (args.pubkeys) {
-        logger.verbose("Filtering by pubkeys from args", {count: args.pubkeys.length});
-        const filteredPubkeys = [];
-
-        for (const pubkeyHex of args.pubkeys) {
-          if (!isValidatePubkeyHex(pubkeyHex)) {
-            throw new YargsError(`Invalid pubkey ${pubkeyHex}`);
-          }
-          const existingPubkey = allPubkeys.find((pubkey) => toPubkeyHex(pubkey) === pubkeyHex);
-          if (!existingPubkey) {
-            logger.warn("Pubkey not found in slashing protection db", {pubkey: pubkeyHex});
-          } else {
-            filteredPubkeys.push(existingPubkey);
-          }
-        }
-
-        pubkeysToExport = filteredPubkeys;
-      }
-
-      logger.info("Starting export for pubkeys found", {count: pubkeysToExport.length});
-      const interchange = await slashingProtection.exportInterchange(
-        genesisValidatorsRoot,
-        pubkeysToExport,
-        formatVersion,
-        logger
-      );
-
-      logger.info("Writing slashing protection data", {file});
-      writeFile600Perm(file, interchange);
-      logger.info("Export completed successfully");
-    },
-  };

package/src/cmds/validator/slashingProtection/import.ts

@@ -1,70 +0,0 @@
-import fs from "node:fs";
-import path from "node:path";
-import {getNodeLogger} from "@lodestar/logger/node";
-import {CliCommand} from "@lodestar/utils";
-import {Interchange} from "@lodestar/validator";
-import {getBeaconConfigFromArgs} from "../../../config/index.js";
-import {GlobalArgs} from "../../../options/index.js";
-import {LogArgs} from "../../../options/logOptions.js";
-import {parseLoggerArgs} from "../../../util/logger.js";
-import {AccountValidatorArgs} from "../options.js";
-import {getValidatorPaths} from "../paths.js";
-import {ISlashingProtectionArgs} from "./options.js";
-import {getGenesisValidatorsRoot, getSlashingProtection} from "./utils.js";
-
-type ImportArgs = {
-  file: string;
-};
-
-export const importCmd: CliCommand<ImportArgs, ISlashingProtectionArgs & AccountValidatorArgs & GlobalArgs & LogArgs> =
-  {
-    command: "import",
-
-    describe: "Import an interchange file.",
-
-    examples: [
-      {
-        command: "validator slashing-protection import --network hoodi --file interchange.json",
-        description: "Import an interchange file to the slashing protection DB",
-      },
-    ],
-
-    options: {
-      file: {
-        description: "The slashing protection interchange file to import (.json).",
-        demandOption: true,
-        type: "string",
-      },
-    },
-
-    handler: async (args) => {
-      const {file} = args;
-
-      const {config, network} = getBeaconConfigFromArgs(args);
-      const validatorPaths = getValidatorPaths(args, network);
-      // slashingProtection commands are fast so do not require logFile feature
-      const logger = getNodeLogger(
-        parseLoggerArgs(args, {defaultLogFilepath: path.join(validatorPaths.dataDir, "validator.log")}, config)
-      );
-
-      const {validatorsDbDir: dbPath} = getValidatorPaths(args, network);
-
-      logger.info("Importing slashing protection data", {dbPath});
-
-      const {slashingProtection, metadata} = await getSlashingProtection(args, network, logger);
-
-      // Fetch genesisValidatorsRoot from:
-      // - existing cached in validator DB
-      // - known genesis data from existing network
-      // - else fetch from beacon node
-      const genesisValidatorsRoot =
-        (await metadata.getGenesisValidatorsRoot()) ?? (await getGenesisValidatorsRoot(args));
-
-      logger.verbose("Reading slashing protection data", {file});
-      const interchangeStr = await fs.promises.readFile(file, "utf8");
-      const interchangeJson = JSON.parse(interchangeStr) as Interchange;
-
-      await slashingProtection.importInterchange(interchangeJson, genesisValidatorsRoot, logger);
-      logger.info("Import completed successfully");
-    },
-  };

package/src/cmds/validator/slashingProtection/index.ts

@@ -1,12 +0,0 @@
-import {CliCommand} from "@lodestar/utils";
-import {AccountValidatorArgs} from "../options.js";
-import {exportCmd} from "./export.js";
-import {importCmd} from "./import.js";
-import {ISlashingProtectionArgs, slashingProtectionOptions} from "./options.js";
-
-export const slashingProtection: CliCommand<ISlashingProtectionArgs, AccountValidatorArgs> = {
-  command: "slashing-protection <command>",
-  describe: "Import or export slashing protection data to or from another client.",
-  options: slashingProtectionOptions,
-  subcommands: [importCmd, exportCmd],
-};

package/src/cmds/validator/slashingProtection/options.ts

@@ -1,15 +0,0 @@
-import {CliCommandOptions} from "@lodestar/utils";
-import {IValidatorCliArgs, validatorOptions} from "../options.js";
-
-export type ISlashingProtectionArgs = Pick<IValidatorCliArgs, "beaconNodes"> & {
-  force?: boolean;
-};
-
-export const slashingProtectionOptions: CliCommandOptions<ISlashingProtectionArgs> = {
-  beaconNodes: validatorOptions.beaconNodes,
-
-  force: {
-    description: "If `genesisValidatorsRoot` can't be fetched from the Beacon node, use a zero hash",
-    type: "boolean",
-  },
-};

package/src/cmds/validator/slashingProtection/utils.ts

@@ -1,56 +0,0 @@
-import {getClient} from "@lodestar/api";
-import {NetworkName, genesisData} from "@lodestar/config/networks";
-import {LevelDbController} from "@lodestar/db/controller/level";
-import {Root} from "@lodestar/types";
-import {Logger, fromHex} from "@lodestar/utils";
-import {MetaDataRepository, SlashingProtection} from "@lodestar/validator";
-import {getBeaconConfigFromArgs} from "../../../config/index.js";
-import {GlobalArgs} from "../../../options/index.js";
-import {getValidatorPaths} from "../paths.js";
-import {ISlashingProtectionArgs} from "./options.js";
-
-/**
- * Returns a new SlashingProtection object instance based on global args.
- */
-export async function getSlashingProtection(
-  args: GlobalArgs,
-  network: string,
-  logger: Logger
-): Promise<{slashingProtection: SlashingProtection; metadata: MetaDataRepository}> {
-  const validatorPaths = getValidatorPaths(args, network);
-  const dbPath = validatorPaths.validatorsDbDir;
-
-  const db = await LevelDbController.create({name: dbPath}, {logger});
-
-  return {
-    slashingProtection: new SlashingProtection(db),
-    metadata: new MetaDataRepository(db),
-  };
-}
-
-/**
- * Returns genesisValidatorsRoot from validator API client.
- */
-export async function getGenesisValidatorsRoot(args: GlobalArgs & ISlashingProtectionArgs): Promise<Root> {
-  const server = args.beaconNodes[0];
-
-  const networkGenesis = genesisData[args.network as NetworkName];
-  if (networkGenesis !== undefined) {
-    return fromHex(networkGenesis.genesisValidatorsRoot);
-  }
-
-  const {config} = getBeaconConfigFromArgs(args);
-  const api = getClient({baseUrl: server}, {config});
-  const genesis = await api.beacon.getGenesis();
-
-  try {
-    genesis.assertOk();
-  } catch (e) {
-    if (args.force) {
-      return Buffer.alloc(32, 0);
-    }
-    throw e;
-  }
-
-  return genesis.value().genesisValidatorsRoot;
-}