@chainsafe/lodestar 1.35.0-dev.c0078a16b5 → 1.35.0-dev.c1880f6940

package/src/util/logger.ts

@@ -0,0 +1,105 @@
+import fs from "node:fs";
+import path from "node:path";
+import {ChainForkConfig} from "@lodestar/config";
+import {LogFormat, TimestampFormatCode, logFormats} from "@lodestar/logger";
+import {LoggerNodeOpts} from "@lodestar/logger/node";
+import {SLOTS_PER_EPOCH} from "@lodestar/params";
+import {LogLevel} from "@lodestar/utils";
+import {GlobalArgs} from "../options/globalOptions.js";
+import {LogArgs} from "../options/logOptions.js";
+
+export const LOG_FILE_DISABLE_KEYWORD = "none";
+
+/**
+ * Setup a CLI logger, common for beacon, validator and dev commands
+ */
+export function parseLoggerArgs(
+  args: LogArgs & Pick<GlobalArgs, "dataDir">,
+  paths: {defaultLogFilepath: string},
+  config: ChainForkConfig,
+  opts?: {hideTimestamp?: boolean}
+): LoggerNodeOpts {
+  return {
+    level: parseLogLevel(args.logLevel),
+    file:
+      args.logFile === LOG_FILE_DISABLE_KEYWORD
+        ? undefined
+        : {
+            filepath: args.logFile ?? paths.defaultLogFilepath,
+            level: parseLogLevel(args.logFileLevel),
+            dailyRotate: args.logFileDailyRotate,
+          },
+    module: args.logPrefix,
+    format: args.logFormat ? parseLogFormat(args.logFormat) : undefined,
+    levelModule: args.logLevelModule && parseLogLevelModule(args.logLevelModule),
+    timestampFormat: opts?.hideTimestamp
+      ? {format: TimestampFormatCode.Hidden}
+      : args.logFormatGenesisTime !== undefined
+        ? {
+            format: TimestampFormatCode.EpochSlot,
+            genesisTime: args.logFormatGenesisTime,
+            secondsPerSlot: config.SECONDS_PER_SLOT,
+            slotsPerEpoch: SLOTS_PER_EPOCH,
+          }
+        : {
+            format: TimestampFormatCode.DateRegular,
+          },
+  };
+}
+
+function parseLogFormat(format: string): LogFormat {
+  if (!logFormats.includes(format as LogFormat)) {
+    throw Error(`Unknown log format ${format}`);
+  }
+  return format as LogFormat;
+}
+
+function parseLogLevel(level: string): LogLevel {
+  if (LogLevel[level as LogLevel] === undefined) {
+    throw Error(`Unknown log level '${level}'`);
+  }
+  return level as LogLevel;
+}
+
+function parseLogLevelModule(logLevelModuleArr: string[]): Record<string, LogLevel> {
+  const levelModule: Record<string, LogLevel> = {};
+  for (const logLevelModule of logLevelModuleArr) {
+    const [module, levelStr] = logLevelModule.split("=");
+    levelModule[module] = parseLogLevel(levelStr);
+  }
+  return levelModule;
+}
+
+/**
+ * Winston is not able to clean old log files if server is offline for a while
+ * so we have to do this manually when starting the node.
+ * See https://github.com/ChainSafe/lodestar/issues/4419
+ */
+export function cleanOldLogFiles(args: LogArgs, paths: {defaultLogFilepath: string}): void {
+  const filepath = args.logFile ?? paths.defaultLogFilepath;
+  const folder = path.dirname(filepath);
+  const filename = path.basename(filepath);
+  const lastIndexDot = filename.lastIndexOf(".");
+  const prefix = filename.substring(0, lastIndexDot);
+  const extension = filename.substring(lastIndexDot + 1, filename.length);
+  const toDelete = fs
+    .readdirSync(folder, {withFileTypes: true})
+    .filter((de) => de.isFile())
+    .map((de) => de.name)
+    .filter((logFileName) => shouldDeleteLogFile(prefix, extension, logFileName, args.logFileDailyRotate))
+    .map((logFileName) => path.join(folder, logFileName));
+  // delete files
+  for (const filename of toDelete) {
+    fs.unlinkSync(filename);
+  }
+}
+
+export function shouldDeleteLogFile(prefix: string, extension: string, logFileName: string, maxFiles: number): boolean {
+  const maxDifferenceMs = maxFiles * 24 * 60 * 60 * 1000;
+  const match = logFileName.match(new RegExp(`${prefix}-([0-9]{4}-[0-9]{2}-[0-9]{2}).${extension}`));
+  // if match[1] exists, it should be the date pattern of YYYY-MM-DD
+  if (match?.[1] && Date.now() - new Date(match[1]).getTime() > maxDifferenceMs) {
+    return true;
+  }
+  return false;
+}

package/src/util/object.ts

@@ -0,0 +1,15 @@
+import {RecursivePartial} from "@lodestar/utils";
+
+/**
+ * Removes (mutates) all properties with a value === undefined, recursively
+ */
+
+// biome-ignore lint/suspicious/noExplicitAny: We need to use `any` type here
+export function removeUndefinedRecursive<T extends {[key: string]: any}>(obj: T): RecursivePartial<T> {
+  for (const key of Object.keys(obj)) {
+    const value = obj[key];
+    if (value && typeof value === "object") removeUndefinedRecursive(value);
+    else if (value === undefined) delete obj[key];
+  }
+  return obj as RecursivePartial<T>;
+}

package/src/util/passphrase.ts

@@ -0,0 +1,25 @@
+import fs from "node:fs";
+import {stripOffNewlines} from "./stripOffNewlines.js";
+
+/**
+ * Utility to read file as UTF8 and strip any trailing new lines.
+ * All passphrase files must be read with this function
+ */
+export function readPassphraseFile(passphraseFile: string): string {
+  const data = fs.readFileSync(passphraseFile, "utf8");
+  const passphrase = stripOffNewlines(data);
+
+  // Validate the passphraseFile contents to prevent the user to create a wallet with a password
+  // that is the contents a random unintended file
+  try {
+    if (passphrase.includes("\n")) throw Error("contains multiple lines");
+    // 512 is an arbitrary high number that should be longer than any actual passphrase
+    if (passphrase.length > 512) throw Error("is really long");
+  } catch (e) {
+    throw new Error(
+      `passphraseFile ${passphraseFile} ${(e as Error).message}. Is this a well-formatted passphraseFile?`
+    );
+  }
+
+  return passphrase;
+}

package/src/util/process.ts

@@ -0,0 +1,25 @@
+const exitSignals = ["SIGTERM", "SIGINT"] as NodeJS.Signals[];
+
+/**
+ * All CLI handlers should register this callback to exit properly and not leave
+ * a process hanging forever. Pass a clean function that will be run until the
+ * user forcibly kills the process by doing CTRL+C again
+ * @param cleanUpFunction
+ */
+export function onGracefulShutdown(
+  cleanUpFunction: () => Promise<void>,
+  logFn: (msg: string) => void = console.log
+): void {
+  for (const signal of exitSignals) {
+    process.once(signal, async function onSignal() {
+      logFn("Stopping gracefully, use Ctrl+C again to force process exit");
+
+      process.on(signal, function onSecondSignal() {
+        logFn("Forcing process exit");
+        process.exit(1);
+      });
+
+      await cleanUpFunction();
+    });
+  }
+}

package/src/util/progress.ts

@@ -0,0 +1,58 @@
+// zero is considered first index in the range
+type NeedleFunc = (needle: number) => void;
+type ProgressFunc = (opts: {current: number; total: number; percentage: number; ratePerSec: number}) => void;
+
+export function showProgress({
+  total,
+  signal,
+  frequencyMs,
+  progress,
+}: {
+  total: number;
+  signal: AbortSignal;
+  frequencyMs: number;
+  progress: ProgressFunc;
+}): NeedleFunc {
+  let current = 0;
+  let last = 0;
+  let lastProcessTime: number = Date.now();
+  let progressIntervalId: NodeJS.Timeout;
+
+  const needle: NeedleFunc = (needle: number) => {
+    // zero is considered first index in the range
+    current = needle + 1;
+
+    if (current >= total) {
+      processProgress();
+    }
+  };
+
+  const processProgress = (): void => {
+    const currentTime = Date.now();
+    const processTime = currentTime - lastProcessTime;
+
+    progress({
+      current,
+      total,
+      ratePerSec: processTime === 0 ? 0 : ((current - last) / processTime) * 1000,
+      percentage: total ? (current / total) * 100 : 100,
+    });
+
+    last = current;
+    lastProcessTime = currentTime;
+
+    if (current >= total) {
+      clearInterval(progressIntervalId);
+    }
+  };
+
+  if (total > 0) {
+    progressIntervalId = setInterval(processProgress, frequencyMs);
+  }
+
+  signal.addEventListener("abort", () => {
+    clearInterval(progressIntervalId);
+  });
+
+  return needle;
+}

package/src/util/proposerConfig.ts

@@ -0,0 +1,136 @@
+import fs from "node:fs";
+import path from "node:path";
+import {routes} from "@lodestar/api";
+import {ValidatorProposerConfig} from "@lodestar/validator";
+import {parseFeeRecipient} from "./feeRecipient.js";
+import {readFile} from "./file.js";
+
+type ProposerConfig = ValidatorProposerConfig["defaultConfig"];
+
+type ProposerConfigFileSection = {
+  graffiti?: string;
+  strict_fee_recipient_check?: string;
+  fee_recipient?: string;
+  builder?: {
+    // boolean are parse as string by the default schema readFile employs
+    // for js-yaml
+    gas_limit?: number;
+    selection?: routes.validator.BuilderSelection;
+    boost_factor?: bigint;
+  };
+};
+
+type ProposerConfigFile = {
+  proposer_config?: {[index: string]: ProposerConfigFileSection};
+  default_config?: ProposerConfigFileSection;
+};
+
+export function parseProposerConfig(
+  configFilePath: string,
+  defaultArgsConfig?: ProposerConfig
+): ValidatorProposerConfig {
+  const configFile = readFile<ProposerConfigFile>(configFilePath, ["yml", "yaml"]);
+  const defaultConfigParsed = parseProposerConfigSection(configFile.default_config || {}, defaultArgsConfig);
+
+  const proposerConfigFile = configFile.proposer_config || {};
+  const proposerConfigParsed: ValidatorProposerConfig["proposerConfig"] = {};
+  for (const pubkeyHex of Object.keys(proposerConfigFile)) {
+    proposerConfigParsed[pubkeyHex] = parseProposerConfigSection(proposerConfigFile[pubkeyHex]);
+  }
+
+  return {
+    proposerConfig: proposerConfigParsed,
+    defaultConfig: defaultConfigParsed,
+  };
+}
+
+function stringtoBool(input: string): boolean {
+  const boolValue = typeof input === "string" ? input === "true" : input;
+  return boolValue;
+}
+
+function parseProposerConfigSection(
+  proposerFileSection: ProposerConfigFileSection,
+  overrideConfig?: ProposerConfig
+): ProposerConfig {
+  const {graffiti, strict_fee_recipient_check, fee_recipient, builder} = proposerFileSection;
+  const {gas_limit, selection: builderSelection, boost_factor} = builder || {};
+
+  if (graffiti !== undefined && typeof graffiti !== "string") {
+    throw Error("graffiti is not 'string");
+  }
+  if (
+    strict_fee_recipient_check !== undefined &&
+    !(strict_fee_recipient_check === "true" || strict_fee_recipient_check === "false")
+  ) {
+    throw Error("strict_fee_recipient_check is not set to boolean");
+  }
+  if (fee_recipient !== undefined && typeof fee_recipient !== "string") {
+    throw Error("fee_recipient is not 'string");
+  }
+  if (gas_limit !== undefined) {
+    if (typeof gas_limit !== "string") {
+      throw Error("(typeof gas_limit !== 'string') 2 ");
+    }
+    if (Number.isNaN(Number(gas_limit))) {
+      throw Error("(Number.isNaN(Number(gas_limit)) 2");
+    }
+  }
+  if (boost_factor !== undefined && typeof boost_factor !== "string") {
+    throw Error("boost_factor is not 'string");
+  }
+
+  return {
+    graffiti: overrideConfig?.graffiti ?? graffiti,
+    strictFeeRecipientCheck:
+      overrideConfig?.strictFeeRecipientCheck ??
+      (strict_fee_recipient_check ? stringtoBool(strict_fee_recipient_check) : undefined),
+    feeRecipient: overrideConfig?.feeRecipient ?? (fee_recipient ? parseFeeRecipient(fee_recipient) : undefined),
+    builder:
+      overrideConfig?.builder || builder
+        ? {
+            gasLimit: overrideConfig?.builder?.gasLimit ?? (gas_limit !== undefined ? Number(gas_limit) : undefined),
+            selection: overrideConfig?.builder?.selection ?? parseBuilderSelection(builderSelection),
+            boostFactor: overrideConfig?.builder?.boostFactor ?? parseBuilderBoostFactor(boost_factor),
+          }
+        : undefined,
+  };
+}
+
+export function readProposerConfigDir(filepath: string, filename: string): ProposerConfigFileSection {
+  const proposerConfigStr = fs.readFileSync(path.join(filepath, filename), "utf8");
+  const proposerConfigJSON = JSON.parse(proposerConfigStr) as ProposerConfigFileSection;
+  return proposerConfigJSON;
+}
+
+export function parseBuilderSelection(builderSelection?: string): routes.validator.BuilderSelection | undefined {
+  if (builderSelection) {
+    switch (builderSelection) {
+      case "default":
+        break;
+      case "maxprofit":
+        break;
+      case "builderalways":
+        break;
+      case "builderonly":
+        break;
+      case "executionalways":
+        break;
+      case "executiononly":
+        break;
+      default:
+        throw Error("Invalid input for builder selection, check help");
+    }
+  }
+  return builderSelection as routes.validator.BuilderSelection;
+}
+
+export function parseBuilderBoostFactor(boostFactor?: string): bigint | undefined {
+  if (boostFactor === undefined) return;
+
+  if (!/^\d+$/.test(boostFactor)) {
+    throw Error("Invalid input for builder boost factor, must be a valid number without decimals");
+  }
+
+  return BigInt(boostFactor);
+}

package/src/util/pruneOldFilesInDir.ts

@@ -0,0 +1,27 @@
+import fs from "node:fs";
+import path from "node:path";
+
+export function pruneOldFilesInDir(dirpath: string, maxAgeMs: number): number {
+  if (!fs.existsSync(dirpath)) {
+    return 0; // Nothing to prune
+  }
+
+  let deletedFileCount = 0;
+  for (const entryName of fs.readdirSync(dirpath)) {
+    const entryPath = path.join(dirpath, entryName);
+
+    const stat = fs.statSync(entryPath);
+    if (stat.isDirectory()) {
+      deletedFileCount += pruneOldFilesInDir(entryPath, maxAgeMs);
+    } else if (stat.isFile() && Date.now() - stat.mtimeMs > maxAgeMs) {
+      fs.unlinkSync(entryPath);
+      deletedFileCount += 1;
+    }
+  }
+
+  // if all files are deleted, delete the directory
+  if (fs.readdirSync(dirpath).length === 0) {
+    fs.rmdirSync(dirpath);
+  }
+  return deletedFileCount;
+}

package/src/util/sleep.ts

@@ -0,0 +1,3 @@
+export function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/src/util/stripOffNewlines.ts

@@ -0,0 +1,6 @@
+/**
+ * Remove trailing new lines '\n' or '\r' if any
+ */
+export function stripOffNewlines(s: string): string {
+  return s.replace(/[\n\r]+$/g, "");
+}

package/src/util/types.ts

@@ -0,0 +1,8 @@
+/**
+ * Typed `Object.keys(o: T)` function, returning `(keyof T)[]`
+ */
+
+// biome-ignore lint/suspicious/noExplicitAny: We need to use `any` type here
+export function ObjectKeys<T extends {[key: string]: any}>(o: T): (keyof T)[] {
+  return Object.keys(o);
+}

package/src/util/version.ts

@@ -0,0 +1,74 @@
+import fs from "node:fs";
+import path from "node:path";
+import {fileURLToPath} from "node:url";
+import {findUpSync} from "find-up";
+import {readAndGetGitData} from "./gitData/index.js";
+
+// Global variable __dirname no longer available in ES6 modules.
+// Solutions: https://stackoverflow.com/questions/46745014/alternative-for-dirname-in-node-js-when-using-es6-modules
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+type VersionJson = {
+  /** "0.28.2" */
+  version: string;
+};
+
+const BRANCH_IGNORE = /^(HEAD|master|unstable|main)$/;
+
+/**
+ * Gathers all information on package version including Git data.
+ * @returns a version string, e.g.
+ * - Stable release: `v0.36.0/80c248bb`
+ * - Dev release: `v0.36.0-dev.80c248bb/80c248bb`
+ * - Test branch: `v0.36.0/developer-feature/80c248bb`
+ */
+export function getVersionData(): {
+  version: string;
+  commit: string;
+} {
+  const parts: string[] = [];
+
+  /** Returns local version from `lerna.json` or `package.json` as `"0.28.2"` */
+  const localVersion = readCliPackageJson() || readVersionFromLernaJson();
+  if (localVersion) {
+    parts.push(`v${localVersion}`);
+  }
+
+  const {branch, commit} = readAndGetGitData();
+
+  // Add branch only if not present and not an ignore value
+  if (branch && !BRANCH_IGNORE.test(branch)) parts.push(branch);
+
+  // Add commit only if present. 7 characters to be consistent with Github
+  if (commit) {
+    const commitShort = commit.slice(0, 7);
+    // Don't add commit if it's already in the version string (dev versions)
+    if (!localVersion || !localVersion.includes(commitShort)) {
+      parts.push(commitShort);
+    }
+  }
+
+  return {
+    // Guard against empty parts array
+    version: parts.length > 0 ? parts.join("/") : "unknown",
+    commit,
+  };
+}
+
+/** Read version information from lerna.json */
+function readVersionFromLernaJson(): string | undefined {
+  const filePath = findUpSync("lerna.json", {cwd: __dirname});
+  if (!filePath) return undefined;
+
+  const lernaJson = JSON.parse(fs.readFileSync(filePath, "utf8")) as VersionJson;
+  return lernaJson.version;
+}
+
+/** Read version information from package.json */
+function readCliPackageJson(): string | undefined {
+  const filePath = findUpSync("package.json", {cwd: __dirname});
+  if (!filePath) return undefined;
+
+  const packageJson = JSON.parse(fs.readFileSync(filePath, "utf8")) as VersionJson;
+  return packageJson.version;
+}