@chainsafe/lodestar 1.35.0-dev.8644a83c62 → 1.35.0-dev.8689cc3545

package/src/paths/global.ts

@@ -0,0 +1,24 @@
+import {GlobalArgs} from "../options/index.js";
+import {getDefaultDataDir} from "./rootDir.js";
+
+export type GlobalPaths = {
+  dataDir: string;
+};
+
+/**
+ * Defines the path structure of the globally used files
+ *
+ * ```bash
+ * $dataDir
+ * └── $paramsFile
+ * ```
+ */
+export function getGlobalPaths(args: Partial<GlobalArgs>, network: string): GlobalPaths {
+  // Set dataDir to network name iff dataDir is not set explicitly
+  const dataDir = args.dataDir || getDefaultDataDir(network);
+  return {
+    dataDir,
+  };
+}
+
+export const defaultGlobalPaths = getGlobalPaths({dataDir: "$dataDir"}, "$network");

package/src/paths/rootDir.ts

@@ -0,0 +1,11 @@
+import os from "node:os";
+import path from "node:path";
+
+/**
+ * Follows XDG Base Directory Specification
+ * https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html#basics
+ */
+export function getDefaultDataDir(network: string): string {
+  const dataHome = process.env.XDG_DATA_HOME || path.join(os.homedir(), ".local", "share");
+  return path.join(dataHome, "lodestar", network);
+}

package/src/util/errors.ts

@@ -0,0 +1,20 @@
+/**
+ * Expected error that shouldn't print a stack trace
+ */
+export class YargsError extends Error {}
+
+export type Result<T> = {err: null; result: T} | {err: Error};
+export async function wrapError<T>(promise: Promise<T>): Promise<Result<T>> {
+  try {
+    return {err: null, result: await promise};
+  } catch (err) {
+    return {err: err as Error};
+  }
+}
+export function wrapFnError<T>(fn: () => T): Result<T> {
+  try {
+    return {err: null, result: fn()};
+  } catch (err) {
+    return {err: err as Error};
+  }
+}

package/src/util/ethers.ts

@@ -0,0 +1,44 @@
+import fs from "node:fs";
+import {ethers} from "ethers";
+
+/**
+ * Returns a connected ethers signer from a variety of provider options
+ */
+export async function getEthersSigner({
+  keystorePath,
+  keystorePassword,
+  rpcUrl,
+  rpcPassword,
+  ipcPath,
+  chainId,
+}: {
+  keystorePath?: string;
+  keystorePassword?: string;
+  rpcUrl?: string;
+  rpcPassword?: string;
+  ipcPath?: string;
+  chainId: number;
+}): Promise<ethers.Signer> {
+  if (keystorePath && keystorePassword) {
+    const keystoreJson = fs.readFileSync(keystorePath, "utf8");
+    const wallet = await ethers.Wallet.fromEncryptedJson(keystoreJson, keystorePassword);
+    const eth1Provider = rpcUrl
+      ? new ethers.JsonRpcProvider(rpcUrl)
+      : new ethers.InfuraProvider({name: "deposit", chainId});
+    return wallet.connect(eth1Provider);
+  }
+
+  if (rpcUrl) {
+    const signer = await new ethers.JsonRpcProvider(rpcUrl).getSigner();
+    if (rpcPassword) {
+      await signer.unlock(rpcPassword);
+    }
+    return signer;
+  }
+
+  if (ipcPath) {
+    return new ethers.IpcSocketProvider(ipcPath).getSigner();
+  }
+
+  throw Error("Must supply either keystorePath, rpcUrl, or ipcPath");
+}

package/src/util/feeRecipient.ts

@@ -0,0 +1,6 @@
+export function parseFeeRecipient(feeRecipientHex: string): string {
+  if (!/^0x[a-fA-F0-9]{40}$/i.test(feeRecipientHex)) {
+    throw Error(`Invalid feeRecipient= ${feeRecipientHex}, expected format: ^0x[a-fA-F0-9]{40}$`);
+  }
+  return feeRecipientHex.toLowerCase();
+}

package/src/util/file.ts

@@ -0,0 +1,167 @@
+import fs from "node:fs";
+import path from "node:path";
+import {Readable} from "node:stream";
+import stream from "node:stream/promises";
+import {ReadableStream as NodeReadableStream} from "node:stream/web";
+import yaml from "js-yaml";
+import {fetch} from "@lodestar/utils";
+
+const {load, dump, FAILSAFE_SCHEMA, Type} = yaml;
+
+import {mkdir} from "./fs.js";
+
+export const yamlSchema = FAILSAFE_SCHEMA.extend({
+  implicit: [
+    new Type("tag:yaml.org,2002:str", {
+      kind: "scalar",
+      construct: function construct(data) {
+        return data !== null ? data : "";
+      },
+    }),
+  ],
+});
+
+export enum FileFormat {
+  json = "json",
+  yaml = "yaml",
+  yml = "yml",
+  toml = "toml",
+}
+
+/**
+ * Parse file contents as Json.
+ */
+export function parse<T>(contents: string, fileFormat: FileFormat): T {
+  switch (fileFormat) {
+    case FileFormat.json:
+      return JSON.parse(contents) as T;
+    case FileFormat.yaml:
+    case FileFormat.yml:
+      return load(contents, {schema: yamlSchema}) as T;
+    default:
+      return contents as unknown as T;
+  }
+}
+
+/**
+ * Stringify file contents.
+ */
+export function stringify(obj: unknown, fileFormat: FileFormat): string {
+  let contents: string;
+  switch (fileFormat) {
+    case FileFormat.json:
+      contents = JSON.stringify(obj, null, 2);
+      break;
+    case FileFormat.yaml:
+    case FileFormat.yml:
+      contents = dump(obj, {schema: yamlSchema});
+      break;
+    default:
+      contents = obj as string;
+  }
+  return contents;
+}
+
+/**
+ * Write a JSON serializable object to a file
+ *
+ * Serialize either to json, yaml, or toml
+ */
+export function writeFile(filepath: string, obj: unknown, options: fs.WriteFileOptions = "utf-8"): void {
+  mkdir(path.dirname(filepath));
+  const fileFormat = path.extname(filepath).substr(1);
+  fs.writeFileSync(filepath, typeof obj === "string" ? obj : stringify(obj, fileFormat as FileFormat), options);
+}
+
+/**
+ * Create a file with `600 (-rw-------)` permissions
+ * *Note*: 600: Owner has full read and write access to the file,
+ * while no other user can access the file
+ */
+export function writeFile600Perm(filepath: string, obj: unknown, options?: fs.WriteFileOptions): void {
+  writeFile(filepath, obj, options);
+  fs.chmodSync(filepath, "0600");
+}
+
+/**
+ * Read a JSON serializable object from a file
+ *
+ * Parse either from json, yaml, or toml
+ * Optional acceptedFormats object can be passed which can be an array of accepted formats, in future can be extended to include parseFn for the accepted formats
+ */
+export function readFile<T>(filepath: string, acceptedFormats?: string[]): T {
+  const fileFormat = path.extname(filepath).substr(1);
+  if (acceptedFormats && !acceptedFormats.includes(fileFormat)) throw new Error(`UnsupportedFileFormat: ${filepath}`);
+  const contents = fs.readFileSync(filepath, "utf-8");
+  return parse(contents, fileFormat as FileFormat);
+}
+
+/**
+ * @see readFile
+ * If `filepath` does not exist returns null
+ */
+export function readFileIfExists<T>(filepath: string, acceptedFormats?: string[]): T | null {
+  try {
+    return readFile(filepath, acceptedFormats);
+  } catch (e) {
+    if ((e as {code: string}).code === "ENOENT") {
+      return null;
+    }
+    throw e;
+  }
+}
+
+/**
+ * Download from URL or copy from local filesystem
+ * @param urlOrPathSrc "/path/to/file.szz" | "https://url.to/file.szz"
+ */
+export async function downloadOrCopyFile(pathDest: string, urlOrPathSrc: string): Promise<void> {
+  if (isUrl(urlOrPathSrc)) {
+    await downloadFile(pathDest, urlOrPathSrc);
+  } else {
+    mkdir(path.dirname(pathDest));
+    await fs.promises.copyFile(urlOrPathSrc, pathDest);
+  }
+}
+
+/**
+ * Downloads a genesis file per network if it does not exist
+ */
+export async function downloadFile(pathDest: string, url: string): Promise<void> {
+  if (!fs.existsSync(pathDest)) {
+    mkdir(path.dirname(pathDest));
+    const res = await fetch(url);
+    if (!res.ok) {
+      throw new Error(`Failed to download file from ${url}: ${res.status} ${res.statusText}`);
+    }
+
+    if (!res.body) {
+      throw new Error("Response body is null");
+    }
+
+    await stream.pipeline(Readable.fromWeb(res.body as NodeReadableStream), fs.createWriteStream(pathDest));
+  }
+}
+
+/**
+ * Download from URL to memory or load from local filesystem
+ * @param urlOrPathSrc "/path/to/file.szz" | "https://url.to/file.szz"
+ */
+export async function downloadOrLoadFile(pathOrUrl: string): Promise<Uint8Array> {
+  if (isUrl(pathOrUrl)) {
+    const res = await fetch(pathOrUrl);
+    if (!res.ok) {
+      throw new Error(`Failed to download file from ${pathOrUrl}: ${res.status} ${res.statusText}`);
+    }
+    const arrayBuffer = await res.arrayBuffer();
+    return new Uint8Array(arrayBuffer);
+  }
+  return fs.promises.readFile(pathOrUrl);
+}
+
+/**
+ * Returns boolean for whether the string is a URL.
+ */
+function isUrl(pathOrUrl: string): boolean {
+  return pathOrUrl.startsWith("http");
+}

package/src/util/format.ts

@@ -0,0 +1,76 @@
+import {PublicKey} from "@chainsafe/blst";
+import {fromHex} from "@lodestar/utils";
+
+/**
+ * 0x prefix a string if not prefixed already
+ */
+export function ensure0xPrefix(hex: string): string {
+  if (!hex.startsWith("0x")) hex = `0x${hex}`;
+  return hex;
+}
+
+export function isValidatePubkeyHex(pubkeyHex: string): boolean {
+  return /^0x[0-9a-fA-F]{96}$/.test(pubkeyHex);
+}
+
+export function getPubkeyHexFromKeystore(keystore: {pubkey?: string}): string {
+  if (!keystore.pubkey) {
+    throw Error("Invalid keystore, must contain .pubkey property");
+  }
+
+  const pubkeyHex = ensure0xPrefix(keystore.pubkey);
+  if (!isValidatePubkeyHex(pubkeyHex)) {
+    throw Error(`Invalid keystore pubkey format ${pubkeyHex}`);
+  }
+
+  return pubkeyHex;
+}
+
+/**
+ * Parse string inclusive range: `0..32`, into an array of all values in that range
+ */
+export function parseRange(range: string): number[] {
+  if (!range.includes("..")) {
+    throw Error(`Invalid range '${range}', must include '..'`);
+  }
+
+  const [from, to] = range.split("..").map((n) => parseInt(n));
+
+  if (Number.isNaN(from)) throw Error(`Invalid range from isNaN '${range}'`);
+  if (Number.isNaN(to)) throw Error(`Invalid range to isNaN '${range}'`);
+  if (from > to) throw Error(`Invalid range from > to '${range}'`);
+
+  const arr: number[] = [];
+  for (let i = from; i <= to; i++) {
+    arr.push(i);
+  }
+
+  return arr;
+}
+
+export function assertValidPubkeysHex(pubkeysHex: string[]): void {
+  for (const pubkeyHex of pubkeysHex) {
+    const pubkeyBytes = fromHex(pubkeyHex);
+    PublicKey.fromBytes(pubkeyBytes, true);
+  }
+}
+
+/**
+ * Parses a file to get a list of bootnodes for a network.
+ * Bootnodes file is expected to contain bootnode ENR's concatenated by newlines, or commas for
+ * parsing plaintext, YAML, JSON and/or env files.
+ */
+export function parseBootnodesFile(bootnodesFile: string): string[] {
+  const enrs = [];
+  for (const line of bootnodesFile.trim().split(/\r?\n/)) {
+    for (const entry of line.split(",")) {
+      const sanitizedEntry = entry.replace(/['",[\]{}.]+/g, "").trim();
+
+      if (sanitizedEntry.includes("enr:-")) {
+        const parsedEnr = `enr:-${sanitizedEntry.split("enr:-")[1].split("#")[0].trim()}`;
+        enrs.push(parsedEnr);
+      }
+    }
+  }
+  return enrs;
+}

package/src/util/fs.ts

@@ -0,0 +1,59 @@
+import fs from "node:fs";
+import path from "node:path";
+
+/**
+ * Maybe create a directory
+ */
+export function mkdir(dirname: string): void {
+  if (!fs.existsSync(dirname)) fs.mkdirSync(dirname, {recursive: true});
+}
+
+/** NodeJS POSIX errors subset */
+type ErrorFs = Error & {code: "ENOENT" | "ENOTDIR"};
+
+/**
+ * Attempts to unlink a file, return true if it is deleted, false if not found
+ */
+export function unlinkSyncMaybe(filepath: string): boolean {
+  try {
+    fs.unlinkSync(filepath);
+    return true;
+  } catch (e) {
+    const {code} = e as ErrorFs;
+    if (code === "ENOENT") return false;
+
+    throw e;
+  }
+}
+
+/**
+ * Attempts rm a dir, return true if it is deleted, false if not found
+ */
+export function rmdirSyncMaybe(dirpath: string): boolean {
+  try {
+    fs.rmSync(dirpath, {recursive: true});
+    return true;
+  } catch (e) {
+    const {code} = e as ErrorFs;
+    // about error codes https://nodejs.org/api/fs.html#fspromisesrmdirpath-options
+    // ENOENT error on Windows and an ENOTDIR
+    if (code === "ENOENT" || code === "ENOTDIR") return false;
+
+    throw e;
+  }
+}
+
+/**
+ * Find all files recursively in `dirPath`
+ */
+export function recursiveLookup(dirPath: string, filepaths: string[] = []): string[] {
+  if (fs.statSync(dirPath).isDirectory()) {
+    for (const filename of fs.readdirSync(dirPath)) {
+      recursiveLookup(path.join(dirPath, filename), filepaths);
+    }
+  } else {
+    filepaths.push(dirPath);
+  }
+
+  return filepaths;
+}

package/src/util/gitData/gitDataPath.ts

@@ -0,0 +1,48 @@
+import fs from "node:fs";
+import path from "node:path";
+import {fileURLToPath} from "node:url";
+
+// Global variable __dirname no longer available in ES6 modules.
+// Solutions: https://stackoverflow.com/questions/46745014/alternative-for-dirname-in-node-js-when-using-es6-modules
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+// Persist git data and distribute through NPM so CLI consumers can know exactly
+// at what commit was this src build. This is used in the metrics and to log initially.
+//
+// - For NPM release (stable): Only the version is persisted. Once must then track the version's tag
+//   in Github to resolve that version to a specific commit. While this is okay, git-data.json gives
+//   a gurantee of the exact commit at build time.
+//
+// - For NPM release (dev): canary commits include the commit, so this feature is not really
+//   necessary. However, it's more cumbersome to have conditional logic on stable / dev.
+//
+// - For build from source: .git folder is available in the context of the built code, so it can extract
+//   branch and commit directly without the need for .git-data.json.
+//
+// - For build from source dockerized: This feature is required to know the branch and commit, since
+//   git data is not persisted past the build. However, .dockerignore prevents .git folder from being
+//   copied into the container's context, so .git-data.json can't be generated.
+
+/**
+ * WARNING!! If you change this path make sure to update:
+ * - 'packages/cli/package.json' -> .files -> `".git-data.json"`
+ */
+export const gitDataPath = path.resolve(__dirname, "../../../.git-data.json");
+
+/** Git data type used to construct version information string and persistence. */
+export type GitData = {
+  /** "developer-feature" */
+  branch: string;
+  /** "80c248bb392f512cc115d95059e22239a17bbd7d" */
+  commit: string;
+};
+
+/** Writes a persistent git data file. */
+export function writeGitDataFile(gitData: GitData): void {
+  fs.writeFileSync(gitDataPath, JSON.stringify(gitData, null, 2));
+}
+
+/** Reads the persistent git data file. */
+export function readGitDataFile(): GitData {
+  return JSON.parse(fs.readFileSync(gitDataPath, "utf8")) as GitData;
+}

package/src/util/gitData/index.ts

@@ -0,0 +1,70 @@
+import {execSync} from "node:child_process";
+// This file is created in the build step and is distributed through NPM
+// MUST be in sync with `-/gitDataPath.ts` and `package.json` files.
+import {GitData, readGitDataFile} from "./gitDataPath.js";
+
+/** Reads git data from a persisted file or local git data at build time. */
+export function readAndGetGitData(): GitData {
+  try {
+    // Gets git data containing current branch and commit info from persistent file.
+    let persistedGitData: Partial<GitData>;
+    try {
+      persistedGitData = readGitDataFile();
+    } catch (_e) {
+      persistedGitData = {};
+    }
+
+    const currentGitData = getGitData();
+
+    return {
+      // If the CLI is run from source, prioritze current git data
+      // over `.git-data.json` file, which might be stale here.
+      branch:
+        currentGitData.branch && currentGitData.branch.length > 0
+          ? currentGitData.branch
+          : (persistedGitData.branch ?? ""),
+      commit:
+        currentGitData.commit && currentGitData.commit.length > 0
+          ? currentGitData.commit
+          : (persistedGitData.commit ?? ""),
+    };
+  } catch (_e) {
+    return {
+      branch: "",
+      commit: "",
+    };
+  }
+}
+
+/** Gets git data containing current branch and commit info from CLI. */
+export function getGitData(): GitData {
+  return {
+    branch: process.env.GIT_BRANCH ?? getBranch(),
+    commit: process.env.GIT_COMMIT ?? getCommit(),
+  };
+}
+
+/** Tries to get branch from git CLI. */
+function getBranch(): string {
+  try {
+    return shellSilent("git rev-parse --abbrev-ref HEAD");
+  } catch (_e) {
+    return "";
+  }
+}
+
+/** Tries to get commit from git from git CLI. */
+function getCommit(): string {
+  try {
+    return shellSilent("git rev-parse --verify HEAD");
+  } catch (_e) {
+    return "";
+  }
+}
+
+/** Silent shell that won't pollute stdout, or stderr */
+function shellSilent(cmd: string): string {
+  return execSync(cmd, {stdio: ["ignore", "pipe", "ignore"]})
+    .toString()
+    .trim();
+}

package/src/util/gitData/writeGitData.ts

@@ -0,0 +1,10 @@
+#!/usr/bin/env node
+
+// For RATIONALE of this file, check packages/cli/src/util/gitData/gitDataPath.ts
+// Persist exact commit in NPM distributions for easier tracking of the build
+
+import {writeGitDataFile} from "./gitDataPath.js";
+import {getGitData} from "./index.js";
+
+// Script to write the git data file (json) used by the build procedures to persist git data.
+writeGitDataFile(getGitData());

package/src/util/index.ts

@@ -0,0 +1,17 @@
+export * from "./errors.js";
+export * from "./ethers.js";
+export * from "./feeRecipient.js";
+export * from "./file.js";
+export * from "./format.js";
+export * from "./fs.js";
+export * from "./gitData/index.js";
+export * from "./jwt.js";
+export * from "./logger.js";
+export * from "./object.js";
+export * from "./passphrase.js";
+export * from "./process.js";
+export * from "./proposerConfig.js";
+export * from "./pruneOldFilesInDir.js";
+export * from "./sleep.js";
+export * from "./stripOffNewlines.js";
+export * from "./types.js";

package/src/util/jwt.ts

@@ -0,0 +1,10 @@
+export function extractJwtHexSecret(jwtSecretContents: string): string {
+  const hexPattern = new RegExp(/^(0x|0X)?(?<jwtSecret>[a-fA-F0-9]+)$/, "g");
+  const jwtSecretHexMatch = hexPattern.exec(jwtSecretContents);
+  const jwtSecret = jwtSecretHexMatch?.groups?.jwtSecret;
+  if (!jwtSecret || jwtSecret.length !== 64) {
+    throw Error(`Need a valid 256 bit hex encoded secret ${jwtSecret} ${jwtSecretContents}`);
+  }
+  // Return the secret in proper hex format
+  return `0x${jwtSecret}`;
+}

package/src/util/lockfile.ts

@@ -0,0 +1,45 @@
+import {lockSync, unlockSync} from "proper-lockfile";
+
+/**
+ * Creates a .lock file for `filepath`, argument passed must not be the lock path
+ * @param filepath File to lock, i.e. `keystore_0001.json`
+ */
+export function lockFilepath(filepath: string): void {
+  try {
+    lockSync(filepath, {
+      // Allows to lock files that do not exist
+      realpath: false,
+    });
+  } catch (e) {
+    if (isLockfileError(e) && (e.code === "ELOCKED" || e.code === "ENOTDIR")) {
+      e.message = `'${filepath}' already in use by another process`;
+    }
+    throw e;
+  }
+}
+
+/**
+ * Deletes a .lock file for `filepath`, argument passed must not be the lock path
+ * @param filepath File to unlock, i.e. `keystore_0001.json`
+ */
+export function unlockFilepath(filepath: string): void {
+  try {
+    unlockSync(filepath, {
+      // Allows to unlock files that do not exist
+      realpath: false,
+    });
+  } catch (e) {
+    if (isLockfileError(e) && e.code === "ENOTACQUIRED") {
+      // Do not throw if the lock file is already deleted
+      return;
+    }
+    throw e;
+  }
+}
+
+// https://github.com/moxystudio/node-proper-lockfile/blob/9f8c303c91998e8404a911dc11c54029812bca69/lib/lockfile.js#L53
+export type LockfileError = Error & {code: "ELOCKED" | "ENOTACQUIRED" | "ENOTDIR"};
+
+function isLockfileError(e: unknown): e is LockfileError {
+  return e instanceof Error && (e as LockfileError).code !== undefined;
+}