@chainsafe/lodestar 1.35.0-dev.c88a6ed255 → 1.35.0-dev.c9deb9b59f

package/src/cmds/validator/keymanager/decryptKeystoreDefinitions.ts

@@ -0,0 +1,189 @@
+import fs from "node:fs";
+import path from "node:path";
+import {Keystore} from "@chainsafe/bls-keystore";
+import {SecretKey} from "@chainsafe/blst";
+import {LogLevel, Logger} from "@lodestar/utils";
+import {SignerLocal, SignerType} from "@lodestar/validator";
+import {lockFilepath, unlockFilepath} from "../../../util/lockfile.js";
+import {DecryptKeystoresThreadPool} from "./decryptKeystores/index.js";
+import {LocalKeystoreDefinition} from "./interface.js";
+import {clearKeystoreCache, loadKeystoreCache, writeKeystoreCache} from "./keystoreCache.js";
+
+export type KeystoreDecryptOptions = {
+  ignoreLockFile?: boolean;
+  onDecrypt?: (index: number) => void;
+  // Try to use the cache file if it exists
+  cacheFilePath?: string;
+  /** Use main thread to decrypt keystores */
+  disableThreadPool?: boolean;
+  logger: Pick<Logger, LogLevel.info | LogLevel.warn | LogLevel.debug>;
+  signal: AbortSignal;
+};
+
+type KeystoreDecryptError = {
+  keystoreFile: string;
+  error: Error;
+};
+
+/**
+ * Decrypt keystore definitions using a thread pool
+ */
+export async function decryptKeystoreDefinitions(
+  keystoreDefinitions: LocalKeystoreDefinition[],
+  opts: KeystoreDecryptOptions
+): Promise<SignerLocal[]> {
+  if (keystoreDefinitions.length === 0) {
+    return [];
+  }
+
+  if (opts.cacheFilePath) {
+    try {
+      const signers = await loadKeystoreCache(opts.cacheFilePath, keystoreDefinitions);
+
+      for (const {keystorePath} of keystoreDefinitions) {
+        lockKeystore(keystorePath, opts);
+      }
+
+      if (opts?.onDecrypt) {
+        opts?.onDecrypt(signers.length - 1);
+      }
+
+      opts.logger.debug("Loaded keystores via keystore cache");
+
+      return signers;
+    } catch (_e) {
+      // Some error loading the cache, ignore and invalidate cache
+      await clearKeystoreCache(opts.cacheFilePath);
+    }
+  }
+
+  const keystoreCount = keystoreDefinitions.length;
+  const signers = new Array<SignerLocal>(keystoreCount);
+  const passwords = new Array<string>(keystoreCount);
+  const errors: KeystoreDecryptError[] = [];
+
+  if (!opts.disableThreadPool) {
+    const decryptKeystores = new DecryptKeystoresThreadPool(keystoreCount, opts.signal);
+
+    for (const [index, definition] of keystoreDefinitions.entries()) {
+      lockKeystore(definition.keystorePath, opts);
+
+      decryptKeystores.queue(
+        definition,
+        (secretKeyBytes: Uint8Array) => {
+          const signer: SignerLocal = {
+            type: SignerType.Local,
+            secretKey: SecretKey.fromBytes(secretKeyBytes),
+          };
+
+          signers[index] = signer;
+          passwords[index] = definition.password;
+
+          if (opts?.onDecrypt) {
+            opts?.onDecrypt(index);
+          }
+        },
+        (error: Error) => {
+          // In-progress tasks can't be canceled, so there's a chance that multiple errors may be caught
+          // add to the list of errors
+          errors.push({keystoreFile: path.basename(definition.keystorePath), error});
+          // cancel all pending tasks, no need to continue decrypting after we hit one error
+          decryptKeystores.cancel();
+        }
+      );
+    }
+
+    await decryptKeystores.completed();
+  } else {
+    // Decrypt keystores in main thread
+    for (const [index, definition] of keystoreDefinitions.entries()) {
+      lockKeystore(definition.keystorePath, opts);
+
+      try {
+        const keystore = Keystore.parse(fs.readFileSync(definition.keystorePath, "utf8"));
+
+        // Memory-hogging function
+        const secretKeyBytes = await keystore.decrypt(definition.password);
+
+        const signer: SignerLocal = {
+          type: SignerType.Local,
+          secretKey: SecretKey.fromBytes(secretKeyBytes),
+        };
+
+        signers[index] = signer;
+        passwords[index] = definition.password;
+
+        if (opts?.onDecrypt) {
+          opts?.onDecrypt(index);
+        }
+      } catch (e) {
+        errors.push({keystoreFile: path.basename(definition.keystorePath), error: e as Error});
+        // stop processing, no need to continue decrypting after we hit one error
+        break;
+      }
+    }
+  }
+
+  if (errors.length > 0) {
+    // If an error occurs, the program isn't going to be running,
+    // so we should unlock all lockfiles we created
+    for (const {keystorePath} of keystoreDefinitions) {
+      unlockFilepath(keystorePath);
+    }
+
+    throw formattedError(errors, signers, keystoreCount);
+  }
+
+  if (opts.cacheFilePath) {
+    await writeKeystoreCache(opts.cacheFilePath, signers, passwords);
+    opts.logger.debug("Written keystores to keystore cache");
+  }
+
+  return signers;
+}
+
+function lockKeystore(keystorePath: string, opts: KeystoreDecryptOptions): void {
+  try {
+    lockFilepath(keystorePath);
+  } catch (e) {
+    if (opts.ignoreLockFile) {
+      opts.logger.warn("Keystore forcefully loaded even though lockfile exists", {
+        path: keystorePath,
+      });
+    } else {
+      throw e;
+    }
+  }
+}
+
+function formattedError(errors: KeystoreDecryptError[], signers: SignerLocal[], keystoreCount: number): Error {
+  // Filter out errors due to terminating the thread pool
+  // https://github.com/ChainSafe/threads.js/blob/df351552cb7d08b8465f5d1e7c543c952d74ac67/src/master/pool.ts#L244
+  const decryptErrors = errors.filter(({error}) => !error.message.startsWith("Pool has been terminated"));
+
+  const errorCount = decryptErrors.length;
+  const decryptedCount = signers.filter(Boolean).length;
+  const abortedCount = keystoreCount - errorCount - decryptedCount;
+
+  let message = "Error importing keystores";
+
+  if (errorCount === 1) {
+    const {keystoreFile, error} = decryptErrors[0];
+    message = `Error importing keystore\n\n${keystoreFile}: ${error.message}`;
+  } else if (errorCount > 1) {
+    message =
+      "Multiple errors importing keystores\n\n" +
+      decryptErrors.map(({keystoreFile, error}) => `${keystoreFile}: ${error.message}`).join("\n");
+  }
+
+  if (abortedCount > 0) {
+    message += `\n\nAborted ${abortedCount} pending keystore import${abortedCount > 1 ? "s" : ""}`;
+  }
+
+  const error = new Error(message);
+
+  // Don't print out stack trace
+  error.stack = message;
+
+  return error;
+}

package/src/cmds/validator/keymanager/decryptKeystores/index.ts

@@ -0,0 +1 @@
+export {DecryptKeystoresThreadPool} from "./threadPool.js";

package/src/cmds/validator/keymanager/decryptKeystores/poolSize.ts

@@ -0,0 +1,16 @@
+let maxPoolSize: number;
+
+try {
+  if (typeof navigator !== "undefined") {
+    maxPoolSize = navigator.hardwareConcurrency ?? 4;
+  } else {
+    maxPoolSize = (await import("node:os")).availableParallelism();
+  }
+} catch (_e) {
+  maxPoolSize = 8;
+}
+
+/**
+ * Cross-platform approx number of logical cores
+ */
+export {maxPoolSize};

package/src/cmds/validator/keymanager/decryptKeystores/threadPool.ts

@@ -0,0 +1,75 @@
+import path from "node:path";
+import {ModuleThread, Pool, QueuedTask, Worker, spawn} from "@chainsafe/threads";
+import {maxPoolSize} from "./poolSize.js";
+import {DecryptKeystoreArgs, DecryptKeystoreWorkerAPI} from "./types.js";
+
+// Worker constructor consider the path relative to the current working directory
+const workerDir =
+  process.env.NODE_ENV === "test" ? "../../../../../lib/cmds/validator/keymanager/decryptKeystores" : "./";
+
+/**
+ * Thread pool to decrypt keystores
+ */
+export class DecryptKeystoresThreadPool {
+  private pool: Pool<ModuleThread<DecryptKeystoreWorkerAPI>>;
+  private tasks: QueuedTask<ModuleThread<DecryptKeystoreWorkerAPI>, Uint8Array>[] = [];
+  private terminatePoolHandler: () => void;
+
+  constructor(
+    keystoreCount: number,
+    private readonly signal: AbortSignal
+  ) {
+    this.pool = Pool(
+      () =>
+        spawn<DecryptKeystoreWorkerAPI>(new Worker(path.join(workerDir, "worker.js")), {
+          // The number below is big enough to almost disable the timeout
+          // which helps during tests run on unpredictably slow hosts
+          timeout: 5 * 60 * 1000,
+        }),
+      {
+        // Adjust worker pool size based on keystore count
+        size: Math.min(keystoreCount, maxPoolSize),
+        // Decrypt keystores in sequence, increasing concurrency does not improve performance
+        concurrency: 1,
+      }
+    );
+    // Terminate worker threads when process receives exit signal
+    this.terminatePoolHandler = () => {
+      void this.pool.terminate(true);
+    };
+    signal.addEventListener("abort", this.terminatePoolHandler, {once: true});
+  }
+
+  /**
+   * Add keystore to the task queue to be decrypted
+   */
+  queue(
+    args: DecryptKeystoreArgs,
+    onDecrypted: (secretKeyBytes: Uint8Array) => void,
+    onError: (e: Error) => void
+  ): void {
+    const task = this.pool.queue((thread) => thread.decryptKeystore(args));
+    this.tasks.push(task);
+    task.then(onDecrypted).catch(onError);
+  }
+
+  /**
+   * Resolves once all queued tasks are completed and terminates worker threads.
+   * Errors during executing can be captured in `onError` handler for each task.
+   */
+  async completed(): Promise<void> {
+    await this.pool.settled(true);
+    await this.pool.terminate();
+    this.signal.removeEventListener("abort", this.terminatePoolHandler);
+  }
+
+  /**
+   * Cancel all pending tasks
+   */
+  cancel(): void {
+    for (const task of this.tasks) {
+      task.cancel();
+    }
+    this.tasks = [];
+  }
+}

package/src/cmds/validator/keymanager/decryptKeystores/types.ts

@@ -0,0 +1,12 @@
+import {KeystoreStr} from "@lodestar/api/keymanager";
+import {LocalKeystoreDefinition} from "../interface.js";
+
+export type DecryptKeystoreWorkerAPI = {
+  decryptKeystore(args: DecryptKeystoreArgs): Promise<Uint8Array>;
+};
+
+export type DecryptKeystoreArgs = LocalKeystoreDefinition | {keystoreStr: KeystoreStr; password: string};
+
+export function isLocalKeystoreDefinition(args: DecryptKeystoreArgs): args is LocalKeystoreDefinition {
+  return (args as LocalKeystoreDefinition).keystorePath !== undefined;
+}

package/src/cmds/validator/keymanager/decryptKeystores/worker.ts

@@ -0,0 +1,24 @@
+import fs from "node:fs";
+import {Keystore} from "@chainsafe/bls-keystore";
+import {Transfer, TransferDescriptor} from "@chainsafe/threads";
+import {expose} from "@chainsafe/threads/worker";
+import {DecryptKeystoreArgs, DecryptKeystoreWorkerAPI, isLocalKeystoreDefinition} from "./types.js";
+
+/**
+ * Decrypt a single keystore, returning the secret key as a Uint8Array
+ *
+ * NOTE: This is a memory (and cpu) -intensive process, since decrypting the keystore involves running a key derivation function (either pbkdf2 or scrypt)
+ */
+export async function decryptKeystore(args: DecryptKeystoreArgs): Promise<TransferDescriptor<Uint8Array>> {
+  const keystore = Keystore.parse(
+    isLocalKeystoreDefinition(args) ? fs.readFileSync(args.keystorePath, "utf8") : args.keystoreStr
+  );
+
+  // Memory-hogging function
+  const secret = await keystore.decrypt(args.password);
+  // Transfer the underlying ArrayBuffer back to the main thread: https://threads.js.org/usage-advanced#transferable-objects
+  // This small performance gain may help in cases where this is run for many keystores
+  return Transfer(secret, [secret.buffer]);
+}
+
+expose({decryptKeystore} as unknown as DecryptKeystoreWorkerAPI);