@lodestar/beacon-node 1.35.0-dev.f80d2d52da → 1.35.0-dev.fd1dac853d

package/src/chain/bls/multithread/poolSize.ts

@@ -0,0 +1,16 @@
+let defaultPoolSize: number;
+
+try {
+  if (typeof navigator !== "undefined") {
+    defaultPoolSize = navigator.hardwareConcurrency ?? 4;
+  } else {
+    defaultPoolSize = (await import("node:os")).availableParallelism();
+  }
+} catch (_e) {
+  defaultPoolSize = 8;
+}
+
+/**
+ * Cross-platform aprox number of logical cores
+ */
+export {defaultPoolSize};

package/src/chain/bls/multithread/types.ts

@@ -0,0 +1,38 @@
+import {VerifySignatureOpts} from "../interface.js";
+
+export type WorkerData = {
+  workerId: number;
+};
+
+export type SerializedSet = {
+  publicKey: Uint8Array;
+  message: Uint8Array;
+  signature: Uint8Array;
+};
+
+export type BlsWorkReq = {
+  opts: VerifySignatureOpts;
+  sets: SerializedSet[];
+};
+
+export enum WorkResultCode {
+  success = "success",
+  error = "error",
+}
+
+export type WorkResultError = {code: WorkResultCode.error; error: Error};
+export type WorkResult<R> = {code: WorkResultCode.success; result: R} | WorkResultError;
+
+export type BlsWorkResult = {
+  /** Ascending integer identifying the worker for metrics */
+  workerId: number;
+  /** Total num of batches that had to be retried */
+  batchRetries: number;
+  /** Total num of sigs that have been successfully verified with batching */
+  batchSigsSuccess: number;
+  /** Time worker function starts - UNIX timestamp in seconds and nanoseconds */
+  workerStartTime: [number, number];
+  /** Time worker function ends - UNIX timestamp in seconds and nanoseconds */
+  workerEndTime: [number, number];
+  results: WorkResult<boolean>[];
+};

package/src/chain/bls/multithread/utils.ts

@@ -0,0 +1,19 @@
+/**
+ * Splits an array into an array of arrays maximizing the size of the smallest chunk.
+ */
+export function chunkifyMaximizeChunkSize<T>(arr: T[], minPerChunk: number): T[][] {
+  const chunkCount = Math.floor(arr.length / minPerChunk);
+  if (chunkCount <= 1) {
+    return [arr];
+  }
+
+  // Prefer less chunks of bigger size
+  const perChunk = Math.ceil(arr.length / chunkCount);
+  const arrArr: T[][] = [];
+
+  for (let i = 0; i < arr.length; i += perChunk) {
+    arrArr.push(arr.slice(i, i + perChunk));
+  }
+
+  return arrArr;
+}

package/src/chain/bls/multithread/worker.ts

@@ -0,0 +1,114 @@
+import worker from "node:worker_threads";
+import {PublicKey} from "@chainsafe/blst";
+import {expose} from "@chainsafe/threads/worker";
+import {SignatureSetDeserialized, verifySignatureSetsMaybeBatch} from "../maybeBatch.js";
+import {BlsWorkReq, BlsWorkResult, SerializedSet, WorkResult, WorkResultCode, WorkerData} from "./types.js";
+import {chunkifyMaximizeChunkSize} from "./utils.js";
+
+/**
+ * Split batchable sets in chunks of minimum size 16.
+ * Batch verify 16 has an aprox cost of 16+1. For 32 it's 32+1. After ~16 the additional savings are not significant.
+ * However, if a sig is invalid the whole batch has to be re-verified. So it's important to keep this number low.
+ * In normal network conditions almost all signatures received by the node are correct.
+ * After observing metrics this number can be reviewed
+ */
+const BATCHABLE_MIN_PER_CHUNK = 16;
+
+// Cloned data from instatiation
+const workerData = worker.workerData as WorkerData;
+if (!workerData) throw Error("workerData must be defined");
+const {workerId} = workerData || {};
+
+expose({
+  async verifyManySignatureSets(workReqArr: BlsWorkReq[]): Promise<BlsWorkResult> {
+    return verifyManySignatureSets(workReqArr);
+  },
+});
+
+function verifyManySignatureSets(workReqArr: BlsWorkReq[]): BlsWorkResult {
+  const [startSec, startNs] = process.hrtime();
+  const results: WorkResult<boolean>[] = [];
+  let batchRetries = 0;
+  let batchSigsSuccess = 0;
+
+  // If there are multiple batchable sets attempt batch verification with them
+  const batchableSets: {idx: number; sets: SignatureSetDeserialized[]}[] = [];
+  const nonBatchableSets: {idx: number; sets: SignatureSetDeserialized[]}[] = [];
+
+  // Split sets between batchable and non-batchable preserving their original index in the req array
+  for (let i = 0; i < workReqArr.length; i++) {
+    const workReq = workReqArr[i];
+    const sets = workReq.sets.map(deserializeSet);
+
+    if (workReq.opts.batchable) {
+      batchableSets.push({idx: i, sets});
+    } else {
+      nonBatchableSets.push({idx: i, sets});
+    }
+  }
+
+  if (batchableSets.length > 0) {
+    // Split batchable into chunks of max size ~ 32 to minimize cost if a sig is wrong
+    const batchableChunks = chunkifyMaximizeChunkSize(batchableSets, BATCHABLE_MIN_PER_CHUNK);
+
+    for (const batchableChunk of batchableChunks) {
+      const allSets: SignatureSetDeserialized[] = [];
+      for (const {sets} of batchableChunk) {
+        for (const set of sets) {
+          allSets.push(set);
+        }
+      }
+
+      try {
+        // Attempt to verify multiple sets at once
+        const isValid = verifySignatureSetsMaybeBatch(allSets);
+
+        if (isValid) {
+          // The entire batch is valid, return success to all
+          for (const {idx, sets} of batchableChunk) {
+            batchSigsSuccess += sets.length;
+            results[idx] = {code: WorkResultCode.success, result: isValid};
+          }
+        } else {
+          batchRetries++;
+          // Re-verify all sigs
+          nonBatchableSets.push(...batchableChunk);
+        }
+      } catch (_e) {
+        // TODO: Ignore this error expecting that the same error will happen when re-verifying the set individually
+        //       It's not ideal but '@chainsafe/blst' may throw errors on some conditions
+        batchRetries++;
+        // Re-verify all sigs
+        nonBatchableSets.push(...batchableChunk);
+      }
+    }
+  }
+
+  for (const {idx, sets} of nonBatchableSets) {
+    try {
+      const isValid = verifySignatureSetsMaybeBatch(sets);
+      results[idx] = {code: WorkResultCode.success, result: isValid};
+    } catch (e) {
+      results[idx] = {code: WorkResultCode.error, error: e as Error};
+    }
+  }
+
+  const [workerEndSec, workerEndNs] = process.hrtime();
+
+  return {
+    workerId,
+    batchRetries,
+    batchSigsSuccess,
+    workerStartTime: [startSec, startNs],
+    workerEndTime: [workerEndSec, workerEndNs],
+    results,
+  };
+}
+
+function deserializeSet(set: SerializedSet): SignatureSetDeserialized {
+  return {
+    publicKey: PublicKey.fromBytes(set.publicKey),
+    message: set.message,
+    signature: set.signature,
+  };
+}

package/src/chain/bls/singleThread.ts

@@ -0,0 +1,87 @@
+import {PublicKey, Signature, aggregatePublicKeys, aggregateSignatures, verify} from "@chainsafe/blst";
+import {ISignatureSet} from "@lodestar/state-transition";
+import {Metrics} from "../../metrics/index.js";
+import {IBlsVerifier} from "./interface.js";
+import {verifySignatureSetsMaybeBatch} from "./maybeBatch.js";
+import {getAggregatedPubkey, getAggregatedPubkeysCount} from "./utils.js";
+
+export class BlsSingleThreadVerifier implements IBlsVerifier {
+  private readonly metrics: Metrics | null;
+
+  constructor({metrics = null}: {metrics: Metrics | null}) {
+    this.metrics = metrics;
+  }
+
+  async verifySignatureSets(sets: ISignatureSet[]): Promise<boolean> {
+    this.metrics?.bls.aggregatedPubkeys.inc(getAggregatedPubkeysCount(sets));
+
+    const setsAggregated = sets.map((set) => ({
+      publicKey: getAggregatedPubkey(set),
+      message: set.signingRoot,
+      signature: set.signature,
+    }));
+
+    // Count time after aggregating
+    const timer = this.metrics?.blsThreadPool.mainThreadDurationInThreadPool.startTimer();
+    const isValid = verifySignatureSetsMaybeBatch(setsAggregated);
+
+    // Don't use a try/catch, only count run without exceptions
+    if (timer) {
+      timer();
+    }
+
+    return isValid;
+  }
+
+  async verifySignatureSetsSameMessage(
+    sets: {publicKey: PublicKey; signature: Uint8Array}[],
+    message: Uint8Array
+  ): Promise<boolean[]> {
+    const timer = this.metrics?.blsThreadPool.mainThreadDurationInThreadPool.startTimer();
+    const pubkey = aggregatePublicKeys(sets.map((set) => set.publicKey));
+    let isAllValid = true;
+    // validate signature = true
+    const signatures = sets.map((set) => {
+      try {
+        return Signature.fromBytes(set.signature, true);
+      } catch (_) {
+        // at least one set has malformed signature
+        isAllValid = false;
+        return null;
+      }
+    });
+
+    if (isAllValid) {
+      const signature = aggregateSignatures(signatures as Signature[]);
+      isAllValid = verify(message, pubkey, signature);
+    }
+
+    let result: boolean[];
+    if (isAllValid) {
+      result = sets.map(() => true);
+    } else {
+      result = sets.map((set, i) => {
+        const sig = signatures[i];
+        if (sig === null) {
+          return false;
+        }
+        return verify(message, set.publicKey, sig);
+      });
+    }
+
+    if (timer) {
+      timer();
+    }
+
+    return result;
+  }
+
+  async close(): Promise<void> {
+    // nothing to do
+  }
+
+  canAcceptWork(): boolean {
+    // Since sigs are verified blocking the main thread, there's no mechanism to throttle
+    return true;
+  }
+}

package/src/chain/bls/utils.ts

@@ -0,0 +1,30 @@
+import {PublicKey, aggregatePublicKeys} from "@chainsafe/blst";
+import {ISignatureSet, SignatureSetType} from "@lodestar/state-transition";
+import {Metrics} from "../../metrics/metrics.js";
+
+export function getAggregatedPubkey(signatureSet: ISignatureSet, metrics: Metrics | null = null): PublicKey {
+  switch (signatureSet.type) {
+    case SignatureSetType.single:
+      return signatureSet.pubkey;
+
+    case SignatureSetType.aggregate: {
+      const timer = metrics?.blsThreadPool.pubkeysAggregationMainThreadDuration.startTimer();
+      const pubkeys = aggregatePublicKeys(signatureSet.pubkeys);
+      timer?.();
+      return pubkeys;
+    }
+
+    default:
+      throw Error("Unknown signature set type");
+  }
+}
+
+export function getAggregatedPubkeysCount(signatureSets: ISignatureSet[]): number {
+  let pubkeysConut = 0;
+  for (const set of signatureSets) {
+    if (set.type === SignatureSetType.aggregate) {
+      pubkeysConut += set.pubkeys.length;
+    }
+  }
+  return pubkeysConut;
+}