@lodestar/beacon-node 1.51.0-dev.b41e3a7363 → 1.51.0-dev.b889ef5768

package/src/chain/blocks/verifyBlocksDataAvailability.ts

@@ -1,3 +1,4 @@
+import {ForkSeq} from "@lodestar/params";
 import {DataAvailabilityStatus} from "@lodestar/state-transition";
 import {TimeoutError} from "@lodestar/utils";
 import {BlockError, BlockErrorCode} from "../errors/index.js";
@@ -42,7 +43,11 @@ export async function verifyBlocksDataAvailability(
       return DataAvailabilityStatus.NotRequired;
     }
     if (blockInput.type === DAType.PreData) {
-      return DataAvailabilityStatus.PreData;
+      // Legacy deneb..electra blob sidecars are not tracked, so these blocks are imported without a DA check.
+      // PreData cannot represent them because the state transition rejects it for post-deneb forks.
+      return ForkSeq[blockInput.forkName] >= ForkSeq.deneb
+        ? DataAvailabilityStatus.NotRequired
+        : DataAvailabilityStatus.PreData;
     }
     if (blockInput.daOutOfRange) {
       return DataAvailabilityStatus.OutOfRange;

package/src/chain/blocks/verifyBlocksExecutionPayloads.ts

@@ -7,21 +7,23 @@ import {
   LVHValidResponse,
   ProtoBlock,
 } from "@lodestar/fork-choice";
-import {ForkSeq} from "@lodestar/params";
+import {ForkPostDeneb, ForkSeq, isForkPostDeneb} from "@lodestar/params";
 import {IBeaconStateView, isExecutionBlockBodyType, isStatePostBellatrix} from "@lodestar/state-transition";
-import {bellatrix, electra} from "@lodestar/types";
+import {SignedBeaconBlock, bellatrix, electra} from "@lodestar/types";
 import {ErrorAborted, Logger, toRootHex} from "@lodestar/utils";
 import {ExecutionPayloadStatus, IExecutionEngine} from "../../execution/engine/interface.js";
 import {Metrics} from "../../metrics/metrics.js";
+import {kzgCommitmentToVersionedHash} from "../../util/blobs.js";
 import {IClock} from "../../util/clock.js";
+import {getBlobKzgCommitments} from "../../util/dataColumns.js";
 import {BlockError, BlockErrorCode} from "../errors/index.js";
 import {BlockProcessOpts} from "../options.js";
-import {isBlockInputBlobs, isBlockInputColumns, isBlockInputNoData} from "./blockInput/blockInput.js";
+import {isBlockInputNoData} from "./blockInput/blockInput.js";
 import {IBlockInput} from "./blockInput/types.js";
 import {ImportBlockOpts} from "./types.js";
 
 export type VerifyBlockExecutionPayloadModules = {
-  executionEngine: IExecutionEngine;
+  executionEngine: Pick<IExecutionEngine, "notifyNewPayload">;
   clock: IClock;
   logger: Logger;
   metrics: Metrics | null;
@@ -166,8 +168,9 @@ export async function verifyBlockExecutionPayload(
 
   // TODO: Handle better notifyNewPayload() returning error is syncing
   const fork = blockInput.forkName;
-  const versionedHashes =
-    isBlockInputBlobs(blockInput) || isBlockInputColumns(blockInput) ? blockInput.getVersionedHashes() : undefined;
+  const versionedHashes = isForkPostDeneb(fork)
+    ? getBlobKzgCommitments(fork, block as SignedBeaconBlock<ForkPostDeneb>).map(kzgCommitmentToVersionedHash)
+    : undefined;
   const parentBlockRoot = ForkSeq[fork] >= ForkSeq.deneb ? block.message.parentRoot : undefined;
   const executionRequests =
     ForkSeq[fork] >= ForkSeq.electra ? (block.message.body as electra.BeaconBlockBody).executionRequests : undefined;

package/src/chain/blocks/verifyBlocksSanityChecks.ts

@@ -1,5 +1,5 @@
 import {ChainForkConfig} from "@lodestar/config";
-import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice";
+import {ExecutionStatus, IForkChoice, ProtoBlock} from "@lodestar/fork-choice";
 import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
 import {RootHex, Slot, isGloasBeaconBlock} from "@lodestar/types";
 import {toRootHex} from "@lodestar/utils";
@@ -118,6 +118,9 @@ export function verifyBlocksSanityChecks(
           parentBlock = parentBlockWithPayload;
         }
       }
+      if (parentBlock.executionStatus === ExecutionStatus.Invalid) {
+        throw new BlockError(block, {code: BlockErrorCode.PARENT_EXECUTION_INVALID, parentRoot});
+      }
       // Parent is known to the fork-choice
       parentBlockSlot = parentBlock.slot;
     }

package/src/chain/blocks/verifyBlocksSignatures.ts

@@ -36,10 +36,9 @@ export async function verifyBlocksSignatures(
     ? preState0.currentSyncCommitteeIndexed
     : new SyncCommitteeCacheEmpty();
 
-  // Verifies signatures after running state transition, so all SyncCommittee signed roots are known at this point.
-  // We must ensure block.slot <= state.slot before running getAllBlockSignatureSets().
-  // NOTE: If in the future multiple blocks signatures are verified at once, all blocks must be in the same epoch
-  // so the attester and proposer shufflings are correct.
+  // Runs concurrently with the state transition. Signature sets are built from preState0, which
+  // verifyBlocksInEpoch() guarantees is in the same epoch as every block, so the proposer and attester shufflings
+  // and the sync committee are the ones the blocks were signed under.
   for (const [i, block] of blocks.entries()) {
     // Use [i] to make clear that the index has to be correct to blame the right block below on BlockError()
     isValidPromises[i] = opts.validSignatures
@@ -50,8 +49,14 @@ export async function verifyBlocksSignatures(
         bls.verifySignatureSets(
           getBlockSignatureSets(config, currentSyncCommitteeIndexed, preState0, block, indexedAttestationsByBlock[i], {
             skipProposerSignature: opts.validProposerSignature,
-          })
+          }),
+          // Prioritize block verification over attestations, since
+          // attestations already buffer up to `MAX_BUFFER_WAIT_MS`
+          {priority: true}
         );
+    // Handlers are attached after the loop, which yields. A rejection in between, e.g. the verifier closing,
+    // must not be unhandled. `rejectFirstInvalidResolveAllValid()` still observes it.
+    isValidPromises[i].catch(() => {});
 
     // getBlockSignatureSets() takes 45ms in benchmarks for 2022Q2 mainnet blocks (100 sigs). When syncing a 32 blocks
     // segments it will block the event loop for 1400 ms, which is too much. This call will allow the event loop to

package/src/chain/blocks/verifyPayloadsDataAvailability.ts

@@ -22,9 +22,7 @@ export async function verifyPayloadsDataAvailability(
   const promises: Promise<gloas.DataColumnSidecar[]>[] = [];
   for (const payloadInput of payloadInputs) {
     if (!payloadInput.hasAllData()) {
-      // Sync and reconstruction keep pulling the data columns from the network while we wait. The wait is
-      // capped at one slot because the payload queue is serial: a longer wait lets consecutive unavailable
-      // payloads push every later payload import further behind with each slot.
+      // Sync and reconstruction keep pulling the data columns from the network while we wait
       promises.push(
         payloadInput.waitForAllData(config.getSlotDurationMs(payloadInput.forkName), signal).catch((e) => {
           if (e instanceof TimeoutError) {

package/src/chain/blocks/writeBlockInputToDb.ts

@@ -3,7 +3,7 @@ import {SignedBeaconBlock} from "@lodestar/types";
 import {toRootHex} from "@lodestar/utils";
 import {getBlobKzgCommitments} from "../../util/dataColumns.js";
 import {BeaconChain} from "../chain.js";
-import {IBlockInput, IDataColumnsInput, isBlockInputBlobs, isBlockInputColumns} from "./blockInput/index.js";
+import {IBlockInput, IDataColumnsInput, isBlockInputColumns} from "./blockInput/index.js";
 import {BLOB_AVAILABILITY_TIMEOUT} from "./verifyBlocksDataAvailability.js";
 
 /**
@@ -12,11 +12,9 @@ import {BLOB_AVAILABILITY_TIMEOUT} from "./verifyBlocksDataAvailability.js";
  *
  * This operation may be performed before, during or after importing to the fork-choice. As long as errors
  * are handled properly for eventual consistency.
- *
- * Block+blobs (pre-fulu) and data columns (fulu+) are written in parallel.
  */
 export async function writeBlockInputToDb(this: BeaconChain, blockInput: IBlockInput): Promise<void> {
-  const promises: Promise<void>[] = [writeBlockAndBlobsToDb.call(this, blockInput)];
+  const promises: Promise<void>[] = [writeBlockToDb.call(this, blockInput)];
 
   if (isBlockInputColumns(blockInput)) {
     promises.push(writeDataColumnsToDb.call(this, blockInput));
@@ -26,7 +24,7 @@ export async function writeBlockInputToDb(this: BeaconChain, blockInput: IBlockI
   this.logger.debug("Persisted blockInput to db", {slot: blockInput.slot, root: blockInput.blockRootHex});
 }
 
-async function writeBlockAndBlobsToDb(this: BeaconChain, blockInput: IBlockInput): Promise<void> {
+async function writeBlockToDb(this: BeaconChain, blockInput: IBlockInput): Promise<void> {
   const block = blockInput.getBlock();
   const slot = block.message.slot;
   const blockRoot = this.config.getForkTypes(slot).BeaconBlock.hashTreeRoot(block.message);
@@ -34,38 +32,17 @@ async function writeBlockAndBlobsToDb(this: BeaconChain, blockInput: IBlockInput
   const numBlobs = isForkPostDeneb(blockInput.forkName)
     ? getBlobKzgCommitments(blockInput.forkName, block as SignedBeaconBlock<ForkPostDeneb>).length
     : undefined;
-  const fnPromises: Promise<void>[] = [];
+  this.logger.debug("Persist block to hot DB", {slot, root: blockRootHex, inputType: blockInput.type, numBlobs});
 
   const blockBytes = this.serializedCache.get(block);
   if (blockBytes) {
     // skip serializing data if we already have it
     this.metrics?.importBlock.persistBlockWithSerializedDataCount.inc();
-    fnPromises.push(this.db.block.putBinary(this.db.block.getId(block), blockBytes));
+    await this.db.block.putBinary(this.db.block.getId(block), blockBytes);
   } else {
     this.metrics?.importBlock.persistBlockNoSerializedDataCount.inc();
-    fnPromises.push(this.db.block.add(block));
+    await this.db.block.add(block);
   }
-
-  this.logger.debug("Persist block to hot DB", {slot, root: blockRootHex, inputType: blockInput.type, numBlobs});
-
-  if (isBlockInputBlobs(blockInput)) {
-    fnPromises.push(
-      (async () => {
-        if (!blockInput.hasAllData()) {
-          await blockInput.waitForAllData(BLOB_AVAILABILITY_TIMEOUT);
-        }
-        const blobSidecars = blockInput.getBlobs();
-        await this.db.blobSidecars.add({blockRoot, slot, blobSidecars});
-        this.logger.debug("Persisted blobSidecars to hot DB", {
-          slot,
-          root: blockRootHex,
-          numBlobs: blobSidecars.length,
-        });
-      })()
-    );
-  }
-
-  await Promise.all(fnPromises);
 }
 
 /**

package/src/chain/bls/interface.ts

@@ -22,6 +22,8 @@ export type VerifySignatureOpts = {
   verifyOnMainThread?: boolean;
   /**
    * Some signature sets are more important than others, and should be verified first.
+   * Non-batchable priority jobs are posted to an idle worker before this call returns, so the caller
+   * can run synchronous work right after without delaying the verification.
    */
   priority?: boolean;
 };

package/src/chain/bls/multithread/index.ts

@@ -7,7 +7,7 @@ import {Worker, spawn} from "@chainsafe/threads";
 self = undefined;
 
 import {ISignatureSet} from "@lodestar/state-transition";
-import {Logger} from "@lodestar/utils";
+import {Logger, defer} from "@lodestar/utils";
 import {Metrics} from "../../../metrics/index.js";
 import {LinkedList} from "../../../util/array.js";
 import {callInNextEventLoop} from "../../../util/eventLoop.js";
@@ -94,7 +94,7 @@ type WorkerStatus =
   | {code: WorkerStatusCode.initializing; initPromise: Promise<WorkerApi>}
   | {code: WorkerStatusCode.initializationError; error: Error}
   | {code: WorkerStatusCode.idle; workerApi: WorkerApi}
-  | {code: WorkerStatusCode.running; workerApi: WorkerApi};
+  | {code: WorkerStatusCode.running; workerApi: WorkerApi; abort: () => void};
 
 type WorkerDescriptor = {
   worker: Worker;
@@ -126,7 +126,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier {
     timeout: NodeJS.Timeout;
   } | null = null;
   private blsVerifyAllMultiThread: boolean;
-  private closed = false;
+  private closePromise: Promise<void> | undefined;
   private workersBusy = 0;
 
   constructor(options: BlsMultiThreadWorkerPoolOptions, modules: BlsMultiThreadWorkerPoolModules) {
@@ -149,6 +149,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier {
 
   canAcceptWork(): boolean {
     return (
+      this.closePromise === undefined &&
       this.workersBusy < blsPoolSize &&
       // TODO: Should also bound the jobs queue?
       this.jobs.length < MAX_JOBS_CAN_ACCEPT_WORK
@@ -156,6 +157,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier {
   }
 
   async verifySignatureSets(sets: ISignatureSet[], opts: VerifySignatureOpts = {}): Promise<boolean> {
+    if (this.closePromise !== undefined) throw new QueueError({code: QueueErrorCode.QUEUE_ABORTED});
     this.metrics?.bls.aggregatedPubkeys.inc(getAggregatedPubkeysCount(sets));
     this.metrics?.blsThreadPool.totalSigSets.inc(sets.length);
     if (opts.priority) {
@@ -208,6 +210,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier {
     message: Uint8Array,
     opts: Omit<VerifySignatureOpts, "verifyOnMainThread"> = {}
   ): Promise<boolean[]> {
+    if (this.closePromise !== undefined) throw new QueueError({code: QueueErrorCode.QUEUE_ABORTED});
     const promises: Promise<boolean[]>[] = [];
     for (const setsChunk of chunkSameMessageSignatureSets(sets)) {
       promises.push(
@@ -229,9 +232,14 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier {
     return results.flat();
   }
 
-  async close(): Promise<void> {
+  close(): Promise<void> {
+    if (this.closePromise !== undefined) return this.closePromise;
     if (this.bufferedJobs) {
       clearTimeout(this.bufferedJobs.timeout);
+      for (const jobs of [this.bufferedJobs.jobs, this.bufferedJobs.prioritizedJobs]) {
+        for (const job of jobs) job.reject(new QueueError({code: QueueErrorCode.QUEUE_ABORTED}));
+      }
+      this.bufferedJobs = null;
     }
 
     // Abort all jobs
@@ -241,15 +249,21 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier {
     this.jobs.clear();
 
     // Terminate all workers. await to ensure no workers are left hanging
-    await Promise.all(
+    this.closePromise = Promise.all(
       Array.from(this.workers.entries()).map(([id, worker]) =>
         // NOTE: 'threads' has not yet updated types, and NodeJS complains with
         // [DEP0132] DeprecationWarning: Passing a callback to worker.terminate() is deprecated. It returns a Promise instead.
-        (worker.worker.terminate() as unknown as Promise<void>).catch((e: Error) => {
-          this.logger.error("Error terminating worker", {id}, e);
-        })
+        (worker.worker.terminate() as unknown as Promise<void>)
+          .catch((e: Error) => {
+            this.logger.error("Error terminating worker", {id}, e);
+          })
+          .finally(() => {
+            // threads does not settle outstanding RPC calls when its worker exits.
+            if (worker.status.code === WorkerStatusCode.running) worker.status.abort();
+          })
       )
-    );
+    ).then(() => {});
+    return this.closePromise;
   }
 
   private createWorkers(poolSize: number): WorkerDescriptor[] {
@@ -295,7 +309,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier {
    * Register BLS work to be done eventually in a worker
    */
   private queueBlsWork(job: JobQueueItem): void {
-    if (this.closed) {
+    if (this.closePromise !== undefined) {
       throw new QueueError({code: QueueErrorCode.QUEUE_ABORTED});
     }
 
@@ -333,16 +347,20 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier {
       }
     }
 
+    // First hand work to an idle worker before returning to the caller.
+    // Deferring the post to the next macrotask would make the worker
+    // sit idle while waiting for the next synchronous work
+    else if (job.opts.priority) {
+      this.jobs.unshift(job);
+      void this.runJob();
+    }
+
     // Push job and schedule to call `runJob` in the next macro event loop cycle.
     // This is useful to allow batching job submitted from a synchronous for loop,
     // and to prevent large stacks since runJob may be called recursively.
     else {
-      if (job.opts.priority) {
-        this.jobs.unshift(job);
-      } else {
-        this.jobs.push(job);
-      }
-      callInNextEventLoop(this.runJob);
+      this.jobs.push(job);
+      callInNextEventLoop(() => void this.runJob());
     }
   }
 
@@ -350,7 +368,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier {
    * Potentially submit jobs to an idle worker, only if there's a worker and jobs
    */
   private runJob = async (): Promise<void> => {
-    if (this.closed) {
+    if (this.closePromise !== undefined) {
       return;
     }
 
@@ -371,13 +389,18 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier {
     // Maybe it's not useful since all data referenced in jobs is likely referenced by others
 
     const workerApi = worker.status.workerApi;
-    worker.status = {code: WorkerStatusCode.running, workerApi};
+    const completion = defer<BlsWorkResult>();
+    worker.status = {
+      code: WorkerStatusCode.running,
+      workerApi,
+      abort: () => completion.reject(new QueueError({code: QueueErrorCode.QUEUE_ABORTED})),
+    };
     this.workersBusy++;
     const jobsStarted: JobQueueItem[] = [];
     let completedJobsCount = 0;
 
     if (this.jobs.length > 0) {
-      callInNextEventLoop(this.runJob);
+      callInNextEventLoop(() => void this.runJob());
     }
 
     try {
@@ -423,7 +446,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier {
       if (workReqs.length === 0) {
         worker.status = {code: WorkerStatusCode.idle, workerApi};
         this.workersBusy--;
-        callInNextEventLoop(this.runJob);
+        callInNextEventLoop(() => void this.runJob());
         return;
       }
 
@@ -437,7 +460,8 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier {
       // If worker communication or result processing throws, all unresolved jobs are rejected.
 
       const [jobStartSec, jobStartNs] = process.hrtime();
-      const workResult = await workerApi.verifyManySignatureSets(workReqs);
+      void workerApi.verifyManySignatureSets(workReqs).then(completion.resolve, completion.reject);
+      const workResult = await completion.promise;
       const [jobEndSec, jobEndNs] = process.hrtime();
       const {workerId, batchRetries, batchSigsSuccess, verificationCalls, workerStartTime, workerEndTime, results} =
         workResult;
@@ -481,7 +505,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier {
       }
     } catch (e) {
       // Worker communications should never reject
-      if (!this.closed) {
+      if (this.closePromise === undefined) {
         this.logger.error("BlsMultiThreadWorkerPool error", {}, e as Error);
       }
       // Reject jobs that were not already completed
@@ -499,7 +523,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier {
     this.workersBusy--;
 
     // Potentially run a new job
-    callInNextEventLoop(this.runJob);
+    callInNextEventLoop(() => void this.runJob());
   };
 
   /**
@@ -542,7 +566,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier {
         this.jobs.unshift(job);
       }
       this.bufferedJobs = null;
-      callInNextEventLoop(this.runJob);
+      callInNextEventLoop(() => void this.runJob());
     }
   };
 

package/src/chain/bls/multithread/types.ts

@@ -1,4 +1,4 @@
-import {type BlsSignatureSet} from "@chainsafe/lodestar-z/bls-verifier";
+import {type BlsSignatureSet} from "@lodestar/native/bls-verifier";
 import {SameMessageSignatureSet, VerifySignatureOpts} from "../interface.js";
 
 export type WorkerData = {

package/src/chain/bls/multithread/worker.ts

@@ -1,10 +1,6 @@
 import worker from "node:worker_threads";
-import {
-  type BlsSignatureSet,
-  verifySignatureSets,
-  verifySignatureSetsSameMessage,
-} from "@chainsafe/lodestar-z/bls-verifier";
 import {expose} from "@chainsafe/threads/worker";
+import {type BlsSignatureSet, verifySignatureSets, verifySignatureSetsSameMessage} from "@lodestar/native/bls-verifier";
 import {
   BlsWorkReq,
   BlsWorkResult,

package/src/chain/bls/singleThread.ts

@@ -1,4 +1,4 @@
-import {verifySignatureSetsSameMessage} from "@chainsafe/lodestar-z/bls-verifier";
+import {verifySignatureSetsSameMessage} from "@lodestar/native/bls-verifier";
 import {ISignatureSet} from "@lodestar/state-transition";
 import {Metrics} from "../../metrics/index.js";
 import {IBlsVerifier, SameMessageSignatureSet} from "./interface.js";

package/src/chain/bls/utils.ts

@@ -2,7 +2,7 @@ import {
   BLS_VERIFIER_MAX_BATCH_SIZE,
   BLS_VERIFIER_MAX_SAME_MESSAGE_BATCH_SIZE,
   verifySignatureSets,
-} from "@chainsafe/lodestar-z/bls-verifier";
+} from "@lodestar/native/bls-verifier";
 import {ISignatureSet, SignatureSetType, toBlsSignatureSet} from "@lodestar/state-transition";
 import {SameMessageSignatureSet} from "./interface.js";