@lodestar/beacon-node 1.40.0-dev.9e8478fc70 → 1.40.0-dev.ade3accfa0

package/src/api/impl/beacon/blocks/index.ts

@@ -133,21 +133,31 @@ export function getBeaconBlockApi({
 
     if (isBlockInputColumns(blockForImport)) {
       for (const dataColumnSidecar of dataColumnSidecars) {
-        blockForImport.addColumn({
-          blockRootHex: blockRoot,
-          columnSidecar: dataColumnSidecar,
-          source: BlockInputSource.api,
-          seenTimestampSec,
-        });
+        blockForImport.addColumn(
+          {
+            blockRootHex: blockRoot,
+            columnSidecar: dataColumnSidecar,
+            source: BlockInputSource.api,
+            seenTimestampSec,
+          },
+          // In multi-BN setups (DVT, fallback), the same block may be published to multiple nodes.
+          // Data columns may arrive via gossip from another node before the API publish completes,
+          // so we allow duplicates here instead of throwing an error.
+          {throwOnDuplicateAdd: false}
+        );
       }
     } else if (isBlockInputBlobs(blockForImport)) {
       for (const blobSidecar of blobSidecars) {
-        blockForImport.addBlob({
-          blockRootHex: blockRoot,
-          blobSidecar,
-          source: BlockInputSource.api,
-          seenTimestampSec,
-        });
+        blockForImport.addBlob(
+          {
+            blockRootHex: blockRoot,
+            blobSidecar,
+            source: BlockInputSource.api,
+            seenTimestampSec,
+          },
+          // Same as above for columns
+          {throwOnDuplicateAdd: false}
+        );
       }
     }
 
@@ -434,11 +444,13 @@ export function getBeaconBlockApi({
         const nonFinalizedBlocks = chain.forkChoice.getBlockSummariesByParentRoot(parentRoot);
         await Promise.all(
           nonFinalizedBlocks.map(async (summary) => {
-            const block = await db.block.get(fromHex(summary.blockRoot));
-            if (block) {
-              const canonical = chain.forkChoice.getCanonicalBlockAtSlot(block.message.slot);
+            const blockResult = await chain.getBlockByRoot(summary.blockRoot);
+            if (blockResult) {
+              const canonical = chain.forkChoice.getCanonicalBlockAtSlot(blockResult.block.message.slot);
               if (canonical) {
-                result.push(toBeaconHeaderResponse(config, block, canonical.blockRoot === summary.blockRoot));
+                result.push(
+                  toBeaconHeaderResponse(config, blockResult.block, canonical.blockRoot === summary.blockRoot)
+                );
                 if (isOptimisticBlock(canonical)) {
                   executionOptimistic = true;
                 }
@@ -492,9 +504,9 @@ export function getBeaconBlockApi({
             finalized = false;
 
             if (summary.blockRoot !== toRootHex(canonicalRoot)) {
-              const block = await db.block.get(fromHex(summary.blockRoot));
-              if (block) {
-                result.push(toBeaconHeaderResponse(config, block));
+              const blockResult = await chain.getBlockByRoot(summary.blockRoot);
+              if (blockResult) {
+                result.push(toBeaconHeaderResponse(config, blockResult.block));
               }
             }
           })

package/src/api/impl/lodestar/index.ts

@@ -154,6 +154,23 @@ export function getLodestarApi({
       await network.disconnectPeer(peerId);
     },
 
+    async addDirectPeer({peer}) {
+      const peerId = await network.addDirectPeer(peer);
+      if (peerId === null) {
+        throw new ApiError(400, `Failed to add direct peer: invalid peer address or ENR "${peer}"`);
+      }
+      return {data: {peerId}};
+    },
+
+    async removeDirectPeer({peerId}) {
+      const removed = await network.removeDirectPeer(peerId);
+      return {data: {removed}};
+    },
+
+    async getDirectPeers() {
+      return {data: await network.getDirectPeers()};
+    },
+
     async getPeers({state, direction}) {
       const peers = (await network.dumpPeers()).filter(
         (nodePeer) =>
@@ -244,6 +261,18 @@ export function getLodestarApi({
         data: chain.validatorMonitor?.getMonitoredValidatorIndices() ?? [],
       };
     },
+
+    async getCustodyInfo() {
+      const {custodyColumns, targetCustodyGroupCount} = chain.custodyConfig;
+
+      return {
+        data: {
+          earliestCustodiedSlot: chain.earliestAvailableSlot,
+          custodyGroupCount: targetCustodyGroupCount,
+          custodyColumns,
+        },
+      };
+    },
   };
 }
 

package/src/api/impl/validator/index.ts

@@ -67,10 +67,11 @@ import {
   SyncCommitteeError,
   SyncCommitteeErrorCode,
 } from "../../../chain/errors/index.js";
-import {ChainEvent, CheckpointHex, CommonBlockBody} from "../../../chain/index.js";
+import {ChainEvent, CommonBlockBody} from "../../../chain/index.js";
 import {PREPARE_NEXT_SLOT_BPS} from "../../../chain/prepareNextSlot.js";
 import {BlockType, ProduceFullDeneb} from "../../../chain/produceBlock/index.js";
 import {RegenCaller} from "../../../chain/regen/index.js";
+import {CheckpointHex} from "../../../chain/stateCache/types.js";
 import {validateApiAggregateAndProof} from "../../../chain/validation/index.js";
 import {validateSyncCommitteeGossipContributionAndProof} from "../../../chain/validation/syncCommitteeContributionAndProof.js";
 import {ZERO_HASH} from "../../../constants/index.js";

package/src/api/rest/base.ts

@@ -1,6 +1,6 @@
 import bearerAuthPlugin from "@fastify/bearer-auth";
 import {fastifyCors} from "@fastify/cors";
-import {FastifyInstance, FastifyRequest, errorCodes, fastify} from "fastify";
+import {FastifyError, FastifyInstance, FastifyRequest, errorCodes, fastify} from "fastify";
 import {parse as parseQueryString} from "qs";
 import {addSszContentTypeParser} from "@lodestar/api/server";
 import {ErrorAborted, Gauge, Histogram, Logger} from "@lodestar/utils";
@@ -73,15 +73,17 @@ export class RestApiServer {
     const server = fastify({
       logger: false,
       ajv: {customOptions: {coerceTypes: "array"}},
-      querystringParser: (str) =>
-        parseQueryString(str, {
-          // Array as comma-separated values must be supported to be OpenAPI spec compliant
-          comma: true,
-          // Drop support for array query strings like `id[0]=1&id[1]=2&id[2]=3` as those are not required to
-          // be OpenAPI spec compliant and results are inconsistent, see https://github.com/ljharb/qs/issues/331.
-          // The schema validation will catch this and throw an error as parsed query string results in an object.
-          parseArrays: false,
-        }),
+      routerOptions: {
+        querystringParser: (str) =>
+          parseQueryString(str, {
+            // Array as comma-separated values must be supported to be OpenAPI spec compliant
+            comma: true,
+            // Drop support for array query strings like `id[0]=1&id[1]=2&id[2]=3` as those are not required to
+            // be OpenAPI spec compliant and results are inconsistent, see https://github.com/ljharb/qs/issues/331.
+            // The schema validation will catch this and throw an error as parsed query string results in an object.
+            parseArrays: false,
+          }),
+      },
       bodyLimit: opts.bodyLimit,
       http: {maxHeaderSize: opts.headerLimit},
     });
@@ -91,10 +93,10 @@ export class RestApiServer {
     this.activeSockets = new HttpActiveSocketsTracker(server.server, metrics);
 
     // To parse our ApiError -> statusCode
-    server.setErrorHandler((err, _req, res) => {
+    server.setErrorHandler<FastifyError | Error>((err, _req, res) => {
       const stacktraces = opts.stacktraces ? err.stack?.split("\n") : undefined;
-      if (err.validation) {
-        const {instancePath, message} = err.validation[0];
+      if ("validation" in err && err.validation) {
+        const {instancePath = "unknown", message} = err.validation?.[0] ?? {};
         const payload: ErrorResponse = {
           code: 400,
           message: `${instancePath.substring(instancePath.lastIndexOf("/") + 1)} ${message}`,

package/src/chain/ColumnReconstructionTracker.ts

@@ -8,12 +8,12 @@ import {ChainEventEmitter} from "./emitter.js";
 /**
  * Minimum time to wait before attempting reconstruction
  */
-const RECONSTRUCTION_DELAY_MIN_MS = 800;
+const RECONSTRUCTION_DELAY_MIN_BPS = 667;
 
 /**
  * Maximum time to wait before attempting reconstruction
  */
-const RECONSTRUCTION_DELAY_MAX_MS = 1200;
+const RECONSTRUCTION_DELAY_MAX_BPS = 1000;
 
 export type ColumnReconstructionTrackerInit = {
   logger: Logger;
@@ -41,11 +41,16 @@ export class ColumnReconstructionTracker {
   /** Track if a reconstruction attempt is in-flight */
   running = false;
 
+  private readonly minDelayMs: number;
+  private readonly maxDelayMs: number;
+
   constructor(init: ColumnReconstructionTrackerInit) {
     this.logger = init.logger;
     this.emitter = init.emitter;
     this.metrics = init.metrics;
     this.config = init.config;
+    this.minDelayMs = this.config.getSlotComponentDurationMs(RECONSTRUCTION_DELAY_MIN_BPS);
+    this.maxDelayMs = this.config.getSlotComponentDurationMs(RECONSTRUCTION_DELAY_MAX_BPS);
   }
 
   triggerColumnReconstruction(blockInput: BlockInputColumns): void {
@@ -61,8 +66,7 @@ export class ColumnReconstructionTracker {
     // just that it has been triggered for this block root.
     this.running = true;
     this.lastBlockRootHex = blockInput.blockRootHex;
-    const delay =
-      RECONSTRUCTION_DELAY_MIN_MS + Math.random() * (RECONSTRUCTION_DELAY_MAX_MS - RECONSTRUCTION_DELAY_MIN_MS);
+    const delay = this.minDelayMs + Math.random() * (this.maxDelayMs - this.minDelayMs);
     sleep(delay)
       .then(() => {
         const logCtx = {slot: blockInput.slot, root: blockInput.blockRootHex};

package/src/chain/archiveStore/archiveStore.ts

@@ -5,7 +5,7 @@ import {callFnWhenAwait} from "@lodestar/utils";
 import {IBeaconDb} from "../../db/index.js";
 import {Metrics} from "../../metrics/metrics.js";
 import {isOptimisticBlock} from "../../util/forkChoice.js";
-import {JobItemQueue} from "../../util/queue/index.js";
+import {JobItemQueue, isQueueErrorAborted} from "../../util/queue/index.js";
 import {ChainEvent} from "../emitter.js";
 import {IBeaconChain} from "../interface.js";
 import {PROCESS_FINALIZED_CHECKPOINT_QUEUE_LENGTH} from "./constants.js";
@@ -165,8 +165,12 @@ export class ArchiveStore {
   //-------------------------------------------------------------------------
   // Event handlers
   //-------------------------------------------------------------------------
-  private onFinalizedCheckpoint = async (finalized: CheckpointWithHex): Promise<void> => {
-    return this.jobQueue.push(finalized);
+  private onFinalizedCheckpoint = (finalized: CheckpointWithHex): void => {
+    this.jobQueue.push(finalized).catch((e) => {
+      if (!isQueueErrorAborted(e)) {
+        this.logger.error("Error queuing finalized checkpoint", {epoch: finalized.epoch}, e as Error);
+      }
+    });
   };
 
   private onCheckpoint = (): void => {
@@ -243,7 +247,9 @@ export class ArchiveStore {
         prunedBlocks: prunedBlocks.length,
       });
     } catch (e) {
-      this.logger.error("Error processing finalized checkpoint", {epoch: finalized.epoch}, e as Error);
+      if (!this.signal.aborted) {
+        this.logger.error("Error processing finalized checkpoint", {epoch: finalized.epoch}, e as Error);
+      }
     }
   };
 }

package/src/chain/archiveStore/historicalState/getHistoricalState.ts

@@ -8,6 +8,7 @@ import {
   createCachedBeaconState,
   stateTransition,
 } from "@lodestar/state-transition";
+import {byteArrayEquals} from "@lodestar/utils";
 import {IBeaconDb} from "../../../db/index.js";
 import {getStateTypeFromBytes} from "../../../util/multifork.js";
 import {HistoricalStateRegenMetrics} from "./metrics.js";
@@ -98,7 +99,7 @@ export async function getHistoricalState(
       throw e;
     }
     blockCount++;
-    if (Buffer.compare(state.hashTreeRoot(), block.message.stateRoot) !== 0) {
+    if (!byteArrayEquals(state.hashTreeRoot(), block.message.stateRoot)) {
       metrics?.regenErrorCount.inc({reason: RegenErrorType.invalidStateRoot});
     }
   }

package/src/chain/blocks/blockInput/blockInput.ts

@@ -1,6 +1,6 @@
-import {ForkName, ForkPostFulu, ForkPreDeneb, ForkPreGloas} from "@lodestar/params";
+import {ForkName, ForkPostFulu, ForkPreDeneb, ForkPreGloas, NUMBER_OF_COLUMNS} from "@lodestar/params";
 import {BeaconBlockBody, BlobIndex, ColumnIndex, SignedBeaconBlock, Slot, deneb, fulu} from "@lodestar/types";
-import {fromHex, prettyBytes, toRootHex, withTimeout} from "@lodestar/utils";
+import {byteArrayEquals, fromHex, prettyBytes, toRootHex, withTimeout} from "@lodestar/utils";
 import {VersionedHashes} from "../../../execution/index.js";
 import {kzgCommitmentToVersionedHash} from "../../../util/blobs.js";
 import {BlockInputError, BlockInputErrorCode} from "./errors.js";
@@ -529,7 +529,7 @@ function blockAndBlobArePaired(block: SignedBeaconBlock<ForkBlobsDA>, blobSideca
   if (!blockCommitment || !blobSidecar.kzgCommitment) {
     return false;
   }
-  return Buffer.compare(blockCommitment, blobSidecar.kzgCommitment) === 0;
+  return byteArrayEquals(blockCommitment, blobSidecar.kzgCommitment);
 }
 
 function assertBlockAndBlobArePaired(
@@ -561,6 +561,7 @@ type BlockInputColumnsState =
   | {
       hasBlock: true;
       hasAllData: true;
+      hasComputedAllData: boolean;
       versionedHashes: VersionedHashes;
       block: SignedBeaconBlock<ForkColumnsDA>;
       source: SourceMeta;
@@ -569,6 +570,7 @@ type BlockInputColumnsState =
   | {
       hasBlock: true;
       hasAllData: false;
+      hasComputedAllData: false;
       versionedHashes: VersionedHashes;
       block: SignedBeaconBlock<ForkColumnsDA>;
       source: SourceMeta;
@@ -576,11 +578,13 @@ type BlockInputColumnsState =
   | {
       hasBlock: false;
       hasAllData: true;
+      hasComputedAllData: boolean;
       versionedHashes: VersionedHashes;
     }
   | {
       hasBlock: false;
       hasAllData: false;
+      hasComputedAllData: false;
       versionedHashes: VersionedHashes;
     };
 /**
@@ -598,6 +602,12 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
   private columnsCache = new Map<ColumnIndex, ColumnWithSource>();
   private readonly sampledColumns: ColumnIndex[];
   private readonly custodyColumns: ColumnIndex[];
+  /**
+   * This promise resolves when all sampled columns are available
+   *
+   * This is different from `dataPromise` which resolves when all data is available or could become available (e.g. through reconstruction)
+   */
+  protected computedDataPromise = createPromise<fulu.DataColumnSidecars>();
 
   private constructor(
     init: BlockInputInit,
@@ -626,6 +636,7 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
     const state = {
       hasBlock: true,
       hasAllData,
+      hasComputedAllData: hasAllData,
       versionedHashes: props.block.message.body.blobKzgCommitments.map(kzgCommitmentToVersionedHash),
       block: props.block,
       source: {
@@ -649,6 +660,7 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
     blockInput.blockPromise.resolve(props.block);
     if (hasAllData) {
       blockInput.dataPromise.resolve([]);
+      blockInput.computedDataPromise.resolve([]);
     }
     return blockInput;
   }
@@ -661,6 +673,7 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
     const state: BlockInputColumnsState = {
       hasBlock: false,
       hasAllData,
+      hasComputedAllData: hasAllData as false,
       versionedHashes: props.columnSidecar.kzgCommitments.map(kzgCommitmentToVersionedHash),
     };
     const init: BlockInputInit = {
@@ -674,6 +687,7 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
     const blockInput = new BlockInputColumns(init, state, props.sampledColumns, props.custodyColumns);
     if (hasAllData) {
       blockInput.dataPromise.resolve([]);
+      blockInput.computedDataPromise.resolve([]);
     }
     return blockInput;
   }
@@ -722,11 +736,14 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
     const hasAllData =
       (props.block.message.body as BeaconBlockBody<ForkPostFulu & ForkPreGloas>).blobKzgCommitments.length === 0 ||
       this.state.hasAllData;
+    const hasComputedAllData =
+      props.block.message.body.blobKzgCommitments.length === 0 || this.state.hasComputedAllData;
 
     this.state = {
       ...this.state,
       hasBlock: true,
       hasAllData,
+      hasComputedAllData,
       block: props.block,
       source: {
         source: props.source,
@@ -774,17 +791,32 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
     this.columnsCache.set(columnSidecar.index, {columnSidecar, source, seenTimestampSec, peerIdStr});
 
     const sampledColumns = this.getSampledColumns();
-    const hasAllData = this.state.hasAllData || sampledColumns.length === this.sampledColumns.length;
+    const hasAllData =
+      // already hasAllData
+      this.state.hasAllData ||
+      // has all sampled columns
+      sampledColumns.length === this.sampledColumns.length ||
+      // has enough columns to reconstruct the rest
+      this.columnsCache.size >= NUMBER_OF_COLUMNS / 2;
+
+    const hasComputedAllData =
+      // has all sampled columns
+      sampledColumns.length === this.sampledColumns.length;
 
     this.state = {
       ...this.state,
       hasAllData: hasAllData || this.state.hasAllData,
+      hasComputedAllData: hasComputedAllData || this.state.hasComputedAllData,
       timeCompleteSec: hasAllData ? seenTimestampSec : undefined,
     } as BlockInputColumnsState;
 
     if (hasAllData && sampledColumns !== null) {
       this.dataPromise.resolve(sampledColumns);
     }
+
+    if (hasComputedAllData && sampledColumns !== null) {
+      this.computedDataPromise.resolve(sampledColumns);
+    }
   }
 
   hasColumn(columnIndex: number): boolean {
@@ -859,4 +891,15 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
       versionedHashes: this.state.versionedHashes,
     };
   }
+
+  hasComputedAllData(): boolean {
+    return this.state.hasComputedAllData;
+  }
+
+  waitForComputedAllData(timeout: number, signal?: AbortSignal): Promise<fulu.DataColumnSidecars> {
+    if (!this.state.hasComputedAllData) {
+      return withTimeout(() => this.computedDataPromise.promise, timeout, signal);
+    }
+    return Promise.resolve(this.getSampledColumns());
+  }
 }

package/src/chain/blocks/importBlock.ts

@@ -30,7 +30,7 @@ import type {BeaconChain} from "../chain.js";
 import {ChainEvent, ReorgEventData} from "../emitter.js";
 import {ForkchoiceCaller} from "../forkChoice/index.js";
 import {REPROCESS_MIN_TIME_TO_NEXT_SLOT_SEC} from "../reprocess.js";
-import {toCheckpointHex} from "../stateCache/index.js";
+import {toCheckpointHex} from "../stateCache/persistentCheckpointsCache.js";
 import {isBlockInputBlobs, isBlockInputColumns} from "./blockInput/blockInput.js";
 import {AttestationImportOpt, FullyVerifiedBlock, ImportBlockOpts} from "./types.js";
 import {getCheckpointFromState} from "./utils/checkpoint.js";
@@ -91,7 +91,15 @@ export async function importBlock(
   }
 
   // 1. Persist block to hot DB (performed asynchronously to avoid blocking head selection)
-  void this.unfinalizedBlockWrites.push([blockInput]);
+  // Wait for space in the write queue to apply backpressure during sync.
+  // Without this, a supernode syncing from behind can accumulate many blocks worth of column
+  // data in memory (up to 128 columns per block) causing OOM before persistence catches up.
+  await this.unfinalizedBlockWrites.waitForSpace();
+  this.unfinalizedBlockWrites.push([blockInput]).catch((e) => {
+    if (!isQueueErrorAborted(e)) {
+      this.logger.error("Error pushing block to unfinalized write queue", {slot: blockSlot}, e as Error);
+    }
+  });
 
   // Without forcefully clearing this cache, we would rely on WeakMap to evict memory which is not reliable
   this.serializedCache.clear();

package/src/chain/blocks/verifyBlock.ts

@@ -145,7 +145,6 @@ export async function verifyBlocksInEpoch(
       opts.skipVerifyBlockSignatures !== true
         ? verifyBlocksSignatures(
             this.config,
-            this.index2pubkey,
             this.bls,
             this.logger,
             this.metrics,

package/src/chain/blocks/verifyBlocksSignatures.ts

@@ -1,5 +1,5 @@
 import {BeaconConfig} from "@lodestar/config";
-import {CachedBeaconStateAllForks, Index2PubkeyCache, getBlockSignatureSets} from "@lodestar/state-transition";
+import {CachedBeaconStateAllForks, getBlockSignatureSets} from "@lodestar/state-transition";
 import {IndexedAttestation, SignedBeaconBlock} from "@lodestar/types";
 import {Logger} from "@lodestar/utils";
 import {Metrics} from "../../metrics/metrics.js";
@@ -17,7 +17,6 @@ import {ImportBlockOpts} from "./types.js";
  */
 export async function verifyBlocksSignatures(
   config: BeaconConfig,
-  index2pubkey: Index2PubkeyCache,
   bls: IBlsVerifier,
   logger: Logger,
   metrics: Metrics | null,
@@ -42,16 +41,9 @@ export async function verifyBlocksSignatures(
       : //
         // Verify signatures per block to track which block is invalid
         bls.verifySignatureSets(
-          getBlockSignatureSets(
-            config,
-            index2pubkey,
-            currentSyncCommitteeIndexed,
-            block,
-            indexedAttestationsByBlock[i],
-            {
-              skipProposerSignature: opts.validProposerSignature,
-            }
-          )
+          getBlockSignatureSets(config, currentSyncCommitteeIndexed, block, indexedAttestationsByBlock[i], {
+            skipProposerSignature: opts.validProposerSignature,
+          })
         );
 
     // getBlockSignatureSets() takes 45ms in benchmarks for 2022Q2 mainnet blocks (100 sigs). When syncing a 32 blocks

package/src/chain/blocks/verifyBlocksStateTransitionOnly.ts

@@ -5,9 +5,8 @@ import {
   StateHashTreeRootSource,
   stateTransition,
 } from "@lodestar/state-transition";
-import {ErrorAborted, Logger} from "@lodestar/utils";
+import {ErrorAborted, Logger, byteArrayEquals} from "@lodestar/utils";
 import {Metrics} from "../../metrics/index.js";
-import {byteArrayEquals} from "../../util/bytes.js";
 import {nextEventLoop} from "../../util/eventLoop.js";
 import {BlockError, BlockErrorCode} from "../errors/index.js";
 import {BlockProcessOpts} from "../options.js";

package/src/chain/blocks/writeBlockInputToDb.ts

@@ -44,6 +44,15 @@ export async function writeBlockInputToDb(this: BeaconChain, blocksInputs: IBloc
 
     // NOTE: Old data is pruned on archive
     if (isBlockInputColumns(blockInput)) {
+      if (!blockInput.hasComputedAllData()) {
+        // Supernodes may only have a subset of the data columns by the time the block begins to be imported
+        // because full data availability can be assumed after NUMBER_OF_COLUMNS / 2 columns are available.
+        // Here, however, all data columns must be fully available/reconstructed before persisting to the DB.
+        await blockInput.waitForComputedAllData(BLOB_AVAILABILITY_TIMEOUT).catch(() => {
+          this.logger.debug("Failed to wait for computed all data", {slot, blockRoot: blockRootHex});
+        });
+      }
+
       const {custodyColumns} = this.custodyConfig;
       const blobsLen = (block.message as fulu.BeaconBlock).body.blobKzgCommitments.length;
       let dataColumnsLen: number;

package/src/chain/bls/multithread/index.ts

@@ -7,7 +7,7 @@ import {Worker, spawn} from "@chainsafe/threads";
 self = undefined;
 
 import {PublicKey} from "@chainsafe/blst";
-import {ISignatureSet} from "@lodestar/state-transition";
+import {ISignatureSet, Index2PubkeyCache} from "@lodestar/state-transition";
 import {Logger} from "@lodestar/utils";
 import {Metrics} from "../../../metrics/index.js";
 import {LinkedList} from "../../../util/array.js";
@@ -34,6 +34,7 @@ const workerDir = process.env.NODE_ENV === "test" ? "../../../../lib/chain/bls/m
 export type BlsMultiThreadWorkerPoolModules = {
   logger: Logger;
   metrics: Metrics | null;
+  index2pubkey: Index2PubkeyCache;
 };
 
 export type BlsMultiThreadWorkerPoolOptions = {
@@ -113,6 +114,7 @@ type WorkerDescriptor = {
 export class BlsMultiThreadWorkerPool implements IBlsVerifier {
   private readonly logger: Logger;
   private readonly metrics: Metrics | null;
+  private readonly index2pubkey: Index2PubkeyCache;
 
   private readonly workers: WorkerDescriptor[];
   private readonly jobs = new LinkedList<JobQueueItem>();
@@ -128,9 +130,10 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier {
   private workersBusy = 0;
 
   constructor(options: BlsMultiThreadWorkerPoolOptions, modules: BlsMultiThreadWorkerPoolModules) {
-    const {logger, metrics} = modules;
+    const {logger, metrics, index2pubkey} = modules;
     this.logger = logger;
     this.metrics = metrics;
+    this.index2pubkey = index2pubkey;
     this.blsVerifyAllMultiThread = options.blsVerifyAllMultiThread ?? false;
 
     // Use compressed for herumi for now.
@@ -170,7 +173,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier {
       try {
         return verifySignatureSetsMaybeBatch(
           sets.map((set) => ({
-            publicKey: getAggregatedPubkey(set),
+            publicKey: getAggregatedPubkey(set, this.index2pubkey),
             message: set.signingRoot.valueOf(),
             signature: set.signature,
           }))
@@ -395,7 +398,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier {
         try {
           // Note: This can throw, must be handled per-job.
           // Pubkey and signature aggregation is defered here
-          workReq = await jobItemWorkReq(job, this.metrics);
+          workReq = await jobItemWorkReq(job, this.index2pubkey, this.metrics);
         } catch (e) {
           this.metrics?.blsThreadPool.errorAggregateSignatureSetsCount.inc({type: job.type});
 

package/src/chain/bls/multithread/jobItem.ts

@@ -1,5 +1,5 @@
 import {PublicKey, asyncAggregateWithRandomness} from "@chainsafe/blst";
-import {ISignatureSet, SignatureSetType} from "@lodestar/state-transition";
+import {ISignatureSet, Index2PubkeyCache, SignatureSetType} from "@lodestar/state-transition";
 import {Metrics} from "../../../metrics/metrics.js";
 import {LinkedList} from "../../../util/array.js";
 import {VerifySignatureOpts} from "../interface.js";
@@ -48,14 +48,18 @@ export function jobItemSigSets(job: JobQueueItem): number {
  * Prepare BlsWorkReq from JobQueueItem
  * WARNING: May throw with untrusted user input
  */
-export async function jobItemWorkReq(job: JobQueueItem, metrics: Metrics | null): Promise<BlsWorkReq> {
+export async function jobItemWorkReq(
+  job: JobQueueItem,
+  index2pubkey: Index2PubkeyCache,
+  metrics: Metrics | null
+): Promise<BlsWorkReq> {
   switch (job.type) {
     case JobQueueItemType.default:
       return {
         opts: job.opts,
         sets: job.sets.map((set) => ({
           // this can throw, handled in the consumer code
-          publicKey: getAggregatedPubkey(set, metrics).toBytes(),
+          publicKey: getAggregatedPubkey(set, index2pubkey, metrics).toBytes(),
           signature: set.signature,
           message: set.signingRoot,
         })),

package/src/chain/bls/singleThread.ts

@@ -1,5 +1,5 @@
 import {PublicKey, Signature, aggregatePublicKeys, aggregateSignatures, verify} from "@chainsafe/blst";
-import {ISignatureSet} from "@lodestar/state-transition";
+import {ISignatureSet, Index2PubkeyCache} from "@lodestar/state-transition";
 import {Metrics} from "../../metrics/index.js";
 import {IBlsVerifier} from "./interface.js";
 import {verifySignatureSetsMaybeBatch} from "./maybeBatch.js";
@@ -7,16 +7,18 @@ import {getAggregatedPubkey, getAggregatedPubkeysCount} from "./utils.js";
 
 export class BlsSingleThreadVerifier implements IBlsVerifier {
   private readonly metrics: Metrics | null;
+  private readonly index2pubkey: Index2PubkeyCache;
 
-  constructor({metrics = null}: {metrics: Metrics | null}) {
+  constructor({metrics = null, index2pubkey}: {metrics: Metrics | null; index2pubkey: Index2PubkeyCache}) {
     this.metrics = metrics;
+    this.index2pubkey = index2pubkey;
   }
 
   async verifySignatureSets(sets: ISignatureSet[]): Promise<boolean> {
     this.metrics?.bls.aggregatedPubkeys.inc(getAggregatedPubkeysCount(sets));
 
     const setsAggregated = sets.map((set) => ({
-      publicKey: getAggregatedPubkey(set),
+      publicKey: getAggregatedPubkey(set, this.index2pubkey, this.metrics),
       message: set.signingRoot,
       signature: set.signature,
     }));