@lodestar/beacon-node 1.40.0-dev.3715a82f3f → 1.40.0-dev.45b04262dc

package/src/chain/ColumnReconstructionTracker.ts

@@ -8,12 +8,12 @@ import {ChainEventEmitter} from "./emitter.js";
 /**
  * Minimum time to wait before attempting reconstruction
  */
-const RECONSTRUCTION_DELAY_MIN_MS = 800;
+const RECONSTRUCTION_DELAY_MIN_BPS = 667;
 
 /**
  * Maximum time to wait before attempting reconstruction
  */
-const RECONSTRUCTION_DELAY_MAX_MS = 1200;
+const RECONSTRUCTION_DELAY_MAX_BPS = 1000;
 
 export type ColumnReconstructionTrackerInit = {
   logger: Logger;
@@ -41,11 +41,16 @@ export class ColumnReconstructionTracker {
   /** Track if a reconstruction attempt is in-flight */
   running = false;
 
+  private readonly minDelayMs: number;
+  private readonly maxDelayMs: number;
+
   constructor(init: ColumnReconstructionTrackerInit) {
     this.logger = init.logger;
     this.emitter = init.emitter;
     this.metrics = init.metrics;
     this.config = init.config;
+    this.minDelayMs = this.config.getSlotComponentDurationMs(RECONSTRUCTION_DELAY_MIN_BPS);
+    this.maxDelayMs = this.config.getSlotComponentDurationMs(RECONSTRUCTION_DELAY_MAX_BPS);
   }
 
   triggerColumnReconstruction(blockInput: BlockInputColumns): void {
@@ -61,8 +66,7 @@ export class ColumnReconstructionTracker {
     // just that it has been triggered for this block root.
     this.running = true;
     this.lastBlockRootHex = blockInput.blockRootHex;
-    const delay =
-      RECONSTRUCTION_DELAY_MIN_MS + Math.random() * (RECONSTRUCTION_DELAY_MAX_MS - RECONSTRUCTION_DELAY_MIN_MS);
+    const delay = this.minDelayMs + Math.random() * (this.maxDelayMs - this.minDelayMs);
     sleep(delay)
       .then(() => {
         const logCtx = {slot: blockInput.slot, root: blockInput.blockRootHex};

package/src/chain/archiveStore/historicalState/getHistoricalState.ts

@@ -8,6 +8,7 @@ import {
   createCachedBeaconState,
   stateTransition,
 } from "@lodestar/state-transition";
+import {byteArrayEquals} from "@lodestar/utils";
 import {IBeaconDb} from "../../../db/index.js";
 import {getStateTypeFromBytes} from "../../../util/multifork.js";
 import {HistoricalStateRegenMetrics} from "./metrics.js";
@@ -98,7 +99,7 @@ export async function getHistoricalState(
       throw e;
     }
     blockCount++;
-    if (Buffer.compare(state.hashTreeRoot(), block.message.stateRoot) !== 0) {
+    if (!byteArrayEquals(state.hashTreeRoot(), block.message.stateRoot)) {
       metrics?.regenErrorCount.inc({reason: RegenErrorType.invalidStateRoot});
     }
   }

package/src/chain/archiveStore/utils/archiveBlocks.ts

@@ -238,6 +238,7 @@ async function migrateBlocksFromHotToColdDb(db: IBeaconDb, blocks: BlockRootSlot
     // load Buffer instead of SignedBeaconBlock to improve performance
     const canonicalBlockEntries: BlockArchiveBatchPutBinaryItem[] = await Promise.all(
       canonicalBlocks.map(async (block) => {
+        // Here we assume the blocks are already in the hot db
         const blockBuffer = await db.block.getBinary(block.root);
         if (!blockBuffer) {
           throw Error(`Block not found for slot ${block.slot} root ${toRootHex(block.root)}`);
@@ -294,6 +295,8 @@ async function migrateBlobSidecarsFromHotToColdDb(
           );
         })
         .map(async (block) => {
+          // Here we assume the blob sidecars are already in the hot db
+          // instead of checking first the block input cache
           const bytes = await db.blobSidecars.getBinary(block.root);
           if (!bytes) {
             throw Error(`No blobSidecars found for slot ${block.slot} root ${toRootHex(block.root)}`);
@@ -343,6 +346,7 @@ async function migrateDataColumnSidecarsFromHotToColdDb(
         continue;
       }
 
+      // Here we assume the data column sidecars are already in the hot db
       const dataColumnSidecarBytes = await fromAsync(db.dataColumnSidecar.valuesStreamBinary(block.root));
       // there could be 0 dataColumnSidecarBytes if block has no blob
       logger.verbose("migrateDataColumnSidecarsFromHotToColdDb", {

package/src/chain/blocks/blockInput/blockInput.ts

@@ -1,6 +1,6 @@
-import {ForkName, ForkPostFulu, ForkPreDeneb, ForkPreGloas} from "@lodestar/params";
+import {ForkName, ForkPostFulu, ForkPreDeneb, ForkPreGloas, NUMBER_OF_COLUMNS} from "@lodestar/params";
 import {BeaconBlockBody, BlobIndex, ColumnIndex, SignedBeaconBlock, Slot, deneb, fulu} from "@lodestar/types";
-import {fromHex, prettyBytes, toRootHex, withTimeout} from "@lodestar/utils";
+import {byteArrayEquals, fromHex, prettyBytes, toRootHex, withTimeout} from "@lodestar/utils";
 import {VersionedHashes} from "../../../execution/index.js";
 import {kzgCommitmentToVersionedHash} from "../../../util/blobs.js";
 import {BlockInputError, BlockInputErrorCode} from "./errors.js";
@@ -412,6 +412,10 @@ export class BlockInputBlobs extends AbstractBlockInput<ForkBlobsDA, deneb.BlobS
     return this.blobsCache.has(blobIndex);
   }
 
+  getBlob(blobIndex: BlobIndex): deneb.BlobSidecar | undefined {
+    return this.blobsCache.get(blobIndex)?.blobSidecar;
+  }
+
   addBlob(
     {blockRootHex, blobSidecar, source, peerIdStr, seenTimestampSec}: AddBlob,
     opts = {throwOnDuplicateAdd: true}
@@ -525,7 +529,7 @@ function blockAndBlobArePaired(block: SignedBeaconBlock<ForkBlobsDA>, blobSideca
   if (!blockCommitment || !blobSidecar.kzgCommitment) {
     return false;
   }
-  return Buffer.compare(blockCommitment, blobSidecar.kzgCommitment) === 0;
+  return byteArrayEquals(blockCommitment, blobSidecar.kzgCommitment);
 }
 
 function assertBlockAndBlobArePaired(
@@ -557,6 +561,7 @@ type BlockInputColumnsState =
   | {
       hasBlock: true;
       hasAllData: true;
+      hasComputedAllData: boolean;
       versionedHashes: VersionedHashes;
       block: SignedBeaconBlock<ForkColumnsDA>;
       source: SourceMeta;
@@ -565,6 +570,7 @@ type BlockInputColumnsState =
   | {
       hasBlock: true;
       hasAllData: false;
+      hasComputedAllData: false;
       versionedHashes: VersionedHashes;
       block: SignedBeaconBlock<ForkColumnsDA>;
       source: SourceMeta;
@@ -572,11 +578,13 @@ type BlockInputColumnsState =
   | {
       hasBlock: false;
       hasAllData: true;
+      hasComputedAllData: boolean;
       versionedHashes: VersionedHashes;
     }
   | {
       hasBlock: false;
       hasAllData: false;
+      hasComputedAllData: false;
       versionedHashes: VersionedHashes;
     };
 /**
@@ -594,6 +602,12 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
   private columnsCache = new Map<ColumnIndex, ColumnWithSource>();
   private readonly sampledColumns: ColumnIndex[];
   private readonly custodyColumns: ColumnIndex[];
+  /**
+   * This promise resolves when all sampled columns are available
+   *
+   * This is different from `dataPromise` which resolves when all data is available or could become available (e.g. through reconstruction)
+   */
+  protected computedDataPromise = createPromise<fulu.DataColumnSidecars>();
 
   private constructor(
     init: BlockInputInit,
@@ -622,6 +636,7 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
     const state = {
       hasBlock: true,
       hasAllData,
+      hasComputedAllData: hasAllData,
       versionedHashes: props.block.message.body.blobKzgCommitments.map(kzgCommitmentToVersionedHash),
       block: props.block,
       source: {
@@ -645,6 +660,7 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
     blockInput.blockPromise.resolve(props.block);
     if (hasAllData) {
       blockInput.dataPromise.resolve([]);
+      blockInput.computedDataPromise.resolve([]);
     }
     return blockInput;
   }
@@ -657,6 +673,7 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
     const state: BlockInputColumnsState = {
       hasBlock: false,
       hasAllData,
+      hasComputedAllData: hasAllData as false,
       versionedHashes: props.columnSidecar.kzgCommitments.map(kzgCommitmentToVersionedHash),
     };
     const init: BlockInputInit = {
@@ -670,6 +687,7 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
     const blockInput = new BlockInputColumns(init, state, props.sampledColumns, props.custodyColumns);
     if (hasAllData) {
       blockInput.dataPromise.resolve([]);
+      blockInput.computedDataPromise.resolve([]);
     }
     return blockInput;
   }
@@ -718,11 +736,14 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
     const hasAllData =
       (props.block.message.body as BeaconBlockBody<ForkPostFulu & ForkPreGloas>).blobKzgCommitments.length === 0 ||
       this.state.hasAllData;
+    const hasComputedAllData =
+      props.block.message.body.blobKzgCommitments.length === 0 || this.state.hasComputedAllData;
 
     this.state = {
       ...this.state,
       hasBlock: true,
       hasAllData,
+      hasComputedAllData,
       block: props.block,
       source: {
         source: props.source,
@@ -770,23 +791,42 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
     this.columnsCache.set(columnSidecar.index, {columnSidecar, source, seenTimestampSec, peerIdStr});
 
     const sampledColumns = this.getSampledColumns();
-    const hasAllData = this.state.hasAllData || sampledColumns.length === this.sampledColumns.length;
+    const hasAllData =
+      // already hasAllData
+      this.state.hasAllData ||
+      // has all sampled columns
+      sampledColumns.length === this.sampledColumns.length ||
+      // has enough columns to reconstruct the rest
+      this.columnsCache.size >= NUMBER_OF_COLUMNS / 2;
+
+    const hasComputedAllData =
+      // has all sampled columns
+      sampledColumns.length === this.sampledColumns.length;
 
     this.state = {
       ...this.state,
       hasAllData: hasAllData || this.state.hasAllData,
+      hasComputedAllData: hasComputedAllData || this.state.hasComputedAllData,
       timeCompleteSec: hasAllData ? seenTimestampSec : undefined,
     } as BlockInputColumnsState;
 
     if (hasAllData && sampledColumns !== null) {
       this.dataPromise.resolve(sampledColumns);
     }
+
+    if (hasComputedAllData && sampledColumns !== null) {
+      this.computedDataPromise.resolve(sampledColumns);
+    }
   }
 
   hasColumn(columnIndex: number): boolean {
     return this.columnsCache.has(columnIndex);
   }
 
+  getColumn(columnIndex: number): fulu.DataColumnSidecar | undefined {
+    return this.columnsCache.get(columnIndex)?.columnSidecar;
+  }
+
   getVersionedHashes(): VersionedHashes {
     return this.state.versionedHashes;
   }
@@ -851,4 +891,15 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
       versionedHashes: this.state.versionedHashes,
     };
   }
+
+  hasComputedAllData(): boolean {
+    return this.state.hasComputedAllData;
+  }
+
+  waitForComputedAllData(timeout: number, signal?: AbortSignal): Promise<fulu.DataColumnSidecars> {
+    if (!this.state.hasComputedAllData) {
+      return withTimeout(() => this.computedDataPromise.promise, timeout, signal);
+    }
+    return Promise.resolve(this.getSampledColumns());
+  }
 }

package/src/chain/blocks/importBlock.ts

@@ -30,11 +30,10 @@ import type {BeaconChain} from "../chain.js";
 import {ChainEvent, ReorgEventData} from "../emitter.js";
 import {ForkchoiceCaller} from "../forkChoice/index.js";
 import {REPROCESS_MIN_TIME_TO_NEXT_SLOT_SEC} from "../reprocess.js";
-import {toCheckpointHex} from "../stateCache/index.js";
+import {toCheckpointHex} from "../stateCache/persistentCheckpointsCache.js";
 import {isBlockInputBlobs, isBlockInputColumns} from "./blockInput/blockInput.js";
 import {AttestationImportOpt, FullyVerifiedBlock, ImportBlockOpts} from "./types.js";
 import {getCheckpointFromState} from "./utils/checkpoint.js";
-import {writeBlockInputToDb} from "./writeBlockInputToDb.js";
 
 /**
  * Fork-choice allows to import attestations from current (0) or past (1) epoch.
@@ -91,11 +90,8 @@ export async function importBlock(
     throw Error("Unavailable block can not be imported in forkchoice");
   }
 
-  // 1. Persist block to hot DB (pre-emptively)
-  // If eagerPersistBlock = true we do that in verifyBlocksInEpoch to batch all I/O operations to save block time to head
-  if (!opts.eagerPersistBlock) {
-    await writeBlockInputToDb.call(this, [blockInput]);
-  }
+  // 1. Persist block to hot DB (performed asynchronously to avoid blocking head selection)
+  void this.unfinalizedBlockWrites.push([blockInput]);
 
   // Without forcefully clearing this cache, we would rely on WeakMap to evict memory which is not reliable
   this.serializedCache.clear();

package/src/chain/blocks/index.ts

@@ -11,7 +11,6 @@ import {FullyVerifiedBlock, ImportBlockOpts} from "./types.js";
 import {assertLinearChainSegment} from "./utils/chainSegment.js";
 import {verifyBlocksInEpoch} from "./verifyBlock.js";
 import {verifyBlocksSanityChecks} from "./verifyBlocksSanityChecks.js";
-import {removeEagerlyPersistedBlockInputs} from "./writeBlockInputToDb.js";
 
 export {AttestationImportOpt, type ImportBlockOpts} from "./types.js";
 
@@ -143,24 +142,6 @@ export async function processBlocks(
       }
     }
 
-    // Clean db if we don't have blocks in forkchoice but already persisted them to db
-    //
-    // NOTE: this function is awaited to ensure that DB size remains constant, otherwise an attacker may bloat the
-    // disk with big malicious payloads. Our sequential block importer will wait for this promise before importing
-    // another block. The removal call error is not propagated since that would halt the chain.
-    //
-    // LOG: Because the error is not propagated and there's a risk of db bloat, the error is logged at warn level
-    // to alert the user of potential db bloat. This error _should_ never happen user must act and report to us
-    if (opts.eagerPersistBlock) {
-      await removeEagerlyPersistedBlockInputs.call(this, blocks).catch((e) => {
-        this.logger.warn(
-          "Error pruning eagerly imported block inputs, DB may grow in size if this error happens frequently",
-          {slot: blocks.map((block) => block.getBlock().message.slot).join(",")},
-          e
-        );
-      });
-    }
-
     throw err;
   }
 }

package/src/chain/blocks/types.ts

@@ -78,8 +78,6 @@ export type ImportBlockOpts = {
   validBlobSidecars?: BlobSidecarValidation;
   /** Seen timestamp seconds */
   seenTimestampSec?: number;
-  /** Set to true if persist block right at verification time */
-  eagerPersistBlock?: boolean;
 };
 
 /**

package/src/chain/blocks/verifyBlock.ts

@@ -21,7 +21,6 @@ import {verifyBlocksDataAvailability} from "./verifyBlocksDataAvailability.js";
 import {SegmentExecStatus, verifyBlocksExecutionPayload} from "./verifyBlocksExecutionPayloads.js";
 import {verifyBlocksSignatures} from "./verifyBlocksSignatures.js";
 import {verifyBlocksStateTransitionOnly} from "./verifyBlocksStateTransitionOnly.js";
-import {writeBlockInputToDb} from "./writeBlockInputToDb.js";
 
 /**
  * Verifies 1 or more blocks are fully valid; from a linear sequence of blocks.
@@ -146,7 +145,6 @@ export async function verifyBlocksInEpoch(
       opts.skipVerifyBlockSignatures !== true
         ? verifyBlocksSignatures(
             this.config,
-            this.index2pubkey,
             this.bls,
             this.logger,
             this.metrics,
@@ -156,13 +154,6 @@ export async function verifyBlocksInEpoch(
             opts
           )
         : Promise.resolve({verifySignaturesTime: Date.now()}),
-
-      // ideally we want to only persist blocks after verifying them however the reality is there are
-      // rarely invalid blocks we'll batch all I/O operation here to reduce the overhead if there's
-      // an error, we'll remove blocks not in forkchoice
-      opts.verifyOnly !== true && opts.eagerPersistBlock
-        ? writeBlockInputToDb.call(this, blockInputs)
-        : Promise.resolve(),
     ]);
 
     if (opts.verifyOnly !== true) {

package/src/chain/blocks/verifyBlocksSignatures.ts

@@ -1,5 +1,5 @@
 import {BeaconConfig} from "@lodestar/config";
-import {CachedBeaconStateAllForks, Index2PubkeyCache, getBlockSignatureSets} from "@lodestar/state-transition";
+import {CachedBeaconStateAllForks, getBlockSignatureSets} from "@lodestar/state-transition";
 import {IndexedAttestation, SignedBeaconBlock} from "@lodestar/types";
 import {Logger} from "@lodestar/utils";
 import {Metrics} from "../../metrics/metrics.js";
@@ -17,7 +17,6 @@ import {ImportBlockOpts} from "./types.js";
  */
 export async function verifyBlocksSignatures(
   config: BeaconConfig,
-  index2pubkey: Index2PubkeyCache,
   bls: IBlsVerifier,
   logger: Logger,
   metrics: Metrics | null,
@@ -42,16 +41,9 @@ export async function verifyBlocksSignatures(
       : //
         // Verify signatures per block to track which block is invalid
         bls.verifySignatureSets(
-          getBlockSignatureSets(
-            config,
-            index2pubkey,
-            currentSyncCommitteeIndexed,
-            block,
-            indexedAttestationsByBlock[i],
-            {
-              skipProposerSignature: opts.validProposerSignature,
-            }
-          )
+          getBlockSignatureSets(config, currentSyncCommitteeIndexed, block, indexedAttestationsByBlock[i], {
+            skipProposerSignature: opts.validProposerSignature,
+          })
         );
 
     // getBlockSignatureSets() takes 45ms in benchmarks for 2022Q2 mainnet blocks (100 sigs). When syncing a 32 blocks

package/src/chain/blocks/verifyBlocksStateTransitionOnly.ts

@@ -5,9 +5,8 @@ import {
   StateHashTreeRootSource,
   stateTransition,
 } from "@lodestar/state-transition";
-import {ErrorAborted, Logger} from "@lodestar/utils";
+import {ErrorAborted, Logger, byteArrayEquals} from "@lodestar/utils";
 import {Metrics} from "../../metrics/index.js";
-import {byteArrayEquals} from "../../util/bytes.js";
 import {nextEventLoop} from "../../util/eventLoop.js";
 import {BlockError, BlockErrorCode} from "../errors/index.js";
 import {BlockProcessOpts} from "../options.js";

package/src/chain/blocks/writeBlockInputToDb.ts

@@ -44,6 +44,15 @@ export async function writeBlockInputToDb(this: BeaconChain, blocksInputs: IBloc
 
     // NOTE: Old data is pruned on archive
     if (isBlockInputColumns(blockInput)) {
+      if (!blockInput.hasComputedAllData()) {
+        // Supernodes may only have a subset of the data columns by the time the block begins to be imported
+        // because full data availability can be assumed after NUMBER_OF_COLUMNS / 2 columns are available.
+        // Here, however, all data columns must be fully available/reconstructed before persisting to the DB.
+        await blockInput.waitForComputedAllData(BLOB_AVAILABILITY_TIMEOUT).catch(() => {
+          this.logger.debug("Failed to wait for computed all data", {slot, blockRoot: blockRootHex});
+        });
+      }
+
       const {custodyColumns} = this.custodyConfig;
       const blobsLen = (block.message as fulu.BeaconBlock).body.blobKzgCommitments.length;
       let dataColumnsLen: number;
@@ -98,35 +107,29 @@ export async function writeBlockInputToDb(this: BeaconChain, blocksInputs: IBloc
   }
 }
 
-/**
- * Prunes eagerly persisted block inputs only if not known to the fork-choice
- */
-export async function removeEagerlyPersistedBlockInputs(this: BeaconChain, blockInputs: IBlockInput[]): Promise<void> {
-  const blockToRemove = [];
-  const blobsToRemove = [];
-  const dataColumnsToRemove = [];
-
-  for (const blockInput of blockInputs) {
-    const block = blockInput.getBlock();
-    const slot = block.message.slot;
-    const blockRoot = this.config.getForkTypes(slot).BeaconBlock.hashTreeRoot(block.message);
-    const blockRootHex = toRootHex(blockRoot);
-    if (!this.forkChoice.hasBlockHex(blockRootHex)) {
-      blockToRemove.push(block);
-
-      if (isBlockInputColumns(blockInput) && blockInput.getCustodyColumns().length > 0) {
-        dataColumnsToRemove.push(blockRoot);
-      } else if (isBlockInputBlobs(blockInput)) {
-        const blobSidecars = blockInput.getBlobs();
-        blobsToRemove.push({blockRoot, slot, blobSidecars});
+export async function persistBlockInputs(this: BeaconChain, blockInputs: IBlockInput[]): Promise<void> {
+  await writeBlockInputToDb
+    .call(this, blockInputs)
+    .catch((e) => {
+      this.logger.debug(
+        "Error persisting block input in hot db",
+        {
+          count: blockInputs.length,
+          slot: blockInputs[0].slot,
+          root: blockInputs[0].blockRootHex,
+        },
+        e
+      );
+    })
+    .finally(() => {
+      for (const blockInput of blockInputs) {
+        this.seenBlockInputCache.prune(blockInput.blockRootHex);
       }
-    }
-  }
-
-  await Promise.all([
-    // TODO: Batch DB operations not with Promise.all but with level db ops
-    this.db.block.batchRemove(blockToRemove),
-    this.db.blobSidecars.batchRemove(blobsToRemove),
-    this.db.dataColumnSidecar.deleteMany(dataColumnsToRemove),
-  ]);
+      if (blockInputs.length === 1) {
+        this.logger.debug("Pruned block input", {
+          slot: blockInputs[0].slot,
+          root: blockInputs[0].blockRootHex,
+        });
+      }
+    });
 }

package/src/chain/bls/multithread/index.ts

@@ -7,7 +7,7 @@ import {Worker, spawn} from "@chainsafe/threads";
 self = undefined;
 
 import {PublicKey} from "@chainsafe/blst";
-import {ISignatureSet} from "@lodestar/state-transition";
+import {ISignatureSet, Index2PubkeyCache} from "@lodestar/state-transition";
 import {Logger} from "@lodestar/utils";
 import {Metrics} from "../../../metrics/index.js";
 import {LinkedList} from "../../../util/array.js";
@@ -34,6 +34,7 @@ const workerDir = process.env.NODE_ENV === "test" ? "../../../../lib/chain/bls/m
 export type BlsMultiThreadWorkerPoolModules = {
   logger: Logger;
   metrics: Metrics | null;
+  index2pubkey: Index2PubkeyCache;
 };
 
 export type BlsMultiThreadWorkerPoolOptions = {
@@ -113,6 +114,7 @@ type WorkerDescriptor = {
 export class BlsMultiThreadWorkerPool implements IBlsVerifier {
   private readonly logger: Logger;
   private readonly metrics: Metrics | null;
+  private readonly index2pubkey: Index2PubkeyCache;
 
   private readonly workers: WorkerDescriptor[];
   private readonly jobs = new LinkedList<JobQueueItem>();
@@ -128,9 +130,10 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier {
   private workersBusy = 0;
 
   constructor(options: BlsMultiThreadWorkerPoolOptions, modules: BlsMultiThreadWorkerPoolModules) {
-    const {logger, metrics} = modules;
+    const {logger, metrics, index2pubkey} = modules;
     this.logger = logger;
     this.metrics = metrics;
+    this.index2pubkey = index2pubkey;
     this.blsVerifyAllMultiThread = options.blsVerifyAllMultiThread ?? false;
 
     // Use compressed for herumi for now.
@@ -170,7 +173,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier {
       try {
         return verifySignatureSetsMaybeBatch(
           sets.map((set) => ({
-            publicKey: getAggregatedPubkey(set),
+            publicKey: getAggregatedPubkey(set, this.index2pubkey),
             message: set.signingRoot.valueOf(),
             signature: set.signature,
           }))
@@ -395,7 +398,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier {
         try {
           // Note: This can throw, must be handled per-job.
           // Pubkey and signature aggregation is defered here
-          workReq = await jobItemWorkReq(job, this.metrics);
+          workReq = await jobItemWorkReq(job, this.index2pubkey, this.metrics);
         } catch (e) {
           this.metrics?.blsThreadPool.errorAggregateSignatureSetsCount.inc({type: job.type});
 

package/src/chain/bls/multithread/jobItem.ts

@@ -1,5 +1,5 @@
 import {PublicKey, asyncAggregateWithRandomness} from "@chainsafe/blst";
-import {ISignatureSet, SignatureSetType} from "@lodestar/state-transition";
+import {ISignatureSet, Index2PubkeyCache, SignatureSetType} from "@lodestar/state-transition";
 import {Metrics} from "../../../metrics/metrics.js";
 import {LinkedList} from "../../../util/array.js";
 import {VerifySignatureOpts} from "../interface.js";
@@ -48,14 +48,18 @@ export function jobItemSigSets(job: JobQueueItem): number {
  * Prepare BlsWorkReq from JobQueueItem
  * WARNING: May throw with untrusted user input
  */
-export async function jobItemWorkReq(job: JobQueueItem, metrics: Metrics | null): Promise<BlsWorkReq> {
+export async function jobItemWorkReq(
+  job: JobQueueItem,
+  index2pubkey: Index2PubkeyCache,
+  metrics: Metrics | null
+): Promise<BlsWorkReq> {
   switch (job.type) {
     case JobQueueItemType.default:
       return {
         opts: job.opts,
         sets: job.sets.map((set) => ({
           // this can throw, handled in the consumer code
-          publicKey: getAggregatedPubkey(set, metrics).toBytes(),
+          publicKey: getAggregatedPubkey(set, index2pubkey, metrics).toBytes(),
           signature: set.signature,
           message: set.signingRoot,
         })),

package/src/chain/bls/singleThread.ts

@@ -1,5 +1,5 @@
 import {PublicKey, Signature, aggregatePublicKeys, aggregateSignatures, verify} from "@chainsafe/blst";
-import {ISignatureSet} from "@lodestar/state-transition";
+import {ISignatureSet, Index2PubkeyCache} from "@lodestar/state-transition";
 import {Metrics} from "../../metrics/index.js";
 import {IBlsVerifier} from "./interface.js";
 import {verifySignatureSetsMaybeBatch} from "./maybeBatch.js";
@@ -7,16 +7,18 @@ import {getAggregatedPubkey, getAggregatedPubkeysCount} from "./utils.js";
 
 export class BlsSingleThreadVerifier implements IBlsVerifier {
   private readonly metrics: Metrics | null;
+  private readonly index2pubkey: Index2PubkeyCache;
 
-  constructor({metrics = null}: {metrics: Metrics | null}) {
+  constructor({metrics = null, index2pubkey}: {metrics: Metrics | null; index2pubkey: Index2PubkeyCache}) {
     this.metrics = metrics;
+    this.index2pubkey = index2pubkey;
   }
 
   async verifySignatureSets(sets: ISignatureSet[]): Promise<boolean> {
     this.metrics?.bls.aggregatedPubkeys.inc(getAggregatedPubkeysCount(sets));
 
     const setsAggregated = sets.map((set) => ({
-      publicKey: getAggregatedPubkey(set),
+      publicKey: getAggregatedPubkey(set, this.index2pubkey, this.metrics),
       message: set.signingRoot,
       signature: set.signature,
     }));

package/src/chain/bls/utils.ts

@@ -1,17 +1,25 @@
 import {PublicKey, aggregatePublicKeys} from "@chainsafe/blst";
-import {ISignatureSet, SignatureSetType} from "@lodestar/state-transition";
+import {ISignatureSet, Index2PubkeyCache, SignatureSetType} from "@lodestar/state-transition";
 import {Metrics} from "../../metrics/metrics.js";
 
-export function getAggregatedPubkey(signatureSet: ISignatureSet, metrics: Metrics | null = null): PublicKey {
+export function getAggregatedPubkey(
+  signatureSet: ISignatureSet,
+  index2pubkey: Index2PubkeyCache,
+  metrics: Metrics | null = null
+): PublicKey {
   switch (signatureSet.type) {
     case SignatureSetType.single:
       return signatureSet.pubkey;
 
+    case SignatureSetType.indexed:
+      return index2pubkey[signatureSet.index];
+
     case SignatureSetType.aggregate: {
       const timer = metrics?.blsThreadPool.pubkeysAggregationMainThreadDuration.startTimer();
-      const pubkeys = aggregatePublicKeys(signatureSet.pubkeys);
+      const pubkeys = signatureSet.indices.map((i) => index2pubkey[i]);
+      const aggregated = aggregatePublicKeys(pubkeys);
       timer?.();
-      return pubkeys;
+      return aggregated;
     }
 
     default:
@@ -20,11 +28,11 @@ export function getAggregatedPubkey(signatureSet: ISignatureSet, metrics: Metric
 }
 
 export function getAggregatedPubkeysCount(signatureSets: ISignatureSet[]): number {
-  let pubkeysConut = 0;
+  let pubkeysCount = 0;
   for (const set of signatureSets) {
     if (set.type === SignatureSetType.aggregate) {
-      pubkeysConut += set.pubkeys.length;
+      pubkeysCount += set.indices.length;
     }
   }
-  return pubkeysConut;
+  return pubkeysCount;
 }