@lodestar/beacon-node 1.39.0 → 1.40.0-dev.1c71f4299a

package/src/chain/archiveStore/utils/archiveBlocks.ts

@@ -73,9 +73,12 @@ export async function archiveBlocks(
     root: fromHex(block.blockRoot),
   }));
 
+  const logCtx = {currentEpoch, finalizedEpoch: finalizedCheckpoint.epoch, finalizedRoot: finalizedCheckpoint.rootHex};
+
   if (finalizedCanonicalBlockRoots.length > 0) {
     await migrateBlocksFromHotToColdDb(db, finalizedCanonicalBlockRoots);
     logger.verbose("Migrated blocks from hot DB to cold DB", {
+      ...logCtx,
       fromSlot: finalizedCanonicalBlockRoots[0].slot,
       toSlot: finalizedCanonicalBlockRoots.at(-1)?.slot,
       size: finalizedCanonicalBlockRoots.length,
@@ -88,7 +91,7 @@ export async function archiveBlocks(
         finalizedCanonicalBlockRoots,
         currentEpoch
       );
-      logger.verbose("Migrated blobSidecars from hot DB to cold DB", {migratedEntries});
+      logger.verbose("Migrated blobSidecars from hot DB to cold DB", {...logCtx, migratedEntries});
     }
 
     if (finalizedPostFulu) {
@@ -99,7 +102,7 @@ export async function archiveBlocks(
         finalizedCanonicalBlockRoots,
         currentEpoch
       );
-      logger.verbose("Migrated dataColumnSidecars from hot DB to cold DB", {migratedEntries});
+      logger.verbose("Migrated dataColumnSidecars from hot DB to cold DB", {...logCtx, migratedEntries});
     }
   }
 
@@ -114,14 +117,14 @@ export async function archiveBlocks(
         nonCanonicalBlockRoots.map(async (root, index) => {
           const block = finalizedNonCanonicalBlocks[index];
           const blockBytes = await db.block.getBinary(root);
-          const logCtx = {slot: block.slot, root: block.blockRoot};
+          const blockLogCtx = {slot: block.slot, root: block.blockRoot};
           if (blockBytes) {
             await persistOrphanedBlock(block.slot, block.blockRoot, blockBytes, {
               persistOrphanedBlocksDir: persistOrphanedBlocksDir ?? "orphaned_blocks",
             });
-            logger.verbose("Persisted orphaned block", logCtx);
+            logger.verbose("Persisted orphaned block", {...logCtx, ...blockLogCtx});
           } else {
-            logger.warn("Tried to persist orphaned block but no block found", logCtx);
+            logger.warn("Tried to persist orphaned block but no block found", {...logCtx, ...blockLogCtx});
           }
         })
       );
@@ -129,17 +132,18 @@ export async function archiveBlocks(
 
     await db.block.batchDelete(nonCanonicalBlockRoots);
     logger.verbose("Deleted non canonical blocks from hot DB", {
+      ...logCtx,
       slots: finalizedNonCanonicalBlocks.map((summary) => summary.slot).join(","),
     });
 
     if (finalizedPostDeneb) {
       await db.blobSidecars.batchDelete(nonCanonicalBlockRoots);
-      logger.verbose("Deleted non canonical blobSidecars from hot DB");
+      logger.verbose("Deleted non canonical blobSidecars from hot DB", logCtx);
     }
 
     if (finalizedPostFulu) {
       await db.dataColumnSidecar.deleteMany(nonCanonicalBlockRoots);
-      logger.verbose("Deleted non canonical dataColumnSidecars from hot DB");
+      logger.verbose("Deleted non canonical dataColumnSidecars from hot DB", logCtx);
     }
   }
 
@@ -154,13 +158,13 @@ export async function archiveBlocks(
         const slotsToDelete = await db.blobSidecarsArchive.keys({lt: computeStartSlotAtEpoch(blobSidecarsMinEpoch)});
         if (slotsToDelete.length > 0) {
           await db.blobSidecarsArchive.batchDelete(slotsToDelete);
-          logger.verbose(`blobSidecars prune: batchDelete range ${slotsToDelete[0]}..${slotsToDelete.at(-1)}`);
+          logger.verbose(`blobSidecars prune: batchDelete range ${slotsToDelete[0]}..${slotsToDelete.at(-1)}`, logCtx);
         } else {
-          logger.verbose(`blobSidecars prune: no entries before epoch ${blobSidecarsMinEpoch}`);
+          logger.verbose(`blobSidecars prune: no entries before epoch ${blobSidecarsMinEpoch}`, logCtx);
         }
       }
     } else {
-      logger.verbose("blobSidecars pruning skipped: archiveDataEpochs set to Infinity");
+      logger.verbose("blobSidecars pruning skipped: archiveDataEpochs set to Infinity", logCtx);
     }
   }
 
@@ -184,20 +188,22 @@ export async function archiveBlocks(
         if (slotsToDelete.length > 0) {
           await db.dataColumnSidecarArchive.deleteMany(slotsToDelete);
           logger.verbose("dataColumnSidecars prune", {
+            ...logCtx,
             slotRange: prettyPrintIndices(slotsToDelete),
             numOfSlots: slotsToDelete.length,
             totalNumOfSidecars: prefixedKeys.length,
           });
         } else {
-          logger.verbose(`dataColumnSidecars prune: no entries before epoch ${dataColumnSidecarsMinEpoch}`);
+          logger.verbose(`dataColumnSidecars prune: no entries before epoch ${dataColumnSidecarsMinEpoch}`, logCtx);
         }
       } else {
         logger.verbose(
-          `dataColumnSidecars pruning skipped: ${dataColumnSidecarsMinEpoch} is before fulu fork epoch ${config.FULU_FORK_EPOCH}`
+          `dataColumnSidecars pruning skipped: ${dataColumnSidecarsMinEpoch} is before fulu fork epoch ${config.FULU_FORK_EPOCH}`,
+          logCtx
         );
       }
     } else {
-      logger.verbose("dataColumnSidecars pruning skipped: archiveDataEpochs set to Infinity");
+      logger.verbose("dataColumnSidecars pruning skipped: archiveDataEpochs set to Infinity", logCtx);
     }
   }
 
@@ -213,8 +219,8 @@ export async function archiveBlocks(
   }
 
   logger.verbose("Archiving of finalized blocks complete", {
+    ...logCtx,
     totalArchived: finalizedCanonicalBlocks.length,
-    finalizedEpoch: finalizedCheckpoint.epoch,
   });
 }
 
@@ -232,6 +238,7 @@ async function migrateBlocksFromHotToColdDb(db: IBeaconDb, blocks: BlockRootSlot
     // load Buffer instead of SignedBeaconBlock to improve performance
     const canonicalBlockEntries: BlockArchiveBatchPutBinaryItem[] = await Promise.all(
       canonicalBlocks.map(async (block) => {
+        // Here we assume the blocks are already in the hot db
         const blockBuffer = await db.block.getBinary(block.root);
         if (!blockBuffer) {
           throw Error(`Block not found for slot ${block.slot} root ${toRootHex(block.root)}`);
@@ -288,6 +295,8 @@ async function migrateBlobSidecarsFromHotToColdDb(
           );
         })
         .map(async (block) => {
+          // Here we assume the blob sidecars are already in the hot db
+          // instead of checking first the block input cache
           const bytes = await db.blobSidecars.getBinary(block.root);
           if (!bytes) {
             throw Error(`No blobSidecars found for slot ${block.slot} root ${toRootHex(block.root)}`);
@@ -337,9 +346,11 @@ async function migrateDataColumnSidecarsFromHotToColdDb(
         continue;
       }
 
+      // Here we assume the data column sidecars are already in the hot db
       const dataColumnSidecarBytes = await fromAsync(db.dataColumnSidecar.valuesStreamBinary(block.root));
       // there could be 0 dataColumnSidecarBytes if block has no blob
       logger.verbose("migrateDataColumnSidecarsFromHotToColdDb", {
+        currentEpoch,
         slot: block.slot,
         root: toRootHex(block.root),
         numSidecars: dataColumnSidecarBytes.length,

package/src/chain/blocks/blockInput/blockInput.ts

@@ -412,6 +412,10 @@ export class BlockInputBlobs extends AbstractBlockInput<ForkBlobsDA, deneb.BlobS
     return this.blobsCache.has(blobIndex);
   }
 
+  getBlob(blobIndex: BlobIndex): deneb.BlobSidecar | undefined {
+    return this.blobsCache.get(blobIndex)?.blobSidecar;
+  }
+
   addBlob(
     {blockRootHex, blobSidecar, source, peerIdStr, seenTimestampSec}: AddBlob,
     opts = {throwOnDuplicateAdd: true}
@@ -787,6 +791,10 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
     return this.columnsCache.has(columnIndex);
   }
 
+  getColumn(columnIndex: number): fulu.DataColumnSidecar | undefined {
+    return this.columnsCache.get(columnIndex)?.columnSidecar;
+  }
+
   getVersionedHashes(): VersionedHashes {
     return this.state.versionedHashes;
   }

package/src/chain/blocks/importBlock.ts

@@ -34,7 +34,6 @@ import {toCheckpointHex} from "../stateCache/index.js";
 import {isBlockInputBlobs, isBlockInputColumns} from "./blockInput/blockInput.js";
 import {AttestationImportOpt, FullyVerifiedBlock, ImportBlockOpts} from "./types.js";
 import {getCheckpointFromState} from "./utils/checkpoint.js";
-import {writeBlockInputToDb} from "./writeBlockInputToDb.js";
 
 /**
  * Fork-choice allows to import attestations from current (0) or past (1) epoch.
@@ -91,11 +90,8 @@ export async function importBlock(
     throw Error("Unavailable block can not be imported in forkchoice");
   }
 
-  // 1. Persist block to hot DB (pre-emptively)
-  // If eagerPersistBlock = true we do that in verifyBlocksInEpoch to batch all I/O operations to save block time to head
-  if (!opts.eagerPersistBlock) {
-    await writeBlockInputToDb.call(this, [blockInput]);
-  }
+  // 1. Persist block to hot DB (performed asynchronously to avoid blocking head selection)
+  void this.unfinalizedBlockWrites.push([blockInput]);
 
   // Without forcefully clearing this cache, we would rely on WeakMap to evict memory which is not reliable
   this.serializedCache.clear();
@@ -418,13 +414,20 @@ export async function importBlock(
     this.logger.verbose("After importBlock caching postState without SSZ cache", {slot: postState.slot});
   }
 
+  // Cache shufflings when crossing an epoch boundary
+  const parentEpoch = computeEpochAtSlot(parentBlockSlot);
+  if (parentEpoch < blockEpoch) {
+    this.shufflingCache.processState(postState);
+    this.logger.verbose("Processed shuffling for next epoch", {parentEpoch, blockEpoch, slot: blockSlot});
+  }
+
   if (blockSlot % SLOTS_PER_EPOCH === 0) {
     // Cache state to preserve epoch transition work
     const checkpointState = postState;
     const cp = getCheckpointFromState(checkpointState);
     this.regen.addCheckpointState(cp, checkpointState);
-    // consumers should not mutate or get the transfered cache
-    this.emitter.emit(ChainEvent.checkpoint, cp, checkpointState.clone(true));
+    // consumers should not mutate state ever
+    this.emitter.emit(ChainEvent.checkpoint, cp, checkpointState);
 
     // Note: in-lined code from previos handler of ChainEvent.checkpoint
     this.logger.verbose("Checkpoint processed", toCheckpointHex(cp));
@@ -584,7 +587,10 @@ export function addAttestationPostElectra(
       true
     );
   } else {
-    const committees = epochCtx.getBeaconCommittees(attestation.data.slot, committeeIndices);
+    const attSlot = attestation.data.slot;
+    const attEpoch = computeEpochAtSlot(attSlot);
+    const decisionRoot = epochCtx.getShufflingDecisionRoot(attEpoch);
+    const committees = this.shufflingCache.getBeaconCommittees(attEpoch, decisionRoot, attSlot, committeeIndices);
     const aggregationBools = attestation.aggregationBits.toBoolArray();
     let offset = 0;
     for (let i = 0; i < committees.length; i++) {

package/src/chain/blocks/index.ts

@@ -11,7 +11,6 @@ import {FullyVerifiedBlock, ImportBlockOpts} from "./types.js";
 import {assertLinearChainSegment} from "./utils/chainSegment.js";
 import {verifyBlocksInEpoch} from "./verifyBlock.js";
 import {verifyBlocksSanityChecks} from "./verifyBlocksSanityChecks.js";
-import {removeEagerlyPersistedBlockInputs} from "./writeBlockInputToDb.js";
 
 export {AttestationImportOpt, type ImportBlockOpts} from "./types.js";
 
@@ -143,24 +142,6 @@ export async function processBlocks(
       }
     }
 
-    // Clean db if we don't have blocks in forkchoice but already persisted them to db
-    //
-    // NOTE: this function is awaited to ensure that DB size remains constant, otherwise an attacker may bloat the
-    // disk with big malicious payloads. Our sequential block importer will wait for this promise before importing
-    // another block. The removal call error is not propagated since that would halt the chain.
-    //
-    // LOG: Because the error is not propagated and there's a risk of db bloat, the error is logged at warn level
-    // to alert the user of potential db bloat. This error _should_ never happen user must act and report to us
-    if (opts.eagerPersistBlock) {
-      await removeEagerlyPersistedBlockInputs.call(this, blocks).catch((e) => {
-        this.logger.warn(
-          "Error pruning eagerly imported block inputs, DB may grow in size if this error happens frequently",
-          {slot: blocks.map((block) => block.getBlock().message.slot).join(",")},
-          e
-        );
-      });
-    }
-
     throw err;
   }
 }

package/src/chain/blocks/types.ts

@@ -78,8 +78,6 @@ export type ImportBlockOpts = {
   validBlobSidecars?: BlobSidecarValidation;
   /** Seen timestamp seconds */
   seenTimestampSec?: number;
-  /** Set to true if persist block right at verification time */
-  eagerPersistBlock?: boolean;
 };
 
 /**

package/src/chain/blocks/verifyBlock.ts

@@ -21,7 +21,6 @@ import {verifyBlocksDataAvailability} from "./verifyBlocksDataAvailability.js";
 import {SegmentExecStatus, verifyBlocksExecutionPayload} from "./verifyBlocksExecutionPayloads.js";
 import {verifyBlocksSignatures} from "./verifyBlocksSignatures.js";
 import {verifyBlocksStateTransitionOnly} from "./verifyBlocksStateTransitionOnly.js";
-import {writeBlockInputToDb} from "./writeBlockInputToDb.js";
 
 /**
  * Verifies 1 or more blocks are fully valid; from a linear sequence of blocks.
@@ -75,6 +74,10 @@ export async function verifyBlocksInEpoch(
       throw new BlockError(block0, {code: BlockErrorCode.PRESTATE_MISSING, error: e as Error});
     });
 
+  // in forky condition, make sure to populate ShufflingCache with regened state
+  // otherwise it may fail to get indexed attestations from shuffling cache later
+  this.shufflingCache.processState(preState0);
+
   if (!isStateValidatorsNodesPopulated(preState0)) {
     this.logger.verbose("verifyBlocksInEpoch preState0 SSZ cache stats", {
       slot: preState0.slot,
@@ -105,9 +108,11 @@ export async function verifyBlocksInEpoch(
     // Store indexed attestations for each block to avoid recomputing them during import
     const indexedAttestationsByBlock: IndexedAttestation[][] = [];
     for (const [i, block] of blocks.entries()) {
-      indexedAttestationsByBlock[i] = block.message.body.attestations.map((attestation) =>
-        preState0.epochCtx.getIndexedAttestation(fork, attestation)
-      );
+      indexedAttestationsByBlock[i] = block.message.body.attestations.map((attestation) => {
+        const attEpoch = computeEpochAtSlot(attestation.data.slot);
+        const decisionRoot = preState0.epochCtx.getShufflingDecisionRoot(attEpoch);
+        return this.shufflingCache.getIndexedAttestation(attEpoch, decisionRoot, fork, attestation);
+      });
     }
 
     // batch all I/O operations to reduce overhead
@@ -150,13 +155,6 @@ export async function verifyBlocksInEpoch(
             opts
           )
         : Promise.resolve({verifySignaturesTime: Date.now()}),
-
-      // ideally we want to only persist blocks after verifying them however the reality is there are
-      // rarely invalid blocks we'll batch all I/O operation here to reduce the overhead if there's
-      // an error, we'll remove blocks not in forkchoice
-      opts.verifyOnly !== true && opts.eagerPersistBlock
-        ? writeBlockInputToDb.call(this, blockInputs)
-        : Promise.resolve(),
     ]);
 
     if (opts.verifyOnly !== true) {

package/src/chain/blocks/verifyBlocksStateTransitionOnly.ts

@@ -59,6 +59,7 @@ export async function verifyBlocksStateTransitionOnly(
         // if block is trusted don't verify proposer or op signature
         verifyProposer: !useBlsBatchVerify && !validSignatures && !validProposerSignature,
         verifySignatures: !useBlsBatchVerify && !validSignatures,
+        dontTransferCache: false,
       },
       {metrics, validatorMonitor}
     );

package/src/chain/blocks/writeBlockInputToDb.ts

@@ -98,35 +98,29 @@ export async function writeBlockInputToDb(this: BeaconChain, blocksInputs: IBloc
   }
 }
 
-/**
- * Prunes eagerly persisted block inputs only if not known to the fork-choice
- */
-export async function removeEagerlyPersistedBlockInputs(this: BeaconChain, blockInputs: IBlockInput[]): Promise<void> {
-  const blockToRemove = [];
-  const blobsToRemove = [];
-  const dataColumnsToRemove = [];
-
-  for (const blockInput of blockInputs) {
-    const block = blockInput.getBlock();
-    const slot = block.message.slot;
-    const blockRoot = this.config.getForkTypes(slot).BeaconBlock.hashTreeRoot(block.message);
-    const blockRootHex = toRootHex(blockRoot);
-    if (!this.forkChoice.hasBlockHex(blockRootHex)) {
-      blockToRemove.push(block);
-
-      if (isBlockInputColumns(blockInput) && blockInput.getCustodyColumns().length > 0) {
-        dataColumnsToRemove.push(blockRoot);
-      } else if (isBlockInputBlobs(blockInput)) {
-        const blobSidecars = blockInput.getBlobs();
-        blobsToRemove.push({blockRoot, slot, blobSidecars});
+export async function persistBlockInputs(this: BeaconChain, blockInputs: IBlockInput[]): Promise<void> {
+  await writeBlockInputToDb
+    .call(this, blockInputs)
+    .catch((e) => {
+      this.logger.debug(
+        "Error persisting block input in hot db",
+        {
+          count: blockInputs.length,
+          slot: blockInputs[0].slot,
+          root: blockInputs[0].blockRootHex,
+        },
+        e
+      );
+    })
+    .finally(() => {
+      for (const blockInput of blockInputs) {
+        this.seenBlockInputCache.prune(blockInput.blockRootHex);
       }
-    }
-  }
-
-  await Promise.all([
-    // TODO: Batch DB operations not with Promise.all but with level db ops
-    this.db.block.batchRemove(blockToRemove),
-    this.db.blobSidecars.batchRemove(blobsToRemove),
-    this.db.dataColumnSidecar.deleteMany(dataColumnsToRemove),
-  ]);
+      if (blockInputs.length === 1) {
+        this.logger.debug("Pruned block input", {
+          slot: blockInputs[0].slot,
+          root: blockInputs[0].blockRootHex,
+        });
+      }
+    });
 }