@lodestar/beacon-node 1.43.0-dev.e341cdc614 → 1.43.0-dev.f758dd5f38

package/src/chain/archiveStore/interface.ts

@@ -1,4 +1,4 @@
-import {CheckpointWithPayloadStatus} from "@lodestar/fork-choice";
+import {CheckpointWithHex} from "@lodestar/fork-choice";
 import {RootHex} from "@lodestar/types";
 import {Metrics} from "../../metrics/metrics.js";
 
@@ -45,9 +45,9 @@ export type FinalizedStats = {
 
 export interface StateArchiveStrategy {
   onCheckpoint(stateRoot: RootHex, metrics?: Metrics | null): Promise<void>;
-  onFinalizedCheckpoint(finalized: CheckpointWithPayloadStatus, metrics?: Metrics | null): Promise<void>;
-  maybeArchiveState(finalized: CheckpointWithPayloadStatus, metrics?: Metrics | null): Promise<void>;
-  archiveState(finalized: CheckpointWithPayloadStatus, metrics?: Metrics | null): Promise<void>;
+  onFinalizedCheckpoint(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise<void>;
+  maybeArchiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise<void>;
+  archiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise<void>;
 }
 
 export interface IArchiveStore {

package/src/chain/archiveStore/strategies/frequencyStateArchiveStrategy.ts

@@ -1,4 +1,4 @@
-import {CheckpointWithPayloadStatus} from "@lodestar/fork-choice";
+import {CheckpointWithHex} from "@lodestar/fork-choice";
 import {SLOTS_PER_EPOCH} from "@lodestar/params";
 import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition";
 import {Epoch, RootHex, Slot} from "@lodestar/types";
@@ -40,7 +40,7 @@ export class FrequencyStateArchiveStrategy implements StateArchiveStrategy {
     private readonly bufferPool?: BufferPool | null
   ) {}
 
-  async onFinalizedCheckpoint(_finalized: CheckpointWithPayloadStatus, _metrics?: Metrics | null): Promise<void> {}
+  async onFinalizedCheckpoint(_finalized: CheckpointWithHex, _metrics?: Metrics | null): Promise<void> {}
   async onCheckpoint(_stateRoot: RootHex, _metrics?: Metrics | null): Promise<void> {}
 
   /**
@@ -55,7 +55,7 @@ export class FrequencyStateArchiveStrategy implements StateArchiveStrategy {
    * epoch - 1024*2    epoch - 1024    epoch - 32    epoch
    * ```
    */
-  async maybeArchiveState(finalized: CheckpointWithPayloadStatus, metrics?: Metrics | null): Promise<void> {
+  async maybeArchiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise<void> {
     let timer = metrics?.processFinalizedCheckpoint.frequencyStateArchive.startTimer();
     const lastStoredSlot = await this.db.stateArchive.lastKey();
     timer?.({step: FrequencyStateArchiveStep.LoadLastStoredSlot});
@@ -104,7 +104,7 @@ export class FrequencyStateArchiveStrategy implements StateArchiveStrategy {
    * Archives finalized states from active bucket to archive bucket.
    * Only the new finalized state is stored to disk
    */
-  async archiveState(finalized: CheckpointWithPayloadStatus, metrics?: Metrics | null): Promise<void> {
+  async archiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise<void> {
     // starting from Mar 2024, the finalized state could be from disk or in memory
     let timer = metrics?.processFinalizedCheckpoint.frequencyStateArchive.startTimer();
     const finalizedHex = {epoch: finalized.epoch, rootHex: finalized.rootHex};

package/src/chain/archiveStore/utils/archiveBlocks.ts

@@ -1,7 +1,7 @@
 import path from "node:path";
 import {ChainForkConfig} from "@lodestar/config";
 import {KeyValue} from "@lodestar/db";
-import {CheckpointWithPayloadStatus, IForkChoice, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice";
+import {CheckpointWithHex, IForkChoice, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice";
 import {ForkSeq, SLOTS_PER_EPOCH} from "@lodestar/params";
 import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition";
 import {Epoch, Slot} from "@lodestar/types";
@@ -52,16 +52,17 @@ export async function archiveBlocks(
   forkChoice: IForkChoice,
   lightclientServer: LightClientServer | undefined,
   logger: Logger,
-  finalizedCheckpoint: CheckpointWithPayloadStatus,
+  finalizedCheckpoint: CheckpointWithHex,
   currentEpoch: Epoch,
   archiveDataEpochs?: number,
   persistOrphanedBlocks?: boolean,
   persistOrphanedBlocksDir?: string
 ): Promise<void> {
-  // Use fork choice to determine the blocks to archive and delete
-  // getAllAncestorBlocks response includes the finalized block, so it's also moved to the cold db
+  // Use fork choice to determine the blocks to archive and delete.
+  // `ancestors` is the canonical walk back from the finalized root, including the previous finalized
+  // block as its last element.
   const {ancestors: finalizedCanonicalBlocks, nonAncestors: finalizedNonCanonicalBlocks} =
-    forkChoice.getAllAncestorAndNonAncestorBlocks(finalizedCheckpoint.rootHex, finalizedCheckpoint.payloadStatus);
+    forkChoice.getAllAncestorAndNonAncestorBlocksDefaultStatus(finalizedCheckpoint.rootHex);
 
   // NOTE: The finalized block will be exactly the first block of `epoch` or previous
   const finalizedPostDeneb = finalizedCheckpoint.epoch >= config.DENEB_FORK_EPOCH;
@@ -76,18 +77,21 @@ export async function archiveBlocks(
   const logCtx = {currentEpoch, finalizedEpoch: finalizedCheckpoint.epoch, finalizedRoot: finalizedCheckpoint.rootHex};
 
   if (finalizedCanonicalBlockRoots.length > 0) {
-    await migrateBlocksFromHotToColdDb(db, finalizedCanonicalBlockRoots);
+    const migratedSlots = await migrateBlocksFromHotToColdDb(db, logger, finalizedCanonicalBlockRoots);
     logger.verbose("Migrated blocks from hot DB to cold DB", {
       ...logCtx,
       fromSlot: finalizedCanonicalBlockRoots[0].slot,
       toSlot: finalizedCanonicalBlockRoots.at(-1)?.slot,
       size: finalizedCanonicalBlockRoots.length,
+      migratedEntries: migratedSlots.length,
+      slotRange: prettyPrintIndices(migratedSlots),
     });
 
     if (finalizedPostDeneb) {
       const migratedEntries = await migrateBlobSidecarsFromHotToColdDb(
         config,
         db,
+        logger,
         finalizedCanonicalBlockRoots,
         currentEpoch
       );
@@ -95,24 +99,32 @@ export async function archiveBlocks(
     }
 
     if (finalizedPostFulu) {
-      const migratedEntries = await migrateDataColumnSidecarsFromHotToColdDb(
+      const migratedSlots = await migrateDataColumnSidecarsFromHotToColdDb(
         config,
         db,
         logger,
-        finalizedCanonicalBlockRoots,
+        finalizedCanonicalBlocks,
         currentEpoch
       );
-      logger.verbose("Migrated dataColumnSidecars from hot DB to cold DB", {...logCtx, migratedEntries});
+      logger.verbose("Migrated dataColumnSidecars from hot DB to cold DB", {
+        ...logCtx,
+        migratedEntries: migratedSlots.length,
+        slotRange: prettyPrintIndices(migratedSlots),
+      });
     }
 
     if (finalizedPostGloas) {
-      const migratedEntries = await migrateExecutionPayloadEnvelopesFromHotToColdDb(
+      const migratedSlots = await migrateExecutionPayloadEnvelopesFromHotToColdDb(
         config,
         db,
         logger,
         finalizedCanonicalBlocks
       );
-      logger.verbose("Migrated executionPayloadEnvelopes from hot DB to cold DB", {...logCtx, migratedEntries});
+      logger.verbose("Migrated executionPayloadEnvelopes from hot DB to cold DB", {
+        ...logCtx,
+        migratedEntries: migratedSlots.length,
+        slotRange: prettyPrintIndices(migratedSlots),
+      });
     }
   }
 
@@ -140,25 +152,29 @@ export async function archiveBlocks(
       );
     }
 
-    await db.block.batchDelete(nonCanonicalBlockRoots);
-    logger.verbose("Deleted non canonical blocks from hot DB", {
+    const nonCanonicalSlots = finalizedNonCanonicalBlocks.map((summary) => summary.slot).sort((a, b) => a - b);
+    const nonCanonicalLogCtx = {
       ...logCtx,
-      slots: finalizedNonCanonicalBlocks.map((summary) => summary.slot).join(","),
-    });
+      count: nonCanonicalBlockRoots.length,
+      slotRange: prettyPrintIndices(nonCanonicalSlots),
+    };
+
+    await db.block.batchDelete(nonCanonicalBlockRoots);
+    logger.verbose("Deleted non canonical blocks from hot DB", nonCanonicalLogCtx);
 
     if (finalizedPostDeneb) {
       await db.blobSidecars.batchDelete(nonCanonicalBlockRoots);
-      logger.verbose("Deleted non canonical blobSidecars from hot DB", logCtx);
+      logger.verbose("Deleted non canonical blobSidecars from hot DB", nonCanonicalLogCtx);
     }
 
     if (finalizedPostFulu) {
       await db.dataColumnSidecar.deleteMany(nonCanonicalBlockRoots);
-      logger.verbose("Deleted non canonical dataColumnSidecars from hot DB", logCtx);
+      logger.verbose("Deleted non canonical dataColumnSidecars from hot DB", nonCanonicalLogCtx);
     }
 
     if (finalizedPostGloas) {
       await db.executionPayloadEnvelope.batchDelete(nonCanonicalBlockRoots);
-      logger.verbose("Deleted non canonical executionPayloadEnvelopes from hot DB", logCtx);
+      logger.verbose("Deleted non canonical executionPayloadEnvelopes from hot DB", nonCanonicalLogCtx);
     }
   }
 
@@ -239,42 +255,51 @@ export async function archiveBlocks(
   });
 }
 
-async function migrateBlocksFromHotToColdDb(db: IBeaconDb, blocks: BlockRootSlot[]): Promise<void> {
-  // Start from `i=0`: 1st block in iterateAncestorBlocks() is the finalized block itself
-  // we move it to blockArchive but forkchoice still have it to check next onBlock calls
-  // the next iterateAncestorBlocks call does not return this block
+async function migrateBlocksFromHotToColdDb(db: IBeaconDb, logger: Logger, blocks: BlockRootSlot[]): Promise<Slot[]> {
+  // The input includes the previous finalized block as the last ancestor; its SignedBeaconBlock
+  // was archived on a previous run and is no longer in hot db. `getBinary` returning null for any
+  // block in the batch is therefore treated as "already migrated, skip" rather than an error.
+  const migratedSlots: Slot[] = [];
   for (let i = 0; i < blocks.length; i += BLOCK_BATCH_SIZE) {
     const toIdx = Math.min(i + BLOCK_BATCH_SIZE, blocks.length);
     const canonicalBlocks = blocks.slice(i, toIdx);
 
-    // processCanonicalBlocks
-    if (canonicalBlocks.length === 0) return;
+    if (canonicalBlocks.length === 0) break;
 
     // load Buffer instead of SignedBeaconBlock to improve performance
-    const canonicalBlockEntries: BlockArchiveBatchPutBinaryItem[] = await Promise.all(
-      canonicalBlocks.map(async (block) => {
-        // Here we assume the blocks are already in the hot db
-        const blockBuffer = await db.block.getBinary(block.root);
-        if (!blockBuffer) {
-          throw Error(`Block not found for slot ${block.slot} root ${toRootHex(block.root)}`);
-        }
-        return {
-          key: block.slot,
-          value: blockBuffer,
-          slot: block.slot,
-          blockRoot: block.root,
-          // TODO: Benchmark if faster to slice Buffer or fromHex()
-          parentRoot: getParentRootFromSignedBlock(blockBuffer),
-        };
-      })
-    );
+    const canonicalBlockEntries = (
+      await Promise.all(
+        canonicalBlocks.map(async (block): Promise<BlockArchiveBatchPutBinaryItem | null> => {
+          const blockBuffer = await db.block.getBinary(block.root);
+          if (!blockBuffer) {
+            logger.debug("Block in forkchoice but missing in hot db, could be already archived", {
+              slot: block.slot,
+              root: toRootHex(block.root),
+            });
+            return null;
+          }
+          return {
+            key: block.slot,
+            value: blockBuffer,
+            slot: block.slot,
+            blockRoot: block.root,
+            // TODO: Benchmark if faster to slice Buffer or fromHex()
+            parentRoot: getParentRootFromSignedBlock(blockBuffer),
+          };
+        })
+      )
+    ).filter((entry): entry is BlockArchiveBatchPutBinaryItem => entry !== null);
+
+    if (canonicalBlockEntries.length === 0) continue;
 
-    // put to blockArchive db and delete block db
     await Promise.all([
       db.blockArchive.batchPutBinary(canonicalBlockEntries),
-      db.block.batchDelete(canonicalBlocks.map((block) => block.root)),
+      db.block.batchDelete(canonicalBlockEntries.map((entry) => entry.blockRoot)),
     ]);
+    for (const entry of canonicalBlockEntries) migratedSlots.push(entry.slot);
   }
+  // Ancestor walk is newest → oldest; sort ascending so `prettyPrintIndices` renders cleanly.
+  return migratedSlots.sort((a, b) => a - b);
 }
 
 /**
@@ -284,6 +309,7 @@ async function migrateBlocksFromHotToColdDb(db: IBeaconDb, blocks: BlockRootSlot
 async function migrateBlobSidecarsFromHotToColdDb(
   config: ChainForkConfig,
   db: IBeaconDb,
+  logger: Logger,
   blocks: BlockRootSlot[],
   currentEpoch: Epoch
 ): Promise<number> {
@@ -296,29 +322,36 @@ async function migrateBlobSidecarsFromHotToColdDb(
     if (canonicalBlocks.length === 0) break;
 
     // load Buffer instead of ssz deserialized to improve performance
-    const canonicalBlobSidecarsEntries: KeyValue<Slot, Uint8Array>[] = await Promise.all(
-      canonicalBlocks
-        .filter((block) => {
-          const blockSlot = block.slot;
-          const blockEpoch = computeEpochAtSlot(blockSlot);
-          const forkSeq = config.getForkSeq(blockSlot);
-          return (
-            forkSeq >= ForkSeq.deneb &&
-            forkSeq < ForkSeq.fulu &&
-            // if block is out of ${config.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS}, skip this step
-            blockEpoch >= currentEpoch - config.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS
-          );
-        })
-        .map(async (block) => {
-          // Here we assume the blob sidecars are already in the hot db
-          // instead of checking first the block input cache
-          const bytes = await db.blobSidecars.getBinary(block.root);
-          if (!bytes) {
-            throw Error(`No blobSidecars found for slot ${block.slot} root ${toRootHex(block.root)}`);
-          }
-          return {key: block.slot, value: bytes};
-        })
-    );
+    const canonicalBlobSidecarsEntries: KeyValue<Slot, Uint8Array>[] = (
+      await Promise.all(
+        canonicalBlocks
+          .filter((block) => {
+            const blockSlot = block.slot;
+            const blockEpoch = computeEpochAtSlot(blockSlot);
+            const forkSeq = config.getForkSeq(blockSlot);
+            return (
+              forkSeq >= ForkSeq.deneb &&
+              forkSeq < ForkSeq.fulu &&
+              // if block is out of ${config.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS}, skip this step
+              blockEpoch >= currentEpoch - config.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS
+            );
+          })
+          .map(async (block): Promise<KeyValue<Slot, Uint8Array> | null> => {
+            // The ancestor walk includes the boundary (previous finalized) block; on first
+            // finalization that boundary is the anchor which has no blob sidecars in hot db.
+            // Treat a null hot-db entry as "nothing to migrate" rather than an error.
+            const bytes = await db.blobSidecars.getBinary(block.root);
+            if (!bytes) {
+              logger.debug("BlobSidecars in forkchoice but missing in hot db, could be already archived", {
+                slot: block.slot,
+                root: toRootHex(block.root),
+              });
+              return null;
+            }
+            return {key: block.slot, value: bytes};
+          })
+      )
+    ).filter((e): e is KeyValue<Slot, Uint8Array> => e !== null);
 
     // put to blockArchive db and delete block db
     await Promise.all([
@@ -332,24 +365,36 @@ async function migrateBlobSidecarsFromHotToColdDb(
 }
 
 // TODO: This function can be simplified further by reducing layers of promises in a loop
+/**
+ * Post-gloas the data columns of a Gloas block are tied to its execution payload envelope —
+ * columns only exist once the FULL variant of the block is in the proto-array. Pre-Gloas (Fulu)
+ * blocks only have a FULL variant, so the `payloadStatus === FULL` filter passes them all.
+ * Blocks whose canonical variant is PENDING/EMPTY are skipped here — their columns will be picked
+ * up on a later run once the FULL variant appears in the ancestor walk.
+ */
 async function migrateDataColumnSidecarsFromHotToColdDb(
   config: ChainForkConfig,
   db: IBeaconDb,
   logger: Logger,
-  blocks: BlockRootSlot[],
+  canonicalBlocks: ProtoBlock[],
   currentEpoch: Epoch
-): Promise<number> {
-  let migratedWrappedDataColumns = 0;
+): Promise<Slot[]> {
+  const columnBlocks = canonicalBlocks.filter(
+    (block) => config.getForkSeq(block.slot) < ForkSeq.gloas || block.payloadStatus === PayloadStatus.FULL
+  );
+  if (columnBlocks.length === 0) return [];
+  const blocks: BlockRootSlot[] = columnBlocks.map((block) => ({slot: block.slot, root: fromHex(block.blockRoot)}));
+
+  const migratedSlots: Slot[] = [];
   for (let i = 0; i < blocks.length; i += BLOB_SIDECAR_BATCH_SIZE) {
     const toIdx = Math.min(i + BLOB_SIDECAR_BATCH_SIZE, blocks.length);
-    const canonicalBlocks = blocks.slice(i, toIdx);
+    const batch = blocks.slice(i, toIdx);
+    if (batch.length === 0) break;
 
-    // processCanonicalBlocks
-    if (canonicalBlocks.length === 0) break;
-    const promises = [];
+    const promises: Promise<void>[] = [];
 
     // load Buffer instead of ssz deserialized to improve performance
-    for (const block of canonicalBlocks) {
+    for (const block of batch) {
       const blockSlot = block.slot;
       const blockEpoch = computeEpochAtSlot(blockSlot);
 
@@ -363,8 +408,16 @@ async function migrateDataColumnSidecarsFromHotToColdDb(
 
       // Here we assume the data column sidecars are already in the hot db
       const dataColumnSidecarBytes = await fromAsync(db.dataColumnSidecar.valuesStreamBinary(block.root));
-      // there could be 0 dataColumnSidecarBytes if block has no blob
-      logger.verbose("migrateDataColumnSidecarsFromHotToColdDb", {
+      if (dataColumnSidecarBytes.length === 0) {
+        // Empty stream: either the block has no blobs, or columns were already archived on a
+        // previous run (boundary block). Nothing to migrate.
+        logger.debug("DataColumnSidecars in forkchoice but missing in hot db, could be already archived", {
+          slot: block.slot,
+          root: toRootHex(block.root),
+        });
+        continue;
+      }
+      logger.verbose("Migrated dataColumnSidecars for block", {
         currentEpoch,
         slot: block.slot,
         root: toRootHex(block.root),
@@ -376,30 +429,32 @@ async function migrateDataColumnSidecarsFromHotToColdDb(
           dataColumnSidecarBytes.map((p) => ({key: p.id, value: p.value}))
         )
       );
-      migratedWrappedDataColumns += dataColumnSidecarBytes.length;
+      migratedSlots.push(block.slot);
     }
 
-    promises.push(db.dataColumnSidecar.deleteMany(canonicalBlocks.map((block) => block.root)));
+    promises.push(db.dataColumnSidecar.deleteMany(batch.map((block) => block.root)));
 
-    // put to blockArchive db and delete block db
     await Promise.all(promises);
   }
 
-  return migratedWrappedDataColumns;
+  // Ancestor walk is newest → oldest; sort ascending so `prettyPrintIndices` renders cleanly.
+  return migratedSlots.sort((a, b) => a - b);
 }
 
+/**
+ * Post-gloas given a finalized checkpoint at a block root, payload of that block root
+ * is not considered finalized, hence they are archived in the next run.
+ */
 async function migrateExecutionPayloadEnvelopesFromHotToColdDb(
   config: ChainForkConfig,
   db: IBeaconDb,
   logger: Logger,
   canonicalBlocks: ProtoBlock[]
-): Promise<number> {
-  let migratedEnvelopes = 0;
-
+): Promise<Slot[]> {
   const payloadBlocks = canonicalBlocks.filter(
-    (block) => config.getForkSeq(block.slot) >= ForkSeq.gloas && block.payloadStatus === PayloadStatus.FULL
+    (block) => config.getForkSeq(block.slot) < ForkSeq.gloas || block.payloadStatus === PayloadStatus.FULL
   );
-  if (payloadBlocks.length === 0) return 0;
+  if (payloadBlocks.length === 0) return [];
   const blocks = payloadBlocks.map((block) => ({slot: block.slot, root: fromHex(block.blockRoot)}));
 
   const envelopeEntries: KeyValue<Slot, Uint8Array>[] = [];
@@ -415,19 +470,23 @@ async function migrateExecutionPayloadEnvelopesFromHotToColdDb(
       envelopeEntries.push({key: blocks[i].slot, value: bytes});
       migratedRoots.push(blocks[i].root);
     } else {
-      logger.debug("Payload in forkchoice but missing in db", {slot: blocks[i].slot, root: toRootHex(blocks[i].root)});
+      logger.debug("ExecutionPayloadEnvelope in forkchoice but missing in hot db, could be already archived", {
+        slot: blocks[i].slot,
+        root: toRootHex(blocks[i].root),
+      });
     }
   }
 
-  if (envelopeEntries.length > 0) {
-    await Promise.all([
-      db.executionPayloadEnvelopeArchive.batchPutBinary(envelopeEntries),
-      db.executionPayloadEnvelope.batchDelete(migratedRoots),
-    ]);
-    migratedEnvelopes = envelopeEntries.length;
-  }
+  if (envelopeEntries.length === 0) return [];
+
+  await Promise.all([
+    db.executionPayloadEnvelopeArchive.batchPutBinary(envelopeEntries),
+    db.executionPayloadEnvelope.batchDelete(migratedRoots),
+  ]);
 
-  return migratedEnvelopes;
+  // Slots are ascending in hot-db key order — sort to guarantee `prettyPrintIndices` output is clean
+  // regardless of ancestor-walk order (newest → oldest).
+  return envelopeEntries.map((entry) => entry.key).sort((a, b) => a - b);
 }
 
 /**