@lodestar/beacon-node 1.51.0-dev.a1662c7de7 → 1.51.0-dev.a6892012c4

package/src/chain/archiveStore/utils/archiveBlocks.ts

@@ -1,11 +1,18 @@
 import path from "node:path";
 import {ChainForkConfig} from "@lodestar/config";
 import {KeyValue} from "@lodestar/db";
-import {CheckpointWithHex, ExecutionStatus, IForkChoice, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice";
+import {
+  CheckpointWithHex,
+  ExecutionStatus,
+  IForkChoice,
+  PayloadStatus,
+  ProtoBlock,
+  isGloasBlock,
+} from "@lodestar/fork-choice";
 import {ForkSeq, SLOTS_PER_EPOCH} from "@lodestar/params";
 import {computeEpochAtSlot, computeStartSlotAtEpoch, getBlobDataRetentionStartEpoch} from "@lodestar/state-transition";
 import {Epoch, Slot} from "@lodestar/types";
-import {Logger, fromAsync, fromHex, prettyPrintIndices, toRootHex} from "@lodestar/utils";
+import {Logger, fromHex, prettyPrintIndices, toRootHex} from "@lodestar/utils";
 import {IBeaconDb} from "../../../db/index.js";
 import {BlockArchiveBatchPutBinaryItem, encodeArchivedHeaderEnvelope} from "../../../db/repositories/index.js";
 import {Metrics} from "../../../metrics/metrics.js";
@@ -49,6 +56,17 @@ async function persistOrphanedBlock(
   await writeIfNotExist(filepath, bytes);
 }
 
+/**
+ * Oldest epoch for which data column sidecars are kept
+ */
+export function getDataColumnSidecarsMinEpoch(
+  config: ChainForkConfig,
+  currentEpoch: Epoch,
+  archiveDataEpochs?: number
+): Epoch {
+  return Math.min(getBlobDataRetentionStartEpoch(config, currentEpoch), currentEpoch - (archiveDataEpochs ?? 0));
+}
+
 /**
  * Archives finalized blocks from active bucket to archive bucket.
  *
@@ -136,23 +154,6 @@ export async function archiveBlocks(
       logger.verbose("Migrated blobSidecars from hot DB to cold DB", {...logCtx, migratedEntries});
     }
 
-    // Keep the normal LevelDB hot-to-cold flow for legacy columns. New columns are written
-    // directly to flat files, so this only moves fallback data left by older versions.
-    if (finalizedPostFulu) {
-      const migratedSlots = await migrateDataColumnSidecarsFromHotToColdDb(
-        config,
-        db,
-        logger,
-        finalizedCanonicalBlocks,
-        currentEpoch
-      );
-      logger.verbose("Migrated legacy dataColumnSidecars from hot DB to cold DB", {
-        ...logCtx,
-        migratedEntries: migratedSlots.length,
-        slotRange: prettyPrintIndices(migratedSlots),
-      });
-    }
-
     if (finalizedPostGloas) {
       const migratedSlots = await migrateExecutionPayloadEnvelopesFromHotToColdDb(
         config,
@@ -169,44 +170,34 @@ export async function archiveBlocks(
     }
   }
 
-  // deleteNonCanonicalBlocks
-  // loop through forkchoice single time
+  const canonicalRoots = new Set(finalizedCanonicalBlocks.map((block) => block.blockRoot));
+  const orphanedBlocks: ProtoBlock[] = [];
+  const orphanedPayloads: ProtoBlock[] = [];
 
-  const nonCanonicalBlockRoots = finalizedNonCanonicalBlocks.map((summary) => fromHex(summary.blockRoot));
-  if (nonCanonicalBlockRoots.length > 0) {
-    if (persistOrphanedBlocks) {
-      // Persist orphaned blocks to disk before deleting them from hot db
-      await Promise.all(
-        nonCanonicalBlockRoots.map(async (root, index) => {
-          const block = finalizedNonCanonicalBlocks[index];
-          const blockBytes = await db.block.getBinary(root);
-          const blockLogCtx = {slot: block.slot, root: block.blockRoot};
-          if (blockBytes) {
-            await persistOrphanedBlock(block.slot, block.blockRoot, blockBytes, {
-              persistOrphanedBlocksDir: persistOrphanedBlocksDir ?? "orphaned_blocks",
-            });
-            logger.verbose("Persisted orphaned block", {...logCtx, ...blockLogCtx});
-          } else {
-            logger.warn("Tried to persist orphaned block but no block found", {...logCtx, ...blockLogCtx});
-          }
-        })
-      );
+  for (const block of finalizedNonCanonicalBlocks) {
+    if (block.payloadStatus === PayloadStatus.FULL) {
+      orphanedPayloads.push(block);
     }
+    // Every Gloas block gets EMPTY before FULL. If only FULL survives pruning, an earlier archive run
+    // already handled the root's block-level data.
+    const isBlockNode = !isGloasBlock(block) || block.payloadStatus === PayloadStatus.EMPTY;
+    if (isBlockNode && !canonicalRoots.has(block.blockRoot)) {
+      orphanedBlocks.push(block);
+    }
+  }
+  const orphanedBlockRoots = orphanedBlocks.map((block) => fromHex(block.blockRoot));
 
-    const nonCanonicalSlots = finalizedNonCanonicalBlocks.map((summary) => summary.slot).sort((a, b) => a - b);
-    const nonCanonicalLogCtx = {
+  if (orphanedPayloads.length > 0) {
+    const orphanedPayloadRoots = orphanedPayloads.map((block) => fromHex(block.blockRoot));
+    const orphanedPayloadLogCtx = {
       ...logCtx,
-      count: nonCanonicalBlockRoots.length,
-      slotRange: prettyPrintIndices(nonCanonicalSlots),
+      count: orphanedPayloads.length,
+      slotRange: prettyPrintIndices(orphanedPayloads.map((block) => block.slot).sort((a, b) => a - b)),
     };
 
-    const columnItems = finalizedNonCanonicalBlocks
-      // Gloas EMPTY and FULL variants share a block root. EMPTY has no sidecars, so deleting by its root could
-      // remove the canonical FULL variant's columns. Pre-Gloas blocks are always FULL.
-      .filter(
-        (summary) => config.getForkSeq(summary.slot) >= ForkSeq.fulu && summary.payloadStatus === PayloadStatus.FULL
-      )
-      .map((summary) => ({slot: summary.slot, blockRoot: summary.blockRoot}));
+    const columnItems = orphanedPayloads
+      .filter((block) => config.getForkSeq(block.slot) >= ForkSeq.fulu)
+      .map((block) => ({slot: block.slot, blockRoot: block.blockRoot}));
     if (columnItems.length > 0) {
       // Delete sidecars first so their block roots remain available to retry cleanup after a failure or crash.
       await db.dataColumns.deleteMany(columnItems);
@@ -217,20 +208,44 @@ export async function archiveBlocks(
       });
     }
 
-    await db.block.batchDelete(nonCanonicalBlockRoots);
-    logger.verbose("Deleted non canonical blocks from hot DB", nonCanonicalLogCtx);
-
     if (finalizedPostDeneb) {
-      await db.blobSidecars.batchDelete(nonCanonicalBlockRoots);
-      logger.verbose("Deleted non canonical blobSidecars from hot DB", nonCanonicalLogCtx);
+      await db.blobSidecars.batchDelete(orphanedPayloadRoots);
+      logger.verbose("Deleted non canonical blobSidecars from hot DB", orphanedPayloadLogCtx);
     }
 
     if (finalizedPostGloas) {
-      await db.executionPayloadEnvelope.batchDelete(nonCanonicalBlockRoots);
-      logger.verbose("Deleted non canonical executionPayloadEnvelopes from hot DB", nonCanonicalLogCtx);
+      await db.executionPayloadEnvelope.batchDelete(orphanedPayloadRoots);
+      logger.verbose("Deleted non canonical executionPayloadEnvelopes from hot DB", orphanedPayloadLogCtx);
     }
   }
 
+  if (orphanedBlocks.length > 0) {
+    if (persistOrphanedBlocks) {
+      // Persist orphaned blocks to disk before deleting them from hot db
+      await Promise.all(
+        orphanedBlocks.map(async (block) => {
+          const blockBytes = await db.block.getBinary(fromHex(block.blockRoot));
+          const blockLogCtx = {slot: block.slot, root: block.blockRoot};
+          if (blockBytes) {
+            await persistOrphanedBlock(block.slot, block.blockRoot, blockBytes, {
+              persistOrphanedBlocksDir: persistOrphanedBlocksDir ?? "orphaned_blocks",
+            });
+            logger.verbose("Persisted orphaned block", {...logCtx, ...blockLogCtx});
+          } else {
+            logger.debug("Tried to persist orphaned block but no block found", {...logCtx, ...blockLogCtx});
+          }
+        })
+      );
+    }
+
+    await db.block.batchDelete(orphanedBlockRoots);
+    logger.verbose("Deleted non canonical blocks from hot DB", {
+      ...logCtx,
+      count: orphanedBlocks.length,
+      slotRange: prettyPrintIndices(orphanedBlocks.map((block) => block.slot).sort((a, b) => a - b)),
+    });
+  }
+
   // Delete expired blobs
   // Keep only `[current_epoch - max(MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS, archiveDataEpochs)]`
   // if archiveDataEpochs set to Infinity do not prune`
@@ -256,10 +271,7 @@ export async function archiveBlocks(
   // Keep only `[min(compute_blob_data_retention_start_epoch(current_epoch), current_epoch - archiveDataEpochs)]`
   if (finalizedPostFulu) {
     if (archiveDataEpochs !== Infinity) {
-      const dataColumnSidecarsMinEpoch = Math.min(
-        getBlobDataRetentionStartEpoch(config, currentEpoch),
-        currentEpoch - (archiveDataEpochs ?? 0)
-      );
+      const dataColumnSidecarsMinEpoch = getDataColumnSidecarsMinEpoch(config, currentEpoch, archiveDataEpochs);
       if (dataColumnSidecarsMinEpoch >= config.FULU_FORK_EPOCH) {
         const columnsPruneSlot = computeStartSlotAtEpoch(dataColumnSidecarsMinEpoch);
         const prunedColumnSlots = await db.dataColumns.pruneBefore(columnsPruneSlot);
@@ -283,9 +295,9 @@ export async function archiveBlocks(
     }
   }
 
-  // Prunning potential checkpoint data
+  // Pruning potential checkpoint data
   const finalizedCanonicalNonCheckpointBlocks = getNonCheckpointBlocks(finalizedCanonicalBlockRoots);
-  const nonCheckpointBlockRoots: Uint8Array[] = [...nonCanonicalBlockRoots];
+  const nonCheckpointBlockRoots: Uint8Array[] = [...orphanedBlockRoots];
   for (const block of finalizedCanonicalNonCheckpointBlocks) {
     nonCheckpointBlockRoots.push(block.root);
   }
@@ -408,72 +420,6 @@ async function migrateBlobSidecarsFromHotToColdDb(
   return migratedWrappedBlobSidecars;
 }
 
-// TODO: This function can be simplified further by reducing layers of promises in a loop
-/**
- * Post-gloas the data columns of a Gloas block are tied to its execution payload envelope.
- * Columns only exist once the FULL variant of the block is in the proto-array. Pre-Gloas
- * blocks only have a FULL variant, so the payload-status filter passes them all.
- */
-async function migrateDataColumnSidecarsFromHotToColdDb(
-  config: ChainForkConfig,
-  db: IBeaconDb,
-  logger: Logger,
-  canonicalBlocks: ProtoBlock[],
-  currentEpoch: Epoch
-): Promise<Slot[]> {
-  const columnBlocks = canonicalBlocks.filter(
-    (block) => config.getForkSeq(block.slot) < ForkSeq.gloas || block.payloadStatus === PayloadStatus.FULL
-  );
-  if (columnBlocks.length === 0) return [];
-  const blocks: BlockRootSlot[] = columnBlocks.map((block) => ({slot: block.slot, root: fromHex(block.blockRoot)}));
-
-  const migratedSlots: Slot[] = [];
-  for (let i = 0; i < blocks.length; i += BLOB_SIDECAR_BATCH_SIZE) {
-    const toIdx = Math.min(i + BLOB_SIDECAR_BATCH_SIZE, blocks.length);
-    const batch = blocks.slice(i, toIdx);
-    if (batch.length === 0) break;
-
-    const promises: Promise<void>[] = [];
-    for (const block of batch) {
-      const blockEpoch = computeEpochAtSlot(block.slot);
-      if (
-        config.getForkSeq(block.slot) < ForkSeq.fulu ||
-        blockEpoch < getBlobDataRetentionStartEpoch(config, currentEpoch)
-      ) {
-        continue;
-      }
-
-      const dataColumnSidecarBytes = await fromAsync(db.dataColumnSidecar.valuesStreamBinary(block.root));
-      if (dataColumnSidecarBytes.length === 0) {
-        logger.debug("DataColumnSidecars in forkchoice but missing in hot db, could be already archived", {
-          slot: block.slot,
-          root: toRootHex(block.root),
-        });
-        continue;
-      }
-      logger.verbose("Migrated legacy dataColumnSidecars for block", {
-        currentEpoch,
-        slot: block.slot,
-        root: toRootHex(block.root),
-        numSidecars: dataColumnSidecarBytes.length,
-      });
-      promises.push(
-        db.dataColumnSidecarArchive.putManyBinary(
-          block.slot,
-          dataColumnSidecarBytes.map((entry) => ({key: entry.id, value: entry.value}))
-        )
-      );
-      migratedSlots.push(block.slot);
-    }
-
-    await Promise.all(promises);
-    await db.dataColumnSidecar.deleteMany(batch.map((block) => block.root));
-  }
-
-  // Ancestor walk is newest to oldest; sort ascending so `prettyPrintIndices` renders cleanly.
-  return migratedSlots.sort((a, b) => a - b);
-}
-
 /**
  * Post-gloas given a finalized checkpoint at a block root, payload of that block root
  * is not considered finalized, hence they are archived in the next run.

package/src/chain/blocks/importBlock.ts

@@ -21,7 +21,7 @@ import {
   IBeaconStateView,
   RootCache,
   computeEpochAtSlot,
-  computeTimeAtSlot,
+  computeTimeAtSlotMs,
   getBeaconCommittees,
   isStatePostAltair,
   isStatePostBellatrix,
@@ -105,8 +105,9 @@ export async function importBlock(
   const currentEpoch = computeEpochAtSlot(currentSlot);
   const blockEpoch = computeEpochAtSlot(blockSlot);
   const prevFinalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch;
-  const receiveDelaySec =
-    fullyVerifiedBlock.seenTimestampSec - computeTimeAtSlot(this.config, blockSlot, postState.genesisTime);
+  const receiveDelayMs =
+    Math.round(fullyVerifiedBlock.seenTimestampSec * 1000) -
+    computeTimeAtSlotMs(this.config, blockSlot, postState.genesisTime * 1000);
   const recvToValLatency = Date.now() / 1000 - (opts.seenTimestampSec ?? Date.now() / 1000);
   const fork = this.config.getForkSeq(blockSlot);
 
@@ -148,12 +149,12 @@ export async function importBlock(
     executionStatus = parentBlock.executionStatus;
   }
 
-  const importDelaySec = this.clock.secFromSlot(blockSlot);
+  const importDelayMs = this.clock.msFromSlot(blockSlot);
   const blockSummary = this.forkChoice.onBlock(
     block.message,
     postState,
-    receiveDelaySec,
-    importDelaySec,
+    receiveDelayMs,
+    importDelayMs,
     currentSlot,
     executionStatus,
     dataAvailabilityStatus
@@ -417,7 +418,7 @@ export async function importBlock(
         //  4) Block meets the criteria of being re-orged out (this is also checked inside the function)
         const result = this.forkChoice.shouldOverrideForkChoiceUpdate(
           blockSummary,
-          this.clock.secFromSlot(currentSlot),
+          this.clock.msFromSlot(currentSlot),
           currentSlot
         );
         shouldOverrideFcu = result.shouldOverrideFcu;

package/src/chain/blocks/importExecutionPayload.ts

@@ -1,3 +1,4 @@
+import {SerializedList} from "@chainsafe/ssz";
 import {routes} from "@lodestar/api";
 import {
   ExecutionStatus,
@@ -36,6 +37,7 @@ export enum PayloadErrorCode {
   MISS_BLOCK_STATE = "PAYLOAD_ERROR_MISS_BLOCK_STATE",
   ENVELOPE_VERIFICATION_ERROR = "PAYLOAD_ERROR_ENVELOPE_VERIFICATION_ERROR",
   INVALID_SIGNATURE = "PAYLOAD_ERROR_INVALID_SIGNATURE",
+  DATA_UNAVAILABLE = "PAYLOAD_ERROR_DATA_UNAVAILABLE",
 }
 
 export type PayloadErrorType =
@@ -63,6 +65,9 @@ export type PayloadErrorType =
     }
   | {
       code: PayloadErrorCode.INVALID_SIGNATURE;
+    }
+  | {
+      code: PayloadErrorCode.DATA_UNAVAILABLE;
     };
 
 export class PayloadError extends Error {
@@ -206,7 +211,7 @@ export async function importExecutionPayload(
     }
     inclusionListTransactions =
       dependentRoot === null
-        ? []
+        ? SerializedList.empty()
         : this.inclusionListStore.getInclusionListTransactions(inclusionListSlot, dependentRoot, true);
   }
 
@@ -382,6 +387,6 @@ export async function processExecutionPayload(
   signal: AbortSignal,
   opts: ImportPayloadOpts = {}
 ): Promise<void> {
-  const {dataAvailabilityStatuses} = await verifyPayloadsDataAvailability([payloadInput], signal);
+  const {dataAvailabilityStatuses} = await verifyPayloadsDataAvailability(this.config, [payloadInput], signal);
   await importExecutionPayload.call(this, payloadInput, dataAvailabilityStatuses[0], opts);
 }

package/src/chain/blocks/index.ts

@@ -147,7 +147,7 @@ export async function processBlocks(
         proposerBalanceDelta: proposerBalanceDeltas[i],
         indexedAttestations: indexedAttestationsByBlock[i],
         // TODO: Make this param mandatory and capture in gossip
-        seenTimestampSec: opts.seenTimestampSec ?? Math.floor(Date.now() / 1000),
+        seenTimestampSec: opts.seenTimestampSec ?? Date.now() / 1000,
       });
     }
 
@@ -292,7 +292,11 @@ async function importPayloadEnvelopesOfKnownBlocks(
     return result;
   }
 
-  const {dataAvailabilityStatuses} = await verifyPayloadsDataAvailability(payloadInputs, new AbortController().signal);
+  const {dataAvailabilityStatuses} = await verifyPayloadsDataAvailability(
+    this.config,
+    payloadInputs,
+    new AbortController().signal
+  );
   for (let i = 0; i < payloadInputs.length; i++) {
     this.logger.debug("Importing payload envelope of known block", {
       slot: payloadInputs[i].slot,

package/src/chain/blocks/payloadEnvelopeProcessor.ts

@@ -13,11 +13,11 @@ const QUEUE_MAX_LENGTH = 16;
  *
  * Jobs are enqueued only on envelope arrival (gossip or API). The envelope may reach us before
  * the sampled data columns; importExecutionPayload awaits `verifyPayloadsDataAvailability`
- * internally, so a queued job can pend for up to `PAYLOAD_DATA_AVAILABILITY_TIMEOUT` while
- * waiting for columns. Duplicate triggers for the same payloadInput are deduped by sharing the
- * in-flight import promise: every caller observes the real import outcome. Resolving duplicates
- * early instead would let BlockInputSync mark a still-pending payload as processed, re-queue it,
- * and spin re-processing the same root until OOM while the import is blocked (e.g. EL syncing).
+ * internally, so a queued job can pend for up to one slot while waiting for columns. Duplicate
+ * triggers for the same payloadInput are deduped by sharing the in-flight import promise: every
+ * caller observes the real import outcome. Resolving duplicates early instead would let
+ * BlockInputSync mark a still-pending payload as processed, re-queue it, and spin re-processing
+ * the same root until OOM while the import is blocked (e.g. EL syncing).
  */
 export class PayloadEnvelopeProcessor {
   readonly jobQueue: JobItemQueue<[PayloadEnvelopeInput, ImportPayloadOpts], void>;

package/src/chain/blocks/verifyBlock.ts

@@ -143,6 +143,7 @@ export async function verifyBlocksInEpoch(
                 ? Array.from(payloadEnvelopes.values()).filter((payloadInput) => payloadInput.hasPayloadEnvelope())
                 : [];
             const {dataAvailabilityStatuses, availableTime} = await verifyPayloadsDataAvailability(
+              this.config,
               payloadInputsForDa,
               abortController.signal
             );

package/src/chain/blocks/verifyBlocksDataAvailability.ts

@@ -1,4 +1,6 @@
 import {DataAvailabilityStatus} from "@lodestar/state-transition";
+import {TimeoutError} from "@lodestar/utils";
+import {BlockError, BlockErrorCode} from "../errors/index.js";
 import {DAData, DAType, IBlockInput} from "./blockInput/index.js";
 
 // we can now wait for full 12 seconds because unavailable block sync will try pulling
@@ -22,7 +24,14 @@ export async function verifyBlocksDataAvailability(
   for (const blockInput of blocks) {
     // block verification is triggered on a verified gossip block so we only need to wait for all data
     if (!blockInput.hasAllData()) {
-      promises.push(blockInput.waitForAllData(BLOB_AVAILABILITY_TIMEOUT, signal));
+      promises.push(
+        blockInput.waitForAllData(BLOB_AVAILABILITY_TIMEOUT, signal).catch((e) => {
+          if (e instanceof TimeoutError) {
+            throw new BlockError(blockInput.getBlock(), {code: BlockErrorCode.DATA_UNAVAILABLE});
+          }
+          throw e;
+        })
+      );
     }
   }
   await Promise.all(promises);

package/src/chain/blocks/verifyPayloadsDataAvailability.ts

@@ -1,18 +1,18 @@
+import {ChainForkConfig} from "@lodestar/config";
 import {DataAvailabilityStatus} from "@lodestar/state-transition";
 import {gloas} from "@lodestar/types";
+import {TimeoutError} from "@lodestar/utils";
 import {PayloadEnvelopeInput} from "../seenCache/seenPayloadEnvelopeInput.js";
-
-// we can now wait for full 12 seconds because sync and reconstruction will try pulling
-// the data columns from the network anyway while the envelope is being processed
-export const PAYLOAD_DATA_AVAILABILITY_TIMEOUT = 12_000;
+import {PayloadError, PayloadErrorCode} from "./importExecutionPayload.js";
 
 /**
  * Verifies that all payload envelope inputs have their data columns available.
- * - Waits a max of PAYLOAD_DATA_AVAILABILITY_TIMEOUT for all data to be available
+ * - Waits a max of one slot duration of the payload's fork for all data to be available
  * - Returns the time at which all data was available
  * - Returns the data availability status for each payload input
  */
 export async function verifyPayloadsDataAvailability(
+  config: ChainForkConfig,
   payloadInputs: PayloadEnvelopeInput[],
   signal: AbortSignal
 ): Promise<{
@@ -22,7 +22,17 @@ export async function verifyPayloadsDataAvailability(
   const promises: Promise<gloas.DataColumnSidecar[]>[] = [];
   for (const payloadInput of payloadInputs) {
     if (!payloadInput.hasAllData()) {
-      promises.push(payloadInput.waitForAllData(PAYLOAD_DATA_AVAILABILITY_TIMEOUT, signal));
+      // Sync and reconstruction keep pulling the data columns from the network while we wait. The wait is
+      // capped at one slot because the payload queue is serial: a longer wait lets consecutive unavailable
+      // payloads push every later payload import further behind with each slot.
+      promises.push(
+        payloadInput.waitForAllData(config.getSlotDurationMs(payloadInput.forkName), signal).catch((e) => {
+          if (e instanceof TimeoutError) {
+            throw new PayloadError(payloadInput, {code: PayloadErrorCode.DATA_UNAVAILABLE});
+          }
+          throw e;
+        })
+      );
     }
   }
   await Promise.all(promises);

package/src/chain/chain.ts

@@ -1486,10 +1486,10 @@ export class BeaconChain implements IBeaconChain {
     this.metrics?.forkChoice.requests.inc();
     const timer = this.metrics?.forkChoice.findHead.startTimer({caller: FindHeadFnName.predictProposerHead});
     const slot = this.clock.currentSlot;
-    const secFromSlot = this.clock.secFromSlot(slot);
+    const msFromSlot = this.clock.msFromSlot(slot);
 
     try {
-      return this.forkChoice.updateAndGetHead({mode: UpdateHeadOpt.GetPredictedProposerHead, secFromSlot, slot}).head;
+      return this.forkChoice.updateAndGetHead({mode: UpdateHeadOpt.GetPredictedProposerHead, msFromSlot, slot}).head;
     } catch (e) {
       this.metrics?.forkChoice.errors.inc({entrypoint: UpdateHeadOpt.GetPredictedProposerHead});
       throw e;
@@ -1501,13 +1501,13 @@ export class BeaconChain implements IBeaconChain {
   getProposerHead(slot: Slot): ProtoBlock {
     this.metrics?.forkChoice.requests.inc();
     const timer = this.metrics?.forkChoice.findHead.startTimer({caller: FindHeadFnName.getProposerHead});
-    const secFromSlot = this.clock.secFromSlot(slot);
+    const msFromSlot = this.clock.msFromSlot(slot);
 
     try {
       // Do not emit head event here, when proposing we rely on the one emitted when importing our own block
       const {head, isHeadTimely, notReorgedReason} = this.forkChoice.updateAndGetHead({
         mode: UpdateHeadOpt.GetProposerHead,
-        secFromSlot,
+        msFromSlot,
         slot,
       });
 

package/src/chain/errors/blockError.ts

@@ -72,7 +72,7 @@ export enum BlockErrorCode {
    * whoever served the block MAY be downscored. Mirrors PayloadErrorCode.EXECUTION_ENGINE_INVALID.
    */
   EXECUTION_ENGINE_INVALID = "BLOCK_ERROR_EXECUTION_INVALID",
-  /** The blobs are unavailable */
+  /** The blob or column data did not arrive within the availability wait */
   DATA_UNAVAILABLE = "BLOCK_ERROR_DATA_UNAVAILABLE",
   /** Block contains too many kzg commitments */
   TOO_MANY_KZG_COMMITMENTS = "BLOCK_ERROR_TOO_MANY_KZG_COMMITMENTS",

package/src/chain/opPools/inclusionListStore.ts

@@ -1,4 +1,4 @@
-import {BitArray} from "@chainsafe/ssz";
+import {BitArray, SerializedList} from "@chainsafe/ssz";
 import {BeaconConfig} from "@lodestar/config";
 import {INCLUSION_LIST_COMMITTEE_SIZE} from "@lodestar/params";
 import {RootHex, Slot, ValidatorIndex, bellatrix, heze, ssz} from "@lodestar/types";
@@ -108,7 +108,7 @@ export class InclusionListStore {
 
   /** Deduplicated transactions from valid, non-equivocating inclusion lists at `(slot, dependentRoot)`. */
   getInclusionListTransactions(slot: Slot, dependentRoot: RootHex, onlyTimely = true): bellatrix.Transactions {
-    const transactions: bellatrix.Transactions = [];
+    const transactions: Uint8Array[] = [];
     const seen = new Set<string>();
 
     for (const {signedInclusionList} of this.getEligible(slot, dependentRoot, onlyTimely)) {
@@ -121,7 +121,7 @@ export class InclusionListStore {
       }
     }
 
-    return transactions;
+    return SerializedList.fromArray(transactions);
   }
 
   /** Bit `i` is set when committee member `i` submitted a valid, non-equivocating inclusion list */

package/src/chain/produceBlock/produceBlockBody.ts

@@ -781,7 +781,6 @@ export async function prepareExecutionPayload(
 
   const payloadIdCached = chain.executionEngine.payloadIdCache.get({
     headBlockHash: toRootHex(parentBlockHash),
-    finalizedBlockHash,
     timestamp: numToQuantity(timestamp),
     prevRandao: toHex(prevRandao),
     suggestedFeeRecipient,

package/src/chain/regen/interface.ts

@@ -25,7 +25,6 @@ export enum RegenCaller {
   publishDeferredVoluntaryExits = "publishDeferredVoluntaryExits",
   validateGossipExecutionPayloadBid = "validateGossipExecutionPayloadBid",
   validateGossipInclusionList = "validateGossipInclusionList",
-  validateApiExecutionPayloadBid = "validateApiExecutionPayloadBid",
   validateGossipPayloadAttestationMessage = "validateGossipPayloadAttestationMessage",
   validateGossipProposerPreferences = "validateGossipProposerPreferences",
   onForkChoiceFinalized = "onForkChoiceFinalized",