@lodestar/beacon-node 1.51.0-dev.7f05c22798 → 1.51.0-dev.7ff0260233

package/src/chain/GetBlobsTracker.ts

@@ -1,10 +1,9 @@
 import {ChainForkConfig} from "@lodestar/config";
 import {computeEpochAtSlot} from "@lodestar/state-transition";
-import {Logger} from "@lodestar/utils";
+import {Logger, pruneSetToMax} from "@lodestar/utils";
 import {BLOB_AND_PROOF_V2_RPC_BYTES} from "../execution/engine/types.js";
 import {IExecutionEngine} from "../execution/index.js";
 import {Metrics} from "../metrics/metrics.js";
-import {callInNextEventLoop} from "../util/eventLoop.js";
 import {
   DataColumnEngineResult,
   getBlobSidecarsFromExecution,
@@ -14,6 +13,11 @@ import {IBlockInput, isBlockInputBlobs} from "./blocks/blockInput/index.js";
 import {PayloadEnvelopeInput} from "./blocks/payloadEnvelopeInput/index.js";
 import {ChainEventEmitter} from "./emitter.js";
 
+/** A `null` answer only changes if the EL receives the transactions later, so retries are few and spaced out */
+export const MAX_GET_BLOBS_ATTEMPTS = 3;
+export const GET_BLOBS_RETRY_INTERVAL_MS = 1000;
+const MAX_TRACKED_BLOCK_ROOTS = 64;
+
 export type GetBlobsTrackerInit = {
   logger: Logger;
   executionEngine: IExecutionEngine;
@@ -32,6 +36,8 @@ export class GetBlobsTracker {
   metrics: Metrics | null;
   config: ChainForkConfig;
   activeReconstructions = new Set<string>();
+  failedAttempts = new Map<string, {count: number; lastAttemptMs: number; retryTimer?: NodeJS.Timeout}>();
+  pendingTriggers = new Set<string>();
   // Preallocate buffers for getBlobsV2 RPC calls
   // See https://github.com/ChainSafe/lodestar/pull/8282 for context
   blobsAndProofsBuffers: {buffers: Uint8Array[]; inUse: boolean}[] = [];
@@ -46,23 +52,55 @@ export class GetBlobsTracker {
 
   triggerGetBlobs(input: IBlockInput | PayloadEnvelopeInput): void {
     if (this.activeReconstructions.has(input.blockRootHex)) {
+      this.pendingTriggers.add(input.blockRootHex);
       return;
     }
 
+    const failed = this.failedAttempts.get(input.blockRootHex);
+    if (failed) {
+      if (failed.count >= MAX_GET_BLOBS_ATTEMPTS) {
+        return;
+      }
+      const waitMs = GET_BLOBS_RETRY_INTERVAL_MS - (Date.now() - failed.lastAttemptMs);
+      if (waitMs > 0) {
+        // Columns tend to arrive in one burst, so keep a single trailing retry instead of relying on a later trigger
+        failed.retryTimer ??= setTimeout(() => {
+          failed.retryTimer = undefined;
+          if (this.failedAttempts.get(input.blockRootHex) === failed && !input.hasAllData()) {
+            this.triggerGetBlobs(input);
+          }
+        }, waitMs).unref();
+        return;
+      }
+    }
+
+    // The request is sent right away, before block processing issues newPayload: ELs that drop a
+    // transaction's blobs once the payload is validated can only answer a request that gets there first
     if (!(input instanceof PayloadEnvelopeInput) && isBlockInputBlobs(input)) {
       // there is not preallocation for blob sidecars like there is for columns sidecars so no need to
       // store the index for the preallocated buffers
       this.activeReconstructions.add(input.blockRootHex);
-      callInNextEventLoop(() => {
-        const logCtx = {slot: input.slot, root: input.blockRootHex};
-        this.logger.verbose("Trigger getBlobsV1 for block", logCtx);
-        getBlobSidecarsFromExecution(this.config, this.executionEngine, this.metrics, this.emitter, input).finally(
-          () => {
-            this.logger.verbose("Completed getBlobsV1 for block", logCtx);
-            this.activeReconstructions.delete(input.blockRootHex);
+      const logCtx = {slot: input.slot, root: input.blockRootHex};
+      this.logger.verbose("Trigger getBlobsV1 for block", logCtx);
+      let requestFailed = false;
+      getBlobSidecarsFromExecution(this.config, this.executionEngine, this.metrics, this.emitter, input)
+        .then(() => {
+          // EL errors and misses are handled inside the helper, blobs still missing means the attempt failed
+          if (!input.hasAllData()) {
+            requestFailed = true;
+            this.recordFailedAttempt(input.blockRootHex);
           }
-        );
-      });
+        })
+        .catch((error) => {
+          this.logger.debug("Error during getBlobsV1 for block", logCtx, error as Error);
+          requestFailed = true;
+          this.recordFailedAttempt(input.blockRootHex);
+        })
+        .finally(() => {
+          this.logger.verbose("Completed getBlobsV1 for block", logCtx);
+          this.activeReconstructions.delete(input.blockRootHex);
+          this.retryPendingTrigger(input, requestFailed);
+        });
 
       return;
     }
@@ -83,34 +121,53 @@ export class GetBlobsTracker {
     }
     timer?.();
 
-    // We don't care about the outcome of this call,
-    // just that it has been triggered for this block root.
     this.activeReconstructions.add(input.blockRootHex);
     this.blobsAndProofsBuffers[freeIndex].inUse = true;
-    callInNextEventLoop(() => {
-      const logCtx = {slot: input.slot, root: input.blockRootHex};
-      this.logger.verbose("Trigger getBlobsV2 for block", logCtx);
-      getDataColumnSidecarsFromExecution(
-        this.config,
-        this.executionEngine,
-        this.emitter,
-        input,
-        this.metrics,
-        this.blobsAndProofsBuffers[freeIndex].buffers
-      )
-        .then((result) => {
-          this.logger.debug("getBlobsV2 result for block", {...logCtx, result});
-          this.metrics?.dataColumns.dataColumnEngineResult.inc({result});
-        })
-        .catch((error) => {
-          this.logger.debug("Error during getBlobsV2 for block", logCtx, error as Error);
-          this.metrics?.dataColumns.dataColumnEngineResult.inc({result: DataColumnEngineResult.Failed});
-        })
-        .finally(() => {
-          this.logger.verbose("Completed getBlobsV2 for block", logCtx);
-          this.activeReconstructions.delete(input.blockRootHex);
-          this.blobsAndProofsBuffers[freeIndex].inUse = false;
-        });
-    });
+    const logCtx = {slot: input.slot, root: input.blockRootHex};
+    this.logger.verbose("Trigger getBlobsV2 for block", logCtx);
+    let requestFailed = false;
+    getDataColumnSidecarsFromExecution(
+      this.config,
+      this.executionEngine,
+      this.emitter,
+      input,
+      this.metrics,
+      this.blobsAndProofsBuffers[freeIndex].buffers
+    )
+      .then((result) => {
+        this.logger.debug("getBlobsV2 result for block", {...logCtx, result});
+        this.metrics?.dataColumns.dataColumnEngineResult.inc({result});
+        if (result === DataColumnEngineResult.NullResponse) {
+          requestFailed = true;
+          this.recordFailedAttempt(input.blockRootHex);
+        }
+      })
+      .catch((error) => {
+        this.logger.debug("Error during getBlobsV2 for block", logCtx, error as Error);
+        this.metrics?.dataColumns.dataColumnEngineResult.inc({result: DataColumnEngineResult.Failed});
+        requestFailed = true;
+        this.recordFailedAttempt(input.blockRootHex);
+      })
+      .finally(() => {
+        this.logger.verbose("Completed getBlobsV2 for block", logCtx);
+        this.activeReconstructions.delete(input.blockRootHex);
+        this.blobsAndProofsBuffers[freeIndex].inUse = false;
+        this.retryPendingTrigger(input, requestFailed);
+      });
+  }
+
+  private retryPendingTrigger(input: IBlockInput | PayloadEnvelopeInput, requestFailed: boolean): void {
+    // Triggers dropped while the request was in flight re-enter once it failed, the attempt that was just
+    // recorded makes this arm the trailing retry instead of sending another request right away
+    if (this.pendingTriggers.delete(input.blockRootHex) && requestFailed) {
+      this.triggerGetBlobs(input);
+    }
+  }
+
+  private recordFailedAttempt(blockRootHex: string): void {
+    const failed = this.failedAttempts.get(blockRootHex);
+    clearTimeout(failed?.retryTimer);
+    this.failedAttempts.set(blockRootHex, {count: (failed?.count ?? 0) + 1, lastAttemptMs: Date.now()});
+    pruneSetToMax(this.failedAttempts, MAX_TRACKED_BLOCK_ROOTS);
   }
 }

package/src/chain/archiveStore/utils/archiveBlocks.ts

@@ -1,11 +1,18 @@
 import path from "node:path";
 import {ChainForkConfig} from "@lodestar/config";
 import {KeyValue} from "@lodestar/db";
-import {CheckpointWithHex, ExecutionStatus, IForkChoice, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice";
+import {
+  CheckpointWithHex,
+  ExecutionStatus,
+  IForkChoice,
+  PayloadStatus,
+  ProtoBlock,
+  isGloasBlock,
+} from "@lodestar/fork-choice";
 import {ForkSeq, SLOTS_PER_EPOCH} from "@lodestar/params";
 import {computeEpochAtSlot, computeStartSlotAtEpoch, getBlobDataRetentionStartEpoch} from "@lodestar/state-transition";
 import {Epoch, Slot} from "@lodestar/types";
-import {Logger, fromAsync, fromHex, prettyPrintIndices, toRootHex} from "@lodestar/utils";
+import {Logger, fromHex, prettyPrintIndices, toRootHex} from "@lodestar/utils";
 import {IBeaconDb} from "../../../db/index.js";
 import {BlockArchiveBatchPutBinaryItem, encodeArchivedHeaderEnvelope} from "../../../db/repositories/index.js";
 import {Metrics} from "../../../metrics/metrics.js";
@@ -49,6 +56,17 @@ async function persistOrphanedBlock(
   await writeIfNotExist(filepath, bytes);
 }
 
+/**
+ * Oldest epoch for which data column sidecars are kept
+ */
+export function getDataColumnSidecarsMinEpoch(
+  config: ChainForkConfig,
+  currentEpoch: Epoch,
+  archiveDataEpochs?: number
+): Epoch {
+  return Math.min(getBlobDataRetentionStartEpoch(config, currentEpoch), currentEpoch - (archiveDataEpochs ?? 0));
+}
+
 /**
  * Archives finalized blocks from active bucket to archive bucket.
  *
@@ -136,23 +154,6 @@ export async function archiveBlocks(
       logger.verbose("Migrated blobSidecars from hot DB to cold DB", {...logCtx, migratedEntries});
     }
 
-    // Keep the normal LevelDB hot-to-cold flow for legacy columns. New columns are written
-    // directly to flat files, so this only moves fallback data left by older versions.
-    if (finalizedPostFulu) {
-      const migratedSlots = await migrateDataColumnSidecarsFromHotToColdDb(
-        config,
-        db,
-        logger,
-        finalizedCanonicalBlocks,
-        currentEpoch
-      );
-      logger.verbose("Migrated legacy dataColumnSidecars from hot DB to cold DB", {
-        ...logCtx,
-        migratedEntries: migratedSlots.length,
-        slotRange: prettyPrintIndices(migratedSlots),
-      });
-    }
-
     if (finalizedPostGloas) {
       const migratedSlots = await migrateExecutionPayloadEnvelopesFromHotToColdDb(
         config,
@@ -169,44 +170,34 @@ export async function archiveBlocks(
     }
   }
 
-  // deleteNonCanonicalBlocks
-  // loop through forkchoice single time
+  const canonicalRoots = new Set(finalizedCanonicalBlocks.map((block) => block.blockRoot));
+  const orphanedBlocks: ProtoBlock[] = [];
+  const orphanedPayloads: ProtoBlock[] = [];
 
-  const nonCanonicalBlockRoots = finalizedNonCanonicalBlocks.map((summary) => fromHex(summary.blockRoot));
-  if (nonCanonicalBlockRoots.length > 0) {
-    if (persistOrphanedBlocks) {
-      // Persist orphaned blocks to disk before deleting them from hot db
-      await Promise.all(
-        nonCanonicalBlockRoots.map(async (root, index) => {
-          const block = finalizedNonCanonicalBlocks[index];
-          const blockBytes = await db.block.getBinary(root);
-          const blockLogCtx = {slot: block.slot, root: block.blockRoot};
-          if (blockBytes) {
-            await persistOrphanedBlock(block.slot, block.blockRoot, blockBytes, {
-              persistOrphanedBlocksDir: persistOrphanedBlocksDir ?? "orphaned_blocks",
-            });
-            logger.verbose("Persisted orphaned block", {...logCtx, ...blockLogCtx});
-          } else {
-            logger.warn("Tried to persist orphaned block but no block found", {...logCtx, ...blockLogCtx});
-          }
-        })
-      );
+  for (const block of finalizedNonCanonicalBlocks) {
+    if (block.payloadStatus === PayloadStatus.FULL) {
+      orphanedPayloads.push(block);
     }
+    // Every Gloas block gets EMPTY before FULL. If only FULL survives pruning, an earlier archive run
+    // already handled the root's block-level data.
+    const isBlockNode = !isGloasBlock(block) || block.payloadStatus === PayloadStatus.EMPTY;
+    if (isBlockNode && !canonicalRoots.has(block.blockRoot)) {
+      orphanedBlocks.push(block);
+    }
+  }
+  const orphanedBlockRoots = orphanedBlocks.map((block) => fromHex(block.blockRoot));
 
-    const nonCanonicalSlots = finalizedNonCanonicalBlocks.map((summary) => summary.slot).sort((a, b) => a - b);
-    const nonCanonicalLogCtx = {
+  if (orphanedPayloads.length > 0) {
+    const orphanedPayloadRoots = orphanedPayloads.map((block) => fromHex(block.blockRoot));
+    const orphanedPayloadLogCtx = {
       ...logCtx,
-      count: nonCanonicalBlockRoots.length,
-      slotRange: prettyPrintIndices(nonCanonicalSlots),
+      count: orphanedPayloads.length,
+      slotRange: prettyPrintIndices(orphanedPayloads.map((block) => block.slot).sort((a, b) => a - b)),
     };
 
-    const columnItems = finalizedNonCanonicalBlocks
-      // Gloas EMPTY and FULL variants share a block root. EMPTY has no sidecars, so deleting by its root could
-      // remove the canonical FULL variant's columns. Pre-Gloas blocks are always FULL.
-      .filter(
-        (summary) => config.getForkSeq(summary.slot) >= ForkSeq.fulu && summary.payloadStatus === PayloadStatus.FULL
-      )
-      .map((summary) => ({slot: summary.slot, blockRoot: summary.blockRoot}));
+    const columnItems = orphanedPayloads
+      .filter((block) => config.getForkSeq(block.slot) >= ForkSeq.fulu)
+      .map((block) => ({slot: block.slot, blockRoot: block.blockRoot}));
     if (columnItems.length > 0) {
       // Delete sidecars first so their block roots remain available to retry cleanup after a failure or crash.
       await db.dataColumns.deleteMany(columnItems);
@@ -217,20 +208,44 @@ export async function archiveBlocks(
       });
     }
 
-    await db.block.batchDelete(nonCanonicalBlockRoots);
-    logger.verbose("Deleted non canonical blocks from hot DB", nonCanonicalLogCtx);
-
     if (finalizedPostDeneb) {
-      await db.blobSidecars.batchDelete(nonCanonicalBlockRoots);
-      logger.verbose("Deleted non canonical blobSidecars from hot DB", nonCanonicalLogCtx);
+      await db.blobSidecars.batchDelete(orphanedPayloadRoots);
+      logger.verbose("Deleted non canonical blobSidecars from hot DB", orphanedPayloadLogCtx);
     }
 
     if (finalizedPostGloas) {
-      await db.executionPayloadEnvelope.batchDelete(nonCanonicalBlockRoots);
-      logger.verbose("Deleted non canonical executionPayloadEnvelopes from hot DB", nonCanonicalLogCtx);
+      await db.executionPayloadEnvelope.batchDelete(orphanedPayloadRoots);
+      logger.verbose("Deleted non canonical executionPayloadEnvelopes from hot DB", orphanedPayloadLogCtx);
     }
   }
 
+  if (orphanedBlocks.length > 0) {
+    if (persistOrphanedBlocks) {
+      // Persist orphaned blocks to disk before deleting them from hot db
+      await Promise.all(
+        orphanedBlocks.map(async (block) => {
+          const blockBytes = await db.block.getBinary(fromHex(block.blockRoot));
+          const blockLogCtx = {slot: block.slot, root: block.blockRoot};
+          if (blockBytes) {
+            await persistOrphanedBlock(block.slot, block.blockRoot, blockBytes, {
+              persistOrphanedBlocksDir: persistOrphanedBlocksDir ?? "orphaned_blocks",
+            });
+            logger.verbose("Persisted orphaned block", {...logCtx, ...blockLogCtx});
+          } else {
+            logger.debug("Tried to persist orphaned block but no block found", {...logCtx, ...blockLogCtx});
+          }
+        })
+      );
+    }
+
+    await db.block.batchDelete(orphanedBlockRoots);
+    logger.verbose("Deleted non canonical blocks from hot DB", {
+      ...logCtx,
+      count: orphanedBlocks.length,
+      slotRange: prettyPrintIndices(orphanedBlocks.map((block) => block.slot).sort((a, b) => a - b)),
+    });
+  }
+
   // Delete expired blobs
   // Keep only `[current_epoch - max(MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS, archiveDataEpochs)]`
   // if archiveDataEpochs set to Infinity do not prune`
@@ -256,10 +271,7 @@ export async function archiveBlocks(
   // Keep only `[min(compute_blob_data_retention_start_epoch(current_epoch), current_epoch - archiveDataEpochs)]`
   if (finalizedPostFulu) {
     if (archiveDataEpochs !== Infinity) {
-      const dataColumnSidecarsMinEpoch = Math.min(
-        getBlobDataRetentionStartEpoch(config, currentEpoch),
-        currentEpoch - (archiveDataEpochs ?? 0)
-      );
+      const dataColumnSidecarsMinEpoch = getDataColumnSidecarsMinEpoch(config, currentEpoch, archiveDataEpochs);
       if (dataColumnSidecarsMinEpoch >= config.FULU_FORK_EPOCH) {
         const columnsPruneSlot = computeStartSlotAtEpoch(dataColumnSidecarsMinEpoch);
         const prunedColumnSlots = await db.dataColumns.pruneBefore(columnsPruneSlot);
@@ -283,9 +295,9 @@ export async function archiveBlocks(
     }
   }
 
-  // Prunning potential checkpoint data
+  // Pruning potential checkpoint data
   const finalizedCanonicalNonCheckpointBlocks = getNonCheckpointBlocks(finalizedCanonicalBlockRoots);
-  const nonCheckpointBlockRoots: Uint8Array[] = [...nonCanonicalBlockRoots];
+  const nonCheckpointBlockRoots: Uint8Array[] = [...orphanedBlockRoots];
   for (const block of finalizedCanonicalNonCheckpointBlocks) {
     nonCheckpointBlockRoots.push(block.root);
   }
@@ -408,72 +420,6 @@ async function migrateBlobSidecarsFromHotToColdDb(
   return migratedWrappedBlobSidecars;
 }
 
-// TODO: This function can be simplified further by reducing layers of promises in a loop
-/**
- * Post-gloas the data columns of a Gloas block are tied to its execution payload envelope.
- * Columns only exist once the FULL variant of the block is in the proto-array. Pre-Gloas
- * blocks only have a FULL variant, so the payload-status filter passes them all.
- */
-async function migrateDataColumnSidecarsFromHotToColdDb(
-  config: ChainForkConfig,
-  db: IBeaconDb,
-  logger: Logger,
-  canonicalBlocks: ProtoBlock[],
-  currentEpoch: Epoch
-): Promise<Slot[]> {
-  const columnBlocks = canonicalBlocks.filter(
-    (block) => config.getForkSeq(block.slot) < ForkSeq.gloas || block.payloadStatus === PayloadStatus.FULL
-  );
-  if (columnBlocks.length === 0) return [];
-  const blocks: BlockRootSlot[] = columnBlocks.map((block) => ({slot: block.slot, root: fromHex(block.blockRoot)}));
-
-  const migratedSlots: Slot[] = [];
-  for (let i = 0; i < blocks.length; i += BLOB_SIDECAR_BATCH_SIZE) {
-    const toIdx = Math.min(i + BLOB_SIDECAR_BATCH_SIZE, blocks.length);
-    const batch = blocks.slice(i, toIdx);
-    if (batch.length === 0) break;
-
-    const promises: Promise<void>[] = [];
-    for (const block of batch) {
-      const blockEpoch = computeEpochAtSlot(block.slot);
-      if (
-        config.getForkSeq(block.slot) < ForkSeq.fulu ||
-        blockEpoch < getBlobDataRetentionStartEpoch(config, currentEpoch)
-      ) {
-        continue;
-      }
-
-      const dataColumnSidecarBytes = await fromAsync(db.dataColumnSidecar.valuesStreamBinary(block.root));
-      if (dataColumnSidecarBytes.length === 0) {
-        logger.debug("DataColumnSidecars in forkchoice but missing in hot db, could be already archived", {
-          slot: block.slot,
-          root: toRootHex(block.root),
-        });
-        continue;
-      }
-      logger.verbose("Migrated legacy dataColumnSidecars for block", {
-        currentEpoch,
-        slot: block.slot,
-        root: toRootHex(block.root),
-        numSidecars: dataColumnSidecarBytes.length,
-      });
-      promises.push(
-        db.dataColumnSidecarArchive.putManyBinary(
-          block.slot,
-          dataColumnSidecarBytes.map((entry) => ({key: entry.id, value: entry.value}))
-        )
-      );
-      migratedSlots.push(block.slot);
-    }
-
-    await Promise.all(promises);
-    await db.dataColumnSidecar.deleteMany(batch.map((block) => block.root));
-  }
-
-  // Ancestor walk is newest to oldest; sort ascending so `prettyPrintIndices` renders cleanly.
-  return migratedSlots.sort((a, b) => a - b);
-}
-
 /**
  * Post-gloas given a finalized checkpoint at a block root, payload of that block root
  * is not considered finalized, hence they are archived in the next run.

package/src/chain/blocks/importBlock.ts

@@ -21,7 +21,7 @@ import {
   IBeaconStateView,
   RootCache,
   computeEpochAtSlot,
-  computeTimeAtSlot,
+  computeTimeAtSlotMs,
   getBeaconCommittees,
   isStatePostAltair,
   isStatePostBellatrix,
@@ -105,8 +105,9 @@ export async function importBlock(
   const currentEpoch = computeEpochAtSlot(currentSlot);
   const blockEpoch = computeEpochAtSlot(blockSlot);
   const prevFinalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch;
-  const receiveDelaySec =
-    fullyVerifiedBlock.seenTimestampSec - computeTimeAtSlot(this.config, blockSlot, postState.genesisTime);
+  const receiveDelayMs =
+    Math.round(fullyVerifiedBlock.seenTimestampSec * 1000) -
+    computeTimeAtSlotMs(this.config, blockSlot, postState.genesisTime * 1000);
   const recvToValLatency = Date.now() / 1000 - (opts.seenTimestampSec ?? Date.now() / 1000);
   const fork = this.config.getForkSeq(blockSlot);
 
@@ -148,12 +149,12 @@ export async function importBlock(
     executionStatus = parentBlock.executionStatus;
   }
 
-  const importDelaySec = this.clock.secFromSlot(blockSlot);
+  const importDelayMs = this.clock.msFromSlot(blockSlot);
   const blockSummary = this.forkChoice.onBlock(
     block.message,
     postState,
-    receiveDelaySec,
-    importDelaySec,
+    receiveDelayMs,
+    importDelayMs,
     currentSlot,
     executionStatus,
     dataAvailabilityStatus
@@ -417,7 +418,7 @@ export async function importBlock(
         //  4) Block meets the criteria of being re-orged out (this is also checked inside the function)
         const result = this.forkChoice.shouldOverrideForkChoiceUpdate(
           blockSummary,
-          this.clock.secFromSlot(currentSlot),
+          this.clock.msFromSlot(currentSlot),
           currentSlot
         );
         shouldOverrideFcu = result.shouldOverrideFcu;
@@ -458,7 +459,8 @@ export async function importBlock(
      * - `headBlockHash !== null` -> Pre BELLATRIX_EPOCH
      * - `headBlockHash !== ZERO_HASH` -> Pre TTD
      */
-    const headBlockHash = this.forkChoice.getHead().executionPayloadBlockHash ?? ZERO_HASH_HEX;
+    const fcuHead = this.forkChoice.getHead();
+    const headBlockHash = fcuHead.executionPayloadBlockHash ?? ZERO_HASH_HEX;
     /**
      * After BELLATRIX_EPOCH and TTD it's okay to send a zero hash block hash for the finalized block. This will happen if
      * the current finalized block does not contain any execution payload at all (pre MERGE_EPOCH) or if it contains a
@@ -468,12 +470,7 @@ export async function importBlock(
     const finalizedBlockHash = getFinalizedExecutionBlockHash(this.forkChoice);
     if (headBlockHash !== ZERO_HASH_HEX) {
       this.executionEngine
-        .notifyForkchoiceUpdate(
-          this.config.getForkName(this.forkChoice.getHead().slot),
-          headBlockHash,
-          safeBlockHash,
-          finalizedBlockHash
-        )
+        .notifyForkchoiceUpdate(this.config.getForkName(fcuHead.slot), headBlockHash, safeBlockHash, finalizedBlockHash)
         .catch((e) => {
           if (!isErrorAborted(e) && !isQueueErrorAborted(e)) {
             this.logger.error("Error pushing notifyForkchoiceUpdate()", {headBlockHash, finalizedBlockHash}, e);

package/src/chain/blocks/importExecutionPayload.ts

@@ -1,3 +1,4 @@
+import {SerializedList} from "@chainsafe/ssz";
 import {routes} from "@lodestar/api";
 import {
   ExecutionStatus,
@@ -5,9 +6,16 @@ import {
   getFinalizedExecutionBlockHash,
   getSafeExecutionBlockHash,
 } from "@lodestar/fork-choice";
-import {DataAvailabilityStatus, isStatePostGloas} from "@lodestar/state-transition";
+import {
+  DataAvailabilityStatus,
+  computeEpochAtSlot,
+  isStatePostGloas,
+  isStatePostHeze,
+} from "@lodestar/state-transition";
+import {RootHex, bellatrix} from "@lodestar/types";
 import {isErrorAborted} from "@lodestar/utils";
 import {ExecutionPayloadStatus} from "../../execution/index.js";
+import {getShufflingDependentRoot} from "../../util/dependentRoot.js";
 import {isQueueErrorAborted} from "../../util/queue/index.js";
 import {BeaconChain} from "../chain.js";
 import {RegenCaller} from "../regen/interface.js";
@@ -29,6 +37,7 @@ export enum PayloadErrorCode {
   MISS_BLOCK_STATE = "PAYLOAD_ERROR_MISS_BLOCK_STATE",
   ENVELOPE_VERIFICATION_ERROR = "PAYLOAD_ERROR_ENVELOPE_VERIFICATION_ERROR",
   INVALID_SIGNATURE = "PAYLOAD_ERROR_INVALID_SIGNATURE",
+  DATA_UNAVAILABLE = "PAYLOAD_ERROR_DATA_UNAVAILABLE",
 }
 
 export type PayloadErrorType =
@@ -56,6 +65,9 @@ export type PayloadErrorType =
     }
   | {
       code: PayloadErrorCode.INVALID_SIGNATURE;
+    }
+  | {
+      code: PayloadErrorCode.DATA_UNAVAILABLE;
     };
 
 export class PayloadError extends Error {
@@ -172,6 +184,37 @@ export async function importExecutionPayload(
     );
   }
 
+  // Timely inclusion lists of the preceding slot, keyed by that slot's dependent root on the block's branch and
+  // resolved from the parent, which is at or before that slot. Gossip validation only accepts lists whose dependent
+  // block is in fork choice, so a dependent root fork choice cannot resolve (below the anchor after checkpoint sync)
+  // keys no stored list
+  let inclusionListTransactions: bellatrix.Transactions | undefined;
+  if (isStatePostHeze(blockState)) {
+    const inclusionListSlot = protoBlock.slot - 1;
+    let dependentRoot: RootHex | null = null;
+    const parentBlock = this.forkChoice.getBlockHexDefaultStatus(protoBlock.parentRoot);
+    if (parentBlock !== null) {
+      try {
+        dependentRoot = getShufflingDependentRoot(
+          this.forkChoice,
+          computeEpochAtSlot(inclusionListSlot),
+          computeEpochAtSlot(parentBlock.slot),
+          parentBlock
+        );
+      } catch (e) {
+        this.logger.debug(
+          "Inclusion list dependent root not in fork choice, importing payload without lists",
+          {slot, root: blockRootHex},
+          e as Error
+        );
+      }
+    }
+    inclusionListTransactions =
+      dependentRoot === null
+        ? SerializedList.empty()
+        : this.inclusionListStore.getInclusionListTransactions(inclusionListSlot, dependentRoot, true);
+  }
+
   // 4a. Run EL and signature verification in parallel
   const logCtx = {slot, root: blockRootHex, executionBlock: envelope.payload.blockNumber};
   this.logger.debug("Call engine api newPayload", logCtx);
@@ -182,7 +225,8 @@ export async function importExecutionPayload(
       envelope.payload,
       payloadInput.getVersionedHashes(),
       envelope.parentBeaconBlockRoot,
-      envelope.executionRequests
+      envelope.executionRequests,
+      inclusionListTransactions
     ),
 
     opts.validSignature === true
@@ -250,6 +294,32 @@ export async function importExecutionPayload(
     }
   });
 
+  // A VALID payload that does not satisfy the inclusion list constraints is still imported, fork choice
+  // only records the verdict so should_extend_payload moves the head to the EMPTY variant, which the next
+  // proposer builds on.
+  // Recorded before the payload enters fork choice. A VALID response must carry a verdict, an
+  // optimistically imported payload counts as satisfied until the engine validates it, anything else as
+  // unsatisfied. A VALID response without a verdict violates the engine API, it is logged and treated as
+  // satisfied so a non-compliant engine does not orphan every payload
+  if (inclusionListTransactions !== undefined) {
+    if (execResult.status === ExecutionPayloadStatus.VALID && execResult.inclusionListSatisfied == null) {
+      this.logger.warn("Execution engine reported no inclusion list verdict, not enforcing inclusion lists", logCtx);
+    }
+    const satisfied =
+      execResult.status === ExecutionPayloadStatus.VALID
+        ? (execResult.inclusionListSatisfied ?? true)
+        : execResult.status === ExecutionPayloadStatus.SYNCING || execResult.status === ExecutionPayloadStatus.ACCEPTED;
+    this.forkChoice.recordPayloadInclusionListSatisfaction(blockRootHex, satisfied);
+    if (!satisfied) {
+      this.logger.verbose("Payload did not satisfy inclusion list constraints", {
+        ...logCtx,
+        blockHash: blockHashHex,
+        status: execResult.status,
+        inclusionListTransactions: inclusionListTransactions.length,
+      });
+    }
+  }
+
   // 6. Update fork choice, transitions the block's PENDING variant to FULL
   const execStatus = toForkChoiceExecutionStatus(execResult.status);
   this.forkChoice.onExecutionPayload(
@@ -317,6 +387,6 @@ export async function processExecutionPayload(
   signal: AbortSignal,
   opts: ImportPayloadOpts = {}
 ): Promise<void> {
-  const {dataAvailabilityStatuses} = await verifyPayloadsDataAvailability([payloadInput], signal);
+  const {dataAvailabilityStatuses} = await verifyPayloadsDataAvailability(this.config, [payloadInput], signal);
   await importExecutionPayload.call(this, payloadInput, dataAvailabilityStatuses[0], opts);
 }

package/src/chain/blocks/index.ts

@@ -147,7 +147,7 @@ export async function processBlocks(
         proposerBalanceDelta: proposerBalanceDeltas[i],
         indexedAttestations: indexedAttestationsByBlock[i],
         // TODO: Make this param mandatory and capture in gossip
-        seenTimestampSec: opts.seenTimestampSec ?? Math.floor(Date.now() / 1000),
+        seenTimestampSec: opts.seenTimestampSec ?? Date.now() / 1000,
       });
     }
 
@@ -292,7 +292,11 @@ async function importPayloadEnvelopesOfKnownBlocks(
     return result;
   }
 
-  const {dataAvailabilityStatuses} = await verifyPayloadsDataAvailability(payloadInputs, new AbortController().signal);
+  const {dataAvailabilityStatuses} = await verifyPayloadsDataAvailability(
+    this.config,
+    payloadInputs,
+    new AbortController().signal
+  );
   for (let i = 0; i < payloadInputs.length; i++) {
     this.logger.debug("Importing payload envelope of known block", {
       slot: payloadInputs[i].slot,