@lodestar/beacon-node 1.51.0-dev.32db66b396 → 1.51.0-dev.36022cc5cb

package/src/chain/blocks/importBlock.ts

@@ -1,11 +1,11 @@
 import {BitArray} from "@chainsafe/ssz";
 import {routes} from "@lodestar/api";
 import {
-  AncestorStatus,
-  ExecutionStatus,
   ForkChoiceError,
   ForkChoiceErrorCode,
+  InvalidBlockCode,
   NotReorgedReason,
+  ProtoBlock,
   getFinalizedExecutionBlockHash,
   getSafeExecutionBlockHash,
 } from "@lodestar/fork-choice";
@@ -21,7 +21,7 @@ import {
   IBeaconStateView,
   RootCache,
   computeEpochAtSlot,
-  computeTimeAtSlot,
+  computeTimeAtSlotMs,
   getBeaconCommittees,
   isStatePostAltair,
   isStatePostBellatrix,
@@ -43,7 +43,8 @@ import {callInNextEventLoop} from "../../util/eventLoop.js";
 import {isOptimisticBlock} from "../../util/forkChoice.js";
 import {isQueueErrorAborted} from "../../util/queue/index.js";
 import type {BeaconChain} from "../chain.js";
-import {ChainEvent, ReorgEventData} from "../emitter.js";
+import {ChainEvent} from "../emitter.js";
+import {BlockError, BlockErrorCode} from "../errors/index.js";
 import {ForkchoiceCaller} from "../forkChoice/index.js";
 import {REPROCESS_MIN_TIME_TO_NEXT_SLOT_BPS} from "../reprocess.js";
 import {toCheckpointHex} from "../stateCache/persistentCheckpointsCache.js";
@@ -94,8 +95,8 @@ export async function importBlock(
   fullyVerifiedBlock: FullyVerifiedBlock,
   opts: ImportBlockOpts
 ): Promise<void> {
-  const {blockInput, postState, parentBlockSlot, dataAvailabilityStatus, indexedAttestations} = fullyVerifiedBlock;
-  let {executionStatus} = fullyVerifiedBlock;
+  const {blockInput, postState, parentBlockSlot, executionStatus, dataAvailabilityStatus, indexedAttestations} =
+    fullyVerifiedBlock;
   const block = blockInput.getBlock();
   const source = blockInput.getBlockSource();
   const {slot: blockSlot} = block.message;
@@ -105,8 +106,9 @@ export async function importBlock(
   const currentEpoch = computeEpochAtSlot(currentSlot);
   const blockEpoch = computeEpochAtSlot(blockSlot);
   const prevFinalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch;
-  const receiveDelaySec =
-    fullyVerifiedBlock.seenTimestampSec - computeTimeAtSlot(this.config, blockSlot, postState.genesisTime);
+  const receiveDelayMs =
+    Math.round(fullyVerifiedBlock.seenTimestampSec * 1000) -
+    computeTimeAtSlotMs(this.config, blockSlot, postState.genesisTime * 1000);
   const recvToValLatency = Date.now() / 1000 - (opts.seenTimestampSec ?? Date.now() / 1000);
   const fork = this.config.getForkSeq(blockSlot);
 
@@ -130,35 +132,30 @@ export async function importBlock(
 
   // Should compute checkpoint balances before forkchoice.onBlock
   this.checkpointBalancesCache.processState(blockRootHex, postState);
-  if (isGloasBeaconBlock(block.message)) {
-    // A gloas block has no payload of its own, it inherits the status of the payload it builds on,
-    // i.e. the parent variant matching the bid's parent block hash. The parent's default (PENDING)
-    // variant does not reflect a SYNCING parent payload, and onBlock would then mark that payload VALID.
-    const parentRootHex = toRootHex(block.message.parentRoot);
-    const parentBlockHashHex = toRootHex(block.message.body.signedExecutionPayloadBid.message.parentBlockHash);
-    const parentBlock = this.forkChoice.getBlockHexAndBlockHash(parentRootHex, parentBlockHashHex);
-    if (parentBlock === null) {
-      throw Error(
-        `Parent block not found in forkChoice, parentRoot=${parentRootHex} parentBlockHash=${parentBlockHashHex}`
-      );
-    }
-    if (parentBlock.executionStatus === ExecutionStatus.Invalid) {
-      throw Error(`Parent block has invalid execution status, parentRoot=${parentRootHex}`);
+  const importDelayMs = this.clock.msFromSlot(blockSlot);
+  let blockSummary: ProtoBlock;
+  try {
+    blockSummary = this.forkChoice.onBlock(
+      block.message,
+      postState,
+      receiveDelayMs,
+      importDelayMs,
+      currentSlot,
+      executionStatus,
+      dataAvailabilityStatus
+    );
+  } catch (e) {
+    // The parent can be invalidated by a forkchoiceUpdated verdict after verifyBlocksSanityChecks passed
+    if (
+      e instanceof ForkChoiceError &&
+      e.type.code === ForkChoiceErrorCode.INVALID_BLOCK &&
+      e.type.err.code === InvalidBlockCode.PARENT_EXECUTION_INVALID
+    ) {
+      throw new BlockError(block, {code: BlockErrorCode.PARENT_EXECUTION_INVALID, parentRoot: e.type.err.root});
     }
-    executionStatus = parentBlock.executionStatus;
+    throw e;
   }
 
-  const importDelaySec = this.clock.secFromSlot(blockSlot);
-  const blockSummary = this.forkChoice.onBlock(
-    block.message,
-    postState,
-    receiveDelaySec,
-    importDelaySec,
-    currentSlot,
-    executionStatus,
-    dataAvailabilityStatus
-  );
-
   // This adds the state necessary to process the next block
   // Some block event handlers require state being in state cache so need to do this before emitting EventType.block
   this.regen.processState(blockRootHex, postState);
@@ -321,18 +318,10 @@ export async function importBlock(
   }
 
   if (newHead.blockRoot !== oldHead.blockRoot) {
-    // Set head state as strong reference
-    this.regen.updateHeadState(newHead, postState);
+    this.onHeadChanged(oldHead, newHead, postState);
 
     const delaySec = this.clock.secFromSlot(newHead.slot);
-    this.logger.verbose("New chain head", {
-      slot: newHead.slot,
-      root: newHead.blockRoot,
-      delaySec,
-    });
-
     if (this.metrics) {
-      this.metrics.headSlot.set(newHead.slot);
       // Only track "recent" blocks. Otherwise sync can distort this metrics heavily.
       // We want to track recent blocks coming from gossip, unknown block sync, and API.
       const headFork = this.config.getForkName(newHead.slot);
@@ -345,32 +334,6 @@ export async function importBlock(
       }
     }
 
-    this.onNewHead(newHead);
-
-    this.metrics?.forkChoice.changedHead.inc();
-
-    const ancestorResult = this.forkChoice.getCommonAncestorDepth(oldHead, newHead);
-    if (ancestorResult.code === AncestorStatus.CommonAncestor) {
-      // CommonAncestor = chain reorg, old head and new head not direct descendants
-
-      const forkChoiceReorgEventData: ReorgEventData = {
-        depth: ancestorResult.depth,
-        epoch: computeEpochAtSlot(newHead.slot),
-        slot: newHead.slot,
-        newHeadBlock: newHead.blockRoot,
-        oldHeadBlock: oldHead.blockRoot,
-        newHeadState: newHead.stateRoot,
-        oldHeadState: oldHead.stateRoot,
-        executionOptimistic: isOptimisticBlock(newHead),
-      };
-
-      this.emitter.emit(routes.events.EventType.chainReorg, forkChoiceReorgEventData);
-      this.logger.verbose("Chain reorg", forkChoiceReorgEventData);
-
-      this.metrics?.forkChoice.reorg.inc();
-      this.metrics?.forkChoice.reorgDistance.observe(ancestorResult.depth);
-    }
-
     // Lightclient server support (only after altair)
     // - Persist state witness
     // - Use block's syncAggregate
@@ -417,7 +380,7 @@ export async function importBlock(
         //  4) Block meets the criteria of being re-orged out (this is also checked inside the function)
         const result = this.forkChoice.shouldOverrideForkChoiceUpdate(
           blockSummary,
-          this.clock.secFromSlot(currentSlot),
+          this.clock.msFromSlot(currentSlot),
           currentSlot
         );
         shouldOverrideFcu = result.shouldOverrideFcu;
@@ -458,7 +421,8 @@ export async function importBlock(
      * - `headBlockHash !== null` -> Pre BELLATRIX_EPOCH
      * - `headBlockHash !== ZERO_HASH` -> Pre TTD
      */
-    const headBlockHash = this.forkChoice.getHead().executionPayloadBlockHash ?? ZERO_HASH_HEX;
+    const fcuHead = this.forkChoice.getHead();
+    const headBlockHash = fcuHead.executionPayloadBlockHash ?? ZERO_HASH_HEX;
     /**
      * After BELLATRIX_EPOCH and TTD it's okay to send a zero hash block hash for the finalized block. This will happen if
      * the current finalized block does not contain any execution payload at all (pre MERGE_EPOCH) or if it contains a
@@ -469,7 +433,8 @@ export async function importBlock(
     if (headBlockHash !== ZERO_HASH_HEX) {
       this.executionEngine
         .notifyForkchoiceUpdate(
-          this.config.getForkName(this.forkChoice.getHead().slot),
+          this.config.getForkName(fcuHead.slot),
+          fcuHead.blockRoot,
           headBlockHash,
           safeBlockHash,
           finalizedBlockHash

package/src/chain/blocks/importExecutionPayload.ts

@@ -1,13 +1,24 @@
+import {SerializedList} from "@chainsafe/ssz";
 import {routes} from "@lodestar/api";
 import {
   ExecutionStatus,
+  ForkChoiceError,
+  ForkChoiceErrorCode,
+  InvalidBlockCode,
   PayloadExecutionStatus,
   getFinalizedExecutionBlockHash,
   getSafeExecutionBlockHash,
 } from "@lodestar/fork-choice";
-import {DataAvailabilityStatus, isStatePostGloas} from "@lodestar/state-transition";
+import {
+  DataAvailabilityStatus,
+  computeEpochAtSlot,
+  isStatePostGloas,
+  isStatePostHeze,
+} from "@lodestar/state-transition";
+import {RootHex, bellatrix} from "@lodestar/types";
 import {isErrorAborted} from "@lodestar/utils";
 import {ExecutionPayloadStatus} from "../../execution/index.js";
+import {getShufflingDependentRoot} from "../../util/dependentRoot.js";
 import {isQueueErrorAborted} from "../../util/queue/index.js";
 import {BeaconChain} from "../chain.js";
 import {RegenCaller} from "../regen/interface.js";
@@ -18,7 +29,6 @@ import {
   verifyExecutionPayloadEnvelope,
   verifyExecutionPayloadEnvelopeSignature,
 } from "./verifyExecutionPayloadEnvelope.js";
-import {verifyPayloadsDataAvailability} from "./verifyPayloadsDataAvailability.js";
 
 const EVENTSTREAM_EMIT_RECENT_EXECUTION_PAYLOAD_SLOTS = 64;
 
@@ -26,9 +36,11 @@ export enum PayloadErrorCode {
   EXECUTION_ENGINE_INVALID = "PAYLOAD_ERROR_EXECUTION_ENGINE_INVALID",
   EXECUTION_ENGINE_ERROR = "PAYLOAD_ERROR_EXECUTION_ENGINE_ERROR",
   BLOCK_NOT_IN_FORK_CHOICE = "PAYLOAD_ERROR_BLOCK_NOT_IN_FORK_CHOICE",
+  BLOCK_EXECUTION_INVALID = "PAYLOAD_ERROR_BLOCK_EXECUTION_INVALID",
   MISS_BLOCK_STATE = "PAYLOAD_ERROR_MISS_BLOCK_STATE",
   ENVELOPE_VERIFICATION_ERROR = "PAYLOAD_ERROR_ENVELOPE_VERIFICATION_ERROR",
   INVALID_SIGNATURE = "PAYLOAD_ERROR_INVALID_SIGNATURE",
+  DATA_UNAVAILABLE = "PAYLOAD_ERROR_DATA_UNAVAILABLE",
 }
 
 export type PayloadErrorType =
@@ -46,6 +58,10 @@ export type PayloadErrorType =
       code: PayloadErrorCode.BLOCK_NOT_IN_FORK_CHOICE;
       blockRootHex: string;
     }
+  | {
+      code: PayloadErrorCode.BLOCK_EXECUTION_INVALID;
+      blockRootHex: string;
+    }
   | {
       code: PayloadErrorCode.MISS_BLOCK_STATE;
       blockRootHex: string;
@@ -56,6 +72,9 @@ export type PayloadErrorType =
     }
   | {
       code: PayloadErrorCode.INVALID_SIGNATURE;
+    }
+  | {
+      code: PayloadErrorCode.DATA_UNAVAILABLE;
     };
 
 export class PayloadError extends Error {
@@ -87,7 +106,8 @@ function toForkChoiceExecutionStatus(status: ExecutionPayloadStatus): PayloadExe
  * are applied on the next block via processParentExecutionPayload.
  *
  * The DA wait must have run upstream (range sync awaits DA in `verifyBlocksInEpoch` for the
- * whole segment; gossip / API path uses the `processExecutionPayload` wrapper below).
+ * whole segment; the gossip / API / unknown-payload sync path waits in `PayloadEnvelopeProcessor`
+ * before the import is queued).
  *
  * Steps:
  * 1. Emit `execution_payload_available` event for payload attestation
@@ -132,6 +152,13 @@ export async function importExecutionPayload(
       blockRootHex,
     });
   }
+  // The payload chain this block builds on was invalidated, a VALID verdict here would contradict it
+  if (protoBlock.executionStatus === ExecutionStatus.Invalid) {
+    throw new PayloadError(payloadInput, {
+      code: PayloadErrorCode.BLOCK_EXECUTION_INVALID,
+      blockRootHex,
+    });
+  }
 
   // 3. Regenerate state for envelope verification
   const blockState = await this.regen
@@ -172,6 +199,37 @@ export async function importExecutionPayload(
     );
   }
 
+  // Timely inclusion lists of the preceding slot, keyed by that slot's dependent root on the block's branch and
+  // resolved from the parent, which is at or before that slot. Gossip validation only accepts lists whose dependent
+  // block is in fork choice, so a dependent root fork choice cannot resolve (below the anchor after checkpoint sync)
+  // keys no stored list
+  let inclusionListTransactions: bellatrix.Transactions | undefined;
+  if (isStatePostHeze(blockState)) {
+    const inclusionListSlot = protoBlock.slot - 1;
+    let dependentRoot: RootHex | null = null;
+    const parentBlock = this.forkChoice.getBlockHexDefaultStatus(protoBlock.parentRoot);
+    if (parentBlock !== null) {
+      try {
+        dependentRoot = getShufflingDependentRoot(
+          this.forkChoice,
+          computeEpochAtSlot(inclusionListSlot),
+          computeEpochAtSlot(parentBlock.slot),
+          parentBlock
+        );
+      } catch (e) {
+        this.logger.debug(
+          "Inclusion list dependent root not in fork choice, importing payload without lists",
+          {slot, root: blockRootHex},
+          e as Error
+        );
+      }
+    }
+    inclusionListTransactions =
+      dependentRoot === null
+        ? SerializedList.empty()
+        : this.inclusionListStore.getInclusionListTransactions(inclusionListSlot, dependentRoot, true);
+  }
+
   // 4a. Run EL and signature verification in parallel
   const logCtx = {slot, root: blockRootHex, executionBlock: envelope.payload.blockNumber};
   this.logger.debug("Call engine api newPayload", logCtx);
@@ -182,7 +240,8 @@ export async function importExecutionPayload(
       envelope.payload,
       payloadInput.getVersionedHashes(),
       envelope.parentBeaconBlockRoot,
-      envelope.executionRequests
+      envelope.executionRequests,
+      inclusionListTransactions
     ),
 
     opts.validSignature === true
@@ -250,27 +309,73 @@ export async function importExecutionPayload(
     }
   });
 
+  // A VALID payload that does not satisfy the inclusion list constraints is still imported, fork choice
+  // only records the verdict so should_extend_payload moves the head to the EMPTY variant, which the next
+  // proposer builds on.
+  // Recorded before the payload enters fork choice. A VALID response must carry a verdict, an
+  // optimistically imported payload counts as satisfied until the engine validates it, anything else as
+  // unsatisfied. A VALID response without a verdict violates the engine API, it is logged and treated as
+  // satisfied so a non-compliant engine does not orphan every payload
+  if (inclusionListTransactions !== undefined) {
+    if (execResult.status === ExecutionPayloadStatus.VALID && execResult.inclusionListSatisfied == null) {
+      this.logger.warn("Execution engine reported no inclusion list verdict, not enforcing inclusion lists", logCtx);
+    }
+    const satisfied =
+      execResult.status === ExecutionPayloadStatus.VALID
+        ? (execResult.inclusionListSatisfied ?? true)
+        : execResult.status === ExecutionPayloadStatus.SYNCING || execResult.status === ExecutionPayloadStatus.ACCEPTED;
+    this.forkChoice.recordPayloadInclusionListSatisfaction(blockRootHex, satisfied);
+    if (!satisfied) {
+      this.logger.verbose("Payload did not satisfy inclusion list constraints", {
+        ...logCtx,
+        blockHash: blockHashHex,
+        status: execResult.status,
+        inclusionListTransactions: inclusionListTransactions.length,
+      });
+    }
+  }
+
   // 6. Update fork choice, transitions the block's PENDING variant to FULL
   const execStatus = toForkChoiceExecutionStatus(execResult.status);
-  this.forkChoice.onExecutionPayload(
-    blockRootHex,
-    blockHashHex,
-    envelope.payload.blockNumber,
-    envelope.payload.gasLimit,
-    execStatus,
-    dataAvailabilityStatus
-  );
+  try {
+    this.forkChoice.onExecutionPayload(
+      blockRootHex,
+      blockHashHex,
+      envelope.payload.blockNumber,
+      envelope.payload.gasLimit,
+      execStatus,
+      dataAvailabilityStatus
+    );
+  } catch (e) {
+    // The payload chain can be invalidated by a forkchoiceUpdated verdict while the engine was awaited
+    if (
+      e instanceof ForkChoiceError &&
+      e.type.code === ForkChoiceErrorCode.INVALID_BLOCK &&
+      e.type.err.code === InvalidBlockCode.PARENT_EXECUTION_INVALID
+    ) {
+      throw new PayloadError(payloadInput, {code: PayloadErrorCode.BLOCK_EXECUTION_INVALID, blockRootHex});
+    }
+    throw e;
+  }
 
   // 7. Queue notifyForkchoiceUpdate to engine api
   const head = this.forkChoice.getHead();
-  if (!this.opts.disableImportExecutionFcU && blockRootHex === head.blockRoot) {
+  if (
+    !this.opts.disableImportExecutionFcU &&
+    blockRootHex === head.blockRoot &&
+    // Skip a payload that does not satisfy the inclusion list constraints, fork choice keeps the EMPTY
+    // variant as head and the next prepare fcU keeps the engine on the parent payload
+    this.forkChoice.isPayloadInclusionListSatisfied(blockRootHex)
+  ) {
     const safeBlockHash = getSafeExecutionBlockHash(this.forkChoice, this.logger);
     const finalizedBlockHash = getFinalizedExecutionBlockHash(this.forkChoice);
-    this.executionEngine.notifyForkchoiceUpdate(fork, blockHashHex, safeBlockHash, finalizedBlockHash).catch((e) => {
-      if (!isErrorAborted(e) && !isQueueErrorAborted(e)) {
-        this.logger.error("Error pushing notifyForkchoiceUpdate()", {blockHashHex, finalizedBlockHash}, e);
-      }
-    });
+    this.executionEngine
+      .notifyForkchoiceUpdate(fork, blockRootHex, blockHashHex, safeBlockHash, finalizedBlockHash)
+      .catch((e) => {
+        if (!isErrorAborted(e) && !isQueueErrorAborted(e)) {
+          this.logger.error("Error pushing notifyForkchoiceUpdate()", {blockHashHex, finalizedBlockHash}, e);
+        }
+      });
   }
 
   // 8. Record metrics for payload envelope and column sources
@@ -302,21 +407,3 @@ export async function importExecutionPayload(
     delaySec,
   });
 }
-
-/**
- * Process an execution payload envelope end-to-end: wait for DA, then import.
- *
- * Used by the PayloadEnvelopeProcessor queue (gossip / API / unknown-payload sync) — i.e.
- * callers that have NOT already awaited DA themselves. Range sync's inline dispatch in
- * processBlocks skips this wrapper and calls `importExecutionPayload` directly, since
- * `verifyBlocksInEpoch` already awaited DA for the segment.
- */
-export async function processExecutionPayload(
-  this: BeaconChain,
-  payloadInput: PayloadEnvelopeInput,
-  signal: AbortSignal,
-  opts: ImportPayloadOpts = {}
-): Promise<void> {
-  const {dataAvailabilityStatuses} = await verifyPayloadsDataAvailability([payloadInput], signal);
-  await importExecutionPayload.call(this, payloadInput, dataAvailabilityStatuses[0], opts);
-}

package/src/chain/blocks/index.ts

@@ -7,7 +7,7 @@ import {nextEventLoop} from "../../util/eventLoop.js";
 import {JobItemQueue, isQueueErrorAborted} from "../../util/queue/index.js";
 import type {BeaconChain} from "../chain.js";
 import {BlockError, BlockErrorCode, isBlockErrorAborted} from "../errors/index.js";
-import {ForkchoiceCaller} from "../forkChoice/index.js";
+import {ExecutionInvalidSource, ForkchoiceCaller} from "../forkChoice/index.js";
 import {BlockProcessOpts} from "../options.js";
 import {IBlockInput} from "./blockInput/types.js";
 import {importBlock, importsBlockAttestations} from "./importBlock.js";
@@ -123,7 +123,10 @@ export async function processBlocks(
     // and we need to further propagate
     if (segmentExecStatus.execAborted !== null) {
       if (segmentExecStatus.invalidSegmentLVH !== undefined) {
-        this.forkChoice.validateLatestHash(segmentExecStatus.invalidSegmentLVH);
+        this.executionEngine.invalidateExecutionPayloads(
+          segmentExecStatus.invalidSegmentLVH,
+          ExecutionInvalidSource.blockSegment
+        );
       }
       throw segmentExecStatus.execAborted.execError;
     }
@@ -147,7 +150,7 @@ export async function processBlocks(
         proposerBalanceDelta: proposerBalanceDeltas[i],
         indexedAttestations: indexedAttestationsByBlock[i],
         // TODO: Make this param mandatory and capture in gossip
-        seenTimestampSec: opts.seenTimestampSec ?? Math.floor(Date.now() / 1000),
+        seenTimestampSec: opts.seenTimestampSec ?? Date.now() / 1000,
       });
     }
 
@@ -292,7 +295,11 @@ async function importPayloadEnvelopesOfKnownBlocks(
     return result;
   }
 
-  const {dataAvailabilityStatuses} = await verifyPayloadsDataAvailability(payloadInputs, new AbortController().signal);
+  const {dataAvailabilityStatuses} = await verifyPayloadsDataAvailability(
+    this.config,
+    payloadInputs,
+    new AbortController().signal
+  );
   for (let i = 0; i < payloadInputs.length; i++) {
     this.logger.debug("Importing payload envelope of known block", {
       slot: payloadInputs[i].slot,

package/src/chain/blocks/payloadEnvelopeProcessor.ts

@@ -1,32 +1,37 @@
+import {DataAvailabilityStatus} from "@lodestar/state-transition";
 import {Metrics} from "../../metrics/metrics.js";
 import {JobItemQueue} from "../../util/queue/index.js";
 import type {BeaconChain} from "../chain.js";
 import {PayloadEnvelopeInput} from "../seenCache/seenPayloadEnvelopeInput.js";
-import {processExecutionPayload} from "./importExecutionPayload.js";
+import {importExecutionPayload} from "./importExecutionPayload.js";
 import {ImportPayloadOpts} from "./types.js";
+import {verifyPayloadsDataAvailability} from "./verifyPayloadsDataAvailability.js";
 
 // TODO GLOAS: Set to be equal to DEFAULT_MAX_PENDING_UNFINALIZED_PAYLOAD_ENVELOPE_WRITES for now
 const QUEUE_MAX_LENGTH = 16;
 
 /**
- * PayloadEnvelopeProcessor processes payload envelope jobs in a queued fashion, one after the other.
+ * PayloadEnvelopeProcessor imports payload envelopes in a queued fashion, one after the other.
  *
- * Jobs are enqueued only on envelope arrival (gossip or API). The envelope may reach us before
- * the sampled data columns; importExecutionPayload awaits `verifyPayloadsDataAvailability`
- * internally, so a queued job can pend for up to `PAYLOAD_DATA_AVAILABILITY_TIMEOUT` while
- * waiting for columns. Duplicate triggers for the same payloadInput are deduped by sharing the
- * in-flight import promise: every caller observes the real import outcome. Resolving duplicates
- * early instead would let BlockInputSync mark a still-pending payload as processed, re-queue it,
- * and spin re-processing the same root until OOM while the import is blocked (e.g. EL syncing).
+ * Jobs are triggered only on envelope arrival (gossip or API). The envelope may reach us before the
+ * sampled data columns, the data availability wait runs before the job is queued. Duplicate
+ * triggers for the same payloadInput are deduped by sharing the in-flight promise: every caller
+ * observes the real import outcome. Resolving duplicates early instead would let
+ * BlockInputSync mark a still-pending payload as processed, re-queue it, and spin re-processing
+ * the same root until OOM while the import is blocked (e.g. EL syncing).
  */
 export class PayloadEnvelopeProcessor {
-  readonly jobQueue: JobItemQueue<[PayloadEnvelopeInput, ImportPayloadOpts], void>;
+  readonly jobQueue: JobItemQueue<[PayloadEnvelopeInput, DataAvailabilityStatus, ImportPayloadOpts], void>;
   private readonly imports = new WeakMap<PayloadEnvelopeInput, Promise<void>>();
 
-  constructor(chain: BeaconChain, metrics: Metrics | null, signal: AbortSignal) {
-    this.jobQueue = new JobItemQueue<[PayloadEnvelopeInput, ImportPayloadOpts], void>(
-      (payloadInput, opts) => {
-        return processExecutionPayload.call(chain, payloadInput, signal, opts);
+  constructor(
+    private readonly chain: BeaconChain,
+    metrics: Metrics | null,
+    private readonly signal: AbortSignal
+  ) {
+    this.jobQueue = new JobItemQueue<[PayloadEnvelopeInput, DataAvailabilityStatus, ImportPayloadOpts], void>(
+      (payloadInput, dataAvailabilityStatus, opts) => {
+        return importExecutionPayload.call(chain, payloadInput, dataAvailabilityStatus, opts);
       },
       {maxLength: QUEUE_MAX_LENGTH, noYieldIfOneItem: true, signal},
       metrics?.payloadEnvelopeProcessorQueue ?? undefined
@@ -35,19 +40,11 @@ export class PayloadEnvelopeProcessor {
 
   async processPayloadEnvelopeJob(payloadInput: PayloadEnvelopeInput, opts: ImportPayloadOpts = {}): Promise<void> {
     const existing = this.imports.get(payloadInput);
-    if (existing) {
+    if (existing !== undefined) {
       return existing;
     }
 
-    await this.jobQueue.waitForSpace();
-
-    // Re-check after await, as another call may have queued this payload.
-    const queued = this.imports.get(payloadInput);
-    if (queued) {
-      return queued;
-    }
-
-    const importPromise = this.jobQueue.push(payloadInput, opts);
+    const importPromise = this.waitForDataAndImport(payloadInput, opts);
     this.imports.set(payloadInput, importPromise);
 
     try {
@@ -59,4 +56,18 @@ export class PayloadEnvelopeProcessor {
       throw e;
     }
   }
+
+  private async waitForDataAndImport(payloadInput: PayloadEnvelopeInput, opts: ImportPayloadOpts): Promise<void> {
+    const {dataAvailabilityStatuses} = await verifyPayloadsDataAvailability(
+      this.chain.config,
+      [payloadInput],
+      this.signal
+    );
+
+    while (this.jobQueue.jobLen >= QUEUE_MAX_LENGTH) {
+      await this.jobQueue.waitForSpace();
+    }
+
+    return this.jobQueue.push(payloadInput, dataAvailabilityStatuses[0], opts);
+  }
 }

package/src/chain/blocks/types.ts

@@ -12,6 +12,11 @@ export enum GossipedInputType {
   dataColumn = "data_column",
 }
 
+export enum InclusionListSource {
+  gossip = "gossip",
+  api = "api",
+}
+
 type DataColumnData = {
   dataColumn: fulu.DataColumnSidecar;
   dataColumnBytes: Uint8Array | null;
@@ -104,7 +109,10 @@ export type FullyVerifiedBlock = {
   parentBlockSlot: Slot;
   proposerBalanceDelta: number;
   dataAvailabilityStatus: DataAvailabilityStatus;
-  /** Pre-computed indexed attestations from signature verification to avoid duplicate work */
+  /**
+   * Pre-computed indexed attestations from signature verification to avoid duplicate work.
+   * Their `attestingIndices` are in committee order, not sorted, so they are not spec-valid `IndexedAttestation`s.
+   */
   indexedAttestations: IndexedAttestation[];
   /** Seen timestamp seconds */
   seenTimestampSec: number;