@lodestar/beacon-node 1.51.0-dev.e22f67e7bc → 1.51.0-dev.e2bf489dd3

package/src/chain/validation/executionPayloadEnvelope.ts

@@ -1,4 +1,4 @@
-import {PayloadStatus} from "@lodestar/fork-choice";
+import {PayloadStatus, ProtoBlock} from "@lodestar/fork-choice";
 import {
   computeStartSlotAtEpoch,
   getExecutionPayloadEnvelopeSignatureSet,
@@ -6,9 +6,15 @@ import {
 } from "@lodestar/state-transition";
 import {gloas, ssz} from "@lodestar/types";
 import {byteArrayEquals, toRootHex} from "@lodestar/utils";
+import {
+  getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized,
+  getSlotFromExecutionPayloadEnvelopeSerialized,
+  getTransactionCountFromExecutionPayloadEnvelopeSerialized,
+} from "../../util/sszBytes.js";
 import {ExecutionPayloadEnvelopeError, ExecutionPayloadEnvelopeErrorCode, GossipAction} from "../errors/index.js";
 import {IBeaconChain} from "../index.js";
 import {RegenCaller} from "../regen/index.js";
+import {PayloadEnvelopeInput} from "../seenCache/seenPayloadEnvelopeInput.js";
 
 export async function validateApiExecutionPayloadEnvelope(
   chain: IBeaconChain,
@@ -24,6 +30,40 @@ export async function validateGossipExecutionPayloadEnvelope(
   return validateExecutionPayloadEnvelope(chain, executionPayloadEnvelope);
 }
 
+/**
+ * Applies the transaction count check to the serialized envelope so that an oversized envelope is dropped
+ * before it is deserialized. Defers to validateGossipExecutionPayloadEnvelope if the data is malformed
+ * or the block is not known.
+ */
+export function validateGossipExecutionPayloadEnvelopeSerialized(
+  chain: IBeaconChain,
+  serializedData: Uint8Array
+): void {
+  const blockRootHex = getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized(serializedData);
+  const slot = getSlotFromExecutionPayloadEnvelopeSerialized(serializedData);
+  const transactionCount = getTransactionCountFromExecutionPayloadEnvelopeSerialized(serializedData);
+  if (blockRootHex === null || slot === null || transactionCount === null) {
+    return;
+  }
+
+  const block = chain.forkChoice.getBlockHexDefaultStatus(blockRootHex);
+  const payloadInput = chain.seenPayloadEnvelopeInputCache.get(blockRootHex);
+  if (!block || !payloadInput) {
+    return;
+  }
+
+  const maxTransactionCount = getMaxTransactionCount(chain, block, payloadInput);
+  if (BigInt(transactionCount) > maxTransactionCount) {
+    throw new ExecutionPayloadEnvelopeError(GossipAction.IGNORE, {
+      code: ExecutionPayloadEnvelopeErrorCode.TOO_MANY_TRANSACTIONS,
+      slot,
+      root: blockRootHex,
+      transactionCount,
+      maxTransactionCount: Number(maxTransactionCount),
+    });
+  }
+}
+
 async function validateExecutionPayloadEnvelope(
   chain: IBeaconChain,
   executionPayloadEnvelope: gloas.SignedExecutionPayloadEnvelope
@@ -123,6 +163,20 @@ async function validateExecutionPayloadEnvelope(
     });
   }
 
+  // [IGNORE] The number of transactions is within the limit implied by the parent payload's gas limit
+  // TODO GLOAS: [REJECT] once https://github.com/ethereum/consensus-specs/pull/5802 is released,
+  // also in validateGossipExecutionPayloadEnvelopeSerialized
+  const maxTransactionCount = getMaxTransactionCount(chain, block, payloadInput);
+  if (BigInt(payload.transactions.length) > maxTransactionCount) {
+    throw new ExecutionPayloadEnvelopeError(GossipAction.IGNORE, {
+      code: ExecutionPayloadEnvelopeErrorCode.TOO_MANY_TRANSACTIONS,
+      slot: payload.slotNumber,
+      root: blockRootHex,
+      transactionCount: payload.transactions.length,
+      maxTransactionCount: Number(maxTransactionCount),
+    });
+  }
+
   // Get the block state to verify the builder's signature.
   const blockState = await chain.regen
     .getState(block.stateRoot, RegenCaller.validateGossipPayloadEnvelope)
@@ -154,3 +208,16 @@ async function validateExecutionPayloadEnvelope(
     });
   }
 }
+
+/**
+ * The bid's gas limit is only tied to the parent payload by bid gossip, so the bound is the gas limit the
+ * EIP-1559 rule allows on top of the parent payload's. The bid is the only bound if that gas limit is unknown.
+ */
+function getMaxTransactionCount(chain: IBeaconChain, block: ProtoBlock, payloadInput: PayloadEnvelopeInput): bigint {
+  const parentGasLimit = block.executionPayloadBlockHash === null ? 0n : BigInt(block.executionPayloadGasLimit);
+  const maxGasLimit =
+    parentGasLimit === 0n
+      ? payloadInput.getBid().gasLimit
+      : parentGasLimit + (parentGasLimit / 1024n > 1n ? parentGasLimit / 1024n : 1n) - 1n;
+  return maxGasLimit / BigInt(chain.config.MIN_TRANSACTION_GAS);
+}

package/src/chain/validation/inclusionList.ts

@@ -31,7 +31,10 @@ export async function validateGossipInclusionList(
   signedInclusionList: heze.SignedInclusionList
 ): Promise<InclusionListValidationResult> {
   const {slot, validatorIndex, transactions, dependentRoot} = signedInclusionList.message;
-  const totalBytes = transactions.reduce((total, transaction) => total + transaction.byteLength, 0);
+  let totalBytes = 0;
+  for (let i = 0; i < transactions.length; i++) {
+    totalBytes += transactions.getSize(i);
+  }
 
   // A pre-heze slot has no inclusion list committee to validate against
   if (!isForkPostHeze(chain.config.getForkName(slot))) {
@@ -75,7 +78,13 @@ export async function validateGossipInclusionList(
   }
 
   // [REJECT] Every transaction in message.transactions is non-empty
-  const emptyTransactionIndex = transactions.findIndex((transaction) => transaction.byteLength === 0);
+  let emptyTransactionIndex = -1;
+  for (let i = 0; i < transactions.length; i++) {
+    if (transactions.getSize(i) === 0) {
+      emptyTransactionIndex = i;
+      break;
+    }
+  }
   if (emptyTransactionIndex !== -1) {
     throw new InclusionListError(GossipAction.REJECT, {
       code: InclusionListErrorCode.EMPTY_TRANSACTION,

package/src/db/beacon.ts

@@ -30,6 +30,9 @@ import {
   VoluntaryExitRepository,
 } from "./repositories/index.js";
 
+/** Below this size, seeks over leftover legacy column data stay cheap */
+const LEGACY_DATA_COLUMNS_COMPACTION_MIN_BYTES = 1024 ** 3;
+
 export type BeaconDbOpts = {
   dataColumnDir: string;
   logger: Logger;
@@ -67,6 +70,7 @@ export class BeaconDb implements IBeaconDb {
   readonly dataColumns: IDataColumnStore;
   lastLegacyArchiveSlot: Slot | null = null;
   private readonly flatFileStore: FlatFileStore;
+  private readonly logger: Logger;
 
   constructor(
     config: ChainForkConfig,
@@ -100,6 +104,7 @@ export class BeaconDb implements IBeaconDb {
     this.syncCommittee = new SyncCommitteeRepository(config, db);
     this.syncCommitteeWitness = new SyncCommitteeWitnessRepository(config, db);
 
+    this.logger = opts.logger;
     this.flatFileStore = new FlatFileStore(opts.dataColumnDir, config, opts.logger, opts.metrics);
     this.dataColumns = new LegacyDataColumnStore(
       this.flatFileStore,
@@ -125,11 +130,50 @@ export class BeaconDb implements IBeaconDb {
     this.flatFileStore.setMetrics(flatFileStoreMetrics);
   }
 
+  /**
+   * Deletes made while an iterator held a snapshot can stay in the bottom LevelDB level, which `compactRange` never
+   * rewrites, so every seek into the legacy column buckets walks them. Writing a key at each end of the pruned range
+   * makes the compaction overlap those files and rewrite them.
+   */
+  async compactLegacyDataColumns(
+    prunedBeforeSlot: Slot,
+    minSizeBytes = LEGACY_DATA_COLUMNS_COMPACTION_MIN_BYTES
+  ): Promise<void> {
+    const archiveBucket = Bucket.allForks_dataColumnSidecarsArchive;
+    const start = encodeKey(Bucket.allForks_dataColumnSidecars, Buffer.alloc(0));
+    const end = encodeKey(archiveBucket, this.dataColumnSidecarArchive.getMinKeyRaw(prunedBeforeSlot));
+    const sizeBytes = await this.db.approximateSize(start, end);
+    if (sizeBytes < minSizeBytes) return;
+
+    // Column indices stay below 0xffff and hot column keys are shorter, so no sentinel matches a real key
+    const upperSentinel =
+      prunedBeforeSlot > 0
+        ? encodeKey(archiveBucket, this.dataColumnSidecarArchive.encodeKeyRaw(prunedBeforeSlot - 1, 0xffff))
+        : encodeKey(Bucket.allForks_dataColumnSidecars, Buffer.alloc(35, 0xff));
+    const sentinels = [start, upperSentinel];
+
+    this.logger.info("Compacting legacy data columns, this may take a while", {sizeBytes});
+    const startTime = Date.now();
+    await this.db.batchPut(sentinels.map((key) => ({key, value: new Uint8Array()})));
+    await this.db.compactRange(start, end);
+    await this.db.batchDelete(sentinels);
+    await this.db.compactRange(start, end);
+    this.logger.verbose("Compacted legacy data columns", {
+      durationMs: Date.now() - startTime,
+      sizeBytes: await this.db.approximateSize(start, end),
+    });
+  }
+
   async pruneHotDb(): Promise<void> {
     // Prune all hot blobs
     await this.blobSidecars.batchDelete(await this.blobSidecars.keys());
     // New columns are persisted in flat files, so this bucket only contains legacy hot data.
     await this.deleteBucketData(Bucket.allForks_dataColumnSidecars);
+    // Compact even if empty to reclaim tombstones left by previous startups.
+    await this.db.compactRange(
+      encodeKey(Bucket.allForks_dataColumnSidecars, Buffer.alloc(0)),
+      encodeKey(Bucket.allForks_dataColumnSidecars + 1, Buffer.alloc(0))
+    );
     // Prune all hot blocks
     // TODO: Enable once it's deemed safe
     // await this.block.batchDelete(await this.block.keys());

package/src/db/dataColumnStore.ts

@@ -13,7 +13,7 @@ export type IndexedDataColumnBytes = {index: ColumnIndex; data: Uint8Array};
 type LegacyHotDataColumnStore = Pick<DataColumnSidecarRepository, "values" | "getManyBinary" | "deleteMany">;
 type LegacyArchiveDataColumnStore = Pick<
   DataColumnSidecarArchiveRepository,
-  "values" | "getManyBinary" | "keys" | "deleteMany"
+  "values" | "getManyBinary" | "pruneBefore"
 >;
 type BlockRootIndex = Pick<BlockArchiveRepository, "getSlotByRoot">;
 
@@ -86,9 +86,7 @@ export class LegacyDataColumnStore implements IDataColumnStore {
 
   async pruneBefore(slot: Slot): Promise<Slot[]> {
     const flatFileSlots = await this.flatFiles.pruneBefore(slot);
-    const prefixedKeys = await this.legacyArchive.keys({lt: {prefix: slot, id: 0}});
-    const slots = [...new Set(prefixedKeys.map(({prefix}) => prefix))];
-    if (slots.length > 0) await this.legacyArchive.deleteMany(slots);
+    const slots = await this.legacyArchive.pruneBefore(slot);
     return [...new Set([...flatFileSlots, ...slots])].sort((a, b) => a - b);
   }
 }

package/src/db/flatFileStore/flatFileStore.ts

@@ -1,4 +1,4 @@
-import fs from "node:fs";
+import {type FileHandle, open, readFile, rm} from "node:fs/promises";
 import path from "node:path";
 import {ChainForkConfig} from "@lodestar/config";
 import {ForkPostFulu} from "@lodestar/params";
@@ -172,7 +172,7 @@ export class FlatFileStore implements IFlatFileStore {
       for (const slot of slotsToPrune) {
         const release = await this.acquireMutationLock(slot);
         try {
-          await fs.promises.rm(path.join(this.dataColumnDir, padSlot(slot)), {recursive: true, force: true});
+          await rm(path.join(this.dataColumnDir, padSlot(slot)), {recursive: true, force: true});
           this.slotIndex.remove(slot);
           this.metrics?.prunedDirectories.inc();
         } finally {
@@ -212,7 +212,7 @@ export class FlatFileStore implements IFlatFileStore {
 
   private async readFile(slot: Slot, rootHex: RootHex): Promise<Uint8Array | null> {
     try {
-      return await fs.promises.readFile(this.filePath(slot, rootHex));
+      return await readFile(this.filePath(slot, rootHex));
     } catch (e) {
       if (!isFsNotFoundError(e)) throw e;
       return null;
@@ -224,9 +224,9 @@ export class FlatFileStore implements IFlatFileStore {
     rootHex: RootHex,
     indices: number[]
   ): Promise<(Uint8Array | undefined)[] | null> {
-    let fd: fs.promises.FileHandle;
+    let fd: FileHandle;
     try {
-      fd = await fs.promises.open(this.filePath(slot, rootHex), "r");
+      fd = await open(this.filePath(slot, rootHex), "r");
     } catch (e) {
       if (!isFsNotFoundError(e)) throw e;
       return null;
@@ -270,7 +270,7 @@ export class FlatFileStore implements IFlatFileStore {
     await observeFlatFileStoreOperation(this.metrics, FlatFileStoreOperation.delete, async () => {
       const release = await this.acquireMutationLock(slot);
       try {
-        await fs.promises.rm(this.filePath(slot, rootHex), {force: true});
+        await rm(this.filePath(slot, rootHex), {force: true});
       } finally {
         release();
       }
@@ -294,7 +294,7 @@ function validateDcolHeader(header: DcolHeader, slot: Slot, rootHex: RootHex): v
   }
 }
 
-async function readExactly(fd: fs.promises.FileHandle, buffer: Uint8Array, position: number): Promise<void> {
+async function readExactly(fd: FileHandle, buffer: Uint8Array, position: number): Promise<void> {
   let totalRead = 0;
   while (totalRead < buffer.length) {
     const {bytesRead} = await fd.read(buffer, totalRead, buffer.length - totalRead, position + totalRead);

package/src/db/flatFileStore/slotIndex.ts

@@ -1,4 +1,5 @@
-import fs from "node:fs";
+import type {Dirent} from "node:fs";
+import {readdir} from "node:fs/promises";
 import {Slot} from "@lodestar/types";
 import {isFsNotFoundError} from "./errors.js";
 import {padSlot} from "./path.js";
@@ -33,9 +34,9 @@ export class SlotIndex {
 
   async rebuildFromDisk(columnsDir: string): Promise<SlotIndexRebuildStats> {
     this.slots.clear();
-    let entries: fs.Dirent[];
+    let entries: Dirent[];
     try {
-      entries = await fs.promises.readdir(columnsDir, {withFileTypes: true});
+      entries = await readdir(columnsDir, {withFileTypes: true});
     } catch (e) {
       if (!isFsNotFoundError(e)) throw e;
       return {slots: 0, ignoredEntries: 0};

package/src/db/interface.ts

@@ -68,6 +68,8 @@ export interface IBeaconDb {
   readonly lastLegacyArchiveSlot: Slot | null;
   init(): Promise<void>;
 
+  compactLegacyDataColumns(prunedBeforeSlot: Slot, minSizeBytes?: number): Promise<void>;
+
   pruneHotDb(): Promise<void>;
 
   deleteDeprecatedEth1Data(): Promise<void>;

package/src/db/repositories/dataColumnSidecarArchive.ts

@@ -15,6 +15,8 @@ const SLOT_BYTE_SIZE = 8;
  * Indexed data by `slot` + `columnIndex`
  */
 export class DataColumnSidecarArchiveRepository extends PrefixedRepository<Slot, ColumnIndex, DataColumnSidecar> {
+  private prunedBeforeSlot: Slot | null = null;
+
   constructor(config: ChainForkConfig, db: Db) {
     const bucket = Bucket.allForks_dataColumnSidecarsArchive;
     // Type won't be used since we select it dynamically based on fork
@@ -70,4 +72,22 @@ export class DataColumnSidecarArchiveRepository extends PrefixedRepository<Slot,
       encodeNumberForDbKey(0, COLUMN_INDEX_BYTE_SIZE),
     ]);
   }
+
+  /**
+   * Deletes all entries before `slot`. Prunes only delete from the start of the bucket, so the first prune
+   * compacts the range to drop tombstones of earlier runs and later prunes resume where the previous one ended.
+   */
+  async pruneBefore(slot: Slot): Promise<Slot[]> {
+    if (this.prunedBeforeSlot === null) {
+      await this.db.compactRange(this.wrapKey(this.getMinKeyRaw(0)), this.wrapKey(this.getMinKeyRaw(slot)));
+    } else if (slot <= this.prunedBeforeSlot) {
+      return [];
+    }
+
+    const keys = await this.keys({gte: {prefix: this.prunedBeforeSlot ?? 0, id: 0}, lt: {prefix: slot, id: 0}});
+    const slots = [...new Set(keys.map(({prefix}) => prefix))];
+    if (slots.length > 0) await this.deleteMany(slots);
+    this.prunedBeforeSlot = slot;
+    return slots;
+  }
 }

package/src/execution/builder/apiClient.ts

@@ -19,21 +19,20 @@ import {IClock} from "../../util/clock.js";
  * Additional duration to account for potential event loop lag which causes
  * builder bids to be rejected even though the response was sent in time.
  */
-const EVENT_LOOP_LAG_BUFFER = 250;
+export const BUILDER_BID_LAG_BUFFER_MS = 250;
 
 /**
- * Duration given to a builder to provide a `SignedExecutionPayloadBid` before the deadline
- * is reached, advertised on each bid request via the `X-Timeout-Ms` header. The p2p bid is
- * selected after the same duration, only considering bids received up to that point.
+ * Default duration given to a builder to provide a `SignedExecutionPayloadBid` before the deadline
+ * is reached, advertised on each bid request via the `X-Timeout-Ms` header. Bids received via
+ * p2p are given the same time to bid up.
  */
-export const BUILDER_BID_DEADLINE_MS = 500;
-
-/** Local timeout for bid requests, event loop lag must not discard bids that arrived in time */
-export const BUILDER_BID_REQUEST_TIMEOUT_MS = BUILDER_BID_DEADLINE_MS + EVENT_LOOP_LAG_BUFFER;
+export const DEFAULT_BUILDER_BID_DEADLINE_MS = 500;
 
 export type BuilderApiClientOpts = {
-  /** Timeout for builder api requests, bid requests always use `BUILDER_BID_REQUEST_TIMEOUT_MS` */
+  /** Timeout for builder api requests, bid requests are bounded by `bidDeadline` instead */
   timeout?: number;
+  /** Time builders are given to provide a bid once block production starts, in milliseconds */
+  bidDeadline?: number;
   // Add User-Agent header to all requests
   userAgent?: string;
 };
@@ -69,6 +68,7 @@ export type BuilderApiBid = {
  * each block production request, clients are dialed on demand based on the entry `url`.
  */
 export class BuilderApiClient {
+  readonly bidDeadline: number;
   private readonly clients = new Map<BuilderUrl, BuilderApi>();
 
   constructor(
@@ -78,7 +78,9 @@ export class BuilderApiClient {
     private readonly bls: IBlsVerifier,
     private readonly metrics: Metrics | null = null,
     private readonly logger?: Logger
-  ) {}
+  ) {
+    this.bidDeadline = opts.bidDeadline ?? DEFAULT_BUILDER_BID_DEADLINE_MS;
+  }
 
   /**
    * Fan out bid requests to the builders named by the entries, one request per unique
@@ -139,9 +141,10 @@ export class BuilderApiClient {
               proposerPubkey,
               requestAuth: entry.auth,
               dateMilliseconds: Date.now(),
-              timeoutMs: BUILDER_BID_DEADLINE_MS,
+              timeoutMs: this.bidDeadline,
             },
-            {timeoutMs: BUILDER_BID_REQUEST_TIMEOUT_MS}
+            // Event loop lag must not discard bids that arrived in time
+            {timeoutMs: this.bidDeadline + BUILDER_BID_LAG_BUFFER_MS}
           );
           const signedBid = res.value();
           if (signedBid === undefined) {
@@ -203,7 +206,7 @@ export class BuilderApiClient {
 
   /**
    * Establish the connection to every known builder outside the bid deadline, a cold TCP/TLS handshake
-   * to a distant builder can exhaust `BUILDER_BID_DEADLINE_MS` on its own, and a long idle connection
+   * to a distant builder can exhaust the bid deadline on its own, and a long idle connection
    * may be half-open and only fail once written to.
    */
   async checkStatus(): Promise<void> {

package/src/execution/builder/http.ts

@@ -20,6 +20,7 @@ import {
 } from "@lodestar/types";
 import {toPrintableUrl} from "@lodestar/utils";
 import {Metrics} from "../../metrics/metrics.js";
+import {DEFAULT_BUILDER_BID_DEADLINE_MS} from "./apiClient.js";
 import {ValidatorRegistration, ValidatorRegistrationCache} from "./cache.js";
 import {IExecutionBuilder} from "./interface.js";
 
@@ -29,6 +30,8 @@ export type ExecutionBuilderHttpOpts = {
   timeout?: number;
   faultInspectionWindow?: number;
   allowedFaults?: number;
+  /** Post-gloas only, time builders are given to provide a bid once block production starts, in milliseconds */
+  bidDeadline?: number;
 
   // Only required for merge-mock runs, no need to expose it to cli
   issueLocalFcUWithFeeRecipient?: string;
@@ -40,6 +43,7 @@ export const defaultExecutionBuilderHttpOpts: ExecutionBuilderHttpOpts = {
   enabled: false,
   url: "http://localhost:8661",
   timeout: 12000,
+  bidDeadline: DEFAULT_BUILDER_BID_DEADLINE_MS,
 };
 
 export enum BuilderStatus {

package/src/execution/engine/blobVersionedHashes.ts

@@ -7,7 +7,7 @@ import {VersionedHashes} from "./interface.js";
  * Skip unrelated fields without decoding or copying them, including untrusted nested access lists.
  * The EL remains responsible for the other transaction validity checks.
  */
-export function isValidBlobVersionedHashes(transactions: Uint8Array[], expected: VersionedHashes): boolean {
+export function isValidBlobVersionedHashes(transactions: Iterable<Uint8Array>, expected: VersionedHashes): boolean {
   let index = 0;
   for (const transaction of transactions) {
     if (transaction.length === 0) return false;

package/src/execution/engine/http.ts

@@ -22,9 +22,11 @@ import {
   ExecutePayloadResponse,
   ExecutionEngineState,
   ExecutionPayloadStatus,
+  ForkchoiceUpdatedResult,
   IExecutionEngine,
   PayloadAttributes,
   PayloadId,
+  PayloadStatusResult,
   VersionedHashes,
 } from "./interface.js";
 import {ErrorJsonRpcResponse, HttpRpcError, JsonRpcHttpClientEvent} from "./jsonRpcHttpClient.js";
@@ -32,7 +34,7 @@ import {PayloadIdCache} from "./payloadIdCache.js";
 import {EngineRestError, EngineRestResponseError, isRetryableEngineRestError} from "./restHttpClient.js";
 import {EngineCapabilities, RestEngineTransport} from "./restTransport.js";
 import {executionForkName} from "./sszTypes.js";
-import {IEngineTransport, PayloadStatusResult} from "./transport.js";
+import {IEngineTransport} from "./transport.js";
 import {ExecutionPayloadBodyV2, serializePayloadAttributes} from "./types.js";
 import {getExecutionEngineState} from "./utils.js";
 
@@ -342,15 +344,16 @@ export class ExecutionEngineHttp implements IExecutionEngine {
     safeBlockHash: RootHex,
     finalizedBlockHash: RootHex,
     payloadAttributes?: PayloadAttributes
-  ): Promise<PayloadId | null> {
-    const {
-      payloadStatus: {status, validationError},
-      payloadId,
-    } = await this.enqueue(() =>
+  ): Promise<ForkchoiceUpdatedResult> {
+    const result = await this.enqueue(() =>
       this.withTransport(fork, undefined, (transport) =>
         transport.forkchoiceUpdated(fork, headBlockHash, safeBlockHash, finalizedBlockHash, payloadAttributes)
       )
     );
+    const {
+      payloadStatus: {status},
+      payloadId,
+    } = result;
 
     this.updateEngineState(getExecutionEngineState({payloadStatus: status, oldState: this.state}));
     this.metrics?.engineNotifyForkchoiceUpdateResult.inc({result: status});
@@ -367,7 +370,6 @@ export class ExecutionEngineHttp implements IExecutionEngine {
           this.payloadIdCache.add(
             {
               headBlockHash,
-              finalizedBlockHash,
               ...payloadAttributesRpc,
               inclusionListTransactions: payloadAttributes.inclusionListTransactions,
             },
@@ -375,21 +377,17 @@ export class ExecutionEngineHttp implements IExecutionEngine {
           );
           void this.prunePayloadIdCache();
         }
-        return payloadId;
+        return result;
 
       case ExecutionPayloadStatus.SYNCING:
         // Throw error on syncing if requested to produce a block, else silently ignore
         if (payloadAttributes) {
           throw Error("Execution Layer Syncing");
         }
-        return null;
+        return result;
 
       case ExecutionPayloadStatus.INVALID:
-        throw Error(
-          `Invalid ${payloadAttributes ? "prepare payload" : "forkchoice request"}, validationError=${
-            validationError ?? ""
-          }`
-        );
+        return result;
 
       default:
         throw Error(`Unknown status ${status}`);

package/src/execution/engine/index.ts

@@ -17,8 +17,8 @@ import {EngineRestHttpClient} from "./restHttpClient.js";
 import {RestEngineTransport, formatClientVersionHeader} from "./restTransport.js";
 import {ExecutionEngineMockJsonRpcClient, JsonRpcBackend} from "./utils.js";
 
-export {ExecutionEngineHttp, ExecutionEngineDisabled, defaultExecutionEngineHttpOpts, engineApiModes};
 export type {EngineApiMode};
+export {ExecutionEngineDisabled, ExecutionEngineHttp, defaultExecutionEngineHttpOpts, engineApiModes};
 
 export type ExecutionEngineOpts =
   | ({mode?: "http"} & ExecutionEngineHttpOpts)

package/src/execution/engine/interface.ts

@@ -25,7 +25,7 @@ import {PayloadId, PayloadIdCache, WithdrawalV1} from "./payloadIdCache.js";
 import {ExecutionPayloadBodyV2} from "./types.js";
 import {DATA} from "./utils.js";
 
-export {PayloadIdCache, type PayloadId, type WithdrawalV1};
+export {type PayloadId, PayloadIdCache, type WithdrawalV1};
 export type ClientVersion = routes.node.ClientVersion;
 export const ClientCode = routes.node.ClientCode;
 
@@ -99,6 +99,19 @@ export type ExecutePayloadResponse =
       validationError: string;
     };
 
+export type PayloadStatusResult = {
+  status: ExecutionPayloadStatus;
+  latestValidHash: RootHex | null;
+  validationError: string | null;
+  /** PayloadStatusV2: whether a VALID payload satisfied the inclusion list constraints, null otherwise */
+  inclusionListSatisfied?: boolean | null;
+};
+
+export type ForkchoiceUpdatedResult = {
+  payloadStatus: PayloadStatusResult;
+  payloadId: PayloadId | null;
+};
+
 export type ForkChoiceUpdateStatus =
   | ExecutionPayloadStatus.VALID
   | ExecutionPayloadStatus.INVALID
@@ -166,7 +179,10 @@ export interface IExecutionEngine {
    * The call of the notify_forkchoice_updated function maps on the POS_FORKCHOICE_UPDATED event defined in the EIP-3675.
    * https://github.com/ethereum/consensus-specs/blob/v1.1.7/specs/merge/fork-choice.md#notify_forkchoice_updated
    *
-   * Should be called in response to fork-choice head and finalized events
+   * Should be called in response to fork-choice head and finalized events.
+   *
+   * An INVALID verdict is returned, not thrown: the caller feeds it back into fork choice together with the
+   * `latestValidHash`. Only SYNCING while building a payload is an error.
    */
   notifyForkchoiceUpdate(
     fork: ForkName,
@@ -174,7 +190,7 @@ export interface IExecutionEngine {
     safeBlockHash: RootHex,
     finalizedBlockHash: RootHex,
     payloadAttributes?: PayloadAttributes
-  ): Promise<PayloadId | null>;
+  ): Promise<ForkchoiceUpdatedResult>;
 
   /**
    * Given the payload_id, get_payload returns the most recent version of the execution payload that has been built

package/src/execution/engine/jsonRpcTransport.ts

@@ -3,9 +3,17 @@ import {ExecutionPayload, ExecutionRequests, Root, RootHex, bellatrix} from "@lo
 import {BlobAndProof} from "@lodestar/types/deneb";
 import {BlobAndProofV2} from "@lodestar/types/fulu";
 import {strip0xPrefix} from "@lodestar/utils";
-import {ClientCode, ClientVersion, PayloadAttributes, PayloadId, VersionedHashes} from "./interface.js";
+import {
+  ClientCode,
+  ClientVersion,
+  ForkchoiceUpdatedResult,
+  PayloadAttributes,
+  PayloadId,
+  PayloadStatusResult,
+  VersionedHashes,
+} from "./interface.js";
 import {IJsonRpcHttpClient, JsonRpcHttpClientEventEmitter, ReqOpts} from "./jsonRpcHttpClient.js";
-import {ForkchoiceUpdatedResult, GetPayloadResult, IEngineTransport, PayloadStatusResult} from "./transport.js";
+import {GetPayloadResult, IEngineTransport} from "./transport.js";
 import {
   BLOB_AND_PROOF_V2_RPC_BYTES,
   EngineApiRpcParamTypes,

package/src/execution/engine/mock.ts

@@ -1,4 +1,5 @@
 import crypto from "node:crypto";
+import {SerializedList} from "@chainsafe/ssz";
 import {ChainConfig} from "@lodestar/config";
 import {
   BLOB_TX_TYPE,
@@ -352,14 +353,14 @@ export class ExecutionEngineMockBackend implements JsonRpcBackend {
       executionPayload.gasUsed = Math.floor(0.5 * INTEROP_GAS_LIMIT);
       executionPayload.timestamp = payloadAttributes.timestamp;
       executionPayload.blockHash = crypto.randomBytes(32);
-      executionPayload.transactions = [];
+      const transactions: Uint8Array[] = [];
 
       // Between 0 and 4 transactions for all forks
       const eip1559TxCount = Math.round(4 * Math.random());
       for (let i = 0; i < eip1559TxCount; i++) {
         const tx = crypto.randomBytes(512);
         tx[0] = TX_TYPE_EIP1559;
-        executionPayload.transactions.push(tx);
+        transactions.push(tx);
       }
 
       const commitments: deneb.KZGCommitment[] = [];
@@ -373,7 +374,7 @@ export class ExecutionEngineMockBackend implements JsonRpcBackend {
           const blob = generateRandomBlob();
           const commitment = kzg.blobToKzgCommitment(blob);
           const {proofs: cellProofs} = kzg.computeCellsAndKzgProofs(blob);
-          executionPayload.transactions.push(transactionForKzgCommitment(commitment));
+          transactions.push(transactionForKzgCommitment(commitment));
           commitments.push(commitment);
           blobs.push(blob);
           proofs.push(...cellProofs);
@@ -385,12 +386,13 @@ export class ExecutionEngineMockBackend implements JsonRpcBackend {
           const blob = generateRandomBlob();
           const commitment = kzg.blobToKzgCommitment(blob);
           const proof = kzg.computeBlobKzgProof(blob, commitment);
-          executionPayload.transactions.push(transactionForKzgCommitment(commitment));
+          transactions.push(transactionForKzgCommitment(commitment));
           commitments.push(commitment);
           blobs.push(blob);
           proofs.push(proof);
         }
       }
+      executionPayload.transactions = SerializedList.fromArray(transactions);
 
       if (ForkSeq[fork] >= ForkSeq.capella) {
         (executionPayload as ExecutionPayload<ForkPostCapella>).withdrawals = ssz.capella.Withdrawals.defaultValue();