@lodestar/beacon-node 1.45.0-dev.51a1c44b27 → 1.45.0-dev.6568180f96

package/lib/api/impl/validator/index.js

@@ -3,41 +3,28 @@ import { ExecutionStatus, PayloadStatus } from "@lodestar/fork-choice";
 import { BUILDER_INDEX_SELF_BUILD, ForkName, ForkSeq, GENESIS_SLOT, SLOTS_PER_EPOCH, SLOTS_PER_HISTORICAL_ROOT, SYNC_COMMITTEE_SUBNET_SIZE, isForkPostBellatrix, isForkPostDeneb, isForkPostElectra, isForkPostFulu, isForkPostGloas, } from "@lodestar/params";
 import { DataAvailabilityStatus, beaconBlockToBlinded, calculateCommitteeAssignments, computeEpochAtSlot, computeStartSlotAtEpoch, computeTimeAtSlot, getCurrentSlot, isStatePostAltair, isStatePostGloas, proposerShufflingDecisionRoot, } from "@lodestar/state-transition";
 import { ProducedBlockSource, getValidatorStatus, ssz, } from "@lodestar/types";
-import { TimeoutError, defer, formatWeiToEth, fromHex, prettyWeiToEth, resolveOrRacePromises, toHex, toRootHex, } from "@lodestar/utils";
+import { GWEI_TO_WEI, TimeoutError, defer, formatWeiToEth, fromHex, prettyWeiToEth, resolveOrRacePromises, toHex, toRootHex, } from "@lodestar/utils";
 import { MAX_BUILDER_BOOST_FACTOR } from "@lodestar/validator";
 import { BlockInputSource } from "../../../chain/blocks/blockInput/types.js";
-import { AttestationError, AttestationErrorCode, GossipAction, SyncCommitteeError, SyncCommitteeErrorCode, } from "../../../chain/errors/index.js";
+import { AttestationError, AttestationErrorCode, GossipAction, ProposerPreferencesError, ProposerPreferencesErrorCode, SyncCommitteeError, SyncCommitteeErrorCode, } from "../../../chain/errors/index.js";
 import { ChainEvent } from "../../../chain/index.js";
 import { PREPARE_NEXT_SLOT_BPS } from "../../../chain/prepareNextSlot.js";
 import { BlockType } from "../../../chain/produceBlock/index.js";
 import { RegenCaller } from "../../../chain/regen/index.js";
 import { validateApiAggregateAndProof } from "../../../chain/validation/index.js";
+import { validateGossipProposerPreferences } from "../../../chain/validation/proposerPreferences.js";
 import { validateSyncCommitteeGossipContributionAndProof } from "../../../chain/validation/syncCommitteeContributionAndProof.js";
 import { ZERO_HASH } from "../../../constants/index.js";
 import { BuilderStatus, NoBidReceived } from "../../../execution/builder/http.js";
 import { validateGossipFnRetryUnknownRoot } from "../../../network/processor/gossipHandlers.js";
-import { SyncState } from "../../../sync/index.js";
 import { callInNextEventLoop } from "../../../util/eventLoop.js";
 import { isOptimisticBlock } from "../../../util/forkChoice.js";
-import { getDefaultGraffiti, toGraffitiBytes } from "../../../util/graffiti.js";
+import { getBlockGraffiti, toGraffitiBytes } from "../../../util/graffiti.js";
 import { getLodestarClientVersion } from "../../../util/metadata.js";
 import { getStateResponseWithRegen } from "../beacon/state/utils.js";
 import { ApiError, IndexedError, NodeIsSyncing, OnlySupportedByDVT } from "../errors.js";
+import { notWhileSyncing } from "../utils.js";
 import { computeSubnetForCommitteesAtSlot, getPubkeysForIndices, selectBlockProductionSource } from "./utils.js";
-/**
- * If the node is within this many epochs from the head, we declare it to be synced regardless of
- * the network sync state.
- *
- * This helps prevent attacks where nodes can convince us that we're syncing some non-existent
- * finalized head.
- *
- * TODO: Lighthouse uses 8 for the attack described above. However, 8 kills Lodestar since validators
- * can trigger regen to fast-forward head state 8 epochs to be immediately invalidated as sync sets
- * a new head. Then the checkpoint state cache grows unbounded with very different states (because
- * they are 8 epochs apart) and causes an OOM. Research a proper solution once regen and the state
- * caches are better.
- */
-export const SYNC_TOLERANCE_EPOCHS = 1;
 /**
  * Cutoff time to wait from start of the slot for execution and builder block production apis to resolve.
  * Post this time, race execution and builder to pick whatever resolves first.
@@ -46,7 +33,7 @@ export const SYNC_TOLERANCE_EPOCHS = 1;
  * A cutoff of 2 seconds gives enough time and if there are unexpected delays it ensures we publish
  * in time as proposals post 4 seconds into the slot will likely be orphaned due to proposer boost reorg.
  *
- * TODO GLOAS: re-evaluate cutoff timing
+ * TODO GLOAS: re-evaluate cutoff timing due to attestation deadline changes in gloas
  */
 const BLOCK_PRODUCTION_RACE_CUTOFF_MS = 2_000;
 /** Overall timeout for execution and block production apis */
@@ -221,31 +208,6 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
         }
         return null;
     }
-    /**
-     * Reject any request while the node is syncing
-     */
-    function notWhileSyncing() {
-        // Consider node synced before or close to genesis
-        if (chain.clock.currentSlot < SLOTS_PER_EPOCH) {
-            return;
-        }
-        const syncState = sync.state;
-        switch (syncState) {
-            case SyncState.SyncingFinalized:
-            case SyncState.SyncingHead: {
-                const currentSlot = chain.clock.currentSlot;
-                const headSlot = chain.forkChoice.getHead().slot;
-                if (currentSlot - headSlot > SYNC_TOLERANCE_EPOCHS * SLOTS_PER_EPOCH) {
-                    throw new NodeIsSyncing(`headSlot ${headSlot} currentSlot ${currentSlot}`);
-                }
-                return;
-            }
-            case SyncState.Synced:
-                return;
-            case SyncState.Stalled:
-                throw new NodeIsSyncing("waiting for peers");
-        }
-    }
     /**
      * Post merge, the CL and EL could be out of step in the sync, and could result in
      * Syncing status of the chain head. To be precise:
@@ -402,7 +364,16 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
     async function produceEngineOrBuilderBlock(slot, randaoReveal, graffiti, 
     // TODO deneb: skip randao verification
     _skipRandaoVerification, builderBoostFactor, { feeRecipient, builderSelection, strictFeeRecipientCheck } = {}) {
-        notWhileSyncing();
+        builderSelection = builderSelection ?? routes.validator.BuilderSelection.MaxProfit;
+        if (builderSelection === routes.validator.BuilderSelection.BuilderOnly) {
+            logger.warn("Builder selection builderonly is no longer supported, treating as builderalways");
+            builderSelection = routes.validator.BuilderSelection.BuilderAlways;
+        }
+        builderBoostFactor = builderBoostFactor ?? BigInt(100);
+        if (builderBoostFactor > MAX_BUILDER_BOOST_FACTOR) {
+            throw new ApiError(400, `Invalid builderBoostFactor=${builderBoostFactor} > MAX_BUILDER_BOOST_FACTOR`);
+        }
+        notWhileSyncing(chain, sync.state);
         await waitForSlot(slot); // Must never request for a future slot > currentSlot
         const parentBlock = chain.getProposerHead(slot);
         const { blockRoot: parentBlockRootHex, slot: parentSlot } = parentBlock;
@@ -410,30 +381,13 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
         notOnOutOfRangeData(parentBlockRoot);
         metrics?.blockProductionSlotDelta.set(slot - parentSlot);
         const fork = config.getForkName(slot);
-        // set some sensible opts
-        // builderSelection will be deprecated and will run in mode MaxProfit if builder is enabled
-        // and the actual selection will be determined using builderBoostFactor passed by the validator
-        builderSelection = builderSelection ?? routes.validator.BuilderSelection.MaxProfit;
-        builderBoostFactor = builderBoostFactor ?? BigInt(100);
-        if (builderBoostFactor > MAX_BUILDER_BOOST_FACTOR) {
-            throw new ApiError(400, `Invalid builderBoostFactor=${builderBoostFactor} > MAX_BUILDER_BOOST_FACTOR`);
-        }
         const isBuilderEnabled = ForkSeq[fork] >= ForkSeq.bellatrix &&
             chain.executionBuilder !== undefined &&
             builderSelection !== routes.validator.BuilderSelection.ExecutionOnly;
-        // At any point either the builder or execution or both flows should be active.
-        //
-        // Ideally such a scenario should be prevented on startup, but proposerSettingsFile or keymanager
-        // configurations could cause a validator pubkey to have builder disabled with builder selection builder only
-        // (TODO: independently make sure such an options update is not successful for a validator pubkey)
-        //
-        // So if builder is disabled ignore builder selection of builder only if caused by user mistake
-        // https://github.com/ChainSafe/lodestar/issues/6338
-        const isEngineEnabled = !isBuilderEnabled || builderSelection !== routes.validator.BuilderSelection.BuilderOnly;
-        if (!isEngineEnabled && !isBuilderEnabled) {
-            throw Error(`Internal Error: Neither builder nor execution proposal flow activated isBuilderEnabled=${isBuilderEnabled} builderSelection=${builderSelection}`);
-        }
-        const graffitiBytes = toGraffitiBytes(graffiti ?? getDefaultGraffiti(getLodestarClientVersion(opts), chain.executionEngine.clientVersion, opts));
+        const graffitiBytes = toGraffitiBytes(getBlockGraffiti(graffiti, getLodestarClientVersion(opts), chain.executionEngine.clientVersion, {
+            private: opts.private,
+            graffitiAppend: chain.opts.graffitiAppend,
+        }));
         const loggerContext = {
             slot,
             parentSlot,
@@ -441,7 +395,6 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
             fork,
             builderSelection,
             isBuilderEnabled,
-            isEngineEnabled,
             strictFeeRecipientCheck,
             // winston logger doesn't like bigint
             builderBoostFactor: `${builderBoostFactor}`,
@@ -462,25 +415,23 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
                 parentBlock,
             })
             : Promise.reject(new Error("Builder disabled"));
-        const enginePromise = isEngineEnabled
-            ? produceEngineBlockContents(slot, randaoReveal, graffitiBytes, {
-                feeRecipient,
-                strictFeeRecipientCheck,
-                commonBlockBodyPromise,
-                parentBlock,
-            }).then((engineBlock) => {
-                // Once the engine returns a block, in the event of either:
-                // - suspected builder censorship
-                // - builder boost factor set to 0 or builder selection `executionalways`
-                // we don't need to wait for builder block as engine block will always be selected
-                if (engineBlock.shouldOverrideBuilder ||
-                    builderBoostFactor === BigInt(0) ||
-                    builderSelection === routes.validator.BuilderSelection.ExecutionAlways) {
-                    controller.abort();
-                }
-                return engineBlock;
-            })
-            : Promise.reject(new Error("Engine disabled"));
+        const enginePromise = produceEngineBlockContents(slot, randaoReveal, graffitiBytes, {
+            feeRecipient,
+            strictFeeRecipientCheck,
+            commonBlockBodyPromise,
+            parentBlock,
+        }).then((engineBlock) => {
+            // Once the engine returns a block, in the event of either:
+            // - suspected builder censorship
+            // - builder boost factor set to 0 or builder selection `executionalways`
+            // we don't need to wait for builder block as engine block will always be selected
+            if (engineBlock.shouldOverrideBuilder ||
+                builderBoostFactor === BigInt(0) ||
+                builderSelection === routes.validator.BuilderSelection.ExecutionAlways) {
+                controller.abort();
+            }
+            return engineBlock;
+        });
         // Calculate cutoff time based on start of the slot
         const cutoffMs = Math.max(0, BLOCK_PRODUCTION_RACE_CUTOFF_MS - chain.clock.msFromSlot(slot));
         logger.verbose("Block production race (builder vs execution) starting", {
@@ -519,25 +470,20 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
         if (builder.status === "pending" && engine.status === "pending") {
             throw Error("Builder and engine both failed to produce the block within timeout");
         }
-        if (builder.status === "pending" && !isEngineEnabled) {
-            throw Error("Builder failed to produce the block within timeout");
-        }
         if (engine.status === "pending" && !isBuilderEnabled) {
             throw Error("Engine failed to produce the block within timeout");
         }
-        if (isEngineEnabled) {
-            if (engine.status === "rejected") {
-                logger.warn("Engine failed to produce the block", {
-                    ...loggerContext,
-                    durationMs: engine.durationMs,
-                }, engine.reason);
-            }
-            else if (engine.status === "pending") {
-                logger.warn("Engine failed to produce the block within cutoff time", {
-                    ...loggerContext,
-                    cutoffMs,
-                });
-            }
+        if (engine.status === "rejected") {
+            logger.warn("Engine failed to produce the block", {
+                ...loggerContext,
+                durationMs: engine.durationMs,
+            }, engine.reason);
+        }
+        else if (engine.status === "pending") {
+            logger.warn("Engine failed to produce the block within cutoff time", {
+                ...loggerContext,
+                cutoffMs,
+            });
         }
         if (isBuilderEnabled) {
             if (builder.status === "rejected") {
@@ -562,11 +508,11 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
             }
         }
         if (builder.status === "rejected" && engine.status === "rejected") {
-            throw Error(`${isBuilderEnabled && isEngineEnabled ? "Builder and engine both" : isBuilderEnabled ? "Builder" : "Engine"} failed to produce the block`);
+            throw Error(`${isBuilderEnabled ? "Builder and engine both" : "Engine"} failed to produce the block`);
         }
         // handle shouldOverrideBuilder separately
-        if (engine.status === "fulfilled" && engine.value.shouldOverrideBuilder) {
-            logger.info("Selected engine block: censorship suspected in builder blocks", {
+        if (engine.status === "fulfilled" && engine.value.shouldOverrideBuilder && isBuilderEnabled) {
+            logger.warn("Selected engine block: censorship suspected in builder blocks", {
                 ...loggerContext,
                 durationMs: engine.durationMs,
                 shouldOverrideBuilder: engine.value.shouldOverrideBuilder,
@@ -579,11 +525,9 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
             return { ...engine.value, executionPayloadBlinded: false, executionPayloadSource: ProducedBlockSource.engine };
         }
         if (builder.status === "fulfilled" && engine.status !== "fulfilled") {
-            const reason = isEngineEnabled === false
-                ? BuilderBlockSelectionReason.EngineDisabled
-                : engine.status === "pending"
-                    ? BuilderBlockSelectionReason.EnginePending
-                    : BuilderBlockSelectionReason.EngineError;
+            const reason = engine.status === "pending"
+                ? BuilderBlockSelectionReason.EnginePending
+                : BuilderBlockSelectionReason.EngineError;
             logger.info("Selected builder block: no engine block produced", {
                 reason,
                 ...loggerContext,
@@ -667,31 +611,50 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
             }
             return { data, meta };
         },
-        async produceBlockV4({ slot, randaoReveal, graffiti, feeRecipient }) {
+        async produceBlockV4({ slot, randaoReveal, graffiti, feeRecipient, includePayload, builderSelection, builderBoostFactor, }) {
             const fork = config.getForkName(slot);
             if (!isForkPostGloas(fork)) {
                 throw new ApiError(400, `produceBlockV4 not supported for pre-gloas fork=${fork}`);
             }
-            notWhileSyncing();
+            builderSelection = builderSelection ?? routes.validator.BuilderSelection.MaxProfit;
+            if (builderSelection === routes.validator.BuilderSelection.BuilderOnly) {
+                logger.warn("Builder selection builderonly is no longer supported, treating as builderalways");
+                builderSelection = routes.validator.BuilderSelection.BuilderAlways;
+            }
+            builderBoostFactor = builderBoostFactor ?? BigInt(100);
+            if (builderBoostFactor > MAX_BUILDER_BOOST_FACTOR) {
+                throw new ApiError(400, `Invalid builderBoostFactor=${builderBoostFactor} > MAX_BUILDER_BOOST_FACTOR`);
+            }
+            notWhileSyncing(chain, sync.state);
             await waitForSlot(slot);
             const parentBlock = chain.getProposerHead(slot);
             const { blockRoot: parentBlockRootHex, slot: parentSlot } = parentBlock;
             const parentBlockRoot = fromHex(parentBlockRootHex);
             notOnOutOfRangeData(parentBlockRoot);
             metrics?.blockProductionSlotDelta.set(slot - parentSlot);
-            const graffitiBytes = toGraffitiBytes(graffiti ?? getDefaultGraffiti(getLodestarClientVersion(opts), chain.executionEngine.clientVersion, opts));
-            // TODO GLOAS: respect builderSelection (MaxProfit, BuilderAlways, ExecutionAlways, etc.) to let
-            // the user control bid source preferences and value comparison. Also add external builder api
-            // support when it is implemented.
+            const graffitiBytes = toGraffitiBytes(getBlockGraffiti(graffiti, getLodestarClientVersion(opts), chain.executionEngine.clientVersion, {
+                private: opts.private,
+                graffitiAppend: chain.opts.graffitiAppend,
+            }));
+            // TODO GLOAS: add external builder api support when it is implemented
             const isBuildingOnFull = chain.forkChoice.shouldBuildOnFull(parentBlock, slot);
             const bidParentBlockHash = isBuildingOnFull ? parentBlock.executionPayloadBlockHash : parentBlock.parentBlockHash;
-            const builderBid = chain.executionPayloadBidPool.getBestBid(slot, bidParentBlockHash, parentBlockRootHex);
+            // Bids are only skipped entirely with executiononly or while the circuit breaker is active,
+            // other engine-preferring selections still build a block with the best bid as fallback in
+            // case local production fails
+            const circuitBreakerActive = chain.builderCircuitBreaker.isActive(slot);
+            const builderBid = builderSelection === routes.validator.BuilderSelection.ExecutionOnly || circuitBreakerActive
+                ? null
+                : chain.executionPayloadBidPool.getBestBid(slot, bidParentBlockHash, parentBlockRootHex);
             const logCtx = {
                 slot,
                 parentSlot,
                 parentBlockRoot: parentBlockRootHex,
                 parentBlockHash: parentBlock.executionPayloadBlockHash,
                 fork,
+                builderSelection,
+                builderBoostFactor,
+                circuitBreakerActive,
                 ...(builderBid !== null
                     ? {
                         bidValue: builderBid.message.value,
@@ -722,32 +685,91 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
                 const t = metrics?.blockProductionTime.startTimer();
                 return fn().finally(() => t?.({ source }));
             };
-            // Always build local block. If builder bid available, also build with it in parallel and prefer it.
-            const [engineResult, bidResult] = await Promise.allSettled([
-                timed(ProducedBlockSource.engine, () => chain.produceBlock(baseAttrs)),
-                builderBid !== null
-                    ? timed(ProducedBlockSource.builder, () => chain.produceBlock({ ...baseAttrs, builderBid }))
-                    : Promise.reject(),
-            ]);
+            // Calculate cutoff time based on start of the slot, ensures a slow local payload build does
+            // not delay the proposal when a builder bid block is available (and vice versa)
+            const cutoffMs = Math.max(0, BLOCK_PRODUCTION_RACE_CUTOFF_MS - chain.clock.msFromSlot(slot));
+            // use abort controller to stop waiting for the bid block if the engine block will be selected
+            const controller = new AbortController();
+            const enginePromise = timed(ProducedBlockSource.engine, () => chain.produceBlock(baseAttrs)).then((engineBlock) => {
+                // No need to wait for the bid block if the engine block will always be selected due to
+                // suspected builder censorship, a builder boost factor of 0 or executionalways selection
+                if (engineBlock.shouldOverrideBuilder ||
+                    builderBoostFactor === BigInt(0) ||
+                    builderSelection === routes.validator.BuilderSelection.ExecutionAlways) {
+                    controller.abort();
+                }
+                return engineBlock;
+            });
+            const bidPromise = builderBid !== null
+                ? timed(ProducedBlockSource.builder, () => chain.produceBlock({ ...baseAttrs, builderBid }))
+                : Promise.reject(new Error("No builder bid available"));
+            const [engineResult, bidResult] = await resolveOrRacePromises([enginePromise, bidPromise], {
+                resolveTimeoutMs: cutoffMs,
+                raceTimeoutMs: BLOCK_PRODUCTION_RACE_TIMEOUT_MS,
+                signal: controller.signal,
+            });
             let bestResult = null;
             let source = ProducedBlockSource.engine;
-            if (builderBid !== null && bidResult.status === "fulfilled") {
+            // handle shouldOverrideBuilder separately
+            if (engineResult.status === "fulfilled" && engineResult.value.shouldOverrideBuilder && builderBid !== null) {
+                source = ProducedBlockSource.engine;
+                bestResult = engineResult;
+                metrics?.blockProductionSelectionResults.inc({
+                    source: ProducedBlockSource.engine,
+                    reason: EngineBlockSelectionReason.BuilderCensorship,
+                });
+                logger.warn("Selected local block: censorship suspected in builder bid", logCtx);
+            }
+            else if (engineResult.status === "fulfilled" && bidResult.status === "fulfilled") {
+                const result = selectBlockProductionSource({
+                    builderSelection,
+                    builderBoostFactor,
+                    engineExecutionPayloadValue: engineResult.value.executionPayloadValue,
+                    // The bid value is the payment to the proposer, in Gwei
+                    builderExecutionPayloadValue: BigInt(builderBid?.message.value ?? 0) * GWEI_TO_WEI,
+                });
+                source = result.source;
+                metrics?.blockProductionSelectionResults.inc(result);
+                logger.info(`Selected ${source} block`, { reason: result.reason, ...logCtx });
+                bestResult = source === ProducedBlockSource.builder ? bidResult : engineResult;
+            }
+            else if (bidResult.status === "fulfilled") {
                 source = ProducedBlockSource.builder;
                 bestResult = bidResult;
-                logger.info("Selected builder bid block", logCtx);
+                const reason = engineResult.status === "pending"
+                    ? BuilderBlockSelectionReason.EnginePending
+                    : BuilderBlockSelectionReason.EngineError;
+                metrics?.blockProductionSelectionResults.inc({ source: ProducedBlockSource.builder, reason });
+                logger.info("Selected builder bid block: no local block produced", {
+                    reason,
+                    ...logCtx,
+                    error: engineResult.status === "rejected" ? engineResult.reason.message : undefined,
+                });
             }
             else if (engineResult.status === "fulfilled") {
                 source = ProducedBlockSource.engine;
                 bestResult = engineResult;
-                if (builderBid !== null) {
-                    logger.warn("Builder bid block production failed, using local block", logCtx);
-                }
+                const reason = builderBid === null
+                    ? EngineBlockSelectionReason.BuilderNoBid
+                    : bidResult.status === "pending"
+                        ? EngineBlockSelectionReason.BuilderPending
+                        : EngineBlockSelectionReason.BuilderError;
+                metrics?.blockProductionSelectionResults.inc({ source: ProducedBlockSource.engine, reason });
+                logger.info("Selected local block: no builder bid block produced", {
+                    reason,
+                    ...logCtx,
+                    error: bidResult.status === "rejected" ? bidResult.reason.message : undefined,
+                });
             }
             if (bestResult === null || bestResult.status !== "fulfilled") {
-                const engineReason = engineResult.status === "rejected" ? engineResult.reason : undefined;
-                const bidReason = builderBid !== null && bidResult.status === "rejected" ? bidResult.reason : undefined;
-                logger.error("Block production failed", { ...logCtx, engineReason, bidReason });
-                throw Error(`Block production failed: engine=${engineReason ?? "n/a"} builder=${bidReason ?? "n/a"}`);
+                const engineReason = engineResult.status === "rejected" ? engineResult.reason : engineResult.status;
+                const bidReason = bidResult.status === "rejected" ? bidResult.reason : bidResult.status;
+                logger.error("Block production failed", {
+                    ...logCtx,
+                    engineReason: String(engineReason),
+                    bidReason: String(bidReason),
+                });
+                throw Error(`Block production failed: engine=${String(engineReason)} builder=${String(bidReason)}`);
             }
             const { block, executionPayloadValue, consensusBlockValue } = bestResult.value;
             metrics?.blockProductionSuccess.inc({ source });
@@ -764,13 +786,40 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
             if (chain.opts.persistProducedBlocks) {
                 void chain.persistBlock(block, "produced_engine_block");
             }
+            // Include the payload for self-builds unless disabled (stateless flow)
+            const isSelfBuild = source === ProducedBlockSource.engine;
+            if (isSelfBuild && includePayload) {
+                const produceResult = chain.blockProductionCache.get(blockRoot);
+                if (produceResult === undefined ||
+                    !isForkPostGloas(produceResult.fork) ||
+                    produceResult.type !== BlockType.Full) {
+                    throw Error(`Missing cached block production result for produced block slot=${slot} blockRoot=${blockRoot}`);
+                }
+                const { executionPayload, executionRequests, blobsBundle, parentBlockRoot } = produceResult;
+                const blockContents = {
+                    block: block,
+                    executionPayloadEnvelope: {
+                        payload: executionPayload,
+                        executionRequests,
+                        builderIndex: BUILDER_INDEX_SELF_BUILD,
+                        beaconBlockRoot: fromHex(blockRoot),
+                        parentBeaconBlockRoot: parentBlockRoot,
+                    },
+                    kzgProofs: blobsBundle.proofs,
+                    blobs: blobsBundle.blobs,
+                };
+                return {
+                    data: blockContents,
+                    meta: { version: fork, consensusBlockValue, executionPayloadValue, executionPayloadIncluded: true },
+                };
+            }
             return {
                 data: block,
-                meta: { version: fork, consensusBlockValue },
+                meta: { version: fork, consensusBlockValue, executionPayloadValue, executionPayloadIncluded: false },
             };
         },
         async produceAttestationData({ committeeIndex, slot }) {
-            notWhileSyncing();
+            notWhileSyncing(chain, sync.state);
             await waitForSlot(slot); // Must never request for a future slot > currentSlot
             // This needs a state in the same epoch as `slot` such that state.currentJustifiedCheckpoint is correct.
             // Note: This may trigger an epoch transition if there skipped slots at the beginning of the epoch.
@@ -841,12 +890,12 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
             if (!isForkPostGloas(fork)) {
                 throw new ApiError(400, `producePayloadAttestationData is not supported before Gloas fork=${fork}`);
             }
-            notWhileSyncing();
+            notWhileSyncing(chain, sync.state);
             await waitForSlot(slot);
             const block = chain.forkChoice.getCanonicalBlockAtSlot(slot);
             if (!block) {
-                // No block is seen at slot. Return 404 so vc can skip casting payload attestation.
-                throw new ApiError(404, `No canonical block found at slot=${slot}`);
+                // No canonical block is seen at slot. Return 204 so vc can skip casting payload attestation.
+                return { data: undefined, meta: { version: fork }, status: 204 };
             }
             const payloadInput = chain.seenPayloadEnvelopeInputCache.get(block.blockRoot);
             // Spec: set payload_present only if the envelope was seen before get_payload_due_ms()
@@ -910,7 +959,7 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
             return { data: contribution };
         },
         async getProposerDuties({ epoch }, _context, opts) {
-            notWhileSyncing();
+            notWhileSyncing(chain, sync.state);
             const currentEpoch = currentEpochWithDisparity();
             const nextEpoch = currentEpoch + 1;
             const startSlot = computeStartSlotAtEpoch(epoch);
@@ -952,7 +1001,7 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
                     state = await chain.getHeadStateAtCurrentEpoch(RegenCaller.getDuties);
                 }
                 else {
-                    const res = await getStateResponseWithRegen(chain, startSlot);
+                    const res = await getStateResponseWithRegen(chain, sync, startSlot);
                     state = res.state instanceof Uint8Array ? chain.getHeadState().loadOtherState(res.state) : res.state;
                     if (state.epoch !== epoch) {
                         throw Error(`Loaded state epoch ${state.epoch} does not match requested epoch ${epoch}`);
@@ -1028,7 +1077,7 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
             return this.getProposerDuties(args, context, { v2: true });
         },
         async getAttesterDuties({ epoch, indices }) {
-            notWhileSyncing();
+            notWhileSyncing(chain, sync.state);
             if (indices.length === 0) {
                 throw new ApiError(400, "No validator to get attester duties");
             }
@@ -1076,7 +1125,7 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
             };
         },
         async getPtcDuties({ epoch, indices }) {
-            notWhileSyncing();
+            notWhileSyncing(chain, sync.state);
             if (indices.length === 0) {
                 throw new ApiError(400, "No validator to get PTC duties");
             }
@@ -1129,7 +1178,7 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
          * @param validatorIndices an array of the validator indices for which to obtain the duties.
          */
         async getSyncCommitteeDuties({ epoch, indices }) {
-            notWhileSyncing();
+            notWhileSyncing(chain, sync.state);
             if (indices.length === 0) {
                 throw new ApiError(400, "No validator to get attester duties");
             }
@@ -1169,7 +1218,7 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
             };
         },
         async getAggregatedAttestationV2({ attestationDataRoot, slot, committeeIndex }) {
-            notWhileSyncing();
+            notWhileSyncing(chain, sync.state);
             await waitForSlot(slot); // Must never request for a future slot > currentSlot
             const dataRootHex = toRootHex(attestationDataRoot);
             const aggregate = chain.attestationPool.getAggregate(slot, dataRootHex, committeeIndex);
@@ -1183,7 +1232,7 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
             };
         },
         async publishAggregateAndProofsV2({ signedAggregateAndProofs }) {
-            notWhileSyncing();
+            notWhileSyncing(chain, sync.state);
             const seenTimestampSec = Date.now() / 1000;
             const failures = [];
             const fork = chain.config.getForkName(chain.clock.currentSlot);
@@ -1229,7 +1278,7 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
          * https://github.com/ethereum/beacon-APIs/pull/137
          */
         async publishContributionAndProofs({ contributionAndProofs }) {
-            notWhileSyncing();
+            notWhileSyncing(chain, sync.state);
             const failures = [];
             await Promise.all(contributionAndProofs.map(async (contributionAndProof, i) => {
                 try {
@@ -1262,7 +1311,7 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
             }
         },
         async prepareBeaconCommitteeSubnet({ subscriptions }) {
-            notWhileSyncing();
+            notWhileSyncing(chain, sync.state);
             await network.prepareBeaconCommitteeSubnets(subscriptions.map(({ validatorIndex, slot, isAggregator, committeesAtSlot, committeeIndex }) => ({
                 validatorIndex: validatorIndex,
                 subnet: computeSubnetForCommitteesAtSlot(slot, committeesAtSlot, committeeIndex),
@@ -1289,7 +1338,7 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
          * https://github.com/ethereum/beacon-APIs/pull/136
          */
         async prepareSyncCommitteeSubnets({ subscriptions }) {
-            notWhileSyncing();
+            notWhileSyncing(chain, sync.state);
             // A `validatorIndex` can be in multiple subnets, so compute the CommitteeSubscription with double for loop
             const subs = [];
             for (const sub of subscriptions) {
@@ -1364,12 +1413,45 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
                 count: filteredRegistrations.length,
             });
         },
+        async submitProposerPreferences({ signedProposerPreferences }) {
+            const failures = [];
+            await Promise.all(signedProposerPreferences.map(async (signed, i) => {
+                try {
+                    await validateGossipProposerPreferences(chain, signed);
+                    chain.proposerPreferencesPool.add(signed);
+                    await network.publishProposerPreferences(signed);
+                    chain.emitter.emit(routes.events.EventType.proposerPreferences, {
+                        version: config.getForkName(signed.message.proposalSlot),
+                        data: signed,
+                    });
+                }
+                catch (e) {
+                    const logCtx = {
+                        slot: signed.message.proposalSlot,
+                        validatorIndex: signed.message.validatorIndex,
+                        dependentRoot: toRootHex(signed.message.dependentRoot),
+                    };
+                    if (e instanceof ProposerPreferencesError && e.type.code === ProposerPreferencesErrorCode.ALREADY_KNOWN) {
+                        logger.debug("Ignoring known signed proposer preferences", logCtx);
+                        return;
+                    }
+                    failures.push({ index: i, message: e.message });
+                    logger.verbose(`Error on submitProposerPreferences [${i}]`, logCtx, e);
+                    if (e instanceof ProposerPreferencesError && e.action === GossipAction.REJECT) {
+                        chain.persistInvalidSszValue(ssz.gloas.SignedProposerPreferences, signed, "api_reject");
+                    }
+                }
+            }));
+            if (failures.length > 0) {
+                throw new IndexedError("Error processing signed proposer preferences", failures);
+            }
+        },
         async getExecutionPayloadEnvelope({ slot, beaconBlockRoot }) {
             const fork = config.getForkName(slot);
             if (!isForkPostGloas(fork)) {
                 throw new ApiError(400, `getExecutionPayloadEnvelope not supported for pre-gloas fork=${fork}`);
             }
-            notWhileSyncing();
+            notWhileSyncing(chain, sync.state);
             await waitForSlot(slot);
             const blockRootHex = toRootHex(beaconBlockRoot);
             const produceResult = chain.blockProductionCache.get(blockRootHex);
@@ -1383,6 +1465,13 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
                 throw Error("Cached block production result is not full block");
             }
             const { executionPayload, executionRequests, parentBlockRoot } = produceResult;
+            if (executionPayload === undefined) {
+                // Blocks committing to a builder bid are cached as full but without payload data
+                throw new ApiError(404, `No local execution payload cached for block root ${blockRootHex}`);
+            }
+            if (executionPayload.slotNumber !== slot) {
+                throw new ApiError(404, `Cached execution payload is for slot=${executionPayload.slotNumber}, requested slot=${slot}`);
+            }
             const envelope = {
                 payload: executionPayload,
                 executionRequests: executionRequests,