@lodestar/beacon-node 1.45.0-dev.fadf0fbb1f → 1.46.0-dev.8fc4e297d2

package/src/api/impl/beacon/blocks/index.ts

@@ -17,6 +17,7 @@ import {
 import {
   computeEpochAtSlot,
   computeTimeAtSlot,
+  isStatePostGloas,
   reconstructSignedBlockContents,
   signedBeaconBlockToBlinded,
   signedBlockToSignedHeader,
@@ -31,6 +32,7 @@ import {
   fulu,
   gloas,
   isDenebBlockContents,
+  isSignedExecutionPayloadEnvelopeContents,
   sszTypesFor,
 } from "@lodestar/types";
 import {fromHex, sleep, toHex, toRootHex} from "@lodestar/utils";
@@ -38,9 +40,16 @@ import {BlockInputSource, isBlockInputBlobs, isBlockInputColumns} from "../../..
 import {PayloadEnvelopeInputSource} from "../../../../chain/blocks/payloadEnvelopeInput/index.js";
 import {ImportBlockOpts} from "../../../../chain/blocks/types.js";
 import {verifyBlocksInEpoch} from "../../../../chain/blocks/verifyBlock.js";
+import {verifyExecutionPayloadEnvelope} from "../../../../chain/blocks/verifyExecutionPayloadEnvelope.js";
 import {BeaconChain} from "../../../../chain/chain.js";
 import {ChainEvent} from "../../../../chain/emitter.js";
-import {BlockError, BlockErrorCode, BlockGossipError} from "../../../../chain/errors/index.js";
+import {
+  BlockError,
+  BlockErrorCode,
+  BlockGossipError,
+  ExecutionPayloadEnvelopeError,
+  ExecutionPayloadEnvelopeErrorCode,
+} from "../../../../chain/errors/index.js";
 import {
   BlockType,
   ProduceFullBellatrix,
@@ -48,6 +57,7 @@ import {
   ProduceFullFulu,
   ProduceFullGloas,
 } from "../../../../chain/produceBlock/index.js";
+import {RegenCaller} from "../../../../chain/regen/index.js";
 import {validateGossipBlock} from "../../../../chain/validation/block.js";
 import {validateApiExecutionPayloadBid} from "../../../../chain/validation/executionPayloadBid.js";
 import {validateApiExecutionPayloadEnvelope} from "../../../../chain/validation/executionPayloadEnvelope.js";
@@ -249,7 +259,7 @@ export function getBeaconBlockApi({
               "api_reject_parent_unknown"
             );
             throw new BlockError(signedBlock, {
-              code: BlockErrorCode.PARENT_UNKNOWN,
+              code: BlockErrorCode.PARENT_BLOCK_UNKNOWN,
               parentRoot: toRootHex(signedBlock.message.parentRoot),
             });
           }
@@ -334,7 +344,8 @@ export function getBeaconBlockApi({
           .catch((e) => {
             if (
               e instanceof BlockError &&
-              (e.type.code === BlockErrorCode.PARENT_UNKNOWN || e.type.code === BlockErrorCode.PARENT_PAYLOAD_UNKNOWN)
+              (e.type.code === BlockErrorCode.PARENT_BLOCK_UNKNOWN ||
+                e.type.code === BlockErrorCode.PARENT_PAYLOAD_UNKNOWN)
             ) {
               chain.emitter.emit(ChainEvent.blockUnknownParent, {
                 blockInput: blockForImport,
@@ -345,7 +356,9 @@ export function getBeaconBlockApi({
             throw e;
           }),
     ];
-    const sentPeersArr = await promiseAllMaybeAsync<number | void>(publishPromises);
+    const sentPeersArr = await promiseAllMaybeAsync<{sentPeers: number; alreadyPublished: boolean} | number | void>(
+      publishPromises
+    );
 
     if (isForkPostGloas(fork)) {
       // After gloas, data columns are not published with the block but when publishing the execution payload envelope
@@ -356,10 +369,13 @@ export function getBeaconBlockApi({
       // + 1 because we publish to beacon_block first
       for (let i = 0; i < dataColumnSidecars.length; i++) {
         // + 1 because we publish to beacon_block first
-        const sentPeers = sentPeersArr[i + 1] as number;
-        // sent peers could be 0 as we set `allowPublishToZeroTopicPeers=true` in network.publishDataColumnSidecar() api
+        const {sentPeers, alreadyPublished} = sentPeersArr[i + 1] as {sentPeers: number; alreadyPublished: boolean};
+        // sent peers could be 0 as we set `allowPublishToZeroTopicPeers=true` in network.publishDataColumnSidecar()
         metrics?.dataColumns.sentPeersPerSubnet.observe(sentPeers);
-        if (sentPeers === 0) {
+        // A duplicate publish (alreadyPublished=true) means the column is already propagating on the network —
+        // expected in self-build flows where peers gossip columns back to us before we publish the envelope.
+        // Only warn when we genuinely failed to reach any peer. See https://github.com/ChainSafe/lodestar/issues/9527.
+        if (sentPeers === 0 && !alreadyPublished) {
           columnsPublishedWithZeroPeers++;
         }
       }
@@ -651,8 +667,22 @@ export function getBeaconBlockApi({
     publishBlockV2,
     publishBlindedBlockV2,
 
-    async publishExecutionPayloadEnvelope({signedExecutionPayloadEnvelope}) {
+    async publishExecutionPayloadEnvelope({signedEnvelopeOrContents, broadcastValidation}) {
       const seenTimestampSec = Date.now() / 1000;
+
+      const blobDataIncluded = isSignedExecutionPayloadEnvelopeContents(signedEnvelopeOrContents);
+      let signedExecutionPayloadEnvelope: gloas.SignedExecutionPayloadEnvelope;
+      // Blobs and KZG proofs submitted alongside the envelope in the stateless flow
+      let submittedContents: {kzgProofs: deneb.KZGProofs; blobs: deneb.Blobs} | null = null;
+
+      if (blobDataIncluded) {
+        signedExecutionPayloadEnvelope = signedEnvelopeOrContents.signedExecutionPayloadEnvelope;
+        submittedContents = {kzgProofs: signedEnvelopeOrContents.kzgProofs, blobs: signedEnvelopeOrContents.blobs};
+      } else {
+        // Stateful flow, blobs and KZG proofs are attached from the block production cache below
+        signedExecutionPayloadEnvelope = signedEnvelopeOrContents;
+      }
+
       const envelope = signedExecutionPayloadEnvelope.message;
       const slot = envelope.payload.slotNumber;
       const fork = config.getForkName(slot);
@@ -663,7 +693,6 @@ export function getBeaconBlockApi({
         throw new ApiError(400, `publishExecutionPayloadEnvelope not supported for pre-gloas fork=${fork}`);
       }
 
-      // TODO GLOAS: review checks, do we want to implement `broadcast_validation`?
       let block = chain.forkChoice.getBlockHex(blockRootHex, PayloadStatus.EMPTY);
       if (block === null) {
         // Only wait if the envelope is for the current slot
@@ -683,39 +712,172 @@ export function getBeaconBlockApi({
         throw new ApiError(400, `Envelope slot ${slot} does not match block slot ${block.slot}`);
       }
 
-      await validateApiExecutionPayloadEnvelope(chain, signedExecutionPayloadEnvelope);
-
       const isSelfBuild = envelope.builderIndex === BUILDER_INDEX_SELF_BUILD;
-      let dataColumnSidecars: gloas.DataColumnSidecar[] = [];
+      const cachedResult = chain.blockProductionCache.get(blockRootHex);
+      // Only use the cached result if it contains payload data, blocks committing to a bid are cached without it
+      const cachedGloasResult =
+        cachedResult !== undefined &&
+        isForkPostGloas(cachedResult.fork) &&
+        cachedResult.type === BlockType.Full &&
+        (cachedResult as ProduceFullGloas).executionPayload !== undefined
+          ? (cachedResult as ProduceFullGloas)
+          : undefined;
+
+      broadcastValidation = broadcastValidation ?? routes.beacon.BroadcastValidation.gossip;
+      const valLogMeta = {
+        slot,
+        blockRoot: blockRootHex,
+        blockHash: blockHashHex,
+        builderIndex: envelope.builderIndex,
+        isSelfBuild,
+        blobDataIncluded,
+        broadcastValidation,
+        ...(submittedContents !== null ? {submittedBlobs: submittedContents.blobs.length} : {}),
+      };
+      try {
+        switch (broadcastValidation) {
+          case routes.beacon.BroadcastValidation.none: {
+            chain.logger.debug("Skipping broadcast validation of execution payload envelope", valLogMeta);
+            break;
+          }
 
-      if (isSelfBuild) {
-        // For self-builds, construct and publish data column sidecars from cached block production data
-        const cachedResult = chain.blockProductionCache.get(blockRootHex) as ProduceFullGloas | undefined;
-        if (cachedResult === undefined) {
-          throw new ApiError(404, `No cached block production result found for block root ${blockRootHex}`);
-        }
-        if (!isForkPostGloas(cachedResult.fork)) {
-          throw new ApiError(400, `Cached block production result is for pre-gloas fork=${cachedResult.fork}`);
+          case routes.beacon.BroadcastValidation.gossip: {
+            await validateApiExecutionPayloadEnvelope(chain, signedExecutionPayloadEnvelope);
+            break;
+          }
+
+          case routes.beacon.BroadcastValidation.consensusAndEquivocation:
+          case routes.beacon.BroadcastValidation.consensus: {
+            await validateApiExecutionPayloadEnvelope(chain, signedExecutionPayloadEnvelope);
+
+            // Verify the envelope against the post-block state
+            const blockState = await chain.regen
+              .getBlockSlotState(block, block.slot, {dontTransferCache: true}, RegenCaller.restApi)
+              .catch((e) => {
+                chain.logger.debug("Failed to regenerate block state for consensus checks", valLogMeta, e as Error);
+                return null;
+              });
+            if (blockState === null || !isStatePostGloas(blockState)) {
+              throw new ApiError(
+                500,
+                `Unable to regenerate block state for consensus checks slot=${slot} blockRoot=${blockRootHex}`
+              );
+            }
+            try {
+              // Signature and executionRequestsRoot are already verified by gossip validation above
+              verifyExecutionPayloadEnvelope(chain.config, blockState, envelope, {verifyExecutionRequestsRoot: false});
+            } catch (error) {
+              chain.logger.error(
+                "Consensus checks failed while publishing execution payload envelope",
+                valLogMeta,
+                error as Error
+              );
+              throw new ApiError(400, (error as Error).message);
+            }
+            chain.logger.debug("Consensus validated while publishing execution payload envelope", valLogMeta);
+
+            // TODO GLOAS: check the block is not a proposer equivocation before publishing the envelope
+            if (broadcastValidation === routes.beacon.BroadcastValidation.consensusAndEquivocation) {
+              const message = `Equivocation checks not yet implemented for broadcastValidation=${broadcastValidation}`;
+              if (chain.opts.broadcastValidationStrictness === "error") {
+                throw Error(message);
+              }
+              chain.logger.warn(message, valLogMeta);
+            }
+            break;
+          }
+
+          default: {
+            const message = `Broadcast validation of ${broadcastValidation} type not implemented yet`;
+            if (chain.opts.broadcastValidationStrictness === "error") {
+              throw Error(message);
+            }
+            chain.logger.warn(message, valLogMeta);
+          }
         }
-        if (cachedResult.type !== BlockType.Full) {
-          throw new ApiError(400, "Cached block production result is not full block");
+      } catch (error) {
+        if (
+          error instanceof ExecutionPayloadEnvelopeError &&
+          error.type.code === ExecutionPayloadEnvelopeErrorCode.ENVELOPE_ALREADY_KNOWN
+        ) {
+          // The envelope may already be known, e.g. received via gossip from another node in a
+          // multi node setup, this is benign and treated as a successful publish (same as blocks)
+          if (submittedContents === null) {
+            chain.logger.debug("Ignoring already-known execution payload envelope during publishing", valLogMeta);
+            return;
+          }
+          // The envelope may have been gossiped without its data columns being published, e.g. if
+          // another beacon node failed mid-publish, still publish columns from the submitted blobs
+          chain.logger.debug("Publishing data columns of already-known execution payload envelope", valLogMeta);
+        } else {
+          throw error;
         }
+      }
 
-        if (cachedResult.cells && cachedResult.blobsBundle.commitments.length > 0) {
-          const timer = metrics?.peerDas.dataColumnSidecarComputationTime.startTimer();
-          const cellsAndProofs = cachedResult.cells.map((rowCells, rowIndex) => ({
-            cells: rowCells,
-            proofs: cachedResult.blobsBundle.proofs.slice(
-              rowIndex * NUMBER_OF_COLUMNS,
-              (rowIndex + 1) * NUMBER_OF_COLUMNS
-            ),
-          }));
-
-          dataColumnSidecars = getGloasDataColumnSidecars(slot, envelope.beaconBlockRoot, cellsAndProofs);
-          timer?.();
+      const payloadInput = chain.seenPayloadEnvelopeInputCache.get(blockRootHex);
+      if (!payloadInput) {
+        // The block is awaited above (queuing if the envelope arrived first), and both the API and
+        // gossip import paths seed the PayloadEnvelopeInput before importing the block, so the input
+        // should exist here.
+        throw new ApiError(404, `PayloadEnvelopeInput not found for slot=${slot} blockRoot=${blockRootHex}`);
+      }
+
+      let dataColumnSidecars: gloas.DataColumnSidecar[] = [];
+      let cells: fulu.Cell[][] | undefined;
+      let kzgProofs: deneb.KZGProofs | undefined;
+      let dataColumnTimer: (() => number) | undefined;
+
+      if (submittedContents !== null) {
+        // Validate submitted blob data against bid commitments before computing data column sidecars
+        const expectedBlobCount = payloadInput.getVersionedHashes().length;
+        if (submittedContents.blobs.length !== expectedBlobCount) {
+          throw new ApiError(
+            400,
+            `Submitted blob count does not match bid commitments submitted=${submittedContents.blobs.length} expected=${expectedBlobCount}`
+          );
+        }
+        const expectedProofCount = expectedBlobCount * NUMBER_OF_COLUMNS;
+        if (submittedContents.kzgProofs.length !== expectedProofCount) {
+          throw new ApiError(
+            400,
+            `Submitted KZG proof count does not match bid commitments submitted=${submittedContents.kzgProofs.length} expected=${expectedProofCount}`
+          );
+        }
+        if (submittedContents.blobs.length > 0) {
+          // If the block was produced by this node, reuse the cached cells and only time the
+          // metric when cells are actually computed from the submitted blobs
+          if (cachedGloasResult?.cells) {
+            cells = cachedGloasResult.cells;
+          } else {
+            dataColumnTimer = metrics?.peerDas.dataColumnSidecarComputationTime.startTimer();
+            cells = submittedContents.blobs.map((blob) => kzg.computeCells(blob));
+          }
+          kzgProofs = submittedContents.kzgProofs;
+        }
+      } else if (cachedGloasResult !== undefined) {
+        if (cachedGloasResult.cells && cachedGloasResult.blobsBundle.commitments.length > 0) {
+          cells = cachedGloasResult.cells;
+          kzgProofs = cachedGloasResult.blobsBundle.proofs;
         }
       } else {
-        // TODO GLOAS: will this api be used by builders or only for self-building?
+        // An envelope without blob data can only be published via the beacon node that cached them at block production
+        if (payloadInput.getVersionedHashes().length > 0) {
+          throw new ApiError(
+            400,
+            `No cached blob data to attach to execution payload envelope for block root ${blockRootHex}`
+          );
+        }
+      }
+
+      if (cells !== undefined && kzgProofs !== undefined && cells.length > 0) {
+        const proofs = kzgProofs;
+        const cellsAndProofs = cells.map((rowCells, rowIndex) => ({
+          cells: rowCells,
+          proofs: proofs.slice(rowIndex * NUMBER_OF_COLUMNS, (rowIndex + 1) * NUMBER_OF_COLUMNS),
+        }));
+
+        dataColumnSidecars = getGloasDataColumnSidecars(slot, envelope.beaconBlockRoot, cellsAndProofs);
+        dataColumnTimer?.();
       }
 
       // If called near a slot boundary (e.g. late in slot N-1), hold briefly so gossip aligns with slot N.
@@ -724,21 +886,18 @@ export function getBeaconBlockApi({
         await sleep(msToBlockSlot);
       }
 
-      const payloadInput = chain.seenPayloadEnvelopeInputCache.get(blockRootHex);
-      if (!payloadInput) {
-        // The block is awaited above (queuing if the envelope arrived first), and both the API and
-        // gossip import paths seed the PayloadEnvelopeInput before importing the block, so the input
-        // should exist here.
-        throw new ApiError(404, `PayloadEnvelopeInput not found for block root ${blockRootHex}`);
+      if (payloadInput.hasPayloadEnvelope()) {
+        // The envelope may have been added while this request was being validated, e.g. via gossip
+        chain.logger.debug("Execution payload envelope already added during publishing", valLogMeta);
+      } else {
+        payloadInput.addPayloadEnvelope({
+          envelope: signedExecutionPayloadEnvelope,
+          source: PayloadEnvelopeInputSource.api,
+          seenTimestampSec,
+          peerIdStr: undefined,
+        });
       }
 
-      payloadInput.addPayloadEnvelope({
-        envelope: signedExecutionPayloadEnvelope,
-        source: PayloadEnvelopeInputSource.api,
-        seenTimestampSec,
-        peerIdStr: undefined,
-      });
-
       if (dataColumnSidecars.length > 0) {
         for (const columnSidecar of dataColumnSidecars) {
           payloadInput.addColumn({
@@ -750,31 +909,28 @@ export function getBeaconBlockApi({
         }
       }
 
-      const valLogMeta = {
-        slot,
-        blockRoot: blockRootHex,
-        blockHash: blockHashHex,
-        builderIndex: envelope.builderIndex,
-        isSelfBuild,
-        dataColumns: dataColumnSidecars.length,
-      };
-
       const delaySec = seenTimestampSec - computeTimeAtSlot(config, slot, chain.genesisTime);
       metrics?.gossipExecutionPayloadEnvelope.elapsedTimeTillReceived.observe({source: OpSource.api}, delaySec);
       chain.validatorMonitor?.registerExecutionPayloadEnvelope(OpSource.api, delaySec, signedExecutionPayloadEnvelope);
 
-      chain.logger.info("Publishing execution payload envelope", valLogMeta);
+      chain.logger.info("Publishing execution payload envelope", {
+        ...valLogMeta,
+        dataColumns: dataColumnSidecars.length,
+      });
 
       const publishPromises = [
         // Gossip the signed execution payload envelope first
         () => network.publishSignedExecutionPayloadEnvelope(signedExecutionPayloadEnvelope),
-        // For self-builds, publish all data column sidecars
+        // Publish all data column sidecars
         ...dataColumnSidecars.map((dataColumnSidecar) => () => network.publishDataColumnSidecar(dataColumnSidecar)),
-        // Import execution payload. Signature already verified above
+        // Import execution payload. Signature is verified during broadcast validation, an already
+        // known envelope was verified by its original source and `none` deliberately skips validation
         () => chain.processExecutionPayload(payloadInput, {validSignature: true}),
       ];
 
-      const publishPromise = promiseAllMaybeAsync<number | void>(publishPromises);
+      const publishPromise = promiseAllMaybeAsync<{sentPeers: number; alreadyPublished: boolean} | number | void>(
+        publishPromises
+      );
 
       chain.emitter.emit(routes.events.EventType.executionPayloadGossip, {
         slot,
@@ -790,9 +946,11 @@ export function getBeaconBlockApi({
         let columnsPublishedWithZeroPeers = 0;
         // Skip first entry (envelope); the final entry is processExecutionPayload(), which returns void.
         for (let i = 0; i < dataColumnSidecars.length; i++) {
-          const sentPeers = sentPeersArr[i + 1] as number;
+          const {sentPeers, alreadyPublished} = sentPeersArr[i + 1] as {sentPeers: number; alreadyPublished: boolean};
           metrics?.dataColumns.sentPeersPerSubnet.observe(sentPeers);
-          if (sentPeers === 0) {
+          // A duplicate publish means the column is already propagating — expected in self-build flows.
+          // Only warn when we genuinely failed to reach any peer. See https://github.com/ChainSafe/lodestar/issues/9527.
+          if (sentPeers === 0 && !alreadyPublished) {
             columnsPublishedWithZeroPeers++;
           }
         }

package/src/api/impl/beacon/index.ts

@@ -7,7 +7,7 @@ import {getBeaconRewardsApi} from "./rewards/index.js";
 import {getBeaconStateApi} from "./state/index.js";
 
 export function getBeaconApi(
-  modules: Pick<ApiModules, "chain" | "config" | "logger" | "metrics" | "network" | "db">
+  modules: Pick<ApiModules, "chain" | "config" | "logger" | "metrics" | "network" | "db" | "sync">
 ): ApplicationMethods<routes.beacon.Endpoints> {
   const block = getBeaconBlockApi(modules);
   const pool = getBeaconPoolApi(modules);

package/src/api/impl/beacon/pool/index.ts

@@ -241,6 +241,11 @@ export function getBeaconPoolApi({
               payloadAttestationMessage.data.blobDataAvailable
             );
 
+            chain.emitter.emit(routes.events.EventType.payloadAttestationMessage, {
+              version: chain.config.getForkName(slot),
+              data: payloadAttestationMessage,
+            });
+
             await network.publishPayloadAttestationMessage(payloadAttestationMessage);
           } catch (e) {
             const logCtx = {

package/src/api/impl/beacon/state/index.ts

@@ -26,11 +26,12 @@ import {
 export function getBeaconStateApi({
   chain,
   config,
-}: Pick<ApiModules, "chain" | "config">): ApplicationMethods<routes.beacon.state.Endpoints> {
+  sync,
+}: Pick<ApiModules, "chain" | "config" | "sync">): ApplicationMethods<routes.beacon.state.Endpoints> {
   async function getState(
     stateId: routes.beacon.StateId
   ): Promise<{state: IBeaconStateView; executionOptimistic: boolean; finalized: boolean}> {
-    const {state, executionOptimistic, finalized} = await getStateResponseWithRegen(chain, stateId);
+    const {state, executionOptimistic, finalized} = await getStateResponseWithRegen(chain, sync, stateId);
 
     return {
       state: state instanceof Uint8Array ? chain.getHeadState().loadOtherState(state) : state,

package/src/api/impl/beacon/state/utils.ts

@@ -14,7 +14,9 @@ import {
 } from "@lodestar/types";
 import {byteArrayEquals, fromHex} from "@lodestar/utils";
 import {IBeaconChain} from "../../../../chain/index.js";
+import {IBeaconSync} from "../../../../sync/index.js";
 import {ApiError, ValidationError} from "../../errors.js";
+import {notWhileSyncing} from "../../utils.js";
 
 export function resolveStateId(
   forkChoice: IForkChoice,
@@ -51,8 +53,17 @@ export function resolveStateId(
 
 export async function getStateResponseWithRegen(
   chain: IBeaconChain,
+  sync: IBeaconSync,
   inStateId: routes.beacon.StateId
 ): Promise<{state: IBeaconStateView | Uint8Array; executionOptimistic: boolean; finalized: boolean}> {
+  // "head", "finalized" and "justified" resolve to already-available cached states, and "genesis" to a
+  // historical DB read - none trigger the forward regen that can walk back past the block-root window
+  // (SLOTS_PER_HISTORICAL_ROOT) and wedge a far-behind node. Keep serving those (node observability,
+  // dashboards, validator client checks) even while syncing; guard only the regen-capable lookups.
+  if (inStateId !== "head" && inStateId !== "finalized" && inStateId !== "justified" && inStateId !== "genesis") {
+    notWhileSyncing(chain, sync.state);
+  }
+
   const stateId = resolveStateId(chain.forkChoice, inStateId);
 
   const res =

package/src/api/impl/debug/index.ts

@@ -39,7 +39,8 @@ function toPayloadStatusName(status: PayloadStatus): "pending" | "empty" | "full
 export function getDebugApi({
   chain,
   config,
-}: Pick<ApiModules, "chain" | "config" | "db">): ApplicationMethods<routes.debug.Endpoints> {
+  sync,
+}: Pick<ApiModules, "chain" | "config" | "db" | "sync">): ApplicationMethods<routes.debug.Endpoints> {
   return {
     async getDebugChainHeadsV2() {
       const heads = chain.forkChoice.getHeads();
@@ -132,7 +133,7 @@ export function getDebugApi({
     },
 
     async getStateV2({stateId}, context) {
-      const {state, executionOptimistic, finalized} = await getStateResponseWithRegen(chain, stateId);
+      const {state, executionOptimistic, finalized} = await getStateResponseWithRegen(chain, sync, stateId);
       let slot: number, data: Uint8Array | BeaconState;
       if (state instanceof Uint8Array) {
         slot = getStateSlotFromBytes(state);

package/src/api/impl/lodestar/index.ts

@@ -217,7 +217,7 @@ export function getLodestarApi({
     },
 
     async getHistoricalSummaries({stateId}) {
-      const {state, executionOptimistic, finalized} = await getStateResponseWithRegen(chain, stateId);
+      const {state, executionOptimistic, finalized} = await getStateResponseWithRegen(chain, sync, stateId);
 
       const stateView = state instanceof Uint8Array ? chain.getHeadState().loadOtherState(state) : state;
 
@@ -332,7 +332,7 @@ export function getLodestarApi({
 
       for (const [epoch, attestationsPerEpoch] of attestations) {
         const slot = computeStartSlotAtEpoch(epoch);
-        const {state} = await getStateResponseWithRegen(chain, slot);
+        const {state} = await getStateResponseWithRegen(chain, sync, slot);
         const stateView = state instanceof Uint8Array ? chain.getHeadState().loadOtherState(state) : state;
         const shuffling = stateView.getShufflingAtEpoch(epoch);
         for (const attestation of attestationsPerEpoch) {

package/src/api/impl/proof/index.ts

@@ -8,7 +8,7 @@ import {ApiModules} from "../types.js";
 
 export function getProofApi(
   opts: ApiOptions,
-  {chain, config}: Pick<ApiModules, "chain" | "config" | "db">
+  {chain, config, sync}: Pick<ApiModules, "chain" | "config" | "db" | "sync">
 ): ApplicationMethods<routes.proof.Endpoints> {
   // It's currently possible to request gigantic proofs (eg: a proof of the entire beacon state)
   // We want some some sort of resistance against this DoS vector.
@@ -21,7 +21,7 @@ export function getProofApi(
         throw new Error("Requested proof is too large.");
       }
 
-      const res = await getStateResponseWithRegen(chain, stateId);
+      const res = await getStateResponseWithRegen(chain, sync, stateId);
 
       const state = res.state instanceof Uint8Array ? chain.getHeadState().loadOtherState(res.state) : res.state;
 

package/src/api/impl/utils.ts

@@ -1,4 +1,54 @@
-import {ApiError} from "./errors.js";
+import {SLOTS_PER_EPOCH} from "@lodestar/params";
+import type {IBeaconChain} from "../../chain/index.js";
+import {SyncState} from "../../sync/index.js";
+import {ApiError, NodeIsSyncing} from "./errors.js";
+
+/**
+ * If the node is within this many epochs from the head, we declare it to be synced regardless of
+ * the network sync state.
+ *
+ * This helps prevent attacks where nodes can convince us that we're syncing some non-existent
+ * finalized head.
+ *
+ * TODO: Lighthouse uses 8 for the attack described above. However, 8 kills Lodestar since validators
+ * can trigger regen to fast-forward head state 8 epochs to be immediately invalidated as sync sets
+ * a new head. Then the checkpoint state cache grows unbounded with very different states (because
+ * they are 8 epochs apart) and causes an OOM. Research a proper solution once regen and the state
+ * caches are better.
+ */
+export const SYNC_TOLERANCE_EPOCHS = 1;
+
+/**
+ * Reject any request while the node is syncing. Used by endpoints that must not serve while the
+ * node is behind — validator duties, and beacon state lookups whose regen could otherwise walk
+ * back past the block-root window (`SLOTS_PER_HISTORICAL_ROOT`) and wedge the node. Throws
+ * {@link NodeIsSyncing} (503).
+ */
+export function notWhileSyncing(chain: IBeaconChain, syncState: SyncState): void {
+  // Consider node synced before or close to genesis
+  if (chain.clock.currentSlot < SLOTS_PER_EPOCH) {
+    return;
+  }
+
+  switch (syncState) {
+    case SyncState.SyncingFinalized:
+    case SyncState.SyncingHead: {
+      const currentSlot = chain.clock.currentSlot;
+      const headSlot = chain.forkChoice.getHead().slot;
+      if (currentSlot - headSlot > SYNC_TOLERANCE_EPOCHS * SLOTS_PER_EPOCH) {
+        throw new NodeIsSyncing(`headSlot ${headSlot} currentSlot ${currentSlot}`);
+      }
+
+      return;
+    }
+
+    case SyncState.Synced:
+      return;
+
+    case SyncState.Stalled:
+      throw new NodeIsSyncing("waiting for peers");
+  }
+}
 
 /**
  * Ensures that the array contains unique values, and throws an ApiError